From 149accb652743c1e8b36e08bd6f0b98c33d22c84 Mon Sep 17 00:00:00 2001 From: Myst <1592048+LeMyst@users.noreply.github.com> Date: Tue, 10 May 2022 21:11:38 +0200 Subject: [PATCH 1/5] Implement new version of FastRun --- README.md | 10 +- pyproject.toml | 1 + test/test_datatypes.py | 2 +- test/test_wbi_fastrun.py | 327 ----- wikibaseintegrator/datatypes/basedatatype.py | 15 +- wikibaseintegrator/datatypes/commonsmedia.py | 21 +- wikibaseintegrator/datatypes/externalid.py | 1 + wikibaseintegrator/datatypes/form.py | 13 +- wikibaseintegrator/datatypes/geoshape.py | 5 + .../datatypes/globecoordinate.py | 37 +- wikibaseintegrator/datatypes/item.py | 38 +- wikibaseintegrator/datatypes/lexeme.py | 11 +- wikibaseintegrator/datatypes/math.py | 33 + .../datatypes/monolingualtext.py | 28 +- .../datatypes/musicalnotation.py | 1 + wikibaseintegrator/datatypes/property.py | 11 +- wikibaseintegrator/datatypes/quantity.py | 30 +- wikibaseintegrator/datatypes/sense.py | 13 +- wikibaseintegrator/datatypes/string.py | 25 +- wikibaseintegrator/datatypes/tabulardata.py | 5 + wikibaseintegrator/datatypes/time.py | 33 +- wikibaseintegrator/datatypes/url.py | 25 +- wikibaseintegrator/entities/baseentity.py | 37 +- wikibaseintegrator/entities/item.py | 2 +- wikibaseintegrator/models/claims.py | 14 +- wikibaseintegrator/models/qualifiers.py | 2 +- wikibaseintegrator/models/snaks.py | 3 + wikibaseintegrator/wbi_config.py | 3 +- wikibaseintegrator/wbi_fastrun.py | 1088 ++++++++--------- 29 files changed, 899 insertions(+), 935 deletions(-) delete mode 100644 test/test_wbi_fastrun.py diff --git a/README.md b/README.md index 8ab0a58c..521fa650 100644 --- a/README.md +++ b/README.md @@ -869,7 +869,15 @@ for entrez_id, ensembl in raw_data.items(): Note: Fastrun mode checks for equality of property/value pairs, qualifiers (not including qualifier attributes), labels, aliases and description, but it ignores references by default! -References can be checked in fast run mode by setting `use_refs` to `True`. +References can be checked in fast run mode by setting `use_references` to `True`. + +# Statistics # + +| Dataset | partial fastrun | fastrun without qualifiers/references | fastrun with qualifiers | fastrun with qualifiers/references | +|:----------------------------|----------------:|--------------------------------------:|------------------------:|-----------------------------------:| +| Communes (34990 elements) | ? | 7min | 30s | 60s | +| Cantons (2042 elements) | ? | ? | ? | ? | +| Départements (100 elements) | 70min | 1s | 30s | 60s | # Debugging # diff --git a/pyproject.toml b/pyproject.toml index c53be093..e18016ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,6 +120,7 @@ disable = [ [tool.pytest.ini_options] log_cli = true +log_cli_level = 'DEBUG' testpaths = ["test"] markers = [ "integration: tests running against a real Wikibase instance (deselected by default, see test/integration/README.md)" diff --git a/test/test_datatypes.py b/test/test_datatypes.py index e2bb420f..cacac655 100644 --- a/test/test_datatypes.py +++ b/test/test_datatypes.py @@ -163,7 +163,7 @@ def test_all_datatypes(self): MonolingualText(text='xxx', language='fr', prop_nr='P7'), Quantity(amount=-5.04, prop_nr='P8'), Quantity(amount=5.06, upper_bound=9.99, lower_bound=-2.22, unit='Q11573', prop_nr='P8'), - CommonsMedia(value='xxx', prop_nr='P9'), + CommonsMedia(value='xxx.jpg', prop_nr='P9'), GlobeCoordinate(latitude=1.2345, longitude=-1.2345, precision=12, prop_nr='P10'), GeoShape(value='Data:xxx.map', prop_nr='P11'), Property(value='P123', prop_nr='P12'), diff --git a/test/test_wbi_fastrun.py b/test/test_wbi_fastrun.py deleted file mode 100644 index 54799266..00000000 --- a/test/test_wbi_fastrun.py +++ /dev/null @@ -1,327 +0,0 @@ -""" -Tests for the fastrun container. The SPARQL endpoint and the property -datatype lookups are served by the simulated Wikibase instance, so the whole -pipeline (query -> reverse lookup -> statement reconstruction -> comparison) -runs offline and deterministically. -""" -from collections import defaultdict -from typing import Any - -import pytest - -from wikibaseintegrator import WikibaseIntegrator, wbi_fastrun -from wikibaseintegrator.datatypes import BaseDataType, ExternalID, Item -from wikibaseintegrator.entities import ItemEntity -from wikibaseintegrator.wbi_enums import ActionIfExists - -from .conftest import literal, load_fixture, uri - -wbi = WikibaseIntegrator() - - -def statement_bindings(wikibase, item_qid: str, prop_nr: str, values: list[dict], refs: dict | None = None) -> list[dict]: - """Build SPARQL bindings shaped like the ones of wbi_fastrun._query_data.""" - bindings = [] - for index, value in enumerate(values): - binding = { - 'sid': uri(f'{wikibase.base_url}/entity/statement/{item_qid}-{prop_nr}-{index}'), - 'item': uri(f'{wikibase.base_url}/entity/{item_qid}'), - 'v': value, - } - if refs: - for ref_prop, ref_value in refs.items(): - bindings.append({ - **binding, - 'ref': uri(f'{wikibase.base_url}/reference/deadbeef{index}'), - 'pr': uri(f'{wikibase.base_url}/prop/reference/{ref_prop}'), - 'rval': ref_value, - }) - else: - bindings.append(binding) - return bindings - - -class TestQueryData: - def test_query_data(self, wikibase): - """The container must query the SPARQL endpoint and store the data in its internal format.""" - wikibase.add_property('P352', 'external-id') - - frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P352')], base_data_type=BaseDataType) - - wikibase.sparql_bindings = statement_bindings(wikibase, 'Q99', 'P352', [literal('P40095')]) - frc._query_data('P352') - - assert 'Q99' in frc.prop_data - assert 'P352' in frc.prop_data['Q99'] - - statement_id = list(frc.prop_data['Q99']['P352'].keys())[0] - statement_data = frc.prop_data['Q99']['P352'][statement_id] - assert all(key in statement_data for key in ('qual', 'ref', 'v')) - assert statement_data['v'] == '"P40095"' - assert frc.rev_lookup['"P40095"'] == {'Q99'} - - def test_query_data_item_and_uri_values(self, wikibase): - wikibase.add_property('P828', 'wikibase-item') - wikibase.add_property('P2888', 'url') - - frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P828')], base_data_type=BaseDataType) - - # wikibase-item value: the entity URI is reduced to its ID - wikibase.sparql_bindings = statement_bindings(wikibase, 'Q99', 'P828', [uri(f'{wikibase.base_url}/entity/Q18228398')]) - frc._query_data('P828') - assert list(frc.prop_data['Q99']['P828'].values())[0]['v'] == 'Q18228398' - - # url value: kept as an URI - wikibase.sparql_bindings = statement_bindings(wikibase, 'Q99', 'P2888', [uri('http://purl.obolibrary.org/obo/DOID_1432')]) - frc._query_data('P2888') - values = {statement['v'] for statement in frc.prop_data['Q99']['P2888'].values()} - assert all(value.startswith(' ?zzP352 .' in frc.base_filter_string - assert f'?item <{wikibase.base_url}/prop/direct/P703>' in frc.base_filter_string - - -# --------------------------------------------------------------------- # -# Offline tests based on hand-crafted container contents. They validate -# the write_required comparison logic itself. -# --------------------------------------------------------------------- # - -class FastRunContainerFakeQueryDataEnsembl(wbi_fastrun.FastRunContainer): - def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) - self.prop_dt_map = {'P248': 'wikibase-item', 'P594': 'external-id'} - self.prop_data['Q14911732'] = {'P594': { - 'fake statement id': { - 'qual': set(), - 'ref': {'fake ref id': { - ('P248', 'Q106833387'), - ('P594', 'ENSG00000123374')}}, - 'unit': '1', - 'v': '"ENSG00000123374"'}}} - self.rev_lookup = defaultdict(set) - self.rev_lookup['"ENSG00000123374"'].add('Q14911732') - - -class FastRunContainerFakeQueryDataEnsemblNoRef(wbi_fastrun.FastRunContainer): - def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) - self.prop_dt_map = {'P248': 'wikibase-item', 'P594': 'external-id'} - self.prop_data['Q14911732'] = {'P594': { - 'fake statement id': { - 'qual': set(), - 'ref': {}, - 'v': 'ENSG00000123374'}}} - self.rev_lookup = defaultdict(set) - self.rev_lookup['"ENSG00000123374"'].add('Q14911732') - - -def test_fastrun_ref_ensembl(): - # fastrun checks refs - frc = FastRunContainerFakeQueryDataEnsembl(base_filter=[BaseDataType(prop_nr='P594'), Item(prop_nr='P703', value='Q15978631')], base_data_type=BaseDataType, use_refs=True) - - # statement has no ref - statements = [ExternalID(value='ENSG00000123374', prop_nr='P594')] - assert frc.write_required(data=statements) - - # statement has the same ref - statements = [ExternalID(value='ENSG00000123374', prop_nr='P594', references=[[Item('Q106833387', prop_nr='P248'), ExternalID('ENSG00000123374', prop_nr='P594')]])] - assert not frc.write_required(data=statements) - - # new statement has a different stated in - statements = [ExternalID(value='ENSG00000123374', prop_nr='P594', references=[[Item('Q99999999999', prop_nr='P248'), ExternalID('ENSG00000123374', prop_nr='P594')]])] - assert frc.write_required(data=statements) - - # fastrun doesn't check references, statement has no reference - frc = FastRunContainerFakeQueryDataEnsemblNoRef(base_filter=[BaseDataType(prop_nr='P594'), Item(prop_nr='P703', value='Q15978631')], base_data_type=BaseDataType, - use_refs=False) - statements = [ExternalID(value='ENSG00000123374', prop_nr='P594')] - assert not frc.write_required(data=statements) - - # fastrun doesn't check references, statement has a reference - frc = FastRunContainerFakeQueryDataEnsemblNoRef(base_filter=[BaseDataType(prop_nr='P594'), Item(prop_nr='P703', value='Q15978631')], base_data_type=BaseDataType, - use_refs=False) - statements = [ExternalID(value='ENSG00000123374', prop_nr='P594', references=[[Item('Q123', prop_nr='P31')]])] - assert not frc.write_required(data=statements) - - -class FakeQueryDataAppendProps(wbi_fastrun.FastRunContainer): - # an item with three values for the same property - def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) - self.prop_dt_map = {'P527': 'wikibase-item', 'P248': 'wikibase-item', 'P594': 'external-id'} - - self.rev_lookup = defaultdict(set) - self.rev_lookup['Q24784025'].add('Q3402672') - self.rev_lookup['Q24743729'].add('Q3402672') - self.rev_lookup['Q24782625'].add('Q3402672') - - self.prop_data['Q3402672'] = {'P527': { - 'Q3402672-11BA231B-857B-498B-AC4F-91D71EE007FD': {'qual': set(), - 'ref': { - '149c9c7ba4e246d9f09ce3ed0cdf7aa721aad5c8': { - ('P248', 'Q3047275'), - }}, - 'v': 'Q24784025'}, - 'Q3402672-15F54AFF-7DCC-4DF6-A32F-73C48619B0B2': {'qual': set(), - 'ref': { - '149c9c7ba4e246d9f09ce3ed0cdf7aa721aad5c8': { - ('P248', 'Q3047275'), - }}, - 'v': 'Q24743729'}, - 'Q3402672-C8F11D55-1B11-44E5-9EAF-637E062825A4': {'qual': set(), - 'ref': { - '149c9c7ba4e246d9f09ce3ed0cdf7aa721aad5c8': { - ('P248', 'Q3047275')}}, - 'v': 'Q24782625'}}} - - -def test_append_props(): - qid = 'Q3402672' - - # don't consider refs - frc = FakeQueryDataAppendProps(base_filter=[BaseDataType(prop_nr='P352'), Item(prop_nr='P703', value='Q15978631')], base_data_type=BaseDataType) - # with append - statements = [Item(value='Q24784025', prop_nr='P527')] - assert frc.write_required(data=statements, action_if_exists=ActionIfExists.APPEND_OR_REPLACE, cqid=qid) is False - # with force append - statements = [Item(value='Q24784025', prop_nr='P527')] - assert frc.write_required(data=statements, action_if_exists=ActionIfExists.FORCE_APPEND, cqid=qid) is True - # without append - statements = [Item(value='Q24784025', prop_nr='P527')] - assert frc.write_required(data=statements, cqid=qid) is True - - # if we are in append mode, and the refs are different, we should write - frc = FakeQueryDataAppendProps(base_filter=[BaseDataType(prop_nr='P352'), Item(prop_nr='P703', value='Q15978631')], base_data_type=BaseDataType, use_refs=True) - # with append - statements = [Item(value='Q24784025', prop_nr='P527')] - assert frc.write_required(data=statements, cqid=qid, action_if_exists=ActionIfExists.APPEND_OR_REPLACE) is True - # without append - statements = [Item(value='Q24784025', prop_nr='P527')] - assert frc.write_required(data=statements, cqid=qid) is True diff --git a/wikibaseintegrator/datatypes/basedatatype.py b/wikibaseintegrator/datatypes/basedatatype.py index 0db1c3d7..f5b99a92 100644 --- a/wikibaseintegrator/datatypes/basedatatype.py +++ b/wikibaseintegrator/datatypes/basedatatype.py @@ -11,6 +11,7 @@ class BaseDataType(Claim): The base class for all Wikibase data types, they inherit from it """ DTYPE = 'base-data-type' + PTYPE = 'property-data-type' subclasses: list[type[BaseDataType]] = [] sparql_query: str = ''' SELECT * WHERE {{ @@ -28,7 +29,14 @@ def __init__(self, prop_nr: int | str | None = None, **kwargs: Any): super().__init__(**kwargs) - self.mainsnak.property_number = prop_nr or None + if isinstance(prop_nr, str): + pattern = re.compile(r'^([a-z][a-z\d+.-]*):([^][<>\"\x00-\x20\x7F])+$') + matches = pattern.match(str(prop_nr)) + + if matches: + prop_nr = prop_nr.rsplit('/', 1)[-1] + + self.mainsnak.property_number = prop_nr # self.subclasses.append(self) # Allow registration of subclasses of BaseDataType into BaseDataType.subclasses @@ -39,7 +47,7 @@ def __init_subclass__(cls, **kwargs): def set_value(self, value: Any | None = None): pass - def get_sparql_value(self) -> str: + def get_sparql_value(self, **kwargs: Any) -> str | None: return '"' + self.mainsnak.datavalue['value'] + '"' def parse_sparql_value(self, value, type='literal', unit='1') -> bool: @@ -61,3 +69,6 @@ def parse_sparql_value(self, value, type='literal', unit='1') -> bool: raise ValueError return True + + def from_sparql_value(self, sparql_value: dict) -> BaseDataType: # type: ignore + pass diff --git a/wikibaseintegrator/datatypes/commonsmedia.py b/wikibaseintegrator/datatypes/commonsmedia.py index c444437d..2f061cdc 100644 --- a/wikibaseintegrator/datatypes/commonsmedia.py +++ b/wikibaseintegrator/datatypes/commonsmedia.py @@ -1,17 +1,30 @@ import re import urllib.parse -from wikibaseintegrator.datatypes.string import String +from wikibaseintegrator.datatypes.url import URL -class CommonsMedia(String): +class CommonsMedia(URL): """ Implements the Wikibase data type for Wikimedia commons media files """ DTYPE = 'commonsMedia' + PTYPE = 'http://wikiba.se/ontology#CommonsMedia' - def get_sparql_value(self) -> str: - return '<' + self.mainsnak.datavalue['value'] + '>' + def set_value(self, value: str | None = None): + assert isinstance(value, str) or value is None, f"Expected str, found {type(value)} ({value})" + + if value: + pattern = re.compile(r'^.+\..+$') + matches = pattern.match(value) + + if not matches: + raise ValueError(f"Invalid CommonsMedia {value}") + + self.mainsnak.datavalue = { + 'value': value, + 'type': 'string' + } def parse_sparql_value(self, value, type='literal', unit='1') -> bool: pattern = re.compile(r'^?$') diff --git a/wikibaseintegrator/datatypes/externalid.py b/wikibaseintegrator/datatypes/externalid.py index c4838138..6b88ca7e 100644 --- a/wikibaseintegrator/datatypes/externalid.py +++ b/wikibaseintegrator/datatypes/externalid.py @@ -6,3 +6,4 @@ class ExternalID(String): Implements the Wikibase data type 'external-id' """ DTYPE = 'external-id' + PTYPE = 'http://wikiba.se/ontology#ExternalId' diff --git a/wikibaseintegrator/datatypes/form.py b/wikibaseintegrator/datatypes/form.py index e8732cfd..a5f069e8 100644 --- a/wikibaseintegrator/datatypes/form.py +++ b/wikibaseintegrator/datatypes/form.py @@ -2,6 +2,8 @@ from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class Form(BaseDataType): @@ -9,6 +11,7 @@ class Form(BaseDataType): Implements the Wikibase data type 'wikibase-form' """ DTYPE = 'wikibase-form' + PTYPE = 'http://wikiba.se/ontology#WikibaseForm' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -55,8 +58,14 @@ def set_value(self, value: str | None = None): 'type': 'wikibase-entityid' } - def get_sparql_value(self) -> str: - return self.mainsnak.datavalue['value']['id'] + # TODO: add from_sparql_value() + + def get_sparql_value(self, **kwargs: Any) -> str | None: + if self.mainsnak.snaktype == WikibaseSnakType.KNOWN_VALUE: + wikibase_url = str(kwargs['wikibase_url'] if 'wikibase_url' in kwargs else config['WIKIBASE_URL']) + return f'<{wikibase_url}/entity/' + self.mainsnak.datavalue['value']['id'] + '>' + + return None def get_lexeme_id(self) -> str: """ diff --git a/wikibaseintegrator/datatypes/geoshape.py b/wikibaseintegrator/datatypes/geoshape.py index e5dc4ca5..903013d4 100644 --- a/wikibaseintegrator/datatypes/geoshape.py +++ b/wikibaseintegrator/datatypes/geoshape.py @@ -9,6 +9,7 @@ class GeoShape(BaseDataType): Implements the Wikibase data type 'geo-shape' """ DTYPE = 'geo-shape' + PTYPE = 'http://wikiba.se/ontology#GeoShape' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -53,3 +54,7 @@ def set_value(self, value: str | None = None): 'value': value, 'type': 'string' } + + # TODO: Does GeoShape need a full URL to wikimedia commons? + def get_sparql_value(self, **kwargs: Any) -> str: + return '<' + self.mainsnak.datavalue['value'] + '>' diff --git a/wikibaseintegrator/datatypes/globecoordinate.py b/wikibaseintegrator/datatypes/globecoordinate.py index d0eb0568..e7667c24 100644 --- a/wikibaseintegrator/datatypes/globecoordinate.py +++ b/wikibaseintegrator/datatypes/globecoordinate.py @@ -1,9 +1,12 @@ +from __future__ import annotations + import re from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType from wikibaseintegrator.models import Claim from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class GlobeCoordinate(BaseDataType): @@ -11,6 +14,7 @@ class GlobeCoordinate(BaseDataType): Implements the Wikibase data type for globe coordinates """ DTYPE = 'globe-coordinate' + PTYPE = 'http://wikiba.se/ontology#GlobeCoordinate' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -75,8 +79,37 @@ def rounded(datavalue: dict) -> dict: return super().__eq__(other) - def get_sparql_value(self) -> str: - return '"Point(' + str(self.mainsnak.datavalue['value']['longitude']) + ' ' + str(self.mainsnak.datavalue['value']['latitude']) + ')"' + def from_sparql_value(self, sparql_value: dict) -> GlobeCoordinate: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of datatype, type and value + :return: True if the parsing is successful + """ + datatype = sparql_value['datatype'] + type = sparql_value['type'] + value = sparql_value['value'] + + if datatype != 'http://www.opengis.net/ont/geosparql#wktLiteral': + raise ValueError('Wrong SPARQL datatype') + + if type != 'literal': + raise ValueError('Wrong SPARQL type') + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + pattern = re.compile(r'^Point\((.*) (.*)\)$') + matches = pattern.match(value) + if not matches: + raise ValueError('Invalid SPARQL value') + + self.set_value(longitude=float(matches.group(1)), latitude=float(matches.group(2))) + + return self + + def get_sparql_value(self, **kwargs: Any) -> str: + return '"Point(' + str(self.mainsnak.datavalue['value']['longitude']) + ' ' + str(self.mainsnak.datavalue['value']['latitude']) + ')"^^geo:wktLiteral' def parse_sparql_value(self, value, type='literal', unit='1') -> bool: pattern = re.compile(r'^"?Point\((.*) (.*)\)"?(?:\^\^geo:wktLiteral)?$') diff --git a/wikibaseintegrator/datatypes/item.py b/wikibaseintegrator/datatypes/item.py index 798ea492..3dbf911f 100644 --- a/wikibaseintegrator/datatypes/item.py +++ b/wikibaseintegrator/datatypes/item.py @@ -1,7 +1,11 @@ +from __future__ import annotations + import re from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class Item(BaseDataType): @@ -9,6 +13,7 @@ class Item(BaseDataType): Implements the Wikibase data type 'wikibase-item' with a value being another item ID """ DTYPE = 'wikibase-item' + PTYPE = 'http://wikiba.se/ontology#WikibaseItem' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -48,5 +53,34 @@ def set_value(self, value: str | int | None = None): 'type': 'wikibase-entityid' } - def get_sparql_value(self) -> str: - return '<{wb_url}/entity/' + self.mainsnak.datavalue['value']['id'] + '>' + def from_sparql_value(self, sparql_value: dict) -> Item: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of type and value + :return: True if the parsing is successful + """ + type = sparql_value['type'] + value = sparql_value['value'] + + if type != 'uri': + raise ValueError('Wrong SPARQL type') + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + pattern = re.compile(r'^.+/([PQLM]\d+)$') + matches = pattern.match(value) + if not matches: + raise ValueError(f"Invalid SPARQL value {value}") + + self.set_value(value=str(matches.group(1))) + + return self + + def get_sparql_value(self, **kwargs: Any) -> str | None: + if self.mainsnak.snaktype == WikibaseSnakType.KNOWN_VALUE: + wikibase_url = str(kwargs['wikibase_url'] if 'wikibase_url' in kwargs else config['WIKIBASE_URL']) + return f'<{wikibase_url}/entity/' + self.mainsnak.datavalue['value']['id'] + '>' + + return None diff --git a/wikibaseintegrator/datatypes/lexeme.py b/wikibaseintegrator/datatypes/lexeme.py index f128c611..f552e097 100644 --- a/wikibaseintegrator/datatypes/lexeme.py +++ b/wikibaseintegrator/datatypes/lexeme.py @@ -2,6 +2,8 @@ from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class Lexeme(BaseDataType): @@ -9,6 +11,7 @@ class Lexeme(BaseDataType): Implements the Wikibase data type 'wikibase-lexeme' """ DTYPE = 'wikibase-lexeme' + PTYPE = 'http://wikiba.se/ontology#WikibaseLexeme' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -48,5 +51,9 @@ def set_value(self, value: str | int | None = None): 'type': 'wikibase-entityid' } - def get_sparql_value(self) -> str: - return self.mainsnak.datavalue['value']['id'] + def get_sparql_value(self, **kwargs: Any) -> str | None: + if self.mainsnak.snaktype == WikibaseSnakType.KNOWN_VALUE: + wikibase_url = str(kwargs['wikibase_url'] if 'wikibase_url' in kwargs else config['WIKIBASE_URL']) + return f'<{wikibase_url}/entity/' + self.mainsnak.datavalue['value']['id'] + '>' + + return None diff --git a/wikibaseintegrator/datatypes/math.py b/wikibaseintegrator/datatypes/math.py index 7ad3f3cc..29c48217 100644 --- a/wikibaseintegrator/datatypes/math.py +++ b/wikibaseintegrator/datatypes/math.py @@ -1,4 +1,9 @@ +from __future__ import annotations + +from typing import Any + from wikibaseintegrator.datatypes.string import String +from wikibaseintegrator.wbi_enums import WikibaseSnakType class Math(String): @@ -6,3 +11,31 @@ class Math(String): Implements the Wikibase data type 'math' for mathematical formula in TEX format """ DTYPE = 'math' + PTYPE = 'http://wikiba.se/ontology#Math' + + def from_sparql_value(self, sparql_value: dict) -> Math: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of type and value + :return: + """ + datatype = sparql_value['datatype'] + type = sparql_value['type'] + value = sparql_value['value'] + + if datatype != 'http://www.w3.org/2001/XMLSchema#dateTime': + raise ValueError('Wrong SPARQL datatype') + + if type != 'literal': + raise ValueError('Wrong SPARQL type') + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + self.set_value(value=value) + + return self + + def get_sparql_value(self, **kwargs: Any) -> str: + return '"' + self.mainsnak.datavalue['value'] + '"^^' diff --git a/wikibaseintegrator/datatypes/monolingualtext.py b/wikibaseintegrator/datatypes/monolingualtext.py index 630a0b9e..f3c41a51 100644 --- a/wikibaseintegrator/datatypes/monolingualtext.py +++ b/wikibaseintegrator/datatypes/monolingualtext.py @@ -1,8 +1,11 @@ +from __future__ import annotations + import re from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class MonolingualText(BaseDataType): @@ -10,6 +13,8 @@ class MonolingualText(BaseDataType): Implements the Wikibase data type for Monolingual Text strings """ DTYPE = 'monolingualtext' + PTYPE = 'http://wikiba.se/ontology#Monolingualtext' + sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -46,7 +51,28 @@ def set_value(self, text: str | None = None, language: str | None = None): 'type': 'monolingualtext' } - def get_sparql_value(self) -> str: + def from_sparql_value(self, sparql_value: dict) -> MonolingualText: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of datatype, type and value + :return: True if the parsing is successful + """ + xml_lang = sparql_value['xml:lang'] + type = sparql_value['type'] + value = sparql_value['value'] + + if type != 'literal': + raise ValueError(f"Wrong SPARQL type {type}") + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + self.set_value(text=value, language=xml_lang) + + return self + + def get_sparql_value(self, **kwargs: Any) -> str: return '"' + self.mainsnak.datavalue['value']['text'].replace('"', r'\"') + '"@' + self.mainsnak.datavalue['value']['language'] def parse_sparql_value(self, value, type='literal', unit='1') -> bool: diff --git a/wikibaseintegrator/datatypes/musicalnotation.py b/wikibaseintegrator/datatypes/musicalnotation.py index 7d38a4b5..61d3ccf2 100644 --- a/wikibaseintegrator/datatypes/musicalnotation.py +++ b/wikibaseintegrator/datatypes/musicalnotation.py @@ -6,6 +6,7 @@ class MusicalNotation(String): Implements the Wikibase data type 'musical-notation' """ DTYPE = 'musical-notation' + PTYPE = 'http://wikiba.se/ontology#MusicalNotation' def set_value(self, value: str | None = None): assert isinstance(value, str) or value is None, f"Expected str, found {type(value)} ({value})" diff --git a/wikibaseintegrator/datatypes/property.py b/wikibaseintegrator/datatypes/property.py index d1d4c312..97c0e018 100644 --- a/wikibaseintegrator/datatypes/property.py +++ b/wikibaseintegrator/datatypes/property.py @@ -2,6 +2,8 @@ from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class Property(BaseDataType): @@ -9,6 +11,7 @@ class Property(BaseDataType): Implements the Wikibase data type 'property' """ DTYPE = 'wikibase-property' + PTYPE = 'http://wikiba.se/ontology#Property' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -49,5 +52,9 @@ def set_value(self, value: str | int | None = None): 'type': 'wikibase-entityid' } - def get_sparql_value(self) -> str: - return self.mainsnak.datavalue['value']['id'] + def get_sparql_value(self, **kwargs: Any) -> str | None: + if self.mainsnak.snaktype == WikibaseSnakType.KNOWN_VALUE: + wikibase_url = str(kwargs['wikibase_url'] if 'wikibase_url' in kwargs else config['WIKIBASE_URL']) + return f'<{wikibase_url}/entity/' + self.mainsnak.datavalue['value']['id'] + '>' + + return None diff --git a/wikibaseintegrator/datatypes/quantity.py b/wikibaseintegrator/datatypes/quantity.py index 0e04113f..558a1ef2 100644 --- a/wikibaseintegrator/datatypes/quantity.py +++ b/wikibaseintegrator/datatypes/quantity.py @@ -1,7 +1,10 @@ +from __future__ import annotations + from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType from wikibaseintegrator.wbi_helpers import format_amount @@ -10,6 +13,7 @@ class Quantity(BaseDataType): Implements the Wikibase data type for quantities """ DTYPE = 'quantity' + PTYPE = 'http://wikiba.se/ontology#Quantity' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -81,7 +85,31 @@ def set_value(self, amount: str | int | float | None = None, upper_bound: str | if not lower_bound: del self.mainsnak.datavalue['value']['lowerBound'] - def get_sparql_value(self) -> str: + def from_sparql_value(self, sparql_value: dict) -> Quantity: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of datatype, type and value + :return: True if the parsing is successful + """ + datatype = sparql_value['datatype'] + type = sparql_value['type'] + value = sparql_value['value'] + + if datatype != 'http://www.w3.org/2001/XMLSchema#decimal': + raise ValueError(f"Wrong SPARQL datatype {datatype}") + + if type != 'literal': + raise ValueError(f"Wrong SPARQL type {type}") + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + self.set_value(amount=value) + + return self + + def get_sparql_value(self, **kwargs: Any) -> str: return '"' + format_amount(self.mainsnak.datavalue['value']['amount']) + '"^^xsd:decimal' def parse_sparql_value(self, value, type='literal', unit='1') -> bool: diff --git a/wikibaseintegrator/datatypes/sense.py b/wikibaseintegrator/datatypes/sense.py index a2f3abfb..91516b69 100644 --- a/wikibaseintegrator/datatypes/sense.py +++ b/wikibaseintegrator/datatypes/sense.py @@ -2,6 +2,8 @@ from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class Sense(BaseDataType): @@ -9,6 +11,7 @@ class Sense(BaseDataType): Implements the Wikibase data type 'wikibase-sense' """ DTYPE = 'wikibase-sense' + PTYPE = 'http://wikiba.se/ontology#WikibaseSense' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -44,8 +47,14 @@ def set_value(self, value: str | None = None): 'type': 'wikibase-entityid' } - def get_sparql_value(self) -> str: - return self.mainsnak.datavalue['value']['id'] + # TODO: add from_sparql_value() + + def get_sparql_value(self, **kwargs: Any) -> str | None: + if self.mainsnak.snaktype == WikibaseSnakType.KNOWN_VALUE: + wikibase_url = str(kwargs['wikibase_url'] if 'wikibase_url' in kwargs else config['WIKIBASE_URL']) + return f'<{wikibase_url}/entity/' + self.mainsnak.datavalue['value']['id'] + '>' + + return None def get_lexeme_id(self) -> str: """ diff --git a/wikibaseintegrator/datatypes/string.py b/wikibaseintegrator/datatypes/string.py index 69b98ee9..e8971911 100644 --- a/wikibaseintegrator/datatypes/string.py +++ b/wikibaseintegrator/datatypes/string.py @@ -1,14 +1,17 @@ +from __future__ import annotations + from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_enums import WikibaseSnakType class String(BaseDataType): """ Implements the Wikibase data type 'string' """ - DTYPE = 'string' + PTYPE = 'http://wikiba.se/ontology#String' def __init__(self, value: str | None = None, **kwargs: Any): """ @@ -31,3 +34,23 @@ def set_value(self, value: str | None = None): 'value': value, 'type': 'string' } + + def from_sparql_value(self, sparql_value: dict) -> String: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of type and value + :return: + """ + type = sparql_value['type'] + value = sparql_value['value'] + + if type != 'literal': + raise ValueError(f"Wrong SPARQL type {type}") + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + self.set_value(value=value) + + return self diff --git a/wikibaseintegrator/datatypes/tabulardata.py b/wikibaseintegrator/datatypes/tabulardata.py index 636ccfa0..ef32043c 100644 --- a/wikibaseintegrator/datatypes/tabulardata.py +++ b/wikibaseintegrator/datatypes/tabulardata.py @@ -9,6 +9,7 @@ class TabularData(BaseDataType): Implements the Wikibase data type 'tabular-data' """ DTYPE = 'tabular-data' + PTYPE = 'http://wikiba.se/ontology#TabularData' def __init__(self, value: str | None = None, **kwargs: Any): """ @@ -34,3 +35,7 @@ def set_value(self, value: str | None = None): 'value': value, 'type': 'string' } + + # TODO: Does TabularData need a full URL to wikimedia commons? + def get_sparql_value(self, **kwargs: Any) -> str: + return '<' + self.mainsnak.datavalue['value'] + '>' diff --git a/wikibaseintegrator/datatypes/time.py b/wikibaseintegrator/datatypes/time.py index 1276d767..be51ea0d 100644 --- a/wikibaseintegrator/datatypes/time.py +++ b/wikibaseintegrator/datatypes/time.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datetime import re from functools import total_ordering @@ -5,7 +7,7 @@ from wikibaseintegrator.datatypes.basedatatype import BaseDataType from wikibaseintegrator.wbi_config import config -from wikibaseintegrator.wbi_enums import WikibaseTimePrecision +from wikibaseintegrator.wbi_enums import WikibaseSnakType, WikibaseTimePrecision @total_ordering @@ -14,6 +16,7 @@ class Time(BaseDataType): Implements the Wikibase data type with date and time values """ DTYPE = 'time' + PTYPE = 'http://wikiba.se/ontology#Time' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -102,8 +105,8 @@ def set_value(self, time: str | None = None, before: int = 0, after: int = 0, pr 'type': 'time' } - def get_sparql_value(self) -> str: - return self.mainsnak.datavalue['value']['time'] + def get_sparql_value(self, **kwargs: Any) -> str: + return '"' + self.mainsnak.datavalue['value']['time'] + '"^^xsd:dateTime' def _time_parts(self) -> tuple[int, int, int]: """ @@ -128,3 +131,27 @@ def get_day(self) -> int: def __lt__(self, other): return self._time_parts() < other._time_parts() + + def from_sparql_value(self, sparql_value: dict) -> Time: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of type and value + :return: + """ + datatype = sparql_value['datatype'] + type = sparql_value['type'] + value = sparql_value['value'] + + if datatype != 'http://www.w3.org/2001/XMLSchema#dateTime': + raise ValueError('Wrong SPARQL datatype') + + if type != 'literal': + raise ValueError('Wrong SPARQL type') + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + self.set_value(time=value) + + return self diff --git a/wikibaseintegrator/datatypes/url.py b/wikibaseintegrator/datatypes/url.py index 3b336085..8b49d258 100644 --- a/wikibaseintegrator/datatypes/url.py +++ b/wikibaseintegrator/datatypes/url.py @@ -1,7 +1,10 @@ +from __future__ import annotations + import re from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_enums import WikibaseSnakType class URL(BaseDataType): @@ -9,6 +12,7 @@ class URL(BaseDataType): Implements the Wikibase data type for URL strings """ DTYPE = 'url' + PTYPE = 'http://wikiba.se/ontology#Url' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -41,7 +45,26 @@ def set_value(self, value: str | None = None): 'type': 'string' } - def get_sparql_value(self) -> str: + def from_sparql_value(self, sparql_value: dict) -> URL: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of type and value + :return: + """ + type = sparql_value['type'] + value = sparql_value['value'] + + if type != 'uri': + raise ValueError(f"Wrong SPARQL type {type}") + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + self.set_value(value=value) + return self + + def get_sparql_value(self, **kwargs: Any) -> str: return '<' + self.mainsnak.datavalue['value'] + '>' def parse_sparql_value(self, value, type='literal', unit='1') -> bool: diff --git a/wikibaseintegrator/entities/baseentity.py b/wikibaseintegrator/entities/baseentity.py index 08ca9f0a..79575ab1 100644 --- a/wikibaseintegrator/entities/baseentity.py +++ b/wikibaseintegrator/entities/baseentity.py @@ -107,9 +107,13 @@ def claims(self) -> Claims: return self.__claims @claims.setter - def claims(self, value: Claims): - if not isinstance(value, Claims): + def claims(self, value: Claim | Claims): + if not isinstance(value, Claims) and not isinstance(value, Claim): raise TypeError + + if isinstance(value, Claim): + value = Claims().add(claims=value) + self.__claims = value def add_claims(self, claims: Claim | list[Claim] | Claims, action_if_exists: ActionIfExists = ActionIfExists.APPEND_OR_REPLACE) -> BaseEntity: @@ -207,6 +211,21 @@ def clear(self, **kwargs: Any) -> dict[str, Any]: """ return self._write(data={}, clear=True, **kwargs) + def get_claims(self, property: str, login: _Login | None = None, allow_anonymous: bool = True, is_bot: bool | None = None, **kwargs: Any): + params = { + 'action': 'wbgetclaims', + 'entity': self.id, + 'property': property, + 'format': 'json' + } + + login = login or self.api.login + is_bot = is_bot if is_bot is not None else self.api.is_bot + + json_data = mediawiki_api_call_helper(data=params, login=login, allow_anonymous=allow_anonymous, is_bot=is_bot, **kwargs) + self.claims.from_json(json_data['claims']) + return self + def _write(self, data: dict | None = None, summary: str | None = None, login: _Login | None = None, allow_anonymous: bool = False, limit_claims: list[str | int] | None = None, clear: bool = False, as_new: bool = False, is_bot: bool | None = None, fields_to_update: list | None | EntityField = None, **kwargs: Any) -> dict[str, Any]: """ @@ -323,13 +342,9 @@ def delete(self, login: _Login | None = None, allow_anonymous: bool = False, is_ return delete_page(title=None, pageid=self.pageid, login=login, allow_anonymous=allow_anonymous, is_bot=is_bot, **kwargs) - def write_required(self, base_filter: list[BaseDataType | list[BaseDataType]] | None = None, action_if_exists: ActionIfExists = ActionIfExists.REPLACE_ALL, - **kwargs: Any) -> bool: + def write_required(self, base_filter: list[BaseDataType | list[BaseDataType]], action_if_exists: ActionIfExists = ActionIfExists.REPLACE_ALL, **kwargs: Any) -> bool: fastrun_container = wbi_fastrun.get_fastrun_container(base_filter=base_filter, **kwargs) - if base_filter is None: - base_filter = [] - # Collect the property numbers targeted by the base_filter. It supports both the simple form (a # BaseDataType) and the property-path form (a list of two BaseDataType), whose anchor is the first property. base_filter_props = set() @@ -339,14 +354,16 @@ def write_required(self, base_filter: list[BaseDataType | list[BaseDataType]] | elif isinstance(prop, list) and prop and isinstance(prop[0], BaseDataType): base_filter_props.add(prop[0].mainsnak.property_number) - claims_to_check = [] + pfilter: set = set() for claim in self.claims: if claim.mainsnak.property_number in base_filter_props: - claims_to_check.append(claim) + pfilter.add(claim.mainsnak.property_number) + + property_filter: list[str] = list(pfilter) # TODO: Add check_language_data - return fastrun_container.write_required(data=claims_to_check, cqid=self.id, action_if_exists=action_if_exists) + return fastrun_container.write_required(claims=self.claims, property_filter=property_filter) def get_entity_url(self, wikibase_url: str | None = None) -> str: from wikibaseintegrator.wbi_config import config diff --git a/wikibaseintegrator/entities/item.py b/wikibaseintegrator/entities/item.py index 2d133adb..8c3e619f 100644 --- a/wikibaseintegrator/entities/item.py +++ b/wikibaseintegrator/entities/item.py @@ -116,7 +116,7 @@ def from_json(self, json_data: dict[str, Any]) -> ItemEntity: def write(self, **kwargs: Any) -> ItemEntity: """ Write the ItemEntity data to the Wikibase instance and return the ItemEntity object returned by the instance. - extend :func:`~wikibaseintegrator.entities.BaseEntity._write` + This function extend :func:`~wikibaseintegrator.entities.baseentity.BaseEntity._write` :param data: The serialized object that is used as the data source. A newly created entity will be assigned an 'id'. :param summary: A summary of the edit diff --git a/wikibaseintegrator/models/claims.py b/wikibaseintegrator/models/claims.py index afff32d3..6c86f14e 100644 --- a/wikibaseintegrator/models/claims.py +++ b/wikibaseintegrator/models/claims.py @@ -164,9 +164,17 @@ def __len__(self): class Claim(BaseModel): + """ + extend :func:`wikibaseintegrator.models.basemodel.BaseModel` + + :param qualifiers: + :param id: + :param rank: + :param references: A References object, a list of Claim object or a list of list of Claim object + """ DTYPE = 'claim' - def __init__(self, qualifiers: Qualifiers | None = None, rank: WikibaseRank | None = None, references: References | list[Claim | list[Claim]] | None = None, + def __init__(self, qualifiers: Qualifiers | None = None, id: str | None = None, rank: WikibaseRank | None = None, references: References | list[Claim | list[Claim]] | None = None, snaktype: WikibaseSnakType = WikibaseSnakType.KNOWN_VALUE) -> None: """ @@ -179,7 +187,7 @@ def __init__(self, qualifiers: Qualifiers | None = None, rank: WikibaseRank | No self.type = 'statement' self.qualifiers = qualifiers or Qualifiers() self.qualifiers_order = [] - self.id = None + self.id = id self.rank = rank or WikibaseRank.NORMAL self.removed = False @@ -437,5 +445,5 @@ def ref_equal(oldref: References, newref: References) -> bool: return any(any(ref_equal(oldref, newref) for oldref in oldrefs) for newref in newrefs) @abstractmethod - def get_sparql_value(self) -> str: + def get_sparql_value(self, **kwargs: Any) -> str | None: pass diff --git a/wikibaseintegrator/models/qualifiers.py b/wikibaseintegrator/models/qualifiers.py index 892ec5aa..56116a59 100644 --- a/wikibaseintegrator/models/qualifiers.py +++ b/wikibaseintegrator/models/qualifiers.py @@ -12,7 +12,7 @@ class Qualifiers(BaseModel): def __init__(self) -> None: - self.qualifiers: dict[str, list[Snak]] = {} + self.qualifiers: dict[str, list[Snak | Claim]] = {} @property def qualifiers(self): diff --git a/wikibaseintegrator/models/snaks.py b/wikibaseintegrator/models/snaks.py index dc6f63bb..8eef0815 100644 --- a/wikibaseintegrator/models/snaks.py +++ b/wikibaseintegrator/models/snaks.py @@ -46,6 +46,9 @@ def get_json(self) -> dict[str, list]: json_data[property].append(snak.get_json()) return json_data + def __eq__(self, other): + return self.snaks == other.snaks + def __iter__(self): iterate = [] for snak in self.snaks.values(): diff --git a/wikibaseintegrator/wbi_config.py b/wikibaseintegrator/wbi_config.py index a2633023..608dde53 100644 --- a/wikibaseintegrator/wbi_config.py +++ b/wikibaseintegrator/wbi_config.py @@ -32,5 +32,6 @@ 'SPARQL_ENDPOINT_URL': 'https://query.wikidata.org/sparql', 'WIKIBASE_URL': 'http://www.wikidata.org', 'DEFAULT_LANGUAGE': 'en', - 'DEFAULT_LEXEME_LANGUAGE': 'Q1860' + 'DEFAULT_LEXEME_LANGUAGE': 'Q1860', + 'SPARQL_QUERY_LIMIT': 10000 } diff --git a/wikibaseintegrator/wbi_fastrun.py b/wikibaseintegrator/wbi_fastrun.py index bdeed124..45d8a6f4 100644 --- a/wikibaseintegrator/wbi_fastrun.py +++ b/wikibaseintegrator/wbi_fastrun.py @@ -1,20 +1,13 @@ from __future__ import annotations -import collections -import copy import logging -from collections import defaultdict -from itertools import chain -from typing import TYPE_CHECKING +import re from wikibaseintegrator.datatypes import BaseDataType -from wikibaseintegrator.models import Claim +from wikibaseintegrator.models import Claim, Claims, Qualifiers, Reference, References from wikibaseintegrator.wbi_config import config -from wikibaseintegrator.wbi_enums import ActionIfExists, WikibaseDatatype -from wikibaseintegrator.wbi_helpers import execute_sparql_query, format_amount - -if TYPE_CHECKING: - from wikibaseintegrator.models import Claims +from wikibaseintegrator.wbi_enums import WikibaseRank +from wikibaseintegrator.wbi_helpers import execute_sparql_query log = logging.getLogger(__name__) @@ -22,644 +15,609 @@ class FastRunContainer: - def __init__(self, base_data_type: type[BaseDataType], mediawiki_api_url: str | None = None, sparql_endpoint_url: str | None = None, wikibase_url: str | None = None, - base_filter: list[BaseDataType | list[BaseDataType]] | None = None, use_refs: bool = False, case_insensitive: bool = False): - self.reconstructed_statements: list[BaseDataType] = [] - self.rev_lookup: defaultdict[str, set[str]] = defaultdict(set) - self.rev_lookup_ci: defaultdict[str, set[str]] = defaultdict(set) - self.prop_data: dict[str, dict] = {} - self.loaded_langs: dict[str, dict] = {} - self.base_filter: list[BaseDataType | list[BaseDataType]] = [] - self.base_filter_string = '' - self.prop_dt_map: dict[str, str | None] = {} - - self.base_data_type: type[BaseDataType] = base_data_type - self.mediawiki_api_url: str = str(mediawiki_api_url or config['MEDIAWIKI_API_URL']) - self.sparql_endpoint_url: str = str(sparql_endpoint_url or config['SPARQL_ENDPOINT_URL']) - self.wikibase_url: str = str(wikibase_url or config['WIKIBASE_URL']) - self.use_refs: bool = use_refs - self.case_insensitive: bool = case_insensitive - - if base_filter and any(base_filter): - self.base_filter = base_filter + """ + + :param base_filter: The default filter to initialize the dataset. A list made of BaseDataType or list of BaseDataType. + :param base_data_type: The default data type to create objects. + :param use_qualifiers: Use qualifiers during fastrun. Enabled by default. + :param use_references: Use references during fastrun. Disabled by default. + :param use_rank: Use rank during fastrun. Disabled by default. + :param cache: Put data returned by WDQS in cache. Enabled by default. + :param case_insensitive: + :param sparql_endpoint_url: SPARLQ endpoint URL. + :param wikibase_url: Wikibase URL used for the concept URI. + """ + + # TODO: Add support for case_insensitive + + data: dict[str, dict[str, list[dict[str, str]]]] + + def __init__(self, base_filter: list[BaseDataType | list[BaseDataType]], base_data_type: type[BaseDataType] | None = None, use_qualifiers: bool = True, + use_references: bool = False, use_rank: bool = False, cache: bool = True, case_insensitive: bool = False, sparql_endpoint_url: str | None = None, + wikibase_url: str | None = None): + + for k in base_filter: + if not isinstance(k, BaseDataType) and not (isinstance(k, list) and len(k) == 2 and isinstance(k[0], BaseDataType) and isinstance(k[1], BaseDataType)): + raise ValueError("base_filter must be an instance of BaseDataType or a list of instances of BaseDataType") + + self.data: dict[str, dict[str, list[dict[str, str]]]] = {} + + self.base_filter = base_filter + self.base_data_type = base_data_type or BaseDataType + self.sparql_endpoint_url = str(sparql_endpoint_url or config['SPARQL_ENDPOINT_URL']) + self.wikibase_url = str(wikibase_url or config['WIKIBASE_URL']) + self.use_qualifiers = use_qualifiers + self.use_references = use_references + self.use_rank = use_rank + self.cache = cache + self.case_insensitive = case_insensitive + self.properties_type: dict[str, str] = {} + + if self.case_insensitive: + raise ValueError("Case insensitive does not work for the moment.") + + def load_statements(self, claims: list[Claim] | Claims | Claim, cache: bool | None = None, wb_url: str | None = None, limit: int | None = None) -> None: + """ + Load the statements related to the given claims into the internal cache of the current object. + + :param claims: A Claim, Claims or list of Claim + :param wb_url: The first part of the concept URI of entities. + :param limit: The limit to request at one time. + :param cache: Put data returned by WDQS in cache. Enabled by default. + :return: + """ + if isinstance(claims, Claim): + claims = [claims] + elif (not isinstance(claims, list) or not all(isinstance(n, Claim) for n in claims)) and not isinstance(claims, Claims): + raise ValueError("claims must be an instance of Claim or Claims or a list of Claim") + + if cache is None: + cache = self.cache + + wb_url = wb_url or self.wikibase_url + + limit = limit or int(config['SPARQL_QUERY_LIMIT']) # type: ignore + + for claim in claims: + prop_nr = claim.mainsnak.property_number + + # Load each property from the Wikibase instance or the cache + if cache and prop_nr in self.data: + logging.debug("Property '%s' found in cache, %s elements", prop_nr, len(self.data[prop_nr])) + continue + + offset = 0 + + # Generate base filter + base_filter_string = '' for k in self.base_filter: if isinstance(k, BaseDataType): + # TODO: Add multiple values for a property (OR-operation) (with the VALUES tag?) if k.mainsnak.datavalue: - self.base_filter_string += '?item <{wb_url}/prop/direct/{prop_nr}> {entity} .\n'.format( - wb_url=self.wikibase_url, prop_nr=k.mainsnak.property_number, entity=k.get_sparql_value().format(wb_url=self.wikibase_url)) - else: - self.base_filter_string += '?item <{wb_url}/prop/direct/{prop_nr}> ?zz{prop_nr} .\n'.format( - wb_url=self.wikibase_url, prop_nr=k.mainsnak.property_number) + base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr}> {entity} .\n'.format( + wb_url=wb_url, prop_nr=k.mainsnak.property_number, entity=k.get_sparql_value(wikibase_url=wb_url)) + elif sum(map(lambda x, other=k: x.mainsnak.property_number == other.mainsnak.property_number, self.base_filter)) == 1: # type: ignore + base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr}> ?zz{prop_nr} .\n'.format( + wb_url=wb_url, prop_nr=k.mainsnak.property_number) elif isinstance(k, list) and len(k) == 2 and isinstance(k[0], BaseDataType) and isinstance(k[1], BaseDataType): if k[0].mainsnak.datavalue: - self.base_filter_string += '?item <{wb_url}/prop/direct/{prop_nr}>/<{wb_url}/prop/direct/{prop_nr2}>* {entity} .\n'.format( - wb_url=self.wikibase_url, prop_nr=k[0].mainsnak.property_number, prop_nr2=k[1].mainsnak.property_number, - entity=k[0].get_sparql_value().format(wb_url=self.wikibase_url)) + base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr}>/<{wb_url}/prop/direct/{prop_nr2}>* {entity} .\n'.format( + wb_url=wb_url, prop_nr=k[0].mainsnak.property_number, prop_nr2=k[1].mainsnak.property_number, + entity=k[0].get_sparql_value(wikibase_url=wb_url)) + # TODO: Remove ?zzPYY if another filter have the same property number, the same as above else: - self.base_filter_string += '?item <{wb_url}/prop/direct/{prop_nr1}>/<{wb_url}/prop/direct/{prop_nr2}>* ?zz{prop_nr1}{prop_nr2} .\n'.format( - wb_url=self.wikibase_url, prop_nr1=k[0].mainsnak.property_number, prop_nr2=k[1].mainsnak.property_number) + base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr1}>/<{wb_url}/prop/direct/{prop_nr2}>* ?zz{prop_nr1}{prop_nr2} .\n'.format( + wb_url=wb_url, prop_nr1=k[0].mainsnak.property_number, prop_nr2=k[1].mainsnak.property_number) else: raise ValueError("base_filter must be an instance of BaseDataType or a list of instances of BaseDataType") - def reconstruct_statements(self, qid: str) -> list[BaseDataType]: - reconstructed_statements: list[BaseDataType] = [] - - if qid not in self.prop_data: - self.reconstructed_statements = reconstructed_statements - return reconstructed_statements - - for prop_nr, dt in self.prop_data[qid].items(): - # get datatypes for qualifier props - q_props = set(chain(*([x[0] for x in d['qual']] for d in dt.values()))) - r_props = set(chain(*(set(chain(*([y[0] for y in x] for x in d['ref'].values()))) for d in dt.values()))) - props = q_props | r_props - for prop in props: - if prop not in self.prop_dt_map: - self.prop_dt_map.update({prop: self.get_prop_datatype(prop)}) - # reconstruct statements from frc (including unit, qualifiers, and refs) - for _, d in dt.items(): - qualifiers = [] - for q in d['qual']: - f = [x for x in self.base_data_type.subclasses if x.DTYPE == self.prop_dt_map[q[0]]][0] - # TODO: Add support for more data type (Time, MonolingualText, GlobeCoordinate) - if self.prop_dt_map[q[0]] == 'quantity': - qualifiers.append(f(value=q[1], prop_nr=q[0], unit=q[2])) - else: - qualifiers.append(f(value=q[1], prop_nr=q[0])) - - references = [] - for _, refs in d['ref'].items(): - this_ref = [] - for ref in refs: - f = [x for x in self.base_data_type.subclasses if x.DTYPE == self.prop_dt_map[ref[0]]][0] - this_ref.append(f(value=ref[1], prop_nr=ref[0])) - references.append(this_ref) - - f = [x for x in self.base_data_type.subclasses if x.DTYPE == self.prop_dt_map[prop_nr]][0] - # TODO: Add support for more data type - if self.prop_dt_map[prop_nr] == 'quantity': - datatype = f(prop_nr=prop_nr, qualifiers=qualifiers, references=references, unit=d['unit']) - datatype.parse_sparql_value(value=d['v'], unit=d['unit']) + qualifiers_filter_string = '' + if self.use_qualifiers: + for qualifier in claim.qualifiers: + fake_json = { + 'mainsnak': qualifier.get_json(), + 'type': qualifier.datatype, + 'id': 'Q0', + 'rank': 'normal' + } + f = [x for x in self.base_data_type.subclasses if x.DTYPE == qualifier.datatype][0]().from_json(json_data=fake_json) + qualifiers_filter_string += f'?sid pq:{qualifier.property_number} {f.get_sparql_value()}.\n' + + # We force a refresh of the data, remove the previous results + self.data[prop_nr] = {} + + while True: + if claim.mainsnak.datavalue and not cache: + query = ''' + #Tool: WikibaseIntegrator wbi_fastrun.load_statements + SELECT ?entity ?sid ?value ?property_type WHERE {{ + # Base filter string + {base_filter_string} + ?entity <{wb_url}/prop/{prop_nr}> ?sid. + <{wb_url}/entity/{prop_nr}> wikibase:propertyType ?property_type. + ?sid <{wb_url}/prop/statement/{prop_nr}> ?value. + ?sid <{wb_url}/prop/statement/{prop_nr}> {value}. + {qualifiers_filter_string} + }} + ORDER BY ?sid + OFFSET {offset} + LIMIT {limit} + ''' + + # Format the query + query = query.format(base_filter_string=base_filter_string, wb_url=wb_url, prop_nr=prop_nr, offset=str(offset), limit=str(limit), + value=claim.get_sparql_value(wikibase_url=wb_url), qualifiers_filter_string=qualifiers_filter_string) else: - datatype = f(prop_nr=prop_nr, qualifiers=qualifiers, references=references) - datatype.parse_sparql_value(value=d['v']) - reconstructed_statements.append(datatype) + query = ''' + #Tool: WikibaseIntegrator wbi_fastrun.load_statements + SELECT ?entity ?sid ?value ?property_type WHERE {{ + # Base filter string + {base_filter_string} + ?entity <{wb_url}/prop/{prop_nr}> ?sid. + <{wb_url}/entity/{prop_nr}> wikibase:propertyType ?property_type. + ?sid <{wb_url}/prop/statement/{prop_nr}> ?value. + {qualifiers_filter_string} + }} + ORDER BY ?sid + OFFSET {offset} + LIMIT {limit} + ''' + + # Format the query + # TODO: Add custom query support + query = query.format(base_filter_string=base_filter_string, wb_url=wb_url, prop_nr=prop_nr, offset=str(offset), limit=str(limit), + qualifiers_filter_string=qualifiers_filter_string) + + offset += limit # We increase the offset for the next iteration + results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] + + for result in results: + entity = result['entity']['value'] + sid = result['sid']['value'] + # value = result['value']['value'] + property_type = result['property_type']['value'] + + # Use casefold for lower case + if self.case_insensitive: + result['value']['value'] = result['value']['value'].casefold() + + f = [x for x in self.base_data_type.subclasses if x.PTYPE == property_type][0]().from_sparql_value(sparql_value=result['value']) + + sparql_value = f.get_sparql_value() + if sparql_value is not None: + if sparql_value not in self.data[prop_nr]: + self.data[prop_nr][sparql_value] = [] - # this isn't used. done for debugging purposes - self.reconstructed_statements = reconstructed_statements - return reconstructed_statements + if prop_nr not in self.properties_type: + self.properties_type[prop_nr] = property_type - def get_items(self, claims: list[Claim] | Claims | Claim, cqid: str | None = None) -> set[str] | None: + self.data[prop_nr][sparql_value].append({'entity': entity, 'sid': sid}) + + if len(results) == 0 or len(results) < limit: + break + + def _load_qualifiers(self, sid: str, limit: int | None = None) -> Qualifiers: """ - Get items ID from a SPARQL endpoint + Load the qualifiers of a statement. - :param claims: A list of claims the entities should have - :param cqid: - :return: a list of entity ID or None - :exception: if there is more than one claim + :param sid: A statement ID. + :param limit: The limit to request at one time. + :return: A Qualifiers object. """ - match_sets = [] + offset = 0 - if isinstance(claims, Claim): - claims = [claims] - elif (not isinstance(claims, list) or not all(isinstance(n, Claim) for n in claims)) and not isinstance(claims, Claims): - raise ValueError("claims must be an instance of Claim or Claims or a list of Claim") + if not isinstance(sid, str): + raise ValueError('sid must be a string') - for claim in claims: - # skip to next if statement has no value or no data type defined, e.g. for deletion objects - if not claim.mainsnak.datavalue and not claim.mainsnak.datatype: - continue + limit = limit or int(config['SPARQL_QUERY_LIMIT']) # type: ignore - prop_nr = claim.mainsnak.property_number + # TODO: Add cache - if prop_nr not in self.prop_dt_map: - log.debug("%s not found in fastrun", prop_nr) + # We force a refresh of the data, remove the previous results + qualifiers: Qualifiers = Qualifiers() + while True: + query = f''' + #Tool: WikibaseIntegrator wbi_fastrun._load_qualifiers + SELECT ?property ?value ?property_type WHERE {{ + VALUES ?sid {{ <{sid}> }} + ?sid ?predicate ?value. + ?property wikibase:qualifier ?predicate. + ?property wikibase:propertyType ?property_type. + }} + ORDER BY ?sid + OFFSET {offset} + LIMIT {limit} + ''' - if isinstance(claim, BaseDataType) and type(claim) != BaseDataType: # pylint: disable=unidiomatic-typecheck - self.prop_dt_map.update({prop_nr: claim.DTYPE}) - else: - self.prop_dt_map.update({prop_nr: self.get_prop_datatype(prop_nr)}) - self._query_data(prop_nr=prop_nr, use_units=self.prop_dt_map[prop_nr] == 'quantity') - - # noinspection PyProtectedMember - current_value = claim.get_sparql_value() - - if self.prop_dt_map[prop_nr] == 'wikibase-item': - current_value = claim.mainsnak.datavalue['value']['id'] - - log.debug(current_value) - # if self.case_insensitive: - # log.debug("case insensitive enabled") - # log.debug(self.rev_lookup_ci) - # else: - # log.debug(self.rev_lookup) - - if current_value in self.rev_lookup: - # quick check for if the value has ever been seen before, if not, write required - match_sets.append(set(self.rev_lookup[current_value])) - elif self.case_insensitive and current_value.casefold() in self.rev_lookup_ci: - match_sets.append(set(self.rev_lookup_ci[current_value.casefold()])) - else: - log.debug("no matches for rev lookup for %s", current_value) - - if not match_sets: - return None - - if cqid: - matching_qids = {cqid} - else: - matching_qids = match_sets[0].intersection(*match_sets[1:]) + # Format the query + # query = query.format(wb_url=wb_url, sid=sid, offset=str(offset), limit=str(limit)) + offset += limit # We increase the offset for the next iteration + results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] + + for result in results: + property = result['property']['value'] + property_type = result['property_type']['value'] + + if property not in self.properties_type: + self.properties_type[property] = property_type + + # Use casefold for lower case + if self.case_insensitive: + result['value']['value'] = result['value']['value'].casefold() + + f = [x for x in self.base_data_type.subclasses if x.PTYPE == property_type][0](prop_nr=property).from_sparql_value(sparql_value=result['value']) + qualifiers.add(f) + + if len(results) == 0 or len(results) < limit: + break - return matching_qids + return qualifiers - def get_item(self, claims: list[Claim] | Claims | Claim, cqid: str | None = None) -> str | None: + def _load_references(self, sid: str, limit: int = 10000) -> References: """ + Load the references of a statement. - :param claims: A list of claims the entity should have - :param cqid: - :return: An entity ID, None if there is more than one. + :param sid: A statement ID. + :param limit: The limit to request at one time. + :return: A References object. """ + offset = 0 - matching_qids: set[str] | None = self.get_items(claims=claims, cqid=cqid) + if not isinstance(sid, str): + raise ValueError('sid must be a string') - if matching_qids is None: - return None + limit = limit or int(config['SPARQL_QUERY_LIMIT']) # type: ignore - # check if there are any items that have all of these values - # if not, a write is required no matter what - if not len(matching_qids) == 1: - log.debug("no matches (%s)", len(matching_qids)) - return None + # TODO: Add cache - return matching_qids.pop() + # We force a refresh of the data, remove the previous results + references: References = References() + while True: + query = f''' + #Tool: WikibaseIntegrator wbi_fastrun._load_references + SELECT ?srid ?ref_property ?ref_value ?property_type WHERE {{ + VALUES ?sid {{ <{sid}> }} + + ?sid prov:wasDerivedFrom ?srid. + ?srid ?ref_predicate ?ref_value. + ?ref_property wikibase:reference ?ref_predicate. + ?ref_property wikibase:propertyType ?property_type. + }} + ORDER BY ?srid + OFFSET {offset} + LIMIT {limit} + ''' - def write_required(self, data: list[Claim], action_if_exists: ActionIfExists = ActionIfExists.REPLACE_ALL, cqid: str | None = None) -> bool: - """ - Check if a write is required + # Format the query + # query = query.format(wb_url=wb_url, sid=sid, offset=str(offset), limit=str(limit)) + offset += limit # We increase the offset for the next iteration + results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] - :param data: - :param action_if_exists: - :param cqid: - :return: Return True if the write is required - """ - del_props = set() - data_props = set() - append_props = [] - if action_if_exists == ActionIfExists.APPEND_OR_REPLACE: - append_props = [x.mainsnak.property_number for x in data] - - for x in data: - if x.mainsnak.datavalue and x.mainsnak.datatype: - data_props.add(x.mainsnak.property_number) - qid = self.get_item(data, cqid) - - if not qid: - return True + reference = {} - reconstructed_statements = self.reconstruct_statements(qid) - tmp_rs = copy.deepcopy(reconstructed_statements) - - # handle append properties - for p in append_props: - app_data = [x for x in data if x.mainsnak.property_number == p] # new statements - rec_app_data = [x for x in tmp_rs if x.mainsnak.property_number == p] # orig statements - comp = [] - for x in app_data: - for y in rec_app_data: - if x.mainsnak.datavalue == y.mainsnak.datavalue: - if y.equals(x, include_ref=self.use_refs) and action_if_exists != ActionIfExists.FORCE_APPEND: - comp.append(True) - - # comp = [True for x in app_data for y in rec_app_data if x.equals(y, include_ref=self.use_refs)] - if len(comp) != len(app_data): - log.debug("failed append: %s", p) - return True - - tmp_rs = [x for x in tmp_rs if x.mainsnak.property_number not in append_props and x.mainsnak.property_number in data_props] - - for date in data: - # ensure that statements meant for deletion get handled properly - reconst_props = {x.mainsnak.property_number for x in tmp_rs} - if not date.mainsnak.datatype and date.mainsnak.property_number in reconst_props: - log.debug("returned from delete prop handling") - return True - - if not date.mainsnak.datavalue or not date.mainsnak.datatype: - # Ignore the deletion statements which are not in the reconstructed statements. - continue + for result in results: + ref_property = result['ref_property']['value'] + srid = result['srid']['value'] + property_type = result['property_type']['value'] - if date.mainsnak.property_number in append_props: - # TODO: check if value already exist and already have the same value - continue + if ref_property not in self.properties_type: + self.properties_type[ref_property] = property_type - if not date.mainsnak.datavalue and not date.mainsnak.datatype: - del_props.add(date.mainsnak.property_number) + # Use casefold for lower case + if self.case_insensitive: + result['value']['value'] = result['value']['value'].casefold() - # this is where the magic happens - # date is a new statement, proposed to be written - # tmp_rs are the reconstructed statements == current state of the item - bool_vec = [] - for x in tmp_rs: - if (x == date or (self.case_insensitive and x.mainsnak.datavalue.casefold() == date.mainsnak.datavalue.casefold())) and x.mainsnak.property_number not in del_props: - bool_vec.append(x.equals(date, include_ref=self.use_refs)) - else: - bool_vec.append(False) - # bool_vec = [x.equals(date, include_ref=self.use_refs, fref=self.ref_comparison_f) and - # x.mainsnak.property_number not in del_props for x in tmp_rs] - - log.debug("bool_vec: %s", bool_vec) - log.debug("-----------------------------------") - for x in tmp_rs: - if x == date and x.mainsnak.property_number not in del_props: - log.debug([x.mainsnak.property_number, x.mainsnak.datavalue, [z.datavalue for z in x.qualifiers]]) - log.debug([date.mainsnak.property_number, date.mainsnak.datavalue, [z.datavalue for z in date.qualifiers]]) - elif x.mainsnak.property_number == date.mainsnak.property_number: - log.debug([x.mainsnak.property_number, x.mainsnak.datavalue, [z.datavalue for z in x.qualifiers]]) - log.debug([date.mainsnak.property_number, date.mainsnak.datavalue, [z.datavalue for z in date.qualifiers]]) - - if not any(bool_vec): - log.debug(len(bool_vec)) - log.debug("fast run failed at %s", date.mainsnak.property_number) - return True - - log.debug("fast run success") - tmp_rs.pop(bool_vec.index(True)) - - if len(tmp_rs) > 0: - log.debug("failed because not zero") - for x in tmp_rs: - log.debug([x.mainsnak.property_number, x.mainsnak.datavalue, [z.mainsnak.datavalue for z in x.qualifiers]]) - log.debug("failed because not zero--END") - return True + f = [x for x in self.base_data_type.subclasses if x.PTYPE == property_type][0](prop_nr=ref_property).from_sparql_value(sparql_value=result['ref_value']) - return False + if srid not in reference: + reference[srid] = Reference() - def init_language_data(self, lang: str, lang_data_type: str) -> None: - """ - Initialize language data store + reference[srid].add(f) - :param lang: language code - :param lang_data_type: 'label', 'description' or 'aliases' - :return: None - """ - if lang not in self.loaded_langs: - self.loaded_langs[lang] = {} + # Add each Reference to the References + for _, ref in reference.items(): + references.add(ref) - if lang_data_type not in self.loaded_langs[lang]: - result = self._query_lang(lang=lang, lang_data_type=lang_data_type) - if result is not None: - data = self._process_lang(result=result) - self.loaded_langs[lang].update({lang_data_type: data}) + if len(results) == 0 or len(results) < limit: + break + + return references - def get_language_data(self, qid: str, lang: str, lang_data_type: str) -> list[str]: + def _load_rank(self, sid: str) -> WikibaseRank | None: """ - get language data for specified qid - - :param qid: Wikibase item id - :param lang: language code - :param lang_data_type: 'label', 'description' or 'aliases' - :return: list of strings - If nothing is found: - If lang_data_type == label: returns [''] - If lang_data_type == description: returns [''] - If lang_data_type == aliases: returns [] + Load the rank of a statement. + + :param sid: A statement ID. + :param limit: The limit to request at one time. + :return: A References object. """ - self.init_language_data(lang, lang_data_type) - current_lang_data = self.loaded_langs[lang][lang_data_type] - all_lang_strings = current_lang_data.get(qid, []) - if not all_lang_strings and lang_data_type in {'label', 'description'}: - all_lang_strings = [''] - return all_lang_strings + if not isinstance(sid, str): + raise ValueError('sid must be a string') - def check_language_data(self, qid: str, lang_data: list, lang: str, lang_data_type: str, action_if_exists: ActionIfExists = ActionIfExists.APPEND_OR_REPLACE) -> bool: - """ - Method to check if certain language data exists as a label, description or aliases - :param qid: Wikibase item id - :param lang_data: list of string values to check - :param lang: language code - :param lang_data_type: What kind of data is it? 'label', 'description' or 'aliases'? - :param action_if_exists: If aliases already exist, APPEND_OR_REPLACE or REPLACE_ALL - :return: boolean - """ - all_lang_strings = {x.strip().casefold() for x in self.get_language_data(qid, lang, lang_data_type)} + # TODO: Add limit? - if action_if_exists == ActionIfExists.REPLACE_ALL: - return collections.Counter(all_lang_strings) != collections.Counter(map(lambda x: x.casefold(), lang_data)) + # TODO: Add cache - for s in lang_data: - if s.strip().casefold() not in all_lang_strings: - log.debug("fastrun failed at: %s, string: %s", lang_data_type, s) - return True + query = f''' + #Tool: WikibaseIntegrator wbi_fastrun._load_rank + SELECT ?rank WHERE {{ + VALUES ?sid {{ <{sid}> }} + ?sid wikibase:rank ?rank. + }} + ''' - return False + results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] - def get_all_data(self) -> dict[str, dict]: - return self.prop_data + for result in results: + rank_raw = result['rank']['value'].rsplit('#', 1)[-1] - def format_query_results(self, r: list, prop_nr: str) -> None: - """ - `r` is the results of the sparql query in _query_data and is modified in place - `prop_nr` is needed to get the property datatype to determine how to format the value - - `r` is a list of dicts. The keys are: - sid: statement ID - item: the subject. the item this statement is on - v: the object. The value for this statement - unit: property unit - pq: qualifier property - qval: qualifier value - qunit: qualifier unit - ref: reference ID - pr: reference property - rval: reference value + if rank_raw == 'PreferredRank': + return WikibaseRank.PREFERRED + elif rank_raw == 'NormalRank': + return WikibaseRank.NORMAL + elif rank_raw == 'DeprecatedRank': + return WikibaseRank.DEPRECATED + + return None + + def _get_property_type(self, prop_nr: str | int) -> str: """ - prop_dt = self.get_prop_datatype(prop_nr) - for i in r: - for value in ['item', 'sid', 'pq', 'pr', 'ref', 'unit', 'qunit']: - if value in i: - if i[value]['value'].startswith(self.wikibase_url): - i[value] = i[value]['value'].split('/')[-1] - else: - # TODO: Dirty fix. If we are not on wikidata, we force unitless (Q199) to '1' - if i[value]['value'] == 'http://www.wikidata.org/entity/Q199': - i[value] = '1' - else: - i[value] = i[value]['value'] - - # make sure datetimes are formatted correctly. - # the correct format is '+%Y-%m-%dT%H:%M:%SZ', but is sometimes missing the plus?? - # some difference between RDF and xsd:dateTime that I don't understand - for value in ['v', 'qval', 'rval']: - if value in i: - if i[value].get("datatype") == 'http://www.w3.org/2001/XMLSchema#dateTime' and not i[value]['value'][0] in '+-': - # if it is a dateTime and doesn't start with plus or minus, add a plus - i[value]['value'] = '+' + i[value]['value'] - - # these three ({'v', 'qval', 'rval'}) are values that can be any data type - # strip off the URI if they are wikibase-items - if 'v' in i: - if i['v']['type'] == 'uri' and prop_dt == 'wikibase-item': - i['v'] = i['v']['value'].split('/')[-1] - elif i['v']['type'] == 'literal' and prop_dt == 'quantity': - i['v'] = format_amount(i['v']['value']) - elif i['v']['type'] == 'literal' and prop_dt == 'monolingualtext': - f = [x for x in self.base_data_type.subclasses if x.DTYPE == prop_dt][0](prop_nr=prop_nr, text=i['v']['value'], language=i['v']['xml:lang']) - i['v'] = f.get_sparql_value() - else: - f = [x for x in self.base_data_type.subclasses if x.DTYPE == prop_dt][0](prop_nr=prop_nr) - if not f.parse_sparql_value(value=i['v']['value'], type=i['v']['type']): - raise ValueError("Can't parse the value with parse_sparql_value()") - i['v'] = f.get_sparql_value() - - # Note: no-value and some-value don't actually show up in the results here - # see for example: select * where { wd:Q7207 p:P40 ?c . ?c ?d ?e } - if not isinstance(i['v'], dict): - self.rev_lookup[i['v']].add(i['item']) - if self.case_insensitive: - self.rev_lookup_ci[i['v'].casefold()].add(i['item']) - - # handle qualifier value - if 'qval' in i: - qual_prop_dt = self.get_prop_datatype(prop_nr=i['pq']) - if i['qval']['type'] == 'uri' and qual_prop_dt == 'wikibase-item': - i['qval'] = i['qval']['value'].split('/')[-1] - elif i['qval']['type'] == 'literal' and qual_prop_dt == 'quantity': - i['qval'] = format_amount(i['qval']['value']) - else: - i['qval'] = i['qval']['value'] - - # handle reference value - if 'rval' in i: - ref_prop_dt = self.get_prop_datatype(prop_nr=i['pr']) - if i['rval']['type'] == 'uri' and ref_prop_dt == 'wikibase-item': - i['rval'] = i['rval']['value'].split('/')[-1] - elif i['rval']['type'] == 'literal' and ref_prop_dt == 'quantity': - i['rval'] = format_amount(i['rval']['value']) - else: - i['rval'] = i['rval']['value'] - - def update_frc_from_query(self, r: list, prop_nr: str) -> None: - # r is the output of format_query_results - # this updates the frc from the query (result of _query_data) - for i in r: - qid = i['item'] - if qid not in self.prop_data: - self.prop_data[qid] = {prop_nr: {}} - if prop_nr not in self.prop_data[qid]: - self.prop_data[qid].update({prop_nr: {}}) - if i['sid'] not in self.prop_data[qid][prop_nr]: - self.prop_data[qid][prop_nr].update({i['sid']: {}}) - # update values for this statement (not including ref) - d = {'v': i['v']} - self.prop_data[qid][prop_nr][i['sid']].update(d) - - if 'qual' not in self.prop_data[qid][prop_nr][i['sid']]: - self.prop_data[qid][prop_nr][i['sid']]['qual'] = set() - if 'pq' in i and 'qval' in i: - if 'qunit' in i: - self.prop_data[qid][prop_nr][i['sid']]['qual'].add((i['pq'], i['qval'], i['qunit'])) - else: - self.prop_data[qid][prop_nr][i['sid']]['qual'].add((i['pq'], i['qval'], '1')) + Obtain the property type of the given property by looking at the SPARQL endpoint. - if 'ref' not in self.prop_data[qid][prop_nr][i['sid']]: - self.prop_data[qid][prop_nr][i['sid']]['ref'] = {} - if 'ref' in i: - if i['ref'] not in self.prop_data[qid][prop_nr][i['sid']]['ref']: - self.prop_data[qid][prop_nr][i['sid']]['ref'][i['ref']] = set() - self.prop_data[qid][prop_nr][i['sid']]['ref'][i['ref']].add((i['pr'], i['rval'])) + :param prop_nr: The property number. + :return: The SPARQL version of the property type. + """ + if isinstance(prop_nr, int): + prop_nr = 'P' + str(prop_nr) + elif prop_nr is not None: + pattern = re.compile(r'^P?([0-9]+)$') + matches = pattern.match(prop_nr) - if 'unit' not in self.prop_data[qid][prop_nr][i['sid']]: - self.prop_data[qid][prop_nr][i['sid']]['unit'] = '1' - if 'unit' in i: - self.prop_data[qid][prop_nr][i['sid']]['unit'] = i['unit'] + if not matches: + raise ValueError('Invalid prop_nr, format must be "P[0-9]+"') - def _query_data(self, prop_nr: str, use_units: bool = False, page_size: int = 10000) -> None: - page_count = 0 + prop_nr = 'P' + str(matches.group(1)) - while True: - # Query header - query = ''' - #Tool: WikibaseIntegrator wbi_fastrun._query_data - SELECT ?sid ?item ?v ?unit ?pq ?qval ?qunit ?ref ?pr ?rval - WHERE - {{ - ''' + query = f'''#Tool: WikibaseIntegrator wbi_fastrun._get_property_type + SELECT ?property_type WHERE {{ wd:{prop_nr} wikibase:propertyType ?property_type. }}''' - # Base filter - query += ''' - {base_filter} + results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'][0]['property_type']['value'] - ?item <{wb_url}/prop/{prop_nr}> ?sid . - ''' + return results - # Amount and unit - if use_units: - query += ''' - {{ - <{wb_url}/entity/{prop_nr}> wikibase:propertyType ?property_type . - FILTER (?property_type != wikibase:Quantity) - ?sid <{wb_url}/prop/statement/{prop_nr}> ?v . - }} - # Get amount and unit for the statement - UNION - {{ - ?sid <{wb_url}/prop/statement/value/{prop_nr}> [wikibase:quantityAmount ?v; wikibase:quantityUnit ?unit] . - }} - ''' - else: - query += ''' - <{wb_url}/entity/{prop_nr}> wikibase:propertyType ?property_type . - ?sid <{wb_url}/prop/statement/{prop_nr}> ?v . - ''' - - # Qualifiers - # Amount and unit - if use_units: - query += ''' - # Get qualifiers - OPTIONAL - {{ - {{ - # Get simple values for qualifiers which are not of type quantity - ?sid ?propQualifier ?qval . - ?pq wikibase:qualifier ?propQualifier . - ?pq wikibase:propertyType ?qualifer_property_type . - FILTER (?qualifer_property_type != wikibase:Quantity) - }} - UNION - {{ - # Get amount and unit for qualifiers of type quantity - ?sid ?pqv [wikibase:quantityAmount ?qval; wikibase:quantityUnit ?qunit] . - ?pq wikibase:qualifierValue ?pqv . - }} - }} - ''' - else: - query += ''' - # Get qualifiers - OPTIONAL - {{ - # Get simple values for qualifiers - ?sid ?propQualifier ?qval . - ?pq wikibase:qualifier ?propQualifier . - ?pq wikibase:propertyType ?qualifer_property_type . - }} - ''' - - # References - if self.use_refs: - query += ''' - # get references - OPTIONAL {{ - ?sid prov:wasDerivedFrom ?ref . - ?ref ?pr ?rval . - [] wikibase:reference ?pr - }} - ''' - # Query footer - query += ''' - }} ORDER BY ?sid OFFSET {offset} LIMIT {page_size} - ''' + def get_entities(self, claims: list[Claim] | Claims | Claim, cache: bool | None = None, query_limit: int | None = None) -> list[str]: + """ + Return a list of entities who correspond to the specified claims. - # Format the query - query = query.format(wb_url=self.wikibase_url, base_filter=self.base_filter_string, prop_nr=prop_nr, offset=str(page_count * page_size), page_size=str(page_size)) + :param claims: A list of claims to query the SPARQL endpoint. + :param cache: Put data returned by WDQS in cache. Enabled by default. + :param query_limit: Limit the amount of results from the SPARQL server + :return: A list of entity ID. + """ + if isinstance(claims, Claim): + claims = [claims] + elif (not isinstance(claims, list) or not all(isinstance(n, Claim) for n in claims)) and not isinstance(claims, Claims): + raise ValueError("claims must be an instance of Claim or Claims or a list of Claim") - results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] - self.format_query_results(results, prop_nr) - self.update_frc_from_query(results, prop_nr) - page_count += 1 + self.load_statements(claims=claims, cache=cache, limit=query_limit) + + result = [] + for base_filter in self.base_filter: + sub_result = set() + if isinstance(base_filter, BaseDataType): # TODO: Manage case where filter is a list of BaseDataType + if not base_filter.mainsnak.datavalue: + for claim in claims: + if base_filter.mainsnak.property_number == claim.mainsnak.property_number: + # Add the returned entities to the result list + if claim.get_sparql_value() in self.data[claim.mainsnak.property_number]: + for rez in self.data[claim.mainsnak.property_number][claim.get_sparql_value()]: # type: ignore + sub_result.add(rez['entity'].rsplit('/', 1)[-1]) + else: + if base_filter.mainsnak.property_number in self.data: + if base_filter.get_sparql_value() in self.data[base_filter.mainsnak.property_number]: + for rez in self.data[base_filter.mainsnak.property_number][base_filter.get_sparql_value()]: # type: ignore + sub_result.add(rez['entity'].rsplit('/', 1)[-1]) + else: + continue + result.append(sub_result) - if len(results) == 0 or len(results) < page_size: - break + if result: + if len(result) > 1: + return list(set(result[0]).intersection(*result[1:])) + return list(result[0]) + else: + return [] - def _query_lang(self, lang: str, lang_data_type: str) -> list[dict[str, dict]] | None: + def write_required(self, claims: list[Claim] | Claims | Claim, entity_filter: list[str] | str | None = None, property_filter: list[str] | str | None = None, + use_qualifiers: bool | None = None, use_references: bool | None = None, use_rank: bool | None = None, cache: bool | None = None, + query_limit: int | None = None) -> bool: """ - :param lang: - :param lang_data_type: + :param claims: + :param entity_filter: Allows you to filter the entities checked. This can be a single entity or a list of entities. + :param property_filter: Allows you to limit the difference comparison to a list of properties + :param use_qualifiers: Use qualifiers during fastrun. Enabled by default. + :param use_references: Use references during fastrun. Disabled by default. + :param use_rank: Use rank during fastrun. Disabled by default. + :param cache: Put data returned by WDQS in cache. Enabled by default. + :param query_limit: Limit the amount of results from the SPARQL server + :return: a boolean True if a write is required. False otherwise. """ - lang_data_type_dict = { - 'label': 'rdfs:label', - 'description': 'schema:description', - 'aliases': 'skos:altLabel' - } + if isinstance(claims, Claim): + claims = [claims] + elif (not isinstance(claims, list) or not all(isinstance(n, Claim) for n in claims)) and not isinstance(claims, Claims): + raise ValueError("claims must be an instance of Claim or Claims or a list of Claim") - query = f''' - #Tool: WikibaseIntegrator wbi_fastrun._query_lang - SELECT ?item ?label WHERE {{ - {self.base_filter_string} + if len(claims) == 0: + raise ValueError("claims must have at least one claim") - OPTIONAL {{ - ?item {lang_data_type_dict[lang_data_type]} ?label FILTER (lang(?label) = "{lang}") . - }} - }} - ''' + if entity_filter is not None and isinstance(entity_filter, str): + entity_filter = [entity_filter] - log.debug(query) - - return execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] - - @staticmethod - def _process_lang(result: list) -> defaultdict[str, set]: - data = defaultdict(set) - for r in result: - qid = r['item']['value'].split("/")[-1] - if 'label' in r: - data[qid].add(r['label']['value']) - return data - - def get_prop_datatype(self, prop_nr: str) -> str | None: - # Memoize in the per-instance prop_dt_map: this is tied to the container's lifetime (no global cache keeping - # containers alive), avoids re-instantiating WikibaseIntegrator and re-querying the API on cache hits, and is - # invalidated by clear(). - if prop_nr not in self.prop_dt_map: - from wikibaseintegrator import WikibaseIntegrator - wbi = WikibaseIntegrator() - datatype = wbi.property.get(prop_nr).datatype - if isinstance(datatype, WikibaseDatatype): - datatype = datatype.value - self.prop_dt_map[prop_nr] = datatype - return self.prop_dt_map[prop_nr] - - def clear(self) -> None: - """ - convenience function to empty this fastrun container - """ - self.prop_dt_map = {} - self.prop_data = {} - self.rev_lookup = defaultdict(set) - self.rev_lookup_ci = defaultdict(set) + if property_filter is not None and isinstance(property_filter, str): + property_filter = [property_filter] - def __repr__(self) -> str: - """A mixin implementing a simple __repr__.""" - return "<{klass} @{id:x} {attrs}>".format( # pylint: disable=consider-using-f-string - klass=self.__class__.__name__, - id=id(self) & 0xFFFFFF, - attrs="\r\n\t ".join(f"{k}={v!r}" for k, v in self.__dict__.items()), - ) + # Generate a property_filter if None is given + if property_filter is None: + property_filter = [claim.mainsnak.property_number for claim in claims] + if use_qualifiers is None: + use_qualifiers = self.use_qualifiers + if use_references is None: + use_references = self.use_references + if use_rank is None: + use_rank = self.use_rank -def get_fastrun_container(base_filter: list[BaseDataType | list[BaseDataType]] | None = None, use_refs: bool = False, case_insensitive: bool = False) -> FastRunContainer: + def contains(in_list, lambda_filter): + for x in in_list: + if lambda_filter(x): + return True + return False + + # Get all the potential statements + statements_to_check: dict[str, list[str]] = {} + for claim in claims: + if claim.mainsnak.property_number in property_filter: + self.load_statements(claims=claim, cache=cache, limit=query_limit) + if claim.mainsnak.property_number in self.data: + if not contains(self.data[claim.mainsnak.property_number], (lambda x, c=claim: x == c.get_sparql_value())): + # Checks if a property with this value does not exist, return True if none exist + logging.debug("Value '%s' does not exist for property '%s'", claim.get_sparql_value(), claim.mainsnak.property_number) + return True + # TODO: Doesn't work in the value already exists in another entity + + sparql_value = claim.get_sparql_value() + if sparql_value: + for statement in self.data[claim.mainsnak.property_number][sparql_value]: + if claim.mainsnak.property_number not in statements_to_check: + statements_to_check[claim.mainsnak.property_number] = [] + statements_to_check[claim.mainsnak.property_number].append(statement['entity']) + + # Generate an intersection between all the statements by property, based on the entity + # Generate only the list of entities + list_entities: list[list[str]] = [] + for _, statements in statements_to_check.items(): + # entities = [statement['entity'] for statement in statements_to_check[property]] + list_entities.append(list(set(statements))) + + # Return the intersection between all the list + common_entities: list = list_entities.pop() + for entities in list_entities: + common_entities = list(set(common_entities).intersection(entities)) + + # If there is none common entities, return True because we need a write + if not common_entities: + logging.debug("There is no common entities") + return True + + # If the property is already found, load it completely to compare deeply + for claim in claims: + # Check if the property is in the filter + if claim.mainsnak.property_number in property_filter: + sparql_value = claim.get_sparql_value() + # If the value exist in the cache + if sparql_value and claim.mainsnak.property_number in self.data and sparql_value in self.data[claim.mainsnak.property_number]: + entity_cache = [statement['entity'].rsplit('/', 1)[-1] for statement in self.data[claim.mainsnak.property_number][sparql_value]] + if entity_filter: + common_cache_filter = [value for value in entity_cache if value in entity_filter] + else: + common_cache_filter = entity_cache + # If there is common entities between the cache and the entity_filter + if common_cache_filter: + for statement in self.data[claim.mainsnak.property_number][sparql_value]: + if entity_filter and statement['entity'].rsplit('/', 1)[-1] not in entity_filter: + continue + + if statement['entity'] in common_entities: + if use_qualifiers: + qualifiers = self._load_qualifiers(statement['sid'], limit=100) + + if len(qualifiers) != len(claim.qualifiers): + logging.debug("Difference in number of qualifiers, '%i' != '%i'", len(qualifiers), len(claim.qualifiers)) + return True + + for qualifier in qualifiers: + if qualifier not in claim.qualifiers: + logging.debug("Difference between two qualifiers") + return True + + if use_references: + references = self._load_references(statement['sid'], limit=100) + + if sum(len(ref) for ref in references) != sum(len(x) for x in claim.references): + logging.debug("Difference in number of references, '%i' != '%i'", sum(len(ref) for ref in references), sum(len(x) for x in claim.references)) + return True + + for reference in references: + if reference not in claim.references: + logging.debug("Difference between two references") + return True + + if use_rank: + rank = self._load_rank(statement['sid']) + + if claim.rank != rank: + logging.debug("Difference with the rank") + return True + else: + logging.debug("No common entities between cache and entity_filter") + return True + # Enable this if the value doesn't exist ? + else: + logging.debug("Value doesn't already exist in an entity") + return True + + return False + + +def get_fastrun_container(base_filter: list[BaseDataType | list[BaseDataType]], use_qualifiers: bool = True, use_references: bool = False, use_rank: bool = False, + cache: bool = True, case_insensitive: bool = False) -> FastRunContainer: + """ + Return a FastRunContainer object, create a new one if it doesn't already exist. + + :param base_filter: The default filter to initialize the dataset. A list made of BaseDataType or list of BaseDataType. + :param use_qualifiers: Use qualifiers during fastrun. Enabled by default. + :param use_references: Use references during fastrun. Disabled by default. + :param use_rank: Use rank during fastrun. Disabled by default. + :param cache: Put data returned by WDQS in cache. Enabled by default. + :param case_insensitive: + :return: a FastRunContainer object + """ if base_filter is None: base_filter = [] # We search if we already have a FastRunContainer with the same parameters to reuse it - fastrun_container = _search_fastrun_store(base_filter=base_filter, use_refs=use_refs, case_insensitive=case_insensitive) + fastrun_container = _search_fastrun_store(base_filter=base_filter, use_qualifiers=use_qualifiers, use_references=use_references, use_rank=use_rank, + case_insensitive=case_insensitive, cache=cache) return fastrun_container -def _search_fastrun_store(base_filter: list[BaseDataType | list[BaseDataType]] | None = None, use_refs: bool = False, case_insensitive: bool = False) -> FastRunContainer: +def _search_fastrun_store(base_filter: list[BaseDataType | list[BaseDataType]], use_qualifiers: bool = True, use_references: bool = False, use_rank: bool = False, + cache: bool = True, case_insensitive: bool = False) -> FastRunContainer: + """ + Search for an existing FastRunContainer with the same parameters or create a new one if it doesn't exist. + + :param base_filter: The default filter to initialize the dataset. A list made of BaseDataType or list of BaseDataType. + :param use_qualifiers: Use qualifiers during fastrun. Enabled by default. + :param use_references: Use references during fastrun. Disabled by default. + :param use_rank: Use rank during fastrun. Disabled by default. + :param cache: Put data returned by WDQS in cache. Enabled by default. + :param case_insensitive: + :return: a FastRunContainer object + """ for fastrun in fastrun_store: - if (fastrun.base_filter == base_filter) and (fastrun.use_refs == use_refs) and (fastrun.case_insensitive == case_insensitive) and ( - fastrun.sparql_endpoint_url == config['SPARQL_ENDPOINT_URL']): + if (fastrun.base_filter == base_filter) and (fastrun.use_qualifiers == use_qualifiers) and (fastrun.use_references == use_references) and ( + fastrun.use_rank == use_rank) and (fastrun.case_insensitive == case_insensitive) and (fastrun.sparql_endpoint_url == config['SPARQL_ENDPOINT_URL']): + fastrun.cache = cache return fastrun # In case nothing was found in the fastrun_store log.info("Create a new FastRunContainer") - fastrun_container = FastRunContainer(base_data_type=BaseDataType, base_filter=base_filter, use_refs=use_refs, case_insensitive=case_insensitive) + fastrun_container = FastRunContainer(base_data_type=BaseDataType, base_filter=base_filter, use_qualifiers=use_qualifiers, use_references=use_references, use_rank=use_rank, + cache=cache, case_insensitive=case_insensitive) fastrun_store.append(fastrun_container) return fastrun_container From ad2b486ae7e2c3ea20c44c71246df76e3488a37b Mon Sep 17 00:00:00 2001 From: Myst <1592048+LeMyst@users.noreply.github.com> Date: Tue, 7 Jul 2026 17:28:58 +0200 Subject: [PATCH 2/5] Complete the new FastRun implementation Features: - Restrict write_required() to the entity being edited: baseentity now passes entity_filter=self.id, so data existing on another entity no longer inhibits a required write - Support action_if_exists in write_required(); FORCE_APPEND always reports a write as required - Port the language data checking (labels, descriptions, aliases) from the old implementation: init_language_data(), get_language_data() and check_language_data(), backed by the shared base filter - Implement the case insensitive mode: string values are compared casefolded, the SPARQL data is keyed casefolded at load time - Add a clear() method and a module docstring Fixes: - write_required() crashed with an IndexError when no claim matched the property filter; it now reports a write as required - The deep comparison returned early on the first mismatching statement: an entity holding duplicate statements (one matching, one not) was wrongly reported as requiring a write. The comparison now looks for at least one entity holding a matching statement for every claim - A load restricted to a value or to qualifiers (cache disabled) was reused as a complete cache, poisoning later comparisons; partial loads are no longer marked as complete - _load_qualifiers()/_load_references() crashed when building datatypes: the full property URI was passed as prop_nr; it is now reduced to the bare property ID - _load_references() duplicated references spanning two result pages - A base filter mixing a valueless property and a property path crashed in the base filter string generation - Datatypes without from_sparql_value() support and unparseable values no longer crash the load; they are skipped with a warning and the statements are reported as requiring a write - Lazily loaded qualifiers, references and ranks are memoized per statement, honoring the cache flag Tests: rebuild test/test_wbi_fastrun.py on the offline MockWikibase infrastructure with a per-query-type SPARQL dispatcher (45 tests). Docs: rewrite the fast run README section around the actual API (base filter forms, write_required() options, language data checking, limitations) and remove the outdated WikidataIntegrator-era example. Co-Authored-By: Claude Fable 5 --- README.md | 136 +++-- test/test_wbi_fastrun.py | 465 ++++++++++++++++ wikibaseintegrator/entities/baseentity.py | 5 +- wikibaseintegrator/wbi_fastrun.py | 640 ++++++++++++++-------- 4 files changed, 990 insertions(+), 256 deletions(-) create mode 100644 test/test_wbi_fastrun.py diff --git a/README.md b/README.md index 521fa650..7d3d1c90 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,9 @@ The main differences between these two libraries are : * Add OAuth 2.0 login method * Add logging module support -But WikibaseIntegrator lack the "fastrun" functionality implemented in WikidataIntegrator. +WikibaseIntegrator also provides a "fast run" mode, which compares the local data with the live data through the SPARQL +endpoint and only writes to the Wikibase instance when something actually changed (see +[Examples (in "fast run" mode)](#examples-in-fast-run-mode)). # Documentation # @@ -799,21 +801,35 @@ for entrez_id, ensembl in raw_data.items(): # Examples (in "fast run" mode) # +The fast run mode is designed for bots synchronising an external resource with a Wikibase instance, where most of the +entities are usually already up to date. Instead of loading every entity through the MediaWiki API to compare it with +the local data, the fast run mode loads the current state of the entities of the data corpus with paginated SPARQL +queries, then decides locally, entity per entity, if a write is required. When nothing changed (the most common case), +no MediaWiki API call is made at all for that entity. + In order to use the fast run mode, you need to know the property/value combination which determines the data corpus you would like to operate on. E.g. for operating on human genes, you need to know that [P351](https://www.wikidata.org/entity/P351) is the NCBI Entrez Gene ID and you also need to know that you are dealing with humans, best represented by the [found in taxon property (P703)](https://www.wikidata.org/entity/P703) with the value [Q15978631](https://www.wikidata.org/entity/Q15978631) for Homo sapiens. -IMPORTANT: In order for the fast run mode to work, the data you provide in the constructor must contain at least one -unique value/id only present on one Wikidata element, e.g. an NCBI entrez gene ID, Uniprot ID, etc. Usually, these would -be the same unique core properties used for defining domains in wbi_core, e.g. for genes, proteins, drugs or your custom -domains. +IMPORTANT: In order for the fast run mode to work on entities without a known entity ID, the data you provide must +contain at least one unique value/id only present on one Wikibase entity, e.g. an NCBI entrez gene ID, Uniprot ID, etc. +This unique value is used to identify the target entity. If the entity ID is already known (e.g. the entity was loaded +with `wbi.item.get()`), the comparison is automatically restricted to that entity and this requirement does not apply. + +## The base filter ## + +The base filter defines the data corpus and is a list of datatype instances (the same classes used to create claims). +Three forms are supported: -Below, the normal mode run example from above, slightly modified, to meet the requirements for the fast run mode. To -enable it, ItemEngine requires two parameters, fast_run=True/False and fast_run_base_filter which is a dictionary -holding the properties to filter for as keys, and the item QIDs as dict values. If the value is not a QID but a literal, -just provide an empty string. For the above example, the dictionary looks like this: +* A property with a value: the entities must have this exact statement, e.g. `Item(prop_nr='P703', value='Q15978631')` + (found in taxon Homo sapiens). +* A property without a value: the entities must have a statement with this property, whatever the value, e.g. + `ExternalID(prop_nr='P351')` (any NCBI Entrez Gene ID). +* A property path, given as a list of two datatype instances: the value is reached through the first property followed + by any number of hops with the second one, e.g. `[Item(prop_nr='P31', value='Q11173'), Item(prop_nr='P279')]` + (instance of (P31) chemical compound (Q11173), directly or through a chain of subclass of (P279)). ```python from wikibaseintegrator.datatypes import ExternalID, Item @@ -821,20 +837,25 @@ from wikibaseintegrator.datatypes import ExternalID, Item fast_run_base_filter = [ExternalID(prop_nr='P351'), Item(prop_nr='P703', value='Q15978631')] ``` -The full example: +## Checking if a write is required ## + +The entry point is `entity.write_required()`. It returns `True` if the local entity differs from the live data and an +actual write is needed, `False` if the entity is already up to date and the write can be skipped. The full example, a +modified version of the mass import bot from the normal mode example: ```python from wikibaseintegrator import WikibaseIntegrator, wbi_login -from wikibaseintegrator.datatypes import ExternalID, Item, String, Time +from wikibaseintegrator.datatypes import ExternalID, Item, Time from wikibaseintegrator.wbi_enums import WikibaseTimePrecision # login object login = wbi_login.OAuth2(consumer_token='', consumer_secret='') -fast_run_base_filter = [ExternalID(prop_nr='P351'), Item(prop_nr='P703', value='Q15978631')] -fast_run = True +wbi = WikibaseIntegrator(login=login) -# We have raw data, which should be written to Wikidata, namely two human NCBI entrez gene IDs mapped to two Ensembl Gene IDs +fast_run_base_filter = [ExternalID(prop_nr='P351'), ExternalID(prop_nr='P704'), Item(prop_nr='P703', value='Q15978631')] + +# We have raw data, which should be written to Wikidata, namely two human NCBI entrez gene IDs mapped to two Ensembl transcript IDs # You can iterate over any data source as long as you can map the values to Wikidata properties. raw_data = { '50943': 'ENST00000376197', @@ -845,38 +866,89 @@ for entrez_id, ensembl in raw_data.items(): # add some references references = [ [ - Item(value='Q20641742', prop_nr='P248') - ], - [ - Time(time='+2020-02-08T00:00:00Z', prop_nr='P813', precision=WikibaseTimePrecision.DAY), - ExternalID(value='1017', prop_nr='P351') + Item(value='Q20641742', prop_nr='P248'), + Time(time='+2020-02-08T00:00:00Z', prop_nr='P813', precision=WikibaseTimePrecision.DAY) ] ] - # data type object - entrez_gene_id = String(value=entrez_id, prop_nr='P351', references=references) - ensembl_transcript_id = String(value=ensembl, prop_nr='P704', references=references) + # data type objects + entrez_gene_id = ExternalID(value=entrez_id, prop_nr='P351', references=references) + ensembl_transcript_id = ExternalID(value=ensembl, prop_nr='P704', references=references) # data goes into a list, because many data objects can be provided to data = [entrez_gene_id, ensembl_transcript_id] - # Search for and then edit/create new item - wb_item = WikibaseIntegrator(login=login).item.new() - wb_item.add_claims(claims=data) - wb_item.init_fastrun(base_filter=fast_run_base_filter) - wb_item.write() + item = wbi.item.new() + item.claims.add(data) + + # Compare the local data with the live data and only write when something differs + if item.write_required(base_filter=fast_run_base_filter): + item.write() ``` -Note: Fastrun mode checks for equality of property/value pairs, qualifiers (not including qualifier attributes), labels, -aliases and description, but it ignores references by default! -References can be checked in fast run mode by setting `use_references` to `True`. +Note: only the claims whose property appears in the base filter are compared. In the example above, the P351 and P704 +claims are checked because both properties are part of the base filter; a claim on any other property would be ignored +by the comparison. A write is reported as not required only when one entity (the entity being edited, when its ID is +known) holds all the compared claims. + +## Options ## + +`write_required()` accepts optional parameters, forwarded to the fastrun container: + +* `use_qualifiers` (default `True`): also compare the qualifiers of the statements. +* `use_references` (default `False`): also compare the references of the statements. By default, references are + ignored. +* `use_rank` (default `False`): also compare the rank of the statements. +* `case_insensitive` (default `False`): compare string values case-insensitively. The comparison of qualifiers, + references and ranks stays case sensitive. +* `cache` (default `True`): keep the data returned by the SPARQL endpoint in memory and reuse it for the next + checks. When disabled, the queries are restricted to the exact values of the claims instead of preloading every + statement of the property. +* `action_if_exists` (default `ActionIfExists.REPLACE_ALL`): the action that will be used for the write. With + `FORCE_APPEND`, the statements are always appended and a write is always reported as required. With the other + actions, the claims must already exist on the entity for the write to be skipped. + +The preloaded data is shared: containers are cached at the module level and reused by every `write_required()` call +using the same base filter and options, so the SPARQL queries are only executed once per property. + +## Checking labels, descriptions and aliases ## + +`write_required()` only compares claims. To check language data (labels, descriptions or aliases), use the fastrun +container directly: + +```python +from wikibaseintegrator import wbi_fastrun +from wikibaseintegrator.datatypes import ExternalID, Item + +frc = wbi_fastrun.get_fastrun_container(base_filter=[ExternalID(prop_nr='P351'), Item(prop_nr='P703', value='Q15978631')]) + +# Resolve the entity ID from a unique value, without any MediaWiki API call +qids = frc.get_entities(claims=[ExternalID(value='50943', prop_nr='P351')]) + +# Returns True if the English label differs from 'CDK7' and a write is required +frc.check_language_data(qid=qids[0], lang_data=['CDK7'], lang='en', lang_data_type='label') +``` + +## Limitations ## + +* The SPARQL endpoint can lag behind the live data (typically a few seconds to a few minutes on Wikidata). A write + that just happened may not be visible yet, and the same write could be reported as required again. +* The attributes missing from the SPARQL simple values are rebuilt with their default value: the precision of a time + value is inferred from the timestamp, the bounds of a quantity are not compared, etc. Statements using non-default + attributes are always reported as requiring a write. +* The datatypes that do not implement `from_sparql_value()` yet (e.g. Form, Sense, Property, Lexeme, GeoShape, + TabularData) cannot be compared: their statements are always reported as requiring a write. +* The qualifiers, references and ranks are loaded lazily with one SPARQL query per statement to check. This is fast + when most entities are up to date, but can slow down the run when many entities hold the same values. + +## Performance statistics ## -# Statistics # +Measured on a synchronisation bot of French administrative entities with Wikidata. The "partial fastrun" column is the +mode without cache, where the queries are restricted to the exact values of the claims: | Dataset | partial fastrun | fastrun without qualifiers/references | fastrun with qualifiers | fastrun with qualifiers/references | |:----------------------------|----------------:|--------------------------------------:|------------------------:|-----------------------------------:| | Communes (34990 elements) | ? | 7min | 30s | 60s | -| Cantons (2042 elements) | ? | ? | ? | ? | | Départements (100 elements) | 70min | 1s | 30s | 60s | # Debugging # diff --git a/test/test_wbi_fastrun.py b/test/test_wbi_fastrun.py new file mode 100644 index 00000000..c11a7901 --- /dev/null +++ b/test/test_wbi_fastrun.py @@ -0,0 +1,465 @@ +""" +Tests for the fastrun container. The SPARQL endpoint is served by the simulated +Wikibase instance, so the whole pipeline (statement loading, lazy qualifier / +reference / rank loading, comparison) runs offline and deterministically. +""" +import re + +import pytest + +from wikibaseintegrator import WikibaseIntegrator, wbi_fastrun +from wikibaseintegrator.datatypes import BaseDataType, ExternalID, Item, String, Time +from wikibaseintegrator.wbi_enums import ActionIfExists, WikibaseRank + +from .conftest import literal, uri + +wbi = WikibaseIntegrator() + +PTYPE_EXTERNAL_ID = 'http://wikiba.se/ontology#ExternalId' +PTYPE_ITEM = 'http://wikiba.se/ontology#WikibaseItem' +PTYPE_STRING = 'http://wikiba.se/ontology#String' +PTYPE_TIME = 'http://wikiba.se/ontology#Time' +XSD_DATETIME = 'http://www.w3.org/2001/XMLSchema#dateTime' + + +class SparqlData: + """ + Serve the SPARQL queries of the fastrun container from in-memory data. + + Each query type issued by the container (statement loading, qualifiers, + references, rank, language data) is recognized through its #Tool comment + and answered from the matching attribute. + """ + + def __init__(self, wikibase): + self.wikibase = wikibase + self.statements: dict[str, list[dict]] = {} # property number -> bindings + self.qualifiers: dict[str, list[dict]] = {} # statement URI -> bindings + self.references: dict[str, list[dict]] = {} # statement URI -> bindings + self.ranks: dict[str, list[dict]] = {} # statement URI -> bindings + self.labels: list[dict] = [] # language data bindings + wikibase.sparql_bindings = self.dispatch + + def dispatch(self, query: str) -> list[dict]: + if 'wbi_fastrun._load_qualifiers' in query: + return self.qualifiers.get(self._sid(query), []) + if 'wbi_fastrun._load_references' in query: + return self.references.get(self._sid(query), []) + if 'wbi_fastrun._load_rank' in query: + return self.ranks.get(self._sid(query), []) + if 'wbi_fastrun._query_lang' in query: + return self.labels + if 'wbi_fastrun.load_statements' in query: + prop_nr = re.search(r'/prop/(P\d+)> \?sid', query).group(1) + return self.statements.get(prop_nr, []) + return [] + + @staticmethod + def _sid(query: str) -> str: + return re.search(r'VALUES \?sid \{ <([^>]+)> \}', query).group(1) + + def statement(self, entity_id: str, prop_nr: str, value: dict, property_type: str, index: int = 0) -> str: + """Register a statement binding and return its statement URI.""" + sid = f'{self.wikibase.base_url}/entity/statement/{entity_id}-{prop_nr}-{index}' + self.statements.setdefault(prop_nr, []).append({ + 'entity': uri(f'{self.wikibase.base_url}/entity/{entity_id}'), + 'sid': uri(sid), + 'value': value, + 'property_type': uri(property_type), + }) + return sid + + def qualifier(self, sid: str, prop_nr: str, value: dict, property_type: str) -> None: + self.qualifiers.setdefault(sid, []).append({ + 'property': uri(f'{self.wikibase.base_url}/entity/{prop_nr}'), + 'value': value, + 'property_type': uri(property_type), + }) + + def reference(self, sid: str, prop_nr: str, value: dict, property_type: str, ref_node: str = 'deadbeef') -> None: + self.references.setdefault(sid, []).append({ + 'srid': uri(f'{self.wikibase.base_url}/reference/{ref_node}'), + 'ref_property': uri(f'{self.wikibase.base_url}/entity/{prop_nr}'), + 'ref_value': value, + 'property_type': uri(property_type), + }) + + def rank(self, sid: str, rank: str) -> None: + self.ranks[sid] = [{'rank': uri(f'http://wikiba.se/ontology#{rank}')}] + + def label(self, entity_id: str, value: str, lang: str) -> None: + self.labels.append({ + 'entity': uri(f'{self.wikibase.base_url}/entity/{entity_id}'), + 'label': literal(value, lang=lang), + }) + + +@pytest.fixture +def sparql_data(wikibase): + return SparqlData(wikibase) + + +@pytest.fixture +def frc(wikibase): + return wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P352')], base_data_type=BaseDataType) + + +class TestLoadStatements: + def test_load_statements(self, wikibase, sparql_data, frc): + """The container must query the SPARQL endpoint and store the data in its internal format.""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + frc.load_statements(claims=ExternalID(value='P40095', prop_nr='P352')) + + expected_key = ExternalID(value='P40095', prop_nr='P352').get_sparql_value() + assert expected_key in frc.data['P352'] + assert frc.data['P352'][expected_key][0]['entity'] == f'{wikibase.base_url}/entity/Q99' + assert frc.properties_type['P352'] == PTYPE_EXTERNAL_ID + assert 'P352' in frc.loaded_complete + + def test_cache_reuse(self, wikibase, sparql_data, frc): + """A complete load must be reused, without a new SPARQL query.""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + frc.load_statements(claims=ExternalID(value='P40095', prop_nr='P352')) + queries_after_first_load = len(wikibase.sparql_queries) + + frc.load_statements(claims=ExternalID(value='something-else', prop_nr='P352')) + assert len(wikibase.sparql_queries) == queries_after_first_load + + def test_partial_load_is_not_reused_as_cache(self, wikibase, sparql_data, frc): + """A load restricted to a value (cache disabled) must not be reused as a complete cache.""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + frc.load_statements(claims=ExternalID(value='P40095', prop_nr='P352'), cache=False) + assert 'P352' not in frc.loaded_complete + # The query is restricted to the value of the claim + assert '"P40095"' in wikibase.sparql_queries[-1] + + # The next cached load must query the endpoint again + queries_after_partial_load = len(wikibase.sparql_queries) + frc.load_statements(claims=ExternalID(value='P40095', prop_nr='P352')) + assert len(wikibase.sparql_queries) > queries_after_partial_load + assert 'P352' in frc.loaded_complete + + def test_unparseable_value_is_skipped(self, wikibase, sparql_data, frc): + """A value the data type can't represent must be skipped, not crash the load.""" + # A timestamp with a time part: its precision can't be inferred by Time.set_value() + sparql_data.statement('Q99', 'P580', literal('2020-02-08T12:34:56Z', datatype=XSD_DATETIME), PTYPE_TIME) + sparql_data.statement('Q99', 'P580', literal('2020-02-08T00:00:00Z', datatype=XSD_DATETIME), PTYPE_TIME, index=1) + + frc.load_statements(claims=Time(time='+2020-02-08T00:00:00Z', prop_nr='P580')) + + assert len(frc.data['P580']) == 1 + + def test_invalid_claims(self, frc): + with pytest.raises(ValueError): + frc.load_statements(claims='not a claim') + + +class TestGetEntities: + def test_get_entities(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.statement('Q100', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID, index=1) + + assert frc.get_entities(claims=ExternalID(value='P40095', prop_nr='P352')) == ['Q100', 'Q99'] + + def test_get_entities_intersection(self, wikibase, sparql_data, frc): + """With several claims, only the entities holding every value are returned.""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.statement('Q100', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID, index=1) + sparql_data.statement('Q99', 'P828', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + + claims = [ExternalID(value='P40095', prop_nr='P352'), Item(value='Q42', prop_nr='P828')] + assert frc.get_entities(claims=claims) == ['Q99'] + + def test_get_entities_no_match(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + assert frc.get_entities(claims=ExternalID(value='unknown-value', prop_nr='P352')) == [] + + +class TestWriteRequired: + def test_not_required_when_value_exists(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')]) is False + + def test_required_when_value_differs(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + assert frc.write_required(claims=[ExternalID(value='something-else', prop_nr='P352')]) is True + + def test_entity_filter(self, wikibase, sparql_data, frc): + """The data exists on Q99: no write is required for Q99, a write is required for another entity.""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + claims = [ExternalID(value='P40095', prop_nr='P352')] + assert frc.write_required(claims=claims, entity_filter='Q99') is False + assert frc.write_required(claims=claims, entity_filter='Q100') is True + # A full entity URI is accepted too + assert frc.write_required(claims=claims, entity_filter=f'{wikibase.base_url}/entity/Q99') is False + + def test_required_when_no_common_entity(self, wikibase, sparql_data, frc): + """The two values exist, but on two different entities: a write is required.""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.statement('Q100', 'P828', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + + claims = [ExternalID(value='P40095', prop_nr='P352'), Item(value='Q42', prop_nr='P828')] + assert frc.write_required(claims=claims) is True + + def test_not_required_when_common_entity(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.statement('Q99', 'P828', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + + claims = [ExternalID(value='P40095', prop_nr='P352'), Item(value='Q42', prop_nr='P828')] + assert frc.write_required(claims=claims) is False + + def test_property_filter(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + claims = [ExternalID(value='P40095', prop_nr='P352'), ExternalID(value='unchecked', prop_nr='P999')] + # P999 is filtered out: only P352 is compared, no write is required + assert frc.write_required(claims=claims, property_filter='P352') is False + + def test_required_when_no_claim_matches_the_property_filter(self, wikibase, sparql_data, frc): + """When nothing can be verified, a write must be reported, not an error raised.""" + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')], property_filter=['P999']) is True + + def test_force_append_always_requires_write(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + claims = [ExternalID(value='P40095', prop_nr='P352')] + assert frc.write_required(claims=claims, action_if_exists=ActionIfExists.FORCE_APPEND) is True + # The check short-circuits without querying the endpoint + assert len(wikibase.sparql_queries) == 0 + + def test_empty_claims(self, frc): + with pytest.raises(ValueError): + frc.write_required(claims=[]) + + +class TestWriteRequiredQualifiers: + def test_missing_qualifier_on_the_claim(self, wikibase, sparql_data, frc): + """The statement has a qualifier, the claim doesn't: a write is required.""" + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.qualifier(sid, 'P642', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')]) is True + + def test_matching_qualifier(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.qualifier(sid, 'P642', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + + claim = ExternalID(value='P40095', prop_nr='P352', qualifiers=[Item(value='Q42', prop_nr='P642')]) + assert frc.write_required(claims=[claim]) is False + + def test_different_qualifier_value(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.qualifier(sid, 'P642', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + + claim = ExternalID(value='P40095', prop_nr='P352', qualifiers=[Item(value='Q43', prop_nr='P642')]) + assert frc.write_required(claims=[claim]) is True + + def test_qualifiers_can_be_ignored(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.qualifier(sid, 'P642', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')], use_qualifiers=False) is False + + def test_duplicate_statements_do_not_mask_a_match(self, wikibase, sparql_data, frc): + """ + The entity holds two statements with the same value, one with a qualifier and one without: a claim matching + either of them requires no write, whatever the statement order. + """ + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID, index=0) + sparql_data.qualifier(sid, 'P642', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID, index=1) + + without_qualifier = ExternalID(value='P40095', prop_nr='P352') + with_qualifier = ExternalID(value='P40095', prop_nr='P352', qualifiers=[Item(value='Q42', prop_nr='P642')]) + assert frc.write_required(claims=[without_qualifier]) is False + assert frc.write_required(claims=[with_qualifier]) is False + + +class TestWriteRequiredReferences: + def test_references_ignored_by_default(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.reference(sid, 'P248', uri(f'{wikibase.base_url}/entity/Q1234'), PTYPE_ITEM) + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')]) is False + + def test_missing_reference_on_the_claim(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.reference(sid, 'P248', uri(f'{wikibase.base_url}/entity/Q1234'), PTYPE_ITEM) + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')], use_references=True) is True + + def test_matching_reference(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.reference(sid, 'P248', uri(f'{wikibase.base_url}/entity/Q1234'), PTYPE_ITEM) + + claim = ExternalID(value='P40095', prop_nr='P352', references=[[Item(value='Q1234', prop_nr='P248')]]) + assert frc.write_required(claims=[claim], use_references=True) is False + + def test_different_reference(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.reference(sid, 'P248', uri(f'{wikibase.base_url}/entity/Q1234'), PTYPE_ITEM) + + claim = ExternalID(value='P40095', prop_nr='P352', references=[[Item(value='Q4321', prop_nr='P248')]]) + assert frc.write_required(claims=[claim], use_references=True) is True + + +class TestWriteRequiredRank: + def test_matching_rank(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.rank(sid, 'NormalRank') + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')], use_rank=True) is False + + def test_different_rank(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.rank(sid, 'PreferredRank') + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')], use_rank=True) is True + + def test_rank_ignored_by_default(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.rank(sid, 'PreferredRank') + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')]) is False + + +class TestCaseInsensitive: + def test_case_insensitive_value(self, wikibase, sparql_data): + sparql_data.statement('Q99', 'P1', literal('FooBar'), PTYPE_STRING) + + frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P1')], base_data_type=BaseDataType, case_insensitive=True) + assert frc.write_required(claims=[String(value='foobar', prop_nr='P1')]) is False + + def test_case_sensitive_by_default(self, wikibase, sparql_data): + sparql_data.statement('Q99', 'P1', literal('FooBar'), PTYPE_STRING) + + frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P1')], base_data_type=BaseDataType) + assert frc.write_required(claims=[String(value='foobar', prop_nr='P1')]) is True + assert frc.write_required(claims=[String(value='FooBar', prop_nr='P1')]) is False + + +class TestEntityWriteRequired: + """write_required() through an entity: the check is restricted to the entity being edited.""" + + def _item(self, item_id: str | None, claims: list) -> 'ItemEntity': # noqa: F821 + item = wbi.item.new() + if item_id: + item.id = item_id + for claim in claims: + item.claims.add(claim) + return item + + def test_not_required_when_the_entity_holds_the_data(self, wikibase, sparql_data): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + item = self._item('Q99', [ExternalID(value='P40095', prop_nr='P352')]) + assert item.write_required(base_filter=[BaseDataType(prop_nr='P352')]) is False + + def test_required_when_the_data_is_on_another_entity(self, wikibase, sparql_data): + """Q100 is edited: the value existing on Q99 must not inhibit the write.""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + item = self._item('Q100', [ExternalID(value='P40095', prop_nr='P352')]) + assert item.write_required(base_filter=[BaseDataType(prop_nr='P352')]) is True + + def test_new_entity_matches_any_entity(self, wikibase, sparql_data): + """An entity without ID matches any entity holding the data (deduplication use case).""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + item = self._item(None, [ExternalID(value='P40095', prop_nr='P352')]) + assert item.write_required(base_filter=[BaseDataType(prop_nr='P352')]) is False + + def test_required_when_no_claim_matches_the_base_filter(self, wikibase, sparql_data): + """No claim of the entity is covered by the base filter: nothing can be verified, a write is required.""" + item = self._item('Q99', [ExternalID(value='P40095', prop_nr='P999')]) + assert item.write_required(base_filter=[BaseDataType(prop_nr='P352')]) is True + + def test_force_append(self, wikibase, sparql_data): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + item = self._item('Q99', [ExternalID(value='P40095', prop_nr='P352')]) + assert item.write_required(base_filter=[BaseDataType(prop_nr='P352')], action_if_exists=ActionIfExists.FORCE_APPEND) is True + + +class TestLanguageData: + def test_language_data_and_check(self, wikibase, sparql_data, frc): + sparql_data.label('Q582', 'Villeurbanne', 'fr') + + assert frc.get_language_data('Q582', 'fr', 'label') == ['Villeurbanne'] + # The comparison is case insensitive + assert frc.check_language_data('Q582', ['villeurbanne'], 'fr', 'label') is False + assert frc.check_language_data('Q582', ['Lyon'], 'fr', 'label') is True + + def test_empty_language_data(self, wikibase, sparql_data, frc): + assert frc.get_language_data('Q582', 'fr', 'label') == [''] + assert frc.get_language_data('Q582', 'fr', 'aliases') == [] + + def test_language_data_is_cached(self, wikibase, sparql_data, frc): + sparql_data.label('Q582', 'Villeurbanne', 'fr') + + frc.get_language_data('Q582', 'fr', 'label') + queries_after_first_load = len(wikibase.sparql_queries) + frc.get_language_data('Q582', 'fr', 'label') + assert len(wikibase.sparql_queries) == queries_after_first_load + + def test_replace_all_language_data(self, wikibase, sparql_data, frc): + sparql_data.label('Q582', 'Villeurbanne', 'fr') + + assert frc.check_language_data('Q582', ['Villeurbanne'], 'fr', 'label', action_if_exists=ActionIfExists.REPLACE_ALL) is False + assert frc.check_language_data('Q582', ['Villeurbanne', 'Lyon'], 'fr', 'label', action_if_exists=ActionIfExists.REPLACE_ALL) is True + + +class TestClear: + def test_clear(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.label('Q582', 'Villeurbanne', 'fr') + + frc.load_statements(claims=ExternalID(value='P40095', prop_nr='P352')) + frc.init_language_data('fr', 'label') + + frc.clear() + + assert not frc.data + assert not frc.loaded_complete + assert not frc.properties_type + assert not frc.loaded_langs + + +class TestFastrunStore: + def test_container_is_reused(self, wikibase): + frc1 = wbi_fastrun.get_fastrun_container(base_filter=[BaseDataType(prop_nr='P352')]) + frc2 = wbi_fastrun.get_fastrun_container(base_filter=[BaseDataType(prop_nr='P352')]) + assert frc1 is frc2 + + def test_different_parameters_create_new_container(self, wikibase): + frc1 = wbi_fastrun.get_fastrun_container(base_filter=[BaseDataType(prop_nr='P352')]) + frc2 = wbi_fastrun.get_fastrun_container(base_filter=[BaseDataType(prop_nr='P921')]) + frc3 = wbi_fastrun.get_fastrun_container(base_filter=[BaseDataType(prop_nr='P352')], use_references=True) + frc4 = wbi_fastrun.get_fastrun_container(base_filter=[BaseDataType(prop_nr='P352')], case_insensitive=True) + assert len({id(frc) for frc in (frc1, frc2, frc3, frc4)}) == 4 + + +class TestBaseFilter: + def test_invalid_base_filter(self): + with pytest.raises(ValueError): + wbi_fastrun.FastRunContainer(base_filter=['P352'], base_data_type=BaseDataType) + + def test_base_filter_string(self, wikibase): + """The three base filter forms must be translated to the expected SPARQL triples.""" + frc = wbi_fastrun.FastRunContainer(base_data_type=BaseDataType, base_filter=[ + BaseDataType(prop_nr='P352'), + Item(value='Q55983715', prop_nr='P703'), + [Item(value='Q3624078', prop_nr='P31'), Item(prop_nr='P279')], + ]) + + base_filter_string = frc._base_filter_string() + assert f'?entity <{wikibase.base_url}/prop/direct/P352> ?zzP352 .' in base_filter_string + assert f'?entity <{wikibase.base_url}/prop/direct/P703> <{wikibase.base_url}/entity/Q55983715> .' in base_filter_string + assert f'?entity <{wikibase.base_url}/prop/direct/P31>/<{wikibase.base_url}/prop/direct/P279>* <{wikibase.base_url}/entity/Q3624078> .' in base_filter_string diff --git a/wikibaseintegrator/entities/baseentity.py b/wikibaseintegrator/entities/baseentity.py index 79575ab1..59891dcc 100644 --- a/wikibaseintegrator/entities/baseentity.py +++ b/wikibaseintegrator/entities/baseentity.py @@ -363,7 +363,10 @@ def write_required(self, base_filter: list[BaseDataType | list[BaseDataType]], a # TODO: Add check_language_data - return fastrun_container.write_required(claims=self.claims, property_filter=property_filter) + # Restrict the check to the entity being edited, when it already has an ID + entity_filter = [self.id] if self.id else None + + return fastrun_container.write_required(claims=self.claims, entity_filter=entity_filter, property_filter=property_filter, action_if_exists=action_if_exists) def get_entity_url(self, wikibase_url: str | None = None) -> str: from wikibaseintegrator.wbi_config import config diff --git a/wikibaseintegrator/wbi_fastrun.py b/wikibaseintegrator/wbi_fastrun.py index 45d8a6f4..9cee9e14 100644 --- a/wikibaseintegrator/wbi_fastrun.py +++ b/wikibaseintegrator/wbi_fastrun.py @@ -1,12 +1,18 @@ +""" +Fast run mode: check locally, through data loaded from the SPARQL endpoint, whether a write to the Wikibase instance +is actually required, so that a synchronisation bot can skip the entities that are already up to date. +""" from __future__ import annotations +import collections import logging import re +from collections import defaultdict from wikibaseintegrator.datatypes import BaseDataType from wikibaseintegrator.models import Claim, Claims, Qualifiers, Reference, References from wikibaseintegrator.wbi_config import config -from wikibaseintegrator.wbi_enums import WikibaseRank +from wikibaseintegrator.wbi_enums import ActionIfExists, WikibaseRank from wikibaseintegrator.wbi_helpers import execute_sparql_query log = logging.getLogger(__name__) @@ -16,20 +22,21 @@ class FastRunContainer: """ + A FastRunContainer loads the statements of the entities matching the base filter from the SPARQL endpoint and + caches them, so that a bot can check whether a write is required without loading every entity through the API. :param base_filter: The default filter to initialize the dataset. A list made of BaseDataType or list of BaseDataType. :param base_data_type: The default data type to create objects. :param use_qualifiers: Use qualifiers during fastrun. Enabled by default. :param use_references: Use references during fastrun. Disabled by default. :param use_rank: Use rank during fastrun. Disabled by default. - :param cache: Put data returned by WDQS in cache. Enabled by default. - :param case_insensitive: - :param sparql_endpoint_url: SPARLQ endpoint URL. + :param cache: Put data returned by the SPARQL endpoint in cache. Enabled by default. + :param case_insensitive: Compare the string values without taking the case into account. The comparison of + qualifiers, references and ranks stays case sensitive. Disabled by default. + :param sparql_endpoint_url: SPARQL endpoint URL. :param wikibase_url: Wikibase URL used for the concept URI. """ - # TODO: Add support for case_insensitive - data: dict[str, dict[str, list[dict[str, str]]]] def __init__(self, base_filter: list[BaseDataType | list[BaseDataType]], base_data_type: type[BaseDataType] | None = None, use_qualifiers: bool = True, @@ -40,7 +47,11 @@ def __init__(self, base_filter: list[BaseDataType | list[BaseDataType]], base_da if not isinstance(k, BaseDataType) and not (isinstance(k, list) and len(k) == 2 and isinstance(k[0], BaseDataType) and isinstance(k[1], BaseDataType)): raise ValueError("base_filter must be an instance of BaseDataType or a list of instances of BaseDataType") + # Statements loaded from the SPARQL endpoint: property number -> value key -> list of {'entity': uri, 'sid': uri} self.data: dict[str, dict[str, list[dict[str, str]]]] = {} + # The properties whose statements are completely loaded in self.data. A load restricted to a value or to + # qualifiers only holds a subset of the statements and must not be reused as a complete cache. + self.loaded_complete: set[str] = set() self.base_filter = base_filter self.base_data_type = base_data_type or BaseDataType @@ -52,18 +63,77 @@ def __init__(self, base_filter: list[BaseDataType | list[BaseDataType]], base_da self.cache = cache self.case_insensitive = case_insensitive self.properties_type: dict[str, str] = {} + self.loaded_langs: dict[str, dict] = {} + + # Per-statement caches for the lazily loaded qualifiers, references and ranks + self._qualifiers_cache: dict[str, Qualifiers] = {} + self._references_cache: dict[str, References] = {} + self._rank_cache: dict[str, WikibaseRank | None] = {} + + @staticmethod + def _entity_id(entity: str) -> str: + """Reduce an entity URI to its bare entity ID. A bare entity ID is returned unchanged.""" + return entity.rsplit('/', 1)[-1] + + def _datatype_class(self, property_type: str) -> type[BaseDataType]: + """ + Return the data type class implementing the given SPARQL property type (e.g. 'http://wikiba.se/ontology#Time' -> Time). - if self.case_insensitive: - raise ValueError("Case insensitive does not work for the moment.") + :param property_type: A property type URI from the wikibase ontology + :exception ValueError: if no class implements the given property type + """ + for subclass in self.base_data_type.subclasses: + if subclass.PTYPE == property_type: + return subclass + raise ValueError(f"No data type class found for the property type '{property_type}'") + + def _value_key(self, claim: Claim) -> str | None: + """The key indexing the value of the given claim in self.data, casefolded when case_insensitive is enabled.""" + value = claim.get_sparql_value() + if value is not None and self.case_insensitive: + value = value.casefold() + return value + + def _base_filter_string(self, wb_url: str | None = None) -> str: + """Generate the SPARQL triples restricting ?entity to the entities matching the base filter.""" + wb_url = wb_url or self.wikibase_url + + base_filter_string = '' + for k in self.base_filter: + if isinstance(k, BaseDataType): + # TODO: Add multiple values for a property (OR-operation) (with the VALUES tag?) + if k.mainsnak.datavalue: + base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr}> {entity} .\n'.format( + wb_url=wb_url, prop_nr=k.mainsnak.property_number, entity=k.get_sparql_value(wikibase_url=wb_url)) + elif sum(1 for x in self.base_filter if isinstance(x, BaseDataType) and x.mainsnak.property_number == k.mainsnak.property_number) == 1: + base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr}> ?zz{prop_nr} .\n'.format( + wb_url=wb_url, prop_nr=k.mainsnak.property_number) + elif isinstance(k, list) and len(k) == 2 and isinstance(k[0], BaseDataType) and isinstance(k[1], BaseDataType): + if k[0].mainsnak.datavalue: + base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr}>/<{wb_url}/prop/direct/{prop_nr2}>* {entity} .\n'.format( + wb_url=wb_url, prop_nr=k[0].mainsnak.property_number, prop_nr2=k[1].mainsnak.property_number, + entity=k[0].get_sparql_value(wikibase_url=wb_url)) + # TODO: Remove ?zzPYY if another filter have the same property number, the same as above + else: + base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr1}>/<{wb_url}/prop/direct/{prop_nr2}>* ?zz{prop_nr1}{prop_nr2} .\n'.format( + wb_url=wb_url, prop_nr1=k[0].mainsnak.property_number, prop_nr2=k[1].mainsnak.property_number) + else: + raise ValueError("base_filter must be an instance of BaseDataType or a list of instances of BaseDataType") + + return base_filter_string def load_statements(self, claims: list[Claim] | Claims | Claim, cache: bool | None = None, wb_url: str | None = None, limit: int | None = None) -> None: """ Load the statements related to the given claims into the internal cache of the current object. + When the cache is enabled, every statement of the property is loaded once and reused afterwards. When the + cache is disabled and the claim carries a value, only the statements holding the same value (and the same + qualifiers if use_qualifiers is enabled) are loaded; such a partial load is never reused as a cache. + :param claims: A Claim, Claims or list of Claim + :param cache: Put data returned by the SPARQL endpoint in cache. Enabled by default. :param wb_url: The first part of the concept URI of entities. :param limit: The limit to request at one time. - :param cache: Put data returned by WDQS in cache. Enabled by default. :return: """ if isinstance(claims, Claim): @@ -82,38 +152,26 @@ def load_statements(self, claims: list[Claim] | Claims | Claim, cache: bool | No prop_nr = claim.mainsnak.property_number # Load each property from the Wikibase instance or the cache - if cache and prop_nr in self.data: - logging.debug("Property '%s' found in cache, %s elements", prop_nr, len(self.data[prop_nr])) + if cache and prop_nr in self.data and prop_nr in self.loaded_complete: + log.debug("Property '%s' found in cache, %s elements", prop_nr, len(self.data[prop_nr])) continue offset = 0 - # Generate base filter - base_filter_string = '' - for k in self.base_filter: - if isinstance(k, BaseDataType): - # TODO: Add multiple values for a property (OR-operation) (with the VALUES tag?) - if k.mainsnak.datavalue: - base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr}> {entity} .\n'.format( - wb_url=wb_url, prop_nr=k.mainsnak.property_number, entity=k.get_sparql_value(wikibase_url=wb_url)) - elif sum(map(lambda x, other=k: x.mainsnak.property_number == other.mainsnak.property_number, self.base_filter)) == 1: # type: ignore - base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr}> ?zz{prop_nr} .\n'.format( - wb_url=wb_url, prop_nr=k.mainsnak.property_number) - elif isinstance(k, list) and len(k) == 2 and isinstance(k[0], BaseDataType) and isinstance(k[1], BaseDataType): - if k[0].mainsnak.datavalue: - base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr}>/<{wb_url}/prop/direct/{prop_nr2}>* {entity} .\n'.format( - wb_url=wb_url, prop_nr=k[0].mainsnak.property_number, prop_nr2=k[1].mainsnak.property_number, - entity=k[0].get_sparql_value(wikibase_url=wb_url)) - # TODO: Remove ?zzPYY if another filter have the same property number, the same as above - else: - base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr1}>/<{wb_url}/prop/direct/{prop_nr2}>* ?zz{prop_nr1}{prop_nr2} .\n'.format( - wb_url=wb_url, prop_nr1=k[0].mainsnak.property_number, prop_nr2=k[1].mainsnak.property_number) - else: - raise ValueError("base_filter must be an instance of BaseDataType or a list of instances of BaseDataType") + base_filter_string = self._base_filter_string(wb_url=wb_url) + + # A partial load restricted to the claim value: only when the cache is disabled, because the result + # can't be reused for other values. The case insensitive mode always needs the complete data, the + # SPARQL comparison is case sensitive. + partial_load = bool(claim.mainsnak.datavalue) and not cache and not self.case_insensitive + # Restrict the statements to the ones holding the same qualifiers as the claim. Only applied to a + # partial load: a complete load must contain every statement, whatever its qualifiers. qualifiers_filter_string = '' - if self.use_qualifiers: + if partial_load and self.use_qualifiers: for qualifier in claim.qualifiers: + if not qualifier.datatype: + continue fake_json = { 'mainsnak': qualifier.get_json(), 'type': qualifier.datatype, @@ -125,9 +183,10 @@ def load_statements(self, claims: list[Claim] | Claims | Claim, cache: bool | No # We force a refresh of the data, remove the previous results self.data[prop_nr] = {} + self.loaded_complete.discard(prop_nr) while True: - if claim.mainsnak.datavalue and not cache: + if partial_load: query = ''' #Tool: WikibaseIntegrator wbi_fastrun.load_statements SELECT ?entity ?sid ?value ?property_type WHERE {{ @@ -156,7 +215,6 @@ def load_statements(self, claims: list[Claim] | Claims | Claim, cache: bool | No ?entity <{wb_url}/prop/{prop_nr}> ?sid. <{wb_url}/entity/{prop_nr}> wikibase:propertyType ?property_type. ?sid <{wb_url}/prop/statement/{prop_nr}> ?value. - {qualifiers_filter_string} }} ORDER BY ?sid OFFSET {offset} @@ -165,8 +223,7 @@ def load_statements(self, claims: list[Claim] | Claims | Claim, cache: bool | No # Format the query # TODO: Add custom query support - query = query.format(base_filter_string=base_filter_string, wb_url=wb_url, prop_nr=prop_nr, offset=str(offset), limit=str(limit), - qualifiers_filter_string=qualifiers_filter_string) + query = query.format(base_filter_string=base_filter_string, wb_url=wb_url, prop_nr=prop_nr, offset=str(offset), limit=str(limit)) offset += limit # We increase the offset for the next iteration results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] @@ -174,17 +231,26 @@ def load_statements(self, claims: list[Claim] | Claims | Claim, cache: bool | No for result in results: entity = result['entity']['value'] sid = result['sid']['value'] - # value = result['value']['value'] property_type = result['property_type']['value'] - # Use casefold for lower case - if self.case_insensitive: - result['value']['value'] = result['value']['value'].casefold() + try: + f = self._datatype_class(property_type)().from_sparql_value(sparql_value=result['value']) + except ValueError as exception: + # A value the data type can't represent (e.g. a timestamp whose precision can't be inferred). + # The value stays out of the dataset, a write will be reported as required for it. + log.warning("Skipping a value of property '%s': %s", prop_nr, exception) + continue - f = [x for x in self.base_data_type.subclasses if x.PTYPE == property_type][0]().from_sparql_value(sparql_value=result['value']) + if f is None: + # The data type does not implement from_sparql_value() yet + log.warning("The data type of property '%s' does not support from_sparql_value(), skipping the value", prop_nr) + continue sparql_value = f.get_sparql_value() if sparql_value is not None: + if self.case_insensitive: + sparql_value = sparql_value.casefold() + if sparql_value not in self.data[prop_nr]: self.data[prop_nr][sparql_value] = [] @@ -196,24 +262,31 @@ def load_statements(self, claims: list[Claim] | Claims | Claim, cache: bool | No if len(results) == 0 or len(results) < limit: break - def _load_qualifiers(self, sid: str, limit: int | None = None) -> Qualifiers: + if not partial_load: + self.loaded_complete.add(prop_nr) + + def _load_qualifiers(self, sid: str, limit: int | None = None, cache: bool | None = None) -> Qualifiers: """ Load the qualifiers of a statement. :param sid: A statement ID. :param limit: The limit to request at one time. + :param cache: Reuse the qualifiers already loaded for this statement. Enabled by default. :return: A Qualifiers object. """ - offset = 0 - if not isinstance(sid, str): raise ValueError('sid must be a string') - limit = limit or int(config['SPARQL_QUERY_LIMIT']) # type: ignore + if cache is None: + cache = self.cache - # TODO: Add cache + if cache and sid in self._qualifiers_cache: + return self._qualifiers_cache[sid] + + offset = 0 + + limit = limit or int(config['SPARQL_QUERY_LIMIT']) # type: ignore - # We force a refresh of the data, remove the previous results qualifiers: Qualifiers = Qualifiers() while True: query = f''' @@ -229,49 +302,60 @@ def _load_qualifiers(self, sid: str, limit: int | None = None) -> Qualifiers: LIMIT {limit} ''' - # Format the query - # query = query.format(wb_url=wb_url, sid=sid, offset=str(offset), limit=str(limit)) offset += limit # We increase the offset for the next iteration results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] for result in results: - property = result['property']['value'] + prop_nr = self._entity_id(result['property']['value']) property_type = result['property_type']['value'] - if property not in self.properties_type: - self.properties_type[property] = property_type + if prop_nr not in self.properties_type: + self.properties_type[prop_nr] = property_type + + try: + f = self._datatype_class(property_type)(prop_nr=prop_nr).from_sparql_value(sparql_value=result['value']) + except ValueError as exception: + # An unparseable qualifier can't be compared, leave it out so that the comparison fails safely + log.warning("Skipping a qualifier of statement '%s': %s", sid, exception) + continue - # Use casefold for lower case - if self.case_insensitive: - result['value']['value'] = result['value']['value'].casefold() + if f is None: + log.warning("The data type of property '%s' does not support from_sparql_value(), skipping the qualifier", prop_nr) + continue - f = [x for x in self.base_data_type.subclasses if x.PTYPE == property_type][0](prop_nr=property).from_sparql_value(sparql_value=result['value']) qualifiers.add(f) if len(results) == 0 or len(results) < limit: break + self._qualifiers_cache[sid] = qualifiers + return qualifiers - def _load_references(self, sid: str, limit: int = 10000) -> References: + def _load_references(self, sid: str, limit: int | None = None, cache: bool | None = None) -> References: """ Load the references of a statement. :param sid: A statement ID. :param limit: The limit to request at one time. + :param cache: Reuse the references already loaded for this statement. Enabled by default. :return: A References object. """ - offset = 0 - if not isinstance(sid, str): raise ValueError('sid must be a string') - limit = limit or int(config['SPARQL_QUERY_LIMIT']) # type: ignore + if cache is None: + cache = self.cache - # TODO: Add cache + if cache and sid in self._references_cache: + return self._references_cache[sid] - # We force a refresh of the data, remove the previous results - references: References = References() + offset = 0 + + limit = limit or int(config['SPARQL_QUERY_LIMIT']) # type: ignore + + # The references are grouped by reference node URI, across the result pages + reference: dict[str, Reference] = {} while True: query = f''' #Tool: WikibaseIntegrator wbi_fastrun._load_references @@ -288,56 +372,60 @@ def _load_references(self, sid: str, limit: int = 10000) -> References: LIMIT {limit} ''' - # Format the query - # query = query.format(wb_url=wb_url, sid=sid, offset=str(offset), limit=str(limit)) offset += limit # We increase the offset for the next iteration results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] - reference = {} - for result in results: - ref_property = result['ref_property']['value'] + prop_nr = self._entity_id(result['ref_property']['value']) srid = result['srid']['value'] property_type = result['property_type']['value'] - if ref_property not in self.properties_type: - self.properties_type[ref_property] = property_type + if prop_nr not in self.properties_type: + self.properties_type[prop_nr] = property_type - # Use casefold for lower case - if self.case_insensitive: - result['value']['value'] = result['value']['value'].casefold() + try: + f = self._datatype_class(property_type)(prop_nr=prop_nr).from_sparql_value(sparql_value=result['ref_value']) + except ValueError as exception: + # An unparseable reference snak can't be compared, leave it out so that the comparison fails safely + log.warning("Skipping a reference snak of statement '%s': %s", sid, exception) + continue - f = [x for x in self.base_data_type.subclasses if x.PTYPE == property_type][0](prop_nr=ref_property).from_sparql_value(sparql_value=result['ref_value']) + if f is None: + log.warning("The data type of property '%s' does not support from_sparql_value(), skipping the reference snak", prop_nr) + continue if srid not in reference: reference[srid] = Reference() reference[srid].add(f) - # Add each Reference to the References - for _, ref in reference.items(): - references.add(ref) - if len(results) == 0 or len(results) < limit: break + references: References = References() + for _, ref in reference.items(): + references.add(ref) + + self._references_cache[sid] = references + return references - def _load_rank(self, sid: str) -> WikibaseRank | None: + def _load_rank(self, sid: str, cache: bool | None = None) -> WikibaseRank | None: """ Load the rank of a statement. :param sid: A statement ID. - :param limit: The limit to request at one time. - :return: A References object. + :param cache: Reuse the rank already loaded for this statement. Enabled by default. + :return: The WikibaseRank of the statement or None if the statement is not found. """ - if not isinstance(sid, str): raise ValueError('sid must be a string') - # TODO: Add limit? + if cache is None: + cache = self.cache - # TODO: Add cache + if cache and sid in self._rank_cache: + return self._rank_cache[sid] query = f''' #Tool: WikibaseIntegrator wbi_fastrun._load_rank @@ -349,17 +437,20 @@ def _load_rank(self, sid: str) -> WikibaseRank | None: results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] + rank: WikibaseRank | None = None for result in results: rank_raw = result['rank']['value'].rsplit('#', 1)[-1] if rank_raw == 'PreferredRank': - return WikibaseRank.PREFERRED + rank = WikibaseRank.PREFERRED elif rank_raw == 'NormalRank': - return WikibaseRank.NORMAL + rank = WikibaseRank.NORMAL elif rank_raw == 'DeprecatedRank': - return WikibaseRank.DEPRECATED + rank = WikibaseRank.DEPRECATED - return None + self._rank_cache[sid] = rank + + return rank def _get_property_type(self, prop_nr: str | int) -> str: """ @@ -391,7 +482,7 @@ def get_entities(self, claims: list[Claim] | Claims | Claim, cache: bool | None Return a list of entities who correspond to the specified claims. :param claims: A list of claims to query the SPARQL endpoint. - :param cache: Put data returned by WDQS in cache. Enabled by default. + :param cache: Put data returned by the SPARQL endpoint in cache. Enabled by default. :param query_limit: Limit the amount of results from the SPARQL server :return: A list of entity ID. """ @@ -400,47 +491,82 @@ def get_entities(self, claims: list[Claim] | Claims | Claim, cache: bool | None elif (not isinstance(claims, list) or not all(isinstance(n, Claim) for n in claims)) and not isinstance(claims, Claims): raise ValueError("claims must be an instance of Claim or Claims or a list of Claim") - self.load_statements(claims=claims, cache=cache, limit=query_limit) - - result = [] - for base_filter in self.base_filter: - sub_result = set() - if isinstance(base_filter, BaseDataType): # TODO: Manage case where filter is a list of BaseDataType - if not base_filter.mainsnak.datavalue: - for claim in claims: - if base_filter.mainsnak.property_number == claim.mainsnak.property_number: - # Add the returned entities to the result list - if claim.get_sparql_value() in self.data[claim.mainsnak.property_number]: - for rez in self.data[claim.mainsnak.property_number][claim.get_sparql_value()]: # type: ignore - sub_result.add(rez['entity'].rsplit('/', 1)[-1]) - else: - if base_filter.mainsnak.property_number in self.data: - if base_filter.get_sparql_value() in self.data[base_filter.mainsnak.property_number]: - for rez in self.data[base_filter.mainsnak.property_number][base_filter.get_sparql_value()]: # type: ignore - sub_result.add(rez['entity'].rsplit('/', 1)[-1]) - else: - continue - result.append(sub_result) + if len(claims) == 0: + raise ValueError("claims must have at least one claim") - if result: - if len(result) > 1: - return list(set(result[0]).intersection(*result[1:])) - return list(result[0]) - else: - return [] + entity_sets: list[set[str]] = [] + for claim in claims: + self.load_statements(claims=claim, cache=cache, limit=query_limit) + value_key = self._value_key(claim) + statements = self.data.get(claim.mainsnak.property_number, {}).get(value_key, []) if value_key is not None else [] + entity_sets.append({self._entity_id(statement['entity']) for statement in statements}) + + return sorted(set.intersection(*entity_sets)) + + def _statement_matches(self, claim: Claim, sid: str, use_qualifiers: bool, use_references: bool, use_rank: bool, cache: bool | None = None) -> bool: + """ + Deeply compare a statement of the Wikibase instance with a local claim holding the same value. + + :param claim: The local claim. + :param sid: The statement ID of the statement of the Wikibase instance. + :param use_qualifiers: Compare the qualifiers. + :param use_references: Compare the references. + :param use_rank: Compare the rank. + :param cache: Reuse the qualifiers, references and rank already loaded for this statement. Enabled by default. + :return: True if the statement matches the claim. + """ + if use_qualifiers: + qualifiers = self._load_qualifiers(sid, cache=cache) + + if qualifiers.count() != claim.qualifiers.count(): + log.debug("Difference in number of qualifiers, '%i' != '%i'", qualifiers.count(), claim.qualifiers.count()) + return False + + for qualifier in qualifiers: + if qualifier not in claim.qualifiers: + log.debug("Difference between two qualifiers") + return False + + if use_references: + references = self._load_references(sid, cache=cache) + + if len(references) != len(claim.references): + log.debug("Difference in number of references, '%i' != '%i'", len(references), len(claim.references)) + return False + + for reference in references: + if reference not in claim.references: + log.debug("Difference between two references") + return False + + if use_rank: + rank = self._load_rank(sid, cache=cache) + + if claim.rank != rank: + log.debug("Difference with the rank") + return False + + return True def write_required(self, claims: list[Claim] | Claims | Claim, entity_filter: list[str] | str | None = None, property_filter: list[str] | str | None = None, - use_qualifiers: bool | None = None, use_references: bool | None = None, use_rank: bool | None = None, cache: bool | None = None, - query_limit: int | None = None) -> bool: + action_if_exists: ActionIfExists = ActionIfExists.REPLACE_ALL, use_qualifiers: bool | None = None, use_references: bool | None = None, + use_rank: bool | None = None, cache: bool | None = None, query_limit: int | None = None) -> bool: """ + Check if a write to the Wikibase instance is required: a write is not required only when at least one entity + already holds, for every claim, a statement with the same value (and the same qualifiers, references and rank, + depending on the flags). - :param claims: + :param claims: The claims proposed to be written. :param entity_filter: Allows you to filter the entities checked. This can be a single entity or a list of entities. - :param property_filter: Allows you to limit the difference comparison to a list of properties + Pass the ID of the entity being edited to check whether this entity specifically already holds the data. + :param property_filter: Allows you to limit the difference comparison to a list of properties. Claims whose + property is not in the filter are ignored. + :param action_if_exists: The action that will be used for the write. With FORCE_APPEND, the statements are + always appended and a write is always required. The other actions check that the claims already exist. :param use_qualifiers: Use qualifiers during fastrun. Enabled by default. :param use_references: Use references during fastrun. Disabled by default. :param use_rank: Use rank during fastrun. Disabled by default. - :param cache: Put data returned by WDQS in cache. Enabled by default. + :param cache: Put data returned by the SPARQL endpoint in cache. Enabled by default. :param query_limit: Limit the amount of results from the SPARQL server :return: a boolean True if a write is required. False otherwise. """ @@ -453,8 +579,16 @@ def write_required(self, claims: list[Claim] | Claims | Claim, entity_filter: li if len(claims) == 0: raise ValueError("claims must have at least one claim") - if entity_filter is not None and isinstance(entity_filter, str): - entity_filter = [entity_filter] + if action_if_exists == ActionIfExists.FORCE_APPEND: + # The new statements are always appended, a write is always required + log.debug("Force append: write required") + return True + + entities_allowed: set[str] | None = None + if entity_filter is not None: + if isinstance(entity_filter, str): + entity_filter = [entity_filter] + entities_allowed = {self._entity_id(entity) for entity in entity_filter} if property_filter is not None and isinstance(property_filter, str): property_filter = [property_filter] @@ -470,107 +604,167 @@ def write_required(self, claims: list[Claim] | Claims | Claim, entity_filter: li if use_rank is None: use_rank = self.use_rank - def contains(in_list, lambda_filter): - for x in in_list: - if lambda_filter(x): - return True - return False + claims_to_check = [claim for claim in claims if claim.mainsnak.property_number in property_filter] + if not claims_to_check: + # Nothing can be verified through the fastrun data + log.debug("No claim matches the property filter: write required") + return True + + # Find, for each claim, the statements holding the same value + candidates: list[tuple[Claim, list[dict[str, str]]]] = [] + for claim in claims_to_check: + self.load_statements(claims=claim, cache=cache, limit=query_limit) - # Get all the potential statements - statements_to_check: dict[str, list[str]] = {} - for claim in claims: - if claim.mainsnak.property_number in property_filter: - self.load_statements(claims=claim, cache=cache, limit=query_limit) - if claim.mainsnak.property_number in self.data: - if not contains(self.data[claim.mainsnak.property_number], (lambda x, c=claim: x == c.get_sparql_value())): - # Checks if a property with this value does not exist, return True if none exist - logging.debug("Value '%s' does not exist for property '%s'", claim.get_sparql_value(), claim.mainsnak.property_number) - return True - # TODO: Doesn't work in the value already exists in another entity - - sparql_value = claim.get_sparql_value() - if sparql_value: - for statement in self.data[claim.mainsnak.property_number][sparql_value]: - if claim.mainsnak.property_number not in statements_to_check: - statements_to_check[claim.mainsnak.property_number] = [] - statements_to_check[claim.mainsnak.property_number].append(statement['entity']) - - # Generate an intersection between all the statements by property, based on the entity - # Generate only the list of entities - list_entities: list[list[str]] = [] - for _, statements in statements_to_check.items(): - # entities = [statement['entity'] for statement in statements_to_check[property]] - list_entities.append(list(set(statements))) - - # Return the intersection between all the list - common_entities: list = list_entities.pop() - for entities in list_entities: - common_entities = list(set(common_entities).intersection(entities)) - - # If there is none common entities, return True because we need a write + value_key = self._value_key(claim) + statements = list(self.data.get(claim.mainsnak.property_number, {}).get(value_key, [])) if value_key is not None else [] + if entities_allowed is not None: + statements = [statement for statement in statements if self._entity_id(statement['entity']) in entities_allowed] + + if not statements: + log.debug("Value '%s' does not exist for property '%s'", claim.get_sparql_value(), claim.mainsnak.property_number) + return True + + candidates.append((claim, statements)) + + # The entities holding every claim value + common_entities = set.intersection(*({self._entity_id(statement['entity']) for statement in statements} for _, statements in candidates)) if not common_entities: - logging.debug("There is no common entities") + log.debug("No entity holds all the claim values: write required") return True - # If the property is already found, load it completely to compare deeply - for claim in claims: - # Check if the property is in the filter - if claim.mainsnak.property_number in property_filter: - sparql_value = claim.get_sparql_value() - # If the value exist in the cache - if sparql_value and claim.mainsnak.property_number in self.data and sparql_value in self.data[claim.mainsnak.property_number]: - entity_cache = [statement['entity'].rsplit('/', 1)[-1] for statement in self.data[claim.mainsnak.property_number][sparql_value]] - if entity_filter: - common_cache_filter = [value for value in entity_cache if value in entity_filter] - else: - common_cache_filter = entity_cache - # If there is common entities between the cache and the entity_filter - if common_cache_filter: - for statement in self.data[claim.mainsnak.property_number][sparql_value]: - if entity_filter and statement['entity'].rsplit('/', 1)[-1] not in entity_filter: - continue - - if statement['entity'] in common_entities: - if use_qualifiers: - qualifiers = self._load_qualifiers(statement['sid'], limit=100) - - if len(qualifiers) != len(claim.qualifiers): - logging.debug("Difference in number of qualifiers, '%i' != '%i'", len(qualifiers), len(claim.qualifiers)) - return True - - for qualifier in qualifiers: - if qualifier not in claim.qualifiers: - logging.debug("Difference between two qualifiers") - return True - - if use_references: - references = self._load_references(statement['sid'], limit=100) - - if sum(len(ref) for ref in references) != sum(len(x) for x in claim.references): - logging.debug("Difference in number of references, '%i' != '%i'", sum(len(ref) for ref in references), sum(len(x) for x in claim.references)) - return True - - for reference in references: - if reference not in claim.references: - logging.debug("Difference between two references") - return True - - if use_rank: - rank = self._load_rank(statement['sid']) - - if claim.rank != rank: - logging.debug("Difference with the rank") - return True - else: - logging.debug("No common entities between cache and entity_filter") - return True - # Enable this if the value doesn't exist ? - else: - logging.debug("Value doesn't already exist in an entity") - return True + # Deep comparison: no write is needed if at least one entity holds, for every claim, a statement also + # matching the qualifiers, references and rank, depending on the flags + for entity in sorted(common_entities): + for claim, statements in candidates: + entity_statements = [statement for statement in statements if self._entity_id(statement['entity']) == entity] + if not any(self._statement_matches(claim, statement['sid'], use_qualifiers=use_qualifiers, use_references=use_references, use_rank=use_rank, cache=cache) + for statement in entity_statements): + break + else: + log.debug("Entity '%s' already holds all the claims: no write required", entity) + return False + + return True + + def init_language_data(self, lang: str, lang_data_type: str) -> None: + """ + Initialize language data store + + :param lang: language code + :param lang_data_type: 'label', 'description' or 'aliases' + :return: None + """ + if lang not in self.loaded_langs: + self.loaded_langs[lang] = {} + + if lang_data_type not in self.loaded_langs[lang]: + result = self._query_lang(lang=lang, lang_data_type=lang_data_type) + if result is not None: + data = self._process_lang(result=result) + self.loaded_langs[lang].update({lang_data_type: data}) + + def get_language_data(self, qid: str, lang: str, lang_data_type: str) -> list[str]: + """ + get language data for specified qid + + :param qid: entity ID + :param lang: language code + :param lang_data_type: 'label', 'description' or 'aliases' + :return: list of strings + If nothing is found: + If lang_data_type == label: returns [''] + If lang_data_type == description: returns [''] + If lang_data_type == aliases: returns [] + """ + self.init_language_data(lang, lang_data_type) + + current_lang_data = self.loaded_langs[lang][lang_data_type] + all_lang_strings = list(current_lang_data.get(self._entity_id(qid), [])) + if not all_lang_strings and lang_data_type in {'label', 'description'}: + all_lang_strings = [''] + return all_lang_strings + + def check_language_data(self, qid: str, lang_data: list, lang: str, lang_data_type: str, action_if_exists: ActionIfExists = ActionIfExists.APPEND_OR_REPLACE) -> bool: + """ + Method to check if certain language data exists as a label, description or aliases + + :param qid: entity ID + :param lang_data: list of string values to check + :param lang: language code + :param lang_data_type: What kind of data is it? 'label', 'description' or 'aliases'? + :param action_if_exists: If aliases already exist, APPEND_OR_REPLACE or REPLACE_ALL + :return: boolean True if a write is required + """ + all_lang_strings = {x.strip().casefold() for x in self.get_language_data(qid, lang, lang_data_type)} + + if action_if_exists == ActionIfExists.REPLACE_ALL: + return collections.Counter(all_lang_strings) != collections.Counter(map(lambda x: x.casefold(), lang_data)) + + for s in lang_data: + if s.strip().casefold() not in all_lang_strings: + log.debug("fastrun failed at: %s, string: %s", lang_data_type, s) + return True return False + def _query_lang(self, lang: str, lang_data_type: str) -> list[dict[str, dict]] | None: + """ + Query the SPARQL endpoint for the language data of the entities matching the base filter. + + :param lang: language code + :param lang_data_type: 'label', 'description' or 'aliases' + """ + + lang_data_type_dict = { + 'label': 'rdfs:label', + 'description': 'schema:description', + 'aliases': 'skos:altLabel' + } + + query = f''' + #Tool: WikibaseIntegrator wbi_fastrun._query_lang + SELECT ?entity ?label WHERE {{ + {self._base_filter_string()} + + OPTIONAL {{ + ?entity {lang_data_type_dict[lang_data_type]} ?label FILTER (lang(?label) = "{lang}") . + }} + }} + ''' + + log.debug(query) + + return execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] + + @staticmethod + def _process_lang(result: list) -> defaultdict[str, set]: + data = defaultdict(set) + for r in result: + qid = r['entity']['value'].split("/")[-1] + if 'label' in r: + data[qid].add(r['label']['value']) + return data + + def clear(self) -> None: + """ + Convenience function to empty the caches of this fastrun container. + """ + self.data = {} + self.loaded_complete = set() + self.properties_type = {} + self.loaded_langs = {} + self._qualifiers_cache = {} + self._references_cache = {} + self._rank_cache = {} + + def __repr__(self) -> str: + """A mixin implementing a simple __repr__.""" + return "<{klass} @{id:x} {attrs}>".format( # pylint: disable=consider-using-f-string + klass=self.__class__.__name__, + id=id(self) & 0xFFFFFF, + attrs="\r\n\t ".join(f"{k}={v!r}" for k, v in self.__dict__.items()), + ) + def get_fastrun_container(base_filter: list[BaseDataType | list[BaseDataType]], use_qualifiers: bool = True, use_references: bool = False, use_rank: bool = False, cache: bool = True, case_insensitive: bool = False) -> FastRunContainer: @@ -581,8 +775,8 @@ def get_fastrun_container(base_filter: list[BaseDataType | list[BaseDataType]], :param use_qualifiers: Use qualifiers during fastrun. Enabled by default. :param use_references: Use references during fastrun. Disabled by default. :param use_rank: Use rank during fastrun. Disabled by default. - :param cache: Put data returned by WDQS in cache. Enabled by default. - :param case_insensitive: + :param cache: Put data returned by the SPARQL endpoint in cache. Enabled by default. + :param case_insensitive: Compare the string values without taking the case into account. Disabled by default. :return: a FastRunContainer object """ if base_filter is None: @@ -604,8 +798,8 @@ def _search_fastrun_store(base_filter: list[BaseDataType | list[BaseDataType]], :param use_qualifiers: Use qualifiers during fastrun. Enabled by default. :param use_references: Use references during fastrun. Disabled by default. :param use_rank: Use rank during fastrun. Disabled by default. - :param cache: Put data returned by WDQS in cache. Enabled by default. - :param case_insensitive: + :param cache: Put data returned by the SPARQL endpoint in cache. Enabled by default. + :param case_insensitive: Compare the string values without taking the case into account. Disabled by default. :return: a FastRunContainer object """ for fastrun in fastrun_store: From 441b26d11fe1f6813873a3b004a5bfea335728d1 Mon Sep 17 00:00:00 2001 From: Myst <1592048+LeMyst@users.noreply.github.com> Date: Tue, 7 Jul 2026 20:35:57 +0200 Subject: [PATCH 3/5] Compare the unit of quantity values in fastrun The simple value of a quantity statement does not carry the unit: two amounts only differing by their unit were considered equal and a required write could be skipped. load_statements() now loads the unit from the value node (OPTIONAL, so only quantities bind it) and the normalized unit becomes part of the comparison key. The unit is normalized to the format of the JSON representation: the RDF export of any Wikibase instance represents a unitless quantity with the Wikidata Q199 entity, which maps back to '1', and the other units are reduced to their bare entity ID. Co-Authored-By: Claude Fable 5 --- test/test_wbi_fastrun.py | 37 ++++++++++++++++++++++--- wikibaseintegrator/wbi_fastrun.py | 46 +++++++++++++++++++++++++------ 2 files changed, 71 insertions(+), 12 deletions(-) diff --git a/test/test_wbi_fastrun.py b/test/test_wbi_fastrun.py index c11a7901..40f465db 100644 --- a/test/test_wbi_fastrun.py +++ b/test/test_wbi_fastrun.py @@ -8,7 +8,7 @@ import pytest from wikibaseintegrator import WikibaseIntegrator, wbi_fastrun -from wikibaseintegrator.datatypes import BaseDataType, ExternalID, Item, String, Time +from wikibaseintegrator.datatypes import BaseDataType, ExternalID, Item, Quantity, String, Time from wikibaseintegrator.wbi_enums import ActionIfExists, WikibaseRank from .conftest import literal, uri @@ -17,9 +17,11 @@ PTYPE_EXTERNAL_ID = 'http://wikiba.se/ontology#ExternalId' PTYPE_ITEM = 'http://wikiba.se/ontology#WikibaseItem' +PTYPE_QUANTITY = 'http://wikiba.se/ontology#Quantity' PTYPE_STRING = 'http://wikiba.se/ontology#String' PTYPE_TIME = 'http://wikiba.se/ontology#Time' XSD_DATETIME = 'http://www.w3.org/2001/XMLSchema#dateTime' +XSD_DECIMAL = 'http://www.w3.org/2001/XMLSchema#decimal' class SparqlData: @@ -58,15 +60,18 @@ def dispatch(self, query: str) -> list[dict]: def _sid(query: str) -> str: return re.search(r'VALUES \?sid \{ <([^>]+)> \}', query).group(1) - def statement(self, entity_id: str, prop_nr: str, value: dict, property_type: str, index: int = 0) -> str: + def statement(self, entity_id: str, prop_nr: str, value: dict, property_type: str, index: int = 0, unit: str | None = None) -> str: """Register a statement binding and return its statement URI.""" sid = f'{self.wikibase.base_url}/entity/statement/{entity_id}-{prop_nr}-{index}' - self.statements.setdefault(prop_nr, []).append({ + binding = { 'entity': uri(f'{self.wikibase.base_url}/entity/{entity_id}'), 'sid': uri(sid), 'value': value, 'property_type': uri(property_type), - }) + } + if unit is not None: + binding['unit'] = uri(unit) + self.statements.setdefault(prop_nr, []).append(binding) return sid def qualifier(self, sid: str, prop_nr: str, value: dict, property_type: str) -> None: @@ -330,6 +335,30 @@ def test_rank_ignored_by_default(self, wikibase, sparql_data, frc): assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')]) is False +class TestWriteRequiredQuantityUnits: + def test_different_unit_requires_write(self, wikibase, sparql_data, frc): + """Two amounts only differing by their unit must not be considered equal.""" + sparql_data.statement('Q99', 'P2046', literal('+42', datatype=XSD_DECIMAL), PTYPE_QUANTITY, unit=f'{wikibase.base_url}/entity/Q712226') + + assert frc.write_required(claims=[Quantity(amount=42, prop_nr='P2046')]) is True + assert frc.write_required(claims=[Quantity(amount=42, unit='Q11573', prop_nr='P2046')]) is True + assert frc.write_required(claims=[Quantity(amount=42, unit='Q712226', prop_nr='P2046')]) is False + + def test_unitless_quantity_q199_normalization(self, wikibase, sparql_data, frc): + """The RDF always represents a unitless quantity with the Wikidata Q199 entity, even on another instance.""" + sparql_data.statement('Q99', 'P1082', literal('+1234', datatype=XSD_DECIMAL), PTYPE_QUANTITY, unit='http://www.wikidata.org/entity/Q199') + + assert frc.write_required(claims=[Quantity(amount=1234, prop_nr='P1082')]) is False + assert frc.write_required(claims=[Quantity(amount=1234, unit='Q712226', prop_nr='P1082')]) is True + + def test_get_entities_with_unit(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P2046', literal('+42', datatype=XSD_DECIMAL), PTYPE_QUANTITY, unit=f'{wikibase.base_url}/entity/Q712226') + sparql_data.statement('Q100', 'P2046', literal('+42', datatype=XSD_DECIMAL), PTYPE_QUANTITY, index=1, unit='http://www.wikidata.org/entity/Q199') + + assert frc.get_entities(claims=Quantity(amount=42, unit='Q712226', prop_nr='P2046')) == ['Q99'] + assert frc.get_entities(claims=Quantity(amount=42, prop_nr='P2046')) == ['Q100'] + + class TestCaseInsensitive: def test_case_insensitive_value(self, wikibase, sparql_data): sparql_data.statement('Q99', 'P1', literal('FooBar'), PTYPE_STRING) diff --git a/wikibaseintegrator/wbi_fastrun.py b/wikibaseintegrator/wbi_fastrun.py index 9cee9e14..8a51e468 100644 --- a/wikibaseintegrator/wbi_fastrun.py +++ b/wikibaseintegrator/wbi_fastrun.py @@ -19,6 +19,10 @@ fastrun_store: list[FastRunContainer] = [] +# The RDF export of a Wikibase instance always represents a quantity without a unit ('1' in the JSON representation) +# with the Wikidata entity Q199 (the number one), whatever the instance. +UNITLESS_UNIT_URIS = ('1', 'http://www.wikidata.org/entity/Q199', 'https://www.wikidata.org/entity/Q199') + class FastRunContainer: """ @@ -87,11 +91,32 @@ def _datatype_class(self, property_type: str) -> type[BaseDataType]: return subclass raise ValueError(f"No data type class found for the property type '{property_type}'") + @classmethod + def _normalize_unit(cls, unit: str) -> str: + """ + Normalize a unit to the format used in the comparison keys: '1' for a unitless quantity (represented by the + Wikidata Q199 entity in the RDF export, whatever the instance) and the bare entity ID otherwise. + + :param unit: A unit URI from the SPARQL results or from the JSON representation, or '1' + """ + if unit in UNITLESS_UNIT_URIS: + return '1' + return cls._entity_id(unit) + def _value_key(self, claim: Claim) -> str | None: - """The key indexing the value of the given claim in self.data, casefolded when case_insensitive is enabled.""" + """ + The key indexing the value of the given claim in self.data, casefolded when case_insensitive is enabled. + The normalized unit is part of the key for quantities: two amounts only differing by their unit must not + be considered equal. + """ value = claim.get_sparql_value() - if value is not None and self.case_insensitive: + if value is None: + return None + if self.case_insensitive: value = value.casefold() + datavalue = claim.mainsnak.datavalue + if isinstance(datavalue, dict) and datavalue.get('type') == 'quantity': + value += '@' + self._normalize_unit(datavalue['value'].get('unit', '1')) return value def _base_filter_string(self, wb_url: str | None = None) -> str: @@ -189,7 +214,7 @@ def load_statements(self, claims: list[Claim] | Claims | Claim, cache: bool | No if partial_load: query = ''' #Tool: WikibaseIntegrator wbi_fastrun.load_statements - SELECT ?entity ?sid ?value ?property_type WHERE {{ + SELECT ?entity ?sid ?value ?property_type ?unit WHERE {{ # Base filter string {base_filter_string} ?entity <{wb_url}/prop/{prop_nr}> ?sid. @@ -197,6 +222,8 @@ def load_statements(self, claims: list[Claim] | Claims | Claim, cache: bool | No ?sid <{wb_url}/prop/statement/{prop_nr}> ?value. ?sid <{wb_url}/prop/statement/{prop_nr}> {value}. {qualifiers_filter_string} + # The unit of a quantity value, only bound for quantities + OPTIONAL {{ ?sid <{wb_url}/prop/statement/value/{prop_nr}> [ wikibase:quantityUnit ?unit ] . }} }} ORDER BY ?sid OFFSET {offset} @@ -209,12 +236,14 @@ def load_statements(self, claims: list[Claim] | Claims | Claim, cache: bool | No else: query = ''' #Tool: WikibaseIntegrator wbi_fastrun.load_statements - SELECT ?entity ?sid ?value ?property_type WHERE {{ + SELECT ?entity ?sid ?value ?property_type ?unit WHERE {{ # Base filter string {base_filter_string} ?entity <{wb_url}/prop/{prop_nr}> ?sid. <{wb_url}/entity/{prop_nr}> wikibase:propertyType ?property_type. ?sid <{wb_url}/prop/statement/{prop_nr}> ?value. + # The unit of a quantity value, only bound for quantities + OPTIONAL {{ ?sid <{wb_url}/prop/statement/value/{prop_nr}> [ wikibase:quantityUnit ?unit ] . }} }} ORDER BY ?sid OFFSET {offset} @@ -246,11 +275,12 @@ def load_statements(self, claims: list[Claim] | Claims | Claim, cache: bool | No log.warning("The data type of property '%s' does not support from_sparql_value(), skipping the value", prop_nr) continue - sparql_value = f.get_sparql_value() - if sparql_value is not None: - if self.case_insensitive: - sparql_value = sparql_value.casefold() + # The simple value of a quantity does not carry the unit, set it from the value node + if 'unit' in result and isinstance(f.mainsnak.datavalue, dict) and f.mainsnak.datavalue.get('type') == 'quantity': + f.mainsnak.datavalue['value']['unit'] = result['unit']['value'] + sparql_value = self._value_key(f) + if sparql_value is not None: if sparql_value not in self.data[prop_nr]: self.data[prop_nr][sparql_value] = [] From 4998cd6b638c56aa5d3e9c1eecb65d0cf96d5cec Mon Sep 17 00:00:00 2001 From: Myst <1592048+LeMyst@users.noreply.github.com> Date: Wed, 8 Jul 2026 21:37:53 +0200 Subject: [PATCH 4/5] Fix mypy errors in the fastrun tests - Guard the re.search() results before reading the match groups - Import ItemEntity instead of a string annotation Co-Authored-By: Claude Fable 5 --- test/test_wbi_fastrun.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/test_wbi_fastrun.py b/test/test_wbi_fastrun.py index 40f465db..45df2981 100644 --- a/test/test_wbi_fastrun.py +++ b/test/test_wbi_fastrun.py @@ -9,6 +9,7 @@ from wikibaseintegrator import WikibaseIntegrator, wbi_fastrun from wikibaseintegrator.datatypes import BaseDataType, ExternalID, Item, Quantity, String, Time +from wikibaseintegrator.entities import ItemEntity from wikibaseintegrator.wbi_enums import ActionIfExists, WikibaseRank from .conftest import literal, uri @@ -52,13 +53,16 @@ def dispatch(self, query: str) -> list[dict]: if 'wbi_fastrun._query_lang' in query: return self.labels if 'wbi_fastrun.load_statements' in query: - prop_nr = re.search(r'/prop/(P\d+)> \?sid', query).group(1) - return self.statements.get(prop_nr, []) + match = re.search(r'/prop/(P\d+)> \?sid', query) + assert match is not None + return self.statements.get(match.group(1), []) return [] @staticmethod def _sid(query: str) -> str: - return re.search(r'VALUES \?sid \{ <([^>]+)> \}', query).group(1) + match = re.search(r'VALUES \?sid \{ <([^>]+)> \}', query) + assert match is not None + return match.group(1) def statement(self, entity_id: str, prop_nr: str, value: dict, property_type: str, index: int = 0, unit: str | None = None) -> str: """Register a statement binding and return its statement URI.""" @@ -377,7 +381,7 @@ def test_case_sensitive_by_default(self, wikibase, sparql_data): class TestEntityWriteRequired: """write_required() through an entity: the check is restricted to the entity being edited.""" - def _item(self, item_id: str | None, claims: list) -> 'ItemEntity': # noqa: F821 + def _item(self, item_id: str | None, claims: list) -> ItemEntity: item = wbi.item.new() if item_id: item.id = item_id From 195ee1d666c69ba7adb67b68c821913110f6e11f Mon Sep 17 00:00:00 2001 From: Myst <1592048+LeMyst@users.noreply.github.com> Date: Wed, 8 Jul 2026 21:44:42 +0200 Subject: [PATCH 5/5] Fix codespell typo: unparseable -> unparsable Co-Authored-By: Claude Sonnet 5 --- wikibaseintegrator/wbi_fastrun.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wikibaseintegrator/wbi_fastrun.py b/wikibaseintegrator/wbi_fastrun.py index 8a51e468..30f52ddb 100644 --- a/wikibaseintegrator/wbi_fastrun.py +++ b/wikibaseintegrator/wbi_fastrun.py @@ -345,7 +345,7 @@ def _load_qualifiers(self, sid: str, limit: int | None = None, cache: bool | Non try: f = self._datatype_class(property_type)(prop_nr=prop_nr).from_sparql_value(sparql_value=result['value']) except ValueError as exception: - # An unparseable qualifier can't be compared, leave it out so that the comparison fails safely + # An unparsable qualifier can't be compared, leave it out so that the comparison fails safely log.warning("Skipping a qualifier of statement '%s': %s", sid, exception) continue @@ -416,7 +416,7 @@ def _load_references(self, sid: str, limit: int | None = None, cache: bool | Non try: f = self._datatype_class(property_type)(prop_nr=prop_nr).from_sparql_value(sparql_value=result['ref_value']) except ValueError as exception: - # An unparseable reference snak can't be compared, leave it out so that the comparison fails safely + # An unparsable reference snak can't be compared, leave it out so that the comparison fails safely log.warning("Skipping a reference snak of statement '%s': %s", sid, exception) continue