diff --git a/README.md b/README.md index 8ab0a58c..7d3d1c90 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,9 @@ The main differences between these two libraries are : * Add OAuth 2.0 login method * Add logging module support -But WikibaseIntegrator lack the "fastrun" functionality implemented in WikidataIntegrator. +WikibaseIntegrator also provides a "fast run" mode, which compares the local data with the live data through the SPARQL +endpoint and only writes to the Wikibase instance when something actually changed (see +[Examples (in "fast run" mode)](#examples-in-fast-run-mode)). # Documentation # @@ -799,21 +801,35 @@ for entrez_id, ensembl in raw_data.items(): # Examples (in "fast run" mode) # +The fast run mode is designed for bots synchronising an external resource with a Wikibase instance, where most of the +entities are usually already up to date. Instead of loading every entity through the MediaWiki API to compare it with +the local data, the fast run mode loads the current state of the entities of the data corpus with paginated SPARQL +queries, then decides locally, entity per entity, if a write is required. When nothing changed (the most common case), +no MediaWiki API call is made at all for that entity. + In order to use the fast run mode, you need to know the property/value combination which determines the data corpus you would like to operate on. E.g. for operating on human genes, you need to know that [P351](https://www.wikidata.org/entity/P351) is the NCBI Entrez Gene ID and you also need to know that you are dealing with humans, best represented by the [found in taxon property (P703)](https://www.wikidata.org/entity/P703) with the value [Q15978631](https://www.wikidata.org/entity/Q15978631) for Homo sapiens. -IMPORTANT: In order for the fast run mode to work, the data you provide in the constructor must contain at least one -unique value/id only present on one Wikidata element, e.g. an NCBI entrez gene ID, Uniprot ID, etc. Usually, these would -be the same unique core properties used for defining domains in wbi_core, e.g. for genes, proteins, drugs or your custom -domains. +IMPORTANT: In order for the fast run mode to work on entities without a known entity ID, the data you provide must +contain at least one unique value/id only present on one Wikibase entity, e.g. an NCBI entrez gene ID, Uniprot ID, etc. +This unique value is used to identify the target entity. If the entity ID is already known (e.g. the entity was loaded +with `wbi.item.get()`), the comparison is automatically restricted to that entity and this requirement does not apply. + +## The base filter ## + +The base filter defines the data corpus and is a list of datatype instances (the same classes used to create claims). +Three forms are supported: -Below, the normal mode run example from above, slightly modified, to meet the requirements for the fast run mode. To -enable it, ItemEngine requires two parameters, fast_run=True/False and fast_run_base_filter which is a dictionary -holding the properties to filter for as keys, and the item QIDs as dict values. If the value is not a QID but a literal, -just provide an empty string. For the above example, the dictionary looks like this: +* A property with a value: the entities must have this exact statement, e.g. `Item(prop_nr='P703', value='Q15978631')` + (found in taxon Homo sapiens). +* A property without a value: the entities must have a statement with this property, whatever the value, e.g. + `ExternalID(prop_nr='P351')` (any NCBI Entrez Gene ID). +* A property path, given as a list of two datatype instances: the value is reached through the first property followed + by any number of hops with the second one, e.g. `[Item(prop_nr='P31', value='Q11173'), Item(prop_nr='P279')]` + (instance of (P31) chemical compound (Q11173), directly or through a chain of subclass of (P279)). ```python from wikibaseintegrator.datatypes import ExternalID, Item @@ -821,20 +837,25 @@ from wikibaseintegrator.datatypes import ExternalID, Item fast_run_base_filter = [ExternalID(prop_nr='P351'), Item(prop_nr='P703', value='Q15978631')] ``` -The full example: +## Checking if a write is required ## + +The entry point is `entity.write_required()`. It returns `True` if the local entity differs from the live data and an +actual write is needed, `False` if the entity is already up to date and the write can be skipped. The full example, a +modified version of the mass import bot from the normal mode example: ```python from wikibaseintegrator import WikibaseIntegrator, wbi_login -from wikibaseintegrator.datatypes import ExternalID, Item, String, Time +from wikibaseintegrator.datatypes import ExternalID, Item, Time from wikibaseintegrator.wbi_enums import WikibaseTimePrecision # login object login = wbi_login.OAuth2(consumer_token='', consumer_secret='') -fast_run_base_filter = [ExternalID(prop_nr='P351'), Item(prop_nr='P703', value='Q15978631')] -fast_run = True +wbi = WikibaseIntegrator(login=login) -# We have raw data, which should be written to Wikidata, namely two human NCBI entrez gene IDs mapped to two Ensembl Gene IDs +fast_run_base_filter = [ExternalID(prop_nr='P351'), ExternalID(prop_nr='P704'), Item(prop_nr='P703', value='Q15978631')] + +# We have raw data, which should be written to Wikidata, namely two human NCBI entrez gene IDs mapped to two Ensembl transcript IDs # You can iterate over any data source as long as you can map the values to Wikidata properties. raw_data = { '50943': 'ENST00000376197', @@ -845,31 +866,90 @@ for entrez_id, ensembl in raw_data.items(): # add some references references = [ [ - Item(value='Q20641742', prop_nr='P248') - ], - [ - Time(time='+2020-02-08T00:00:00Z', prop_nr='P813', precision=WikibaseTimePrecision.DAY), - ExternalID(value='1017', prop_nr='P351') + Item(value='Q20641742', prop_nr='P248'), + Time(time='+2020-02-08T00:00:00Z', prop_nr='P813', precision=WikibaseTimePrecision.DAY) ] ] - # data type object - entrez_gene_id = String(value=entrez_id, prop_nr='P351', references=references) - ensembl_transcript_id = String(value=ensembl, prop_nr='P704', references=references) + # data type objects + entrez_gene_id = ExternalID(value=entrez_id, prop_nr='P351', references=references) + ensembl_transcript_id = ExternalID(value=ensembl, prop_nr='P704', references=references) # data goes into a list, because many data objects can be provided to data = [entrez_gene_id, ensembl_transcript_id] - # Search for and then edit/create new item - wb_item = WikibaseIntegrator(login=login).item.new() - wb_item.add_claims(claims=data) - wb_item.init_fastrun(base_filter=fast_run_base_filter) - wb_item.write() + item = wbi.item.new() + item.claims.add(data) + + # Compare the local data with the live data and only write when something differs + if item.write_required(base_filter=fast_run_base_filter): + item.write() ``` -Note: Fastrun mode checks for equality of property/value pairs, qualifiers (not including qualifier attributes), labels, -aliases and description, but it ignores references by default! -References can be checked in fast run mode by setting `use_refs` to `True`. +Note: only the claims whose property appears in the base filter are compared. In the example above, the P351 and P704 +claims are checked because both properties are part of the base filter; a claim on any other property would be ignored +by the comparison. A write is reported as not required only when one entity (the entity being edited, when its ID is +known) holds all the compared claims. + +## Options ## + +`write_required()` accepts optional parameters, forwarded to the fastrun container: + +* `use_qualifiers` (default `True`): also compare the qualifiers of the statements. +* `use_references` (default `False`): also compare the references of the statements. By default, references are + ignored. +* `use_rank` (default `False`): also compare the rank of the statements. +* `case_insensitive` (default `False`): compare string values case-insensitively. The comparison of qualifiers, + references and ranks stays case sensitive. +* `cache` (default `True`): keep the data returned by the SPARQL endpoint in memory and reuse it for the next + checks. When disabled, the queries are restricted to the exact values of the claims instead of preloading every + statement of the property. +* `action_if_exists` (default `ActionIfExists.REPLACE_ALL`): the action that will be used for the write. With + `FORCE_APPEND`, the statements are always appended and a write is always reported as required. With the other + actions, the claims must already exist on the entity for the write to be skipped. + +The preloaded data is shared: containers are cached at the module level and reused by every `write_required()` call +using the same base filter and options, so the SPARQL queries are only executed once per property. + +## Checking labels, descriptions and aliases ## + +`write_required()` only compares claims. To check language data (labels, descriptions or aliases), use the fastrun +container directly: + +```python +from wikibaseintegrator import wbi_fastrun +from wikibaseintegrator.datatypes import ExternalID, Item + +frc = wbi_fastrun.get_fastrun_container(base_filter=[ExternalID(prop_nr='P351'), Item(prop_nr='P703', value='Q15978631')]) + +# Resolve the entity ID from a unique value, without any MediaWiki API call +qids = frc.get_entities(claims=[ExternalID(value='50943', prop_nr='P351')]) + +# Returns True if the English label differs from 'CDK7' and a write is required +frc.check_language_data(qid=qids[0], lang_data=['CDK7'], lang='en', lang_data_type='label') +``` + +## Limitations ## + +* The SPARQL endpoint can lag behind the live data (typically a few seconds to a few minutes on Wikidata). A write + that just happened may not be visible yet, and the same write could be reported as required again. +* The attributes missing from the SPARQL simple values are rebuilt with their default value: the precision of a time + value is inferred from the timestamp, the bounds of a quantity are not compared, etc. Statements using non-default + attributes are always reported as requiring a write. +* The datatypes that do not implement `from_sparql_value()` yet (e.g. Form, Sense, Property, Lexeme, GeoShape, + TabularData) cannot be compared: their statements are always reported as requiring a write. +* The qualifiers, references and ranks are loaded lazily with one SPARQL query per statement to check. This is fast + when most entities are up to date, but can slow down the run when many entities hold the same values. + +## Performance statistics ## + +Measured on a synchronisation bot of French administrative entities with Wikidata. The "partial fastrun" column is the +mode without cache, where the queries are restricted to the exact values of the claims: + +| Dataset | partial fastrun | fastrun without qualifiers/references | fastrun with qualifiers | fastrun with qualifiers/references | +|:----------------------------|----------------:|--------------------------------------:|------------------------:|-----------------------------------:| +| Communes (34990 elements) | ? | 7min | 30s | 60s | +| Départements (100 elements) | 70min | 1s | 30s | 60s | # Debugging # diff --git a/pyproject.toml b/pyproject.toml index c53be093..e18016ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,6 +120,7 @@ disable = [ [tool.pytest.ini_options] log_cli = true +log_cli_level = 'DEBUG' testpaths = ["test"] markers = [ "integration: tests running against a real Wikibase instance (deselected by default, see test/integration/README.md)" diff --git a/test/test_datatypes.py b/test/test_datatypes.py index e2bb420f..cacac655 100644 --- a/test/test_datatypes.py +++ b/test/test_datatypes.py @@ -163,7 +163,7 @@ def test_all_datatypes(self): MonolingualText(text='xxx', language='fr', prop_nr='P7'), Quantity(amount=-5.04, prop_nr='P8'), Quantity(amount=5.06, upper_bound=9.99, lower_bound=-2.22, unit='Q11573', prop_nr='P8'), - CommonsMedia(value='xxx', prop_nr='P9'), + CommonsMedia(value='xxx.jpg', prop_nr='P9'), GlobeCoordinate(latitude=1.2345, longitude=-1.2345, precision=12, prop_nr='P10'), GeoShape(value='Data:xxx.map', prop_nr='P11'), Property(value='P123', prop_nr='P12'), diff --git a/test/test_wbi_fastrun.py b/test/test_wbi_fastrun.py index 54799266..45df2981 100644 --- a/test/test_wbi_fastrun.py +++ b/test/test_wbi_fastrun.py @@ -1,327 +1,498 @@ """ -Tests for the fastrun container. The SPARQL endpoint and the property -datatype lookups are served by the simulated Wikibase instance, so the whole -pipeline (query -> reverse lookup -> statement reconstruction -> comparison) -runs offline and deterministically. +Tests for the fastrun container. The SPARQL endpoint is served by the simulated +Wikibase instance, so the whole pipeline (statement loading, lazy qualifier / +reference / rank loading, comparison) runs offline and deterministically. """ -from collections import defaultdict -from typing import Any +import re import pytest from wikibaseintegrator import WikibaseIntegrator, wbi_fastrun -from wikibaseintegrator.datatypes import BaseDataType, ExternalID, Item +from wikibaseintegrator.datatypes import BaseDataType, ExternalID, Item, Quantity, String, Time from wikibaseintegrator.entities import ItemEntity -from wikibaseintegrator.wbi_enums import ActionIfExists +from wikibaseintegrator.wbi_enums import ActionIfExists, WikibaseRank -from .conftest import literal, load_fixture, uri +from .conftest import literal, uri wbi = WikibaseIntegrator() - -def statement_bindings(wikibase, item_qid: str, prop_nr: str, values: list[dict], refs: dict | None = None) -> list[dict]: - """Build SPARQL bindings shaped like the ones of wbi_fastrun._query_data.""" - bindings = [] - for index, value in enumerate(values): +PTYPE_EXTERNAL_ID = 'http://wikiba.se/ontology#ExternalId' +PTYPE_ITEM = 'http://wikiba.se/ontology#WikibaseItem' +PTYPE_QUANTITY = 'http://wikiba.se/ontology#Quantity' +PTYPE_STRING = 'http://wikiba.se/ontology#String' +PTYPE_TIME = 'http://wikiba.se/ontology#Time' +XSD_DATETIME = 'http://www.w3.org/2001/XMLSchema#dateTime' +XSD_DECIMAL = 'http://www.w3.org/2001/XMLSchema#decimal' + + +class SparqlData: + """ + Serve the SPARQL queries of the fastrun container from in-memory data. + + Each query type issued by the container (statement loading, qualifiers, + references, rank, language data) is recognized through its #Tool comment + and answered from the matching attribute. + """ + + def __init__(self, wikibase): + self.wikibase = wikibase + self.statements: dict[str, list[dict]] = {} # property number -> bindings + self.qualifiers: dict[str, list[dict]] = {} # statement URI -> bindings + self.references: dict[str, list[dict]] = {} # statement URI -> bindings + self.ranks: dict[str, list[dict]] = {} # statement URI -> bindings + self.labels: list[dict] = [] # language data bindings + wikibase.sparql_bindings = self.dispatch + + def dispatch(self, query: str) -> list[dict]: + if 'wbi_fastrun._load_qualifiers' in query: + return self.qualifiers.get(self._sid(query), []) + if 'wbi_fastrun._load_references' in query: + return self.references.get(self._sid(query), []) + if 'wbi_fastrun._load_rank' in query: + return self.ranks.get(self._sid(query), []) + if 'wbi_fastrun._query_lang' in query: + return self.labels + if 'wbi_fastrun.load_statements' in query: + match = re.search(r'/prop/(P\d+)> \?sid', query) + assert match is not None + return self.statements.get(match.group(1), []) + return [] + + @staticmethod + def _sid(query: str) -> str: + match = re.search(r'VALUES \?sid \{ <([^>]+)> \}', query) + assert match is not None + return match.group(1) + + def statement(self, entity_id: str, prop_nr: str, value: dict, property_type: str, index: int = 0, unit: str | None = None) -> str: + """Register a statement binding and return its statement URI.""" + sid = f'{self.wikibase.base_url}/entity/statement/{entity_id}-{prop_nr}-{index}' binding = { - 'sid': uri(f'{wikibase.base_url}/entity/statement/{item_qid}-{prop_nr}-{index}'), - 'item': uri(f'{wikibase.base_url}/entity/{item_qid}'), - 'v': value, + 'entity': uri(f'{self.wikibase.base_url}/entity/{entity_id}'), + 'sid': uri(sid), + 'value': value, + 'property_type': uri(property_type), } - if refs: - for ref_prop, ref_value in refs.items(): - bindings.append({ - **binding, - 'ref': uri(f'{wikibase.base_url}/reference/deadbeef{index}'), - 'pr': uri(f'{wikibase.base_url}/prop/reference/{ref_prop}'), - 'rval': ref_value, - }) - else: - bindings.append(binding) - return bindings - - -class TestQueryData: - def test_query_data(self, wikibase): + if unit is not None: + binding['unit'] = uri(unit) + self.statements.setdefault(prop_nr, []).append(binding) + return sid + + def qualifier(self, sid: str, prop_nr: str, value: dict, property_type: str) -> None: + self.qualifiers.setdefault(sid, []).append({ + 'property': uri(f'{self.wikibase.base_url}/entity/{prop_nr}'), + 'value': value, + 'property_type': uri(property_type), + }) + + def reference(self, sid: str, prop_nr: str, value: dict, property_type: str, ref_node: str = 'deadbeef') -> None: + self.references.setdefault(sid, []).append({ + 'srid': uri(f'{self.wikibase.base_url}/reference/{ref_node}'), + 'ref_property': uri(f'{self.wikibase.base_url}/entity/{prop_nr}'), + 'ref_value': value, + 'property_type': uri(property_type), + }) + + def rank(self, sid: str, rank: str) -> None: + self.ranks[sid] = [{'rank': uri(f'http://wikiba.se/ontology#{rank}')}] + + def label(self, entity_id: str, value: str, lang: str) -> None: + self.labels.append({ + 'entity': uri(f'{self.wikibase.base_url}/entity/{entity_id}'), + 'label': literal(value, lang=lang), + }) + + +@pytest.fixture +def sparql_data(wikibase): + return SparqlData(wikibase) + + +@pytest.fixture +def frc(wikibase): + return wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P352')], base_data_type=BaseDataType) + + +class TestLoadStatements: + def test_load_statements(self, wikibase, sparql_data, frc): """The container must query the SPARQL endpoint and store the data in its internal format.""" - wikibase.add_property('P352', 'external-id') + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) - frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P352')], base_data_type=BaseDataType) + frc.load_statements(claims=ExternalID(value='P40095', prop_nr='P352')) - wikibase.sparql_bindings = statement_bindings(wikibase, 'Q99', 'P352', [literal('P40095')]) - frc._query_data('P352') + expected_key = ExternalID(value='P40095', prop_nr='P352').get_sparql_value() + assert expected_key in frc.data['P352'] + assert frc.data['P352'][expected_key][0]['entity'] == f'{wikibase.base_url}/entity/Q99' + assert frc.properties_type['P352'] == PTYPE_EXTERNAL_ID + assert 'P352' in frc.loaded_complete - assert 'Q99' in frc.prop_data - assert 'P352' in frc.prop_data['Q99'] + def test_cache_reuse(self, wikibase, sparql_data, frc): + """A complete load must be reused, without a new SPARQL query.""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) - statement_id = list(frc.prop_data['Q99']['P352'].keys())[0] - statement_data = frc.prop_data['Q99']['P352'][statement_id] - assert all(key in statement_data for key in ('qual', 'ref', 'v')) - assert statement_data['v'] == '"P40095"' - assert frc.rev_lookup['"P40095"'] == {'Q99'} + frc.load_statements(claims=ExternalID(value='P40095', prop_nr='P352')) + queries_after_first_load = len(wikibase.sparql_queries) - def test_query_data_item_and_uri_values(self, wikibase): - wikibase.add_property('P828', 'wikibase-item') - wikibase.add_property('P2888', 'url') + frc.load_statements(claims=ExternalID(value='something-else', prop_nr='P352')) + assert len(wikibase.sparql_queries) == queries_after_first_load - frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P828')], base_data_type=BaseDataType) + def test_partial_load_is_not_reused_as_cache(self, wikibase, sparql_data, frc): + """A load restricted to a value (cache disabled) must not be reused as a complete cache.""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) - # wikibase-item value: the entity URI is reduced to its ID - wikibase.sparql_bindings = statement_bindings(wikibase, 'Q99', 'P828', [uri(f'{wikibase.base_url}/entity/Q18228398')]) - frc._query_data('P828') - assert list(frc.prop_data['Q99']['P828'].values())[0]['v'] == 'Q18228398' + frc.load_statements(claims=ExternalID(value='P40095', prop_nr='P352'), cache=False) + assert 'P352' not in frc.loaded_complete + # The query is restricted to the value of the claim + assert '"P40095"' in wikibase.sparql_queries[-1] - # url value: kept as an URI - wikibase.sparql_bindings = statement_bindings(wikibase, 'Q99', 'P2888', [uri('http://purl.obolibrary.org/obo/DOID_1432')]) - frc._query_data('P2888') - values = {statement['v'] for statement in frc.prop_data['Q99']['P2888'].values()} - assert all(value.startswith(' queries_after_partial_load + assert 'P352' in frc.loaded_complete - def test_query_data_ref(self, wikibase): - wikibase.add_property('P352', 'external-id') - wikibase.add_property('P248', 'wikibase-item') + def test_unparseable_value_is_skipped(self, wikibase, sparql_data, frc): + """A value the data type can't represent must be skipped, not crash the load.""" + # A timestamp with a time part: its precision can't be inferred by Time.set_value() + sparql_data.statement('Q99', 'P580', literal('2020-02-08T12:34:56Z', datatype=XSD_DATETIME), PTYPE_TIME) + sparql_data.statement('Q99', 'P580', literal('2020-02-08T00:00:00Z', datatype=XSD_DATETIME), PTYPE_TIME, index=1) - frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P352')], base_data_type=BaseDataType, use_refs=True) + frc.load_statements(claims=Time(time='+2020-02-08T00:00:00Z', prop_nr='P580')) - wikibase.sparql_bindings = statement_bindings(wikibase, 'Q99', 'P352', [literal('P40095')], refs={'P248': uri(f'{wikibase.base_url}/entity/Q1234')}) - frc._query_data('P352') + assert len(frc.data['P580']) == 1 - statement_id = list(frc.prop_data['Q99']['P352'].keys())[0] - statement_data = frc.prop_data['Q99']['P352'][statement_id] - assert len(statement_data['ref']) == 1 - reference = list(statement_data['ref'].values())[0] - assert ('P248', 'Q1234') in reference + def test_invalid_claims(self, frc): + with pytest.raises(ValueError): + frc.load_statements(claims='not a claim') -class TestWriteRequiredEndToEnd: - """Full fastrun pipeline: SPARQL query, reconstruction and comparison.""" +class TestGetEntities: + def test_get_entities(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.statement('Q100', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID, index=1) - @pytest.fixture - def frc(self, wikibase): - # According to the SPARQL endpoint, Q582 already holds P352 = 'P40095'. - wikibase.add_property('P352', 'external-id') - wikibase.sparql_bindings = statement_bindings(wikibase, 'Q582', 'P352', [literal('P40095')]) - return wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P352'), Item(prop_nr='P703', value='Q27510868')], base_data_type=BaseDataType) + assert frc.get_entities(claims=ExternalID(value='P40095', prop_nr='P352')) == ['Q100', 'Q99'] - def test_write_not_required_when_data_matches(self, frc): - assert frc.write_required(data=[ExternalID(value='P40095', prop_nr='P352')]) is False + def test_get_entities_intersection(self, wikibase, sparql_data, frc): + """With several claims, only the entities holding every value are returned.""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.statement('Q100', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID, index=1) + sparql_data.statement('Q99', 'P828', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) - def test_write_required_when_value_differs(self, frc): - assert frc.write_required(data=[ExternalID(value='DIFFERENT', prop_nr='P352')]) is True + claims = [ExternalID(value='P40095', prop_nr='P352'), Item(value='Q42', prop_nr='P828')] + assert frc.get_entities(claims=claims) == ['Q99'] - def test_write_required_via_entity(self, wikibase, frc, item_q582): - """BaseEntity.write_required goes through the shared fastrun store.""" - wbi_fastrun.fastrun_store.append(frc) + def test_get_entities_no_match(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) - item = ItemEntity().from_json(load_fixture('item_Q582')) - item.claims.add(ExternalID(value='P40095', prop_nr='P352')) - assert item.write_required(base_filter=[BaseDataType(prop_nr='P352'), Item(prop_nr='P703', value='Q27510868')]) is False + assert frc.get_entities(claims=ExternalID(value='unknown-value', prop_nr='P352')) == [] - item.claims.add(ExternalID(value='CHANGED', prop_nr='P352')) - assert item.write_required(base_filter=[BaseDataType(prop_nr='P352'), Item(prop_nr='P703', value='Q27510868')]) is True - def test_write_required_with_property_path_base_filter(self, wikibase, item_q582): - """ - A property-path base_filter (a list of BaseDataType) must still select the matching claims. +class TestWriteRequired: + def test_not_required_when_value_exists(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')]) is False + + def test_required_when_value_differs(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + assert frc.write_required(claims=[ExternalID(value='something-else', prop_nr='P352')]) is True + + def test_entity_filter(self, wikibase, sparql_data, frc): + """The data exists on Q99: no write is required for Q99, a write is required for another entity.""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + claims = [ExternalID(value='P40095', prop_nr='P352')] + assert frc.write_required(claims=claims, entity_filter='Q99') is False + assert frc.write_required(claims=claims, entity_filter='Q100') is True + # A full entity URI is accepted too + assert frc.write_required(claims=claims, entity_filter=f'{wikibase.base_url}/entity/Q99') is False + + def test_required_when_no_common_entity(self, wikibase, sparql_data, frc): + """The two values exist, but on two different entities: a write is required.""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.statement('Q100', 'P828', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + + claims = [ExternalID(value='P40095', prop_nr='P352'), Item(value='Q42', prop_nr='P828')] + assert frc.write_required(claims=claims) is True + + def test_not_required_when_common_entity(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.statement('Q99', 'P828', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) - Regression: the anchor property (here P352) was never matched, so a write was always wrongly reported. + claims = [ExternalID(value='P40095', prop_nr='P352'), Item(value='Q42', prop_nr='P828')] + assert frc.write_required(claims=claims) is False + + def test_property_filter(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + claims = [ExternalID(value='P40095', prop_nr='P352'), ExternalID(value='unchecked', prop_nr='P999')] + # P999 is filtered out: only P352 is compared, no write is required + assert frc.write_required(claims=claims, property_filter='P352') is False + + def test_required_when_no_claim_matches_the_property_filter(self, wikibase, sparql_data, frc): + """When nothing can be verified, a write must be reported, not an error raised.""" + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')], property_filter=['P999']) is True + + def test_force_append_always_requires_write(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + claims = [ExternalID(value='P40095', prop_nr='P352')] + assert frc.write_required(claims=claims, action_if_exists=ActionIfExists.FORCE_APPEND) is True + # The check short-circuits without querying the endpoint + assert len(wikibase.sparql_queries) == 0 + + def test_empty_claims(self, frc): + with pytest.raises(ValueError): + frc.write_required(claims=[]) + + +class TestWriteRequiredQualifiers: + def test_missing_qualifier_on_the_claim(self, wikibase, sparql_data, frc): + """The statement has a qualifier, the claim doesn't: a write is required.""" + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.qualifier(sid, 'P642', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')]) is True + + def test_matching_qualifier(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.qualifier(sid, 'P642', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + + claim = ExternalID(value='P40095', prop_nr='P352', qualifiers=[Item(value='Q42', prop_nr='P642')]) + assert frc.write_required(claims=[claim]) is False + + def test_different_qualifier_value(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.qualifier(sid, 'P642', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + + claim = ExternalID(value='P40095', prop_nr='P352', qualifiers=[Item(value='Q43', prop_nr='P642')]) + assert frc.write_required(claims=[claim]) is True + + def test_qualifiers_can_be_ignored(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.qualifier(sid, 'P642', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')], use_qualifiers=False) is False + + def test_duplicate_statements_do_not_mask_a_match(self, wikibase, sparql_data, frc): + """ + The entity holds two statements with the same value, one with a qualifier and one without: a claim matching + either of them requires no write, whatever the statement order. """ - wikibase.add_property('P352', 'external-id') - wikibase.add_property('P703', 'wikibase-item') - wikibase.sparql_bindings = statement_bindings(wikibase, 'Q582', 'P352', [literal('P40095')]) + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID, index=0) + sparql_data.qualifier(sid, 'P642', uri(f'{wikibase.base_url}/entity/Q42'), PTYPE_ITEM) + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID, index=1) - base_filter = [[BaseDataType(prop_nr='P352'), BaseDataType(prop_nr='P703')]] - wbi_fastrun.fastrun_store.append(wbi_fastrun.FastRunContainer(base_filter=base_filter, base_data_type=BaseDataType)) + without_qualifier = ExternalID(value='P40095', prop_nr='P352') + with_qualifier = ExternalID(value='P40095', prop_nr='P352', qualifiers=[Item(value='Q42', prop_nr='P642')]) + assert frc.write_required(claims=[without_qualifier]) is False + assert frc.write_required(claims=[with_qualifier]) is False - item = ItemEntity().from_json(load_fixture('item_Q582')) - item.claims.add(ExternalID(value='P40095', prop_nr='P352')) - assert item.write_required(base_filter=base_filter) is False - item.claims.add(ExternalID(value='CHANGED', prop_nr='P352')) - assert item.write_required(base_filter=base_filter) is True +class TestWriteRequiredReferences: + def test_references_ignored_by_default(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.reference(sid, 'P248', uri(f'{wikibase.base_url}/entity/Q1234'), PTYPE_ITEM) + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')]) is False -class TestPropDatatype: - def test_get_prop_datatype_is_cached(self, wikibase): - wikibase.add_property('P352', 'external-id') - frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P352')], base_data_type=BaseDataType) + def test_missing_reference_on_the_claim(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.reference(sid, 'P248', uri(f'{wikibase.base_url}/entity/Q1234'), PTYPE_ITEM) - assert frc.get_prop_datatype('P352') == 'external-id' - calls = len([r for r in wikibase.requests if r.get('action') == 'wbgetentities']) + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')], use_references=True) is True - # A second lookup must be served from the per-instance cache, without any additional API request - assert frc.get_prop_datatype('P352') == 'external-id' - assert len([r for r in wikibase.requests if r.get('action') == 'wbgetentities']) == calls + def test_matching_reference(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.reference(sid, 'P248', uri(f'{wikibase.base_url}/entity/Q1234'), PTYPE_ITEM) - # clear() invalidates the cache - frc.clear() - assert 'P352' not in frc.prop_dt_map + claim = ExternalID(value='P40095', prop_nr='P352', references=[[Item(value='Q1234', prop_nr='P248')]]) + assert frc.write_required(claims=[claim], use_references=True) is False + + def test_different_reference(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.reference(sid, 'P248', uri(f'{wikibase.base_url}/entity/Q1234'), PTYPE_ITEM) + + claim = ExternalID(value='P40095', prop_nr='P352', references=[[Item(value='Q4321', prop_nr='P248')]]) + assert frc.write_required(claims=[claim], use_references=True) is True + + +class TestWriteRequiredRank: + def test_matching_rank(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.rank(sid, 'NormalRank') + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')], use_rank=True) is False + + def test_different_rank(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.rank(sid, 'PreferredRank') + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')], use_rank=True) is True + + def test_rank_ignored_by_default(self, wikibase, sparql_data, frc): + sid = sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.rank(sid, 'PreferredRank') + + assert frc.write_required(claims=[ExternalID(value='P40095', prop_nr='P352')]) is False + + +class TestWriteRequiredQuantityUnits: + def test_different_unit_requires_write(self, wikibase, sparql_data, frc): + """Two amounts only differing by their unit must not be considered equal.""" + sparql_data.statement('Q99', 'P2046', literal('+42', datatype=XSD_DECIMAL), PTYPE_QUANTITY, unit=f'{wikibase.base_url}/entity/Q712226') + + assert frc.write_required(claims=[Quantity(amount=42, prop_nr='P2046')]) is True + assert frc.write_required(claims=[Quantity(amount=42, unit='Q11573', prop_nr='P2046')]) is True + assert frc.write_required(claims=[Quantity(amount=42, unit='Q712226', prop_nr='P2046')]) is False + + def test_unitless_quantity_q199_normalization(self, wikibase, sparql_data, frc): + """The RDF always represents a unitless quantity with the Wikidata Q199 entity, even on another instance.""" + sparql_data.statement('Q99', 'P1082', literal('+1234', datatype=XSD_DECIMAL), PTYPE_QUANTITY, unit='http://www.wikidata.org/entity/Q199') + + assert frc.write_required(claims=[Quantity(amount=1234, prop_nr='P1082')]) is False + assert frc.write_required(claims=[Quantity(amount=1234, unit='Q712226', prop_nr='P1082')]) is True + + def test_get_entities_with_unit(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P2046', literal('+42', datatype=XSD_DECIMAL), PTYPE_QUANTITY, unit=f'{wikibase.base_url}/entity/Q712226') + sparql_data.statement('Q100', 'P2046', literal('+42', datatype=XSD_DECIMAL), PTYPE_QUANTITY, index=1, unit='http://www.wikidata.org/entity/Q199') + + assert frc.get_entities(claims=Quantity(amount=42, unit='Q712226', prop_nr='P2046')) == ['Q99'] + assert frc.get_entities(claims=Quantity(amount=42, prop_nr='P2046')) == ['Q100'] + + +class TestCaseInsensitive: + def test_case_insensitive_value(self, wikibase, sparql_data): + sparql_data.statement('Q99', 'P1', literal('FooBar'), PTYPE_STRING) + + frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P1')], base_data_type=BaseDataType, case_insensitive=True) + assert frc.write_required(claims=[String(value='foobar', prop_nr='P1')]) is False + + def test_case_sensitive_by_default(self, wikibase, sparql_data): + sparql_data.statement('Q99', 'P1', literal('FooBar'), PTYPE_STRING) + + frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P1')], base_data_type=BaseDataType) + assert frc.write_required(claims=[String(value='foobar', prop_nr='P1')]) is True + assert frc.write_required(claims=[String(value='FooBar', prop_nr='P1')]) is False + + +class TestEntityWriteRequired: + """write_required() through an entity: the check is restricted to the entity being edited.""" + + def _item(self, item_id: str | None, claims: list) -> ItemEntity: + item = wbi.item.new() + if item_id: + item.id = item_id + for claim in claims: + item.claims.add(claim) + return item + + def test_not_required_when_the_entity_holds_the_data(self, wikibase, sparql_data): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + item = self._item('Q99', [ExternalID(value='P40095', prop_nr='P352')]) + assert item.write_required(base_filter=[BaseDataType(prop_nr='P352')]) is False + + def test_required_when_the_data_is_on_another_entity(self, wikibase, sparql_data): + """Q100 is edited: the value existing on Q99 must not inhibit the write.""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + item = self._item('Q100', [ExternalID(value='P40095', prop_nr='P352')]) + assert item.write_required(base_filter=[BaseDataType(prop_nr='P352')]) is True + + def test_new_entity_matches_any_entity(self, wikibase, sparql_data): + """An entity without ID matches any entity holding the data (deduplication use case).""" + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + item = self._item(None, [ExternalID(value='P40095', prop_nr='P352')]) + assert item.write_required(base_filter=[BaseDataType(prop_nr='P352')]) is False + + def test_required_when_no_claim_matches_the_base_filter(self, wikibase, sparql_data): + """No claim of the entity is covered by the base filter: nothing can be verified, a write is required.""" + item = self._item('Q99', [ExternalID(value='P40095', prop_nr='P999')]) + assert item.write_required(base_filter=[BaseDataType(prop_nr='P352')]) is True + + def test_force_append(self, wikibase, sparql_data): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + + item = self._item('Q99', [ExternalID(value='P40095', prop_nr='P352')]) + assert item.write_required(base_filter=[BaseDataType(prop_nr='P352')], action_if_exists=ActionIfExists.FORCE_APPEND) is True class TestLanguageData: - def test_language_data_and_check(self, wikibase): - frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P352')], base_data_type=BaseDataType) + def test_language_data_and_check(self, wikibase, sparql_data, frc): + sparql_data.label('Q582', 'Villeurbanne', 'fr') + + assert frc.get_language_data('Q582', 'fr', 'label') == ['Villeurbanne'] + # The comparison is case insensitive + assert frc.check_language_data('Q582', ['villeurbanne'], 'fr', 'label') is False + assert frc.check_language_data('Q582', ['Lyon'], 'fr', 'label') is True + + def test_empty_language_data(self, wikibase, sparql_data, frc): + assert frc.get_language_data('Q582', 'fr', 'label') == [''] + assert frc.get_language_data('Q582', 'fr', 'aliases') == [] + + def test_language_data_is_cached(self, wikibase, sparql_data, frc): + sparql_data.label('Q582', 'Villeurbanne', 'fr') - wikibase.sparql_bindings = [ - {'item': uri(f'{wikibase.base_url}/entity/Q99'), 'label': literal('Earth', lang='en')}, - ] + frc.get_language_data('Q582', 'fr', 'label') + queries_after_first_load = len(wikibase.sparql_queries) + frc.get_language_data('Q582', 'fr', 'label') + assert len(wikibase.sparql_queries) == queries_after_first_load - assert list(frc.get_language_data('Q99', 'en', 'label')) == ['Earth'] - # Same language data: no write required - assert frc.check_language_data('Q99', ['Earth'], 'en', 'label') is False - # Different language data: write required - assert frc.check_language_data('Q99', ['not the Earth'], 'en', 'label') is True + def test_replace_all_language_data(self, wikibase, sparql_data, frc): + sparql_data.label('Q582', 'Villeurbanne', 'fr') - def test_empty_language_data(self, wikibase): - frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P352')], base_data_type=BaseDataType) - wikibase.sparql_bindings = [] + assert frc.check_language_data('Q582', ['Villeurbanne'], 'fr', 'label', action_if_exists=ActionIfExists.REPLACE_ALL) is False + assert frc.check_language_data('Q582', ['Villeurbanne', 'Lyon'], 'fr', 'label', action_if_exists=ActionIfExists.REPLACE_ALL) is True + + +class TestClear: + def test_clear(self, wikibase, sparql_data, frc): + sparql_data.statement('Q99', 'P352', literal('P40095'), PTYPE_EXTERNAL_ID) + sparql_data.label('Q582', 'Villeurbanne', 'fr') + + frc.load_statements(claims=ExternalID(value='P40095', prop_nr='P352')) + frc.init_language_data('fr', 'label') + + frc.clear() - assert frc.get_language_data('Q99', 'ak', 'label') == [''] - assert frc.get_language_data('Q99', 'ak', 'description') == [''] - assert frc.get_language_data('Q99', 'ak', 'aliases') == [] - assert frc.check_language_data('Q99', [''], 'ak', 'description') is False - assert frc.check_language_data('Q99', [], 'ak', 'aliases') is False + assert not frc.data + assert not frc.loaded_complete + assert not frc.properties_type + assert not frc.loaded_langs class TestFastrunStore: def test_container_is_reused(self, wikibase): - base_filter = [BaseDataType(prop_nr='P352')] - first = wbi_fastrun.get_fastrun_container(base_filter=base_filter) - second = wbi_fastrun.get_fastrun_container(base_filter=base_filter) - assert first is second + frc1 = wbi_fastrun.get_fastrun_container(base_filter=[BaseDataType(prop_nr='P352')]) + frc2 = wbi_fastrun.get_fastrun_container(base_filter=[BaseDataType(prop_nr='P352')]) + assert frc1 is frc2 def test_different_parameters_create_new_container(self, wikibase): - base_filter = [BaseDataType(prop_nr='P352')] - first = wbi_fastrun.get_fastrun_container(base_filter=base_filter) - second = wbi_fastrun.get_fastrun_container(base_filter=base_filter, use_refs=True) - assert first is not second + frc1 = wbi_fastrun.get_fastrun_container(base_filter=[BaseDataType(prop_nr='P352')]) + frc2 = wbi_fastrun.get_fastrun_container(base_filter=[BaseDataType(prop_nr='P921')]) + frc3 = wbi_fastrun.get_fastrun_container(base_filter=[BaseDataType(prop_nr='P352')], use_references=True) + frc4 = wbi_fastrun.get_fastrun_container(base_filter=[BaseDataType(prop_nr='P352')], case_insensitive=True) + assert len({id(frc) for frc in (frc1, frc2, frc3, frc4)}) == 4 -class TestBaseFilters: +class TestBaseFilter: def test_invalid_base_filter(self): with pytest.raises(ValueError): - wbi_fastrun.FastRunContainer(base_filter=['not a datatype'], base_data_type=BaseDataType) + wbi_fastrun.FastRunContainer(base_filter=['P352'], base_data_type=BaseDataType) def test_base_filter_string(self, wikibase): - frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P352'), Item(prop_nr='P703', value='Q27510868')], base_data_type=BaseDataType) - assert f'?item <{wikibase.base_url}/prop/direct/P352> ?zzP352 .' in frc.base_filter_string - assert f'?item <{wikibase.base_url}/prop/direct/P703>' in frc.base_filter_string - - -# --------------------------------------------------------------------- # -# Offline tests based on hand-crafted container contents. They validate -# the write_required comparison logic itself. -# --------------------------------------------------------------------- # - -class FastRunContainerFakeQueryDataEnsembl(wbi_fastrun.FastRunContainer): - def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) - self.prop_dt_map = {'P248': 'wikibase-item', 'P594': 'external-id'} - self.prop_data['Q14911732'] = {'P594': { - 'fake statement id': { - 'qual': set(), - 'ref': {'fake ref id': { - ('P248', 'Q106833387'), - ('P594', 'ENSG00000123374')}}, - 'unit': '1', - 'v': '"ENSG00000123374"'}}} - self.rev_lookup = defaultdict(set) - self.rev_lookup['"ENSG00000123374"'].add('Q14911732') - - -class FastRunContainerFakeQueryDataEnsemblNoRef(wbi_fastrun.FastRunContainer): - def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) - self.prop_dt_map = {'P248': 'wikibase-item', 'P594': 'external-id'} - self.prop_data['Q14911732'] = {'P594': { - 'fake statement id': { - 'qual': set(), - 'ref': {}, - 'v': 'ENSG00000123374'}}} - self.rev_lookup = defaultdict(set) - self.rev_lookup['"ENSG00000123374"'].add('Q14911732') - - -def test_fastrun_ref_ensembl(): - # fastrun checks refs - frc = FastRunContainerFakeQueryDataEnsembl(base_filter=[BaseDataType(prop_nr='P594'), Item(prop_nr='P703', value='Q15978631')], base_data_type=BaseDataType, use_refs=True) - - # statement has no ref - statements = [ExternalID(value='ENSG00000123374', prop_nr='P594')] - assert frc.write_required(data=statements) - - # statement has the same ref - statements = [ExternalID(value='ENSG00000123374', prop_nr='P594', references=[[Item('Q106833387', prop_nr='P248'), ExternalID('ENSG00000123374', prop_nr='P594')]])] - assert not frc.write_required(data=statements) - - # new statement has a different stated in - statements = [ExternalID(value='ENSG00000123374', prop_nr='P594', references=[[Item('Q99999999999', prop_nr='P248'), ExternalID('ENSG00000123374', prop_nr='P594')]])] - assert frc.write_required(data=statements) - - # fastrun doesn't check references, statement has no reference - frc = FastRunContainerFakeQueryDataEnsemblNoRef(base_filter=[BaseDataType(prop_nr='P594'), Item(prop_nr='P703', value='Q15978631')], base_data_type=BaseDataType, - use_refs=False) - statements = [ExternalID(value='ENSG00000123374', prop_nr='P594')] - assert not frc.write_required(data=statements) - - # fastrun doesn't check references, statement has a reference - frc = FastRunContainerFakeQueryDataEnsemblNoRef(base_filter=[BaseDataType(prop_nr='P594'), Item(prop_nr='P703', value='Q15978631')], base_data_type=BaseDataType, - use_refs=False) - statements = [ExternalID(value='ENSG00000123374', prop_nr='P594', references=[[Item('Q123', prop_nr='P31')]])] - assert not frc.write_required(data=statements) - - -class FakeQueryDataAppendProps(wbi_fastrun.FastRunContainer): - # an item with three values for the same property - def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) - self.prop_dt_map = {'P527': 'wikibase-item', 'P248': 'wikibase-item', 'P594': 'external-id'} - - self.rev_lookup = defaultdict(set) - self.rev_lookup['Q24784025'].add('Q3402672') - self.rev_lookup['Q24743729'].add('Q3402672') - self.rev_lookup['Q24782625'].add('Q3402672') - - self.prop_data['Q3402672'] = {'P527': { - 'Q3402672-11BA231B-857B-498B-AC4F-91D71EE007FD': {'qual': set(), - 'ref': { - '149c9c7ba4e246d9f09ce3ed0cdf7aa721aad5c8': { - ('P248', 'Q3047275'), - }}, - 'v': 'Q24784025'}, - 'Q3402672-15F54AFF-7DCC-4DF6-A32F-73C48619B0B2': {'qual': set(), - 'ref': { - '149c9c7ba4e246d9f09ce3ed0cdf7aa721aad5c8': { - ('P248', 'Q3047275'), - }}, - 'v': 'Q24743729'}, - 'Q3402672-C8F11D55-1B11-44E5-9EAF-637E062825A4': {'qual': set(), - 'ref': { - '149c9c7ba4e246d9f09ce3ed0cdf7aa721aad5c8': { - ('P248', 'Q3047275')}}, - 'v': 'Q24782625'}}} - - -def test_append_props(): - qid = 'Q3402672' - - # don't consider refs - frc = FakeQueryDataAppendProps(base_filter=[BaseDataType(prop_nr='P352'), Item(prop_nr='P703', value='Q15978631')], base_data_type=BaseDataType) - # with append - statements = [Item(value='Q24784025', prop_nr='P527')] - assert frc.write_required(data=statements, action_if_exists=ActionIfExists.APPEND_OR_REPLACE, cqid=qid) is False - # with force append - statements = [Item(value='Q24784025', prop_nr='P527')] - assert frc.write_required(data=statements, action_if_exists=ActionIfExists.FORCE_APPEND, cqid=qid) is True - # without append - statements = [Item(value='Q24784025', prop_nr='P527')] - assert frc.write_required(data=statements, cqid=qid) is True - - # if we are in append mode, and the refs are different, we should write - frc = FakeQueryDataAppendProps(base_filter=[BaseDataType(prop_nr='P352'), Item(prop_nr='P703', value='Q15978631')], base_data_type=BaseDataType, use_refs=True) - # with append - statements = [Item(value='Q24784025', prop_nr='P527')] - assert frc.write_required(data=statements, cqid=qid, action_if_exists=ActionIfExists.APPEND_OR_REPLACE) is True - # without append - statements = [Item(value='Q24784025', prop_nr='P527')] - assert frc.write_required(data=statements, cqid=qid) is True + """The three base filter forms must be translated to the expected SPARQL triples.""" + frc = wbi_fastrun.FastRunContainer(base_data_type=BaseDataType, base_filter=[ + BaseDataType(prop_nr='P352'), + Item(value='Q55983715', prop_nr='P703'), + [Item(value='Q3624078', prop_nr='P31'), Item(prop_nr='P279')], + ]) + + base_filter_string = frc._base_filter_string() + assert f'?entity <{wikibase.base_url}/prop/direct/P352> ?zzP352 .' in base_filter_string + assert f'?entity <{wikibase.base_url}/prop/direct/P703> <{wikibase.base_url}/entity/Q55983715> .' in base_filter_string + assert f'?entity <{wikibase.base_url}/prop/direct/P31>/<{wikibase.base_url}/prop/direct/P279>* <{wikibase.base_url}/entity/Q3624078> .' in base_filter_string diff --git a/wikibaseintegrator/datatypes/basedatatype.py b/wikibaseintegrator/datatypes/basedatatype.py index 0db1c3d7..f5b99a92 100644 --- a/wikibaseintegrator/datatypes/basedatatype.py +++ b/wikibaseintegrator/datatypes/basedatatype.py @@ -11,6 +11,7 @@ class BaseDataType(Claim): The base class for all Wikibase data types, they inherit from it """ DTYPE = 'base-data-type' + PTYPE = 'property-data-type' subclasses: list[type[BaseDataType]] = [] sparql_query: str = ''' SELECT * WHERE {{ @@ -28,7 +29,14 @@ def __init__(self, prop_nr: int | str | None = None, **kwargs: Any): super().__init__(**kwargs) - self.mainsnak.property_number = prop_nr or None + if isinstance(prop_nr, str): + pattern = re.compile(r'^([a-z][a-z\d+.-]*):([^][<>\"\x00-\x20\x7F])+$') + matches = pattern.match(str(prop_nr)) + + if matches: + prop_nr = prop_nr.rsplit('/', 1)[-1] + + self.mainsnak.property_number = prop_nr # self.subclasses.append(self) # Allow registration of subclasses of BaseDataType into BaseDataType.subclasses @@ -39,7 +47,7 @@ def __init_subclass__(cls, **kwargs): def set_value(self, value: Any | None = None): pass - def get_sparql_value(self) -> str: + def get_sparql_value(self, **kwargs: Any) -> str | None: return '"' + self.mainsnak.datavalue['value'] + '"' def parse_sparql_value(self, value, type='literal', unit='1') -> bool: @@ -61,3 +69,6 @@ def parse_sparql_value(self, value, type='literal', unit='1') -> bool: raise ValueError return True + + def from_sparql_value(self, sparql_value: dict) -> BaseDataType: # type: ignore + pass diff --git a/wikibaseintegrator/datatypes/commonsmedia.py b/wikibaseintegrator/datatypes/commonsmedia.py index c444437d..2f061cdc 100644 --- a/wikibaseintegrator/datatypes/commonsmedia.py +++ b/wikibaseintegrator/datatypes/commonsmedia.py @@ -1,17 +1,30 @@ import re import urllib.parse -from wikibaseintegrator.datatypes.string import String +from wikibaseintegrator.datatypes.url import URL -class CommonsMedia(String): +class CommonsMedia(URL): """ Implements the Wikibase data type for Wikimedia commons media files """ DTYPE = 'commonsMedia' + PTYPE = 'http://wikiba.se/ontology#CommonsMedia' - def get_sparql_value(self) -> str: - return '<' + self.mainsnak.datavalue['value'] + '>' + def set_value(self, value: str | None = None): + assert isinstance(value, str) or value is None, f"Expected str, found {type(value)} ({value})" + + if value: + pattern = re.compile(r'^.+\..+$') + matches = pattern.match(value) + + if not matches: + raise ValueError(f"Invalid CommonsMedia {value}") + + self.mainsnak.datavalue = { + 'value': value, + 'type': 'string' + } def parse_sparql_value(self, value, type='literal', unit='1') -> bool: pattern = re.compile(r'^?$') diff --git a/wikibaseintegrator/datatypes/externalid.py b/wikibaseintegrator/datatypes/externalid.py index c4838138..6b88ca7e 100644 --- a/wikibaseintegrator/datatypes/externalid.py +++ b/wikibaseintegrator/datatypes/externalid.py @@ -6,3 +6,4 @@ class ExternalID(String): Implements the Wikibase data type 'external-id' """ DTYPE = 'external-id' + PTYPE = 'http://wikiba.se/ontology#ExternalId' diff --git a/wikibaseintegrator/datatypes/form.py b/wikibaseintegrator/datatypes/form.py index e8732cfd..a5f069e8 100644 --- a/wikibaseintegrator/datatypes/form.py +++ b/wikibaseintegrator/datatypes/form.py @@ -2,6 +2,8 @@ from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class Form(BaseDataType): @@ -9,6 +11,7 @@ class Form(BaseDataType): Implements the Wikibase data type 'wikibase-form' """ DTYPE = 'wikibase-form' + PTYPE = 'http://wikiba.se/ontology#WikibaseForm' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -55,8 +58,14 @@ def set_value(self, value: str | None = None): 'type': 'wikibase-entityid' } - def get_sparql_value(self) -> str: - return self.mainsnak.datavalue['value']['id'] + # TODO: add from_sparql_value() + + def get_sparql_value(self, **kwargs: Any) -> str | None: + if self.mainsnak.snaktype == WikibaseSnakType.KNOWN_VALUE: + wikibase_url = str(kwargs['wikibase_url'] if 'wikibase_url' in kwargs else config['WIKIBASE_URL']) + return f'<{wikibase_url}/entity/' + self.mainsnak.datavalue['value']['id'] + '>' + + return None def get_lexeme_id(self) -> str: """ diff --git a/wikibaseintegrator/datatypes/geoshape.py b/wikibaseintegrator/datatypes/geoshape.py index e5dc4ca5..903013d4 100644 --- a/wikibaseintegrator/datatypes/geoshape.py +++ b/wikibaseintegrator/datatypes/geoshape.py @@ -9,6 +9,7 @@ class GeoShape(BaseDataType): Implements the Wikibase data type 'geo-shape' """ DTYPE = 'geo-shape' + PTYPE = 'http://wikiba.se/ontology#GeoShape' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -53,3 +54,7 @@ def set_value(self, value: str | None = None): 'value': value, 'type': 'string' } + + # TODO: Does GeoShape need a full URL to wikimedia commons? + def get_sparql_value(self, **kwargs: Any) -> str: + return '<' + self.mainsnak.datavalue['value'] + '>' diff --git a/wikibaseintegrator/datatypes/globecoordinate.py b/wikibaseintegrator/datatypes/globecoordinate.py index d0eb0568..e7667c24 100644 --- a/wikibaseintegrator/datatypes/globecoordinate.py +++ b/wikibaseintegrator/datatypes/globecoordinate.py @@ -1,9 +1,12 @@ +from __future__ import annotations + import re from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType from wikibaseintegrator.models import Claim from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class GlobeCoordinate(BaseDataType): @@ -11,6 +14,7 @@ class GlobeCoordinate(BaseDataType): Implements the Wikibase data type for globe coordinates """ DTYPE = 'globe-coordinate' + PTYPE = 'http://wikiba.se/ontology#GlobeCoordinate' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -75,8 +79,37 @@ def rounded(datavalue: dict) -> dict: return super().__eq__(other) - def get_sparql_value(self) -> str: - return '"Point(' + str(self.mainsnak.datavalue['value']['longitude']) + ' ' + str(self.mainsnak.datavalue['value']['latitude']) + ')"' + def from_sparql_value(self, sparql_value: dict) -> GlobeCoordinate: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of datatype, type and value + :return: True if the parsing is successful + """ + datatype = sparql_value['datatype'] + type = sparql_value['type'] + value = sparql_value['value'] + + if datatype != 'http://www.opengis.net/ont/geosparql#wktLiteral': + raise ValueError('Wrong SPARQL datatype') + + if type != 'literal': + raise ValueError('Wrong SPARQL type') + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + pattern = re.compile(r'^Point\((.*) (.*)\)$') + matches = pattern.match(value) + if not matches: + raise ValueError('Invalid SPARQL value') + + self.set_value(longitude=float(matches.group(1)), latitude=float(matches.group(2))) + + return self + + def get_sparql_value(self, **kwargs: Any) -> str: + return '"Point(' + str(self.mainsnak.datavalue['value']['longitude']) + ' ' + str(self.mainsnak.datavalue['value']['latitude']) + ')"^^geo:wktLiteral' def parse_sparql_value(self, value, type='literal', unit='1') -> bool: pattern = re.compile(r'^"?Point\((.*) (.*)\)"?(?:\^\^geo:wktLiteral)?$') diff --git a/wikibaseintegrator/datatypes/item.py b/wikibaseintegrator/datatypes/item.py index 798ea492..3dbf911f 100644 --- a/wikibaseintegrator/datatypes/item.py +++ b/wikibaseintegrator/datatypes/item.py @@ -1,7 +1,11 @@ +from __future__ import annotations + import re from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class Item(BaseDataType): @@ -9,6 +13,7 @@ class Item(BaseDataType): Implements the Wikibase data type 'wikibase-item' with a value being another item ID """ DTYPE = 'wikibase-item' + PTYPE = 'http://wikiba.se/ontology#WikibaseItem' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -48,5 +53,34 @@ def set_value(self, value: str | int | None = None): 'type': 'wikibase-entityid' } - def get_sparql_value(self) -> str: - return '<{wb_url}/entity/' + self.mainsnak.datavalue['value']['id'] + '>' + def from_sparql_value(self, sparql_value: dict) -> Item: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of type and value + :return: True if the parsing is successful + """ + type = sparql_value['type'] + value = sparql_value['value'] + + if type != 'uri': + raise ValueError('Wrong SPARQL type') + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + pattern = re.compile(r'^.+/([PQLM]\d+)$') + matches = pattern.match(value) + if not matches: + raise ValueError(f"Invalid SPARQL value {value}") + + self.set_value(value=str(matches.group(1))) + + return self + + def get_sparql_value(self, **kwargs: Any) -> str | None: + if self.mainsnak.snaktype == WikibaseSnakType.KNOWN_VALUE: + wikibase_url = str(kwargs['wikibase_url'] if 'wikibase_url' in kwargs else config['WIKIBASE_URL']) + return f'<{wikibase_url}/entity/' + self.mainsnak.datavalue['value']['id'] + '>' + + return None diff --git a/wikibaseintegrator/datatypes/lexeme.py b/wikibaseintegrator/datatypes/lexeme.py index f128c611..f552e097 100644 --- a/wikibaseintegrator/datatypes/lexeme.py +++ b/wikibaseintegrator/datatypes/lexeme.py @@ -2,6 +2,8 @@ from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class Lexeme(BaseDataType): @@ -9,6 +11,7 @@ class Lexeme(BaseDataType): Implements the Wikibase data type 'wikibase-lexeme' """ DTYPE = 'wikibase-lexeme' + PTYPE = 'http://wikiba.se/ontology#WikibaseLexeme' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -48,5 +51,9 @@ def set_value(self, value: str | int | None = None): 'type': 'wikibase-entityid' } - def get_sparql_value(self) -> str: - return self.mainsnak.datavalue['value']['id'] + def get_sparql_value(self, **kwargs: Any) -> str | None: + if self.mainsnak.snaktype == WikibaseSnakType.KNOWN_VALUE: + wikibase_url = str(kwargs['wikibase_url'] if 'wikibase_url' in kwargs else config['WIKIBASE_URL']) + return f'<{wikibase_url}/entity/' + self.mainsnak.datavalue['value']['id'] + '>' + + return None diff --git a/wikibaseintegrator/datatypes/math.py b/wikibaseintegrator/datatypes/math.py index 7ad3f3cc..29c48217 100644 --- a/wikibaseintegrator/datatypes/math.py +++ b/wikibaseintegrator/datatypes/math.py @@ -1,4 +1,9 @@ +from __future__ import annotations + +from typing import Any + from wikibaseintegrator.datatypes.string import String +from wikibaseintegrator.wbi_enums import WikibaseSnakType class Math(String): @@ -6,3 +11,31 @@ class Math(String): Implements the Wikibase data type 'math' for mathematical formula in TEX format """ DTYPE = 'math' + PTYPE = 'http://wikiba.se/ontology#Math' + + def from_sparql_value(self, sparql_value: dict) -> Math: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of type and value + :return: + """ + datatype = sparql_value['datatype'] + type = sparql_value['type'] + value = sparql_value['value'] + + if datatype != 'http://www.w3.org/2001/XMLSchema#dateTime': + raise ValueError('Wrong SPARQL datatype') + + if type != 'literal': + raise ValueError('Wrong SPARQL type') + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + self.set_value(value=value) + + return self + + def get_sparql_value(self, **kwargs: Any) -> str: + return '"' + self.mainsnak.datavalue['value'] + '"^^' diff --git a/wikibaseintegrator/datatypes/monolingualtext.py b/wikibaseintegrator/datatypes/monolingualtext.py index 630a0b9e..f3c41a51 100644 --- a/wikibaseintegrator/datatypes/monolingualtext.py +++ b/wikibaseintegrator/datatypes/monolingualtext.py @@ -1,8 +1,11 @@ +from __future__ import annotations + import re from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class MonolingualText(BaseDataType): @@ -10,6 +13,8 @@ class MonolingualText(BaseDataType): Implements the Wikibase data type for Monolingual Text strings """ DTYPE = 'monolingualtext' + PTYPE = 'http://wikiba.se/ontology#Monolingualtext' + sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -46,7 +51,28 @@ def set_value(self, text: str | None = None, language: str | None = None): 'type': 'monolingualtext' } - def get_sparql_value(self) -> str: + def from_sparql_value(self, sparql_value: dict) -> MonolingualText: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of datatype, type and value + :return: True if the parsing is successful + """ + xml_lang = sparql_value['xml:lang'] + type = sparql_value['type'] + value = sparql_value['value'] + + if type != 'literal': + raise ValueError(f"Wrong SPARQL type {type}") + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + self.set_value(text=value, language=xml_lang) + + return self + + def get_sparql_value(self, **kwargs: Any) -> str: return '"' + self.mainsnak.datavalue['value']['text'].replace('"', r'\"') + '"@' + self.mainsnak.datavalue['value']['language'] def parse_sparql_value(self, value, type='literal', unit='1') -> bool: diff --git a/wikibaseintegrator/datatypes/musicalnotation.py b/wikibaseintegrator/datatypes/musicalnotation.py index 7d38a4b5..61d3ccf2 100644 --- a/wikibaseintegrator/datatypes/musicalnotation.py +++ b/wikibaseintegrator/datatypes/musicalnotation.py @@ -6,6 +6,7 @@ class MusicalNotation(String): Implements the Wikibase data type 'musical-notation' """ DTYPE = 'musical-notation' + PTYPE = 'http://wikiba.se/ontology#MusicalNotation' def set_value(self, value: str | None = None): assert isinstance(value, str) or value is None, f"Expected str, found {type(value)} ({value})" diff --git a/wikibaseintegrator/datatypes/property.py b/wikibaseintegrator/datatypes/property.py index d1d4c312..97c0e018 100644 --- a/wikibaseintegrator/datatypes/property.py +++ b/wikibaseintegrator/datatypes/property.py @@ -2,6 +2,8 @@ from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class Property(BaseDataType): @@ -9,6 +11,7 @@ class Property(BaseDataType): Implements the Wikibase data type 'property' """ DTYPE = 'wikibase-property' + PTYPE = 'http://wikiba.se/ontology#Property' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -49,5 +52,9 @@ def set_value(self, value: str | int | None = None): 'type': 'wikibase-entityid' } - def get_sparql_value(self) -> str: - return self.mainsnak.datavalue['value']['id'] + def get_sparql_value(self, **kwargs: Any) -> str | None: + if self.mainsnak.snaktype == WikibaseSnakType.KNOWN_VALUE: + wikibase_url = str(kwargs['wikibase_url'] if 'wikibase_url' in kwargs else config['WIKIBASE_URL']) + return f'<{wikibase_url}/entity/' + self.mainsnak.datavalue['value']['id'] + '>' + + return None diff --git a/wikibaseintegrator/datatypes/quantity.py b/wikibaseintegrator/datatypes/quantity.py index 0e04113f..558a1ef2 100644 --- a/wikibaseintegrator/datatypes/quantity.py +++ b/wikibaseintegrator/datatypes/quantity.py @@ -1,7 +1,10 @@ +from __future__ import annotations + from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType from wikibaseintegrator.wbi_helpers import format_amount @@ -10,6 +13,7 @@ class Quantity(BaseDataType): Implements the Wikibase data type for quantities """ DTYPE = 'quantity' + PTYPE = 'http://wikiba.se/ontology#Quantity' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -81,7 +85,31 @@ def set_value(self, amount: str | int | float | None = None, upper_bound: str | if not lower_bound: del self.mainsnak.datavalue['value']['lowerBound'] - def get_sparql_value(self) -> str: + def from_sparql_value(self, sparql_value: dict) -> Quantity: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of datatype, type and value + :return: True if the parsing is successful + """ + datatype = sparql_value['datatype'] + type = sparql_value['type'] + value = sparql_value['value'] + + if datatype != 'http://www.w3.org/2001/XMLSchema#decimal': + raise ValueError(f"Wrong SPARQL datatype {datatype}") + + if type != 'literal': + raise ValueError(f"Wrong SPARQL type {type}") + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + self.set_value(amount=value) + + return self + + def get_sparql_value(self, **kwargs: Any) -> str: return '"' + format_amount(self.mainsnak.datavalue['value']['amount']) + '"^^xsd:decimal' def parse_sparql_value(self, value, type='literal', unit='1') -> bool: diff --git a/wikibaseintegrator/datatypes/sense.py b/wikibaseintegrator/datatypes/sense.py index a2f3abfb..91516b69 100644 --- a/wikibaseintegrator/datatypes/sense.py +++ b/wikibaseintegrator/datatypes/sense.py @@ -2,6 +2,8 @@ from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_config import config +from wikibaseintegrator.wbi_enums import WikibaseSnakType class Sense(BaseDataType): @@ -9,6 +11,7 @@ class Sense(BaseDataType): Implements the Wikibase data type 'wikibase-sense' """ DTYPE = 'wikibase-sense' + PTYPE = 'http://wikiba.se/ontology#WikibaseSense' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -44,8 +47,14 @@ def set_value(self, value: str | None = None): 'type': 'wikibase-entityid' } - def get_sparql_value(self) -> str: - return self.mainsnak.datavalue['value']['id'] + # TODO: add from_sparql_value() + + def get_sparql_value(self, **kwargs: Any) -> str | None: + if self.mainsnak.snaktype == WikibaseSnakType.KNOWN_VALUE: + wikibase_url = str(kwargs['wikibase_url'] if 'wikibase_url' in kwargs else config['WIKIBASE_URL']) + return f'<{wikibase_url}/entity/' + self.mainsnak.datavalue['value']['id'] + '>' + + return None def get_lexeme_id(self) -> str: """ diff --git a/wikibaseintegrator/datatypes/string.py b/wikibaseintegrator/datatypes/string.py index 69b98ee9..e8971911 100644 --- a/wikibaseintegrator/datatypes/string.py +++ b/wikibaseintegrator/datatypes/string.py @@ -1,14 +1,17 @@ +from __future__ import annotations + from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_enums import WikibaseSnakType class String(BaseDataType): """ Implements the Wikibase data type 'string' """ - DTYPE = 'string' + PTYPE = 'http://wikiba.se/ontology#String' def __init__(self, value: str | None = None, **kwargs: Any): """ @@ -31,3 +34,23 @@ def set_value(self, value: str | None = None): 'value': value, 'type': 'string' } + + def from_sparql_value(self, sparql_value: dict) -> String: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of type and value + :return: + """ + type = sparql_value['type'] + value = sparql_value['value'] + + if type != 'literal': + raise ValueError(f"Wrong SPARQL type {type}") + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + self.set_value(value=value) + + return self diff --git a/wikibaseintegrator/datatypes/tabulardata.py b/wikibaseintegrator/datatypes/tabulardata.py index 636ccfa0..ef32043c 100644 --- a/wikibaseintegrator/datatypes/tabulardata.py +++ b/wikibaseintegrator/datatypes/tabulardata.py @@ -9,6 +9,7 @@ class TabularData(BaseDataType): Implements the Wikibase data type 'tabular-data' """ DTYPE = 'tabular-data' + PTYPE = 'http://wikiba.se/ontology#TabularData' def __init__(self, value: str | None = None, **kwargs: Any): """ @@ -34,3 +35,7 @@ def set_value(self, value: str | None = None): 'value': value, 'type': 'string' } + + # TODO: Does TabularData need a full URL to wikimedia commons? + def get_sparql_value(self, **kwargs: Any) -> str: + return '<' + self.mainsnak.datavalue['value'] + '>' diff --git a/wikibaseintegrator/datatypes/time.py b/wikibaseintegrator/datatypes/time.py index 1276d767..be51ea0d 100644 --- a/wikibaseintegrator/datatypes/time.py +++ b/wikibaseintegrator/datatypes/time.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datetime import re from functools import total_ordering @@ -5,7 +7,7 @@ from wikibaseintegrator.datatypes.basedatatype import BaseDataType from wikibaseintegrator.wbi_config import config -from wikibaseintegrator.wbi_enums import WikibaseTimePrecision +from wikibaseintegrator.wbi_enums import WikibaseSnakType, WikibaseTimePrecision @total_ordering @@ -14,6 +16,7 @@ class Time(BaseDataType): Implements the Wikibase data type with date and time values """ DTYPE = 'time' + PTYPE = 'http://wikiba.se/ontology#Time' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -102,8 +105,8 @@ def set_value(self, time: str | None = None, before: int = 0, after: int = 0, pr 'type': 'time' } - def get_sparql_value(self) -> str: - return self.mainsnak.datavalue['value']['time'] + def get_sparql_value(self, **kwargs: Any) -> str: + return '"' + self.mainsnak.datavalue['value']['time'] + '"^^xsd:dateTime' def _time_parts(self) -> tuple[int, int, int]: """ @@ -128,3 +131,27 @@ def get_day(self) -> int: def __lt__(self, other): return self._time_parts() < other._time_parts() + + def from_sparql_value(self, sparql_value: dict) -> Time: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of type and value + :return: + """ + datatype = sparql_value['datatype'] + type = sparql_value['type'] + value = sparql_value['value'] + + if datatype != 'http://www.w3.org/2001/XMLSchema#dateTime': + raise ValueError('Wrong SPARQL datatype') + + if type != 'literal': + raise ValueError('Wrong SPARQL type') + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + self.set_value(time=value) + + return self diff --git a/wikibaseintegrator/datatypes/url.py b/wikibaseintegrator/datatypes/url.py index 3b336085..8b49d258 100644 --- a/wikibaseintegrator/datatypes/url.py +++ b/wikibaseintegrator/datatypes/url.py @@ -1,7 +1,10 @@ +from __future__ import annotations + import re from typing import Any from wikibaseintegrator.datatypes.basedatatype import BaseDataType +from wikibaseintegrator.wbi_enums import WikibaseSnakType class URL(BaseDataType): @@ -9,6 +12,7 @@ class URL(BaseDataType): Implements the Wikibase data type for URL strings """ DTYPE = 'url' + PTYPE = 'http://wikiba.se/ontology#Url' sparql_query = ''' SELECT * WHERE {{ ?item_id <{wb_url}/prop/{pid}> ?s . @@ -41,7 +45,26 @@ def set_value(self, value: str | None = None): 'type': 'string' } - def get_sparql_value(self) -> str: + def from_sparql_value(self, sparql_value: dict) -> URL: + """ + Parse data returned by a SPARQL endpoint and set the value to the object + + :param sparql_value: A SPARQL value composed of type and value + :return: + """ + type = sparql_value['type'] + value = sparql_value['value'] + + if type != 'uri': + raise ValueError(f"Wrong SPARQL type {type}") + + if value.startswith('http://www.wikidata.org/.well-known/genid/'): + self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE + else: + self.set_value(value=value) + return self + + def get_sparql_value(self, **kwargs: Any) -> str: return '<' + self.mainsnak.datavalue['value'] + '>' def parse_sparql_value(self, value, type='literal', unit='1') -> bool: diff --git a/wikibaseintegrator/entities/baseentity.py b/wikibaseintegrator/entities/baseentity.py index 08ca9f0a..59891dcc 100644 --- a/wikibaseintegrator/entities/baseentity.py +++ b/wikibaseintegrator/entities/baseentity.py @@ -107,9 +107,13 @@ def claims(self) -> Claims: return self.__claims @claims.setter - def claims(self, value: Claims): - if not isinstance(value, Claims): + def claims(self, value: Claim | Claims): + if not isinstance(value, Claims) and not isinstance(value, Claim): raise TypeError + + if isinstance(value, Claim): + value = Claims().add(claims=value) + self.__claims = value def add_claims(self, claims: Claim | list[Claim] | Claims, action_if_exists: ActionIfExists = ActionIfExists.APPEND_OR_REPLACE) -> BaseEntity: @@ -207,6 +211,21 @@ def clear(self, **kwargs: Any) -> dict[str, Any]: """ return self._write(data={}, clear=True, **kwargs) + def get_claims(self, property: str, login: _Login | None = None, allow_anonymous: bool = True, is_bot: bool | None = None, **kwargs: Any): + params = { + 'action': 'wbgetclaims', + 'entity': self.id, + 'property': property, + 'format': 'json' + } + + login = login or self.api.login + is_bot = is_bot if is_bot is not None else self.api.is_bot + + json_data = mediawiki_api_call_helper(data=params, login=login, allow_anonymous=allow_anonymous, is_bot=is_bot, **kwargs) + self.claims.from_json(json_data['claims']) + return self + def _write(self, data: dict | None = None, summary: str | None = None, login: _Login | None = None, allow_anonymous: bool = False, limit_claims: list[str | int] | None = None, clear: bool = False, as_new: bool = False, is_bot: bool | None = None, fields_to_update: list | None | EntityField = None, **kwargs: Any) -> dict[str, Any]: """ @@ -323,13 +342,9 @@ def delete(self, login: _Login | None = None, allow_anonymous: bool = False, is_ return delete_page(title=None, pageid=self.pageid, login=login, allow_anonymous=allow_anonymous, is_bot=is_bot, **kwargs) - def write_required(self, base_filter: list[BaseDataType | list[BaseDataType]] | None = None, action_if_exists: ActionIfExists = ActionIfExists.REPLACE_ALL, - **kwargs: Any) -> bool: + def write_required(self, base_filter: list[BaseDataType | list[BaseDataType]], action_if_exists: ActionIfExists = ActionIfExists.REPLACE_ALL, **kwargs: Any) -> bool: fastrun_container = wbi_fastrun.get_fastrun_container(base_filter=base_filter, **kwargs) - if base_filter is None: - base_filter = [] - # Collect the property numbers targeted by the base_filter. It supports both the simple form (a # BaseDataType) and the property-path form (a list of two BaseDataType), whose anchor is the first property. base_filter_props = set() @@ -339,14 +354,19 @@ def write_required(self, base_filter: list[BaseDataType | list[BaseDataType]] | elif isinstance(prop, list) and prop and isinstance(prop[0], BaseDataType): base_filter_props.add(prop[0].mainsnak.property_number) - claims_to_check = [] + pfilter: set = set() for claim in self.claims: if claim.mainsnak.property_number in base_filter_props: - claims_to_check.append(claim) + pfilter.add(claim.mainsnak.property_number) + + property_filter: list[str] = list(pfilter) # TODO: Add check_language_data - return fastrun_container.write_required(data=claims_to_check, cqid=self.id, action_if_exists=action_if_exists) + # Restrict the check to the entity being edited, when it already has an ID + entity_filter = [self.id] if self.id else None + + return fastrun_container.write_required(claims=self.claims, entity_filter=entity_filter, property_filter=property_filter, action_if_exists=action_if_exists) def get_entity_url(self, wikibase_url: str | None = None) -> str: from wikibaseintegrator.wbi_config import config diff --git a/wikibaseintegrator/entities/item.py b/wikibaseintegrator/entities/item.py index 2d133adb..8c3e619f 100644 --- a/wikibaseintegrator/entities/item.py +++ b/wikibaseintegrator/entities/item.py @@ -116,7 +116,7 @@ def from_json(self, json_data: dict[str, Any]) -> ItemEntity: def write(self, **kwargs: Any) -> ItemEntity: """ Write the ItemEntity data to the Wikibase instance and return the ItemEntity object returned by the instance. - extend :func:`~wikibaseintegrator.entities.BaseEntity._write` + This function extend :func:`~wikibaseintegrator.entities.baseentity.BaseEntity._write` :param data: The serialized object that is used as the data source. A newly created entity will be assigned an 'id'. :param summary: A summary of the edit diff --git a/wikibaseintegrator/models/claims.py b/wikibaseintegrator/models/claims.py index afff32d3..6c86f14e 100644 --- a/wikibaseintegrator/models/claims.py +++ b/wikibaseintegrator/models/claims.py @@ -164,9 +164,17 @@ def __len__(self): class Claim(BaseModel): + """ + extend :func:`wikibaseintegrator.models.basemodel.BaseModel` + + :param qualifiers: + :param id: + :param rank: + :param references: A References object, a list of Claim object or a list of list of Claim object + """ DTYPE = 'claim' - def __init__(self, qualifiers: Qualifiers | None = None, rank: WikibaseRank | None = None, references: References | list[Claim | list[Claim]] | None = None, + def __init__(self, qualifiers: Qualifiers | None = None, id: str | None = None, rank: WikibaseRank | None = None, references: References | list[Claim | list[Claim]] | None = None, snaktype: WikibaseSnakType = WikibaseSnakType.KNOWN_VALUE) -> None: """ @@ -179,7 +187,7 @@ def __init__(self, qualifiers: Qualifiers | None = None, rank: WikibaseRank | No self.type = 'statement' self.qualifiers = qualifiers or Qualifiers() self.qualifiers_order = [] - self.id = None + self.id = id self.rank = rank or WikibaseRank.NORMAL self.removed = False @@ -437,5 +445,5 @@ def ref_equal(oldref: References, newref: References) -> bool: return any(any(ref_equal(oldref, newref) for oldref in oldrefs) for newref in newrefs) @abstractmethod - def get_sparql_value(self) -> str: + def get_sparql_value(self, **kwargs: Any) -> str | None: pass diff --git a/wikibaseintegrator/models/qualifiers.py b/wikibaseintegrator/models/qualifiers.py index 892ec5aa..56116a59 100644 --- a/wikibaseintegrator/models/qualifiers.py +++ b/wikibaseintegrator/models/qualifiers.py @@ -12,7 +12,7 @@ class Qualifiers(BaseModel): def __init__(self) -> None: - self.qualifiers: dict[str, list[Snak]] = {} + self.qualifiers: dict[str, list[Snak | Claim]] = {} @property def qualifiers(self): diff --git a/wikibaseintegrator/models/snaks.py b/wikibaseintegrator/models/snaks.py index dc6f63bb..8eef0815 100644 --- a/wikibaseintegrator/models/snaks.py +++ b/wikibaseintegrator/models/snaks.py @@ -46,6 +46,9 @@ def get_json(self) -> dict[str, list]: json_data[property].append(snak.get_json()) return json_data + def __eq__(self, other): + return self.snaks == other.snaks + def __iter__(self): iterate = [] for snak in self.snaks.values(): diff --git a/wikibaseintegrator/wbi_config.py b/wikibaseintegrator/wbi_config.py index a2633023..608dde53 100644 --- a/wikibaseintegrator/wbi_config.py +++ b/wikibaseintegrator/wbi_config.py @@ -32,5 +32,6 @@ 'SPARQL_ENDPOINT_URL': 'https://query.wikidata.org/sparql', 'WIKIBASE_URL': 'http://www.wikidata.org', 'DEFAULT_LANGUAGE': 'en', - 'DEFAULT_LEXEME_LANGUAGE': 'Q1860' + 'DEFAULT_LEXEME_LANGUAGE': 'Q1860', + 'SPARQL_QUERY_LIMIT': 10000 } diff --git a/wikibaseintegrator/wbi_fastrun.py b/wikibaseintegrator/wbi_fastrun.py index bdeed124..30f52ddb 100644 --- a/wikibaseintegrator/wbi_fastrun.py +++ b/wikibaseintegrator/wbi_fastrun.py @@ -1,298 +1,680 @@ +""" +Fast run mode: check locally, through data loaded from the SPARQL endpoint, whether a write to the Wikibase instance +is actually required, so that a synchronisation bot can skip the entities that are already up to date. +""" from __future__ import annotations import collections -import copy import logging +import re from collections import defaultdict -from itertools import chain -from typing import TYPE_CHECKING from wikibaseintegrator.datatypes import BaseDataType -from wikibaseintegrator.models import Claim +from wikibaseintegrator.models import Claim, Claims, Qualifiers, Reference, References from wikibaseintegrator.wbi_config import config -from wikibaseintegrator.wbi_enums import ActionIfExists, WikibaseDatatype -from wikibaseintegrator.wbi_helpers import execute_sparql_query, format_amount - -if TYPE_CHECKING: - from wikibaseintegrator.models import Claims +from wikibaseintegrator.wbi_enums import ActionIfExists, WikibaseRank +from wikibaseintegrator.wbi_helpers import execute_sparql_query log = logging.getLogger(__name__) fastrun_store: list[FastRunContainer] = [] +# The RDF export of a Wikibase instance always represents a quantity without a unit ('1' in the JSON representation) +# with the Wikidata entity Q199 (the number one), whatever the instance. +UNITLESS_UNIT_URIS = ('1', 'http://www.wikidata.org/entity/Q199', 'https://www.wikidata.org/entity/Q199') + class FastRunContainer: - def __init__(self, base_data_type: type[BaseDataType], mediawiki_api_url: str | None = None, sparql_endpoint_url: str | None = None, wikibase_url: str | None = None, - base_filter: list[BaseDataType | list[BaseDataType]] | None = None, use_refs: bool = False, case_insensitive: bool = False): - self.reconstructed_statements: list[BaseDataType] = [] - self.rev_lookup: defaultdict[str, set[str]] = defaultdict(set) - self.rev_lookup_ci: defaultdict[str, set[str]] = defaultdict(set) - self.prop_data: dict[str, dict] = {} + """ + A FastRunContainer loads the statements of the entities matching the base filter from the SPARQL endpoint and + caches them, so that a bot can check whether a write is required without loading every entity through the API. + + :param base_filter: The default filter to initialize the dataset. A list made of BaseDataType or list of BaseDataType. + :param base_data_type: The default data type to create objects. + :param use_qualifiers: Use qualifiers during fastrun. Enabled by default. + :param use_references: Use references during fastrun. Disabled by default. + :param use_rank: Use rank during fastrun. Disabled by default. + :param cache: Put data returned by the SPARQL endpoint in cache. Enabled by default. + :param case_insensitive: Compare the string values without taking the case into account. The comparison of + qualifiers, references and ranks stays case sensitive. Disabled by default. + :param sparql_endpoint_url: SPARQL endpoint URL. + :param wikibase_url: Wikibase URL used for the concept URI. + """ + + data: dict[str, dict[str, list[dict[str, str]]]] + + def __init__(self, base_filter: list[BaseDataType | list[BaseDataType]], base_data_type: type[BaseDataType] | None = None, use_qualifiers: bool = True, + use_references: bool = False, use_rank: bool = False, cache: bool = True, case_insensitive: bool = False, sparql_endpoint_url: str | None = None, + wikibase_url: str | None = None): + + for k in base_filter: + if not isinstance(k, BaseDataType) and not (isinstance(k, list) and len(k) == 2 and isinstance(k[0], BaseDataType) and isinstance(k[1], BaseDataType)): + raise ValueError("base_filter must be an instance of BaseDataType or a list of instances of BaseDataType") + + # Statements loaded from the SPARQL endpoint: property number -> value key -> list of {'entity': uri, 'sid': uri} + self.data: dict[str, dict[str, list[dict[str, str]]]] = {} + # The properties whose statements are completely loaded in self.data. A load restricted to a value or to + # qualifiers only holds a subset of the statements and must not be reused as a complete cache. + self.loaded_complete: set[str] = set() + + self.base_filter = base_filter + self.base_data_type = base_data_type or BaseDataType + self.sparql_endpoint_url = str(sparql_endpoint_url or config['SPARQL_ENDPOINT_URL']) + self.wikibase_url = str(wikibase_url or config['WIKIBASE_URL']) + self.use_qualifiers = use_qualifiers + self.use_references = use_references + self.use_rank = use_rank + self.cache = cache + self.case_insensitive = case_insensitive + self.properties_type: dict[str, str] = {} self.loaded_langs: dict[str, dict] = {} - self.base_filter: list[BaseDataType | list[BaseDataType]] = [] - self.base_filter_string = '' - self.prop_dt_map: dict[str, str | None] = {} - - self.base_data_type: type[BaseDataType] = base_data_type - self.mediawiki_api_url: str = str(mediawiki_api_url or config['MEDIAWIKI_API_URL']) - self.sparql_endpoint_url: str = str(sparql_endpoint_url or config['SPARQL_ENDPOINT_URL']) - self.wikibase_url: str = str(wikibase_url or config['WIKIBASE_URL']) - self.use_refs: bool = use_refs - self.case_insensitive: bool = case_insensitive - - if base_filter and any(base_filter): - self.base_filter = base_filter - for k in self.base_filter: - if isinstance(k, BaseDataType): - if k.mainsnak.datavalue: - self.base_filter_string += '?item <{wb_url}/prop/direct/{prop_nr}> {entity} .\n'.format( - wb_url=self.wikibase_url, prop_nr=k.mainsnak.property_number, entity=k.get_sparql_value().format(wb_url=self.wikibase_url)) - else: - self.base_filter_string += '?item <{wb_url}/prop/direct/{prop_nr}> ?zz{prop_nr} .\n'.format( - wb_url=self.wikibase_url, prop_nr=k.mainsnak.property_number) - elif isinstance(k, list) and len(k) == 2 and isinstance(k[0], BaseDataType) and isinstance(k[1], BaseDataType): - if k[0].mainsnak.datavalue: - self.base_filter_string += '?item <{wb_url}/prop/direct/{prop_nr}>/<{wb_url}/prop/direct/{prop_nr2}>* {entity} .\n'.format( - wb_url=self.wikibase_url, prop_nr=k[0].mainsnak.property_number, prop_nr2=k[1].mainsnak.property_number, - entity=k[0].get_sparql_value().format(wb_url=self.wikibase_url)) - else: - self.base_filter_string += '?item <{wb_url}/prop/direct/{prop_nr1}>/<{wb_url}/prop/direct/{prop_nr2}>* ?zz{prop_nr1}{prop_nr2} .\n'.format( - wb_url=self.wikibase_url, prop_nr1=k[0].mainsnak.property_number, prop_nr2=k[1].mainsnak.property_number) - else: - raise ValueError("base_filter must be an instance of BaseDataType or a list of instances of BaseDataType") - - def reconstruct_statements(self, qid: str) -> list[BaseDataType]: - reconstructed_statements: list[BaseDataType] = [] - - if qid not in self.prop_data: - self.reconstructed_statements = reconstructed_statements - return reconstructed_statements - - for prop_nr, dt in self.prop_data[qid].items(): - # get datatypes for qualifier props - q_props = set(chain(*([x[0] for x in d['qual']] for d in dt.values()))) - r_props = set(chain(*(set(chain(*([y[0] for y in x] for x in d['ref'].values()))) for d in dt.values()))) - props = q_props | r_props - for prop in props: - if prop not in self.prop_dt_map: - self.prop_dt_map.update({prop: self.get_prop_datatype(prop)}) - # reconstruct statements from frc (including unit, qualifiers, and refs) - for _, d in dt.items(): - qualifiers = [] - for q in d['qual']: - f = [x for x in self.base_data_type.subclasses if x.DTYPE == self.prop_dt_map[q[0]]][0] - # TODO: Add support for more data type (Time, MonolingualText, GlobeCoordinate) - if self.prop_dt_map[q[0]] == 'quantity': - qualifiers.append(f(value=q[1], prop_nr=q[0], unit=q[2])) - else: - qualifiers.append(f(value=q[1], prop_nr=q[0])) - - references = [] - for _, refs in d['ref'].items(): - this_ref = [] - for ref in refs: - f = [x for x in self.base_data_type.subclasses if x.DTYPE == self.prop_dt_map[ref[0]]][0] - this_ref.append(f(value=ref[1], prop_nr=ref[0])) - references.append(this_ref) - - f = [x for x in self.base_data_type.subclasses if x.DTYPE == self.prop_dt_map[prop_nr]][0] - # TODO: Add support for more data type - if self.prop_dt_map[prop_nr] == 'quantity': - datatype = f(prop_nr=prop_nr, qualifiers=qualifiers, references=references, unit=d['unit']) - datatype.parse_sparql_value(value=d['v'], unit=d['unit']) - else: - datatype = f(prop_nr=prop_nr, qualifiers=qualifiers, references=references) - datatype.parse_sparql_value(value=d['v']) - reconstructed_statements.append(datatype) - # this isn't used. done for debugging purposes - self.reconstructed_statements = reconstructed_statements - return reconstructed_statements + # Per-statement caches for the lazily loaded qualifiers, references and ranks + self._qualifiers_cache: dict[str, Qualifiers] = {} + self._references_cache: dict[str, References] = {} + self._rank_cache: dict[str, WikibaseRank | None] = {} + + @staticmethod + def _entity_id(entity: str) -> str: + """Reduce an entity URI to its bare entity ID. A bare entity ID is returned unchanged.""" + return entity.rsplit('/', 1)[-1] + + def _datatype_class(self, property_type: str) -> type[BaseDataType]: + """ + Return the data type class implementing the given SPARQL property type (e.g. 'http://wikiba.se/ontology#Time' -> Time). + + :param property_type: A property type URI from the wikibase ontology + :exception ValueError: if no class implements the given property type + """ + for subclass in self.base_data_type.subclasses: + if subclass.PTYPE == property_type: + return subclass + raise ValueError(f"No data type class found for the property type '{property_type}'") + + @classmethod + def _normalize_unit(cls, unit: str) -> str: + """ + Normalize a unit to the format used in the comparison keys: '1' for a unitless quantity (represented by the + Wikidata Q199 entity in the RDF export, whatever the instance) and the bare entity ID otherwise. - def get_items(self, claims: list[Claim] | Claims | Claim, cqid: str | None = None) -> set[str] | None: + :param unit: A unit URI from the SPARQL results or from the JSON representation, or '1' + """ + if unit in UNITLESS_UNIT_URIS: + return '1' + return cls._entity_id(unit) + + def _value_key(self, claim: Claim) -> str | None: + """ + The key indexing the value of the given claim in self.data, casefolded when case_insensitive is enabled. + The normalized unit is part of the key for quantities: two amounts only differing by their unit must not + be considered equal. """ - Get items ID from a SPARQL endpoint + value = claim.get_sparql_value() + if value is None: + return None + if self.case_insensitive: + value = value.casefold() + datavalue = claim.mainsnak.datavalue + if isinstance(datavalue, dict) and datavalue.get('type') == 'quantity': + value += '@' + self._normalize_unit(datavalue['value'].get('unit', '1')) + return value + + def _base_filter_string(self, wb_url: str | None = None) -> str: + """Generate the SPARQL triples restricting ?entity to the entities matching the base filter.""" + wb_url = wb_url or self.wikibase_url + + base_filter_string = '' + for k in self.base_filter: + if isinstance(k, BaseDataType): + # TODO: Add multiple values for a property (OR-operation) (with the VALUES tag?) + if k.mainsnak.datavalue: + base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr}> {entity} .\n'.format( + wb_url=wb_url, prop_nr=k.mainsnak.property_number, entity=k.get_sparql_value(wikibase_url=wb_url)) + elif sum(1 for x in self.base_filter if isinstance(x, BaseDataType) and x.mainsnak.property_number == k.mainsnak.property_number) == 1: + base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr}> ?zz{prop_nr} .\n'.format( + wb_url=wb_url, prop_nr=k.mainsnak.property_number) + elif isinstance(k, list) and len(k) == 2 and isinstance(k[0], BaseDataType) and isinstance(k[1], BaseDataType): + if k[0].mainsnak.datavalue: + base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr}>/<{wb_url}/prop/direct/{prop_nr2}>* {entity} .\n'.format( + wb_url=wb_url, prop_nr=k[0].mainsnak.property_number, prop_nr2=k[1].mainsnak.property_number, + entity=k[0].get_sparql_value(wikibase_url=wb_url)) + # TODO: Remove ?zzPYY if another filter have the same property number, the same as above + else: + base_filter_string += '?entity <{wb_url}/prop/direct/{prop_nr1}>/<{wb_url}/prop/direct/{prop_nr2}>* ?zz{prop_nr1}{prop_nr2} .\n'.format( + wb_url=wb_url, prop_nr1=k[0].mainsnak.property_number, prop_nr2=k[1].mainsnak.property_number) + else: + raise ValueError("base_filter must be an instance of BaseDataType or a list of instances of BaseDataType") - :param claims: A list of claims the entities should have - :param cqid: - :return: a list of entity ID or None - :exception: if there is more than one claim + return base_filter_string + + def load_statements(self, claims: list[Claim] | Claims | Claim, cache: bool | None = None, wb_url: str | None = None, limit: int | None = None) -> None: """ - match_sets = [] + Load the statements related to the given claims into the internal cache of the current object. + + When the cache is enabled, every statement of the property is loaded once and reused afterwards. When the + cache is disabled and the claim carries a value, only the statements holding the same value (and the same + qualifiers if use_qualifiers is enabled) are loaded; such a partial load is never reused as a cache. + :param claims: A Claim, Claims or list of Claim + :param cache: Put data returned by the SPARQL endpoint in cache. Enabled by default. + :param wb_url: The first part of the concept URI of entities. + :param limit: The limit to request at one time. + :return: + """ if isinstance(claims, Claim): claims = [claims] elif (not isinstance(claims, list) or not all(isinstance(n, Claim) for n in claims)) and not isinstance(claims, Claims): raise ValueError("claims must be an instance of Claim or Claims or a list of Claim") - for claim in claims: - # skip to next if statement has no value or no data type defined, e.g. for deletion objects - if not claim.mainsnak.datavalue and not claim.mainsnak.datatype: - continue + if cache is None: + cache = self.cache + + wb_url = wb_url or self.wikibase_url + + limit = limit or int(config['SPARQL_QUERY_LIMIT']) # type: ignore + for claim in claims: prop_nr = claim.mainsnak.property_number - if prop_nr not in self.prop_dt_map: - log.debug("%s not found in fastrun", prop_nr) + # Load each property from the Wikibase instance or the cache + if cache and prop_nr in self.data and prop_nr in self.loaded_complete: + log.debug("Property '%s' found in cache, %s elements", prop_nr, len(self.data[prop_nr])) + continue - if isinstance(claim, BaseDataType) and type(claim) != BaseDataType: # pylint: disable=unidiomatic-typecheck - self.prop_dt_map.update({prop_nr: claim.DTYPE}) + offset = 0 + + base_filter_string = self._base_filter_string(wb_url=wb_url) + + # A partial load restricted to the claim value: only when the cache is disabled, because the result + # can't be reused for other values. The case insensitive mode always needs the complete data, the + # SPARQL comparison is case sensitive. + partial_load = bool(claim.mainsnak.datavalue) and not cache and not self.case_insensitive + + # Restrict the statements to the ones holding the same qualifiers as the claim. Only applied to a + # partial load: a complete load must contain every statement, whatever its qualifiers. + qualifiers_filter_string = '' + if partial_load and self.use_qualifiers: + for qualifier in claim.qualifiers: + if not qualifier.datatype: + continue + fake_json = { + 'mainsnak': qualifier.get_json(), + 'type': qualifier.datatype, + 'id': 'Q0', + 'rank': 'normal' + } + f = [x for x in self.base_data_type.subclasses if x.DTYPE == qualifier.datatype][0]().from_json(json_data=fake_json) + qualifiers_filter_string += f'?sid pq:{qualifier.property_number} {f.get_sparql_value()}.\n' + + # We force a refresh of the data, remove the previous results + self.data[prop_nr] = {} + self.loaded_complete.discard(prop_nr) + + while True: + if partial_load: + query = ''' + #Tool: WikibaseIntegrator wbi_fastrun.load_statements + SELECT ?entity ?sid ?value ?property_type ?unit WHERE {{ + # Base filter string + {base_filter_string} + ?entity <{wb_url}/prop/{prop_nr}> ?sid. + <{wb_url}/entity/{prop_nr}> wikibase:propertyType ?property_type. + ?sid <{wb_url}/prop/statement/{prop_nr}> ?value. + ?sid <{wb_url}/prop/statement/{prop_nr}> {value}. + {qualifiers_filter_string} + # The unit of a quantity value, only bound for quantities + OPTIONAL {{ ?sid <{wb_url}/prop/statement/value/{prop_nr}> [ wikibase:quantityUnit ?unit ] . }} + }} + ORDER BY ?sid + OFFSET {offset} + LIMIT {limit} + ''' + + # Format the query + query = query.format(base_filter_string=base_filter_string, wb_url=wb_url, prop_nr=prop_nr, offset=str(offset), limit=str(limit), + value=claim.get_sparql_value(wikibase_url=wb_url), qualifiers_filter_string=qualifiers_filter_string) else: - self.prop_dt_map.update({prop_nr: self.get_prop_datatype(prop_nr)}) - self._query_data(prop_nr=prop_nr, use_units=self.prop_dt_map[prop_nr] == 'quantity') - - # noinspection PyProtectedMember - current_value = claim.get_sparql_value() - - if self.prop_dt_map[prop_nr] == 'wikibase-item': - current_value = claim.mainsnak.datavalue['value']['id'] - - log.debug(current_value) - # if self.case_insensitive: - # log.debug("case insensitive enabled") - # log.debug(self.rev_lookup_ci) - # else: - # log.debug(self.rev_lookup) - - if current_value in self.rev_lookup: - # quick check for if the value has ever been seen before, if not, write required - match_sets.append(set(self.rev_lookup[current_value])) - elif self.case_insensitive and current_value.casefold() in self.rev_lookup_ci: - match_sets.append(set(self.rev_lookup_ci[current_value.casefold()])) - else: - log.debug("no matches for rev lookup for %s", current_value) + query = ''' + #Tool: WikibaseIntegrator wbi_fastrun.load_statements + SELECT ?entity ?sid ?value ?property_type ?unit WHERE {{ + # Base filter string + {base_filter_string} + ?entity <{wb_url}/prop/{prop_nr}> ?sid. + <{wb_url}/entity/{prop_nr}> wikibase:propertyType ?property_type. + ?sid <{wb_url}/prop/statement/{prop_nr}> ?value. + # The unit of a quantity value, only bound for quantities + OPTIONAL {{ ?sid <{wb_url}/prop/statement/value/{prop_nr}> [ wikibase:quantityUnit ?unit ] . }} + }} + ORDER BY ?sid + OFFSET {offset} + LIMIT {limit} + ''' + + # Format the query + # TODO: Add custom query support + query = query.format(base_filter_string=base_filter_string, wb_url=wb_url, prop_nr=prop_nr, offset=str(offset), limit=str(limit)) + + offset += limit # We increase the offset for the next iteration + results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] + + for result in results: + entity = result['entity']['value'] + sid = result['sid']['value'] + property_type = result['property_type']['value'] + + try: + f = self._datatype_class(property_type)().from_sparql_value(sparql_value=result['value']) + except ValueError as exception: + # A value the data type can't represent (e.g. a timestamp whose precision can't be inferred). + # The value stays out of the dataset, a write will be reported as required for it. + log.warning("Skipping a value of property '%s': %s", prop_nr, exception) + continue + + if f is None: + # The data type does not implement from_sparql_value() yet + log.warning("The data type of property '%s' does not support from_sparql_value(), skipping the value", prop_nr) + continue + + # The simple value of a quantity does not carry the unit, set it from the value node + if 'unit' in result and isinstance(f.mainsnak.datavalue, dict) and f.mainsnak.datavalue.get('type') == 'quantity': + f.mainsnak.datavalue['value']['unit'] = result['unit']['value'] + + sparql_value = self._value_key(f) + if sparql_value is not None: + if sparql_value not in self.data[prop_nr]: + self.data[prop_nr][sparql_value] = [] + + if prop_nr not in self.properties_type: + self.properties_type[prop_nr] = property_type + + self.data[prop_nr][sparql_value].append({'entity': entity, 'sid': sid}) + + if len(results) == 0 or len(results) < limit: + break + + if not partial_load: + self.loaded_complete.add(prop_nr) + + def _load_qualifiers(self, sid: str, limit: int | None = None, cache: bool | None = None) -> Qualifiers: + """ + Load the qualifiers of a statement. - if not match_sets: - return None + :param sid: A statement ID. + :param limit: The limit to request at one time. + :param cache: Reuse the qualifiers already loaded for this statement. Enabled by default. + :return: A Qualifiers object. + """ + if not isinstance(sid, str): + raise ValueError('sid must be a string') + + if cache is None: + cache = self.cache + + if cache and sid in self._qualifiers_cache: + return self._qualifiers_cache[sid] + + offset = 0 + + limit = limit or int(config['SPARQL_QUERY_LIMIT']) # type: ignore + + qualifiers: Qualifiers = Qualifiers() + while True: + query = f''' + #Tool: WikibaseIntegrator wbi_fastrun._load_qualifiers + SELECT ?property ?value ?property_type WHERE {{ + VALUES ?sid {{ <{sid}> }} + ?sid ?predicate ?value. + ?property wikibase:qualifier ?predicate. + ?property wikibase:propertyType ?property_type. + }} + ORDER BY ?sid + OFFSET {offset} + LIMIT {limit} + ''' - if cqid: - matching_qids = {cqid} - else: - matching_qids = match_sets[0].intersection(*match_sets[1:]) + offset += limit # We increase the offset for the next iteration + results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] + + for result in results: + prop_nr = self._entity_id(result['property']['value']) + property_type = result['property_type']['value'] + + if prop_nr not in self.properties_type: + self.properties_type[prop_nr] = property_type + + try: + f = self._datatype_class(property_type)(prop_nr=prop_nr).from_sparql_value(sparql_value=result['value']) + except ValueError as exception: + # An unparsable qualifier can't be compared, leave it out so that the comparison fails safely + log.warning("Skipping a qualifier of statement '%s': %s", sid, exception) + continue + + if f is None: + log.warning("The data type of property '%s' does not support from_sparql_value(), skipping the qualifier", prop_nr) + continue - return matching_qids + qualifiers.add(f) - def get_item(self, claims: list[Claim] | Claims | Claim, cqid: str | None = None) -> str | None: + if len(results) == 0 or len(results) < limit: + break + + self._qualifiers_cache[sid] = qualifiers + + return qualifiers + + def _load_references(self, sid: str, limit: int | None = None, cache: bool | None = None) -> References: """ + Load the references of a statement. - :param claims: A list of claims the entity should have - :param cqid: - :return: An entity ID, None if there is more than one. + :param sid: A statement ID. + :param limit: The limit to request at one time. + :param cache: Reuse the references already loaded for this statement. Enabled by default. + :return: A References object. """ + if not isinstance(sid, str): + raise ValueError('sid must be a string') - matching_qids: set[str] | None = self.get_items(claims=claims, cqid=cqid) + if cache is None: + cache = self.cache - if matching_qids is None: - return None + if cache and sid in self._references_cache: + return self._references_cache[sid] - # check if there are any items that have all of these values - # if not, a write is required no matter what - if not len(matching_qids) == 1: - log.debug("no matches (%s)", len(matching_qids)) - return None + offset = 0 + + limit = limit or int(config['SPARQL_QUERY_LIMIT']) # type: ignore + + # The references are grouped by reference node URI, across the result pages + reference: dict[str, Reference] = {} + while True: + query = f''' + #Tool: WikibaseIntegrator wbi_fastrun._load_references + SELECT ?srid ?ref_property ?ref_value ?property_type WHERE {{ + VALUES ?sid {{ <{sid}> }} + + ?sid prov:wasDerivedFrom ?srid. + ?srid ?ref_predicate ?ref_value. + ?ref_property wikibase:reference ?ref_predicate. + ?ref_property wikibase:propertyType ?property_type. + }} + ORDER BY ?srid + OFFSET {offset} + LIMIT {limit} + ''' + + offset += limit # We increase the offset for the next iteration + results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] + + for result in results: + prop_nr = self._entity_id(result['ref_property']['value']) + srid = result['srid']['value'] + property_type = result['property_type']['value'] + + if prop_nr not in self.properties_type: + self.properties_type[prop_nr] = property_type - return matching_qids.pop() + try: + f = self._datatype_class(property_type)(prop_nr=prop_nr).from_sparql_value(sparql_value=result['ref_value']) + except ValueError as exception: + # An unparsable reference snak can't be compared, leave it out so that the comparison fails safely + log.warning("Skipping a reference snak of statement '%s': %s", sid, exception) + continue - def write_required(self, data: list[Claim], action_if_exists: ActionIfExists = ActionIfExists.REPLACE_ALL, cqid: str | None = None) -> bool: + if f is None: + log.warning("The data type of property '%s' does not support from_sparql_value(), skipping the reference snak", prop_nr) + continue + + if srid not in reference: + reference[srid] = Reference() + + reference[srid].add(f) + + if len(results) == 0 or len(results) < limit: + break + + references: References = References() + for _, ref in reference.items(): + references.add(ref) + + self._references_cache[sid] = references + + return references + + def _load_rank(self, sid: str, cache: bool | None = None) -> WikibaseRank | None: """ - Check if a write is required + Load the rank of a statement. - :param data: - :param action_if_exists: - :param cqid: - :return: Return True if the write is required + :param sid: A statement ID. + :param cache: Reuse the rank already loaded for this statement. Enabled by default. + :return: The WikibaseRank of the statement or None if the statement is not found. """ - del_props = set() - data_props = set() - append_props = [] - if action_if_exists == ActionIfExists.APPEND_OR_REPLACE: - append_props = [x.mainsnak.property_number for x in data] + if not isinstance(sid, str): + raise ValueError('sid must be a string') - for x in data: - if x.mainsnak.datavalue and x.mainsnak.datatype: - data_props.add(x.mainsnak.property_number) - qid = self.get_item(data, cqid) + if cache is None: + cache = self.cache - if not qid: - return True + if cache and sid in self._rank_cache: + return self._rank_cache[sid] - reconstructed_statements = self.reconstruct_statements(qid) - tmp_rs = copy.deepcopy(reconstructed_statements) - - # handle append properties - for p in append_props: - app_data = [x for x in data if x.mainsnak.property_number == p] # new statements - rec_app_data = [x for x in tmp_rs if x.mainsnak.property_number == p] # orig statements - comp = [] - for x in app_data: - for y in rec_app_data: - if x.mainsnak.datavalue == y.mainsnak.datavalue: - if y.equals(x, include_ref=self.use_refs) and action_if_exists != ActionIfExists.FORCE_APPEND: - comp.append(True) - - # comp = [True for x in app_data for y in rec_app_data if x.equals(y, include_ref=self.use_refs)] - if len(comp) != len(app_data): - log.debug("failed append: %s", p) - return True + query = f''' + #Tool: WikibaseIntegrator wbi_fastrun._load_rank + SELECT ?rank WHERE {{ + VALUES ?sid {{ <{sid}> }} + ?sid wikibase:rank ?rank. + }} + ''' - tmp_rs = [x for x in tmp_rs if x.mainsnak.property_number not in append_props and x.mainsnak.property_number in data_props] + results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] - for date in data: - # ensure that statements meant for deletion get handled properly - reconst_props = {x.mainsnak.property_number for x in tmp_rs} - if not date.mainsnak.datatype and date.mainsnak.property_number in reconst_props: - log.debug("returned from delete prop handling") - return True + rank: WikibaseRank | None = None + for result in results: + rank_raw = result['rank']['value'].rsplit('#', 1)[-1] - if not date.mainsnak.datavalue or not date.mainsnak.datatype: - # Ignore the deletion statements which are not in the reconstructed statements. - continue + if rank_raw == 'PreferredRank': + rank = WikibaseRank.PREFERRED + elif rank_raw == 'NormalRank': + rank = WikibaseRank.NORMAL + elif rank_raw == 'DeprecatedRank': + rank = WikibaseRank.DEPRECATED - if date.mainsnak.property_number in append_props: - # TODO: check if value already exist and already have the same value - continue + self._rank_cache[sid] = rank - if not date.mainsnak.datavalue and not date.mainsnak.datatype: - del_props.add(date.mainsnak.property_number) + return rank - # this is where the magic happens - # date is a new statement, proposed to be written - # tmp_rs are the reconstructed statements == current state of the item - bool_vec = [] - for x in tmp_rs: - if (x == date or (self.case_insensitive and x.mainsnak.datavalue.casefold() == date.mainsnak.datavalue.casefold())) and x.mainsnak.property_number not in del_props: - bool_vec.append(x.equals(date, include_ref=self.use_refs)) - else: - bool_vec.append(False) - # bool_vec = [x.equals(date, include_ref=self.use_refs, fref=self.ref_comparison_f) and - # x.mainsnak.property_number not in del_props for x in tmp_rs] - - log.debug("bool_vec: %s", bool_vec) - log.debug("-----------------------------------") - for x in tmp_rs: - if x == date and x.mainsnak.property_number not in del_props: - log.debug([x.mainsnak.property_number, x.mainsnak.datavalue, [z.datavalue for z in x.qualifiers]]) - log.debug([date.mainsnak.property_number, date.mainsnak.datavalue, [z.datavalue for z in date.qualifiers]]) - elif x.mainsnak.property_number == date.mainsnak.property_number: - log.debug([x.mainsnak.property_number, x.mainsnak.datavalue, [z.datavalue for z in x.qualifiers]]) - log.debug([date.mainsnak.property_number, date.mainsnak.datavalue, [z.datavalue for z in date.qualifiers]]) - - if not any(bool_vec): - log.debug(len(bool_vec)) - log.debug("fast run failed at %s", date.mainsnak.property_number) + def _get_property_type(self, prop_nr: str | int) -> str: + """ + Obtain the property type of the given property by looking at the SPARQL endpoint. + + :param prop_nr: The property number. + :return: The SPARQL version of the property type. + """ + if isinstance(prop_nr, int): + prop_nr = 'P' + str(prop_nr) + elif prop_nr is not None: + pattern = re.compile(r'^P?([0-9]+)$') + matches = pattern.match(prop_nr) + + if not matches: + raise ValueError('Invalid prop_nr, format must be "P[0-9]+"') + + prop_nr = 'P' + str(matches.group(1)) + + query = f'''#Tool: WikibaseIntegrator wbi_fastrun._get_property_type + SELECT ?property_type WHERE {{ wd:{prop_nr} wikibase:propertyType ?property_type. }}''' + + results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'][0]['property_type']['value'] + + return results + + def get_entities(self, claims: list[Claim] | Claims | Claim, cache: bool | None = None, query_limit: int | None = None) -> list[str]: + """ + Return a list of entities who correspond to the specified claims. + + :param claims: A list of claims to query the SPARQL endpoint. + :param cache: Put data returned by the SPARQL endpoint in cache. Enabled by default. + :param query_limit: Limit the amount of results from the SPARQL server + :return: A list of entity ID. + """ + if isinstance(claims, Claim): + claims = [claims] + elif (not isinstance(claims, list) or not all(isinstance(n, Claim) for n in claims)) and not isinstance(claims, Claims): + raise ValueError("claims must be an instance of Claim or Claims or a list of Claim") + + if len(claims) == 0: + raise ValueError("claims must have at least one claim") + + entity_sets: list[set[str]] = [] + for claim in claims: + self.load_statements(claims=claim, cache=cache, limit=query_limit) + value_key = self._value_key(claim) + statements = self.data.get(claim.mainsnak.property_number, {}).get(value_key, []) if value_key is not None else [] + entity_sets.append({self._entity_id(statement['entity']) for statement in statements}) + + return sorted(set.intersection(*entity_sets)) + + def _statement_matches(self, claim: Claim, sid: str, use_qualifiers: bool, use_references: bool, use_rank: bool, cache: bool | None = None) -> bool: + """ + Deeply compare a statement of the Wikibase instance with a local claim holding the same value. + + :param claim: The local claim. + :param sid: The statement ID of the statement of the Wikibase instance. + :param use_qualifiers: Compare the qualifiers. + :param use_references: Compare the references. + :param use_rank: Compare the rank. + :param cache: Reuse the qualifiers, references and rank already loaded for this statement. Enabled by default. + :return: True if the statement matches the claim. + """ + if use_qualifiers: + qualifiers = self._load_qualifiers(sid, cache=cache) + + if qualifiers.count() != claim.qualifiers.count(): + log.debug("Difference in number of qualifiers, '%i' != '%i'", qualifiers.count(), claim.qualifiers.count()) + return False + + for qualifier in qualifiers: + if qualifier not in claim.qualifiers: + log.debug("Difference between two qualifiers") + return False + + if use_references: + references = self._load_references(sid, cache=cache) + + if len(references) != len(claim.references): + log.debug("Difference in number of references, '%i' != '%i'", len(references), len(claim.references)) + return False + + for reference in references: + if reference not in claim.references: + log.debug("Difference between two references") + return False + + if use_rank: + rank = self._load_rank(sid, cache=cache) + + if claim.rank != rank: + log.debug("Difference with the rank") + return False + + return True + + def write_required(self, claims: list[Claim] | Claims | Claim, entity_filter: list[str] | str | None = None, property_filter: list[str] | str | None = None, + action_if_exists: ActionIfExists = ActionIfExists.REPLACE_ALL, use_qualifiers: bool | None = None, use_references: bool | None = None, + use_rank: bool | None = None, cache: bool | None = None, query_limit: int | None = None) -> bool: + """ + Check if a write to the Wikibase instance is required: a write is not required only when at least one entity + already holds, for every claim, a statement with the same value (and the same qualifiers, references and rank, + depending on the flags). + + :param claims: The claims proposed to be written. + :param entity_filter: Allows you to filter the entities checked. This can be a single entity or a list of entities. + Pass the ID of the entity being edited to check whether this entity specifically already holds the data. + :param property_filter: Allows you to limit the difference comparison to a list of properties. Claims whose + property is not in the filter are ignored. + :param action_if_exists: The action that will be used for the write. With FORCE_APPEND, the statements are + always appended and a write is always required. The other actions check that the claims already exist. + :param use_qualifiers: Use qualifiers during fastrun. Enabled by default. + :param use_references: Use references during fastrun. Disabled by default. + :param use_rank: Use rank during fastrun. Disabled by default. + :param cache: Put data returned by the SPARQL endpoint in cache. Enabled by default. + :param query_limit: Limit the amount of results from the SPARQL server + :return: a boolean True if a write is required. False otherwise. + """ + + if isinstance(claims, Claim): + claims = [claims] + elif (not isinstance(claims, list) or not all(isinstance(n, Claim) for n in claims)) and not isinstance(claims, Claims): + raise ValueError("claims must be an instance of Claim or Claims or a list of Claim") + + if len(claims) == 0: + raise ValueError("claims must have at least one claim") + + if action_if_exists == ActionIfExists.FORCE_APPEND: + # The new statements are always appended, a write is always required + log.debug("Force append: write required") + return True + + entities_allowed: set[str] | None = None + if entity_filter is not None: + if isinstance(entity_filter, str): + entity_filter = [entity_filter] + entities_allowed = {self._entity_id(entity) for entity in entity_filter} + + if property_filter is not None and isinstance(property_filter, str): + property_filter = [property_filter] + + # Generate a property_filter if None is given + if property_filter is None: + property_filter = [claim.mainsnak.property_number for claim in claims] + + if use_qualifiers is None: + use_qualifiers = self.use_qualifiers + if use_references is None: + use_references = self.use_references + if use_rank is None: + use_rank = self.use_rank + + claims_to_check = [claim for claim in claims if claim.mainsnak.property_number in property_filter] + if not claims_to_check: + # Nothing can be verified through the fastrun data + log.debug("No claim matches the property filter: write required") + return True + + # Find, for each claim, the statements holding the same value + candidates: list[tuple[Claim, list[dict[str, str]]]] = [] + for claim in claims_to_check: + self.load_statements(claims=claim, cache=cache, limit=query_limit) + + value_key = self._value_key(claim) + statements = list(self.data.get(claim.mainsnak.property_number, {}).get(value_key, [])) if value_key is not None else [] + if entities_allowed is not None: + statements = [statement for statement in statements if self._entity_id(statement['entity']) in entities_allowed] + + if not statements: + log.debug("Value '%s' does not exist for property '%s'", claim.get_sparql_value(), claim.mainsnak.property_number) return True - log.debug("fast run success") - tmp_rs.pop(bool_vec.index(True)) + candidates.append((claim, statements)) - if len(tmp_rs) > 0: - log.debug("failed because not zero") - for x in tmp_rs: - log.debug([x.mainsnak.property_number, x.mainsnak.datavalue, [z.mainsnak.datavalue for z in x.qualifiers]]) - log.debug("failed because not zero--END") + # The entities holding every claim value + common_entities = set.intersection(*({self._entity_id(statement['entity']) for statement in statements} for _, statements in candidates)) + if not common_entities: + log.debug("No entity holds all the claim values: write required") return True - return False + # Deep comparison: no write is needed if at least one entity holds, for every claim, a statement also + # matching the qualifiers, references and rank, depending on the flags + for entity in sorted(common_entities): + for claim, statements in candidates: + entity_statements = [statement for statement in statements if self._entity_id(statement['entity']) == entity] + if not any(self._statement_matches(claim, statement['sid'], use_qualifiers=use_qualifiers, use_references=use_references, use_rank=use_rank, cache=cache) + for statement in entity_statements): + break + else: + log.debug("Entity '%s' already holds all the claims: no write required", entity) + return False + + return True def init_language_data(self, lang: str, lang_data_type: str) -> None: """ @@ -315,7 +697,7 @@ def get_language_data(self, qid: str, lang: str, lang_data_type: str) -> list[st """ get language data for specified qid - :param qid: Wikibase item id + :param qid: entity ID :param lang: language code :param lang_data_type: 'label', 'description' or 'aliases' :return: list of strings @@ -327,7 +709,7 @@ def get_language_data(self, qid: str, lang: str, lang_data_type: str) -> list[st self.init_language_data(lang, lang_data_type) current_lang_data = self.loaded_langs[lang][lang_data_type] - all_lang_strings = current_lang_data.get(qid, []) + all_lang_strings = list(current_lang_data.get(self._entity_id(qid), [])) if not all_lang_strings and lang_data_type in {'label', 'description'}: all_lang_strings = [''] return all_lang_strings @@ -335,12 +717,13 @@ def get_language_data(self, qid: str, lang: str, lang_data_type: str) -> list[st def check_language_data(self, qid: str, lang_data: list, lang: str, lang_data_type: str, action_if_exists: ActionIfExists = ActionIfExists.APPEND_OR_REPLACE) -> bool: """ Method to check if certain language data exists as a label, description or aliases - :param qid: Wikibase item id + + :param qid: entity ID :param lang_data: list of string values to check :param lang: language code :param lang_data_type: What kind of data is it? 'label', 'description' or 'aliases'? :param action_if_exists: If aliases already exist, APPEND_OR_REPLACE or REPLACE_ALL - :return: boolean + :return: boolean True if a write is required """ all_lang_strings = {x.strip().casefold() for x in self.get_language_data(qid, lang, lang_data_type)} @@ -354,230 +737,12 @@ def check_language_data(self, qid: str, lang_data: list, lang: str, lang_data_ty return False - def get_all_data(self) -> dict[str, dict]: - return self.prop_data - - def format_query_results(self, r: list, prop_nr: str) -> None: - """ - `r` is the results of the sparql query in _query_data and is modified in place - `prop_nr` is needed to get the property datatype to determine how to format the value - - `r` is a list of dicts. The keys are: - sid: statement ID - item: the subject. the item this statement is on - v: the object. The value for this statement - unit: property unit - pq: qualifier property - qval: qualifier value - qunit: qualifier unit - ref: reference ID - pr: reference property - rval: reference value - """ - prop_dt = self.get_prop_datatype(prop_nr) - for i in r: - for value in ['item', 'sid', 'pq', 'pr', 'ref', 'unit', 'qunit']: - if value in i: - if i[value]['value'].startswith(self.wikibase_url): - i[value] = i[value]['value'].split('/')[-1] - else: - # TODO: Dirty fix. If we are not on wikidata, we force unitless (Q199) to '1' - if i[value]['value'] == 'http://www.wikidata.org/entity/Q199': - i[value] = '1' - else: - i[value] = i[value]['value'] - - # make sure datetimes are formatted correctly. - # the correct format is '+%Y-%m-%dT%H:%M:%SZ', but is sometimes missing the plus?? - # some difference between RDF and xsd:dateTime that I don't understand - for value in ['v', 'qval', 'rval']: - if value in i: - if i[value].get("datatype") == 'http://www.w3.org/2001/XMLSchema#dateTime' and not i[value]['value'][0] in '+-': - # if it is a dateTime and doesn't start with plus or minus, add a plus - i[value]['value'] = '+' + i[value]['value'] - - # these three ({'v', 'qval', 'rval'}) are values that can be any data type - # strip off the URI if they are wikibase-items - if 'v' in i: - if i['v']['type'] == 'uri' and prop_dt == 'wikibase-item': - i['v'] = i['v']['value'].split('/')[-1] - elif i['v']['type'] == 'literal' and prop_dt == 'quantity': - i['v'] = format_amount(i['v']['value']) - elif i['v']['type'] == 'literal' and prop_dt == 'monolingualtext': - f = [x for x in self.base_data_type.subclasses if x.DTYPE == prop_dt][0](prop_nr=prop_nr, text=i['v']['value'], language=i['v']['xml:lang']) - i['v'] = f.get_sparql_value() - else: - f = [x for x in self.base_data_type.subclasses if x.DTYPE == prop_dt][0](prop_nr=prop_nr) - if not f.parse_sparql_value(value=i['v']['value'], type=i['v']['type']): - raise ValueError("Can't parse the value with parse_sparql_value()") - i['v'] = f.get_sparql_value() - - # Note: no-value and some-value don't actually show up in the results here - # see for example: select * where { wd:Q7207 p:P40 ?c . ?c ?d ?e } - if not isinstance(i['v'], dict): - self.rev_lookup[i['v']].add(i['item']) - if self.case_insensitive: - self.rev_lookup_ci[i['v'].casefold()].add(i['item']) - - # handle qualifier value - if 'qval' in i: - qual_prop_dt = self.get_prop_datatype(prop_nr=i['pq']) - if i['qval']['type'] == 'uri' and qual_prop_dt == 'wikibase-item': - i['qval'] = i['qval']['value'].split('/')[-1] - elif i['qval']['type'] == 'literal' and qual_prop_dt == 'quantity': - i['qval'] = format_amount(i['qval']['value']) - else: - i['qval'] = i['qval']['value'] - - # handle reference value - if 'rval' in i: - ref_prop_dt = self.get_prop_datatype(prop_nr=i['pr']) - if i['rval']['type'] == 'uri' and ref_prop_dt == 'wikibase-item': - i['rval'] = i['rval']['value'].split('/')[-1] - elif i['rval']['type'] == 'literal' and ref_prop_dt == 'quantity': - i['rval'] = format_amount(i['rval']['value']) - else: - i['rval'] = i['rval']['value'] - - def update_frc_from_query(self, r: list, prop_nr: str) -> None: - # r is the output of format_query_results - # this updates the frc from the query (result of _query_data) - for i in r: - qid = i['item'] - if qid not in self.prop_data: - self.prop_data[qid] = {prop_nr: {}} - if prop_nr not in self.prop_data[qid]: - self.prop_data[qid].update({prop_nr: {}}) - if i['sid'] not in self.prop_data[qid][prop_nr]: - self.prop_data[qid][prop_nr].update({i['sid']: {}}) - # update values for this statement (not including ref) - d = {'v': i['v']} - self.prop_data[qid][prop_nr][i['sid']].update(d) - - if 'qual' not in self.prop_data[qid][prop_nr][i['sid']]: - self.prop_data[qid][prop_nr][i['sid']]['qual'] = set() - if 'pq' in i and 'qval' in i: - if 'qunit' in i: - self.prop_data[qid][prop_nr][i['sid']]['qual'].add((i['pq'], i['qval'], i['qunit'])) - else: - self.prop_data[qid][prop_nr][i['sid']]['qual'].add((i['pq'], i['qval'], '1')) - - if 'ref' not in self.prop_data[qid][prop_nr][i['sid']]: - self.prop_data[qid][prop_nr][i['sid']]['ref'] = {} - if 'ref' in i: - if i['ref'] not in self.prop_data[qid][prop_nr][i['sid']]['ref']: - self.prop_data[qid][prop_nr][i['sid']]['ref'][i['ref']] = set() - self.prop_data[qid][prop_nr][i['sid']]['ref'][i['ref']].add((i['pr'], i['rval'])) - - if 'unit' not in self.prop_data[qid][prop_nr][i['sid']]: - self.prop_data[qid][prop_nr][i['sid']]['unit'] = '1' - if 'unit' in i: - self.prop_data[qid][prop_nr][i['sid']]['unit'] = i['unit'] - - def _query_data(self, prop_nr: str, use_units: bool = False, page_size: int = 10000) -> None: - page_count = 0 - - while True: - # Query header - query = ''' - #Tool: WikibaseIntegrator wbi_fastrun._query_data - SELECT ?sid ?item ?v ?unit ?pq ?qval ?qunit ?ref ?pr ?rval - WHERE - {{ - ''' - - # Base filter - query += ''' - {base_filter} - - ?item <{wb_url}/prop/{prop_nr}> ?sid . - ''' - - # Amount and unit - if use_units: - query += ''' - {{ - <{wb_url}/entity/{prop_nr}> wikibase:propertyType ?property_type . - FILTER (?property_type != wikibase:Quantity) - ?sid <{wb_url}/prop/statement/{prop_nr}> ?v . - }} - # Get amount and unit for the statement - UNION - {{ - ?sid <{wb_url}/prop/statement/value/{prop_nr}> [wikibase:quantityAmount ?v; wikibase:quantityUnit ?unit] . - }} - ''' - else: - query += ''' - <{wb_url}/entity/{prop_nr}> wikibase:propertyType ?property_type . - ?sid <{wb_url}/prop/statement/{prop_nr}> ?v . - ''' - - # Qualifiers - # Amount and unit - if use_units: - query += ''' - # Get qualifiers - OPTIONAL - {{ - {{ - # Get simple values for qualifiers which are not of type quantity - ?sid ?propQualifier ?qval . - ?pq wikibase:qualifier ?propQualifier . - ?pq wikibase:propertyType ?qualifer_property_type . - FILTER (?qualifer_property_type != wikibase:Quantity) - }} - UNION - {{ - # Get amount and unit for qualifiers of type quantity - ?sid ?pqv [wikibase:quantityAmount ?qval; wikibase:quantityUnit ?qunit] . - ?pq wikibase:qualifierValue ?pqv . - }} - }} - ''' - else: - query += ''' - # Get qualifiers - OPTIONAL - {{ - # Get simple values for qualifiers - ?sid ?propQualifier ?qval . - ?pq wikibase:qualifier ?propQualifier . - ?pq wikibase:propertyType ?qualifer_property_type . - }} - ''' - - # References - if self.use_refs: - query += ''' - # get references - OPTIONAL {{ - ?sid prov:wasDerivedFrom ?ref . - ?ref ?pr ?rval . - [] wikibase:reference ?pr - }} - ''' - # Query footer - query += ''' - }} ORDER BY ?sid OFFSET {offset} LIMIT {page_size} - ''' - - # Format the query - query = query.format(wb_url=self.wikibase_url, base_filter=self.base_filter_string, prop_nr=prop_nr, offset=str(page_count * page_size), page_size=str(page_size)) - - results = execute_sparql_query(query=query, endpoint=self.sparql_endpoint_url)['results']['bindings'] - self.format_query_results(results, prop_nr) - self.update_frc_from_query(results, prop_nr) - page_count += 1 - - if len(results) == 0 or len(results) < page_size: - break - def _query_lang(self, lang: str, lang_data_type: str) -> list[dict[str, dict]] | None: """ + Query the SPARQL endpoint for the language data of the entities matching the base filter. - :param lang: - :param lang_data_type: + :param lang: language code + :param lang_data_type: 'label', 'description' or 'aliases' """ lang_data_type_dict = { @@ -588,11 +753,11 @@ def _query_lang(self, lang: str, lang_data_type: str) -> list[dict[str, dict]] | query = f''' #Tool: WikibaseIntegrator wbi_fastrun._query_lang - SELECT ?item ?label WHERE {{ - {self.base_filter_string} + SELECT ?entity ?label WHERE {{ + {self._base_filter_string()} OPTIONAL {{ - ?item {lang_data_type_dict[lang_data_type]} ?label FILTER (lang(?label) = "{lang}") . + ?entity {lang_data_type_dict[lang_data_type]} ?label FILTER (lang(?label) = "{lang}") . }} }} ''' @@ -605,32 +770,22 @@ def _query_lang(self, lang: str, lang_data_type: str) -> list[dict[str, dict]] | def _process_lang(result: list) -> defaultdict[str, set]: data = defaultdict(set) for r in result: - qid = r['item']['value'].split("/")[-1] + qid = r['entity']['value'].split("/")[-1] if 'label' in r: data[qid].add(r['label']['value']) return data - def get_prop_datatype(self, prop_nr: str) -> str | None: - # Memoize in the per-instance prop_dt_map: this is tied to the container's lifetime (no global cache keeping - # containers alive), avoids re-instantiating WikibaseIntegrator and re-querying the API on cache hits, and is - # invalidated by clear(). - if prop_nr not in self.prop_dt_map: - from wikibaseintegrator import WikibaseIntegrator - wbi = WikibaseIntegrator() - datatype = wbi.property.get(prop_nr).datatype - if isinstance(datatype, WikibaseDatatype): - datatype = datatype.value - self.prop_dt_map[prop_nr] = datatype - return self.prop_dt_map[prop_nr] - def clear(self) -> None: """ - convenience function to empty this fastrun container + Convenience function to empty the caches of this fastrun container. """ - self.prop_dt_map = {} - self.prop_data = {} - self.rev_lookup = defaultdict(set) - self.rev_lookup_ci = defaultdict(set) + self.data = {} + self.loaded_complete = set() + self.properties_type = {} + self.loaded_langs = {} + self._qualifiers_cache = {} + self._references_cache = {} + self._rank_cache = {} def __repr__(self) -> str: """A mixin implementing a simple __repr__.""" @@ -641,25 +796,52 @@ def __repr__(self) -> str: ) -def get_fastrun_container(base_filter: list[BaseDataType | list[BaseDataType]] | None = None, use_refs: bool = False, case_insensitive: bool = False) -> FastRunContainer: +def get_fastrun_container(base_filter: list[BaseDataType | list[BaseDataType]], use_qualifiers: bool = True, use_references: bool = False, use_rank: bool = False, + cache: bool = True, case_insensitive: bool = False) -> FastRunContainer: + """ + Return a FastRunContainer object, create a new one if it doesn't already exist. + + :param base_filter: The default filter to initialize the dataset. A list made of BaseDataType or list of BaseDataType. + :param use_qualifiers: Use qualifiers during fastrun. Enabled by default. + :param use_references: Use references during fastrun. Disabled by default. + :param use_rank: Use rank during fastrun. Disabled by default. + :param cache: Put data returned by the SPARQL endpoint in cache. Enabled by default. + :param case_insensitive: Compare the string values without taking the case into account. Disabled by default. + :return: a FastRunContainer object + """ if base_filter is None: base_filter = [] # We search if we already have a FastRunContainer with the same parameters to reuse it - fastrun_container = _search_fastrun_store(base_filter=base_filter, use_refs=use_refs, case_insensitive=case_insensitive) + fastrun_container = _search_fastrun_store(base_filter=base_filter, use_qualifiers=use_qualifiers, use_references=use_references, use_rank=use_rank, + case_insensitive=case_insensitive, cache=cache) return fastrun_container -def _search_fastrun_store(base_filter: list[BaseDataType | list[BaseDataType]] | None = None, use_refs: bool = False, case_insensitive: bool = False) -> FastRunContainer: +def _search_fastrun_store(base_filter: list[BaseDataType | list[BaseDataType]], use_qualifiers: bool = True, use_references: bool = False, use_rank: bool = False, + cache: bool = True, case_insensitive: bool = False) -> FastRunContainer: + """ + Search for an existing FastRunContainer with the same parameters or create a new one if it doesn't exist. + + :param base_filter: The default filter to initialize the dataset. A list made of BaseDataType or list of BaseDataType. + :param use_qualifiers: Use qualifiers during fastrun. Enabled by default. + :param use_references: Use references during fastrun. Disabled by default. + :param use_rank: Use rank during fastrun. Disabled by default. + :param cache: Put data returned by the SPARQL endpoint in cache. Enabled by default. + :param case_insensitive: Compare the string values without taking the case into account. Disabled by default. + :return: a FastRunContainer object + """ for fastrun in fastrun_store: - if (fastrun.base_filter == base_filter) and (fastrun.use_refs == use_refs) and (fastrun.case_insensitive == case_insensitive) and ( - fastrun.sparql_endpoint_url == config['SPARQL_ENDPOINT_URL']): + if (fastrun.base_filter == base_filter) and (fastrun.use_qualifiers == use_qualifiers) and (fastrun.use_references == use_references) and ( + fastrun.use_rank == use_rank) and (fastrun.case_insensitive == case_insensitive) and (fastrun.sparql_endpoint_url == config['SPARQL_ENDPOINT_URL']): + fastrun.cache = cache return fastrun # In case nothing was found in the fastrun_store log.info("Create a new FastRunContainer") - fastrun_container = FastRunContainer(base_data_type=BaseDataType, base_filter=base_filter, use_refs=use_refs, case_insensitive=case_insensitive) + fastrun_container = FastRunContainer(base_data_type=BaseDataType, base_filter=base_filter, use_qualifiers=use_qualifiers, use_references=use_references, use_rank=use_rank, + cache=cache, case_insensitive=case_insensitive) fastrun_store.append(fastrun_container) return fastrun_container