diff --git a/README.md b/README.md index 8ab0a58c..aefcd389 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,11 @@ wikibaseintegrator~=0.11.3 - [Modify an existing item](#modify-an-existing-item) - [A bot for Mass Import](#a-bot-for-mass-import) - [Examples (in "fast run" mode)](#examples-in-fast-run-mode) + - [The base filter](#the-base-filter) + - [Checking if a write is required](#checking-if-a-write-is-required) + - [Options](#options) + - [Checking labels, descriptions and aliases](#checking-labels-descriptions-and-aliases) + - [Limitations](#limitations) - [Debugging](#debugging) # WikibaseIntegrator / WikidataIntegrator # @@ -91,7 +96,8 @@ The main differences between these two libraries are : * Add OAuth 2.0 login method * Add logging module support -But WikibaseIntegrator lack the "fastrun" functionality implemented in WikidataIntegrator. +WikibaseIntegrator also provides a rewritten version of the "fast run" mode of WikidataIntegrator, which avoids +unnecessary API writes (see [Examples (in "fast run" mode)](#examples-in-fast-run-mode)). # Documentation # @@ -799,21 +805,35 @@ for entrez_id, ensembl in raw_data.items(): # Examples (in "fast run" mode) # +The fast run mode is designed for bots synchronising an external resource with a Wikibase instance, where most of the +entities are usually already up to date. Instead of loading every entity through the MediaWiki API to compare it with +the local data, the fast run mode preloads the current state of all the entities of the data corpus with a few paginated +SPARQL queries, then decides locally, entity per entity, if a write is required. When nothing changed (the most common +case), no MediaWiki API call is made at all for that entity. + In order to use the fast run mode, you need to know the property/value combination which determines the data corpus you would like to operate on. E.g. for operating on human genes, you need to know that [P351](https://www.wikidata.org/entity/P351) is the NCBI Entrez Gene ID and you also need to know that you are dealing with humans, best represented by the [found in taxon property (P703)](https://www.wikidata.org/entity/P703) with the value [Q15978631](https://www.wikidata.org/entity/Q15978631) for Homo sapiens. -IMPORTANT: In order for the fast run mode to work, the data you provide in the constructor must contain at least one -unique value/id only present on one Wikidata element, e.g. an NCBI entrez gene ID, Uniprot ID, etc. Usually, these would -be the same unique core properties used for defining domains in wbi_core, e.g. for genes, proteins, drugs or your custom -domains. +IMPORTANT: In order for the fast run mode to work on entities without a known entity ID, the data you provide must +contain at least one unique value/id only present on one Wikibase entity, e.g. an NCBI entrez gene ID, Uniprot ID, etc. +This unique value is used to identify the target entity. If the entity ID is already known (e.g. the entity was loaded +with `wbi.item.get()`), this requirement does not apply. + +## The base filter ## + +The base filter defines the data corpus and is a list of datatype instances (the same classes used to create claims). +Three forms are supported: -Below, the normal mode run example from above, slightly modified, to meet the requirements for the fast run mode. To -enable it, ItemEngine requires two parameters, fast_run=True/False and fast_run_base_filter which is a dictionary -holding the properties to filter for as keys, and the item QIDs as dict values. If the value is not a QID but a literal, -just provide an empty string. For the above example, the dictionary looks like this: +* A property with a value: the entities must have this exact statement, e.g. `Item(prop_nr='P703', value='Q15978631')` + (found in taxon Homo sapiens). +* A property without a value: the entities must have a statement with this property, whatever the value, e.g. + `ExternalID(prop_nr='P351')` (any NCBI Entrez Gene ID). +* A property path, given as a list of two datatype instances: the value is reached through the first property followed + by any number of hops with the second one, e.g. `[Item(prop_nr='P31', value='Q11173'), Item(prop_nr='P279')]` + (instance of (P31) chemical compound (Q11173), directly or through a chain of subclass of (P279)). ```python from wikibaseintegrator.datatypes import ExternalID, Item @@ -821,20 +841,25 @@ from wikibaseintegrator.datatypes import ExternalID, Item fast_run_base_filter = [ExternalID(prop_nr='P351'), Item(prop_nr='P703', value='Q15978631')] ``` -The full example: +## Checking if a write is required ## + +The entry point is `entity.write_required()`. It returns `True` if the local entity differs from the live data and an +actual write is needed, `False` if the entity is already up to date and the write can be skipped. The full example, a +modified version of the mass import bot from the normal mode example: ```python from wikibaseintegrator import WikibaseIntegrator, wbi_login -from wikibaseintegrator.datatypes import ExternalID, Item, String, Time +from wikibaseintegrator.datatypes import ExternalID, Item, Time from wikibaseintegrator.wbi_enums import WikibaseTimePrecision # login object login = wbi_login.OAuth2(consumer_token='', consumer_secret='') -fast_run_base_filter = [ExternalID(prop_nr='P351'), Item(prop_nr='P703', value='Q15978631')] -fast_run = True +wbi = WikibaseIntegrator(login=login) -# We have raw data, which should be written to Wikidata, namely two human NCBI entrez gene IDs mapped to two Ensembl Gene IDs +fast_run_base_filter = [ExternalID(prop_nr='P351'), ExternalID(prop_nr='P704'), Item(prop_nr='P703', value='Q15978631')] + +# We have raw data, which should be written to Wikidata, namely two human NCBI entrez gene IDs mapped to two Ensembl transcript IDs # You can iterate over any data source as long as you can map the values to Wikidata properties. raw_data = { '50943': 'ENST00000376197', @@ -845,31 +870,75 @@ for entrez_id, ensembl in raw_data.items(): # add some references references = [ [ - Item(value='Q20641742', prop_nr='P248') - ], - [ - Time(time='+2020-02-08T00:00:00Z', prop_nr='P813', precision=WikibaseTimePrecision.DAY), - ExternalID(value='1017', prop_nr='P351') + Item(value='Q20641742', prop_nr='P248'), + Time(time='+2020-02-08T00:00:00Z', prop_nr='P813', precision=WikibaseTimePrecision.DAY) ] ] - # data type object - entrez_gene_id = String(value=entrez_id, prop_nr='P351', references=references) - ensembl_transcript_id = String(value=ensembl, prop_nr='P704', references=references) + # data type objects + entrez_gene_id = ExternalID(value=entrez_id, prop_nr='P351', references=references) + ensembl_transcript_id = ExternalID(value=ensembl, prop_nr='P704', references=references) # data goes into a list, because many data objects can be provided to data = [entrez_gene_id, ensembl_transcript_id] - # Search for and then edit/create new item - wb_item = WikibaseIntegrator(login=login).item.new() - wb_item.add_claims(claims=data) - wb_item.init_fastrun(base_filter=fast_run_base_filter) - wb_item.write() + item = wbi.item.new() + item.claims.add(data) + + # Compare the local data with the live data and only write when something differs + if item.write_required(base_filter=fast_run_base_filter): + item.write() ``` -Note: Fastrun mode checks for equality of property/value pairs, qualifiers (not including qualifier attributes), labels, -aliases and description, but it ignores references by default! -References can be checked in fast run mode by setting `use_refs` to `True`. +Note: only the claims whose property appears in the base filter are compared. In the example above, the P351 and P704 +claims are checked because both properties are part of the base filter; a claim on any other property would be ignored +by the comparison. + +## Options ## + +`write_required()` accepts optional parameters: + +* `use_refs` (default `False`): also compare the references of the statements. By default, references are ignored. +* `case_insensitive` (default `False`): compare string values case-insensitively. +* `action_if_exists` (default `ActionIfExists.REPLACE_ALL`): how the local statements are compared with the live ones. + With `REPLACE_ALL`, the entity must hold exactly the provided statements for the filtered properties. With + `APPEND_OR_REPLACE`, the provided statements must already exist on the entity, but additional existing statements are + allowed. With `FORCE_APPEND`, a write is always reported as required. + +The preloaded data is shared: containers are cached at the module level and reused by every `write_required()` call +using the same base filter and options, so the SPARQL queries are only executed once per property. + +## Checking labels, descriptions and aliases ## + +`write_required()` only compares claims. To check language data (labels, descriptions or aliases), use the fastrun +container directly: + +```python +from wikibaseintegrator import wbi_fastrun +from wikibaseintegrator.datatypes import ExternalID, Item + +frc = wbi_fastrun.get_fastrun_container(base_filter=[ExternalID(prop_nr='P351'), Item(prop_nr='P703', value='Q15978631')]) + +# Resolve the entity ID from a unique value, without any MediaWiki API call +qid = frc.get_item(claims=[ExternalID(value='50943', prop_nr='P351')]) + +# Returns True if the English label differs from 'CDK7' and a write is required +frc.check_language_data(qid=qid, lang_data=['CDK7'], lang='en', lang_data_type='label') +``` + +`check_language_data()` accepts `'label'`, `'description'` or `'aliases'` as `lang_data_type`, and the same +`action_if_exists` logic as above (`APPEND_OR_REPLACE` or `REPLACE_ALL`). + +## Limitations ## + +* The comparison is based on the SPARQL endpoint, which can lag behind the live data (e.g. the Wikidata Query Service + replication lag). A decision taken on stale data results, at worst, in one unnecessary write or one missed update + until the next run. +* The whole data corpus selected by the base filter is loaded in memory. Choose a base filter that selects a reasonably + sized subset of the instance. +* The statement attributes that are not exposed in the SPARQL simple values (time precision, globe coordinate + precision, quantity bounds...) are reconstructed with their default values: statements using non-default values for + these attributes are always reported as requiring a write. # Debugging # diff --git a/test/test_datatypes.py b/test/test_datatypes.py index e2bb420f..b6d8148c 100644 --- a/test/test_datatypes.py +++ b/test/test_datatypes.py @@ -87,6 +87,22 @@ def test_large_year_parsing(self): # Ordering keeps working across the 4/5-digit boundary assert Time(time='+9999-01-01T00:00:00Z', prop_nr='P5') < time + def test_parse_sparql_value(self): + # A bare RDF timestamp: the missing '+' is added and the precision is inferred + time = Time(prop_nr='P5') + assert time.parse_sparql_value('2020-02-08T00:00:00Z') is True + assert time.mainsnak.datavalue['value']['time'] == '+2020-02-08T00:00:00Z' + assert time.mainsnak.datavalue['value']['precision'] == WikibaseTimePrecision.DAY.value + + # A quoted literal with its datatype suffix + time = Time(prop_nr='P5') + assert time.parse_sparql_value('"+2020-02-00T00:00:00Z"^^xsd:dateTime') is True + assert time.mainsnak.datavalue['value']['precision'] == WikibaseTimePrecision.MONTH.value + + # A timestamp with a time part can't be represented, its precision can't be inferred + assert Time(prop_nr='P5').parse_sparql_value('2020-02-08T12:34:56Z') is False + assert Time(prop_nr='P5').parse_sparql_value('not a timestamp') is False + class TestRank: def test_rank_parsing(self): diff --git a/test/test_wbi_fastrun.py b/test/test_wbi_fastrun.py index 54799266..be93941d 100644 --- a/test/test_wbi_fastrun.py +++ b/test/test_wbi_fastrun.py @@ -10,7 +10,7 @@ import pytest from wikibaseintegrator import WikibaseIntegrator, wbi_fastrun -from wikibaseintegrator.datatypes import BaseDataType, ExternalID, Item +from wikibaseintegrator.datatypes import BaseDataType, ExternalID, GlobeCoordinate, Item, MonolingualText, Quantity, Time from wikibaseintegrator.entities import ItemEntity from wikibaseintegrator.wbi_enums import ActionIfExists @@ -77,6 +77,47 @@ def test_query_data_item_and_uri_values(self, wikibase): values = {statement['v'] for statement in frc.prop_data['Q99']['P2888'].values()} assert all(value.startswith(' str: return self.mainsnak.datavalue['value']['time'] + def parse_sparql_value(self, value, type='literal', unit='1') -> bool: + pattern = re.compile(r'^"?([+-]?[0-9]{1,16}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)"?(?:\^\^xsd:dateTime)?$') + matches = pattern.match(value) + if not matches: + return False + + try: + # The RDF value does not carry the precision, set_value() infers it from the timestamp + self.set_value(time=matches.group(1)) + except ValueError: + # The precision cannot be inferred (e.g. a timestamp with a time part), set_value() cannot represent it + return False + + return True + def _time_parts(self) -> tuple[int, int, int]: """ Split the timestamp into (year, month, day). diff --git a/wikibaseintegrator/wbi_fastrun.py b/wikibaseintegrator/wbi_fastrun.py index bdeed124..fb5d2161 100644 --- a/wikibaseintegrator/wbi_fastrun.py +++ b/wikibaseintegrator/wbi_fastrun.py @@ -20,6 +20,10 @@ fastrun_store: list[FastRunContainer] = [] +# The RDF export of a Wikibase instance always represents a quantity without a unit ('1' in the JSON representation) +# with the Wikidata entity Q199 (the number one), whatever the instance. +UNITLESS_UNIT_URI = 'http://www.wikidata.org/entity/Q199' + class FastRunContainer: def __init__(self, base_data_type: type[BaseDataType], mediawiki_api_url: str | None = None, sparql_endpoint_url: str | None = None, wikibase_url: str | None = None, @@ -61,6 +65,32 @@ def __init__(self, base_data_type: type[BaseDataType], mediawiki_api_url: str | else: raise ValueError("base_filter must be an instance of BaseDataType or a list of instances of BaseDataType") + def _get_datatype_class(self, datatype: str | None) -> type[BaseDataType]: + """ + Return the data type class implementing the given Wikibase datatype (e.g. 'external-id' -> ExternalID). + + :param datatype: A Wikibase datatype name + :exception ValueError: if no class implements the given datatype + """ + for subclass in self.base_data_type.subclasses: + if subclass.DTYPE == datatype: + return subclass + raise ValueError(f"No data type class found for datatype '{datatype}'") + + def _reconstruct_snak_datatype(self, prop_nr: str, value: str, unit: str = '1') -> BaseDataType: + """ + Rebuild a datatype object from the raw SPARQL value stored in prop_data, used for qualifiers and references. + + :param prop_nr: The property number of the snak + :param value: The raw SPARQL value + :param unit: The unit entity ID if the value is a quantity + :exception ValueError: if the value can't be parsed by the datatype class + """ + datatype = self._get_datatype_class(self.prop_dt_map[prop_nr])(prop_nr=prop_nr) + if not datatype.parse_sparql_value(value=value, unit=unit): + raise ValueError(f"Can't parse the value '{value}' of property {prop_nr} with parse_sparql_value()") + return datatype + def reconstruct_statements(self, qid: str) -> list[BaseDataType]: reconstructed_statements: list[BaseDataType] = [] @@ -77,32 +107,20 @@ def reconstruct_statements(self, qid: str) -> list[BaseDataType]: if prop not in self.prop_dt_map: self.prop_dt_map.update({prop: self.get_prop_datatype(prop)}) # reconstruct statements from frc (including unit, qualifiers, and refs) + # Note: attributes missing from the SPARQL simple values (time precision, globe coordinate precision, + # quantity bounds...) are rebuilt with their default value, so statements using non-default attributes + # are always reported as requiring a write. for _, d in dt.items(): - qualifiers = [] - for q in d['qual']: - f = [x for x in self.base_data_type.subclasses if x.DTYPE == self.prop_dt_map[q[0]]][0] - # TODO: Add support for more data type (Time, MonolingualText, GlobeCoordinate) - if self.prop_dt_map[q[0]] == 'quantity': - qualifiers.append(f(value=q[1], prop_nr=q[0], unit=q[2])) - else: - qualifiers.append(f(value=q[1], prop_nr=q[0])) + qualifiers = [self._reconstruct_snak_datatype(prop_nr=q[0], value=q[1], unit=q[2]) for q in d['qual']] references = [] for _, refs in d['ref'].items(): - this_ref = [] - for ref in refs: - f = [x for x in self.base_data_type.subclasses if x.DTYPE == self.prop_dt_map[ref[0]]][0] - this_ref.append(f(value=ref[1], prop_nr=ref[0])) - references.append(this_ref) - - f = [x for x in self.base_data_type.subclasses if x.DTYPE == self.prop_dt_map[prop_nr]][0] - # TODO: Add support for more data type - if self.prop_dt_map[prop_nr] == 'quantity': - datatype = f(prop_nr=prop_nr, qualifiers=qualifiers, references=references, unit=d['unit']) - datatype.parse_sparql_value(value=d['v'], unit=d['unit']) - else: - datatype = f(prop_nr=prop_nr, qualifiers=qualifiers, references=references) - datatype.parse_sparql_value(value=d['v']) + references.append([self._reconstruct_snak_datatype(prop_nr=ref[0], value=ref[1]) for ref in refs]) + + f = self._get_datatype_class(self.prop_dt_map[prop_nr]) + datatype = f(prop_nr=prop_nr, qualifiers=qualifiers, references=references) + if not datatype.parse_sparql_value(value=d['v'], unit=d.get('unit', '1')): + raise ValueError(f"Can't parse the value '{d['v']}' of property {prop_nr} with parse_sparql_value()") reconstructed_statements.append(datatype) # this isn't used. done for debugging purposes @@ -114,9 +132,8 @@ def get_items(self, claims: list[Claim] | Claims | Claim, cqid: str | None = Non Get items ID from a SPARQL endpoint :param claims: A list of claims the entities should have - :param cqid: - :return: a list of entity ID or None - :exception: if there is more than one claim + :param cqid: If given, this entity ID is returned instead of the IDs found by the value lookup + :return: a set of entity IDs or None if no entity matches the claims """ match_sets = [] @@ -127,7 +144,7 @@ def get_items(self, claims: list[Claim] | Claims | Claim, cqid: str | None = Non for claim in claims: # skip to next if statement has no value or no data type defined, e.g. for deletion objects - if not claim.mainsnak.datavalue and not claim.mainsnak.datatype: + if not claim.mainsnak.datavalue or not claim.mainsnak.datatype: continue prop_nr = claim.mainsnak.property_number @@ -141,18 +158,16 @@ def get_items(self, claims: list[Claim] | Claims | Claim, cqid: str | None = Non self.prop_dt_map.update({prop_nr: self.get_prop_datatype(prop_nr)}) self._query_data(prop_nr=prop_nr, use_units=self.prop_dt_map[prop_nr] == 'quantity') - # noinspection PyProtectedMember - current_value = claim.get_sparql_value() - + # The value must be formatted the same way as the rev_lookup keys built in format_query_results() if self.prop_dt_map[prop_nr] == 'wikibase-item': current_value = claim.mainsnak.datavalue['value']['id'] + elif self.prop_dt_map[prop_nr] == 'quantity': + # rev_lookup stores plain amounts (e.g. '+42'), not the full SPARQL literal returned by get_sparql_value() + current_value = format_amount(claim.mainsnak.datavalue['value']['amount']) + else: + current_value = claim.get_sparql_value() log.debug(current_value) - # if self.case_insensitive: - # log.debug("case insensitive enabled") - # log.debug(self.rev_lookup_ci) - # else: - # log.debug(self.rev_lookup) if current_value in self.rev_lookup: # quick check for if the value has ever been seen before, if not, write required @@ -202,12 +217,16 @@ def write_required(self, data: list[Claim], action_if_exists: ActionIfExists = A :param cqid: :return: Return True if the write is required """ - del_props = set() data_props = set() - append_props = [] - if action_if_exists == ActionIfExists.APPEND_OR_REPLACE: + append_props: list[str] = [] + if action_if_exists in (ActionIfExists.APPEND_OR_REPLACE, ActionIfExists.FORCE_APPEND): append_props = [x.mainsnak.property_number for x in data] + if append_props and action_if_exists == ActionIfExists.FORCE_APPEND: + # The new statements are always appended, so a write is always required + log.debug("force append: write required") + return True + for x in data: if x.mainsnak.datavalue and x.mainsnak.datatype: data_props.add(x.mainsnak.property_number) @@ -219,67 +238,48 @@ def write_required(self, data: list[Claim], action_if_exists: ActionIfExists = A reconstructed_statements = self.reconstruct_statements(qid) tmp_rs = copy.deepcopy(reconstructed_statements) - # handle append properties - for p in append_props: + # handle append properties: every new statement must already exist on the item with the same value + # (and the same references if use_refs is enabled), otherwise a write is required + for p in set(append_props): app_data = [x for x in data if x.mainsnak.property_number == p] # new statements rec_app_data = [x for x in tmp_rs if x.mainsnak.property_number == p] # orig statements - comp = [] - for x in app_data: - for y in rec_app_data: - if x.mainsnak.datavalue == y.mainsnak.datavalue: - if y.equals(x, include_ref=self.use_refs) and action_if_exists != ActionIfExists.FORCE_APPEND: - comp.append(True) - - # comp = [True for x in app_data for y in rec_app_data if x.equals(y, include_ref=self.use_refs)] - if len(comp) != len(app_data): - log.debug("failed append: %s", p) - return True + for new_statement in app_data: + if not any(original.mainsnak.datavalue == new_statement.mainsnak.datavalue and original.equals(new_statement, include_ref=self.use_refs) + for original in rec_app_data): + log.debug("failed append: %s", p) + return True tmp_rs = [x for x in tmp_rs if x.mainsnak.property_number not in append_props and x.mainsnak.property_number in data_props] - for date in data: + for statement in data: # ensure that statements meant for deletion get handled properly reconst_props = {x.mainsnak.property_number for x in tmp_rs} - if not date.mainsnak.datatype and date.mainsnak.property_number in reconst_props: + if not statement.mainsnak.datatype and statement.mainsnak.property_number in reconst_props: log.debug("returned from delete prop handling") return True - if not date.mainsnak.datavalue or not date.mainsnak.datatype: + if not statement.mainsnak.datavalue or not statement.mainsnak.datatype: # Ignore the deletion statements which are not in the reconstructed statements. continue - if date.mainsnak.property_number in append_props: - # TODO: check if value already exist and already have the same value + if statement.mainsnak.property_number in append_props: continue - if not date.mainsnak.datavalue and not date.mainsnak.datatype: - del_props.add(date.mainsnak.property_number) - # this is where the magic happens - # date is a new statement, proposed to be written + # statement is a new statement, proposed to be written # tmp_rs are the reconstructed statements == current state of the item - bool_vec = [] - for x in tmp_rs: - if (x == date or (self.case_insensitive and x.mainsnak.datavalue.casefold() == date.mainsnak.datavalue.casefold())) and x.mainsnak.property_number not in del_props: - bool_vec.append(x.equals(date, include_ref=self.use_refs)) - else: - bool_vec.append(False) - # bool_vec = [x.equals(date, include_ref=self.use_refs, fref=self.ref_comparison_f) and - # x.mainsnak.property_number not in del_props for x in tmp_rs] + bool_vec = [self._statements_equal(x, statement) for x in tmp_rs] - log.debug("bool_vec: %s", bool_vec) - log.debug("-----------------------------------") - for x in tmp_rs: - if x == date and x.mainsnak.property_number not in del_props: - log.debug([x.mainsnak.property_number, x.mainsnak.datavalue, [z.datavalue for z in x.qualifiers]]) - log.debug([date.mainsnak.property_number, date.mainsnak.datavalue, [z.datavalue for z in date.qualifiers]]) - elif x.mainsnak.property_number == date.mainsnak.property_number: - log.debug([x.mainsnak.property_number, x.mainsnak.datavalue, [z.datavalue for z in x.qualifiers]]) - log.debug([date.mainsnak.property_number, date.mainsnak.datavalue, [z.datavalue for z in date.qualifiers]]) + if log.isEnabledFor(logging.DEBUG): + log.debug("bool_vec: %s", bool_vec) + log.debug("-----------------------------------") + for x in tmp_rs: + if x.mainsnak.property_number == statement.mainsnak.property_number: + log.debug([x.mainsnak.property_number, x.mainsnak.datavalue, [z.datavalue for z in x.qualifiers]]) + log.debug([statement.mainsnak.property_number, statement.mainsnak.datavalue, [z.datavalue for z in statement.qualifiers]]) if not any(bool_vec): - log.debug(len(bool_vec)) - log.debug("fast run failed at %s", date.mainsnak.property_number) + log.debug("fast run failed at %s (%s candidate statements)", statement.mainsnak.property_number, len(bool_vec)) return True log.debug("fast run success") @@ -294,6 +294,31 @@ def write_required(self, data: list[Claim], action_if_exists: ActionIfExists = A return False + def _statements_equal(self, statement: BaseDataType, new_statement: Claim) -> bool: + """ + Compare a reconstructed statement with a new statement, including the references if use_refs is enabled. + When case_insensitive is enabled, string values differing only by case are considered equal. + + :param statement: A statement reconstructed from the SPARQL data (the current state of the entity) + :param new_statement: The statement proposed to be written + :return: True if both statements are considered equal + """ + if statement.equals(new_statement, include_ref=self.use_refs): + return True + + if not self.case_insensitive or statement.mainsnak.property_number != new_statement.mainsnak.property_number: + return False + + current_value = (statement.mainsnak.datavalue or {}).get('value') + new_value = (new_statement.mainsnak.datavalue or {}).get('value') + if not isinstance(current_value, str) or not isinstance(new_value, str) or current_value.casefold() != new_value.casefold(): + return False + + if not statement.has_equal_qualifiers(new_statement): + return False + + return not self.use_refs or Claim.refs_equal(statement, new_statement) + def init_language_data(self, lang: str, lang_data_type: str) -> None: """ Initialize language data store @@ -357,6 +382,20 @@ def check_language_data(self, qid: str, lang_data: list, lang: str, lang_data_ty def get_all_data(self) -> dict[str, dict]: return self.prop_data + def _normalize_unit(self, unit_uri: str) -> str: + """ + Normalize a unit URI from the SPARQL results to the format used in the JSON representation: the entity ID for + the units local to the instance, and '1' for unitless quantities, which the RDF export always represents with + the Wikidata entity Q199, whatever the Wikibase instance. + + :param unit_uri: The unit URI from the SPARQL results + """ + if unit_uri == UNITLESS_UNIT_URI: + return '1' + if unit_uri.startswith(self.wikibase_url): + return unit_uri.split('/')[-1] + return unit_uri + def format_query_results(self, r: list, prop_nr: str) -> None: """ `r` is the results of the sparql query in _query_data and is modified in place @@ -376,16 +415,17 @@ def format_query_results(self, r: list, prop_nr: str) -> None: """ prop_dt = self.get_prop_datatype(prop_nr) for i in r: - for value in ['item', 'sid', 'pq', 'pr', 'ref', 'unit', 'qunit']: + for value in ['item', 'sid', 'pq', 'pr', 'ref']: if value in i: if i[value]['value'].startswith(self.wikibase_url): i[value] = i[value]['value'].split('/')[-1] else: - # TODO: Dirty fix. If we are not on wikidata, we force unitless (Q199) to '1' - if i[value]['value'] == 'http://www.wikidata.org/entity/Q199': - i[value] = '1' - else: - i[value] = i[value]['value'] + i[value] = i[value]['value'] + + # normalize the unit URIs to entity IDs and the unitless unit to '1' + for value in ['unit', 'qunit']: + if value in i: + i[value] = self._normalize_unit(i[value]['value']) # make sure datetimes are formatted correctly. # the correct format is '+%Y-%m-%dT%H:%M:%SZ', but is sometimes missing the plus?? @@ -404,10 +444,10 @@ def format_query_results(self, r: list, prop_nr: str) -> None: elif i['v']['type'] == 'literal' and prop_dt == 'quantity': i['v'] = format_amount(i['v']['value']) elif i['v']['type'] == 'literal' and prop_dt == 'monolingualtext': - f = [x for x in self.base_data_type.subclasses if x.DTYPE == prop_dt][0](prop_nr=prop_nr, text=i['v']['value'], language=i['v']['xml:lang']) + f = self._get_datatype_class(prop_dt)(prop_nr=prop_nr, text=i['v']['value'], language=i['v']['xml:lang']) i['v'] = f.get_sparql_value() else: - f = [x for x in self.base_data_type.subclasses if x.DTYPE == prop_dt][0](prop_nr=prop_nr) + f = self._get_datatype_class(prop_dt)(prop_nr=prop_nr) if not f.parse_sparql_value(value=i['v']['value'], type=i['v']['type']): raise ValueError("Can't parse the value with parse_sparql_value()") i['v'] = f.get_sparql_value() @@ -426,6 +466,9 @@ def format_query_results(self, r: list, prop_nr: str) -> None: i['qval'] = i['qval']['value'].split('/')[-1] elif i['qval']['type'] == 'literal' and qual_prop_dt == 'quantity': i['qval'] = format_amount(i['qval']['value']) + elif i['qval']['type'] == 'literal' and qual_prop_dt == 'monolingualtext' and 'xml:lang' in i['qval']: + # keep the language, it is needed to reconstruct the qualifier + i['qval'] = self._get_datatype_class(qual_prop_dt)(prop_nr=i['pq'], text=i['qval']['value'], language=i['qval']['xml:lang']).get_sparql_value() else: i['qval'] = i['qval']['value'] @@ -436,6 +479,9 @@ def format_query_results(self, r: list, prop_nr: str) -> None: i['rval'] = i['rval']['value'].split('/')[-1] elif i['rval']['type'] == 'literal' and ref_prop_dt == 'quantity': i['rval'] = format_amount(i['rval']['value']) + elif i['rval']['type'] == 'literal' and ref_prop_dt == 'monolingualtext' and 'xml:lang' in i['rval']: + # keep the language, it is needed to reconstruct the reference + i['rval'] = self._get_datatype_class(ref_prop_dt)(prop_nr=i['pr'], text=i['rval']['value'], language=i['rval']['xml:lang']).get_sparql_value() else: i['rval'] = i['rval']['value'] @@ -570,7 +616,7 @@ def _query_data(self, prop_nr: str, use_units: bool = False, page_size: int = 10 self.update_frc_from_query(results, prop_nr) page_count += 1 - if len(results) == 0 or len(results) < page_size: + if len(results) < page_size: break def _query_lang(self, lang: str, lang_data_type: str) -> list[dict[str, dict]] | None: @@ -631,6 +677,7 @@ def clear(self) -> None: self.prop_data = {} self.rev_lookup = defaultdict(set) self.rev_lookup_ci = defaultdict(set) + self.loaded_langs = {} def __repr__(self) -> str: """A mixin implementing a simple __repr__."""