diff --git a/examples/configurations/database-sources/prod-nexus-sources.yml b/examples/configurations/database-sources/prod-nexus-sources.yml new file mode 100644 index 00000000..8928abd2 --- /dev/null +++ b/examples/configurations/database-sources/prod-nexus-sources.yml @@ -0,0 +1,96 @@ +Model: + name: RdfModel + origin: store + source: BlueBrainNexus + context: + iri: "https://bbp.neuroshapes.org" + bucket: "neurosciencegraph/datamodels" +Store: + name: BlueBrainNexus + endpoint: https://bbp.epfl.ch/nexus/v1 + searchendpoints: + sparql: + endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" + elastic: + endpoint: "https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset" + mapping: "https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset" + default_str_keyword_field: "keyword" + vocabulary: + metadata: + iri: "https://bluebrain.github.io/nexus/contexts/metadata.json" + local_iri: "https://bluebrainnexus.io/contexts/metadata.json" + namespace: "https://bluebrain.github.io/nexus/vocabulary/" + deprecated_property: "https://bluebrain.github.io/nexus/vocabulary/deprecated" + project_property: "https://bluebrain.github.io/nexus/vocabulary/project" + max_connection: 50 + versioned_id_template: "{x.id}?rev={x._store_metadata._rev}" + file_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson + +Resolvers: + ontology: + - resolver: OntologyResolver + origin: store + source: BlueBrainNexus + targets: + - identifier: terms + bucket: neurosciencegraph/datamodels + searchendpoints: + sparql: + endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" + result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/term-to-resource-mapping.hjson + agent: + - resolver: AgentResolver + origin: store + source: BlueBrainNexus + targets: + - identifier: agents + bucket: bbp/agents + searchendpoints: + sparql: + endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" + result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/agent-to-resource-mapping.hjson + +Formatters: + identifier: https://bbp.epfl.ch/neurosciencegraph/data/{}/{} + identifier_neuroelectro: http://neuroelectro.org/api/1/{}/{} + identifier_neurolex: http://neurolex.org/wiki/{} + +Databases: + UniProt: + origin: store + source: SPARQLStore + searchendpoints: + sparql: + endpoint: "https://sparql.uniprot.org/sparql" + model: + name: RdfModel + origin: directory + source: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/UniProt + context: + iri: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/UniProt/jsonld_context.json + NeuroElectro: + origin: store + source: BlueBrainNexus + bucket: bbp/neuroelectro + model: + name: RdfModel + origin: directory + source: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/NeuroElectro + context: + iri: /Users/cgonzale/Documents/code/nexus-forge/examples/models/neuroshapes_context.json + NeuroMorpho: + origin: web_service + source: http://neuromorpho.org/api/neuron + health: http://neuromorpho.org/api/health + files_download: + endpoint: https://neuromorpho.org/dableFiles + Accept: "*/*" + max_connection: 50 + content_type: application/json;charset=UTF-8 + accept: "*/*" + model: + name: RdfModel + origin: directory + source: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/Neuromorpho + context: + iri: /Users/cgonzale/Documents/code/nexus-forge/examples/database-sources/Neuromorpho/jsonld_context.json \ No newline at end of file diff --git a/examples/database-sources/DemoDB/mappings/DictionaryMappings/Example.hjson b/examples/database-sources/DemoDB/mappings/DictionaryMappings/Example.hjson new file mode 100644 index 00000000..0706644e --- /dev/null +++ b/examples/database-sources/DemoDB/mappings/DictionaryMappings/Example.hjson @@ -0,0 +1,9 @@ +{ + id: forge.format('identifier', "examples", x.id.split('/')[-1]) + type: [ + Entity + ] + name: x.name + label: x.label + subject: forge.resolve(x.subject, scope='ontology') if forge.resolve(x.subject, scope='ontology') is not None else {"label": x.subject} +} \ No newline at end of file diff --git a/examples/database-sources/MouseLight/mappings/DictionaryMapping/NeuronMorphology.hjson b/examples/database-sources/MouseLight/mappings/DictionaryMapping/NeuronMorphology.hjson new file mode 100644 index 00000000..a723c936 --- /dev/null +++ b/examples/database-sources/MouseLight/mappings/DictionaryMapping/NeuronMorphology.hjson @@ -0,0 +1,73 @@ +{ + type: [ + Dataset + NeuronMorphology + ] + id: forge.format("identifier", "neuronmorphologies/mouselight", x.neurons[0]["idString"]) + brainLocation: + { + type: BrainLocation + brainRegion: + { + id: f"http://api.brain-map.org/api/v2/data/Structure/{x.neurons[0]['soma']['allenId']}" + label: x.neurons[0]["allenLabel"] + } + coordinatesInBrainAtlas: + { + valueX: x.neurons[0]["soma"]["x"] + valueY: x.neurons[0]["soma"]["y"] + valueZ: x.neurons[0]["soma"]["z"] + } + } + contribution: + { + type: Contribution + agent: + { + type: Organization + id: https://www.grid.ac/institutes/grid.443970.d + label: "Janelia Research Campus" + } + } + distribution: forge.attach(f"./mouselight/{x.neurons[0]['idString']}.swc", content_type="application/swc") + identifier: x.neurons[0]["idString"] + name: x.neurons[0]["idString"] + generation: + { + type: Generation + activity: + { + type: "nsg:NeuronMorphologyReconstruction" + hadProtocol:{ + + } + } + } + subject: + { + type: Subject + species: { + id: http://purl.obolibrary.org/obo/NCBITaxon_10090 + label: Mus musculus + } + strain: { + label: x.neurons[0]["sample"]["strain"] + } + } + license: + { + type: License + id: https://mouselight.janelia.org + } + objectOfStudy: + { + type: ObjectOfStudy + id: http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells + label: Single Cell + } + dateCreated: x.neurons[0]["sample"]["date"] + version: x.neurons[0]["annotationSpace"]["version"] + description: x.neurons[0]["annotationSpace"]["description"] + virus: x.neurons[0]["label"]["virus"] + fluorophore: x.neurons[0]["label"]["fluorophore"] +} \ No newline at end of file diff --git a/examples/database-sources/MouseLight/metadata.json b/examples/database-sources/MouseLight/metadata.json new file mode 100644 index 00000000..5f4f52e0 --- /dev/null +++ b/examples/database-sources/MouseLight/metadata.json @@ -0,0 +1,8 @@ +{ + "description": "The MouseLight project generates datasets of whole mouse brains imaged at submicron resolution that allows reconstructions of complete axonal arbors of individual neurons across the entire mouse brain. ", + "url": "https://www.janelia.org/project-team/mouselight/neuronbrowser", + "license" : { + "id": "https://creativecommons.org/licenses/by-nc/4.0", + "label": "CC BY-NC 4.0" + } +} \ No newline at end of file diff --git a/examples/database-sources/NeuroElectro/jsonld_context.json b/examples/database-sources/NeuroElectro/jsonld_context.json new file mode 100644 index 00000000..1dfd8b46 --- /dev/null +++ b/examples/database-sources/NeuroElectro/jsonld_context.json @@ -0,0 +1,158 @@ +{ + "@context": { + "@vocab": "http://example.org/vocab/", + "Activity": { + "@id": "prov:Activity" + }, + "Agent": { + "@id": "prov:Agent" + }, + "Class": { + "@id": "owl:Class" + }, + "Entity": { + "@id": "prov:Entity" + }, + "Association": { + "@id": "schema:Association" + }, + "BrainRegion": { + "@id": "nsg:BrainRegion" + }, + "Building": { + "@id": "http://schema.org/Building" + }, + "Employee": { + "@id": "http://www.example.com/Employee" + }, + "Organization": { + "@id": "schema:Organization" + }, + "Person": { + "@id": "schema:Person" + }, + "PostalAddress": { + "@id": "schema:PostalAddress" + }, + "address": { + "@id": "schema:address" + }, + "agent": { + "@id": "schema:agent" + }, + "birthDate": { + "@id": "schema:birthDate", + "@type": "xsd:date" + }, + "citation": { + "@id": "schema:citation" + }, + "contractor": { + "@id": "schema:contractor" + }, + "deathDate": { + "@id": "schema:deathDate", + "@type": "xsd:date" + }, + "department": { + "@id": "schema:department" + }, + "description": { + "@id": "schema:description" + }, + "endedAtTime": { + "@id": "prov:endedAtTime" + }, + "familyName": { + "@id": "schema:familyName" + }, + "founder": { + "@id": "schema:founder" + }, + "gender": { + "@id": "schema:gender" + }, + "generated": { + "@id": "prov:generated" + }, + "geo": { + "@id": "schema:geo" + }, + "givenName": { + "@id": "schema:givenName" + }, + "image": { + "@id": "http://schema.org/image", + "@type": "@id" + }, + "isDefinedBy": { + "@id": "rdfs:isDefinedBy", + "@type": "@id" + }, + "label": "rdfs:label", + "latitude": { + "@id": "schema:latitude", + "@type": "xsd:float" + }, + "longitude": { + "@id": "schema:longitude" + }, + "name": { + "@id": "schema:name" + }, + "notation": "skos:notation", + "nsg": "https://neuroshapes.org/", + "owl": "http://www.w3.org/2002/07/owl#", + "postalCode": { + "@id": "schema:postalCode" + }, + "prefLabel": "skos:prefLabel", + "prov": "http://www.w3.org/ns/prov#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "schema": "http://schema.org/", + "skos": "http://www.w3.org/2004/02/skos/core#", + "startDate": { + "@id": "schema:startDate", + "@type": "xsd:date" + }, + "startedAtTime": { + "@id": "prov:startedAtTime", + "@type": "xsd:dateTime" + }, + "status": { + "@id": "schema:status" + }, + "streetAddress": { + "@id": "schema:streetAddress" + }, + "subClassOf": { + "@id": "rdfs:subClassOf", + "@type": "@id" + }, + "supervisor": { + "@id": "schema:supervisor", + "@type": "@id" + }, + "type": { + "@id": "rdf:type", + "@type": "@id" + }, + "used": { + "@id": "prov:used" + }, + "validated": { + "@id": "schema:validated" + }, + "value": { + "@id": "schema:value" + }, + "wasStartedBy": { + "@id": "prov:wasStartedBy", + "@type": "@id" + }, + "xsd": "http://www.w3.org/2001/XMLSchema#", + "foaf": "http://xmlns.com/foaf/0.1/" + }, + "@id": "http://context.example.org" + } \ No newline at end of file diff --git a/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ElectrophysiologicalFeatureAnnotation.hjson b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ElectrophysiologicalFeatureAnnotation.hjson new file mode 100644 index 00000000..4b9ba001 --- /dev/null +++ b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ElectrophysiologicalFeatureAnnotation.hjson @@ -0,0 +1,48 @@ +{ + "id": forge.format('identifier', "electrophysiologicalfeatureannotations", x.bbpID) + "type": [ + "Entity" + "Annotation" + ] + "description": f"This resource contains Electrophysiological Feature Annotations for {x.NeuronName}" + "hasBody": x.AnnotationBody + "hasTarget": { + "type": "AnnotationTarget" + "hasSelector": x.Selector + "hasSource": { + "id": f"https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/{str(x.ArticleID)}" + "type": "ScholarlyArticle" + } + } + "subject": { + "type": "Subject" + "species": x.resolvedSpecies + "strain": x.resolvedStrain + } + "brainLocation": { + "type": "BrainLocation" + "atlasSpatialReferenceSystem": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/allen_ccfv3_spatial_reference_system" + "type": [ + "AtlasSpatialReferenceSystem" + "BrainAtlasSpatialReferenceSystem" + ] + } + "brainRegion": x.resolvedBrainRegion + } + "atlasRelease": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/831a626a-c0ae-4691-8ce8-cfb7491345d9" + "type": [ + "BrainAtlasRelease" + "AtlasRelease" + ] + } + "isRegisteredIn": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/allen_ccfv3_spatial_reference_system" + "type": [ + "AtlasSpatialReferenceSystem" + "BrainAtlasSpatialReferenceSystem" + ] + } + "name": f"Annotation for {x.NeuronPrefName}" +} diff --git a/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ParameterAnnotation.hjson b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ParameterAnnotation.hjson new file mode 100644 index 00000000..99be0646 --- /dev/null +++ b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ParameterAnnotation.hjson @@ -0,0 +1,51 @@ +{ + "id": forge.format('identifier', "protocolannotations", x.expID) + "description": f"This resource contains experimental parameters used to measure the electrophysiological features reported in article_{x.ArticleID}" + "type": [ + "Entity" + "Protocol" + "ParameterAnnotation" + "Annotation" + ] + "hasBody": x.AnnotationBody + "hasTarget": { + "type": "AnnotationTarget" + "hasSource": { + "id": f"https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/{str(x.ArticleID)}" + "type": "ScholarlyArticle" + } + } + "brainLocation": { + "type": "BrainLocation" + "atlasSpatialReferenceSystem": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/allen_ccfv3_spatial_reference_system" + "type": [ + "AtlasSpatialReferenceSystem" + "BrainAtlasSpatialReferenceSystem" + ] + } + "brainRegion": x.resolvedBrainRegion + } + "atlasRelease": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/831a626a-c0ae-4691-8ce8-cfb7491345d9" + "type": [ + "BrainAtlasRelease" + "AtlasRelease" + ] + } + "isRegisteredIn": { + "id": "https://bbp.epfl.ch/neurosciencegraph/data/allen_ccfv3_spatial_reference_system" + "type": [ + "AtlasSpatialReferenceSystem" + "BrainAtlasSpatialReferenceSystem" + ] + } + "subject": { + "type": "Subject" + "species": x.resolvedSpecies + "strain": x.resolvedStrain + "age": x.ageObject + } + "name": f"Experimental parameters from article with PubMed ID {x.Pmid}" + "note": x.note +} diff --git a/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ParameterBody.hjson b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ParameterBody.hjson new file mode 100644 index 00000000..f65975e2 --- /dev/null +++ b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ParameterBody.hjson @@ -0,0 +1,13 @@ +{ + "type": [ + "AnnotationBody" + "Parameter" + ] + "isMeasurementOf": + { + "label": x.parameter_label + } + "value": x.value + "unitCode": x.units + "comment": x.comment +} \ No newline at end of file diff --git a/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ScholarlyArticle.hjson b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ScholarlyArticle.hjson new file mode 100644 index 00000000..49cd796a --- /dev/null +++ b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/ScholarlyArticle.hjson @@ -0,0 +1,21 @@ +{ + type: [ + Entity + ScholarlyArticle + ] + id: forge.format("identifier", "scholarlyarticles", x.id) + abstract: x.abstract + author: x.authors_shaped + title: x.title + name: f"article_{x.id}" + isPartOf: { + type: Periodical + name: x.journal + publisher: x.publisher + issn: x.issn + } + identifier: x.identifiers + datePublished: x.date_issued + sameAs: x.full_text_link + url: x.full_text_link +} \ No newline at end of file diff --git a/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/SeriesBody.hjson b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/SeriesBody.hjson new file mode 100644 index 00000000..392b11e2 --- /dev/null +++ b/examples/database-sources/NeuroElectro/mappings/DictionaryMapping/SeriesBody.hjson @@ -0,0 +1,32 @@ +{ + "type": "AnnotationBody" + "isMeasurementOf": { + "id": x.ephys_id + "label": x.resolved_name + } + "value": + { + "series": [ + { + "statistic": "raw" + "unitCode": x.units + "value": x.value['raw'] + } + { + "statistic": "error" + "unitCode": x.units + "value": x.value['err'] + } + { + "statistic": "N" + "unitCode": x.units + "value": x.value['n'] + } + { + "statistic": "standard deviation" + "unitCode": x.units + "value": x.value['sd'] + } + ] + } +} \ No newline at end of file diff --git a/examples/database-sources/NeuroElectro/metadata.json b/examples/database-sources/NeuroElectro/metadata.json new file mode 100644 index 00000000..b04a23d7 --- /dev/null +++ b/examples/database-sources/NeuroElectro/metadata.json @@ -0,0 +1,8 @@ +{ + "description": "The goal of the NeuroElectro Project is to extract information about the electrophysiological properties (e.g. resting membrane potentials and membrane time constants) of diverse neuron types from the existing literature and place it into a centralized database.", + "url": "https://neuroelectro.org", + "license" : { + "id": "https://creativecommons.org/licenses/by-sa/4.0", + "label": "CC BY-SA 4.0" + } +} \ No newline at end of file diff --git a/examples/database-sources/NeuroMorpho/jsonld_context.json b/examples/database-sources/NeuroMorpho/jsonld_context.json new file mode 100644 index 00000000..8754369a --- /dev/null +++ b/examples/database-sources/NeuroMorpho/jsonld_context.json @@ -0,0 +1,164 @@ +{ + "@context": { + "@vocab": "http://example.org/vocab/", + "owl": "http://www.w3.org/2002/07/owl#", + "owl2xml": "http://www.w3.org/2006/12/owl2-xml#", + "swrlb": "http://www.w3.org/2003/11/swrlb#", + "protege": "http://protege.stanford.edu/plugins/owl/protege#", + "swrl": "http://www.w3.org/2003/11/swrl#", + "schema": "http://schema.org/", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "foaf": "http://xmlns.com/foaf/0.1/", + "nsg": "https://neuroshapes.org/", + "id": { + "@id": "http://www.geneontology.org/formats/oboInOwl#id" + }, + "Activity": { + "@id": "prov:Activity" + }, + "Agent": { + "@id": "prov:Agent" + }, + "Class": { + "@id": "owl:Class" + }, + "Entity": { + "@id": "prov:Entity" + }, + "Association": { + "@id": "schema:Association" + }, + "BrainRegion": { + "@id": "nsg:BrainRegion" + }, + "Building": { + "@id": "http://schema.org/Building" + }, + "Employee": { + "@id": "http://www.example.com/Employee" + }, + "Organization": { + "@id": "schema:Organization" + }, + "Person": { + "@id": "schema:Person" + }, + "PostalAddress": { + "@id": "schema:PostalAddress" + }, + "address": { + "@id": "schema:address" + }, + "agent": { + "@id": "schema:agent" + }, + "birthDate": { + "@id": "schema:birthDate", + "@type": "xsd:date" + }, + "citation": { + "@id": "schema:citation" + }, + "contractor": { + "@id": "schema:contractor" + }, + "deathDate": { + "@id": "schema:deathDate", + "@type": "xsd:date" + }, + "department": { + "@id": "schema:department" + }, + "description": { + "@id": "schema:description" + }, + "endedAtTime": { + "@id": "prov:endedAtTime" + }, + "familyName": { + "@id": "schema:familyName" + }, + "founder": { + "@id": "schema:founder" + }, + "gender": { + "@id": "schema:gender" + }, + "generated": { + "@id": "prov:generated" + }, + "geo": { + "@id": "schema:geo" + }, + "givenName": { + "@id": "schema:givenName" + }, + "image": { + "@id": "http://schema.org/image", + "@type": "@id" + }, + "isDefinedBy": { + "@id": "rdfs:isDefinedBy", + "@type": "@id" + }, + "label": "rdfs:label", + "latitude": { + "@id": "schema:latitude", + "@type": "xsd:float" + }, + "longitude": { + "@id": "schema:longitude" + }, + "name": { + "@id": "schema:name" + }, + "notation": "skos:notation", + "postalCode": { + "@id": "schema:postalCode" + }, + "prefLabel": "skos:prefLabel", + "startDate": { + "@id": "schema:startDate", + "@type": "xsd:date" + }, + "startedAtTime": { + "@id": "prov:startedAtTime", + "@type": "xsd:dateTime" + }, + "status": { + "@id": "schema:status" + }, + "streetAddress": { + "@id": "schema:streetAddress" + }, + "subClassOf": { + "@id": "rdfs:subClassOf", + "@type": "@id" + }, + "supervisor": { + "@id": "schema:supervisor", + "@type": "@id" + }, + "type": { + "@id": "rdf:type", + "@type": "@id" + }, + "used": { + "@id": "prov:used" + }, + "validated": { + "@id": "schema:validated" + }, + "value": { + "@id": "schema:value" + }, + "wasStartedBy": { + "@id": "prov:wasStartedBy", + "@type": "@id" + } + }, + "@id": "http://context.example.org" + } \ No newline at end of file diff --git a/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson b/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson new file mode 100644 index 00000000..7c328891 --- /dev/null +++ b/examples/database-sources/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson @@ -0,0 +1,61 @@ +{ + type: [ + Dataset + NeuronMorphology + Entity + ] + id: forge.format('identifier', 'neuronmorphologies/neuromorpho', f'{x.bbpID}') + brainLocation: { + type: BrainLocation + brainRegion: forge.resolve(x.brain_region[0], scope='ontology', strategy='EXACT_CASEINSENSITIVE_MATCH') + } + contribution: + { + type: Contribution + agent: + { + id: https://ror.org/02jqj7156 + type: Organization + label: George Mason University + } + } + dateCreated: + { + type: "xsd:dateTime" + @value: f"{x.date_formatted}" + } + identifier: { + propertyID: NeuroMorhoID + value: x.neuron_id + } + archive: x.archive + name: x.neuron_name + generation: + { + type: Generation + activity: + { + type: "nsg:NeuronMorphologyReconstruction" + } + } + subject: + { + type: Subject + species: forge.resolve(x.species, scope='ontology', strategy='EXACT_CASEINSENSITIVE_MATCH') + strain: forge.resolve(x.scientific_name, scope='ontology', strategy='EXACT_CASEINSENSITIVE_MATCH') + } + license: + { + type: License + id: https://neuromorpho.org + } + objectOfStudy: + { + type: ObjectOfStudy + id: http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells + label: Single Cell + } + description: f"Morphology of {x.neuron_name} obtained from NeuroMorpho API." + stain: x.stain + version: "8.4.7" +} \ No newline at end of file diff --git a/examples/database-sources/NeuroMorpho/metadata.json b/examples/database-sources/NeuroMorpho/metadata.json new file mode 100644 index 00000000..3eb312ce --- /dev/null +++ b/examples/database-sources/NeuroMorpho/metadata.json @@ -0,0 +1,8 @@ +{ + "description": "The goal of NeuroMorpho.Org is to provide dense coverage of available reconstruction data for the neuroscience community. Data sharing through NeuroMorpho.Org enables the full and continuing research potential of existing digital reconstruction data.", + "url": "https://neuromorpho.org", + "license" : { + "id": "https://creativecommons.org/licenses/by/4.0/", + "label": "CC BY 4.0" + } +} \ No newline at end of file diff --git a/examples/database-sources/UniProt/jsonld_context.json b/examples/database-sources/UniProt/jsonld_context.json new file mode 100644 index 00000000..0d12ce3e --- /dev/null +++ b/examples/database-sources/UniProt/jsonld_context.json @@ -0,0 +1,29 @@ +{ + "@context": { + "up": "http://purl.uniprot.org/core/", + "owl": "http://www.w3.org/2002/07/owl#", + "owl2xml": "http://www.w3.org/2006/12/owl2-xml#", + "swrlb": "http://www.w3.org/2003/11/swrlb#", + "protege": "http://protege.stanford.edu/plugins/owl/protege#", + "swrl": "http://www.w3.org/2003/11/swrl#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "dc11": "http://purl.org/dc/terms/", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "foaf": "http://xmlns.com/foaf/0.1/", + "id": { + "@id": "http://www.geneontology.org/formats/oboInOwl#id" + }, + "Protein": { + "@id": "up:Protein" + }, + "Gene": { + "@id": "up:Gene" + }, + "type": { + "@id": "rdf:type" + }, + "@id": "https://bbp.epfl.ch/jsonldcontext/db/uniprot" + } +} \ No newline at end of file diff --git a/examples/database-sources/UniProt/mappings/DictionaryMapping/Gene.hjson b/examples/database-sources/UniProt/mappings/DictionaryMapping/Gene.hjson new file mode 100644 index 00000000..11caa996 --- /dev/null +++ b/examples/database-sources/UniProt/mappings/DictionaryMapping/Gene.hjson @@ -0,0 +1,90 @@ +{ + "id": f"https://bbp.epfl.ch/neurosciencegraph/data/genes/{x.uid}" + "type": [ + "Entity" + "Gene" + ] + "description": x.description + "enables": [ + { + "id": "GO:0017046" + "type": "Class" + "label": "peptide hormone binding" + } + { + "id": "GO:0008528" + "type": "Class" + "label": "G protein-coupled peptide receptor activity" + } + ] + "identifier": [ + { + "propertyID": "ArrayExpress" + "value": "ENSMUSG00000025127" + } + { + "propertyID": "EntrezGene" + "value": "14527" + } + { + "propertyID": "MGI" + "value": "MGI:99572" + } + { + "propertyID": "Uniprot_gn" + "value": "Q61606" + } + { + "propertyID": "WikiGene" + "value": "14527" + } + { + "propertyID": "Ensembl" + "value": "ENSMUSG00000025127" + } + ] + "involvedIn": [ + { + "id": "GO:0007189" + "type": "Class" + "label": "adenylate cyclase-activating G protein-coupled receptor signaling pathway" + } + { + "id": "GO:0007188" + "type": "Class" + "label": "adenylate cyclase-modulating G protein-coupled receptor signaling pathway" + } + { + "id": "GO:0071377" + "type": "Class" + "label": "cellular response to glucagon stimulus" + } + ] + "label": "Gcgr" + "locatedIn": [ + { + "id": "GO:0005887" + "type": "Class" + "label": "integral component of plasma membrane" + } + { + "id": "GO:0016020" + "type": "Class" + "label": "membrane" + } + { + "id": "GO:0016021" + "type": "Class" + "label": "integral component of membrane" + } + ] + "name": "Gcgr" + "prefLabel": "Gcgr" + "subject": { + "@type": "Subject" + "species": { + "id": "NCBITaxon:10090" + "label": "Mus musculus" + } + } +} \ No newline at end of file diff --git a/examples/database-sources/UniProt/mappings/DictionaryMapping/Protein.hjson b/examples/database-sources/UniProt/mappings/DictionaryMapping/Protein.hjson new file mode 100644 index 00000000..d5b6aee1 --- /dev/null +++ b/examples/database-sources/UniProt/mappings/DictionaryMapping/Protein.hjson @@ -0,0 +1,18 @@ +{ + "id": forge.format('identifier', "proteins", x.id.split('/')[-1]) + "type": [ + "Entity" + "Protein" + ] + "encodedBy": { + "id": x.gene + "label": x.gene_label + } + "identifier": { + "propertyID": "UniProtKB" + "value": x.id.split('/')[-1] + } + "name": f"Protein {x.id.split('/')[-1]} from UniProtKB" + "label": x.label + "subject": forge.resolve(x.subject, scope='ontology') if forge.resolve(x.subject, scope='ontology') is not None else {"label": x.subject} +} \ No newline at end of file diff --git a/examples/database-sources/UniProt/metadata.json b/examples/database-sources/UniProt/metadata.json new file mode 100644 index 00000000..b04a23d7 --- /dev/null +++ b/examples/database-sources/UniProt/metadata.json @@ -0,0 +1,8 @@ +{ + "description": "The goal of the NeuroElectro Project is to extract information about the electrophysiological properties (e.g. resting membrane potentials and membrane time constants) of diverse neuron types from the existing literature and place it into a centralized database.", + "url": "https://neuroelectro.org", + "license" : { + "id": "https://creativecommons.org/licenses/by-sa/4.0", + "label": "CC BY-SA 4.0" + } +} \ No newline at end of file diff --git a/examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/NeuronMorphology.hjson b/examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/NeuronMorphology.hjson new file mode 100644 index 00000000..fe4adecb --- /dev/null +++ b/examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/NeuronMorphology.hjson @@ -0,0 +1,64 @@ +{ + # There is no NeuronMorphologyShape nor NeuronMorphology type in Neuroshapes at the moment (09.08.2019). + # Using ReconstructedPatchedCellShape meanwhile. + type: NeuronMorphology + id: forge.format("identifier", "neuronmorphologies", x.specimen__id) + # This property is not part of the ReconstructedPatchedCell shape at the moment (09.08.2019). + apicalDendrite: x.tag__apical + brainLocation: + { + type: BrainLocation + brainRegion: + { + id: f"http://api.brain-map.org/api/v2/data/Structure/{x.structure__id}" + label: x.structure__acronym + } + coordinatesInBrainAtlas: + { + valueX: x.csl__x + valueY: x.csl__y + valueZ: x.csl__z + } + layer: forge.resolve(f"layer {x.structure__layer}", scope="terms", target="structure-layer") + } + contribution: + { + type: Contribution + agent: + { + # 'Organization' is a subclass of 'Agent'. + type: Organization + id: https://www.grid.ac/institutes/grid.417881.3 + } + } + derivation: + [ + { + type: Derivation + entity: + { + # 'Subject' is a subclass of 'Entity'. + type: Subject + id: forge.format("identifier", "subjects", x.donor__id) + } + } + { + type: Derivation + entity: + { + # 'PatchedCell' is a subclass of 'Entity'. + type: PatchedCell + id: forge.format("identifier", "patchedcells", x.specimen__id) + } + } + ] + distribution: forge.attach(f"./allen_cell_types_database/specimen_{x.specimen__id}/reconstruction.swc", content_type="application/swc") + identifier: x.specimen__id + name: x.specimen__name + # This property is not part of the ReconstructedPatchedCell shape at the moment (09.08.2019). + subject: + { + type: Subject + id: forge.format("identifier", "subjects", x.donor__id) + } +} \ No newline at end of file diff --git a/examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/PatchedCell.hjson b/examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/PatchedCell.hjson new file mode 100644 index 00000000..a7c266cb --- /dev/null +++ b/examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/PatchedCell.hjson @@ -0,0 +1,41 @@ +{ + type: PatchedCell + id: forge.format("identifier", "patchedcells", x.specimen__id) + brainLocation: + { + type: BrainLocation + brainRegion: + { + id: f"http://api.brain-map.org/api/v2/data/Structure/{x.structure__id}" + label: x.structure__acronym + } + } + contribution: + { + type: Contribution + agent: + { + # 'Organization' is a subclass of 'Agent'. + type: Organization + id: https://www.grid.ac/institutes/grid.417881.3 + } + } + derivation: + { + type: Derivation + entity: + { + # 'Subject' is a subclass of 'Entity'. + type: Subject + id: forge.format("identifier", "subjects", x.donor__id) + } + } + identifier: x.specimen__id + name: x.specimen__name + # This property is not part of the PatchedCell shape at the moment (09.08.2019). + subject: + { + type: Subject + id: forge.format("identifier", "subjects", x.donor__id) + } +} \ No newline at end of file diff --git a/examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/Subject.hjson b/examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/Subject.hjson new file mode 100644 index 00000000..175493d0 --- /dev/null +++ b/examples/database-sources/allen-cell-types-database/mappings/DictionaryMapping/Subject.hjson @@ -0,0 +1,8 @@ +{ + type: Subject + id: forge.format("identifier", "subjects", x.donor__id) + identifier: x.donor__id + name: x.donor__name + sex: forge.resolve(x.donor__sex, scope="terms", target="sex") + species: forge.resolve(x.donor__species, scope="terms", target="species") +} \ No newline at end of file diff --git a/examples/database-sources/scientists-database/DictionaryMapping/Association.hjson b/examples/database-sources/scientists-database/DictionaryMapping/Association.hjson new file mode 100644 index 00000000..9c35618e --- /dev/null +++ b/examples/database-sources/scientists-database/DictionaryMapping/Association.hjson @@ -0,0 +1,12 @@ +{ + type: Association + agent: + { + id: forge.format("identifier", "persons", x.id) + type: Person + additionalName: x.middle_name + gender: forge.resolve(x.gender, scope="terms") + name: x.name + } + distribution: forge.attach(f"../../data/scientists-database/{'_'.join(x.name.lower().split())}.txt") +} \ No newline at end of file diff --git a/examples/database-sources/scientists-database/DictionaryMapping/Contribution.hjson b/examples/database-sources/scientists-database/DictionaryMapping/Contribution.hjson new file mode 100644 index 00000000..63f02a77 --- /dev/null +++ b/examples/database-sources/scientists-database/DictionaryMapping/Contribution.hjson @@ -0,0 +1,12 @@ +{ + type: Contribution + agent: + { + id: forge.format("identifier", "persons", x.id) + type: Person + additionalName: x.middle_name + gender: forge.resolve(x.gender, scope="terms") + name: x.name + } + distribution: forge.attach(f"../../data/scientists-database/{'_'.join(x.name.lower().split())}.txt") +} \ No newline at end of file diff --git a/examples/notebooks/getting-started/17 - Database-sources.ipynb b/examples/notebooks/getting-started/17 - Database-sources.ipynb new file mode 100644 index 00000000..15e10a49 --- /dev/null +++ b/examples/notebooks/getting-started/17 - Database-sources.ipynb @@ -0,0 +1,1163 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Querying external database sources of interest\n", + "\n", + "* Enable users to integrate data from external databases of interest within BBP KG\n", + "* While using the Nexus Forge interface and BMO vocabulary as much as possible as\n", + "* While benefiting from out of the box (meta)data transformation to make them ready for BBP internal pipelines and applications\n", + "* Demo with Mouselight, NeuroElectro, UniProt" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2019-09-23T18:50:20.068658Z", + "start_time": "2019-09-23T18:50:19.054054Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import uuid\n", + "import json\n", + "\n", + "from kgforge.core import KnowledgeGraphForge\n", + "from kgforge.specializations.resources import Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# import getpass\n", + "# TOKEN = getpass.getpass()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "endpoint = \"https://staging.nise.bbp.epfl.ch/nexus/v1\"\n", + "BUCKET = \"bbp/atlas\"\n", + "forge = KnowledgeGraphForge(\"../../configurations/database-sources/prod-nexus-sources.yml\",\n", + " # endpoint=endpoint,\n", + " bucket=BUCKET, debug=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# List of Data sources" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available Database sources:\n", + "UniProt\n", + "NeuroElectro\n", + "NeuroMorpho\n" + ] + } + ], + "source": [ + "forge.db_sources(pretty=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "sources = forge.db_sources()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "data = {\n", + " 'origin': 'store',\n", + " 'source': 'DemoStore',\n", + " 'model': { \n", + " 'name': 'DemoModel',\n", + " 'origin': 'directory',\n", + " 'source': \"../../../tests/data/demo-model/\" \n", + " }\n", + "}\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from kgforge.specializations.databases import StoreDatabase\n", + "ds = StoreDatabase(forge, name=\"DemoDB\", **data)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "forge.add_db_source(ds)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available Database sources:\n", + "UniProt\n", + "NeuroElectro\n", + "NeuroMorpho\n", + "DemoDB\n" + ] + } + ], + "source": [ + "forge.db_sources(pretty=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data source metadata" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "neuroelectro = sources['NeuroElectro']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get data mappings (hold transformations logic) per data type\n", + "\n", + "* Data mappings are used to transform results obtained from the external data sources so that they are ready for consumption by BBP tools\n", + "* Perform automatic ontology linking" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Managed mappings for the data source per entity type and mapping type:\n", + " - ElectrophysiologicalFeatureAnnotation:\n", + " * DictionaryMapping\n", + " - ParameterAnnotation:\n", + " * DictionaryMapping\n", + " - ParameterBody:\n", + " * DictionaryMapping\n", + " - ScholarlyArticle:\n", + " * DictionaryMapping\n", + " - SeriesBody:\n", + " * DictionaryMapping\n" + ] + } + ], + "source": [ + "forge.mappings(source=\"NeuroElectro\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Managed mappings for the data source per entity type and mapping type:\n", + " - Gene:\n", + " * DictionaryMapping\n", + " - Protein:\n", + " * DictionaryMapping\n" + ] + } + ], + "source": [ + "forge.mappings('UniProt')" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "from kgforge.specializations.mappings import DictionaryMapping\n", + "mapping = forge.mapping(entity=\"ScholarlyArticle\", source=\"NeuroElectro\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " id: forge.format(\"identifier\", \"scholarlyarticles\", x.id)\n", + " type:\n", + " [\n", + " Entity\n", + " ScholarlyArticle\n", + " ]\n", + " abstract: x.abstract\n", + " author: x.authors_shaped\n", + " datePublished: x.date_issued\n", + " identifier: x.identifiers\n", + " isPartOf:\n", + " {\n", + " type: Periodical\n", + " issn: x.issn\n", + " name: x.journal\n", + " publisher: x.publisher\n", + " }\n", + " name: f\"article_{x.id}\"\n", + " sameAs: x.full_text_link\n", + " title: x.title\n", + " url: x.full_text_link\n", + "}\n" + ] + } + ], + "source": [ + "print(mapping)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available Database sources:\n", + "UniProt\n" + ] + } + ], + "source": [ + "forge.db_sources(type_='Gene', pretty=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Search and Access data from data source\n", + "\n", + "* Mapping are automatically applied to search results\n", + "* takes a mn for now => working on making it faster " + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Submitted query:\n", + " PREFIX bmc: \n", + " PREFIX bmo: \n", + " PREFIX commonshapes: \n", + " PREFIX datashapes: \n", + " PREFIX dc: \n", + " PREFIX dcat: \n", + " PREFIX dcterms: \n", + " PREFIX mba: \n", + " PREFIX nsg: \n", + " PREFIX nxv: \n", + " PREFIX oa: \n", + " PREFIX obo: \n", + " PREFIX owl: \n", + " PREFIX prov: \n", + " PREFIX rdf: \n", + " PREFIX rdfs: \n", + " PREFIX schema: \n", + " PREFIX sh: \n", + " PREFIX shsh: \n", + " PREFIX skos: \n", + " PREFIX vann: \n", + " PREFIX void: \n", + " PREFIX xml: \n", + " PREFIX xsd: \n", + " PREFIX : \n", + " SELECT ?id ?_constrainedBy ?_createdAt ?_createdBy ?_deprecated ?_incoming ?_outgoing ?_project ?_rev ?_schemaProject ?_self ?_updatedAt ?_updatedBy WHERE { Graph ?g {?id rdf:type schema:ScholarlyArticle;\n", + " ?_constrainedBy;\n", + " ?_createdAt;\n", + " ?_createdBy;\n", + " ?_deprecated;\n", + " ?_incoming;\n", + " ?_outgoing;\n", + " ?_project;\n", + " ?_rev;\n", + " ?_schemaProject;\n", + " ?_self;\n", + " ?_updatedAt;\n", + " ?_updatedBy . \n", + " Filter (?_deprecated = 'false'^^xsd:boolean)\n", + " Filter (?_project = )}} LIMIT 2\n", + "\n" + ] + } + ], + "source": [ + "filters = {\"type\":\"ScholarlyArticle\"}\n", + "#map=True, use_cache=True, # download=True\n", + "resources = forge.search(filters, db_source=\"NeuroElectro\", limit=2, debug=True) \n", + "# Add function for checking datsource health => reqsuire health url from db\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(resources)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " context: https://bbp.neuroshapes.org\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/scholarlyarticles/35177\n", + " type:\n", + " [\n", + " Entity\n", + " ScholarlyArticle\n", + " ]\n", + " abstract: On the one hand, neuronal activity can cause changes in pH; on the other hand, changes in pH can modulate neuronal activity. Consequently, the pH of the brain is regulated at various levels. Here we show that steady-state pH and acid extrusion were diminished in cultured hippocampal neurons of mice with a targeted disruption of the Na(+)-driven Cl(-)/HCO(3)(-) exchanger Slc4a8. Because Slc4a8 was found to predominantly localize to presynaptic nerve endings, we hypothesize that Slc4a8 is a key regulator of presynaptic pH. Supporting this hypothesis, spontaneous glutamate release in the CA1 pyramidal layer was reduced but could be rescued by increasing the intracellular pH. The reduced excitability in vitro correlated with an increased seizure threshold in vivo. Together with the altered kinetics of stimulated synaptic vesicle release, these data suggest that Slc4a8 modulates glutamate release in a pH-dependent manner.\n", + " author:\n", + " [\n", + " {\n", + " type: Person\n", + " familyName: Sinning\n", + " givenName: Anne\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Liebmann\n", + " givenName: Lutz\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Kougioumtzes\n", + " givenName: Alexandra\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Westermann\n", + " givenName: Martin\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Bruehl\n", + " givenName: Claus\n", + " }\n", + " {\n", + " type: Person\n", + " familyName: Hübner\n", + " givenName: Christian A\n", + " }\n", + " ]\n", + " datePublished: 2011-5-18\n", + " identifier:\n", + " [\n", + " {\n", + " propertyID: PMID\n", + " value: 21593314\n", + " }\n", + " {\n", + " propertyID: doi\n", + " value: 10.1523/JNEUROSCI.0269-11.2011\n", + " }\n", + " ]\n", + " isPartOf:\n", + " {\n", + " type: Periodical\n", + " issn: 0270-6474\n", + " name: The Journal of neuroscience : the official journal of the Society for Neuroscience\n", + " publisher: Society for Neuroscience\n", + " }\n", + " name: article_35177\n", + " sameAs: http://www.jneurosci.org/content/31/20/7300.long\n", + " title: Synaptic glutamate release is modulated by the Na+ -driven Cl-/HCO₃⁻ exchanger Slc4a8.\n", + " url: http://www.jneurosci.org/content/31/20/7300.long\n", + "}\n" + ] + } + ], + "source": [ + "print(resources[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "uquery = \"\"\"\n", + "PREFIX up: \n", + "SELECT ?protein\n", + "WHERE {\n", + " ?protein a up:Protein ;\n", + " up:reviewed true.\n", + "}\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Submitted query:\n", + " \n", + " PREFIX up: \n", + " SELECT ?protein\n", + " WHERE {\n", + " ?protein a up:Protein ;\n", + " up:reviewed true.\n", + " }\n", + " LIMIT 10\n", + "\n" + ] + } + ], + "source": [ + "uresources = forge.sparql(query=uquery, db_source='UniProt', limit=10, debug=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "10" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(uresources)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, _inner_sync=False, protein='http://purl.uniprot.org/uniprot/A0B137')" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "uresources[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Use Filters to search" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "from kgforge.core.wrappings.paths import Filter, FilterOperator" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Submitted query:\n", + " PREFIX up: \n", + " PREFIX owl: \n", + " PREFIX owl2xml: \n", + " PREFIX swrlb: \n", + " PREFIX protege: \n", + " PREFIX swrl: \n", + " PREFIX xsd: \n", + " PREFIX skos: \n", + " PREFIX rdfs: \n", + " PREFIX dc11: \n", + " PREFIX rdf: \n", + " PREFIX foaf: \n", + " SELECT ?id WHERE {?id rdf:type up:Protein;\n", + " up:reviewed ?v1 . \n", + " FILTER(?v1 = 'true'^^xsd:boolean)\n", + " } LIMIT 10\n", + "\n" + ] + } + ], + "source": [ + "\n", + "proteins = forge.search({'type': 'Protein', 'up:reviewed': True}, db_source='UniProt', limit=10, debug=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.uniprot.org/uniprot/A0B137', _inner_sync=False)" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "proteins[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Map resources" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "uniprot = sources['UniProt']" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "complete_query = \"\"\"\n", + "PREFIX up: \n", + "SELECT ?id ?gene ?label ?subject ?gene_label\n", + "WHERE {\n", + " ?id a up:Protein ;\n", + " up:reviewed true ;\n", + " up:encodedBy ?gene ;\n", + " up:recommendedName / up:fullName ?label ;\n", + " up:organism / up:scientificName ?subject .\n", + " ?gene skos:prefLabel ?gene_label . \n", + "}\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "raw_proteins = uniprot.sparql(complete_query)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "new_resource = uniprot.map(raw_proteins[0], 'Protein')" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[\n", + " {\n", + " \"@context\": \"https://bbp.neuroshapes.org\",\n", + " \"@id\": \"https://bbp.epfl.ch/neurosciencegraph/data/proteins/P0DJN9\",\n", + " \"@type\": [\n", + " \"Entity\",\n", + " \"Protein\"\n", + " ],\n", + " \"encodedBy\": {\n", + " \"@id\": \"http://purl.uniprot.org/uniprot/P0DJN9#gene-MD5A00DD99270221B359AB0AE338E423668\",\n", + " \"label\": \"acsF\"\n", + " },\n", + " \"identifier\": {\n", + " \"propertyID\": \"UniProtKB\",\n", + " \"value\": \"P0DJN9\"\n", + " },\n", + " \"name\": \"Protein P0DJN9 from UniProtKB\",\n", + " \"label\": \"Aerobic magnesium-protoporphyrin IX monomethyl ester [oxidative] cyclase\",\n", + " \"subject\": {\n", + " \"label\": \"Rubrivivax gelatinosus\"\n", + " }\n", + " }\n", + "]\n" + ] + } + ], + "source": [ + "\n", + "print(json.dumps(forge.as_jsonld(new_resource), indent=4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### same result could be obtain from a dictionary and a DictionaryMapping instance" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[\n", + " {\n", + " \"@context\": \"https://bbp.neuroshapes.org\",\n", + " \"@id\": \"https://bbp.epfl.ch/neurosciencegraph/data/proteins/P0DJN9\",\n", + " \"@type\": [\n", + " \"Entity\",\n", + " \"Protein\"\n", + " ],\n", + " \"encodedBy\": {\n", + " \"@id\": \"http://purl.uniprot.org/uniprot/P0DJN9#gene-MD5A00DD99270221B359AB0AE338E423668\",\n", + " \"label\": \"acsF\"\n", + " },\n", + " \"identifier\": {\n", + " \"propertyID\": \"UniProtKB\",\n", + " \"value\": \"P0DJN9\"\n", + " },\n", + " \"name\": \"Protein P0DJN9 from UniProtKB\",\n", + " \"label\": \"Aerobic magnesium-protoporphyrin IX monomethyl ester [oxidative] cyclase\",\n", + " \"subject\": {\n", + " \"label\": \"Rubrivivax gelatinosus\"\n", + " }\n", + " }\n", + "]\n" + ] + } + ], + "source": [ + "dict_resource = forge.as_json(raw_proteins[0])\n", + "mapping = DictionaryMapping.load(\"../../database-sources/UniProt/mappings/DictionaryMapping/Protein.hjson\")\n", + "print(json.dumps(forge.as_jsonld(uniprot.map(dict_resource, mapping)), indent=4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Query the NeuroMorpho WebService" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "neuromorpho = sources['NeuroMorpho']" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "nmo_filters = {\"species\": \"rat,mouse,human\", \"response_loc\": [\"_embedded\", \"neuronResources\"]}" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/miniconda3/envs/kgforge/lib/python3.7/site-packages/urllib3/connectionpool.py:1052: InsecureRequestWarning: Unverified HTTPS request is being made to host 'neuromorpho.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n", + " InsecureRequestWarning,\n" + ] + } + ], + "source": [ + "nmo_resources = forge.search(nmo_filters, db_source='NeuroMorpho', size=3, searchendpoint='select_query', q=\"species:mouse\")" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Resource(_last_action=None, _validated=False, _synchronized=False, _store_metadata=None, id='http://purl.obolibrary.org/obo/UBERON_0002301', type='Class', label='Neocortical Layer', _inner_sync=False, prefLabel='Neocortical Layer', subClassOf='bmo:BrainLayer')" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "forge.resolve(nmo_resources[0].brain_region[0], scope='ontology', strategy='BEST_MATCH')" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['neocortex', 'occipital', 'layer 6']" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nmo_resources[0].brain_region" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Format date" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "for resource in nmo_resources:\n", + " resource.bbpID = str(uuid.uuid4())\n", + " date_ints = [int(p) for p in resource.deposition_date.split('-')]\n", + " date_str = datetime(*date_ints)\n", + " resource.date_formatted = date_str.strftime(\"%Y-%m-%dT%H:%M:%S\")" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [], + "source": [ + "new_morphologies = neuromorpho.map(nmo_resources, 'NeuronMorphology')" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "format_file = neuromorpho.service.files_download['endpoint'] + \"/{}/CNG version/{}.CNG.swc\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Attach file as distribution with morphologies" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "import morphio" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/miniconda3/envs/kgforge/lib/python3.7/site-packages/urllib3/connectionpool.py:1052: InsecureRequestWarning: Unverified HTTPS request is being made to host 'neuromorpho.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n", + " InsecureRequestWarning,\n", + "/opt/miniconda3/envs/kgforge/lib/python3.7/site-packages/urllib3/connectionpool.py:1052: InsecureRequestWarning: Unverified HTTPS request is being made to host 'neuromorpho.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n", + " InsecureRequestWarning,\n", + "/opt/miniconda3/envs/kgforge/lib/python3.7/site-packages/urllib3/connectionpool.py:1052: InsecureRequestWarning: Unverified HTTPS request is being made to host 'neuromorpho.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n", + " InsecureRequestWarning,\n" + ] + } + ], + "source": [ + "for morphology in new_morphologies:\n", + " url = format_file.format(morphology.archive.lower(), morphology.name)\n", + " base_name = ''.join(url.split('.')[:-2])\n", + " file_path = f\"./downloaded/{base_name.split('/')[-1]}\"\n", + " swc = file_path + '.swc'\n", + " neuromorpho.download(url, swc)\n", + " neuromorpho.attach_file(morphology, swc, content_type='application/swc')\n", + " # Generate other morphology files\n", + " m = morphio.mut.Morphology(swc)\n", + " for extension in ['h5', 'asc']:\n", + " path = f\"{file_path}.{extension}\"\n", + " m.write(path)\n", + " neuromorpho.attach_file(morphology, path, content_type=f'application/{extension}')\n", + " m.write(file_path+'.asc')" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " _register_one\n", + " False\n", + " RegistrationError: 400 Client Error: Bad Request for url: https://bbp.epfl.ch/nexus/v1/resources/bbp/atlas/datashapes%3Aneuronmorphology\n" + ] + } + ], + "source": [ + "forge.register(new_morphologies[0], schema_id=\"datashapes:neuronmorphology\")" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [], + "source": [ + "# forge.validate(new_morphologies[0], type_=\"NeuronMorphology\")" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/neuronmorphologies/neuromorpho/0b869f02-85b3-4c49-9cb3-5d6f12bbde28\n", + " type:\n", + " [\n", + " Dataset\n", + " NeuronMorphology\n", + " ]\n", + " archive: Scanziani\n", + " brainLocation:\n", + " {\n", + " type: BrainLocation\n", + " }\n", + " contribution:\n", + " {\n", + " type: Contribution\n", + " agent:\n", + " {\n", + " id: https://ror.org/02jqj7156\n", + " type: Organization\n", + " label: George Mason University\n", + " }\n", + " }\n", + " dateCreated:\n", + " {\n", + " type: xsd:dateTime\n", + " @value: 2012-07-01T00:00:00\n", + " }\n", + " description: Morphology of TypeA-10 obtained from NeuroMorpho API.\n", + " distribution:\n", + " [\n", + " {\n", + " type: DataDownload\n", + " atLocation:\n", + " {\n", + " type: Location\n", + " location: file:///gpfs/bbp.cscs.ch/data/project/proj39/nexus/bbp/atlas/0/4/8/5/1/9/5/c/TypeA-10.swc\n", + " store:\n", + " {\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/2f120131-0bbc-4a7e-b84b-52f7f00eec9e\n", + " type: RemoteDiskStorage\n", + " _rev: 1\n", + " }\n", + " }\n", + " contentSize:\n", + " {\n", + " unitCode: bytes\n", + " value: 80865\n", + " }\n", + " contentUrl: https://bbp.epfl.ch/nexus/v1/files/bbp/atlas/85d5c8d9-788d-4777-b62f-825b2ff12a51\n", + " digest:\n", + " {\n", + " algorithm: SHA-256\n", + " value: d0e898900e129fe41173eb0fe36bd84bd87a2100acb303e0ee96bf584eccfae0\n", + " }\n", + " encodingFormat: application/swc\n", + " name: TypeA-10.swc\n", + " }\n", + " {\n", + " type: DataDownload\n", + " atLocation:\n", + " {\n", + " type: Location\n", + " location: file:///gpfs/bbp.cscs.ch/data/project/proj39/nexus/bbp/atlas/c/f/7/d/4/6/8/7/TypeA-10.h5\n", + " store:\n", + " {\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/2f120131-0bbc-4a7e-b84b-52f7f00eec9e\n", + " type: RemoteDiskStorage\n", + " _rev: 1\n", + " }\n", + " }\n", + " contentSize:\n", + " {\n", + " unitCode: bytes\n", + " value: 45952\n", + " }\n", + " contentUrl: https://bbp.epfl.ch/nexus/v1/files/bbp/atlas/03edd654-0c39-4069-9b24-c7e3d68cd3ad\n", + " digest:\n", + " {\n", + " algorithm: SHA-256\n", + " value: 6cd5a3786e5502ce9443616089924ca8d22718a765d8e8202a5619f34b4afc9e\n", + " }\n", + " encodingFormat: application/h5\n", + " name: TypeA-10.h5\n", + " }\n", + " {\n", + " type: DataDownload\n", + " atLocation:\n", + " {\n", + " type: Location\n", + " location: file:///gpfs/bbp.cscs.ch/data/project/proj39/nexus/bbp/atlas/a/0/4/e/6/9/d/9/TypeA-10.asc\n", + " store:\n", + " {\n", + " id: https://bbp.epfl.ch/neurosciencegraph/data/2f120131-0bbc-4a7e-b84b-52f7f00eec9e\n", + " type: RemoteDiskStorage\n", + " _rev: 1\n", + " }\n", + " }\n", + " contentSize:\n", + " {\n", + " unitCode: bytes\n", + " value: 147432\n", + " }\n", + " contentUrl: https://bbp.epfl.ch/nexus/v1/files/bbp/atlas/00a51e52-b9f5-47e6-b61f-c640820d01bf\n", + " digest:\n", + " {\n", + " algorithm: SHA-256\n", + " value: a2469aceb35eb4569f9b2826ff099fb362f25bd5501ba468b269e8d1a1a3d93c\n", + " }\n", + " encodingFormat: application/asc\n", + " name: TypeA-10.asc\n", + " }\n", + " ]\n", + " generation:\n", + " {\n", + " type: Generation\n", + " activity:\n", + " {\n", + " type: nsg:NeuronMorphologyReconstruction\n", + " }\n", + " }\n", + " identifier:\n", + " {\n", + " propertyID: NeuroMorhoID\n", + " value: 10047\n", + " }\n", + " license:\n", + " {\n", + " id: https://neuromorpho.org\n", + " type: License\n", + " }\n", + " name: TypeA-10\n", + " objectOfStudy:\n", + " {\n", + " id: http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\n", + " type: ObjectOfStudy\n", + " label: Single Cell\n", + " }\n", + " stain: biocytin\n", + " subject:\n", + " {\n", + " type: Subject\n", + " species:\n", + " {\n", + " id: http://purl.obolibrary.org/obo/NCBITaxon_10088\n", + " type: Class\n", + " label: Mouse\n", + " subClassOf: obo:NCBITaxon_9989\n", + " }\n", + " }\n", + " version: 8.4.7\n", + "}\n" + ] + } + ], + "source": [ + "print(new_morphologies[0])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.7.13 ('kgforge')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.13" + }, + "vscode": { + "interpreter": { + "hash": "9ac393a5ddd595f2c78ea58b15bf8d269850a4413729cbea5c5fae9013762763" + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/kgforge/core/archetypes/__init__.py b/kgforge/core/archetypes/__init__.py index abda5bcd..7ba09666 100644 --- a/kgforge/core/archetypes/__init__.py +++ b/kgforge/core/archetypes/__init__.py @@ -19,3 +19,4 @@ from .store import Store from .model import Model from .resolver import Resolver +from .database import Database diff --git a/kgforge/core/archetypes/database.py b/kgforge/core/archetypes/database.py new file mode 100644 index 00000000..5f22f419 --- /dev/null +++ b/kgforge/core/archetypes/database.py @@ -0,0 +1,144 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any, Optional, Callable, Type, List, Union + +from kgforge.core import Resource +from kgforge.core.commons.execution import not_supported +from kgforge.core.archetypes import Mapping, Model +from kgforge.core.commons.attributes import repr_class +from kgforge.core.commons.exceptions import ConfigurationError +from kgforge.core.commons.dictionaries import use_values +from kgforge.core.commons.imports import import_class + + +class Database(ABC): + + # POLICY Methods of archetypes, except __init__, should not have optional arguments. + + # POLICY Implementations should be declared in kgforge/specializations/databases/__init__.py. + # POLICY Implementations should not add methods but private functions in the file. + # POLICY Implementations should pass tests/specializations/databases/test_databases.py. + + def __init__(self, forge : Optional["KnowledgeGraphForge"], source: str, **config) -> None: + # POLICY Resolver data access should be lazy, unless it takes less than a second. + # POLICY There could be data caching but it should be aware of changes made in the source. + self._forge: Optional["KnowledgeGraphForge"] = forge + # Model + model_config = config.pop("model") + if model_config["origin"] == "store": + use_values( + model_config, + config, + ["endpoint", "token", "bucket", "vocabulary"], + ) + model_name = model_config.pop("name") + model = import_class(model_name, "models") + self._model: Model = model(**model_config) + self.context = self._model.context() + if 'model_context' not in config: + config['model_context'] = self._model.context() + self.source: str = source + self.service: Any = self._initialize_service(self.source, **config) + + def __repr__(self) -> str: + return repr_class(self) + + def map(self, resources : Union[List[Union[Resource, str]], Union[Resource, str]], + type_ : Optional[Union[str, "DictionaryMapping"]] = None) -> Optional[Union[Resource, List[Resource]]]: + mappings = self._model.mappings(self._model.source, False) + mapped_resources = [] + resources = (resources if isinstance(resources, list) else [resources]) + for resource in resources: + if isinstance(resource, Resource): + resource_dict = self._forge.as_json(resource) + else: + resource_dict = resource + resource = self._forge.from_json(resource_dict) + if type_ is None: + try: + type_ = resource.type + except AttributeError: + mapped_resources.append(resource) + elif isinstance(type_, Mapping): + mapped_resources.append(self._forge.map(resource_dict, type_)) + elif type_ in mappings: + # type_ is the entity here + mapping_class : Type[Mapping] = import_class(mappings[type_][0], "mappings") + mapping = self._model.mapping(type_, self._model.source, mapping_class) + mapped_resources.append(self._forge.map(resource_dict, mapping)) + else: + mapped_resources.append(resource) + return mapped_resources + + def types(self): + # TODO: add other datatypes used, for instance, inside the mappings + return list(self._model.mappings(self._model.source, False).keys()) + + @abstractmethod + def search(self, resolvers, *filters, **params) -> Resource: + pass + + @abstractmethod + def sparql(self, query: str, debug: bool = False, limit: Optional[int] = None, + offset: Optional[int] = None,**params) -> Resource: + pass + + @abstractmethod + def elastic(self, **params) -> Resource: + pass + + @property + @abstractmethod + def health(self) -> Callable: + """Health of the database.""" + pass + + # Utils. + + def _initialize_service(self, source: str, **source_config) -> Any: + # Resolver data could be accessed from a directory, a web service, or a Store. + # Initialize the access to the resolver data according to the source type. + # POLICY Should not use 'self'. This is not a function only for the specialization to work. + origin = source_config.pop("origin") + if origin == "directory": + dirpath = Path(source) + return self._service_from_directory(dirpath, **source_config) + elif origin == "web_service": + source_config['service_context'] = self.context + return self._service_from_web_service(source, **source_config) + elif origin == "store": + store = import_class(source, "stores") + if source != 'DemoStore': + source_config['store_context'] = self.context + return self._service_from_store(store, **source_config) + else: + raise ConfigurationError(f"unrecognized DataBase origin '{origin}'") + + @staticmethod + def _service_from_directory(dirpath: Path, **source_config) -> Any: + not_supported() + + @staticmethod + @abstractmethod + def _service_from_web_service(endpoint: str, **source_config) -> Any: + pass + + @staticmethod + @abstractmethod + def _service_from_store(store: Callable, **store_config) -> Any: + pass + diff --git a/kgforge/core/archetypes/store.py b/kgforge/core/archetypes/store.py index b65435d5..cebd4138 100644 --- a/kgforge/core/archetypes/store.py +++ b/kgforge/core/archetypes/store.py @@ -15,7 +15,9 @@ import re import time +from pyld import jsonld from abc import ABC, abstractmethod +from rdflib import Graph from pathlib import Path from typing import Any, Dict, List, Match, Optional, Tuple, Union, Type @@ -34,6 +36,7 @@ QueryingError, ) from kgforge.core.commons.execution import not_supported, run +from kgforge.core.conversions.rdf import from_jsonld from kgforge.core.reshaping import collect_values # NB: Do not 'from kgforge.core.archetypes import Resolver' to avoid cyclic dependency. @@ -605,3 +608,40 @@ def replace(match: Match) -> str: pfx = "\n".join([pfx, f"PREFIX : <{context.vocab}>"]) return f"{pfx}\n{qr}" + + +def resources_from_construct_query(data, context) -> List[Resource]: + """Build resources from a CONSTRUCT query.""" + subject_triples = {} + for r in data["results"]["bindings"]: + subject = r["subject"]["value"] + s = f"<{r['subject']['value']}>" + p = f"<{r['predicate']['value']}>" + if r["object"]["type"] == "uri": + o = f"<{r['object']['value']}>" + else: + if "datatype" in r["object"]: + o = f"\"{r['object']['value']}\"^^{r['object']['datatype']}" + else: + o = f"\"{r['object']['value']}\"" + if subject in subject_triples: + subject_triples[subject] += f"\n{s} {p} {o} . " + else: + subject_triples[subject] = f"{s} {p} {o} . " + + def triples_to_resource(iri, triples): + graph = Graph().parse(data=triples, format="nt") + data_expanded = json.loads(graph.serialize(format="json-ld")) + data_expanded = json.loads(graph.serialize(format="json-ld")) + frame = {"@id": iri} + data_framed = jsonld.frame(data_expanded, frame) + compacted = jsonld.compact(data_framed, context.document) + resource = from_jsonld(compacted) + resource.context = ( + context.iri + if context.is_http_iri() + else context.document["@context"] + ) + return resource + + return [triples_to_resource(s, t) for s, t in subject_triples.items()] \ No newline at end of file diff --git a/kgforge/core/commons/dictionaries.py b/kgforge/core/commons/dictionaries.py index 434531cc..ab3ed5fd 100644 --- a/kgforge/core/commons/dictionaries.py +++ b/kgforge/core/commons/dictionaries.py @@ -21,12 +21,17 @@ def with_defaults(original: Dict, other: Dict, original_key: str, other_key: str """Update 'original' with 'other' 'keys' unless keys value is different in both dictionaries.""" if original[original_key] == other[other_key]: - for x in keys: - if x not in original and x in other: - original[x] = other[x] + use_values(original, other, keys) + + +def use_values(original: Dict, other: Dict, + keys: List[str]) -> None: + for x in keys: + if x not in original and x in other: + original[x] = other[x] def update_dict(original: Dict, other: Dict) -> Dict: original_copy = copy.deepcopy(original) original_copy.update(other) - return original_copy + return original_copy \ No newline at end of file diff --git a/kgforge/core/conversions/rdf.py b/kgforge/core/conversions/rdf.py index 357822b6..03321a17 100644 --- a/kgforge/core/conversions/rdf.py +++ b/kgforge/core/conversions/rdf.py @@ -407,11 +407,10 @@ def _unpack_from_list(data): return node -def _add_ld_keys( - rsc: [Resource, Dict], - context: Optional[Union[Dict, List, str]], - base: Optional[str], -) -> Union[Dict, List[str]]: +def _add_ld_keys(rsc: [Resource, Dict], + context: Optional[Union[Dict, List, str]], + base: Optional[str], + ) -> Union[Dict, List[str]]: local_attrs = {} local_context = None json_arrays = [] diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index db28427c..969c696f 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -21,12 +21,13 @@ from rdflib import Graph from kgforge.core import Resource +from kgforge.core.commons.context import Context +from kgforge.core.archetypes import Mapping, Model, Resolver, Store, Database from kgforge.core.commons.files import load_file_as_byte -from kgforge.core.archetypes import Mapping, Model, Resolver, Store from kgforge.core.commons.actions import LazyAction from kgforge.core.commons.dictionaries import with_defaults from kgforge.core.commons.exceptions import ResolvingError -from kgforge.core.commons.execution import catch +from kgforge.core.commons.execution import catch, not_supported from kgforge.core.commons.imports import import_class from kgforge.core.commons.strategies import ResolvingStrategy from kgforge.core.commons.formatter import Formatter @@ -43,6 +44,7 @@ from kgforge.core.wrappings.paths import PathsWrapper, wrap_paths from kgforge.specializations.mappers import DictionaryMapper from kgforge.specializations.mappings import DictionaryMapping +from kgforge.specializations.databases import StoreDatabase, WebServiceDatabase class KnowledgeGraphForge: @@ -223,6 +225,7 @@ def __init__(self, configuration: Union[str, Dict], **kwargs) -> None: model_name = model_config.pop("name") model = import_class(model_name, "models") self._model: Model = model(**model_config) + model_config.update(name=model_name) # Store. store_config.update(model_context=self._model.context()) @@ -518,6 +521,50 @@ def sources(self, pretty: bool = True) -> Optional[List[str]]: """ return self._model.sources(pretty) + @catch + def db_sources(self, type_: Optional[List[str]] = None, + pretty: bool = False) -> Optional[List[str]]: + """ + Print(pretty=True) or return (pretty=False) configured data sources. + :param pretty: a boolean + :return: Optional[List[str]] + """ + if type_ is None: + sources = self._db_sources + else: + sources = {} + if isinstance(type_, list): + for type in type_: + for db in self._db_sources: + try: + types = self._db_sources[db].types() + if type in types: + sources[db] = self._db_sources[db] + except ValueError: + # skiping db without mappings + continue + else: + for db in self._db_sources: + try: + types = self._db_sources[db].types() + if type_ in types: + sources[db] = self._db_sources[db] + except ValueError: + # skiping db without mappings + continue + if not sources: + print("No Database sources were found for the given type(s)") + if pretty: + print(*["Available Database sources:", *sources], sep="\n") + else: + return sources + + def add_db_source(self, db_source: Database) -> None: + """ + Add a DatabaseSource to the KG. + """ + self._db_sources[db_source.name] = db_source + @catch def mappings( self, source: str, pretty: bool = True @@ -530,7 +577,11 @@ def mappings( :param pretty: a boolean :return: Optional[Dict[str, List[str]]] """ - return self._model.mappings(source, pretty) + if source in self._db_sources: + db = self._db_sources[source] + return db._model.mappings(db._model.source, pretty) + else: + return self._model.mappings(source, pretty) @catch def mapping( @@ -544,7 +595,11 @@ def mapping( :param type: a Mapping class :return: Mapping """ - return self._model.mapping(entity, source, type) + if source in self._db_sources: + db = self._db_sources[source] + return db._model.mapping(entity, db._model.source, type) + else: + return self._model.mapping(entity, source, type) @catch def map( @@ -632,7 +687,14 @@ def search(self, *filters, **params) -> List[Resource]: resolvers = ( list(self._resolvers.values()) if self._resolvers is not None else None ) - return self._store.search(resolvers, *filters, **params) + db_source = params.pop('db_source', None) + if db_source: + if db_source in self.db_sources(): + return self._db_sources[db_source].search(resolvers, *filters, **params) + else: + raise AttributeError('Selected database was not declared within forge.') + else: + return self._store.search(resolvers, *filters, **params) @catch def sparql( @@ -653,7 +715,14 @@ def sparql( :param params: a dictionary of parameters. Supported params are: rewrite (whether to rewrite the sparql query or run it as is) :return: List[Resource] """ - return self._store.sparql(query, debug, limit, offset, **params) + db_source = params.pop('db_source', None) + if db_source: + if db_source in self.db_sources(): + return self._db_sources[db_source].sparql(query, debug, limit, offset, **params) + else: + raise AttributeError('Selected database was not declared within forge.') + else: + return self._store.sparql(query, debug, limit, offset, **params) @catch def elastic( @@ -662,6 +731,7 @@ def elastic( debug: bool = False, limit: Optional[int] = None, offset: Optional[int] = None, + db_source: Optional[str] = None, ) -> List[Resource]: """ Search for resources using an ElasticSearch DSL query. See ElasticSearch DSL docs: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html. @@ -672,7 +742,13 @@ def elastic( :param offset: how many results to skip from the first one :return: List[Resource] """ - return self._store.elastic(query, debug, limit, offset) + if db_source: + if db_source in self.db_sources(): + return self._db_sources[db_source].elastic(query, debug, limit, offset) + else: + raise AttributeError('Selected database was not declared within forge.') + else: + return self._store.elastic(query, debug, limit, offset) @catch def download( @@ -939,6 +1015,31 @@ def get_store_context(self): def get_model_context(self): """Expose the context used in the model.""" return self._model.context() + + def create_db_sources(self, all_config: Optional[Dict[str, Dict[str, str]]], + store_config : Optional[Dict[str, Dict[str, str]]], + model_context: Context + ) -> Dict[str, Database]: + dbs = {} + for name, config in all_config.items(): + origin = config['origin'] + source = config['source'] + if origin == 'store': + # Reuse complete configuration of the store when Nexus is called + if source == store_config['name'] == 'BlueBrainNexus': + store_copy = deepcopy(store_config) + with_defaults(config, store_copy, + "source", "name", + list(store_copy.keys())) + config['model_context'] = model_context + config['name'] = name + dbs[name] = StoreDatabase(self, **config) + elif origin == 'web_service': + config['name'] = name + dbs[name] = WebServiceDatabase(self, **config) + else: + raise NotImplementedError(f'Database from {origin} is not yet implemented.') + return dbs def prepare_resolvers( diff --git a/kgforge/specializations/databases/__init__.py b/kgforge/specializations/databases/__init__.py new file mode 100644 index 00000000..a4b12dc8 --- /dev/null +++ b/kgforge/specializations/databases/__init__.py @@ -0,0 +1,16 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +from .store_database import StoreDatabase +from .webservice_database import WebServiceDatabase \ No newline at end of file diff --git a/kgforge/specializations/databases/store_database.py b/kgforge/specializations/databases/store_database.py new file mode 100644 index 00000000..e9bbb8e2 --- /dev/null +++ b/kgforge/specializations/databases/store_database.py @@ -0,0 +1,97 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . +from typing import Callable, Optional, Any + +from kgforge.core.archetypes import Store, Database +from kgforge.core.commons.execution import not_supported +from kgforge.specializations.stores.bluebrain_nexus import BlueBrainNexus +from kgforge.specializations.databases.utils import type_from_filters + + +class StoreDatabase(Database): + """A high-level class to retrieve and create Database-related Resources.""" + + + _REQUIRED = ("name", "origin", "source", "model") + + def __init__(self, forge: Optional["KnowledgeGraphForge"], + **config) -> None: + """ + The properties defining the StoreDatabase are: + + :param forge: To use forge utilities + name: - REQUIRED + origin: <'store'> - REQUIRED + source: - REQUIRED + bucket: + endpoint: + token: + source: + context: <'directory', 'url', or 'store'> + bucket: + iri: + """ + self._check_properties(**config) + self.name = config.pop('name') + source = config.pop('source') + super().__init__(forge, source, **config) + + def _check_properties(self, **info): + properties = info.keys() + for r in self._REQUIRED: + if r not in properties: + raise ValueError(f'Missing {r} from the properties to define the DatabasSource') + + def search(self, resolvers, *filters, **params): + """Search within the database. + + :param keep_original: bool + """ + keep_original = params.pop('keep_original', True) + unmapped_resources = self.service.search(resolvers, *filters, **params) + if isinstance(self.service, BlueBrainNexus) or keep_original: + return unmapped_resources + else: + # Try to find the type of the resources within the filters + resource_type = type_from_filters(*filters) + return self.map_resources(unmapped_resources, resource_type=resource_type) + + def sparql(self, query: str, debug: bool = False, limit: Optional[int] = None, + offset: Optional[int] = None,**params): + """Use SPARQL within the database. + + :param keep_original: bool + """ + keep_original = params.pop('keep_original', True) + unmapped_resources = self.service.sparql(query, debug, limit, offset, **params) + if keep_original: + return unmapped_resources + else: + return self.map_resources(unmapped_resources) + + def elastic(**params): + not_supported() + + @staticmethod + def _service_from_web_service(endpoint: str, **source_config) -> Any: + not_supported() + + @staticmethod + def _service_from_store(store: Callable, **store_config) -> Store: + return store(**store_config) + + def health(self) -> Callable: + not_supported() \ No newline at end of file diff --git a/kgforge/specializations/databases/utils.py b/kgforge/specializations/databases/utils.py new file mode 100644 index 00000000..43f86d70 --- /dev/null +++ b/kgforge/specializations/databases/utils.py @@ -0,0 +1,88 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +import os +import json +import requests +from typing import Optional, Any + +from kgforge.core import Resource +from kgforge.core.commons.exceptions import DownloadingError, QueryingError + + +def type_from_filters(*filters) -> Optional[str]: + """Returns the first `type` found in filters.""" + resource_type = None + filters = filters[0] + if isinstance(filters, dict): + if 'type' in filters: + resource_type = filters['type'] + else: + # check filters grouping + if isinstance(filters, (list, tuple)): + filters = [filter for filter in filters] + else: + filters = [filters] + for filter in filters: + if 'type' in filter.path and filter.operator is "__eq__": + resource_type = filter.value + break + return resource_type + +def resources_from_results(results): + """Returns Resources from standard response bindings.""" + return [ + Resource(**{k: json.loads(str(v["value"]).lower()) if v['type'] =='literal' and + ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#boolean') + else (int(v["value"]) if v['type'] =='literal' and + ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#integer') + else v["value"] + ) + for k, v in x.items()} ) + for x in results + ] + +def resources_from_request(url, headers, **params): + """Perform a HTTP request + params: + ------- + response_loc : list[str] + The nested location of the relevat metadata in the + response. + Example: NeuroMorpho uses response["_embedded"]["neuronResources"] + which should be given as: response_loc = ["_embedded", "neuronResources"] + """ + response_location = params.pop('response_loc', None) + try: + response = requests.get( + url, params=params, headers=headers, verify=False + ) + response.raise_for_status() + except Exception as e: + raise QueryingError(e) + else: + data = response.json() + if response_location: + # Get the resources directly from a location in the response + if isinstance(response_location, str): + results = data[response_location] + elif isinstance(response_location, (list, tuple)): + for inner in response_location: + data = data[inner] + results = data + return [Resource(**result) for result in results] + else: + # Standard response format + results = data["results"]["bindings"] + return resources_from_results(results) \ No newline at end of file diff --git a/kgforge/specializations/databases/webservice_database.py b/kgforge/specializations/databases/webservice_database.py new file mode 100644 index 00000000..d1e73570 --- /dev/null +++ b/kgforge/specializations/databases/webservice_database.py @@ -0,0 +1,197 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . +import copy +import requests +import asyncio +from aiohttp import ClientSession +from requests.exceptions import SSLError +from typing import Callable, List, Optional, Any, Union + +from kgforge.core import Resource +from kgforge.core.archetypes import Database, Store +from kgforge.core.commons.exceptions import ConfigurationError, DownloadingError +from kgforge.core.commons.execution import not_supported, catch +from kgforge.specializations.databases.utils import type_from_filters +from kgforge.specializations.stores.bluebrain_nexus import BlueBrainNexus, _error_message +from kgforge.specializations.stores.databases import WebService +from kgforge.specializations.resources.datasets import _set + + +class WebServiceDatabase(Database): + """A high-level class to retrieve and create Database-related Resources.""" + + + _REQUIRED = ("name", "origin", "source", "model") + + def __init__(self, forge: Optional["KnowledgeGraphForge"], + **config) -> None: + """ + The properties defining the WebServiceDatabase are: + + :param forge: To use forge utilities + name: - REQUIRED + origin: <'web_service'> - REQUIRED + source: - REQUIRED + bucket: + endpoint: + token: + source: + context: <'directory', 'url', or 'store'> + bucket: + iri: + """ + self._check_properties(**config) + self.name = config.pop('name') + source = config.pop('source') + self._health = config.pop('health', None) + super().__init__(forge, source, **config) + + def _check_properties(self, **info): + properties = info.keys() + for r in self._REQUIRED: + if r not in properties: + raise ValueError(f'Missing {r} from the properties to define the DatabasSource') + + def search(self, resolvers, *filters, **params): + """Search within the database. + + :param keep_original: bool + """ + keep_original = params.pop('keep_original', True) + unmapped_resources = self.service.search(resolvers, *filters, **params) + if isinstance(self.service, BlueBrainNexus) or keep_original: + return unmapped_resources + else: + # Try to find the type of the resources within the filters + resource_type = type_from_filters(*filters) + return self.map_resources(unmapped_resources, resource_type=resource_type) + + def sparql(self, query: str, debug: bool = False, limit: Optional[int] = None, + offset: Optional[int] = None,**params): + not_supported() + + def elastic(**params): + not_supported() + + @catch + def attach_file(self, resource, path: str, url: str = None, + content_type: str = None, + as_part: bool=False, save: bool = False) -> None: + """Add (different) files as parts or distribution + + :param resource: Resource to which attach files + :param path: DirPath or URL - Location of the file + :param content_type: the file type + :param as_part: indicate how to add the file. + Default adds the file as distribution + :param save: If path is url, clean up the downloaded files + """ + if url: + self._download_one(url, path) + action = self._forge.attach(path, content_type) + if as_part: + distribution = Resource(distribution=action) + _set(resource, "hasPart", distribution) + else: + _set(resource, "distribution", action) + + @catch + def download(self, urls: Union[str, List[str]], + paths: Union[str, List[str]], overwrite: bool = False) -> None: + # path: DirPath. + """Download files """ + if isinstance(urls, list): + # Check consistancy between urls and paths + if not isinstance(paths, list): + raise TypeError("Given multiple urls, paths should also be a list.") + if len(paths) != len(urls): + raise ValueError(f"Missmatch between urls ({len(urls)}) and paths ({len(paths)}), \ + they should be the same amount.") + self._download_many(urls, paths) + else: + self._download_one(urls, paths) + + def _download_many(self, urls: List[str], + paths: List[str]) -> None: + async def _bulk(): + loop = asyncio.get_event_loop() + semaphore = asyncio.Semaphore(self.service.max_connection) + async with ClientSession(headers=self.service.headers_download) as session: + tasks = ( + _create_task(x, y, loop, semaphore, session) + for x, y in zip(urls, paths) + ) + return await asyncio.gather(*tasks) + + def _create_task(url, path, loop, semaphore, session): + return loop.create_task( + _download(url, path, semaphore, session) + ) + + async def _download(url, path, semaphore, session): + async with semaphore: + params_download = copy.deepcopy(self.service.params.get('download', {})) + async with session.get(url, params=params_download) as response: + try: + response.raise_for_status() + except Exception as e: + raise DownloadingError( + f"Downloading:{_error_message(e)}" + ) + else: + with open(path, "wb") as f: + data = await response.read() + f.write(data) + + return asyncio.run(_bulk()) + + def _download_one(self, url: str, path: str) -> None: + try: + params_download = copy.deepcopy(self.service.params.get('download', {})) + response = requests.get( + url=url, + headers=self.service.headers_download, + params=params_download, + verify=False + ) + response.raise_for_status() + except Exception as e: + raise DownloadingError( + f"Downloading from failed :{_error_message(e)}" + ) + else: + with open(path, "wb") as f: + for chunk in response.iter_content(chunk_size=4096): + f.write(chunk) + + def health(self) -> Callable: + if self._health: + try: + response = requests.get(self._health) + except SSLError: + response = requests.get(self._health, verify=False) + return response.json() + else: + raise ConfigurationError('Health information not reachable with given configuration. \ + Define health in configuration arguments or set _health.') + + @staticmethod + def _service_from_web_service(endpoint: str, **source_config) -> Any: + return WebService(endpoint, **source_config) + + @staticmethod + def _service_from_store(store: Callable, **store_config) -> Store: + not_supported() \ No newline at end of file diff --git a/kgforge/specializations/models/rdf_model.py b/kgforge/specializations/models/rdf_model.py index be8e81b1..3f87c94e 100644 --- a/kgforge/specializations/models/rdf_model.py +++ b/kgforge/specializations/models/rdf_model.py @@ -21,11 +21,12 @@ from rdflib.namespace import XSD from kgforge.core import Resource +from kgforge.core.archetypes import Mapping +from kgforge.core.commons.execution import run from kgforge.core.archetypes import Model, Store from kgforge.core.commons.actions import Action from kgforge.core.commons.context import Context from kgforge.core.commons.exceptions import ValidationError -from kgforge.core.commons.execution import run from kgforge.specializations.models.rdf.collectors import NodeProperties from kgforge.specializations.models.rdf.directory_service import DirectoryService from kgforge.specializations.models.rdf.service import RdfService @@ -59,6 +60,7 @@ XSD.dateTime: datetime.datetime(9999, 12, 31).isoformat(), } + DEFAULT_TYPE_ORDER = [str, float, int, bool, datetime.date, datetime.time] @@ -87,6 +89,26 @@ def _generate_context(self) -> Context: if document: return Context(document) + # Mappings. + + def _mappings(self, source: str) -> Dict[str, List[str]]: + dirpath = Path(source, "mappings") + mappings = {} + if dirpath.is_dir(): + for x in dirpath.glob("*/*.hjson"): + mappings.setdefault(x.stem, []).append(x.parent.name) + else: + raise ValueError(f"unrecognized source {dirpath}") + return mappings + + def mapping(self, entity: str, source: str, type: Callable) -> Mapping: + filename = f"{entity}.hjson" + filepath = Path(source, "mappings", type.__name__, filename) + if filepath.is_file(): + return type.load(filepath) + else: + raise ValueError(f"unrecognized entity type or source file: {filepath}") + # Templates. def _template(self, type: str, only_required: bool) -> Dict: diff --git a/kgforge/specializations/resources/datasets.py b/kgforge/specializations/resources/datasets.py index baa35954..b3564729 100644 --- a/kgforge/specializations/resources/datasets.py +++ b/kgforge/specializations/resources/datasets.py @@ -12,13 +12,13 @@ # You should have received a copy of the GNU Lesser General Public License # along with Blue Brain Nexus Forge. If not, see . -from typing import List, Union +from typing import List, Union, Optional from warnings import warn from kgforge.core import Resource from kgforge.core.commons.actions import LazyAction from kgforge.core.commons.execution import catch, not_supported -from kgforge.core.forge import KnowledgeGraphForge +# Do not 'from kgforge.core import KnowledgeGraphForge' to avoid cyclic dependency # POLICY _set() should be only called as the last statement to ensure atomicity in case of errors. @@ -33,9 +33,9 @@ class Dataset(Resource): # No catching of exceptions so that no incomplete instance is created if an error occurs. # This is a best practice in Python for __init__(). - def __init__(self, forge: KnowledgeGraphForge, type: str = "Dataset", **properties) -> None: + def __init__(self, forge: Optional["KnowledgeGraphForge"], type: str = "Dataset", **properties) -> None: super().__init__(**properties) - self._forge: KnowledgeGraphForge = forge + self._forge: Optional["KnowledgeGraphForge"] = forge self.type: str = type @catch @@ -164,7 +164,7 @@ def download(self, path: str = ".", source: str = "distributions", follow: str = self._forge.download(self, follow, path, overwrite, cross_bucket, content_type) @classmethod - def from_resource(cls, forge: KnowledgeGraphForge, data: Union[Resource, List[Resource]], + def from_resource(cls, forge: Optional["KnowledgeGraphForge"], data: Union[Resource, List[Resource]], store_metadata: bool = False): def _(d): resource_json = forge.as_json(d) @@ -175,9 +175,9 @@ def _(d): return [_(d) for d in data] if isinstance(data, List) else _(data) -def _set(dataset: Dataset, attr: str, data: Union[Resource, List[Resource], LazyAction]) -> None: - if hasattr(dataset, attr): - value = getattr(dataset, attr) +def _set(resource: Resource, attr: str, data: Union[Resource, List[Resource], LazyAction]) -> None: + if hasattr(resource, attr): + value = getattr(resource, attr) if isinstance(value, List): if isinstance(data, List): value.extend(data) @@ -188,6 +188,6 @@ def _set(dataset: Dataset, attr: str, data: Union[Resource, List[Resource], Lazy new = [value, *data] else: new = [value, data] - setattr(dataset, attr, new) + setattr(resource, attr, new) else: - setattr(dataset, attr, data) + setattr(resource, attr, data) diff --git a/kgforge/specializations/stores/__init__.py b/kgforge/specializations/stores/__init__.py index ec3d912f..6af3cba1 100644 --- a/kgforge/specializations/stores/__init__.py +++ b/kgforge/specializations/stores/__init__.py @@ -14,3 +14,4 @@ from .bluebrain_nexus import BlueBrainNexus from .demo_store import DemoStore +from .sparql import SPARQLStore \ No newline at end of file diff --git a/kgforge/specializations/stores/bluebrain_nexus.py b/kgforge/specializations/stores/bluebrain_nexus.py index 7294295c..81a66cbb 100644 --- a/kgforge/specializations/stores/bluebrain_nexus.py +++ b/kgforge/specializations/stores/bluebrain_nexus.py @@ -60,7 +60,8 @@ from kgforge.specializations.mappers import DictionaryMapper from kgforge.specializations.mappings import DictionaryMapping from kgforge.specializations.stores.nexus.service import BatchAction, Service, _error_message - +from kgforge.core.archetypes.store import resources_from_construct_query +from kgforge.core.commons.es_query_builder import ESQueryBuilder class CategoryDataType(Enum): DATETIME = "datetime" diff --git a/kgforge/specializations/stores/databases/__init__.py b/kgforge/specializations/stores/databases/__init__.py new file mode 100644 index 00000000..8b95aec5 --- /dev/null +++ b/kgforge/specializations/stores/databases/__init__.py @@ -0,0 +1,16 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +from .service import Service +from .web_service import WebService \ No newline at end of file diff --git a/kgforge/specializations/stores/databases/service.py b/kgforge/specializations/stores/databases/service.py new file mode 100644 index 00000000..2e8b9d79 --- /dev/null +++ b/kgforge/specializations/stores/databases/service.py @@ -0,0 +1,83 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +import asyncio +import copy +from difflib import context_diff +import json +import re +from pathlib import Path +from asyncio import Task +from collections import namedtuple +from copy import deepcopy +from enum import Enum +from typing import Callable, Dict, List, Optional, Union, Tuple +from urllib.error import URLError +from urllib.parse import quote_plus, urlparse + +import nest_asyncio +import nexussdk as nexus +import requests +from aiohttp import ClientSession, hdrs +from numpy import nan +from requests import HTTPError + +from kgforge.core import Resource +from kgforge.core.commons.context import Context +from kgforge.core.conversions.rdf import ( + _from_jsonld_one, + recursive_resolve, +) +from kgforge.core.wrappings.dict import wrap_dict + + +class Service: + + def __init__( + self, + endpoint: str, + model_context: Context, + store_context: Context, + max_connection: int, + searchendpoints: Dict, + content_type: str, + accept: str, + **params, + ): + + self.endpoint = endpoint + self.model_context = model_context + self.context_cache: Dict = dict() + self.context = store_context + self.max_connection = max_connection + self.params = copy.deepcopy(params) + + self.headers = {"Content-Type": content_type, "Accept": accept} + + try: + sparql_config = searchendpoints["sparql"] + self.headers_sparql = { + "Content-Type": sparql_config["Content-Type"] + if sparql_config and "Content-Type" in sparql_config + else "text/plain", + "Accept": sparql_config["Accept"] + if sparql_config and "Accept" in sparql_config + else "application/sparql-results+json", + } + + self.sparql_endpoint = dict() + self.sparql_endpoint["endpoint"] = searchendpoints["sparql"]["endpoint"] + self.sparql_endpoint["type"] = "sparql" + except Exception: + raise ValueError(f"Store configuration error: sparql searchendpoint missing") \ No newline at end of file diff --git a/kgforge/specializations/stores/databases/web_service.py b/kgforge/specializations/stores/databases/web_service.py new file mode 100644 index 00000000..d9a3ef32 --- /dev/null +++ b/kgforge/specializations/stores/databases/web_service.py @@ -0,0 +1,79 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . +import copy +from typing import Dict, List, Optional, Union, Tuple + +from kgforge.core.commons.context import Context +from kgforge.core.commons.exceptions import ConfigurationError +from kgforge.specializations.databases.utils import resources_from_request +from kgforge.specializations.stores.bluebrain_nexus import _error_message + + +class WebService: + + def __init__( + self, + endpoint: str, + model_context: Context, + service_context: Context, + max_connection: int, + content_type: str, + accept: str, + **params, + ): + + self.endpoint = endpoint + self.model_context = model_context + self.context_cache: Dict = dict() + self.context = service_context + self.max_connection = max_connection + self.files_download = params.pop('files_download', None) + self.search_endpoints = params.pop('searchendpoints', None) + if self.search_endpoints: + for endpoint in self.search_endpoints: + if not 'endpoint' in self.search_endpoints[endpoint]: + raise ConfigurationError(f"Missing endpoint searchenpoints") + self.params = copy.deepcopy(params) + + self.headers = {"Content-Type": content_type, "Accept": accept} + + self.headers_download = { + "Content-Type": self.files_download["Content-Type"] + if self.files_download and "Content-Type" in self.files_download + else "text/plain", + "Accept": self.files_download["Accept"] + if self.files_download and "Accept" in self.files_download + else "text/plain", + } + + def search(self, resolvers, *filters, **params): + # resolvers are not used, just passed because of previous methods shapes + filter_params = filters[0] + if not isinstance(filter_params, dict): + raise NotImplementedError('Currently only the use of a dictionary is implemented') + searchendpoint = params.pop('searchendpoint', None) + if searchendpoint: + if self.search_endpoints is None: + raise ConfigurationError(f"No searchendpoints were given " \ + "in the initial configuration.") + try: + endpoint = self.search_endpoints[searchendpoint]['endpoint'] + except KeyError: + raise ConfigurationError(f"The {searchendpoint} searchpoint was not given "\ + "in the initial configuration.") + + else: + endpoint = self.endpoint + query_params = {**filter_params, **params} + return resources_from_request(endpoint, self.headers, **query_params) \ No newline at end of file diff --git a/kgforge/specializations/stores/sparql.py b/kgforge/specializations/stores/sparql.py new file mode 100644 index 00000000..d0ea2ac8 --- /dev/null +++ b/kgforge/specializations/stores/sparql.py @@ -0,0 +1,280 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +import json +from pathlib import Path +from typing import Dict, List, Callable, Optional, Union, Any +from rdflib.plugins.sparql.parser import Query +from SPARQLWrapper import SPARQLWrapper, JSON + +from kgforge.core import Resource +from kgforge.core.archetypes import Resolver, Store +from kgforge.core.archetypes.store import _replace_in_sparql, rewrite_sparql, resources_from_construct_query +from kgforge.core.commons.context import Context +from kgforge.core.wrappings.dict import DictWrapper +from kgforge.specializations.stores.databases import Service +from kgforge.core.wrappings.paths import create_filters_from_dict +from kgforge.core.commons.exceptions import QueryingError +from kgforge.core.commons.execution import not_supported +from kgforge.specializations.mappers import DictionaryMapper +from kgforge.specializations.mappings import DictionaryMapping +from kgforge.core.commons.sparql_query_builder import SPARQLQueryBuilder +from kgforge.specializations.stores.bluebrain_nexus import ( + CategoryDataType, + _create_select_query +) + + +class SPARQLStore(Store): + """A Store specialized for SPARQL queries, supporting only Reading (searching) methods.""" + + def __init__(self, endpoint: Optional[str] = None, bucket: Optional[str] = None, + token: Optional[str] = None, versioned_id_template: Optional[str] = None, + file_resource_mapping: Optional[str] = None, + model_context: Optional[Context] = None, + searchendpoints: Optional[Dict] = None, **store_config) -> None: + super().__init__(endpoint, bucket, token, versioned_id_template, file_resource_mapping, + model_context, searchendpoints, **store_config) + + + # [C]RUD + @property + def mapping(self) -> Optional[Callable]: + return DictionaryMapping + + @property + def mapper(self) -> Optional[DictionaryMapper]: + return DictionaryMapper + + def register(self, data: Union[Resource, List[Resource]], schema_id: str = None + ) -> None: + # Replace None by self._register_many to switch to optimized bulk registration. + not_supported() + + def _register_one(self, resource: Resource, schema_id: str) -> None: + not_supported() + + def upload(self, path: str, content_type: str) -> Union[Resource, List[Resource]]: + not_supported() + + def _upload(path: Path, content_type: str) -> Union[Any, List[Any]]: + not_supported() + + def _upload_many(self, paths: List[Path], content_type: str) -> List[Any]: + not_supported() + + def _download_one( + self, + url: str, + path: str, + store_metadata: Optional[DictWrapper], + cross_bucket: bool, + ) -> None: + # path: FilePath. + # TODO define dowloading method + # POLICY Should notify of failures with exception DownloadingError including a message. + not_supported() + + # C[R]UD + + def retrieve( + self, id: str, version: Optional[Union[int, str]], cross_bucket: bool, **params + ) -> Resource: + not_supported() + + def _retrieve_filename(self, id: str) -> str: + not_supported() + + # CR[U]D. + + def update( + self, data: Union[Resource, List[Resource]], schema_id: Optional[str] + ) -> None: + # Replace None by self._update_many to switch to optimized bulk update. + not_supported() + + def _update_one(self, resource: Resource, schema_id: Optional[str]) -> None: + not_supported() + + def tag(self, data: Union[Resource, List[Resource]], value: str) -> None: + # Replace None by self._tag_many to switch to optimized bulk tagging. + # POLICY If tagging modify the resource, run() should have status='_synchronized'. + not_supported() + + # CRU[D]. + + def deprecate(self, data: Union[Resource, List[Resource]]) -> None: + # Replace None by self._deprecate_many to switch to optimized bulk deprecation. + not_supported() + + # Querying. + + def search( + self, resolvers: Optional[List["Resolver"]]=None, *filters, **params + ) -> List[Resource]: + # Positional arguments in 'filters' are instances of type Filter from wrappings/paths.py + # A dictionary can be provided for filters: + # - {'key1': 'val', 'key2': {'key3': 'val'}} will be translated to + # - [Filter(operator='__eq__', path=['key1'], value='val'), Filter(operator='__eq__', path=['key2', 'key3'], value='val')] + # Keyword arguments in 'params' could be: + # - debug: bool, + # - limit: int, + # - offset: int, + # - deprecated: bool, + # - resolving: str, with values in ('exact', 'fuzzy'), + # - lookup: str, with values in ('current', 'children'). + # POLICY Should use sparql() and contain 'search_endpoint'. + # POLICY Resource _store_metadata should be set using wrappers.dict.wrap_dict(). + # POLICY Resource _synchronized should be set to True. + # pass + # if self.model_context is None: + # raise ValueError("context model missing") + debug = params.get("debug", False) + limit = params.get("limit", 100) + offset = params.get("offset", None) + distinct = params.get("distinct", False) + includes = params.get("includes", None) + excludes = params.get("excludes", None) + search_endpoint = params.get( + "search_endpoint", self.service.sparql_endpoint["type"] + ) + if search_endpoint not in [ + self.service.sparql_endpoint["type"], + ]: + raise ValueError( + f"The provided search_endpoint value '{search_endpoint}' is not supported, only 'sparql'" + ) + if "filters" in params: + raise ValueError("A 'filters' key was provided as params. Filters should be provided as iterable to be unpacked.") + + + if filters and isinstance(filters[0], dict): + filters = create_filters_from_dict(filters[0]) + filters = list(filters) if not isinstance(filters, list) else filters + + if includes or excludes: + raise ValueError( + "Field inclusion and exclusion are not supported when using SPARQL" + ) + + query_statements, query_filters = SPARQLQueryBuilder.build( + None, resolvers, self.model_context, *filters + ) + statements = ";\n ".join(query_statements) + _filters = ".\n".join(query_filters) + '\n' + _vars = ["?id"] + query = _create_select_query( + _vars, f"?id {statements} . \n {_filters}", distinct, False + ) + # support @id and @type + resources = self.sparql(query, debug=debug, limit=limit, offset=offset) + return resources + + def sparql( + self, query: str, debug: bool = False, limit: int = None, offset: int = None, **params + ) -> List[Resource]: + rewrite = params.get("rewrite", True) + qr = ( + rewrite_sparql(query, self.context, self.service.context) + if self.context is not None and rewrite + else query + ) + if rewrite: + qr = _replace_in_sparql(qr, "LIMIT", limit, 100, r" LIMIT \d+") + qr = _replace_in_sparql(qr, "OFFSET", offset, 0, r" OFFSET \d+") + if debug: + self._debug_query(qr) + return self._sparql(qr, limit, offset, **params) + + def _sparql(self, query: str, limit: int, offset: int = None, **params) -> List[Resource]: + try: + wrapper = SPARQLWrapper(self.service.sparql_endpoint["endpoint"]) + wrapper.setQuery(query) + wrapper.setReturnFormat(JSON) + response = wrapper.query() + except Exception as e: + raise QueryingError(e) + else: + data = response.convert() + # FIXME workaround to parse a CONSTRUCT query, this fix depends on + # https://github.com/BlueBrain/nexus/issues/1155 + _, q_comp = Query.parseString(query) + if q_comp.name == "ConstructQuery": + context = self.model_context or self.context + return resources_from_construct_query(data, context) + else: + # SELECT QUERY + results = data["results"]["bindings"] + return self.resources_from_results(results) + + @staticmethod + def resources_from_results(results): + return [ + Resource(**{k: json.loads(str(v["value"]).lower()) if v['type'] =='literal' and + ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#boolean') + else (int(v["value"]) if v['type'] =='literal' and + ('datatype' in v and v['datatype']=='http://www.w3.org/2001/XMLSchema#integer') + else v["value"] + ) + for k, v in x.items()} ) + for x in results + ] + + def elastic( + self, query: str, debug: bool, limit: int, offset: int + ) -> List[Resource]: + not_supported() + + # Versioning. + + def freeze(self, data: Union[Resource, List[Resource]]) -> None: + not_supported() + + def _freeze_one(self, resource: Resource) -> None: + not_supported() + + # Utils. + + def _initialize_service( + self, + endpoint: Optional[str], + bucket: Optional[str], + token: Optional[str], + searchendpoints: Optional[Dict], + **store_config, + ) -> Any: + try: + max_connection = store_config.pop("max_connection", 50) + if max_connection <= 0: + raise ValueError( + f"max_connection value should be great than 0 but {max_connection} is provided" + ) + content_type = store_config.pop("Content-Type", "application/ld+json") + accept = store_config.pop("Accept", "application/ld+json") + params = store_config.pop("params", {}) + store_context = store_config.pop('store_context', None) + + except Exception as ve: + raise ValueError(f"Store configuration error: {ve}") + else: + return Service(endpoint=endpoint, model_context=self.model_context, store_context=store_context, max_connection=max_connection, + searchendpoints=searchendpoints, content_type=content_type, accept=accept, **params) + + @staticmethod + def _debug_query(query): + if isinstance(query, Dict): + print("Submitted query:", query) + else: + print(*["Submitted query:", *query.splitlines()], sep="\n ") + print() \ No newline at end of file diff --git a/setup.py b/setup.py index e1e6cf4b..7ff5a8f0 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,8 @@ "nest-asyncio>=1.5.1", "pyparsing>=2.0.2", "owlrl>=5.2.3", - "elasticsearch_dsl==7.4.0" + "elasticsearch_dsl==7.4.0", + "SPARQLWrapper==2.0.0" ], extras_require={ "dev": [ diff --git a/tests/conftest.py b/tests/conftest.py index 99122160..e1352a4e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -422,7 +422,9 @@ def _make_jsonld( MODEL = "DemoModel" STORE = "DemoStore" RESOLVER = "DemoResolver" - +DATABASE = "StoreDatabase" +DBNAME = "DBpedia" +SPARQL_ENDPOINT = "http://dbpedia.org/sparql" @pytest.fixture( params=[ @@ -456,9 +458,18 @@ def store(request): def resolver(request): return request.param +@pytest.fixture( + params=[ + DATABASE, + f"{DATABASE} from kgforge.specializations.databases", + f"{DATABASE} from kgforge.specializations.databases.store_database", + ] +) +def database(request): + return request.param @pytest.fixture -def config(model, store, resolver): +def config(model, store, resolver, database): return { "Model": { "name": model, @@ -497,6 +508,27 @@ def config(model, store, resolver): }, ], }, + "Databases": { + DBNAME: + { + "origin": "store", + "source": "SPARQLStore", + "searchendpoints":{ + "sparql":{ + "endpoint": SPARQL_ENDPOINT + }, + + }, + "model": { + "name": model, + "origin": "directory", + "source": "tests/data/demo-model/", + "context":{ + "iri": "test/data/dbpedia" + } + }, + }, + } } diff --git a/tests/data/dbpedia/jsonld_context.json b/tests/data/dbpedia/jsonld_context.json new file mode 100644 index 00000000..09950988 --- /dev/null +++ b/tests/data/dbpedia/jsonld_context.json @@ -0,0 +1,33 @@ +{ + "@context": { + "owl": "http://www.w3.org/2002/07/owl#", + "owl2xml": "http://www.w3.org/2006/12/owl2-xml#", + "swrlb": "http://www.w3.org/2003/11/swrlb#", + "protege": "http://protege.stanford.edu/plugins/owl/protege#", + "swrl": "http://www.w3.org/2003/11/swrl#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "foaf": "http://xmlns.com/foaf/0.1/", + "dbpedia2": "http://dbpedia.org/property/", + "dbpedia": "http://dbpedia.org/", + "dbo": "http://dbpedia.org/ontology/", + "id": { + "@id": "http://www.geneontology.org/formats/oboInOwl#id" + }, + "MusicalArtist": { + "@id": "dbo:MusicalArtist" + }, + "birthDate": { + "@id": "dbo:birthDate" + }, + "birthPlace": { + "@id": "dbo:birthPlace" + }, + "type": { + "@id": "rdf:type" + }, + "@id": "https://bbp.epfl.ch/jsonldcontext/db/dbpedia" + } +} \ No newline at end of file diff --git a/tests/specializations/databases/test_database.py b/tests/specializations/databases/test_database.py new file mode 100644 index 00000000..1524ee48 --- /dev/null +++ b/tests/specializations/databases/test_database.py @@ -0,0 +1,224 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +# Placeholder for the test suite for actions. +import copy +import pytest + +from kgforge.core import Resource, KnowledgeGraphForge +from kgforge.core.commons.context import Context +from kgforge.core.wrappings.paths import Filter +from kgforge.specializations.databases import WebServiceDatabase +from kgforge.specializations.databases.store_database import StoreDatabase +from kgforge.specializations.databases.utils import type_from_filters +from kgforge.core.commons.exceptions import ConfigurationError, QueryingError +from kgforge.core.wrappings.dict import DictWrapper, wrap_dict +import json + +from tests.specializations.stores.test_sparql import dict_context + + +@pytest.mark.parametrize( + "filters,expected", + [ + pytest.param( + ({'type': 'Person'}), + ('Person'), + id="dictionary_type", + ), + pytest.param( + ({'id': 'my_id'}), + (None), + id="dictionary_notype", + ), + pytest.param( + (Filter(['id'], '__eq__', 'some_id')), + (None), + id="one_filter_notype", + ), + pytest.param( + (Filter(['type'], '__eq__', 'Person')), + ('Person'), + id="one_filter_type", + ), + pytest.param( + (Filter(['id'], '__eq__', 'some_id'), + Filter(['label'], '__eq__', 'some_label')), + None, + id="two_filters_notype", + ), + pytest.param( + (Filter(['id'], '__eq__', 'some_id'), + Filter(['type'], '__eq__', 'Person')), + 'Person', + id="two_filters_type", + ), + pytest.param( + (Filter(['type'], '__eq__', 'Protein'), + Filter(['type'], '__eq__', 'Person')), + 'Protein', + id="two_filters_two_types", + ), + ] +) +def test_type_from_filters(filters, expected): + assert type_from_filters(filters) == expected + + +@pytest.fixture +def context(dict_context): + return Context(dict_context) + + +def forge(db_config, model_config): + config = { + "Model": model_config, + "Store": { + "name": "DemoStore", + }, + "Databases": { + 'dbpedia': db_config + } + } + return KnowledgeGraphForge(config) + +@pytest.fixture +def model_config(): + return { + "name": "DemoModel", + "origin": "directory", + "source": "tests/data/demo-model/" + } + +@pytest.fixture +def storedb_config(): + return { + "origin": "store", + "source": "SPARQLStore", + "searchendpoints":{ + "sparql":{ + "endpoint": "http://dbpedia.org/sparql" + }, + + }, + "model": { + "name": "DemoModel", + "origin": "directory", + "source": "tests/data/demo-model/", + "context":{ + "iri": "test/data/dbpedia" + } + } + } +class TestStoreDB: + + @pytest.fixture + def forge_with_store(self, storedb_config, model_config): + return forge(storedb_config, model_config) + + def test_database_config(self, forge_with_store, storedb_config, model_config): + with pytest.raises(ValueError): + StoreDatabase(storedb_config) # Missing forge + with pytest.raises(ValueError): + StoreDatabase(forge_with_store, **storedb_config) # Missing name + with pytest.raises(ValueError): + # Missing model + StoreDatabase(forge_with_store, **{'name': 'mydb', 'origin': 'store', + 'source':'tests/data/demo-store/'}) + with pytest.raises(ConfigurationError): + StoreDatabase(forge_with_store, **{'name': 'mydb', 'origin': 'store', + 'source':'tests/data/demo-store/', + 'model': model_config}) + + def test_database_directory(self, forge_with_store, model_config): + with pytest.raises(Exception): + StoreDatabase(forge_with_store, **{'name': 'mydb', 'origin': 'directory', + 'source':'tests/data/demo-store/', + 'model': model_config}) + + def test_database_wrong_web_service(self, forge_with_store, model_config): + with pytest.raises(Exception): + StoreDatabase(forge_with_store, **{'name': 'mydb', 'origin': 'web_service', + 'source':'tests/data/demo-store/', + 'model': model_config}) + + +@pytest.fixture +def ws_db_config(): + return { + "name": "uniprot", + "origin": "web_service", + "source": "https://rest.uniprot.org/uniprotkb/", + "model": { + "name": "DemoModel", + "origin": "directory", + "source": "tests/data/demo-model/", + "context":{ + "iri": "examples/database-sources/UniProt" + } + }, + "max_connection": 10, + "content_type": "application/json;charset=UTF-8", + "accept": "*/*" + } + +class TestWebServiceDB: + @pytest.fixture + def forge_with_wsdb(self, ws_db_config, model_config): + return forge(ws_db_config, model_config) + + def test_database_config(self, forge_with_wsdb, ws_db_config, model_config): + with pytest.raises(ValueError): + WebServiceDatabase(ws_db_config) # Missing forge + with pytest.raises(ValueError): + # Missing model + WebServiceDatabase(forge_with_wsdb, **{'name': 'mydb', 'origin': 'web_service', + 'source':'tests/data/demo-store/', 'max_connection': 5, + 'content_type': 'application/json', 'accept': '*/*'}) + with pytest.raises(ValueError): + # Missing max_connection + WebServiceDatabase(forge_with_wsdb, **{'name': 'mydb', 'origin': 'web_service', + 'source':'tests/data/demo-store/', + 'content_type': 'application/json', 'accept': '*/*'}) + with pytest.raises(ValueError): + # Missing content_type + WebServiceDatabase(forge_with_wsdb, **{'name': 'mydb', 'origin': 'web_service', + 'source':'tests/data/demo-store/', 'max_connection': 5, + 'content_type': 'application/json'}) + + def test_database_directory(self, forge_with_wsdb, model_config): + with pytest.raises(Exception): + WebServiceDatabase(forge_with_wsdb, **{'name':'mydb', 'origin': 'directory', + 'source':'tests/data/demo-store/', + 'model': model_config}) + def test_database_wrong_store(self, forge_with_wsdb, model_config): + with pytest.raises(Exception): + WebServiceDatabase(forge_with_wsdb, **{'name': 'mydb', 'origin': 'store', + 'source':'tests/data/demo-store/', + 'model': model_config}) + + def test_searchendpoint_config(self, forge_with_wsdb, ws_db_config): + tmp_config = copy.deepcopy(ws_db_config) + tmp_config['searchendpoints'] = {'some_endpoint': {'wrong': 'True'}} + with pytest.raises(ConfigurationError): + WebServiceDatabase(forge_with_wsdb, **tmp_config) + + def test_search(self, forge_with_wsdb, ws_db_config): + tmp_config = copy.deepcopy(ws_db_config) + tmp_config['searchendpoints'] = {'some_endpoint': {'endpoint': "https://my_endpoint.com/API/"}} + db = WebServiceDatabase(forge_with_wsdb, **tmp_config) + with pytest.raises(ConfigurationError): + db.search([], {'type': 'Protein'}, searchendpoint='uniprot') + with pytest.raises(QueryingError): + db.search([], {'type': 'Protein'}) \ No newline at end of file diff --git a/tests/specializations/stores/test_sparql.py b/tests/specializations/stores/test_sparql.py new file mode 100644 index 00000000..9d393dba --- /dev/null +++ b/tests/specializations/stores/test_sparql.py @@ -0,0 +1,87 @@ +# +# Blue Brain Nexus Forge is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Blue Brain Nexus Forge is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +# General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Blue Brain Nexus Forge. If not, see . + +import pytest + +from kgforge.core.commons.context import Context +from kgforge.specializations.stores.sparql import SPARQLStore + + +SEARCH_ENDPOINT = {"sparql": {"endpoint": "http://dbpedia.org/sparql"}} + +@pytest.fixture +def dict_context(): + document = { + "@context": { + "owl": "http://www.w3.org/2002/07/owl#", + "owl2xml": "http://www.w3.org/2006/12/owl2-xml#", + "swrlb": "http://www.w3.org/2003/11/swrlb#", + "protege": "http://protege.stanford.edu/plugins/owl/protege#", + "swrl": "http://www.w3.org/2003/11/swrl#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "foaf": "http://xmlns.com/foaf/0.1/", + "dbpedia2": "http://dbpedia.org/property/", + "dbpedia": "http://dbpedia.org/", + "dbo": "http://dbpedia.org/ontology/", + "dbr": "http://dbpedia.org/resource/", + "id": { + "@id": "http://www.geneontology.org/formats/oboInOwl#id" + }, + "MusicalArtist": { + "@id": "dbo:MusicalArtist" + }, + "birthDate": { + "@id": "dbo:birthDate" + }, + "birthPlace": { + "@id": "dbo:birthPlace" + }, + "type": { + "@id": "rdf:type" + }, + "Berlin": { + "@id": "dbr:Berlin" + }, + "@id": "https://bbp.epfl.ch/jsonldcontext/db/dbpedia" + } + } + return document + +@pytest.fixture +def store_context(dict_context): + return Context(dict_context) + +@pytest.fixture +def sparql_store(store_context): + return SPARQLStore( + model_context=store_context, + searchendpoints=SEARCH_ENDPOINT, + store_context=store_context + ) + +def test_config_error(): + with pytest.raises(ValueError): + SPARQLStore(endpoint="test", bucket="invalid") + +def test_config(sparql_store, store_context): + assert sparql_store.bucket == None + assert sparql_store.endpoint == None + assert sparql_store.context == store_context + +def test_search_params(sparql_store): + with pytest.raises(ValueError): + sparql_store.search(filters=[None]) \ No newline at end of file