diff --git a/.gitignore b/.gitignore index a08c0b8..c8d4804 100644 --- a/.gitignore +++ b/.gitignore @@ -174,3 +174,4 @@ cython_debug/ .pypirc README.pdf +.DS_STORE diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/README.org b/README.org index f4c2572..24c3e54 100644 --- a/README.org +++ b/README.org @@ -3,3 +3,15 @@ A database schema for storing arbitrary quantitative measurements. See the [[file:./docs/api.org][REST API documentation]] for details on the external query interface. + +* Installation +add the passwords to ~/.pgpass as +``` +host:port:*:quantdb-admin:password +host:port:*:quantdb-user:password +host:port:*:quantdb-test-admin:password +host:port:*:quantdb-test-user:password +``` + +* Notes +macOS will need the config.yaml in `Library/Application Support/quantdb/config.yaml` diff --git a/README.pdf b/README.pdf new file mode 100644 index 0000000..0431664 Binary files /dev/null and b/README.pdf differ diff --git a/bin/prepare_test_db.sh b/bin/prepare_test_db.sh new file mode 100644 index 0000000..b526bbe --- /dev/null +++ b/bin/prepare_test_db.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# quantdb-prepare-test-db.sh +# This script prepares a test PostgreSQL database with the same schema as the main DB and populates it with the first 10 rows of each table. +# Usage: ./bin/prepare_test_db.sh [MAIN_DB] [TEST_DB] [PGUSER] [PGHOST] [PGPORT] + +set -e +eval "$(conda shell.bash hook)" +conda activate quantdb + +# If you want to use the remote MAIN_DB connection info from client.py, extract it using Python +REMOTE_DB_URI=$(python -c "from quantdb.client import get_session; s = get_session(echo=False); print(str(s.get_bind().url))") +echo "Established Connection" + +MAIN_DB=${1:-quantdb} +TEST_DB=${2:-quantdb_test} +PGUSER=${3:-quantdb-test-admin} +PGHOST=${4:-localhost} +PGPORT=${5:-5432} + +# Parse the remote DB URI for pg_dump +REMOTE_PGUSER=$(echo "$REMOTE_DB_URI" | sed -E 's|.*://([^:]+):.*|\1|') +REMOTE_PGPASSWORD=$(python -c "from quantdb.config import auth; print(auth.get('db-password') or '')") +REMOTE_PGHOST=$(echo "$REMOTE_DB_URI" | sed -E 's|.*@([^:/]+).*|\1|') +REMOTE_PGPORT=$(echo "$REMOTE_DB_URI" | sed -E 's|.*:([0-9]+)/.*|\1|') +REMOTE_DBNAME=$(echo "$REMOTE_DB_URI" | sed -E 's|.*/([^?]+).*|\1|') + +# Export password for pg_dump if available +if [ -n "$REMOTE_PGPASSWORD" ]; then + export PGPASSWORD="$REMOTE_PGPASSWORD" +fi + +# Ensure the PGUSER has a database (needed for psql connection) +psql -U $PGUSER -h $PGHOST -p $PGPORT -d postgres -tc "SELECT 1 FROM pg_database WHERE datname = '$PGUSER'" | grep -q 1 || createdb -U $PGUSER -h $PGHOST -p $PGPORT $PGUSER + +# Drop and recreate the test database +psql -U $PGUSER -h $PGHOST -p $PGPORT -c "DROP DATABASE IF EXISTS $TEST_DB;" +psql -U $PGUSER -h $PGHOST -p $PGPORT -c "CREATE DATABASE $TEST_DB;" + +# Dump schema only from remote MAIN_DB and restore to local test DB +pg_dump -U $REMOTE_PGUSER -h $REMOTE_PGHOST -p $REMOTE_PGPORT -s $REMOTE_DBNAME | psql -U $PGUSER -h $PGHOST -p $PGPORT $TEST_DB + +# Get all table names from the remote MAIN_DB +TABLES=$(psql -U $REMOTE_PGUSER -h $REMOTE_PGHOST -p $REMOTE_PGPORT -d $REMOTE_DBNAME -Atc "SELECT tablename FROM pg_tables WHERE schemaname='public';") + +echo "Tables in remote DB: $TABLES" + +# For each table, copy the first 10 rows from remote MAIN_DB to local test DB +for TABLE in $TABLES; do + COLS=$(psql -U $REMOTE_PGUSER -h $REMOTE_PGHOST -p $REMOTE_PGPORT -d $REMOTE_DBNAME -Atc "SELECT string_agg('"' || column_name || '"', ',') FROM information_schema.columns WHERE table_name='$TABLE' AND table_schema='public';") + psql -U $PGUSER -h $PGHOST -p $PGPORT -d $TEST_DB -c "INSERT INTO \"$TABLE\" ($COLS) SELECT $COLS FROM dblink('host=$REMOTE_PGHOST user=$REMOTE_PGUSER dbname=$REMOTE_DBNAME port=$REMOTE_PGPORT','SELECT $COLS FROM \"$TABLE\" LIMIT 10') AS t($COLS);" 2>/dev/null || true +done + +unset PGPASSWORD + +echo "Test database '$TEST_DB' prepared with schema and first 10 rows of each table from remote '$REMOTE_DBNAME'." diff --git a/ingestion/f006.py b/ingestion/f006.py new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml index 2b360b8..0ceac3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,14 +13,16 @@ dependencies = [ "orthauth ~= 0.0.18", "cython", "psycopg2-binary", - "pre-commit ~= 4.1.0" + "pre-commit ~= 4.1.0", + "pytest==8.3.5", + "pandas==2.2.3", ] name = "quantdb" version = "0.1.0" description = "Quantitative DB" dynamic = ["readme"] packages = [{ include = "quantdb" }] -requires-python = ">=3.10" +requires-python = ">=3.10, <=3.13" [tool.setuptools.dynamic] readme = {file = ["README.org"], content-type = "text/plain"} diff --git a/quantdb/__init__.py b/quantdb/__init__.py index e69de29..9d48db4 100644 --- a/quantdb/__init__.py +++ b/quantdb/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/quantdb/client.py b/quantdb/client.py new file mode 100644 index 0000000..74eb0ef --- /dev/null +++ b/quantdb/client.py @@ -0,0 +1,48 @@ +from sqlalchemy import create_engine, text +from sqlalchemy.orm import Session + +from quantdb.config import auth +from quantdb.utils import dbUri + + +def get_session(echo: bool = True, test: bool = False) -> Session: + """ + Get a SQLAlchemy session for the main or test database. + + Parameters + ---------- + echo : bool, optional + If True, SQLAlchemy will log all statements. + test : bool, optional + If True, connect to the test database (quantdb_test or as configured). + + Returns + ------- + session : sqlalchemy.orm.Session + The database session. + """ + # pull in the db connection info + if test: + # For testing, ALWAYS use local postgres instance regardless of main db config + dbkwargs = { + 'dbuser': 'quantdb-test-admin', + 'host': 'localhost', # FORCE localhost for all testing + 'port': 5432, + 'database': auth.get('test-db-database') or 'quantdb_test', + 'password': 'tom-is-cool', # Use hardcoded password for test database + } + else: + # For non-test, use regular database configuration + dbkwargs = {k: auth.get(f'db-{k}') for k in ('user', 'host', 'port', 'database')} + dbkwargs['dbuser'] = dbkwargs.pop('user') + print(dbkwargs) + engine = create_engine(dbUri(**dbkwargs)) # type: ignore + engine.echo = echo + print(engine, dbkwargs) + session = Session(engine) + return session + + +if __name__ == '__main__': + session = get_session(echo=False) + print(session.execute(text('SELECT * FROM information_schema.tables')).fetchall()) diff --git a/quantdb/config.py b/quantdb/config.py index a50ec33..e42b009 100644 --- a/quantdb/config.py +++ b/quantdb/config.py @@ -1,3 +1,16 @@ +from dataclasses import dataclass + import orthauth as oa +from quantdb.utils import dbUri + auth = oa.configure_here('auth-config.py', __name__) + + +@dataclass +class Settings: + """Settings for the app.""" + + db_params = {k: auth.get(f'db-{k}') for k in ('user', 'host', 'port', 'database')} + db_params['dbuser'] = db_params.pop('user') + SQLALCHEMY_DATABASE_URI = dbUri(**db_params) diff --git a/quantdb/generic_ingest.py b/quantdb/generic_ingest.py new file mode 100644 index 0000000..ceba8b0 --- /dev/null +++ b/quantdb/generic_ingest.py @@ -0,0 +1,275 @@ +import ast +import uuid + +import pandas as pd +from sqlalchemy import PrimaryKeyConstraint, UniqueConstraint, inspect +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from sqlalchemy.sql.elements import ClauseElement + +from quantdb.client import get_session +from quantdb.models import Objects + +# df = pd.read_csv("data/CadaverVNMorphology_OutputMetrics.csv", index_col=0) +# df = df.T.reset_index(drop=True) +# df["id_sub"] = "sub-" + df.sub_sam +# df.index = df.index + 1 +# df.head() + + +def object_as_dict(obj): + """ + Convert SQLAlchemy ORM object to a dictionary of attributes. + + Parameters + ---------- + obj : SQLAlchemy ORM object + The object to convert to a dictionary. + + Returns + ------- + dict + Dictionary containing non-null column attributes of the object. + UUID objects are converted to strings for consistency. + """ + result = {} + for c in inspect(obj).mapper.column_attrs: + value = getattr(obj, c.key) + if value is not None: + # Convert UUID objects to strings for consistency + if hasattr(value, 'hex') and hasattr(value, 'version'): # Check if it's a UUID object + result[c.key] = str(value) + else: + result[c.key] = value + return result + + +def get_or_create(session, obj, back_populate=None): + """ + Retrieve an existing instance of the object from the database, or create it if it does not exist. + + Parameters + ---------- + session : sqlalchemy.orm.Session + The database session to use for querying and committing. + obj : SQLAlchemy model instance + The object to retrieve or create. + back_populate : dict, optional + Dictionary mapping attribute names to values to back-populate after creation. + + Returns + ------- + instance : SQLAlchemy model instance + The retrieved or newly created instance with UUID attributes converted to strings. + + Notes + ----- + The function will roll back the transaction and attempt to retrieve the object + if an exception occurs during creation. + """ + model = obj.__class__ + data = object_as_dict(obj) + instance = session.query(model).filter_by(**data).one_or_none() + if instance: + # Convert UUID attributes to strings for consistency + _convert_uuids_to_strings(instance) + return instance + else: + params = {k: v for k, v in data.items() if not isinstance(v, ClauseElement)} + instance = model(**params) + try: + session.add(instance) + session.commit() + if back_populate: + for attr, value in back_populate.items(): + setattr(instance, attr, value) + session.commit() + # Convert UUID attributes to strings for consistency + _convert_uuids_to_strings(instance) + except Exception: + session.rollback() + instance = session.query(model).filter_by(**data).one() + _convert_uuids_to_strings(instance) + return instance + else: + return instance + + +def _convert_uuids_to_strings(obj): + """ + Convert UUID attributes to strings in-place for an ORM object. + + Parameters + ---------- + obj : SQLAlchemy ORM object + The object to modify. + """ + for c in inspect(obj).mapper.column_attrs: + value = getattr(obj, c.key) + if value is not None and hasattr(value, 'hex') and hasattr(value, 'version'): + setattr(obj, c.key, str(value)) + + +def get_constraint_columns(model): + """ + Returns the columns of the unique constraints and primary key constraints for a SQLAlchemy model. + + Parameters + ---------- + model : SQLAlchemy model class + The SQLAlchemy model class to inspect for constraints. + + Returns + ------- + list of list of str + A list of lists, where each inner list contains the column names + that form a unique constraint or primary key constraint. + """ + constraints = [ + constraint + for constraint in model.__mapper__.tables[0].constraints + if isinstance(constraint, (UniqueConstraint, PrimaryKeyConstraint)) + ] + + return [[c.name for c in list(constraint.columns)] for constraint in constraints] + + +def query_by_constraints(db, obj): + """ + Query for an existing object in the database using its unique constraints. + + Parameters + ---------- + db : sqlalchemy.orm.Session + The database session to use for querying. + obj : SQLAlchemy model instance + The object to find in the database. + + Returns + ------- + SQLAlchemy model instance or None + The existing instance if found with UUID attributes converted to strings, None otherwise. + """ + filter_criteria = object_as_dict(obj) + + existing_obj = None + for cols in get_constraint_columns(obj.__class__): + if set(cols) != (set(cols) & set(filter_criteria.keys())): + continue + + constraint_filter = {key: value for key, value in filter_criteria.items() if key in cols} + + existing_obj = db.query(type(obj)).filter_by(**constraint_filter).first() + if existing_obj: + # Convert UUID attributes to strings for consistency + _convert_uuids_to_strings(existing_obj) + return existing_obj + + +def back_populate_tables(db: Session, obj) -> object: + """ + Recursively back-populates objects in the database, + retrieving existing objects instead of failing on duplicates. + + Parameters + ---------- + db : sqlalchemy.orm.Session + The database session. + obj : SQLAlchemy model instance + The ORM model object. + + Returns + ------- + object + The ORM model object, with its related objects back-populated and UUID attributes converted to strings. + + Notes + ----- + This function: + - Iterates through all relationships of the object + - Handles both initial population and recursive calls + - Performs a complete rollback if any commit fails + - Filters by all primary key columns of the object + - Handles cases where the child object is not a valid ORM instance + - Converts UUID attributes to strings for consistency + """ + mapper = obj.__mapper__ # Get the mapper for the object + # print(obj) + try: + # Iterate through all relationships of the object + for relationship_prop in mapper.relationships: + if relationship_prop.direction.name == 'MANYTOONE' or relationship_prop.direction.name == 'MANYTOMANY': + parent = getattr(obj, relationship_prop.key) + if parent: + filter_criteria = object_as_dict(parent) + existing_obj = db.query(type(parent)).filter_by(**filter_criteria).first() + if existing_obj: + _convert_uuids_to_strings(existing_obj) + for key, value in object_as_dict(existing_obj).items(): + if hasattr(parent, key): + setattr(parent, key, value) + # print("populated", object_as_dict(parent)) + setattr(obj, relationship_prop.key, parent) + # db.add(parent) + # db.commit() + back_populate_tables(db, parent) + + # print("object", type(obj)) + filter_criteria = object_as_dict(obj) + print(type(obj), filter_criteria) + existing_obj = db.query(type(obj)).filter_by(**filter_criteria).first() + print('filter criteria', filter_criteria) + print('found:', existing_obj) + + if not existing_obj: + existing_obj = query_by_constraints(db, obj) + + # print("filter using", filter_criteria) + # existing_obj = db.query(type(obj)).filter_by(**filter_criteria).first() + # print("is existing??", existing_obj) + + if not existing_obj: + db.add(obj) + # db.add(obj) + # db.flush() + db.commit() + # db.refresh(obj) + print('added new') + else: + _convert_uuids_to_strings(existing_obj) + db.merge(obj) + db.commit() + for key, value in object_as_dict(existing_obj).items(): + if hasattr(obj, key): + setattr(obj, key, value) + print('updated existing') + # print("after update", object_as_dict(existing_obj)) + + except IntegrityError: + db.rollback() # Rollback the transaction in case of an error + # print(f"Error during commit: {e}") + raise # Re-raise the exception to indicate failure + + # Convert UUID attributes to strings for consistency in the returned object + _convert_uuids_to_strings(obj) + return obj # Return the object with its related objects back-populated + + +if __name__ == '__main__': + + session = get_session(echo=False) + + object_dataset = Objects( + id='55c5b69c-a5b8-4881-a105-e4048af26fa5', + id_type='dataset', + id_file=None, + id_internal=None, + ) + object_package = Objects( + id='20720c2e-83fb-4454-bef1-1ce6a97fa748', + id_type='package', + id_file=1094489, + id_internal=None, + objects_=object_dataset, + ) + back_populate_tables(session, object_package) diff --git a/quantdb/ingest.py b/quantdb/ingest.py index 19abfce..85c37a1 100644 --- a/quantdb/ingest.py +++ b/quantdb/ingest.py @@ -872,7 +872,6 @@ def make_values_quant(this_dataset_updated_uuid, i, luinst): make_values_cat, make_values_quant, ) - # this is where things get annoying with needing selects on instance measured @@ -1315,7 +1314,6 @@ def ingest_demo_jp2(session, source_local=True, do_insert=True, commit=False, de def ingest_reva_ft_all(session, source_local=False, do_insert=True, batch=False, commit=False, dev=False): - dataset_uuids = ( 'aa43eda8-b29a-4c25-9840-ecbd57598afc', # f001 # the rest have uuid1 issues :/ all in the undefined folder it seems, might be able to fix with a reupload diff --git a/quantdb/models.py b/quantdb/models.py new file mode 100644 index 0000000..5fbc6ee --- /dev/null +++ b/quantdb/models.py @@ -0,0 +1,1456 @@ +import uuid +from typing import List, Optional + +from sqlalchemy import ( + CheckConstraint, + Column, + DateTime, + Enum, + ForeignKey, + ForeignKeyConstraint, + Identity, + Index, + Integer, + Numeric, + PrimaryKeyConstraint, + String, + Table, + Text, + UniqueConstraint, + Uuid, + event, + text, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import ( + Mapped, + declarative_base, + foreign, + mapped_column, + relationship, + validates, +) +from sqlalchemy.orm.base import Mapped + +Base = declarative_base() +metadata = Base.metadata + + +class Addresses(Base): + """Addresses of data + + Parameters + ---------- + id : int + Unique identifier for the address. + addr_type : str, optional + Type of address. + value_type : str, optional + Type of value. + addr_field : str, optional + Address field. + curator_note : str, optional + Curator note. + + Example + ------- + #/path-metadata/data/#int/dataset_relative_path + """ + + __tablename__ = 'addresses' + __table_args__ = ( + PrimaryKeyConstraint('id', name='addresses_pkey'), + UniqueConstraint( + 'addr_type', + 'addr_field', + 'value_type', + name='addresses_addr_type_addr_field_value_type_key', + ), + ) + + id = mapped_column( + Integer, + Identity( + start=1, + increment=1, + minvalue=1, + maxvalue=2147483647, + cycle=False, + cache=1, + ), + ) + addr_type = mapped_column( + Enum( + 'constant', + 'tabular-header', + 'tabular-alt-header', + 'workbook-sheet-tabular-header', + 'workbook-sheet-tabular-alt-header', + 'json-path-with-types', + 'file-system-extracted', + 'arbitrary-function', + name='address_type', + ) + ) + value_type = mapped_column( + Enum('single', 'multi', name='field_value_type'), + server_default=text("'single'::field_value_type"), + ) + addr_field = mapped_column(Text) + curator_note = mapped_column(Text) + + obj_desc_inst: Mapped[List['ObjDescInst']] = relationship( + 'ObjDescInst', + uselist=True, + foreign_keys='[ObjDescInst.addr_desc_inst]', + back_populates='addresses', + viewonly=True, + cascade='all, delete-orphan', + ) + obj_desc_inst_: Mapped[List['ObjDescInst']] = relationship( + 'ObjDescInst', + uselist=True, + foreign_keys='[ObjDescInst.addr_field]', + back_populates='addresses_field', + viewonly=True, + cascade='all, delete-orphan', + ) + obj_desc_cat: Mapped[List['ObjDescCat']] = relationship( + 'ObjDescCat', + uselist=True, + foreign_keys='[ObjDescCat.addr_desc_inst]', + back_populates='addresses', + viewonly=True, + cascade='all, delete-orphan', + ) + obj_desc_cat_: Mapped[List['ObjDescCat']] = relationship( + 'ObjDescCat', + uselist=True, + foreign_keys='[ObjDescCat.addr_field]', + back_populates='addresses_', + viewonly=True, + cascade='all, delete-orphan', + ) + obj_desc_quant: Mapped[List['ObjDescQuant']] = relationship( + 'ObjDescQuant', + uselist=True, + foreign_keys='[ObjDescQuant.addr_aspect]', + back_populates='addresses_aspect', + viewonly=True, + cascade='all, delete-orphan', + ) + obj_desc_quant_: Mapped[List['ObjDescQuant']] = relationship( + 'ObjDescQuant', + uselist=True, + foreign_keys='[ObjDescQuant.addr_desc_inst]', + back_populates='addresses', + viewonly=True, + cascade='all, delete-orphan', + ) + obj_desc_quant1: Mapped[List['ObjDescQuant']] = relationship( + 'ObjDescQuant', + uselist=True, + foreign_keys='[ObjDescQuant.addr_field]', + back_populates='addresses_field', + viewonly=True, + cascade='all, delete-orphan', + ) + obj_desc_quant2: Mapped[List['ObjDescQuant']] = relationship( + 'ObjDescQuant', + uselist=True, + foreign_keys='[ObjDescQuant.addr_unit]', + back_populates='addresses_unit', + viewonly=True, + cascade='all, delete-orphan', + ) + + +class Aspects(Base): + """Aspects of data + + Parameters + ---------- + id : int + Unique identifier for the aspect. + label : str + Label of the aspect. + iri : str + IRI of the aspect. + description : str, optional + Description of the aspect. + + Example + ------- + label: "distance" + iri: "http://uri.interlex.org/tgbugs/uris/readable/aspect/distance" + """ + + __tablename__ = 'aspects' + __table_args__ = ( + PrimaryKeyConstraint('id', name='aspects_pkey'), + UniqueConstraint('iri', name='aspects_iri_key'), + UniqueConstraint('label', name='aspects_label_key'), + ) + + id = mapped_column( + Integer, + Identity( + start=1, + increment=1, + minvalue=1, + maxvalue=2147483647, + cycle=False, + cache=1, + ), + ) + label = mapped_column(Text, nullable=False) + iri = mapped_column(Text, nullable=False) + description = mapped_column(Text) + + # aspects: Mapped["Aspects"] = relationship( + # "Aspects", + # secondary="aspect_parent", + # primaryjoin=lambda: Aspects.id == t_aspect_parent.c.id, + # secondaryjoin=lambda: Aspects.id == t_aspect_parent.c.parent, + # back_populates="aspects_", + # ) + # aspects_: Mapped["Aspects"] = relationship( + # "Aspects", + # secondary="aspect_parent", + # primaryjoin=lambda: Aspects.id == t_aspect_parent.c.parent, + # secondaryjoin=lambda: Aspects.id == t_aspect_parent.c.id, + # back_populates="aspects", + # ) + descriptors_quant: Mapped[List['DescriptorsQuant']] = relationship( + 'DescriptorsQuant', + uselist=True, + back_populates='aspects', + viewonly=True, + cascade='all, delete-orphan', + sync_backref=False, + ) + + +class ControlledTerms(Base): + """Controlled terms + + Parameters + ---------- + id : int + Unique identifier for the controlled term. + label : str + Label of the controlled term. + iri : str + IRI of the controlled term. + """ + + __tablename__ = 'controlled_terms' + __table_args__ = ( + PrimaryKeyConstraint('id', name='controlled_terms_pkey'), + UniqueConstraint('iri', name='controlled_terms_iri_key'), + UniqueConstraint('label', name='controlled_terms_label_key'), + ) + + id = mapped_column( + Integer, + Identity( + start=1, + increment=1, + minvalue=1, + maxvalue=2147483647, + cycle=False, + cache=1, + ), + ) + label = mapped_column(Text, nullable=False) + iri = mapped_column(Text, nullable=False) + + values_cat: Mapped[List['ValuesCat']] = relationship('ValuesCat', uselist=True, back_populates='controlled_terms') + + +class DescriptorsInst(Base): + """Descriptors Instance + + Parameters + ---------- + id : int + Unique identifier for the descriptor. + label : str + Label of the descriptor. + iri : str + IRI of the descriptor. + description : str, optional + Description of the descriptor. + + """ + + __tablename__ = 'descriptors_inst' + __table_args__ = ( + PrimaryKeyConstraint('id', name='descriptors_inst_pkey'), + UniqueConstraint('iri', name='descriptors_inst_iri_key'), + UniqueConstraint('label', name='descriptors_inst_label_key'), + ) + + id = mapped_column( + Integer, + Identity( + start=1, + increment=1, + minvalue=1, + maxvalue=2147483647, + cycle=False, + cache=1, + ), + ) + label = mapped_column(Text, nullable=False) + iri = mapped_column(Text, nullable=False) + description = mapped_column(Text) + + # descriptors_inst: Mapped["DescriptorsInst"] = relationship( + # "DescriptorsInst", + # secondary="class_parent", + # primaryjoin=lambda: DescriptorsInst.id == t_class_parent.c.id, + # secondaryjoin=lambda: DescriptorsInst.id == t_class_parent.c.parent, + # back_populates="descriptors_inst_", + # ) + # descriptors_inst_: Mapped["DescriptorsInst"] = relationship( + # "DescriptorsInst", + # secondary="class_parent", + # primaryjoin=lambda: DescriptorsInst.id == t_class_parent.c.parent, + # secondaryjoin=lambda: DescriptorsInst.id == t_class_parent.c.id, + # back_populates="descriptors_inst", + # ) + descriptors_cat: Mapped[List['DescriptorsCat']] = relationship( + 'DescriptorsCat', uselist=True, back_populates='descriptors_inst' + ) + descriptors_quant: Mapped[List['DescriptorsQuant']] = relationship( + 'DescriptorsQuant', + uselist=True, + back_populates='descriptors_inst', + viewonly=True, + cascade='all, delete-orphan', + ) + obj_desc_inst: Mapped[List['ObjDescInst']] = relationship( + 'ObjDescInst', + uselist=True, + back_populates='descriptors_inst', + viewonly=True, + cascade='all, delete-orphan', + ) + values_inst: Mapped[List['ValuesInst']] = relationship( + 'ValuesInst', + uselist=True, + back_populates='descriptors_inst', + viewonly=True, + cascade='all, delete-orphan', + ) + values_cat: Mapped[List['ValuesCat']] = relationship( + 'ValuesCat', + uselist=True, + back_populates='descriptors_inst', + viewonly=True, + cascade='all, delete-orphan', + ) + values_quant: Mapped[List['ValuesQuant']] = relationship( + 'ValuesQuant', + uselist=True, + back_populates='descriptors_inst', + viewonly=True, + cascade='all, delete-orphan', + ) + + +class Objects(Base): + """ + Represents an object in the database with various relationships and constraints. + + Attributes + ---------- + id : Uuid + Unique identifier for the object. + id_type : Enum + Type of the remote ID, can be one of 'organization', 'dataset', 'collection', 'package', or 'quantdb'. + id_file : Integer + Identifier for the file associated with the object. + id_internal : Uuid + Internal identifier for the object. + + Relationships + ------------- + objects : Mapped["Objects"] + Relationship to other objects through the 'dataset_object' table. + objects_ : Mapped["Objects"] + Reverse relationship to other objects through the 'dataset_object' table. + obj_desc_inst : Mapped[List["ObjDescInst"]] + Relationship to object description instances. + values_inst : Mapped[List["ValuesInst"]] + Relationship to value instances. + obj_desc_cat : Mapped[List["ObjDescCat"]] + Relationship to object description categories. + obj_desc_quant : Mapped[List["ObjDescQuant"]] + Relationship to object description quantities. + values_cat : Mapped[List["ValuesCat"]] + Relationship to value categories. + values_quant : Mapped[List["ValuesQuant"]] + Relationship to value quantities. + + Constraints + ----------- + __table_args__ : tuple + Contains various constraints and indexes for the table: + - CheckConstraint to ensure 'id_file' is not NULL if 'id_type' is 'package'. + - CheckConstraint to ensure 'id_internal' is not NULL and 'id' equals 'id_internal' if 'id_type' is 'quantdb'. + - PrimaryKeyConstraint on 'id'. + - Index on 'id_internal'. + """ + + __tablename__ = 'objects' + __table_args__ = ( + # CheckConstraint( + # "id_type <> 'package'::remote_id_type OR id_file IS NOT NULL", + # name="constraint_objects_remote_id_type_id_package", + # ), + # CheckConstraint( + # "id_type <> 'quantdb'::remote_id_type OR id_internal IS NOT NULL AND id = id_internal", + # name="constraint_objects_remote_id_type_id_internal", + # ), + # ForeignKeyConstraint( + # ["id_internal"], + # ["objects_internal.id"], + # name="objects_id_internal_fkey", + # ), + PrimaryKeyConstraint('id', name='objects_pkey'), + Index('idx_objects_id_internal', 'id_internal'), + ) + + id = mapped_column(Uuid) + id_type = mapped_column( + Enum( + 'organization', + 'dataset', + 'collection', + 'package', + 'quantdb', + name='remote_id_type', + ), + nullable=False, + ) + id_file = mapped_column(Integer, nullable=True) + id_internal = mapped_column(Uuid, nullable=True) + + # objects_internal: Mapped[Optional["ObjectsInternal"]] = relationship( + # "ObjectsInternal", foreign_keys=[id_internal], back_populates="objects" + # ) + objects: Mapped['Objects'] = relationship( + 'Objects', + secondary='dataset_object', + primaryjoin=lambda: Objects.id == t_dataset_object.c.dataset, + secondaryjoin=lambda: Objects.id == t_dataset_object.c.object, + # back_populates="objects_", + # cascade="all, delete-orphan", + ) + objects_: Mapped['Objects'] = relationship( + 'Objects', + secondary='dataset_object', + primaryjoin=lambda: Objects.id == t_dataset_object.c.object, + secondaryjoin=lambda: Objects.id == t_dataset_object.c.dataset, + # back_populates="objects", + # cascade="all, delete-orphan", + ) + # objects_internal_: Mapped[List["ObjectsInternal"]] = relationship( + # "ObjectsInternal", + # uselist=True, + # foreign_keys="[ObjectsInternal.dataset]", + # back_populates="objects_", + # ) + obj_desc_inst: Mapped[List['ObjDescInst']] = relationship( + 'ObjDescInst', + uselist=True, + back_populates='objects', + viewonly=True, + cascade='all, delete-orphan', + ) + obj_desc_cat: Mapped[List['ObjDescCat']] = relationship( + 'ObjDescCat', + uselist=True, + back_populates='objects', + viewonly=True, + cascade='all, delete-orphan', + ) + obj_desc_quant: Mapped[List['ObjDescQuant']] = relationship( + 'ObjDescQuant', + uselist=True, + back_populates='objects', + viewonly=True, + cascade='all, delete-orphan', + ) + values_inst: Mapped[List['ValuesInst']] = relationship( + 'ValuesInst', + uselist=True, + back_populates='objects', + viewonly=True, + cascade='all, delete-orphan', + ) + values_cat: Mapped[List['ValuesCat']] = relationship( + 'ValuesCat', + uselist=True, + back_populates='objects', + viewonly=True, + cascade='all, delete-orphan', + ) + values_quant: Mapped[List['ValuesQuant']] = relationship( + 'ValuesQuant', + uselist=True, + back_populates='objects', + viewonly=True, + cascade='all, delete-orphan', + ) + + def __repr__(self): + return f'' + + @staticmethod + def preprocess_id(id_value: str) -> str: + """ + Preprocess the id to remove 'N:dataset:' if it exists. + + Parameters + ---------- + id_value : str + The original id value. + + Returns + ------- + str + The preprocessed id value. + """ + if isinstance(id_value, str): + if id_value.startswith('N:dataset:'): + return id_value.replace('N:dataset:', '') + # if isinstance(id_value, str): + # id_value = uuid.UUID(id_value) + return id_value + + @validates('id') + def validate_id(self, key: str, id_value: str) -> str: + """ + Validate and preprocess the id before storing it. + + Parameters + ---------- + key : str + The key being validated. + id_value : str + The original id value. + + Returns + ------- + str + The preprocessed id value. + """ + return self.preprocess_id(id_value) + + +# class ObjectsInternal(Base): +# __tablename__ = "objects_internal" +# __table_args__ = ( +# CheckConstraint( +# "type <> 'path-metadata'::oi_type OR updated_transitive IS NOT NULL AND dataset IS NOT NULL", +# name="constraint_objects_internal_type_updated_transitive", +# ), +# ForeignKeyConstraint( +# ["dataset"], ["objects.id"], name="constraint_oi_dataset_fk" +# ), +# PrimaryKeyConstraint("id", name="objects_internal_pkey"), +# UniqueConstraint( +# "dataset", +# "updated_transitive", +# name="objects_internal_dataset_updated_transitive_key", +# ), +# ) + +# id = mapped_column(Uuid, server_default=text("gen_random_uuid()")) +# type = mapped_column( +# Enum("path-metadata", "other", name="oi_type"), +# server_default=text("'other'::oi_type"), +# ) +# dataset = mapped_column(Uuid) +# updated_transitive = mapped_column(DateTime) +# label = mapped_column(Text) +# curator_note = mapped_column(Text) + +# BUG: cycles! +# objects: Mapped[List["Objects"]] = relationship( +# "Objects", +# uselist=True, +# foreign_keys="[Objects.id_internal]", +# back_populates="objects_internal", +# ) +# objects_: Mapped[Optional["Objects"]] = relationship( +# "Objects", foreign_keys=[dataset], back_populates="objects_internal_" +# ) + + +class Units(Base): + __tablename__ = 'units' + __table_args__ = ( + PrimaryKeyConstraint('id', name='units_pkey'), + UniqueConstraint('iri', name='units_iri_key'), + UniqueConstraint('label', name='units_label_key'), + ) + + id = mapped_column( + Integer, + Identity( + start=1, + increment=1, + minvalue=1, + maxvalue=2147483647, + cycle=False, + cache=1, + ), + primary_key=True, + ) + label = mapped_column(Text, nullable=False) + iri = mapped_column(Text, nullable=False) + + descriptors_quant: Mapped[List['DescriptorsQuant']] = relationship( + 'DescriptorsQuant', uselist=True, back_populates='units' + ) + + +t_aspect_parent = Table( + 'aspect_parent', + metadata, + Column('id', Integer, nullable=False), + Column('parent', Integer, nullable=False), + ForeignKeyConstraint(['id'], ['aspects.id'], name='aspect_parent_id_fkey'), + ForeignKeyConstraint(['parent'], ['aspects.id'], name='aspect_parent_parent_fkey'), + PrimaryKeyConstraint('id', 'parent', name='aspect_parent_pkey'), +) + + +t_class_parent = Table( + 'class_parent', + metadata, + Column('id', Integer, nullable=False), + Column('parent', Integer, nullable=False), + ForeignKeyConstraint(['id'], ['descriptors_inst.id'], name='class_parent_id_fkey'), + ForeignKeyConstraint(['parent'], ['descriptors_inst.id'], name='class_parent_parent_fkey'), + PrimaryKeyConstraint('id', 'parent', name='class_parent_pkey'), +) + + +t_dataset_object = Table( + 'dataset_object', + metadata, + Column('dataset', Uuid, nullable=False), + Column('object', Uuid, nullable=False), + ForeignKeyConstraint(['dataset'], ['objects.id'], name='dataset_object_dataset_fkey'), + ForeignKeyConstraint(['object'], ['objects.id'], name='dataset_object_object_fkey'), + PrimaryKeyConstraint('dataset', 'object', name='dataset_object_pkey'), +) + + +class DescriptorsCat(Base): + __tablename__ = 'descriptors_cat' + __table_args__ = ( + ForeignKeyConstraint( + ['domain'], + ['descriptors_inst.id'], + name='descriptors_cat_domain_fkey', + ), + PrimaryKeyConstraint('id', name='descriptors_cat_pkey'), + UniqueConstraint( + 'domain', + 'range', + 'label', + name='descriptors_cat_domain_range_label_key', + ), + ) + + id = mapped_column( + Integer, + Identity( + start=1, + increment=1, + minvalue=1, + maxvalue=2147483647, + cycle=False, + cache=1, + ), + ) + domain = mapped_column(Integer) + range = mapped_column(Enum('open', 'controlled', name='cat_range_type')) + label = mapped_column(Text) + description = mapped_column(Text) + curator_note = mapped_column(Text) + + descriptors_inst: Mapped[Optional['DescriptorsInst']] = relationship( + 'DescriptorsInst', back_populates='descriptors_cat' + ) + obj_desc_cat: Mapped[List['ObjDescCat']] = relationship( + 'ObjDescCat', uselist=True, back_populates='descriptors_cat' + ) + values_cat: Mapped[List['ValuesCat']] = relationship('ValuesCat', uselist=True, back_populates='descriptors_cat') + + +class DescriptorsQuant(Base): + """ + DescriptorsQuant model representing quantitative descriptors. + Attributes + ---------- + id : int + Primary key of the descriptor. + shape : str + Shape of the descriptor, default is 'scalar'. + label : str + Label of the descriptor. + aggregation_type : str + Type of aggregation, default is 'instance'. + unit : int + Foreign key referencing the unit. + aspect : int + Foreign key referencing the aspect. + domain : int + Foreign key referencing the domain. + description : str + Description of the descriptor. + curator_note : str + Notes from the curator. + Relationships + ------------- + aspects : Optional[Aspects] + Relationship to the Aspects model. + descriptors_inst : Optional[DescriptorsInst] + Relationship to the DescriptorsInst model. + units : Optional[Units] + Relationship to the Units model. + obj_desc_quant : List[ObjDescQuant] + Relationship to the ObjDescQuant model. + values_quant : List[ValuesQuant] + Relationship to the ValuesQuant model. + """ + + __tablename__ = 'descriptors_quant' + __table_args__ = ( + ForeignKeyConstraint(['aspect'], ['aspects.id'], name='descriptors_quant_aspect_fkey'), + ForeignKeyConstraint( + ['domain'], + ['descriptors_inst.id'], + name='descriptors_quant_domain_fkey', + ), + ForeignKeyConstraint(['unit'], ['units.id'], name='descriptors_quant_unit_fkey'), + PrimaryKeyConstraint('id', name='descriptors_quant_pkey'), + UniqueConstraint('label', name='descriptors_quant_label_key'), + UniqueConstraint( + 'unit', + 'aspect', + 'domain', + 'shape', + 'aggregation_type', + name='descriptors_quant_unit_aspect_domain_shape_aggregation_type_key', + ), + ) + + id = mapped_column( + Integer, + Identity( + start=1, + increment=1, + minvalue=1, + maxvalue=2147483647, + cycle=False, + cache=1, + ), + ) + shape = mapped_column( + Enum('scalar', name='quant_shape'), + nullable=False, + server_default=text("'scalar'::quant_shape"), + ) + label = mapped_column(Text, nullable=False) + aggregation_type = mapped_column( + Enum( + 'instance', + 'function', + 'summary', + 'mean', + 'media', + 'mode', + 'sum', + 'min', + 'max', + name='quant_agg_type', + ), + nullable=False, + server_default=text("'instance'::quant_agg_type"), + ) + unit = mapped_column(Integer) + aspect = mapped_column(Integer, ForeignKey('aspects.id')) + domain = mapped_column(Integer) + description = mapped_column(Text) + curator_note = mapped_column(Text) + + aspects: Mapped[Optional['Aspects']] = relationship( + 'Aspects', + back_populates='descriptors_quant', + # viewonly=True, + # cascade="all, delete-orphan", + ) + descriptors_inst: Mapped[Optional['DescriptorsInst']] = relationship( + 'DescriptorsInst', + back_populates='descriptors_quant', + # viewonly=True, + # cascade="all, delete-orphan", + ) + units: Mapped[Optional['Units']] = relationship('Units', back_populates='descriptors_quant') + obj_desc_quant: Mapped[List['ObjDescQuant']] = relationship( + 'ObjDescQuant', + uselist=True, + back_populates='descriptors_quant', + # viewonly=True, + # cascade="all, delete-orphan", + ) + values_quant: Mapped[List['ValuesQuant']] = relationship( + 'ValuesQuant', uselist=True, back_populates='descriptors_quant' + ) + + +class ObjDescInst(Base): + __tablename__ = 'obj_desc_inst' + __table_args__ = ( + ForeignKeyConstraint( + ['addr_desc_inst'], + ['addresses.id'], + name='obj_desc_inst_addr_desc_inst_fkey', + ), + ForeignKeyConstraint( + ['addr_field'], + ['addresses.id'], + name='obj_desc_inst_addr_field_fkey', + ), + ForeignKeyConstraint( + ['desc_inst'], + ['descriptors_inst.id'], + name='obj_desc_inst_desc_inst_fkey', + ), + ForeignKeyConstraint(['object'], ['objects.id'], name='obj_desc_inst_object_fkey'), + PrimaryKeyConstraint('object', 'desc_inst', name='obj_desc_inst_pkey'), + ) + + object = mapped_column(Uuid, nullable=False) + desc_inst = mapped_column(Integer, nullable=False) + addr_field = mapped_column(Integer, nullable=False) + addr_desc_inst = mapped_column(Integer) + expect = mapped_column(Integer) + + addresses: Mapped[Optional['Addresses']] = relationship( + 'Addresses', + foreign_keys=[addr_desc_inst], + back_populates='obj_desc_inst', + viewonly=True, + cascade='all, delete-orphan', + ) + addresses_field: Mapped['Addresses'] = relationship( + 'Addresses', + foreign_keys=[addr_field], + back_populates='obj_desc_inst_', + viewonly=True, + cascade='all, delete-orphan', + ) + descriptors_inst: Mapped['DescriptorsInst'] = relationship( + 'DescriptorsInst', + back_populates='obj_desc_inst', + viewonly=True, + cascade='all, delete-orphan', + ) + objects: Mapped['Objects'] = relationship( + 'Objects', + back_populates='obj_desc_inst', + viewonly=True, + cascade='all, delete-orphan', + ) + values_cat: Mapped[List['ValuesCat']] = relationship( + 'ValuesCat', + uselist=True, + back_populates='obj_desc_inst', + viewonly=True, + cascade='all, delete-orphan', + ) + values_quant: Mapped[List['ValuesQuant']] = relationship( + 'ValuesQuant', + uselist=True, + back_populates='obj_desc_inst', + viewonly=True, + cascade='all, delete-orphan', + ) + + +class ValuesInst(Base): + """ + Represents an instance of values in the database. + + Attributes + ---------- + id : int + The primary key of the values instance. + type : str + The type of the instance, which can be 'subject', 'sample', or 'below'. + id_sub : str + The subject identifier. + desc_inst : int + The descriptor instance identifier. + dataset : UUID + The dataset identifier. + id_formal : str + The formal identifier. + local_identifier : str + The local identifier. + id_sam : str + The sample identifier. + objects : Optional[Objects] + The related objects. + descriptors_inst : Optional[DescriptorsInst] + The related descriptor instances. + values_inst : ValuesInst + The related values instances through the 'equiv_inst' table. + values_inst_ : ValuesInst + The related values instances through the 'equiv_inst' table. + values_inst1 : ValuesInst + The related values instances through the 'instance_parent' table. + values_inst2 : ValuesInst + The related values instances through the 'instance_parent' table. + values_cat : List[ValuesCat] + The related categorical values. + values_quant : List[ValuesQuant] + The related quantitative values. + + Relationships + ------------- + objects : relationship + Relationship to the Objects table. + descriptors_inst : relationship + Relationship to the DescriptorsInst table. + values_inst : relationship + Self-referential relationship through the 'equiv_inst' table (left to right). + values_inst_ : relationship + Self-referential relationship through the 'equiv_inst' table (right to left). + values_inst1 : relationship + Self-referential relationship through the 'instance_parent' table (id to parent). + values_inst2 : relationship + Self-referential relationship through the 'instance_parent' table (parent to id). + values_cat : relationship + Relationship to the ValuesCat table. + values_quant : relationship + Relationship to the ValuesQuant table. + """ + + __tablename__ = 'values_inst' + __table_args__ = ( + CheckConstraint("id_sam ~ '^sam-'::text", name='values_inst_id_sam_check'), + CheckConstraint("id_sub ~ '^sub-'::text", name='values_inst_id_sub_check'), + CheckConstraint( + "type <> 'below'::instance_type OR id_sub IS NOT NULL OR id_sam IS NOT NULL", + name='constraint_values_inst_type_below', + ), + CheckConstraint( + "type <> 'sample'::instance_type OR id_sam IS NOT NULL AND id_sam = id_formal", + name='constraint_values_inst_type_id_sam', + ), + CheckConstraint( + "type <> 'subject'::instance_type OR id_sub = id_formal AND id_sam IS NULL", + name='constraint_values_inst_type_id_sub', + ), + CheckConstraint( + "type = 'below'::instance_type AND NOT id_formal ~ '^(sub|sam)-'::text OR type = 'subject'::instance_type AND id_formal ~ '^sub-'::text OR type = 'sample'::instance_type AND id_formal ~ '^sam-'::text", + name='constraint_values_inst_type_id_formal', + ), + ForeignKeyConstraint(['dataset'], ['objects.id'], name='values_inst_dataset_fkey'), + ForeignKeyConstraint( + ['desc_inst'], + ['descriptors_inst.id'], + name='values_inst_desc_inst_fkey', + ), + PrimaryKeyConstraint('id', name='values_inst_pkey'), + UniqueConstraint('dataset', 'id_formal', name='values_inst_dataset_id_formal_key'), + Index('idx_values_inst_dataset', 'dataset'), + Index('idx_values_inst_dataset_id_formal', 'dataset', 'id_formal'), + Index('idx_values_inst_dataset_id_sam', 'dataset', 'id_sam'), + Index('idx_values_inst_dataset_id_sub', 'dataset', 'id_sub'), + Index('idx_values_inst_desc_inst', 'desc_inst'), + Index('idx_values_inst_id_formal', 'id_formal'), + Index('idx_values_inst_id_sam', 'id_sam'), + Index('idx_values_inst_id_sub', 'id_sub'), + ) + + id = mapped_column( + Integer, + Identity( + start=1, + increment=1, + minvalue=1, + maxvalue=2147483647, + cycle=False, + cache=1, + ), + ) + type = mapped_column(Enum('subject', 'sample', 'below', name='instance_type'), nullable=False) + id_sub = mapped_column(Text, nullable=False) + desc_inst = mapped_column(Integer) + dataset = mapped_column(Uuid, ForeignKey('objects.id')) + id_formal = mapped_column(Text) + local_identifier = mapped_column(Text) + id_sam = mapped_column(Text) + + objects: Mapped[Optional['Objects']] = relationship( + 'Objects', + back_populates='values_inst', + viewonly=True, + cascade='all, delete-orphan', + ) + descriptors_inst: Mapped[Optional['DescriptorsInst']] = relationship( + 'DescriptorsInst', + back_populates='values_inst', + viewonly=True, + # cascade="all, delete-orphan", + # single_parent=True, + ) + # values_inst: Mapped["ValuesInst"] = relationship( + # "ValuesInst", + # secondary="equiv_inst", + # primaryjoin=lambda: ValuesInst.id == t_equiv_inst.c.left_thing, + # secondaryjoin=lambda: ValuesInst.id == t_equiv_inst.c.right_thing, + # back_populates="values_inst_", + # ) + # values_inst_: Mapped["ValuesInst"] = relationship( + # "ValuesInst", + # secondary="equiv_inst", + # primaryjoin=lambda: ValuesInst.id == t_equiv_inst.c.right_thing, + # secondaryjoin=lambda: ValuesInst.id == t_equiv_inst.c.left_thing, + # back_populates="values_inst", + # ) + # values_inst1: Mapped["ValuesInst"] = relationship( + # "ValuesInst", + # secondary="instance_parent", + # primaryjoin=lambda: ValuesInst.id == t_instance_parent.c.id, + # secondaryjoin=lambda: ValuesInst.id == t_instance_parent.c.parent, + # back_populates="values_inst2", + # ) + # values_inst2: Mapped["ValuesInst"] = relationship( + # "ValuesInst", + # secondary="instance_parent", + # primaryjoin=lambda: ValuesInst.id == t_instance_parent.c.parent, + # secondaryjoin=lambda: ValuesInst.id == t_instance_parent.c.id, + # back_populates="values_inst1", + # ) + values_cat: Mapped[List['ValuesCat']] = relationship('ValuesCat', uselist=True, back_populates='values_inst') + values_quant: Mapped[List['ValuesQuant']] = relationship( + 'ValuesQuant', + uselist=True, + back_populates='values_inst', + viewonly=True, + cascade='all, delete-orphan', + ) + + +t_equiv_inst = Table( + 'equiv_inst', + metadata, + Column('left_thing', Integer, nullable=False), + Column('right_thing', Integer, nullable=False), + CheckConstraint('left_thing <> right_thing', name='sse_no_self'), + ForeignKeyConstraint(['left_thing'], ['values_inst.id'], name='equiv_inst_left_thing_fkey'), + ForeignKeyConstraint(['right_thing'], ['values_inst.id'], name='equiv_inst_right_thing_fkey'), + PrimaryKeyConstraint('left_thing', 'right_thing', name='equiv_inst_pkey'), +) + + +t_instance_parent = Table( + 'instance_parent', + metadata, + Column('id', Integer, nullable=False), + Column('parent', Integer, nullable=False), + ForeignKeyConstraint(['id'], ['values_inst.id'], name='instance_parent_id_fkey'), + ForeignKeyConstraint(['parent'], ['values_inst.id'], name='instance_parent_parent_fkey'), + PrimaryKeyConstraint('id', 'parent', name='instance_parent_pkey'), +) + + +class ObjDescCat(Base): + __tablename__ = 'obj_desc_cat' + __table_args__ = ( + ForeignKeyConstraint( + ['addr_desc_inst'], + ['addresses.id'], + name='obj_desc_cat_addr_desc_inst_fkey', + ), + ForeignKeyConstraint( + ['addr_field'], + ['addresses.id'], + name='obj_desc_cat_addr_field_fkey', + ), + ForeignKeyConstraint( + ['desc_cat'], + ['descriptors_cat.id'], + name='obj_desc_cat_desc_cat_fkey', + ), + ForeignKeyConstraint(['object'], ['objects.id'], name='obj_desc_cat_object_fkey'), + PrimaryKeyConstraint('object', 'desc_cat', name='obj_desc_cat_pkey'), + ) + + object = mapped_column(Uuid, nullable=False) + desc_cat = mapped_column(Integer, nullable=False) + addr_field = mapped_column(Integer, nullable=False) + addr_desc_inst = mapped_column(Integer) + expect = mapped_column(Integer) + + addresses: Mapped[Optional['Addresses']] = relationship( + 'Addresses', + foreign_keys=[addr_desc_inst], + back_populates='obj_desc_cat', + ) + addresses_: Mapped['Addresses'] = relationship( + 'Addresses', foreign_keys=[addr_field], back_populates='obj_desc_cat_' + ) + descriptors_cat: Mapped['DescriptorsCat'] = relationship('DescriptorsCat', back_populates='obj_desc_cat') + objects: Mapped['Objects'] = relationship('Objects', back_populates='obj_desc_cat') + values_cat: Mapped[List['ValuesCat']] = relationship('ValuesCat', uselist=True, back_populates='obj_desc_cat') + + +class ObjDescQuant(Base): + __tablename__ = 'obj_desc_quant' + __table_args__ = ( + ForeignKeyConstraint( + ['addr_aspect'], + ['addresses.id'], + name='obj_desc_quant_addr_aspect_fkey', + ), + ForeignKeyConstraint( + ['addr_desc_inst'], + ['addresses.id'], + name='obj_desc_quant_addr_desc_inst_fkey', + ), + ForeignKeyConstraint( + ['addr_field'], + ['addresses.id'], + name='obj_desc_quant_addr_field_fkey', + ), + ForeignKeyConstraint( + ['addr_unit'], + ['addresses.id'], + name='obj_desc_quant_addr_unit_fkey', + ), + ForeignKeyConstraint( + ['desc_quant'], + ['descriptors_quant.id'], + name='obj_desc_quant_desc_quant_fkey', + ), + ForeignKeyConstraint(['object'], ['objects.id'], name='obj_desc_quant_object_fkey'), + PrimaryKeyConstraint('object', 'desc_quant', name='obj_desc_quant_pkey'), + ) + + object = mapped_column(Uuid, nullable=False) + desc_quant = mapped_column(Integer, nullable=False) + addr_field = mapped_column(Integer, nullable=False) + addr_unit = mapped_column(Integer) + addr_aspect = mapped_column(Integer) + addr_desc_inst = mapped_column(Integer) + expect = mapped_column(Integer) + + addresses: Mapped[Optional['Addresses']] = relationship( + 'Addresses', + foreign_keys=[addr_desc_inst], + back_populates='obj_desc_quant_', + viewonly=True, + cascade='all, delete-orphan', + ) + addresses_aspect: Mapped[Optional['Addresses']] = relationship( + 'Addresses', + foreign_keys=[addr_aspect], + back_populates='obj_desc_quant', + viewonly=True, + cascade='all, delete-orphan', + ) + addresses_field: Mapped['Addresses'] = relationship( + 'Addresses', + foreign_keys=[addr_field], + back_populates='obj_desc_quant1', + viewonly=True, + cascade='all, delete-orphan', + ) + addresses_unit: Mapped[Optional['Addresses']] = relationship( + 'Addresses', + foreign_keys=[addr_unit], + back_populates='obj_desc_quant2', + viewonly=True, + cascade='all, delete-orphan', + ) + descriptors_quant: Mapped['DescriptorsQuant'] = relationship( + 'DescriptorsQuant', + back_populates='obj_desc_quant', + viewonly=True, + cascade='all, delete-orphan', + ) + objects: Mapped['Objects'] = relationship( + 'Objects', + back_populates='obj_desc_quant', + viewonly=True, + cascade='all, delete-orphan', + ) + values_quant: Mapped[List['ValuesQuant']] = relationship( + 'ValuesQuant', + uselist=True, + back_populates='obj_desc_quant', + viewonly=True, + cascade='all, delete-orphan', + ) + + +class ValuesCat(Base): + __tablename__ = 'values_cat' + __table_args__ = ( + CheckConstraint( + 'value_open IS NOT NULL OR value_controlled IS NOT NULL', + name='constraint_values_cat_some_value', + ), + ForeignKeyConstraint( + ['desc_cat'], + ['descriptors_cat.id'], + name='values_cat_desc_cat_fkey', + ), + ForeignKeyConstraint( + ['desc_inst'], + ['descriptors_inst.id'], + name='values_cat_desc_inst_fkey', + ), + ForeignKeyConstraint(['instance'], ['values_inst.id'], name='values_cat_instance_fkey'), + ForeignKeyConstraint( + ['object', 'desc_cat'], + ['obj_desc_cat.object', 'obj_desc_cat.desc_cat'], + name='values_cat_object_desc_cat_fkey', + ), + ForeignKeyConstraint( + ['object', 'desc_inst'], + ['obj_desc_inst.object', 'obj_desc_inst.desc_inst'], + name='values_cat_object_desc_inst_fkey', + ), + ForeignKeyConstraint(['object'], ['objects.id'], name='values_cat_object_fkey'), + ForeignKeyConstraint( + ['value_controlled'], + ['controlled_terms.id'], + name='values_cat_value_controlled_fkey', + ), + PrimaryKeyConstraint('id', name='values_cat_pkey'), + Index('idx_values_cat_desc_cat', 'desc_cat'), + Index('idx_values_cat_desc_inst', 'desc_inst'), + Index('idx_values_cat_instance', 'instance'), + Index('idx_values_cat_object', 'object'), + Index('idx_values_cat_value_controlled', 'value_controlled'), + ) + + id = mapped_column( + Integer, + Identity( + start=1, + increment=1, + minvalue=1, + maxvalue=2147483647, + cycle=False, + cache=1, + ), + ) + object = mapped_column(Uuid, nullable=False) + desc_inst = mapped_column(Integer, nullable=False) + desc_cat = mapped_column(Integer, nullable=False) + value_open = mapped_column(Text) + value_controlled = mapped_column(Integer) + instance = mapped_column(Integer) + + descriptors_cat: Mapped['DescriptorsCat'] = relationship('DescriptorsCat', back_populates='values_cat') + descriptors_inst: Mapped['DescriptorsInst'] = relationship('DescriptorsInst', back_populates='values_cat') + values_inst: Mapped[Optional['ValuesInst']] = relationship('ValuesInst', back_populates='values_cat') + obj_desc_cat: Mapped['ObjDescCat'] = relationship('ObjDescCat', back_populates='values_cat') + obj_desc_inst: Mapped['ObjDescInst'] = relationship('ObjDescInst', back_populates='values_cat') + objects: Mapped['Objects'] = relationship('Objects', back_populates='values_cat') + controlled_terms: Mapped[Optional['ControlledTerms']] = relationship('ControlledTerms', back_populates='values_cat') + + +class ValuesQuant(Base): + __tablename__ = 'values_quant' + __table_args__ = ( + ForeignKeyConstraint( + ['desc_inst'], + ['descriptors_inst.id'], + name='values_quant_desc_inst_fkey', + ), + ForeignKeyConstraint( + ['desc_quant'], + ['descriptors_quant.id'], + name='values_quant_desc_quant_fkey', + ), + ForeignKeyConstraint(['instance'], ['values_inst.id'], name='values_quant_instance_fkey'), + ForeignKeyConstraint( + ['object', 'desc_inst'], + ['obj_desc_inst.object', 'obj_desc_inst.desc_inst'], + name='values_quant_object_desc_inst_fkey', + ), + ForeignKeyConstraint( + ['object', 'desc_quant'], + ['obj_desc_quant.object', 'obj_desc_quant.desc_quant'], + name='values_quant_object_desc_quant_fkey', + ), + ForeignKeyConstraint(['object'], ['objects.id'], name='values_quant_object_fkey'), + PrimaryKeyConstraint('id', name='values_quant_pkey'), + Index('idx_values_quant_desc_inst', 'desc_inst'), + Index('idx_values_quant_desc_quant', 'desc_quant'), + Index('idx_values_quant_instance', 'instance'), + Index('idx_values_quant_object', 'object'), + ) + + id = mapped_column( + Integer, + Identity( + start=1, + increment=1, + minvalue=1, + maxvalue=2147483647, + cycle=False, + cache=1, + ), + ) + value = mapped_column(Numeric, nullable=False) + object = mapped_column( + Uuid, ForeignKey('objects.id'), nullable=False + ) # TODO: object is not obj id, but File Obj ID that | should be same + desc_inst = mapped_column(Integer, nullable=False) + desc_quant = mapped_column(Integer, nullable=False) + value_blob = mapped_column(JSONB, nullable=False) + instance = mapped_column(Integer) + orig_value = mapped_column(String) + orig_units = mapped_column(String) + + descriptors_inst: Mapped['DescriptorsInst'] = relationship( + 'DescriptorsInst', + back_populates='values_quant', + viewonly=True, + cascade='all, delete-orphan', + ) + descriptors_quant: Mapped['DescriptorsQuant'] = relationship( + 'DescriptorsQuant', + back_populates='values_quant', + viewonly=True, + cascade='all, delete-orphan', + ) + values_inst: Mapped[Optional['ValuesInst']] = relationship( + 'ValuesInst', + back_populates='values_quant', + viewonly=True, + cascade='all, delete-orphan', + ) + obj_desc_inst: Mapped['ObjDescInst'] = relationship( + 'ObjDescInst', + back_populates='values_quant', + viewonly=True, + cascade='all, delete-orphan', + # sync_backref=False, + ) + obj_desc_quant: Mapped['ObjDescQuant'] = relationship( + 'ObjDescQuant', + back_populates='values_quant', + viewonly=True, + cascade='all, delete-orphan', + # sync_backref=False, + ) + objects: Mapped['Objects'] = relationship( + 'Objects', + back_populates='values_quant', + viewonly=True, + cascade='all, delete-orphan', + ) + + @staticmethod + def update_desc_inst(target: 'ValuesQuant', value, oldvalue, initiator) -> None: + """ + Automatically update desc_inst when descriptors_inst parent is updated. + """ + if target.descriptors_inst: + target.desc_inst = target.descriptors_inst.id + + @staticmethod + def register_listeners() -> None: + """ + Register event listeners for the ValuesQuant model. + """ + event.listen(ValuesQuant.descriptors_inst, 'set', ValuesQuant.update_desc_inst) + + +# @event.listens_for(ValuesQuant.descriptors_inst, "set") +# def set_desc_inst_from_descriptor(target, value, oldvalue, initiator): +# print(target, value, oldvalue, initiator) +# if value is not None: +# target.desc_inst = value.id + + +def update_all_children(target, value, oldvalue, initiator): + # Loop through all relationships of the target (parent) object + for relationship_prop in target.__mapper__.relationships: + related_objects = getattr(target, relationship_prop.key) + + # If there are related objects, update their attributes + if related_objects is not None: + # Handle single and multiple relationships + related_objects = related_objects if isinstance(related_objects, list) else [related_objects] + for related_obj in related_objects: + # Update attributes on related objects as needed + # For demonstration, we'll update an attribute based on parent_value + if hasattr(related_obj, 'child_value'): + related_obj.child_value = f'Updated from parent_value: {value}' + if hasattr(related_obj, 'other_child_value'): + related_obj.other_child_value = f'Updated from parent_value: {value}' + + +# # Attach the listener to all attributes on the Parent model +# for column in ValuesQuant.__table__.columns: +# event.listen(getattr(ValuesQuant, column.name), "set", update_all_children) + + +# @event.listens_for(ValuesQuant.descriptors_inst, "set") +# def update_desc_inst(target, value, oldvalue, initiator): +# if value is not None: +# target.desc_inst = value.id + + +# Helper function to find the foreign key field associated with a relationship +def get_foreign_key_field(model, relationship_key): + """Get the local foreign key field name for a given relationship key in the model.""" + for constraint in model.__table__.constraints: + if isinstance(constraint, ForeignKeyConstraint): + for element in constraint.elements: + # Match the local column to the relationship key + if element.column.table.name == relationship_key: + return element.parent.name + return None + + +# General listener function for updating foreign keys dynamically +def update_foreign_key(target, value, oldvalue, initiator): + if value is not None: + # Use the initiator key to find the appropriate foreign key field + relationship_key = initiator.key # Use initiator.key to get the relationship name + foreign_key_field = get_foreign_key_field(target.__class__, relationship_key) + + # Set the foreign key field if it exists on the target + if foreign_key_field and hasattr(target, foreign_key_field): + setattr(target, foreign_key_field, value.id) + + +# Function to attach the listener to all relationships in a given model +def add_dynamic_listeners(base): + for cls in base.__subclasses__(): + for relationship_prop in cls.__mapper__.relationships: + event.listen(getattr(cls, relationship_prop.key), 'set', update_foreign_key) + + +# Attach listeners to all models in the Base +add_dynamic_listeners(Base) diff --git a/quantdb/router.py b/quantdb/router.py new file mode 100644 index 0000000..71b0c03 --- /dev/null +++ b/quantdb/router.py @@ -0,0 +1,21 @@ +from typing import Literal + +import uvicorn +from fastapi import FastAPI +from fastapi.middleware.wsgi import WSGIMiddleware +from fastapi.staticfiles import StaticFiles + +from quantdb.api_server import app as quantdb_flask_app + +app = FastAPI() +app.mount('/quantdb', WSGIMiddleware(quantdb_flask_app)) + + +# Root URL +@app.get('/') +def index() -> Literal['Hello']: + return 'Hello' + + +if __name__ == '__main__': + uvicorn.run('router:app', host='localhost', port=8000, reload=True) diff --git a/quantdb/utils.py b/quantdb/utils.py index 35069ec..3097575 100644 --- a/quantdb/utils.py +++ b/quantdb/utils.py @@ -27,17 +27,21 @@ def makeSimpleLogger(name, level=logging.INFO): def setPS1(script__file__): try: text = 'Running ' + os.path.basename(script__file__) - os.sys.stdout.write('\x1b]2;{}\x07\n'.format(text)) + sys.stdout.write('\x1b]2;{}\x07\n'.format(text)) except AttributeError as e: log.exception(e) -def dbUri(dbuser, host, port, database): +def dbUri(dbuser, host, port, database, password=None): if hasattr(sys, 'pypy_version_info'): dialect = 'psycopg2cffi' else: dialect = 'psycopg2' - return f'postgresql+{dialect}://{dbuser}@{host}:{port}/{database}' + + if password: + return f'postgresql+{dialect}://{dbuser}:{password}@{host}:{port}/{database}' + else: + return f'postgresql+{dialect}://{dbuser}@{host}:{port}/{database}' # from pyontutils.utils_fast import isoformat diff --git a/sql/database.sql b/sql/database.sql index 7f1634a..4d7608a 100644 --- a/sql/database.sql +++ b/sql/database.sql @@ -10,8 +10,10 @@ CREATE DATABASE :database -- quantdb WITH OWNER = 'quantdb-admin' ENCODING = 'UTF8' --TABLESPACE = pg_default -- leave this out for now since rds doesn't really support it - LC_COLLATE = 'en_US.UTF-8' -- this was a gentoo locale issue check ${LANG} - LC_CTYPE = 'en_US.UTF-8' + -- Use template0 to avoid collation conflicts, or inherit from template1 + TEMPLATE = template0 + LC_COLLATE = 'C' -- Use C collation for maximum compatibility + LC_CTYPE = 'C' CONNECTION LIMIT = -1; REVOKE "quantdb-admin" FROM CURRENT_USER; diff --git a/sql/tables.sql b/sql/tables.sql index b532801..58e4e1d 100644 --- a/sql/tables.sql +++ b/sql/tables.sql @@ -956,7 +956,7 @@ BEGIN RETURN QUERY WITH RECURSIVE tree(child, parents) AS ( SELECT parent, ARRAY[]::integer[] FROM -- start from nodes with no parents -(SELECT ap0.parent FROM aspect_parent AS ap0 EXCEPT SELECT ap1.id FROM aspect_parent AS ap1) +(SELECT ap0.parent FROM aspect_parent AS ap0 EXCEPT SELECT ap1.id FROM aspect_parent AS ap1) AS root_nodes UNION ALL SELECT ap.id, ap.parent || t.parents FROM aspect_parent AS ap @@ -1075,7 +1075,7 @@ BEGIN RETURN QUERY WITH RECURSIVE tree(child, parents) AS ( SELECT parent, ARRAY[]::integer[] FROM -- start from nodes with no parents -(SELECT cp0.parent FROM class_parent AS cp0 EXCEPT SELECT cp1.id FROM class_parent AS cp1) +(SELECT cp0.parent FROM class_parent AS cp0 EXCEPT SELECT cp1.id FROM class_parent AS cp1) AS root_nodes UNION ALL SELECT cp.id, cp.parent || t.parents FROM class_parent AS cp @@ -1170,7 +1170,7 @@ BEGIN RETURN QUERY WITH RECURSIVE tree(child, parents) AS ( SELECT parent, ARRAY[]::integer[] FROM -- start from nodes with no parents -(SELECT ip0.parent FROM instance_parent AS ip0 EXCEPT SELECT ip1.id FROM instance_parent AS ip1) +(SELECT ip0.parent FROM instance_parent AS ip0 EXCEPT SELECT ip1.id FROM instance_parent AS ip1) AS root_nodes UNION ALL SELECT ip.id, ip.parent || t.parents FROM instance_parent AS ip @@ -1205,8 +1205,8 @@ WHERE im.id = NEW.id) SELECT -- FIXME surely there is a more efficient way to do this (((SELECT count(*) FROM samples) > 0 OR (SELECT count(*) from subjects) > 0) AND (SELECT count(*) FROM parents) = 0) AS one, ( (SELECT count(*) FROM samples) = 0 AND (SELECT count(*) from subjects) = 0 AND (SELECT count(*) FROM parents) > 0) AS two, -( (SELECT count(*) FROM (SELECT * FROM parents INTERSECT SELECT * FROM subjects)) = (SELECT count(*) FROM subjects) -AND (SELECT count(*) FROM (SELECT * FROM parents INTERSECT SELECT * FROM samples)) = (SELECT count(*) FROM samples)) AS three +( (SELECT count(*) FROM (SELECT * FROM parents INTERSECT SELECT * FROM subjects) AS parent_subject_intersect) = (SELECT count(*) FROM subjects) +AND (SELECT count(*) FROM (SELECT * FROM parents INTERSECT SELECT * FROM samples) AS parent_sample_intersect) = (SELECT count(*) FROM samples)) AS three /* -- we cannot check against parents in this last case because it can contain intermediates therefore we can only check parents contain all referenced subjects and samples, going the other way around we know we will be missing any intervening samples, but those will have been checked when they were inserted AND (SELECT count(*) FROM (SELECT * FROM parents INTERSECT SELECT * FROM (SELECT * FROM subjects UNION SELECT * FROM samples))) = (SELECT count(*) FROM parents)) diff --git a/sql/test_database.sql b/sql/test_database.sql index 1bb2fc3..dbb1cae 100644 --- a/sql/test_database.sql +++ b/sql/test_database.sql @@ -14,8 +14,10 @@ CREATE DATABASE :test_database -- quantdb WITH OWNER = 'quantdb-test-admin' ENCODING = 'UTF8' --TABLESPACE = pg_default -- leave this out for now since rds doesn't really support it - LC_COLLATE = 'en_US.UTF-8' -- this was a gentoo locale issue check ${LANG} - LC_CTYPE = 'en_US.UTF-8' + -- Use template0 to avoid collation conflicts, or inherit from template1 + TEMPLATE = template0 + LC_COLLATE = 'C' -- Use C collation for maximum compatibility + LC_CTYPE = 'C' CONNECTION LIMIT = -1; REVOKE "quantdb-test-admin" FROM CURRENT_USER; diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..e3a6802 --- /dev/null +++ b/test/README.md @@ -0,0 +1,85 @@ +# Test Database Setup + +This directory contains pytest setup for creating and verifying a local test PostgreSQL database using the `dbsetup` script. + +## Purpose + +This test setup is focused **solely** on: +1. Creating a test PostgreSQL database using the `bin/dbsetup` script +2. Verifying that the expected tables were created +3. Ensuring basic database connectivity works + +**No other functionality is tested** - this is just for basic database setup verification. + +## Prerequisites + +1. **PostgreSQL server** must be running locally on port 5432 (or configured port) +2. **postgres superuser** must be available for database creation +3. **Python dependencies** must be installed (see `pyproject.toml`) + +## Configuration + +The test database configuration is **hardcoded** to always use a local PostgreSQL instance: +- Database: `quantdb_test` (can be overridden by `QUANTDB_TEST_DB_DATABASE`) +- Host: `localhost` (**ALWAYS localhost for testing - ignores all other config**) +- Port: `5432` (**ALWAYS 5432 for testing**) +- User: `quantdb-test-admin` (**ALWAYS this user for testing**) +- Password: `tom-is-cool` (**hardcoded for test database**) + +**Important**: Testing will NEVER use remote databases (like AWS RDS). All test database connections are forced to use localhost regardless of environment variables or configuration files. + +**Note**: Tables are created in the `quantdb` schema, not the `public` schema. The database users are configured with `search_path = quantdb, public` which makes `quantdb` the default schema for table creation. + +## Running Tests + +### Run all database setup tests: +```bash +pytest test/test_database_setup.py -v +``` + +### Run with more detailed output: +```bash +pytest test/test_database_setup.py -v -s +``` + +### Run only the table creation test: +```bash +pytest test/test_database_setup.py::TestDatabaseSetup::test_tables_created -v +``` + +## What the Tests Do + +1. **`test_database_connection`**: Verifies basic database connectivity +2. **`test_tables_created`**: Checks that core tables (objects, units, aspects, etc.) exist in the `quantdb` schema +3. **`test_database_schema_info`**: Queries schema information from both `quantdb` and `public` schemas +4. **`test_database_functions_created`**: Verifies custom functions were created in the `quantdb` schema +5. **`test_database_enums_created`**: Checks that custom enum types exist + +## Test Flow + +1. The `setup_test_database` fixture runs the `bin/dbsetup` script +2. The `test_session` fixture creates a database connection +3. The `verify_database_tables` fixture queries and lists all tables +4. Individual tests verify different aspects of the database setup + +## Troubleshooting + +- **Connection errors**: Check that PostgreSQL is running and accessible +- **Permission errors**: Ensure the postgres user has proper permissions +- **Script errors**: Check that `bin/dbsetup` is executable and all SQL files exist +- **Timeout errors**: Database creation is taking longer than 2 minutes + +## Expected Tables + +The tests verify these core tables exist: +- `objects` - Main object table +- `units` - Unit definitions +- `aspects` - Aspect definitions +- `descriptors_inst` - Instance descriptors +- `descriptors_cat` - Categorical descriptors +- `descriptors_quant` - Quantitative descriptors +- `values_inst` - Instance values +- `values_cat` - Categorical values +- `values_quant` - Quantitative values +- `addresses` - Address mappings +- `controlled_terms` - Controlled vocabulary diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..1535124 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,143 @@ +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from sqlalchemy import create_engine, text +from sqlalchemy.exc import OperationalError + +from quantdb.client import get_session +from quantdb.config import auth + + +@pytest.fixture(scope='session') +def project_root(): + """Get the project root directory.""" + return Path(__file__).parent.parent + + +@pytest.fixture(scope='session') +def test_database_config(): + """Get test database configuration.""" + config = { + 'user': 'quantdb-test-admin', + 'host': 'localhost', # ALWAYS use localhost for testing, never remote + 'port': 5432, + 'database': auth.get('test-db-database') or 'quantdb_test', + } + return config + + +@pytest.fixture(scope='session') +def setup_test_database(project_root, test_database_config): + """ + Set up the test database using the dbsetup script. + This fixture runs once per test session. + """ + print(f'\n=== Setting up test database ===') + print(f"Database: {test_database_config['database']}") + print(f"Host: {test_database_config['host']}") + print(f"Port: {test_database_config['port']}") + print(f"User: {test_database_config['user']}") + + # Path to dbsetup script + dbsetup_script = project_root / 'bin' / 'dbsetup' + + if not dbsetup_script.exists(): + pytest.fail(f'dbsetup script not found at: {dbsetup_script}') + + # Make sure the script is executable + os.chmod(dbsetup_script, 0o755) + + try: + # Set up environment with hardcoded test password + test_env = os.environ.copy() + test_env['PGPASSWORD'] = 'tom-is-cool' # Hardcoded test password + + # Run dbsetup with test database parameters + cmd = [ + str(dbsetup_script), + str(test_database_config['port']), + test_database_config['database'], + test_database_config['host'], + ] + + print(f"Running command: {' '.join(cmd)}") + print(f'Using hardcoded test password: tom-is-cool') + + # Run the dbsetup script with the password environment variable + result = subprocess.run( + cmd, cwd=project_root, capture_output=True, text=True, timeout=120, env=test_env + ) # 2 minute timeout + + if result.returncode != 0: + print(f'STDOUT: {result.stdout}') + print(f'STDERR: {result.stderr}') + pytest.fail(f'dbsetup failed with return code {result.returncode}') + + print(f'dbsetup completed successfully') + print(f'STDOUT: {result.stdout}') + + # Return the configuration for use by other fixtures + return test_database_config + + except subprocess.TimeoutExpired: + pytest.fail('dbsetup script timed out after 2 minutes') + except Exception as e: + pytest.fail(f'Failed to run dbsetup: {e}') + + +@pytest.fixture(scope='session') +def test_session(setup_test_database): + """ + Get a SQLAlchemy session connected to the test database. + Depends on setup_test_database to ensure the database is created first. + """ + try: + session = get_session(echo=False, test=True) + yield session + session.close() + except Exception as e: + pytest.fail(f'Failed to create test database session: {e}') + + +@pytest.fixture(scope='session') +def verify_database_tables(test_session): + """ + Verify that the expected database tables were created. + Returns a list of table names found in the database. + """ + try: + # Query for all tables in the quantdb and public schemas + result = test_session.execute( + text( + """ + SELECT table_schema, table_name + FROM information_schema.tables + WHERE table_schema IN ('quantdb', 'public') + ORDER BY table_schema, table_name + """ + ) + ) + + rows = result.fetchall() + tables_by_schema = {} + all_tables = [] + + for schema, table in rows: + if schema not in tables_by_schema: + tables_by_schema[schema] = [] + tables_by_schema[schema].append(table) + all_tables.append(table) + + print(f'\n=== Found {len(all_tables)} tables in test database ===') + for schema, tables in tables_by_schema.items(): + print(f"Schema '{schema}': {len(tables)} tables") + for table in tables: + print(f' - {table}') + + return all_tables + + except Exception as e: + pytest.fail(f'Failed to query database tables: {e}') diff --git a/test/pytest.ini b/test/pytest.ini new file mode 100644 index 0000000..2ee4565 --- /dev/null +++ b/test/pytest.ini @@ -0,0 +1,13 @@ +[tool:pytest] +testpaths = test +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = + -v + --tb=short + --strict-markers + --disable-warnings +markers = + database: marks tests as database tests (deselect with '-m "not database"') + slow: marks tests as slow (deselect with '-m "not slow"') diff --git a/test/test_database_setup.py b/test/test_database_setup.py new file mode 100644 index 0000000..13a2b93 --- /dev/null +++ b/test/test_database_setup.py @@ -0,0 +1,111 @@ +""" +Test database setup and table creation. + +This test module focuses solely on verifying that: +1. The test database can be created using the dbsetup script +2. The expected tables are present in the database +3. The database connection works properly + +No other functionality is tested - this is just for basic database setup verification. +""" + +import pytest +from sqlalchemy import text + + +class TestDatabaseSetup: + """Test the basic database setup and table creation.""" + + def test_database_connection(self, test_session): + """Test that we can connect to the test database.""" + # Try a simple query to verify the connection works + result = test_session.execute(text('SELECT 1 as test_value')) + row = result.fetchone() + assert row[0] == 1, 'Basic database connection failed' + + def test_tables_created(self, verify_database_tables): + """Test that the expected core tables were created.""" + tables = verify_database_tables + + # Verify we have tables (the list should not be empty) + assert len(tables) > 0, 'No tables found in the database' + + # Check for some core tables that should exist based on sql/tables.sql + expected_core_tables = [ + 'objects', + 'units', + 'aspects', + 'descriptors_inst', + 'descriptors_cat', + 'descriptors_quant', + 'values_inst', + 'values_cat', + 'values_quant', + 'addresses', + 'controlled_terms', + ] + + missing_tables = [] + for expected_table in expected_core_tables: + if expected_table not in tables: + missing_tables.append(expected_table) + + assert len(missing_tables) == 0, f'Missing expected tables: {missing_tables}' + + print(f'✓ All {len(expected_core_tables)} core tables found in database') + + def test_database_schema_info(self, test_session): + """Test that we can query basic schema information.""" + # Check that we can query table information in both schemas + result = test_session.execute( + text( + """ + SELECT COUNT(*) as table_count + FROM information_schema.tables + WHERE table_schema IN ('quantdb', 'public') + """ + ) + ) + + table_count = result.fetchone()[0] + assert table_count > 0, 'No tables found in quantdb or public schemas' + + print(f'✓ Found {table_count} tables in quantdb and public schemas') + + def test_database_functions_created(self, test_session): + """Test that database functions were created.""" + # Check for some of the custom functions from tables.sql in both schemas + result = test_session.execute( + text( + """ + SELECT COUNT(*) as function_count + FROM information_schema.routines + WHERE routine_schema IN ('quantdb', 'public') + AND routine_type = 'FUNCTION' + """ + ) + ) + + function_count = result.fetchone()[0] + # We expect several functions to be created based on tables.sql + assert function_count > 0, 'No custom functions found in database' + + print(f'✓ Found {function_count} custom functions in database') + + def test_database_enums_created(self, test_session): + """Test that custom enum types were created.""" + result = test_session.execute( + text( + """ + SELECT COUNT(*) as enum_count + FROM pg_type + WHERE typtype = 'e' + """ + ) + ) + + enum_count = result.fetchone()[0] + # We expect several enum types based on tables.sql (remote_id_type, instance_type, etc.) + assert enum_count > 0, 'No custom enum types found in database' + + print(f'✓ Found {enum_count} custom enum types in database') diff --git a/test/test_generic_ingest.py b/test/test_generic_ingest.py new file mode 100644 index 0000000..011402f --- /dev/null +++ b/test/test_generic_ingest.py @@ -0,0 +1,84 @@ +import pytest + +from quantdb.generic_ingest import ( + back_populate_tables, + get_constraint_columns, + get_or_create, + object_as_dict, + query_by_constraints, +) +from quantdb.models import Objects + + +def test_object_as_dict(): + obj = Objects(id='00000000-0000-0000-0000-000000000001', id_type='dataset', id_file=None, id_internal=None) + d = object_as_dict(obj) + assert d['id'] == '00000000-0000-0000-0000-000000000001' + assert d['id_type'] == 'dataset' + + +def test_get_or_create_creates_and_gets(test_session): + obj = Objects(id='00000000-0000-0000-0000-000000000002', id_type='dataset', id_file=None, id_internal=None) + instance = get_or_create(test_session, obj) + assert getattr(instance, 'id', None) == '00000000-0000-0000-0000-000000000002' + # Should return the same instance if called again + instance2 = get_or_create(test_session, obj) + assert getattr(instance, 'id', None) == getattr(instance2, 'id', None) + + +def test_get_constraint_columns(): + cols = get_constraint_columns(Objects) + assert any('id' in col for col in cols) + + +def test_query_by_constraints(test_session): + obj = Objects(id='00000000-0000-0000-0000-000000000003', id_type='dataset', id_file=None, id_internal=None) + test_session.add(obj) + test_session.commit() + # Should find the object by unique constraint + found = query_by_constraints( + test_session, + Objects(id='00000000-0000-0000-0000-000000000003', id_type='dataset', id_file=None, id_internal=None), + ) + assert found is not None + assert getattr(found, 'id', None) == '00000000-0000-0000-0000-000000000003' + assert getattr(found, 'id_type', None) == 'dataset' + + +def test_back_populate_tables_adds_and_merges(test_session): + obj = Objects(id='00000000-0000-0000-0000-000000000004', id_type='dataset', id_file=None, id_internal=None) + # Should add new + out = back_populate_tables(test_session, obj) + assert getattr(out, 'id', None) == '00000000-0000-0000-0000-000000000004' + # Should merge existing + obj2 = Objects(id='00000000-0000-0000-0000-000000000004', id_type='dataset', id_file=None, id_internal=None) + out2 = back_populate_tables(test_session, obj2) + assert getattr(out2, 'id', None) == getattr(out, 'id', None) + + +def test_get_or_create_back_populate(test_session): + obj = Objects(id='00000000-0000-0000-0000-000000000005', id_type='dataset', id_file=None, id_internal=None) + instance = get_or_create(test_session, obj, back_populate={'id_file': 12345}) + assert getattr(instance, 'id_file', None) == 12345 + + +def test_print_first_row_of_each_entity(test_session): + """ + Prints the first row of each mapped entity in the test database. + """ + import inspect as pyinspect + + from quantdb import models + + printed = False + for name, cls in vars(models).items(): + if pyinspect.isclass(cls) and hasattr(cls, '__table__') and hasattr(cls, '__mapper__'): + try: + row = test_session.query(cls).first() + if row: + print(f'First row for {name}: {row}') + printed = True + except Exception as e: + print(f'Could not query {name}: {e}') + if not printed: + print('No rows found in any entity.')