Test Database

.gitignore

@@ -174,3 +174,4 @@ cython_debug/
 .pypirc
 
 README.pdf
+.DS_STORE

README.org

@@ -3,3 +3,15 @@ A database schema for storing arbitrary quantitative measurements.
 
 See the [[file:./docs/api.org][REST API documentation]] for details on
 the external query interface.
+
+* Installation
+add the passwords to ~/.pgpass as
+```
+host:port:*:quantdb-admin:password
+host:port:*:quantdb-user:password
+host:port:*:quantdb-test-admin:password
+host:port:*:quantdb-test-user:password
+```
+
+* Notes
+macOS will need the config.yaml in `Library/Application Support/quantdb/config.yaml`

bin/prepare_test_db.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# quantdb-prepare-test-db.sh
+# This script prepares a test PostgreSQL database with the same schema as the main DB and populates it with the first 10 rows of each table.
+# Usage: ./bin/prepare_test_db.sh [MAIN_DB] [TEST_DB] [PGUSER] [PGHOST] [PGPORT]
+
+set -e
+eval "$(conda shell.bash hook)"
+conda activate quantdb
+
+# If you want to use the remote MAIN_DB connection info from client.py, extract it using Python
+REMOTE_DB_URI=$(python -c "from quantdb.client import get_session; s = get_session(echo=False); print(str(s.get_bind().url))")
+echo "Established Connection"
+
+MAIN_DB=${1:-quantdb}
+TEST_DB=${2:-quantdb_test}
+PGUSER=${3:-quantdb-test-admin}
+PGHOST=${4:-localhost}
+PGPORT=${5:-5432}
+
+# Parse the remote DB URI for pg_dump
+REMOTE_PGUSER=$(echo "$REMOTE_DB_URI" | sed -E 's|.*://([^:]+):.*|\1|')
+REMOTE_PGPASSWORD=$(python -c "from quantdb.config import auth; print(auth.get('db-password') or '')")
+REMOTE_PGHOST=$(echo "$REMOTE_DB_URI" | sed -E 's|.*@([^:/]+).*|\1|')
+REMOTE_PGPORT=$(echo "$REMOTE_DB_URI" | sed -E 's|.*:([0-9]+)/.*|\1|')
+REMOTE_DBNAME=$(echo "$REMOTE_DB_URI" | sed -E 's|.*/([^?]+).*|\1|')
+
+# Export password for pg_dump if available
+if [ -n "$REMOTE_PGPASSWORD" ]; then
+    export PGPASSWORD="$REMOTE_PGPASSWORD"
+fi
+
+# Ensure the PGUSER has a database (needed for psql connection)
+psql -U $PGUSER -h $PGHOST -p $PGPORT -d postgres -tc "SELECT 1 FROM pg_database WHERE datname = '$PGUSER'" | grep -q 1 || createdb -U $PGUSER -h $PGHOST -p $PGPORT $PGUSER
+
+# Drop and recreate the test database
+psql -U $PGUSER -h $PGHOST -p $PGPORT -c "DROP DATABASE IF EXISTS $TEST_DB;"
+psql -U $PGUSER -h $PGHOST -p $PGPORT -c "CREATE DATABASE $TEST_DB;"
+
+# Dump schema only from remote MAIN_DB and restore to local test DB
+pg_dump -U $REMOTE_PGUSER -h $REMOTE_PGHOST -p $REMOTE_PGPORT -s $REMOTE_DBNAME | psql -U $PGUSER -h $PGHOST -p $PGPORT $TEST_DB
+
+# Get all table names from the remote MAIN_DB
+TABLES=$(psql -U $REMOTE_PGUSER -h $REMOTE_PGHOST -p $REMOTE_PGPORT -d $REMOTE_DBNAME -Atc "SELECT tablename FROM pg_tables WHERE schemaname='public';")
+
+echo "Tables in remote DB: $TABLES"
+
+# For each table, copy the first 10 rows from remote MAIN_DB to local test DB
+for TABLE in $TABLES; do
+    COLS=$(psql -U $REMOTE_PGUSER -h $REMOTE_PGHOST -p $REMOTE_PGPORT -d $REMOTE_DBNAME -Atc "SELECT string_agg('"' || column_name || '"', ',') FROM information_schema.columns WHERE table_name='$TABLE' AND table_schema='public';")
+    psql -U $PGUSER -h $PGHOST -p $PGPORT -d $TEST_DB -c "INSERT INTO \"$TABLE\" ($COLS) SELECT $COLS FROM dblink('host=$REMOTE_PGHOST user=$REMOTE_PGUSER dbname=$REMOTE_DBNAME port=$REMOTE_PGPORT','SELECT $COLS FROM \"$TABLE\" LIMIT 10') AS t($COLS);" 2>/dev/null || true
+done
+
+unset PGPASSWORD
+
+echo "Test database '$TEST_DB' prepared with schema and first 10 rows of each table from remote '$REMOTE_DBNAME'."

pyproject.toml

@@ -13,14 +13,16 @@ dependencies = [
     "orthauth ~= 0.0.18",
     "cython",
     "psycopg2-binary",
-    "pre-commit ~= 4.1.0"
+    "pre-commit ~= 4.1.0",
+    "pytest==8.3.5",
+    "pandas==2.2.3",
 ]
 name = "quantdb"
 version = "0.1.0"
 description = "Quantitative DB"
 dynamic = ["readme"]
 packages = [{ include = "quantdb" }]
-requires-python = ">=3.10"
+requires-python = ">=3.10, <=3.13"
 
 [tool.setuptools.dynamic]
 readme = {file = ["README.org"], content-type = "text/plain"}

quantdb/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

quantdb/client.py

@@ -0,0 +1,48 @@
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import Session
+
+from quantdb.config import auth
+from quantdb.utils import dbUri
+
+
+def get_session(echo: bool = True, test: bool = False) -> Session:
+    """
+    Get a SQLAlchemy session for the main or test database.
+
+    Parameters
+    ----------
+    echo : bool, optional
+        If True, SQLAlchemy will log all statements.
+    test : bool, optional
+        If True, connect to the test database (quantdb_test or as configured).
+
+    Returns
+    -------
+    session : sqlalchemy.orm.Session
+        The database session.
+    """
+    # pull in the db connection info
+    if test:
+        # For testing, ALWAYS use local postgres instance regardless of main db config
+        dbkwargs = {
+            'dbuser': 'quantdb-test-admin',
+            'host': 'localhost',  # FORCE localhost for all testing
+            'port': 5432,
+            'database': auth.get('test-db-database') or 'quantdb_test',
+            'password': 'tom-is-cool',  # Use hardcoded password for test database
+        }
+    else:
+        # For non-test, use regular database configuration
+        dbkwargs = {k: auth.get(f'db-{k}') for k in ('user', 'host', 'port', 'database')}
+        dbkwargs['dbuser'] = dbkwargs.pop('user')
+    print(dbkwargs)
+    engine = create_engine(dbUri(**dbkwargs))  # type: ignore
+    engine.echo = echo
+    print(engine, dbkwargs)
+    session = Session(engine)
+    return session
+
+
+if __name__ == '__main__':
+    session = get_session(echo=False)
+    print(session.execute(text('SELECT * FROM information_schema.tables')).fetchall())

quantdb/config.py

@@ -1,3 +1,16 @@
+from dataclasses import dataclass
+
 import orthauth as oa
 
+from quantdb.utils import dbUri
+
 auth = oa.configure_here('auth-config.py', __name__)
+
+
+@dataclass
+class Settings:
+    """Settings for the app."""
+
+    db_params = {k: auth.get(f'db-{k}') for k in ('user', 'host', 'port', 'database')}
+    db_params['dbuser'] = db_params.pop('user')
+    SQLALCHEMY_DATABASE_URI = dbUri(**db_params)