diff --git a/docs/user_guide.md b/docs/user_guide.md index 15b927b0..9977ff14 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -321,6 +321,54 @@ and you can see all the stored metadata against a remote simulation using: simdb remote info ``` +To filter simulations and return selected quantities from their metadata, use +`data-query`. Each quantity has a response name and a metadata path: + +```bash +simdb remote data-query \ + 'summary.global_quantities.ip.value=gt:-5' \ + 'summary.global_quantities.ip.value=lt:10' \ + --quantity 'ip=summary.global_quantities.ip.value' \ + --quantity 'q95=summary.global_quantities.q_95.value' +``` + +The corresponding REST endpoint accepts the same operation as a structured request: + +```text +POST /v1.2/simulations/data/query +``` + +```json +{ + "filters": [ + { + "field": "summary.global_quantities.ip.value", + "operator": "gt", + "value": -5 + }, + { + "field": "summary.global_quantities.ip.value", + "operator": "lt", + "value": 10 + } + ], + "quantities": [ + { + "name": "ip", + "path": "summary.global_quantities.ip.value", + "source": "metadata" + } + ], + "page": 1, + "limit": 100 +} +``` + +Numeric arrays are stored in metadata as their minimum and maximum. For range metadata, +`gt` and `ge` compare the minimum while `lt` and `le` compare the maximum, so combining +lower and upper bounds requires every original value to be inside them. Use `agt`, +`age`, `alt`, or `ale` when any value may satisfy a bound. + ## Accessing Simulation Metadata via the SimDB Dashboard You can view a simulation's metadata directly in the SimDB dashboard using its UUID. diff --git a/src/simdb/cli/commands/remote.py b/src/simdb/cli/commands/remote.py index 4401464b..d7758c8a 100644 --- a/src/simdb/cli/commands/remote.py +++ b/src/simdb/cli/commands/remote.py @@ -1,3 +1,4 @@ +import json import re import shutil import sys @@ -12,6 +13,7 @@ from simdb.cli.remote_api import RemoteAPI from simdb.database.models.simulation import Simulation from simdb.notifications import Notification +from simdb.query import QueryType, parse_query_arg from . import check_meta_args, pass_config from .utils import print_simulations, print_trace @@ -534,6 +536,85 @@ def remote_query( ) +@remote.command("data-query", cls=remote_command_cls()) +@pass_api +@click.argument("constraints", nargs=-1) +@click.option( + "-q", + "--quantity", + "quantities", + multiple=True, + required=True, + metavar="NAME=METADATA_PATH", + help="Stored metadata quantity to return. May be provided multiple times.", +) +@click.option( + "-l", + "--limit", + default=100, + show_default=True, + callback=validate_positive, + help="Maximum simulations returned per page.", +) +@click.option( + "--page", + default=1, + show_default=True, + callback=validate_positive, + help="One-indexed result page.", +) +def remote_data_query( + api: RemoteAPI, + constraints: Tuple[str, ...], + quantities: Tuple[str, ...], + limit: int, + page: int, +): + """Filter simulations and return quantities from remote database metadata. + + CONSTRAINTS use the same ``NAME=[operator:]VALUE`` syntax as ``remote query``. + """ + filters = [] + for constraint in constraints: + if "=" not in constraint: + raise click.ClickException(f"Invalid constraint {constraint}.") + field, expression = constraint.split("=", 1) + value, operator = parse_query_arg(expression) + if operator == QueryType.NONE: + raise click.ClickException( + f"Invalid constraint {constraint}; a value or operator is required." + ) + try: + typed_value = json.loads(value) + except json.JSONDecodeError: + typed_value = value + filters.append( + { + "field": field, + "operator": operator.name.lower(), + "value": None if operator == QueryType.EXIST else typed_value, + } + ) + + requested_quantities = [] + for quantity in quantities: + if "=" not in quantity: + raise click.ClickException( + f"Invalid quantity {quantity}; expected NAME=METADATA_PATH." + ) + name, path = quantity.split("=", 1) + if not name or not path: + raise click.ClickException( + f"Invalid quantity {quantity}; expected NAME=METADATA_PATH." + ) + requested_quantities.append({"name": name, "path": path, "source": "metadata"}) + + result = api.query_simulation_data( + filters, requested_quantities, limit=limit, page=page + ) + click.echo(json.dumps(result, indent=2)) + + @remote.group(cls=RemoteSubGroup) def token(): """Manage user authentication tokens.""" diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 99f87363..70a37f14 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -714,6 +714,27 @@ def query_simulations( @versioned_method("v1.2", "v1.3") @try_request + def query_simulation_data( + self, + filters: List[Dict[str, Any]], + quantities: List[Dict[str, str]], + limit: int = 100, + page: int = 1, + ) -> Dict[str, Any]: + """Filter remote simulations and select stored metadata quantities.""" + response = self.post( + "simulations/data/query", + { + "filters": filters, + "quantities": quantities, + "limit": limit, + "page": page, + }, + ) + return response.json() + + @versioned_method("v1.2") + @try_request def delete_simulation(self, sim_id: str) -> Dict: res = self.delete("simulation/" + sim_id, {}) return res.json() diff --git a/src/simdb/database/database.py b/src/simdb/database/database.py index 356bb73e..7ee2195f 100644 --- a/src/simdb/database/database.py +++ b/src/simdb/database/database.py @@ -465,7 +465,7 @@ def _build_json_filter( try: cmp_float = float(compare_value) - except ValueError: + except (TypeError, ValueError): cmp_float = None def _string_cmp(op): @@ -473,6 +473,19 @@ def _string_cmp(op): return op(json_access, compare_value) return op(func.cast(json_access, String), compare_value) + def _boolean_cmp(op): + expected = compare_value.lower() + if dialect == "postgresql": + return sql_and( + func.jsonb_typeof(json_obj) == "boolean", + op(json_access, expected), + ) + json_type = func.json_type(column, f'$."{key}"') + return sql_and( + json_type.in_(["true", "false"]), + op(json_type, expected), + ) + def _number_cmp(cmp_op): if dialect == "postgresql": return sql_and( @@ -486,27 +499,43 @@ def _number_cmp(cmp_op): cmp_op(), ) - def _num_with_op(cmp_op): + def _scalar_or_range_cmp(scalar_cmp, range_cmp): if cmp_float is None: return None - return _number_cmp(cmp_op) + return sql_or(_number_cmp(scalar_cmp), range_cmp()) if query_type == QueryType.EQ: + if compare_value.lower() in {"true", "false"}: + return _boolean_cmp(lambda a, b: a == b) return _string_cmp(lambda a, b: a == b) elif query_type == QueryType.NE: + if compare_value.lower() in {"true", "false"}: + return _boolean_cmp(lambda a, b: a != b) return _string_cmp(lambda a, b: a != b) elif query_type == QueryType.IN: return _string_cmp(lambda a, b: a.ilike(f"%{b}%")) elif query_type == QueryType.NI: return _string_cmp(lambda a, b: a.notilike(f"%{b}%")) elif query_type == QueryType.GT: - return _num_with_op(lambda: sql_cast(json_access, Float) > cmp_float) + return _scalar_or_range_cmp( + lambda: sql_cast(json_access, Float) > cmp_float, + lambda: sql_cast(json_min, Float) > cmp_float, + ) elif query_type == QueryType.GE: - return _num_with_op(lambda: sql_cast(json_access, Float) >= cmp_float) + return _scalar_or_range_cmp( + lambda: sql_cast(json_access, Float) >= cmp_float, + lambda: sql_cast(json_min, Float) >= cmp_float, + ) elif query_type == QueryType.LT: - return _num_with_op(lambda: sql_cast(json_access, Float) < cmp_float) + return _scalar_or_range_cmp( + lambda: sql_cast(json_access, Float) < cmp_float, + lambda: sql_cast(json_max, Float) < cmp_float, + ) elif query_type == QueryType.LE: - return _num_with_op(lambda: sql_cast(json_access, Float) <= cmp_float) + return _scalar_or_range_cmp( + lambda: sql_cast(json_access, Float) <= cmp_float, + lambda: sql_cast(json_max, Float) <= cmp_float, + ) elif query_type == QueryType.AGT: if cmp_float is not None: return sql_cast(json_max, Float) > cmp_float diff --git a/src/simdb/remote/apis/simulation_data_query.py b/src/simdb/remote/apis/simulation_data_query.py new file mode 100644 index 00000000..ff6f7aa0 --- /dev/null +++ b/src/simdb/remote/apis/simulation_data_query.py @@ -0,0 +1,88 @@ +"""Database-backed simulation filtering and quantity selection endpoint.""" + +from typing import Annotated + +from flask_restx import Namespace, Resource + +from simdb.query import QueryType +from simdb.remote.core.auth import User, requires_auth +from simdb.remote.core.pydantic_utils import Body, pydantic_validate +from simdb.remote.core.typing import current_app +from simdb.remote.models import ( + PaginatedResponse, + SimulationDataQuantityResult, + SimulationDataQueryItem, + SimulationDataQueryRequest, +) + +api = Namespace("simulation-data-query", path="/") + + +def _filter_value(value) -> str: + if isinstance(value, bool): + return str(value).lower() + return "" if value is None else str(value) + + +@api.route("/simulations/data/query") +class SimulationDataQuery(Resource): + @requires_auth() + @pydantic_validate(api) + def post( + self, + user: User, + body: Annotated[SimulationDataQueryRequest, Body()], + ) -> PaginatedResponse[SimulationDataQueryItem]: + """Filter simulations and retrieve quantities from stored metadata.""" + constraints = [ + ( + item.field, + _filter_value(item.value), + QueryType[item.operator.upper()], + ) + for item in body.filters + ] + metadata_paths = list(dict.fromkeys(item.path for item in body.quantities)) + + count, rows = current_app.db.query_meta_data( + constraints, + metadata_paths, + limit=body.limit, + page=body.page, + sort_by=body.sort_by, + sort_asc=body.sort_asc, + ) + + results = [] + for row in rows: + metadata = { + item["element"]: item["value"] for item in row.get("metadata", []) + } + quantities = {} + missing = [] + for requested in body.quantities: + if requested.path not in metadata: + missing.append(requested.name) + continue + quantities[requested.name] = SimulationDataQuantityResult( + source=requested.source, + path=requested.path, + value=metadata[requested.path], + ) + + results.append( + SimulationDataQueryItem( + uuid=row["uuid"], + alias=row["alias"], + datetime=row["datetime"], + quantities=quantities, + missing=missing, + ) + ) + + return PaginatedResponse[SimulationDataQueryItem]( + count=count, + page=body.page, + limit=body.limit, + results=results, + ) diff --git a/src/simdb/remote/apis/v1_2/__init__.py b/src/simdb/remote/apis/v1_2/__init__.py index 920f303b..e8c17fc4 100644 --- a/src/simdb/remote/apis/v1_2/__init__.py +++ b/src/simdb/remote/apis/v1_2/__init__.py @@ -4,6 +4,7 @@ from simdb.remote.apis.files import api as file_ns from simdb.remote.apis.metadata import api as metadata_ns +from simdb.remote.apis.simulation_data_query import api as simulation_data_query_ns from simdb.remote.apis.watchers import api as watcher_ns from simdb.remote.core.auth import TokenAuthenticator, User, requires_auth from simdb.remote.core.pydantic_utils import pydantic_validate @@ -31,7 +32,7 @@ ) api.add_namespace(sim_ns) -namespaces = [metadata_ns, watcher_ns, file_ns, sim_ns] +namespaces = [metadata_ns, watcher_ns, file_ns, sim_ns, simulation_data_query_ns] @api.route("/staging_dir", defaults={"sim_hex": None}) diff --git a/src/simdb/remote/models.py b/src/simdb/remote/models.py index eb3827ae..f7647bd3 100644 --- a/src/simdb/remote/models.py +++ b/src/simdb/remote/models.py @@ -390,6 +390,158 @@ class SimulationListItem(BaseModel): """Simulation metadata.""" +MetadataQueryOperator = Literal[ + "eq", + "ne", + "in", + "ni", + "gt", + "ge", + "lt", + "le", + "agt", + "age", + "alt", + "ale", + "exist", +] +"""Operators supported by database metadata queries.""" + + +class SimulationDataFilter(BaseModel): + """A filter applied to stored simulation metadata.""" + + field: str = Field(min_length=1) + """Metadata key to filter on.""" + operator: MetadataQueryOperator = "eq" + """Comparison operator.""" + value: Optional[Union[str, int, float, bool]] = None + """Comparison value. Omit only when using the ``exist`` operator.""" + + @model_validator(mode="after") + def validate_filter(self): + self.field = self.field.strip() + if not self.field: + raise ValueError("field must not be blank") + + if self.operator != "exist" and self.value is None: + raise ValueError("value is required unless operator is 'exist'") + + scalar_operators = {"eq", "ne", "in", "ni", "gt", "ge", "lt", "le"} + allowed_reserved_operators = { + "alias": {"eq", "ne", "in", "ni", "exist"}, + "uuid": {"eq", "ne", "in", "ni", "exist"}, + "creation_date": {"eq", "ne", "gt", "ge", "lt", "le", "exist"}, + } + allowed = allowed_reserved_operators.get(self.field) + if allowed is not None and self.operator not in allowed: + raise ValueError( + f"operator '{self.operator}' is not supported for field '{self.field}'" + ) + + if isinstance(self.value, bool) and self.operator not in {"eq", "ne"}: + raise ValueError("Boolean filter values support only 'eq' and 'ne'") + + numeric_operators = { + "gt", + "ge", + "lt", + "le", + "agt", + "age", + "alt", + "ale", + } + if self.operator in numeric_operators and self.field != "creation_date": + assert self.value is not None + try: + float(self.value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"operator '{self.operator}' requires a numeric value" + ) from exc + + if self.field == "uuid" and self.operator in {"eq", "ne"}: + try: + UUID(str(self.value)) + except ValueError as exc: + raise ValueError("invalid UUID filter value") from exc + + if ( + self.field == "creation_date" + and self.operator in scalar_operators + and self.operator not in {"in", "ni"} + ): + try: + dt.strptime(str(self.value).replace("_", ":"), "%Y-%m-%d %H:%M:%S") + except ValueError as exc: + raise ValueError("creation_date must use YYYY-MM-DD HH_MM_SS") from exc + return self + + +class SimulationDataQuantity(BaseModel): + """A quantity requested for every matching simulation.""" + + name: str = Field(min_length=1) + """Name used for this quantity in the response.""" + path: str = Field(min_length=1) + """Stored metadata key. May later identify another supported data source.""" + source: Literal["metadata"] = "metadata" + """Quantity provider. Only database metadata is currently supported.""" + + +class SimulationDataQueryRequest(BaseModel): + """Filter simulations and select quantities from stored metadata.""" + + filters: List[SimulationDataFilter] = Field(default_factory=list, max_length=50) + """Metadata constraints. All filters are combined with logical AND.""" + quantities: List[SimulationDataQuantity] = Field(min_length=1, max_length=50) + """Quantities returned for each matching simulation.""" + page: int = Field(1, ge=1) + """One-indexed result page.""" + limit: int = Field(100, ge=1, le=1000) + """Maximum simulations returned on this page.""" + sort_by: Literal["", "alias", "uuid", "datetime"] = "" + """Optional simulation field used for sorting.""" + sort_asc: bool = False + """Sort ascending when true, descending otherwise.""" + + @model_validator(mode="after") + def require_unique_quantity_names(self): + names = [quantity.name for quantity in self.quantities] + if len(names) != len(set(names)): + raise ValueError("quantity names must be unique") + return self + + +class SimulationDataQuantityResult(BaseModel): + """Value returned by a quantity provider.""" + + source: Literal["metadata"] + """Provider used to retrieve the quantity.""" + path: str + """Provider-specific quantity path.""" + value: MetadataValue + """Retrieved value.""" + + +class SimulationDataQueryItem(BaseModel): + """Selected quantities for one matching simulation.""" + + uuid: CustomUUID + """Simulation UUID.""" + alias: Optional[str] = None + """Simulation alias.""" + datetime: str + """Simulation creation timestamp.""" + quantities: Dict[str, SimulationDataQuantityResult] = Field(default_factory=dict) + """Retrieved quantities, keyed by their requested names.""" + missing: List[str] = Field(default_factory=list) + """Requested quantity names not present in this simulation's metadata.""" + errors: Dict[str, str] = Field(default_factory=dict) + """Provider errors keyed by requested quantity name.""" + + T = TypeVar("T") """Type variable for generic paginated responses.""" diff --git a/tests/cli/test_cli_remote_command.py b/tests/cli/test_cli_remote_command.py index 0e626042..ad80af19 100644 --- a/tests/cli/test_cli_remote_command.py +++ b/tests/cli/test_cli_remote_command.py @@ -335,3 +335,76 @@ def test_remote_query_command_with_verbose( assert args == (constraints, (), 100) assert kwargs == {} assert get_api_version.called + + +@mock.patch("simdb.cli.remote_api.RemoteAPI.get_server_authentication") +@mock.patch("simdb.cli.remote_api.RemoteAPI.get_endpoints") +@mock.patch("simdb.cli.remote_api.RemoteAPI.get_api_version") +@mock.patch("simdb.cli.remote_api.RemoteAPI.get_server_version") +@mock.patch("simdb.cli.remote_api.RemoteAPI.query_simulation_data") +def test_remote_data_query_command( + query_simulation_data, + get_server_version, + get_api_version, + get_endpoints, + get_server_authentication, +): + get_endpoints.return_value = ["v1", "v1.1", "v1.1.1", "v1.2"] + get_api_version.return_value = "1.2" + get_server_version.return_value = "0.11" + get_server_authentication.return_value = "None" + query_simulation_data.return_value = { + "count": 1, + "page": 1, + "limit": 25, + "results": [{"alias": "inside-range", "quantities": {"ip": 2.0}}], + } + + config_file = config_test_file() + runner = CliRunner() + result = runner.invoke( + cli, + [ + f"--config-file={config_file}", + "remote", + "data-query", + "--limit", + "25", + "--quantity", + "ip=summary.global_quantities.ip.value", + "summary.global_quantities.ip.value=gt:-5", + "summary.global_quantities.ip.value=lt:10", + "summary.valid=eq:true", + ], + ) + + assert result.exception is None + assert '"inside-range"' in result.output + query_simulation_data.assert_called_once_with( + [ + { + "field": "summary.global_quantities.ip.value", + "operator": "gt", + "value": -5, + }, + { + "field": "summary.global_quantities.ip.value", + "operator": "lt", + "value": 10, + }, + { + "field": "summary.valid", + "operator": "eq", + "value": True, + }, + ], + [ + { + "name": "ip", + "path": "summary.global_quantities.ip.value", + "source": "metadata", + } + ], + limit=25, + page=1, + ) diff --git a/tests/database/test_metadata_queries.py b/tests/database/test_metadata_queries.py index f7630055..4a86fe68 100644 --- a/tests/database/test_metadata_queries.py +++ b/tests/database/test_metadata_queries.py @@ -127,6 +127,30 @@ def test_query_meta_exist_comparator(self, db): assert len(results) == 1 assert results[0].find_meta("status") == ["passed"] + @pytest.mark.parametrize( + ("query_type", "value", "expected_alias"), + [ + (QueryType.EQ, "true", "enabled"), + (QueryType.EQ, "false", "disabled"), + (QueryType.NE, "true", "disabled"), + ], + ) + def test_query_meta_boolean_comparator(self, db, query_type, value, expected_alias): + db.insert_simulation( + create_simulation(alias="enabled", metadata={"flag": True}) + ) + db.insert_simulation( + create_simulation(alias="disabled", metadata={"flag": False}) + ) + db.insert_simulation( + create_simulation(alias="string-value", metadata={"flag": "true"}) + ) + db.session.commit() + + results = db.query_meta([("flag", value, query_type)]) + + assert [result.alias for result in results] == [expected_alias] + def test_query_meta_agt_comparator(self, db): sim1 = create_simulation( alias="sim1", metadata={"range": {"min": 1.0, "max": 5.0}} @@ -228,6 +252,54 @@ def test_query_meta_range_combined_constraints(self, db): aliases = {r.alias for r in results} assert aliases == {"sim1", "sim3"} + def test_query_meta_all_values_inside_bounds(self, db): + sim1 = create_simulation( + alias="inside", metadata={"range": {"min": -4.0, "max": 9.0}} + ) + sim2 = create_simulation( + alias="above", metadata={"range": {"min": 0.0, "max": 12.0}} + ) + sim3 = create_simulation( + alias="below", metadata={"range": {"min": -8.0, "max": 2.0}} + ) + db.insert_simulation(sim1) + db.insert_simulation(sim2) + db.insert_simulation(sim3) + db.session.commit() + + results = db.query_meta( + [ + ("range", "-5", QueryType.GT), + ("range", "10", QueryType.LT), + ] + ) + + assert [result.alias for result in results] == ["inside"] + + @pytest.mark.parametrize( + ("query_type", "bound", "nonmatching_value"), + [ + (QueryType.GT, "1", 0.0), + (QueryType.GE, "2", 1.0), + (QueryType.LT, "3", 4.0), + (QueryType.LE, "2", 3.0), + ], + ) + def test_query_meta_scalar_bounds_remain_supported( + self, db, query_type, bound, nonmatching_value + ): + matching = create_simulation(alias="matching", metadata={"value": 2.0}) + not_matching = create_simulation( + alias="not-matching", metadata={"value": nonmatching_value} + ) + db.insert_simulation(matching) + db.insert_simulation(not_matching) + db.session.commit() + + results = db.query_meta([("value", bound, query_type)]) + + assert [result.alias for result in results] == ["matching"] + class TestListSimulationData: def test_list_simulation_data_empty_database(self, db):