added simulation data query endpoint [in work]

docs/user_guide.md

@@ -321,6 +321,54 @@ and you can see all the stored metadata against a remote simulation using:
 simdb remote info <SIM_ID>
 ```
 
+To filter simulations and return selected quantities from their metadata, use
+`data-query`. Each quantity has a response name and a metadata path:
+
+```bash
+simdb remote data-query \
+  'summary.global_quantities.ip.value=gt:-5' \
+  'summary.global_quantities.ip.value=lt:10' \
+  --quantity 'ip=summary.global_quantities.ip.value' \
+  --quantity 'q95=summary.global_quantities.q_95.value'
+```
+
+The corresponding REST endpoint accepts the same operation as a structured request:
+
+```text
+POST /v1.2/simulations/data/query
+```
+
+```json
+{
+  "filters": [
+    {
+      "field": "summary.global_quantities.ip.value",
+      "operator": "gt",
+      "value": -5
+    },
+    {
+      "field": "summary.global_quantities.ip.value",
+      "operator": "lt",
+      "value": 10
+    }
+  ],
+  "quantities": [
+    {
+      "name": "ip",
+      "path": "summary.global_quantities.ip.value",
+      "source": "metadata"
+    }
+  ],
+  "page": 1,
+  "limit": 100
+}
+```
+
+Numeric arrays are stored in metadata as their minimum and maximum. For range metadata,
+`gt` and `ge` compare the minimum while `lt` and `le` compare the maximum, so combining
+lower and upper bounds requires every original value to be inside them. Use `agt`,
+`age`, `alt`, or `ale` when any value may satisfy a bound.
+
 ## Accessing Simulation Metadata via the SimDB Dashboard
 
 You can view a simulation's metadata directly in the SimDB dashboard using its UUID.

src/simdb/cli/commands/remote.py

@@ -1,3 +1,4 @@
+import json
 import re
 import shutil
 import sys
@@ -12,6 +13,7 @@
 from simdb.cli.remote_api import RemoteAPI
 from simdb.database.models.simulation import Simulation
 from simdb.notifications import Notification
+from simdb.query import QueryType, parse_query_arg
 
 from . import check_meta_args, pass_config
 from .utils import print_simulations, print_trace
@@ -534,6 +536,85 @@ def remote_query(
     )
 
 
+@remote.command("data-query", cls=remote_command_cls())
+@pass_api
+@click.argument("constraints", nargs=-1)
+@click.option(
+    "-q",
+    "--quantity",
+    "quantities",
+    multiple=True,
+    required=True,
+    metavar="NAME=METADATA_PATH",
+    help="Stored metadata quantity to return. May be provided multiple times.",
+)
+@click.option(
+    "-l",
+    "--limit",
+    default=100,
+    show_default=True,
+    callback=validate_positive,
+    help="Maximum simulations returned per page.",
+)
+@click.option(
+    "--page",
+    default=1,
+    show_default=True,
+    callback=validate_positive,
+    help="One-indexed result page.",
+)
+def remote_data_query(
+    api: RemoteAPI,
+    constraints: Tuple[str, ...],
+    quantities: Tuple[str, ...],
+    limit: int,
+    page: int,
+):
+    """Filter simulations and return quantities from remote database metadata.
+
+    CONSTRAINTS use the same ``NAME=[operator:]VALUE`` syntax as ``remote query``.
+    """
+    filters = []
+    for constraint in constraints:
+        if "=" not in constraint:
+            raise click.ClickException(f"Invalid constraint {constraint}.")
+        field, expression = constraint.split("=", 1)
+        value, operator = parse_query_arg(expression)
+        if operator == QueryType.NONE:
+            raise click.ClickException(
+                f"Invalid constraint {constraint}; a value or operator is required."
+            )
+        try:
+            typed_value = json.loads(value)
+        except json.JSONDecodeError:
+            typed_value = value
+        filters.append(
+            {
+                "field": field,
+                "operator": operator.name.lower(),
+                "value": None if operator == QueryType.EXIST else typed_value,
+            }
+        )
+
+    requested_quantities = []
+    for quantity in quantities:
+        if "=" not in quantity:
+            raise click.ClickException(
+                f"Invalid quantity {quantity}; expected NAME=METADATA_PATH."
+            )
+        name, path = quantity.split("=", 1)
+        if not name or not path:
+            raise click.ClickException(
+                f"Invalid quantity {quantity}; expected NAME=METADATA_PATH."
+            )
+        requested_quantities.append({"name": name, "path": path, "source": "metadata"})
+
+    result = api.query_simulation_data(
+        filters, requested_quantities, limit=limit, page=page
+    )
+    click.echo(json.dumps(result, indent=2))
+
+
 @remote.group(cls=RemoteSubGroup)
 def token():
     """Manage user authentication tokens."""

src/simdb/cli/remote_api.py

@@ -714,6 +714,27 @@ def query_simulations(
 
     @versioned_method("v1.2", "v1.3")
     @try_request
+    def query_simulation_data(
+        self,
+        filters: List[Dict[str, Any]],
+        quantities: List[Dict[str, str]],
+        limit: int = 100,
+        page: int = 1,
+    ) -> Dict[str, Any]:
+        """Filter remote simulations and select stored metadata quantities."""
+        response = self.post(
+            "simulations/data/query",
+            {
+                "filters": filters,
+                "quantities": quantities,
+                "limit": limit,
+                "page": page,
+            },
+        )
+        return response.json()
+
+    @versioned_method("v1.2")
+    @try_request
     def delete_simulation(self, sim_id: str) -> Dict:
         res = self.delete("simulation/" + sim_id, {})
         return res.json()

src/simdb/database/database.py

@@ -465,14 +465,27 @@ def _build_json_filter(
 
         try:
             cmp_float = float(compare_value)
-        except ValueError:
+        except (TypeError, ValueError):
             cmp_float = None
 
         def _string_cmp(op):
             if dialect == "postgresql":
                 return op(json_access, compare_value)
             return op(func.cast(json_access, String), compare_value)
 
+        def _boolean_cmp(op):
+            expected = compare_value.lower()
+            if dialect == "postgresql":
+                return sql_and(
+                    func.jsonb_typeof(json_obj) == "boolean",
+                    op(json_access, expected),
+                )
+            json_type = func.json_type(column, f'$."{key}"')
+            return sql_and(
+                json_type.in_(["true", "false"]),
+                op(json_type, expected),
+            )
+
         def _number_cmp(cmp_op):
             if dialect == "postgresql":
                 return sql_and(
@@ -486,27 +499,43 @@ def _number_cmp(cmp_op):
                 cmp_op(),
             )
 
-        def _num_with_op(cmp_op):
+        def _scalar_or_range_cmp(scalar_cmp, range_cmp):
             if cmp_float is None:
                 return None
-            return _number_cmp(cmp_op)
+            return sql_or(_number_cmp(scalar_cmp), range_cmp())
 
         if query_type == QueryType.EQ:
+            if compare_value.lower() in {"true", "false"}:
+                return _boolean_cmp(lambda a, b: a == b)
             return _string_cmp(lambda a, b: a == b)
         elif query_type == QueryType.NE:
+            if compare_value.lower() in {"true", "false"}:
+                return _boolean_cmp(lambda a, b: a != b)
             return _string_cmp(lambda a, b: a != b)
         elif query_type == QueryType.IN:
             return _string_cmp(lambda a, b: a.ilike(f"%{b}%"))
         elif query_type == QueryType.NI:
             return _string_cmp(lambda a, b: a.notilike(f"%{b}%"))
         elif query_type == QueryType.GT:
-            return _num_with_op(lambda: sql_cast(json_access, Float) > cmp_float)
+            return _scalar_or_range_cmp(
+                lambda: sql_cast(json_access, Float) > cmp_float,
+                lambda: sql_cast(json_min, Float) > cmp_float,
+            )
         elif query_type == QueryType.GE:
-            return _num_with_op(lambda: sql_cast(json_access, Float) >= cmp_float)
+            return _scalar_or_range_cmp(
+                lambda: sql_cast(json_access, Float) >= cmp_float,
+                lambda: sql_cast(json_min, Float) >= cmp_float,
+            )
         elif query_type == QueryType.LT:
-            return _num_with_op(lambda: sql_cast(json_access, Float) < cmp_float)
+            return _scalar_or_range_cmp(
+                lambda: sql_cast(json_access, Float) < cmp_float,
+                lambda: sql_cast(json_max, Float) < cmp_float,
+            )
         elif query_type == QueryType.LE:
-            return _num_with_op(lambda: sql_cast(json_access, Float) <= cmp_float)
+            return _scalar_or_range_cmp(
+                lambda: sql_cast(json_access, Float) <= cmp_float,
+                lambda: sql_cast(json_max, Float) <= cmp_float,
+            )
         elif query_type == QueryType.AGT:
             if cmp_float is not None:
                 return sql_cast(json_max, Float) > cmp_float

src/simdb/remote/apis/simulation_data_query.py

@@ -0,0 +1,88 @@
+"""Database-backed simulation filtering and quantity selection endpoint."""
+
+from typing import Annotated
+
+from flask_restx import Namespace, Resource
+
+from simdb.query import QueryType
+from simdb.remote.core.auth import User, requires_auth
+from simdb.remote.core.pydantic_utils import Body, pydantic_validate
+from simdb.remote.core.typing import current_app
+from simdb.remote.models import (
+    PaginatedResponse,
+    SimulationDataQuantityResult,
+    SimulationDataQueryItem,
+    SimulationDataQueryRequest,
+)
+
+api = Namespace("simulation-data-query", path="/")
+
+
+def _filter_value(value) -> str:
+    if isinstance(value, bool):
+        return str(value).lower()
+    return "" if value is None else str(value)
+
+
+@api.route("/simulations/data/query")
+class SimulationDataQuery(Resource):
+    @requires_auth()
+    @pydantic_validate(api)
+    def post(
+        self,
+        user: User,
+        body: Annotated[SimulationDataQueryRequest, Body()],
+    ) -> PaginatedResponse[SimulationDataQueryItem]:
+        """Filter simulations and retrieve quantities from stored metadata."""
+        constraints = [
+            (
+                item.field,
+                _filter_value(item.value),
+                QueryType[item.operator.upper()],
+            )
+            for item in body.filters
+        ]
+        metadata_paths = list(dict.fromkeys(item.path for item in body.quantities))
+
+        count, rows = current_app.db.query_meta_data(
+            constraints,
+            metadata_paths,
+            limit=body.limit,
+            page=body.page,
+            sort_by=body.sort_by,
+            sort_asc=body.sort_asc,
+        )
+
+        results = []
+        for row in rows:
+            metadata = {
+                item["element"]: item["value"] for item in row.get("metadata", [])
+            }
+            quantities = {}
+            missing = []
+            for requested in body.quantities:
+                if requested.path not in metadata:
+                    missing.append(requested.name)
+                    continue
+                quantities[requested.name] = SimulationDataQuantityResult(
+                    source=requested.source,
+                    path=requested.path,
+                    value=metadata[requested.path],
+                )
+
+            results.append(
+                SimulationDataQueryItem(
+                    uuid=row["uuid"],
+                    alias=row["alias"],
+                    datetime=row["datetime"],
+                    quantities=quantities,
+                    missing=missing,
+                )
+            )
+
+        return PaginatedResponse[SimulationDataQueryItem](
+            count=count,
+            page=body.page,
+            limit=body.limit,
+            results=results,
+        )

src/simdb/remote/apis/v1_2/__init__.py

@@ -4,6 +4,7 @@
 
 from simdb.remote.apis.files import api as file_ns
 from simdb.remote.apis.metadata import api as metadata_ns
+from simdb.remote.apis.simulation_data_query import api as simulation_data_query_ns
 from simdb.remote.apis.watchers import api as watcher_ns
 from simdb.remote.core.auth import TokenAuthenticator, User, requires_auth
 from simdb.remote.core.pydantic_utils import pydantic_validate
@@ -31,7 +32,7 @@
 )
 
 api.add_namespace(sim_ns)
-namespaces = [metadata_ns, watcher_ns, file_ns, sim_ns]
+namespaces = [metadata_ns, watcher_ns, file_ns, sim_ns, simulation_data_query_ns]
 
 
 @api.route("/staging_dir", defaults={"sim_hex": None})