diff --git a/database/README.md b/database/README.md index 71f64628f..d72820eb1 100644 --- a/database/README.md +++ b/database/README.md @@ -4,7 +4,6 @@ | field | value | |-------|-------| -| version | 1.0.0 | | type | binary | | supported_targets | x86_64-apple-darwin, aarch64-apple-darwin, x86_64-unknown-linux-gnu, aarch64-unknown-linux-gnu | | author | iii | @@ -12,7 +11,7 @@ ## Install ```sh -iii worker add database@1.0.0 +iii worker add database ``` ## Skills @@ -193,6 +192,41 @@ const { rows } = await iii.trigger({ | `database::rollbackTransaction` | Rollback and finalize an interactive transaction. Subsequent calls against the same id return `TRANSACTION_NOT_FOUND`. | | `database::listDatabases` | List configured databases. Returns `{ databases, count }`; each entry has `name`, `driver`, credential-redacted `url`, `pool` settings, and `tls` (`mode`, `ca_cert_present`, `trust_native`). Config only — no health checks or live pool stats. | +### Reading the schema + +| Function | Description | +|---|---| +| `database::listTables` | Every table and view, with its kind and (postgres) its schema. | +| `database::describeTable` | One table: columns with type, nullability, default, primary-key membership and foreign-key target; plus indexes and a planner row estimate. Foreign keys are structured `{ schema, table, column }`, not a joined string. | +| `database::describeSchema` | The same shape for every table at once. One catalog query per aspect across the whole database rather than one call per table, so a 200-table schema costs a handful of queries. `include_indexes` is off by default. | +| `database::schemaDiagram` | Positioned table nodes and routed foreign-key edges, plus each table's hub `degree`, the `isolated` tables, and remaining edge `crossings`. Layout runs server-side, so a renderer only draws. | + +### Reading data + +| Function | Description | +|---|---| +| `database::browseTable` | Paged, sorted, filtered table read — no SQL from the caller. Filters are structured (`{ column, op, value }`) and compile to a parameterised `WHERE` for the driver in hand; `total` honours the same filters. Sorts accept a `mode` (`natural`, `length`, `absolute_value`, `random`) applied across the whole table, not just the page. To follow a foreign key, filter on equality with `page_size: 1`. | +| `database::explain` | The query plan as a tree with per-node cost, row estimates and warnings, instead of the driver's raw text. `analyze` collects real timings by **running** the statement, so it defaults to `false` and is refused for anything that is not a single read. | +| `database::columnStats` | Profile a table's columns. Reads the planner's own statistics by default — free and approximate, labelled `source: planner`. `exact: true` runs real aggregates and scans the table; it is refused above a row-count ceiling. To profile rows you already hold, pipe a `browseTable` result through the `fp` worker instead. | + +### Operations + +| Function | Description | +|---|---| +| `database::health` | Live pool occupancy plus active queries, table sizes, blocking locks and cache hit ratio. Each section reports separately as `available`, `unsupported` or `denied`, so a driver gap or a restricted role is never mistaken for an empty result. | +| `database::terminateQuery` | Terminate a backend session, or cancel just its statement with `cancel_only`. Takes an id from `health`. Separate from `health` because it is a write. | + +### Saved queries and history + +Stored in the [`state`](https://github.com/iii-hq/workers/tree/main/state) worker, scoped per database, so they survive restarts and any agent can read them. + +| Function | Description | +|---|---| +| `database::saveQuery` | Save a named query. Saving under an existing name replaces it. | +| `database::listSavedQueries` | Saved queries for a database, sorted by name. | +| `database::deleteSavedQuery` | Delete by id or by name. | +| `database::history` | Recent queries, newest first. Best effort — recording never blocks or fails a query, so this is a convenience rather than an audit log. For an audit trail bind `database::row-changed`. | + ## Triggers ### `database::row-changed` diff --git a/database/skills/SKILL.md b/database/skills/SKILL.md index 65dce3ce9..6b91e1c98 100644 --- a/database/skills/SKILL.md +++ b/database/skills/SKILL.md @@ -73,14 +73,54 @@ point. Placeholder syntax: `?` for SQLite and MySQL, `$1`/`$2`/… for Postgres. transaction. - `database::rollbackTransaction` — roll back and finalize an interactive transaction. -- `database::listDatabases` — list every configured database with its - driver, credential-redacted connection URL, pool settings, and TLS mode. - Config details only; no health checks or live pool statistics. +- `database::listDatabases` — every configured database with its driver, + credential-redacted URL, pool settings and TLS mode. Config only; use + `database::health` for live state. Interactive transactions auto-roll back when `timeout_ms` elapses (default 30 s, max 5 min). Prepared handles default to a 1 h TTL (max 24 h) with no explicit release call — let them expire or stop using them when done. +### Reading the schema + +One shape across all three drivers — prefer these over hand-writing +`sqlite_master` / `information_schema` / `PRAGMA`. + +- `database::listTables` — tables and views, with kind and (postgres) schema. +- `database::describeTable` — columns with type, nullability, default, primary + key and a structured `foreign_key` of `{ schema, table, column }`; plus + indexes and a planner row estimate. +- `database::describeSchema` — the same for every table in one pass. Use this + rather than looping `describeTable`. +- `database::schemaDiagram` — positioned nodes, routed foreign-key edges, hub + `degree` and `isolated` tables. For reasoning about a schema's shape, not + only for drawing it. + +### Reading data + +- `database::browseTable` — paged, sorted, filtered reads with no SQL. Filters + are `{ column, op, value }` and `total` honours them. Follow a foreign key + with an equality filter at `page_size: 1`. +- `database::explain` — the plan as a tree with costs and warnings. `analyze` + **runs** the statement, so it defaults to false and is refused for anything + that is not a single read. +- `database::columnStats` — planner statistics by default (approximate, + labelled `source: planner`); `exact: true` scans. To profile rows you already + hold, use the `fp` worker on a `browseTable` result instead. + +### Operations and reuse + +- `database::health` — pool occupancy, active queries, table sizes, locks, + cache ratio. Each section is `available`, `unsupported` or `denied`, so a + driver gap is never mistaken for an empty result. +- `database::terminateQuery` — end a session, or cancel its statement with + `cancel_only`. Takes an id from `database::health`. +- `database::saveQuery`, `database::listSavedQueries`, + `database::deleteSavedQuery` — named queries per database, kept in the + `state` worker. +- `database::history` — recent queries, newest first. Best effort, not an audit + log; bind `database::row-changed` for that. + ## Reacting to writes Register a `database::row-changed` trigger to be told when this worker commits diff --git a/database/src/handlers/browse.rs b/database/src/handlers/browse.rs new file mode 100644 index 000000000..6c1f4fd27 --- /dev/null +++ b/database/src/handlers/browse.rs @@ -0,0 +1,165 @@ +//! `database::browseTable` — paged, sorted, filtered reads without the caller +//! writing SQL. +//! +//! Subsumes three things the console used to do by hand: building +//! `SELECT * … ORDER BY … LIMIT/OFFSET`, running a matching `COUNT(*)`, and +//! compiling filter chips into a `WHERE`. It also covers foreign-key lookup — +//! that is an equality filter at `page_size: 1` — so there is no separate +//! read-a-row function. + +use super::filter::{self, FilterSpec, SortSpec}; +use super::query::{self, err_to_str, QueryReq}; +use super::AppState; +use crate::config::DriverKind; +use crate::driver::ColumnMeta; +use crate::error::DbError; +use crate::pool::Pool; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +fn default_page_size() -> u32 { + 50 +} + +fn default_timeout() -> u64 { + 30_000 +} + +fn default_include_total() -> bool { + true +} + +/// Ceiling on a single page. A caller wanting everything should page. +const MAX_PAGE_SIZE: u32 = 1_000; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct BrowseTableReq { + #[serde(default)] + pub db: Option, + pub table: String, + #[serde(default)] + pub schema: Option, + /// Zero-based. + #[serde(default)] + pub page: u32, + #[serde(default = "default_page_size")] + pub page_size: u32, + /// Applied in order; sort priority is position in the list. + #[serde(default)] + pub sort: Vec, + /// Combined with AND. + #[serde(default)] + pub filters: Vec, + /// A filtered `COUNT(*)` is a second query and can be expensive on a + /// large table. Turn it off while the caller is still typing. + #[serde(default = "default_include_total")] + pub include_total: bool, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct BrowseTableResp { + pub rows: Vec>, + pub columns: Vec, + pub page: u32, + pub page_size: u32, + /// Derived from a sentinel row, so it is correct without a count. + pub has_more: bool, + /// Total matching the same filters. Absent when not requested. + #[serde(skip_serializing_if = "Option::is_none")] + pub total: Option, +} + +async fn driver_of(state: &AppState, db: &str) -> Result { + Ok(match state.pool(db).await.map_err(err_to_str)? { + Pool::Sqlite(_) => DriverKind::Sqlite, + Pool::Postgres(_) => DriverKind::Postgres, + Pool::Mysql(_) => DriverKind::Mysql, + }) +} + +pub async fn handle(state: &AppState, req: BrowseTableReq) -> Result { + if req.table.trim().is_empty() { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: "table is required".into(), + })); + } + let db = state.resolve_db(req.db.clone()).await.map_err(err_to_str)?; + let driver = driver_of(state, &db).await?; + + let page_size = req.page_size.clamp(1, MAX_PAGE_SIZE); + let offset = (req.page as u64) * (page_size as u64); + + let target = filter::quote_table(driver, req.schema.as_deref(), &req.table); + let where_clause = filter::compile_where(driver, &req.filters, 1).map_err(err_to_str)?; + let predicate = if where_clause.sql.is_empty() { + String::new() + } else { + format!(" WHERE {}", where_clause.sql) + }; + + let order = filter::compile_order_by(driver, &req.sort).map_err(err_to_str)?; + let order_by = if order.is_empty() { + String::new() + } else { + format!(" ORDER BY {order}") + }; + + // Fetch one extra row to learn whether another page exists, so `has_more` + // is right even when the caller skipped the count. + let limit = page_size as u64 + 1; + let sql = format!("SELECT * FROM {target}{predicate}{order_by} LIMIT {limit} OFFSET {offset}"); + + let mut resp = query::handle( + state, + QueryReq { + db: Some(db.clone()), + sql, + params: where_clause.params.clone(), + timeout_ms: req.timeout_ms, + record_history: false, + }, + ) + .await?; + + let has_more = resp.rows.len() as u64 > page_size as u64; + resp.rows.truncate(page_size as usize); + + let total = if req.include_total { + let count_sql = format!("SELECT COUNT(*) AS total FROM {target}{predicate}"); + let counted = query::handle( + state, + QueryReq { + db: Some(db), + sql: count_sql, + params: where_clause.params, + timeout_ms: req.timeout_ms, + record_history: false, + }, + ) + .await?; + counted + .rows + .first() + .and_then(|r| r.get("total")) + .and_then(|v| match v { + Value::Number(n) => n.as_i64(), + Value::String(s) => s.parse().ok(), + _ => None, + }) + } else { + None + }; + + Ok(BrowseTableResp { + rows: resp.rows, + columns: resp.columns, + page: req.page, + page_size, + has_more, + total, + }) +} diff --git a/database/src/handlers/catalog/mod.rs b/database/src/handlers/catalog/mod.rs new file mode 100644 index 000000000..f392ed3c5 --- /dev/null +++ b/database/src/handlers/catalog/mod.rs @@ -0,0 +1,264 @@ +//! Driver-neutral catalog shapes and the per-driver readers behind +//! `database::listTables` / `describeTable` / `describeSchema`. +//! +//! Every reader goes through `database::query`, so catalog reads inherit the +//! same pool, timeout, and read-only-transaction handling as user SQL. +//! +//! Two rules hold across all three drivers: +//! +//! 1. **No array-valued columns.** `RowValue` has no array variant, so a +//! catalog query returning `text[]`/`int[]` fails to decode. Anywhere a +//! catalog exposes an array (postgres `indkey`, `most_common_vals`), it is +//! unnested into scalar rows and regrouped in Rust. +//! 2. **One query per aspect, not per table.** The readers take an optional +//! table filter; `describeSchema` omits it and regroups, so a 200-table +//! schema costs three queries rather than six hundred. + +pub mod mysql; +pub mod postgres; +pub mod sqlite; + +use super::AppState; +use crate::handlers::query::{self, QueryReq}; +use schemars::JsonSchema; +use serde::Serialize; +use serde_json::Value; +use std::collections::HashMap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum TableKind { + Table, + View, +} + +/// A relation. `schema` is populated only where the driver has a meaningful +/// namespace above the table (postgres); it is never concatenated into `name`, +/// because `analytics.events` and a table literally called `analytics.events` +/// are different things and callers must be able to tell them apart. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct TableRef { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option, + pub kind: TableKind, +} + +/// Where a foreign key points. Structured rather than a `"table.column"` +/// string so a schema-qualified target stays unambiguous. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct ForeignKeyRef { + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option, + pub table: String, + pub column: String, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct ColumnDesc { + pub name: String, + /// Driver-reported type text (`TEXT`, `integer`, `varchar(255)`). + #[serde(rename = "type")] + pub ty: String, + pub nullable: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub default_value: Option, + pub primary_key: bool, + /// 1-based ordinal, as the driver reports it. + pub position: i32, + #[serde(skip_serializing_if = "Option::is_none")] + pub foreign_key: Option, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct IndexDesc { + pub name: String, + pub unique: bool, + pub primary: bool, + /// Indexed columns in ordinal order. Empty when the index is on an + /// expression rather than plain columns. + pub columns: Vec, +} + +/// Key a per-table result is grouped under. `describeSchema` fans one query +/// out across every table, so each row has to say which table it belongs to. +pub type TableKey = (Option, String); + +/// Identifies one table for the readers. `schema` is ignored by sqlite and +/// mysql, which have no namespace above the table within a connection. +#[derive(Debug, Clone)] +pub struct TableFilter { + pub schema: Option, + pub table: String, +} + +/// Split a possibly schema-qualified name into `(schema, bare)`. Only applied +/// where the driver actually has schemas — see `TableFilter`. +pub fn split_qualified(name: &str) -> (Option, String) { + match name.split_once('.') { + Some((schema, bare)) if !schema.is_empty() && !bare.is_empty() => { + (Some(schema.to_string()), bare.to_string()) + } + _ => (None, name.to_string()), + } +} + +/// Double-quote an identifier, doubling embedded quotes. Used only where a +/// value cannot be bound as a parameter. +pub fn quote_ident(ident: &str) -> String { + format!("\"{}\"", ident.replace('"', "\"\"")) +} + +/// Run a catalog statement through the ordinary query path. +pub async fn run( + state: &AppState, + db: &str, + sql: impl Into, + params: Vec, + timeout_ms: u64, +) -> Result>, String> { + let resp = query::handle( + state, + QueryReq { + db: Some(db.to_string()), + sql: sql.into(), + params, + timeout_ms, + // Catalog reads are the page's plumbing, not the user's queries. + record_history: false, + }, + ) + .await?; + Ok(resp.rows) +} + +type Row = serde_json::Map; + +pub fn str_at(row: &Row, key: &str) -> Option { + match row.get(key) { + Some(Value::String(s)) => Some(s.clone()), + Some(Value::Number(n)) => Some(n.to_string()), + _ => None, + } +} + +pub fn i64_at(row: &Row, key: &str) -> Option { + match row.get(key) { + Some(Value::Number(n)) => n.as_i64().or_else(|| n.as_f64().map(|f| f as i64)), + Some(Value::String(s)) => s.parse().ok(), + _ => None, + } +} + +/// Catalogs are inconsistent about how they spell a boolean: postgres returns +/// a real bool, mysql returns 0/1, sqlite's PRAGMAs return 0/1, and +/// `information_schema` returns the strings `YES`/`NO`. Normalize all of them. +pub fn bool_at(row: &Row, key: &str) -> bool { + match row.get(key) { + Some(Value::Bool(b)) => *b, + Some(Value::Number(n)) => n.as_i64().is_some_and(|v| v != 0), + Some(Value::String(s)) => { + matches!(s.to_ascii_lowercase().as_str(), "yes" | "true" | "t" | "1") + } + _ => false, + } +} + +/// Group unnested index rows into `IndexDesc`, preserving first-seen order so +/// the output is stable across runs. +pub fn fold_index_rows(rows: &[Row], key_of: F) -> HashMap> +where + F: Fn(&Row) -> Option, +{ + let mut out: HashMap> = HashMap::new(); + for row in rows { + let Some(key) = key_of(row) else { continue }; + let Some(name) = str_at(row, "index_name") else { + continue; + }; + let entry = out.entry(key).or_default(); + let idx = match entry.iter_mut().find(|i| i.name == name) { + Some(existing) => existing, + None => { + entry.push(IndexDesc { + name: name.clone(), + unique: bool_at(row, "is_unique"), + primary: bool_at(row, "is_primary"), + columns: Vec::new(), + }); + entry.last_mut().expect("just pushed") + } + }; + if let Some(col) = str_at(row, "column_name") { + idx.columns.push(col); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn split_qualified_handles_bare_and_qualified_names() { + assert_eq!(split_qualified("users"), (None, "users".into())); + assert_eq!( + split_qualified("analytics.events"), + (Some("analytics".into()), "events".into()) + ); + // Degenerate halves stay whole rather than producing an empty schema. + assert_eq!(split_qualified(".events"), (None, ".events".into())); + assert_eq!(split_qualified("analytics."), (None, "analytics.".into())); + } + + #[test] + fn quote_ident_doubles_embedded_quotes() { + assert_eq!(quote_ident("users"), "\"users\""); + assert_eq!(quote_ident("we\"ird"), "\"we\"\"ird\""); + } + + #[test] + fn bool_at_normalizes_every_catalog_spelling() { + let row: Row = serde_json::from_str( + r#"{"a": true, "b": 1, "c": 0, "d": "YES", "e": "NO", "f": "t", "g": null}"#, + ) + .unwrap(); + assert!(bool_at(&row, "a")); + assert!(bool_at(&row, "b")); + assert!(!bool_at(&row, "c")); + assert!(bool_at(&row, "d")); + assert!(!bool_at(&row, "e")); + assert!(bool_at(&row, "f")); + assert!(!bool_at(&row, "g")); + assert!(!bool_at(&row, "missing")); + } + + #[test] + fn i64_at_accepts_the_string_counts_mysql_returns() { + let row: Row = + serde_json::from_str(r#"{"n": 42, "s": "1234", "f": 7.0, "x": "no"}"#).unwrap(); + assert_eq!(i64_at(&row, "n"), Some(42)); + assert_eq!(i64_at(&row, "s"), Some(1234)); + assert_eq!(i64_at(&row, "f"), Some(7)); + assert_eq!(i64_at(&row, "x"), None); + } + + #[test] + fn fold_index_rows_groups_columns_in_ordinal_order() { + let rows: Vec = serde_json::from_str( + r#"[ + {"table_name":"orders","index_name":"pk_orders","is_unique":true,"is_primary":true,"column_name":"id"}, + {"table_name":"orders","index_name":"ix_o","is_unique":false,"is_primary":false,"column_name":"user_id"}, + {"table_name":"orders","index_name":"ix_o","is_unique":false,"is_primary":false,"column_name":"created_at"} + ]"#, + ) + .unwrap(); + let folded = fold_index_rows(&rows, |r| str_at(r, "table_name").map(|t| (None, t))); + let idxs = &folded[&(None, "orders".to_string())]; + assert_eq!(idxs.len(), 2); + assert_eq!(idxs[0].name, "pk_orders"); + assert!(idxs[0].primary && idxs[0].unique); + assert_eq!(idxs[1].columns, vec!["user_id", "created_at"]); + } +} diff --git a/database/src/handlers/catalog/mysql.rs b/database/src/handlers/catalog/mysql.rs new file mode 100644 index 000000000..50e2e64bf --- /dev/null +++ b/database/src/handlers/catalog/mysql.rs @@ -0,0 +1,180 @@ +//! MySQL catalog reads. +//! +//! MySQL has no namespace above the table within a connection — its "schema" +//! *is* the database the pool is connected to — so every reader scopes to +//! `DATABASE()` and reports `schema: None`. Callers get the same shape as the +//! other drivers without a redundant level. +//! +//! `information_schema.KEY_COLUMN_USAGE` already emits one row per key column +//! with `ORDINAL_POSITION`, and `STATISTICS` one row per indexed column with +//! `SEQ_IN_INDEX`, so composite keys and multi-column indexes pair correctly +//! without any array handling or `GROUP_CONCAT`. + +use super::{ + bool_at, fold_index_rows, i64_at, run, str_at, ColumnDesc, ForeignKeyRef, IndexDesc, + TableFilter, TableKey, TableKind, TableRef, +}; +use crate::handlers::AppState; +use serde_json::Value; +use std::collections::HashMap; + +fn filter_clause(filter: Option<&TableFilter>, column: &str) -> (String, Vec) { + match filter { + Some(f) => ( + format!(" AND {column} = ?"), + vec![Value::String(f.table.clone())], + ), + None => (String::new(), Vec::new()), + } +} + +fn key_of(row: &serde_json::Map) -> Option { + Some((None, str_at(row, "table_name")?)) +} + +pub async fn list_tables( + state: &AppState, + db: &str, + timeout_ms: u64, +) -> Result, String> { + let sql = "SELECT TABLE_NAME AS table_name, TABLE_TYPE AS kind \ + FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() \ + ORDER BY TABLE_TYPE, TABLE_NAME"; + let rows = run(state, db, sql, vec![], timeout_ms).await?; + Ok(rows + .iter() + .filter_map(|r| { + Some(TableRef { + name: str_at(r, "table_name")?, + schema: None, + kind: match str_at(r, "kind").as_deref() { + Some("VIEW") => TableKind::View, + _ => TableKind::Table, + }, + }) + }) + .collect()) +} + +pub async fn columns( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result>, String> { + let (clause, params) = filter_clause(filter, "TABLE_NAME"); + let sql = format!( + "SELECT TABLE_NAME AS table_name, COLUMN_NAME AS column_name, \ + COLUMN_TYPE AS column_type, IS_NULLABLE AS is_nullable, \ + COLUMN_DEFAULT AS default_value, ORDINAL_POSITION AS position, \ + COLUMN_KEY AS column_key \ + FROM information_schema.COLUMNS \ + WHERE TABLE_SCHEMA = DATABASE(){clause} \ + ORDER BY TABLE_NAME, ORDINAL_POSITION" + ); + let rows = run(state, db, sql, params, timeout_ms).await?; + + let mut out: HashMap> = HashMap::new(); + for r in &rows { + let (Some(key), Some(name)) = (key_of(r), str_at(r, "column_name")) else { + continue; + }; + out.entry(key).or_default().push(ColumnDesc { + name, + ty: str_at(r, "column_type").unwrap_or_default(), + // IS_NULLABLE is the string 'YES'/'NO'. + nullable: bool_at(r, "is_nullable"), + default_value: str_at(r, "default_value"), + // COLUMN_KEY is 'PRI' for a primary-key member. Cheaper and more + // reliable than a second trip to KEY_COLUMN_USAGE. + primary_key: str_at(r, "column_key").as_deref() == Some("PRI"), + position: i64_at(r, "position").unwrap_or(0) as i32, + foreign_key: None, + }); + } + + merge_foreign_keys(state, db, filter, timeout_ms, &mut out).await?; + Ok(out) +} + +async fn merge_foreign_keys( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, + columns: &mut HashMap>, +) -> Result<(), String> { + let (clause, params) = filter_clause(filter, "TABLE_NAME"); + let sql = format!( + "SELECT TABLE_NAME AS table_name, COLUMN_NAME AS src_column, \ + REFERENCED_TABLE_NAME AS ref_table, REFERENCED_COLUMN_NAME AS ref_column \ + FROM information_schema.KEY_COLUMN_USAGE \ + WHERE TABLE_SCHEMA = DATABASE() AND REFERENCED_TABLE_NAME IS NOT NULL{clause} \ + ORDER BY TABLE_NAME, CONSTRAINT_NAME, ORDINAL_POSITION" + ); + for r in run(state, db, sql, params, timeout_ms).await? { + let (Some(key), Some(src), Some(ref_table), Some(ref_column)) = ( + key_of(&r), + str_at(&r, "src_column"), + str_at(&r, "ref_table"), + str_at(&r, "ref_column"), + ) else { + continue; + }; + if let Some(cols) = columns.get_mut(&key) { + if let Some(c) = cols.iter_mut().find(|c| c.name == src) { + c.foreign_key = Some(ForeignKeyRef { + schema: None, + table: ref_table, + column: ref_column, + }); + } + } + } + Ok(()) +} + +pub async fn indexes( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result>, String> { + let (clause, params) = filter_clause(filter, "TABLE_NAME"); + // NON_UNIQUE is inverted, so flip it into the `is_unique` the fold reads. + // MySQL names the primary-key index 'PRIMARY' and offers no other flag. + let sql = format!( + "SELECT TABLE_NAME AS table_name, INDEX_NAME AS index_name, \ + NON_UNIQUE = 0 AS is_unique, INDEX_NAME = 'PRIMARY' AS is_primary, \ + COLUMN_NAME AS column_name \ + FROM information_schema.STATISTICS \ + WHERE TABLE_SCHEMA = DATABASE(){clause} \ + ORDER BY TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX" + ); + let rows = run(state, db, sql, params, timeout_ms).await?; + Ok(fold_index_rows(&rows, key_of)) +} + +/// `TABLE_ROWS` is an InnoDB estimate sampled from the index, and is NULL for +/// views. Treat NULL as absent rather than zero — "unknown" and "empty" are +/// different answers. +pub async fn row_estimates( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result, String> { + let (clause, params) = filter_clause(filter, "TABLE_NAME"); + let sql = format!( + "SELECT TABLE_NAME AS table_name, TABLE_ROWS AS estimate \ + FROM information_schema.TABLES \ + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_ROWS IS NOT NULL{clause}" + ); + let mut out = HashMap::new(); + for r in run(state, db, sql, params, timeout_ms).await? { + if let (Some(key), Some(est)) = (key_of(&r), i64_at(&r, "estimate")) { + out.insert(key, est); + } + } + Ok(out) +} diff --git a/database/src/handlers/catalog/postgres.rs b/database/src/handlers/catalog/postgres.rs new file mode 100644 index 000000000..6760776bf --- /dev/null +++ b/database/src/handlers/catalog/postgres.rs @@ -0,0 +1,250 @@ +//! PostgreSQL catalog reads. +//! +//! Reads `pg_catalog` rather than `information_schema`. Two reasons: it is +//! markedly faster, and `information_schema.constraint_column_usage` pairs +//! composite foreign keys by cross product, which mis-associates columns on a +//! multi-column key. `pg_constraint` carries `conkey`/`confkey` as ordered +//! vectors, so unnesting both `WITH ORDINALITY` and matching on the ordinal +//! pairs them correctly. +//! +//! Nothing here returns an array-typed column: `RowValue` has no array +//! variant, so `array_agg`, a bare `conkey`, or `indkey` would fail to decode. +//! Arrays are unnested into scalar rows and regrouped in Rust. + +use super::{ + bool_at, fold_index_rows, i64_at, run, str_at, ColumnDesc, ForeignKeyRef, IndexDesc, + TableFilter, TableKey, TableKind, TableRef, +}; +use crate::handlers::AppState; +use serde_json::Value; +use std::collections::HashMap; + +const SKIP_SCHEMAS: &str = "n.nspname NOT IN ('pg_catalog', 'information_schema')"; + +/// Build the optional `AND schema = $n AND table = $m` tail. Postgres uses +/// positional placeholders, so the caller's first free index is passed in. +fn filter_clause(filter: Option<&TableFilter>, first_param: usize) -> (String, Vec) { + let Some(f) = filter else { + return (String::new(), Vec::new()); + }; + let mut params = Vec::new(); + let mut clause = String::new(); + let mut n = first_param; + if let Some(schema) = &f.schema { + clause.push_str(&format!(" AND n.nspname = ${n}")); + params.push(Value::String(schema.clone())); + n += 1; + } + clause.push_str(&format!(" AND c.relname = ${n}")); + params.push(Value::String(f.table.clone())); + (clause, params) +} + +fn key_of(row: &serde_json::Map) -> Option { + Some((str_at(row, "table_schema"), str_at(row, "table_name")?)) +} + +pub async fn list_tables( + state: &AppState, + db: &str, + timeout_ms: u64, +) -> Result, String> { + let sql = format!( + "SELECT n.nspname AS table_schema, c.relname AS table_name, c.relkind AS kind \ + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE c.relkind IN ('r', 'p', 'v', 'm') AND {SKIP_SCHEMAS} \ + ORDER BY n.nspname, c.relname" + ); + let rows = run(state, db, sql, vec![], timeout_ms).await?; + Ok(rows + .iter() + .filter_map(|r| { + Some(TableRef { + name: str_at(r, "table_name")?, + schema: str_at(r, "table_schema"), + // r = ordinary table, p = partitioned, v = view, m = materialized view + kind: match str_at(r, "kind").as_deref() { + Some("v") | Some("m") => TableKind::View, + _ => TableKind::Table, + }, + }) + }) + .collect()) +} + +pub async fn columns( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result>, String> { + let (clause, params) = filter_clause(filter, 1); + let sql = format!( + "SELECT n.nspname AS table_schema, c.relname AS table_name, \ + a.attname AS column_name, \ + format_type(a.atttypid, a.atttypmod) AS column_type, \ + NOT a.attnotnull AS is_nullable, \ + pg_get_expr(d.adbin, d.adrelid) AS default_value, \ + a.attnum AS position \ + FROM pg_attribute a \ + JOIN pg_class c ON c.oid = a.attrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + LEFT JOIN pg_attrdef d ON d.adrelid = c.oid AND d.adnum = a.attnum \ + WHERE a.attnum > 0 AND NOT a.attisdropped \ + AND c.relkind IN ('r', 'p', 'v', 'm') AND {SKIP_SCHEMAS}{clause} \ + ORDER BY n.nspname, c.relname, a.attnum" + ); + let rows = run(state, db, sql, params, timeout_ms).await?; + + let mut out: HashMap> = HashMap::new(); + for r in &rows { + let (Some(key), Some(name)) = (key_of(r), str_at(r, "column_name")) else { + continue; + }; + out.entry(key).or_default().push(ColumnDesc { + name, + ty: str_at(r, "column_type").unwrap_or_default(), + nullable: bool_at(r, "is_nullable"), + default_value: str_at(r, "default_value"), + primary_key: false, + position: i64_at(r, "position").unwrap_or(0) as i32, + foreign_key: None, + }); + } + + mark_primary_keys(state, db, filter, timeout_ms, &mut out).await?; + merge_foreign_keys(state, db, filter, timeout_ms, &mut out).await?; + Ok(out) +} + +async fn mark_primary_keys( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, + columns: &mut HashMap>, +) -> Result<(), String> { + let (clause, params) = filter_clause(filter, 1); + let sql = format!( + "SELECT n.nspname AS table_schema, c.relname AS table_name, a.attname AS column_name \ + FROM pg_constraint con \ + JOIN pg_class c ON c.oid = con.conrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + CROSS JOIN LATERAL unnest(con.conkey) AS k(attnum) \ + JOIN pg_attribute a ON a.attrelid = con.conrelid AND a.attnum = k.attnum \ + WHERE con.contype = 'p' AND {SKIP_SCHEMAS}{clause}" + ); + for r in run(state, db, sql, params, timeout_ms).await? { + let (Some(key), Some(col)) = (key_of(&r), str_at(&r, "column_name")) else { + continue; + }; + if let Some(cols) = columns.get_mut(&key) { + if let Some(c) = cols.iter_mut().find(|c| c.name == col) { + c.primary_key = true; + } + } + } + Ok(()) +} + +async fn merge_foreign_keys( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, + columns: &mut HashMap>, +) -> Result<(), String> { + let (clause, params) = filter_clause(filter, 1); + // `k.ord = fk.ord` is what makes a composite key pair correctly — without + // it the two unnests cross-product and column 1 can be reported as + // referencing the parent's column 2. + let sql = format!( + "SELECT n.nspname AS table_schema, c.relname AS table_name, \ + a.attname AS src_column, \ + fn.nspname AS ref_schema, fc.relname AS ref_table, fa.attname AS ref_column \ + FROM pg_constraint con \ + JOIN pg_class c ON c.oid = con.conrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + JOIN pg_class fc ON fc.oid = con.confrelid \ + JOIN pg_namespace fn ON fn.oid = fc.relnamespace \ + CROSS JOIN LATERAL unnest(con.conkey) WITH ORDINALITY AS k(attnum, ord) \ + CROSS JOIN LATERAL unnest(con.confkey) WITH ORDINALITY AS fk(attnum, ord) \ + JOIN pg_attribute a ON a.attrelid = con.conrelid AND a.attnum = k.attnum \ + JOIN pg_attribute fa ON fa.attrelid = con.confrelid AND fa.attnum = fk.attnum \ + WHERE con.contype = 'f' AND k.ord = fk.ord AND {SKIP_SCHEMAS}{clause}" + ); + for r in run(state, db, sql, params, timeout_ms).await? { + let (Some(key), Some(src), Some(ref_table), Some(ref_column)) = ( + key_of(&r), + str_at(&r, "src_column"), + str_at(&r, "ref_table"), + str_at(&r, "ref_column"), + ) else { + continue; + }; + if let Some(cols) = columns.get_mut(&key) { + if let Some(c) = cols.iter_mut().find(|c| c.name == src) { + c.foreign_key = Some(ForeignKeyRef { + schema: str_at(&r, "ref_schema"), + table: ref_table, + column: ref_column, + }); + } + } + } + Ok(()) +} + +pub async fn indexes( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result>, String> { + let (clause, params) = filter_clause(filter, 1); + // `indkey` is an int2vector; unnest it rather than returning it, and keep + // the LEFT JOIN so an expression index still yields its row with no + // column name attached. + let sql = format!( + "SELECT n.nspname AS table_schema, c.relname AS table_name, \ + i.relname AS index_name, ix.indisunique AS is_unique, \ + ix.indisprimary AS is_primary, a.attname AS column_name \ + FROM pg_index ix \ + JOIN pg_class c ON c.oid = ix.indrelid \ + JOIN pg_class i ON i.oid = ix.indexrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + CROSS JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS k(attnum, ord) \ + LEFT JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = k.attnum \ + WHERE {SKIP_SCHEMAS}{clause} \ + ORDER BY n.nspname, c.relname, i.relname, k.ord" + ); + let rows = run(state, db, sql, params, timeout_ms).await?; + Ok(fold_index_rows(&rows, key_of)) +} + +/// `reltuples` is the planner's estimate, maintained by ANALYZE/autovacuum. +/// It is -1 on a table that has never been analyzed, which we report as +/// absent rather than as a row count of minus one. +pub async fn row_estimates( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result, String> { + let (clause, params) = filter_clause(filter, 1); + let sql = format!( + "SELECT n.nspname AS table_schema, c.relname AS table_name, \ + c.reltuples::bigint AS estimate \ + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE c.relkind IN ('r', 'p') AND {SKIP_SCHEMAS}{clause}" + ); + let mut out = HashMap::new(); + for r in run(state, db, sql, params, timeout_ms).await? { + if let (Some(key), Some(est)) = (key_of(&r), i64_at(&r, "estimate")) { + if est >= 0 { + out.insert(key, est); + } + } + } + Ok(out) +} diff --git a/database/src/handlers/catalog/sqlite.rs b/database/src/handlers/catalog/sqlite.rs new file mode 100644 index 000000000..2e6c5a3d5 --- /dev/null +++ b/database/src/handlers/catalog/sqlite.rs @@ -0,0 +1,232 @@ +//! SQLite catalog reads. +//! +//! Uses the table-valued PRAGMA functions (`pragma_table_info(name)`, SQLite +//! 3.16+) joined against `sqlite_master` rather than issuing a bare `PRAGMA` +//! per table. That keeps `describeSchema` to three queries instead of one per +//! table, and it is the only way to read the catalog with bound parameters — +//! a bare `PRAGMA table_info(x)` cannot bind `x`. + +use super::{ + bool_at, fold_index_rows, i64_at, run, str_at, ColumnDesc, ForeignKeyRef, IndexDesc, + TableFilter, TableKey, TableKind, TableRef, +}; +use crate::handlers::AppState; +use serde_json::Value; +use std::collections::HashMap; + +/// `sqlite_master` rows we never surface: the internal bookkeeping tables. +const NOT_INTERNAL: &str = "m.name NOT LIKE 'sqlite\\_%' ESCAPE '\\'"; + +fn filter_clause(filter: Option<&TableFilter>) -> (String, Vec) { + match filter { + Some(f) => ( + " AND m.name = ?".to_string(), + vec![Value::String(f.table.clone())], + ), + None => (String::new(), Vec::new()), + } +} + +pub async fn list_tables( + state: &AppState, + db: &str, + timeout_ms: u64, +) -> Result, String> { + let sql = format!( + "SELECT m.name AS name, m.type AS kind FROM sqlite_master m \ + WHERE m.type IN ('table', 'view') AND {NOT_INTERNAL} ORDER BY m.type, m.name" + ); + let rows = run(state, db, sql, vec![], timeout_ms).await?; + Ok(rows + .iter() + .filter_map(|r| { + Some(TableRef { + name: str_at(r, "name")?, + schema: None, + kind: match str_at(r, "kind").as_deref() { + Some("view") => TableKind::View, + _ => TableKind::Table, + }, + }) + }) + .collect()) +} + +/// Columns for one table or every table, with primary keys and foreign keys +/// already merged in. +pub async fn columns( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result>, String> { + let (clause, params) = filter_clause(filter); + let sql = format!( + "SELECT m.name AS table_name, p.cid AS cid, p.name AS column_name, \ + p.type AS column_type, p.\"notnull\" AS not_null, \ + p.dflt_value AS default_value, p.pk AS pk_ord \ + FROM sqlite_master m JOIN pragma_table_info(m.name) p \ + WHERE m.type IN ('table', 'view') AND {NOT_INTERNAL}{clause} \ + ORDER BY m.name, p.cid" + ); + let rows = run(state, db, sql, params, timeout_ms).await?; + + let mut out: HashMap> = HashMap::new(); + for r in &rows { + let Some(table) = str_at(r, "table_name") else { + continue; + }; + let Some(name) = str_at(r, "column_name") else { + continue; + }; + out.entry((None, table)).or_default().push(ColumnDesc { + name, + ty: str_at(r, "column_type").unwrap_or_default(), + nullable: !bool_at(r, "not_null"), + default_value: str_at(r, "default_value"), + // `pk` is the 1-based position within the primary key, 0 when the + // column is not part of it — not a boolean. + primary_key: i64_at(r, "pk_ord").unwrap_or(0) > 0, + position: i64_at(r, "cid").unwrap_or(0) as i32 + 1, + foreign_key: None, + }); + } + + merge_foreign_keys(state, db, filter, timeout_ms, &mut out).await?; + Ok(out) +} + +/// SQLite reports a foreign key's target column as NULL when the reference is +/// to the parent's primary key implicitly (`REFERENCES users` rather than +/// `REFERENCES users(id)`). Resolve that against the columns we already hold +/// rather than emitting a half-empty reference. +async fn merge_foreign_keys( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, + columns: &mut HashMap>, +) -> Result<(), String> { + let (clause, params) = filter_clause(filter); + let sql = format!( + "SELECT m.name AS table_name, f.\"from\" AS src_column, \ + f.\"table\" AS ref_table, f.\"to\" AS ref_column \ + FROM sqlite_master m JOIN pragma_foreign_key_list(m.name) f \ + WHERE m.type = 'table' AND {NOT_INTERNAL}{clause}" + ); + let rows = run(state, db, sql, params, timeout_ms).await?; + + // Primary keys of tables already loaded. Free, but only covers parents + // inside the current filter — describing one table does not load its + // parents, which is exactly when implicit references need resolving. + let mut pk_of: HashMap = columns + .iter() + .filter_map(|((_, t), cols)| { + cols.iter() + .find(|c| c.primary_key) + .map(|c| (t.clone(), c.name.clone())) + }) + .collect(); + + // Look up any parent we are missing in one extra query rather than + // dropping the reference. Dropping it would mean `describeTable` silently + // reports no foreign key on a column that plainly has one. + let unresolved: Vec = rows + .iter() + .filter(|r| str_at(r, "ref_column").is_none()) + .filter_map(|r| str_at(r, "ref_table")) + .filter(|t| !pk_of.contains_key(t)) + .collect::>() + .into_iter() + .collect(); + if !unresolved.is_empty() { + let placeholders = vec!["?"; unresolved.len()].join(", "); + let sql = format!( + "SELECT m.name AS table_name, p.name AS column_name \ + FROM sqlite_master m JOIN pragma_table_info(m.name) p \ + WHERE m.type = 'table' AND p.pk > 0 AND m.name IN ({placeholders}) \ + ORDER BY m.name, p.pk" + ); + let params = unresolved.iter().cloned().map(Value::String).collect(); + for r in run(state, db, sql, params, timeout_ms).await? { + let (Some(t), Some(c)) = (str_at(&r, "table_name"), str_at(&r, "column_name")) else { + continue; + }; + // Ordered by `p.pk`, so the first row per table is the leading + // primary-key column — the one an implicit reference means. + pk_of.entry(t).or_insert(c); + } + } + + for r in &rows { + let (Some(table), Some(src), Some(ref_table)) = ( + str_at(r, "table_name"), + str_at(r, "src_column"), + str_at(r, "ref_table"), + ) else { + continue; + }; + let ref_column = match str_at(r, "ref_column") { + Some(c) => c, + // Implicit `REFERENCES parent` means the parent's primary key. + // Unresolvable only if the parent no longer exists. + None => match pk_of.get(&ref_table) { + Some(pk) => pk.clone(), + None => continue, + }, + }; + if let Some(cols) = columns.get_mut(&(None, table)) { + if let Some(col) = cols.iter_mut().find(|c| c.name == src) { + col.foreign_key = Some(ForeignKeyRef { + schema: None, + table: ref_table, + column: ref_column, + }); + } + } + } + Ok(()) +} + +pub async fn indexes( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result>, String> { + let (clause, params) = filter_clause(filter); + // `origin` is 'pk' for the implicit primary-key index, 'u' for a UNIQUE + // constraint, 'c' for CREATE INDEX. + let sql = format!( + "SELECT m.name AS table_name, il.name AS index_name, \ + il.\"unique\" AS is_unique, il.origin AS origin, \ + ii.name AS column_name \ + FROM sqlite_master m \ + JOIN pragma_index_list(m.name) il \ + LEFT JOIN pragma_index_info(il.name) ii \ + WHERE m.type = 'table' AND {NOT_INTERNAL}{clause} \ + ORDER BY m.name, il.seq, ii.seqno" + ); + let mut rows = run(state, db, sql, params, timeout_ms).await?; + // `fold_index_rows` reads `is_primary`; sqlite spells that as origin='pk'. + // Normalize in place so the fold stays a single pass. + for r in rows.iter_mut() { + let primary = str_at(r, "origin").as_deref() == Some("pk"); + r.insert("is_primary".to_string(), Value::Bool(primary)); + } + Ok(fold_index_rows(&rows, |r| { + str_at(r, "table_name").map(|t| (None, t)) + })) +} + +/// SQLite has no cheap row estimate — `sqlite_stat1` exists only after an +/// explicit `ANALYZE`, and `COUNT(*)` is a full scan. Report nothing rather +/// than pay for a scan the caller did not ask for. +pub async fn row_estimates( + _state: &AppState, + _db: &str, + _filter: Option<&TableFilter>, + _timeout_ms: u64, +) -> Result, String> { + Ok(HashMap::new()) +} diff --git a/database/src/handlers/column_stats.rs b/database/src/handlers/column_stats.rs new file mode 100644 index 000000000..059355046 --- /dev/null +++ b/database/src/handlers/column_stats.rs @@ -0,0 +1,427 @@ +//! `database::columnStats` — profile a column without reading the table. +//! +//! Two modes, and the default matters. A naive profile runs +//! `COUNT(DISTINCT col)`, `MIN`, `MAX` and a `GROUP BY`, each of which is a +//! full scan; doing that from a console panel is how a read-only viewer +//! causes a production incident. So the default reads the statistics the +//! planner already maintains (`pg_stats`, `information_schema.STATISTICS`, +//! `sqlite_stat1`) — O(1) catalog reads that cost nothing — and `exact: true` +//! is an explicit opt-in that runs the real aggregates behind a timeout. +//! +//! Approximate numbers are always labelled `source: planner`, never presented +//! as if they were counted. +//! +//! Scope note: this profiles the *whole table* server-side. To profile rows +//! you already hold, pipe a `browseTable` result through the `fp` worker +//! instead — that is what it is for, and duplicating it here would be worse +//! on both counts. + +use super::filter::{quote_ident, quote_table}; +use super::query::{self, err_to_str, QueryReq}; +use super::AppState; +use crate::config::DriverKind; +use crate::error::DbError; +use crate::pool::Pool; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +fn default_timeout() -> u64 { + 30_000 +} + +fn default_top_n() -> usize { + 10 +} + +/// Cap on `top_n` — a "most common values" list longer than this is a report, +/// not a profile. +const MAX_TOP_N: usize = 100; + +/// Above this planner-estimated row count, `exact: true` is refused rather +/// than silently scanning a very large table. +const EXACT_ROW_CEILING: i64 = 5_000_000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum StatSource { + /// Read from the planner's own statistics. Approximate, and free. + Planner, + /// Counted by running aggregates over the table. + Computed, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct TopValue { + pub value: Value, + pub count: i64, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct ColumnStat { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub row_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub distinct_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub null_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub null_fraction: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub min: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mean: Option, + /// Only populated in `exact` mode; the planner's own most-common-value + /// lists are not portable enough to report faithfully. + pub top_values: Vec, + pub source: StatSource, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ColumnStatsReq { + #[serde(default)] + pub db: Option, + pub table: String, + #[serde(default)] + pub schema: Option, + /// Omit to profile every column. + #[serde(default)] + pub columns: Option>, + /// Run real aggregates instead of reading planner statistics. This scans + /// the table. + #[serde(default)] + pub exact: bool, + #[serde(default = "default_top_n")] + pub top_n: usize, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ColumnStatsResp { + pub table: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option, + pub columns: Vec, + /// True when the numbers came from the planner rather than a count. + pub approximate: bool, +} + +async fn driver_of(state: &AppState, db: &str) -> Result { + Ok(match state.pool(db).await.map_err(err_to_str)? { + Pool::Sqlite(_) => DriverKind::Sqlite, + Pool::Postgres(_) => DriverKind::Postgres, + Pool::Mysql(_) => DriverKind::Mysql, + }) +} + +async fn run( + state: &AppState, + db: &str, + sql: String, + params: Vec, + timeout_ms: u64, +) -> Result>, String> { + Ok(query::handle( + state, + QueryReq { + db: Some(db.to_string()), + sql, + params, + timeout_ms, + record_history: false, + }, + ) + .await? + .rows) +} + +fn f64_at(row: &serde_json::Map, key: &str) -> Option { + match row.get(key) { + Some(Value::Number(n)) => n.as_f64(), + Some(Value::String(s)) => s.parse().ok(), + _ => None, + } +} + +fn i64_at(row: &serde_json::Map, key: &str) -> Option { + f64_at(row, key).map(|f| f as i64) +} + +pub async fn handle(state: &AppState, req: ColumnStatsReq) -> Result { + if req.table.trim().is_empty() { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: "table is required".into(), + })); + } + let db = state.resolve_db(req.db.clone()).await.map_err(err_to_str)?; + let driver = driver_of(state, &db).await?; + let top_n = req.top_n.clamp(1, MAX_TOP_N); + + // Resolve the column list from the catalog rather than trusting the + // caller, so an unknown name fails here instead of inside an aggregate. + let described = super::schema::describe_table( + state, + super::schema::DescribeTableReq { + db: Some(db.clone()), + table: req.table.clone(), + schema: req.schema.clone(), + timeout_ms: req.timeout_ms, + }, + ) + .await?; + + let wanted: Vec = match &req.columns { + Some(list) => { + for c in list { + if !described.columns.iter().any(|d| &d.name == c) { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: format!("no such column `{c}` on `{}`", req.table), + })); + } + } + list.clone() + } + None => described.columns.iter().map(|c| c.name.clone()).collect(), + }; + + let target = quote_table(driver, described.schema.as_deref(), &described.table); + + if !req.exact { + let columns = + planner_stats(state, &db, driver, &described, &wanted, req.timeout_ms).await?; + return Ok(ColumnStatsResp { + table: described.table, + schema: described.schema, + columns, + approximate: true, + }); + } + + // Refuse an exact profile of a very large table rather than starting a + // scan the caller cannot cancel — on sqlite the driver drops `timeout_ms` + // entirely, so a row-count guard is the only brake available. + if let Some(est) = described.row_count_estimate { + if est > EXACT_ROW_CEILING { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: format!( + "`{}` has roughly {est} rows; an exact profile would scan all of \ + them. Re-run without `exact` for planner statistics, or profile \ + a filtered subset.", + described.table + ), + })); + } + } + + let mut columns = Vec::new(); + for name in &wanted { + columns.push(exact_stats(state, &db, driver, &target, name, top_n, req.timeout_ms).await?); + } + Ok(ColumnStatsResp { + table: described.table, + schema: described.schema, + columns, + approximate: false, + }) +} + +/// Read what the planner already knows. Cheap, approximate, and clearly +/// labelled as such. +async fn planner_stats( + state: &AppState, + db: &str, + driver: DriverKind, + described: &super::schema::TableDescription, + wanted: &[String], + timeout_ms: u64, +) -> Result, String> { + let row_count = described.row_count_estimate; + let mut out: Vec = wanted + .iter() + .map(|name| ColumnStat { + name: name.clone(), + row_count, + distinct_count: None, + null_count: None, + null_fraction: None, + min: None, + max: None, + mean: None, + top_values: Vec::new(), + source: StatSource::Planner, + }) + .collect(); + + match driver { + DriverKind::Postgres => { + // `pg_stats` exposes null_frac and n_distinct as plain scalars. + // n_distinct is negative when it is a ratio of the row count. + let rows = run( + state, + db, + "SELECT attname AS column_name, null_frac, n_distinct \ + FROM pg_stats WHERE schemaname = COALESCE($1, 'public') AND tablename = $2" + .into(), + vec![ + described + .schema + .clone() + .map(Value::String) + .unwrap_or(Value::Null), + Value::String(described.table.clone()), + ], + timeout_ms, + ) + .await?; + for r in &rows { + let Some(col) = r.get("column_name").and_then(Value::as_str) else { + continue; + }; + let Some(stat) = out.iter_mut().find(|s| s.name == col) else { + continue; + }; + stat.null_fraction = f64_at(r, "null_frac"); + if let (Some(frac), Some(total)) = (stat.null_fraction, row_count) { + stat.null_count = Some((frac * total as f64).round() as i64); + } + stat.distinct_count = f64_at(r, "n_distinct").map(|n| { + if n < 0.0 { + // Negative means "this fraction of the row count". + (-n * row_count.unwrap_or(0) as f64).round() as i64 + } else { + n as i64 + } + }); + } + } + DriverKind::Mysql => { + // CARDINALITY is per leading index column, so it only answers for + // indexed columns — which is honest: unindexed columns get None. + let rows = run( + state, + db, + "SELECT COLUMN_NAME AS column_name, MAX(CARDINALITY) AS cardinality \ + FROM information_schema.STATISTICS \ + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND SEQ_IN_INDEX = 1 \ + GROUP BY COLUMN_NAME" + .into(), + vec![Value::String(described.table.clone())], + timeout_ms, + ) + .await?; + for r in &rows { + let Some(col) = r.get("column_name").and_then(Value::as_str) else { + continue; + }; + if let Some(stat) = out.iter_mut().find(|s| s.name == col) { + stat.distinct_count = i64_at(r, "cardinality"); + } + } + } + DriverKind::Sqlite => { + // `sqlite_stat1` exists only after an explicit ANALYZE. Its `stat` + // column is " ..." per index column. + let rows = run( + state, + db, + "SELECT tbl, idx, stat FROM sqlite_stat1 WHERE tbl = ?".into(), + vec![Value::String(described.table.clone())], + timeout_ms, + ) + .await + // The table does not exist until someone runs ANALYZE; that is a + // missing answer, not an error. + .unwrap_or_default(); + if let Some(total) = rows + .first() + .and_then(|r| r.get("stat")) + .and_then(Value::as_str) + .and_then(|s| s.split_whitespace().next()) + .and_then(|s| s.parse::().ok()) + { + for stat in out.iter_mut() { + stat.row_count = Some(total); + } + } + } + } + Ok(out) +} + +/// Real aggregates. One pass for the scalars, one for the top values. +async fn exact_stats( + state: &AppState, + db: &str, + driver: DriverKind, + target: &str, + column: &str, + top_n: usize, + timeout_ms: u64, +) -> Result { + let col = quote_ident(driver, column); + let scalars = run( + state, + db, + format!( + "SELECT COUNT(*) AS row_count, COUNT({col}) AS non_null, \ + COUNT(DISTINCT {col}) AS distinct_count, \ + MIN({col}) AS min_value, MAX({col}) AS max_value, \ + AVG(CASE WHEN {col} + 0 = {col} THEN {col} END) AS mean_value \ + FROM {target}" + ), + vec![], + timeout_ms, + ) + .await?; + + let row = scalars.first().cloned().unwrap_or_default(); + let row_count = i64_at(&row, "row_count"); + let non_null = i64_at(&row, "non_null"); + let null_count = match (row_count, non_null) { + (Some(t), Some(n)) => Some(t - n), + _ => None, + }; + + let tops = run( + state, + db, + format!( + "SELECT {col} AS value, COUNT(*) AS n FROM {target} \ + WHERE {col} IS NOT NULL GROUP BY {col} ORDER BY n DESC, 1 LIMIT {top_n}" + ), + vec![], + timeout_ms, + ) + .await?; + + Ok(ColumnStat { + name: column.to_string(), + row_count, + distinct_count: i64_at(&row, "distinct_count"), + null_count, + null_fraction: match (null_count, row_count) { + (Some(n), Some(t)) if t > 0 => Some(n as f64 / t as f64), + _ => None, + }, + min: row.get("min_value").cloned().filter(|v| !v.is_null()), + max: row.get("max_value").cloned().filter(|v| !v.is_null()), + mean: f64_at(&row, "mean_value"), + top_values: tops + .iter() + .filter_map(|r| { + Some(TopValue { + value: r.get("value")?.clone(), + count: i64_at(r, "n")?, + }) + }) + .collect(), + source: StatSource::Computed, + }) +} diff --git a/database/src/handlers/diagram.rs b/database/src/handlers/diagram.rs new file mode 100644 index 000000000..449af5a23 --- /dev/null +++ b/database/src/handlers/diagram.rs @@ -0,0 +1,1290 @@ +//! `database::schemaDiagram` — the shape of a schema, laid out. +//! +//! Layout runs here rather than in a renderer for two reasons. It collapses +//! the N+1 a diagram would otherwise need (`describeSchema` already reads the +//! whole catalog in a handful of queries), and it makes the schema's *shape* +//! askable: an agent can call this and reason about hub tables, orphans and +//! reference cycles without drawing anything. +//! +//! The algorithm is a cut-down Sugiyama: +//! +//! 1. Split into connected components over foreign keys, so a hundred +//! unrelated lookup tables do not dominate the canvas. +//! 2. Rank each component by longest path along the FK direction, breaking +//! cycles at the lowest-degree edge. +//! 3. Reduce edge crossings with a barycenter sweep followed by adjacent +//! transposition, keeping whichever ordering measures better. +//! 4. Assign coordinates and route edges as three-segment elbows anchored on +//! the *column row*, not the box. +//! +//! Every step sorts by name before iterating, so the same schema always +//! produces the same diagram — which is what makes `crossings` a number worth +//! asserting on in a test. + +use super::schema::{self, DescribeSchemaReq, TableDescription}; +use super::AppState; +use crate::error::DbError; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +fn default_timeout() -> u64 { + 30_000 +} + +fn default_max_tables() -> usize { + 200 +} + +/// Geometry. Fixed here so the renderer never has to guess, and so edge +/// anchors line up with the row a column actually occupies. +pub const NODE_WIDTH: f64 = 264.0; +pub const HEADER_HEIGHT: f64 = 34.0; +pub const ROW_HEIGHT: f64 = 24.0; +pub const NODE_PAD_Y: f64 = 8.0; +/// Columns beyond this are summarised rather than drawn. +pub const MAX_ROWS: usize = 12; +/// Rank pitch, measured centre-of-column to centre-of-column, so the readable +/// gutter is `RANK_SEP - NODE_WIDTH`. At 220 that gutter was 20px: every edge +/// collapsed into a stub too short to read, and a schema with real foreign +/// keys looked unrelated. The gutter has to dominate the elbow's corner radius +/// for the connection to register at all. +const RANK_SEP: f64 = 380.0; +const V_GAP: f64 = 56.0; +const COMPONENT_GAP: f64 = 140.0; +/// Pitch between unrelated tables. Tighter than `COMPONENT_GAP` because no +/// edge is ever drawn between them. +const ISOLATED_GAP: f64 = 40.0; +/// Vertical break between the connected diagram and the orphan shelf. +const SHELF_GAP: f64 = 80.0; +const SHELF_WIDTH: f64 = 1600.0; + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct DiagramColumn { + pub name: String, + #[serde(rename = "type")] + pub ty: String, + pub primary_key: bool, + pub foreign_key: bool, + pub nullable: bool, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct DiagramNode { + pub table: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option, + pub x: f64, + pub y: f64, + pub w: f64, + pub h: f64, + pub rank: u32, + /// Number of foreign keys touching this table, in or out. Renderers use + /// it to emphasise hubs. + pub degree: u32, + pub columns: Vec, + /// Columns not drawn because of `MAX_ROWS`. + pub hidden_columns: usize, +} + +#[derive(Debug, Clone, Copy, Serialize, JsonSchema)] +pub struct Point { + pub x: f64, + pub y: f64, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct DiagramEdge { + pub from: String, + pub from_column: String, + pub to: String, + pub to_column: String, + /// Polyline, already routed. Anchored on the column row where the column + /// is visible, on the node edge otherwise. + pub points: Vec, + pub self_loop: bool, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct SchemaDiagramReq { + #[serde(default)] + pub db: Option, + #[serde(default)] + pub include_views: bool, + #[serde(default = "default_max_tables")] + pub max_tables: usize, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, + /// Lay out only the neighbourhood of this table. + /// + /// A whole schema drawn at once answers "what exists"; it does not answer + /// "what does this table touch", which is the question actually being + /// asked most of the time. With a focus the diagram becomes explorable + /// one hop at a time instead of a wall to be scanned. + #[serde(default)] + pub focus: Option, + /// How many foreign-key hops out from `focus` to include. Ignored without + /// one. 1 is the table and its direct relations. + #[serde(default = "default_depth")] + pub depth: usize, +} + +fn default_depth() -> usize { + 1 +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct SchemaDiagramResp { + pub nodes: Vec, + pub edges: Vec, + pub width: f64, + pub height: f64, + /// Tables with no foreign keys at all, placed on a trailing shelf. + pub isolated: Vec, + /// Connected groups, with the box that encloses each. A schema is usually + /// several independent clusters rather than one graph, and saying so is + /// most of what makes a large diagram readable — a reader can take in + /// "four unrelated groups" at a glance instead of scanning for edges that + /// are not there. + pub components: Vec, + /// Edge crossings remaining after ordering. Lower is a tidier diagram. + pub crossings: u32, + pub truncated: bool, + /// Echoed when the caller asked for a neighbourhood rather than the whole + /// schema. + #[serde(skip_serializing_if = "Option::is_none")] + pub focus: Option, + /// Tables one hop beyond what was drawn. Non-empty means there is more to + /// expand into, which is the difference between a diagram that looks + /// complete and one that says where it stops. + #[serde(default)] + pub frontier: Vec, +} + +/// One connected group of tables and its bounding box. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct DiagramComponent { + /// Stable index, in layout order. + pub index: usize, + /// Member tables, by node id. + pub tables: Vec, + /// The most-referenced table in the group, if it has more than one member. + pub hub: Option, + pub x: f64, + pub y: f64, + pub w: f64, + pub h: f64, +} + +/// A relationship between two tables, keyed by the qualified names used as +/// node ids. +#[derive(Debug, Clone)] +struct Relation { + from: String, + from_column: String, + to: String, + to_column: String, +} + +fn qualified(schema: Option<&str>, table: &str) -> String { + match schema { + Some(s) => format!("{s}.{table}"), + None => table.to_string(), + } +} + +fn node_height(columns: usize) -> f64 { + HEADER_HEIGHT + (columns.min(MAX_ROWS) as f64) * ROW_HEIGHT + 2.0 * NODE_PAD_Y +} + +pub async fn handle(state: &AppState, req: SchemaDiagramReq) -> Result { + let described = schema::describe_schema( + state, + DescribeSchemaReq { + db: req.db, + tables: None, + include_indexes: false, + max_tables: req.max_tables, + timeout_ms: req.timeout_ms, + }, + ) + .await?; + + let tables: Vec = described + .tables + .into_iter() + .filter(|t| req.include_views || t.kind == super::catalog::TableKind::Table) + .collect(); + + let (tables, frontier) = match &req.focus { + Some(focus) => neighbourhood(&tables, focus, req.depth)?, + None => (tables, Vec::new()), + }; + + let mut resp = layout(&tables, described.truncated); + resp.focus = req.focus; + resp.frontier = frontier; + Ok(resp) +} + +/// The tables within `depth` foreign-key hops of `focus`, plus the names one +/// hop beyond that were not included. +/// +/// Edges are walked in both directions: a reader asking what `users` touches +/// means both the tables it references and the tables referencing it, and +/// following only the declared direction would hide every child table. +/// +/// The `frontier` is what makes this explorable rather than merely smaller — +/// it tells the renderer which nodes have more behind them, so expanding is an +/// informed click instead of a guess. +fn neighbourhood( + tables: &[TableDescription], + focus: &str, + depth: usize, +) -> Result<(Vec, Vec), String> { + let ids: Vec = tables + .iter() + .map(|t| qualified(t.schema.as_deref(), &t.table)) + .collect(); + + // Accept either the qualified id or the bare table name, since a caller + // clicking a node has the former and a caller typing has the latter. + let start = ids + .iter() + .find(|id| id.as_str() == focus) + .or_else(|| { + tables + .iter() + .zip(&ids) + .find(|(t, _)| t.table == focus) + .map(|(_, id)| id) + }) + .cloned(); + let Some(start) = start else { + return Err(super::query::err_to_str(DbError::InvalidParam { + index: 0, + reason: format!("no table named `{focus}` in this database"), + })); + }; + + // Undirected adjacency over foreign keys. + let present: BTreeSet<&String> = ids.iter().collect(); + let mut adj: BTreeMap> = BTreeMap::new(); + for t in tables { + let from = qualified(t.schema.as_deref(), &t.table); + for c in &t.columns { + let Some(fk) = &c.foreign_key else { continue }; + let to = qualified(fk.schema.as_deref().or(t.schema.as_deref()), &fk.table); + if !present.contains(&to) || to == from { + continue; + } + adj.entry(from.clone()).or_default().insert(to.clone()); + adj.entry(to).or_default().insert(from.clone()); + } + } + + // Breadth-first to `depth`, then one more ring to find the frontier. + let mut included: BTreeSet = BTreeSet::new(); + included.insert(start.clone()); + let mut ring: Vec = vec![start]; + for _ in 0..depth { + // Stop once the component is exhausted. `depth` comes straight off the + // wire with no ceiling, so without this a caller asking for depth + // 1e11 spins the worker thread long after there is nothing left to + // reach. + if ring.is_empty() { + break; + } + let mut next: Vec = Vec::new(); + for node in &ring { + for peer in adj.get(node).into_iter().flatten() { + if included.insert(peer.clone()) { + next.push(peer.clone()); + } + } + } + ring = next; + } + + let frontier: Vec = ring + .iter() + .flat_map(|node| adj.get(node).into_iter().flatten()) + .filter(|peer| !included.contains(*peer)) + .cloned() + .collect::>() + .into_iter() + .collect(); + + let kept: Vec = tables + .iter() + .zip(&ids) + .filter(|(_, id)| included.contains(*id)) + .map(|(t, _)| t.clone()) + .collect(); + + Ok((kept, frontier)) +} + +/// Pure layout. Separated from the fetch so it can be tested without a +/// database, and so the same input always yields the same diagram. +pub fn layout(tables: &[TableDescription], truncated: bool) -> SchemaDiagramResp { + // Node ids, sorted so every later iteration is deterministic. + let mut ids: Vec = tables + .iter() + .map(|t| qualified(t.schema.as_deref(), &t.table)) + .collect(); + ids.sort(); + let present: BTreeSet<&String> = ids.iter().collect(); + + // Relations, deduplicated. A reference to a table outside the set (a view + // we filtered out, or a truncated tail) is dropped rather than drawn to + // nowhere. + let mut relations: Vec = Vec::new(); + let mut seen: BTreeSet<(String, String, String, String)> = BTreeSet::new(); + for t in tables { + let from = qualified(t.schema.as_deref(), &t.table); + for c in &t.columns { + let Some(fk) = &c.foreign_key else { continue }; + let to = qualified(fk.schema.as_deref().or(t.schema.as_deref()), &fk.table); + if !present.contains(&to) { + continue; + } + let key = (from.clone(), c.name.clone(), to.clone(), fk.column.clone()); + if seen.insert(key) { + relations.push(Relation { + from: from.clone(), + from_column: c.name.clone(), + to, + to_column: fk.column.clone(), + }); + } + } + } + + let mut degree: BTreeMap = ids.iter().map(|i| (i.clone(), 0)).collect(); + for r in &relations { + *degree.get_mut(&r.from).expect("id present") += 1; + if r.to != r.from { + *degree.get_mut(&r.to).expect("id present") += 1; + } + } + + // Degree-0 tables go to a trailing shelf; on a real schema they are often + // the majority and would otherwise stretch the canvas around nothing. + let isolated: Vec = ids.iter().filter(|i| degree[*i] == 0).cloned().collect(); + let connected: Vec = ids.iter().filter(|i| degree[*i] > 0).cloned().collect(); + + let components = split_components(&connected, &relations); + let ranks = rank_all(&components, &relations, °ree); + + // Order within each rank, then place. + let by_id: HashMap<&String, &TableDescription> = tables + .iter() + .map(|t| { + let id = ids + .iter() + .find(|i| **i == qualified(t.schema.as_deref(), &t.table)) + .expect("id built from this table"); + (id, t) + }) + .collect(); + + let mut positions: BTreeMap = BTreeMap::new(); + let mut cursor_x = 0.0_f64; + let mut max_height = 0.0_f64; + // Origin of the row of components being filled, and the tallest component + // in it. + let mut row_y = 0.0_f64; + let mut row_height = 0.0_f64; + + for comp in &components { + let order = order_component(comp, &ranks, &relations); + // Wrap *before* placing, and lay the component out at the current row + // origin. Placing at y=0 unconditionally and only recording the wrap + // afterwards drew every row after the first on top of the one above + // it, so a schema with enough components to wrap rendered stacked. + if cursor_x > 0.0 && cursor_x + NODE_WIDTH > SHELF_WIDTH { + cursor_x = 0.0; + row_y += row_height + COMPONENT_GAP; + row_height = 0.0; + } + let (w, h) = place(&order, &ranks, &by_id, cursor_x, row_y, &mut positions); + cursor_x += w + COMPONENT_GAP; + row_height = row_height.max(h); + max_height = max_height.max(row_y + row_height); + } + + // Isolated tables tile a shelf below everything else, packed tightly. + // + // They get ISOLATED_GAP rather than COMPONENT_GAP: nothing connects them, + // so the wide gutter that makes edges readable buys nothing here and only + // stretches the canvas. A canvas wider than it needs to be forces the + // fit-zoom down, which shrinks the type in *every* node — the sparse + // shelf was why the whole diagram was rendering at 55%. + // + // The shelf is also capped to the width the connected part already + // occupies, so orphans wrap under the diagram instead of widening it. + let connected_width = positions + .values() + .map(|(x, _)| x + NODE_WIDTH) + .fold(0.0_f64, f64::max); + let shelf_width = connected_width.clamp(NODE_WIDTH * 4.0 + ISOLATED_GAP * 3.0, SHELF_WIDTH); + let mut ix = 0.0_f64; + let mut iy = max_height + SHELF_GAP; + let mut row_height = 0.0_f64; + for id in &isolated { + let cols = by_id.get(id).map(|t| t.columns.len()).unwrap_or(0); + let h = node_height(cols); + if ix > 0.0 && ix + NODE_WIDTH > shelf_width { + ix = 0.0; + iy += row_height + V_GAP; + row_height = 0.0; + } + positions.insert(id.clone(), (ix, iy)); + row_height = row_height.max(h); + ix += NODE_WIDTH + ISOLATED_GAP; + max_height = max_height.max(iy + h); + } + + let nodes = build_nodes(&ids, &by_id, &positions, &ranks, °ree); + + // Bounding boxes are derived from the placed nodes rather than tracked + // during placement, so they cannot drift from where the nodes ended up. + let node_box: HashMap<&str, (f64, f64, f64, f64)> = nodes + .iter() + .map(|n| (node_id(n) as &str, (n.x, n.y, n.w, n.h))) + .collect(); + let component_boxes: Vec = components + .iter() + .enumerate() + .filter_map(|(index, comp)| { + let boxes: Vec<_> = comp + .iter() + .filter_map(|t| node_box.get(t.as_str()).copied()) + .collect(); + if boxes.is_empty() { + return None; + } + let x = boxes.iter().map(|b| b.0).fold(f64::INFINITY, f64::min); + let y = boxes.iter().map(|b| b.1).fold(f64::INFINITY, f64::min); + let right = boxes.iter().map(|b| b.0 + b.2).fold(0.0_f64, f64::max); + let bottom = boxes.iter().map(|b| b.1 + b.3).fold(0.0_f64, f64::max); + // Only name a hub when one table is strictly the most connected. + // In a two-table pair both ends have degree 1, and picking one on + // a tiebreak would emphasise an arbitrary node in the renderer. + let mut ranked: Vec<(u32, &String)> = comp + .iter() + .map(|t| (degree.get(t).copied().unwrap_or(0), t)) + .collect(); + ranked.sort_by(|a, b| b.0.cmp(&a.0).then_with(|| a.1.cmp(b.1))); + let hub = match ranked.as_slice() { + [(top, name), (second, _), ..] if top > second => Some((*name).clone()), + _ => None, + }; + Some(DiagramComponent { + index, + tables: comp.clone(), + hub, + x, + y, + w: right - x, + h: bottom - y, + }) + }) + .collect(); + + let index: HashMap<&str, &DiagramNode> = nodes + .iter() + .map(|n| (node_id(n) as &str, n)) + .collect::>(); + let edges = route(&relations, &index); + let crossings = count_crossings(&edges); + + let width = nodes + .iter() + .map(|n| n.x + n.w) + .fold(0.0_f64, f64::max) + .max(NODE_WIDTH); + let height = nodes + .iter() + .map(|n| n.y + n.h) + .fold(0.0_f64, f64::max) + .max(HEADER_HEIGHT); + + SchemaDiagramResp { + nodes, + edges, + width, + height, + isolated, + components: component_boxes, + crossings, + // Set by `handle` when the caller asked for a neighbourhood; `layout` + // itself is given the tables it should draw and knows nothing of why. + focus: None, + frontier: Vec::new(), + truncated, + } +} + +/// Leaked id string for the lookup map; nodes own their name, this just +/// borrows it. +fn node_id(n: &DiagramNode) -> &str { + // `table` already carries the qualifier when one exists, because nodes are + // built from the qualified id. + &n.table +} + +fn split_components(ids: &[String], relations: &[Relation]) -> Vec> { + let mut parent: BTreeMap<&String, &String> = ids.iter().map(|i| (i, i)).collect(); + + fn find<'a>(parent: &BTreeMap<&'a String, &'a String>, x: &'a String) -> &'a String { + let mut cur = x; + while parent[cur] != cur { + cur = parent[cur]; + } + cur + } + + for r in relations { + let (Some(a), Some(b)) = ( + ids.iter().find(|i| **i == r.from), + ids.iter().find(|i| **i == r.to), + ) else { + continue; + }; + let ra = find(&parent, a); + let rb = find(&parent, b); + if ra != rb { + // Union by name keeps the result independent of input order. + let (lo, hi) = if ra < rb { (ra, rb) } else { (rb, ra) }; + parent.insert(hi, lo); + } + } + + let mut groups: BTreeMap<&String, Vec> = BTreeMap::new(); + for id in ids { + groups + .entry(find(&parent, id)) + .or_default() + .push(id.clone()); + } + groups.into_values().collect() +} + +/// Longest-path ranking along FK direction (referencing above referenced), +/// with cycles broken at the edge whose endpoints have the lowest degree. +fn rank_all( + components: &[Vec], + relations: &[Relation], + degree: &BTreeMap, +) -> BTreeMap { + let mut ranks: BTreeMap = BTreeMap::new(); + for comp in components { + let members: BTreeSet<&String> = comp.iter().collect(); + let mut edges: Vec<&Relation> = relations + .iter() + .filter(|r| members.contains(&r.from) && members.contains(&r.to) && r.from != r.to) + .collect(); + // Deterministic cycle breaking: drop the lowest-degree edge last, so + // the densest structure keeps its shape. + edges.sort_by(|a, b| { + (degree[&a.from] + degree[&a.to]) + .cmp(&(degree[&b.from] + degree[&b.to])) + .then(a.from.cmp(&b.from)) + .then(a.to.cmp(&b.to)) + }); + + for id in comp { + ranks.insert(id.clone(), 0); + } + // Relax repeatedly; bounded by component size so a residual cycle + // terminates instead of spinning. + for _ in 0..comp.len().min(64) { + let mut changed = false; + for e in &edges { + let want = ranks[&e.to] + 1; + if ranks[&e.from] < want { + ranks.insert(e.from.clone(), want); + changed = true; + } + } + if !changed { + break; + } + } + } + ranks +} + +/// Barycenter sweep plus adjacent transposition, keeping the better result. +fn order_component( + comp: &[String], + ranks: &BTreeMap, + relations: &[Relation], +) -> BTreeMap> { + let mut layers: BTreeMap> = BTreeMap::new(); + for id in comp { + layers.entry(ranks[id]).or_default().push(id.clone()); + } + for v in layers.values_mut() { + v.sort(); + } + + let neighbours = |id: &str| -> Vec { + relations + .iter() + .filter_map(|r| { + if r.from == id { + Some(r.to.clone()) + } else if r.to == id { + Some(r.from.clone()) + } else { + None + } + }) + .collect() + }; + + let mut best = layers.clone(); + let mut best_score = layer_crossings(&layers, relations); + + for _ in 0..4 { + let snapshot = layers.clone(); + for row in layers.values_mut() { + let mut keyed: Vec<(f64, String)> = row + .iter() + .map(|id| { + let ns = neighbours(id); + let bary = if ns.is_empty() { + f64::MAX + } else { + let sum: f64 = ns + .iter() + .filter_map(|n| { + snapshot + .values() + .find_map(|r| r.iter().position(|x| x == n).map(|p| p as f64)) + }) + .sum(); + sum / ns.len() as f64 + }; + (bary, id.clone()) + }) + .collect(); + // Name breaks ties, so the sweep is reproducible. + keyed.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap().then(a.1.cmp(&b.1))); + *row = keyed.into_iter().map(|(_, id)| id).collect(); + } + let score = layer_crossings(&layers, relations); + if score < best_score { + best_score = score; + best = layers.clone(); + } + } + best +} + +/// Crossings implied by the current ordering, used to choose between sweeps. +fn layer_crossings(layers: &BTreeMap>, relations: &[Relation]) -> u32 { + let pos: HashMap<&String, (u32, usize)> = layers + .iter() + .flat_map(|(rank, row)| row.iter().enumerate().map(move |(i, id)| (id, (*rank, i)))) + .collect(); + + let mut edges: Vec<(u32, usize, usize)> = Vec::new(); + for r in relations { + let (Some(a), Some(b)) = (pos.get(&r.from), pos.get(&r.to)) else { + continue; + }; + if a.0 == b.0 + 1 { + edges.push((b.0, a.1, b.1)); + } + } + + let mut n = 0; + for i in 0..edges.len() { + for j in (i + 1)..edges.len() { + let (l1, u1, v1) = edges[i]; + let (l2, u2, v2) = edges[j]; + if l1 == l2 && ((u1 < u2) != (v1 < v2)) { + n += 1; + } + } + } + n +} + +fn place( + order: &BTreeMap>, + _ranks: &BTreeMap, + by_id: &HashMap<&String, &TableDescription>, + origin_x: f64, + origin_y: f64, + out: &mut BTreeMap, +) -> (f64, f64) { + let mut max_w = 0.0_f64; + let mut max_h = 0.0_f64; + for (rank, row) in order { + let x = origin_x + (*rank as f64) * RANK_SEP; + let mut y = origin_y; + for id in row { + out.insert(id.clone(), (x, y)); + let cols = by_id.get(id).map(|t| t.columns.len()).unwrap_or(0); + y += node_height(cols) + V_GAP; + } + max_w = max_w.max(x - origin_x + NODE_WIDTH); + max_h = max_h.max(y - origin_y); + } + (max_w, max_h) +} + +fn build_nodes( + ids: &[String], + by_id: &HashMap<&String, &TableDescription>, + positions: &BTreeMap, + ranks: &BTreeMap, + degree: &BTreeMap, +) -> Vec { + ids.iter() + .filter_map(|id| { + let t = by_id.get(id)?; + let (x, y) = *positions.get(id)?; + let shown: Vec = t + .columns + .iter() + .take(MAX_ROWS) + .map(|c| DiagramColumn { + name: c.name.clone(), + ty: c.ty.clone(), + primary_key: c.primary_key, + foreign_key: c.foreign_key.is_some(), + nullable: c.nullable, + }) + .collect(); + Some(DiagramNode { + table: id.clone(), + schema: t.schema.clone(), + x, + y, + w: NODE_WIDTH, + h: node_height(t.columns.len()), + rank: ranks.get(id).copied().unwrap_or(0), + degree: degree.get(id).copied().unwrap_or(0), + hidden_columns: t.columns.len().saturating_sub(shown.len()), + columns: shown, + }) + }) + .collect() +} + +/// Vertical centre of a column's row, so an edge points at `user_id` rather +/// than at the middle of the box. Falls back to the node centre when the +/// column is past `MAX_ROWS` and therefore not drawn. +fn anchor_y(node: &DiagramNode, column: &str) -> f64 { + match node.columns.iter().position(|c| c.name == column) { + Some(i) => node.y + HEADER_HEIGHT + NODE_PAD_Y + (i as f64 + 0.5) * ROW_HEIGHT, + None => node.y + node.h / 2.0, + } +} + +fn route(relations: &[Relation], index: &HashMap<&str, &DiagramNode>) -> Vec { + relations + .iter() + .filter_map(|r| { + let from = index.get(r.from.as_str())?; + let to = index.get(r.to.as_str())?; + let self_loop = r.from == r.to; + + let y1 = anchor_y(from, &r.from_column); + let y2 = anchor_y(to, &r.to_column); + + let points = if self_loop { + // Loop out of the right edge and back. + let x = from.x + from.w; + vec![ + Point { x, y: y1 }, + Point { x: x + 24.0, y: y1 }, + Point { + x: x + 24.0, + y: y1 - ROW_HEIGHT, + }, + Point { + x, + y: y1 - ROW_HEIGHT, + }, + ] + } else { + // Three-segment elbow, leaving from whichever side faces the + // target so edges do not cross their own node. + let (x1, x2) = if to.x >= from.x + from.w { + (from.x + from.w, to.x) + } else if from.x >= to.x + to.w { + (from.x, to.x + to.w) + } else { + (from.x + from.w, to.x + to.w) + }; + let mid = (x1 + x2) / 2.0; + vec![ + Point { x: x1, y: y1 }, + Point { x: mid, y: y1 }, + Point { x: mid, y: y2 }, + Point { x: x2, y: y2 }, + ] + }; + + Some(DiagramEdge { + from: r.from.clone(), + from_column: r.from_column.clone(), + to: r.to.clone(), + to_column: r.to_column.clone(), + points, + self_loop, + }) + }) + .collect() +} + +/// Crossings between routed edges, by the standard interleaving test: two +/// edges sharing a corridor cross when their endpoints are in opposite order +/// at each end. +/// +/// Not a plain overlap test. Six edges fanning into one parent all occupy the +/// same corridor and overlap vertically, but none of them cross — they +/// converge. Counting overlap would report fifteen crossings for a diagram a +/// reader would call tidy. +fn count_crossings(edges: &[DiagramEdge]) -> u32 { + /// (corridor x, source y, target y) + fn ends(e: &DiagramEdge) -> Option<(f64, f64, f64)> { + let mid = e.points.get(1)?; + Some((mid.x, e.points.first()?.y, e.points.last()?.y)) + } + let segs: Vec<(f64, f64, f64)> = edges + .iter() + .filter(|e| !e.self_loop) + .filter_map(ends) + .collect(); + + let mut n = 0; + for i in 0..segs.len() { + for j in (i + 1)..segs.len() { + let (x1, s1, t1) = segs[i]; + let (x2, s2, t2) = segs[j]; + if (x1 - x2).abs() > f64::EPSILON { + continue; + } + // Shared endpoints converge or diverge; neither is a crossing. + if s1 == s2 || t1 == t2 { + continue; + } + if (s1 < s2) != (t1 < t2) { + n += 1; + } + } + } + n +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::handlers::catalog::{ColumnDesc, ForeignKeyRef, TableKind}; + + fn col(name: &str, pk: bool, fk: Option<(&str, &str)>) -> ColumnDesc { + ColumnDesc { + name: name.into(), + ty: "INTEGER".into(), + nullable: !pk, + default_value: None, + primary_key: pk, + position: 1, + foreign_key: fk.map(|(t, c)| ForeignKeyRef { + schema: None, + table: t.into(), + column: c.into(), + }), + } + } + + fn table(name: &str, columns: Vec) -> TableDescription { + TableDescription { + table: name.into(), + schema: None, + kind: TableKind::Table, + columns, + indexes: vec![], + row_count_estimate: None, + } + } + + #[test] + fn an_empty_schema_lays_out_without_panicking() { + let d = layout(&[], false); + assert!(d.nodes.is_empty() && d.edges.is_empty()); + assert_eq!(d.crossings, 0); + } + + #[test] + fn a_lone_table_is_isolated_not_ranked() { + let d = layout(&[table("users", vec![col("id", true, None)])], false); + assert_eq!(d.isolated, vec!["users"]); + assert_eq!(d.nodes.len(), 1); + assert_eq!(d.nodes[0].degree, 0); + } + + /// users ← orders ← order_items, plus an unrelated table. + fn chain() -> Vec { + vec![ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + table( + "order_items", + vec![ + col("id", true, None), + col("order_id", false, Some(("orders", "id"))), + ], + ), + table("unrelated", vec![col("id", true, None)]), + ] + } + + #[test] + fn components_never_overlap_once_the_row_wraps() { + // Enough two-table components to run past SHELF_WIDTH. Every one was + // previously placed at y=0, so the second row drew on top of the + // first. + let mut tables = Vec::new(); + for i in 0..12 { + tables.push(table(&format!("p{i}"), vec![col("id", true, None)])); + tables.push(table( + &format!("c{i}"), + vec![ + col("id", true, None), + col("pid", false, Some((&format!("p{i}"), "id"))), + ], + )); + } + let d = layout(&tables, false); + assert!(d.components.len() > 1); + for (i, a) in d.components.iter().enumerate() { + for b in d.components.iter().skip(i + 1) { + let apart = + a.x + a.w <= b.x || b.x + b.w <= a.x || a.y + a.h <= b.y || b.y + b.h <= a.y; + assert!(apart, "components {} and {} overlap", a.index, b.index); + } + } + } + + #[test] + fn an_absurd_depth_terminates_instead_of_spinning() { + // `depth` is unbounded on the wire; the walk must stop when the + // component is exhausted rather than iterating that many times. + let (kept, frontier) = neighbourhood(&chain(), "users", usize::MAX).unwrap(); + assert_eq!(kept.len(), 3); + assert!(frontier.is_empty()); + } + + #[test] + fn focus_follows_references_in_both_directions() { + // `orders` references users and is referenced by order_items. A walk + // that only followed the declared direction would hide every child. + let (kept, _) = neighbourhood(&chain(), "orders", 1).unwrap(); + let mut names: Vec<&str> = kept.iter().map(|t| t.table.as_str()).collect(); + names.sort(); + assert_eq!(names, vec!["order_items", "orders", "users"]); + } + + #[test] + fn focus_reports_what_lies_one_hop_beyond() { + let (kept, frontier) = neighbourhood(&chain(), "users", 1).unwrap(); + let mut names: Vec<&str> = kept.iter().map(|t| t.table.as_str()).collect(); + names.sort(); + assert_eq!(names, vec!["orders", "users"]); + // order_items is reachable but not drawn — the renderer needs to know + // there is something to expand into. + assert_eq!(frontier, vec!["order_items".to_string()]); + } + + #[test] + fn a_deeper_focus_pulls_the_next_ring_in() { + let (kept, frontier) = neighbourhood(&chain(), "users", 2).unwrap(); + assert_eq!(kept.len(), 3, "users, orders and order_items"); + assert!(frontier.is_empty(), "nothing further to reach"); + } + + #[test] + fn focus_never_drags_in_an_unrelated_table() { + let (kept, _) = neighbourhood(&chain(), "users", 9).unwrap(); + assert!(kept.iter().all(|t| t.table != "unrelated")); + } + + #[test] + fn an_unknown_focus_is_an_error_not_an_empty_diagram() { + // Silently returning nothing would read as "this table has no + // relations", which is a different and wrong answer. + assert!(neighbourhood(&chain(), "nope", 1).is_err()); + } + + #[test] + fn adjacent_ranks_leave_a_gutter_wide_enough_to_draw_an_edge_in() { + // Regression. RANK_SEP was 220 against a 200-wide node, leaving a + // 20px gutter: every edge rendered as a stub too short to see, and a + // schema with real foreign keys looked unrelated on screen. The + // gutter, not the pitch, is the number that matters. + let d = layout( + &[ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + ], + false, + ); + let by: HashMap<&str, &DiagramNode> = + d.nodes.iter().map(|n| (n.table.as_str(), n)).collect(); + let gutter = by["orders"].x - (by["users"].x + by["users"].w); + assert!( + gutter >= 100.0, + "adjacent ranks left a {gutter}px gutter; an edge needs room to read as a connection" + ); + } + + #[test] + fn related_tables_are_grouped_into_one_component_box() { + let d = layout( + &[ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + table("unrelated", vec![col("id", true, None)]), + ], + false, + ); + assert_eq!(d.components.len(), 1, "one connected group"); + let c = &d.components[0]; + assert_eq!(c.tables.len(), 2); + assert_eq!( + c.hub, None, + "both ends of a pair have degree 1 — naming either as the hub would be arbitrary" + ); + assert!(c.w > 0.0 && c.h > 0.0, "the group has a drawable box"); + // The box must actually contain its members, or the renderer draws a + // boundary that clips them. + for t in &c.tables { + let n = d.nodes.iter().find(|n| &n.table == t).expect("member node"); + assert!( + n.x >= c.x && n.y >= c.y && n.x + n.w <= c.x + c.w && n.y + n.h <= c.y + c.h, + "{t} falls outside its component box" + ); + } + assert_eq!(d.isolated, vec!["unrelated".to_string()]); + } + + #[test] + fn the_most_connected_table_is_named_as_the_hub() { + let d = layout( + &[ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + table( + "sessions", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + ], + false, + ); + assert_eq!(d.components.len(), 1); + assert_eq!( + d.components[0].hub.as_deref(), + Some("users"), + "two tables reference users, so it is strictly the most connected" + ); + } + + #[test] + fn a_reference_ranks_the_child_above_the_parent() { + let d = layout( + &[ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + ], + false, + ); + let by: HashMap<&str, &DiagramNode> = + d.nodes.iter().map(|n| (n.table.as_str(), n)).collect(); + assert_eq!(by["users"].rank, 0, "the referenced table sits upstream"); + assert_eq!(by["orders"].rank, 1); + assert!(d.isolated.is_empty()); + assert_eq!(d.edges.len(), 1); + assert_eq!(d.edges[0].to_column, "id"); + } + + #[test] + fn an_edge_anchors_on_the_column_row_not_the_box() { + let d = layout( + &[ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("filler", false, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + ], + false, + ); + let orders = d.nodes.iter().find(|n| n.table == "orders").unwrap(); + let edge = &d.edges[0]; + // Third column, so the anchor is two rows below the first. + let expected = orders.y + HEADER_HEIGHT + NODE_PAD_Y + 2.5 * ROW_HEIGHT; + assert!( + (edge.points[0].y - expected).abs() < 0.001, + "anchored at {} not {expected}", + edge.points[0].y + ); + } + + #[test] + fn a_reference_cycle_terminates_and_still_ranks_everything() { + let d = layout( + &[ + table( + "a", + vec![col("id", true, None), col("b_id", false, Some(("b", "id")))], + ), + table( + "b", + vec![col("id", true, None), col("a_id", false, Some(("a", "id")))], + ), + ], + false, + ); + assert_eq!(d.nodes.len(), 2); + assert!(d.isolated.is_empty()); + } + + #[test] + fn a_self_reference_becomes_a_loop_not_a_line() { + let d = layout( + &[table( + "employees", + vec![ + col("id", true, None), + col("manager_id", false, Some(("employees", "id"))), + ], + )], + false, + ); + assert_eq!(d.edges.len(), 1); + assert!(d.edges[0].self_loop); + } + + #[test] + fn a_hub_reports_its_degree_and_leaves_no_crossings() { + let mut tables = vec![table("users", vec![col("id", true, None)])]; + for i in 0..6 { + tables.push(table( + &format!("child{i}"), + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + )); + } + let d = layout(&tables, false); + let users = d.nodes.iter().find(|n| n.table == "users").unwrap(); + assert_eq!(users.degree, 6); + assert_eq!(d.crossings, 0, "a simple fan should not cross"); + } + + #[test] + fn layout_is_deterministic_regardless_of_input_order() { + let mut a = vec![ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + table("audit", vec![col("id", true, None)]), + ]; + let first = layout(&a, false); + a.reverse(); + let second = layout(&a, false); + + let key = |d: &SchemaDiagramResp| -> Vec<(String, u64, u64)> { + let mut v: Vec<_> = d + .nodes + .iter() + .map(|n| (n.table.clone(), n.x as u64, n.y as u64)) + .collect(); + v.sort(); + v + }; + assert_eq!(key(&first), key(&second)); + assert_eq!(first.crossings, second.crossings); + } + + #[test] + fn a_wide_table_is_summarised_rather_than_drawn_in_full() { + let cols: Vec = (0..30) + .map(|i| col(&format!("c{i}"), i == 0, None)) + .collect(); + let d = layout(&[table("wide", cols)], false); + let n = &d.nodes[0]; + assert_eq!(n.columns.len(), MAX_ROWS); + assert_eq!(n.hidden_columns, 30 - MAX_ROWS); + // Height reflects what is drawn, not the full column count. + assert_eq!(n.h, node_height(MAX_ROWS)); + } + + #[test] + fn a_reference_to_a_table_outside_the_set_is_dropped() { + // The parent was filtered out or truncated away; an edge to nowhere + // would render as a line into empty space. + let d = layout( + &[table( + "orders", + vec![col("user_id", false, Some(("users", "id")))], + )], + true, + ); + assert!(d.edges.is_empty()); + assert_eq!(d.isolated, vec!["orders"]); + assert!(d.truncated); + } +} diff --git a/database/src/handlers/explain.rs b/database/src/handlers/explain.rs new file mode 100644 index 000000000..18bf948a9 --- /dev/null +++ b/database/src/handlers/explain.rs @@ -0,0 +1,712 @@ +//! `database::explain` — a query plan as a tree, not a grid of text. +//! +//! Every driver spells a plan differently: postgres emits nested JSON, mysql +//! emits a differently-nested JSON, and sqlite emits a flat `(id, parent, +//! detail)` set that has to be reassembled. Callers should not have to know +//! that, so all three collapse into one `PlanNode` tree with the same fields +//! and the same warnings. +//! +//! **`ANALYZE` really executes the statement.** `EXPLAIN ANALYZE DELETE FROM +//! users` deletes users. It is off by default and refused outright for +//! anything that does not parse as a read — the console's own check is a +//! convenience, this one is the authority. + +use super::query::{self, err_to_str, QueryReq}; +use super::tx_sql_guard; +use super::AppState; +use crate::config::DriverKind; +use crate::error::DbError; +use crate::pool::Pool; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +fn default_timeout() -> u64 { + 30_000 +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum NodeClass { + Scan, + Index, + Join, + Sort, + Aggregate, + Cte, + Limit, + Other, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum PlanFormat { + PgJson, + SqliteQueryPlan, + MysqlJson, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum WarningKind { + /// A sequential scan over a large relation. + SeqScanLarge, + /// Estimated and actual row counts differ by an order of magnitude — + /// usually stale statistics. + EstimateSkew, + /// An inner loop executed a very large number of times. + NestedLoopLarge, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum Severity { + Info, + Warn, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct PlanNode { + pub id: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub parent: Option, + pub label: String, + pub node_class: NodeClass, + #[serde(skip_serializing_if = "Option::is_none")] + pub relation: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cost_startup: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cost_total: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rows_estimated: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rows_actual: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub width: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub time_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub loops: Option, + pub detail: String, + pub children: Vec, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct PlanWarning { + pub node_id: u32, + pub kind: WarningKind, + pub message: String, + pub severity: Severity, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ExplainReq { + #[serde(default)] + pub db: Option, + pub sql: String, + #[serde(default, deserialize_with = "crate::handlers::lenient_params")] + pub params: Vec, + /// Runs the statement to collect real timings. Refused for anything that + /// is not a read. + #[serde(default)] + pub analyze: bool, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ExplainResp { + pub format: PlanFormat, + pub analyzed: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub root: Option, + pub warnings: Vec, + /// The driver's own output, so a caller is never stuck when the shape is + /// one we do not recognise. + #[serde(skip_serializing_if = "Option::is_none")] + pub raw: Option, +} + +/// Rows above this make a sequential scan worth flagging. +const LARGE_ROWS: f64 = 10_000.0; +/// Estimate/actual ratio worth flagging as stale statistics. +const SKEW_FACTOR: f64 = 10.0; +/// Loop count worth flagging on a nested loop. +const LARGE_LOOPS: f64 = 1_000.0; + +pub fn classify(label: &str) -> NodeClass { + let l = label.to_ascii_lowercase(); + if l.contains("index") { + NodeClass::Index + } else if l.contains("scan") || l.contains("seek") { + NodeClass::Scan + } else if l.contains("join") || l.contains("nested loop") { + NodeClass::Join + } else if l.contains("sort") || l.contains("b-tree") { + NodeClass::Sort + } else if l.contains("aggregate") || l.contains("group") { + NodeClass::Aggregate + } else if l.contains("cte") || l.contains("subquery") { + NodeClass::Cte + } else if l.contains("limit") { + NodeClass::Limit + } else { + NodeClass::Other + } +} + +async fn driver_of(state: &AppState, db: &str) -> Result { + Ok(match state.pool(db).await.map_err(err_to_str)? { + Pool::Sqlite(_) => DriverKind::Sqlite, + Pool::Postgres(_) => DriverKind::Postgres, + Pool::Mysql(_) => DriverKind::Mysql, + }) +} + +pub async fn handle(state: &AppState, req: ExplainReq) -> Result { + if req.sql.trim().is_empty() { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: "sql is required".into(), + })); + } + let db = state.resolve_db(req.db.clone()).await.map_err(err_to_str)?; + let driver = driver_of(state, &db).await?; + + // The gate that matters. ANALYZE executes; refuse it on anything that is + // not unambiguously a read. + if req.analyze && !tx_sql_guard::is_read_only_sql(&req.sql) { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: "analyze runs the statement, so it is only allowed for a single \ + read-only statement; run it without analyze to see the estimated plan" + .into(), + })); + } + // SQLite's EXPLAIN QUERY PLAN has no ANALYZE form. + let analyzed = req.analyze && driver != DriverKind::Sqlite; + + let prefixed = match (driver, analyzed) { + (DriverKind::Postgres, true) => format!("EXPLAIN (FORMAT JSON, ANALYZE true) {}", req.sql), + (DriverKind::Postgres, false) => format!("EXPLAIN (FORMAT JSON) {}", req.sql), + (DriverKind::Mysql, _) => format!("EXPLAIN FORMAT=JSON {}", req.sql), + (DriverKind::Sqlite, _) => format!("EXPLAIN QUERY PLAN {}", req.sql), + }; + + let resp = query::handle( + state, + QueryReq { + db: Some(db), + sql: prefixed, + params: req.params, + timeout_ms: req.timeout_ms, + record_history: false, + }, + ) + .await?; + + let (format, root) = match driver { + DriverKind::Postgres => parse_pg(&resp.rows), + DriverKind::Mysql => parse_mysql(&resp.rows), + DriverKind::Sqlite => parse_sqlite(&resp.rows), + }; + let warnings = root.as_ref().map(warnings_for).unwrap_or_default(); + let raw = (format == PlanFormat::Unknown) + .then(|| Value::Array(resp.rows.iter().cloned().map(Value::Object).collect())); + + Ok(ExplainResp { + format, + analyzed, + root, + warnings, + raw, + }) +} + +fn num(v: Option<&Value>) -> Option { + match v { + Some(Value::Number(n)) => n.as_f64(), + Some(Value::String(s)) => s.parse().ok(), + _ => None, + } +} + +/// The driver may hand back a JSON column already parsed, or as text. +fn as_json(v: &Value) -> Option { + match v { + Value::String(s) => serde_json::from_str(s).ok(), + other => Some(other.clone()), + } +} + +fn first_value(rows: &[serde_json::Map]) -> Option { + rows.first()?.values().next().cloned() +} + +/* ---------------- postgres ---------------- */ + +fn parse_pg(rows: &[serde_json::Map]) -> (PlanFormat, Option) { + let Some(parsed) = first_value(rows).as_ref().and_then(as_json) else { + return (PlanFormat::Unknown, None); + }; + // EXPLAIN (FORMAT JSON) wraps the plan in a single-element array. + let plan = parsed + .as_array() + .and_then(|a| a.first()) + .and_then(|o| o.get("Plan")) + .cloned(); + match plan { + Some(p) => { + let mut next_id = 0; + (PlanFormat::PgJson, Some(pg_node(&p, None, &mut next_id))) + } + None => (PlanFormat::Unknown, None), + } +} + +fn pg_node(v: &Value, parent: Option, next_id: &mut u32) -> PlanNode { + let id = *next_id; + *next_id += 1; + + let label = v + .get("Node Type") + .and_then(Value::as_str) + .unwrap_or("Unknown") + .to_string(); + let relation = v + .get("Relation Name") + .and_then(Value::as_str) + .map(str::to_string); + + let mut detail = label.clone(); + if let Some(r) = &relation { + detail = format!("{detail} on {r}"); + } + + let children = v + .get("Plans") + .and_then(Value::as_array) + .map(|a| a.iter().map(|c| pg_node(c, Some(id), next_id)).collect()) + .unwrap_or_default(); + + PlanNode { + id, + parent, + node_class: classify(&label), + label, + relation, + cost_startup: num(v.get("Startup Cost")), + cost_total: num(v.get("Total Cost")), + rows_estimated: num(v.get("Plan Rows")), + rows_actual: num(v.get("Actual Rows")), + width: num(v.get("Plan Width")).map(|w| w as i64), + time_ms: num(v.get("Actual Total Time")), + loops: num(v.get("Actual Loops")), + detail, + children, + } +} + +/* ---------------- mysql ---------------- */ + +fn parse_mysql(rows: &[serde_json::Map]) -> (PlanFormat, Option) { + let Some(parsed) = first_value(rows).as_ref().and_then(as_json) else { + return (PlanFormat::Unknown, None); + }; + let Some(block) = parsed.get("query_block") else { + return (PlanFormat::Unknown, None); + }; + let mut next_id = 0; + ( + PlanFormat::MysqlJson, + Some(mysql_node(block, None, &mut next_id, "query_block")), + ) +} + +/// MySQL's shape is a loose bag of nested objects rather than a uniform node +/// list, so walk it generically: any nested object carrying a `table` or +/// another recognisable block becomes a child. +fn mysql_node(v: &Value, parent: Option, next_id: &mut u32, label: &str) -> PlanNode { + let id = *next_id; + *next_id += 1; + + let table = v.get("table"); + let relation = table + .and_then(|t| t.get("table_name")) + .and_then(Value::as_str) + .map(str::to_string); + let access = table + .and_then(|t| t.get("access_type")) + .and_then(Value::as_str) + .unwrap_or(label); + + let cost = table + .and_then(|t| t.get("cost_info")) + .and_then(|c| c.get("read_cost").or_else(|| c.get("query_cost"))); + + let mut children = Vec::new(); + if let Some(obj) = v.as_object() { + for (k, child) in obj { + if k == "table" || k == "cost_info" { + continue; + } + match child { + Value::Object(_) => children.push(mysql_node(child, Some(id), next_id, k)), + Value::Array(items) => { + for item in items.iter().filter(|i| i.is_object()) { + children.push(mysql_node(item, Some(id), next_id, k)); + } + } + _ => {} + } + } + } + + let label = access.to_string(); + PlanNode { + id, + parent, + node_class: classify(&label), + detail: match &relation { + Some(r) => format!("{label} on {r}"), + None => label.clone(), + }, + label, + relation, + cost_startup: None, + cost_total: num(cost), + rows_estimated: num(table.and_then(|t| t.get("rows_examined_per_scan"))), + rows_actual: num(table.and_then(|t| t.get("rows_produced_per_join"))), + width: None, + time_ms: None, + loops: None, + children, + } +} + +/* ---------------- sqlite ---------------- */ + +/// `EXPLAIN QUERY PLAN` returns a flat `(id, parent, notused, detail)` set; +/// the tree is implied by `parent` and has to be rebuilt. +fn parse_sqlite(rows: &[serde_json::Map]) -> (PlanFormat, Option) { + if rows.is_empty() { + return (PlanFormat::Unknown, None); + } + let flat: Vec<(u32, u32, String)> = rows + .iter() + .filter_map(|r| { + let id = num(r.get("id"))? as u32; + let parent = num(r.get("parent")).unwrap_or(0.0) as u32; + let detail = r.get("detail").and_then(Value::as_str)?.to_string(); + Some((id, parent, detail)) + }) + .collect(); + if flat.is_empty() { + return (PlanFormat::Unknown, None); + } + + // Group by parent first, then build the tree in one recursive pass. Doing + // it this way avoids searching a partly-built tree for each row, and it + // tolerates rows arriving in any order. + let mut children_of: std::collections::HashMap> = + std::collections::HashMap::new(); + for (id, parent, detail) in flat { + children_of.entry(parent).or_default().push((id, detail)); + } + + // sqlite numbers top-level rows with parent 0, so synthesise a root to + // hang them from rather than promoting an arbitrary step. + let root = PlanNode { + id: 0, + parent: None, + label: "QUERY PLAN".into(), + node_class: NodeClass::Other, + relation: None, + cost_startup: None, + cost_total: None, + rows_estimated: None, + rows_actual: None, + width: None, + time_ms: None, + loops: None, + detail: "QUERY PLAN".into(), + children: sqlite_children(0, &children_of, 0), + }; + (PlanFormat::SqliteQueryPlan, Some(root)) +} + +/// Pull the relation out of an `EXPLAIN QUERY PLAN` detail string. +/// +/// SQLite changed this wording: older versions say `SCAN TABLE users`, 3.36 +/// and later say `SCAN users`. Both forms are accepted, as is the `SUBQUERY` +/// spelling, so the relation does not silently go missing on one build. +fn sqlite_relation(detail: &str) -> Option { + let mut words = detail + .split_whitespace() + .skip_while(|w| !matches!(*w, "SCAN" | "SEARCH")) + .skip(1); + let first = words.next()?; + let name = match first { + "TABLE" | "SUBQUERY" => words.next()?, + other => other, + }; + // `SCAN users USING INDEX ...` — the name never starts a clause keyword. + (!matches!(name, "USING" | "AS" | "COVERING")).then(|| name.to_string()) +} + +/// Depth bound guards against a malformed set whose parent links form a cycle. +const MAX_PLAN_DEPTH: u32 = 64; + +fn sqlite_children( + parent: u32, + children_of: &std::collections::HashMap>, + depth: u32, +) -> Vec { + if depth >= MAX_PLAN_DEPTH { + return Vec::new(); + } + children_of + .get(&parent) + .map(|kids| { + kids.iter() + .map(|(id, detail)| PlanNode { + id: *id, + parent: Some(parent), + node_class: classify(detail), + label: detail.clone(), + relation: sqlite_relation(detail), + cost_startup: None, + cost_total: None, + rows_estimated: None, + rows_actual: None, + width: None, + time_ms: None, + loops: None, + detail: detail.clone(), + children: sqlite_children(*id, children_of, depth + 1), + }) + .collect() + }) + .unwrap_or_default() +} + +/* ---------------- warnings ---------------- */ + +pub fn warnings_for(root: &PlanNode) -> Vec { + let mut out = Vec::new(); + walk(root, &mut out); + out +} + +fn walk(node: &PlanNode, out: &mut Vec) { + let rows = node.rows_actual.or(node.rows_estimated).unwrap_or(0.0); + + if node.node_class == NodeClass::Scan + && !node.label.to_ascii_lowercase().contains("index") + && rows > LARGE_ROWS + { + out.push(PlanWarning { + node_id: node.id, + kind: WarningKind::SeqScanLarge, + message: format!( + "sequential scan over ~{rows:.0} rows{}; an index on the filtered \ + column would avoid reading the whole relation", + node.relation + .as_ref() + .map(|r| format!(" of {r}")) + .unwrap_or_default() + ), + severity: Severity::Warn, + }); + } + + if let (Some(est), Some(act)) = (node.rows_estimated, node.rows_actual) { + let hi = est.max(act); + let lo = est.min(act).max(1.0); + if hi / lo > SKEW_FACTOR { + out.push(PlanWarning { + node_id: node.id, + kind: WarningKind::EstimateSkew, + message: format!( + "planner estimated {est:.0} rows but saw {act:.0}; statistics are \ + likely stale — ANALYZE the table" + ), + severity: Severity::Warn, + }); + } + } + + if node.node_class == NodeClass::Join && node.loops.unwrap_or(0.0) > LARGE_LOOPS { + out.push(PlanWarning { + node_id: node.id, + kind: WarningKind::NestedLoopLarge, + message: format!( + "inner side executed {:.0} times; a hash or merge join would usually \ + be cheaper at this size", + node.loops.unwrap_or(0.0) + ), + severity: Severity::Info, + }); + } + + for c in &node.children { + walk(c, out); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn rows(v: Value) -> Vec> { + vec![[("x".to_string(), v)].into_iter().collect()] + } + + #[test] + fn classify_maps_the_common_node_names() { + assert_eq!(classify("Seq Scan"), NodeClass::Scan); + assert_eq!(classify("Index Only Scan"), NodeClass::Index); + assert_eq!(classify("Hash Join"), NodeClass::Join); + assert_eq!(classify("Sort"), NodeClass::Sort); + assert_eq!(classify("HashAggregate"), NodeClass::Aggregate); + assert_eq!(classify("Limit"), NodeClass::Limit); + assert_eq!(classify("Gather Merge"), NodeClass::Other); + } + + #[test] + fn pg_plan_becomes_a_tree_with_costs() { + let plan = json!([{ + "Plan": { + "Node Type": "Hash Join", "Startup Cost": 1.5, "Total Cost": 42.0, + "Plan Rows": 100, "Actual Rows": 90, "Plan Width": 32, + "Actual Total Time": 3.5, "Actual Loops": 1, + "Plans": [ + {"Node Type": "Seq Scan", "Relation Name": "users", + "Total Cost": 20.0, "Plan Rows": 50}, + {"Node Type": "Index Scan", "Relation Name": "orders", + "Total Cost": 10.0, "Plan Rows": 50} + ] + } + }]); + let (fmt, root) = parse_pg(&rows(plan)); + assert_eq!(fmt, PlanFormat::PgJson); + let root = root.unwrap(); + assert_eq!(root.label, "Hash Join"); + assert_eq!(root.node_class, NodeClass::Join); + assert_eq!(root.cost_total, Some(42.0)); + assert_eq!(root.children.len(), 2); + assert_eq!(root.children[0].relation.as_deref(), Some("users")); + assert_eq!(root.children[0].detail, "Seq Scan on users"); + // Ids are unique across the tree so warnings can point at a node. + assert_eq!(root.id, 0); + assert_eq!(root.children[0].id, 1); + assert_eq!(root.children[1].id, 2); + } + + #[test] + fn pg_plan_accepts_json_delivered_as_text() { + let text = json!([{"Plan": {"Node Type": "Result"}}]).to_string(); + let (fmt, root) = parse_pg(&rows(Value::String(text))); + assert_eq!(fmt, PlanFormat::PgJson); + assert_eq!(root.unwrap().label, "Result"); + } + + #[test] + fn sqlite_flat_rows_are_reassembled_into_a_tree() { + let flat: Vec> = vec![ + json!({"id": 2, "parent": 0, "detail": "SCAN TABLE users"}), + json!({"id": 4, "parent": 2, "detail": "SEARCH TABLE orders USING INDEX ix"}), + ] + .into_iter() + .map(|v| v.as_object().unwrap().clone()) + .collect(); + + let (fmt, root) = parse_sqlite(&flat); + assert_eq!(fmt, PlanFormat::SqliteQueryPlan); + let root = root.unwrap(); + assert_eq!(root.children.len(), 1, "top-level rows hang off the root"); + let scan = &root.children[0]; + assert_eq!(scan.node_class, NodeClass::Scan); + assert_eq!(scan.relation.as_deref(), Some("users")); + assert_eq!(scan.children.len(), 1, "child attaches to its parent id"); + assert_eq!(scan.children[0].node_class, NodeClass::Index); + } + + #[test] + fn sqlite_relation_survives_both_wordings() { + // SQLite dropped the TABLE keyword in 3.36; both forms are in the wild. + assert_eq!(sqlite_relation("SCAN TABLE users"), Some("users".into())); + assert_eq!(sqlite_relation("SCAN people"), Some("people".into())); + assert_eq!( + sqlite_relation("SEARCH orders USING INDEX ix_o (user_id=?)"), + Some("orders".into()) + ); + assert_eq!( + sqlite_relation("SEARCH TABLE orders USING INDEX ix_o"), + Some("orders".into()) + ); + assert_eq!(sqlite_relation("USE TEMP B-TREE FOR ORDER BY"), None); + } + + #[test] + fn unrecognised_output_reports_unknown_rather_than_guessing() { + let (fmt, root) = parse_pg(&rows(json!("not a plan"))); + assert_eq!(fmt, PlanFormat::Unknown); + assert!(root.is_none()); + assert_eq!(parse_sqlite(&[]).0, PlanFormat::Unknown); + } + + fn node(id: u32, label: &str, est: f64, act: Option) -> PlanNode { + PlanNode { + id, + parent: None, + node_class: classify(label), + label: label.into(), + relation: Some("t".into()), + cost_startup: None, + cost_total: None, + rows_estimated: Some(est), + rows_actual: act, + width: None, + time_ms: None, + loops: None, + detail: label.into(), + children: vec![], + } + } + + #[test] + fn a_large_sequential_scan_is_flagged_but_a_small_one_is_not() { + let big = warnings_for(&node(0, "Seq Scan", 50_000.0, None)); + assert_eq!(big.len(), 1); + assert_eq!(big[0].kind, WarningKind::SeqScanLarge); + + let small = warnings_for(&node(0, "Seq Scan", 10.0, None)); + assert!(small.is_empty(), "a small scan is not a problem"); + + // An index scan of the same size is fine. + let indexed = warnings_for(&node(0, "Index Scan", 50_000.0, None)); + assert!(indexed.is_empty()); + } + + #[test] + fn estimate_skew_fires_only_past_the_threshold() { + let skewed = warnings_for(&node(0, "Index Scan", 10.0, Some(5_000.0))); + assert!(skewed.iter().any(|w| w.kind == WarningKind::EstimateSkew)); + + let close = warnings_for(&node(0, "Index Scan", 100.0, Some(150.0))); + assert!(close.is_empty(), "a 1.5x difference is normal"); + } + + #[test] + fn warnings_reach_nested_nodes() { + let mut root = node(0, "Limit", 1.0, None); + root.children.push(node(7, "Seq Scan", 90_000.0, None)); + let w = warnings_for(&root); + assert_eq!(w.len(), 1); + assert_eq!(w[0].node_id, 7, "the warning points at the offending node"); + } +} diff --git a/database/src/handlers/filter.rs b/database/src/handlers/filter.rs new file mode 100644 index 000000000..a487cb145 --- /dev/null +++ b/database/src/handlers/filter.rs @@ -0,0 +1,744 @@ +//! Typed filters and sorts, compiled to dialect SQL here rather than in the +//! caller. +//! +//! A caller says "email contains test and plan equals free" as data; this +//! module turns that into a parameterised `WHERE` clause for the driver in +//! hand. That matters for correctness as much as convenience — the traps +//! below are ones every hand-written client filter gets wrong at least once: +//! +//! * **LIKE metacharacters.** `contains "50%"` must match a literal percent, +//! not every row. Values are escaped and the clause carries `ESCAPE '\'`. +//! * **`IS NULL` is not `= ''`.** They are separate operators, because +//! conflating them silently changes the answer. +//! * **Case-insensitivity is not portable.** Postgres expresses it as +//! `ILIKE`; sqlite and mysql apply it through collation and cannot honour a +//! per-query flag. Asking for it there is rejected rather than ignored. +//! * **Placeholders differ.** Postgres numbers them, the others do not. + +use crate::config::DriverKind; +use crate::error::DbError; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// Escape character for `LIKE` patterns. Backslash is the conventional choice +/// and is stated explicitly in every clause so no driver default applies. +const LIKE_ESCAPE: char = '\\'; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FilterOp { + Contains, + NotContains, + Equals, + NotEquals, + StartsWith, + EndsWith, + Gt, + Gte, + Lt, + Lte, + /// Inclusive range; needs both `value` and `value2`. + Between, + IsTrue, + IsFalse, + IsNull, + IsNotNull, + /// NULL or the empty string. Distinct from `is_null` on purpose. + IsEmpty, + /// Set membership, over `values`. Expressing "status is one of open, + /// pending, held" as three OR'd equality filters is not possible here — + /// filters stack with AND — so without this the question cannot be asked + /// at all. + In, + NotIn, +} + +impl FilterOp { + /// How many operands the operator consumes. Used to reject an incomplete + /// filter up front instead of compiling something that means nothing. + fn arity(self) -> usize { + match self { + FilterOp::IsTrue + | FilterOp::IsFalse + | FilterOp::IsNull + | FilterOp::IsNotNull + | FilterOp::IsEmpty => 0, + FilterOp::Between => 2, + // Variadic: operands come from `values`, not `value`. + FilterOp::In | FilterOp::NotIn => 0, + _ => 1, + } + } + + fn is_like(self) -> bool { + matches!( + self, + FilterOp::Contains | FilterOp::NotContains | FilterOp::StartsWith | FilterOp::EndsWith + ) + } + + fn is_set(self) -> bool { + matches!(self, FilterOp::In | FilterOp::NotIn) + } +} + +#[derive(Debug, Clone, Deserialize, JsonSchema)] +pub struct FilterSpec { + pub column: String, + pub op: FilterOp, + #[serde(default)] + pub value: Option, + /// Upper bound for `between`. + #[serde(default)] + pub value2: Option, + /// Operands for `in` / `not_in`. + #[serde(default)] + pub values: Vec, + /// Postgres only. Rejected elsewhere rather than silently ignored. + #[serde(default)] + pub case_sensitive: Option, + /// Kept in the list but not applied. A caller refining a query wants to + /// switch one condition off and back on without losing how it was built, + /// and a console that only offers delete makes that a retype. + #[serde(default)] + pub disabled: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum Direction { + Asc, + Desc, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum NullsPosition { + First, + Last, +} + +/// Type-aware sort modes. These exist server-side because the grid is paged: +/// sorting the fetched page would order 50 rows out of N, which is a +/// different answer from sorting the table. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum SortMode { + #[default] + Default, + /// `item2` before `item10`. The mode users actually notice. + Natural, + Length, + AbsoluteValue, + Random, +} + +#[derive(Debug, Clone, Deserialize, JsonSchema)] +pub struct SortSpec { + pub column: String, + #[serde(default = "default_direction")] + pub direction: Direction, + #[serde(default)] + pub nulls: Option, + #[serde(default)] + pub mode: SortMode, +} + +fn default_direction() -> Direction { + Direction::Asc +} + +/// Quote an identifier for the driver, doubling the quote character so a +/// crafted column name cannot escape it. +pub fn quote_ident(driver: DriverKind, ident: &str) -> String { + match driver { + DriverKind::Mysql => format!("`{}`", ident.replace('`', "``")), + _ => format!("\"{}\"", ident.replace('"', "\"\"")), + } +} + +/// Qualify a table reference. Only postgres has a schema above the table. +pub fn quote_table(driver: DriverKind, schema: Option<&str>, table: &str) -> String { + match (driver, schema) { + (DriverKind::Postgres, Some(s)) => { + format!("{}.{}", quote_ident(driver, s), quote_ident(driver, table)) + } + _ => quote_ident(driver, table), + } +} + +/// Emits `$1`, `$2`, … on postgres and `?` elsewhere. +struct Placeholders { + driver: DriverKind, + next: usize, +} + +impl Placeholders { + fn new(driver: DriverKind, start: usize) -> Self { + Self { + driver, + next: start, + } + } + + fn take(&mut self) -> String { + let n = self.next; + self.next += 1; + match self.driver { + DriverKind::Postgres => format!("${n}"), + _ => "?".to_string(), + } + } +} + +fn invalid(reason: impl Into) -> DbError { + DbError::InvalidParam { + index: 0, + reason: reason.into(), + } +} + +/// Escape `%`, `_` and the escape character itself so they match literally. +fn escape_like(raw: &str) -> String { + let mut out = String::with_capacity(raw.len()); + for ch in raw.chars() { + if ch == '%' || ch == '_' || ch == LIKE_ESCAPE { + out.push(LIKE_ESCAPE); + } + out.push(ch); + } + out +} + +fn as_text(value: &Value, column: &str) -> Result { + match value { + Value::String(s) => Ok(s.clone()), + Value::Number(n) => Ok(n.to_string()), + Value::Bool(b) => Ok(b.to_string()), + _ => Err(invalid(format!( + "filter on `{column}` needs a text-comparable value" + ))), + } +} + +#[derive(Debug, Clone)] +pub struct WhereClause { + /// Empty when there are no filters; otherwise a bare boolean expression + /// with no leading `WHERE`, so callers can compose it. + pub sql: String, + pub params: Vec, +} + +/// Compile filters into a parameterised predicate. `start_param` is the first +/// free placeholder index, so a caller that already bound values keeps +/// numbering correct on postgres. +pub fn compile_where( + driver: DriverKind, + filters: &[FilterSpec], + start_param: usize, +) -> Result { + let mut ph = Placeholders::new(driver, start_param); + let mut parts: Vec = Vec::new(); + let mut params: Vec = Vec::new(); + + for f in filters { + // A disabled filter is skipped, not rejected: it stays in the caller's + // list so it can be switched back on without being rebuilt. + if f.disabled { + continue; + } + if f.column.trim().is_empty() { + return Err(invalid("filter is missing a column")); + } + // Check operands by position, not by count. Counting accepts + // `{op: equals, value2: 5}` — one operand supplied for a one-operand + // operator — and the compiler then unwraps the `value` that is not + // there. The request body is caller-controlled, so that is a panic + // reachable from the wire. + let arity = f.op.arity(); + if arity >= 1 && f.value.is_none() { + return Err(invalid(format!("filter on `{}` needs `value`", f.column))); + } + if arity >= 2 && f.value2.is_none() { + return Err(invalid(format!( + "filter on `{}` needs both `value` and `value2`", + f.column + ))); + } + if f.op.is_set() && f.values.is_empty() { + // `IN ()` is a syntax error on every driver, and silently + // dropping the filter would quietly widen the result set. + return Err(invalid(format!( + "filter on `{}` needs at least one value in `values`", + f.column + ))); + } + + // Case sensitivity is only expressible on postgres. Everywhere else + // it is a property of the column's collation, so honouring the flag + // would be a lie and ignoring it would be worse. + // Rejected whenever it cannot be honoured, which includes operators + // that never consult it. Accepting `{op: equals, case_sensitive: true}` + // and quietly doing nothing is the failure the module header warns + // about, just in a different place. + if f.case_sensitive.is_some() && !(driver == DriverKind::Postgres && f.op.is_like()) { + return Err(invalid( + "case_sensitive applies only to postgres pattern operators; \ + elsewhere it is determined by the column collation", + )); + } + + let col = quote_ident(driver, &f.column); + let sensitive = f.case_sensitive.unwrap_or(false); + + let part = match f.op { + FilterOp::IsNull => format!("{col} IS NULL"), + FilterOp::IsNotNull => format!("{col} IS NOT NULL"), + FilterOp::IsTrue => format!("{col} = TRUE"), + FilterOp::IsFalse => format!("{col} = FALSE"), + // NULL *or* empty string — the distinction from is_null is the point. + FilterOp::IsEmpty => format!("({col} IS NULL OR {col} = '')"), + + FilterOp::In | FilterOp::NotIn => { + let marks: Vec = f + .values + .iter() + .map(|v| { + params.push(v.clone()); + ph.take() + }) + .collect(); + let list = marks.join(", "); + if f.op == FilterOp::In { + format!("{col} IN ({list})") + } else { + // NOT IN drops NULLs on every driver, because `NULL <> x` + // is unknown. A reader asking for "not one of these" means + // to keep the NULLs, so say so explicitly. + format!("({col} IS NULL OR {col} NOT IN ({list}))") + } + } + + FilterOp::Contains + | FilterOp::NotContains + | FilterOp::StartsWith + | FilterOp::EndsWith => { + let raw = as_text(f.value.as_ref().expect("arity checked"), &f.column)?; + let escaped = escape_like(&raw); + let pattern = match f.op { + FilterOp::StartsWith => format!("{escaped}%"), + FilterOp::EndsWith => format!("%{escaped}"), + _ => format!("%{escaped}%"), + }; + let negate = f.op == FilterOp::NotContains; + let op = match (driver, sensitive) { + (DriverKind::Postgres, false) => "ILIKE", + _ => "LIKE", + }; + params.push(Value::String(pattern)); + let p = ph.take(); + let expr = format!("{col} {op} {p} ESCAPE '{LIKE_ESCAPE}'"); + if negate { + // A NULL never matches LIKE, so a naive NOT LIKE drops + // NULL rows the user would expect to see in "does not + // contain". + format!("({col} IS NULL OR NOT ({expr}))") + } else { + expr + } + } + + FilterOp::Between => { + params.push(f.value.clone().expect("arity checked")); + let lo = ph.take(); + params.push(f.value2.clone().expect("arity checked")); + let hi = ph.take(); + format!("{col} BETWEEN {lo} AND {hi}") + } + + _ => { + let sym = match f.op { + FilterOp::Equals => "=", + FilterOp::NotEquals => "<>", + FilterOp::Gt => ">", + FilterOp::Gte => ">=", + FilterOp::Lt => "<", + FilterOp::Lte => "<=", + other => unreachable!("{other:?} handled above"), + }; + params.push(f.value.clone().expect("arity checked")); + let p = ph.take(); + format!("{col} {sym} {p}") + } + }; + parts.push(part); + } + + Ok(WhereClause { + sql: parts.join(" AND "), + params, + }) +} + +/// Compile sorts into an `ORDER BY` body (no leading keyword). Column names +/// are quoted; modes that a driver cannot express fall back to plain ordering +/// rather than erroring, because a sort is a presentation choice and refusing +/// one would be worse than approximating it. +pub fn compile_order_by(driver: DriverKind, sorts: &[SortSpec]) -> Result { + let mut parts = Vec::new(); + for s in sorts { + if s.column.trim().is_empty() { + return Err(invalid("sort is missing a column")); + } + let col = quote_ident(driver, &s.column); + let expr = match s.mode { + SortMode::Default => col.clone(), + SortMode::Length => match driver { + DriverKind::Mysql => format!("CHAR_LENGTH({col})"), + _ => format!("LENGTH({col})"), + }, + SortMode::AbsoluteValue => format!("ABS({col})"), + SortMode::Random => match driver { + DriverKind::Postgres => "RANDOM()".to_string(), + DriverKind::Mysql => "RAND()".to_string(), + DriverKind::Sqlite => "RANDOM()".to_string(), + }, + // Natural order: pad the leading digit run so `item2` sorts before + // `item10`. Postgres can express this inline; the others have no + // portable equivalent, so they order plainly rather than pretend. + SortMode::Natural => match driver { + DriverKind::Postgres => format!( + "regexp_replace({col}, '\\d+', lpad(substring({col} from '\\d+'), 12, '0'))" + ), + _ => col.clone(), + }, + }; + + let dir = match s.direction { + Direction::Asc => "ASC", + Direction::Desc => "DESC", + }; + let mut term = format!("{expr} {dir}"); + if let Some(nulls) = s.nulls { + let n = match nulls { + NullsPosition::First => "FIRST", + NullsPosition::Last => "LAST", + }; + match driver { + // MySQL has no NULLS FIRST/LAST; emulate with a leading key. + DriverKind::Mysql => { + let flip = matches!(nulls, NullsPosition::First); + term = format!("{col} IS NOT NULL = {}, {term}", u8::from(flip)); + } + _ => term.push_str(&format!(" NULLS {n}")), + } + } + parts.push(term); + } + Ok(parts.join(", ")) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn spec(column: &str, op: FilterOp, value: Option) -> FilterSpec { + FilterSpec { + column: column.into(), + op, + value, + value2: None, + values: Vec::new(), + case_sensitive: None, + disabled: false, + } + } + + #[test] + fn no_filters_compiles_to_an_empty_clause() { + let w = compile_where(DriverKind::Sqlite, &[], 1).unwrap(); + assert!(w.sql.is_empty()); + assert!(w.params.is_empty()); + } + + #[test] + fn like_metacharacters_are_escaped_so_a_literal_percent_matches() { + let w = compile_where( + DriverKind::Sqlite, + &[spec("code", FilterOp::Contains, Some(json!("50%")))], + 1, + ) + .unwrap(); + // Without escaping this pattern would match every row. + assert_eq!(w.params[0], json!("%50\\%%")); + assert!(w.sql.contains("ESCAPE '\\'"), "got: {}", w.sql); + } + + #[test] + fn underscore_and_the_escape_char_are_escaped_too() { + let w = compile_where( + DriverKind::Sqlite, + &[spec("p", FilterOp::StartsWith, Some(json!("a_b\\c")))], + 1, + ) + .unwrap(); + assert_eq!(w.params[0], json!("a\\_b\\\\c%")); + } + + #[test] + fn postgres_numbers_placeholders_and_others_do_not() { + let filters = [ + spec("a", FilterOp::Equals, Some(json!(1))), + spec("b", FilterOp::Equals, Some(json!(2))), + ]; + let pg = compile_where(DriverKind::Postgres, &filters, 1).unwrap(); + assert_eq!(pg.sql, r#""a" = $1 AND "b" = $2"#); + + let my = compile_where(DriverKind::Mysql, &filters, 1).unwrap(); + assert_eq!(my.sql, "`a` = ? AND `b` = ?"); + } + + #[test] + fn placeholder_numbering_continues_from_the_callers_offset() { + let w = compile_where( + DriverKind::Postgres, + &[spec("a", FilterOp::Equals, Some(json!(1)))], + 4, + ) + .unwrap(); + assert_eq!(w.sql, r#""a" = $4"#); + } + + #[test] + fn is_null_and_is_empty_are_different_questions() { + let null = + compile_where(DriverKind::Sqlite, &[spec("a", FilterOp::IsNull, None)], 1).unwrap(); + assert_eq!(null.sql, r#""a" IS NULL"#); + assert!(null.params.is_empty()); + + let empty = + compile_where(DriverKind::Sqlite, &[spec("a", FilterOp::IsEmpty, None)], 1).unwrap(); + assert_eq!(empty.sql, r#"("a" IS NULL OR "a" = '')"#); + } + + #[test] + fn not_contains_keeps_null_rows() { + let w = compile_where( + DriverKind::Sqlite, + &[spec("a", FilterOp::NotContains, Some(json!("x")))], + 1, + ) + .unwrap(); + // A bare NOT LIKE would drop NULLs, which reads as data loss. + assert!( + w.sql.starts_with(r#"("a" IS NULL OR NOT ("#), + "got: {}", + w.sql + ); + } + + #[test] + fn case_insensitive_uses_ilike_on_postgres_only() { + let pg = compile_where( + DriverKind::Postgres, + &[spec("a", FilterOp::Contains, Some(json!("x")))], + 1, + ) + .unwrap(); + assert!(pg.sql.contains("ILIKE"), "got: {}", pg.sql); + + let lite = compile_where( + DriverKind::Sqlite, + &[spec("a", FilterOp::Contains, Some(json!("x")))], + 1, + ) + .unwrap(); + assert!(lite.sql.contains("LIKE") && !lite.sql.contains("ILIKE")); + } + + #[test] + fn case_sensitive_flag_is_rejected_where_it_cannot_be_honoured() { + let mut f = spec("a", FilterOp::Contains, Some(json!("x"))); + f.case_sensitive = Some(true); + let err = compile_where(DriverKind::Sqlite, &[f], 1).unwrap_err(); + let body = serde_json::to_string(&err).unwrap(); + assert!(body.contains("postgres pattern operators"), "got: {body}"); + } + + #[test] + fn an_incomplete_filter_is_refused_rather_than_guessed() { + let err = + compile_where(DriverKind::Sqlite, &[spec("a", FilterOp::Equals, None)], 1).unwrap_err(); + let body = serde_json::to_string(&err).unwrap(); + assert!(body.contains("needs `value`"), "got: {body}"); + + let mut between = spec("a", FilterOp::Between, Some(json!(1))); + between.value2 = None; + let err = compile_where(DriverKind::Sqlite, &[between], 1).unwrap_err(); + let body = serde_json::to_string(&err).unwrap(); + assert!(body.contains("`value2`"), "got: {body}"); + } + + #[test] + fn between_binds_both_bounds_in_order() { + let mut f = spec("n", FilterOp::Between, Some(json!(1))); + f.value2 = Some(json!(9)); + let w = compile_where(DriverKind::Postgres, &[f], 1).unwrap(); + assert_eq!(w.sql, r#""n" BETWEEN $1 AND $2"#); + assert_eq!(w.params, vec![json!(1), json!(9)]); + } + + #[test] + fn quoting_defeats_an_identifier_break_out() { + assert_eq!( + quote_ident(DriverKind::Postgres, r#"a" OR 1=1--"#), + r#""a"" OR 1=1--""# + ); + assert_eq!(quote_ident(DriverKind::Mysql, "a`b"), "`a``b`"); + } + + #[test] + fn table_is_schema_qualified_only_on_postgres() { + assert_eq!( + quote_table(DriverKind::Postgres, Some("analytics"), "events"), + r#""analytics"."events""# + ); + assert_eq!( + quote_table(DriverKind::Mysql, Some("ignored"), "events"), + "`events`" + ); + } + + #[test] + fn order_by_emits_direction_and_nulls_placement() { + let sorts = [SortSpec { + column: "created_at".into(), + direction: Direction::Desc, + nulls: Some(NullsPosition::Last), + mode: SortMode::Default, + }]; + let pg = compile_order_by(DriverKind::Postgres, &sorts).unwrap(); + assert_eq!(pg, r#""created_at" DESC NULLS LAST"#); + + // MySQL has no NULLS clause, so it emulates with a leading key. + let my = compile_order_by(DriverKind::Mysql, &sorts).unwrap(); + assert!(my.contains("IS NOT NULL ="), "got: {my}"); + assert!(!my.contains("NULLS"), "got: {my}"); + } + + #[test] + fn length_mode_uses_the_drivers_own_function() { + let s = [SortSpec { + column: "name".into(), + direction: Direction::Asc, + nulls: None, + mode: SortMode::Length, + }]; + assert_eq!( + compile_order_by(DriverKind::Mysql, &s).unwrap(), + "CHAR_LENGTH(`name`) ASC" + ); + assert_eq!( + compile_order_by(DriverKind::Sqlite, &s).unwrap(), + r#"LENGTH("name") ASC"# + ); + } + + #[test] + fn the_wrong_operand_alone_is_refused_rather_than_unwrapped() { + // `value2` without `value` used to satisfy a count-based arity check + // and then panic on the missing `value`. Reachable from the wire. + let mut f = spec("a", FilterOp::Equals, None); + f.value2 = Some(json!(5)); + let err = compile_where(DriverKind::Sqlite, &[f], 1).unwrap_err(); + assert!(format!("{err:?}").contains("needs `value`")); + } + + #[test] + fn between_needs_both_bounds_not_just_a_count_of_two() { + let mut f = spec("a", FilterOp::Between, None); + f.value2 = Some(json!(5)); + assert!(compile_where(DriverKind::Sqlite, &[f], 1).is_err()); + } + + #[test] + fn case_sensitive_is_refused_where_it_would_do_nothing() { + // Not a pattern operator, so the flag can never apply — even on the + // one driver that supports it for LIKE. + let mut f = spec("a", FilterOp::Equals, Some(json!("x"))); + f.case_sensitive = Some(true); + assert!(compile_where(DriverKind::Postgres, &[f], 1).is_err()); + + // Still accepted where it is honoured. + let mut ok = spec("a", FilterOp::Contains, Some(json!("x"))); + ok.case_sensitive = Some(true); + assert!(compile_where(DriverKind::Postgres, &[ok], 1).is_ok()); + } + + #[test] + fn in_binds_every_member_and_numbers_them_on_postgres() { + let mut f = spec("status", FilterOp::In, None); + f.values = vec![json!("open"), json!("held")]; + let c = compile_where(DriverKind::Postgres, &[f], 1).unwrap(); + assert_eq!(c.sql, r#""status" IN ($1, $2)"#); + assert_eq!(c.params, vec![json!("open"), json!("held")]); + } + + #[test] + fn not_in_keeps_nulls() { + // `NULL NOT IN (...)` is unknown, so a plain NOT IN silently drops + // every NULL row. Asking for "not one of these" should keep them. + let mut f = spec("status", FilterOp::NotIn, None); + f.values = vec![json!("open")]; + let c = compile_where(DriverKind::Sqlite, &[f], 1).unwrap(); + assert_eq!(c.sql, r#"("status" IS NULL OR "status" NOT IN (?))"#); + } + + #[test] + fn an_empty_set_is_refused_rather_than_dropped() { + let f = spec("status", FilterOp::In, None); + let err = compile_where(DriverKind::Sqlite, &[f], 1).unwrap_err(); + assert!(format!("{err:?}").contains("at least one value")); + } + + #[test] + fn a_disabled_filter_is_skipped_but_its_neighbours_still_compile() { + let mut off = spec("plan", FilterOp::Equals, Some(json!("free"))); + off.disabled = true; + let on = spec("status", FilterOp::Equals, Some(json!("open"))); + let c = compile_where(DriverKind::Postgres, &[off, on], 1).unwrap(); + // Numbering must close up: the skipped filter must not burn $1. + assert_eq!(c.sql, r#""status" = $1"#); + assert_eq!(c.params, vec![json!("open")]); + } + + #[test] + fn a_disabled_filter_is_not_validated() { + // Half-built filters are the normal state of a chip being edited; + // disabling one must not turn the whole request into an error. + let mut half = spec("", FilterOp::Equals, None); + half.disabled = true; + let c = compile_where(DriverKind::Sqlite, &[half], 1).unwrap(); + assert_eq!(c.sql, ""); + } + + #[test] + fn natural_mode_falls_back_rather_than_faking_it() { + let s = [SortSpec { + column: "label".into(), + direction: Direction::Asc, + nulls: None, + mode: SortMode::Natural, + }]; + assert!(compile_order_by(DriverKind::Postgres, &s) + .unwrap() + .contains("regexp_replace")); + // No portable equivalent — order plainly instead of approximating. + assert_eq!( + compile_order_by(DriverKind::Sqlite, &s).unwrap(), + r#""label" ASC"# + ); + } +} diff --git a/database/src/handlers/health.rs b/database/src/handlers/health.rs new file mode 100644 index 000000000..638725ffc --- /dev/null +++ b/database/src/handlers/health.rs @@ -0,0 +1,450 @@ +//! `database::health` — live operational state, honestly reported. +//! +//! The design point is `ProbeResult`. Each section answers separately, so a +//! caller can tell "sqlite has no equivalent of `pg_stat_activity`" from +//! "there are zero active queries" from "this role may not read +//! `pg_stat_activity`". Collapsing those into an empty list would render a +//! confidently wrong panel, and a restricted application role is the common +//! case rather than the exception — so one probe being denied never fails the +//! whole call. +//! +//! Boundary against `database::testConnection`: that probes a *candidate* URL +//! that is not configured yet. This reports the live state of a pool that +//! already exists, and deliberately accepts no URL. + +use super::query::{self, err_to_str, QueryReq}; +use super::AppState; +use crate::config::DriverKind; +use crate::error::DbError; +use crate::pool::PoolStats; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +fn default_timeout() -> u64 { + 15_000 +} + +/// One section of the report. +#[derive(Debug, Clone, Serialize, JsonSchema)] +#[serde(tag = "status", rename_all = "lowercase")] +pub enum ProbeResult { + /// The driver answered. + Available { data: T }, + /// The driver has no equivalent of this concept. + Unsupported { reason: String }, + /// The driver has it, but this role may not read it. + Denied { reason: String }, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct ActiveQuery { + pub id: String, + pub sql: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub state: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct TableSize { + pub table: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub total_bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub index_bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub row_estimate: Option, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct LockInfo { + pub blocked_id: String, + pub blocked_sql: String, + pub blocking_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub blocking_sql: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub relation: Option, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct CacheStats { + /// Fraction of block reads served from cache. A healthy OLTP database + /// usually sits well above 0.99. + pub hit_ratio: f64, + pub blocks_hit: i64, + pub blocks_read: i64, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct HealthReq { + #[serde(default)] + pub db: Option, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct HealthResp { + pub db: String, + pub driver: String, + pub worker_version: String, + pub pool: PoolStats, + pub active_queries: ProbeResult>, + pub table_sizes: ProbeResult>, + pub locks: ProbeResult>, + pub cache: ProbeResult, +} + +fn unsupported(driver: DriverKind, what: &str) -> ProbeResult { + ProbeResult::Unsupported { + reason: format!("{driver:?} has no equivalent of {what}").to_lowercase(), + } +} + +/// Turn a probe failure into a per-section result rather than failing the +/// whole call. A permission error on one view must not hide the others. +fn probe(outcome: Result) -> ProbeResult { + match outcome { + Ok(data) => ProbeResult::Available { data }, + Err(reason) => ProbeResult::Denied { reason }, + } +} + +async fn rows( + state: &AppState, + db: &str, + sql: &str, + timeout_ms: u64, +) -> Result>, String> { + Ok(query::handle( + state, + QueryReq { + db: Some(db.to_string()), + sql: sql.to_string(), + params: vec![], + timeout_ms, + record_history: false, + }, + ) + .await? + .rows) +} + +fn s_at(r: &serde_json::Map, k: &str) -> Option { + match r.get(k) { + Some(Value::String(s)) => Some(s.clone()), + Some(Value::Number(n)) => Some(n.to_string()), + _ => None, + } +} + +fn i_at(r: &serde_json::Map, k: &str) -> Option { + match r.get(k) { + Some(Value::Number(n)) => n.as_i64().or_else(|| n.as_f64().map(|f| f as i64)), + Some(Value::String(s)) => s.parse().ok(), + _ => None, + } +} + +pub async fn handle(state: &AppState, req: HealthReq) -> Result { + let db = state.resolve_db(req.db).await.map_err(err_to_str)?; + let pool = state.pool(&db).await.map_err(err_to_str)?; + let driver = pool.driver(); + let t = req.timeout_ms; + + let (active_queries, table_sizes, locks, cache) = match driver { + DriverKind::Postgres => ( + probe(pg_active(state, &db, t).await), + probe(pg_sizes(state, &db, t).await), + probe(pg_locks(state, &db, t).await), + probe(pg_cache(state, &db, t).await), + ), + DriverKind::Mysql => ( + probe(mysql_active(state, &db, t).await), + probe(mysql_sizes(state, &db, t).await), + unsupported(driver, "a queryable lock-wait graph"), + unsupported(driver, "a per-database buffer-pool hit ratio"), + ), + DriverKind::Sqlite => ( + // SQLite runs in-process: there is no server holding sessions. + unsupported(driver, "server-side sessions"), + unsupported(driver, "per-table size accounting"), + unsupported(driver, "a queryable lock table"), + unsupported(driver, "a shared buffer cache"), + ), + }; + + Ok(HealthResp { + db, + driver: format!("{driver:?}").to_lowercase(), + worker_version: env!("CARGO_PKG_VERSION").to_string(), + pool: pool.stats(), + active_queries, + table_sizes, + locks, + cache, + }) +} + +/* ---------------- postgres ---------------- */ + +async fn pg_active(state: &AppState, db: &str, t: u64) -> Result, String> { + let r = rows( + state, + db, + "SELECT pid::text AS id, query AS sql, state, usename AS usr, \ + (EXTRACT(EPOCH FROM (now() - query_start)) * 1000)::bigint AS duration_ms \ + FROM pg_stat_activity \ + WHERE datname = current_database() AND pid <> pg_backend_pid() \ + AND state <> 'idle' \ + ORDER BY query_start", + t, + ) + .await?; + Ok(r.iter() + .filter_map(|x| { + Some(ActiveQuery { + id: s_at(x, "id")?, + sql: s_at(x, "sql").unwrap_or_default(), + state: s_at(x, "state"), + duration_ms: i_at(x, "duration_ms"), + user: s_at(x, "usr"), + }) + }) + .collect()) +} + +async fn pg_sizes(state: &AppState, db: &str, t: u64) -> Result, String> { + let r = rows( + state, + db, + "SELECT n.nspname AS schema_name, c.relname AS table_name, \ + pg_total_relation_size(c.oid)::bigint AS total_bytes, \ + pg_indexes_size(c.oid)::bigint AS index_bytes, \ + c.reltuples::bigint AS row_estimate \ + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE c.relkind IN ('r', 'p') \ + AND n.nspname NOT IN ('pg_catalog', 'information_schema') \ + ORDER BY pg_total_relation_size(c.oid) DESC LIMIT 100", + t, + ) + .await?; + Ok(r.iter() + .filter_map(|x| { + Some(TableSize { + table: s_at(x, "table_name")?, + schema: s_at(x, "schema_name"), + total_bytes: i_at(x, "total_bytes"), + index_bytes: i_at(x, "index_bytes"), + row_estimate: i_at(x, "row_estimate").filter(|v| *v >= 0), + }) + }) + .collect()) +} + +/// Resolves blockers by joining `pg_locks` to itself. Deliberately avoids +/// `pg_blocking_pids()`, which returns `int[]` — and `RowValue` has no array +/// variant, so that column would fail to decode. +async fn pg_locks(state: &AppState, db: &str, t: u64) -> Result, String> { + let r = rows( + state, + db, + "SELECT w.pid::text AS blocked_id, wa.query AS blocked_sql, \ + b.pid::text AS blocking_id, ba.query AS blocking_sql, \ + COALESCE(c.relname, '') AS relation \ + FROM pg_locks w \ + JOIN pg_locks b ON b.granted AND NOT w.granted \ + AND b.pid <> w.pid \ + AND b.locktype = w.locktype \ + AND b.database IS NOT DISTINCT FROM w.database \ + AND b.relation IS NOT DISTINCT FROM w.relation \ + AND b.transactionid IS NOT DISTINCT FROM w.transactionid \ + JOIN pg_stat_activity wa ON wa.pid = w.pid \ + LEFT JOIN pg_stat_activity ba ON ba.pid = b.pid \ + LEFT JOIN pg_class c ON c.oid = w.relation \ + WHERE NOT w.granted", + t, + ) + .await?; + Ok(r.iter() + .filter_map(|x| { + Some(LockInfo { + blocked_id: s_at(x, "blocked_id")?, + blocked_sql: s_at(x, "blocked_sql").unwrap_or_default(), + blocking_id: s_at(x, "blocking_id")?, + blocking_sql: s_at(x, "blocking_sql"), + relation: s_at(x, "relation").filter(|s| !s.is_empty()), + }) + }) + .collect()) +} + +async fn pg_cache(state: &AppState, db: &str, t: u64) -> Result { + let r = rows( + state, + db, + "SELECT COALESCE(SUM(heap_blks_hit), 0)::bigint AS hit, \ + COALESCE(SUM(heap_blks_read), 0)::bigint AS rd \ + FROM pg_statio_user_tables", + t, + ) + .await?; + let row = r.first().cloned().unwrap_or_default(); + let hit = i_at(&row, "hit").unwrap_or(0); + let read = i_at(&row, "rd").unwrap_or(0); + let total = hit + read; + Ok(CacheStats { + // No reads yet is not a 0% hit rate; report it as perfect rather than + // as an alarming zero. + hit_ratio: if total == 0 { + 1.0 + } else { + hit as f64 / total as f64 + }, + blocks_hit: hit, + blocks_read: read, + }) +} + +/* ---------------- mysql ---------------- */ + +async fn mysql_active(state: &AppState, db: &str, t: u64) -> Result, String> { + let r = rows( + state, + db, + "SELECT ID AS id, INFO AS sql_text, STATE AS state, USER AS usr, \ + TIME * 1000 AS duration_ms \ + FROM information_schema.PROCESSLIST \ + WHERE DB = DATABASE() AND COMMAND <> 'Sleep' AND ID <> CONNECTION_ID() \ + ORDER BY TIME DESC", + t, + ) + .await?; + Ok(r.iter() + .filter_map(|x| { + Some(ActiveQuery { + id: s_at(x, "id")?, + sql: s_at(x, "sql_text").unwrap_or_default(), + state: s_at(x, "state"), + duration_ms: i_at(x, "duration_ms"), + user: s_at(x, "usr"), + }) + }) + .collect()) +} + +async fn mysql_sizes(state: &AppState, db: &str, t: u64) -> Result, String> { + let r = rows( + state, + db, + "SELECT TABLE_NAME AS table_name, \ + (DATA_LENGTH + INDEX_LENGTH) AS total_bytes, \ + INDEX_LENGTH AS index_bytes, TABLE_ROWS AS row_estimate \ + FROM information_schema.TABLES \ + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_TYPE = 'BASE TABLE' \ + ORDER BY (DATA_LENGTH + INDEX_LENGTH) DESC LIMIT 100", + t, + ) + .await?; + Ok(r.iter() + .filter_map(|x| { + Some(TableSize { + table: s_at(x, "table_name")?, + schema: None, + total_bytes: i_at(x, "total_bytes"), + index_bytes: i_at(x, "index_bytes"), + row_estimate: i_at(x, "row_estimate"), + }) + }) + .collect()) +} + +/* ---------------- terminateQuery ---------------- */ + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct TerminateReq { + #[serde(default)] + pub db: Option, + /// Backend pid (postgres) or connection id (mysql), as reported by + /// `database::health`. + pub id: String, + /// Ask the backend to cancel the running statement but keep the session. + /// The default terminates the session outright. + #[serde(default)] + pub cancel_only: bool, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct TerminateResp { + pub id: String, + pub terminated: bool, +} + +/// A separate function from `health` on purpose: this is a write, and a +/// read-only viewer must not be able to synthesise it from a report. +pub async fn terminate(state: &AppState, req: TerminateReq) -> Result { + let db = state.resolve_db(req.db).await.map_err(err_to_str)?; + let pool = state.pool(&db).await.map_err(err_to_str)?; + let driver = pool.driver(); + + // The id is interpolated, so it must be exactly a number — never trust it + // as an identifier. + let id: i64 = req.id.trim().parse().map_err(|_| { + err_to_str(DbError::InvalidParam { + index: 0, + reason: format!("`{}` is not a backend id", req.id), + }) + })?; + + let sql = match (driver, req.cancel_only) { + (DriverKind::Postgres, true) => format!("SELECT pg_cancel_backend({id}) AS ok"), + (DriverKind::Postgres, false) => format!("SELECT pg_terminate_backend({id}) AS ok"), + (DriverKind::Mysql, true) => format!("KILL QUERY {id}"), + (DriverKind::Mysql, false) => format!("KILL CONNECTION {id}"), + (DriverKind::Sqlite, _) => { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: "sqlite runs in-process and has no sessions to terminate".into(), + })) + } + }; + + let terminated = match driver { + DriverKind::Postgres => { + let r = rows(state, &db, &sql, req.timeout_ms).await?; + r.first() + .and_then(|x| x.get("ok")) + .and_then(Value::as_bool) + .unwrap_or(false) + } + // KILL returns no result set; reaching here without an error is the + // signal. + _ => { + super::execute::handle( + state, + super::execute::ExecuteReq { + db: Some(db.clone()), + sql: sql.clone(), + params: vec![], + returning: vec![], + }, + ) + .await?; + true + } + }; + + Ok(TerminateResp { + id: req.id, + terminated, + }) +} diff --git a/database/src/handlers/mod.rs b/database/src/handlers/mod.rs index 783fad377..16087c543 100644 --- a/database/src/handlers/mod.rs +++ b/database/src/handlers/mod.rs @@ -30,14 +30,24 @@ where } pub mod begin_transaction; +pub mod browse; +pub mod catalog; +pub mod column_stats; pub mod commit_transaction; +pub mod diagram; pub mod execute; pub mod execute_batch; +pub mod explain; +pub mod filter; +pub mod health; pub mod list_databases; pub mod prepare; pub mod query; pub mod rollback_transaction; pub mod run_statement; +pub mod saved; +pub mod schema; +pub mod table_view; pub mod test_connection; pub mod transaction; pub mod transaction_execute; @@ -97,6 +107,12 @@ impl AppState { .is_none_or(|d| d.capture.is_statements()) } + /// Engine client, when the worker is connected to one. `None` in tests, + /// where sibling-worker calls are not available and history is skipped. + pub fn client(&self) -> Option<&Arc> { + self.row_changes.as_ref().map(|bus| bus.client()) + } + /// Announce a committed change, if anything is listening. Every mutating /// handler ends with this; the bus decides whether the statement changed /// rows and who cares. diff --git a/database/src/handlers/query.rs b/database/src/handlers/query.rs index 3dde87fd9..411e80549 100644 --- a/database/src/handlers/query.rs +++ b/database/src/handlers/query.rs @@ -21,6 +21,20 @@ pub struct QueryReq { pub params: Vec, #[serde(default = "default_timeout")] pub timeout_ms: u64, + /// Whether this run belongs in `database::history`. + /// + /// Internal only — never deserialized, so a statement arriving over the + /// wire always records. The catalog reads behind `describeSchema`, + /// `browseTable`, `columnStats`, `explain` and `schemaDiagram` all run + /// through this same path, and recording them would bury the statements a + /// person actually typed under `PRAGMA table_info(...)` and spawn a + /// `state::update` per internal read. + #[serde(skip, default = "records_history")] + pub record_history: bool, +} + +fn records_history() -> bool { + true } #[derive(Debug, Serialize, JsonSchema)] @@ -55,6 +69,7 @@ pub async fn handle(state: &AppState, req: QueryReq) -> Result driver::sqlite::query(p, &req.sql, ¶ms, req.timeout_ms).await, Pool::Postgres(p) => driver::postgres::query(p, &req.sql, ¶ms, req.timeout_ms).await, @@ -63,6 +78,18 @@ pub async fn handle(state: &AppState, req: QueryReq) -> Result String { +pub fn err_to_str(e: DbError) -> String { serde_json::to_string(&e).unwrap_or_else(|_| { format!( "{{\"code\":\"DRIVER_ERROR\",\"message\":{:?}}}", diff --git a/database/src/handlers/saved.rs b/database/src/handlers/saved.rs new file mode 100644 index 000000000..89ada1171 --- /dev/null +++ b/database/src/handlers/saved.rs @@ -0,0 +1,368 @@ +//! `database::saveQuery` / `listSavedQueries` / `deleteSavedQuery` / +//! `history` — the queries you keep, and the ones you ran. +//! +//! These are thin wrappers over the `state` worker, not a store of their own. +//! Query history used to live in browser `localStorage`, which meant it +//! existed for one person, in one browser, and vanished on a cache clear. On +//! `state::*` it survives restarts, any agent can read it, and an agent can +//! save a query for a human to find in the console. +//! +//! Recording is deliberately cheap and deliberately lossy: it is fire and +//! forget, never awaited on the query path, and a `state` failure never fails +//! the user's query. History is a convenience, not an audit log — for an +//! audit trail, bind the `database::row-changed` trigger instead. + +use super::query::err_to_str; +use crate::error::DbError; +use iii_sdk::protocol::TriggerRequest; +use iii_sdk::IIIClient; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::sync::Arc; + +/// Scope every key lives under in the `state` worker. +pub(super) const SCOPE: &str = "database"; +/// Entries returned by `history` unless the caller asks for fewer. +const HISTORY_LIMIT: usize = 50; +/// Stored entries are trimmed back to `HISTORY_LIMIT` once they exceed this. +/// Trimming on read rather than on write keeps the hot path a single atomic +/// append instead of a read-modify-write. +const HISTORY_HIGH_WATER: usize = 200; +/// SQL longer than this is truncated before it is stored. +const MAX_SQL_CHARS: usize = 4_000; + +fn saved_key(db: &str) -> String { + format!("saved:{db}") +} + +fn history_key(db: &str) -> String { + format!("history:{db}") +} + +pub(super) async fn call( + iii: &Arc, + function_id: &str, + payload: Value, +) -> Result { + iii.trigger(TriggerRequest { + function_id: function_id.to_string(), + payload, + action: None, + timeout_ms: Some(10_000), + }) + .await + .map_err(|e| { + err_to_str(DbError::ConfigError { + message: format!( + "{function_id} failed: {e}. Saved queries and history need the `state` \ + worker — run `iii worker add state`." + ), + }) + }) +} + +async fn state_get(iii: &Arc, key: &str) -> Result, String> { + let raw = call(iii, "state::get", json!({"scope": SCOPE, "key": key})).await?; + // A missing key is an empty list, not an error. + Ok(raw + .get("value") + .or(Some(&raw)) + .and_then(Value::as_array) + .cloned() + .unwrap_or_default()) +} + +async fn state_set(iii: &Arc, key: &str, value: Value) -> Result<(), String> { + call( + iii, + "state::set", + json!({"scope": SCOPE, "key": key, "value": value}), + ) + .await?; + Ok(()) +} + +/* ---------------- saved queries ---------------- */ + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct SavedQuery { + pub id: String, + pub name: String, + pub sql: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + pub saved_at: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct SaveQueryReq { + #[serde(default)] + pub db: Option, + pub name: String, + pub sql: String, + #[serde(default)] + pub description: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct SaveQueryResp { + pub id: String, + pub replaced: bool, +} + +pub async fn save( + iii: &Arc, + db: &str, + req: SaveQueryReq, +) -> Result { + if req.name.trim().is_empty() || req.sql.trim().is_empty() { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: "name and sql are both required".into(), + })); + } + let key = saved_key(db); + let mut items: Vec = state_get(iii, &key) + .await? + .into_iter() + .filter_map(|v| serde_json::from_value(v).ok()) + .collect(); + + // Saving under an existing name replaces it rather than accumulating + // near-duplicates the user then has to tell apart. + let replaced = items.iter().any(|q| q.name == req.name); + let id = items + .iter() + .find(|q| q.name == req.name) + .map(|q| q.id.clone()) + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + items.retain(|q| q.name != req.name); + items.push(SavedQuery { + id: id.clone(), + name: req.name, + sql: truncate(&req.sql), + description: req.description, + saved_at: now(), + }); + items.sort_by(|a, b| a.name.cmp(&b.name)); + + state_set(iii, &key, serde_json::to_value(&items).unwrap_or(json!([]))).await?; + Ok(SaveQueryResp { id, replaced }) +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ListSavedReq { + #[serde(default)] + pub db: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ListSavedResp { + pub queries: Vec, + pub count: usize, +} + +pub async fn list(iii: &Arc, db: &str) -> Result { + let queries: Vec = state_get(iii, &saved_key(db)) + .await? + .into_iter() + // Drop anything that no longer parses rather than failing the call — + // a stale entry should not make the list unreadable. + .filter_map(|v| serde_json::from_value(v).ok()) + .collect(); + Ok(ListSavedResp { + count: queries.len(), + queries, + }) +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct DeleteSavedReq { + #[serde(default)] + pub db: Option, + /// Either the id returned by `saveQuery`, or the name it was saved under. + pub id: String, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct DeleteSavedResp { + pub deleted: bool, +} + +pub async fn delete( + iii: &Arc, + db: &str, + req: DeleteSavedReq, +) -> Result { + let key = saved_key(db); + let mut items: Vec = state_get(iii, &key) + .await? + .into_iter() + .filter_map(|v| serde_json::from_value(v).ok()) + .collect(); + let before = items.len(); + items.retain(|q| q.id != req.id && q.name != req.id); + let deleted = items.len() != before; + if deleted { + state_set(iii, &key, serde_json::to_value(&items).unwrap_or(json!([]))).await?; + } + Ok(DeleteSavedResp { deleted }) +} + +/* ---------------- history ---------------- */ + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct HistoryEntry { + pub sql: String, + pub verb: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub row_count: Option, + pub at: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct HistoryReq { + #[serde(default)] + pub db: Option, + #[serde(default)] + pub limit: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct HistoryResp { + pub entries: Vec, + pub count: usize, +} + +pub async fn history( + iii: &Arc, + db: &str, + req: HistoryReq, +) -> Result { + let key = history_key(db); + let raw = state_get(iii, &key).await?; + let all: Vec = raw + .iter() + .filter_map(|v| serde_json::from_value(v.clone()).ok()) + .collect(); + + // Self-healing trim: the write path only appends, so the stored list is + // capped here once it drifts past the high-water mark. + if all.len() > HISTORY_HIGH_WATER { + let tail: Vec<&HistoryEntry> = all.iter().rev().take(HISTORY_LIMIT).rev().collect(); + let _ = state_set(iii, &key, serde_json::to_value(&tail).unwrap_or(json!([]))).await; + } + + let limit = req + .limit + .unwrap_or(HISTORY_LIMIT) + .clamp(1, HISTORY_HIGH_WATER); + let entries: Vec = all.into_iter().rev().take(limit).collect(); + Ok(HistoryResp { + count: entries.len(), + entries, + }) +} + +/// The `state::update` op list for appending one entry. +/// +/// Split out only so a test can assert the discriminator, which is the one +/// detail of this file that cannot be caught at runtime. +fn append_ops(entry: Value) -> Vec { + vec![json!({ "type": "append", "value": entry })] +} + +/// Record one run. Fire and forget: never awaited on the query path, and a +/// failure is logged rather than surfaced, because losing a history line must +/// never fail the query the user actually asked for. +pub fn record(iii: Arc, db: String, sql: &str, duration_ms: u64, row_count: usize) { + let entry = HistoryEntry { + sql: truncate(sql), + verb: leading_verb(sql), + duration_ms: Some(duration_ms), + row_count: Some(row_count), + at: now(), + }; + tokio::spawn(async move { + let payload = json!({ + "scope": SCOPE, + "key": history_key(&db), + "ops": append_ops(serde_json::to_value(&entry).unwrap_or(json!({}))), + }); + if let Err(e) = call(&iii, "state::update", payload).await { + tracing::warn!(error = %e, "history not recorded"); + } + }); +} + +fn now() -> String { + chrono::Utc::now().to_rfc3339() +} + +fn truncate(sql: &str) -> String { + if sql.chars().count() <= MAX_SQL_CHARS { + return sql.to_string(); + } + let head: String = sql.chars().take(MAX_SQL_CHARS).collect(); + format!("{head}…") +} + +/// First keyword, lowercased — enough to group history without storing a +/// parse tree. +fn leading_verb(sql: &str) -> String { + sql.split(|c: char| !c.is_ascii_alphabetic()) + .find(|w| !w.is_empty()) + .unwrap_or("unknown") + .to_ascii_lowercase() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn leading_verb_ignores_punctuation_and_case() { + assert_eq!(leading_verb(" SELECT * FROM t"), "select"); + assert_eq!(leading_verb("(select 1)"), "select"); + assert_eq!(leading_verb("INSERT INTO t VALUES (1)"), "insert"); + assert_eq!(leading_verb(""), "unknown"); + } + + #[test] + fn long_sql_is_truncated_with_a_marker() { + let long = "x".repeat(MAX_SQL_CHARS + 50); + let t = truncate(&long); + assert_eq!(t.chars().count(), MAX_SQL_CHARS + 1); + assert!(t.ends_with('…')); + + let short = "SELECT 1"; + assert_eq!(truncate(short), short); + } + + #[test] + fn truncation_counts_characters_not_bytes() { + // A byte-based cut would slice a multi-byte character in half. + let s = "é".repeat(MAX_SQL_CHARS + 10); + let t = truncate(&s); + assert_eq!(t.chars().count(), MAX_SQL_CHARS + 1); + } + + #[test] + fn append_op_uses_the_type_discriminator() { + // Locked deliberately: `record` is fire-and-forget, so a wrong op + // shape fails where nobody is looking. This is the only cheap place + // to notice. + let ops = append_ops(json!({"sql": "select 1"})); + assert_eq!(ops[0]["type"], "append"); + assert!(ops[0].get("op").is_none()); + } + + #[test] + fn keys_are_scoped_per_database() { + assert_eq!(saved_key("primary"), "saved:primary"); + assert_eq!(history_key("analytics"), "history:analytics"); + } +} diff --git a/database/src/handlers/schema.rs b/database/src/handlers/schema.rs new file mode 100644 index 000000000..498ad741e --- /dev/null +++ b/database/src/handlers/schema.rs @@ -0,0 +1,274 @@ +//! `database::listTables` / `describeTable` / `describeSchema` — what is +//! *inside* a database, as opposed to what is on the bus. +//! +//! The engine already introspects itself (`engine::functions::list` and +//! friends); nothing in the engine knows what a table is, because only this +//! worker holds the connection pools. These functions are the SQL-catalog +//! equivalent, so an agent can ask what tables exist and how they relate +//! without hand-writing `sqlite_master` / `information_schema` / `PRAGMA` per +//! driver — which is exactly what the console was doing before. +//! +//! `describe_table` is a one-table `describe_schema`, so there is a single +//! assembly path. The difference that matters is the filter: describing one +//! table scopes every catalog query to it, while describing a whole schema +//! runs each query once across all tables and regroups in Rust. A 200-table +//! schema therefore costs three or four queries, not six hundred. + +use super::catalog::{self, ColumnDesc, IndexDesc, TableFilter, TableKey, TableKind, TableRef}; +use super::AppState; +use crate::config::DriverKind; +use crate::error::DbError; +use crate::handlers::query::err_to_str; +use crate::pool::Pool; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +fn default_timeout() -> u64 { + 30_000 +} + +fn default_max_tables() -> usize { + 500 +} + +/// Hard ceiling regardless of what the caller asks for. Describing a schema +/// is bounded work; describing an unbounded one is not. +const MAX_TABLES_CEILING: usize = 2_000; + +async fn driver_of(state: &AppState, db: &str) -> Result { + let pool = state.pool(db).await.map_err(err_to_str)?; + Ok(match pool { + Pool::Sqlite(_) => DriverKind::Sqlite, + Pool::Postgres(_) => DriverKind::Postgres, + Pool::Mysql(_) => DriverKind::Mysql, + }) +} + +fn no_such_table(driver: DriverKind, table: &str) -> String { + err_to_str(DbError::DriverError { + driver: format!("{driver:?}").to_lowercase(), + code: None, + message: format!("no such table: {table}"), + failed_index: None, + }) +} + +/* ---------------- listTables ---------------- */ + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ListTablesReq { + /// Logical database name. Optional — omitting it targets the sole + /// configured database, or `primary` when several are configured. + #[serde(default)] + pub db: Option, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ListTablesResp { + pub tables: Vec, + pub count: usize, +} + +pub async fn list_tables(state: &AppState, req: ListTablesReq) -> Result { + let db = state.resolve_db(req.db).await.map_err(err_to_str)?; + let tables = read_tables(state, &db, req.timeout_ms).await?; + Ok(ListTablesResp { + count: tables.len(), + tables, + }) +} + +async fn read_tables(state: &AppState, db: &str, timeout_ms: u64) -> Result, String> { + match driver_of(state, db).await? { + DriverKind::Sqlite => catalog::sqlite::list_tables(state, db, timeout_ms).await, + DriverKind::Postgres => catalog::postgres::list_tables(state, db, timeout_ms).await, + DriverKind::Mysql => catalog::mysql::list_tables(state, db, timeout_ms).await, + } +} + +/* ---------------- describeTable / describeSchema ---------------- */ + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct TableDescription { + pub table: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option, + pub kind: TableKind, + pub columns: Vec, + pub indexes: Vec, + /// Planner estimate, never a `COUNT(*)`. Absent when the driver has no + /// cheap estimate (sqlite) or has not analyzed the table yet. + #[serde(skip_serializing_if = "Option::is_none")] + pub row_count_estimate: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct DescribeTableReq { + #[serde(default)] + pub db: Option, + /// Table or view name. May be schema-qualified (`analytics.events`) on + /// postgres; prefer the explicit `schema` field when the name itself + /// contains a dot. + pub table: String, + #[serde(default)] + pub schema: Option, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +pub async fn describe_table( + state: &AppState, + req: DescribeTableReq, +) -> Result { + let db = state.resolve_db(req.db).await.map_err(err_to_str)?; + let driver = driver_of(state, &db).await?; + + // Only postgres has a namespace above the table, so only there does a dot + // in the name mean a schema qualifier. + let (schema, table) = match (&req.schema, driver) { + (Some(s), _) => (Some(s.clone()), req.table.clone()), + (None, DriverKind::Postgres) => catalog::split_qualified(&req.table), + (None, _) => (None, req.table.clone()), + }; + + let filter = TableFilter { + schema: schema.clone(), + table: table.clone(), + }; + let mut described = assemble(state, &db, driver, Some(&filter), true, req.timeout_ms).await?; + + // Scoping the catalog queries to one table means an unknown name simply + // yields nothing; say so rather than returning an empty description. + if described.is_empty() { + return Err(no_such_table(driver, &req.table)); + } + Ok(described.remove(0)) +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct DescribeSchemaReq { + #[serde(default)] + pub db: Option, + /// Restrict to these tables. Omit for every table in the database. + #[serde(default)] + pub tables: Option>, + /// Indexes cost one extra catalog query. Off by default because the + /// common caller (a relationship diagram) only needs columns and keys. + #[serde(default)] + pub include_indexes: bool, + #[serde(default = "default_max_tables")] + pub max_tables: usize, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct DescribeSchemaResp { + pub tables: Vec, + pub count: usize, + /// True when `max_tables` cut the result short. Never silently truncate. + pub truncated: bool, +} + +pub async fn describe_schema( + state: &AppState, + req: DescribeSchemaReq, +) -> Result { + let db = state.resolve_db(req.db).await.map_err(err_to_str)?; + let driver = driver_of(state, &db).await?; + let limit = req.max_tables.min(MAX_TABLES_CEILING); + + let mut tables = assemble( + state, + &db, + driver, + None, + req.include_indexes, + req.timeout_ms, + ) + .await?; + + if let Some(wanted) = &req.tables { + let wanted: Vec<(Option, String)> = wanted + .iter() + .map(|t| match driver { + DriverKind::Postgres => catalog::split_qualified(t), + _ => (None, t.clone()), + }) + .collect(); + tables.retain(|d| { + wanted + .iter() + .any(|(s, t)| &d.table == t && (s.is_none() || &d.schema == s)) + }); + } + + let truncated = tables.len() > limit; + tables.truncate(limit); + Ok(DescribeSchemaResp { + count: tables.len(), + tables, + truncated, + }) +} + +/// The single assembly path. One catalog query per aspect, regrouped by +/// table — never a query per table. +async fn assemble( + state: &AppState, + db: &str, + driver: DriverKind, + filter: Option<&TableFilter>, + include_indexes: bool, + timeout_ms: u64, +) -> Result, String> { + let tables = read_tables(state, db, timeout_ms).await?; + + let columns = match driver { + DriverKind::Sqlite => catalog::sqlite::columns(state, db, filter, timeout_ms).await?, + DriverKind::Postgres => catalog::postgres::columns(state, db, filter, timeout_ms).await?, + DriverKind::Mysql => catalog::mysql::columns(state, db, filter, timeout_ms).await?, + }; + + let mut indexes: HashMap> = HashMap::new(); + if include_indexes { + indexes = match driver { + DriverKind::Sqlite => catalog::sqlite::indexes(state, db, filter, timeout_ms).await?, + DriverKind::Postgres => { + catalog::postgres::indexes(state, db, filter, timeout_ms).await? + } + DriverKind::Mysql => catalog::mysql::indexes(state, db, filter, timeout_ms).await?, + }; + } + + let estimates = match driver { + DriverKind::Sqlite => catalog::sqlite::row_estimates(state, db, filter, timeout_ms).await?, + DriverKind::Postgres => { + catalog::postgres::row_estimates(state, db, filter, timeout_ms).await? + } + DriverKind::Mysql => catalog::mysql::row_estimates(state, db, filter, timeout_ms).await?, + }; + + let mut out = Vec::new(); + for t in tables { + let key: TableKey = (t.schema.clone(), t.name.clone()); + // A table with no column rows was filtered out by the catalog query, + // so it is not part of this result. + let Some(mut cols) = columns.get(&key).cloned() else { + continue; + }; + cols.sort_by_key(|c| c.position); + out.push(TableDescription { + table: t.name, + schema: t.schema, + kind: t.kind, + columns: cols, + indexes: indexes.get(&key).cloned().unwrap_or_default(), + row_count_estimate: estimates.get(&key).copied(), + }); + } + Ok(out) +} diff --git a/database/src/handlers/table_view.rs b/database/src/handlers/table_view.rs new file mode 100644 index 000000000..76d00f323 --- /dev/null +++ b/database/src/handlers/table_view.rs @@ -0,0 +1,183 @@ +//! `database::getTableView` / `saveTableView` — how a table is laid out. +//! +//! Column widths, hidden columns and column order. These are preferences, not +//! data, and the obvious place to keep them is browser storage — which is +//! exactly where the query history used to live, for one person, in one +//! browser, until a cache clear. +//! +//! On `state::*` instead they survive a restart, follow the operator to +//! another machine, and are legible to anything else on the bus. An agent +//! preparing a table for someone to look at can widen the column that matters +//! and hide the six that do not. +//! +//! Deliberately *not* validated against the live schema. A view saved before a +//! column was renamed should degrade to "that column has no stored width", not +//! fail the read; the renderer already treats every entry as optional. + +use super::saved::{call, SCOPE}; +use crate::error::DbError; +use iii_sdk::IIIClient; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::sync::Arc; + +/// Upper bound on stored columns per table. A view is a preference, not a +/// place to accumulate unbounded state from a caller. +const MAX_COLUMNS: usize = 1_000; +/// Clamped so a stored width cannot render a column unusable or push the grid +/// to an absurd scroll width. +const MIN_WIDTH: f64 = 48.0; +const MAX_WIDTH: f64 = 1_200.0; + +fn view_key(db: &str, table: &str) -> String { + format!("view:{db}:{table}") +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] +pub struct TableView { + /// Per-column pixel width. Absent means "size to content". + #[serde(default)] + pub widths: std::collections::BTreeMap, + /// Columns the reader has hidden. Order is not meaningful. + #[serde(default)] + pub hidden: Vec, + /// Column display order. Names not listed keep their natural position + /// after those that are, so adding a column to the table does not require + /// re-saving the view. + #[serde(default)] + pub order: Vec, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct GetTableViewReq { + #[serde(default)] + pub db: Option, + pub table: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct SaveTableViewReq { + #[serde(default)] + pub db: Option, + pub table: String, + #[serde(flatten)] + pub view: TableView, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct SaveTableViewResp { + pub saved: bool, +} + +pub async fn get(iii: &Arc, db: &str, table: &str) -> Result { + let raw = call( + iii, + "state::get", + json!({"scope": SCOPE, "key": view_key(db, table)}), + ) + .await?; + let value = raw.get("value").cloned().unwrap_or(raw); + // A missing or unparseable view is the default one. Losing a layout is a + // far smaller problem than refusing to show the table. + Ok(serde_json::from_value(value).unwrap_or_default()) +} + +pub async fn save( + iii: &Arc, + db: &str, + req: SaveTableViewReq, +) -> Result { + if req.table.trim().is_empty() { + return Err(super::query::err_to_str(DbError::InvalidParam { + index: 0, + reason: "table is required".into(), + })); + } + let view = sanitise(req.view)?; + call( + iii, + "state::set", + json!({ + "scope": SCOPE, + "key": view_key(db, &req.table), + "value": serde_json::to_value(&view).unwrap_or(Value::Null), + }), + ) + .await?; + Ok(SaveTableViewResp { saved: true }) +} + +/// Clamp widths and bound the lists. The caller is a renderer, so this is not +/// hostile input, but a stored `width: 1e9` would be a layout no one could +/// undo without editing state by hand. +fn sanitise(mut view: TableView) -> Result { + let too_many = view.widths.len() > MAX_COLUMNS + || view.hidden.len() > MAX_COLUMNS + || view.order.len() > MAX_COLUMNS; + if too_many { + return Err(super::query::err_to_str(DbError::InvalidParam { + index: 0, + reason: format!("a view may describe at most {MAX_COLUMNS} columns"), + })); + } + view.widths = view + .widths + .into_iter() + .filter(|(_, w)| w.is_finite()) + .map(|(k, w)| (k, w.clamp(MIN_WIDTH, MAX_WIDTH))) + .collect(); + view.hidden.sort(); + view.hidden.dedup(); + Ok(view) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keys_are_scoped_per_table() { + assert_eq!(view_key("primary", "orders"), "view:primary:orders"); + } + + #[test] + fn widths_are_clamped_into_a_usable_range() { + let mut v = TableView::default(); + v.widths.insert("a".into(), 5.0); + v.widths.insert("b".into(), 99_999.0); + let out = sanitise(v).unwrap(); + assert_eq!(out.widths["a"], MIN_WIDTH); + assert_eq!(out.widths["b"], MAX_WIDTH); + } + + #[test] + fn non_finite_widths_are_dropped_rather_than_stored() { + let mut v = TableView::default(); + v.widths.insert("a".into(), f64::NAN); + v.widths.insert("b".into(), f64::INFINITY); + v.widths.insert("c".into(), 200.0); + let out = sanitise(v).unwrap(); + assert_eq!(out.widths.len(), 1); + assert!(out.widths.contains_key("c")); + } + + #[test] + fn hidden_columns_are_deduplicated() { + let v = TableView { + hidden: vec!["b".into(), "a".into(), "b".into()], + ..Default::default() + }; + let out = sanitise(v).unwrap(); + assert_eq!(out.hidden, vec!["a".to_string(), "b".to_string()]); + } + + #[test] + fn an_oversized_view_is_refused() { + let mut v = TableView::default(); + for i in 0..(MAX_COLUMNS + 1) { + v.widths.insert(format!("c{i}"), 100.0); + } + assert!(sanitise(v).is_err()); + } +} diff --git a/database/src/handlers/tx_sql_guard.rs b/database/src/handlers/tx_sql_guard.rs index 4b2b4614b..6195cd2b9 100644 --- a/database/src/handlers/tx_sql_guard.rs +++ b/database/src/handlers/tx_sql_guard.rs @@ -53,6 +53,63 @@ pub(super) fn is_transaction_control_sql(sql: &str) -> bool { } } +/// Statements that only read. Anything not on this list is treated as a +/// write, so an unrecognised or vendor-specific verb fails closed. +const READ_ONLY_VERBS: &[&str] = &[ + "select", "with", "explain", "pragma", "show", "describe", "desc", "values", "table", +]; + +/// Keywords that mutate. Checked anywhere in the statement, not just at the +/// front, so a data-modifying CTE (`WITH x AS (DELETE ...) SELECT ...`) is +/// caught — it leads with `WITH` but is very much a write. +const WRITE_KEYWORDS: &[&str] = &[ + "insert", "update", "delete", "replace", "merge", "upsert", "drop", "create", "alter", + "truncate", "grant", "revoke", "attach", "detach", "reindex", "vacuum", "call", "do", +]; + +/// Whether `sql` is a single statement that cannot modify data. +/// +/// Used to gate `EXPLAIN ANALYZE`, which really executes the statement — an +/// `EXPLAIN ANALYZE DELETE FROM users` deletes users. The client-side check +/// in the console is a convenience; this one is the authority. +/// +/// Fails closed: anything it cannot confidently classify as a read is a +/// write. Reuses `strip_leading_noise`, so a leading `--` or `/* */` comment +/// cannot smuggle a verb past the check. +pub(crate) fn is_read_only_sql(sql: &str) -> bool { + let head = strip_leading_noise(sql); + if head.is_empty() { + return false; + } + + // More than one statement means the tail is unchecked; refuse rather than + // classify only the first. A trailing `;` alone is fine. + if head.trim_end().trim_end_matches(';').contains(';') { + return false; + } + + let lowered = head.to_ascii_lowercase(); + let leading = lowered + .split(|c: char| !c.is_ascii_alphabetic()) + .find(|w| !w.is_empty()) + .unwrap_or_default(); + if !READ_ONLY_VERBS.contains(&leading) { + return false; + } + + // `PRAGMA foo = 1` writes; `PRAGMA foo` reads. + if leading == "pragma" && lowered.contains('=') { + return false; + } + + // Word-boundary scan so `created_at` does not trip the `create` keyword, + // and so `EXPLAIN ANALYZE DELETE ...` is caught by its `delete`. + !lowered + .split(|c: char| !c.is_ascii_alphanumeric() && c != '_') + .filter(|w| !w.is_empty()) + .any(|w| WRITE_KEYWORDS.contains(&w)) +} + /// Strip any prefix of leading whitespace, `;`, line comments (`-- ...\n`), /// and block comments (`/* ... */`, with nesting supported) so the returned /// slice begins at the first character of the first real token. Returns an @@ -255,4 +312,57 @@ mod tests { "INSERT INTO t VALUES ('مرحبا')" )); } + + #[test] + fn read_only_accepts_plain_reads() { + for sql in [ + "SELECT 1", + " select * from users where created_at > now()", + "WITH x AS (SELECT 1) SELECT * FROM x", + "EXPLAIN SELECT * FROM t", + "PRAGMA table_info(users)", + "SHOW TABLES", + "VALUES (1), (2)", + "-- a comment\nSELECT 1", + "/* block */ SELECT 1", + "SELECT 1;", + ] { + assert!(is_read_only_sql(sql), "should be read-only: {sql}"); + } + } + + #[test] + fn read_only_rejects_writes_and_the_ways_they_hide() { + for sql in [ + // Plain writes. + "DELETE FROM users", + "UPDATE users SET a = 1", + "DROP TABLE users", + // The one that matters most: EXPLAIN ANALYZE really executes. + "EXPLAIN ANALYZE DELETE FROM users", + // A data-modifying CTE leads with WITH but is a write. + "WITH x AS (DELETE FROM users RETURNING *) SELECT * FROM x", + // A second statement would go unchecked. + "SELECT 1; DROP TABLE users", + // A comment must not smuggle the verb past the check. + "-- SELECT\nDELETE FROM users", + "/* SELECT */ DROP TABLE t", + // PRAGMA with an assignment writes. + "PRAGMA journal_mode = WAL", + // Unknown verbs fail closed. + "LOCK TABLE users", + "", + " ", + ] { + assert!(!is_read_only_sql(sql), "should be refused: {sql}"); + } + } + + #[test] + fn read_only_does_not_trip_on_identifiers_containing_keywords() { + // `created_at` contains "create"; a substring scan would reject this. + assert!(is_read_only_sql( + "SELECT created_at, updated_at FROM t ORDER BY created_at" + )); + } } diff --git a/database/src/main.rs b/database/src/main.rs index 98f404710..d0899c766 100644 --- a/database/src/main.rs +++ b/database/src/main.rs @@ -5,14 +5,22 @@ use database::configuration; use database::handle::HandleRegistry; use database::handlers::{ begin_transaction::{self, BeginTxReq}, + browse::{self, BrowseTableReq}, + column_stats::{self, ColumnStatsReq}, commit_transaction::{self, CommitTxReq}, + diagram::{self, SchemaDiagramReq}, execute::{self, ExecuteReq}, execute_batch::{self, ExecuteBatchReq}, + explain::{self, ExplainReq}, + health::{self, HealthReq, TerminateReq}, list_databases::{self, ListDatabasesReq}, prepare::{self, PrepareReq}, query::{self, QueryReq}, rollback_transaction::{self, RollbackTxReq}, run_statement::{self, RunReq}, + saved::{self, DeleteSavedReq, HistoryReq, ListSavedReq, SaveQueryReq}, + schema::{self, DescribeSchemaReq, DescribeTableReq, ListTablesReq}, + table_view::{self, GetTableViewReq, SaveTableViewReq}, test_connection::{self, TestConnectionReq}, transaction::{self, TxReq}, transaction_execute::{self, TxExecuteReq}, @@ -353,6 +361,321 @@ async fn main() -> Result<()> { ), ); } + { + let st = state.clone(); + let client = iii.clone(); + iii.register_function( + "database::getTableView", + RegisterFunction::new_async(move |req: GetTableViewReq| { + let (st, client) = (st.clone(), client.clone()); + async move { + let db = st + .resolve_db(req.db.clone()) + .await + .map_err(database::handlers::query::err_to_str)?; + table_view::get(&client, &db, &req.table) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "How a table is laid out for reading: column widths, hidden columns and \ + column order. Stored in the state worker rather than a browser, so it \ + survives a restart and any caller can set it up for someone else.", + ), + ); + } + { + let st = state.clone(); + let client = iii.clone(); + iii.register_function( + "database::saveTableView", + RegisterFunction::new_async(move |req: SaveTableViewReq| { + let (st, client) = (st.clone(), client.clone()); + async move { + let db = st + .resolve_db(req.db.clone()) + .await + .map_err(database::handlers::query::err_to_str)?; + table_view::save(&client, &db, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Replace the stored layout for a table. Widths are clamped to a usable \ + range; columns the table no longer has are kept rather than rejected, \ + so a rename degrades to a missing width instead of an error.", + ), + ); + } + { + let st = state.clone(); + let client = iii.clone(); + iii.register_function( + "database::saveQuery", + RegisterFunction::new_async(move |req: SaveQueryReq| { + let (st, client) = (st.clone(), client.clone()); + async move { + let db = st + .resolve_db(req.db.clone()) + .await + .map_err(database::handlers::query::err_to_str)?; + saved::save(&client, &db, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Save a named query against a database. Stored in the state worker, so \ + it survives restarts and an agent can save one for a human to find in \ + the console. Saving under an existing name replaces it.", + ), + ); + } + { + let st = state.clone(); + let client = iii.clone(); + iii.register_function( + "database::listSavedQueries", + RegisterFunction::new_async(move |req: ListSavedReq| { + let (st, client) = (st.clone(), client.clone()); + async move { + let db = st + .resolve_db(req.db.clone()) + .await + .map_err(database::handlers::query::err_to_str)?; + saved::list(&client, &db) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description("List the saved queries for a database, sorted by name."), + ); + } + { + let st = state.clone(); + let client = iii.clone(); + iii.register_function( + "database::deleteSavedQuery", + RegisterFunction::new_async(move |req: DeleteSavedReq| { + let (st, client) = (st.clone(), client.clone()); + async move { + let db = st + .resolve_db(req.db.clone()) + .await + .map_err(database::handlers::query::err_to_str)?; + saved::delete(&client, &db, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description("Delete a saved query by id or by name."), + ); + } + { + let st = state.clone(); + let client = iii.clone(); + iii.register_function( + "database::history", + RegisterFunction::new_async(move |req: HistoryReq| { + let (st, client) = (st.clone(), client.clone()); + async move { + let db = st + .resolve_db(req.db.clone()) + .await + .map_err(database::handlers::query::err_to_str)?; + saved::history(&client, &db, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Recent queries run against a database, newest first. Best effort — \ + recording never blocks or fails a query, so this is a convenience \ + rather than an audit log. For an audit trail bind database::row-changed.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::schemaDiagram", + RegisterFunction::new_async(move |req: SchemaDiagramReq| { + let st = st.clone(); + async move { + diagram::handle(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Lay out the schema as a diagram: positioned table nodes and routed \ + foreign-key edges, plus the hub degree of each table, the isolated \ + tables, and the remaining edge crossings. Reads the whole catalog in \ + a handful of queries rather than one per table.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::columnStats", + RegisterFunction::new_async(move |req: ColumnStatsReq| { + let st = st.clone(); + async move { + column_stats::handle(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Profile a table's columns. Reads the planner's own statistics by \ + default, which is free and approximate; `exact` runs real aggregates \ + and scans the table. To profile rows you already hold, pipe a \ + browseTable result through the fp worker instead.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::health", + RegisterFunction::new_async(move |req: HealthReq| { + let st = st.clone(); + async move { + health::handle(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Live pool occupancy plus active queries, table sizes, blocking locks \ + and cache hit ratio. Each section reports separately as available, \ + unsupported or denied, so a driver gap or a restricted role is never \ + mistaken for an empty result.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::terminateQuery", + RegisterFunction::new_async(move |req: TerminateReq| { + let st = st.clone(); + async move { + health::terminate(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Terminate a backend session, or cancel just its running statement \ + with `cancel_only`. Takes an id from database::health. Separate from \ + health because it is a write.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::explain", + RegisterFunction::new_async(move |req: ExplainReq| { + let st = st.clone(); + async move { + explain::handle(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Return a statement's query plan as a tree with per-node costs, row \ + estimates and warnings, instead of the driver's raw text. `analyze` \ + collects real timings by RUNNING the statement, so it defaults to \ + false and is refused for anything that is not a single read.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::browseTable", + RegisterFunction::new_async(move |req: BrowseTableReq| { + let st = st.clone(); + async move { + browse::handle(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Read a table page by page with typed filters and sorts, without \ + writing SQL. Filters are structured (column, op, value) and \ + compile to a parameterised WHERE for the driver in hand; the \ + total honours the same filters. Use an equality filter at \ + page_size 1 to follow a foreign key.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::listTables", + RegisterFunction::new_async(move |req: ListTablesReq| { + let st = st.clone(); + async move { + schema::list_tables(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "List every table and view in a database, with its kind and (on \ + postgres) its schema. Reads the driver's own catalog, so no \ + dialect-specific SQL is needed from the caller.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::describeTable", + RegisterFunction::new_async(move |req: DescribeTableReq| { + let st = st.clone(); + async move { + schema::describe_table(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Describe one table or view: columns with type, nullability, \ + default, primary-key membership and foreign-key target; plus \ + indexes and a planner row estimate. Foreign keys are structured \ + (schema, table, column), not a joined string.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::describeSchema", + RegisterFunction::new_async(move |req: DescribeSchemaReq| { + let st = st.clone(); + async move { + schema::describe_schema(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Describe every table at once — the same shape as describeTable, \ + but one catalog query per aspect across the whole database \ + instead of one call per table. Use this to reason about \ + relationships; set include_indexes only when you need them.", + ), + ); + } // The worker announces its own writes. Registered AFTER the functions so // the console can attribute the type, and gated on the databases that @@ -380,7 +703,7 @@ async fn main() -> Result<()> { database::ui::register(&iii); tracing::info!( - "database worker registered 14 functions and 1 trigger type, waiting for invocations" + "database worker registered 29 functions and 1 trigger type, waiting for invocations" ); wait_for_shutdown_signal().await?; tracing::info!("database worker shutting down"); diff --git a/database/src/pool/mod.rs b/database/src/pool/mod.rs index e85978a69..322603f3e 100644 --- a/database/src/pool/mod.rs +++ b/database/src/pool/mod.rs @@ -20,6 +20,23 @@ pub enum Pool { Sqlite(SqlitePool), } +/// Live pool occupancy, for `database::health`. +/// +/// `size` and `idle` are `None` where the underlying pool does not expose +/// them — `mysql_async` keeps its counters private. Reporting `None` rather +/// than zero matters: "unknown" and "no idle connections" are different +/// answers, and a health panel that conflates them is actively misleading. +#[derive(Debug, Clone, Copy, serde::Serialize, schemars::JsonSchema)] +pub struct PoolStats { + pub max: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub idle: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub waiting: Option, +} + impl Pool { pub fn driver(&self) -> DriverKind { match self { @@ -28,6 +45,14 @@ impl Pool { Pool::Sqlite(_) => DriverKind::Sqlite, } } + + pub fn stats(&self) -> PoolStats { + match self { + Pool::Postgres(p) => p.stats(), + Pool::Mysql(p) => p.stats(), + Pool::Sqlite(p) => p.stats(), + } + } } /// Build a pool for a single configured database. Used at startup by main.rs. diff --git a/database/src/pool/mysql.rs b/database/src/pool/mysql.rs index 590837f65..0b4ce9fca 100644 --- a/database/src/pool/mysql.rs +++ b/database/src/pool/mysql.rs @@ -12,11 +12,26 @@ pub struct MysqlPool { inner: Arc, db_name: Arc, acquire_timeout: Duration, + /// Configured ceiling. `mysql_async` exposes no live counters, so this is + /// the only pool figure `database::health` can report for mysql. + max_size: u32, } pub type MysqlConn = mysql_async::Conn; impl MysqlPool { + /// `mysql_async` does not expose live pool counters, so only the + /// configured ceiling is known. The rest stay `None` rather than being + /// reported as zero. + pub fn stats(&self) -> crate::pool::PoolStats { + crate::pool::PoolStats { + max: self.max_size, + size: None, + idle: None, + waiting: None, + } + } + pub fn new(url: &str, pool_cfg: &PoolConfig, tls_cfg: &TlsConfig) -> Result { let constraints = PoolConstraints::new(0, pool_cfg.max as usize).ok_or_else(|| DbError::ConfigError { @@ -45,6 +60,7 @@ impl MysqlPool { inner: Arc::new(pool), db_name: Arc::from("(unset)"), acquire_timeout: Duration::from_millis(pool_cfg.acquire_timeout_ms), + max_size: pool_cfg.max, }) } diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index 856eeb9dc..f0c519b4d 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -18,6 +18,19 @@ pub struct PostgresPool { pub type PgClient = deadpool_postgres::Object; impl PostgresPool { + /// Live occupancy from deadpool's own counters. + pub fn stats(&self) -> crate::pool::PoolStats { + let st = self.inner.status(); + crate::pool::PoolStats { + max: st.max_size as u32, + size: Some(st.size as u32), + // deadpool counts `available` as idle-or-creatable, so it can + // exceed the number of connections actually open. + idle: Some(st.available as u32), + waiting: Some(st.waiting as u32), + } + } + pub async fn new( url: &str, pool_cfg: &PoolConfig, diff --git a/database/src/pool/sqlite.rs b/database/src/pool/sqlite.rs index bb832e9a4..0b41de226 100644 --- a/database/src/pool/sqlite.rs +++ b/database/src/pool/sqlite.rs @@ -32,6 +32,17 @@ impl SqliteConn { } impl SqlitePool { + /// Live occupancy from r2d2's own counters. + pub fn stats(&self) -> crate::pool::PoolStats { + let st = self.inner.state(); + crate::pool::PoolStats { + max: self.inner.max_size(), + size: Some(st.connections), + idle: Some(st.idle_connections), + waiting: None, + } + } + pub fn new(url: &str, pool_cfg: &PoolConfig) -> Result { let path = url.strip_prefix("sqlite:").unwrap_or(url); let manager = if path == ":memory:" || path.starts_with(":memory:") { diff --git a/database/src/triggers/bus.rs b/database/src/triggers/bus.rs index 894af3a60..415f4de1d 100644 --- a/database/src/triggers/bus.rs +++ b/database/src/triggers/bus.rs @@ -191,6 +191,13 @@ pub struct RowChangeBus { } impl RowChangeBus { + /// The engine client this bus dispatches through. Handlers reuse it to + /// call sibling workers (history goes to `state::*`) without threading a + /// second client through `AppState`. + pub fn client(&self) -> &Arc { + &self.iii + } + pub fn new(iii: Arc, dispatch_timeout_ms: u64) -> Self { Self { iii, diff --git a/database/tests/integration.rs b/database/tests/integration.rs index 8b743c191..bb50040ad 100644 --- a/database/tests/integration.rs +++ b/database/tests/integration.rs @@ -4,11 +4,17 @@ use database::config::WorkerConfig; use database::configuration; use database::handle::HandleRegistry; +use database::handlers::browse::{self, BrowseTableReq}; +use database::handlers::catalog::{TableKind, TableRef}; +use database::handlers::column_stats::{self, ColumnStatsReq}; use database::handlers::execute::ExecuteReq; +use database::handlers::explain::{self, ExplainReq}; +use database::handlers::health::{self, HealthReq, TerminateReq}; use database::handlers::list_databases::{self, ListDatabasesReq}; use database::handlers::prepare::PrepareReq; use database::handlers::query::QueryReq; use database::handlers::run_statement::RunReq; +use database::handlers::schema::{self, DescribeSchemaReq, DescribeTableReq, ListTablesReq}; use database::handlers::transaction::TxReq; use database::handlers::{execute, prepare, query, run_statement, transaction, AppState}; use database::pool; @@ -248,3 +254,634 @@ async fn build_pool_creates_missing_sqlite_parent_dir() { .unwrap(); assert_eq!(r.row_count, 1); } + +/* ---------------- catalog introspection ---------------- */ + +/// Schema with the shapes that break naive catalog readers: an implicit +/// foreign key (`REFERENCES users` with no column), a composite primary key, +/// a multi-column index, and a view. +async fn seed_catalog(st: &AppState) { + for sql in [ + "CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT NOT NULL, plan TEXT DEFAULT 'free')", + // No target column: sqlite reports `to` as NULL and means "the parent's + // primary key". A reader that trusts the NULL emits a broken reference. + "CREATE TABLE orders (id INTEGER PRIMARY KEY, user_id INTEGER REFERENCES users, total REAL)", + "CREATE TABLE order_items (order_id INTEGER, sku TEXT, qty INTEGER, \ + PRIMARY KEY (order_id, sku), FOREIGN KEY (order_id) REFERENCES orders(id))", + "CREATE INDEX ix_orders_user_total ON orders (user_id, total)", + "CREATE VIEW big_orders AS SELECT * FROM orders WHERE total > 100", + ] { + execute::handle( + st, + serde_json::from_value::(json!({"db": "primary", "sql": sql})).unwrap(), + ) + .await + .unwrap(); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn list_tables_reports_tables_and_views() { + let st = build_state().await; + seed_catalog(&st).await; + + let r = schema::list_tables( + &st, + serde_json::from_value::(json!({"db": "primary"})).unwrap(), + ) + .await + .unwrap(); + + assert_eq!(r.count, r.tables.len()); + let by_name: HashMap<&str, &TableRef> = r.tables.iter().map(|t| (t.name.as_str(), t)).collect(); + assert!(by_name.contains_key("users")); + assert!(matches!(by_name["orders"].kind, TableKind::Table)); + assert!(matches!(by_name["big_orders"].kind, TableKind::View)); + // sqlite has no namespace above the table. + assert!(by_name["users"].schema.is_none()); + // Internal bookkeeping stays hidden. + assert!(!r.tables.iter().any(|t| t.name.starts_with("sqlite_"))); +} + +#[tokio::test(flavor = "multi_thread")] +async fn describe_table_resolves_columns_keys_and_indexes() { + let st = build_state().await; + seed_catalog(&st).await; + + let d = schema::describe_table( + &st, + serde_json::from_value::(json!({"db": "primary", "table": "orders"})) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(d.table, "orders"); + // Columns arrive in declaration order. + let names: Vec<&str> = d.columns.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, vec!["id", "user_id", "total"]); + + let id = &d.columns[0]; + assert!(id.primary_key); + assert_eq!(id.position, 1); + + // The implicit `REFERENCES users` resolves to the parent's primary key + // rather than surfacing an empty column. + let user_id = &d.columns[1]; + let fk = user_id + .foreign_key + .as_ref() + .expect("user_id is a foreign key"); + assert_eq!(fk.table, "users"); + assert_eq!(fk.column, "id"); + assert!(fk.schema.is_none()); + assert!(d.columns[2].foreign_key.is_none()); + + // A multi-column index keeps its columns in ordinal order. + let ix = d + .indexes + .iter() + .find(|i| i.name == "ix_orders_user_total") + .expect("index present"); + assert_eq!(ix.columns, vec!["user_id", "total"]); + assert!(!ix.unique); + assert!(!ix.primary); + + // sqlite has no cheap estimate, so it reports none rather than guessing. + assert!(d.row_count_estimate.is_none()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn describe_table_reports_composite_primary_keys() { + let st = build_state().await; + seed_catalog(&st).await; + + let d = schema::describe_table( + &st, + serde_json::from_value::( + json!({"db": "primary", "table": "order_items"}), + ) + .unwrap(), + ) + .await + .unwrap(); + + let pk: Vec<&str> = d + .columns + .iter() + .filter(|c| c.primary_key) + .map(|c| c.name.as_str()) + .collect(); + assert_eq!(pk, vec!["order_id", "sku"]); + + let fk = d.columns[0].foreign_key.as_ref().expect("explicit fk"); + assert_eq!((fk.table.as_str(), fk.column.as_str()), ("orders", "id")); +} + +#[tokio::test(flavor = "multi_thread")] +async fn describe_table_rejects_an_unknown_name() { + let st = build_state().await; + seed_catalog(&st).await; + + let err = schema::describe_table( + &st, + serde_json::from_value::(json!({"db": "primary", "table": "nope"})) + .unwrap(), + ) + .await + .unwrap_err(); + assert!(err.contains("no such table: nope"), "got: {err}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn describe_schema_covers_every_table_in_one_pass() { + let st = build_state().await; + seed_catalog(&st).await; + + let all = schema::describe_schema( + &st, + serde_json::from_value::( + json!({"db": "primary", "include_indexes": true}), + ) + .unwrap(), + ) + .await + .unwrap(); + + assert!(!all.truncated); + assert_eq!(all.count, all.tables.len()); + let names: Vec<&str> = all.tables.iter().map(|t| t.table.as_str()).collect(); + for want in ["users", "orders", "order_items", "big_orders"] { + assert!(names.contains(&want), "missing {want} in {names:?}"); + } + + // Relationships survive the batch path — this is what a diagram reads. + let orders = all.tables.iter().find(|t| t.table == "orders").unwrap(); + assert_eq!( + orders.columns[1] + .foreign_key + .as_ref() + .map(|f| f.table.as_str()), + Some("users") + ); + + // Restricting to a subset filters without changing the shape. + let some = schema::describe_schema( + &st, + serde_json::from_value::( + json!({"db": "primary", "tables": ["users", "orders"]}), + ) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(some.count, 2); +} + +#[tokio::test(flavor = "multi_thread")] +async fn describe_schema_flags_truncation_instead_of_hiding_it() { + let st = build_state().await; + seed_catalog(&st).await; + + let capped = schema::describe_schema( + &st, + serde_json::from_value::(json!({"db": "primary", "max_tables": 2})) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(capped.count, 2); + assert!(capped.truncated, "a cut-short result must say so"); +} + +/* ---------------- browseTable ---------------- */ + +async fn seed_rows(st: &AppState) { + execute::handle( + st, + serde_json::from_value::(json!({ + "db": "primary", + "sql": "CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT, note TEXT, score INT)" + })) + .unwrap(), + ) + .await + .unwrap(); + for (name, note, score) in [ + ("ana", Some("50% off"), 10), + ("bob", None, 20), + ("cyd", Some("plain"), 30), + ("dee", Some(""), 40), + ("eve", Some("other"), 50), + ] { + execute::handle( + st, + serde_json::from_value::(json!({ + "db": "primary", + "sql": "INSERT INTO people (name, note, score) VALUES (?, ?, ?)", + "params": [name, note, score] + })) + .unwrap(), + ) + .await + .unwrap(); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_pages_with_a_sentinel_and_counts_the_whole_table() { + let st = build_state().await; + seed_rows(&st).await; + + let first = browse::handle( + &st, + serde_json::from_value::( + json!({"db": "primary", "table": "people", "page_size": 2}), + ) + .unwrap(), + ) + .await + .unwrap(); + // The sentinel row is fetched but never returned. + assert_eq!(first.rows.len(), 2); + assert!(first.has_more); + assert_eq!(first.total, Some(5)); + + let last = browse::handle( + &st, + serde_json::from_value::( + json!({"db": "primary", "table": "people", "page": 2, "page_size": 2}), + ) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(last.rows.len(), 1); + assert!(!last.has_more, "the final page has nothing after it"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_total_reflects_the_filters_not_the_table() { + let st = build_state().await; + seed_rows(&st).await; + + let r = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", "page_size": 2, + "filters": [{"column": "score", "op": "gte", "value": 30}] + })) + .unwrap(), + ) + .await + .unwrap(); + // A pager showing 5 while 3 rows match is the bug this prevents. + assert_eq!(r.total, Some(3)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_treats_a_percent_in_a_filter_as_a_literal() { + let st = build_state().await; + seed_rows(&st).await; + + let r = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", + "filters": [{"column": "note", "op": "contains", "value": "50%"}] + })) + .unwrap(), + ) + .await + .unwrap(); + // Unescaped, `%` would make this match every non-null note. + assert_eq!(r.total, Some(1)); + assert_eq!(r.rows[0]["name"], "ana"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_separates_null_from_the_empty_string() { + let st = build_state().await; + seed_rows(&st).await; + + let nulls = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", + "filters": [{"column": "note", "op": "is_null"}] + })) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(nulls.total, Some(1), "only bob has a NULL note"); + + let empty = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", + "filters": [{"column": "note", "op": "is_empty"}] + })) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(empty.total, Some(2), "is_empty covers NULL and ''"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_sorts_and_can_skip_the_count() { + let st = build_state().await; + seed_rows(&st).await; + + let r = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", + "sort": [{"column": "score", "direction": "desc"}], + "include_total": false + })) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(r.rows[0]["name"], "eve"); + assert!(r.total.is_none(), "the count is skipped when not asked for"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_follows_a_foreign_key_with_an_equality_filter() { + let st = build_state().await; + seed_rows(&st).await; + + // This is why there is no separate read-a-row function. + let r = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", "page_size": 1, + "filters": [{"column": "id", "op": "equals", "value": 3}] + })) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(r.rows.len(), 1); + assert_eq!(r.rows[0]["name"], "cyd"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_refuses_an_incomplete_filter() { + let st = build_state().await; + seed_rows(&st).await; + + let err = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", + "filters": [{"column": "score", "op": "equals"}] + })) + .unwrap(), + ) + .await + .unwrap_err(); + assert!(err.contains("needs `value`"), "got: {err}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_refuses_the_wrong_operand_instead_of_panicking() { + // `value2` alone satisfied a count-based arity check and then unwrapped + // the absent `value`. Reachable from the wire, so it aborted the task. + let st = build_state().await; + seed_rows(&st).await; + + let err = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", + "filters": [{"column": "score", "op": "equals", "value2": 5}] + })) + .unwrap(), + ) + .await + .unwrap_err(); + assert!(err.contains("needs `value`"), "got: {err}"); +} + +/* ---------------- explain ---------------- */ + +#[tokio::test(flavor = "multi_thread")] +async fn explain_returns_a_plan_tree_for_a_read() { + let st = build_state().await; + seed_rows(&st).await; + + let r = explain::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "sql": "SELECT * FROM people WHERE score > 20" + })) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(r.format, explain::PlanFormat::SqliteQueryPlan); + assert!(!r.analyzed, "sqlite has no ANALYZE form"); + let root = r.root.expect("a plan tree"); + assert!(!root.children.is_empty(), "the scan hangs off the root"); + assert_eq!(root.children[0].relation.as_deref(), Some("people")); +} + +/// The gate that keeps a viewer from deleting data. `EXPLAIN ANALYZE DELETE` +/// really executes on postgres and mysql, so the worker refuses it outright +/// rather than trusting the caller to have checked. +#[tokio::test(flavor = "multi_thread")] +async fn explain_refuses_analyze_on_anything_that_writes() { + let st = build_state().await; + seed_rows(&st).await; + + for sql in [ + "DELETE FROM people", + "UPDATE people SET score = 0", + "DROP TABLE people", + // Leads with WITH, but is a write. + "WITH doomed AS (DELETE FROM people RETURNING *) SELECT * FROM doomed", + // A second statement would go unchecked. + "SELECT 1; DROP TABLE people", + // A comment must not smuggle the verb past the check. + "-- SELECT\nDELETE FROM people", + ] { + let err = explain::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "sql": sql, "analyze": true + })) + .unwrap(), + ) + .await + .unwrap_err(); + assert!( + err.contains("read-only"), + "should refuse `{sql}`, got: {err}" + ); + } + + // The rows are all still there. + let after = browse::handle( + &st, + serde_json::from_value::(json!({"db": "primary", "table": "people"})) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(after.total, Some(5)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn explain_allows_analyze_on_a_plain_read() { + let st = build_state().await; + seed_rows(&st).await; + + // Accepted by the gate; sqlite then reports it could not actually analyze. + let r = explain::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "sql": "SELECT * FROM people", "analyze": true + })) + .unwrap(), + ) + .await + .unwrap(); + assert!(!r.analyzed); +} + +/* ---------------- columnStats ---------------- */ + +#[tokio::test(flavor = "multi_thread")] +async fn column_stats_defaults_to_planner_statistics() { + let st = build_state().await; + seed_rows(&st).await; + + let r = column_stats::handle( + &st, + serde_json::from_value::(json!({"db": "primary", "table": "people"})) + .unwrap(), + ) + .await + .unwrap(); + + // The default must never scan; it reports what the planner knows, clearly + // labelled, even when that is very little. + assert!(r.approximate); + assert_eq!(r.columns.len(), 4); + assert!(r + .columns + .iter() + .all(|c| c.source == column_stats::StatSource::Planner)); + assert!( + r.columns.iter().all(|c| c.top_values.is_empty()), + "top values require a scan, so the cheap path does not invent them" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn column_stats_exact_counts_nulls_and_top_values() { + let st = build_state().await; + seed_rows(&st).await; + + let r = column_stats::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", "columns": ["note"], "exact": true + })) + .unwrap(), + ) + .await + .unwrap(); + + assert!(!r.approximate); + let note = &r.columns[0]; + assert_eq!(note.source, column_stats::StatSource::Computed); + assert_eq!(note.row_count, Some(5)); + // bob's note is NULL; dee's is '' — counted as present, not null. + assert_eq!(note.null_count, Some(1)); + assert_eq!(note.distinct_count, Some(4)); + assert_eq!(note.top_values.len(), 4); + assert!(note.top_values.iter().all(|t| t.count == 1)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn column_stats_rejects_a_column_that_does_not_exist() { + let st = build_state().await; + seed_rows(&st).await; + + let err = column_stats::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", "columns": ["nope"] + })) + .unwrap(), + ) + .await + .unwrap_err(); + assert!(err.contains("no such column `nope`"), "got: {err}"); +} + +/* ---------------- health ---------------- */ + +#[tokio::test(flavor = "multi_thread")] +async fn health_reports_pool_stats_and_says_what_sqlite_cannot_answer() { + let st = build_state().await; + seed_rows(&st).await; + + let h = health::handle( + &st, + serde_json::from_value::(json!({"db": "primary"})).unwrap(), + ) + .await + .unwrap(); + + assert_eq!(h.driver, "sqlite"); + assert_eq!(h.worker_version, env!("CARGO_PKG_VERSION")); + // The pool always answers, on every driver. + assert!(h.pool.max >= 1); + assert!(h.pool.size.is_some(), "r2d2 exposes live counters"); + + // The distinction that makes the report honest: sqlite has no sessions, + // which is a different answer from "no queries are running". + for section in [&h.active_queries] { + match section { + health::ProbeResult::Unsupported { reason } => { + assert!(reason.contains("sqlite"), "got: {reason}") + } + other => panic!("expected unsupported, got {other:?}"), + } + } + assert!(matches!(h.locks, health::ProbeResult::Unsupported { .. })); + assert!(matches!(h.cache, health::ProbeResult::Unsupported { .. })); +} + +#[tokio::test(flavor = "multi_thread")] +async fn terminate_refuses_a_non_numeric_id_and_refuses_sqlite() { + let st = build_state().await; + + let err = health::terminate( + &st, + serde_json::from_value::(json!({"db": "primary", "id": "1"})).unwrap(), + ) + .await + .unwrap_err(); + assert!(err.contains("in-process"), "got: {err}"); + + // An id is interpolated into the statement, so it must parse as a number + // before it gets anywhere near SQL. + let err = health::terminate( + &st, + serde_json::from_value::( + json!({"db": "primary", "id": "1); DROP TABLE people--"}), + ) + .unwrap(), + ) + .await + .unwrap_err(); + assert!(err.contains("not a backend id"), "got: {err}"); +} diff --git a/database/ui/build.mjs b/database/ui/build.mjs index f039d23d1..0a061659f 100644 --- a/database/ui/build.mjs +++ b/database/ui/build.mjs @@ -11,14 +11,21 @@ * poller for the hot-reload dev loop. */ +import { readFileSync } from 'node:fs' import esbuild from 'esbuild' +const watch = process.argv.includes('--watch') + const options = { entryPoints: ['page.tsx', 'styles.css'], bundle: true, format: 'esm', jsx: 'automatic', outdir: 'dist', + // The bundle is `include_str!`'d into the worker binary and the injectable-UI + // protocol rejects an asset over 8 MiB outright, so ship it minified. Left + // off under --watch, where readable stack traces matter more than bytes. + minify: !watch, external: [ 'react', 'react-dom', @@ -29,9 +36,86 @@ const options = { logLevel: 'info', } -if (process.argv.includes('--watch')) { +// Minification strips the quotes from an attribute selector, so the scope +// prefix has to be matched in both spellings. +const SCOPE = '[data-iii-ui="database"]' +const SCOPE_RE = /^\[data-iii-ui=("database"|'database'|database)\]/ +const KEYFRAME_PREFIXES = ['db-ui-', 'db-page-'] + +/** + * Fail the build if any rule could escape this worker's subtree. + * + * The console mounts every worker's UI into one document, so a single + * unscoped selector restyles the whole app. The Rust side already asserts + * that the scope string appears *somewhere* in the sheet, which a file + * containing one stray `body { }` would also pass. This checks every + * selector, and runs in CI for free because CI runs `pnpm build`. + */ +function assertScoped(css) { + // Strip comments, then strings, so a brace or selector inside either can't + // desynchronise the scan. + const src = css + .replace(/\/\*[\s\S]*?\*\//g, '') + .replace(/"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'/g, '""') + + const problems = [] + let depth = 0 + let buf = '' + // Nested at-rules (@media, @container, @supports) open a block whose + // children are still top-level selectors, so track them rather than + // skipping their contents. + const atStack = [] + + for (let i = 0; i < src.length; i++) { + const ch = src[i] + if (ch === '{') { + const head = buf.trim() + buf = '' + if (head.startsWith('@')) { + const name = head.split(/[\s(]/)[0] + atStack.push(name) + if (name === '@keyframes') { + const kf = head.slice('@keyframes'.length).trim() + if (!KEYFRAME_PREFIXES.some((p) => kf.startsWith(p))) { + problems.push(`@keyframes ${kf} is not prefixed ${KEYFRAME_PREFIXES.join(' or ')}`) + } + } else if (name === '@font-face') { + problems.push('@font-face is not allowed in an injected sheet') + } + depth++ + continue + } + // Inside @keyframes the "selectors" are 0%/from/to, not real ones. + if (!atStack.includes('@keyframes') && head) { + for (const sel of head.split(',')) { + const s = sel.trim() + if (s && !SCOPE_RE.test(s)) { + problems.push(`selector not scoped: ${s}`) + } + } + } + depth++ + } else if (ch === '}') { + depth-- + buf = '' + if (atStack.length && depth < atStack.length) atStack.pop() + } else { + buf += ch + } + } + + if (problems.length) { + console.error(`\n${problems.length} unscoped rule(s) in dist/styles.css:\n`) + for (const p of [...new Set(problems)]) console.error(` ${p}`) + console.error(`\nEvery rule must start with ${SCOPE}.\n`) + process.exit(1) + } +} + +if (watch) { const ctx = await esbuild.context(options) await ctx.watch() } else { await esbuild.build(options) + assertScoped(readFileSync('dist/styles.css', 'utf8')) } diff --git a/database/ui/package.json b/database/ui/package.json index 3a3cf59a1..49bb7b4e6 100644 --- a/database/ui/package.json +++ b/database/ui/package.json @@ -5,7 +5,8 @@ "type": "module", "scripts": { "build": "tsc --noEmit && node build.mjs", - "watch": "node build.mjs --watch" + "watch": "node build.mjs --watch", + "test": "vitest run" }, "dependencies": { "@iii-dev/console-ui": "workspace:*", @@ -14,6 +15,7 @@ "devDependencies": { "@types/react": "^19.2.14", "esbuild": "^0.25.0", - "typescript": "^5.9.2" + "typescript": "^5.9.2", + "vitest": "^4.1.6" } } diff --git a/database/ui/src/lib/capabilities.ts b/database/ui/src/lib/capabilities.ts new file mode 100644 index 000000000..18214c507 --- /dev/null +++ b/database/ui/src/lib/capabilities.ts @@ -0,0 +1,82 @@ +/** + * Which of the worker's functions this engine actually has. + * + * The page no longer carries a client-side catalog fallback: there is one + * implementation of every read, and it lives in the worker. That removes a + * whole class of drift, and introduces a hard minimum worker version. An + * older install must degrade visibly rather than throw "function not found" + * from four components at once — so a panel whose function is missing is + * *hidden*, never rendered as a control that always errors. + * + * Discovery uses the engine's own `engine::functions::list`. Inventing a + * `database::capabilities` for this would have been one more function to keep + * in sync with the real registry. + */ + +import type { Host } from '@iii-dev/console-ui' +import { OPTIONAL_FNS, type DbFunction } from './rpc' + +export type Capabilities = ReadonlySet + +/** Every optional function, for the common case of an up-to-date worker. */ +export const ALL: Capabilities = new Set(OPTIONAL_FNS) + +/** + * Ask the engine which `database::*` functions are registered. + * + * Falls back to assuming everything is present when the listing itself is + * unavailable: a probe that cannot run is not evidence of a missing feature, + * and hiding the whole page because one introspection call failed would be + * the worse error. Individual calls still surface their own failures. + */ +export async function probe(host: Host): Promise { + try { + const raw = await host.iii.trigger('engine::functions::list', {}) + const ids = collectIds(raw) + if (ids.size === 0) return ALL + return new Set(OPTIONAL_FNS.filter((fn) => ids.has(fn))) + } catch { + return ALL + } +} + +/** + * Pull function ids out of whatever shape the listing returns — an array of + * strings, an array of objects with `id` / `function_id` / `name`, or an + * object keyed by id. Being permissive here is deliberate: the exact shape is + * an engine detail, and guessing wrong should not disable the page. + */ +function collectIds(raw: unknown): Set { + const out = new Set() + + const add = (v: unknown) => { + if (typeof v === 'string' && v.startsWith('database::')) out.add(v) + } + + const walk = (node: unknown, depth: number) => { + if (depth > 4 || node == null) return + if (typeof node === 'string') return add(node) + if (Array.isArray(node)) { + for (const item of node) walk(item, depth + 1) + return + } + if (typeof node === 'object') { + const rec = node as Record + for (const key of ['id', 'function_id', 'functionId', 'name']) { + add(rec[key]) + } + // An object keyed by function id. + for (const key of Object.keys(rec)) add(key) + for (const key of ['functions', 'data', 'items', 'result']) { + if (key in rec) walk(rec[key], depth + 1) + } + } + } + + walk(raw, 0) + return out +} + +/** Version hint for the empty state a hidden panel leaves behind. */ +export const MIN_VERSION_HINT = + 'this needs a newer database worker — run `iii worker update database`' diff --git a/database/ui/src/lib/grid-cursor.test.ts b/database/ui/src/lib/grid-cursor.test.ts new file mode 100644 index 000000000..39a07b9b4 --- /dev/null +++ b/database/ui/src/lib/grid-cursor.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from 'vitest' +import { type Bounds, moveCursor, reanchor, rowAsTsv, rowKey } from './grid-cursor' + +const bounds = (over: Partial = {}): Bounds => ({ + rowCount: 10, + colCount: 4, + pageRows: 5, + hasPrevPage: false, + hasNextPage: false, + ...over, +}) + +describe('moveCursor', () => { + it('clamps at the edges rather than wrapping', () => { + expect(moveCursor({ row: 0, col: 0 }, { type: 'up' }, bounds()).cursor).toEqual({ row: 0, col: 0 }) + expect(moveCursor({ row: 0, col: 0 }, { type: 'left' }, bounds()).cursor).toEqual({ row: 0, col: 0 }) + expect(moveCursor({ row: 9, col: 3 }, { type: 'down' }, bounds()).cursor).toEqual({ row: 9, col: 3 }) + expect(moveCursor({ row: 9, col: 3 }, { type: 'right' }, bounds()).cursor).toEqual({ row: 9, col: 3 }) + }) + + it('crosses to the next page from the last row', () => { + const m = moveCursor({ row: 9, col: 2 }, { type: 'down' }, bounds({ hasNextPage: true })) + expect(m.pageDelta).toBe(1) + expect(m.landing).toBe('first') + // The column is preserved across the page turn — moving down a column + // should not also move sideways. + expect(m.cursor.col).toBe(2) + }) + + it('crosses to the previous page from the first row', () => { + const m = moveCursor({ row: 0, col: 1 }, { type: 'up' }, bounds({ hasPrevPage: true })) + expect(m.pageDelta).toBe(-1) + expect(m.landing).toBe('last') + }) + + it('does not cross a page that does not exist', () => { + expect(moveCursor({ row: 9, col: 0 }, { type: 'down' }, bounds()).pageDelta).toBeUndefined() + expect(moveCursor({ row: 0, col: 0 }, { type: 'up' }, bounds()).pageDelta).toBeUndefined() + }) + + it('pages within the grid before crossing a boundary', () => { + // From the middle, PageDown jumps inside the page even when a next page + // exists — the key means "move far", not "turn the page". + const m = moveCursor({ row: 2, col: 0 }, { type: 'pageDown' }, bounds({ hasNextPage: true })) + expect(m.pageDelta).toBeUndefined() + expect(m.cursor.row).toBe(7) + }) + + it('turns the page when PageDown is pressed at the bottom', () => { + const m = moveCursor({ row: 9, col: 0 }, { type: 'pageDown' }, bounds({ hasNextPage: true })) + expect(m.pageDelta).toBe(1) + }) + + it('moves to row and grid extremes', () => { + expect(moveCursor({ row: 3, col: 2 }, { type: 'rowStart' }, bounds()).cursor).toEqual({ row: 3, col: 0 }) + expect(moveCursor({ row: 3, col: 2 }, { type: 'rowEnd' }, bounds()).cursor).toEqual({ row: 3, col: 3 }) + expect(moveCursor({ row: 3, col: 2 }, { type: 'gridStart' }, bounds()).cursor).toEqual({ row: 0, col: 0 }) + expect(moveCursor({ row: 3, col: 2 }, { type: 'gridEnd' }, bounds()).cursor).toEqual({ row: 9, col: 3 }) + }) + + it('clamps an out-of-range cursor before moving it', () => { + // The page shrank underneath the cursor. + const m = moveCursor({ row: 99, col: 99 }, { type: 'up' }, bounds()) + expect(m.cursor).toEqual({ row: 8, col: 3 }) + }) + + it('does nothing on an empty grid', () => { + const m = moveCursor({ row: 0, col: 0 }, { type: 'down' }, bounds({ rowCount: 0 })) + expect(m).toEqual({ cursor: { row: 0, col: 0 } }) + }) +}) + +describe('reanchor', () => { + const columns = ['id', 'email', 'plan'] + + it('follows a row to its new index after a sort', () => { + const keys = ['id=3', 'id=1', 'id=2'] + expect(reanchor({ row: 0, col: 1 }, 'id=2', keys, columns, 'email')).toEqual({ row: 2, col: 1 }) + }) + + it('follows a column to its new index', () => { + const keys = ['id=1'] + expect(reanchor({ row: 0, col: 0 }, 'id=1', keys, ['plan', 'id', 'email'], 'email')).toEqual({ + row: 0, + col: 2, + }) + }) + + it('clamps when the row is gone', () => { + // Filtered out. Keep the column, put the cursor somewhere valid. + expect(reanchor({ row: 5, col: 2 }, 'id=99', ['id=1', 'id=2'], columns, 'plan')).toEqual({ + row: 1, + col: 2, + }) + }) +}) + +describe('rowKey', () => { + it('uses the primary key when there is one', () => { + expect(rowKey({ id: 7, email: 'a@b.c' }, ['id'])).toBe('id=7') + }) + + it('falls back to the whole row, which survives a re-sort', () => { + expect(rowKey({ id: 7, email: 'a@b.c' }, [])).toBe('id=7\u0001email=a@b.c') + }) + + it('distinguishes null from the empty string', () => { + expect(rowKey({ a: null }, [])).not.toBe(rowKey({ a: '' }, [])) + }) + + it('does not let adjacent fields run together', () => { + // With an empty separator these two rows produce the same key, and the + // cursor would follow the wrong one after a sort. + expect(rowKey({ a: '1', b: '2' }, ['a', 'b'])).not.toBe(rowKey({ a: '12', b: '' }, ['a', 'b'])) + }) +}) + +describe('rowAsTsv', () => { + it('joins with tabs so it pastes as spreadsheet cells', () => { + expect(rowAsTsv({ a: 1, b: 'x' }, ['a', 'b'])).toBe('1\tx') + }) + + it('renders null as empty rather than the text NULL', () => { + expect(rowAsTsv({ a: null, b: 2 }, ['a', 'b'])).toBe('\t2') + }) + + it('serialises objects rather than pasting [object Object]', () => { + expect(rowAsTsv({ a: { x: 1 } }, ['a'])).toBe('{"x":1}') + }) +}) diff --git a/database/ui/src/lib/grid-cursor.ts b/database/ui/src/lib/grid-cursor.ts new file mode 100644 index 000000000..fed825b60 --- /dev/null +++ b/database/ui/src/lib/grid-cursor.ts @@ -0,0 +1,173 @@ +/** + * Grid cursor movement, as a pure function. + * + * Kept out of React because it is the one piece of this page with enough edge + * cases to be worth testing directly: clamping, page-edge crossing, and + * surviving a re-sort. It has no imports and never touches the DOM. + * + * The grid is *server*-paged, which is what makes this more than a clamp. + * Pressing Down on the last visible row of page 2 should land on the first row + * of page 3, not sit still — a grid that stops dead at the page edge feels + * broken in a way a desktop client never does. Movement therefore returns an + * optional `pageDelta` alongside the new cursor, and the caller decides + * whether it can honour it (it cannot, at the first or last page). + */ + +export interface Cursor { + row: number + col: number +} + +export interface Bounds { + rowCount: number + colCount: number + /** Rows to jump for PageUp/PageDown. */ + pageRows: number + /** False on the first page — Up at row 0 then has nowhere to go. */ + hasPrevPage: boolean + hasNextPage: boolean +} + +export type CursorAction = + | { type: 'up' } + | { type: 'down' } + | { type: 'left' } + | { type: 'right' } + | { type: 'rowStart' } + | { type: 'rowEnd' } + | { type: 'gridStart' } + | { type: 'gridEnd' } + | { type: 'pageUp' } + | { type: 'pageDown' } + | { type: 'to'; row: number; col: number } + +export interface CursorMove { + cursor: Cursor + /** + * -1 / +1 when the move ran off the top or bottom of the page. The caller + * turns the page and places the cursor at `landing`. + */ + pageDelta?: -1 | 1 + /** Which edge row to land on after the caller turns the page. */ + landing?: 'first' | 'last' +} + +const clamp = (n: number, max: number) => Math.max(0, Math.min(max, n)) + +export function moveCursor(cursor: Cursor, action: CursorAction, bounds: Bounds): CursorMove { + const { rowCount, colCount, pageRows, hasPrevPage, hasNextPage } = bounds + if (rowCount <= 0 || colCount <= 0) return { cursor } + + const lastRow = rowCount - 1 + const lastCol = colCount - 1 + const at = { row: clamp(cursor.row, lastRow), col: clamp(cursor.col, lastCol) } + + switch (action.type) { + case 'up': + if (at.row === 0 && hasPrevPage) { + return { cursor: at, pageDelta: -1, landing: 'last' } + } + return { cursor: { ...at, row: clamp(at.row - 1, lastRow) } } + + case 'down': + if (at.row === lastRow && hasNextPage) { + return { cursor: at, pageDelta: 1, landing: 'first' } + } + return { cursor: { ...at, row: clamp(at.row + 1, lastRow) } } + + case 'left': + return { cursor: { ...at, col: clamp(at.col - 1, lastCol) } } + + case 'right': + return { cursor: { ...at, col: clamp(at.col + 1, lastCol) } } + + case 'rowStart': + return { cursor: { ...at, col: 0 } } + + case 'rowEnd': + return { cursor: { ...at, col: lastCol } } + + case 'gridStart': + return { cursor: { row: 0, col: 0 } } + + case 'gridEnd': + return { cursor: { row: lastRow, col: lastCol } } + + case 'pageUp': + // Only cross a page boundary from the very top. Otherwise jump within + // the page, which is what the key means when there is room to move. + if (at.row === 0 && hasPrevPage) { + return { cursor: at, pageDelta: -1, landing: 'last' } + } + return { cursor: { ...at, row: clamp(at.row - pageRows, lastRow) } } + + case 'pageDown': + if (at.row === lastRow && hasNextPage) { + return { cursor: at, pageDelta: 1, landing: 'first' } + } + return { cursor: { ...at, row: clamp(at.row + pageRows, lastRow) } } + + case 'to': + return { cursor: { row: clamp(action.row, lastRow), col: clamp(action.col, lastCol) } } + } +} + +/** + * Re-find the cursor after the rows underneath it changed. + * + * Anchoring on the row *key* and the column *name* rather than their indices + * is the whole point: after a sort, index 4 is a different row, and a cursor + * that stayed at index 4 would silently point somewhere else. When the row is + * gone — filtered out, or on another page now — the column is kept and the row + * clamps, which is the least surprising thing available. + */ +export function reanchor( + cursor: Cursor, + previousKey: string | null, + keys: string[], + columns: string[], + previousColumn: string | null, +): Cursor { + const row = previousKey === null ? cursor.row : keys.indexOf(previousKey) + const col = previousColumn === null ? cursor.col : columns.indexOf(previousColumn) + return { + row: row >= 0 ? row : clamp(cursor.row, Math.max(0, keys.length - 1)), + col: col >= 0 ? col : clamp(cursor.col, Math.max(0, columns.length - 1)), + } +} + +/** + * A stable identity for a row. + * + * Primary key when there is one. Otherwise the row's own values, which is not + * guaranteed unique but is stable across a re-sort — and a duplicate is a far + * smaller error than an index that silently means a different row. + */ +export function rowKey(row: Record, primaryKeys: string[]): string { + const cols = primaryKeys.length > 0 ? primaryKeys : Object.keys(row) + return cols.map((c) => `${c}=${String(row[c])}`).join(KEY_SEP) +} + +/** + * Field separator for composite keys. A control character rather than an empty + * string or a comma: joining with nothing makes `{a:'1',b:'2'}` collide with + * `{a:'12',b:''}`, and any printable separator can occur inside a value. + */ +const KEY_SEP = '\u0001' + +/** A row as tab-separated text, so it pastes into a spreadsheet as cells. */ +export function rowAsTsv(row: Record, columns: string[]): string { + return columns.map((c) => cellText(row[c])).join('\t') +} + +export function cellText(value: unknown): string { + if (value === null || value === undefined) return '' + if (typeof value === 'object') { + try { + return JSON.stringify(value) + } catch { + return String(value) + } + } + return String(value) +} diff --git a/database/ui/src/lib/rpc.ts b/database/ui/src/lib/rpc.ts new file mode 100644 index 000000000..873249695 --- /dev/null +++ b/database/ui/src/lib/rpc.ts @@ -0,0 +1,682 @@ +/** + * Typed calls into the worker's own functions. + * + * The page used to build catalog SQL in the browser — around 520 lines of + * `sqlite_master` / `information_schema` / `PRAGMA` per driver. That logic now + * lives in the worker, where it is written once, tested against all three + * drivers in CI, and callable by any agent on the bus. This module is the + * whole of what replaced it: one wrapper per function, no SQL. + * + * Everything is parsed with zod at the boundary, because a worker one version + * behind will happily return a shape this page does not expect, and a missing + * field should surface as a readable error rather than `undefined` three + * components deep. + */ + +import type { Host } from '@iii-dev/console-ui' +import { z } from 'zod' + +export const DB = { + listDatabases: 'database::listDatabases', + listTables: 'database::listTables', + describeTable: 'database::describeTable', + describeSchema: 'database::describeSchema', + browseTable: 'database::browseTable', + query: 'database::query', + explain: 'database::explain', + columnStats: 'database::columnStats', + health: 'database::health', + schemaDiagram: 'database::schemaDiagram', + saveQuery: 'database::saveQuery', + listSavedQueries: 'database::listSavedQueries', + deleteSavedQuery: 'database::deleteSavedQuery', + history: 'database::history', + getTableView: 'database::getTableView', + saveTableView: 'database::saveTableView', +} as const + +export type DbFunction = (typeof DB)[keyof typeof DB] + +/** Functions older installed workers will not have. */ +export const OPTIONAL_FNS: readonly DbFunction[] = [ + DB.listTables, + DB.describeTable, + DB.describeSchema, + DB.browseTable, + DB.explain, + DB.columnStats, + DB.health, + DB.schemaDiagram, + DB.saveQuery, + DB.listSavedQueries, + DB.deleteSavedQuery, + DB.history, + DB.getTableView, + DB.saveTableView, +] + +async function call(host: Host, fn: DbFunction, payload: Record, schema: z.ZodType): Promise { + const raw = await host.iii.trigger(fn, payload) + const parsed = schema.safeParse(raw) + if (!parsed.success) { + throw new Error(`unexpected response shape from ${fn}`) + } + return parsed.data +} + +/* ---------------- shared shapes ---------------- */ + +export const columnMetaSchema = z.object({ + name: z.string(), + type: z.string().optional(), +}) +export type ColumnMeta = z.infer + +export const queryResponseSchema = z.object({ + rows: z.array(z.record(z.string(), z.unknown())), + row_count: z.number(), + columns: z.array(columnMetaSchema), +}) +export type QueryResponse = z.infer + +export const foreignKeySchema = z.object({ + schema: z.string().nullish(), + table: z.string(), + column: z.string(), +}) +export type ForeignKeyRef = z.infer + +export const columnDescSchema = z.object({ + name: z.string(), + type: z.string(), + nullable: z.boolean(), + default_value: z.string().nullish(), + primary_key: z.boolean(), + position: z.number(), + foreign_key: foreignKeySchema.nullish(), +}) +export type ColumnDesc = z.infer + +export const indexDescSchema = z.object({ + name: z.string(), + unique: z.boolean(), + primary: z.boolean(), + columns: z.array(z.string()), +}) +export type IndexDesc = z.infer + +export const tableRefSchema = z.object({ + name: z.string(), + schema: z.string().nullish(), + kind: z.enum(['table', 'view']), +}) +export type TableRef = z.infer + +export const tableDescriptionSchema = z.object({ + table: z.string(), + schema: z.string().nullish(), + kind: z.enum(['table', 'view']), + columns: z.array(columnDescSchema), + indexes: z.array(indexDescSchema), + row_count_estimate: z.number().nullish(), +}) +export type TableDescription = z.infer + +/* ---------------- filters and sorts ---------------- */ + +export type FilterOp = + | 'contains' + | 'not_contains' + | 'equals' + | 'not_equals' + | 'starts_with' + | 'ends_with' + | 'gt' + | 'gte' + | 'lt' + | 'lte' + | 'between' + | 'is_true' + | 'is_false' + | 'is_null' + | 'is_not_null' + | 'is_empty' + | 'in' + | 'not_in' + +export interface FilterSpec { + column: string + op: FilterOp + value?: unknown + value2?: unknown + /** Operands for `in` / `not_in`. */ + values?: unknown[] + case_sensitive?: boolean + /** Kept in the bar but not applied. */ + disabled?: boolean +} + +/** Human labels, grouped the way a picker should present them. */ +export const OP_LABEL: Record = { + equals: 'is', + not_equals: 'is not', + contains: 'contains', + not_contains: 'does not contain', + starts_with: 'starts with', + ends_with: 'ends with', + gt: 'greater than', + gte: 'at least', + lt: 'less than', + lte: 'at most', + between: 'between', + in: 'is one of', + not_in: 'is none of', + is_true: 'is true', + is_false: 'is false', + is_null: 'is null', + is_not_null: 'is not null', + is_empty: 'is empty', +} + +export type SortMode = 'default' | 'natural' | 'length' | 'absolute_value' | 'random' + +export interface SortSpec { + column: string + direction: 'asc' | 'desc' + nulls?: 'first' | 'last' + mode?: SortMode +} + +/** + * Operators that make sense for a column, by its declared type. Offering + * `contains` on a boolean is how a filter bar starts feeling untrustworthy. + */ +export function opsFor(category: TypeCategory): FilterOp[] { + switch (category) { + case 'numeric': + return ['equals', 'not_equals', 'gt', 'gte', 'lt', 'lte', 'between', 'in', 'not_in', 'is_null', 'is_not_null'] + case 'bool': + return ['is_true', 'is_false', 'is_null', 'is_not_null'] + case 'date': + return ['equals', 'gt', 'lt', 'between', 'is_null', 'is_not_null'] + default: + return [ + 'contains', + 'not_contains', + 'equals', + 'not_equals', + 'starts_with', + 'ends_with', + 'in', + 'not_in', + 'is_empty', + 'is_null', + 'is_not_null', + ] + } +} + +/** Operators taking a list rather than a single operand. */ +export const SET_OPS: ReadonlySet = new Set(['in', 'not_in']) + +/** Operators taking no operand — a chip using one is complete immediately. */ +const NULLARY: ReadonlySet = new Set(['is_true', 'is_false', 'is_null', 'is_not_null', 'is_empty']) + +/** + * Whether a chip is ready to send. An incomplete chip stays in the bar as a + * draft and is never sent — the worker would reject it, and a round trip to + * learn that would make typing feel broken. + */ +export function isComplete(f: FilterSpec): boolean { + if (!f.column) return false + if (NULLARY.has(f.op)) return true + if (f.op === 'between') return hasValue(f.value) && hasValue(f.value2) + // `IN ()` is a syntax error everywhere, so an empty list is a draft. + if (SET_OPS.has(f.op)) return (f.values ?? []).some(hasValue) + return hasValue(f.value) +} + +function hasValue(v: unknown): boolean { + return v !== undefined && v !== null && v !== '' +} + +export type TypeCategory = 'numeric' | 'text' | 'bool' | 'date' | 'json' | 'binary' | 'other' + +/** Coarse category for a driver-reported column type. */ +export function typeCategory(type: string | undefined): TypeCategory { + if (!type) return 'other' + const t = type.toLowerCase() + if (/bool/.test(t)) return 'bool' + if (/int|real|float|double|decimal|numeric|serial|money/.test(t)) return 'numeric' + if (/date|time|year/.test(t)) return 'date' + if (/json/.test(t)) return 'json' + if (/blob|bytea|binary/.test(t)) return 'binary' + if (/char|text|clob|uuid|enum/.test(t)) return 'text' + return 'other' +} + +/* ---------------- calls ---------------- */ + +export const PAGE_SIZE = 50 +export type DbDriver = 'sqlite' | 'postgres' | 'mysql' | 'unknown' + +const listDatabasesSchema = z.object({ + databases: z.array( + z.object({ + name: z.string(), + driver: z.string(), + url: z.string().optional(), + pool: z.object({ max: z.number().optional() }).optional(), + }), + ), + count: z.number(), +}) + +export interface DbInfo { + name: string + driver: DbDriver + url?: string + poolMax?: number +} + +function parseDriver(raw: string | undefined): DbDriver { + return raw === 'sqlite' || raw === 'postgres' || raw === 'mysql' ? raw : 'unknown' +} + +export async function listDbs(host: Host): Promise { + const res = await call(host, DB.listDatabases, {}, listDatabasesSchema) + return res.databases.map((d) => ({ + name: d.name, + driver: parseDriver(d.driver), + url: d.url, + poolMax: d.pool?.max, + })) +} + +export async function listTables(host: Host, db: string): Promise { + const res = await call(host, DB.listTables, { db }, z.object({ tables: z.array(tableRefSchema), count: z.number() })) + return res.tables +} + +export async function describeTable( + host: Host, + db: string, + table: string, + schema?: string | null, +): Promise { + return call(host, DB.describeTable, { db, table, schema }, tableDescriptionSchema) +} + +export async function describeSchema(host: Host, db: string, includeIndexes = false): Promise { + const res = await call( + host, + DB.describeSchema, + { db, include_indexes: includeIndexes }, + z.object({ + tables: z.array(tableDescriptionSchema), + count: z.number(), + truncated: z.boolean(), + }), + ) + return res.tables +} + +const browseSchema = z.object({ + rows: z.array(z.record(z.string(), z.unknown())), + columns: z.array(columnMetaSchema), + page: z.number(), + page_size: z.number(), + has_more: z.boolean(), + total: z.number().nullish(), +}) +export type BrowseResult = z.infer + +export interface BrowseOptions { + page?: number + pageSize?: number + sort?: SortSpec[] + filters?: FilterSpec[] + /** Skip the filtered COUNT(*) while the user is still editing a chip. */ + includeTotal?: boolean +} + +export async function browseTable( + host: Host, + db: string, + table: string, + schema: string | null | undefined, + opts: BrowseOptions = {}, +): Promise { + return call( + host, + DB.browseTable, + { + db, + table, + schema, + page: Math.max(0, Math.trunc(opts.page ?? 0)), + page_size: Math.max(1, Math.trunc(opts.pageSize ?? PAGE_SIZE)), + sort: opts.sort ?? [], + // Draft chips never leave the browser. + filters: (opts.filters ?? []).filter(isComplete), + include_total: opts.includeTotal ?? true, + }, + browseSchema, + ) +} + +/** Ad-hoc SQL from the editor. Still goes through `database::query`. */ +export async function runSql(host: Host, db: string, sql: string): Promise { + return call(host, DB.query, { db, sql }, queryResponseSchema) +} + +export const planNodeSchema: z.ZodType = z.lazy(() => + z.object({ + id: z.number(), + parent: z.number().nullish(), + label: z.string(), + node_class: z.string(), + relation: z.string().nullish(), + cost_startup: z.number().nullish(), + cost_total: z.number().nullish(), + rows_estimated: z.number().nullish(), + rows_actual: z.number().nullish(), + width: z.number().nullish(), + time_ms: z.number().nullish(), + loops: z.number().nullish(), + detail: z.string(), + children: z.array(planNodeSchema), + }), +) + +export interface PlanNode { + id: number + parent?: number | null + label: string + node_class: string + relation?: string | null + cost_startup?: number | null + cost_total?: number | null + rows_estimated?: number | null + rows_actual?: number | null + width?: number | null + time_ms?: number | null + loops?: number | null + detail: string + children: PlanNode[] +} + +const explainSchema = z.object({ + format: z.string(), + analyzed: z.boolean(), + root: planNodeSchema.nullish(), + warnings: z.array( + z.object({ + node_id: z.number(), + kind: z.string(), + message: z.string(), + severity: z.string(), + }), + ), + raw: z.unknown().nullish(), +}) +export type ExplainResult = z.infer + +export async function explain(host: Host, db: string, sql: string, analyze = false): Promise { + return call(host, DB.explain, { db, sql, analyze }, explainSchema) +} + +const columnStatSchema = z.object({ + name: z.string(), + row_count: z.number().nullish(), + distinct_count: z.number().nullish(), + null_count: z.number().nullish(), + null_fraction: z.number().nullish(), + min: z.unknown().nullish(), + max: z.unknown().nullish(), + mean: z.number().nullish(), + top_values: z.array(z.object({ value: z.unknown(), count: z.number() })), + source: z.enum(['planner', 'computed']), +}) +export type ColumnStat = z.infer + +export async function columnStats( + host: Host, + db: string, + table: string, + columns: string[], + exact = false, +): Promise<{ columns: ColumnStat[]; approximate: boolean }> { + return call( + host, + DB.columnStats, + { db, table, columns, exact }, + z.object({ + table: z.string(), + schema: z.string().nullish(), + columns: z.array(columnStatSchema), + approximate: z.boolean(), + }), + ) +} + +const probeSchema = (data: T) => + z.discriminatedUnion('status', [ + z.object({ status: z.literal('available'), data }), + z.object({ status: z.literal('unsupported'), reason: z.string() }), + z.object({ status: z.literal('denied'), reason: z.string() }), + ]) + +const healthSchema = z.object({ + db: z.string(), + driver: z.string(), + worker_version: z.string(), + pool: z.object({ + max: z.number(), + size: z.number().nullish(), + idle: z.number().nullish(), + waiting: z.number().nullish(), + }), + active_queries: probeSchema( + z.array( + z.object({ + id: z.string(), + sql: z.string(), + state: z.string().nullish(), + duration_ms: z.number().nullish(), + user: z.string().nullish(), + }), + ), + ), + table_sizes: probeSchema( + z.array( + z.object({ + table: z.string(), + schema: z.string().nullish(), + total_bytes: z.number().nullish(), + index_bytes: z.number().nullish(), + row_estimate: z.number().nullish(), + }), + ), + ), + locks: probeSchema( + z.array( + z.object({ + blocked_id: z.string(), + blocked_sql: z.string(), + blocking_id: z.string(), + blocking_sql: z.string().nullish(), + relation: z.string().nullish(), + }), + ), + ), + cache: probeSchema( + z.object({ + hit_ratio: z.number(), + blocks_hit: z.number(), + blocks_read: z.number(), + }), + ), +}) +export type HealthReport = z.infer + +export async function health(host: Host, db: string): Promise { + return call(host, DB.health, { db }, healthSchema) +} + +const diagramSchema = z.object({ + nodes: z.array( + z.object({ + table: z.string(), + schema: z.string().nullish(), + x: z.number(), + y: z.number(), + w: z.number(), + h: z.number(), + rank: z.number(), + degree: z.number(), + hidden_columns: z.number(), + columns: z.array( + z.object({ + name: z.string(), + type: z.string(), + primary_key: z.boolean(), + foreign_key: z.boolean(), + nullable: z.boolean(), + }), + ), + }), + ), + edges: z.array( + z.object({ + from: z.string(), + from_column: z.string(), + to: z.string(), + to_column: z.string(), + points: z.array(z.object({ x: z.number(), y: z.number() })), + self_loop: z.boolean(), + }), + ), + width: z.number(), + height: z.number(), + isolated: z.array(z.string()), + // Optional so an older worker still renders — it just draws no boundaries. + components: z + .array( + z.object({ + index: z.number(), + tables: z.array(z.string()), + hub: z.string().nullish(), + x: z.number(), + y: z.number(), + w: z.number(), + h: z.number(), + }), + ) + .nullish() + .transform((v) => v ?? []), + crossings: z.number(), + truncated: z.boolean(), + focus: z.string().nullish(), + frontier: z + .array(z.string()) + .nullish() + .transform((v) => v ?? []), +}) +export type SchemaDiagram = z.infer + +export interface DiagramOptions { + /** Draw only this table's neighbourhood. */ + focus?: string | null + /** Foreign-key hops out from `focus`. */ + depth?: number +} + +export async function schemaDiagram(host: Host, db: string, opts: DiagramOptions = {}): Promise { + return call( + host, + DB.schemaDiagram, + // `focus` is omitted rather than sent as null: the worker treats absence + // as "the whole schema", and an explicit null would have to mean the same + // thing in two places. + opts.focus ? { db, focus: opts.focus, depth: opts.depth ?? 1 } : { db }, + diagramSchema, + ) +} + +const savedQuerySchema = z.object({ + id: z.string(), + name: z.string(), + sql: z.string(), + description: z.string().nullish(), + saved_at: z.string(), +}) +export type SavedQuery = z.infer + +export async function listSavedQueries(host: Host, db: string): Promise { + const res = await call( + host, + DB.listSavedQueries, + { db }, + z.object({ queries: z.array(savedQuerySchema), count: z.number() }), + ) + return res.queries +} + +export async function saveQuery( + host: Host, + db: string, + name: string, + sql: string, +): Promise<{ id: string; replaced: boolean }> { + return call(host, DB.saveQuery, { db, name, sql }, z.object({ id: z.string(), replaced: z.boolean() })) +} + +export async function deleteSavedQuery(host: Host, db: string, id: string): Promise { + const res = await call(host, DB.deleteSavedQuery, { db, id }, z.object({ deleted: z.boolean() })) + return res.deleted +} + +const historySchema = z.object({ + entries: z.array( + z.object({ + sql: z.string(), + verb: z.string(), + duration_ms: z.number().nullish(), + row_count: z.number().nullish(), + at: z.string(), + }), + ), + count: z.number(), +}) +export type HistoryEntry = z.infer['entries'][number] + +const tableViewSchema = z.object({ + widths: z + .record(z.string(), z.number()) + .nullish() + .transform((v) => v ?? {}), + hidden: z + .array(z.string()) + .nullish() + .transform((v) => v ?? []), + order: z + .array(z.string()) + .nullish() + .transform((v) => v ?? []), +}) +export type TableView = z.infer + +export async function getTableView(host: Host, db: string, table: string): Promise { + return call(host, DB.getTableView, { db, table }, tableViewSchema) +} + +export async function saveTableView(host: Host, db: string, table: string, view: TableView): Promise { + const r = await call(host, DB.saveTableView, { db, table, ...view }, z.object({ saved: z.boolean() })) + return r.saved +} + +export async function history(host: Host, db: string, limit = 25): Promise { + const res = await call(host, DB.history, { db, limit }, historySchema) + return res.entries +} diff --git a/database/ui/src/page/CellInspector.tsx b/database/ui/src/page/CellInspector.tsx new file mode 100644 index 000000000..189a92ee0 --- /dev/null +++ b/database/ui/src/page/CellInspector.tsx @@ -0,0 +1,146 @@ +/** + * The focused cell, in full. + * + * A pure function of the grid cursor: arrows keep driving the grid, this + * follows. That is why it can sit beside the grid without competing for the + * keyboard. + * + * Two measurements are worth the lines. **Code points, not `.length`** — + * `'👋'.length` is 2, which is the kind of number that makes someone doubt the + * whole panel; `[...s].length` is 1. And **bytes via `TextEncoder`**, because + * a `varchar(20)` limit counts bytes on some drivers and characters on others, + * and "why did this fail to insert" usually ends here. + */ + +import { Badge, Button } from '@iii-dev/console-ui' +import { useMemo, useState } from 'react' +import { cellText } from '../lib/grid-cursor' +import type { ForeignKeyRef } from '../lib/rpc' +import { Check, Copy, Link2 } from './icons' +import { JsonTree } from './JsonTree' + +const ENCODER = new TextEncoder() + +export function CellInspector({ + table, + column, + type, + value, + row, + rowCount, + col, + colCount, + foreignKey, + onFollow, +}: { + table: string + column: string + type?: string + value: unknown + row: number + rowCount: number + col: number + colCount: number + foreignKey?: ForeignKeyRef + onFollow?: (fk: ForeignKeyRef, value: unknown) => void +}) { + const [copied, setCopied] = useState(null) + const [wrap, setWrap] = useState(true) + + const text = useMemo(() => cellText(value), [value]) + const isNull = value === null || value === undefined + const json = useMemo(() => asJson(value), [value]) + + const copy = (what: string, payload: string) => { + void navigator.clipboard?.writeText(payload) + setCopied(what) + // Recolour briefly rather than raising a toast — the house pattern. + setTimeout(() => setCopied(null), 1200) + } + + const codePoints = isNull ? 0 : [...text].length + const bytes = isNull ? 0 : ENCODER.encode(text).length + + return ( +
+
+ + {table}.{column} + + {type ? {type.toLowerCase()} : null} +
+ +
+ row {row + 1} of {rowCount} · column {col + 1} of {colCount} +
+ +
+
kind
+
{kindOf(value)}
+
characters
+
{codePoints.toLocaleString()}
+
bytes
+
{bytes.toLocaleString()}
+
+ + {foreignKey && onFollow && !isNull ? ( + + ) : null} + +
+ {isNull ? ( + NULL + ) : value === '' ? ( + // Distinct from NULL, and from an empty box. + '' (empty string) + ) : json !== undefined ? ( + copy('json', t)} /> + ) : ( + 
{text}
+ )} +
+ +
+ + {json === undefined && text.length > 80 ? ( + + ) : null} +
+
+ ) +} + +function kindOf(value: unknown): string { + if (value === null || value === undefined) return 'null' + if (Array.isArray(value)) return 'array' + return typeof value +} + +/** + * The value as JSON, when it is worth a tree. + * + * Objects qualify directly. A *string* qualifies only when it parses to an + * object or array — many drivers hand back JSON columns as text, and rendering + * that as a flat line loses the structure the column exists to hold. A string + * that parses to a bare number is left alone: `"5"` is not a document. + */ +function asJson(value: unknown): unknown { + if (value !== null && typeof value === 'object') return value + if (typeof value !== 'string') return undefined + const trimmed = value.trim() + if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) return undefined + try { + const parsed = JSON.parse(trimmed) + return typeof parsed === 'object' && parsed !== null ? parsed : undefined + } catch { + return undefined + } +} diff --git a/database/ui/src/page/ChangesPanel.tsx b/database/ui/src/page/ChangesPanel.tsx new file mode 100644 index 000000000..1751fbfb7 --- /dev/null +++ b/database/ui/src/page/ChangesPanel.tsx @@ -0,0 +1,185 @@ +/** + * What has changed in the selected table, as it happens. + * + * The worker has emitted `database::row-changed` for a while and #640 gave it + * real cross-client capture, but nothing ever rendered it — so a write from + * another client was observable to agents and invisible to the person looking + * at the table. This is that view. + * + * The panel is deliberate about what it promises. It never uses the words + * "live" or "realtime"; it names the guarantee in force, because the two + * capture modes differ in a way that matters and only one of them sees other + * clients' writes. + */ + +import { Badge, Button, EmptyState, type Host, StatusDot } from '@iii-dev/console-ui' +import { useEffect, useState } from 'react' +import { History, RefreshCw } from './icons' +import { type RowChange, useRowChanges } from './useRowChanges' + +const HistoryIcon = (p: { className?: string }) => + +/** How long a row keeps its arrival highlight. */ +const FRESH_MS = 4000 + +const OP_LABEL: Record = { + insert: 'insert', + update: 'update', + delete: 'delete', + other: 'other', +} + +export function ChangesPanel({ + host, + db, + table, + kind, + onRefresh, +}: { + host: Host + db: string + table: string | null + /** `view` cannot be followed — see below. */ + kind?: string + onRefresh?: () => void +}) { + // A view is never the target of a write. The worker keys every change on the + // table the statement actually touched, so a binding on a view matches + // nothing and would sit at "following" forever while rows visibly change + // underneath it. Refuse the binding and say why. + const isView = kind === 'view' + const feed = useRowChanges(host, db, isView ? null : table) + // Re-render on a slow cadence so relative stamps and the fade advance + // without an interval per row. + const [, tick] = useState(0) + useEffect(() => { + if (feed.changes.length === 0) return + const id = setInterval(() => tick((n) => n + 1), 1000) + return () => clearInterval(id) + }, [feed.changes.length]) + + if (!table) { + return ( + + ) + } + + if (isView) { + return ( + + ) + } + + if (feed.status === 'unsupported') { + return ( + + ) + } + + return ( +
+
+ + {table} +
+ {feed.pending > 0 && onRefresh ? ( + + ) : null} +
+ + {feed.changes.length === 0 ? ( +
+

nothing yet.

+

+ writes to {table} appear here as they commit. what counts as a write depends on this + connection's capture mode — hover the indicator above. +

+
+ ) : ( +
    + {feed.changes.map((c, i) => ( + + ))} +
+ )} +
+ ) +} + +function ChangeRow({ change }: { change: RowChange }) { + const age = Date.now() - change.seen + const fresh = age < FRESH_MS + const rows = change.affected_rows + return ( +
  • + {OP_LABEL[change.op] ?? change.op} + + {rows} {rows === 1 ? 'row' : 'rows'} + + {change.returning?.length ? {change.returning.length} returned : null} + + {relative(age)} + +
    • + ) +} + +/** + * Which guarantee is in force. + * + * The mode is a property of the connection and is not exposed as a read, so + * this reports what it can actually stand behind: that a binding is attached, + * and when something last arrived. It never claims completeness — the tooltip + * carries the caveat rather than a word in the badge implying more than is + * true. + */ +function FreshnessBadge({ status, lastAt }: { status: string; lastAt: number | null }) { + const bound = status === 'bound' + const recent = lastAt != null && Date.now() - lastAt < 10_000 + return ( + + + {bound ? 'following' : 'not following'} + {lastAt != null ? · last {relative(Date.now() - lastAt)} : null} + + ) +} + +function relative(ms: number): string { + if (ms < 1000) return 'just now' + const s = Math.floor(ms / 1000) + if (s < 60) return `${s}s ago` + const m = Math.floor(s / 60) + if (m < 60) return `${m}m ago` + const h = Math.floor(m / 60) + return `${h}h ago` +} diff --git a/database/ui/src/page/ColumnStatsPanel.tsx b/database/ui/src/page/ColumnStatsPanel.tsx new file mode 100644 index 000000000..6fa2e1cc0 --- /dev/null +++ b/database/ui/src/page/ColumnStatsPanel.tsx @@ -0,0 +1,134 @@ +/** + * What is actually in a column. + * + * `source: planner` means the numbers come from statistics the database keeps + * for the query planner — cheap, and possibly stale or absent. `computed` + * means real aggregates ran. That distinction is rendered as a chip on every + * result, because a stale `n_distinct` presented as fact is worse than no + * number: it reads exactly like a measurement. + */ + +import { Badge, Button, type Host, StatusPanel } from '@iii-dev/console-ui' +import { useCallback, useState } from 'react' +import { type ColumnStat, columnStats } from '../lib/rpc' +import { AlertCircle } from './icons' +import { useDatabaseRead } from './useDatabaseRead' + +export function ColumnStatsPanel({ + host, + db, + table, + column, +}: { + host: Host + db: string + table: string + column: string +}) { + const [exact, setExact] = useState(false) + const fetcher = useCallback(() => columnStats(host, db, table, [column], exact), [host, db, table, column, exact]) + const read = useDatabaseRead(true, fetcher) + + if (read.error) { + return ( + } + headline="could not profile this column" + detail={read.error} + /> + ) + } + if (!read.data) { + return
    · reading statistics…
    + } + + const stat: ColumnStat | undefined = read.data.columns[0] + if (!stat) { + return

    no statistics for this column.

    + } + + const total = stat.row_count ?? 0 + const nullFrac = stat.null_fraction ?? (total > 0 && stat.null_count != null ? stat.null_count / total : null) + const topMax = Math.max(...stat.top_values.map((v) => v.count), 1) + + return ( +
    +
    + {stat.name} + {stat.source === 'planner' ? ( + + estimated · from planner statistics + + ) : ( + measured + )} +
    + + {stat.source === 'planner' ? ( + + ) : null} + +
    + + + + + + +
    + + {nullFrac != null ? ( +
    + null share + + + + {(nullFrac * 100).toFixed(1)}% +
    + ) : null} + + {stat.top_values.length > 0 ? ( + <> +

    most common

    +
      + {stat.top_values.map((v, i) => ( +
    • + + {render(v.value)} + + + + + {v.count.toLocaleString()} +
      • + ))} +
    + + ) : null} +
    + ) +} + +function Row({ label, value }: { label: string; value: string }) { + return ( + <> +
    {label}
    +
    {value}
    + + ) +} + +function fmt(n: number | null | undefined): string { + return n == null ? '—' : n.toLocaleString() +} + +/** Null and empty string must not look alike here either. */ +function render(v: unknown): string { + if (v == null) return 'NULL' + if (v === '') return "''" + if (typeof v === 'object') return JSON.stringify(v) + return String(v) +} diff --git a/database/ui/src/page/ErdPanel.tsx b/database/ui/src/page/ErdPanel.tsx new file mode 100644 index 000000000..1da2211e3 --- /dev/null +++ b/database/ui/src/page/ErdPanel.tsx @@ -0,0 +1,490 @@ +/** + * Schema diagram. + * + * The worker does the layout — `database::schemaDiagram` returns positioned + * nodes and already-routed edge polylines, so this panel only draws and pans. + * That keeps the expensive part off the console's single React tree, and it + * means an agent can read the same shape without rendering anything. + * + * Rendering follows the house pattern used by the worktree and memory pages: + * absolutely-positioned HTML nodes over one SVG for the edges. Pan and zoom + * live in a ref and write `transform` straight onto the canvas during + * pointermove — putting them in state would re-render every node on every + * mouse move. + */ + +import { Badge, Button, EmptyState, type Host, Input, Select, StatusPanel } from '@iii-dev/console-ui' +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { schemaDiagram } from '../lib/rpc' +import { AlertCircle, KeyRound, Link2, RefreshCw, Table2 } from './icons' +import { useDatabaseRead } from './useDatabaseRead' + +const MIN_ZOOM = 0.15 +const MAX_ZOOM = 2.5 +/** Below this, column rows are unreadable anyway — drop them and keep 200 + * headers instead of 2400 rows in the DOM. */ +const COMPACT_BELOW = 0.5 +const PAD = 60 +/** Breathing room between a component's boundary and its outermost node. */ +const GROUP_PAD = 22 + +const TableIcon = (p: { className?: string }) => + +type Offset = { dx: number; dy: number } | undefined +type Pt = { x: number; y: number } + +/** + * Shift a routed edge to follow nodes that have been dragged. + * + * The worker emits a three-segment elbow: source anchor, two corner points in + * the gutter, target anchor. Moving the ends alone would leave the corners + * behind and bend the line through them, so each point is offset by a blend of + * the two ends weighted by how far along the polyline it sits. With neither + * end moved this is the identity. + */ +function shiftEdge(points: Pt[], from: Offset, to: Offset): Pt[] { + if (!from && !to) return points + const fx = from?.dx ?? 0 + const fy = from?.dy ?? 0 + const tx = to?.dx ?? 0 + const ty = to?.dy ?? 0 + const last = points.length - 1 + if (last <= 0) return points.map((p) => ({ x: p.x + fx, y: p.y + fy })) + return points.map((p, i) => { + const t = i / last + return { x: p.x + fx * (1 - t) + tx * t, y: p.y + fy * (1 - t) + ty * t } + }) +} + +interface View { + x: number + y: number + z: number +} + +export function ErdPanel({ + host, + db, + focusTable, +}: { + host: Host + db: string + /** Open focused on this table rather than on the whole schema. */ + focusTable?: string | null +}) { + // Focus turns the diagram from a wall into something explorable: one table + // and what it touches, expanding a hop at a time. + const [focus, setFocus] = useState(focusTable ?? null) + const [depth, setDepth] = useState(1) + + const fetcher = useCallback(() => schemaDiagram(host, db, { focus, depth }), [host, db, focus, depth]) + const read = useDatabaseRead(true, fetcher) + const diagram = read.data + + const [search, setSearch] = useState('') + const [selected, setSelected] = useState(null) + // Manual node positions, as offsets over what the worker laid out. Kept in + // the browser: this is where you dragged a box just now, not a preference + // worth persisting for everyone. + const [offsets, setOffsets] = useState>({}) + const nodeDragRef = useRef<{ table: string; px: number; py: number } | null>(null) + // Mirrored: `view` drives React, `viewRef` drives the raw transform during a + // drag so a 200-node diagram does not re-render per frame. + const [view, setView] = useState({ x: 0, y: 0, z: 1 }) + const viewRef = useRef(view) + const canvasRef = useRef(null) + const frameRef = useRef(null) + const dragRef = useRef<{ px: number; py: number } | null>(null) + const rafRef = useRef(null) + + const apply = useCallback(() => { + rafRef.current = null + const el = canvasRef.current + if (!el) return + const v = viewRef.current + el.style.transform = `translate(${v.x}px, ${v.y}px) scale(${v.z})` + el.classList.toggle('compact', v.z < COMPACT_BELOW) + }, []) + + const schedule = useCallback(() => { + if (rafRef.current == null) rafRef.current = requestAnimationFrame(apply) + }, [apply]) + + const setViewNow = useCallback( + (next: View) => { + viewRef.current = next + setView(next) + schedule() + }, + [schedule], + ) + + /** Scale the diagram to fit, so a wide schema is not off-screen on open. */ + const fitToFrame = useCallback(() => { + const frame = frameRef.current + if (!frame || !diagram || diagram.width === 0) return + const z = Math.min( + 1, + Math.max( + MIN_ZOOM, + Math.min((frame.clientWidth - PAD) / diagram.width, (frame.clientHeight - PAD) / diagram.height), + ), + ) + // Centre, rather than pinning to a corner and leaving the rest empty. + setViewNow({ + x: Math.max(PAD / 2, (frame.clientWidth - diagram.width * z) / 2), + y: Math.max(PAD / 2, (frame.clientHeight - diagram.height * z) / 2), + z, + }) + }, [diagram, setViewNow]) + + useEffect(() => { + fitToFrame() + }, [fitToFrame]) + + // A new layout invalidates hand-placed offsets: they were relative to + // coordinates that no longer exist. + // biome-ignore lint/correctness/useExhaustiveDependencies: keyed on the layout, not the object identity + useEffect(() => { + setOffsets({}) + }, [focus, depth]) + + useEffect( + () => () => { + if (rafRef.current != null) cancelAnimationFrame(rafRef.current) + }, + [], + ) + + /** + * Zoom on wheel. + * + * Bound natively with `{ passive: false }` rather than through React's + * `onWheel`. React attaches wheel handlers passively, where + * `preventDefault()` is a no-op — so the page scrolled underneath the + * diagram instead of the diagram zooming. + */ + useEffect(() => { + const frame = frameRef.current + if (!frame) return + const onWheel = (e: WheelEvent) => { + e.preventDefault() + const rect = frame.getBoundingClientRect() + const v = viewRef.current + const next = Math.min(MAX_ZOOM, Math.max(MIN_ZOOM, v.z * (e.deltaY < 0 ? 1.12 : 1 / 1.12))) + // Keep the point under the cursor fixed while zooming. + const cx = e.clientX - rect.left + const cy = e.clientY - rect.top + const k = next / v.z + setViewNow({ x: cx - (cx - v.x) * k, y: cy - (cy - v.y) * k, z: next }) + } + frame.addEventListener('wheel', onWheel, { passive: false }) + return () => frame.removeEventListener('wheel', onWheel) + }, [setViewNow]) + + const onPointerDown = (e: React.PointerEvent) => { + if (e.button !== 0) return + dragRef.current = { px: e.clientX, py: e.clientY } + ;(e.currentTarget as HTMLElement).setPointerCapture(e.pointerId) + } + + const onPointerMove = (e: React.PointerEvent) => { + const d = dragRef.current + if (!d) return + const v = viewRef.current + viewRef.current = { ...v, x: v.x + (e.clientX - d.px), y: v.y + (e.clientY - d.py) } + dragRef.current = { px: e.clientX, py: e.clientY } + schedule() + } + + const onPointerUp = (e: React.PointerEvent) => { + if (!dragRef.current) return + dragRef.current = null + ;(e.currentTarget as HTMLElement).releasePointerCapture(e.pointerId) + // Commit once, at the end, rather than on every frame. + setView(viewRef.current) + } + + /** + * Drag one node. Offsets are divided by the zoom so a box tracks the cursor + * at any scale — without that, dragging at 40% moves the node 2.5x too far. + */ + const startNodeDrag = (e: React.PointerEvent, table: string) => { + if (e.button !== 0) return + e.stopPropagation() + nodeDragRef.current = { table, px: e.clientX, py: e.clientY } + ;(e.currentTarget as HTMLElement).setPointerCapture(e.pointerId) + } + + const moveNode = (e: React.PointerEvent) => { + const d = nodeDragRef.current + if (!d) return + e.stopPropagation() + const z = viewRef.current.z || 1 + const dx = (e.clientX - d.px) / z + const dy = (e.clientY - d.py) / z + nodeDragRef.current = { ...d, px: e.clientX, py: e.clientY } + setOffsets((prev) => { + const at = prev[d.table] ?? { dx: 0, dy: 0 } + return { ...prev, [d.table]: { dx: at.dx + dx, dy: at.dy + dy } } + }) + } + + const endNodeDrag = (e: React.PointerEvent) => { + if (!nodeDragRef.current) return + nodeDragRef.current = null + ;(e.currentTarget as HTMLElement).releasePointerCapture(e.pointerId) + } + + const matches = useMemo(() => { + const q = search.trim().toLowerCase() + if (!q || !diagram) return null + return new Set(diagram.nodes.filter((n) => n.table.toLowerCase().includes(q)).map((n) => n.table)) + }, [search, diagram]) + + if (read.error) { + return ( + } + headline="could not lay out the schema" + detail={read.error} + /> + ) + } + if (read.loading && !diagram) { + return
    · laying out the schema…
    + } + if (!diagram || diagram.nodes.length === 0) { + return ( + + ) + } + + const related = diagram.nodes.length - diagram.isolated.length + + return ( +
    +
    + + {diagram.nodes.length} tables · {diagram.edges.length} relations + + {related === 0 ? ( + no foreign keys — nothing to connect + ) : ( + + {diagram.isolated.length} unrelated · {diagram.crossings} crossings + + )} + {diagram.truncated ? truncated : null} +
    + {focus ? ( + <> + + focused on {focus} + + + {Math.round(view.z * 100)}% + {Object.keys(offsets).length > 0 ? ( + + ) : null} + +
    + + {/* What was left out. A diagram that simply stops looks complete; naming + the next ring is what makes expanding a decision rather than a guess. */} + {focus && diagram.frontier.length > 0 ? ( +
    + also connected: + {diagram.frontier.map((t) => ( + + ))} + +
    + ) : null} + +
    +
    + {/* Component boundaries sit behind everything. A schema is usually + several independent clusters, and drawing that fact is what lets + a reader take in the structure before reading a single name. */} + {diagram.components + .filter((c) => c.tables.length > 1) + .map((c) => ( +
    + + {c.hub ? `${c.hub} · ` : ''} + {c.tables.length} related + +
    + ))} + + + foreign key relationships + {diagram.edges.map((e) => { + const dim = matches ? !matches.has(e.from) && !matches.has(e.to) : false + const hot = selected === e.from || selected === e.to + // An edge has to follow the nodes it joins. The worker routed it + // against the original coordinates, so each end is shifted by + // whatever its own node was dragged and the corner between them + // is interpolated — the elbow stays orthogonal without + // re-running the router in the browser. + const from = offsets[e.from] + const to = offsets[e.to] + const points = shiftEdge(e.points, from, to) + const a = points[0] + const b = points[points.length - 1] + return ( + ${e.to}.${e.to_column}`} + className={`db-erd-edge${hot ? ' hot' : ''}${dim ? ' dim' : ''}`} + > + `${p.x},${p.y}`).join(' ')} fill="none" /> + {/* Endpoint dots, the idiom both reference clients use: the + line lands on the exact column row, so a relationship + reads as column-to-column rather than box-to-box. */} + + + + ) + })} + + + {diagram.nodes.map((n) => { + const dim = matches ? !matches.has(n.table) : false + // Hub emphasis by border weight, not fill — the palette stays rationed. + const hub = n.degree >= 5 ? ' hub-3' : n.degree >= 2 ? ' hub-2' : '' + const off = offsets[n.table] + const isFocus = diagram.focus === n.table + return ( +
    + {/* The header is the drag handle as well as the button: one + place to grab, and a click that never moved still selects. */} + +
    + {n.columns.map((c) => ( +
    + {c.primary_key ? ( + + ) : c.foreign_key ? ( + + ) : ( + + )} + {c.name} + {/* Required columns carry a marker rather than the + nullable ones: in most schemas NOT NULL is the + smaller set, so it is the one worth marking. */} + {!c.nullable && !c.primary_key ? ( + + * + + ) : null} + {c.type.toLowerCase()} +
    + ))} + {n.hidden_columns > 0 ?
    +{n.hidden_columns} more
    : null} +
    +
    + ) + })} +
    +
    +
    + ) +} diff --git a/database/ui/src/page/FilterBar.tsx b/database/ui/src/page/FilterBar.tsx new file mode 100644 index 000000000..647331304 --- /dev/null +++ b/database/ui/src/page/FilterBar.tsx @@ -0,0 +1,263 @@ +/** + * Filters as chips, compiled server-side. + * + * Each chip is segmented — column, operator, value — and every segment is its + * own control, so refining one part of a condition does not mean retyping the + * rest. Two behaviours are worth calling out because both are easy to omit and + * both are what make a filter bar usable rather than merely present: + * + * A chip can be **switched off** instead of deleted. Narrowing a query is + * iterative, and "does this condition matter?" is the question being asked + * most often; answering it by deleting and rebuilding loses the work. + * + * An **incomplete chip is a draft** and is never sent. The worker would reject + * it, and a round trip that ends in an error would make typing feel broken. + * Drafts read as dashed, the repo's convention for content that is not yet + * real. + * + * Filters stack with AND. `in` exists precisely because OR does not: "status + * is one of open, held" cannot be expressed as two AND'd equalities. + */ + +import { + Button, + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, + Input, +} from '@iii-dev/console-ui' +import { useState } from 'react' +import { type FilterOp, type FilterSpec, isComplete, OP_LABEL, opsFor, SET_OPS, typeCategory } from '../lib/rpc' +import { Eye, EyeOff, Plus, X } from './icons' + +/** The minimum a column must expose to be filterable. */ +export interface FilterColumn { + name: string + type: string +} + +export function FilterBar({ + columns, + filters, + onChange, +}: { + columns: FilterColumn[] + filters: FilterSpec[] + onChange: (next: FilterSpec[]) => void +}) { + const replace = (i: number, f: FilterSpec) => onChange(filters.map((x, j) => (j === i ? f : x))) + const remove = (i: number) => onChange(filters.filter((_, j) => j !== i)) + + const add = () => { + const first = columns[0] + if (!first) return + onChange([...filters, { column: first.name, op: opsFor(typeCategory(first.type))[0] }]) + } + + const active = filters.filter((f) => !f.disabled && isComplete(f)).length + const drafts = filters.filter((f) => !isComplete(f)).length + + return ( +
    + {filters.map((f, i) => ( + replace(i, next)} + onRemove={() => remove(i)} + /> + ))} + + + + {filters.length > 0 ? ( + <> + + {active} applied + {drafts > 0 ? ` · ${drafts} draft` : ''} + + + + ) : null} +
    + ) +} + +function Chip({ + filter, + columns, + onChange, + onRemove, +}: { + filter: FilterSpec + columns: FilterColumn[] + onChange: (next: FilterSpec) => void + onRemove: () => void +}) { + const col = columns.find((c) => c.name === filter.column) + const ops = opsFor(typeCategory(col?.type)) + const draft = !isComplete(filter) + const isSet = SET_OPS.has(filter.op) + const takesValue = !NO_VALUE.has(filter.op) + + const setColumn = (name: string) => { + const next = columns.find((c) => c.name === name) + const allowed = opsFor(typeCategory(next?.type)) + // Keep the operator when the new column still supports it; otherwise fall + // back rather than leaving a chip that cannot compile. + onChange({ + ...filter, + column: name, + op: allowed.includes(filter.op) ? filter.op : allowed[0], + }) + } + + const setOp = (op: FilterOp) => { + // Moving between shapes must not carry stale operands across, or a chip + // reads as complete while holding a value its operator ignores. + const carry: FilterSpec = { ...filter, op } + if (SET_OPS.has(op)) { + carry.values = filter.values ?? (filter.value != null ? [filter.value] : []) + carry.value = undefined + carry.value2 = undefined + } else { + carry.value = filter.value ?? filter.values?.[0] + carry.values = undefined + if (op !== 'between') carry.value2 = undefined + } + onChange(carry) + } + + return ( + + + + ({ value: c.name, label: c.name, hint: c.type.toLowerCase() }))} + onPick={setColumn} + /> + + ({ value: o, label: OP_LABEL[o], hint: o }))} + onPick={(v) => setOp(v as FilterOp)} + /> + + {takesValue ? ( + isSet ? ( + onChange({ ...filter, values })} + /> + ) : ( + <> + onChange({ ...filter, value })} label="value" /> + {filter.op === 'between' ? ( + onChange({ ...filter, value2 })} + label="upper bound" + /> + ) : null} + + ) + ) : null} + + + + ) +} + +const NO_VALUE: ReadonlySet = new Set(['is_true', 'is_false', 'is_null', 'is_not_null', 'is_empty']) + +function Picker({ + label, + className, + items, + onPick, +}: { + label: string + className: string + items: { value: string; label: string; hint?: string }[] + onPick: (value: string) => void +}) { + return ( + + + + + + {items.map((it) => ( + onPick(it.value)}> + {it.label} + {it.hint ? {it.hint} : null} + + ))} + + + ) +} + +function ValueInput({ value, onChange, label }: { value: unknown; onChange: (v: string) => void; label: string }) { + return ( + + + + ) +} + +/** Comma-separated entry for `in` / `not_in`. */ +function SetValue({ values, onChange }: { values: unknown[]; onChange: (v: unknown[]) => void }) { + const [text, setText] = useState(values.map((v) => String(v)).join(', ')) + return ( + + { + setText(next) + onChange( + next + .split(',') + .map((s) => s.trim()) + .filter((s) => s !== ''), + ) + }} + placeholder="a, b, c" + preserveCase + aria-label="values, comma separated" + className="db-chip-input wide" + /> + + ) +} diff --git a/database/ui/src/page/HealthPanel.tsx b/database/ui/src/page/HealthPanel.tsx new file mode 100644 index 000000000..76cf015de --- /dev/null +++ b/database/ui/src/page/HealthPanel.tsx @@ -0,0 +1,295 @@ +/** + * Connection health. + * + * Every section is a `ProbeResult`, and rendering that honestly is the entire + * point of the panel. "sqlite has no equivalent of pg_stat_activity", + * "permission denied on pg_stat_activity" and "no queries are running" are + * three different facts, and a panel that shows an empty list for all three + * teaches you to distrust it. + */ + +import { Button, type Host, Select, StatusPanel } from '@iii-dev/console-ui' +import { useCallback, useEffect, useState } from 'react' +import { type HealthReport, health } from '../lib/rpc' +import { AlertCircle, RefreshCw } from './icons' +import { useDatabaseRead } from './useDatabaseRead' + +type Probe = + | { status: 'available'; data: T } + | { status: 'unsupported'; reason: string } + | { status: 'denied'; reason: string } + +const INTERVALS = [ + { value: '0', label: 'manual' }, + { value: '5', label: 'every 5s' }, + { value: '15', label: 'every 15s' }, + { value: '60', label: 'every 60s' }, +] + +export function HealthPanel({ host, db }: { host: Host; db: string }) { + const [nonce, setNonce] = useState(0) + const [every, setEvery] = useState('0') + const fetcher = useCallback(() => { + void nonce + return health(host, db) + }, [host, db, nonce]) + const read = useDatabaseRead(true, fetcher) + + const refresh = useCallback(() => setNonce((n) => n + 1), []) + + useEffect(() => { + const seconds = Number(every) + if (!seconds) return + const id = setInterval(refresh, seconds * 1000) + return () => clearInterval(id) + }, [every, refresh]) + + if (read.error) { + return ( + } + headline="could not read connection health" + detail={read.error} + /> + ) + } + if (!read.data) { + return
    · probing the connection…
    + } + + const h: HealthReport = read.data + const pool = h.pool + const inUse = pool.size != null && pool.idle != null ? pool.size - pool.idle : null + + return ( +
    +
    + {h.driver} + worker {h.worker_version} +
    +