From a0c16b282b111b7d88a9c4a1c2ff628fed2128b6 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Thu, 30 Jul 2026 16:51:24 +0100 Subject: [PATCH] (MOT-4283) feat(database): read-side function surface and console renderer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The database worker learns to describe itself, and the console page becomes a renderer on top of that rather than a second implementation. Twelve documented functions become twenty-nine. Introspection (listTables, describeTable, describeSchema) returns structured foreign keys rather than a lossy "table.column" string, which is what makes relational navigation and the diagram possible. browseTable takes typed filters and sort specs, so a caller gets paged, filtered, sorted reads without writing dialect SQL. explain parses three plan formats into one tree and computes its own warnings. columnStats reads planner statistics by default and only runs real aggregates when asked. health reports each probe as available, unsupported or denied, so "sqlite has no pg_stat_activity" cannot be mistaken for "nothing is running". schemaDiagram does the layout — component split, ranking, crossing reduction, routed elbows — and returns positioned nodes and the connected groups. Saved queries and history live on the state builtin instead of one browser's localStorage. Filters cover set membership and can be switched off. Stacking is AND, so "is one of open, held" is not expressible as two equalities and needs `in` outright; `not_in` keeps NULL rows, which the plain form silently drops. A disabled filter is skipped rather than validated, because a half-built condition parked mid-edit should not fail the request around it. The page drops roughly 520 lines of per-driver catalog SQL and calls those functions instead. Alongside the existing data and sql modes it gains a schema diagram, a connection health view, and a view of the writes landing in the selected table, which the row-changed trigger has emitted for some time with nothing rendering it. Explain draws a plan tree with a per-node cost bar rather than a grid of text. The grid becomes navigable. Cells were individually focusable buttons, which on a fifty-row page put a thousand tab stops between the filter bar and the pager; they are now one roving-tabindex grid where Tab leaves and the arrows move within. The cursor crosses page boundaries, and survives a sort by re-finding its row key rather than its index. Movement is a pure function with its own tests. Following a foreign key filters the referenced table and leaves a breadcrumb, and the cell inspector reads long values, counts code points rather than UTF-16 units, and renders JSON as a tree instead of a flat line. The grid is customisable, and the layout is not private to one browser. Columns resize by dragging their edge, reorder by dragging their header, and hide from the header itself; widths, order and hidden columns persist through getTableView / saveTableView on the state builtin. Keeping them there rather than in localStorage is the same argument as for history: a layout survives a restart, follows the operator to another machine, and an agent can set a table up for a person to read. Widths are clamped server-side so a stored value cannot produce a layout that has to be undone by hand, and a view is never validated against the live schema — after a rename a column loses its width rather than the whole layout failing to load. Two things the change feed refuses to do, both because the honest answer is narrower than it looks: it never says "live", since a connection using statement capture sees only this worker's writes; and it will not follow a view, since changes are keyed on the table a statement writes to and such a binding would sit at "following" forever while rows changed underneath it. Panels whose function the worker lacks are hidden rather than rendered as controls that always error, discovered once per mount through engine::functions::list. --- database/README.md | 38 +- database/skills/SKILL.md | 46 +- database/src/handlers/browse.rs | 165 +++ database/src/handlers/catalog/mod.rs | 264 +++++ database/src/handlers/catalog/mysql.rs | 180 +++ database/src/handlers/catalog/postgres.rs | 250 ++++ database/src/handlers/catalog/sqlite.rs | 232 ++++ database/src/handlers/column_stats.rs | 427 +++++++ database/src/handlers/diagram.rs | 1290 +++++++++++++++++++++ database/src/handlers/explain.rs | 712 ++++++++++++ database/src/handlers/filter.rs | 744 ++++++++++++ database/src/handlers/health.rs | 450 +++++++ database/src/handlers/mod.rs | 16 + database/src/handlers/query.rs | 29 +- database/src/handlers/saved.rs | 368 ++++++ database/src/handlers/schema.rs | 274 +++++ database/src/handlers/table_view.rs | 183 +++ database/src/handlers/tx_sql_guard.rs | 110 ++ database/src/main.rs | 325 +++++- database/src/pool/mod.rs | 25 + database/src/pool/mysql.rs | 16 + database/src/pool/postgres.rs | 13 + database/src/pool/sqlite.rs | 11 + database/src/triggers/bus.rs | 7 + database/tests/integration.rs | 637 ++++++++++ database/ui/build.mjs | 86 +- database/ui/package.json | 6 +- database/ui/src/lib/capabilities.ts | 82 ++ database/ui/src/lib/grid-cursor.test.ts | 130 +++ database/ui/src/lib/grid-cursor.ts | 173 +++ database/ui/src/lib/rpc.ts | 682 +++++++++++ database/ui/src/page/CellInspector.tsx | 146 +++ database/ui/src/page/ChangesPanel.tsx | 185 +++ database/ui/src/page/ColumnStatsPanel.tsx | 134 +++ database/ui/src/page/ErdPanel.tsx | 490 ++++++++ database/ui/src/page/FilterBar.tsx | 263 +++++ database/ui/src/page/HealthPanel.tsx | 295 +++++ database/ui/src/page/JsonTree.tsx | 160 +++ database/ui/src/page/PlanTree.tsx | 181 +++ database/ui/src/page/RowDetail.tsx | 55 +- database/ui/src/page/SqlPanel.tsx | 113 +- database/ui/src/page/TableDataPanel.tsx | 257 +++- database/ui/src/page/db-data.ts | 701 +++-------- database/ui/src/page/icons.tsx | 17 + database/ui/src/page/index.tsx | 218 +++- database/ui/src/page/result-grid.tsx | 260 +++-- database/ui/src/page/useGridKeyboard.ts | 207 ++++ database/ui/src/page/useRowChanges.ts | 132 +++ database/ui/src/page/useTableView.ts | 185 +++ database/ui/styles.css | 725 +++++++++++- pnpm-lock.yaml | 3 + 51 files changed, 11883 insertions(+), 815 deletions(-) create mode 100644 database/src/handlers/browse.rs create mode 100644 database/src/handlers/catalog/mod.rs create mode 100644 database/src/handlers/catalog/mysql.rs create mode 100644 database/src/handlers/catalog/postgres.rs create mode 100644 database/src/handlers/catalog/sqlite.rs create mode 100644 database/src/handlers/column_stats.rs create mode 100644 database/src/handlers/diagram.rs create mode 100644 database/src/handlers/explain.rs create mode 100644 database/src/handlers/filter.rs create mode 100644 database/src/handlers/health.rs create mode 100644 database/src/handlers/saved.rs create mode 100644 database/src/handlers/schema.rs create mode 100644 database/src/handlers/table_view.rs create mode 100644 database/ui/src/lib/capabilities.ts create mode 100644 database/ui/src/lib/grid-cursor.test.ts create mode 100644 database/ui/src/lib/grid-cursor.ts create mode 100644 database/ui/src/lib/rpc.ts create mode 100644 database/ui/src/page/CellInspector.tsx create mode 100644 database/ui/src/page/ChangesPanel.tsx create mode 100644 database/ui/src/page/ColumnStatsPanel.tsx create mode 100644 database/ui/src/page/ErdPanel.tsx create mode 100644 database/ui/src/page/FilterBar.tsx create mode 100644 database/ui/src/page/HealthPanel.tsx create mode 100644 database/ui/src/page/JsonTree.tsx create mode 100644 database/ui/src/page/PlanTree.tsx create mode 100644 database/ui/src/page/useGridKeyboard.ts create mode 100644 database/ui/src/page/useRowChanges.ts create mode 100644 database/ui/src/page/useTableView.ts diff --git a/database/README.md b/database/README.md index 71f64628f..d72820eb1 100644 --- a/database/README.md +++ b/database/README.md @@ -4,7 +4,6 @@ | field | value | |-------|-------| -| version | 1.0.0 | | type | binary | | supported_targets | x86_64-apple-darwin, aarch64-apple-darwin, x86_64-unknown-linux-gnu, aarch64-unknown-linux-gnu | | author | iii | @@ -12,7 +11,7 @@ ## Install ```sh -iii worker add database@1.0.0 +iii worker add database ``` ## Skills @@ -193,6 +192,41 @@ const { rows } = await iii.trigger({ | `database::rollbackTransaction` | Rollback and finalize an interactive transaction. Subsequent calls against the same id return `TRANSACTION_NOT_FOUND`. | | `database::listDatabases` | List configured databases. Returns `{ databases, count }`; each entry has `name`, `driver`, credential-redacted `url`, `pool` settings, and `tls` (`mode`, `ca_cert_present`, `trust_native`). Config only — no health checks or live pool stats. | +### Reading the schema + +| Function | Description | +|---|---| +| `database::listTables` | Every table and view, with its kind and (postgres) its schema. | +| `database::describeTable` | One table: columns with type, nullability, default, primary-key membership and foreign-key target; plus indexes and a planner row estimate. Foreign keys are structured `{ schema, table, column }`, not a joined string. | +| `database::describeSchema` | The same shape for every table at once. One catalog query per aspect across the whole database rather than one call per table, so a 200-table schema costs a handful of queries. `include_indexes` is off by default. | +| `database::schemaDiagram` | Positioned table nodes and routed foreign-key edges, plus each table's hub `degree`, the `isolated` tables, and remaining edge `crossings`. Layout runs server-side, so a renderer only draws. | + +### Reading data + +| Function | Description | +|---|---| +| `database::browseTable` | Paged, sorted, filtered table read — no SQL from the caller. Filters are structured (`{ column, op, value }`) and compile to a parameterised `WHERE` for the driver in hand; `total` honours the same filters. Sorts accept a `mode` (`natural`, `length`, `absolute_value`, `random`) applied across the whole table, not just the page. To follow a foreign key, filter on equality with `page_size: 1`. | +| `database::explain` | The query plan as a tree with per-node cost, row estimates and warnings, instead of the driver's raw text. `analyze` collects real timings by **running** the statement, so it defaults to `false` and is refused for anything that is not a single read. | +| `database::columnStats` | Profile a table's columns. Reads the planner's own statistics by default — free and approximate, labelled `source: planner`. `exact: true` runs real aggregates and scans the table; it is refused above a row-count ceiling. To profile rows you already hold, pipe a `browseTable` result through the `fp` worker instead. | + +### Operations + +| Function | Description | +|---|---| +| `database::health` | Live pool occupancy plus active queries, table sizes, blocking locks and cache hit ratio. Each section reports separately as `available`, `unsupported` or `denied`, so a driver gap or a restricted role is never mistaken for an empty result. | +| `database::terminateQuery` | Terminate a backend session, or cancel just its statement with `cancel_only`. Takes an id from `health`. Separate from `health` because it is a write. | + +### Saved queries and history + +Stored in the [`state`](https://github.com/iii-hq/workers/tree/main/state) worker, scoped per database, so they survive restarts and any agent can read them. + +| Function | Description | +|---|---| +| `database::saveQuery` | Save a named query. Saving under an existing name replaces it. | +| `database::listSavedQueries` | Saved queries for a database, sorted by name. | +| `database::deleteSavedQuery` | Delete by id or by name. | +| `database::history` | Recent queries, newest first. Best effort — recording never blocks or fails a query, so this is a convenience rather than an audit log. For an audit trail bind `database::row-changed`. | + ## Triggers ### `database::row-changed` diff --git a/database/skills/SKILL.md b/database/skills/SKILL.md index 65dce3ce9..6b91e1c98 100644 --- a/database/skills/SKILL.md +++ b/database/skills/SKILL.md @@ -73,14 +73,54 @@ point. Placeholder syntax: `?` for SQLite and MySQL, `$1`/`$2`/… for Postgres. transaction. - `database::rollbackTransaction` — roll back and finalize an interactive transaction. -- `database::listDatabases` — list every configured database with its - driver, credential-redacted connection URL, pool settings, and TLS mode. - Config details only; no health checks or live pool statistics. +- `database::listDatabases` — every configured database with its driver, + credential-redacted URL, pool settings and TLS mode. Config only; use + `database::health` for live state. Interactive transactions auto-roll back when `timeout_ms` elapses (default 30 s, max 5 min). Prepared handles default to a 1 h TTL (max 24 h) with no explicit release call — let them expire or stop using them when done. +### Reading the schema + +One shape across all three drivers — prefer these over hand-writing +`sqlite_master` / `information_schema` / `PRAGMA`. + +- `database::listTables` — tables and views, with kind and (postgres) schema. +- `database::describeTable` — columns with type, nullability, default, primary + key and a structured `foreign_key` of `{ schema, table, column }`; plus + indexes and a planner row estimate. +- `database::describeSchema` — the same for every table in one pass. Use this + rather than looping `describeTable`. +- `database::schemaDiagram` — positioned nodes, routed foreign-key edges, hub + `degree` and `isolated` tables. For reasoning about a schema's shape, not + only for drawing it. + +### Reading data + +- `database::browseTable` — paged, sorted, filtered reads with no SQL. Filters + are `{ column, op, value }` and `total` honours them. Follow a foreign key + with an equality filter at `page_size: 1`. +- `database::explain` — the plan as a tree with costs and warnings. `analyze` + **runs** the statement, so it defaults to false and is refused for anything + that is not a single read. +- `database::columnStats` — planner statistics by default (approximate, + labelled `source: planner`); `exact: true` scans. To profile rows you already + hold, use the `fp` worker on a `browseTable` result instead. + +### Operations and reuse + +- `database::health` — pool occupancy, active queries, table sizes, locks, + cache ratio. Each section is `available`, `unsupported` or `denied`, so a + driver gap is never mistaken for an empty result. +- `database::terminateQuery` — end a session, or cancel its statement with + `cancel_only`. Takes an id from `database::health`. +- `database::saveQuery`, `database::listSavedQueries`, + `database::deleteSavedQuery` — named queries per database, kept in the + `state` worker. +- `database::history` — recent queries, newest first. Best effort, not an audit + log; bind `database::row-changed` for that. + ## Reacting to writes Register a `database::row-changed` trigger to be told when this worker commits diff --git a/database/src/handlers/browse.rs b/database/src/handlers/browse.rs new file mode 100644 index 000000000..6c1f4fd27 --- /dev/null +++ b/database/src/handlers/browse.rs @@ -0,0 +1,165 @@ +//! `database::browseTable` — paged, sorted, filtered reads without the caller +//! writing SQL. +//! +//! Subsumes three things the console used to do by hand: building +//! `SELECT * … ORDER BY … LIMIT/OFFSET`, running a matching `COUNT(*)`, and +//! compiling filter chips into a `WHERE`. It also covers foreign-key lookup — +//! that is an equality filter at `page_size: 1` — so there is no separate +//! read-a-row function. + +use super::filter::{self, FilterSpec, SortSpec}; +use super::query::{self, err_to_str, QueryReq}; +use super::AppState; +use crate::config::DriverKind; +use crate::driver::ColumnMeta; +use crate::error::DbError; +use crate::pool::Pool; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +fn default_page_size() -> u32 { + 50 +} + +fn default_timeout() -> u64 { + 30_000 +} + +fn default_include_total() -> bool { + true +} + +/// Ceiling on a single page. A caller wanting everything should page. +const MAX_PAGE_SIZE: u32 = 1_000; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct BrowseTableReq { + #[serde(default)] + pub db: Option, + pub table: String, + #[serde(default)] + pub schema: Option, + /// Zero-based. + #[serde(default)] + pub page: u32, + #[serde(default = "default_page_size")] + pub page_size: u32, + /// Applied in order; sort priority is position in the list. + #[serde(default)] + pub sort: Vec, + /// Combined with AND. + #[serde(default)] + pub filters: Vec, + /// A filtered `COUNT(*)` is a second query and can be expensive on a + /// large table. Turn it off while the caller is still typing. + #[serde(default = "default_include_total")] + pub include_total: bool, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct BrowseTableResp { + pub rows: Vec>, + pub columns: Vec, + pub page: u32, + pub page_size: u32, + /// Derived from a sentinel row, so it is correct without a count. + pub has_more: bool, + /// Total matching the same filters. Absent when not requested. + #[serde(skip_serializing_if = "Option::is_none")] + pub total: Option, +} + +async fn driver_of(state: &AppState, db: &str) -> Result { + Ok(match state.pool(db).await.map_err(err_to_str)? { + Pool::Sqlite(_) => DriverKind::Sqlite, + Pool::Postgres(_) => DriverKind::Postgres, + Pool::Mysql(_) => DriverKind::Mysql, + }) +} + +pub async fn handle(state: &AppState, req: BrowseTableReq) -> Result { + if req.table.trim().is_empty() { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: "table is required".into(), + })); + } + let db = state.resolve_db(req.db.clone()).await.map_err(err_to_str)?; + let driver = driver_of(state, &db).await?; + + let page_size = req.page_size.clamp(1, MAX_PAGE_SIZE); + let offset = (req.page as u64) * (page_size as u64); + + let target = filter::quote_table(driver, req.schema.as_deref(), &req.table); + let where_clause = filter::compile_where(driver, &req.filters, 1).map_err(err_to_str)?; + let predicate = if where_clause.sql.is_empty() { + String::new() + } else { + format!(" WHERE {}", where_clause.sql) + }; + + let order = filter::compile_order_by(driver, &req.sort).map_err(err_to_str)?; + let order_by = if order.is_empty() { + String::new() + } else { + format!(" ORDER BY {order}") + }; + + // Fetch one extra row to learn whether another page exists, so `has_more` + // is right even when the caller skipped the count. + let limit = page_size as u64 + 1; + let sql = format!("SELECT * FROM {target}{predicate}{order_by} LIMIT {limit} OFFSET {offset}"); + + let mut resp = query::handle( + state, + QueryReq { + db: Some(db.clone()), + sql, + params: where_clause.params.clone(), + timeout_ms: req.timeout_ms, + record_history: false, + }, + ) + .await?; + + let has_more = resp.rows.len() as u64 > page_size as u64; + resp.rows.truncate(page_size as usize); + + let total = if req.include_total { + let count_sql = format!("SELECT COUNT(*) AS total FROM {target}{predicate}"); + let counted = query::handle( + state, + QueryReq { + db: Some(db), + sql: count_sql, + params: where_clause.params, + timeout_ms: req.timeout_ms, + record_history: false, + }, + ) + .await?; + counted + .rows + .first() + .and_then(|r| r.get("total")) + .and_then(|v| match v { + Value::Number(n) => n.as_i64(), + Value::String(s) => s.parse().ok(), + _ => None, + }) + } else { + None + }; + + Ok(BrowseTableResp { + rows: resp.rows, + columns: resp.columns, + page: req.page, + page_size, + has_more, + total, + }) +} diff --git a/database/src/handlers/catalog/mod.rs b/database/src/handlers/catalog/mod.rs new file mode 100644 index 000000000..f392ed3c5 --- /dev/null +++ b/database/src/handlers/catalog/mod.rs @@ -0,0 +1,264 @@ +//! Driver-neutral catalog shapes and the per-driver readers behind +//! `database::listTables` / `describeTable` / `describeSchema`. +//! +//! Every reader goes through `database::query`, so catalog reads inherit the +//! same pool, timeout, and read-only-transaction handling as user SQL. +//! +//! Two rules hold across all three drivers: +//! +//! 1. **No array-valued columns.** `RowValue` has no array variant, so a +//! catalog query returning `text[]`/`int[]` fails to decode. Anywhere a +//! catalog exposes an array (postgres `indkey`, `most_common_vals`), it is +//! unnested into scalar rows and regrouped in Rust. +//! 2. **One query per aspect, not per table.** The readers take an optional +//! table filter; `describeSchema` omits it and regroups, so a 200-table +//! schema costs three queries rather than six hundred. + +pub mod mysql; +pub mod postgres; +pub mod sqlite; + +use super::AppState; +use crate::handlers::query::{self, QueryReq}; +use schemars::JsonSchema; +use serde::Serialize; +use serde_json::Value; +use std::collections::HashMap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum TableKind { + Table, + View, +} + +/// A relation. `schema` is populated only where the driver has a meaningful +/// namespace above the table (postgres); it is never concatenated into `name`, +/// because `analytics.events` and a table literally called `analytics.events` +/// are different things and callers must be able to tell them apart. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct TableRef { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option, + pub kind: TableKind, +} + +/// Where a foreign key points. Structured rather than a `"table.column"` +/// string so a schema-qualified target stays unambiguous. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct ForeignKeyRef { + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option, + pub table: String, + pub column: String, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct ColumnDesc { + pub name: String, + /// Driver-reported type text (`TEXT`, `integer`, `varchar(255)`). + #[serde(rename = "type")] + pub ty: String, + pub nullable: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub default_value: Option, + pub primary_key: bool, + /// 1-based ordinal, as the driver reports it. + pub position: i32, + #[serde(skip_serializing_if = "Option::is_none")] + pub foreign_key: Option, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct IndexDesc { + pub name: String, + pub unique: bool, + pub primary: bool, + /// Indexed columns in ordinal order. Empty when the index is on an + /// expression rather than plain columns. + pub columns: Vec, +} + +/// Key a per-table result is grouped under. `describeSchema` fans one query +/// out across every table, so each row has to say which table it belongs to. +pub type TableKey = (Option, String); + +/// Identifies one table for the readers. `schema` is ignored by sqlite and +/// mysql, which have no namespace above the table within a connection. +#[derive(Debug, Clone)] +pub struct TableFilter { + pub schema: Option, + pub table: String, +} + +/// Split a possibly schema-qualified name into `(schema, bare)`. Only applied +/// where the driver actually has schemas — see `TableFilter`. +pub fn split_qualified(name: &str) -> (Option, String) { + match name.split_once('.') { + Some((schema, bare)) if !schema.is_empty() && !bare.is_empty() => { + (Some(schema.to_string()), bare.to_string()) + } + _ => (None, name.to_string()), + } +} + +/// Double-quote an identifier, doubling embedded quotes. Used only where a +/// value cannot be bound as a parameter. +pub fn quote_ident(ident: &str) -> String { + format!("\"{}\"", ident.replace('"', "\"\"")) +} + +/// Run a catalog statement through the ordinary query path. +pub async fn run( + state: &AppState, + db: &str, + sql: impl Into, + params: Vec, + timeout_ms: u64, +) -> Result>, String> { + let resp = query::handle( + state, + QueryReq { + db: Some(db.to_string()), + sql: sql.into(), + params, + timeout_ms, + // Catalog reads are the page's plumbing, not the user's queries. + record_history: false, + }, + ) + .await?; + Ok(resp.rows) +} + +type Row = serde_json::Map; + +pub fn str_at(row: &Row, key: &str) -> Option { + match row.get(key) { + Some(Value::String(s)) => Some(s.clone()), + Some(Value::Number(n)) => Some(n.to_string()), + _ => None, + } +} + +pub fn i64_at(row: &Row, key: &str) -> Option { + match row.get(key) { + Some(Value::Number(n)) => n.as_i64().or_else(|| n.as_f64().map(|f| f as i64)), + Some(Value::String(s)) => s.parse().ok(), + _ => None, + } +} + +/// Catalogs are inconsistent about how they spell a boolean: postgres returns +/// a real bool, mysql returns 0/1, sqlite's PRAGMAs return 0/1, and +/// `information_schema` returns the strings `YES`/`NO`. Normalize all of them. +pub fn bool_at(row: &Row, key: &str) -> bool { + match row.get(key) { + Some(Value::Bool(b)) => *b, + Some(Value::Number(n)) => n.as_i64().is_some_and(|v| v != 0), + Some(Value::String(s)) => { + matches!(s.to_ascii_lowercase().as_str(), "yes" | "true" | "t" | "1") + } + _ => false, + } +} + +/// Group unnested index rows into `IndexDesc`, preserving first-seen order so +/// the output is stable across runs. +pub fn fold_index_rows(rows: &[Row], key_of: F) -> HashMap> +where + F: Fn(&Row) -> Option, +{ + let mut out: HashMap> = HashMap::new(); + for row in rows { + let Some(key) = key_of(row) else { continue }; + let Some(name) = str_at(row, "index_name") else { + continue; + }; + let entry = out.entry(key).or_default(); + let idx = match entry.iter_mut().find(|i| i.name == name) { + Some(existing) => existing, + None => { + entry.push(IndexDesc { + name: name.clone(), + unique: bool_at(row, "is_unique"), + primary: bool_at(row, "is_primary"), + columns: Vec::new(), + }); + entry.last_mut().expect("just pushed") + } + }; + if let Some(col) = str_at(row, "column_name") { + idx.columns.push(col); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn split_qualified_handles_bare_and_qualified_names() { + assert_eq!(split_qualified("users"), (None, "users".into())); + assert_eq!( + split_qualified("analytics.events"), + (Some("analytics".into()), "events".into()) + ); + // Degenerate halves stay whole rather than producing an empty schema. + assert_eq!(split_qualified(".events"), (None, ".events".into())); + assert_eq!(split_qualified("analytics."), (None, "analytics.".into())); + } + + #[test] + fn quote_ident_doubles_embedded_quotes() { + assert_eq!(quote_ident("users"), "\"users\""); + assert_eq!(quote_ident("we\"ird"), "\"we\"\"ird\""); + } + + #[test] + fn bool_at_normalizes_every_catalog_spelling() { + let row: Row = serde_json::from_str( + r#"{"a": true, "b": 1, "c": 0, "d": "YES", "e": "NO", "f": "t", "g": null}"#, + ) + .unwrap(); + assert!(bool_at(&row, "a")); + assert!(bool_at(&row, "b")); + assert!(!bool_at(&row, "c")); + assert!(bool_at(&row, "d")); + assert!(!bool_at(&row, "e")); + assert!(bool_at(&row, "f")); + assert!(!bool_at(&row, "g")); + assert!(!bool_at(&row, "missing")); + } + + #[test] + fn i64_at_accepts_the_string_counts_mysql_returns() { + let row: Row = + serde_json::from_str(r#"{"n": 42, "s": "1234", "f": 7.0, "x": "no"}"#).unwrap(); + assert_eq!(i64_at(&row, "n"), Some(42)); + assert_eq!(i64_at(&row, "s"), Some(1234)); + assert_eq!(i64_at(&row, "f"), Some(7)); + assert_eq!(i64_at(&row, "x"), None); + } + + #[test] + fn fold_index_rows_groups_columns_in_ordinal_order() { + let rows: Vec = serde_json::from_str( + r#"[ + {"table_name":"orders","index_name":"pk_orders","is_unique":true,"is_primary":true,"column_name":"id"}, + {"table_name":"orders","index_name":"ix_o","is_unique":false,"is_primary":false,"column_name":"user_id"}, + {"table_name":"orders","index_name":"ix_o","is_unique":false,"is_primary":false,"column_name":"created_at"} + ]"#, + ) + .unwrap(); + let folded = fold_index_rows(&rows, |r| str_at(r, "table_name").map(|t| (None, t))); + let idxs = &folded[&(None, "orders".to_string())]; + assert_eq!(idxs.len(), 2); + assert_eq!(idxs[0].name, "pk_orders"); + assert!(idxs[0].primary && idxs[0].unique); + assert_eq!(idxs[1].columns, vec!["user_id", "created_at"]); + } +} diff --git a/database/src/handlers/catalog/mysql.rs b/database/src/handlers/catalog/mysql.rs new file mode 100644 index 000000000..50e2e64bf --- /dev/null +++ b/database/src/handlers/catalog/mysql.rs @@ -0,0 +1,180 @@ +//! MySQL catalog reads. +//! +//! MySQL has no namespace above the table within a connection — its "schema" +//! *is* the database the pool is connected to — so every reader scopes to +//! `DATABASE()` and reports `schema: None`. Callers get the same shape as the +//! other drivers without a redundant level. +//! +//! `information_schema.KEY_COLUMN_USAGE` already emits one row per key column +//! with `ORDINAL_POSITION`, and `STATISTICS` one row per indexed column with +//! `SEQ_IN_INDEX`, so composite keys and multi-column indexes pair correctly +//! without any array handling or `GROUP_CONCAT`. + +use super::{ + bool_at, fold_index_rows, i64_at, run, str_at, ColumnDesc, ForeignKeyRef, IndexDesc, + TableFilter, TableKey, TableKind, TableRef, +}; +use crate::handlers::AppState; +use serde_json::Value; +use std::collections::HashMap; + +fn filter_clause(filter: Option<&TableFilter>, column: &str) -> (String, Vec) { + match filter { + Some(f) => ( + format!(" AND {column} = ?"), + vec![Value::String(f.table.clone())], + ), + None => (String::new(), Vec::new()), + } +} + +fn key_of(row: &serde_json::Map) -> Option { + Some((None, str_at(row, "table_name")?)) +} + +pub async fn list_tables( + state: &AppState, + db: &str, + timeout_ms: u64, +) -> Result, String> { + let sql = "SELECT TABLE_NAME AS table_name, TABLE_TYPE AS kind \ + FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() \ + ORDER BY TABLE_TYPE, TABLE_NAME"; + let rows = run(state, db, sql, vec![], timeout_ms).await?; + Ok(rows + .iter() + .filter_map(|r| { + Some(TableRef { + name: str_at(r, "table_name")?, + schema: None, + kind: match str_at(r, "kind").as_deref() { + Some("VIEW") => TableKind::View, + _ => TableKind::Table, + }, + }) + }) + .collect()) +} + +pub async fn columns( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result>, String> { + let (clause, params) = filter_clause(filter, "TABLE_NAME"); + let sql = format!( + "SELECT TABLE_NAME AS table_name, COLUMN_NAME AS column_name, \ + COLUMN_TYPE AS column_type, IS_NULLABLE AS is_nullable, \ + COLUMN_DEFAULT AS default_value, ORDINAL_POSITION AS position, \ + COLUMN_KEY AS column_key \ + FROM information_schema.COLUMNS \ + WHERE TABLE_SCHEMA = DATABASE(){clause} \ + ORDER BY TABLE_NAME, ORDINAL_POSITION" + ); + let rows = run(state, db, sql, params, timeout_ms).await?; + + let mut out: HashMap> = HashMap::new(); + for r in &rows { + let (Some(key), Some(name)) = (key_of(r), str_at(r, "column_name")) else { + continue; + }; + out.entry(key).or_default().push(ColumnDesc { + name, + ty: str_at(r, "column_type").unwrap_or_default(), + // IS_NULLABLE is the string 'YES'/'NO'. + nullable: bool_at(r, "is_nullable"), + default_value: str_at(r, "default_value"), + // COLUMN_KEY is 'PRI' for a primary-key member. Cheaper and more + // reliable than a second trip to KEY_COLUMN_USAGE. + primary_key: str_at(r, "column_key").as_deref() == Some("PRI"), + position: i64_at(r, "position").unwrap_or(0) as i32, + foreign_key: None, + }); + } + + merge_foreign_keys(state, db, filter, timeout_ms, &mut out).await?; + Ok(out) +} + +async fn merge_foreign_keys( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, + columns: &mut HashMap>, +) -> Result<(), String> { + let (clause, params) = filter_clause(filter, "TABLE_NAME"); + let sql = format!( + "SELECT TABLE_NAME AS table_name, COLUMN_NAME AS src_column, \ + REFERENCED_TABLE_NAME AS ref_table, REFERENCED_COLUMN_NAME AS ref_column \ + FROM information_schema.KEY_COLUMN_USAGE \ + WHERE TABLE_SCHEMA = DATABASE() AND REFERENCED_TABLE_NAME IS NOT NULL{clause} \ + ORDER BY TABLE_NAME, CONSTRAINT_NAME, ORDINAL_POSITION" + ); + for r in run(state, db, sql, params, timeout_ms).await? { + let (Some(key), Some(src), Some(ref_table), Some(ref_column)) = ( + key_of(&r), + str_at(&r, "src_column"), + str_at(&r, "ref_table"), + str_at(&r, "ref_column"), + ) else { + continue; + }; + if let Some(cols) = columns.get_mut(&key) { + if let Some(c) = cols.iter_mut().find(|c| c.name == src) { + c.foreign_key = Some(ForeignKeyRef { + schema: None, + table: ref_table, + column: ref_column, + }); + } + } + } + Ok(()) +} + +pub async fn indexes( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result>, String> { + let (clause, params) = filter_clause(filter, "TABLE_NAME"); + // NON_UNIQUE is inverted, so flip it into the `is_unique` the fold reads. + // MySQL names the primary-key index 'PRIMARY' and offers no other flag. + let sql = format!( + "SELECT TABLE_NAME AS table_name, INDEX_NAME AS index_name, \ + NON_UNIQUE = 0 AS is_unique, INDEX_NAME = 'PRIMARY' AS is_primary, \ + COLUMN_NAME AS column_name \ + FROM information_schema.STATISTICS \ + WHERE TABLE_SCHEMA = DATABASE(){clause} \ + ORDER BY TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX" + ); + let rows = run(state, db, sql, params, timeout_ms).await?; + Ok(fold_index_rows(&rows, key_of)) +} + +/// `TABLE_ROWS` is an InnoDB estimate sampled from the index, and is NULL for +/// views. Treat NULL as absent rather than zero — "unknown" and "empty" are +/// different answers. +pub async fn row_estimates( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result, String> { + let (clause, params) = filter_clause(filter, "TABLE_NAME"); + let sql = format!( + "SELECT TABLE_NAME AS table_name, TABLE_ROWS AS estimate \ + FROM information_schema.TABLES \ + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_ROWS IS NOT NULL{clause}" + ); + let mut out = HashMap::new(); + for r in run(state, db, sql, params, timeout_ms).await? { + if let (Some(key), Some(est)) = (key_of(&r), i64_at(&r, "estimate")) { + out.insert(key, est); + } + } + Ok(out) +} diff --git a/database/src/handlers/catalog/postgres.rs b/database/src/handlers/catalog/postgres.rs new file mode 100644 index 000000000..6760776bf --- /dev/null +++ b/database/src/handlers/catalog/postgres.rs @@ -0,0 +1,250 @@ +//! PostgreSQL catalog reads. +//! +//! Reads `pg_catalog` rather than `information_schema`. Two reasons: it is +//! markedly faster, and `information_schema.constraint_column_usage` pairs +//! composite foreign keys by cross product, which mis-associates columns on a +//! multi-column key. `pg_constraint` carries `conkey`/`confkey` as ordered +//! vectors, so unnesting both `WITH ORDINALITY` and matching on the ordinal +//! pairs them correctly. +//! +//! Nothing here returns an array-typed column: `RowValue` has no array +//! variant, so `array_agg`, a bare `conkey`, or `indkey` would fail to decode. +//! Arrays are unnested into scalar rows and regrouped in Rust. + +use super::{ + bool_at, fold_index_rows, i64_at, run, str_at, ColumnDesc, ForeignKeyRef, IndexDesc, + TableFilter, TableKey, TableKind, TableRef, +}; +use crate::handlers::AppState; +use serde_json::Value; +use std::collections::HashMap; + +const SKIP_SCHEMAS: &str = "n.nspname NOT IN ('pg_catalog', 'information_schema')"; + +/// Build the optional `AND schema = $n AND table = $m` tail. Postgres uses +/// positional placeholders, so the caller's first free index is passed in. +fn filter_clause(filter: Option<&TableFilter>, first_param: usize) -> (String, Vec) { + let Some(f) = filter else { + return (String::new(), Vec::new()); + }; + let mut params = Vec::new(); + let mut clause = String::new(); + let mut n = first_param; + if let Some(schema) = &f.schema { + clause.push_str(&format!(" AND n.nspname = ${n}")); + params.push(Value::String(schema.clone())); + n += 1; + } + clause.push_str(&format!(" AND c.relname = ${n}")); + params.push(Value::String(f.table.clone())); + (clause, params) +} + +fn key_of(row: &serde_json::Map) -> Option { + Some((str_at(row, "table_schema"), str_at(row, "table_name")?)) +} + +pub async fn list_tables( + state: &AppState, + db: &str, + timeout_ms: u64, +) -> Result, String> { + let sql = format!( + "SELECT n.nspname AS table_schema, c.relname AS table_name, c.relkind AS kind \ + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE c.relkind IN ('r', 'p', 'v', 'm') AND {SKIP_SCHEMAS} \ + ORDER BY n.nspname, c.relname" + ); + let rows = run(state, db, sql, vec![], timeout_ms).await?; + Ok(rows + .iter() + .filter_map(|r| { + Some(TableRef { + name: str_at(r, "table_name")?, + schema: str_at(r, "table_schema"), + // r = ordinary table, p = partitioned, v = view, m = materialized view + kind: match str_at(r, "kind").as_deref() { + Some("v") | Some("m") => TableKind::View, + _ => TableKind::Table, + }, + }) + }) + .collect()) +} + +pub async fn columns( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result>, String> { + let (clause, params) = filter_clause(filter, 1); + let sql = format!( + "SELECT n.nspname AS table_schema, c.relname AS table_name, \ + a.attname AS column_name, \ + format_type(a.atttypid, a.atttypmod) AS column_type, \ + NOT a.attnotnull AS is_nullable, \ + pg_get_expr(d.adbin, d.adrelid) AS default_value, \ + a.attnum AS position \ + FROM pg_attribute a \ + JOIN pg_class c ON c.oid = a.attrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + LEFT JOIN pg_attrdef d ON d.adrelid = c.oid AND d.adnum = a.attnum \ + WHERE a.attnum > 0 AND NOT a.attisdropped \ + AND c.relkind IN ('r', 'p', 'v', 'm') AND {SKIP_SCHEMAS}{clause} \ + ORDER BY n.nspname, c.relname, a.attnum" + ); + let rows = run(state, db, sql, params, timeout_ms).await?; + + let mut out: HashMap> = HashMap::new(); + for r in &rows { + let (Some(key), Some(name)) = (key_of(r), str_at(r, "column_name")) else { + continue; + }; + out.entry(key).or_default().push(ColumnDesc { + name, + ty: str_at(r, "column_type").unwrap_or_default(), + nullable: bool_at(r, "is_nullable"), + default_value: str_at(r, "default_value"), + primary_key: false, + position: i64_at(r, "position").unwrap_or(0) as i32, + foreign_key: None, + }); + } + + mark_primary_keys(state, db, filter, timeout_ms, &mut out).await?; + merge_foreign_keys(state, db, filter, timeout_ms, &mut out).await?; + Ok(out) +} + +async fn mark_primary_keys( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, + columns: &mut HashMap>, +) -> Result<(), String> { + let (clause, params) = filter_clause(filter, 1); + let sql = format!( + "SELECT n.nspname AS table_schema, c.relname AS table_name, a.attname AS column_name \ + FROM pg_constraint con \ + JOIN pg_class c ON c.oid = con.conrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + CROSS JOIN LATERAL unnest(con.conkey) AS k(attnum) \ + JOIN pg_attribute a ON a.attrelid = con.conrelid AND a.attnum = k.attnum \ + WHERE con.contype = 'p' AND {SKIP_SCHEMAS}{clause}" + ); + for r in run(state, db, sql, params, timeout_ms).await? { + let (Some(key), Some(col)) = (key_of(&r), str_at(&r, "column_name")) else { + continue; + }; + if let Some(cols) = columns.get_mut(&key) { + if let Some(c) = cols.iter_mut().find(|c| c.name == col) { + c.primary_key = true; + } + } + } + Ok(()) +} + +async fn merge_foreign_keys( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, + columns: &mut HashMap>, +) -> Result<(), String> { + let (clause, params) = filter_clause(filter, 1); + // `k.ord = fk.ord` is what makes a composite key pair correctly — without + // it the two unnests cross-product and column 1 can be reported as + // referencing the parent's column 2. + let sql = format!( + "SELECT n.nspname AS table_schema, c.relname AS table_name, \ + a.attname AS src_column, \ + fn.nspname AS ref_schema, fc.relname AS ref_table, fa.attname AS ref_column \ + FROM pg_constraint con \ + JOIN pg_class c ON c.oid = con.conrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + JOIN pg_class fc ON fc.oid = con.confrelid \ + JOIN pg_namespace fn ON fn.oid = fc.relnamespace \ + CROSS JOIN LATERAL unnest(con.conkey) WITH ORDINALITY AS k(attnum, ord) \ + CROSS JOIN LATERAL unnest(con.confkey) WITH ORDINALITY AS fk(attnum, ord) \ + JOIN pg_attribute a ON a.attrelid = con.conrelid AND a.attnum = k.attnum \ + JOIN pg_attribute fa ON fa.attrelid = con.confrelid AND fa.attnum = fk.attnum \ + WHERE con.contype = 'f' AND k.ord = fk.ord AND {SKIP_SCHEMAS}{clause}" + ); + for r in run(state, db, sql, params, timeout_ms).await? { + let (Some(key), Some(src), Some(ref_table), Some(ref_column)) = ( + key_of(&r), + str_at(&r, "src_column"), + str_at(&r, "ref_table"), + str_at(&r, "ref_column"), + ) else { + continue; + }; + if let Some(cols) = columns.get_mut(&key) { + if let Some(c) = cols.iter_mut().find(|c| c.name == src) { + c.foreign_key = Some(ForeignKeyRef { + schema: str_at(&r, "ref_schema"), + table: ref_table, + column: ref_column, + }); + } + } + } + Ok(()) +} + +pub async fn indexes( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result>, String> { + let (clause, params) = filter_clause(filter, 1); + // `indkey` is an int2vector; unnest it rather than returning it, and keep + // the LEFT JOIN so an expression index still yields its row with no + // column name attached. + let sql = format!( + "SELECT n.nspname AS table_schema, c.relname AS table_name, \ + i.relname AS index_name, ix.indisunique AS is_unique, \ + ix.indisprimary AS is_primary, a.attname AS column_name \ + FROM pg_index ix \ + JOIN pg_class c ON c.oid = ix.indrelid \ + JOIN pg_class i ON i.oid = ix.indexrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + CROSS JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS k(attnum, ord) \ + LEFT JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = k.attnum \ + WHERE {SKIP_SCHEMAS}{clause} \ + ORDER BY n.nspname, c.relname, i.relname, k.ord" + ); + let rows = run(state, db, sql, params, timeout_ms).await?; + Ok(fold_index_rows(&rows, key_of)) +} + +/// `reltuples` is the planner's estimate, maintained by ANALYZE/autovacuum. +/// It is -1 on a table that has never been analyzed, which we report as +/// absent rather than as a row count of minus one. +pub async fn row_estimates( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result, String> { + let (clause, params) = filter_clause(filter, 1); + let sql = format!( + "SELECT n.nspname AS table_schema, c.relname AS table_name, \ + c.reltuples::bigint AS estimate \ + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE c.relkind IN ('r', 'p') AND {SKIP_SCHEMAS}{clause}" + ); + let mut out = HashMap::new(); + for r in run(state, db, sql, params, timeout_ms).await? { + if let (Some(key), Some(est)) = (key_of(&r), i64_at(&r, "estimate")) { + if est >= 0 { + out.insert(key, est); + } + } + } + Ok(out) +} diff --git a/database/src/handlers/catalog/sqlite.rs b/database/src/handlers/catalog/sqlite.rs new file mode 100644 index 000000000..2e6c5a3d5 --- /dev/null +++ b/database/src/handlers/catalog/sqlite.rs @@ -0,0 +1,232 @@ +//! SQLite catalog reads. +//! +//! Uses the table-valued PRAGMA functions (`pragma_table_info(name)`, SQLite +//! 3.16+) joined against `sqlite_master` rather than issuing a bare `PRAGMA` +//! per table. That keeps `describeSchema` to three queries instead of one per +//! table, and it is the only way to read the catalog with bound parameters — +//! a bare `PRAGMA table_info(x)` cannot bind `x`. + +use super::{ + bool_at, fold_index_rows, i64_at, run, str_at, ColumnDesc, ForeignKeyRef, IndexDesc, + TableFilter, TableKey, TableKind, TableRef, +}; +use crate::handlers::AppState; +use serde_json::Value; +use std::collections::HashMap; + +/// `sqlite_master` rows we never surface: the internal bookkeeping tables. +const NOT_INTERNAL: &str = "m.name NOT LIKE 'sqlite\\_%' ESCAPE '\\'"; + +fn filter_clause(filter: Option<&TableFilter>) -> (String, Vec) { + match filter { + Some(f) => ( + " AND m.name = ?".to_string(), + vec![Value::String(f.table.clone())], + ), + None => (String::new(), Vec::new()), + } +} + +pub async fn list_tables( + state: &AppState, + db: &str, + timeout_ms: u64, +) -> Result, String> { + let sql = format!( + "SELECT m.name AS name, m.type AS kind FROM sqlite_master m \ + WHERE m.type IN ('table', 'view') AND {NOT_INTERNAL} ORDER BY m.type, m.name" + ); + let rows = run(state, db, sql, vec![], timeout_ms).await?; + Ok(rows + .iter() + .filter_map(|r| { + Some(TableRef { + name: str_at(r, "name")?, + schema: None, + kind: match str_at(r, "kind").as_deref() { + Some("view") => TableKind::View, + _ => TableKind::Table, + }, + }) + }) + .collect()) +} + +/// Columns for one table or every table, with primary keys and foreign keys +/// already merged in. +pub async fn columns( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result>, String> { + let (clause, params) = filter_clause(filter); + let sql = format!( + "SELECT m.name AS table_name, p.cid AS cid, p.name AS column_name, \ + p.type AS column_type, p.\"notnull\" AS not_null, \ + p.dflt_value AS default_value, p.pk AS pk_ord \ + FROM sqlite_master m JOIN pragma_table_info(m.name) p \ + WHERE m.type IN ('table', 'view') AND {NOT_INTERNAL}{clause} \ + ORDER BY m.name, p.cid" + ); + let rows = run(state, db, sql, params, timeout_ms).await?; + + let mut out: HashMap> = HashMap::new(); + for r in &rows { + let Some(table) = str_at(r, "table_name") else { + continue; + }; + let Some(name) = str_at(r, "column_name") else { + continue; + }; + out.entry((None, table)).or_default().push(ColumnDesc { + name, + ty: str_at(r, "column_type").unwrap_or_default(), + nullable: !bool_at(r, "not_null"), + default_value: str_at(r, "default_value"), + // `pk` is the 1-based position within the primary key, 0 when the + // column is not part of it — not a boolean. + primary_key: i64_at(r, "pk_ord").unwrap_or(0) > 0, + position: i64_at(r, "cid").unwrap_or(0) as i32 + 1, + foreign_key: None, + }); + } + + merge_foreign_keys(state, db, filter, timeout_ms, &mut out).await?; + Ok(out) +} + +/// SQLite reports a foreign key's target column as NULL when the reference is +/// to the parent's primary key implicitly (`REFERENCES users` rather than +/// `REFERENCES users(id)`). Resolve that against the columns we already hold +/// rather than emitting a half-empty reference. +async fn merge_foreign_keys( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, + columns: &mut HashMap>, +) -> Result<(), String> { + let (clause, params) = filter_clause(filter); + let sql = format!( + "SELECT m.name AS table_name, f.\"from\" AS src_column, \ + f.\"table\" AS ref_table, f.\"to\" AS ref_column \ + FROM sqlite_master m JOIN pragma_foreign_key_list(m.name) f \ + WHERE m.type = 'table' AND {NOT_INTERNAL}{clause}" + ); + let rows = run(state, db, sql, params, timeout_ms).await?; + + // Primary keys of tables already loaded. Free, but only covers parents + // inside the current filter — describing one table does not load its + // parents, which is exactly when implicit references need resolving. + let mut pk_of: HashMap = columns + .iter() + .filter_map(|((_, t), cols)| { + cols.iter() + .find(|c| c.primary_key) + .map(|c| (t.clone(), c.name.clone())) + }) + .collect(); + + // Look up any parent we are missing in one extra query rather than + // dropping the reference. Dropping it would mean `describeTable` silently + // reports no foreign key on a column that plainly has one. + let unresolved: Vec = rows + .iter() + .filter(|r| str_at(r, "ref_column").is_none()) + .filter_map(|r| str_at(r, "ref_table")) + .filter(|t| !pk_of.contains_key(t)) + .collect::>() + .into_iter() + .collect(); + if !unresolved.is_empty() { + let placeholders = vec!["?"; unresolved.len()].join(", "); + let sql = format!( + "SELECT m.name AS table_name, p.name AS column_name \ + FROM sqlite_master m JOIN pragma_table_info(m.name) p \ + WHERE m.type = 'table' AND p.pk > 0 AND m.name IN ({placeholders}) \ + ORDER BY m.name, p.pk" + ); + let params = unresolved.iter().cloned().map(Value::String).collect(); + for r in run(state, db, sql, params, timeout_ms).await? { + let (Some(t), Some(c)) = (str_at(&r, "table_name"), str_at(&r, "column_name")) else { + continue; + }; + // Ordered by `p.pk`, so the first row per table is the leading + // primary-key column — the one an implicit reference means. + pk_of.entry(t).or_insert(c); + } + } + + for r in &rows { + let (Some(table), Some(src), Some(ref_table)) = ( + str_at(r, "table_name"), + str_at(r, "src_column"), + str_at(r, "ref_table"), + ) else { + continue; + }; + let ref_column = match str_at(r, "ref_column") { + Some(c) => c, + // Implicit `REFERENCES parent` means the parent's primary key. + // Unresolvable only if the parent no longer exists. + None => match pk_of.get(&ref_table) { + Some(pk) => pk.clone(), + None => continue, + }, + }; + if let Some(cols) = columns.get_mut(&(None, table)) { + if let Some(col) = cols.iter_mut().find(|c| c.name == src) { + col.foreign_key = Some(ForeignKeyRef { + schema: None, + table: ref_table, + column: ref_column, + }); + } + } + } + Ok(()) +} + +pub async fn indexes( + state: &AppState, + db: &str, + filter: Option<&TableFilter>, + timeout_ms: u64, +) -> Result>, String> { + let (clause, params) = filter_clause(filter); + // `origin` is 'pk' for the implicit primary-key index, 'u' for a UNIQUE + // constraint, 'c' for CREATE INDEX. + let sql = format!( + "SELECT m.name AS table_name, il.name AS index_name, \ + il.\"unique\" AS is_unique, il.origin AS origin, \ + ii.name AS column_name \ + FROM sqlite_master m \ + JOIN pragma_index_list(m.name) il \ + LEFT JOIN pragma_index_info(il.name) ii \ + WHERE m.type = 'table' AND {NOT_INTERNAL}{clause} \ + ORDER BY m.name, il.seq, ii.seqno" + ); + let mut rows = run(state, db, sql, params, timeout_ms).await?; + // `fold_index_rows` reads `is_primary`; sqlite spells that as origin='pk'. + // Normalize in place so the fold stays a single pass. + for r in rows.iter_mut() { + let primary = str_at(r, "origin").as_deref() == Some("pk"); + r.insert("is_primary".to_string(), Value::Bool(primary)); + } + Ok(fold_index_rows(&rows, |r| { + str_at(r, "table_name").map(|t| (None, t)) + })) +} + +/// SQLite has no cheap row estimate — `sqlite_stat1` exists only after an +/// explicit `ANALYZE`, and `COUNT(*)` is a full scan. Report nothing rather +/// than pay for a scan the caller did not ask for. +pub async fn row_estimates( + _state: &AppState, + _db: &str, + _filter: Option<&TableFilter>, + _timeout_ms: u64, +) -> Result, String> { + Ok(HashMap::new()) +} diff --git a/database/src/handlers/column_stats.rs b/database/src/handlers/column_stats.rs new file mode 100644 index 000000000..059355046 --- /dev/null +++ b/database/src/handlers/column_stats.rs @@ -0,0 +1,427 @@ +//! `database::columnStats` — profile a column without reading the table. +//! +//! Two modes, and the default matters. A naive profile runs +//! `COUNT(DISTINCT col)`, `MIN`, `MAX` and a `GROUP BY`, each of which is a +//! full scan; doing that from a console panel is how a read-only viewer +//! causes a production incident. So the default reads the statistics the +//! planner already maintains (`pg_stats`, `information_schema.STATISTICS`, +//! `sqlite_stat1`) — O(1) catalog reads that cost nothing — and `exact: true` +//! is an explicit opt-in that runs the real aggregates behind a timeout. +//! +//! Approximate numbers are always labelled `source: planner`, never presented +//! as if they were counted. +//! +//! Scope note: this profiles the *whole table* server-side. To profile rows +//! you already hold, pipe a `browseTable` result through the `fp` worker +//! instead — that is what it is for, and duplicating it here would be worse +//! on both counts. + +use super::filter::{quote_ident, quote_table}; +use super::query::{self, err_to_str, QueryReq}; +use super::AppState; +use crate::config::DriverKind; +use crate::error::DbError; +use crate::pool::Pool; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +fn default_timeout() -> u64 { + 30_000 +} + +fn default_top_n() -> usize { + 10 +} + +/// Cap on `top_n` — a "most common values" list longer than this is a report, +/// not a profile. +const MAX_TOP_N: usize = 100; + +/// Above this planner-estimated row count, `exact: true` is refused rather +/// than silently scanning a very large table. +const EXACT_ROW_CEILING: i64 = 5_000_000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum StatSource { + /// Read from the planner's own statistics. Approximate, and free. + Planner, + /// Counted by running aggregates over the table. + Computed, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct TopValue { + pub value: Value, + pub count: i64, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct ColumnStat { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub row_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub distinct_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub null_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub null_fraction: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub min: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mean: Option, + /// Only populated in `exact` mode; the planner's own most-common-value + /// lists are not portable enough to report faithfully. + pub top_values: Vec, + pub source: StatSource, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ColumnStatsReq { + #[serde(default)] + pub db: Option, + pub table: String, + #[serde(default)] + pub schema: Option, + /// Omit to profile every column. + #[serde(default)] + pub columns: Option>, + /// Run real aggregates instead of reading planner statistics. This scans + /// the table. + #[serde(default)] + pub exact: bool, + #[serde(default = "default_top_n")] + pub top_n: usize, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ColumnStatsResp { + pub table: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option, + pub columns: Vec, + /// True when the numbers came from the planner rather than a count. + pub approximate: bool, +} + +async fn driver_of(state: &AppState, db: &str) -> Result { + Ok(match state.pool(db).await.map_err(err_to_str)? { + Pool::Sqlite(_) => DriverKind::Sqlite, + Pool::Postgres(_) => DriverKind::Postgres, + Pool::Mysql(_) => DriverKind::Mysql, + }) +} + +async fn run( + state: &AppState, + db: &str, + sql: String, + params: Vec, + timeout_ms: u64, +) -> Result>, String> { + Ok(query::handle( + state, + QueryReq { + db: Some(db.to_string()), + sql, + params, + timeout_ms, + record_history: false, + }, + ) + .await? + .rows) +} + +fn f64_at(row: &serde_json::Map, key: &str) -> Option { + match row.get(key) { + Some(Value::Number(n)) => n.as_f64(), + Some(Value::String(s)) => s.parse().ok(), + _ => None, + } +} + +fn i64_at(row: &serde_json::Map, key: &str) -> Option { + f64_at(row, key).map(|f| f as i64) +} + +pub async fn handle(state: &AppState, req: ColumnStatsReq) -> Result { + if req.table.trim().is_empty() { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: "table is required".into(), + })); + } + let db = state.resolve_db(req.db.clone()).await.map_err(err_to_str)?; + let driver = driver_of(state, &db).await?; + let top_n = req.top_n.clamp(1, MAX_TOP_N); + + // Resolve the column list from the catalog rather than trusting the + // caller, so an unknown name fails here instead of inside an aggregate. + let described = super::schema::describe_table( + state, + super::schema::DescribeTableReq { + db: Some(db.clone()), + table: req.table.clone(), + schema: req.schema.clone(), + timeout_ms: req.timeout_ms, + }, + ) + .await?; + + let wanted: Vec = match &req.columns { + Some(list) => { + for c in list { + if !described.columns.iter().any(|d| &d.name == c) { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: format!("no such column `{c}` on `{}`", req.table), + })); + } + } + list.clone() + } + None => described.columns.iter().map(|c| c.name.clone()).collect(), + }; + + let target = quote_table(driver, described.schema.as_deref(), &described.table); + + if !req.exact { + let columns = + planner_stats(state, &db, driver, &described, &wanted, req.timeout_ms).await?; + return Ok(ColumnStatsResp { + table: described.table, + schema: described.schema, + columns, + approximate: true, + }); + } + + // Refuse an exact profile of a very large table rather than starting a + // scan the caller cannot cancel — on sqlite the driver drops `timeout_ms` + // entirely, so a row-count guard is the only brake available. + if let Some(est) = described.row_count_estimate { + if est > EXACT_ROW_CEILING { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: format!( + "`{}` has roughly {est} rows; an exact profile would scan all of \ + them. Re-run without `exact` for planner statistics, or profile \ + a filtered subset.", + described.table + ), + })); + } + } + + let mut columns = Vec::new(); + for name in &wanted { + columns.push(exact_stats(state, &db, driver, &target, name, top_n, req.timeout_ms).await?); + } + Ok(ColumnStatsResp { + table: described.table, + schema: described.schema, + columns, + approximate: false, + }) +} + +/// Read what the planner already knows. Cheap, approximate, and clearly +/// labelled as such. +async fn planner_stats( + state: &AppState, + db: &str, + driver: DriverKind, + described: &super::schema::TableDescription, + wanted: &[String], + timeout_ms: u64, +) -> Result, String> { + let row_count = described.row_count_estimate; + let mut out: Vec = wanted + .iter() + .map(|name| ColumnStat { + name: name.clone(), + row_count, + distinct_count: None, + null_count: None, + null_fraction: None, + min: None, + max: None, + mean: None, + top_values: Vec::new(), + source: StatSource::Planner, + }) + .collect(); + + match driver { + DriverKind::Postgres => { + // `pg_stats` exposes null_frac and n_distinct as plain scalars. + // n_distinct is negative when it is a ratio of the row count. + let rows = run( + state, + db, + "SELECT attname AS column_name, null_frac, n_distinct \ + FROM pg_stats WHERE schemaname = COALESCE($1, 'public') AND tablename = $2" + .into(), + vec![ + described + .schema + .clone() + .map(Value::String) + .unwrap_or(Value::Null), + Value::String(described.table.clone()), + ], + timeout_ms, + ) + .await?; + for r in &rows { + let Some(col) = r.get("column_name").and_then(Value::as_str) else { + continue; + }; + let Some(stat) = out.iter_mut().find(|s| s.name == col) else { + continue; + }; + stat.null_fraction = f64_at(r, "null_frac"); + if let (Some(frac), Some(total)) = (stat.null_fraction, row_count) { + stat.null_count = Some((frac * total as f64).round() as i64); + } + stat.distinct_count = f64_at(r, "n_distinct").map(|n| { + if n < 0.0 { + // Negative means "this fraction of the row count". + (-n * row_count.unwrap_or(0) as f64).round() as i64 + } else { + n as i64 + } + }); + } + } + DriverKind::Mysql => { + // CARDINALITY is per leading index column, so it only answers for + // indexed columns — which is honest: unindexed columns get None. + let rows = run( + state, + db, + "SELECT COLUMN_NAME AS column_name, MAX(CARDINALITY) AS cardinality \ + FROM information_schema.STATISTICS \ + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND SEQ_IN_INDEX = 1 \ + GROUP BY COLUMN_NAME" + .into(), + vec![Value::String(described.table.clone())], + timeout_ms, + ) + .await?; + for r in &rows { + let Some(col) = r.get("column_name").and_then(Value::as_str) else { + continue; + }; + if let Some(stat) = out.iter_mut().find(|s| s.name == col) { + stat.distinct_count = i64_at(r, "cardinality"); + } + } + } + DriverKind::Sqlite => { + // `sqlite_stat1` exists only after an explicit ANALYZE. Its `stat` + // column is " ..." per index column. + let rows = run( + state, + db, + "SELECT tbl, idx, stat FROM sqlite_stat1 WHERE tbl = ?".into(), + vec![Value::String(described.table.clone())], + timeout_ms, + ) + .await + // The table does not exist until someone runs ANALYZE; that is a + // missing answer, not an error. + .unwrap_or_default(); + if let Some(total) = rows + .first() + .and_then(|r| r.get("stat")) + .and_then(Value::as_str) + .and_then(|s| s.split_whitespace().next()) + .and_then(|s| s.parse::().ok()) + { + for stat in out.iter_mut() { + stat.row_count = Some(total); + } + } + } + } + Ok(out) +} + +/// Real aggregates. One pass for the scalars, one for the top values. +async fn exact_stats( + state: &AppState, + db: &str, + driver: DriverKind, + target: &str, + column: &str, + top_n: usize, + timeout_ms: u64, +) -> Result { + let col = quote_ident(driver, column); + let scalars = run( + state, + db, + format!( + "SELECT COUNT(*) AS row_count, COUNT({col}) AS non_null, \ + COUNT(DISTINCT {col}) AS distinct_count, \ + MIN({col}) AS min_value, MAX({col}) AS max_value, \ + AVG(CASE WHEN {col} + 0 = {col} THEN {col} END) AS mean_value \ + FROM {target}" + ), + vec![], + timeout_ms, + ) + .await?; + + let row = scalars.first().cloned().unwrap_or_default(); + let row_count = i64_at(&row, "row_count"); + let non_null = i64_at(&row, "non_null"); + let null_count = match (row_count, non_null) { + (Some(t), Some(n)) => Some(t - n), + _ => None, + }; + + let tops = run( + state, + db, + format!( + "SELECT {col} AS value, COUNT(*) AS n FROM {target} \ + WHERE {col} IS NOT NULL GROUP BY {col} ORDER BY n DESC, 1 LIMIT {top_n}" + ), + vec![], + timeout_ms, + ) + .await?; + + Ok(ColumnStat { + name: column.to_string(), + row_count, + distinct_count: i64_at(&row, "distinct_count"), + null_count, + null_fraction: match (null_count, row_count) { + (Some(n), Some(t)) if t > 0 => Some(n as f64 / t as f64), + _ => None, + }, + min: row.get("min_value").cloned().filter(|v| !v.is_null()), + max: row.get("max_value").cloned().filter(|v| !v.is_null()), + mean: f64_at(&row, "mean_value"), + top_values: tops + .iter() + .filter_map(|r| { + Some(TopValue { + value: r.get("value")?.clone(), + count: i64_at(r, "n")?, + }) + }) + .collect(), + source: StatSource::Computed, + }) +} diff --git a/database/src/handlers/diagram.rs b/database/src/handlers/diagram.rs new file mode 100644 index 000000000..449af5a23 --- /dev/null +++ b/database/src/handlers/diagram.rs @@ -0,0 +1,1290 @@ +//! `database::schemaDiagram` — the shape of a schema, laid out. +//! +//! Layout runs here rather than in a renderer for two reasons. It collapses +//! the N+1 a diagram would otherwise need (`describeSchema` already reads the +//! whole catalog in a handful of queries), and it makes the schema's *shape* +//! askable: an agent can call this and reason about hub tables, orphans and +//! reference cycles without drawing anything. +//! +//! The algorithm is a cut-down Sugiyama: +//! +//! 1. Split into connected components over foreign keys, so a hundred +//! unrelated lookup tables do not dominate the canvas. +//! 2. Rank each component by longest path along the FK direction, breaking +//! cycles at the lowest-degree edge. +//! 3. Reduce edge crossings with a barycenter sweep followed by adjacent +//! transposition, keeping whichever ordering measures better. +//! 4. Assign coordinates and route edges as three-segment elbows anchored on +//! the *column row*, not the box. +//! +//! Every step sorts by name before iterating, so the same schema always +//! produces the same diagram — which is what makes `crossings` a number worth +//! asserting on in a test. + +use super::schema::{self, DescribeSchemaReq, TableDescription}; +use super::AppState; +use crate::error::DbError; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +fn default_timeout() -> u64 { + 30_000 +} + +fn default_max_tables() -> usize { + 200 +} + +/// Geometry. Fixed here so the renderer never has to guess, and so edge +/// anchors line up with the row a column actually occupies. +pub const NODE_WIDTH: f64 = 264.0; +pub const HEADER_HEIGHT: f64 = 34.0; +pub const ROW_HEIGHT: f64 = 24.0; +pub const NODE_PAD_Y: f64 = 8.0; +/// Columns beyond this are summarised rather than drawn. +pub const MAX_ROWS: usize = 12; +/// Rank pitch, measured centre-of-column to centre-of-column, so the readable +/// gutter is `RANK_SEP - NODE_WIDTH`. At 220 that gutter was 20px: every edge +/// collapsed into a stub too short to read, and a schema with real foreign +/// keys looked unrelated. The gutter has to dominate the elbow's corner radius +/// for the connection to register at all. +const RANK_SEP: f64 = 380.0; +const V_GAP: f64 = 56.0; +const COMPONENT_GAP: f64 = 140.0; +/// Pitch between unrelated tables. Tighter than `COMPONENT_GAP` because no +/// edge is ever drawn between them. +const ISOLATED_GAP: f64 = 40.0; +/// Vertical break between the connected diagram and the orphan shelf. +const SHELF_GAP: f64 = 80.0; +const SHELF_WIDTH: f64 = 1600.0; + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct DiagramColumn { + pub name: String, + #[serde(rename = "type")] + pub ty: String, + pub primary_key: bool, + pub foreign_key: bool, + pub nullable: bool, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct DiagramNode { + pub table: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option, + pub x: f64, + pub y: f64, + pub w: f64, + pub h: f64, + pub rank: u32, + /// Number of foreign keys touching this table, in or out. Renderers use + /// it to emphasise hubs. + pub degree: u32, + pub columns: Vec, + /// Columns not drawn because of `MAX_ROWS`. + pub hidden_columns: usize, +} + +#[derive(Debug, Clone, Copy, Serialize, JsonSchema)] +pub struct Point { + pub x: f64, + pub y: f64, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct DiagramEdge { + pub from: String, + pub from_column: String, + pub to: String, + pub to_column: String, + /// Polyline, already routed. Anchored on the column row where the column + /// is visible, on the node edge otherwise. + pub points: Vec, + pub self_loop: bool, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct SchemaDiagramReq { + #[serde(default)] + pub db: Option, + #[serde(default)] + pub include_views: bool, + #[serde(default = "default_max_tables")] + pub max_tables: usize, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, + /// Lay out only the neighbourhood of this table. + /// + /// A whole schema drawn at once answers "what exists"; it does not answer + /// "what does this table touch", which is the question actually being + /// asked most of the time. With a focus the diagram becomes explorable + /// one hop at a time instead of a wall to be scanned. + #[serde(default)] + pub focus: Option, + /// How many foreign-key hops out from `focus` to include. Ignored without + /// one. 1 is the table and its direct relations. + #[serde(default = "default_depth")] + pub depth: usize, +} + +fn default_depth() -> usize { + 1 +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct SchemaDiagramResp { + pub nodes: Vec, + pub edges: Vec, + pub width: f64, + pub height: f64, + /// Tables with no foreign keys at all, placed on a trailing shelf. + pub isolated: Vec, + /// Connected groups, with the box that encloses each. A schema is usually + /// several independent clusters rather than one graph, and saying so is + /// most of what makes a large diagram readable — a reader can take in + /// "four unrelated groups" at a glance instead of scanning for edges that + /// are not there. + pub components: Vec, + /// Edge crossings remaining after ordering. Lower is a tidier diagram. + pub crossings: u32, + pub truncated: bool, + /// Echoed when the caller asked for a neighbourhood rather than the whole + /// schema. + #[serde(skip_serializing_if = "Option::is_none")] + pub focus: Option, + /// Tables one hop beyond what was drawn. Non-empty means there is more to + /// expand into, which is the difference between a diagram that looks + /// complete and one that says where it stops. + #[serde(default)] + pub frontier: Vec, +} + +/// One connected group of tables and its bounding box. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct DiagramComponent { + /// Stable index, in layout order. + pub index: usize, + /// Member tables, by node id. + pub tables: Vec, + /// The most-referenced table in the group, if it has more than one member. + pub hub: Option, + pub x: f64, + pub y: f64, + pub w: f64, + pub h: f64, +} + +/// A relationship between two tables, keyed by the qualified names used as +/// node ids. +#[derive(Debug, Clone)] +struct Relation { + from: String, + from_column: String, + to: String, + to_column: String, +} + +fn qualified(schema: Option<&str>, table: &str) -> String { + match schema { + Some(s) => format!("{s}.{table}"), + None => table.to_string(), + } +} + +fn node_height(columns: usize) -> f64 { + HEADER_HEIGHT + (columns.min(MAX_ROWS) as f64) * ROW_HEIGHT + 2.0 * NODE_PAD_Y +} + +pub async fn handle(state: &AppState, req: SchemaDiagramReq) -> Result { + let described = schema::describe_schema( + state, + DescribeSchemaReq { + db: req.db, + tables: None, + include_indexes: false, + max_tables: req.max_tables, + timeout_ms: req.timeout_ms, + }, + ) + .await?; + + let tables: Vec = described + .tables + .into_iter() + .filter(|t| req.include_views || t.kind == super::catalog::TableKind::Table) + .collect(); + + let (tables, frontier) = match &req.focus { + Some(focus) => neighbourhood(&tables, focus, req.depth)?, + None => (tables, Vec::new()), + }; + + let mut resp = layout(&tables, described.truncated); + resp.focus = req.focus; + resp.frontier = frontier; + Ok(resp) +} + +/// The tables within `depth` foreign-key hops of `focus`, plus the names one +/// hop beyond that were not included. +/// +/// Edges are walked in both directions: a reader asking what `users` touches +/// means both the tables it references and the tables referencing it, and +/// following only the declared direction would hide every child table. +/// +/// The `frontier` is what makes this explorable rather than merely smaller — +/// it tells the renderer which nodes have more behind them, so expanding is an +/// informed click instead of a guess. +fn neighbourhood( + tables: &[TableDescription], + focus: &str, + depth: usize, +) -> Result<(Vec, Vec), String> { + let ids: Vec = tables + .iter() + .map(|t| qualified(t.schema.as_deref(), &t.table)) + .collect(); + + // Accept either the qualified id or the bare table name, since a caller + // clicking a node has the former and a caller typing has the latter. + let start = ids + .iter() + .find(|id| id.as_str() == focus) + .or_else(|| { + tables + .iter() + .zip(&ids) + .find(|(t, _)| t.table == focus) + .map(|(_, id)| id) + }) + .cloned(); + let Some(start) = start else { + return Err(super::query::err_to_str(DbError::InvalidParam { + index: 0, + reason: format!("no table named `{focus}` in this database"), + })); + }; + + // Undirected adjacency over foreign keys. + let present: BTreeSet<&String> = ids.iter().collect(); + let mut adj: BTreeMap> = BTreeMap::new(); + for t in tables { + let from = qualified(t.schema.as_deref(), &t.table); + for c in &t.columns { + let Some(fk) = &c.foreign_key else { continue }; + let to = qualified(fk.schema.as_deref().or(t.schema.as_deref()), &fk.table); + if !present.contains(&to) || to == from { + continue; + } + adj.entry(from.clone()).or_default().insert(to.clone()); + adj.entry(to).or_default().insert(from.clone()); + } + } + + // Breadth-first to `depth`, then one more ring to find the frontier. + let mut included: BTreeSet = BTreeSet::new(); + included.insert(start.clone()); + let mut ring: Vec = vec![start]; + for _ in 0..depth { + // Stop once the component is exhausted. `depth` comes straight off the + // wire with no ceiling, so without this a caller asking for depth + // 1e11 spins the worker thread long after there is nothing left to + // reach. + if ring.is_empty() { + break; + } + let mut next: Vec = Vec::new(); + for node in &ring { + for peer in adj.get(node).into_iter().flatten() { + if included.insert(peer.clone()) { + next.push(peer.clone()); + } + } + } + ring = next; + } + + let frontier: Vec = ring + .iter() + .flat_map(|node| adj.get(node).into_iter().flatten()) + .filter(|peer| !included.contains(*peer)) + .cloned() + .collect::>() + .into_iter() + .collect(); + + let kept: Vec = tables + .iter() + .zip(&ids) + .filter(|(_, id)| included.contains(*id)) + .map(|(t, _)| t.clone()) + .collect(); + + Ok((kept, frontier)) +} + +/// Pure layout. Separated from the fetch so it can be tested without a +/// database, and so the same input always yields the same diagram. +pub fn layout(tables: &[TableDescription], truncated: bool) -> SchemaDiagramResp { + // Node ids, sorted so every later iteration is deterministic. + let mut ids: Vec = tables + .iter() + .map(|t| qualified(t.schema.as_deref(), &t.table)) + .collect(); + ids.sort(); + let present: BTreeSet<&String> = ids.iter().collect(); + + // Relations, deduplicated. A reference to a table outside the set (a view + // we filtered out, or a truncated tail) is dropped rather than drawn to + // nowhere. + let mut relations: Vec = Vec::new(); + let mut seen: BTreeSet<(String, String, String, String)> = BTreeSet::new(); + for t in tables { + let from = qualified(t.schema.as_deref(), &t.table); + for c in &t.columns { + let Some(fk) = &c.foreign_key else { continue }; + let to = qualified(fk.schema.as_deref().or(t.schema.as_deref()), &fk.table); + if !present.contains(&to) { + continue; + } + let key = (from.clone(), c.name.clone(), to.clone(), fk.column.clone()); + if seen.insert(key) { + relations.push(Relation { + from: from.clone(), + from_column: c.name.clone(), + to, + to_column: fk.column.clone(), + }); + } + } + } + + let mut degree: BTreeMap = ids.iter().map(|i| (i.clone(), 0)).collect(); + for r in &relations { + *degree.get_mut(&r.from).expect("id present") += 1; + if r.to != r.from { + *degree.get_mut(&r.to).expect("id present") += 1; + } + } + + // Degree-0 tables go to a trailing shelf; on a real schema they are often + // the majority and would otherwise stretch the canvas around nothing. + let isolated: Vec = ids.iter().filter(|i| degree[*i] == 0).cloned().collect(); + let connected: Vec = ids.iter().filter(|i| degree[*i] > 0).cloned().collect(); + + let components = split_components(&connected, &relations); + let ranks = rank_all(&components, &relations, °ree); + + // Order within each rank, then place. + let by_id: HashMap<&String, &TableDescription> = tables + .iter() + .map(|t| { + let id = ids + .iter() + .find(|i| **i == qualified(t.schema.as_deref(), &t.table)) + .expect("id built from this table"); + (id, t) + }) + .collect(); + + let mut positions: BTreeMap = BTreeMap::new(); + let mut cursor_x = 0.0_f64; + let mut max_height = 0.0_f64; + // Origin of the row of components being filled, and the tallest component + // in it. + let mut row_y = 0.0_f64; + let mut row_height = 0.0_f64; + + for comp in &components { + let order = order_component(comp, &ranks, &relations); + // Wrap *before* placing, and lay the component out at the current row + // origin. Placing at y=0 unconditionally and only recording the wrap + // afterwards drew every row after the first on top of the one above + // it, so a schema with enough components to wrap rendered stacked. + if cursor_x > 0.0 && cursor_x + NODE_WIDTH > SHELF_WIDTH { + cursor_x = 0.0; + row_y += row_height + COMPONENT_GAP; + row_height = 0.0; + } + let (w, h) = place(&order, &ranks, &by_id, cursor_x, row_y, &mut positions); + cursor_x += w + COMPONENT_GAP; + row_height = row_height.max(h); + max_height = max_height.max(row_y + row_height); + } + + // Isolated tables tile a shelf below everything else, packed tightly. + // + // They get ISOLATED_GAP rather than COMPONENT_GAP: nothing connects them, + // so the wide gutter that makes edges readable buys nothing here and only + // stretches the canvas. A canvas wider than it needs to be forces the + // fit-zoom down, which shrinks the type in *every* node — the sparse + // shelf was why the whole diagram was rendering at 55%. + // + // The shelf is also capped to the width the connected part already + // occupies, so orphans wrap under the diagram instead of widening it. + let connected_width = positions + .values() + .map(|(x, _)| x + NODE_WIDTH) + .fold(0.0_f64, f64::max); + let shelf_width = connected_width.clamp(NODE_WIDTH * 4.0 + ISOLATED_GAP * 3.0, SHELF_WIDTH); + let mut ix = 0.0_f64; + let mut iy = max_height + SHELF_GAP; + let mut row_height = 0.0_f64; + for id in &isolated { + let cols = by_id.get(id).map(|t| t.columns.len()).unwrap_or(0); + let h = node_height(cols); + if ix > 0.0 && ix + NODE_WIDTH > shelf_width { + ix = 0.0; + iy += row_height + V_GAP; + row_height = 0.0; + } + positions.insert(id.clone(), (ix, iy)); + row_height = row_height.max(h); + ix += NODE_WIDTH + ISOLATED_GAP; + max_height = max_height.max(iy + h); + } + + let nodes = build_nodes(&ids, &by_id, &positions, &ranks, °ree); + + // Bounding boxes are derived from the placed nodes rather than tracked + // during placement, so they cannot drift from where the nodes ended up. + let node_box: HashMap<&str, (f64, f64, f64, f64)> = nodes + .iter() + .map(|n| (node_id(n) as &str, (n.x, n.y, n.w, n.h))) + .collect(); + let component_boxes: Vec = components + .iter() + .enumerate() + .filter_map(|(index, comp)| { + let boxes: Vec<_> = comp + .iter() + .filter_map(|t| node_box.get(t.as_str()).copied()) + .collect(); + if boxes.is_empty() { + return None; + } + let x = boxes.iter().map(|b| b.0).fold(f64::INFINITY, f64::min); + let y = boxes.iter().map(|b| b.1).fold(f64::INFINITY, f64::min); + let right = boxes.iter().map(|b| b.0 + b.2).fold(0.0_f64, f64::max); + let bottom = boxes.iter().map(|b| b.1 + b.3).fold(0.0_f64, f64::max); + // Only name a hub when one table is strictly the most connected. + // In a two-table pair both ends have degree 1, and picking one on + // a tiebreak would emphasise an arbitrary node in the renderer. + let mut ranked: Vec<(u32, &String)> = comp + .iter() + .map(|t| (degree.get(t).copied().unwrap_or(0), t)) + .collect(); + ranked.sort_by(|a, b| b.0.cmp(&a.0).then_with(|| a.1.cmp(b.1))); + let hub = match ranked.as_slice() { + [(top, name), (second, _), ..] if top > second => Some((*name).clone()), + _ => None, + }; + Some(DiagramComponent { + index, + tables: comp.clone(), + hub, + x, + y, + w: right - x, + h: bottom - y, + }) + }) + .collect(); + + let index: HashMap<&str, &DiagramNode> = nodes + .iter() + .map(|n| (node_id(n) as &str, n)) + .collect::>(); + let edges = route(&relations, &index); + let crossings = count_crossings(&edges); + + let width = nodes + .iter() + .map(|n| n.x + n.w) + .fold(0.0_f64, f64::max) + .max(NODE_WIDTH); + let height = nodes + .iter() + .map(|n| n.y + n.h) + .fold(0.0_f64, f64::max) + .max(HEADER_HEIGHT); + + SchemaDiagramResp { + nodes, + edges, + width, + height, + isolated, + components: component_boxes, + crossings, + // Set by `handle` when the caller asked for a neighbourhood; `layout` + // itself is given the tables it should draw and knows nothing of why. + focus: None, + frontier: Vec::new(), + truncated, + } +} + +/// Leaked id string for the lookup map; nodes own their name, this just +/// borrows it. +fn node_id(n: &DiagramNode) -> &str { + // `table` already carries the qualifier when one exists, because nodes are + // built from the qualified id. + &n.table +} + +fn split_components(ids: &[String], relations: &[Relation]) -> Vec> { + let mut parent: BTreeMap<&String, &String> = ids.iter().map(|i| (i, i)).collect(); + + fn find<'a>(parent: &BTreeMap<&'a String, &'a String>, x: &'a String) -> &'a String { + let mut cur = x; + while parent[cur] != cur { + cur = parent[cur]; + } + cur + } + + for r in relations { + let (Some(a), Some(b)) = ( + ids.iter().find(|i| **i == r.from), + ids.iter().find(|i| **i == r.to), + ) else { + continue; + }; + let ra = find(&parent, a); + let rb = find(&parent, b); + if ra != rb { + // Union by name keeps the result independent of input order. + let (lo, hi) = if ra < rb { (ra, rb) } else { (rb, ra) }; + parent.insert(hi, lo); + } + } + + let mut groups: BTreeMap<&String, Vec> = BTreeMap::new(); + for id in ids { + groups + .entry(find(&parent, id)) + .or_default() + .push(id.clone()); + } + groups.into_values().collect() +} + +/// Longest-path ranking along FK direction (referencing above referenced), +/// with cycles broken at the edge whose endpoints have the lowest degree. +fn rank_all( + components: &[Vec], + relations: &[Relation], + degree: &BTreeMap, +) -> BTreeMap { + let mut ranks: BTreeMap = BTreeMap::new(); + for comp in components { + let members: BTreeSet<&String> = comp.iter().collect(); + let mut edges: Vec<&Relation> = relations + .iter() + .filter(|r| members.contains(&r.from) && members.contains(&r.to) && r.from != r.to) + .collect(); + // Deterministic cycle breaking: drop the lowest-degree edge last, so + // the densest structure keeps its shape. + edges.sort_by(|a, b| { + (degree[&a.from] + degree[&a.to]) + .cmp(&(degree[&b.from] + degree[&b.to])) + .then(a.from.cmp(&b.from)) + .then(a.to.cmp(&b.to)) + }); + + for id in comp { + ranks.insert(id.clone(), 0); + } + // Relax repeatedly; bounded by component size so a residual cycle + // terminates instead of spinning. + for _ in 0..comp.len().min(64) { + let mut changed = false; + for e in &edges { + let want = ranks[&e.to] + 1; + if ranks[&e.from] < want { + ranks.insert(e.from.clone(), want); + changed = true; + } + } + if !changed { + break; + } + } + } + ranks +} + +/// Barycenter sweep plus adjacent transposition, keeping the better result. +fn order_component( + comp: &[String], + ranks: &BTreeMap, + relations: &[Relation], +) -> BTreeMap> { + let mut layers: BTreeMap> = BTreeMap::new(); + for id in comp { + layers.entry(ranks[id]).or_default().push(id.clone()); + } + for v in layers.values_mut() { + v.sort(); + } + + let neighbours = |id: &str| -> Vec { + relations + .iter() + .filter_map(|r| { + if r.from == id { + Some(r.to.clone()) + } else if r.to == id { + Some(r.from.clone()) + } else { + None + } + }) + .collect() + }; + + let mut best = layers.clone(); + let mut best_score = layer_crossings(&layers, relations); + + for _ in 0..4 { + let snapshot = layers.clone(); + for row in layers.values_mut() { + let mut keyed: Vec<(f64, String)> = row + .iter() + .map(|id| { + let ns = neighbours(id); + let bary = if ns.is_empty() { + f64::MAX + } else { + let sum: f64 = ns + .iter() + .filter_map(|n| { + snapshot + .values() + .find_map(|r| r.iter().position(|x| x == n).map(|p| p as f64)) + }) + .sum(); + sum / ns.len() as f64 + }; + (bary, id.clone()) + }) + .collect(); + // Name breaks ties, so the sweep is reproducible. + keyed.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap().then(a.1.cmp(&b.1))); + *row = keyed.into_iter().map(|(_, id)| id).collect(); + } + let score = layer_crossings(&layers, relations); + if score < best_score { + best_score = score; + best = layers.clone(); + } + } + best +} + +/// Crossings implied by the current ordering, used to choose between sweeps. +fn layer_crossings(layers: &BTreeMap>, relations: &[Relation]) -> u32 { + let pos: HashMap<&String, (u32, usize)> = layers + .iter() + .flat_map(|(rank, row)| row.iter().enumerate().map(move |(i, id)| (id, (*rank, i)))) + .collect(); + + let mut edges: Vec<(u32, usize, usize)> = Vec::new(); + for r in relations { + let (Some(a), Some(b)) = (pos.get(&r.from), pos.get(&r.to)) else { + continue; + }; + if a.0 == b.0 + 1 { + edges.push((b.0, a.1, b.1)); + } + } + + let mut n = 0; + for i in 0..edges.len() { + for j in (i + 1)..edges.len() { + let (l1, u1, v1) = edges[i]; + let (l2, u2, v2) = edges[j]; + if l1 == l2 && ((u1 < u2) != (v1 < v2)) { + n += 1; + } + } + } + n +} + +fn place( + order: &BTreeMap>, + _ranks: &BTreeMap, + by_id: &HashMap<&String, &TableDescription>, + origin_x: f64, + origin_y: f64, + out: &mut BTreeMap, +) -> (f64, f64) { + let mut max_w = 0.0_f64; + let mut max_h = 0.0_f64; + for (rank, row) in order { + let x = origin_x + (*rank as f64) * RANK_SEP; + let mut y = origin_y; + for id in row { + out.insert(id.clone(), (x, y)); + let cols = by_id.get(id).map(|t| t.columns.len()).unwrap_or(0); + y += node_height(cols) + V_GAP; + } + max_w = max_w.max(x - origin_x + NODE_WIDTH); + max_h = max_h.max(y - origin_y); + } + (max_w, max_h) +} + +fn build_nodes( + ids: &[String], + by_id: &HashMap<&String, &TableDescription>, + positions: &BTreeMap, + ranks: &BTreeMap, + degree: &BTreeMap, +) -> Vec { + ids.iter() + .filter_map(|id| { + let t = by_id.get(id)?; + let (x, y) = *positions.get(id)?; + let shown: Vec = t + .columns + .iter() + .take(MAX_ROWS) + .map(|c| DiagramColumn { + name: c.name.clone(), + ty: c.ty.clone(), + primary_key: c.primary_key, + foreign_key: c.foreign_key.is_some(), + nullable: c.nullable, + }) + .collect(); + Some(DiagramNode { + table: id.clone(), + schema: t.schema.clone(), + x, + y, + w: NODE_WIDTH, + h: node_height(t.columns.len()), + rank: ranks.get(id).copied().unwrap_or(0), + degree: degree.get(id).copied().unwrap_or(0), + hidden_columns: t.columns.len().saturating_sub(shown.len()), + columns: shown, + }) + }) + .collect() +} + +/// Vertical centre of a column's row, so an edge points at `user_id` rather +/// than at the middle of the box. Falls back to the node centre when the +/// column is past `MAX_ROWS` and therefore not drawn. +fn anchor_y(node: &DiagramNode, column: &str) -> f64 { + match node.columns.iter().position(|c| c.name == column) { + Some(i) => node.y + HEADER_HEIGHT + NODE_PAD_Y + (i as f64 + 0.5) * ROW_HEIGHT, + None => node.y + node.h / 2.0, + } +} + +fn route(relations: &[Relation], index: &HashMap<&str, &DiagramNode>) -> Vec { + relations + .iter() + .filter_map(|r| { + let from = index.get(r.from.as_str())?; + let to = index.get(r.to.as_str())?; + let self_loop = r.from == r.to; + + let y1 = anchor_y(from, &r.from_column); + let y2 = anchor_y(to, &r.to_column); + + let points = if self_loop { + // Loop out of the right edge and back. + let x = from.x + from.w; + vec![ + Point { x, y: y1 }, + Point { x: x + 24.0, y: y1 }, + Point { + x: x + 24.0, + y: y1 - ROW_HEIGHT, + }, + Point { + x, + y: y1 - ROW_HEIGHT, + }, + ] + } else { + // Three-segment elbow, leaving from whichever side faces the + // target so edges do not cross their own node. + let (x1, x2) = if to.x >= from.x + from.w { + (from.x + from.w, to.x) + } else if from.x >= to.x + to.w { + (from.x, to.x + to.w) + } else { + (from.x + from.w, to.x + to.w) + }; + let mid = (x1 + x2) / 2.0; + vec![ + Point { x: x1, y: y1 }, + Point { x: mid, y: y1 }, + Point { x: mid, y: y2 }, + Point { x: x2, y: y2 }, + ] + }; + + Some(DiagramEdge { + from: r.from.clone(), + from_column: r.from_column.clone(), + to: r.to.clone(), + to_column: r.to_column.clone(), + points, + self_loop, + }) + }) + .collect() +} + +/// Crossings between routed edges, by the standard interleaving test: two +/// edges sharing a corridor cross when their endpoints are in opposite order +/// at each end. +/// +/// Not a plain overlap test. Six edges fanning into one parent all occupy the +/// same corridor and overlap vertically, but none of them cross — they +/// converge. Counting overlap would report fifteen crossings for a diagram a +/// reader would call tidy. +fn count_crossings(edges: &[DiagramEdge]) -> u32 { + /// (corridor x, source y, target y) + fn ends(e: &DiagramEdge) -> Option<(f64, f64, f64)> { + let mid = e.points.get(1)?; + Some((mid.x, e.points.first()?.y, e.points.last()?.y)) + } + let segs: Vec<(f64, f64, f64)> = edges + .iter() + .filter(|e| !e.self_loop) + .filter_map(ends) + .collect(); + + let mut n = 0; + for i in 0..segs.len() { + for j in (i + 1)..segs.len() { + let (x1, s1, t1) = segs[i]; + let (x2, s2, t2) = segs[j]; + if (x1 - x2).abs() > f64::EPSILON { + continue; + } + // Shared endpoints converge or diverge; neither is a crossing. + if s1 == s2 || t1 == t2 { + continue; + } + if (s1 < s2) != (t1 < t2) { + n += 1; + } + } + } + n +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::handlers::catalog::{ColumnDesc, ForeignKeyRef, TableKind}; + + fn col(name: &str, pk: bool, fk: Option<(&str, &str)>) -> ColumnDesc { + ColumnDesc { + name: name.into(), + ty: "INTEGER".into(), + nullable: !pk, + default_value: None, + primary_key: pk, + position: 1, + foreign_key: fk.map(|(t, c)| ForeignKeyRef { + schema: None, + table: t.into(), + column: c.into(), + }), + } + } + + fn table(name: &str, columns: Vec) -> TableDescription { + TableDescription { + table: name.into(), + schema: None, + kind: TableKind::Table, + columns, + indexes: vec![], + row_count_estimate: None, + } + } + + #[test] + fn an_empty_schema_lays_out_without_panicking() { + let d = layout(&[], false); + assert!(d.nodes.is_empty() && d.edges.is_empty()); + assert_eq!(d.crossings, 0); + } + + #[test] + fn a_lone_table_is_isolated_not_ranked() { + let d = layout(&[table("users", vec![col("id", true, None)])], false); + assert_eq!(d.isolated, vec!["users"]); + assert_eq!(d.nodes.len(), 1); + assert_eq!(d.nodes[0].degree, 0); + } + + /// users ← orders ← order_items, plus an unrelated table. + fn chain() -> Vec { + vec![ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + table( + "order_items", + vec![ + col("id", true, None), + col("order_id", false, Some(("orders", "id"))), + ], + ), + table("unrelated", vec![col("id", true, None)]), + ] + } + + #[test] + fn components_never_overlap_once_the_row_wraps() { + // Enough two-table components to run past SHELF_WIDTH. Every one was + // previously placed at y=0, so the second row drew on top of the + // first. + let mut tables = Vec::new(); + for i in 0..12 { + tables.push(table(&format!("p{i}"), vec![col("id", true, None)])); + tables.push(table( + &format!("c{i}"), + vec![ + col("id", true, None), + col("pid", false, Some((&format!("p{i}"), "id"))), + ], + )); + } + let d = layout(&tables, false); + assert!(d.components.len() > 1); + for (i, a) in d.components.iter().enumerate() { + for b in d.components.iter().skip(i + 1) { + let apart = + a.x + a.w <= b.x || b.x + b.w <= a.x || a.y + a.h <= b.y || b.y + b.h <= a.y; + assert!(apart, "components {} and {} overlap", a.index, b.index); + } + } + } + + #[test] + fn an_absurd_depth_terminates_instead_of_spinning() { + // `depth` is unbounded on the wire; the walk must stop when the + // component is exhausted rather than iterating that many times. + let (kept, frontier) = neighbourhood(&chain(), "users", usize::MAX).unwrap(); + assert_eq!(kept.len(), 3); + assert!(frontier.is_empty()); + } + + #[test] + fn focus_follows_references_in_both_directions() { + // `orders` references users and is referenced by order_items. A walk + // that only followed the declared direction would hide every child. + let (kept, _) = neighbourhood(&chain(), "orders", 1).unwrap(); + let mut names: Vec<&str> = kept.iter().map(|t| t.table.as_str()).collect(); + names.sort(); + assert_eq!(names, vec!["order_items", "orders", "users"]); + } + + #[test] + fn focus_reports_what_lies_one_hop_beyond() { + let (kept, frontier) = neighbourhood(&chain(), "users", 1).unwrap(); + let mut names: Vec<&str> = kept.iter().map(|t| t.table.as_str()).collect(); + names.sort(); + assert_eq!(names, vec!["orders", "users"]); + // order_items is reachable but not drawn — the renderer needs to know + // there is something to expand into. + assert_eq!(frontier, vec!["order_items".to_string()]); + } + + #[test] + fn a_deeper_focus_pulls_the_next_ring_in() { + let (kept, frontier) = neighbourhood(&chain(), "users", 2).unwrap(); + assert_eq!(kept.len(), 3, "users, orders and order_items"); + assert!(frontier.is_empty(), "nothing further to reach"); + } + + #[test] + fn focus_never_drags_in_an_unrelated_table() { + let (kept, _) = neighbourhood(&chain(), "users", 9).unwrap(); + assert!(kept.iter().all(|t| t.table != "unrelated")); + } + + #[test] + fn an_unknown_focus_is_an_error_not_an_empty_diagram() { + // Silently returning nothing would read as "this table has no + // relations", which is a different and wrong answer. + assert!(neighbourhood(&chain(), "nope", 1).is_err()); + } + + #[test] + fn adjacent_ranks_leave_a_gutter_wide_enough_to_draw_an_edge_in() { + // Regression. RANK_SEP was 220 against a 200-wide node, leaving a + // 20px gutter: every edge rendered as a stub too short to see, and a + // schema with real foreign keys looked unrelated on screen. The + // gutter, not the pitch, is the number that matters. + let d = layout( + &[ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + ], + false, + ); + let by: HashMap<&str, &DiagramNode> = + d.nodes.iter().map(|n| (n.table.as_str(), n)).collect(); + let gutter = by["orders"].x - (by["users"].x + by["users"].w); + assert!( + gutter >= 100.0, + "adjacent ranks left a {gutter}px gutter; an edge needs room to read as a connection" + ); + } + + #[test] + fn related_tables_are_grouped_into_one_component_box() { + let d = layout( + &[ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + table("unrelated", vec![col("id", true, None)]), + ], + false, + ); + assert_eq!(d.components.len(), 1, "one connected group"); + let c = &d.components[0]; + assert_eq!(c.tables.len(), 2); + assert_eq!( + c.hub, None, + "both ends of a pair have degree 1 — naming either as the hub would be arbitrary" + ); + assert!(c.w > 0.0 && c.h > 0.0, "the group has a drawable box"); + // The box must actually contain its members, or the renderer draws a + // boundary that clips them. + for t in &c.tables { + let n = d.nodes.iter().find(|n| &n.table == t).expect("member node"); + assert!( + n.x >= c.x && n.y >= c.y && n.x + n.w <= c.x + c.w && n.y + n.h <= c.y + c.h, + "{t} falls outside its component box" + ); + } + assert_eq!(d.isolated, vec!["unrelated".to_string()]); + } + + #[test] + fn the_most_connected_table_is_named_as_the_hub() { + let d = layout( + &[ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + table( + "sessions", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + ], + false, + ); + assert_eq!(d.components.len(), 1); + assert_eq!( + d.components[0].hub.as_deref(), + Some("users"), + "two tables reference users, so it is strictly the most connected" + ); + } + + #[test] + fn a_reference_ranks_the_child_above_the_parent() { + let d = layout( + &[ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + ], + false, + ); + let by: HashMap<&str, &DiagramNode> = + d.nodes.iter().map(|n| (n.table.as_str(), n)).collect(); + assert_eq!(by["users"].rank, 0, "the referenced table sits upstream"); + assert_eq!(by["orders"].rank, 1); + assert!(d.isolated.is_empty()); + assert_eq!(d.edges.len(), 1); + assert_eq!(d.edges[0].to_column, "id"); + } + + #[test] + fn an_edge_anchors_on_the_column_row_not_the_box() { + let d = layout( + &[ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("filler", false, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + ], + false, + ); + let orders = d.nodes.iter().find(|n| n.table == "orders").unwrap(); + let edge = &d.edges[0]; + // Third column, so the anchor is two rows below the first. + let expected = orders.y + HEADER_HEIGHT + NODE_PAD_Y + 2.5 * ROW_HEIGHT; + assert!( + (edge.points[0].y - expected).abs() < 0.001, + "anchored at {} not {expected}", + edge.points[0].y + ); + } + + #[test] + fn a_reference_cycle_terminates_and_still_ranks_everything() { + let d = layout( + &[ + table( + "a", + vec![col("id", true, None), col("b_id", false, Some(("b", "id")))], + ), + table( + "b", + vec![col("id", true, None), col("a_id", false, Some(("a", "id")))], + ), + ], + false, + ); + assert_eq!(d.nodes.len(), 2); + assert!(d.isolated.is_empty()); + } + + #[test] + fn a_self_reference_becomes_a_loop_not_a_line() { + let d = layout( + &[table( + "employees", + vec![ + col("id", true, None), + col("manager_id", false, Some(("employees", "id"))), + ], + )], + false, + ); + assert_eq!(d.edges.len(), 1); + assert!(d.edges[0].self_loop); + } + + #[test] + fn a_hub_reports_its_degree_and_leaves_no_crossings() { + let mut tables = vec![table("users", vec![col("id", true, None)])]; + for i in 0..6 { + tables.push(table( + &format!("child{i}"), + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + )); + } + let d = layout(&tables, false); + let users = d.nodes.iter().find(|n| n.table == "users").unwrap(); + assert_eq!(users.degree, 6); + assert_eq!(d.crossings, 0, "a simple fan should not cross"); + } + + #[test] + fn layout_is_deterministic_regardless_of_input_order() { + let mut a = vec![ + table("users", vec![col("id", true, None)]), + table( + "orders", + vec![ + col("id", true, None), + col("user_id", false, Some(("users", "id"))), + ], + ), + table("audit", vec![col("id", true, None)]), + ]; + let first = layout(&a, false); + a.reverse(); + let second = layout(&a, false); + + let key = |d: &SchemaDiagramResp| -> Vec<(String, u64, u64)> { + let mut v: Vec<_> = d + .nodes + .iter() + .map(|n| (n.table.clone(), n.x as u64, n.y as u64)) + .collect(); + v.sort(); + v + }; + assert_eq!(key(&first), key(&second)); + assert_eq!(first.crossings, second.crossings); + } + + #[test] + fn a_wide_table_is_summarised_rather_than_drawn_in_full() { + let cols: Vec = (0..30) + .map(|i| col(&format!("c{i}"), i == 0, None)) + .collect(); + let d = layout(&[table("wide", cols)], false); + let n = &d.nodes[0]; + assert_eq!(n.columns.len(), MAX_ROWS); + assert_eq!(n.hidden_columns, 30 - MAX_ROWS); + // Height reflects what is drawn, not the full column count. + assert_eq!(n.h, node_height(MAX_ROWS)); + } + + #[test] + fn a_reference_to_a_table_outside_the_set_is_dropped() { + // The parent was filtered out or truncated away; an edge to nowhere + // would render as a line into empty space. + let d = layout( + &[table( + "orders", + vec![col("user_id", false, Some(("users", "id")))], + )], + true, + ); + assert!(d.edges.is_empty()); + assert_eq!(d.isolated, vec!["orders"]); + assert!(d.truncated); + } +} diff --git a/database/src/handlers/explain.rs b/database/src/handlers/explain.rs new file mode 100644 index 000000000..18bf948a9 --- /dev/null +++ b/database/src/handlers/explain.rs @@ -0,0 +1,712 @@ +//! `database::explain` — a query plan as a tree, not a grid of text. +//! +//! Every driver spells a plan differently: postgres emits nested JSON, mysql +//! emits a differently-nested JSON, and sqlite emits a flat `(id, parent, +//! detail)` set that has to be reassembled. Callers should not have to know +//! that, so all three collapse into one `PlanNode` tree with the same fields +//! and the same warnings. +//! +//! **`ANALYZE` really executes the statement.** `EXPLAIN ANALYZE DELETE FROM +//! users` deletes users. It is off by default and refused outright for +//! anything that does not parse as a read — the console's own check is a +//! convenience, this one is the authority. + +use super::query::{self, err_to_str, QueryReq}; +use super::tx_sql_guard; +use super::AppState; +use crate::config::DriverKind; +use crate::error::DbError; +use crate::pool::Pool; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +fn default_timeout() -> u64 { + 30_000 +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum NodeClass { + Scan, + Index, + Join, + Sort, + Aggregate, + Cte, + Limit, + Other, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum PlanFormat { + PgJson, + SqliteQueryPlan, + MysqlJson, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum WarningKind { + /// A sequential scan over a large relation. + SeqScanLarge, + /// Estimated and actual row counts differ by an order of magnitude — + /// usually stale statistics. + EstimateSkew, + /// An inner loop executed a very large number of times. + NestedLoopLarge, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum Severity { + Info, + Warn, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct PlanNode { + pub id: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub parent: Option, + pub label: String, + pub node_class: NodeClass, + #[serde(skip_serializing_if = "Option::is_none")] + pub relation: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cost_startup: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cost_total: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rows_estimated: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rows_actual: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub width: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub time_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub loops: Option, + pub detail: String, + pub children: Vec, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct PlanWarning { + pub node_id: u32, + pub kind: WarningKind, + pub message: String, + pub severity: Severity, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ExplainReq { + #[serde(default)] + pub db: Option, + pub sql: String, + #[serde(default, deserialize_with = "crate::handlers::lenient_params")] + pub params: Vec, + /// Runs the statement to collect real timings. Refused for anything that + /// is not a read. + #[serde(default)] + pub analyze: bool, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ExplainResp { + pub format: PlanFormat, + pub analyzed: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub root: Option, + pub warnings: Vec, + /// The driver's own output, so a caller is never stuck when the shape is + /// one we do not recognise. + #[serde(skip_serializing_if = "Option::is_none")] + pub raw: Option, +} + +/// Rows above this make a sequential scan worth flagging. +const LARGE_ROWS: f64 = 10_000.0; +/// Estimate/actual ratio worth flagging as stale statistics. +const SKEW_FACTOR: f64 = 10.0; +/// Loop count worth flagging on a nested loop. +const LARGE_LOOPS: f64 = 1_000.0; + +pub fn classify(label: &str) -> NodeClass { + let l = label.to_ascii_lowercase(); + if l.contains("index") { + NodeClass::Index + } else if l.contains("scan") || l.contains("seek") { + NodeClass::Scan + } else if l.contains("join") || l.contains("nested loop") { + NodeClass::Join + } else if l.contains("sort") || l.contains("b-tree") { + NodeClass::Sort + } else if l.contains("aggregate") || l.contains("group") { + NodeClass::Aggregate + } else if l.contains("cte") || l.contains("subquery") { + NodeClass::Cte + } else if l.contains("limit") { + NodeClass::Limit + } else { + NodeClass::Other + } +} + +async fn driver_of(state: &AppState, db: &str) -> Result { + Ok(match state.pool(db).await.map_err(err_to_str)? { + Pool::Sqlite(_) => DriverKind::Sqlite, + Pool::Postgres(_) => DriverKind::Postgres, + Pool::Mysql(_) => DriverKind::Mysql, + }) +} + +pub async fn handle(state: &AppState, req: ExplainReq) -> Result { + if req.sql.trim().is_empty() { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: "sql is required".into(), + })); + } + let db = state.resolve_db(req.db.clone()).await.map_err(err_to_str)?; + let driver = driver_of(state, &db).await?; + + // The gate that matters. ANALYZE executes; refuse it on anything that is + // not unambiguously a read. + if req.analyze && !tx_sql_guard::is_read_only_sql(&req.sql) { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: "analyze runs the statement, so it is only allowed for a single \ + read-only statement; run it without analyze to see the estimated plan" + .into(), + })); + } + // SQLite's EXPLAIN QUERY PLAN has no ANALYZE form. + let analyzed = req.analyze && driver != DriverKind::Sqlite; + + let prefixed = match (driver, analyzed) { + (DriverKind::Postgres, true) => format!("EXPLAIN (FORMAT JSON, ANALYZE true) {}", req.sql), + (DriverKind::Postgres, false) => format!("EXPLAIN (FORMAT JSON) {}", req.sql), + (DriverKind::Mysql, _) => format!("EXPLAIN FORMAT=JSON {}", req.sql), + (DriverKind::Sqlite, _) => format!("EXPLAIN QUERY PLAN {}", req.sql), + }; + + let resp = query::handle( + state, + QueryReq { + db: Some(db), + sql: prefixed, + params: req.params, + timeout_ms: req.timeout_ms, + record_history: false, + }, + ) + .await?; + + let (format, root) = match driver { + DriverKind::Postgres => parse_pg(&resp.rows), + DriverKind::Mysql => parse_mysql(&resp.rows), + DriverKind::Sqlite => parse_sqlite(&resp.rows), + }; + let warnings = root.as_ref().map(warnings_for).unwrap_or_default(); + let raw = (format == PlanFormat::Unknown) + .then(|| Value::Array(resp.rows.iter().cloned().map(Value::Object).collect())); + + Ok(ExplainResp { + format, + analyzed, + root, + warnings, + raw, + }) +} + +fn num(v: Option<&Value>) -> Option { + match v { + Some(Value::Number(n)) => n.as_f64(), + Some(Value::String(s)) => s.parse().ok(), + _ => None, + } +} + +/// The driver may hand back a JSON column already parsed, or as text. +fn as_json(v: &Value) -> Option { + match v { + Value::String(s) => serde_json::from_str(s).ok(), + other => Some(other.clone()), + } +} + +fn first_value(rows: &[serde_json::Map]) -> Option { + rows.first()?.values().next().cloned() +} + +/* ---------------- postgres ---------------- */ + +fn parse_pg(rows: &[serde_json::Map]) -> (PlanFormat, Option) { + let Some(parsed) = first_value(rows).as_ref().and_then(as_json) else { + return (PlanFormat::Unknown, None); + }; + // EXPLAIN (FORMAT JSON) wraps the plan in a single-element array. + let plan = parsed + .as_array() + .and_then(|a| a.first()) + .and_then(|o| o.get("Plan")) + .cloned(); + match plan { + Some(p) => { + let mut next_id = 0; + (PlanFormat::PgJson, Some(pg_node(&p, None, &mut next_id))) + } + None => (PlanFormat::Unknown, None), + } +} + +fn pg_node(v: &Value, parent: Option, next_id: &mut u32) -> PlanNode { + let id = *next_id; + *next_id += 1; + + let label = v + .get("Node Type") + .and_then(Value::as_str) + .unwrap_or("Unknown") + .to_string(); + let relation = v + .get("Relation Name") + .and_then(Value::as_str) + .map(str::to_string); + + let mut detail = label.clone(); + if let Some(r) = &relation { + detail = format!("{detail} on {r}"); + } + + let children = v + .get("Plans") + .and_then(Value::as_array) + .map(|a| a.iter().map(|c| pg_node(c, Some(id), next_id)).collect()) + .unwrap_or_default(); + + PlanNode { + id, + parent, + node_class: classify(&label), + label, + relation, + cost_startup: num(v.get("Startup Cost")), + cost_total: num(v.get("Total Cost")), + rows_estimated: num(v.get("Plan Rows")), + rows_actual: num(v.get("Actual Rows")), + width: num(v.get("Plan Width")).map(|w| w as i64), + time_ms: num(v.get("Actual Total Time")), + loops: num(v.get("Actual Loops")), + detail, + children, + } +} + +/* ---------------- mysql ---------------- */ + +fn parse_mysql(rows: &[serde_json::Map]) -> (PlanFormat, Option) { + let Some(parsed) = first_value(rows).as_ref().and_then(as_json) else { + return (PlanFormat::Unknown, None); + }; + let Some(block) = parsed.get("query_block") else { + return (PlanFormat::Unknown, None); + }; + let mut next_id = 0; + ( + PlanFormat::MysqlJson, + Some(mysql_node(block, None, &mut next_id, "query_block")), + ) +} + +/// MySQL's shape is a loose bag of nested objects rather than a uniform node +/// list, so walk it generically: any nested object carrying a `table` or +/// another recognisable block becomes a child. +fn mysql_node(v: &Value, parent: Option, next_id: &mut u32, label: &str) -> PlanNode { + let id = *next_id; + *next_id += 1; + + let table = v.get("table"); + let relation = table + .and_then(|t| t.get("table_name")) + .and_then(Value::as_str) + .map(str::to_string); + let access = table + .and_then(|t| t.get("access_type")) + .and_then(Value::as_str) + .unwrap_or(label); + + let cost = table + .and_then(|t| t.get("cost_info")) + .and_then(|c| c.get("read_cost").or_else(|| c.get("query_cost"))); + + let mut children = Vec::new(); + if let Some(obj) = v.as_object() { + for (k, child) in obj { + if k == "table" || k == "cost_info" { + continue; + } + match child { + Value::Object(_) => children.push(mysql_node(child, Some(id), next_id, k)), + Value::Array(items) => { + for item in items.iter().filter(|i| i.is_object()) { + children.push(mysql_node(item, Some(id), next_id, k)); + } + } + _ => {} + } + } + } + + let label = access.to_string(); + PlanNode { + id, + parent, + node_class: classify(&label), + detail: match &relation { + Some(r) => format!("{label} on {r}"), + None => label.clone(), + }, + label, + relation, + cost_startup: None, + cost_total: num(cost), + rows_estimated: num(table.and_then(|t| t.get("rows_examined_per_scan"))), + rows_actual: num(table.and_then(|t| t.get("rows_produced_per_join"))), + width: None, + time_ms: None, + loops: None, + children, + } +} + +/* ---------------- sqlite ---------------- */ + +/// `EXPLAIN QUERY PLAN` returns a flat `(id, parent, notused, detail)` set; +/// the tree is implied by `parent` and has to be rebuilt. +fn parse_sqlite(rows: &[serde_json::Map]) -> (PlanFormat, Option) { + if rows.is_empty() { + return (PlanFormat::Unknown, None); + } + let flat: Vec<(u32, u32, String)> = rows + .iter() + .filter_map(|r| { + let id = num(r.get("id"))? as u32; + let parent = num(r.get("parent")).unwrap_or(0.0) as u32; + let detail = r.get("detail").and_then(Value::as_str)?.to_string(); + Some((id, parent, detail)) + }) + .collect(); + if flat.is_empty() { + return (PlanFormat::Unknown, None); + } + + // Group by parent first, then build the tree in one recursive pass. Doing + // it this way avoids searching a partly-built tree for each row, and it + // tolerates rows arriving in any order. + let mut children_of: std::collections::HashMap> = + std::collections::HashMap::new(); + for (id, parent, detail) in flat { + children_of.entry(parent).or_default().push((id, detail)); + } + + // sqlite numbers top-level rows with parent 0, so synthesise a root to + // hang them from rather than promoting an arbitrary step. + let root = PlanNode { + id: 0, + parent: None, + label: "QUERY PLAN".into(), + node_class: NodeClass::Other, + relation: None, + cost_startup: None, + cost_total: None, + rows_estimated: None, + rows_actual: None, + width: None, + time_ms: None, + loops: None, + detail: "QUERY PLAN".into(), + children: sqlite_children(0, &children_of, 0), + }; + (PlanFormat::SqliteQueryPlan, Some(root)) +} + +/// Pull the relation out of an `EXPLAIN QUERY PLAN` detail string. +/// +/// SQLite changed this wording: older versions say `SCAN TABLE users`, 3.36 +/// and later say `SCAN users`. Both forms are accepted, as is the `SUBQUERY` +/// spelling, so the relation does not silently go missing on one build. +fn sqlite_relation(detail: &str) -> Option { + let mut words = detail + .split_whitespace() + .skip_while(|w| !matches!(*w, "SCAN" | "SEARCH")) + .skip(1); + let first = words.next()?; + let name = match first { + "TABLE" | "SUBQUERY" => words.next()?, + other => other, + }; + // `SCAN users USING INDEX ...` — the name never starts a clause keyword. + (!matches!(name, "USING" | "AS" | "COVERING")).then(|| name.to_string()) +} + +/// Depth bound guards against a malformed set whose parent links form a cycle. +const MAX_PLAN_DEPTH: u32 = 64; + +fn sqlite_children( + parent: u32, + children_of: &std::collections::HashMap>, + depth: u32, +) -> Vec { + if depth >= MAX_PLAN_DEPTH { + return Vec::new(); + } + children_of + .get(&parent) + .map(|kids| { + kids.iter() + .map(|(id, detail)| PlanNode { + id: *id, + parent: Some(parent), + node_class: classify(detail), + label: detail.clone(), + relation: sqlite_relation(detail), + cost_startup: None, + cost_total: None, + rows_estimated: None, + rows_actual: None, + width: None, + time_ms: None, + loops: None, + detail: detail.clone(), + children: sqlite_children(*id, children_of, depth + 1), + }) + .collect() + }) + .unwrap_or_default() +} + +/* ---------------- warnings ---------------- */ + +pub fn warnings_for(root: &PlanNode) -> Vec { + let mut out = Vec::new(); + walk(root, &mut out); + out +} + +fn walk(node: &PlanNode, out: &mut Vec) { + let rows = node.rows_actual.or(node.rows_estimated).unwrap_or(0.0); + + if node.node_class == NodeClass::Scan + && !node.label.to_ascii_lowercase().contains("index") + && rows > LARGE_ROWS + { + out.push(PlanWarning { + node_id: node.id, + kind: WarningKind::SeqScanLarge, + message: format!( + "sequential scan over ~{rows:.0} rows{}; an index on the filtered \ + column would avoid reading the whole relation", + node.relation + .as_ref() + .map(|r| format!(" of {r}")) + .unwrap_or_default() + ), + severity: Severity::Warn, + }); + } + + if let (Some(est), Some(act)) = (node.rows_estimated, node.rows_actual) { + let hi = est.max(act); + let lo = est.min(act).max(1.0); + if hi / lo > SKEW_FACTOR { + out.push(PlanWarning { + node_id: node.id, + kind: WarningKind::EstimateSkew, + message: format!( + "planner estimated {est:.0} rows but saw {act:.0}; statistics are \ + likely stale — ANALYZE the table" + ), + severity: Severity::Warn, + }); + } + } + + if node.node_class == NodeClass::Join && node.loops.unwrap_or(0.0) > LARGE_LOOPS { + out.push(PlanWarning { + node_id: node.id, + kind: WarningKind::NestedLoopLarge, + message: format!( + "inner side executed {:.0} times; a hash or merge join would usually \ + be cheaper at this size", + node.loops.unwrap_or(0.0) + ), + severity: Severity::Info, + }); + } + + for c in &node.children { + walk(c, out); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn rows(v: Value) -> Vec> { + vec![[("x".to_string(), v)].into_iter().collect()] + } + + #[test] + fn classify_maps_the_common_node_names() { + assert_eq!(classify("Seq Scan"), NodeClass::Scan); + assert_eq!(classify("Index Only Scan"), NodeClass::Index); + assert_eq!(classify("Hash Join"), NodeClass::Join); + assert_eq!(classify("Sort"), NodeClass::Sort); + assert_eq!(classify("HashAggregate"), NodeClass::Aggregate); + assert_eq!(classify("Limit"), NodeClass::Limit); + assert_eq!(classify("Gather Merge"), NodeClass::Other); + } + + #[test] + fn pg_plan_becomes_a_tree_with_costs() { + let plan = json!([{ + "Plan": { + "Node Type": "Hash Join", "Startup Cost": 1.5, "Total Cost": 42.0, + "Plan Rows": 100, "Actual Rows": 90, "Plan Width": 32, + "Actual Total Time": 3.5, "Actual Loops": 1, + "Plans": [ + {"Node Type": "Seq Scan", "Relation Name": "users", + "Total Cost": 20.0, "Plan Rows": 50}, + {"Node Type": "Index Scan", "Relation Name": "orders", + "Total Cost": 10.0, "Plan Rows": 50} + ] + } + }]); + let (fmt, root) = parse_pg(&rows(plan)); + assert_eq!(fmt, PlanFormat::PgJson); + let root = root.unwrap(); + assert_eq!(root.label, "Hash Join"); + assert_eq!(root.node_class, NodeClass::Join); + assert_eq!(root.cost_total, Some(42.0)); + assert_eq!(root.children.len(), 2); + assert_eq!(root.children[0].relation.as_deref(), Some("users")); + assert_eq!(root.children[0].detail, "Seq Scan on users"); + // Ids are unique across the tree so warnings can point at a node. + assert_eq!(root.id, 0); + assert_eq!(root.children[0].id, 1); + assert_eq!(root.children[1].id, 2); + } + + #[test] + fn pg_plan_accepts_json_delivered_as_text() { + let text = json!([{"Plan": {"Node Type": "Result"}}]).to_string(); + let (fmt, root) = parse_pg(&rows(Value::String(text))); + assert_eq!(fmt, PlanFormat::PgJson); + assert_eq!(root.unwrap().label, "Result"); + } + + #[test] + fn sqlite_flat_rows_are_reassembled_into_a_tree() { + let flat: Vec> = vec![ + json!({"id": 2, "parent": 0, "detail": "SCAN TABLE users"}), + json!({"id": 4, "parent": 2, "detail": "SEARCH TABLE orders USING INDEX ix"}), + ] + .into_iter() + .map(|v| v.as_object().unwrap().clone()) + .collect(); + + let (fmt, root) = parse_sqlite(&flat); + assert_eq!(fmt, PlanFormat::SqliteQueryPlan); + let root = root.unwrap(); + assert_eq!(root.children.len(), 1, "top-level rows hang off the root"); + let scan = &root.children[0]; + assert_eq!(scan.node_class, NodeClass::Scan); + assert_eq!(scan.relation.as_deref(), Some("users")); + assert_eq!(scan.children.len(), 1, "child attaches to its parent id"); + assert_eq!(scan.children[0].node_class, NodeClass::Index); + } + + #[test] + fn sqlite_relation_survives_both_wordings() { + // SQLite dropped the TABLE keyword in 3.36; both forms are in the wild. + assert_eq!(sqlite_relation("SCAN TABLE users"), Some("users".into())); + assert_eq!(sqlite_relation("SCAN people"), Some("people".into())); + assert_eq!( + sqlite_relation("SEARCH orders USING INDEX ix_o (user_id=?)"), + Some("orders".into()) + ); + assert_eq!( + sqlite_relation("SEARCH TABLE orders USING INDEX ix_o"), + Some("orders".into()) + ); + assert_eq!(sqlite_relation("USE TEMP B-TREE FOR ORDER BY"), None); + } + + #[test] + fn unrecognised_output_reports_unknown_rather_than_guessing() { + let (fmt, root) = parse_pg(&rows(json!("not a plan"))); + assert_eq!(fmt, PlanFormat::Unknown); + assert!(root.is_none()); + assert_eq!(parse_sqlite(&[]).0, PlanFormat::Unknown); + } + + fn node(id: u32, label: &str, est: f64, act: Option) -> PlanNode { + PlanNode { + id, + parent: None, + node_class: classify(label), + label: label.into(), + relation: Some("t".into()), + cost_startup: None, + cost_total: None, + rows_estimated: Some(est), + rows_actual: act, + width: None, + time_ms: None, + loops: None, + detail: label.into(), + children: vec![], + } + } + + #[test] + fn a_large_sequential_scan_is_flagged_but_a_small_one_is_not() { + let big = warnings_for(&node(0, "Seq Scan", 50_000.0, None)); + assert_eq!(big.len(), 1); + assert_eq!(big[0].kind, WarningKind::SeqScanLarge); + + let small = warnings_for(&node(0, "Seq Scan", 10.0, None)); + assert!(small.is_empty(), "a small scan is not a problem"); + + // An index scan of the same size is fine. + let indexed = warnings_for(&node(0, "Index Scan", 50_000.0, None)); + assert!(indexed.is_empty()); + } + + #[test] + fn estimate_skew_fires_only_past_the_threshold() { + let skewed = warnings_for(&node(0, "Index Scan", 10.0, Some(5_000.0))); + assert!(skewed.iter().any(|w| w.kind == WarningKind::EstimateSkew)); + + let close = warnings_for(&node(0, "Index Scan", 100.0, Some(150.0))); + assert!(close.is_empty(), "a 1.5x difference is normal"); + } + + #[test] + fn warnings_reach_nested_nodes() { + let mut root = node(0, "Limit", 1.0, None); + root.children.push(node(7, "Seq Scan", 90_000.0, None)); + let w = warnings_for(&root); + assert_eq!(w.len(), 1); + assert_eq!(w[0].node_id, 7, "the warning points at the offending node"); + } +} diff --git a/database/src/handlers/filter.rs b/database/src/handlers/filter.rs new file mode 100644 index 000000000..a487cb145 --- /dev/null +++ b/database/src/handlers/filter.rs @@ -0,0 +1,744 @@ +//! Typed filters and sorts, compiled to dialect SQL here rather than in the +//! caller. +//! +//! A caller says "email contains test and plan equals free" as data; this +//! module turns that into a parameterised `WHERE` clause for the driver in +//! hand. That matters for correctness as much as convenience — the traps +//! below are ones every hand-written client filter gets wrong at least once: +//! +//! * **LIKE metacharacters.** `contains "50%"` must match a literal percent, +//! not every row. Values are escaped and the clause carries `ESCAPE '\'`. +//! * **`IS NULL` is not `= ''`.** They are separate operators, because +//! conflating them silently changes the answer. +//! * **Case-insensitivity is not portable.** Postgres expresses it as +//! `ILIKE`; sqlite and mysql apply it through collation and cannot honour a +//! per-query flag. Asking for it there is rejected rather than ignored. +//! * **Placeholders differ.** Postgres numbers them, the others do not. + +use crate::config::DriverKind; +use crate::error::DbError; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// Escape character for `LIKE` patterns. Backslash is the conventional choice +/// and is stated explicitly in every clause so no driver default applies. +const LIKE_ESCAPE: char = '\\'; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FilterOp { + Contains, + NotContains, + Equals, + NotEquals, + StartsWith, + EndsWith, + Gt, + Gte, + Lt, + Lte, + /// Inclusive range; needs both `value` and `value2`. + Between, + IsTrue, + IsFalse, + IsNull, + IsNotNull, + /// NULL or the empty string. Distinct from `is_null` on purpose. + IsEmpty, + /// Set membership, over `values`. Expressing "status is one of open, + /// pending, held" as three OR'd equality filters is not possible here — + /// filters stack with AND — so without this the question cannot be asked + /// at all. + In, + NotIn, +} + +impl FilterOp { + /// How many operands the operator consumes. Used to reject an incomplete + /// filter up front instead of compiling something that means nothing. + fn arity(self) -> usize { + match self { + FilterOp::IsTrue + | FilterOp::IsFalse + | FilterOp::IsNull + | FilterOp::IsNotNull + | FilterOp::IsEmpty => 0, + FilterOp::Between => 2, + // Variadic: operands come from `values`, not `value`. + FilterOp::In | FilterOp::NotIn => 0, + _ => 1, + } + } + + fn is_like(self) -> bool { + matches!( + self, + FilterOp::Contains | FilterOp::NotContains | FilterOp::StartsWith | FilterOp::EndsWith + ) + } + + fn is_set(self) -> bool { + matches!(self, FilterOp::In | FilterOp::NotIn) + } +} + +#[derive(Debug, Clone, Deserialize, JsonSchema)] +pub struct FilterSpec { + pub column: String, + pub op: FilterOp, + #[serde(default)] + pub value: Option, + /// Upper bound for `between`. + #[serde(default)] + pub value2: Option, + /// Operands for `in` / `not_in`. + #[serde(default)] + pub values: Vec, + /// Postgres only. Rejected elsewhere rather than silently ignored. + #[serde(default)] + pub case_sensitive: Option, + /// Kept in the list but not applied. A caller refining a query wants to + /// switch one condition off and back on without losing how it was built, + /// and a console that only offers delete makes that a retype. + #[serde(default)] + pub disabled: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum Direction { + Asc, + Desc, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum NullsPosition { + First, + Last, +} + +/// Type-aware sort modes. These exist server-side because the grid is paged: +/// sorting the fetched page would order 50 rows out of N, which is a +/// different answer from sorting the table. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum SortMode { + #[default] + Default, + /// `item2` before `item10`. The mode users actually notice. + Natural, + Length, + AbsoluteValue, + Random, +} + +#[derive(Debug, Clone, Deserialize, JsonSchema)] +pub struct SortSpec { + pub column: String, + #[serde(default = "default_direction")] + pub direction: Direction, + #[serde(default)] + pub nulls: Option, + #[serde(default)] + pub mode: SortMode, +} + +fn default_direction() -> Direction { + Direction::Asc +} + +/// Quote an identifier for the driver, doubling the quote character so a +/// crafted column name cannot escape it. +pub fn quote_ident(driver: DriverKind, ident: &str) -> String { + match driver { + DriverKind::Mysql => format!("`{}`", ident.replace('`', "``")), + _ => format!("\"{}\"", ident.replace('"', "\"\"")), + } +} + +/// Qualify a table reference. Only postgres has a schema above the table. +pub fn quote_table(driver: DriverKind, schema: Option<&str>, table: &str) -> String { + match (driver, schema) { + (DriverKind::Postgres, Some(s)) => { + format!("{}.{}", quote_ident(driver, s), quote_ident(driver, table)) + } + _ => quote_ident(driver, table), + } +} + +/// Emits `$1`, `$2`, … on postgres and `?` elsewhere. +struct Placeholders { + driver: DriverKind, + next: usize, +} + +impl Placeholders { + fn new(driver: DriverKind, start: usize) -> Self { + Self { + driver, + next: start, + } + } + + fn take(&mut self) -> String { + let n = self.next; + self.next += 1; + match self.driver { + DriverKind::Postgres => format!("${n}"), + _ => "?".to_string(), + } + } +} + +fn invalid(reason: impl Into) -> DbError { + DbError::InvalidParam { + index: 0, + reason: reason.into(), + } +} + +/// Escape `%`, `_` and the escape character itself so they match literally. +fn escape_like(raw: &str) -> String { + let mut out = String::with_capacity(raw.len()); + for ch in raw.chars() { + if ch == '%' || ch == '_' || ch == LIKE_ESCAPE { + out.push(LIKE_ESCAPE); + } + out.push(ch); + } + out +} + +fn as_text(value: &Value, column: &str) -> Result { + match value { + Value::String(s) => Ok(s.clone()), + Value::Number(n) => Ok(n.to_string()), + Value::Bool(b) => Ok(b.to_string()), + _ => Err(invalid(format!( + "filter on `{column}` needs a text-comparable value" + ))), + } +} + +#[derive(Debug, Clone)] +pub struct WhereClause { + /// Empty when there are no filters; otherwise a bare boolean expression + /// with no leading `WHERE`, so callers can compose it. + pub sql: String, + pub params: Vec, +} + +/// Compile filters into a parameterised predicate. `start_param` is the first +/// free placeholder index, so a caller that already bound values keeps +/// numbering correct on postgres. +pub fn compile_where( + driver: DriverKind, + filters: &[FilterSpec], + start_param: usize, +) -> Result { + let mut ph = Placeholders::new(driver, start_param); + let mut parts: Vec = Vec::new(); + let mut params: Vec = Vec::new(); + + for f in filters { + // A disabled filter is skipped, not rejected: it stays in the caller's + // list so it can be switched back on without being rebuilt. + if f.disabled { + continue; + } + if f.column.trim().is_empty() { + return Err(invalid("filter is missing a column")); + } + // Check operands by position, not by count. Counting accepts + // `{op: equals, value2: 5}` — one operand supplied for a one-operand + // operator — and the compiler then unwraps the `value` that is not + // there. The request body is caller-controlled, so that is a panic + // reachable from the wire. + let arity = f.op.arity(); + if arity >= 1 && f.value.is_none() { + return Err(invalid(format!("filter on `{}` needs `value`", f.column))); + } + if arity >= 2 && f.value2.is_none() { + return Err(invalid(format!( + "filter on `{}` needs both `value` and `value2`", + f.column + ))); + } + if f.op.is_set() && f.values.is_empty() { + // `IN ()` is a syntax error on every driver, and silently + // dropping the filter would quietly widen the result set. + return Err(invalid(format!( + "filter on `{}` needs at least one value in `values`", + f.column + ))); + } + + // Case sensitivity is only expressible on postgres. Everywhere else + // it is a property of the column's collation, so honouring the flag + // would be a lie and ignoring it would be worse. + // Rejected whenever it cannot be honoured, which includes operators + // that never consult it. Accepting `{op: equals, case_sensitive: true}` + // and quietly doing nothing is the failure the module header warns + // about, just in a different place. + if f.case_sensitive.is_some() && !(driver == DriverKind::Postgres && f.op.is_like()) { + return Err(invalid( + "case_sensitive applies only to postgres pattern operators; \ + elsewhere it is determined by the column collation", + )); + } + + let col = quote_ident(driver, &f.column); + let sensitive = f.case_sensitive.unwrap_or(false); + + let part = match f.op { + FilterOp::IsNull => format!("{col} IS NULL"), + FilterOp::IsNotNull => format!("{col} IS NOT NULL"), + FilterOp::IsTrue => format!("{col} = TRUE"), + FilterOp::IsFalse => format!("{col} = FALSE"), + // NULL *or* empty string — the distinction from is_null is the point. + FilterOp::IsEmpty => format!("({col} IS NULL OR {col} = '')"), + + FilterOp::In | FilterOp::NotIn => { + let marks: Vec = f + .values + .iter() + .map(|v| { + params.push(v.clone()); + ph.take() + }) + .collect(); + let list = marks.join(", "); + if f.op == FilterOp::In { + format!("{col} IN ({list})") + } else { + // NOT IN drops NULLs on every driver, because `NULL <> x` + // is unknown. A reader asking for "not one of these" means + // to keep the NULLs, so say so explicitly. + format!("({col} IS NULL OR {col} NOT IN ({list}))") + } + } + + FilterOp::Contains + | FilterOp::NotContains + | FilterOp::StartsWith + | FilterOp::EndsWith => { + let raw = as_text(f.value.as_ref().expect("arity checked"), &f.column)?; + let escaped = escape_like(&raw); + let pattern = match f.op { + FilterOp::StartsWith => format!("{escaped}%"), + FilterOp::EndsWith => format!("%{escaped}"), + _ => format!("%{escaped}%"), + }; + let negate = f.op == FilterOp::NotContains; + let op = match (driver, sensitive) { + (DriverKind::Postgres, false) => "ILIKE", + _ => "LIKE", + }; + params.push(Value::String(pattern)); + let p = ph.take(); + let expr = format!("{col} {op} {p} ESCAPE '{LIKE_ESCAPE}'"); + if negate { + // A NULL never matches LIKE, so a naive NOT LIKE drops + // NULL rows the user would expect to see in "does not + // contain". + format!("({col} IS NULL OR NOT ({expr}))") + } else { + expr + } + } + + FilterOp::Between => { + params.push(f.value.clone().expect("arity checked")); + let lo = ph.take(); + params.push(f.value2.clone().expect("arity checked")); + let hi = ph.take(); + format!("{col} BETWEEN {lo} AND {hi}") + } + + _ => { + let sym = match f.op { + FilterOp::Equals => "=", + FilterOp::NotEquals => "<>", + FilterOp::Gt => ">", + FilterOp::Gte => ">=", + FilterOp::Lt => "<", + FilterOp::Lte => "<=", + other => unreachable!("{other:?} handled above"), + }; + params.push(f.value.clone().expect("arity checked")); + let p = ph.take(); + format!("{col} {sym} {p}") + } + }; + parts.push(part); + } + + Ok(WhereClause { + sql: parts.join(" AND "), + params, + }) +} + +/// Compile sorts into an `ORDER BY` body (no leading keyword). Column names +/// are quoted; modes that a driver cannot express fall back to plain ordering +/// rather than erroring, because a sort is a presentation choice and refusing +/// one would be worse than approximating it. +pub fn compile_order_by(driver: DriverKind, sorts: &[SortSpec]) -> Result { + let mut parts = Vec::new(); + for s in sorts { + if s.column.trim().is_empty() { + return Err(invalid("sort is missing a column")); + } + let col = quote_ident(driver, &s.column); + let expr = match s.mode { + SortMode::Default => col.clone(), + SortMode::Length => match driver { + DriverKind::Mysql => format!("CHAR_LENGTH({col})"), + _ => format!("LENGTH({col})"), + }, + SortMode::AbsoluteValue => format!("ABS({col})"), + SortMode::Random => match driver { + DriverKind::Postgres => "RANDOM()".to_string(), + DriverKind::Mysql => "RAND()".to_string(), + DriverKind::Sqlite => "RANDOM()".to_string(), + }, + // Natural order: pad the leading digit run so `item2` sorts before + // `item10`. Postgres can express this inline; the others have no + // portable equivalent, so they order plainly rather than pretend. + SortMode::Natural => match driver { + DriverKind::Postgres => format!( + "regexp_replace({col}, '\\d+', lpad(substring({col} from '\\d+'), 12, '0'))" + ), + _ => col.clone(), + }, + }; + + let dir = match s.direction { + Direction::Asc => "ASC", + Direction::Desc => "DESC", + }; + let mut term = format!("{expr} {dir}"); + if let Some(nulls) = s.nulls { + let n = match nulls { + NullsPosition::First => "FIRST", + NullsPosition::Last => "LAST", + }; + match driver { + // MySQL has no NULLS FIRST/LAST; emulate with a leading key. + DriverKind::Mysql => { + let flip = matches!(nulls, NullsPosition::First); + term = format!("{col} IS NOT NULL = {}, {term}", u8::from(flip)); + } + _ => term.push_str(&format!(" NULLS {n}")), + } + } + parts.push(term); + } + Ok(parts.join(", ")) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn spec(column: &str, op: FilterOp, value: Option) -> FilterSpec { + FilterSpec { + column: column.into(), + op, + value, + value2: None, + values: Vec::new(), + case_sensitive: None, + disabled: false, + } + } + + #[test] + fn no_filters_compiles_to_an_empty_clause() { + let w = compile_where(DriverKind::Sqlite, &[], 1).unwrap(); + assert!(w.sql.is_empty()); + assert!(w.params.is_empty()); + } + + #[test] + fn like_metacharacters_are_escaped_so_a_literal_percent_matches() { + let w = compile_where( + DriverKind::Sqlite, + &[spec("code", FilterOp::Contains, Some(json!("50%")))], + 1, + ) + .unwrap(); + // Without escaping this pattern would match every row. + assert_eq!(w.params[0], json!("%50\\%%")); + assert!(w.sql.contains("ESCAPE '\\'"), "got: {}", w.sql); + } + + #[test] + fn underscore_and_the_escape_char_are_escaped_too() { + let w = compile_where( + DriverKind::Sqlite, + &[spec("p", FilterOp::StartsWith, Some(json!("a_b\\c")))], + 1, + ) + .unwrap(); + assert_eq!(w.params[0], json!("a\\_b\\\\c%")); + } + + #[test] + fn postgres_numbers_placeholders_and_others_do_not() { + let filters = [ + spec("a", FilterOp::Equals, Some(json!(1))), + spec("b", FilterOp::Equals, Some(json!(2))), + ]; + let pg = compile_where(DriverKind::Postgres, &filters, 1).unwrap(); + assert_eq!(pg.sql, r#""a" = $1 AND "b" = $2"#); + + let my = compile_where(DriverKind::Mysql, &filters, 1).unwrap(); + assert_eq!(my.sql, "`a` = ? AND `b` = ?"); + } + + #[test] + fn placeholder_numbering_continues_from_the_callers_offset() { + let w = compile_where( + DriverKind::Postgres, + &[spec("a", FilterOp::Equals, Some(json!(1)))], + 4, + ) + .unwrap(); + assert_eq!(w.sql, r#""a" = $4"#); + } + + #[test] + fn is_null_and_is_empty_are_different_questions() { + let null = + compile_where(DriverKind::Sqlite, &[spec("a", FilterOp::IsNull, None)], 1).unwrap(); + assert_eq!(null.sql, r#""a" IS NULL"#); + assert!(null.params.is_empty()); + + let empty = + compile_where(DriverKind::Sqlite, &[spec("a", FilterOp::IsEmpty, None)], 1).unwrap(); + assert_eq!(empty.sql, r#"("a" IS NULL OR "a" = '')"#); + } + + #[test] + fn not_contains_keeps_null_rows() { + let w = compile_where( + DriverKind::Sqlite, + &[spec("a", FilterOp::NotContains, Some(json!("x")))], + 1, + ) + .unwrap(); + // A bare NOT LIKE would drop NULLs, which reads as data loss. + assert!( + w.sql.starts_with(r#"("a" IS NULL OR NOT ("#), + "got: {}", + w.sql + ); + } + + #[test] + fn case_insensitive_uses_ilike_on_postgres_only() { + let pg = compile_where( + DriverKind::Postgres, + &[spec("a", FilterOp::Contains, Some(json!("x")))], + 1, + ) + .unwrap(); + assert!(pg.sql.contains("ILIKE"), "got: {}", pg.sql); + + let lite = compile_where( + DriverKind::Sqlite, + &[spec("a", FilterOp::Contains, Some(json!("x")))], + 1, + ) + .unwrap(); + assert!(lite.sql.contains("LIKE") && !lite.sql.contains("ILIKE")); + } + + #[test] + fn case_sensitive_flag_is_rejected_where_it_cannot_be_honoured() { + let mut f = spec("a", FilterOp::Contains, Some(json!("x"))); + f.case_sensitive = Some(true); + let err = compile_where(DriverKind::Sqlite, &[f], 1).unwrap_err(); + let body = serde_json::to_string(&err).unwrap(); + assert!(body.contains("postgres pattern operators"), "got: {body}"); + } + + #[test] + fn an_incomplete_filter_is_refused_rather_than_guessed() { + let err = + compile_where(DriverKind::Sqlite, &[spec("a", FilterOp::Equals, None)], 1).unwrap_err(); + let body = serde_json::to_string(&err).unwrap(); + assert!(body.contains("needs `value`"), "got: {body}"); + + let mut between = spec("a", FilterOp::Between, Some(json!(1))); + between.value2 = None; + let err = compile_where(DriverKind::Sqlite, &[between], 1).unwrap_err(); + let body = serde_json::to_string(&err).unwrap(); + assert!(body.contains("`value2`"), "got: {body}"); + } + + #[test] + fn between_binds_both_bounds_in_order() { + let mut f = spec("n", FilterOp::Between, Some(json!(1))); + f.value2 = Some(json!(9)); + let w = compile_where(DriverKind::Postgres, &[f], 1).unwrap(); + assert_eq!(w.sql, r#""n" BETWEEN $1 AND $2"#); + assert_eq!(w.params, vec![json!(1), json!(9)]); + } + + #[test] + fn quoting_defeats_an_identifier_break_out() { + assert_eq!( + quote_ident(DriverKind::Postgres, r#"a" OR 1=1--"#), + r#""a"" OR 1=1--""# + ); + assert_eq!(quote_ident(DriverKind::Mysql, "a`b"), "`a``b`"); + } + + #[test] + fn table_is_schema_qualified_only_on_postgres() { + assert_eq!( + quote_table(DriverKind::Postgres, Some("analytics"), "events"), + r#""analytics"."events""# + ); + assert_eq!( + quote_table(DriverKind::Mysql, Some("ignored"), "events"), + "`events`" + ); + } + + #[test] + fn order_by_emits_direction_and_nulls_placement() { + let sorts = [SortSpec { + column: "created_at".into(), + direction: Direction::Desc, + nulls: Some(NullsPosition::Last), + mode: SortMode::Default, + }]; + let pg = compile_order_by(DriverKind::Postgres, &sorts).unwrap(); + assert_eq!(pg, r#""created_at" DESC NULLS LAST"#); + + // MySQL has no NULLS clause, so it emulates with a leading key. + let my = compile_order_by(DriverKind::Mysql, &sorts).unwrap(); + assert!(my.contains("IS NOT NULL ="), "got: {my}"); + assert!(!my.contains("NULLS"), "got: {my}"); + } + + #[test] + fn length_mode_uses_the_drivers_own_function() { + let s = [SortSpec { + column: "name".into(), + direction: Direction::Asc, + nulls: None, + mode: SortMode::Length, + }]; + assert_eq!( + compile_order_by(DriverKind::Mysql, &s).unwrap(), + "CHAR_LENGTH(`name`) ASC" + ); + assert_eq!( + compile_order_by(DriverKind::Sqlite, &s).unwrap(), + r#"LENGTH("name") ASC"# + ); + } + + #[test] + fn the_wrong_operand_alone_is_refused_rather_than_unwrapped() { + // `value2` without `value` used to satisfy a count-based arity check + // and then panic on the missing `value`. Reachable from the wire. + let mut f = spec("a", FilterOp::Equals, None); + f.value2 = Some(json!(5)); + let err = compile_where(DriverKind::Sqlite, &[f], 1).unwrap_err(); + assert!(format!("{err:?}").contains("needs `value`")); + } + + #[test] + fn between_needs_both_bounds_not_just_a_count_of_two() { + let mut f = spec("a", FilterOp::Between, None); + f.value2 = Some(json!(5)); + assert!(compile_where(DriverKind::Sqlite, &[f], 1).is_err()); + } + + #[test] + fn case_sensitive_is_refused_where_it_would_do_nothing() { + // Not a pattern operator, so the flag can never apply — even on the + // one driver that supports it for LIKE. + let mut f = spec("a", FilterOp::Equals, Some(json!("x"))); + f.case_sensitive = Some(true); + assert!(compile_where(DriverKind::Postgres, &[f], 1).is_err()); + + // Still accepted where it is honoured. + let mut ok = spec("a", FilterOp::Contains, Some(json!("x"))); + ok.case_sensitive = Some(true); + assert!(compile_where(DriverKind::Postgres, &[ok], 1).is_ok()); + } + + #[test] + fn in_binds_every_member_and_numbers_them_on_postgres() { + let mut f = spec("status", FilterOp::In, None); + f.values = vec![json!("open"), json!("held")]; + let c = compile_where(DriverKind::Postgres, &[f], 1).unwrap(); + assert_eq!(c.sql, r#""status" IN ($1, $2)"#); + assert_eq!(c.params, vec![json!("open"), json!("held")]); + } + + #[test] + fn not_in_keeps_nulls() { + // `NULL NOT IN (...)` is unknown, so a plain NOT IN silently drops + // every NULL row. Asking for "not one of these" should keep them. + let mut f = spec("status", FilterOp::NotIn, None); + f.values = vec![json!("open")]; + let c = compile_where(DriverKind::Sqlite, &[f], 1).unwrap(); + assert_eq!(c.sql, r#"("status" IS NULL OR "status" NOT IN (?))"#); + } + + #[test] + fn an_empty_set_is_refused_rather_than_dropped() { + let f = spec("status", FilterOp::In, None); + let err = compile_where(DriverKind::Sqlite, &[f], 1).unwrap_err(); + assert!(format!("{err:?}").contains("at least one value")); + } + + #[test] + fn a_disabled_filter_is_skipped_but_its_neighbours_still_compile() { + let mut off = spec("plan", FilterOp::Equals, Some(json!("free"))); + off.disabled = true; + let on = spec("status", FilterOp::Equals, Some(json!("open"))); + let c = compile_where(DriverKind::Postgres, &[off, on], 1).unwrap(); + // Numbering must close up: the skipped filter must not burn $1. + assert_eq!(c.sql, r#""status" = $1"#); + assert_eq!(c.params, vec![json!("open")]); + } + + #[test] + fn a_disabled_filter_is_not_validated() { + // Half-built filters are the normal state of a chip being edited; + // disabling one must not turn the whole request into an error. + let mut half = spec("", FilterOp::Equals, None); + half.disabled = true; + let c = compile_where(DriverKind::Sqlite, &[half], 1).unwrap(); + assert_eq!(c.sql, ""); + } + + #[test] + fn natural_mode_falls_back_rather_than_faking_it() { + let s = [SortSpec { + column: "label".into(), + direction: Direction::Asc, + nulls: None, + mode: SortMode::Natural, + }]; + assert!(compile_order_by(DriverKind::Postgres, &s) + .unwrap() + .contains("regexp_replace")); + // No portable equivalent — order plainly instead of approximating. + assert_eq!( + compile_order_by(DriverKind::Sqlite, &s).unwrap(), + r#""label" ASC"# + ); + } +} diff --git a/database/src/handlers/health.rs b/database/src/handlers/health.rs new file mode 100644 index 000000000..638725ffc --- /dev/null +++ b/database/src/handlers/health.rs @@ -0,0 +1,450 @@ +//! `database::health` — live operational state, honestly reported. +//! +//! The design point is `ProbeResult`. Each section answers separately, so a +//! caller can tell "sqlite has no equivalent of `pg_stat_activity`" from +//! "there are zero active queries" from "this role may not read +//! `pg_stat_activity`". Collapsing those into an empty list would render a +//! confidently wrong panel, and a restricted application role is the common +//! case rather than the exception — so one probe being denied never fails the +//! whole call. +//! +//! Boundary against `database::testConnection`: that probes a *candidate* URL +//! that is not configured yet. This reports the live state of a pool that +//! already exists, and deliberately accepts no URL. + +use super::query::{self, err_to_str, QueryReq}; +use super::AppState; +use crate::config::DriverKind; +use crate::error::DbError; +use crate::pool::PoolStats; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +fn default_timeout() -> u64 { + 15_000 +} + +/// One section of the report. +#[derive(Debug, Clone, Serialize, JsonSchema)] +#[serde(tag = "status", rename_all = "lowercase")] +pub enum ProbeResult { + /// The driver answered. + Available { data: T }, + /// The driver has no equivalent of this concept. + Unsupported { reason: String }, + /// The driver has it, but this role may not read it. + Denied { reason: String }, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct ActiveQuery { + pub id: String, + pub sql: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub state: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct TableSize { + pub table: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub total_bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub index_bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub row_estimate: Option, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct LockInfo { + pub blocked_id: String, + pub blocked_sql: String, + pub blocking_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub blocking_sql: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub relation: Option, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct CacheStats { + /// Fraction of block reads served from cache. A healthy OLTP database + /// usually sits well above 0.99. + pub hit_ratio: f64, + pub blocks_hit: i64, + pub blocks_read: i64, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct HealthReq { + #[serde(default)] + pub db: Option, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct HealthResp { + pub db: String, + pub driver: String, + pub worker_version: String, + pub pool: PoolStats, + pub active_queries: ProbeResult>, + pub table_sizes: ProbeResult>, + pub locks: ProbeResult>, + pub cache: ProbeResult, +} + +fn unsupported(driver: DriverKind, what: &str) -> ProbeResult { + ProbeResult::Unsupported { + reason: format!("{driver:?} has no equivalent of {what}").to_lowercase(), + } +} + +/// Turn a probe failure into a per-section result rather than failing the +/// whole call. A permission error on one view must not hide the others. +fn probe(outcome: Result) -> ProbeResult { + match outcome { + Ok(data) => ProbeResult::Available { data }, + Err(reason) => ProbeResult::Denied { reason }, + } +} + +async fn rows( + state: &AppState, + db: &str, + sql: &str, + timeout_ms: u64, +) -> Result>, String> { + Ok(query::handle( + state, + QueryReq { + db: Some(db.to_string()), + sql: sql.to_string(), + params: vec![], + timeout_ms, + record_history: false, + }, + ) + .await? + .rows) +} + +fn s_at(r: &serde_json::Map, k: &str) -> Option { + match r.get(k) { + Some(Value::String(s)) => Some(s.clone()), + Some(Value::Number(n)) => Some(n.to_string()), + _ => None, + } +} + +fn i_at(r: &serde_json::Map, k: &str) -> Option { + match r.get(k) { + Some(Value::Number(n)) => n.as_i64().or_else(|| n.as_f64().map(|f| f as i64)), + Some(Value::String(s)) => s.parse().ok(), + _ => None, + } +} + +pub async fn handle(state: &AppState, req: HealthReq) -> Result { + let db = state.resolve_db(req.db).await.map_err(err_to_str)?; + let pool = state.pool(&db).await.map_err(err_to_str)?; + let driver = pool.driver(); + let t = req.timeout_ms; + + let (active_queries, table_sizes, locks, cache) = match driver { + DriverKind::Postgres => ( + probe(pg_active(state, &db, t).await), + probe(pg_sizes(state, &db, t).await), + probe(pg_locks(state, &db, t).await), + probe(pg_cache(state, &db, t).await), + ), + DriverKind::Mysql => ( + probe(mysql_active(state, &db, t).await), + probe(mysql_sizes(state, &db, t).await), + unsupported(driver, "a queryable lock-wait graph"), + unsupported(driver, "a per-database buffer-pool hit ratio"), + ), + DriverKind::Sqlite => ( + // SQLite runs in-process: there is no server holding sessions. + unsupported(driver, "server-side sessions"), + unsupported(driver, "per-table size accounting"), + unsupported(driver, "a queryable lock table"), + unsupported(driver, "a shared buffer cache"), + ), + }; + + Ok(HealthResp { + db, + driver: format!("{driver:?}").to_lowercase(), + worker_version: env!("CARGO_PKG_VERSION").to_string(), + pool: pool.stats(), + active_queries, + table_sizes, + locks, + cache, + }) +} + +/* ---------------- postgres ---------------- */ + +async fn pg_active(state: &AppState, db: &str, t: u64) -> Result, String> { + let r = rows( + state, + db, + "SELECT pid::text AS id, query AS sql, state, usename AS usr, \ + (EXTRACT(EPOCH FROM (now() - query_start)) * 1000)::bigint AS duration_ms \ + FROM pg_stat_activity \ + WHERE datname = current_database() AND pid <> pg_backend_pid() \ + AND state <> 'idle' \ + ORDER BY query_start", + t, + ) + .await?; + Ok(r.iter() + .filter_map(|x| { + Some(ActiveQuery { + id: s_at(x, "id")?, + sql: s_at(x, "sql").unwrap_or_default(), + state: s_at(x, "state"), + duration_ms: i_at(x, "duration_ms"), + user: s_at(x, "usr"), + }) + }) + .collect()) +} + +async fn pg_sizes(state: &AppState, db: &str, t: u64) -> Result, String> { + let r = rows( + state, + db, + "SELECT n.nspname AS schema_name, c.relname AS table_name, \ + pg_total_relation_size(c.oid)::bigint AS total_bytes, \ + pg_indexes_size(c.oid)::bigint AS index_bytes, \ + c.reltuples::bigint AS row_estimate \ + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE c.relkind IN ('r', 'p') \ + AND n.nspname NOT IN ('pg_catalog', 'information_schema') \ + ORDER BY pg_total_relation_size(c.oid) DESC LIMIT 100", + t, + ) + .await?; + Ok(r.iter() + .filter_map(|x| { + Some(TableSize { + table: s_at(x, "table_name")?, + schema: s_at(x, "schema_name"), + total_bytes: i_at(x, "total_bytes"), + index_bytes: i_at(x, "index_bytes"), + row_estimate: i_at(x, "row_estimate").filter(|v| *v >= 0), + }) + }) + .collect()) +} + +/// Resolves blockers by joining `pg_locks` to itself. Deliberately avoids +/// `pg_blocking_pids()`, which returns `int[]` — and `RowValue` has no array +/// variant, so that column would fail to decode. +async fn pg_locks(state: &AppState, db: &str, t: u64) -> Result, String> { + let r = rows( + state, + db, + "SELECT w.pid::text AS blocked_id, wa.query AS blocked_sql, \ + b.pid::text AS blocking_id, ba.query AS blocking_sql, \ + COALESCE(c.relname, '') AS relation \ + FROM pg_locks w \ + JOIN pg_locks b ON b.granted AND NOT w.granted \ + AND b.pid <> w.pid \ + AND b.locktype = w.locktype \ + AND b.database IS NOT DISTINCT FROM w.database \ + AND b.relation IS NOT DISTINCT FROM w.relation \ + AND b.transactionid IS NOT DISTINCT FROM w.transactionid \ + JOIN pg_stat_activity wa ON wa.pid = w.pid \ + LEFT JOIN pg_stat_activity ba ON ba.pid = b.pid \ + LEFT JOIN pg_class c ON c.oid = w.relation \ + WHERE NOT w.granted", + t, + ) + .await?; + Ok(r.iter() + .filter_map(|x| { + Some(LockInfo { + blocked_id: s_at(x, "blocked_id")?, + blocked_sql: s_at(x, "blocked_sql").unwrap_or_default(), + blocking_id: s_at(x, "blocking_id")?, + blocking_sql: s_at(x, "blocking_sql"), + relation: s_at(x, "relation").filter(|s| !s.is_empty()), + }) + }) + .collect()) +} + +async fn pg_cache(state: &AppState, db: &str, t: u64) -> Result { + let r = rows( + state, + db, + "SELECT COALESCE(SUM(heap_blks_hit), 0)::bigint AS hit, \ + COALESCE(SUM(heap_blks_read), 0)::bigint AS rd \ + FROM pg_statio_user_tables", + t, + ) + .await?; + let row = r.first().cloned().unwrap_or_default(); + let hit = i_at(&row, "hit").unwrap_or(0); + let read = i_at(&row, "rd").unwrap_or(0); + let total = hit + read; + Ok(CacheStats { + // No reads yet is not a 0% hit rate; report it as perfect rather than + // as an alarming zero. + hit_ratio: if total == 0 { + 1.0 + } else { + hit as f64 / total as f64 + }, + blocks_hit: hit, + blocks_read: read, + }) +} + +/* ---------------- mysql ---------------- */ + +async fn mysql_active(state: &AppState, db: &str, t: u64) -> Result, String> { + let r = rows( + state, + db, + "SELECT ID AS id, INFO AS sql_text, STATE AS state, USER AS usr, \ + TIME * 1000 AS duration_ms \ + FROM information_schema.PROCESSLIST \ + WHERE DB = DATABASE() AND COMMAND <> 'Sleep' AND ID <> CONNECTION_ID() \ + ORDER BY TIME DESC", + t, + ) + .await?; + Ok(r.iter() + .filter_map(|x| { + Some(ActiveQuery { + id: s_at(x, "id")?, + sql: s_at(x, "sql_text").unwrap_or_default(), + state: s_at(x, "state"), + duration_ms: i_at(x, "duration_ms"), + user: s_at(x, "usr"), + }) + }) + .collect()) +} + +async fn mysql_sizes(state: &AppState, db: &str, t: u64) -> Result, String> { + let r = rows( + state, + db, + "SELECT TABLE_NAME AS table_name, \ + (DATA_LENGTH + INDEX_LENGTH) AS total_bytes, \ + INDEX_LENGTH AS index_bytes, TABLE_ROWS AS row_estimate \ + FROM information_schema.TABLES \ + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_TYPE = 'BASE TABLE' \ + ORDER BY (DATA_LENGTH + INDEX_LENGTH) DESC LIMIT 100", + t, + ) + .await?; + Ok(r.iter() + .filter_map(|x| { + Some(TableSize { + table: s_at(x, "table_name")?, + schema: None, + total_bytes: i_at(x, "total_bytes"), + index_bytes: i_at(x, "index_bytes"), + row_estimate: i_at(x, "row_estimate"), + }) + }) + .collect()) +} + +/* ---------------- terminateQuery ---------------- */ + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct TerminateReq { + #[serde(default)] + pub db: Option, + /// Backend pid (postgres) or connection id (mysql), as reported by + /// `database::health`. + pub id: String, + /// Ask the backend to cancel the running statement but keep the session. + /// The default terminates the session outright. + #[serde(default)] + pub cancel_only: bool, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct TerminateResp { + pub id: String, + pub terminated: bool, +} + +/// A separate function from `health` on purpose: this is a write, and a +/// read-only viewer must not be able to synthesise it from a report. +pub async fn terminate(state: &AppState, req: TerminateReq) -> Result { + let db = state.resolve_db(req.db).await.map_err(err_to_str)?; + let pool = state.pool(&db).await.map_err(err_to_str)?; + let driver = pool.driver(); + + // The id is interpolated, so it must be exactly a number — never trust it + // as an identifier. + let id: i64 = req.id.trim().parse().map_err(|_| { + err_to_str(DbError::InvalidParam { + index: 0, + reason: format!("`{}` is not a backend id", req.id), + }) + })?; + + let sql = match (driver, req.cancel_only) { + (DriverKind::Postgres, true) => format!("SELECT pg_cancel_backend({id}) AS ok"), + (DriverKind::Postgres, false) => format!("SELECT pg_terminate_backend({id}) AS ok"), + (DriverKind::Mysql, true) => format!("KILL QUERY {id}"), + (DriverKind::Mysql, false) => format!("KILL CONNECTION {id}"), + (DriverKind::Sqlite, _) => { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: "sqlite runs in-process and has no sessions to terminate".into(), + })) + } + }; + + let terminated = match driver { + DriverKind::Postgres => { + let r = rows(state, &db, &sql, req.timeout_ms).await?; + r.first() + .and_then(|x| x.get("ok")) + .and_then(Value::as_bool) + .unwrap_or(false) + } + // KILL returns no result set; reaching here without an error is the + // signal. + _ => { + super::execute::handle( + state, + super::execute::ExecuteReq { + db: Some(db.clone()), + sql: sql.clone(), + params: vec![], + returning: vec![], + }, + ) + .await?; + true + } + }; + + Ok(TerminateResp { + id: req.id, + terminated, + }) +} diff --git a/database/src/handlers/mod.rs b/database/src/handlers/mod.rs index 783fad377..16087c543 100644 --- a/database/src/handlers/mod.rs +++ b/database/src/handlers/mod.rs @@ -30,14 +30,24 @@ where } pub mod begin_transaction; +pub mod browse; +pub mod catalog; +pub mod column_stats; pub mod commit_transaction; +pub mod diagram; pub mod execute; pub mod execute_batch; +pub mod explain; +pub mod filter; +pub mod health; pub mod list_databases; pub mod prepare; pub mod query; pub mod rollback_transaction; pub mod run_statement; +pub mod saved; +pub mod schema; +pub mod table_view; pub mod test_connection; pub mod transaction; pub mod transaction_execute; @@ -97,6 +107,12 @@ impl AppState { .is_none_or(|d| d.capture.is_statements()) } + /// Engine client, when the worker is connected to one. `None` in tests, + /// where sibling-worker calls are not available and history is skipped. + pub fn client(&self) -> Option<&Arc> { + self.row_changes.as_ref().map(|bus| bus.client()) + } + /// Announce a committed change, if anything is listening. Every mutating /// handler ends with this; the bus decides whether the statement changed /// rows and who cares. diff --git a/database/src/handlers/query.rs b/database/src/handlers/query.rs index 3dde87fd9..411e80549 100644 --- a/database/src/handlers/query.rs +++ b/database/src/handlers/query.rs @@ -21,6 +21,20 @@ pub struct QueryReq { pub params: Vec, #[serde(default = "default_timeout")] pub timeout_ms: u64, + /// Whether this run belongs in `database::history`. + /// + /// Internal only — never deserialized, so a statement arriving over the + /// wire always records. The catalog reads behind `describeSchema`, + /// `browseTable`, `columnStats`, `explain` and `schemaDiagram` all run + /// through this same path, and recording them would bury the statements a + /// person actually typed under `PRAGMA table_info(...)` and spawn a + /// `state::update` per internal read. + #[serde(skip, default = "records_history")] + pub record_history: bool, +} + +fn records_history() -> bool { + true } #[derive(Debug, Serialize, JsonSchema)] @@ -55,6 +69,7 @@ pub async fn handle(state: &AppState, req: QueryReq) -> Result driver::sqlite::query(p, &req.sql, ¶ms, req.timeout_ms).await, Pool::Postgres(p) => driver::postgres::query(p, &req.sql, ¶ms, req.timeout_ms).await, @@ -63,6 +78,18 @@ pub async fn handle(state: &AppState, req: QueryReq) -> Result String { +pub fn err_to_str(e: DbError) -> String { serde_json::to_string(&e).unwrap_or_else(|_| { format!( "{{\"code\":\"DRIVER_ERROR\",\"message\":{:?}}}", diff --git a/database/src/handlers/saved.rs b/database/src/handlers/saved.rs new file mode 100644 index 000000000..89ada1171 --- /dev/null +++ b/database/src/handlers/saved.rs @@ -0,0 +1,368 @@ +//! `database::saveQuery` / `listSavedQueries` / `deleteSavedQuery` / +//! `history` — the queries you keep, and the ones you ran. +//! +//! These are thin wrappers over the `state` worker, not a store of their own. +//! Query history used to live in browser `localStorage`, which meant it +//! existed for one person, in one browser, and vanished on a cache clear. On +//! `state::*` it survives restarts, any agent can read it, and an agent can +//! save a query for a human to find in the console. +//! +//! Recording is deliberately cheap and deliberately lossy: it is fire and +//! forget, never awaited on the query path, and a `state` failure never fails +//! the user's query. History is a convenience, not an audit log — for an +//! audit trail, bind the `database::row-changed` trigger instead. + +use super::query::err_to_str; +use crate::error::DbError; +use iii_sdk::protocol::TriggerRequest; +use iii_sdk::IIIClient; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::sync::Arc; + +/// Scope every key lives under in the `state` worker. +pub(super) const SCOPE: &str = "database"; +/// Entries returned by `history` unless the caller asks for fewer. +const HISTORY_LIMIT: usize = 50; +/// Stored entries are trimmed back to `HISTORY_LIMIT` once they exceed this. +/// Trimming on read rather than on write keeps the hot path a single atomic +/// append instead of a read-modify-write. +const HISTORY_HIGH_WATER: usize = 200; +/// SQL longer than this is truncated before it is stored. +const MAX_SQL_CHARS: usize = 4_000; + +fn saved_key(db: &str) -> String { + format!("saved:{db}") +} + +fn history_key(db: &str) -> String { + format!("history:{db}") +} + +pub(super) async fn call( + iii: &Arc, + function_id: &str, + payload: Value, +) -> Result { + iii.trigger(TriggerRequest { + function_id: function_id.to_string(), + payload, + action: None, + timeout_ms: Some(10_000), + }) + .await + .map_err(|e| { + err_to_str(DbError::ConfigError { + message: format!( + "{function_id} failed: {e}. Saved queries and history need the `state` \ + worker — run `iii worker add state`." + ), + }) + }) +} + +async fn state_get(iii: &Arc, key: &str) -> Result, String> { + let raw = call(iii, "state::get", json!({"scope": SCOPE, "key": key})).await?; + // A missing key is an empty list, not an error. + Ok(raw + .get("value") + .or(Some(&raw)) + .and_then(Value::as_array) + .cloned() + .unwrap_or_default()) +} + +async fn state_set(iii: &Arc, key: &str, value: Value) -> Result<(), String> { + call( + iii, + "state::set", + json!({"scope": SCOPE, "key": key, "value": value}), + ) + .await?; + Ok(()) +} + +/* ---------------- saved queries ---------------- */ + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct SavedQuery { + pub id: String, + pub name: String, + pub sql: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + pub saved_at: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct SaveQueryReq { + #[serde(default)] + pub db: Option, + pub name: String, + pub sql: String, + #[serde(default)] + pub description: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct SaveQueryResp { + pub id: String, + pub replaced: bool, +} + +pub async fn save( + iii: &Arc, + db: &str, + req: SaveQueryReq, +) -> Result { + if req.name.trim().is_empty() || req.sql.trim().is_empty() { + return Err(err_to_str(DbError::InvalidParam { + index: 0, + reason: "name and sql are both required".into(), + })); + } + let key = saved_key(db); + let mut items: Vec = state_get(iii, &key) + .await? + .into_iter() + .filter_map(|v| serde_json::from_value(v).ok()) + .collect(); + + // Saving under an existing name replaces it rather than accumulating + // near-duplicates the user then has to tell apart. + let replaced = items.iter().any(|q| q.name == req.name); + let id = items + .iter() + .find(|q| q.name == req.name) + .map(|q| q.id.clone()) + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + items.retain(|q| q.name != req.name); + items.push(SavedQuery { + id: id.clone(), + name: req.name, + sql: truncate(&req.sql), + description: req.description, + saved_at: now(), + }); + items.sort_by(|a, b| a.name.cmp(&b.name)); + + state_set(iii, &key, serde_json::to_value(&items).unwrap_or(json!([]))).await?; + Ok(SaveQueryResp { id, replaced }) +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ListSavedReq { + #[serde(default)] + pub db: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ListSavedResp { + pub queries: Vec, + pub count: usize, +} + +pub async fn list(iii: &Arc, db: &str) -> Result { + let queries: Vec = state_get(iii, &saved_key(db)) + .await? + .into_iter() + // Drop anything that no longer parses rather than failing the call — + // a stale entry should not make the list unreadable. + .filter_map(|v| serde_json::from_value(v).ok()) + .collect(); + Ok(ListSavedResp { + count: queries.len(), + queries, + }) +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct DeleteSavedReq { + #[serde(default)] + pub db: Option, + /// Either the id returned by `saveQuery`, or the name it was saved under. + pub id: String, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct DeleteSavedResp { + pub deleted: bool, +} + +pub async fn delete( + iii: &Arc, + db: &str, + req: DeleteSavedReq, +) -> Result { + let key = saved_key(db); + let mut items: Vec = state_get(iii, &key) + .await? + .into_iter() + .filter_map(|v| serde_json::from_value(v).ok()) + .collect(); + let before = items.len(); + items.retain(|q| q.id != req.id && q.name != req.id); + let deleted = items.len() != before; + if deleted { + state_set(iii, &key, serde_json::to_value(&items).unwrap_or(json!([]))).await?; + } + Ok(DeleteSavedResp { deleted }) +} + +/* ---------------- history ---------------- */ + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct HistoryEntry { + pub sql: String, + pub verb: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub row_count: Option, + pub at: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct HistoryReq { + #[serde(default)] + pub db: Option, + #[serde(default)] + pub limit: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct HistoryResp { + pub entries: Vec, + pub count: usize, +} + +pub async fn history( + iii: &Arc, + db: &str, + req: HistoryReq, +) -> Result { + let key = history_key(db); + let raw = state_get(iii, &key).await?; + let all: Vec = raw + .iter() + .filter_map(|v| serde_json::from_value(v.clone()).ok()) + .collect(); + + // Self-healing trim: the write path only appends, so the stored list is + // capped here once it drifts past the high-water mark. + if all.len() > HISTORY_HIGH_WATER { + let tail: Vec<&HistoryEntry> = all.iter().rev().take(HISTORY_LIMIT).rev().collect(); + let _ = state_set(iii, &key, serde_json::to_value(&tail).unwrap_or(json!([]))).await; + } + + let limit = req + .limit + .unwrap_or(HISTORY_LIMIT) + .clamp(1, HISTORY_HIGH_WATER); + let entries: Vec = all.into_iter().rev().take(limit).collect(); + Ok(HistoryResp { + count: entries.len(), + entries, + }) +} + +/// The `state::update` op list for appending one entry. +/// +/// Split out only so a test can assert the discriminator, which is the one +/// detail of this file that cannot be caught at runtime. +fn append_ops(entry: Value) -> Vec { + vec![json!({ "type": "append", "value": entry })] +} + +/// Record one run. Fire and forget: never awaited on the query path, and a +/// failure is logged rather than surfaced, because losing a history line must +/// never fail the query the user actually asked for. +pub fn record(iii: Arc, db: String, sql: &str, duration_ms: u64, row_count: usize) { + let entry = HistoryEntry { + sql: truncate(sql), + verb: leading_verb(sql), + duration_ms: Some(duration_ms), + row_count: Some(row_count), + at: now(), + }; + tokio::spawn(async move { + let payload = json!({ + "scope": SCOPE, + "key": history_key(&db), + "ops": append_ops(serde_json::to_value(&entry).unwrap_or(json!({}))), + }); + if let Err(e) = call(&iii, "state::update", payload).await { + tracing::warn!(error = %e, "history not recorded"); + } + }); +} + +fn now() -> String { + chrono::Utc::now().to_rfc3339() +} + +fn truncate(sql: &str) -> String { + if sql.chars().count() <= MAX_SQL_CHARS { + return sql.to_string(); + } + let head: String = sql.chars().take(MAX_SQL_CHARS).collect(); + format!("{head}…") +} + +/// First keyword, lowercased — enough to group history without storing a +/// parse tree. +fn leading_verb(sql: &str) -> String { + sql.split(|c: char| !c.is_ascii_alphabetic()) + .find(|w| !w.is_empty()) + .unwrap_or("unknown") + .to_ascii_lowercase() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn leading_verb_ignores_punctuation_and_case() { + assert_eq!(leading_verb(" SELECT * FROM t"), "select"); + assert_eq!(leading_verb("(select 1)"), "select"); + assert_eq!(leading_verb("INSERT INTO t VALUES (1)"), "insert"); + assert_eq!(leading_verb(""), "unknown"); + } + + #[test] + fn long_sql_is_truncated_with_a_marker() { + let long = "x".repeat(MAX_SQL_CHARS + 50); + let t = truncate(&long); + assert_eq!(t.chars().count(), MAX_SQL_CHARS + 1); + assert!(t.ends_with('…')); + + let short = "SELECT 1"; + assert_eq!(truncate(short), short); + } + + #[test] + fn truncation_counts_characters_not_bytes() { + // A byte-based cut would slice a multi-byte character in half. + let s = "é".repeat(MAX_SQL_CHARS + 10); + let t = truncate(&s); + assert_eq!(t.chars().count(), MAX_SQL_CHARS + 1); + } + + #[test] + fn append_op_uses_the_type_discriminator() { + // Locked deliberately: `record` is fire-and-forget, so a wrong op + // shape fails where nobody is looking. This is the only cheap place + // to notice. + let ops = append_ops(json!({"sql": "select 1"})); + assert_eq!(ops[0]["type"], "append"); + assert!(ops[0].get("op").is_none()); + } + + #[test] + fn keys_are_scoped_per_database() { + assert_eq!(saved_key("primary"), "saved:primary"); + assert_eq!(history_key("analytics"), "history:analytics"); + } +} diff --git a/database/src/handlers/schema.rs b/database/src/handlers/schema.rs new file mode 100644 index 000000000..498ad741e --- /dev/null +++ b/database/src/handlers/schema.rs @@ -0,0 +1,274 @@ +//! `database::listTables` / `describeTable` / `describeSchema` — what is +//! *inside* a database, as opposed to what is on the bus. +//! +//! The engine already introspects itself (`engine::functions::list` and +//! friends); nothing in the engine knows what a table is, because only this +//! worker holds the connection pools. These functions are the SQL-catalog +//! equivalent, so an agent can ask what tables exist and how they relate +//! without hand-writing `sqlite_master` / `information_schema` / `PRAGMA` per +//! driver — which is exactly what the console was doing before. +//! +//! `describe_table` is a one-table `describe_schema`, so there is a single +//! assembly path. The difference that matters is the filter: describing one +//! table scopes every catalog query to it, while describing a whole schema +//! runs each query once across all tables and regroups in Rust. A 200-table +//! schema therefore costs three or four queries, not six hundred. + +use super::catalog::{self, ColumnDesc, IndexDesc, TableFilter, TableKey, TableKind, TableRef}; +use super::AppState; +use crate::config::DriverKind; +use crate::error::DbError; +use crate::handlers::query::err_to_str; +use crate::pool::Pool; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +fn default_timeout() -> u64 { + 30_000 +} + +fn default_max_tables() -> usize { + 500 +} + +/// Hard ceiling regardless of what the caller asks for. Describing a schema +/// is bounded work; describing an unbounded one is not. +const MAX_TABLES_CEILING: usize = 2_000; + +async fn driver_of(state: &AppState, db: &str) -> Result { + let pool = state.pool(db).await.map_err(err_to_str)?; + Ok(match pool { + Pool::Sqlite(_) => DriverKind::Sqlite, + Pool::Postgres(_) => DriverKind::Postgres, + Pool::Mysql(_) => DriverKind::Mysql, + }) +} + +fn no_such_table(driver: DriverKind, table: &str) -> String { + err_to_str(DbError::DriverError { + driver: format!("{driver:?}").to_lowercase(), + code: None, + message: format!("no such table: {table}"), + failed_index: None, + }) +} + +/* ---------------- listTables ---------------- */ + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ListTablesReq { + /// Logical database name. Optional — omitting it targets the sole + /// configured database, or `primary` when several are configured. + #[serde(default)] + pub db: Option, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ListTablesResp { + pub tables: Vec, + pub count: usize, +} + +pub async fn list_tables(state: &AppState, req: ListTablesReq) -> Result { + let db = state.resolve_db(req.db).await.map_err(err_to_str)?; + let tables = read_tables(state, &db, req.timeout_ms).await?; + Ok(ListTablesResp { + count: tables.len(), + tables, + }) +} + +async fn read_tables(state: &AppState, db: &str, timeout_ms: u64) -> Result, String> { + match driver_of(state, db).await? { + DriverKind::Sqlite => catalog::sqlite::list_tables(state, db, timeout_ms).await, + DriverKind::Postgres => catalog::postgres::list_tables(state, db, timeout_ms).await, + DriverKind::Mysql => catalog::mysql::list_tables(state, db, timeout_ms).await, + } +} + +/* ---------------- describeTable / describeSchema ---------------- */ + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub struct TableDescription { + pub table: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option, + pub kind: TableKind, + pub columns: Vec, + pub indexes: Vec, + /// Planner estimate, never a `COUNT(*)`. Absent when the driver has no + /// cheap estimate (sqlite) or has not analyzed the table yet. + #[serde(skip_serializing_if = "Option::is_none")] + pub row_count_estimate: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct DescribeTableReq { + #[serde(default)] + pub db: Option, + /// Table or view name. May be schema-qualified (`analytics.events`) on + /// postgres; prefer the explicit `schema` field when the name itself + /// contains a dot. + pub table: String, + #[serde(default)] + pub schema: Option, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +pub async fn describe_table( + state: &AppState, + req: DescribeTableReq, +) -> Result { + let db = state.resolve_db(req.db).await.map_err(err_to_str)?; + let driver = driver_of(state, &db).await?; + + // Only postgres has a namespace above the table, so only there does a dot + // in the name mean a schema qualifier. + let (schema, table) = match (&req.schema, driver) { + (Some(s), _) => (Some(s.clone()), req.table.clone()), + (None, DriverKind::Postgres) => catalog::split_qualified(&req.table), + (None, _) => (None, req.table.clone()), + }; + + let filter = TableFilter { + schema: schema.clone(), + table: table.clone(), + }; + let mut described = assemble(state, &db, driver, Some(&filter), true, req.timeout_ms).await?; + + // Scoping the catalog queries to one table means an unknown name simply + // yields nothing; say so rather than returning an empty description. + if described.is_empty() { + return Err(no_such_table(driver, &req.table)); + } + Ok(described.remove(0)) +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct DescribeSchemaReq { + #[serde(default)] + pub db: Option, + /// Restrict to these tables. Omit for every table in the database. + #[serde(default)] + pub tables: Option>, + /// Indexes cost one extra catalog query. Off by default because the + /// common caller (a relationship diagram) only needs columns and keys. + #[serde(default)] + pub include_indexes: bool, + #[serde(default = "default_max_tables")] + pub max_tables: usize, + #[serde(default = "default_timeout")] + pub timeout_ms: u64, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct DescribeSchemaResp { + pub tables: Vec, + pub count: usize, + /// True when `max_tables` cut the result short. Never silently truncate. + pub truncated: bool, +} + +pub async fn describe_schema( + state: &AppState, + req: DescribeSchemaReq, +) -> Result { + let db = state.resolve_db(req.db).await.map_err(err_to_str)?; + let driver = driver_of(state, &db).await?; + let limit = req.max_tables.min(MAX_TABLES_CEILING); + + let mut tables = assemble( + state, + &db, + driver, + None, + req.include_indexes, + req.timeout_ms, + ) + .await?; + + if let Some(wanted) = &req.tables { + let wanted: Vec<(Option, String)> = wanted + .iter() + .map(|t| match driver { + DriverKind::Postgres => catalog::split_qualified(t), + _ => (None, t.clone()), + }) + .collect(); + tables.retain(|d| { + wanted + .iter() + .any(|(s, t)| &d.table == t && (s.is_none() || &d.schema == s)) + }); + } + + let truncated = tables.len() > limit; + tables.truncate(limit); + Ok(DescribeSchemaResp { + count: tables.len(), + tables, + truncated, + }) +} + +/// The single assembly path. One catalog query per aspect, regrouped by +/// table — never a query per table. +async fn assemble( + state: &AppState, + db: &str, + driver: DriverKind, + filter: Option<&TableFilter>, + include_indexes: bool, + timeout_ms: u64, +) -> Result, String> { + let tables = read_tables(state, db, timeout_ms).await?; + + let columns = match driver { + DriverKind::Sqlite => catalog::sqlite::columns(state, db, filter, timeout_ms).await?, + DriverKind::Postgres => catalog::postgres::columns(state, db, filter, timeout_ms).await?, + DriverKind::Mysql => catalog::mysql::columns(state, db, filter, timeout_ms).await?, + }; + + let mut indexes: HashMap> = HashMap::new(); + if include_indexes { + indexes = match driver { + DriverKind::Sqlite => catalog::sqlite::indexes(state, db, filter, timeout_ms).await?, + DriverKind::Postgres => { + catalog::postgres::indexes(state, db, filter, timeout_ms).await? + } + DriverKind::Mysql => catalog::mysql::indexes(state, db, filter, timeout_ms).await?, + }; + } + + let estimates = match driver { + DriverKind::Sqlite => catalog::sqlite::row_estimates(state, db, filter, timeout_ms).await?, + DriverKind::Postgres => { + catalog::postgres::row_estimates(state, db, filter, timeout_ms).await? + } + DriverKind::Mysql => catalog::mysql::row_estimates(state, db, filter, timeout_ms).await?, + }; + + let mut out = Vec::new(); + for t in tables { + let key: TableKey = (t.schema.clone(), t.name.clone()); + // A table with no column rows was filtered out by the catalog query, + // so it is not part of this result. + let Some(mut cols) = columns.get(&key).cloned() else { + continue; + }; + cols.sort_by_key(|c| c.position); + out.push(TableDescription { + table: t.name, + schema: t.schema, + kind: t.kind, + columns: cols, + indexes: indexes.get(&key).cloned().unwrap_or_default(), + row_count_estimate: estimates.get(&key).copied(), + }); + } + Ok(out) +} diff --git a/database/src/handlers/table_view.rs b/database/src/handlers/table_view.rs new file mode 100644 index 000000000..76d00f323 --- /dev/null +++ b/database/src/handlers/table_view.rs @@ -0,0 +1,183 @@ +//! `database::getTableView` / `saveTableView` — how a table is laid out. +//! +//! Column widths, hidden columns and column order. These are preferences, not +//! data, and the obvious place to keep them is browser storage — which is +//! exactly where the query history used to live, for one person, in one +//! browser, until a cache clear. +//! +//! On `state::*` instead they survive a restart, follow the operator to +//! another machine, and are legible to anything else on the bus. An agent +//! preparing a table for someone to look at can widen the column that matters +//! and hide the six that do not. +//! +//! Deliberately *not* validated against the live schema. A view saved before a +//! column was renamed should degrade to "that column has no stored width", not +//! fail the read; the renderer already treats every entry as optional. + +use super::saved::{call, SCOPE}; +use crate::error::DbError; +use iii_sdk::IIIClient; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::sync::Arc; + +/// Upper bound on stored columns per table. A view is a preference, not a +/// place to accumulate unbounded state from a caller. +const MAX_COLUMNS: usize = 1_000; +/// Clamped so a stored width cannot render a column unusable or push the grid +/// to an absurd scroll width. +const MIN_WIDTH: f64 = 48.0; +const MAX_WIDTH: f64 = 1_200.0; + +fn view_key(db: &str, table: &str) -> String { + format!("view:{db}:{table}") +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] +pub struct TableView { + /// Per-column pixel width. Absent means "size to content". + #[serde(default)] + pub widths: std::collections::BTreeMap, + /// Columns the reader has hidden. Order is not meaningful. + #[serde(default)] + pub hidden: Vec, + /// Column display order. Names not listed keep their natural position + /// after those that are, so adding a column to the table does not require + /// re-saving the view. + #[serde(default)] + pub order: Vec, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct GetTableViewReq { + #[serde(default)] + pub db: Option, + pub table: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct SaveTableViewReq { + #[serde(default)] + pub db: Option, + pub table: String, + #[serde(flatten)] + pub view: TableView, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct SaveTableViewResp { + pub saved: bool, +} + +pub async fn get(iii: &Arc, db: &str, table: &str) -> Result { + let raw = call( + iii, + "state::get", + json!({"scope": SCOPE, "key": view_key(db, table)}), + ) + .await?; + let value = raw.get("value").cloned().unwrap_or(raw); + // A missing or unparseable view is the default one. Losing a layout is a + // far smaller problem than refusing to show the table. + Ok(serde_json::from_value(value).unwrap_or_default()) +} + +pub async fn save( + iii: &Arc, + db: &str, + req: SaveTableViewReq, +) -> Result { + if req.table.trim().is_empty() { + return Err(super::query::err_to_str(DbError::InvalidParam { + index: 0, + reason: "table is required".into(), + })); + } + let view = sanitise(req.view)?; + call( + iii, + "state::set", + json!({ + "scope": SCOPE, + "key": view_key(db, &req.table), + "value": serde_json::to_value(&view).unwrap_or(Value::Null), + }), + ) + .await?; + Ok(SaveTableViewResp { saved: true }) +} + +/// Clamp widths and bound the lists. The caller is a renderer, so this is not +/// hostile input, but a stored `width: 1e9` would be a layout no one could +/// undo without editing state by hand. +fn sanitise(mut view: TableView) -> Result { + let too_many = view.widths.len() > MAX_COLUMNS + || view.hidden.len() > MAX_COLUMNS + || view.order.len() > MAX_COLUMNS; + if too_many { + return Err(super::query::err_to_str(DbError::InvalidParam { + index: 0, + reason: format!("a view may describe at most {MAX_COLUMNS} columns"), + })); + } + view.widths = view + .widths + .into_iter() + .filter(|(_, w)| w.is_finite()) + .map(|(k, w)| (k, w.clamp(MIN_WIDTH, MAX_WIDTH))) + .collect(); + view.hidden.sort(); + view.hidden.dedup(); + Ok(view) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keys_are_scoped_per_table() { + assert_eq!(view_key("primary", "orders"), "view:primary:orders"); + } + + #[test] + fn widths_are_clamped_into_a_usable_range() { + let mut v = TableView::default(); + v.widths.insert("a".into(), 5.0); + v.widths.insert("b".into(), 99_999.0); + let out = sanitise(v).unwrap(); + assert_eq!(out.widths["a"], MIN_WIDTH); + assert_eq!(out.widths["b"], MAX_WIDTH); + } + + #[test] + fn non_finite_widths_are_dropped_rather_than_stored() { + let mut v = TableView::default(); + v.widths.insert("a".into(), f64::NAN); + v.widths.insert("b".into(), f64::INFINITY); + v.widths.insert("c".into(), 200.0); + let out = sanitise(v).unwrap(); + assert_eq!(out.widths.len(), 1); + assert!(out.widths.contains_key("c")); + } + + #[test] + fn hidden_columns_are_deduplicated() { + let v = TableView { + hidden: vec!["b".into(), "a".into(), "b".into()], + ..Default::default() + }; + let out = sanitise(v).unwrap(); + assert_eq!(out.hidden, vec!["a".to_string(), "b".to_string()]); + } + + #[test] + fn an_oversized_view_is_refused() { + let mut v = TableView::default(); + for i in 0..(MAX_COLUMNS + 1) { + v.widths.insert(format!("c{i}"), 100.0); + } + assert!(sanitise(v).is_err()); + } +} diff --git a/database/src/handlers/tx_sql_guard.rs b/database/src/handlers/tx_sql_guard.rs index 4b2b4614b..6195cd2b9 100644 --- a/database/src/handlers/tx_sql_guard.rs +++ b/database/src/handlers/tx_sql_guard.rs @@ -53,6 +53,63 @@ pub(super) fn is_transaction_control_sql(sql: &str) -> bool { } } +/// Statements that only read. Anything not on this list is treated as a +/// write, so an unrecognised or vendor-specific verb fails closed. +const READ_ONLY_VERBS: &[&str] = &[ + "select", "with", "explain", "pragma", "show", "describe", "desc", "values", "table", +]; + +/// Keywords that mutate. Checked anywhere in the statement, not just at the +/// front, so a data-modifying CTE (`WITH x AS (DELETE ...) SELECT ...`) is +/// caught — it leads with `WITH` but is very much a write. +const WRITE_KEYWORDS: &[&str] = &[ + "insert", "update", "delete", "replace", "merge", "upsert", "drop", "create", "alter", + "truncate", "grant", "revoke", "attach", "detach", "reindex", "vacuum", "call", "do", +]; + +/// Whether `sql` is a single statement that cannot modify data. +/// +/// Used to gate `EXPLAIN ANALYZE`, which really executes the statement — an +/// `EXPLAIN ANALYZE DELETE FROM users` deletes users. The client-side check +/// in the console is a convenience; this one is the authority. +/// +/// Fails closed: anything it cannot confidently classify as a read is a +/// write. Reuses `strip_leading_noise`, so a leading `--` or `/* */` comment +/// cannot smuggle a verb past the check. +pub(crate) fn is_read_only_sql(sql: &str) -> bool { + let head = strip_leading_noise(sql); + if head.is_empty() { + return false; + } + + // More than one statement means the tail is unchecked; refuse rather than + // classify only the first. A trailing `;` alone is fine. + if head.trim_end().trim_end_matches(';').contains(';') { + return false; + } + + let lowered = head.to_ascii_lowercase(); + let leading = lowered + .split(|c: char| !c.is_ascii_alphabetic()) + .find(|w| !w.is_empty()) + .unwrap_or_default(); + if !READ_ONLY_VERBS.contains(&leading) { + return false; + } + + // `PRAGMA foo = 1` writes; `PRAGMA foo` reads. + if leading == "pragma" && lowered.contains('=') { + return false; + } + + // Word-boundary scan so `created_at` does not trip the `create` keyword, + // and so `EXPLAIN ANALYZE DELETE ...` is caught by its `delete`. + !lowered + .split(|c: char| !c.is_ascii_alphanumeric() && c != '_') + .filter(|w| !w.is_empty()) + .any(|w| WRITE_KEYWORDS.contains(&w)) +} + /// Strip any prefix of leading whitespace, `;`, line comments (`-- ...\n`), /// and block comments (`/* ... */`, with nesting supported) so the returned /// slice begins at the first character of the first real token. Returns an @@ -255,4 +312,57 @@ mod tests { "INSERT INTO t VALUES ('مرحبا')" )); } + + #[test] + fn read_only_accepts_plain_reads() { + for sql in [ + "SELECT 1", + " select * from users where created_at > now()", + "WITH x AS (SELECT 1) SELECT * FROM x", + "EXPLAIN SELECT * FROM t", + "PRAGMA table_info(users)", + "SHOW TABLES", + "VALUES (1), (2)", + "-- a comment\nSELECT 1", + "/* block */ SELECT 1", + "SELECT 1;", + ] { + assert!(is_read_only_sql(sql), "should be read-only: {sql}"); + } + } + + #[test] + fn read_only_rejects_writes_and_the_ways_they_hide() { + for sql in [ + // Plain writes. + "DELETE FROM users", + "UPDATE users SET a = 1", + "DROP TABLE users", + // The one that matters most: EXPLAIN ANALYZE really executes. + "EXPLAIN ANALYZE DELETE FROM users", + // A data-modifying CTE leads with WITH but is a write. + "WITH x AS (DELETE FROM users RETURNING *) SELECT * FROM x", + // A second statement would go unchecked. + "SELECT 1; DROP TABLE users", + // A comment must not smuggle the verb past the check. + "-- SELECT\nDELETE FROM users", + "/* SELECT */ DROP TABLE t", + // PRAGMA with an assignment writes. + "PRAGMA journal_mode = WAL", + // Unknown verbs fail closed. + "LOCK TABLE users", + "", + " ", + ] { + assert!(!is_read_only_sql(sql), "should be refused: {sql}"); + } + } + + #[test] + fn read_only_does_not_trip_on_identifiers_containing_keywords() { + // `created_at` contains "create"; a substring scan would reject this. + assert!(is_read_only_sql( + "SELECT created_at, updated_at FROM t ORDER BY created_at" + )); + } } diff --git a/database/src/main.rs b/database/src/main.rs index 98f404710..d0899c766 100644 --- a/database/src/main.rs +++ b/database/src/main.rs @@ -5,14 +5,22 @@ use database::configuration; use database::handle::HandleRegistry; use database::handlers::{ begin_transaction::{self, BeginTxReq}, + browse::{self, BrowseTableReq}, + column_stats::{self, ColumnStatsReq}, commit_transaction::{self, CommitTxReq}, + diagram::{self, SchemaDiagramReq}, execute::{self, ExecuteReq}, execute_batch::{self, ExecuteBatchReq}, + explain::{self, ExplainReq}, + health::{self, HealthReq, TerminateReq}, list_databases::{self, ListDatabasesReq}, prepare::{self, PrepareReq}, query::{self, QueryReq}, rollback_transaction::{self, RollbackTxReq}, run_statement::{self, RunReq}, + saved::{self, DeleteSavedReq, HistoryReq, ListSavedReq, SaveQueryReq}, + schema::{self, DescribeSchemaReq, DescribeTableReq, ListTablesReq}, + table_view::{self, GetTableViewReq, SaveTableViewReq}, test_connection::{self, TestConnectionReq}, transaction::{self, TxReq}, transaction_execute::{self, TxExecuteReq}, @@ -353,6 +361,321 @@ async fn main() -> Result<()> { ), ); } + { + let st = state.clone(); + let client = iii.clone(); + iii.register_function( + "database::getTableView", + RegisterFunction::new_async(move |req: GetTableViewReq| { + let (st, client) = (st.clone(), client.clone()); + async move { + let db = st + .resolve_db(req.db.clone()) + .await + .map_err(database::handlers::query::err_to_str)?; + table_view::get(&client, &db, &req.table) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "How a table is laid out for reading: column widths, hidden columns and \ + column order. Stored in the state worker rather than a browser, so it \ + survives a restart and any caller can set it up for someone else.", + ), + ); + } + { + let st = state.clone(); + let client = iii.clone(); + iii.register_function( + "database::saveTableView", + RegisterFunction::new_async(move |req: SaveTableViewReq| { + let (st, client) = (st.clone(), client.clone()); + async move { + let db = st + .resolve_db(req.db.clone()) + .await + .map_err(database::handlers::query::err_to_str)?; + table_view::save(&client, &db, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Replace the stored layout for a table. Widths are clamped to a usable \ + range; columns the table no longer has are kept rather than rejected, \ + so a rename degrades to a missing width instead of an error.", + ), + ); + } + { + let st = state.clone(); + let client = iii.clone(); + iii.register_function( + "database::saveQuery", + RegisterFunction::new_async(move |req: SaveQueryReq| { + let (st, client) = (st.clone(), client.clone()); + async move { + let db = st + .resolve_db(req.db.clone()) + .await + .map_err(database::handlers::query::err_to_str)?; + saved::save(&client, &db, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Save a named query against a database. Stored in the state worker, so \ + it survives restarts and an agent can save one for a human to find in \ + the console. Saving under an existing name replaces it.", + ), + ); + } + { + let st = state.clone(); + let client = iii.clone(); + iii.register_function( + "database::listSavedQueries", + RegisterFunction::new_async(move |req: ListSavedReq| { + let (st, client) = (st.clone(), client.clone()); + async move { + let db = st + .resolve_db(req.db.clone()) + .await + .map_err(database::handlers::query::err_to_str)?; + saved::list(&client, &db) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description("List the saved queries for a database, sorted by name."), + ); + } + { + let st = state.clone(); + let client = iii.clone(); + iii.register_function( + "database::deleteSavedQuery", + RegisterFunction::new_async(move |req: DeleteSavedReq| { + let (st, client) = (st.clone(), client.clone()); + async move { + let db = st + .resolve_db(req.db.clone()) + .await + .map_err(database::handlers::query::err_to_str)?; + saved::delete(&client, &db, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description("Delete a saved query by id or by name."), + ); + } + { + let st = state.clone(); + let client = iii.clone(); + iii.register_function( + "database::history", + RegisterFunction::new_async(move |req: HistoryReq| { + let (st, client) = (st.clone(), client.clone()); + async move { + let db = st + .resolve_db(req.db.clone()) + .await + .map_err(database::handlers::query::err_to_str)?; + saved::history(&client, &db, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Recent queries run against a database, newest first. Best effort — \ + recording never blocks or fails a query, so this is a convenience \ + rather than an audit log. For an audit trail bind database::row-changed.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::schemaDiagram", + RegisterFunction::new_async(move |req: SchemaDiagramReq| { + let st = st.clone(); + async move { + diagram::handle(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Lay out the schema as a diagram: positioned table nodes and routed \ + foreign-key edges, plus the hub degree of each table, the isolated \ + tables, and the remaining edge crossings. Reads the whole catalog in \ + a handful of queries rather than one per table.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::columnStats", + RegisterFunction::new_async(move |req: ColumnStatsReq| { + let st = st.clone(); + async move { + column_stats::handle(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Profile a table's columns. Reads the planner's own statistics by \ + default, which is free and approximate; `exact` runs real aggregates \ + and scans the table. To profile rows you already hold, pipe a \ + browseTable result through the fp worker instead.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::health", + RegisterFunction::new_async(move |req: HealthReq| { + let st = st.clone(); + async move { + health::handle(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Live pool occupancy plus active queries, table sizes, blocking locks \ + and cache hit ratio. Each section reports separately as available, \ + unsupported or denied, so a driver gap or a restricted role is never \ + mistaken for an empty result.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::terminateQuery", + RegisterFunction::new_async(move |req: TerminateReq| { + let st = st.clone(); + async move { + health::terminate(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Terminate a backend session, or cancel just its running statement \ + with `cancel_only`. Takes an id from database::health. Separate from \ + health because it is a write.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::explain", + RegisterFunction::new_async(move |req: ExplainReq| { + let st = st.clone(); + async move { + explain::handle(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Return a statement's query plan as a tree with per-node costs, row \ + estimates and warnings, instead of the driver's raw text. `analyze` \ + collects real timings by RUNNING the statement, so it defaults to \ + false and is refused for anything that is not a single read.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::browseTable", + RegisterFunction::new_async(move |req: BrowseTableReq| { + let st = st.clone(); + async move { + browse::handle(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Read a table page by page with typed filters and sorts, without \ + writing SQL. Filters are structured (column, op, value) and \ + compile to a parameterised WHERE for the driver in hand; the \ + total honours the same filters. Use an equality filter at \ + page_size 1 to follow a foreign key.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::listTables", + RegisterFunction::new_async(move |req: ListTablesReq| { + let st = st.clone(); + async move { + schema::list_tables(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "List every table and view in a database, with its kind and (on \ + postgres) its schema. Reads the driver's own catalog, so no \ + dialect-specific SQL is needed from the caller.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::describeTable", + RegisterFunction::new_async(move |req: DescribeTableReq| { + let st = st.clone(); + async move { + schema::describe_table(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Describe one table or view: columns with type, nullability, \ + default, primary-key membership and foreign-key target; plus \ + indexes and a planner row estimate. Foreign keys are structured \ + (schema, table, column), not a joined string.", + ), + ); + } + { + let st = state.clone(); + iii.register_function( + "database::describeSchema", + RegisterFunction::new_async(move |req: DescribeSchemaReq| { + let st = st.clone(); + async move { + schema::describe_schema(&st, req) + .await + .map_err(iii_sdk::errors::Error::from) + } + }) + .description( + "Describe every table at once — the same shape as describeTable, \ + but one catalog query per aspect across the whole database \ + instead of one call per table. Use this to reason about \ + relationships; set include_indexes only when you need them.", + ), + ); + } // The worker announces its own writes. Registered AFTER the functions so // the console can attribute the type, and gated on the databases that @@ -380,7 +703,7 @@ async fn main() -> Result<()> { database::ui::register(&iii); tracing::info!( - "database worker registered 14 functions and 1 trigger type, waiting for invocations" + "database worker registered 29 functions and 1 trigger type, waiting for invocations" ); wait_for_shutdown_signal().await?; tracing::info!("database worker shutting down"); diff --git a/database/src/pool/mod.rs b/database/src/pool/mod.rs index e85978a69..322603f3e 100644 --- a/database/src/pool/mod.rs +++ b/database/src/pool/mod.rs @@ -20,6 +20,23 @@ pub enum Pool { Sqlite(SqlitePool), } +/// Live pool occupancy, for `database::health`. +/// +/// `size` and `idle` are `None` where the underlying pool does not expose +/// them — `mysql_async` keeps its counters private. Reporting `None` rather +/// than zero matters: "unknown" and "no idle connections" are different +/// answers, and a health panel that conflates them is actively misleading. +#[derive(Debug, Clone, Copy, serde::Serialize, schemars::JsonSchema)] +pub struct PoolStats { + pub max: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub idle: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub waiting: Option, +} + impl Pool { pub fn driver(&self) -> DriverKind { match self { @@ -28,6 +45,14 @@ impl Pool { Pool::Sqlite(_) => DriverKind::Sqlite, } } + + pub fn stats(&self) -> PoolStats { + match self { + Pool::Postgres(p) => p.stats(), + Pool::Mysql(p) => p.stats(), + Pool::Sqlite(p) => p.stats(), + } + } } /// Build a pool for a single configured database. Used at startup by main.rs. diff --git a/database/src/pool/mysql.rs b/database/src/pool/mysql.rs index 590837f65..0b4ce9fca 100644 --- a/database/src/pool/mysql.rs +++ b/database/src/pool/mysql.rs @@ -12,11 +12,26 @@ pub struct MysqlPool { inner: Arc, db_name: Arc, acquire_timeout: Duration, + /// Configured ceiling. `mysql_async` exposes no live counters, so this is + /// the only pool figure `database::health` can report for mysql. + max_size: u32, } pub type MysqlConn = mysql_async::Conn; impl MysqlPool { + /// `mysql_async` does not expose live pool counters, so only the + /// configured ceiling is known. The rest stay `None` rather than being + /// reported as zero. + pub fn stats(&self) -> crate::pool::PoolStats { + crate::pool::PoolStats { + max: self.max_size, + size: None, + idle: None, + waiting: None, + } + } + pub fn new(url: &str, pool_cfg: &PoolConfig, tls_cfg: &TlsConfig) -> Result { let constraints = PoolConstraints::new(0, pool_cfg.max as usize).ok_or_else(|| DbError::ConfigError { @@ -45,6 +60,7 @@ impl MysqlPool { inner: Arc::new(pool), db_name: Arc::from("(unset)"), acquire_timeout: Duration::from_millis(pool_cfg.acquire_timeout_ms), + max_size: pool_cfg.max, }) } diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index 856eeb9dc..f0c519b4d 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -18,6 +18,19 @@ pub struct PostgresPool { pub type PgClient = deadpool_postgres::Object; impl PostgresPool { + /// Live occupancy from deadpool's own counters. + pub fn stats(&self) -> crate::pool::PoolStats { + let st = self.inner.status(); + crate::pool::PoolStats { + max: st.max_size as u32, + size: Some(st.size as u32), + // deadpool counts `available` as idle-or-creatable, so it can + // exceed the number of connections actually open. + idle: Some(st.available as u32), + waiting: Some(st.waiting as u32), + } + } + pub async fn new( url: &str, pool_cfg: &PoolConfig, diff --git a/database/src/pool/sqlite.rs b/database/src/pool/sqlite.rs index bb832e9a4..0b41de226 100644 --- a/database/src/pool/sqlite.rs +++ b/database/src/pool/sqlite.rs @@ -32,6 +32,17 @@ impl SqliteConn { } impl SqlitePool { + /// Live occupancy from r2d2's own counters. + pub fn stats(&self) -> crate::pool::PoolStats { + let st = self.inner.state(); + crate::pool::PoolStats { + max: self.inner.max_size(), + size: Some(st.connections), + idle: Some(st.idle_connections), + waiting: None, + } + } + pub fn new(url: &str, pool_cfg: &PoolConfig) -> Result { let path = url.strip_prefix("sqlite:").unwrap_or(url); let manager = if path == ":memory:" || path.starts_with(":memory:") { diff --git a/database/src/triggers/bus.rs b/database/src/triggers/bus.rs index 894af3a60..415f4de1d 100644 --- a/database/src/triggers/bus.rs +++ b/database/src/triggers/bus.rs @@ -191,6 +191,13 @@ pub struct RowChangeBus { } impl RowChangeBus { + /// The engine client this bus dispatches through. Handlers reuse it to + /// call sibling workers (history goes to `state::*`) without threading a + /// second client through `AppState`. + pub fn client(&self) -> &Arc { + &self.iii + } + pub fn new(iii: Arc, dispatch_timeout_ms: u64) -> Self { Self { iii, diff --git a/database/tests/integration.rs b/database/tests/integration.rs index 8b743c191..bb50040ad 100644 --- a/database/tests/integration.rs +++ b/database/tests/integration.rs @@ -4,11 +4,17 @@ use database::config::WorkerConfig; use database::configuration; use database::handle::HandleRegistry; +use database::handlers::browse::{self, BrowseTableReq}; +use database::handlers::catalog::{TableKind, TableRef}; +use database::handlers::column_stats::{self, ColumnStatsReq}; use database::handlers::execute::ExecuteReq; +use database::handlers::explain::{self, ExplainReq}; +use database::handlers::health::{self, HealthReq, TerminateReq}; use database::handlers::list_databases::{self, ListDatabasesReq}; use database::handlers::prepare::PrepareReq; use database::handlers::query::QueryReq; use database::handlers::run_statement::RunReq; +use database::handlers::schema::{self, DescribeSchemaReq, DescribeTableReq, ListTablesReq}; use database::handlers::transaction::TxReq; use database::handlers::{execute, prepare, query, run_statement, transaction, AppState}; use database::pool; @@ -248,3 +254,634 @@ async fn build_pool_creates_missing_sqlite_parent_dir() { .unwrap(); assert_eq!(r.row_count, 1); } + +/* ---------------- catalog introspection ---------------- */ + +/// Schema with the shapes that break naive catalog readers: an implicit +/// foreign key (`REFERENCES users` with no column), a composite primary key, +/// a multi-column index, and a view. +async fn seed_catalog(st: &AppState) { + for sql in [ + "CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT NOT NULL, plan TEXT DEFAULT 'free')", + // No target column: sqlite reports `to` as NULL and means "the parent's + // primary key". A reader that trusts the NULL emits a broken reference. + "CREATE TABLE orders (id INTEGER PRIMARY KEY, user_id INTEGER REFERENCES users, total REAL)", + "CREATE TABLE order_items (order_id INTEGER, sku TEXT, qty INTEGER, \ + PRIMARY KEY (order_id, sku), FOREIGN KEY (order_id) REFERENCES orders(id))", + "CREATE INDEX ix_orders_user_total ON orders (user_id, total)", + "CREATE VIEW big_orders AS SELECT * FROM orders WHERE total > 100", + ] { + execute::handle( + st, + serde_json::from_value::(json!({"db": "primary", "sql": sql})).unwrap(), + ) + .await + .unwrap(); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn list_tables_reports_tables_and_views() { + let st = build_state().await; + seed_catalog(&st).await; + + let r = schema::list_tables( + &st, + serde_json::from_value::(json!({"db": "primary"})).unwrap(), + ) + .await + .unwrap(); + + assert_eq!(r.count, r.tables.len()); + let by_name: HashMap<&str, &TableRef> = r.tables.iter().map(|t| (t.name.as_str(), t)).collect(); + assert!(by_name.contains_key("users")); + assert!(matches!(by_name["orders"].kind, TableKind::Table)); + assert!(matches!(by_name["big_orders"].kind, TableKind::View)); + // sqlite has no namespace above the table. + assert!(by_name["users"].schema.is_none()); + // Internal bookkeeping stays hidden. + assert!(!r.tables.iter().any(|t| t.name.starts_with("sqlite_"))); +} + +#[tokio::test(flavor = "multi_thread")] +async fn describe_table_resolves_columns_keys_and_indexes() { + let st = build_state().await; + seed_catalog(&st).await; + + let d = schema::describe_table( + &st, + serde_json::from_value::(json!({"db": "primary", "table": "orders"})) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(d.table, "orders"); + // Columns arrive in declaration order. + let names: Vec<&str> = d.columns.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, vec!["id", "user_id", "total"]); + + let id = &d.columns[0]; + assert!(id.primary_key); + assert_eq!(id.position, 1); + + // The implicit `REFERENCES users` resolves to the parent's primary key + // rather than surfacing an empty column. + let user_id = &d.columns[1]; + let fk = user_id + .foreign_key + .as_ref() + .expect("user_id is a foreign key"); + assert_eq!(fk.table, "users"); + assert_eq!(fk.column, "id"); + assert!(fk.schema.is_none()); + assert!(d.columns[2].foreign_key.is_none()); + + // A multi-column index keeps its columns in ordinal order. + let ix = d + .indexes + .iter() + .find(|i| i.name == "ix_orders_user_total") + .expect("index present"); + assert_eq!(ix.columns, vec!["user_id", "total"]); + assert!(!ix.unique); + assert!(!ix.primary); + + // sqlite has no cheap estimate, so it reports none rather than guessing. + assert!(d.row_count_estimate.is_none()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn describe_table_reports_composite_primary_keys() { + let st = build_state().await; + seed_catalog(&st).await; + + let d = schema::describe_table( + &st, + serde_json::from_value::( + json!({"db": "primary", "table": "order_items"}), + ) + .unwrap(), + ) + .await + .unwrap(); + + let pk: Vec<&str> = d + .columns + .iter() + .filter(|c| c.primary_key) + .map(|c| c.name.as_str()) + .collect(); + assert_eq!(pk, vec!["order_id", "sku"]); + + let fk = d.columns[0].foreign_key.as_ref().expect("explicit fk"); + assert_eq!((fk.table.as_str(), fk.column.as_str()), ("orders", "id")); +} + +#[tokio::test(flavor = "multi_thread")] +async fn describe_table_rejects_an_unknown_name() { + let st = build_state().await; + seed_catalog(&st).await; + + let err = schema::describe_table( + &st, + serde_json::from_value::(json!({"db": "primary", "table": "nope"})) + .unwrap(), + ) + .await + .unwrap_err(); + assert!(err.contains("no such table: nope"), "got: {err}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn describe_schema_covers_every_table_in_one_pass() { + let st = build_state().await; + seed_catalog(&st).await; + + let all = schema::describe_schema( + &st, + serde_json::from_value::( + json!({"db": "primary", "include_indexes": true}), + ) + .unwrap(), + ) + .await + .unwrap(); + + assert!(!all.truncated); + assert_eq!(all.count, all.tables.len()); + let names: Vec<&str> = all.tables.iter().map(|t| t.table.as_str()).collect(); + for want in ["users", "orders", "order_items", "big_orders"] { + assert!(names.contains(&want), "missing {want} in {names:?}"); + } + + // Relationships survive the batch path — this is what a diagram reads. + let orders = all.tables.iter().find(|t| t.table == "orders").unwrap(); + assert_eq!( + orders.columns[1] + .foreign_key + .as_ref() + .map(|f| f.table.as_str()), + Some("users") + ); + + // Restricting to a subset filters without changing the shape. + let some = schema::describe_schema( + &st, + serde_json::from_value::( + json!({"db": "primary", "tables": ["users", "orders"]}), + ) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(some.count, 2); +} + +#[tokio::test(flavor = "multi_thread")] +async fn describe_schema_flags_truncation_instead_of_hiding_it() { + let st = build_state().await; + seed_catalog(&st).await; + + let capped = schema::describe_schema( + &st, + serde_json::from_value::(json!({"db": "primary", "max_tables": 2})) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(capped.count, 2); + assert!(capped.truncated, "a cut-short result must say so"); +} + +/* ---------------- browseTable ---------------- */ + +async fn seed_rows(st: &AppState) { + execute::handle( + st, + serde_json::from_value::(json!({ + "db": "primary", + "sql": "CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT, note TEXT, score INT)" + })) + .unwrap(), + ) + .await + .unwrap(); + for (name, note, score) in [ + ("ana", Some("50% off"), 10), + ("bob", None, 20), + ("cyd", Some("plain"), 30), + ("dee", Some(""), 40), + ("eve", Some("other"), 50), + ] { + execute::handle( + st, + serde_json::from_value::(json!({ + "db": "primary", + "sql": "INSERT INTO people (name, note, score) VALUES (?, ?, ?)", + "params": [name, note, score] + })) + .unwrap(), + ) + .await + .unwrap(); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_pages_with_a_sentinel_and_counts_the_whole_table() { + let st = build_state().await; + seed_rows(&st).await; + + let first = browse::handle( + &st, + serde_json::from_value::( + json!({"db": "primary", "table": "people", "page_size": 2}), + ) + .unwrap(), + ) + .await + .unwrap(); + // The sentinel row is fetched but never returned. + assert_eq!(first.rows.len(), 2); + assert!(first.has_more); + assert_eq!(first.total, Some(5)); + + let last = browse::handle( + &st, + serde_json::from_value::( + json!({"db": "primary", "table": "people", "page": 2, "page_size": 2}), + ) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(last.rows.len(), 1); + assert!(!last.has_more, "the final page has nothing after it"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_total_reflects_the_filters_not_the_table() { + let st = build_state().await; + seed_rows(&st).await; + + let r = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", "page_size": 2, + "filters": [{"column": "score", "op": "gte", "value": 30}] + })) + .unwrap(), + ) + .await + .unwrap(); + // A pager showing 5 while 3 rows match is the bug this prevents. + assert_eq!(r.total, Some(3)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_treats_a_percent_in_a_filter_as_a_literal() { + let st = build_state().await; + seed_rows(&st).await; + + let r = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", + "filters": [{"column": "note", "op": "contains", "value": "50%"}] + })) + .unwrap(), + ) + .await + .unwrap(); + // Unescaped, `%` would make this match every non-null note. + assert_eq!(r.total, Some(1)); + assert_eq!(r.rows[0]["name"], "ana"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_separates_null_from_the_empty_string() { + let st = build_state().await; + seed_rows(&st).await; + + let nulls = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", + "filters": [{"column": "note", "op": "is_null"}] + })) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(nulls.total, Some(1), "only bob has a NULL note"); + + let empty = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", + "filters": [{"column": "note", "op": "is_empty"}] + })) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(empty.total, Some(2), "is_empty covers NULL and ''"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_sorts_and_can_skip_the_count() { + let st = build_state().await; + seed_rows(&st).await; + + let r = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", + "sort": [{"column": "score", "direction": "desc"}], + "include_total": false + })) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(r.rows[0]["name"], "eve"); + assert!(r.total.is_none(), "the count is skipped when not asked for"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_follows_a_foreign_key_with_an_equality_filter() { + let st = build_state().await; + seed_rows(&st).await; + + // This is why there is no separate read-a-row function. + let r = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", "page_size": 1, + "filters": [{"column": "id", "op": "equals", "value": 3}] + })) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(r.rows.len(), 1); + assert_eq!(r.rows[0]["name"], "cyd"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_refuses_an_incomplete_filter() { + let st = build_state().await; + seed_rows(&st).await; + + let err = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", + "filters": [{"column": "score", "op": "equals"}] + })) + .unwrap(), + ) + .await + .unwrap_err(); + assert!(err.contains("needs `value`"), "got: {err}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn browse_refuses_the_wrong_operand_instead_of_panicking() { + // `value2` alone satisfied a count-based arity check and then unwrapped + // the absent `value`. Reachable from the wire, so it aborted the task. + let st = build_state().await; + seed_rows(&st).await; + + let err = browse::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", + "filters": [{"column": "score", "op": "equals", "value2": 5}] + })) + .unwrap(), + ) + .await + .unwrap_err(); + assert!(err.contains("needs `value`"), "got: {err}"); +} + +/* ---------------- explain ---------------- */ + +#[tokio::test(flavor = "multi_thread")] +async fn explain_returns_a_plan_tree_for_a_read() { + let st = build_state().await; + seed_rows(&st).await; + + let r = explain::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "sql": "SELECT * FROM people WHERE score > 20" + })) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(r.format, explain::PlanFormat::SqliteQueryPlan); + assert!(!r.analyzed, "sqlite has no ANALYZE form"); + let root = r.root.expect("a plan tree"); + assert!(!root.children.is_empty(), "the scan hangs off the root"); + assert_eq!(root.children[0].relation.as_deref(), Some("people")); +} + +/// The gate that keeps a viewer from deleting data. `EXPLAIN ANALYZE DELETE` +/// really executes on postgres and mysql, so the worker refuses it outright +/// rather than trusting the caller to have checked. +#[tokio::test(flavor = "multi_thread")] +async fn explain_refuses_analyze_on_anything_that_writes() { + let st = build_state().await; + seed_rows(&st).await; + + for sql in [ + "DELETE FROM people", + "UPDATE people SET score = 0", + "DROP TABLE people", + // Leads with WITH, but is a write. + "WITH doomed AS (DELETE FROM people RETURNING *) SELECT * FROM doomed", + // A second statement would go unchecked. + "SELECT 1; DROP TABLE people", + // A comment must not smuggle the verb past the check. + "-- SELECT\nDELETE FROM people", + ] { + let err = explain::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "sql": sql, "analyze": true + })) + .unwrap(), + ) + .await + .unwrap_err(); + assert!( + err.contains("read-only"), + "should refuse `{sql}`, got: {err}" + ); + } + + // The rows are all still there. + let after = browse::handle( + &st, + serde_json::from_value::(json!({"db": "primary", "table": "people"})) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(after.total, Some(5)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn explain_allows_analyze_on_a_plain_read() { + let st = build_state().await; + seed_rows(&st).await; + + // Accepted by the gate; sqlite then reports it could not actually analyze. + let r = explain::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "sql": "SELECT * FROM people", "analyze": true + })) + .unwrap(), + ) + .await + .unwrap(); + assert!(!r.analyzed); +} + +/* ---------------- columnStats ---------------- */ + +#[tokio::test(flavor = "multi_thread")] +async fn column_stats_defaults_to_planner_statistics() { + let st = build_state().await; + seed_rows(&st).await; + + let r = column_stats::handle( + &st, + serde_json::from_value::(json!({"db": "primary", "table": "people"})) + .unwrap(), + ) + .await + .unwrap(); + + // The default must never scan; it reports what the planner knows, clearly + // labelled, even when that is very little. + assert!(r.approximate); + assert_eq!(r.columns.len(), 4); + assert!(r + .columns + .iter() + .all(|c| c.source == column_stats::StatSource::Planner)); + assert!( + r.columns.iter().all(|c| c.top_values.is_empty()), + "top values require a scan, so the cheap path does not invent them" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn column_stats_exact_counts_nulls_and_top_values() { + let st = build_state().await; + seed_rows(&st).await; + + let r = column_stats::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", "columns": ["note"], "exact": true + })) + .unwrap(), + ) + .await + .unwrap(); + + assert!(!r.approximate); + let note = &r.columns[0]; + assert_eq!(note.source, column_stats::StatSource::Computed); + assert_eq!(note.row_count, Some(5)); + // bob's note is NULL; dee's is '' — counted as present, not null. + assert_eq!(note.null_count, Some(1)); + assert_eq!(note.distinct_count, Some(4)); + assert_eq!(note.top_values.len(), 4); + assert!(note.top_values.iter().all(|t| t.count == 1)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn column_stats_rejects_a_column_that_does_not_exist() { + let st = build_state().await; + seed_rows(&st).await; + + let err = column_stats::handle( + &st, + serde_json::from_value::(json!({ + "db": "primary", "table": "people", "columns": ["nope"] + })) + .unwrap(), + ) + .await + .unwrap_err(); + assert!(err.contains("no such column `nope`"), "got: {err}"); +} + +/* ---------------- health ---------------- */ + +#[tokio::test(flavor = "multi_thread")] +async fn health_reports_pool_stats_and_says_what_sqlite_cannot_answer() { + let st = build_state().await; + seed_rows(&st).await; + + let h = health::handle( + &st, + serde_json::from_value::(json!({"db": "primary"})).unwrap(), + ) + .await + .unwrap(); + + assert_eq!(h.driver, "sqlite"); + assert_eq!(h.worker_version, env!("CARGO_PKG_VERSION")); + // The pool always answers, on every driver. + assert!(h.pool.max >= 1); + assert!(h.pool.size.is_some(), "r2d2 exposes live counters"); + + // The distinction that makes the report honest: sqlite has no sessions, + // which is a different answer from "no queries are running". + for section in [&h.active_queries] { + match section { + health::ProbeResult::Unsupported { reason } => { + assert!(reason.contains("sqlite"), "got: {reason}") + } + other => panic!("expected unsupported, got {other:?}"), + } + } + assert!(matches!(h.locks, health::ProbeResult::Unsupported { .. })); + assert!(matches!(h.cache, health::ProbeResult::Unsupported { .. })); +} + +#[tokio::test(flavor = "multi_thread")] +async fn terminate_refuses_a_non_numeric_id_and_refuses_sqlite() { + let st = build_state().await; + + let err = health::terminate( + &st, + serde_json::from_value::(json!({"db": "primary", "id": "1"})).unwrap(), + ) + .await + .unwrap_err(); + assert!(err.contains("in-process"), "got: {err}"); + + // An id is interpolated into the statement, so it must parse as a number + // before it gets anywhere near SQL. + let err = health::terminate( + &st, + serde_json::from_value::( + json!({"db": "primary", "id": "1); DROP TABLE people--"}), + ) + .unwrap(), + ) + .await + .unwrap_err(); + assert!(err.contains("not a backend id"), "got: {err}"); +} diff --git a/database/ui/build.mjs b/database/ui/build.mjs index f039d23d1..0a061659f 100644 --- a/database/ui/build.mjs +++ b/database/ui/build.mjs @@ -11,14 +11,21 @@ * poller for the hot-reload dev loop. */ +import { readFileSync } from 'node:fs' import esbuild from 'esbuild' +const watch = process.argv.includes('--watch') + const options = { entryPoints: ['page.tsx', 'styles.css'], bundle: true, format: 'esm', jsx: 'automatic', outdir: 'dist', + // The bundle is `include_str!`'d into the worker binary and the injectable-UI + // protocol rejects an asset over 8 MiB outright, so ship it minified. Left + // off under --watch, where readable stack traces matter more than bytes. + minify: !watch, external: [ 'react', 'react-dom', @@ -29,9 +36,86 @@ const options = { logLevel: 'info', } -if (process.argv.includes('--watch')) { +// Minification strips the quotes from an attribute selector, so the scope +// prefix has to be matched in both spellings. +const SCOPE = '[data-iii-ui="database"]' +const SCOPE_RE = /^\[data-iii-ui=("database"|'database'|database)\]/ +const KEYFRAME_PREFIXES = ['db-ui-', 'db-page-'] + +/** + * Fail the build if any rule could escape this worker's subtree. + * + * The console mounts every worker's UI into one document, so a single + * unscoped selector restyles the whole app. The Rust side already asserts + * that the scope string appears *somewhere* in the sheet, which a file + * containing one stray `body { }` would also pass. This checks every + * selector, and runs in CI for free because CI runs `pnpm build`. + */ +function assertScoped(css) { + // Strip comments, then strings, so a brace or selector inside either can't + // desynchronise the scan. + const src = css + .replace(/\/\*[\s\S]*?\*\//g, '') + .replace(/"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'/g, '""') + + const problems = [] + let depth = 0 + let buf = '' + // Nested at-rules (@media, @container, @supports) open a block whose + // children are still top-level selectors, so track them rather than + // skipping their contents. + const atStack = [] + + for (let i = 0; i < src.length; i++) { + const ch = src[i] + if (ch === '{') { + const head = buf.trim() + buf = '' + if (head.startsWith('@')) { + const name = head.split(/[\s(]/)[0] + atStack.push(name) + if (name === '@keyframes') { + const kf = head.slice('@keyframes'.length).trim() + if (!KEYFRAME_PREFIXES.some((p) => kf.startsWith(p))) { + problems.push(`@keyframes ${kf} is not prefixed ${KEYFRAME_PREFIXES.join(' or ')}`) + } + } else if (name === '@font-face') { + problems.push('@font-face is not allowed in an injected sheet') + } + depth++ + continue + } + // Inside @keyframes the "selectors" are 0%/from/to, not real ones. + if (!atStack.includes('@keyframes') && head) { + for (const sel of head.split(',')) { + const s = sel.trim() + if (s && !SCOPE_RE.test(s)) { + problems.push(`selector not scoped: ${s}`) + } + } + } + depth++ + } else if (ch === '}') { + depth-- + buf = '' + if (atStack.length && depth < atStack.length) atStack.pop() + } else { + buf += ch + } + } + + if (problems.length) { + console.error(`\n${problems.length} unscoped rule(s) in dist/styles.css:\n`) + for (const p of [...new Set(problems)]) console.error(` ${p}`) + console.error(`\nEvery rule must start with ${SCOPE}.\n`) + process.exit(1) + } +} + +if (watch) { const ctx = await esbuild.context(options) await ctx.watch() } else { await esbuild.build(options) + assertScoped(readFileSync('dist/styles.css', 'utf8')) } diff --git a/database/ui/package.json b/database/ui/package.json index 3a3cf59a1..49bb7b4e6 100644 --- a/database/ui/package.json +++ b/database/ui/package.json @@ -5,7 +5,8 @@ "type": "module", "scripts": { "build": "tsc --noEmit && node build.mjs", - "watch": "node build.mjs --watch" + "watch": "node build.mjs --watch", + "test": "vitest run" }, "dependencies": { "@iii-dev/console-ui": "workspace:*", @@ -14,6 +15,7 @@ "devDependencies": { "@types/react": "^19.2.14", "esbuild": "^0.25.0", - "typescript": "^5.9.2" + "typescript": "^5.9.2", + "vitest": "^4.1.6" } } diff --git a/database/ui/src/lib/capabilities.ts b/database/ui/src/lib/capabilities.ts new file mode 100644 index 000000000..18214c507 --- /dev/null +++ b/database/ui/src/lib/capabilities.ts @@ -0,0 +1,82 @@ +/** + * Which of the worker's functions this engine actually has. + * + * The page no longer carries a client-side catalog fallback: there is one + * implementation of every read, and it lives in the worker. That removes a + * whole class of drift, and introduces a hard minimum worker version. An + * older install must degrade visibly rather than throw "function not found" + * from four components at once — so a panel whose function is missing is + * *hidden*, never rendered as a control that always errors. + * + * Discovery uses the engine's own `engine::functions::list`. Inventing a + * `database::capabilities` for this would have been one more function to keep + * in sync with the real registry. + */ + +import type { Host } from '@iii-dev/console-ui' +import { OPTIONAL_FNS, type DbFunction } from './rpc' + +export type Capabilities = ReadonlySet + +/** Every optional function, for the common case of an up-to-date worker. */ +export const ALL: Capabilities = new Set(OPTIONAL_FNS) + +/** + * Ask the engine which `database::*` functions are registered. + * + * Falls back to assuming everything is present when the listing itself is + * unavailable: a probe that cannot run is not evidence of a missing feature, + * and hiding the whole page because one introspection call failed would be + * the worse error. Individual calls still surface their own failures. + */ +export async function probe(host: Host): Promise { + try { + const raw = await host.iii.trigger('engine::functions::list', {}) + const ids = collectIds(raw) + if (ids.size === 0) return ALL + return new Set(OPTIONAL_FNS.filter((fn) => ids.has(fn))) + } catch { + return ALL + } +} + +/** + * Pull function ids out of whatever shape the listing returns — an array of + * strings, an array of objects with `id` / `function_id` / `name`, or an + * object keyed by id. Being permissive here is deliberate: the exact shape is + * an engine detail, and guessing wrong should not disable the page. + */ +function collectIds(raw: unknown): Set { + const out = new Set() + + const add = (v: unknown) => { + if (typeof v === 'string' && v.startsWith('database::')) out.add(v) + } + + const walk = (node: unknown, depth: number) => { + if (depth > 4 || node == null) return + if (typeof node === 'string') return add(node) + if (Array.isArray(node)) { + for (const item of node) walk(item, depth + 1) + return + } + if (typeof node === 'object') { + const rec = node as Record + for (const key of ['id', 'function_id', 'functionId', 'name']) { + add(rec[key]) + } + // An object keyed by function id. + for (const key of Object.keys(rec)) add(key) + for (const key of ['functions', 'data', 'items', 'result']) { + if (key in rec) walk(rec[key], depth + 1) + } + } + } + + walk(raw, 0) + return out +} + +/** Version hint for the empty state a hidden panel leaves behind. */ +export const MIN_VERSION_HINT = + 'this needs a newer database worker — run `iii worker update database`' diff --git a/database/ui/src/lib/grid-cursor.test.ts b/database/ui/src/lib/grid-cursor.test.ts new file mode 100644 index 000000000..39a07b9b4 --- /dev/null +++ b/database/ui/src/lib/grid-cursor.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from 'vitest' +import { type Bounds, moveCursor, reanchor, rowAsTsv, rowKey } from './grid-cursor' + +const bounds = (over: Partial = {}): Bounds => ({ + rowCount: 10, + colCount: 4, + pageRows: 5, + hasPrevPage: false, + hasNextPage: false, + ...over, +}) + +describe('moveCursor', () => { + it('clamps at the edges rather than wrapping', () => { + expect(moveCursor({ row: 0, col: 0 }, { type: 'up' }, bounds()).cursor).toEqual({ row: 0, col: 0 }) + expect(moveCursor({ row: 0, col: 0 }, { type: 'left' }, bounds()).cursor).toEqual({ row: 0, col: 0 }) + expect(moveCursor({ row: 9, col: 3 }, { type: 'down' }, bounds()).cursor).toEqual({ row: 9, col: 3 }) + expect(moveCursor({ row: 9, col: 3 }, { type: 'right' }, bounds()).cursor).toEqual({ row: 9, col: 3 }) + }) + + it('crosses to the next page from the last row', () => { + const m = moveCursor({ row: 9, col: 2 }, { type: 'down' }, bounds({ hasNextPage: true })) + expect(m.pageDelta).toBe(1) + expect(m.landing).toBe('first') + // The column is preserved across the page turn — moving down a column + // should not also move sideways. + expect(m.cursor.col).toBe(2) + }) + + it('crosses to the previous page from the first row', () => { + const m = moveCursor({ row: 0, col: 1 }, { type: 'up' }, bounds({ hasPrevPage: true })) + expect(m.pageDelta).toBe(-1) + expect(m.landing).toBe('last') + }) + + it('does not cross a page that does not exist', () => { + expect(moveCursor({ row: 9, col: 0 }, { type: 'down' }, bounds()).pageDelta).toBeUndefined() + expect(moveCursor({ row: 0, col: 0 }, { type: 'up' }, bounds()).pageDelta).toBeUndefined() + }) + + it('pages within the grid before crossing a boundary', () => { + // From the middle, PageDown jumps inside the page even when a next page + // exists — the key means "move far", not "turn the page". + const m = moveCursor({ row: 2, col: 0 }, { type: 'pageDown' }, bounds({ hasNextPage: true })) + expect(m.pageDelta).toBeUndefined() + expect(m.cursor.row).toBe(7) + }) + + it('turns the page when PageDown is pressed at the bottom', () => { + const m = moveCursor({ row: 9, col: 0 }, { type: 'pageDown' }, bounds({ hasNextPage: true })) + expect(m.pageDelta).toBe(1) + }) + + it('moves to row and grid extremes', () => { + expect(moveCursor({ row: 3, col: 2 }, { type: 'rowStart' }, bounds()).cursor).toEqual({ row: 3, col: 0 }) + expect(moveCursor({ row: 3, col: 2 }, { type: 'rowEnd' }, bounds()).cursor).toEqual({ row: 3, col: 3 }) + expect(moveCursor({ row: 3, col: 2 }, { type: 'gridStart' }, bounds()).cursor).toEqual({ row: 0, col: 0 }) + expect(moveCursor({ row: 3, col: 2 }, { type: 'gridEnd' }, bounds()).cursor).toEqual({ row: 9, col: 3 }) + }) + + it('clamps an out-of-range cursor before moving it', () => { + // The page shrank underneath the cursor. + const m = moveCursor({ row: 99, col: 99 }, { type: 'up' }, bounds()) + expect(m.cursor).toEqual({ row: 8, col: 3 }) + }) + + it('does nothing on an empty grid', () => { + const m = moveCursor({ row: 0, col: 0 }, { type: 'down' }, bounds({ rowCount: 0 })) + expect(m).toEqual({ cursor: { row: 0, col: 0 } }) + }) +}) + +describe('reanchor', () => { + const columns = ['id', 'email', 'plan'] + + it('follows a row to its new index after a sort', () => { + const keys = ['id=3', 'id=1', 'id=2'] + expect(reanchor({ row: 0, col: 1 }, 'id=2', keys, columns, 'email')).toEqual({ row: 2, col: 1 }) + }) + + it('follows a column to its new index', () => { + const keys = ['id=1'] + expect(reanchor({ row: 0, col: 0 }, 'id=1', keys, ['plan', 'id', 'email'], 'email')).toEqual({ + row: 0, + col: 2, + }) + }) + + it('clamps when the row is gone', () => { + // Filtered out. Keep the column, put the cursor somewhere valid. + expect(reanchor({ row: 5, col: 2 }, 'id=99', ['id=1', 'id=2'], columns, 'plan')).toEqual({ + row: 1, + col: 2, + }) + }) +}) + +describe('rowKey', () => { + it('uses the primary key when there is one', () => { + expect(rowKey({ id: 7, email: 'a@b.c' }, ['id'])).toBe('id=7') + }) + + it('falls back to the whole row, which survives a re-sort', () => { + expect(rowKey({ id: 7, email: 'a@b.c' }, [])).toBe('id=7\u0001email=a@b.c') + }) + + it('distinguishes null from the empty string', () => { + expect(rowKey({ a: null }, [])).not.toBe(rowKey({ a: '' }, [])) + }) + + it('does not let adjacent fields run together', () => { + // With an empty separator these two rows produce the same key, and the + // cursor would follow the wrong one after a sort. + expect(rowKey({ a: '1', b: '2' }, ['a', 'b'])).not.toBe(rowKey({ a: '12', b: '' }, ['a', 'b'])) + }) +}) + +describe('rowAsTsv', () => { + it('joins with tabs so it pastes as spreadsheet cells', () => { + expect(rowAsTsv({ a: 1, b: 'x' }, ['a', 'b'])).toBe('1\tx') + }) + + it('renders null as empty rather than the text NULL', () => { + expect(rowAsTsv({ a: null, b: 2 }, ['a', 'b'])).toBe('\t2') + }) + + it('serialises objects rather than pasting [object Object]', () => { + expect(rowAsTsv({ a: { x: 1 } }, ['a'])).toBe('{"x":1}') + }) +}) diff --git a/database/ui/src/lib/grid-cursor.ts b/database/ui/src/lib/grid-cursor.ts new file mode 100644 index 000000000..fed825b60 --- /dev/null +++ b/database/ui/src/lib/grid-cursor.ts @@ -0,0 +1,173 @@ +/** + * Grid cursor movement, as a pure function. + * + * Kept out of React because it is the one piece of this page with enough edge + * cases to be worth testing directly: clamping, page-edge crossing, and + * surviving a re-sort. It has no imports and never touches the DOM. + * + * The grid is *server*-paged, which is what makes this more than a clamp. + * Pressing Down on the last visible row of page 2 should land on the first row + * of page 3, not sit still — a grid that stops dead at the page edge feels + * broken in a way a desktop client never does. Movement therefore returns an + * optional `pageDelta` alongside the new cursor, and the caller decides + * whether it can honour it (it cannot, at the first or last page). + */ + +export interface Cursor { + row: number + col: number +} + +export interface Bounds { + rowCount: number + colCount: number + /** Rows to jump for PageUp/PageDown. */ + pageRows: number + /** False on the first page — Up at row 0 then has nowhere to go. */ + hasPrevPage: boolean + hasNextPage: boolean +} + +export type CursorAction = + | { type: 'up' } + | { type: 'down' } + | { type: 'left' } + | { type: 'right' } + | { type: 'rowStart' } + | { type: 'rowEnd' } + | { type: 'gridStart' } + | { type: 'gridEnd' } + | { type: 'pageUp' } + | { type: 'pageDown' } + | { type: 'to'; row: number; col: number } + +export interface CursorMove { + cursor: Cursor + /** + * -1 / +1 when the move ran off the top or bottom of the page. The caller + * turns the page and places the cursor at `landing`. + */ + pageDelta?: -1 | 1 + /** Which edge row to land on after the caller turns the page. */ + landing?: 'first' | 'last' +} + +const clamp = (n: number, max: number) => Math.max(0, Math.min(max, n)) + +export function moveCursor(cursor: Cursor, action: CursorAction, bounds: Bounds): CursorMove { + const { rowCount, colCount, pageRows, hasPrevPage, hasNextPage } = bounds + if (rowCount <= 0 || colCount <= 0) return { cursor } + + const lastRow = rowCount - 1 + const lastCol = colCount - 1 + const at = { row: clamp(cursor.row, lastRow), col: clamp(cursor.col, lastCol) } + + switch (action.type) { + case 'up': + if (at.row === 0 && hasPrevPage) { + return { cursor: at, pageDelta: -1, landing: 'last' } + } + return { cursor: { ...at, row: clamp(at.row - 1, lastRow) } } + + case 'down': + if (at.row === lastRow && hasNextPage) { + return { cursor: at, pageDelta: 1, landing: 'first' } + } + return { cursor: { ...at, row: clamp(at.row + 1, lastRow) } } + + case 'left': + return { cursor: { ...at, col: clamp(at.col - 1, lastCol) } } + + case 'right': + return { cursor: { ...at, col: clamp(at.col + 1, lastCol) } } + + case 'rowStart': + return { cursor: { ...at, col: 0 } } + + case 'rowEnd': + return { cursor: { ...at, col: lastCol } } + + case 'gridStart': + return { cursor: { row: 0, col: 0 } } + + case 'gridEnd': + return { cursor: { row: lastRow, col: lastCol } } + + case 'pageUp': + // Only cross a page boundary from the very top. Otherwise jump within + // the page, which is what the key means when there is room to move. + if (at.row === 0 && hasPrevPage) { + return { cursor: at, pageDelta: -1, landing: 'last' } + } + return { cursor: { ...at, row: clamp(at.row - pageRows, lastRow) } } + + case 'pageDown': + if (at.row === lastRow && hasNextPage) { + return { cursor: at, pageDelta: 1, landing: 'first' } + } + return { cursor: { ...at, row: clamp(at.row + pageRows, lastRow) } } + + case 'to': + return { cursor: { row: clamp(action.row, lastRow), col: clamp(action.col, lastCol) } } + } +} + +/** + * Re-find the cursor after the rows underneath it changed. + * + * Anchoring on the row *key* and the column *name* rather than their indices + * is the whole point: after a sort, index 4 is a different row, and a cursor + * that stayed at index 4 would silently point somewhere else. When the row is + * gone — filtered out, or on another page now — the column is kept and the row + * clamps, which is the least surprising thing available. + */ +export function reanchor( + cursor: Cursor, + previousKey: string | null, + keys: string[], + columns: string[], + previousColumn: string | null, +): Cursor { + const row = previousKey === null ? cursor.row : keys.indexOf(previousKey) + const col = previousColumn === null ? cursor.col : columns.indexOf(previousColumn) + return { + row: row >= 0 ? row : clamp(cursor.row, Math.max(0, keys.length - 1)), + col: col >= 0 ? col : clamp(cursor.col, Math.max(0, columns.length - 1)), + } +} + +/** + * A stable identity for a row. + * + * Primary key when there is one. Otherwise the row's own values, which is not + * guaranteed unique but is stable across a re-sort — and a duplicate is a far + * smaller error than an index that silently means a different row. + */ +export function rowKey(row: Record, primaryKeys: string[]): string { + const cols = primaryKeys.length > 0 ? primaryKeys : Object.keys(row) + return cols.map((c) => `${c}=${String(row[c])}`).join(KEY_SEP) +} + +/** + * Field separator for composite keys. A control character rather than an empty + * string or a comma: joining with nothing makes `{a:'1',b:'2'}` collide with + * `{a:'12',b:''}`, and any printable separator can occur inside a value. + */ +const KEY_SEP = '\u0001' + +/** A row as tab-separated text, so it pastes into a spreadsheet as cells. */ +export function rowAsTsv(row: Record, columns: string[]): string { + return columns.map((c) => cellText(row[c])).join('\t') +} + +export function cellText(value: unknown): string { + if (value === null || value === undefined) return '' + if (typeof value === 'object') { + try { + return JSON.stringify(value) + } catch { + return String(value) + } + } + return String(value) +} diff --git a/database/ui/src/lib/rpc.ts b/database/ui/src/lib/rpc.ts new file mode 100644 index 000000000..873249695 --- /dev/null +++ b/database/ui/src/lib/rpc.ts @@ -0,0 +1,682 @@ +/** + * Typed calls into the worker's own functions. + * + * The page used to build catalog SQL in the browser — around 520 lines of + * `sqlite_master` / `information_schema` / `PRAGMA` per driver. That logic now + * lives in the worker, where it is written once, tested against all three + * drivers in CI, and callable by any agent on the bus. This module is the + * whole of what replaced it: one wrapper per function, no SQL. + * + * Everything is parsed with zod at the boundary, because a worker one version + * behind will happily return a shape this page does not expect, and a missing + * field should surface as a readable error rather than `undefined` three + * components deep. + */ + +import type { Host } from '@iii-dev/console-ui' +import { z } from 'zod' + +export const DB = { + listDatabases: 'database::listDatabases', + listTables: 'database::listTables', + describeTable: 'database::describeTable', + describeSchema: 'database::describeSchema', + browseTable: 'database::browseTable', + query: 'database::query', + explain: 'database::explain', + columnStats: 'database::columnStats', + health: 'database::health', + schemaDiagram: 'database::schemaDiagram', + saveQuery: 'database::saveQuery', + listSavedQueries: 'database::listSavedQueries', + deleteSavedQuery: 'database::deleteSavedQuery', + history: 'database::history', + getTableView: 'database::getTableView', + saveTableView: 'database::saveTableView', +} as const + +export type DbFunction = (typeof DB)[keyof typeof DB] + +/** Functions older installed workers will not have. */ +export const OPTIONAL_FNS: readonly DbFunction[] = [ + DB.listTables, + DB.describeTable, + DB.describeSchema, + DB.browseTable, + DB.explain, + DB.columnStats, + DB.health, + DB.schemaDiagram, + DB.saveQuery, + DB.listSavedQueries, + DB.deleteSavedQuery, + DB.history, + DB.getTableView, + DB.saveTableView, +] + +async function call(host: Host, fn: DbFunction, payload: Record, schema: z.ZodType): Promise { + const raw = await host.iii.trigger(fn, payload) + const parsed = schema.safeParse(raw) + if (!parsed.success) { + throw new Error(`unexpected response shape from ${fn}`) + } + return parsed.data +} + +/* ---------------- shared shapes ---------------- */ + +export const columnMetaSchema = z.object({ + name: z.string(), + type: z.string().optional(), +}) +export type ColumnMeta = z.infer + +export const queryResponseSchema = z.object({ + rows: z.array(z.record(z.string(), z.unknown())), + row_count: z.number(), + columns: z.array(columnMetaSchema), +}) +export type QueryResponse = z.infer + +export const foreignKeySchema = z.object({ + schema: z.string().nullish(), + table: z.string(), + column: z.string(), +}) +export type ForeignKeyRef = z.infer + +export const columnDescSchema = z.object({ + name: z.string(), + type: z.string(), + nullable: z.boolean(), + default_value: z.string().nullish(), + primary_key: z.boolean(), + position: z.number(), + foreign_key: foreignKeySchema.nullish(), +}) +export type ColumnDesc = z.infer + +export const indexDescSchema = z.object({ + name: z.string(), + unique: z.boolean(), + primary: z.boolean(), + columns: z.array(z.string()), +}) +export type IndexDesc = z.infer + +export const tableRefSchema = z.object({ + name: z.string(), + schema: z.string().nullish(), + kind: z.enum(['table', 'view']), +}) +export type TableRef = z.infer + +export const tableDescriptionSchema = z.object({ + table: z.string(), + schema: z.string().nullish(), + kind: z.enum(['table', 'view']), + columns: z.array(columnDescSchema), + indexes: z.array(indexDescSchema), + row_count_estimate: z.number().nullish(), +}) +export type TableDescription = z.infer + +/* ---------------- filters and sorts ---------------- */ + +export type FilterOp = + | 'contains' + | 'not_contains' + | 'equals' + | 'not_equals' + | 'starts_with' + | 'ends_with' + | 'gt' + | 'gte' + | 'lt' + | 'lte' + | 'between' + | 'is_true' + | 'is_false' + | 'is_null' + | 'is_not_null' + | 'is_empty' + | 'in' + | 'not_in' + +export interface FilterSpec { + column: string + op: FilterOp + value?: unknown + value2?: unknown + /** Operands for `in` / `not_in`. */ + values?: unknown[] + case_sensitive?: boolean + /** Kept in the bar but not applied. */ + disabled?: boolean +} + +/** Human labels, grouped the way a picker should present them. */ +export const OP_LABEL: Record = { + equals: 'is', + not_equals: 'is not', + contains: 'contains', + not_contains: 'does not contain', + starts_with: 'starts with', + ends_with: 'ends with', + gt: 'greater than', + gte: 'at least', + lt: 'less than', + lte: 'at most', + between: 'between', + in: 'is one of', + not_in: 'is none of', + is_true: 'is true', + is_false: 'is false', + is_null: 'is null', + is_not_null: 'is not null', + is_empty: 'is empty', +} + +export type SortMode = 'default' | 'natural' | 'length' | 'absolute_value' | 'random' + +export interface SortSpec { + column: string + direction: 'asc' | 'desc' + nulls?: 'first' | 'last' + mode?: SortMode +} + +/** + * Operators that make sense for a column, by its declared type. Offering + * `contains` on a boolean is how a filter bar starts feeling untrustworthy. + */ +export function opsFor(category: TypeCategory): FilterOp[] { + switch (category) { + case 'numeric': + return ['equals', 'not_equals', 'gt', 'gte', 'lt', 'lte', 'between', 'in', 'not_in', 'is_null', 'is_not_null'] + case 'bool': + return ['is_true', 'is_false', 'is_null', 'is_not_null'] + case 'date': + return ['equals', 'gt', 'lt', 'between', 'is_null', 'is_not_null'] + default: + return [ + 'contains', + 'not_contains', + 'equals', + 'not_equals', + 'starts_with', + 'ends_with', + 'in', + 'not_in', + 'is_empty', + 'is_null', + 'is_not_null', + ] + } +} + +/** Operators taking a list rather than a single operand. */ +export const SET_OPS: ReadonlySet = new Set(['in', 'not_in']) + +/** Operators taking no operand — a chip using one is complete immediately. */ +const NULLARY: ReadonlySet = new Set(['is_true', 'is_false', 'is_null', 'is_not_null', 'is_empty']) + +/** + * Whether a chip is ready to send. An incomplete chip stays in the bar as a + * draft and is never sent — the worker would reject it, and a round trip to + * learn that would make typing feel broken. + */ +export function isComplete(f: FilterSpec): boolean { + if (!f.column) return false + if (NULLARY.has(f.op)) return true + if (f.op === 'between') return hasValue(f.value) && hasValue(f.value2) + // `IN ()` is a syntax error everywhere, so an empty list is a draft. + if (SET_OPS.has(f.op)) return (f.values ?? []).some(hasValue) + return hasValue(f.value) +} + +function hasValue(v: unknown): boolean { + return v !== undefined && v !== null && v !== '' +} + +export type TypeCategory = 'numeric' | 'text' | 'bool' | 'date' | 'json' | 'binary' | 'other' + +/** Coarse category for a driver-reported column type. */ +export function typeCategory(type: string | undefined): TypeCategory { + if (!type) return 'other' + const t = type.toLowerCase() + if (/bool/.test(t)) return 'bool' + if (/int|real|float|double|decimal|numeric|serial|money/.test(t)) return 'numeric' + if (/date|time|year/.test(t)) return 'date' + if (/json/.test(t)) return 'json' + if (/blob|bytea|binary/.test(t)) return 'binary' + if (/char|text|clob|uuid|enum/.test(t)) return 'text' + return 'other' +} + +/* ---------------- calls ---------------- */ + +export const PAGE_SIZE = 50 +export type DbDriver = 'sqlite' | 'postgres' | 'mysql' | 'unknown' + +const listDatabasesSchema = z.object({ + databases: z.array( + z.object({ + name: z.string(), + driver: z.string(), + url: z.string().optional(), + pool: z.object({ max: z.number().optional() }).optional(), + }), + ), + count: z.number(), +}) + +export interface DbInfo { + name: string + driver: DbDriver + url?: string + poolMax?: number +} + +function parseDriver(raw: string | undefined): DbDriver { + return raw === 'sqlite' || raw === 'postgres' || raw === 'mysql' ? raw : 'unknown' +} + +export async function listDbs(host: Host): Promise { + const res = await call(host, DB.listDatabases, {}, listDatabasesSchema) + return res.databases.map((d) => ({ + name: d.name, + driver: parseDriver(d.driver), + url: d.url, + poolMax: d.pool?.max, + })) +} + +export async function listTables(host: Host, db: string): Promise { + const res = await call(host, DB.listTables, { db }, z.object({ tables: z.array(tableRefSchema), count: z.number() })) + return res.tables +} + +export async function describeTable( + host: Host, + db: string, + table: string, + schema?: string | null, +): Promise { + return call(host, DB.describeTable, { db, table, schema }, tableDescriptionSchema) +} + +export async function describeSchema(host: Host, db: string, includeIndexes = false): Promise { + const res = await call( + host, + DB.describeSchema, + { db, include_indexes: includeIndexes }, + z.object({ + tables: z.array(tableDescriptionSchema), + count: z.number(), + truncated: z.boolean(), + }), + ) + return res.tables +} + +const browseSchema = z.object({ + rows: z.array(z.record(z.string(), z.unknown())), + columns: z.array(columnMetaSchema), + page: z.number(), + page_size: z.number(), + has_more: z.boolean(), + total: z.number().nullish(), +}) +export type BrowseResult = z.infer + +export interface BrowseOptions { + page?: number + pageSize?: number + sort?: SortSpec[] + filters?: FilterSpec[] + /** Skip the filtered COUNT(*) while the user is still editing a chip. */ + includeTotal?: boolean +} + +export async function browseTable( + host: Host, + db: string, + table: string, + schema: string | null | undefined, + opts: BrowseOptions = {}, +): Promise { + return call( + host, + DB.browseTable, + { + db, + table, + schema, + page: Math.max(0, Math.trunc(opts.page ?? 0)), + page_size: Math.max(1, Math.trunc(opts.pageSize ?? PAGE_SIZE)), + sort: opts.sort ?? [], + // Draft chips never leave the browser. + filters: (opts.filters ?? []).filter(isComplete), + include_total: opts.includeTotal ?? true, + }, + browseSchema, + ) +} + +/** Ad-hoc SQL from the editor. Still goes through `database::query`. */ +export async function runSql(host: Host, db: string, sql: string): Promise { + return call(host, DB.query, { db, sql }, queryResponseSchema) +} + +export const planNodeSchema: z.ZodType = z.lazy(() => + z.object({ + id: z.number(), + parent: z.number().nullish(), + label: z.string(), + node_class: z.string(), + relation: z.string().nullish(), + cost_startup: z.number().nullish(), + cost_total: z.number().nullish(), + rows_estimated: z.number().nullish(), + rows_actual: z.number().nullish(), + width: z.number().nullish(), + time_ms: z.number().nullish(), + loops: z.number().nullish(), + detail: z.string(), + children: z.array(planNodeSchema), + }), +) + +export interface PlanNode { + id: number + parent?: number | null + label: string + node_class: string + relation?: string | null + cost_startup?: number | null + cost_total?: number | null + rows_estimated?: number | null + rows_actual?: number | null + width?: number | null + time_ms?: number | null + loops?: number | null + detail: string + children: PlanNode[] +} + +const explainSchema = z.object({ + format: z.string(), + analyzed: z.boolean(), + root: planNodeSchema.nullish(), + warnings: z.array( + z.object({ + node_id: z.number(), + kind: z.string(), + message: z.string(), + severity: z.string(), + }), + ), + raw: z.unknown().nullish(), +}) +export type ExplainResult = z.infer + +export async function explain(host: Host, db: string, sql: string, analyze = false): Promise { + return call(host, DB.explain, { db, sql, analyze }, explainSchema) +} + +const columnStatSchema = z.object({ + name: z.string(), + row_count: z.number().nullish(), + distinct_count: z.number().nullish(), + null_count: z.number().nullish(), + null_fraction: z.number().nullish(), + min: z.unknown().nullish(), + max: z.unknown().nullish(), + mean: z.number().nullish(), + top_values: z.array(z.object({ value: z.unknown(), count: z.number() })), + source: z.enum(['planner', 'computed']), +}) +export type ColumnStat = z.infer + +export async function columnStats( + host: Host, + db: string, + table: string, + columns: string[], + exact = false, +): Promise<{ columns: ColumnStat[]; approximate: boolean }> { + return call( + host, + DB.columnStats, + { db, table, columns, exact }, + z.object({ + table: z.string(), + schema: z.string().nullish(), + columns: z.array(columnStatSchema), + approximate: z.boolean(), + }), + ) +} + +const probeSchema = (data: T) => + z.discriminatedUnion('status', [ + z.object({ status: z.literal('available'), data }), + z.object({ status: z.literal('unsupported'), reason: z.string() }), + z.object({ status: z.literal('denied'), reason: z.string() }), + ]) + +const healthSchema = z.object({ + db: z.string(), + driver: z.string(), + worker_version: z.string(), + pool: z.object({ + max: z.number(), + size: z.number().nullish(), + idle: z.number().nullish(), + waiting: z.number().nullish(), + }), + active_queries: probeSchema( + z.array( + z.object({ + id: z.string(), + sql: z.string(), + state: z.string().nullish(), + duration_ms: z.number().nullish(), + user: z.string().nullish(), + }), + ), + ), + table_sizes: probeSchema( + z.array( + z.object({ + table: z.string(), + schema: z.string().nullish(), + total_bytes: z.number().nullish(), + index_bytes: z.number().nullish(), + row_estimate: z.number().nullish(), + }), + ), + ), + locks: probeSchema( + z.array( + z.object({ + blocked_id: z.string(), + blocked_sql: z.string(), + blocking_id: z.string(), + blocking_sql: z.string().nullish(), + relation: z.string().nullish(), + }), + ), + ), + cache: probeSchema( + z.object({ + hit_ratio: z.number(), + blocks_hit: z.number(), + blocks_read: z.number(), + }), + ), +}) +export type HealthReport = z.infer + +export async function health(host: Host, db: string): Promise { + return call(host, DB.health, { db }, healthSchema) +} + +const diagramSchema = z.object({ + nodes: z.array( + z.object({ + table: z.string(), + schema: z.string().nullish(), + x: z.number(), + y: z.number(), + w: z.number(), + h: z.number(), + rank: z.number(), + degree: z.number(), + hidden_columns: z.number(), + columns: z.array( + z.object({ + name: z.string(), + type: z.string(), + primary_key: z.boolean(), + foreign_key: z.boolean(), + nullable: z.boolean(), + }), + ), + }), + ), + edges: z.array( + z.object({ + from: z.string(), + from_column: z.string(), + to: z.string(), + to_column: z.string(), + points: z.array(z.object({ x: z.number(), y: z.number() })), + self_loop: z.boolean(), + }), + ), + width: z.number(), + height: z.number(), + isolated: z.array(z.string()), + // Optional so an older worker still renders — it just draws no boundaries. + components: z + .array( + z.object({ + index: z.number(), + tables: z.array(z.string()), + hub: z.string().nullish(), + x: z.number(), + y: z.number(), + w: z.number(), + h: z.number(), + }), + ) + .nullish() + .transform((v) => v ?? []), + crossings: z.number(), + truncated: z.boolean(), + focus: z.string().nullish(), + frontier: z + .array(z.string()) + .nullish() + .transform((v) => v ?? []), +}) +export type SchemaDiagram = z.infer + +export interface DiagramOptions { + /** Draw only this table's neighbourhood. */ + focus?: string | null + /** Foreign-key hops out from `focus`. */ + depth?: number +} + +export async function schemaDiagram(host: Host, db: string, opts: DiagramOptions = {}): Promise { + return call( + host, + DB.schemaDiagram, + // `focus` is omitted rather than sent as null: the worker treats absence + // as "the whole schema", and an explicit null would have to mean the same + // thing in two places. + opts.focus ? { db, focus: opts.focus, depth: opts.depth ?? 1 } : { db }, + diagramSchema, + ) +} + +const savedQuerySchema = z.object({ + id: z.string(), + name: z.string(), + sql: z.string(), + description: z.string().nullish(), + saved_at: z.string(), +}) +export type SavedQuery = z.infer + +export async function listSavedQueries(host: Host, db: string): Promise { + const res = await call( + host, + DB.listSavedQueries, + { db }, + z.object({ queries: z.array(savedQuerySchema), count: z.number() }), + ) + return res.queries +} + +export async function saveQuery( + host: Host, + db: string, + name: string, + sql: string, +): Promise<{ id: string; replaced: boolean }> { + return call(host, DB.saveQuery, { db, name, sql }, z.object({ id: z.string(), replaced: z.boolean() })) +} + +export async function deleteSavedQuery(host: Host, db: string, id: string): Promise { + const res = await call(host, DB.deleteSavedQuery, { db, id }, z.object({ deleted: z.boolean() })) + return res.deleted +} + +const historySchema = z.object({ + entries: z.array( + z.object({ + sql: z.string(), + verb: z.string(), + duration_ms: z.number().nullish(), + row_count: z.number().nullish(), + at: z.string(), + }), + ), + count: z.number(), +}) +export type HistoryEntry = z.infer['entries'][number] + +const tableViewSchema = z.object({ + widths: z + .record(z.string(), z.number()) + .nullish() + .transform((v) => v ?? {}), + hidden: z + .array(z.string()) + .nullish() + .transform((v) => v ?? []), + order: z + .array(z.string()) + .nullish() + .transform((v) => v ?? []), +}) +export type TableView = z.infer + +export async function getTableView(host: Host, db: string, table: string): Promise { + return call(host, DB.getTableView, { db, table }, tableViewSchema) +} + +export async function saveTableView(host: Host, db: string, table: string, view: TableView): Promise { + const r = await call(host, DB.saveTableView, { db, table, ...view }, z.object({ saved: z.boolean() })) + return r.saved +} + +export async function history(host: Host, db: string, limit = 25): Promise { + const res = await call(host, DB.history, { db, limit }, historySchema) + return res.entries +} diff --git a/database/ui/src/page/CellInspector.tsx b/database/ui/src/page/CellInspector.tsx new file mode 100644 index 000000000..189a92ee0 --- /dev/null +++ b/database/ui/src/page/CellInspector.tsx @@ -0,0 +1,146 @@ +/** + * The focused cell, in full. + * + * A pure function of the grid cursor: arrows keep driving the grid, this + * follows. That is why it can sit beside the grid without competing for the + * keyboard. + * + * Two measurements are worth the lines. **Code points, not `.length`** — + * `'👋'.length` is 2, which is the kind of number that makes someone doubt the + * whole panel; `[...s].length` is 1. And **bytes via `TextEncoder`**, because + * a `varchar(20)` limit counts bytes on some drivers and characters on others, + * and "why did this fail to insert" usually ends here. + */ + +import { Badge, Button } from '@iii-dev/console-ui' +import { useMemo, useState } from 'react' +import { cellText } from '../lib/grid-cursor' +import type { ForeignKeyRef } from '../lib/rpc' +import { Check, Copy, Link2 } from './icons' +import { JsonTree } from './JsonTree' + +const ENCODER = new TextEncoder() + +export function CellInspector({ + table, + column, + type, + value, + row, + rowCount, + col, + colCount, + foreignKey, + onFollow, +}: { + table: string + column: string + type?: string + value: unknown + row: number + rowCount: number + col: number + colCount: number + foreignKey?: ForeignKeyRef + onFollow?: (fk: ForeignKeyRef, value: unknown) => void +}) { + const [copied, setCopied] = useState(null) + const [wrap, setWrap] = useState(true) + + const text = useMemo(() => cellText(value), [value]) + const isNull = value === null || value === undefined + const json = useMemo(() => asJson(value), [value]) + + const copy = (what: string, payload: string) => { + void navigator.clipboard?.writeText(payload) + setCopied(what) + // Recolour briefly rather than raising a toast — the house pattern. + setTimeout(() => setCopied(null), 1200) + } + + const codePoints = isNull ? 0 : [...text].length + const bytes = isNull ? 0 : ENCODER.encode(text).length + + return ( +
+
+ + {table}.{column} + + {type ? {type.toLowerCase()} : null} +
+ +
+ row {row + 1} of {rowCount} · column {col + 1} of {colCount} +
+ +
+
kind
+
{kindOf(value)}
+
characters
+
{codePoints.toLocaleString()}
+
bytes
+
{bytes.toLocaleString()}
+
+ + {foreignKey && onFollow && !isNull ? ( + + ) : null} + +
+ {isNull ? ( + NULL + ) : value === '' ? ( + // Distinct from NULL, and from an empty box. + '' (empty string) + ) : json !== undefined ? ( + copy('json', t)} /> + ) : ( + 
{text}
+ )} +
+ +
+ + {json === undefined && text.length > 80 ? ( + + ) : null} +
+
+ ) +} + +function kindOf(value: unknown): string { + if (value === null || value === undefined) return 'null' + if (Array.isArray(value)) return 'array' + return typeof value +} + +/** + * The value as JSON, when it is worth a tree. + * + * Objects qualify directly. A *string* qualifies only when it parses to an + * object or array — many drivers hand back JSON columns as text, and rendering + * that as a flat line loses the structure the column exists to hold. A string + * that parses to a bare number is left alone: `"5"` is not a document. + */ +function asJson(value: unknown): unknown { + if (value !== null && typeof value === 'object') return value + if (typeof value !== 'string') return undefined + const trimmed = value.trim() + if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) return undefined + try { + const parsed = JSON.parse(trimmed) + return typeof parsed === 'object' && parsed !== null ? parsed : undefined + } catch { + return undefined + } +} diff --git a/database/ui/src/page/ChangesPanel.tsx b/database/ui/src/page/ChangesPanel.tsx new file mode 100644 index 000000000..1751fbfb7 --- /dev/null +++ b/database/ui/src/page/ChangesPanel.tsx @@ -0,0 +1,185 @@ +/** + * What has changed in the selected table, as it happens. + * + * The worker has emitted `database::row-changed` for a while and #640 gave it + * real cross-client capture, but nothing ever rendered it — so a write from + * another client was observable to agents and invisible to the person looking + * at the table. This is that view. + * + * The panel is deliberate about what it promises. It never uses the words + * "live" or "realtime"; it names the guarantee in force, because the two + * capture modes differ in a way that matters and only one of them sees other + * clients' writes. + */ + +import { Badge, Button, EmptyState, type Host, StatusDot } from '@iii-dev/console-ui' +import { useEffect, useState } from 'react' +import { History, RefreshCw } from './icons' +import { type RowChange, useRowChanges } from './useRowChanges' + +const HistoryIcon = (p: { className?: string }) => + +/** How long a row keeps its arrival highlight. */ +const FRESH_MS = 4000 + +const OP_LABEL: Record = { + insert: 'insert', + update: 'update', + delete: 'delete', + other: 'other', +} + +export function ChangesPanel({ + host, + db, + table, + kind, + onRefresh, +}: { + host: Host + db: string + table: string | null + /** `view` cannot be followed — see below. */ + kind?: string + onRefresh?: () => void +}) { + // A view is never the target of a write. The worker keys every change on the + // table the statement actually touched, so a binding on a view matches + // nothing and would sit at "following" forever while rows visibly change + // underneath it. Refuse the binding and say why. + const isView = kind === 'view' + const feed = useRowChanges(host, db, isView ? null : table) + // Re-render on a slow cadence so relative stamps and the fade advance + // without an interval per row. + const [, tick] = useState(0) + useEffect(() => { + if (feed.changes.length === 0) return + const id = setInterval(() => tick((n) => n + 1), 1000) + return () => clearInterval(id) + }, [feed.changes.length]) + + if (!table) { + return ( + + ) + } + + if (isView) { + return ( + + ) + } + + if (feed.status === 'unsupported') { + return ( + + ) + } + + return ( +
+
+ + {table} +
+ {feed.pending > 0 && onRefresh ? ( + + ) : null} +
+ + {feed.changes.length === 0 ? ( +
+

nothing yet.

+

+ writes to {table} appear here as they commit. what counts as a write depends on this + connection's capture mode — hover the indicator above. +

+
+ ) : ( +
    + {feed.changes.map((c, i) => ( + + ))} +
+ )} +
+ ) +} + +function ChangeRow({ change }: { change: RowChange }) { + const age = Date.now() - change.seen + const fresh = age < FRESH_MS + const rows = change.affected_rows + return ( +
  • + {OP_LABEL[change.op] ?? change.op} + + {rows} {rows === 1 ? 'row' : 'rows'} + + {change.returning?.length ? {change.returning.length} returned : null} + + {relative(age)} + +
    • + ) +} + +/** + * Which guarantee is in force. + * + * The mode is a property of the connection and is not exposed as a read, so + * this reports what it can actually stand behind: that a binding is attached, + * and when something last arrived. It never claims completeness — the tooltip + * carries the caveat rather than a word in the badge implying more than is + * true. + */ +function FreshnessBadge({ status, lastAt }: { status: string; lastAt: number | null }) { + const bound = status === 'bound' + const recent = lastAt != null && Date.now() - lastAt < 10_000 + return ( + + + {bound ? 'following' : 'not following'} + {lastAt != null ? · last {relative(Date.now() - lastAt)} : null} + + ) +} + +function relative(ms: number): string { + if (ms < 1000) return 'just now' + const s = Math.floor(ms / 1000) + if (s < 60) return `${s}s ago` + const m = Math.floor(s / 60) + if (m < 60) return `${m}m ago` + const h = Math.floor(m / 60) + return `${h}h ago` +} diff --git a/database/ui/src/page/ColumnStatsPanel.tsx b/database/ui/src/page/ColumnStatsPanel.tsx new file mode 100644 index 000000000..6fa2e1cc0 --- /dev/null +++ b/database/ui/src/page/ColumnStatsPanel.tsx @@ -0,0 +1,134 @@ +/** + * What is actually in a column. + * + * `source: planner` means the numbers come from statistics the database keeps + * for the query planner — cheap, and possibly stale or absent. `computed` + * means real aggregates ran. That distinction is rendered as a chip on every + * result, because a stale `n_distinct` presented as fact is worse than no + * number: it reads exactly like a measurement. + */ + +import { Badge, Button, type Host, StatusPanel } from '@iii-dev/console-ui' +import { useCallback, useState } from 'react' +import { type ColumnStat, columnStats } from '../lib/rpc' +import { AlertCircle } from './icons' +import { useDatabaseRead } from './useDatabaseRead' + +export function ColumnStatsPanel({ + host, + db, + table, + column, +}: { + host: Host + db: string + table: string + column: string +}) { + const [exact, setExact] = useState(false) + const fetcher = useCallback(() => columnStats(host, db, table, [column], exact), [host, db, table, column, exact]) + const read = useDatabaseRead(true, fetcher) + + if (read.error) { + return ( + } + headline="could not profile this column" + detail={read.error} + /> + ) + } + if (!read.data) { + return
    · reading statistics…
    + } + + const stat: ColumnStat | undefined = read.data.columns[0] + if (!stat) { + return

    no statistics for this column.

    + } + + const total = stat.row_count ?? 0 + const nullFrac = stat.null_fraction ?? (total > 0 && stat.null_count != null ? stat.null_count / total : null) + const topMax = Math.max(...stat.top_values.map((v) => v.count), 1) + + return ( +
    +
    + {stat.name} + {stat.source === 'planner' ? ( + + estimated · from planner statistics + + ) : ( + measured + )} +
    + + {stat.source === 'planner' ? ( + + ) : null} + +
    + + + + + + +
    + + {nullFrac != null ? ( +
    + null share + + + + {(nullFrac * 100).toFixed(1)}% +
    + ) : null} + + {stat.top_values.length > 0 ? ( + <> +

    most common

    +
      + {stat.top_values.map((v, i) => ( +
    • + + {render(v.value)} + + + + + {v.count.toLocaleString()} +
      • + ))} +
    + + ) : null} +
    + ) +} + +function Row({ label, value }: { label: string; value: string }) { + return ( + <> +
    {label}
    +
    {value}
    + + ) +} + +function fmt(n: number | null | undefined): string { + return n == null ? '—' : n.toLocaleString() +} + +/** Null and empty string must not look alike here either. */ +function render(v: unknown): string { + if (v == null) return 'NULL' + if (v === '') return "''" + if (typeof v === 'object') return JSON.stringify(v) + return String(v) +} diff --git a/database/ui/src/page/ErdPanel.tsx b/database/ui/src/page/ErdPanel.tsx new file mode 100644 index 000000000..1da2211e3 --- /dev/null +++ b/database/ui/src/page/ErdPanel.tsx @@ -0,0 +1,490 @@ +/** + * Schema diagram. + * + * The worker does the layout — `database::schemaDiagram` returns positioned + * nodes and already-routed edge polylines, so this panel only draws and pans. + * That keeps the expensive part off the console's single React tree, and it + * means an agent can read the same shape without rendering anything. + * + * Rendering follows the house pattern used by the worktree and memory pages: + * absolutely-positioned HTML nodes over one SVG for the edges. Pan and zoom + * live in a ref and write `transform` straight onto the canvas during + * pointermove — putting them in state would re-render every node on every + * mouse move. + */ + +import { Badge, Button, EmptyState, type Host, Input, Select, StatusPanel } from '@iii-dev/console-ui' +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { schemaDiagram } from '../lib/rpc' +import { AlertCircle, KeyRound, Link2, RefreshCw, Table2 } from './icons' +import { useDatabaseRead } from './useDatabaseRead' + +const MIN_ZOOM = 0.15 +const MAX_ZOOM = 2.5 +/** Below this, column rows are unreadable anyway — drop them and keep 200 + * headers instead of 2400 rows in the DOM. */ +const COMPACT_BELOW = 0.5 +const PAD = 60 +/** Breathing room between a component's boundary and its outermost node. */ +const GROUP_PAD = 22 + +const TableIcon = (p: { className?: string }) => + +type Offset = { dx: number; dy: number } | undefined +type Pt = { x: number; y: number } + +/** + * Shift a routed edge to follow nodes that have been dragged. + * + * The worker emits a three-segment elbow: source anchor, two corner points in + * the gutter, target anchor. Moving the ends alone would leave the corners + * behind and bend the line through them, so each point is offset by a blend of + * the two ends weighted by how far along the polyline it sits. With neither + * end moved this is the identity. + */ +function shiftEdge(points: Pt[], from: Offset, to: Offset): Pt[] { + if (!from && !to) return points + const fx = from?.dx ?? 0 + const fy = from?.dy ?? 0 + const tx = to?.dx ?? 0 + const ty = to?.dy ?? 0 + const last = points.length - 1 + if (last <= 0) return points.map((p) => ({ x: p.x + fx, y: p.y + fy })) + return points.map((p, i) => { + const t = i / last + return { x: p.x + fx * (1 - t) + tx * t, y: p.y + fy * (1 - t) + ty * t } + }) +} + +interface View { + x: number + y: number + z: number +} + +export function ErdPanel({ + host, + db, + focusTable, +}: { + host: Host + db: string + /** Open focused on this table rather than on the whole schema. */ + focusTable?: string | null +}) { + // Focus turns the diagram from a wall into something explorable: one table + // and what it touches, expanding a hop at a time. + const [focus, setFocus] = useState(focusTable ?? null) + const [depth, setDepth] = useState(1) + + const fetcher = useCallback(() => schemaDiagram(host, db, { focus, depth }), [host, db, focus, depth]) + const read = useDatabaseRead(true, fetcher) + const diagram = read.data + + const [search, setSearch] = useState('') + const [selected, setSelected] = useState(null) + // Manual node positions, as offsets over what the worker laid out. Kept in + // the browser: this is where you dragged a box just now, not a preference + // worth persisting for everyone. + const [offsets, setOffsets] = useState>({}) + const nodeDragRef = useRef<{ table: string; px: number; py: number } | null>(null) + // Mirrored: `view` drives React, `viewRef` drives the raw transform during a + // drag so a 200-node diagram does not re-render per frame. + const [view, setView] = useState({ x: 0, y: 0, z: 1 }) + const viewRef = useRef(view) + const canvasRef = useRef(null) + const frameRef = useRef(null) + const dragRef = useRef<{ px: number; py: number } | null>(null) + const rafRef = useRef(null) + + const apply = useCallback(() => { + rafRef.current = null + const el = canvasRef.current + if (!el) return + const v = viewRef.current + el.style.transform = `translate(${v.x}px, ${v.y}px) scale(${v.z})` + el.classList.toggle('compact', v.z < COMPACT_BELOW) + }, []) + + const schedule = useCallback(() => { + if (rafRef.current == null) rafRef.current = requestAnimationFrame(apply) + }, [apply]) + + const setViewNow = useCallback( + (next: View) => { + viewRef.current = next + setView(next) + schedule() + }, + [schedule], + ) + + /** Scale the diagram to fit, so a wide schema is not off-screen on open. */ + const fitToFrame = useCallback(() => { + const frame = frameRef.current + if (!frame || !diagram || diagram.width === 0) return + const z = Math.min( + 1, + Math.max( + MIN_ZOOM, + Math.min((frame.clientWidth - PAD) / diagram.width, (frame.clientHeight - PAD) / diagram.height), + ), + ) + // Centre, rather than pinning to a corner and leaving the rest empty. + setViewNow({ + x: Math.max(PAD / 2, (frame.clientWidth - diagram.width * z) / 2), + y: Math.max(PAD / 2, (frame.clientHeight - diagram.height * z) / 2), + z, + }) + }, [diagram, setViewNow]) + + useEffect(() => { + fitToFrame() + }, [fitToFrame]) + + // A new layout invalidates hand-placed offsets: they were relative to + // coordinates that no longer exist. + // biome-ignore lint/correctness/useExhaustiveDependencies: keyed on the layout, not the object identity + useEffect(() => { + setOffsets({}) + }, [focus, depth]) + + useEffect( + () => () => { + if (rafRef.current != null) cancelAnimationFrame(rafRef.current) + }, + [], + ) + + /** + * Zoom on wheel. + * + * Bound natively with `{ passive: false }` rather than through React's + * `onWheel`. React attaches wheel handlers passively, where + * `preventDefault()` is a no-op — so the page scrolled underneath the + * diagram instead of the diagram zooming. + */ + useEffect(() => { + const frame = frameRef.current + if (!frame) return + const onWheel = (e: WheelEvent) => { + e.preventDefault() + const rect = frame.getBoundingClientRect() + const v = viewRef.current + const next = Math.min(MAX_ZOOM, Math.max(MIN_ZOOM, v.z * (e.deltaY < 0 ? 1.12 : 1 / 1.12))) + // Keep the point under the cursor fixed while zooming. + const cx = e.clientX - rect.left + const cy = e.clientY - rect.top + const k = next / v.z + setViewNow({ x: cx - (cx - v.x) * k, y: cy - (cy - v.y) * k, z: next }) + } + frame.addEventListener('wheel', onWheel, { passive: false }) + return () => frame.removeEventListener('wheel', onWheel) + }, [setViewNow]) + + const onPointerDown = (e: React.PointerEvent) => { + if (e.button !== 0) return + dragRef.current = { px: e.clientX, py: e.clientY } + ;(e.currentTarget as HTMLElement).setPointerCapture(e.pointerId) + } + + const onPointerMove = (e: React.PointerEvent) => { + const d = dragRef.current + if (!d) return + const v = viewRef.current + viewRef.current = { ...v, x: v.x + (e.clientX - d.px), y: v.y + (e.clientY - d.py) } + dragRef.current = { px: e.clientX, py: e.clientY } + schedule() + } + + const onPointerUp = (e: React.PointerEvent) => { + if (!dragRef.current) return + dragRef.current = null + ;(e.currentTarget as HTMLElement).releasePointerCapture(e.pointerId) + // Commit once, at the end, rather than on every frame. + setView(viewRef.current) + } + + /** + * Drag one node. Offsets are divided by the zoom so a box tracks the cursor + * at any scale — without that, dragging at 40% moves the node 2.5x too far. + */ + const startNodeDrag = (e: React.PointerEvent, table: string) => { + if (e.button !== 0) return + e.stopPropagation() + nodeDragRef.current = { table, px: e.clientX, py: e.clientY } + ;(e.currentTarget as HTMLElement).setPointerCapture(e.pointerId) + } + + const moveNode = (e: React.PointerEvent) => { + const d = nodeDragRef.current + if (!d) return + e.stopPropagation() + const z = viewRef.current.z || 1 + const dx = (e.clientX - d.px) / z + const dy = (e.clientY - d.py) / z + nodeDragRef.current = { ...d, px: e.clientX, py: e.clientY } + setOffsets((prev) => { + const at = prev[d.table] ?? { dx: 0, dy: 0 } + return { ...prev, [d.table]: { dx: at.dx + dx, dy: at.dy + dy } } + }) + } + + const endNodeDrag = (e: React.PointerEvent) => { + if (!nodeDragRef.current) return + nodeDragRef.current = null + ;(e.currentTarget as HTMLElement).releasePointerCapture(e.pointerId) + } + + const matches = useMemo(() => { + const q = search.trim().toLowerCase() + if (!q || !diagram) return null + return new Set(diagram.nodes.filter((n) => n.table.toLowerCase().includes(q)).map((n) => n.table)) + }, [search, diagram]) + + if (read.error) { + return ( + } + headline="could not lay out the schema" + detail={read.error} + /> + ) + } + if (read.loading && !diagram) { + return
    · laying out the schema…
    + } + if (!diagram || diagram.nodes.length === 0) { + return ( + + ) + } + + const related = diagram.nodes.length - diagram.isolated.length + + return ( +
    +
    + + {diagram.nodes.length} tables · {diagram.edges.length} relations + + {related === 0 ? ( + no foreign keys — nothing to connect + ) : ( + + {diagram.isolated.length} unrelated · {diagram.crossings} crossings + + )} + {diagram.truncated ? truncated : null} +
    + {focus ? ( + <> + + focused on {focus} + + + {Math.round(view.z * 100)}% + {Object.keys(offsets).length > 0 ? ( + + ) : null} + +
    + + {/* What was left out. A diagram that simply stops looks complete; naming + the next ring is what makes expanding a decision rather than a guess. */} + {focus && diagram.frontier.length > 0 ? ( +
    + also connected: + {diagram.frontier.map((t) => ( + + ))} + +
    + ) : null} + +
    +
    + {/* Component boundaries sit behind everything. A schema is usually + several independent clusters, and drawing that fact is what lets + a reader take in the structure before reading a single name. */} + {diagram.components + .filter((c) => c.tables.length > 1) + .map((c) => ( +
    + + {c.hub ? `${c.hub} · ` : ''} + {c.tables.length} related + +
    + ))} + + + foreign key relationships + {diagram.edges.map((e) => { + const dim = matches ? !matches.has(e.from) && !matches.has(e.to) : false + const hot = selected === e.from || selected === e.to + // An edge has to follow the nodes it joins. The worker routed it + // against the original coordinates, so each end is shifted by + // whatever its own node was dragged and the corner between them + // is interpolated — the elbow stays orthogonal without + // re-running the router in the browser. + const from = offsets[e.from] + const to = offsets[e.to] + const points = shiftEdge(e.points, from, to) + const a = points[0] + const b = points[points.length - 1] + return ( + ${e.to}.${e.to_column}`} + className={`db-erd-edge${hot ? ' hot' : ''}${dim ? ' dim' : ''}`} + > + `${p.x},${p.y}`).join(' ')} fill="none" /> + {/* Endpoint dots, the idiom both reference clients use: the + line lands on the exact column row, so a relationship + reads as column-to-column rather than box-to-box. */} + + + + ) + })} + + + {diagram.nodes.map((n) => { + const dim = matches ? !matches.has(n.table) : false + // Hub emphasis by border weight, not fill — the palette stays rationed. + const hub = n.degree >= 5 ? ' hub-3' : n.degree >= 2 ? ' hub-2' : '' + const off = offsets[n.table] + const isFocus = diagram.focus === n.table + return ( +
    + {/* The header is the drag handle as well as the button: one + place to grab, and a click that never moved still selects. */} + +
    + {n.columns.map((c) => ( +
    + {c.primary_key ? ( + + ) : c.foreign_key ? ( + + ) : ( + + )} + {c.name} + {/* Required columns carry a marker rather than the + nullable ones: in most schemas NOT NULL is the + smaller set, so it is the one worth marking. */} + {!c.nullable && !c.primary_key ? ( + + * + + ) : null} + {c.type.toLowerCase()} +
    + ))} + {n.hidden_columns > 0 ?
    +{n.hidden_columns} more
    : null} +
    +
    + ) + })} +
    +
    +
    + ) +} diff --git a/database/ui/src/page/FilterBar.tsx b/database/ui/src/page/FilterBar.tsx new file mode 100644 index 000000000..647331304 --- /dev/null +++ b/database/ui/src/page/FilterBar.tsx @@ -0,0 +1,263 @@ +/** + * Filters as chips, compiled server-side. + * + * Each chip is segmented — column, operator, value — and every segment is its + * own control, so refining one part of a condition does not mean retyping the + * rest. Two behaviours are worth calling out because both are easy to omit and + * both are what make a filter bar usable rather than merely present: + * + * A chip can be **switched off** instead of deleted. Narrowing a query is + * iterative, and "does this condition matter?" is the question being asked + * most often; answering it by deleting and rebuilding loses the work. + * + * An **incomplete chip is a draft** and is never sent. The worker would reject + * it, and a round trip that ends in an error would make typing feel broken. + * Drafts read as dashed, the repo's convention for content that is not yet + * real. + * + * Filters stack with AND. `in` exists precisely because OR does not: "status + * is one of open, held" cannot be expressed as two AND'd equalities. + */ + +import { + Button, + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, + Input, +} from '@iii-dev/console-ui' +import { useState } from 'react' +import { type FilterOp, type FilterSpec, isComplete, OP_LABEL, opsFor, SET_OPS, typeCategory } from '../lib/rpc' +import { Eye, EyeOff, Plus, X } from './icons' + +/** The minimum a column must expose to be filterable. */ +export interface FilterColumn { + name: string + type: string +} + +export function FilterBar({ + columns, + filters, + onChange, +}: { + columns: FilterColumn[] + filters: FilterSpec[] + onChange: (next: FilterSpec[]) => void +}) { + const replace = (i: number, f: FilterSpec) => onChange(filters.map((x, j) => (j === i ? f : x))) + const remove = (i: number) => onChange(filters.filter((_, j) => j !== i)) + + const add = () => { + const first = columns[0] + if (!first) return + onChange([...filters, { column: first.name, op: opsFor(typeCategory(first.type))[0] }]) + } + + const active = filters.filter((f) => !f.disabled && isComplete(f)).length + const drafts = filters.filter((f) => !isComplete(f)).length + + return ( +
    + {filters.map((f, i) => ( + replace(i, next)} + onRemove={() => remove(i)} + /> + ))} + + + + {filters.length > 0 ? ( + <> + + {active} applied + {drafts > 0 ? ` · ${drafts} draft` : ''} + + + + ) : null} +
    + ) +} + +function Chip({ + filter, + columns, + onChange, + onRemove, +}: { + filter: FilterSpec + columns: FilterColumn[] + onChange: (next: FilterSpec) => void + onRemove: () => void +}) { + const col = columns.find((c) => c.name === filter.column) + const ops = opsFor(typeCategory(col?.type)) + const draft = !isComplete(filter) + const isSet = SET_OPS.has(filter.op) + const takesValue = !NO_VALUE.has(filter.op) + + const setColumn = (name: string) => { + const next = columns.find((c) => c.name === name) + const allowed = opsFor(typeCategory(next?.type)) + // Keep the operator when the new column still supports it; otherwise fall + // back rather than leaving a chip that cannot compile. + onChange({ + ...filter, + column: name, + op: allowed.includes(filter.op) ? filter.op : allowed[0], + }) + } + + const setOp = (op: FilterOp) => { + // Moving between shapes must not carry stale operands across, or a chip + // reads as complete while holding a value its operator ignores. + const carry: FilterSpec = { ...filter, op } + if (SET_OPS.has(op)) { + carry.values = filter.values ?? (filter.value != null ? [filter.value] : []) + carry.value = undefined + carry.value2 = undefined + } else { + carry.value = filter.value ?? filter.values?.[0] + carry.values = undefined + if (op !== 'between') carry.value2 = undefined + } + onChange(carry) + } + + return ( + + + + ({ value: c.name, label: c.name, hint: c.type.toLowerCase() }))} + onPick={setColumn} + /> + + ({ value: o, label: OP_LABEL[o], hint: o }))} + onPick={(v) => setOp(v as FilterOp)} + /> + + {takesValue ? ( + isSet ? ( + onChange({ ...filter, values })} + /> + ) : ( + <> + onChange({ ...filter, value })} label="value" /> + {filter.op === 'between' ? ( + onChange({ ...filter, value2 })} + label="upper bound" + /> + ) : null} + + ) + ) : null} + + + + ) +} + +const NO_VALUE: ReadonlySet = new Set(['is_true', 'is_false', 'is_null', 'is_not_null', 'is_empty']) + +function Picker({ + label, + className, + items, + onPick, +}: { + label: string + className: string + items: { value: string; label: string; hint?: string }[] + onPick: (value: string) => void +}) { + return ( + + + + + + {items.map((it) => ( + onPick(it.value)}> + {it.label} + {it.hint ? {it.hint} : null} + + ))} + + + ) +} + +function ValueInput({ value, onChange, label }: { value: unknown; onChange: (v: string) => void; label: string }) { + return ( + + + + ) +} + +/** Comma-separated entry for `in` / `not_in`. */ +function SetValue({ values, onChange }: { values: unknown[]; onChange: (v: unknown[]) => void }) { + const [text, setText] = useState(values.map((v) => String(v)).join(', ')) + return ( + + { + setText(next) + onChange( + next + .split(',') + .map((s) => s.trim()) + .filter((s) => s !== ''), + ) + }} + placeholder="a, b, c" + preserveCase + aria-label="values, comma separated" + className="db-chip-input wide" + /> + + ) +} diff --git a/database/ui/src/page/HealthPanel.tsx b/database/ui/src/page/HealthPanel.tsx new file mode 100644 index 000000000..76cf015de --- /dev/null +++ b/database/ui/src/page/HealthPanel.tsx @@ -0,0 +1,295 @@ +/** + * Connection health. + * + * Every section is a `ProbeResult`, and rendering that honestly is the entire + * point of the panel. "sqlite has no equivalent of pg_stat_activity", + * "permission denied on pg_stat_activity" and "no queries are running" are + * three different facts, and a panel that shows an empty list for all three + * teaches you to distrust it. + */ + +import { Button, type Host, Select, StatusPanel } from '@iii-dev/console-ui' +import { useCallback, useEffect, useState } from 'react' +import { type HealthReport, health } from '../lib/rpc' +import { AlertCircle, RefreshCw } from './icons' +import { useDatabaseRead } from './useDatabaseRead' + +type Probe = + | { status: 'available'; data: T } + | { status: 'unsupported'; reason: string } + | { status: 'denied'; reason: string } + +const INTERVALS = [ + { value: '0', label: 'manual' }, + { value: '5', label: 'every 5s' }, + { value: '15', label: 'every 15s' }, + { value: '60', label: 'every 60s' }, +] + +export function HealthPanel({ host, db }: { host: Host; db: string }) { + const [nonce, setNonce] = useState(0) + const [every, setEvery] = useState('0') + const fetcher = useCallback(() => { + void nonce + return health(host, db) + }, [host, db, nonce]) + const read = useDatabaseRead(true, fetcher) + + const refresh = useCallback(() => setNonce((n) => n + 1), []) + + useEffect(() => { + const seconds = Number(every) + if (!seconds) return + const id = setInterval(refresh, seconds * 1000) + return () => clearInterval(id) + }, [every, refresh]) + + if (read.error) { + return ( + } + headline="could not read connection health" + detail={read.error} + /> + ) + } + if (!read.data) { + return
    · probing the connection…
    + } + + const h: HealthReport = read.data + const pool = h.pool + const inUse = pool.size != null && pool.idle != null ? pool.size - pool.idle : null + + return ( +
    +
    + {h.driver} + worker {h.worker_version} +
    +