diff --git a/CHANGELOG.md b/CHANGELOG.md index f4b07d4..cce72c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Auto-resolve `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` from `~/.secrets` when not in environment (fixes doctor + judge in non-shell contexts) - Multi-path example dataset resolution in `evals doctor` (works globally installed) - `--module`, `--export`, `--command`, `--mcp-command`, `--tool` options on `evals run` and `evals ci run` -- `evals sync push/pull/status` commands via `@hasna/cloud` SDK +- `evals sync push/pull/status` commands via the legacy shared sync SDK - Shell completion: `evals completion bash` / `evals completion zsh` - React SPA dashboard served by `evals-serve` - Pass^k metric (`repeat: N`, `passThreshold`) on eval cases @@ -51,7 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - OpenAI v6 `tool_calls` type change (`function` property access) ### Changed -- Upgraded all dependencies to latest: `@anthropic-ai/sdk@0.82`, `openai@6`, `zod@4`, `commander@14`, `typescript@6`, `@modelcontextprotocol/sdk@1.29`, `@hasna/cloud@1.30` +- Upgraded all dependencies to latest at the time, including the legacy shared sync package that has since been removed from active runtime use ## [0.1.0] - 2026-03-27 diff --git a/README.md b/README.md index da6c429..95dc487 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,19 @@ evals mcp register --gemini # Gemini (~/.gemini/settings.json) evals mcp register --all # all three at once ``` +## Storage Sync + +Runs and baselines are stored locally in SQLite. Set a remote PostgreSQL storage URL to sync them: + +```bash +export HASNA_EVALS_DATABASE_URL="postgres://..." + +evals sync status +evals sync push +evals sync pull +evals sync sync +``` + --- ## CI / GitHub Actions diff --git a/hasna-attachments-1.0.24.tgz b/hasna-attachments-1.0.24.tgz new file mode 100644 index 0000000..7af7c0e Binary files /dev/null and b/hasna-attachments-1.0.24.tgz differ diff --git a/package.json b/package.json index 23c4d8a..4d61a34 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hasna/evals", - "version": "0.1.21", + "version": "0.1.25", "description": "Open source AI evaluation framework — LLM-as-judge + assertion-based evals for any AI app. CLI + MCP server.", "type": "module", "main": "dist/index.js", @@ -14,6 +14,10 @@ ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js" + }, + "./storage": { + "types": "./dist/storage.d.ts", + "import": "./dist/storage.js" } }, "files": [ @@ -24,7 +28,7 @@ "README.md" ], "scripts": { - "build": "cd dashboard && bun run build && cd .. && bun build src/cli/index.ts --outdir dist/cli --target bun --external ink --external react --external chalk --external @modelcontextprotocol/sdk && bun build src/mcp/index.ts --outdir dist/mcp --target bun --external @modelcontextprotocol/sdk && bun build src/server/index.ts --outdir dist/server --target bun && bun build src/index.ts --outdir dist --target bun && tsc --emitDeclarationOnly --outDir dist", + "build": "rm -rf dist && cd dashboard && bun run build && cd .. && bun build src/cli/index.ts --outdir dist/cli --target bun --external ink --external react --external chalk --external @modelcontextprotocol/sdk && bun build src/mcp/index.ts --outdir dist/mcp --target bun --external @modelcontextprotocol/sdk && bun build src/server/index.ts --outdir dist/server --target bun && bun build src/index.ts src/storage.ts --outdir dist --target bun && tsc --emitDeclarationOnly --outDir dist", "build:dashboard": "cd dashboard && bun run build", "typecheck": "tsc --noEmit", "test": "bun test", @@ -65,16 +69,18 @@ "license": "Apache-2.0", "dependencies": { "@anthropic-ai/sdk": "^0.82.0", - "@hasna/cloud": "^0.1.30", "@modelcontextprotocol/sdk": "^1.29.0", "ajv": "^8.18.0", "chalk": "^5.4.1", "commander": "^14.0.3", "openai": "^6.33.0", - "zod": "^4.3.6" + "pg": "^8.13.3", + "zod": "^4.3.6", + "@hasna/events": "^0.1.6" }, "devDependencies": { "@types/bun": "^1.2.4", + "@types/pg": "^8.20.0", "typescript": "^6.0.2" } } diff --git a/src/adapters/anthropic-openai.test.ts b/src/adapters/anthropic-openai.test.ts index 2dcdf22..7715027 100644 --- a/src/adapters/anthropic-openai.test.ts +++ b/src/adapters/anthropic-openai.test.ts @@ -146,13 +146,10 @@ describe("OpenAI adapter", () => { }); }); -// ─── resolveKey in judge (via env injection) ───────────────────────────────── +// ─── judge import smoke ────────────────────────────────────────────────────── -describe("judge.ts resolveKey — secrets fallback", () => { - test("ANTHROPIC_API_KEY is injected from secrets on module load", async () => { - // The judge module runs resolveKey eagerly on load. - // In CI or clean envs it reads from ~/.secrets if available. - // We just verify the module imports without throwing. +describe("judge.ts", () => { + test("imports without resolving secrets on module load", async () => { const { runJudge } = await import("../core/judge.js"); expect(typeof runJudge).toBe("function"); }); diff --git a/src/cli/cli.test.ts b/src/cli/cli.test.ts index 7cc61bc..4d70451 100644 --- a/src/cli/cli.test.ts +++ b/src/cli/cli.test.ts @@ -162,3 +162,33 @@ describe("evals completion", () => { expect(zsh.stdout).toContain("sync:"); }); }); + +describe("evals sync", () => { + test("reports remote storage status with canonical env names", async () => { + const { stdout, exitCode } = await runCli(["sync", "status", "--json"]); + expect(exitCode).toBe(0); + + const status = JSON.parse(stdout) as { + configured: boolean; + mode: string; + env: string[]; + activeEnv: string | null; + service: string; + tables: string[]; + }; + + expect(status.configured).toBe(false); + expect(status.mode).toBe("local"); + expect(status.env).toEqual(["HASNA_EVALS_DATABASE_URL", "EVALS_DATABASE_URL"]); + expect(status.activeEnv).toBe(null); + expect(status.service).toBe("evals"); + expect(status.tables).toEqual(["runs", "baselines"]); + }); + + test("describes sync using storage terminology", async () => { + const { stdout, exitCode } = await runCli(["sync", "--help"]); + expect(exitCode).toBe(0); + expect(stdout).toContain("remote PostgreSQL storage"); + expect(stdout).not.toContain("cloud"); + }); +}); diff --git a/src/cli/commands/completion.ts b/src/cli/commands/completion.ts index 3e75180..7258175 100644 --- a/src/cli/commands/completion.ts +++ b/src/cli/commands/completion.ts @@ -55,7 +55,7 @@ _evals() { 'doctor:Health check' 'mcp:MCP server management' 'completion:Print shell completion script' - 'sync:Sync eval runs and datasets with cloud' + 'sync:Sync eval runs and datasets with remote storage' ) _arguments -C \\ diff --git a/src/cli/commands/doctor.ts b/src/cli/commands/doctor.ts index 47de167..0ac66fb 100644 --- a/src/cli/commands/doctor.ts +++ b/src/cli/commands/doctor.ts @@ -1,39 +1,4 @@ import { Command } from "commander"; -import { existsSync, readFileSync } from "fs"; -import { homedir } from "os"; -import { join } from "path"; - -/** - * Try to resolve an API key from process.env first, then from the - * ~/.secrets/hasnaxyz//live.env file as a fallback. - * This handles cases where the CLI is invoked outside a shell session - * that sources the secrets (e.g. cron, agent spawns, MCP calls). - */ -function resolveApiKey(envVar: string, secretsPath: string, secretsKey: string): string | undefined { - // 1. Direct env var - if (process.env[envVar]) return process.env[envVar]; - - // 2. ~/.secrets fallback - const fullPath = join(homedir(), ".secrets", secretsPath); - if (existsSync(fullPath)) { - try { - const content = readFileSync(fullPath, "utf8"); - for (const line of content.split("\n")) { - const trimmed = line.trim(); - if (trimmed.startsWith(secretsKey + "=")) { - const value = trimmed.slice(secretsKey.length + 1).replace(/^["']|["']$/g, ""); - if (value) { - // Auto-inject for this process so downstream calls work too - process.env[envVar] = value; - return value; - } - } - } - } catch { /* ignore read errors */ } - } - - return undefined; -} export function doctorCommand(): Command { return new Command("doctor") @@ -42,24 +7,16 @@ export function doctorCommand(): Command { .action(async (opts: { json?: boolean }) => { const checks: Array<{ name: string; ok: boolean; hint?: string }> = []; - // Check Anthropic API key — env var or ~/.secrets fallback - const anthropicKey = resolveApiKey( - "ANTHROPIC_API_KEY", - "hasnaxyz/anthropic/live.env", - "HASNAXYZ_ANTHROPIC_LIVE_API_KEY" - ); + // Check Anthropic API key. + const anthropicKey = process.env["ANTHROPIC_API_KEY"]; checks.push({ name: "ANTHROPIC_API_KEY", ok: !!anthropicKey, - hint: "export ANTHROPIC_API_KEY= (or add to ~/.secrets/hasnaxyz/anthropic/live.env)", + hint: "export ANTHROPIC_API_KEY=", }); - // Check OpenAI API key (optional) — env var or ~/.secrets fallback - const openaiKey = resolveApiKey( - "OPENAI_API_KEY", - "hasnaxyz/openai/live.env", - "HASNAXYZ_OPENAI_LIVE_API_KEY" - ); + // Check OpenAI API key (optional). + const openaiKey = process.env["OPENAI_API_KEY"]; checks.push({ name: "OPENAI_API_KEY (optional)", ok: !!openaiKey, diff --git a/src/cli/commands/sync.ts b/src/cli/commands/sync.ts index 979fb64..633bed6 100644 --- a/src/cli/commands/sync.ts +++ b/src/cli/commands/sync.ts @@ -1,92 +1,128 @@ import { Command } from "commander"; +import { + STORAGE_TABLES, + getStoragePg, + getStorageStatus, + storagePull, + storagePush, + storageSync, + getSyncMetaAll, + runStorageMigrations, + type SyncResult, +} from "../../db/storage-sync.js"; +import { PG_MIGRATIONS } from "../../db/pg-migrations.js"; + +function parseTables(value?: string): string[] | undefined { + if (!value) return undefined; + return value.split(",").map((table) => table.trim()).filter(Boolean); +} + +function printResults(results: SyncResult[], label: string): void { + const total = results.reduce((sum, result) => sum + result.rowsWritten, 0); + for (const result of results) { + const errors = result.errors.length > 0 ? ` (${result.errors.join("; ")})` : ""; + console.log(` ${result.table}: ${result.rowsWritten}/${result.rowsRead} rows ${label}${errors}`); + } + console.log(`\x1b[32m✓ ${total} rows ${label}\x1b[0m`); +} export function syncCommand(): Command { const cmd = new Command("sync") - .description("Sync eval runs and datasets with cloud"); + .description("Sync eval runs and baselines with remote PostgreSQL storage"); cmd .command("push") - .description("Push local runs and datasets to cloud") + .description("Push local runs and baselines to remote storage") .option("--dry-run", "Show what would be pushed without doing it") - .action(async (opts: { dryRun?: boolean }) => { + .option("--tables ", "Comma-separated table names") + .action(async (opts: { dryRun?: boolean; tables?: string }) => { try { - const { syncPush } = await import("@hasna/cloud"); - void await import("../../db/store.js"); // ensure DB initialized - if (opts.dryRun) { - console.log("Dry run — would push evals database to cloud."); + console.log(`Dry run — would push tables: ${(parseTables(opts.tables) ?? [...STORAGE_TABLES]).join(", ")}`); return; } - console.log("Pushing to cloud..."); - const { SqliteAdapter, PgAdapterAsync, getConnectionString, getDbPath } = await import("@hasna/cloud"); - const connStr = getConnectionString("evals"); - const dbPath = getDbPath("evals"); - const local = new SqliteAdapter(dbPath); - const remote = new PgAdapterAsync(connStr); - const results = await syncPush(local, remote, { tables: ["runs", "baselines"] }); - const total = results.reduce((s, r) => s + r.rowsWritten, 0); - console.log(`\x1b[32m✓ Pushed ${total} rows\x1b[0m`); + const results = await storagePush({ tables: parseTables(opts.tables) }); + printResults(results, "pushed"); } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ERR_MODULE_NOT_FOUND" || - String(err).includes("not found")) { - console.error("Cloud sync requires @hasna/cloud. Run: bun install"); - } else { - console.error(`Sync failed: ${err instanceof Error ? err.message : String(err)}`); - } + console.error(`Sync failed: ${err instanceof Error ? err.message : String(err)}`); process.exit(1); } }); cmd .command("pull") - .description("Pull runs and datasets from cloud") + .description("Pull runs and baselines from remote storage") .option("--dry-run", "Show what would be pulled without doing it") - .action(async (opts: { dryRun?: boolean }) => { + .option("--tables ", "Comma-separated table names") + .action(async (opts: { dryRun?: boolean; tables?: string }) => { try { - const { syncPull } = await import("@hasna/cloud"); - void await import("../../db/store.js"); // ensure DB initialized - if (opts.dryRun) { - console.log("Dry run — would pull evals data from cloud."); + console.log(`Dry run — would pull tables: ${(parseTables(opts.tables) ?? [...STORAGE_TABLES]).join(", ")}`); return; } - console.log("Pulling from cloud..."); - const { SqliteAdapter, PgAdapterAsync, getConnectionString, getDbPath } = await import("@hasna/cloud"); - const connStr = getConnectionString("evals"); - const dbPath = getDbPath("evals"); - const local = new SqliteAdapter(dbPath); - const remote = new PgAdapterAsync(connStr); - const results = await syncPull(remote, local, { tables: ["runs", "baselines"] }); - const total = results.reduce((s, r) => s + r.rowsWritten, 0); - console.log(`\x1b[32m✓ Pulled ${total} rows\x1b[0m`); + const results = await storagePull({ tables: parseTables(opts.tables) }); + printResults(results, "pulled"); } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ERR_MODULE_NOT_FOUND" || - String(err).includes("not found")) { - console.error("Cloud sync requires @hasna/cloud. Run: bun install"); - } else { - console.error(`Sync failed: ${err instanceof Error ? err.message : String(err)}`); - } + console.error(`Sync failed: ${err instanceof Error ? err.message : String(err)}`); process.exit(1); } }); cmd - .command("status") - .description("Show cloud sync status") - .action(async () => { + .command("sync") + .description("Bidirectional sync: pull then push") + .option("--tables ", "Comma-separated table names") + .action(async (opts: { tables?: string }) => { + try { + const result = await storageSync({ tables: parseTables(opts.tables) }); + printResults(result.pull, "pulled"); + printResults(result.push, "pushed"); + } catch (err) { + console.error(`Sync failed: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + + cmd + .command("migrate") + .description("Apply PostgreSQL migrations for evals") + .option("--dry-run", "Print SQL without executing") + .action(async (opts: { dryRun?: boolean }) => { try { - const { getCloudConfig } = await import("@hasna/cloud"); - const config = await getCloudConfig(); - if (!config) { - console.log("Cloud sync not configured. Run: evals sync push"); + if (opts.dryRun) { + for (const sql of PG_MIGRATIONS) console.log(sql); return; } - console.log(`\x1b[32m✓ Cloud sync configured\x1b[0m`); - console.log(` Service: evals`); - } catch { - console.log("Cloud sync not configured."); + const pg = await getStoragePg(); + await runStorageMigrations(pg); + await pg.close(); + console.log("\x1b[32m✓ Migrations applied\x1b[0m"); + } catch (err) { + console.error(`Migration failed: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + + cmd + .command("status") + .description("Show storage sync status") + .option("--json", "Output as JSON") + .action((opts: { json?: boolean }) => { + const status = getStorageStatus(); + if (opts.json) { + console.log(JSON.stringify(status, null, 2)); + return; + } + console.log(`Storage configured: ${status.configured ? "yes" : "no"}`); + console.log(`Mode: ${status.mode}`); + console.log(`Env: ${status.env.join(", ")}`); + console.log(`Tables: ${status.tables.join(", ")}`); + const sync = getSyncMetaAll(); + if (sync.length === 0) console.log("Sync: no local sync history"); + for (const entry of sync) { + console.log(` ${entry.table_name} ${entry.direction}: ${entry.last_synced_at ?? "never"}`); } }); diff --git a/src/cli/index.ts b/src/cli/index.ts index 6b21ffc..a5bee6c 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,5 +1,6 @@ #!/usr/bin/env bun import { Command } from "commander"; +import { registerEventsCommands } from "@hasna/events/commander"; import { runCommand } from "./commands/run.js"; import { ciCommand } from "./commands/ci.js"; import { judgeCommand } from "./commands/judge.js"; @@ -34,5 +35,7 @@ program.addCommand(mcpCommand()); program.addCommand(captureCommand()); program.addCommand(completionCommand()); program.addCommand(syncCommand()); +registerEventsCommands(program, { source: "evals" }); + program.parse(process.argv); diff --git a/src/core/judge.ts b/src/core/judge.ts index 3c786a4..a1c4363 100644 --- a/src/core/judge.ts +++ b/src/core/judge.ts @@ -1,32 +1,7 @@ import Anthropic from "@anthropic-ai/sdk"; import OpenAI from "openai"; -import { existsSync, readFileSync } from "fs"; -import { homedir } from "os"; -import { join } from "path"; import type { JudgeConfig, JudgeResult, Verdict } from "../types/index.js"; -/** - * Resolve an API key from env or ~/.secrets fallback. - * Auto-injects into process.env so the SDK picks it up. - */ -function resolveKey(envVar: string, secretsRelPath: string, secretsKey: string): string | undefined { - if (process.env[envVar]) return process.env[envVar]; - const p = join(homedir(), ".secrets", secretsRelPath); - if (existsSync(p)) { - for (const line of readFileSync(p, "utf8").split("\n")) { - if (line.trim().startsWith(secretsKey + "=")) { - const v = line.trim().slice(secretsKey.length + 1).replace(/^["']|["']$/g, ""); - if (v) { process.env[envVar] = v; return v; } - } - } - } - return undefined; -} - -// Eagerly resolve on module load so all SDK instances pick it up -resolveKey("ANTHROPIC_API_KEY", "hasnaxyz/anthropic/live.env", "HASNAXYZ_ANTHROPIC_LIVE_API_KEY"); -resolveKey("OPENAI_API_KEY", "hasnaxyz/openai/live.env", "HASNAXYZ_OPENAI_LIVE_API_KEY"); - // Judge is ALWAYS temperature=0 — deterministic, consistent verdicts. // Chain-of-thought reasoning is REQUIRED before verdict (never score first). // Only three verdicts: PASS / FAIL / UNKNOWN (no numeric scales). diff --git a/src/db/pg-migrations.ts b/src/db/pg-migrations.ts new file mode 100644 index 0000000..7cf969b --- /dev/null +++ b/src/db/pg-migrations.ts @@ -0,0 +1,21 @@ +/** + * PostgreSQL migrations for open-evals remote storage sync. + */ + +export const PG_MIGRATIONS: string[] = [ + `CREATE TABLE IF NOT EXISTS runs ( + id TEXT PRIMARY KEY, + created_at TEXT NOT NULL, + dataset TEXT NOT NULL, + stats TEXT NOT NULL, + adapter TEXT, + data TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS baselines ( + name TEXT PRIMARY KEY, + run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE, + created_at TEXT NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS runs_created_at ON runs(created_at DESC)`, + `CREATE INDEX IF NOT EXISTS runs_dataset ON runs(dataset)`, +]; diff --git a/src/db/remote-storage.ts b/src/db/remote-storage.ts new file mode 100644 index 0000000..2a37076 --- /dev/null +++ b/src/db/remote-storage.ts @@ -0,0 +1,40 @@ +import pg from "pg"; +import type { Pool } from "pg"; + +function translatePlaceholders(sql: string): string { + let index = 0; + return sql.replace(/\?/g, () => `$${++index}`); +} + +function normalizeParams(params: unknown[]): unknown[] { + const flat = params.length === 1 && Array.isArray(params[0]) ? params[0] : params; + return flat.map((value) => value === undefined ? null : value); +} + +function sslConfigFor(connectionString: string): { rejectUnauthorized: boolean } | undefined { + return connectionString.includes("sslmode=require") || connectionString.includes("ssl=true") + ? { rejectUnauthorized: false } + : undefined; +} + +export class PgAdapterAsync { + private readonly pool: Pool; + + constructor(connectionString: string) { + this.pool = new pg.Pool({ connectionString, ssl: sslConfigFor(connectionString) }); + } + + async run(sql: string, ...params: unknown[]): Promise<{ changes: number }> { + const result = await this.pool.query(translatePlaceholders(sql), normalizeParams(params)); + return { changes: result.rowCount ?? 0 }; + } + + async all(sql: string, ...params: unknown[]): Promise { + const result = await this.pool.query(translatePlaceholders(sql), normalizeParams(params)); + return result.rows; + } + + async close(): Promise { + await this.pool.end(); + } +} diff --git a/src/db/storage-sync.test.ts b/src/db/storage-sync.test.ts new file mode 100644 index 0000000..f0a3685 --- /dev/null +++ b/src/db/storage-sync.test.ts @@ -0,0 +1,67 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { + getStorageDatabaseEnv, + getStorageDatabaseUrl, + getStorageMode, + parseStorageTables, +} from "./storage-sync"; + +const ENV_NAMES = [ + "HASNA_EVALS_DATABASE_URL", + "EVALS_DATABASE_URL", + "HASNA_EVALS_STORAGE_MODE", + "EVALS_STORAGE_MODE", +] as const; + +const ORIGINAL_ENV = new Map( + ENV_NAMES.map((name) => [name, process.env[name]]), +); + +describe("evals storage sync configuration", () => { + beforeEach(() => { + for (const name of ENV_NAMES) delete process.env[name]; + }); + + afterEach(() => { + for (const name of ENV_NAMES) { + const value = ORIGINAL_ENV.get(name); + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + test("prefers canonical storage database env over fallback env", () => { + process.env["EVALS_DATABASE_URL"] = "postgres://fallback"; + process.env["HASNA_EVALS_DATABASE_URL"] = "postgres://canonical"; + + expect(getStorageDatabaseUrl()).toBe("postgres://canonical"); + expect(getStorageDatabaseEnv()).toEqual({ + name: "HASNA_EVALS_DATABASE_URL", + }); + }); + + test("accepts fallback storage database env", () => { + process.env["EVALS_DATABASE_URL"] = "postgres://fallback"; + + expect(getStorageDatabaseUrl()).toBe("postgres://fallback"); + expect(getStorageDatabaseEnv()).toEqual({ + name: "EVALS_DATABASE_URL", + }); + }); + + test("uses storage mode envs", () => { + expect(getStorageMode()).toBe("local"); + + process.env["EVALS_DATABASE_URL"] = "postgres://remote"; + expect(getStorageMode()).toBe("hybrid"); + + process.env["HASNA_EVALS_STORAGE_MODE"] = "remote"; + expect(getStorageMode()).toBe("remote"); + }); + + test("parses and validates storage table filters", () => { + expect(parseStorageTables()).toEqual(["runs", "baselines"]); + expect(parseStorageTables([" runs ", "baselines"])).toEqual(["runs", "baselines"]); + expect(() => parseStorageTables(["missing"])).toThrow("Unknown evals sync table"); + }); +}); diff --git a/src/db/storage-sync.ts b/src/db/storage-sync.ts new file mode 100644 index 0000000..f8c691f --- /dev/null +++ b/src/db/storage-sync.ts @@ -0,0 +1,277 @@ +import type { Database } from "bun:sqlite"; +import { getDatabase } from "./store.js"; +import { PG_MIGRATIONS } from "./pg-migrations.js"; +import { PgAdapterAsync } from "./remote-storage.js"; + +export const STORAGE_TABLES = ["runs", "baselines"] as const; +export const EVALS_STORAGE_TABLES = STORAGE_TABLES; + +type StorageTable = (typeof STORAGE_TABLES)[number]; +type Row = Record; +export type StorageMode = "local" | "hybrid" | "remote"; + +export const EVALS_STORAGE_ENV = "HASNA_EVALS_DATABASE_URL"; +export const EVALS_STORAGE_FALLBACK_ENV = "EVALS_DATABASE_URL"; +export const EVALS_STORAGE_MODE_ENV = "HASNA_EVALS_STORAGE_MODE"; +export const EVALS_STORAGE_MODE_FALLBACK_ENV = "EVALS_STORAGE_MODE"; +export const STORAGE_DATABASE_ENV = [EVALS_STORAGE_ENV, EVALS_STORAGE_FALLBACK_ENV] as const; +export const STORAGE_MODE_ENV = [EVALS_STORAGE_MODE_ENV, EVALS_STORAGE_MODE_FALLBACK_ENV] as const; + +const PRIMARY_KEYS: Record = { + runs: ["id"], + baselines: ["name"], +}; + +export interface SyncResult { + table: string; + rowsRead: number; + rowsWritten: number; + errors: string[]; +} + +export interface SyncMeta { + table_name: string; + last_synced_at: string | null; + direction: "push" | "pull"; +} + +export interface StorageEnv { + name: string; +} + +export interface StorageStatus { + configured: boolean; + mode: StorageMode; + env: typeof STORAGE_DATABASE_ENV; + activeEnv: string | null; + service: "evals"; + tables: readonly StorageTable[]; + sync: SyncMeta[]; +} + +function readEnv(name: string): string | null { + const value = process.env[name]?.trim(); + return value ? value : null; +} + +export function getStorageDatabaseEnv(): StorageEnv | null { + for (const name of STORAGE_DATABASE_ENV) { + if (readEnv(name)) return { name }; + } + return null; +} + +export function getStorageDatabaseUrl(): string | null { + const env = getStorageDatabaseEnv(); + return env ? readEnv(env.name) : null; +} + +function normalizeStorageMode(value: string): StorageMode { + const normalized = value.trim().toLowerCase(); + if (normalized === "local" || normalized === "hybrid" || normalized === "remote") { + return normalized; + } + throw new Error(`Unknown evals storage mode: ${value}`); +} + +export function getStorageMode(): StorageMode { + for (const name of STORAGE_MODE_ENV) { + const value = readEnv(name); + if (value) return normalizeStorageMode(value); + } + return getStorageDatabaseUrl() ? "hybrid" : "local"; +} + +export function getStorageStatus(): StorageStatus { + const activeEnv = getStorageDatabaseEnv(); + return { + configured: Boolean(activeEnv), + mode: getStorageMode(), + env: STORAGE_DATABASE_ENV, + activeEnv: activeEnv?.name ?? null, + service: "evals", + tables: STORAGE_TABLES, + sync: getSyncMetaAll(), + }; +} + +export async function getStoragePg(): Promise { + const url = getStorageDatabaseUrl(); + if (!url) { + throw new Error("Missing HASNA_EVALS_DATABASE_URL or EVALS_DATABASE_URL"); + } + return new PgAdapterAsync(url); +} + +export async function runStorageMigrations(remote: PgAdapterAsync): Promise { + for (const sql of PG_MIGRATIONS) await remote.run(sql); +} + +export async function storagePush(options?: { tables?: string[] }): Promise { + const remote = await getStoragePg(); + const db = getDatabase(); + try { + await runStorageMigrations(remote); + const results: SyncResult[] = []; + for (const table of parseStorageTables(options?.tables)) results.push(await pushTable(db, remote, table)); + recordSyncMeta(db, "push", results); + return results; + } finally { + await remote.close(); + } +} + +export async function storagePull(options?: { tables?: string[] }): Promise { + const remote = await getStoragePg(); + const db = getDatabase(); + try { + await runStorageMigrations(remote); + const results: SyncResult[] = []; + for (const table of parseStorageTables(options?.tables)) results.push(await pullTable(remote, db, table)); + recordSyncMeta(db, "pull", results); + return results; + } finally { + await remote.close(); + } +} + +export async function storageSync(options?: { tables?: string[] }): Promise<{ pull: SyncResult[]; push: SyncResult[] }> { + const pull = await storagePull(options); + const push = await storagePush(options); + return { pull, push }; +} + +export function getSyncMetaAll(): SyncMeta[] { + const db = getDatabase(); + ensureSyncMetaTable(db); + return db.query("SELECT table_name, last_synced_at, direction FROM _evals_sync_meta ORDER BY table_name, direction").all(); +} + +export function parseStorageTables(tables?: string[]): StorageTable[] { + if (!tables || tables.length === 0) return [...STORAGE_TABLES]; + const allowed = new Set(STORAGE_TABLES); + const requested = tables.map((table) => table.trim()).filter(Boolean); + const invalid = requested.filter((table) => !allowed.has(table)); + if (invalid.length > 0) throw new Error(`Unknown evals sync table(s): ${invalid.join(", ")}`); + return requested as StorageTable[]; +} + +export const resolveTables = parseStorageTables; + +async function pushTable(db: Database, remote: PgAdapterAsync, table: StorageTable): Promise { + const result: SyncResult = { table, rowsRead: 0, rowsWritten: 0, errors: [] }; + try { + const rows = db.query(`SELECT * FROM ${quoteIdent(table)}`).all(); + result.rowsRead = rows.length; + if (rows.length === 0) return result; + const remoteColumns = await getRemoteColumns(remote, table); + const columns = filterRemoteColumns(remoteColumns, Object.keys(rows[0]!)); + result.rowsWritten = await upsertPg(remote, table, columns, rows, remoteColumns); + } catch (error) { + result.errors.push(error instanceof Error ? error.message : String(error)); + } + return result; +} + +async function pullTable(remote: PgAdapterAsync, db: Database, table: StorageTable): Promise { + const result: SyncResult = { table, rowsRead: 0, rowsWritten: 0, errors: [] }; + try { + const rows = await remote.all(`SELECT * FROM ${quoteIdent(table)}`) as Row[]; + result.rowsRead = rows.length; + if (rows.length === 0) return result; + const columns = filterLocalColumns(db, table, Object.keys(rows[0]!)); + result.rowsWritten = upsertSqlite(db, table, columns, rows); + } catch (error) { + result.errors.push(error instanceof Error ? error.message : String(error)); + } + return result; +} + +async function getRemoteColumns(remote: PgAdapterAsync, table: string): Promise> { + const rows = await remote.all( + "SELECT column_name, data_type FROM information_schema.columns WHERE table_schema = 'public' AND table_name = ?", + table, + ) as Array<{ column_name: string; data_type: string }>; + return new Map(rows.map((row) => [row.column_name, row.data_type])); +} + +function filterRemoteColumns(remoteColumns: Map, columns: string[]): string[] { + if (remoteColumns.size === 0) return columns; + return columns.filter((column) => remoteColumns.has(column)); +} + +function filterLocalColumns(db: Database, table: string, columns: string[]): string[] { + const rows = db.query<{ name: string }, []>(`PRAGMA table_info(${quoteIdent(table)})`).all(); + const allowed = new Set(rows.map((row) => row.name)); + return columns.filter((column) => allowed.has(column)); +} + +async function upsertPg(remote: PgAdapterAsync, table: StorageTable, columns: string[], rows: Row[], remoteColumns: Map): Promise { + if (columns.length === 0) return 0; + const primaryKeys = PRIMARY_KEYS[table]; + const columnList = columns.map(quoteIdent).join(", "); + const placeholders = columns.map(() => "?").join(", "); + const keyList = primaryKeys.map(quoteIdent).join(", "); + const updateColumns = columns.filter((column) => !primaryKeys.includes(column)); + const fallbackKey = primaryKeys[0]!; + const setClause = updateColumns.length > 0 + ? updateColumns.map((column) => `${quoteIdent(column)} = EXCLUDED.${quoteIdent(column)}`).join(", ") + : `${quoteIdent(fallbackKey)} = EXCLUDED.${quoteIdent(fallbackKey)}`; + for (const row of rows) { + await remote.run( + `INSERT INTO ${quoteIdent(table)} (${columnList}) VALUES (${placeholders}) ON CONFLICT (${keyList}) DO UPDATE SET ${setClause}`, + ...columns.map((column) => coerceValue(row[column], remoteColumns.get(column))), + ); + } + return rows.length; +} + +function upsertSqlite(db: Database, table: StorageTable, columns: string[], rows: Row[]): number { + if (columns.length === 0) return 0; + const primaryKeys = PRIMARY_KEYS[table]; + const columnList = columns.map(quoteIdent).join(", "); + const placeholders = columns.map(() => "?").join(", "); + const keyList = primaryKeys.map(quoteIdent).join(", "); + const updateColumns = columns.filter((column) => !primaryKeys.includes(column)); + const fallbackKey = primaryKeys[0]!; + const setClause = updateColumns.length > 0 + ? updateColumns.map((column) => `${quoteIdent(column)} = excluded.${quoteIdent(column)}`).join(", ") + : `${quoteIdent(fallbackKey)} = excluded.${quoteIdent(fallbackKey)}`; + const statement = db.prepare(`INSERT INTO ${quoteIdent(table)} (${columnList}) VALUES (${placeholders}) ON CONFLICT (${keyList}) DO UPDATE SET ${setClause}`); + for (const row of rows) statement.run(...columns.map((column) => coerceValue(row[column]))); + return rows.length; +} + +function recordSyncMeta(db: Database, direction: "push" | "pull", results: SyncResult[]): void { + ensureSyncMetaTable(db); + const timestamp = new Date().toISOString(); + const statement = db.prepare( + "INSERT INTO _evals_sync_meta (table_name, last_synced_at, direction) VALUES (?, ?, ?) ON CONFLICT(table_name, direction) DO UPDATE SET last_synced_at = excluded.last_synced_at", + ); + for (const result of results) { + if (result.errors.length > 0) continue; + statement.run(result.table, timestamp, direction); + } +} + +function ensureSyncMetaTable(db: Database): void { + db.exec("CREATE TABLE IF NOT EXISTS _evals_sync_meta (table_name TEXT NOT NULL, last_synced_at TEXT, direction TEXT NOT NULL CHECK(direction IN ('push', 'pull')), PRIMARY KEY (table_name, direction))"); +} + +function quoteIdent(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"`; +} + +function coerceValue(value: unknown, dataType?: string): string | number | bigint | boolean | null | Uint8Array { + if (value === undefined || value === null) return null; + if (dataType === "boolean") { + if (typeof value === "boolean") return value; + if (typeof value === "number") return value !== 0; + if (typeof value === "string") return value === "1" || value.toLowerCase() === "true"; + } + if (typeof value === "string" || typeof value === "number" || typeof value === "bigint" || typeof value === "boolean") return value; + if (value instanceof Date) return value.toISOString(); + if (Buffer.isBuffer(value) || value instanceof Uint8Array) return value; + if (typeof value === "object") return JSON.stringify(value); + return String(value); +} diff --git a/src/index.ts b/src/index.ts index 830f97c..dc2f85c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,3 +6,6 @@ export * from "./core/judge.js"; export * from "./core/reporter.js"; export * from "./datasets/loader.js"; export * from "./db/store.js"; +export * from "./db/pg-migrations.js"; +export * from "./db/remote-storage.js"; +export * from "./db/storage-sync.js"; diff --git a/src/mcp/index.ts b/src/mcp/index.ts index f33d6b3..3a8042b 100644 --- a/src/mcp/index.ts +++ b/src/mcp/index.ts @@ -8,6 +8,7 @@ import { judgeOnce } from "../core/judge.js"; import { loadDataset } from "../datasets/loader.js"; import { toJson, toMarkdown, compareRuns } from "../core/reporter.js"; import { saveRun, getRun, listRuns } from "../db/store.js"; +import { getStorageStatus, storagePull, storagePush, storageSync } from "../db/storage-sync.js"; import { writeFileSync, appendFileSync } from "fs"; import type { EvalCase, AdapterConfig } from "../types/index.js"; @@ -146,6 +147,35 @@ const tools = [ required: ["description"], }, }, + { + name: "evals_storage_status", + description: "Show evals remote storage sync configuration and local sync history", + inputSchema: { type: "object", properties: {} }, + }, + { + name: "evals_storage_push", + description: "Push local eval runs and baselines to remote PostgreSQL storage", + inputSchema: { + type: "object", + properties: { tables: { type: "array", items: { type: "string" } } }, + }, + }, + { + name: "evals_storage_pull", + description: "Pull eval runs and baselines from remote PostgreSQL storage", + inputSchema: { + type: "object", + properties: { tables: { type: "array", items: { type: "string" } } }, + }, + }, + { + name: "evals_storage_sync", + description: "Bidirectional evals storage sync: pull then push", + inputSchema: { + type: "object", + properties: { tables: { type: "array", items: { type: "string" } } }, + }, + }, ]; // ─── Tool handlers ──────────────────────────────────────────────────────────── @@ -291,6 +321,23 @@ server.setRequestHandler(CallToolRequestSchema, async (req) => { return { content: [{ type: "text", text: `Generated ${lines.length} cases → ${output}` }] }; } + case "evals_storage_status": + return { + content: [{ + type: "text", + text: JSON.stringify(getStorageStatus(), null, 2), + }], + }; + + case "evals_storage_push": + return { content: [{ type: "text", text: JSON.stringify(await storagePush({ tables: a["tables"] as string[] | undefined }), null, 2) }] }; + + case "evals_storage_pull": + return { content: [{ type: "text", text: JSON.stringify(await storagePull({ tables: a["tables"] as string[] | undefined }), null, 2) }] }; + + case "evals_storage_sync": + return { content: [{ type: "text", text: JSON.stringify(await storageSync({ tables: a["tables"] as string[] | undefined }), null, 2) }] }; + default: return { content: [{ type: "text", text: `Unknown tool: ${name}` }] }; } diff --git a/src/mcp/mcp.test.ts b/src/mcp/mcp.test.ts index c492773..ee60fcc 100644 --- a/src/mcp/mcp.test.ts +++ b/src/mcp/mcp.test.ts @@ -1,4 +1,6 @@ import { describe, test, expect, mock, beforeEach, afterEach } from "bun:test"; +import { readFileSync } from "fs"; +import { join } from "path"; import type { EvalRun } from "../types/index.js"; // Only mock external API — not internal modules @@ -160,3 +162,20 @@ describe("MCP Zod validation patterns", () => { expect(verdicts.length).toBe(3); }); }); + +describe("MCP storage tool surface", () => { + test("uses storage tool names instead of legacy cloud names", () => { + const source = readFileSync(join(import.meta.dir, "index.ts"), "utf8"); + + expect(source).toContain("evals_storage_status"); + expect(source).toContain("evals_storage_push"); + expect(source).toContain("evals_storage_pull"); + expect(source).toContain("evals_storage_sync"); + expect(source).toContain("remote PostgreSQL storage"); + + expect(source).not.toContain("evals_cloud_status"); + expect(source).not.toContain("evals_cloud_push"); + expect(source).not.toContain("evals_cloud_pull"); + expect(source).not.toContain("evals_cloud_sync"); + }); +}); diff --git a/src/storage.ts b/src/storage.ts new file mode 100644 index 0000000..3efcf5d --- /dev/null +++ b/src/storage.ts @@ -0,0 +1,25 @@ +export { + EVALS_STORAGE_ENV, + EVALS_STORAGE_FALLBACK_ENV, + EVALS_STORAGE_MODE_ENV, + EVALS_STORAGE_MODE_FALLBACK_ENV, + STORAGE_DATABASE_ENV, + STORAGE_MODE_ENV, + EVALS_STORAGE_TABLES, + STORAGE_TABLES, + getStorageDatabaseEnv, + getStorageDatabaseUrl, + getStorageMode, + getStoragePg, + getStorageStatus, + getSyncMetaAll, + parseStorageTables, + resolveTables, + runStorageMigrations, + storagePull, + storagePush, + storageSync, +} from "./db/storage-sync.js"; +export type { StorageEnv, StorageMode, StorageStatus, SyncMeta, SyncResult } from "./db/storage-sync.js"; +export { PgAdapterAsync } from "./db/remote-storage.js"; +export { PG_MIGRATIONS } from "./db/pg-migrations.js";