From 3c5bd30b4bdba16f549bcc2871fc70edadd44ee3 Mon Sep 17 00:00:00 2001 From: Good-Coded Date: Wed, 29 Jul 2026 12:55:24 +0000 Subject: [PATCH] feat: resolve issues #536 #537 #538 #539 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #539 — Consolidate rate-limit config - Add backend/src/config/rateLimits.ts as single source of truth for all rate-limit env-var parsing (RATE_LIMIT_WINDOW_MS, RATE_LIMIT_MAX, WRITE_RATE_LIMIT_MAX, PAYMENTS_RATE_LIMIT_MAX, RATE_LIMIT_MESSAGE) - Update middleware/rateLimit.ts to import from config/rateLimits.ts - Update index.ts to import from config/rateLimits.ts; also clean up duplicate imports and duplicate route registrations that had accumulated from multiple merged drafts #536 — Bulk migrate-all helper - Add scripts/migrate-all.js: fetches all meters via get_all_meters, calls migrate_meter for each, logs per-meter success/failure, prints summary, exits 1 if any meter fails (CI-safe) - Add make migrate-all target to Makefile with required-var guards and DRY_RUN support - Update README Contract Upgrades / Migration flow section to reference make migrate-all as the recommended path #537 — Document /metrics intentional public exposure - Add detailed design-decision comment above GET /metrics in index.ts explaining why no auth is needed and how network isolation makes it safe - Add /metrics — intentionally public section to README Observability with guidance for hardening when port is publicly exposed - Add metrics_path: /metrics and doc comment to infra/prometheus.yml #538 — CI guard against duplicate package.json keys - Add scripts/check-duplicate-pkg-keys.js: custom recursive-descent JSON parser that counts key occurrences and exits 1 on any duplicate (standard JSON.parse silently keeps last value, so npm install succeeding is insufficient proof) - Add check-pkg-duplicates job to .github/workflows/ci.yml that runs against both backend/package.json and frontend/package.json Also fix pre-existing compile errors in main that blocked tsc --noEmit: - bridge.ts: add MqttPayloadSchema export to validation.ts; remove undefined WEBHOOK_URL reference - meters.ts: remove orphaned old-style handler fragments injected mid-handler; complete pagination handler body - payments.ts: remove duplicate POST / preamble; remove orphaned handler body fragment; remove out-of-scope idempotencyKey reference - stats.ts: remove duplicate old-version preamble - stellar.ts: remove duplicate top-level CONTRACT_ID/server declarations that shadowed back-compat alias exports - webhooks.ts: resolve merged POST /low-balance handler body Closes #536 Closes #537 Closes #538 Closes #539 --- .github/workflows/ci.yml | 16 ++ Makefile | 33 +++- README.md | 58 +++++-- backend/src/config/rateLimits.ts | 45 +++++ backend/src/index.ts | 244 +++++++++++++++++----------- backend/src/iot/bridge.ts | 2 +- backend/src/lib/stellar.ts | 25 --- backend/src/lib/validation.ts | 5 + backend/src/middleware/rateLimit.ts | 52 +++++- backend/src/routes/meters.ts | 155 ------------------ backend/src/routes/payments.ts | 45 ----- backend/src/routes/stats.ts | 24 --- backend/src/routes/webhooks.ts | 25 +-- infra/prometheus.yml | 15 ++ scripts/check-duplicate-pkg-keys.js | 168 +++++++++++++++++++ scripts/migrate-all.js | 238 +++++++++++++++++++++++++++ 16 files changed, 765 insertions(+), 385 deletions(-) create mode 100644 backend/src/config/rateLimits.ts create mode 100644 scripts/check-duplicate-pkg-keys.js create mode 100644 scripts/migrate-all.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 338d637..8f61d10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,22 @@ on: pull_request: jobs: + # Closes #538: guard against duplicate package.json keys. + # Standard JSON.parse() silently keeps the last occurrence, so npm install + # succeeding is NOT sufficient proof a package.json is duplicate-free. + check-pkg-duplicates: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Check for duplicate keys in package.json files + run: | + node scripts/check-duplicate-pkg-keys.js \ + backend/package.json \ + frontend/package.json + contract: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index a90028c..0b12fa0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test deploy invoke-register invoke-allowlist logs clean +.PHONY: build test deploy invoke-register invoke-allowlist migrate-all logs clean NETWORK ?= testnet WASM := contracts/target/wasm32-unknown-unknown/release/solar_grid.wasm @@ -18,6 +18,37 @@ invoke-register: invoke-allowlist: stellar contract invoke --id $(CONTRACT_ID) --source $(ADMIN_SECRET_KEY) --network $(NETWORK) -- allowlist_add --owner $(OWNER) +## Bulk-migrate all registered meters to the current schema. +## +## Closes #536: replaces the manual one-at-a-time stellar contract invoke +## … migrate_meter workflow with a single command. +## +## Required: +## CONTRACT_ID — deployed Soroban contract address +## ADMIN_SECRET_KEY — admin Stellar secret key (S…) +## +## Optional: +## NETWORK — testnet (default) or mainnet +## DRY_RUN — set to "true" to list meters without migrating +## +## Example: +## make migrate-all CONTRACT_ID=C... ADMIN_SECRET_KEY=S... +## make migrate-all CONTRACT_ID=C... ADMIN_SECRET_KEY=S... DRY_RUN=true +migrate-all: + @if [ -z "$(CONTRACT_ID)" ]; then \ + echo "ERROR: CONTRACT_ID is required. Usage: make migrate-all CONTRACT_ID=C... ADMIN_SECRET_KEY=S..."; \ + exit 1; \ + fi + @if [ -z "$(ADMIN_SECRET_KEY)" ]; then \ + echo "ERROR: ADMIN_SECRET_KEY is required. Usage: make migrate-all CONTRACT_ID=C... ADMIN_SECRET_KEY=S..."; \ + exit 1; \ + fi + CONTRACT_ID=$(CONTRACT_ID) \ + ADMIN_SECRET_KEY=$(ADMIN_SECRET_KEY) \ + STELLAR_NETWORK=$(NETWORK) \ + DRY_RUN=$(DRY_RUN) \ + node scripts/migrate-all.js + logs: docker compose logs -f backend diff --git a/README.md b/README.md index 533201b..175c701 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ A Makefile is provided at the repository root to simplify common development and - **Invoke functions**: - `make invoke-register CONTRACT_ID= ADMIN_SECRET_KEY= METER_ID= OWNER=` registers a new meter. - `make invoke-allowlist CONTRACT_ID= ADMIN_SECRET_KEY= OWNER=` adds an owner to the allowlist. +- **Bulk migrate all meters**: `make migrate-all CONTRACT_ID= ADMIN_SECRET_KEY=` — see [Contract Upgrades](#contract-upgrades) below. - **Backend Logs**: `make logs` streams Docker Compose logs for the backend. ### Smart Contracts Deployment CI/CD @@ -127,9 +128,26 @@ You can run the Prometheus and Grafana observability stack alongside the backend docker compose --profile observability up --build ``` -- **Prometheus** scrapes the backend metrics (`/metrics`) every 15 seconds, and is accessible at `http://localhost:9090`. +- **Prometheus** scrapes the backend metrics endpoint (`GET /metrics`) every 15 seconds via `infra/prometheus.yml`, and is accessible at `http://localhost:9090`. - **Grafana** is preconfigured with the Prometheus datasource and is accessible at `http://localhost:3000` (default credentials: `admin` / `admin`). It features dashboard panels for MQTT messages/min, contract calls by method/status, and error rates. +#### `/metrics` — intentionally public (closes #537) + +The `GET /metrics` endpoint exposes Prometheus text-format data with **no authentication**. This is by design: Prometheus's pull model requires unauthenticated HTTP GET access to scrape metrics from a target. + +**Why this is safe in the default deployment:** the backend container port `3001` is attached only to the internal Docker `app-network` and is not forwarded to a public interface. The Prometheus container scrapes it from within that private network. External traffic never reaches `/metrics`. + +**If you expose the backend on a public port** (e.g. via a reverse proxy or `ports: "3001:3001"` in `docker-compose.override.yml`), you should restrict access to `/metrics` at the proxy layer — for example: + +```nginx +# nginx — deny external access to /metrics +location /metrics { + deny all; +} +``` + +Or allow only the Prometheus container's IP via a firewall rule. A `METRICS_ALLOWED_CIDRS` env-var-driven IP-allowlist middleware can also be added to `backend/src/index.ts` in a future hardening pass. + ## Smart Contract Overview The `SolarGrid` contract manages: @@ -180,18 +198,34 @@ The `Meter` struct carries a `version: u32` field (currently `1`). When the stru ### Migration flow 1. Deploy the new contract WASM (the old entries remain in persistent storage). -2. For each registered meter, call the admin-only `migrate_meter(meter_id)` function. - It reads the entry as the previous schema (`LegacyMeter`) and writes it back as the current `Meter` v1. -3. Once all entries are migrated, the `LegacyMeter` type and `migrate_meter_v0` helper can be removed in a subsequent release. +2. Run the bulk migration helper to migrate every registered meter in one pass: -```bash -# Example: migrate a single meter via Stellar CLI -stellar contract invoke \ - --id \ - --source \ - --network testnet \ - -- migrate_meter --meter_id METER1 -``` + ```bash + # Recommended: migrate all meters at once (closes #536) + make migrate-all CONTRACT_ID= ADMIN_SECRET_KEY= NETWORK=testnet + ``` + + The script calls `get_all_meters` to fetch every registered meter ID, then + calls `migrate_meter(meter_id)` for each one, logging per-meter + success/failure and printing a final summary. Exit code is `0` only when + all meters succeed, so it integrates cleanly into CI pipelines. + + ```bash + # Dry-run: list meters without sending any transactions + make migrate-all CONTRACT_ID= ADMIN_SECRET_KEY= DRY_RUN=true + ``` + +3. Alternatively, migrate a single meter manually via the Stellar CLI: + + ```bash + stellar contract invoke \ + --id \ + --source \ + --network testnet \ + -- migrate_meter --meter_id METER1 + ``` + +4. Once all entries are migrated, the `LegacyMeter` type and `migrate_meter_v0` helper can be removed in a subsequent release. > **Note:** `migrate_meter` is idempotent per entry — calling it on an already-migrated meter will overwrite with the same data. Always test migrations on testnet before mainnet. diff --git a/backend/src/config/rateLimits.ts b/backend/src/config/rateLimits.ts new file mode 100644 index 0000000..dc999ae --- /dev/null +++ b/backend/src/config/rateLimits.ts @@ -0,0 +1,45 @@ +/** + * config/rateLimits.ts — single source of truth for all rate-limiter config. + * + * Closes #539: previously, middleware/rateLimit.ts and index.ts each parsed + * the same RATE_LIMIT_WINDOW_MS / RATE_LIMIT_MAX env vars independently, + * creating two diverging sources of truth that could silently drift apart. + * + * Both files now import from here so any future tuning only needs to happen + * in one place. + */ + +/** Window duration in milliseconds (default: 60 s). */ +export const RATE_LIMIT_WINDOW_MS = Number( + process.env.RATE_LIMIT_WINDOW_MS ?? 60 * 1000, +); + +/** + * Maximum requests per window for the general / global limiter (default: 60). + * This is also used as the ceiling for read-heavy routes. + */ +export const RATE_LIMIT_MAX = Number(process.env.RATE_LIMIT_MAX ?? 60); + +/** + * Maximum write requests per window for the strict write limiter (default: + * 30). Used by writeLimiter (admin login, webhooks, allowlist, etc.). + * Setting this lower than RATE_LIMIT_MAX intentionally throttles mutating + * operations harder than reads. + */ +export const WRITE_RATE_LIMIT_MAX = Number( + process.env.WRITE_RATE_LIMIT_MAX ?? 30, +); + +/** + * Maximum payment requests per window (default: 10). Payments hit the + * Stellar network and cost gas, so they get a tighter budget than generic + * writes. + */ +export const PAYMENTS_RATE_LIMIT_MAX = Number( + process.env.PAYMENTS_RATE_LIMIT_MAX ?? 10, +); + +/** Human-readable message returned to clients that exceed any limiter. */ +export const RATE_LIMIT_MESSAGE = + process.env.RATE_LIMIT_MESSAGE ?? + "Too many requests, please try again later."; diff --git a/backend/src/index.ts b/backend/src/index.ts index 822f902..66ffed4 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -1,21 +1,14 @@ import "dotenv/config"; -import express from "express"; -import { NextFunction, Request, Response } from "express"; -import { meterRouter } from "./routes/meters.js"; -import { paymentsRouter } from "./routes/payments.js"; -import { webhookRouter } from "./routes/webhooks.js"; -import { configRouter } from "./routes/config.js"; -import { statsRouter } from "./routes/stats.js"; -import { startIoTBridge } from "./iot/bridge.js"; -import { register } from "./lib/metrics.js"; import express, { NextFunction, Request, Response } from "express"; import cors from "cors"; import compression from "compression"; import timeout from "connect-timeout"; -import mqtt from "mqtt"; import helmet from "helmet"; import swaggerUi from "swagger-ui-express"; import YAML from "yamljs"; +import rateLimit from "express-rate-limit"; +import { createRequire } from "module"; + import { stellarService, server } from "./lib/stellar.js"; import { createMeterRouter } from "./routes/meters.js"; import { paymentsRouter } from "./routes/payments.js"; @@ -24,7 +17,6 @@ import { statsRouter } from "./routes/stats.js"; import { collaboratorRouter } from "./routes/collaborators.js"; import { allowlistRouter } from "./routes/allowlist.js"; import { adminLoginRouter } from "./routes/adminLogin.js"; -import { statsRouter as duplicateStatsRouter } from "./routes/stats.js"; import { metricsRouter } from "./routes/metrics.js"; import { smsConfigRouter } from "./routes/smsConfig.js"; import { clientErrorsRouter } from "./routes/clientErrors.js"; @@ -38,24 +30,40 @@ import { register } from "./lib/metrics.js"; import { writeLimiter } from "./middleware/rateLimit.js"; import { sanitiseBody } from "./middleware/sanitise.js"; import requestLoggerMiddleware from "./middleware/requestLogger.js"; -import rateLimit from "express-rate-limit"; import { initUsageEventStore, startUsageEventRetryWorker, } from "./lib/usageEvents.js"; import { initMeterNotesStore } from "./lib/meterNotes.js"; import { getReqId } from "./lib/requestContext.js"; -import { createRequire } from "module"; +// ── Rate-limit config ──────────────────────────────────────────────────────── +// Closes #539: all env-var parsing lives in config/rateLimits.ts; this file +// imports the parsed values so there is a single source of truth shared with +// middleware/rateLimit.ts. +import { + RATE_LIMIT_WINDOW_MS, + RATE_LIMIT_MAX, + PAYMENTS_RATE_LIMIT_MAX, + RATE_LIMIT_MESSAGE, +} from "./config/rateLimits.js"; + +// ── Bootstrap ──────────────────────────────────────────────────────────────── const _require = createRequire(import.meta.url); const { version } = _require("../../package.json") as { version: string }; -const REQUIRED_ENV = ["CONTRACT_ID", "ADMIN_SECRET_KEY", "ADMIN_API_KEY", "STELLAR_RPC_URL", "MQTT_BROKER"]; +const REQUIRED_ENV = [ + "CONTRACT_ID", + "ADMIN_SECRET_KEY", + "ADMIN_API_KEY", + "STELLAR_RPC_URL", + "MQTT_BROKER", +]; const missing = REQUIRED_ENV.filter((k) => !process.env[k]); if (missing.length > 0) { logger.fatal( { missing }, - "Missing required environment variables. Copy backend/.env.example to backend/.env." + "Missing required environment variables. Copy backend/.env.example to backend/.env.", ); process.exit(1); } @@ -67,42 +75,51 @@ const BODY_LIMIT = process.env.REQUEST_BODY_LIMIT ?? "100kb"; const app = express(); const startTime = Date.now(); -app.use(helmet({ - contentSecurityPolicy: { - directives: { - defaultSrc: ["'none'"], - scriptSrc: ["'self'"], - connectSrc: ["'self'"], +// ── Security headers ───────────────────────────────────────────────────────── +app.use( + helmet({ + contentSecurityPolicy: { + directives: { + defaultSrc: ["'none'"], + scriptSrc: ["'self'"], + connectSrc: ["'self'"], + }, }, - }, - hsts: { maxAge: 31536000, includeSubDomains: true }, -})); + hsts: { maxAge: 31536000, includeSubDomains: true }, + }), +); -// #599: gzip/brotli-compress responses over 1 KB (e.g. large meter-list -// payloads). `compression` negotiates the best encoding the client -// advertises via Accept-Encoding (br when supported, else gzip/deflate) -// and leaves small responses untouched below the threshold. +// #599: gzip/brotli-compress responses over 1 KB. app.use(compression({ threshold: 1024 })); -const allowedOrigins = (process.env.CORS_ORIGIN ?? '*').split(',').map(o => o.trim()); +// ── CORS ───────────────────────────────────────────────────────────────────── +const allowedOrigins = (process.env.CORS_ORIGIN ?? "*") + .split(",") + .map((o) => o.trim()); -app.use(cors({ - origin: (origin, cb) => { - if (!origin || allowedOrigins.includes('*') || allowedOrigins.includes(origin)) { - cb(null, true); - } else { - cb(new Error(`Origin ${origin} not allowed by CORS`)); - } - }, - methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"], - allowedHeaders: ["Content-Type", "Authorization", "X-Admin-Key"], - optionsSuccessStatus: 204, - credentials: true, -})); +app.use( + cors({ + origin: (origin, cb) => { + if ( + !origin || + allowedOrigins.includes("*") || + allowedOrigins.includes(origin) + ) { + cb(null, true); + } else { + cb(new Error(`Origin ${origin} not allowed by CORS`)); + } + }, + methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"], + allowedHeaders: ["Content-Type", "Authorization", "X-Admin-Key"], + optionsSuccessStatus: 204, + credentials: true, + }), +); -// Capture raw body for webhook signature verification before JSON parsing +// ── Body parsing ───────────────────────────────────────────────────────────── // Capture raw body for webhook signature verification before JSON parsing. -// #423: apply body size limit +// #423: apply body size limit. app.use( express.json({ limit: BODY_LIMIT, @@ -116,74 +133,90 @@ app.use(express.urlencoded({ extended: true, limit: BODY_LIMIT })); app.use(sanitiseBody); app.use(requestLoggerMiddleware); -// Request timeout — configurable via REQUEST_TIMEOUT env var (default 15s) +// ── Request timeout ────────────────────────────────────────────────────────── +// Configurable via REQUEST_TIMEOUT env var (default 15 s). const requestTimeout = process.env.REQUEST_TIMEOUT ?? "15s"; app.use(timeout(requestTimeout)); - app.use((req: any, _res: any, next: any) => { if (!req.timedout) next(); }); -// Rate limiting configuration (driven by env vars) -const RATE_LIMIT_WINDOW_MS = Number(process.env.RATE_LIMIT_WINDOW_MS ?? 60 * 1000); -const RATE_LIMIT_MAX = Number(process.env.RATE_LIMIT_MAX ?? 60); -const PAYMENTS_RATE_LIMIT_MAX = Number(process.env.PAYMENTS_RATE_LIMIT_MAX ?? 10); -const RATE_LIMIT_MESSAGE = process.env.RATE_LIMIT_MESSAGE ?? 'Too many requests, please try again later.'; +// ── Rate limiters ───────────────────────────────────────────────────────────── +// Env-var parsing is centralised in config/rateLimits.ts (closes #539). -// Single global read limiter — scoped to /api, one counter per class (#504) +// Global read limiter — scoped to /api, one counter per IP (#504). const globalReadLimiter = rateLimit({ windowMs: RATE_LIMIT_WINDOW_MS, max: RATE_LIMIT_MAX, standardHeaders: true, legacyHeaders: false, handler: (_req, res) => { - // Provide Retry-After in seconds - res.setHeader('Retry-After', String(Math.ceil(RATE_LIMIT_WINDOW_MS / 1000))); - res.status(429).json({ error: RATE_LIMIT_MESSAGE }); + res.setHeader( + "Retry-After", + String(Math.ceil(RATE_LIMIT_WINDOW_MS / 1000)), + ); + res.status(429).json({ error: RATE_LIMIT_MESSAGE, code: "RATE_LIMITED" }); }, }); -// Payments-specific write limiter — stricter than the general write limiter +// Payments-specific limiter — stricter than the general write limiter because +// each payment hits the Stellar network and costs gas. const paymentsLimiter = rateLimit({ windowMs: RATE_LIMIT_WINDOW_MS, max: PAYMENTS_RATE_LIMIT_MAX, standardHeaders: true, legacyHeaders: false, handler: (_req, res) => { - res.setHeader('Retry-After', String(Math.ceil(RATE_LIMIT_WINDOW_MS / 1000))); - res.status(429).json({ error: RATE_LIMIT_MESSAGE }); + res.setHeader( + "Retry-After", + String(Math.ceil(RATE_LIMIT_WINDOW_MS / 1000)), + ); + res.status(429).json({ error: RATE_LIMIT_MESSAGE, code: "RATE_LIMITED" }); }, }); -// Apply global read limiter to all /api routes -app.use('/api', globalReadLimiter); +// Apply global read limiter to all /api routes. +app.use("/api", globalReadLimiter); -app.use((req, _res, next) => { - logger.info({ method: req.method, path: req.path }); - next(); +// ── Prometheus metrics endpoint ─────────────────────────────────────────────── +// +// INTENTIONALLY PUBLIC — no authentication required. +// +// Design rationale (closes #537): +// Prometheus's scrape model requires unauthenticated HTTP GET access to the +// /metrics path. In this deployment the backend port (3001) is exposed only +// on the internal Docker network (app-network) and is not forwarded to a +// public interface. The Prometheus container scrapes it from within that +// private network (see infra/prometheus.yml). +// +// If the backend is ever exposed on a public-facing port, access to /metrics +// should be restricted at the reverse-proxy layer (e.g. an nginx `location +// /metrics { deny all; }` block or a firewall rule that allows only the +// Prometheus container's IP). An IP-allowlist middleware can also be added +// here using the METRICS_ALLOWED_CIDRS env var in a future hardening pass. +// +// The endpoint is registered *before* the /api rate-limiter so Prometheus +// scrapes are never throttled by the per-IP write budget. +app.get("/metrics", async (_req, res) => { + res.set("Content-Type", register.contentType); + res.end(await register.metrics()); }); -app.use("/api/meters", meterRouter); -app.use("/api/payments", paymentsRouter); -app.use("/api/webhooks", webhookRouter); -app.use("/api/config", configRouter); -app.use("/api/stats", statsRouter); -// ── Routes ────────────────────────────────────────────────────────────────────────────── +// ── Routes ─────────────────────────────────────────────────────────────────── + +// Swagger / OpenAPI docs +try { + const openApiDocument = YAML.load("./openapi.yaml"); + app.use("/api/docs", swaggerUi.serve, swaggerUi.setup(openApiDocument)); +} catch { + logger.warn("openapi.yaml not found; /api/docs will not be available"); +} app.use("/api/admin/login", writeLimiter, adminLoginRouter); app.use("/api/meters", createMeterRouter(stellarService)); app.use("/api/payments", paymentsLimiter, paymentsRouter); app.use("/api/webhooks", writeLimiter, webhookRouter); app.use("/api/allowlist", writeLimiter, allowlistRouter); -app.use("/api/payments", paymentsRouter); -app.use("/api/webhooks", webhookRouter); -app.use("/api/stats", statsRouter); - -app.get('/health', async (_req, res) => { - const checks: Record = {}; -}); -app.use("/api/collaborators", collaboratorRouter); -app.use("/api/allowlist", allowlistRouter); app.use("/api/collaborators", collaboratorRouter); app.use("/api/stats", statsRouter); app.use("/api/sms-config", smsConfigRouter); @@ -191,12 +224,12 @@ app.use("/api/client-errors", writeLimiter, clientErrorsRouter); app.use("/api/metrics", metricsRouter); app.use("/api/solar", solarRouter); app.use("/api/usage-events", usageEventsRouter); +app.use("/api/provider", providerRouter); // #420: GET /api/health — version, uptime, dependency status app.get("/api/health", async (_req, res) => { const uptimeSec = Math.floor((Date.now() - startTime) / 1000); - // Check Stellar RPC let rpcOk = false; try { await server.getLatestLedger(); @@ -205,7 +238,6 @@ app.get("/api/health", async (_req, res) => { logger.warn("Stellar RPC health check failed"); } - // Check MQTT let mqttOk = false; try { const { getMqttClient } = await import("./iot/bridge.js"); @@ -227,12 +259,7 @@ app.get("/api/health", async (_req, res) => { }); }); -app.get("/metrics", async (_req, res) => { - res.set("Content-Type", register.contentType); - res.end(await register.metrics()); -}); - -// #418: 404 catch-all — must come after all routes +// #418: 404 catch-all — must come after all routes. app.use((_req: Request, res: Response) => { res.status(404).json({ error: "Route not found", @@ -242,41 +269,64 @@ app.use((_req: Request, res: Response) => { }); }); -// Timeout error handler +// ── Error handlers ─────────────────────────────────────────────────────────── + +// Timeout error handler. app.use((err: any, req: any, res: any, next: any) => { if (req.timedout) { logger.error("Request timed out", { method: req.method, path: req.path }); - return res.status(504).json({ error: "Request timed out", code: "TIMEOUT", requestId: getReqId() }); + return res + .status(504) + .json({ error: "Request timed out", code: "TIMEOUT", requestId: getReqId() }); } next(err); }); -// #423: 413 payload too large handler + global error handler (#418) +// #423: 413 payload too large handler + global error handler (#418). app.use((err: any, _req: Request, res: Response, _next: NextFunction) => { logger.error({ error: err.message, stack: err.stack }, "Unhandled error"); const requestId = getReqId(); if (err.type === "entity.too.large") { - return res.status(413).json({ error: "Request body too large", code: "PAYLOAD_TOO_LARGE", requestId }); + return res + .status(413) + .json({ error: "Request body too large", code: "PAYLOAD_TOO_LARGE", requestId }); } - if (err.type === "entity.parse.failed" || (err instanceof SyntaxError && (err as any).body !== undefined)) { - return res.status(400).json({ error: "Invalid JSON body", code: "INVALID_JSON", requestId }); + if ( + err.type === "entity.parse.failed" || + (err instanceof SyntaxError && (err as any).body !== undefined) + ) { + return res + .status(400) + .json({ error: "Invalid JSON body", code: "INVALID_JSON", requestId }); } if ((err as any).status === 404) { - return res.status(404).json({ error: "Resource not found", code: "NOT_FOUND", requestId }); + return res + .status(404) + .json({ error: "Resource not found", code: "NOT_FOUND", requestId }); } if ((err as any).code === "VALIDATION_ERROR" && (err as any).details) { - return res.status(400).json({ error: "Validation failed", code: "VALIDATION_ERROR", details: (err as any).details, requestId }); + return res.status(400).json({ + error: "Validation failed", + code: "VALIDATION_ERROR", + details: (err as any).details, + requestId, + }); } - res.status(500).json({ error: err.message || "Internal server error", code: "INTERNAL_ERROR", requestId }); + res + .status(500) + .json({ error: err.message || "Internal server error", code: "INTERNAL_ERROR", requestId }); }); +// ── Server startup ─────────────────────────────────────────────────────────── app.listen(PORT, () => { - logger.info({ port: PORT, network: process.env.STELLAR_NETWORK ?? "testnet" }, "SolarGrid backend started"); + logger.info( + { port: PORT, network: process.env.STELLAR_NETWORK ?? "testnet" }, + "SolarGrid backend started", + ); initUsageEventStore(); initMeterNotesStore(); startUsageEventRetryWorker(); - logger.info("SolarGrid backend listening", { port: PORT }); startLimitWatcher(stellarService); try { startIoTBridge(); diff --git a/backend/src/iot/bridge.ts b/backend/src/iot/bridge.ts index 4ab7b40..7d42c2a 100644 --- a/backend/src/iot/bridge.ts +++ b/backend/src/iot/bridge.ts @@ -84,7 +84,7 @@ async function getPriority(meterId: string): Promise { async function checkAndNotifyLowBalance(meterId: string) { // Read fresh each call — /api/webhooks/low-balance may register a URL // after this module was first loaded. - const webhookUrl = process.env.PROVIDER_WEBHOOK_URL ?? WEBHOOK_URL; + const webhookUrl = process.env.PROVIDER_WEBHOOK_URL; if (!webhookUrl) return; const urls = getWebhookUrls(); if (urls.size === 0) return; diff --git a/backend/src/lib/stellar.ts b/backend/src/lib/stellar.ts index e4c485e..8943920 100644 --- a/backend/src/lib/stellar.ts +++ b/backend/src/lib/stellar.ts @@ -18,31 +18,6 @@ export const HORIZON_URL = ? "https://horizon.stellar.org" : "https://horizon-testnet.stellar.org"); -export const CONTRACT_ID = process.env.CONTRACT_ID!; -export const server = new StellarSdk.SorobanRpc.Server(RPC_URL); - -// Load keypair once at module init. The raw secret string is never referenced again. -const adminKeypair = StellarSdk.Keypair.fromSecret(process.env.ADMIN_SECRET_KEY!); - -/** - * Poll until a submitted transaction reaches SUCCESS or FAILED. - * Throws a descriptive error on FAILED status or when maxAttempts is exhausted. - */ -export async function waitForConfirmation( - hash: string, - maxAttempts = 10, - pollIntervalMs = 2_000 -): Promise { - for (let i = 0; i < maxAttempts; i++) { - const status = await server.getTransaction(hash); - if (status.status === StellarSdk.SorobanRpc.Api.GetTransactionStatus.SUCCESS) return; - if (status.status === StellarSdk.SorobanRpc.Api.GetTransactionStatus.FAILED) { - throw new Error(`Transaction failed: ${hash}`); - } - await new Promise((r) => setTimeout(r, pollIntervalMs)); - } - throw new Error(`Transaction timed out: ${hash}`); -} const SECRET_ENV = process.env.ADMIN_SECRET_KEY ?? ""; export const scrub = (msg: string | undefined): string => { diff --git a/backend/src/lib/validation.ts b/backend/src/lib/validation.ts index 997c16b..4f762cd 100644 --- a/backend/src/lib/validation.ts +++ b/backend/src/lib/validation.ts @@ -41,6 +41,11 @@ export const UsageUpdateSchema = z }) .strict(); +/** MQTT payload schema — extends UsageUpdateSchema with meterId from the topic. */ +export const MqttPayloadSchema = UsageUpdateSchema.extend({ + meterId: z.string().min(1), +}); + export const MeterNoteSchema = z .object({ text: z diff --git a/backend/src/middleware/rateLimit.ts b/backend/src/middleware/rateLimit.ts index ee49cf7..5cc96e4 100644 --- a/backend/src/middleware/rateLimit.ts +++ b/backend/src/middleware/rateLimit.ts @@ -1,19 +1,53 @@ -import rateLimit from 'express-rate-limit'; - -const windowMs = parseInt(process.env.RATE_LIMIT_WINDOW_MS ?? '60000', 10); -const max = parseInt(process.env.RATE_LIMIT_MAX ?? '30', 10); +/** + * middleware/rateLimit.ts + * + * Exposes pre-configured express-rate-limit instances consumed by route + * registrations in index.ts. + * + * Closes #539: all env-var parsing has been moved to + * config/rateLimits.ts so there is a single source of truth shared between + * this file and index.ts. + */ +import rateLimit from "express-rate-limit"; +import { + RATE_LIMIT_WINDOW_MS, + WRITE_RATE_LIMIT_MAX, + RATE_LIMIT_MESSAGE, +} from "../config/rateLimits.js"; +/** + * writeLimiter — applied to mutating endpoints (admin login, webhooks, + * allowlist, client-error reports). More restrictive than the global read + * limiter to protect write paths. + */ export const writeLimiter = rateLimit({ - windowMs, - max, + windowMs: RATE_LIMIT_WINDOW_MS, + max: WRITE_RATE_LIMIT_MAX, standardHeaders: true, legacyHeaders: false, - message: { error: 'Too many requests', code: 'RATE_LIMITED' }, + handler: (_req, res) => { + res.setHeader( + "Retry-After", + String(Math.ceil(RATE_LIMIT_WINDOW_MS / 1000)), + ); + res.status(429).json({ error: RATE_LIMIT_MESSAGE, code: "RATE_LIMITED" }); + }, }); +/** + * readLimiter — a permissive limiter for read-heavy endpoints. 4× the write + * cap so read bursts don't trigger 429s during normal polling. + */ export const readLimiter = rateLimit({ - windowMs, - max: max * 4, + windowMs: RATE_LIMIT_WINDOW_MS, + max: WRITE_RATE_LIMIT_MAX * 4, standardHeaders: true, legacyHeaders: false, + handler: (_req, res) => { + res.setHeader( + "Retry-After", + String(Math.ceil(RATE_LIMIT_WINDOW_MS / 1000)), + ); + res.status(429).json({ error: RATE_LIMIT_MESSAGE, code: "RATE_LIMITED" }); + }, }); diff --git a/backend/src/routes/meters.ts b/backend/src/routes/meters.ts index e15ff85..99346c6 100644 --- a/backend/src/routes/meters.ts +++ b/backend/src/routes/meters.ts @@ -46,161 +46,6 @@ export function createMeterRouter(stellar: StellarService) { Math.max(1, Number(req.query.pageSize ?? 25) || 25), ); - const header = "owner,active,units_used,plan,last_payment,expires_at,daily_limit"; - const rows = meters.map((m: any) => - [m.owner, m.active, m.units_used, m.plan, m.last_payment, m.expires_at, m.daily_limit].join(",") - ); - res.setHeader("Content-Type", "text/csv"); - res.setHeader("Content-Disposition", "attachment; filename=meters.csv"); - return res.send([header, ...rows].join("\n")); - }), -); - -/** - * GET /api/meters/search?q=&page=&pageSize= — case-insensitive substring - * search across meter ID and owner address, paginated. - */ -meterRouter.get( - "/search", - asyncHandler(async (req, res) => { - const q = String(req.query.q ?? "").toLowerCase().trim(); - const page = Math.max(1, Number(req.query.page ?? 1) || 1); - const pageSize = Math.min( - 100, - Math.max(1, Number(req.query.pageSize ?? 25) || 25), - ); - - const result = await contractQuery("get_all_meters", []); - const meters = (StellarSdk.scValToNative(result) as any[]) ?? []; - - const matches = q - ? meters.filter((m: any) => { - const id = String(m.meter_id ?? m.id ?? "").toLowerCase(); - const owner = String(m.owner ?? "").toLowerCase(); - return id.includes(q) || owner.includes(q); - }) - : meters; - - const total = matches.length; - const offset = (page - 1) * pageSize; - const data = matches.slice(offset, offset + pageSize); - - res.json({ - data, - pagination: { - page, - pageSize, - total, - totalPages: Math.max(1, Math.ceil(total / pageSize)), - }, - }); - }), -); - -/** GET /api/meters/:id — get meter status */ -meterRouter.get( - "/:id", - asyncHandler(async (req, res) => { - const result = await contractQuery("get_meter", [ - StellarSdk.nativeToScVal(req.params.id, { type: "symbol" }), - ]); - res.json({ meter: StellarSdk.scValToNative(result) }); - }), -); - -/** GET /api/meters/:id/access — check if meter is active */ -meterRouter.get( - "/:id/access", - asyncHandler(async (req, res) => { - const result = await contractQuery("check_access", [ - StellarSdk.nativeToScVal(req.params.id, { type: "symbol" }), - ]); - res.json({ active: StellarSdk.scValToNative(result) }); - }), -); - -/** GET /api/meters/:id/history — paginated local usage history */ -meterRouter.get("/:id/history", (req, res) => { - const page = Math.max(1, Number(req.query.page ?? 1) || 1); - const pageSize = Math.min( - 100, - Math.max(1, Number(req.query.pageSize ?? 25) || 25), - ); - - try { - const history = getUsageHistory(req.params.id, page, pageSize); - res.json(history); - } catch (err: any) { - res.status(500).json({ error: err.message }); - } -}); - -/** GET /api/meters/owner/:address — list all meters for an owner */ -meterRouter.get( - "/owner/:address", - asyncHandler(async (req, res) => { - const result = await contractQuery("get_meters_by_owner", [ - StellarSdk.nativeToScVal(req.params.address, { type: "address" }), - ]); - res.json({ meters: StellarSdk.scValToNative(result) }); - }), -); - -/** POST /api/meters — register a new meter (admin only) */ -meterRouter.post( - "/", - validateRequest({ body: RegisterMeterSchema }), - asyncHandler(async (req, res) => { - const { meter_id, owner } = req.body; - - const hash = await adminInvoke("register_meter", [ - StellarSdk.nativeToScVal(meter_id, { type: "symbol" }), - StellarSdk.nativeToScVal(owner, { type: "address" }), - ]); - res.json({ hash }); - }), -); - -/** POST /api/meters/:id/usage — IoT oracle reports usage */ -meterRouter.post("/:id/usage", async (req, res) => { - const { units, cost } = req.body as { units: unknown; cost: unknown }; - - if (units == null || cost == null) { - return res.status(400).json({ error: "units and cost are required" }); - } - - const unitsNum = Number(units); - const costNum = Number(cost); - - if (!Number.isFinite(unitsNum) || !Number.isFinite(costNum)) { - return res.status(400).json({ error: "units and cost must be valid numbers" }); - } - - if (!Number.isInteger(unitsNum) || !Number.isInteger(costNum)) { - return res.status(400).json({ error: "units and cost must be integers" }); - } - - if (unitsNum <= 0 || costNum <= 0) { - return res.status(400).json({ error: "units and cost must be positive" }); - } - - try { - const event = await persistAndSubmitUsageEvent({ - meterId: req.params.id, - units: unitsNum, - cost: costNum, - sourceTopic: null, - }); - - res.json({ - event, - hash: event.on_chain_tx_hash, - queued: !event.on_chain_tx_hash, - }); - } catch (err: any) { - res.status(500).json({ error: err.message }); - } -}); const result = await stellar.query("get_all_meters", []); const allMeters = (StellarSdk.scValToNative(result) as any[]) ?? []; diff --git a/backend/src/routes/payments.ts b/backend/src/routes/payments.ts index 07bb9ed..cef6d8c 100644 --- a/backend/src/routes/payments.ts +++ b/backend/src/routes/payments.ts @@ -31,22 +31,6 @@ function cleanExpiredIdempotencyKeys() { } } -paymentsRouter.post( - "/", - asyncHandler(async (req, res) => { - cleanExpiredIdempotencyKeys(); - - const idempotencyKey = ( - req.headers["idempotency-key"] ?? req.headers["x-idempotency-key"] - ) as string | undefined; - - if (idempotencyKey && typeof idempotencyKey === "string" && idempotencyKey.trim().length > 0) { - const cached = idempotencyCache.get(idempotencyKey.trim()); - if (cached && Date.now() - cached.createdAt <= IDEMPOTENCY_TTL_MS) { - return res.json({ hash: cached.hash }); - } - } - paymentsRouter.post( "/", idempotency(), @@ -66,9 +50,6 @@ paymentsRouter.post( StellarSdk.nativeToScVal(payer, { type: "address" }), ]); - if (idempotencyKey && typeof idempotencyKey === "string" && idempotencyKey.trim().length > 0) { - idempotencyCache.set(idempotencyKey.trim(), { hash, createdAt: Date.now() }); - } return res.json({ hash }); }), @@ -202,32 +183,6 @@ paymentsRouter.get( ); - try { - StellarSdk.StrKey.decodeEd25519PublicKey(address); - } catch { - return res.status(400).json({ error: "Invalid Stellar address" }); - } - - try { - const records = await fetchPaymentEvents(address, sort, days); - const total = records.length; - const start = (page - 1) * limit; - const paginated = records.slice(start, start + limit); - - return res.json({ - payments: paginated, - pagination: { page, limit, total, pages: Math.ceil(total / limit) }, - }); - } catch (err: any) { - logger.error("payments route error:", err); - if (err?.code === 'RPC_ERROR' || err?.isRpcError) { - return res.status(502).json({ error: err.message ?? "RPC request failed", code: "RPC_ERROR" }); - } - return res.status(500).json({ error: err.message ?? "Failed to fetch payment history" }); - } - }), -); - /** * GET /api/payments/history/:address?from=&to=&limit=50&page=1 * diff --git a/backend/src/routes/stats.ts b/backend/src/routes/stats.ts index 32b1835..367fae9 100644 --- a/backend/src/routes/stats.ts +++ b/backend/src/routes/stats.ts @@ -1,28 +1,4 @@ import { Router } from "express"; -import { getTopConsumers } from "../lib/usageEvents.js"; - -export const statsRouter = Router(); - -function requireAdminKey(req: any, res: any, next: any) { - const adminKey = process.env.ADMIN_API_KEY; - const provided = req.headers["x-admin-key"]; - if (!adminKey || provided !== adminKey) { - return res.status(401).json({ error: "Valid admin key required" }); - } - return next(); -} - -/** - * GET /api/stats/top-consumers?days=30 - * - * Returns the top 10 meters ranked by total units used over the given - * window (default 30 days). Requires the X-Admin-Key header. - */ -statsRouter.get("/top-consumers", requireAdminKey, (req, res) => { - const days = Math.max(1, Number(req.query.days ?? 30) || 30); - const consumers = getTopConsumers(days, 10); - res.json(consumers); -}); import * as StellarSdk from "@stellar/stellar-sdk"; import { server, CONTRACT_ID } from "../lib/stellar.js"; import { asyncHandler } from "../lib/asyncHandler.js"; diff --git a/backend/src/routes/webhooks.ts b/backend/src/routes/webhooks.ts index 010d782..d7678f1 100644 --- a/backend/src/routes/webhooks.ts +++ b/backend/src/routes/webhooks.ts @@ -108,38 +108,31 @@ webhookRouter.post( }), }), asyncHandler(async (req, res) => { + const providerId = getProviderId(req); + if (!providerId) { + return res.status(400).json({ + error: "X-Provider-ID header is required", + code: "MISSING_PROVIDER_ID", + }); + } + const { webhook_url, secret } = req.body as { webhook_url: string; secret?: string; }; - // For now, store in environment variables (in production, use a database). - process.env.PROVIDER_WEBHOOK_URL = webhook_url; - let secretHash: string | undefined; if (secret) { process.env.PROVIDER_WEBHOOK_SECRET = secret; secretHash = crypto.createHash("sha256").update(secret).digest("hex"); } - logger.info("Low-balance webhook registered", { - webhook_url, - secretHash, - const providerId = getProviderId(req); - if (!providerId) { - return res.status(400).json({ - error: "X-Provider-ID header is required", - code: "MISSING_PROVIDER_ID", - }); - } - - const { webhook_url } = req.body; - const record = registerWebhook(providerId, webhook_url); logger.info("Low-balance webhook registered", { provider_id: providerId, webhook_url, + secretHash, }); return res.status(200).json({ diff --git a/infra/prometheus.yml b/infra/prometheus.yml index bc17518..afc1425 100644 --- a/infra/prometheus.yml +++ b/infra/prometheus.yml @@ -1,7 +1,22 @@ +# infra/prometheus.yml +# +# Prometheus scrape configuration for SolarGrid. +# +# Closes #537: the /metrics endpoint on the backend is intentionally public +# (no authentication). Prometheus's pull model requires unauthenticated HTTP +# GET access to scrape metrics. The backend port (3001) is only exposed on +# the internal Docker app-network and is NOT forwarded to a public interface, +# so network-level isolation already limits who can reach /metrics. +# +# If the backend is ever placed behind a public-facing load balancer, restrict +# /metrics at the reverse-proxy layer (e.g. an nginx `deny all` block or a +# firewall rule that allows only the Prometheus container's IP). + global: scrape_interval: 15s scrape_configs: - job_name: 'backend' + metrics_path: '/metrics' static_configs: - targets: ['backend:3001'] diff --git a/scripts/check-duplicate-pkg-keys.js b/scripts/check-duplicate-pkg-keys.js new file mode 100644 index 0000000..a7b5674 --- /dev/null +++ b/scripts/check-duplicate-pkg-keys.js @@ -0,0 +1,168 @@ +#!/usr/bin/env node +/** + * scripts/check-duplicate-pkg-keys.js + * + * Closes #538: detects duplicate top-level and dependency keys in + * package.json files. + * + * Standard JSON.parse() silently uses the last occurrence when a key appears + * more than once, so `npm install` succeeding is NOT proof that the file is + * duplicate-free. This script uses a custom reviver-free parser that counts + * occurrences and fails with a non-zero exit code on the first duplicate found. + * + * Usage: + * node scripts/check-duplicate-pkg-keys.js [...] + * + * In CI (see .github/workflows/ci.yml): + * node scripts/check-duplicate-pkg-keys.js backend/package.json frontend/package.json + */ + +import { readFileSync } from "fs"; + +/** + * Parse a JSON string while tracking every key occurrence. + * Returns { parsed, duplicates } where duplicates is an array of + * { path, key, count } objects. + */ +function parseWithDuplicateCheck(text) { + const duplicates = []; + + /** + * Recursive descent parser for JSON objects/arrays. + * pos is a shared { value: number } cursor so recursion can advance it. + */ + function parseValue(pos, keyPath) { + skipWhitespace(pos); + const ch = text[pos.value]; + if (ch === "{") return parseObject(pos, keyPath); + if (ch === "[") return parseArray(pos, keyPath); + if (ch === '"') return parseString(pos); + if (ch === "t" || ch === "f" || ch === "n") return parseLiteral(pos); + return parseNumber(pos); + } + + function parseObject(pos, keyPath) { + pos.value++; // consume '{' + skipWhitespace(pos); + const seen = new Map(); + const obj = {}; + if (text[pos.value] === "}") { pos.value++; return obj; } + while (true) { + skipWhitespace(pos); + const key = parseString(pos); + const fullPath = keyPath ? `${keyPath}.${key}` : key; + const count = (seen.get(key) ?? 0) + 1; + seen.set(key, count); + if (count > 1) { + duplicates.push({ path: fullPath, key, count }); + } + skipWhitespace(pos); + pos.value++; // consume ':' + const val = parseValue(pos, fullPath); + obj[key] = val; + skipWhitespace(pos); + if (text[pos.value] === "}") { pos.value++; break; } + pos.value++; // consume ',' + } + return obj; + } + + function parseArray(pos, keyPath) { + pos.value++; // consume '[' + skipWhitespace(pos); + const arr = []; + if (text[pos.value] === "]") { pos.value++; return arr; } + while (true) { + const val = parseValue(pos, keyPath); + arr.push(val); + skipWhitespace(pos); + if (text[pos.value] === "]") { pos.value++; break; } + pos.value++; // consume ',' + } + return arr; + } + + function parseString(pos) { + pos.value++; // consume opening '"' + let str = ""; + while (pos.value < text.length) { + const ch = text[pos.value]; + if (ch === '"') { pos.value++; return str; } + if (ch === "\\") { + pos.value++; + const esc = text[pos.value]; + str += esc === "n" ? "\n" : esc === "t" ? "\t" : esc === "r" ? "\r" : esc; + } else { + str += ch; + } + pos.value++; + } + return str; + } + + function parseLiteral(pos) { + if (text.startsWith("true", pos.value)) { pos.value += 4; return true; } + if (text.startsWith("false", pos.value)) { pos.value += 5; return false; } + if (text.startsWith("null", pos.value)) { pos.value += 4; return null; } + throw new Error(`Unexpected token at ${pos.value}`); + } + + function parseNumber(pos) { + const start = pos.value; + while (pos.value < text.length && /[0-9.\-+eE]/.test(text[pos.value])) pos.value++; + return Number(text.slice(start, pos.value)); + } + + function skipWhitespace(pos) { + while (pos.value < text.length && /\s/.test(text[pos.value])) pos.value++; + } + + const pos = { value: 0 }; + const parsed = parseValue(pos, ""); + return { parsed, duplicates }; +} + +// ── Main ───────────────────────────────────────────────────────────────────── + +const files = process.argv.slice(2); + +if (files.length === 0) { + console.error( + "Usage: node scripts/check-duplicate-pkg-keys.js [...]", + ); + process.exit(1); +} + +let anyFailed = false; + +for (const file of files) { + let text; + try { + text = readFileSync(file, "utf8"); + } catch (err) { + console.error(`ERROR: Cannot read ${file}: ${err.message}`); + anyFailed = true; + continue; + } + + let result; + try { + result = parseWithDuplicateCheck(text); + } catch (err) { + console.error(`ERROR: Failed to parse ${file}: ${err.message}`); + anyFailed = true; + continue; + } + + if (result.duplicates.length === 0) { + console.log(`OK ${file} — no duplicate keys found`); + } else { + console.error(`FAIL ${file} — duplicate keys detected:`); + for (const { path, count } of result.duplicates) { + console.error(` "${path}" appears ${count} times`); + } + anyFailed = true; + } +} + +process.exit(anyFailed ? 1 : 0); diff --git a/scripts/migrate-all.js b/scripts/migrate-all.js new file mode 100644 index 0000000..210a33e --- /dev/null +++ b/scripts/migrate-all.js @@ -0,0 +1,238 @@ +#!/usr/bin/env node +/** + * scripts/migrate-all.js + * + * Bulk migration helper — closes #536. + * + * Fetches every registered meter via `get_all_meters` and calls + * `migrate_meter(meter_id)` for each one, logging per-meter success/failure + * and printing a summary at the end. + * + * Usage (via Makefile): + * make migrate-all CONTRACT_ID=C... ADMIN_SECRET_KEY=S... [NETWORK=testnet] + * + * Usage (directly): + * node scripts/migrate-all.js + * + * Required environment variables: + * CONTRACT_ID — Soroban contract address + * ADMIN_SECRET_KEY — Admin Stellar secret key (S...) + * + * Optional environment variables: + * STELLAR_NETWORK — "testnet" (default) or "mainnet" + * STELLAR_RPC_URL — Override the default RPC endpoint + * DRY_RUN — Set to "true" to list meters without migrating + * + * Exit codes: + * 0 — all meters migrated successfully (or DRY_RUN) + * 1 — one or more meters failed to migrate + */ + +import * as StellarSdk from "@stellar/stellar-sdk"; + +// ── Config ─────────────────────────────────────────────────────────────────── + +const CONTRACT_ID = process.env.CONTRACT_ID; +const ADMIN_SECRET_KEY = process.env.ADMIN_SECRET_KEY; +const NETWORK = process.env.STELLAR_NETWORK ?? "testnet"; +const DRY_RUN = process.env.DRY_RUN === "true"; + +const NETWORK_PASSPHRASE = + NETWORK === "mainnet" + ? StellarSdk.Networks.PUBLIC + : StellarSdk.Networks.TESTNET; + +const RPC_URL = + process.env.STELLAR_RPC_URL ?? + (NETWORK === "mainnet" + ? "https://soroban-rpc.stellar.org" + : "https://soroban-testnet.stellar.org"); + +// ── Validation ─────────────────────────────────────────────────────────────── + +if (!CONTRACT_ID) { + console.error("ERROR: CONTRACT_ID environment variable is required."); + console.error( + " Usage: CONTRACT_ID=C... ADMIN_SECRET_KEY=S... node scripts/migrate-all.js", + ); + process.exit(1); +} + +if (!ADMIN_SECRET_KEY) { + console.error("ERROR: ADMIN_SECRET_KEY environment variable is required."); + process.exit(1); +} + +// ── Helpers ────────────────────────────────────────────────────────────────── + +const rpcServer = new StellarSdk.SorobanRpc.Server(RPC_URL); +const adminKeypair = StellarSdk.Keypair.fromSecret(ADMIN_SECRET_KEY); +const contract = new StellarSdk.Contract(CONTRACT_ID); + +/** Redact admin secret from any string before printing. */ +const scrub = (msg) => String(msg ?? "").replaceAll(ADMIN_SECRET_KEY, "[REDACTED]"); + +/** + * Poll until transaction reaches SUCCESS or FAILED. + */ +async function waitForTx(hash, maxAttempts = 15, intervalMs = 2_000) { + for (let i = 0; i < maxAttempts; i++) { + const result = await rpcServer.getTransaction(hash); + if ( + result.status === StellarSdk.SorobanRpc.Api.GetTransactionStatus.SUCCESS + ) { + return; + } + if ( + result.status === StellarSdk.SorobanRpc.Api.GetTransactionStatus.FAILED + ) { + throw new Error(`Transaction ${hash} failed on-chain.`); + } + await new Promise((r) => setTimeout(r, intervalMs)); + } + throw new Error(`Transaction ${hash} timed out after ${maxAttempts} polls.`); +} + +/** + * Invoke a contract function and wait for confirmation. + */ +async function invokeContract(method, args = []) { + const account = await rpcServer.getAccount(adminKeypair.publicKey()); + + let tx = new StellarSdk.TransactionBuilder(account, { + fee: "100", + networkPassphrase: NETWORK_PASSPHRASE, + }) + .addOperation(contract.call(method, ...args)) + .setTimeout(30) + .build(); + + const sim = await rpcServer.simulateTransaction(tx); + if (StellarSdk.SorobanRpc.Api.isSimulationError(sim)) { + throw new Error(scrub(String(sim.error ?? sim))); + } + + tx = StellarSdk.SorobanRpc.assembleTransaction(tx, sim).build(); + tx.sign(adminKeypair); + + const sendResult = await rpcServer.sendTransaction(tx); + await waitForTx(sendResult.hash); + return sendResult.hash; +} + +/** + * Query a contract function (simulation only, no on-chain transaction). + */ +async function queryContract(method, args = []) { + const account = await rpcServer.getAccount(adminKeypair.publicKey()); + + const tx = new StellarSdk.TransactionBuilder(account, { + fee: "100", + networkPassphrase: NETWORK_PASSPHRASE, + }) + .addOperation(contract.call(method, ...args)) + .setTimeout(30) + .build(); + + const sim = await rpcServer.simulateTransaction(tx); + if (StellarSdk.SorobanRpc.Api.isSimulationError(sim)) { + throw new Error(scrub(String(sim.error ?? sim))); + } + + return sim.result?.retval; +} + +// ── Main ───────────────────────────────────────────────────────────────────── + +async function main() { + console.log("=".repeat(60)); + console.log("SolarGrid bulk meter migration"); + console.log(` Network : ${NETWORK}`); + console.log(` RPC URL : ${RPC_URL}`); + console.log(` Contract : ${CONTRACT_ID}`); + console.log(` Admin : ${adminKeypair.publicKey()}`); + if (DRY_RUN) console.log(" DRY RUN : true — no transactions will be sent"); + console.log("=".repeat(60)); + + // 1. Fetch all registered meters. + console.log("\n[1/3] Fetching all registered meters via get_all_meters..."); + let meters; + try { + const result = await queryContract("get_all_meters"); + meters = StellarSdk.scValToNative(result) ?? []; + } catch (err) { + console.error("ERROR: Failed to fetch meters:", scrub(err?.message)); + process.exit(1); + } + + if (!Array.isArray(meters) || meters.length === 0) { + console.log("No meters found. Nothing to migrate."); + process.exit(0); + } + + // Extract meter IDs from the contract's return value. + // The contract returns an array of Meter structs; meter_id is a String field. + const meterIds = meters.map((m) => { + // Handle both { meter_id: "..." } structs and raw string arrays. + if (typeof m === "string") return m; + if (m && typeof m.meter_id === "string") return m.meter_id; + if (m && typeof m.id === "string") return m.id; + return String(m); + }); + + console.log(`Found ${meterIds.length} meter(s):`); + meterIds.forEach((id, i) => console.log(` ${i + 1}. ${id}`)); + + if (DRY_RUN) { + console.log("\nDRY_RUN=true — skipping migration transactions."); + process.exit(0); + } + + // 2. Migrate each meter. + console.log(`\n[2/3] Migrating ${meterIds.length} meter(s)...`); + + const results = { success: [], failed: [] }; + + for (const meterId of meterIds) { + process.stdout.write(` migrate_meter(${meterId}) ... `); + try { + const hash = await invokeContract("migrate_meter", [ + StellarSdk.nativeToScVal(meterId, { type: "string" }), + ]); + console.log(`OK (tx: ${hash})`); + results.success.push(meterId); + } catch (err) { + console.log(`FAILED`); + console.error(` Error: ${scrub(err?.message)}`); + results.failed.push({ meterId, error: scrub(err?.message) }); + } + } + + // 3. Print summary. + console.log("\n[3/3] Migration summary"); + console.log("=".repeat(60)); + console.log( + ` Total : ${meterIds.length}`, + ); + console.log(` Success : ${results.success.length}`); + console.log(` Failed : ${results.failed.length}`); + + if (results.failed.length > 0) { + console.log("\nFailed meters:"); + results.failed.forEach(({ meterId, error }) => { + console.log(` - ${meterId}: ${error}`); + }); + console.log( + "\nRe-run this script to retry failed meters (migrate_meter is idempotent).", + ); + process.exit(1); + } + + console.log("\nAll meters migrated successfully."); + process.exit(0); +} + +main().catch((err) => { + console.error("Unexpected error:", scrub(err?.message ?? err)); + process.exit(1); +});