diff --git a/CHANGELOG.md b/CHANGELOG.md index 91cc4b5..9e12d88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.15.0] - 2026-09-07 + +### Added +- `attestation_responses` is registered as a hub-mirror table. +- The bootstrap mirror is vendored instead of living only on the file master. + +### Fixed +- The replication-source read asks the hub for unredacted config secrets. +- Replica columns widen from the shared utf8mb4 definition. +- mariadb moved off the cleartext-credential advisory range with the floor pinned in the dependency gate. + +### Changed +- The vendored coin registry is resynced from the hub. + ## [0.12.0] - 2026-08-30 ### Added diff --git a/README.md b/README.md index c198d8a..bbe763f 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ # XChain Sync

- Version - Tests + Version + Tests Node License

@@ -129,11 +129,11 @@ npm run api |---|---| | `npm run api` | Start the sync service | | `bin/run-db-tiers.sh` | Run the DB-backed tiers against a throwaway MariaDB it starts and drops | -| `npm test` | Run unit tests (1,975 tests) | +| `npm test` | Run unit tests (1,993 tests) | | `npm run ci` | Unit tests plus the security tier, exits on completion | | `npm run coverage` | Unit tests under `c8` coverage instrumentation | | `npm run test:boundary` | Boundary condition tests (consensus constants, 7 tests) | -| `npm run test:regression` | Unit tests tagged `@regression` (501 tests) | +| `npm run test:regression` | Unit tests tagged `@regression` (506 tests) | | `npm run test:smoke` | Smoke tests (server + client startup, config loading, 17 tests) | | `npm run test:integration` | Integration tests (requires MariaDB + running indexer, 102 tests) | | `npm run test:e2e` | End-to-end tests (full server/client lifecycle, 72 tests) | diff --git a/deploy/sync-bootstraps/README.md b/deploy/sync-bootstraps/README.md new file mode 100644 index 0000000..c178a39 --- /dev/null +++ b/deploy/sync-bootstraps/README.md @@ -0,0 +1,105 @@ +# sync-bootstraps + +Serving side of bootstrap distribution. + +Two pieces, deployed to different machines: + +| File | Deployed to | Typical path | +|---|---|---| +| `sync-bootstraps.sh` | the file master | `/usr/local/sbin/sync-bootstraps.sh`, mode 755, root-owned | +| `latest.php` | each serving host | inside the served payload tree, with its `.htaccess` | + +`latest.php` resolves `latest.tgz` / `latest.tgz.sig` to the newest archive that +has a paired signature, ordered by the UTC timestamp in the filename. + +## The thing that makes this easy to get wrong + +**The file master does not serve the bootstraps.** The public hostname resolves +to the serving tier, and the master's docroot entry for the payload directory is +a symlink to the real tree. So on the master a freshly published archive looks +published. It is not public until this script has mirrored it out. + +The generators push only as far as the master and log a success line when they +get there. That line means "reached the master", not "reached the internet". +Verify against the serving tier, never the master. + +## Cron + +Install in root's crontab on the file master: + +```cron +# Mirror the bootstrap tree out to the serving tier. Hourly, NOT daily: see the +# SCHEDULE block in the script. The PID lock makes an overlapping run a no-op. +30 * * * * /usr/local/sbin/sync-bootstraps.sh 1>/dev/null 2>/dev/null +``` + +Hourly is a correctness requirement, not tuning. The monthly tracker publish +runs for hours. A daily slot that happens to fall before it means the tier +serves the previous month's bootstrap until the next day's pass, with every +component reporting success. That is exactly what happened while this ran daily. + +Do not replace hourly with a slot chosen to be "after" the publish. The publish +window grows with the chain, so frequency is the guarantee and timing is not. + +## Environment + +The script carries no site topology. Set these where cron or the unit can see +them; the script refuses to start if any required one is unset. + +| Variable | Meaning | +|---|---| +| `BOOTSTRAP_SRC` | payload tree on the master, trailing slash | +| `BOOTSTRAP_SITE_SRC` | docroot on the master, trailing slash | +| `BOOTSTRAP_DEST` | payload path on the targets, trailing slash | +| `BOOTSTRAP_SITE_DEST` | docroot on the targets, trailing slash | +| `BOOTSTRAP_TARGETS` | space-separated serving hosts | +| `BOOTSTRAP_SSH_KEY` | private key authorised on the targets | +| `BOOTSTRAP_REMOTE_USER` | ssh user on the targets (default `www`) | +| `BOOTSTRAP_PAYLOAD_DIRNAME` | payload dir name, excluded from the site leg (default `bootstraps`) | +| `BWLIMIT` | rsync bandwidth cap (default `60M`) | +| `BOOTSTRAP_LOCKFILE`, `BOOTSTRAP_LOGFILE` | lock and log paths | + +Use a dedicated key, not one shared with other fanouts, and restrict it to the +master's address in `authorized_keys`. + +## Operating notes + +- Targets are mirrored **sequentially**. A full pass is the size of the whole + tree per target, so total wall time is roughly one target's transfer time + times the number of targets. +- `BWLIMIT` throttles the master, which typically also carries the hub and the + replication master. Raising it doubles throughput about linearly; raise it + deliberately, and remember it applies per run, not per target. +- **Do not edit this script on the machine while a run is in flight.** bash + reads a script lazily by byte offset, so an in-place edit can corrupt the + running execution. Stop the run, edit, restart. +- Killing a run hard strands rsync's hidden `..XXXXXX` temp instead of + moving it into `.rsync-partial`, so that file restarts from zero next pass. + The stray temp is extraneous and `--delete-after` sweeps it. +- If you stop a run to restart it with different settings, confirm the old rsync + is actually gone before relaunching. Two concurrent passes against the same + target will both transfer, doubling load on the master. `pgrep` takes an + extended regex, so a pattern like `'a\|b'` matches nothing and will tell you + the process is gone when it is not. Check with `ps` and kill by PID. +- Retention is **not** coordinated with the master. The publisher prunes at + publish time, so between that prune and the next mirror pass the serving tier + is the only holder of the superseded archive and is still serving links to it. + +## Verifying a publish actually reached the public + +Check every address the hostname resolves to. A round-robin will otherwise mask +a host that missed the mirror: + +```bash +HOST= +ARCHIVE=//.tar.gz +for ip in $(dig +short "$HOST"); do + curl -s -o /dev/null -w "$ip %{http_code}\n" \ + --resolve "$HOST:443:$ip" \ + "https://$HOST/bootstraps/xchain-utxo-tracker/$ARCHIVE" +done +``` + +All addresses must return 200 for the same archive. Anything else means the +mirror has not finished, and consumers are getting different answers depending +on which host they reach. diff --git a/deploy/sync-bootstraps/sync-bootstraps.sh b/deploy/sync-bootstraps/sync-bootstraps.sh new file mode 100755 index 0000000..d917515 --- /dev/null +++ b/deploy/sync-bootstraps/sync-bootstraps.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# sync-bootstraps.sh - mirror the bootstrap tree from the file master out to the +# hosts that actually serve it. +# +# Deployment specifics (target hosts, key, paths) come from the environment so +# this file carries no site topology. Set them in the unit or cron environment. +# +# SCHEDULE: hourly, and it must stay at least that frequent. +# +# THIS SCRIPT IS THE ONLY PATH FROM THE FILE MASTER TO THE SERVING TIER. The +# master is not in the serving path: the bootstrap hostname resolves to the web +# tier, and the master's own docroot entry is a symlink to the payload tree, so +# a fresh archive is visible there while the public still cannot get it. +# Inspecting the master therefore proves nothing about what is served, and +# neither does the publisher's success line, which only means "reached the +# master". +# +# This ran daily for a while, in a slot that happened to fall about an hour +# BEFORE the monthly tracker publish began. The publish runs for hours, so every +# month this fired against the previous month's tree, exited rc=0, and left the +# public tier advertising a month-old bootstrap until the next day's run. Do not +# move it back to a daily slot, and do not pick a slot you believe is "after" +# the publish: that window is hours long and grows with the chain, so frequency +# is the guarantee, timing is not. +# +# RETENTION IS NOT COORDINATED with the master. The publisher prunes to KEEP +# archives at publish time, so between that prune and the next mirror pass the +# serving tier is the ONLY holder of the superseded archive while still +# advertising links to it. --delete-after is what finally retires them. +# +# Notes: +# - --partial-dir keeps a resumable partial OUT of the served tree, so a +# download can never hit a half-written multi-GB tarball; the finished file +# appears by atomic rename. It only protects an INTERRUPTED transfer: rsync +# writes to a hidden ..XXXXXX temp while running and moves it into +# .rsync-partial on a clean interrupt, so a hard kill (SIGKILL, or a SIGTERM +# it cannot service) strands that temp and the next run restarts the file +# from zero. The stray temp is extraneous and --delete-after sweeps it, but +# the transfer work is lost. +# - --exclude='*.part' because the publisher uploads each archive as +# .part and only then renames it into place. Without this, an hourly +# pass landing inside the publish window hauls a partial upload of up to +# ~160 GB to every target and deletes it on the next pass. --partial-dir +# does NOT cover this: those are the PUBLISHER's temp files sitting in the +# source tree, not rsync's own. +# - --bwlimit protects the master, which also carries the hub and the +# replication master. Raise it deliberately, not by default. +# - --delete keeps the mirror exact. Safe here because the source holds one +# snapshot per coin/network and the browser assets live in the same tree. +# - The SITE leg mirrors the docroot-level chrome (index.html, stylesheets, +# assets/, listing templates) that the payload leg never touches. Without it +# the tier serves the bootstraps page with no stylesheet and a 404ing logo. +# `--exclude=$PAYLOAD_DIRNAME` keeps the source's symlink from clobbering +# the targets' real payload directory, and --delete never removes an +# excluded path, so the payload tree is safe from this leg. +set -u + +# --- deployment configuration (override in the environment) ------------------ +SRC="${BOOTSTRAP_SRC:?set BOOTSTRAP_SRC to the payload tree on the file master, with trailing slash}" +SITE_SRC="${BOOTSTRAP_SITE_SRC:?set BOOTSTRAP_SITE_SRC to the docroot on the file master, with trailing slash}" +DEST_PATH="${BOOTSTRAP_DEST:?set BOOTSTRAP_DEST to the payload path on the targets, with trailing slash}" +SITE_DEST="${BOOTSTRAP_SITE_DEST:?set BOOTSTRAP_SITE_DEST to the docroot on the targets, with trailing slash}" +TARGETS="${BOOTSTRAP_TARGETS:?set BOOTSTRAP_TARGETS to a space-separated list of serving hosts}" +SSH_KEY="${BOOTSTRAP_SSH_KEY:?set BOOTSTRAP_SSH_KEY to the private key authorised on the targets}" +REMOTE_USER="${BOOTSTRAP_REMOTE_USER:-www}" +PAYLOAD_DIRNAME="${BOOTSTRAP_PAYLOAD_DIRNAME:-bootstraps}" +BWLIMIT="${BWLIMIT:-60M}" +LOCKFILE="${BOOTSTRAP_LOCKFILE:-/var/tmp/sync-bootstraps.lock}" +LOGFILE="${BOOTSTRAP_LOGFILE:-/var/log/sync-bootstraps.log}" + +log(){ echo "[$(date -u '+%F %T UTC')] $*" >> "$LOGFILE"; } + +# PID lock with stale-lock recovery: a crashed run must not block every future +# one. +if [ -f "$LOCKFILE" ]; then + OLDPID=$(cat "$LOCKFILE" 2>/dev/null) + if [ -n "$OLDPID" ] && kill -0 "$OLDPID" 2>/dev/null; then + log "already running as pid $OLDPID; exiting" + exit 0 + fi + log "clearing stale lock (pid ${OLDPID:-unknown} is gone)" +fi +echo $$ > "$LOCKFILE" +trap 'rm -f "$LOCKFILE"' EXIT + +if [ ! -d "$SRC" ]; then + log "FATAL: source $SRC missing" + exit 1 +fi +if [ ! -d "$SITE_SRC" ]; then + log "FATAL: site source $SITE_SRC missing" + exit 1 +fi + +log "=== start (bwlimit=$BWLIMIT, source $(du -sh "$SRC" 2>/dev/null | cut -f1)) ===" +RC_TOTAL=0 +for T in $TARGETS; do + log "--- $T: site files starting" + START=$(date +%s) + rsync -a --delete-after \ + --exclude="$PAYLOAD_DIRNAME" \ + --exclude=.rsync-partial \ + --stats \ + -e "ssh -i $SSH_KEY -o BatchMode=yes -o StrictHostKeyChecking=accept-new" \ + "$SITE_SRC" "${REMOTE_USER}@${T}:${SITE_DEST}" >> "$LOGFILE" 2>&1 + RC=$? + ELAPSED=$(( $(date +%s) - START )) + if [ $RC -eq 0 ]; then + log "--- $T: site files OK in ${ELAPSED}s" + else + log "--- $T: site files FAILED rc=$RC after ${ELAPSED}s" + RC_TOTAL=$RC + fi + + log "--- $T: starting" + START=$(date +%s) + rsync -a --delete-after \ + --partial-dir=.rsync-partial \ + --exclude='*.part' \ + --bwlimit="$BWLIMIT" \ + --stats \ + -e "ssh -i $SSH_KEY -o BatchMode=yes -o StrictHostKeyChecking=accept-new" \ + "$SRC" "${REMOTE_USER}@${T}:${DEST_PATH}" >> "$LOGFILE" 2>&1 + RC=$? + ELAPSED=$(( $(date +%s) - START )) + if [ $RC -eq 0 ]; then + log "--- $T: OK in ${ELAPSED}s" + else + log "--- $T: FAILED rc=$RC after ${ELAPSED}s" + RC_TOTAL=$RC + fi +done +log "=== done (worst rc=$RC_TOTAL) ===" +exit $RC_TOTAL diff --git a/package-lock.json b/package-lock.json index 6090c8f..aabae60 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "xchain-sync", - "version": "0.12.0", + "version": "0.15.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "xchain-sync", - "version": "0.12.0", + "version": "0.15.0", "license": "AGPL-3.0-or-later", "dependencies": { "axios": "^1.18.1", diff --git a/package.json b/package.json index f782625..09a8bac 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "xchain-sync", "description": "Database replication service for the XChain Platform: syncs indexer and decoder databases to validators and consumers via REST snapshots and WebSocket streaming", - "version": "0.12.0", + "version": "0.15.0", "license": "AGPL-3.0-or-later", "repository": { "type": "git", diff --git a/src/ClientApplier.js b/src/ClientApplier.js index f5141ee..b10600a 100644 --- a/src/ClientApplier.js +++ b/src/ClientApplier.js @@ -100,13 +100,20 @@ class ClientApplier { // full-dump re-send on an incremental catch-up idempotent. 'merkle_epochs', // validator_rewards has a UNIQUE key (source_id, signing_pubkey_id, - // reward_type, round_reference). The recovery-redriven collector + // reward_type, round_reference, round_qualifier). The recovery-redriven collector // (recoveryRewards.js) can re-inject a backdated survivor row via BOTH the // live per-block and incremental-snapshot channels when their windows overlap; // INSERT IGNORE makes that re-injection idempotent. Safe for the normal path // (each row streams once in its earn-block; mirrors createValidatorReward's // own INSERT IGNORE on the source). - 'validator_rewards' + 'validator_rewards', + // rollcalls / rollcall_absences ride the bootstrap full dump AND stream by + // close_block, so an overlapping window re-delivers a row already applied. + // Each is pinned at close and never re-derived, so the re-delivery is + // identical and IGNORE is a no-op; a plain INSERT would abort the apply + // transaction on the duplicate PK. Same reasoning as validator_rewards above. + 'rollcalls', + 'rollcall_absences' ]); // Mutable aggregates that the indexer full-dump re-sends with their CURRENT @@ -480,10 +487,14 @@ class ClientApplier { // roots must go even when state commitment is inactive on this node. Swallow // only schema gaps (1146 table missing on decoder / older schemas, 1054 missing // column); any other error must propagate so the outer catch rolls the txn back. + // Bind this.coinTicker, NOT this.chain: the rows carry the TICKER (every writer + // in stateCommitment.js is called with this.coinTicker, and SyncService passes + // the hub's full lowercase coin name into the constructor), so the full name + // matches zero rows and the cleanup silently no-ops on every production chain. try { await this.db.doQuery( 'DELETE FROM state_tree_roots WHERE chain = ? AND network = ? AND block_index >= ?', - [this.chain, this.network, snapshotData.block_height]); + [this.coinTicker, this.network, snapshotData.block_height]); } catch(e){ if(e.errno !== 1146 && e.errno !== 1054) throw e; } @@ -729,8 +740,19 @@ class ClientApplier { // (bootstrap snapshot, or the pre-flag-day ANCHOR write) kept it forever, strictly // AHEAD of the source and invisible to the source-ahead-only count check. The log // row carries the loser's full UNIQUE identity (source_id, signing_pubkey_id, - // reward_type, round_reference), so this is a keyed delete with no winner predicate - // to reproduce; rows the source still holds (winners) never match a pre-image. + // reward_type, round_reference, round_qualifier), so this is a keyed delete with no + // winner predicate to reproduce; rows the source still holds (winners) never match a + // pre-image. + // + // round_qualifier is load-bearing here, not decoration. The archive leg keys + // round_reference on MATCH_BATCH_SEQ, a dense hub counter a rebase reissues, so two + // genuinely distinct archive rewards can share all four older columns and differ only + // in qualifier (the snapshot_block). Keyed on the four alone this DELETE also reaches + // the OTHER snapshot's row and destroys a reward the source still holds: the exact + // inverse of the drift the mirror exists to close, and silent, because + // validator_rewards declares no hash class (tableLifecycle.js). Both columns are NOT + // NULL DEFAULT 0 on both tables, so this predicate is a plain `=` rather than the + // NULL-safe `<=>` that nullable round_reference needs. // Runs AFTER the insert loop (the log rows of this apply are in place) and INSIDE // the apply transaction. The reverse twin is ClientRollback's RB-ANCHOR restore, // which re-INSERTs these pre-images when the reconcile block is orphaned. `scopeSql` @@ -743,6 +765,7 @@ class ClientApplier { "JOIN anchor_reward_reconcile_log d " + " ON d.source_id = vr.source_id AND d.signing_pubkey_id = vr.signing_pubkey_id " + " AND d.reward_type = vr.reward_type AND d.round_reference <=> vr.round_reference " + + " AND d.round_qualifier = vr.round_qualifier " + "WHERE " + scopeSql, scopeArgs); } catch(e){ @@ -750,6 +773,17 @@ class ClientApplier { // to skip: such a replica received no log rows either. Anything else must // abort the apply so the block is retried, never applied half-mirrored. if(e && e.errno !== 1146 && e.errno !== 1054) throw e; + // A replica whose log table or validator_rewards predates round_qualifier now + // raises 1054 on the whole statement, so the mirror stops rather than deleting + // on the stale four-column key. That leaves the replica AHEAD, which the + // source-ahead-only count check cannot see, so say it once per apply instead of + // skipping in silence; schema replication (ensureReplicatedColumns) adds the + // column on the next pass and the mirror resumes. + if(e && e.errno === 1054) + console.warn('anchor-reward reconcile mirror skipped: an identity column ' + + '(round_qualifier) is missing from validator_rewards or ' + + 'anchor_reward_reconcile_log on this replica, so reconcile losers stay ' + + 'until schema replication adds it'); } } diff --git a/src/ClientRollback.js b/src/ClientRollback.js index 4f498fa..ba9bf9f 100644 --- a/src/ClientRollback.js +++ b/src/ClientRollback.js @@ -27,13 +27,19 @@ const lifecycle = require('./tableLifecycle'); const replicatedTables = require('./replicatedTables'); const { activationDelayBlocks, gasTickSymbol } = require('./consensus-constants'); const { ARCHIVE_HEAD_VERSIONS_SQL } = require('./stateHash'); +const { archiveAuthorScopeJoin } = require('./archive_rollback_author_scope_activation'); class ClientRollback { - constructor(db, util, coin) { + constructor(db, util, coin, network) { this.db = db; this.util = util; + // The replica's own network, the key for the publisher-scoped archive reset below. + // An omitted network reads as inactive, correct only while every threshold is inert: + // the guard in test/unit/rollback-coverage.test.js fails the moment one is armed. + this.network = network || null; + // Frozen per-chain STAKING.ACTIVATION_DELAY_BLOCKS, needed to mirror the source // indexer's reorg deactivation_block re-NULL resets (see _rollbackIndexer). A wrong // or zero delay would wrongly clear legitimately-earned deactivations, so a coin that @@ -341,19 +347,23 @@ class ClientRollback { // delete below drops orphaned-range rows but never re-streams the surviving // mutated row, so the replica keeps the slashed amount and diverges from the // source after a reorg. Mirror the source restore (xchain-indexer - // rollback.js): copy back the EARLIEST orphaned debit's `prev_amount` per row - // This is a pure string copy, byte-identical to the source (no arithmetic). Keys - // only on block_index/stake_action_index, so it ports cleanly (no - // ACTIVATION_DELAY_BLOCKS dependency). + // rollback.js): copy back the HIGHEST orphaned `prev_amount` per row. + // The restored value is a pure string copy, byte-identical to the source (no + // arithmetic on the amount itself). Keys only on block_index/stake_action_index, + // so it ports cleanly (no ACTIVATION_DELAY_BLOCKS dependency). // - // Same-block tiebreak is (execution_index, slash_position), the EXECUTE's - // on-chain action_index plus the emission-loop index, the deterministic total - // order the source uses for contract_emissions, NOT the AUTO_INCREMENT `id`. + // The debits on one stake row form a strictly decreasing chain and the orphaned + // range is a suffix of it, so the maximum `prev_amount` IS the value the row held + // before the first orphaned debit. Position columns alone cannot express that + // order, because a re-entrant nested EXECUTE slashes FIRST under a HIGHER + // action_index than its parent frame. (execution_index, slash_position) stays as + // the tiebreak for numerically equal amounts, NOT the AUTO_INCREMENT `id`. // This MUST byte-match the source indexer or a reorg retracting a block with // ≥2 contract slashes on one stake row restores a divergent amount on the // replica vs the source (stake-weight fork). for(let slashTbl of ['contract_stakes', 'contract_unstakes']){ try { + // await this.db.doQuery( "UPDATE " + slashTbl + " t " + "JOIN contract_slash_debits d ON d.stake_action_index = t.action_index " + @@ -364,13 +374,16 @@ class ClientRollback { " WHERE e.target_table = d.target_table " + " AND e.stake_action_index = d.stake_action_index " + " AND e.block_index >= ? " + - " AND (e.block_index < d.block_index " + - " OR (e.block_index = d.block_index " + - " AND (e.execution_index < d.execution_index " + - " OR (e.execution_index = d.execution_index " + - " AND e.slash_position < d.slash_position)))))", + " AND (CAST(e.prev_amount AS DECIMAL(60,18)) > CAST(d.prev_amount AS DECIMAL(60,18)) " + + " OR (CAST(e.prev_amount AS DECIMAL(60,18)) = CAST(d.prev_amount AS DECIMAL(60,18)) " + + " AND (e.block_index < d.block_index " + + " OR (e.block_index = d.block_index " + + " AND (e.execution_index < d.execution_index " + + " OR (e.execution_index = d.execution_index " + + " AND e.slash_position < d.slash_position)))))))", [slashTbl, block_index, block_index] ); + // } catch(e){ // Schema-gap errors (missing table/column on older replicas) are safe to skip. // All other errors (deadlock, lock-wait, connection drop) must abort the reorg-reset. @@ -459,11 +472,21 @@ class ClientRollback { // MATERIALIZATION block (reward_derive_block_index) is itself inside the // orphaned range is NOT restored: its earn-block survives, but a replay to // reorg_block-1 never derived it, so restoring it would mint an orphan. + // round_qualifier rides the pre-image like every other key column (the twin + // at xchain-indexer/src/rollback.js carries it in both the column list and the + // projection): it is part of the reward's UNIQUE identity, snapshot_block for + // the archive leg whose round_reference is a reissuable hub counter. Dropped, + // the restore re-INSERTs the loser under the schema default 0, a DIFFERENT row + // from the one the reconcile deleted, which either collides with whatever + // legacy row already holds that key and is swallowed by INSERT IGNORE, or + // lands as a wrong-identity duplicate. Either way the real loser stays + // unrestored and the replica forks SUM(validator_rewards) from the source. try { await this.db.doQuery( "INSERT IGNORE INTO validator_rewards " + - "(source_id, signing_pubkey_id, reward_type, round_reference, amount, block_index, derive_block_index) " + + "(source_id, signing_pubkey_id, reward_type, round_reference, round_qualifier, amount, block_index, derive_block_index) " + "SELECT d.source_id, d.signing_pubkey_id, d.reward_type, d.round_reference, " + + " d.round_qualifier, " + " d.amount, d.reward_block_index, d.reward_derive_block_index " + " FROM anchor_reward_reconcile_log d " + " WHERE d.block_index >= ? AND d.reward_block_index < ? " + @@ -644,11 +667,16 @@ class ClientRollback { // 'unverified', the conservative re-verification state. Runs BEFORE the delete. if(firstActionIndex !== null){ try { + // Author scope, flag-day gated and INERT on every network today; mirror of + // the source indexer's term. Rationale and the arming precondition live in + // archive_rollback_author_scope_activation.js. + let authorScope = archiveAuthorScopeJoin(block_index, this.network); await this.db.doQuery( "UPDATE anchor_actions p " + "JOIN index_statuses ps ON ps.id = p.status_id AND ps.status = 'invalid_archive' " + "JOIN anchor_actions c ON c.version = 2 AND c.match_batch_seq = p.match_batch_seq AND c.action_index >= ? " + "JOIN index_statuses cs ON cs.id = c.status_id AND cs.status = 'valid' " + + authorScope + "JOIN index_statuses us ON us.status = 'unverified' " + "SET p.status_id = us.id " + "WHERE p.version " + ARCHIVE_HEAD_VERSIONS_SQL + " AND p.action_index < ?", diff --git a/src/ClientSync.js b/src/ClientSync.js index 65ed705..38a4a9d 100644 --- a/src/ClientSync.js +++ b/src/ClientSync.js @@ -234,6 +234,18 @@ class ClientSync { // Count of sources that agreed on the most recently applied block (for /status). this._lastSourcesAgreeing = null; + // Upstream replication evidence, per CONNECTED source index. A server's status + // event carries its own DB tip (source_block_height) and the verdict its + // ServerPoller reached on its own database (replica_stale, replica_seconds_behind). + // Discarding it left this follower publishing lag_blocks 0 against a server whose + // SQL replica had stopped applying hours earlier: both of that server's heights + // freeze together, so we catch up to the frozen tip while the heartbeats keep + // source_height_stale false. Rewritten on EVERY status event, not only when the + // height advances, because a stalled upstream is exactly the case where it never + // advances again; dropped when the socket closes, so a disconnected source's last + // verdict is never mistaken for current evidence. + this._upstreamStatus = new Map(); // sourceIndex -> { sourceHeight, stale, secondsBehind } + // Applied-block heartbeat state. After committing each live block we report // our applied height back to the source servers so operators can observe // this validator's lag via the server's /status endpoint. Debounced to avoid @@ -2203,6 +2215,10 @@ class ClientSync { ws.on('close', () => { console.log('WebSocket disconnected from ' + source); + // A source we are no longer connected to is not evidence about anything. + // Keeping its last verdict would let a disconnected server go on certifying + // its own freshness, which is the shape of the bug this map exists to fix. + this._upstreamStatus.delete(sourceIndex); this._scheduleReconnect(source, sourceIndex); }); @@ -2261,6 +2277,9 @@ class ClientSync { (this.lastKnownServerBlock === null || event.block_height > this.lastKnownServerBlock)){ this.lastKnownServerBlock = event.block_height; } + // Keep the server's own replication verdict. Unconditional: the height guard + // above is exactly what a stalled upstream stops satisfying. + this._recordUpstreamStatus(sourceIndex, event); // Check for gaps on status update. Use a strict '>' (not '>='): a // server exactly one block ahead is the normal steady state (that // next block arrives over the live WS stream), so only a shortfall of @@ -2664,6 +2683,47 @@ class ClientSync { return (Date.now() - this._lastWsEventAt) > this.config['CLIENT_SOURCE_STALE_MS']; } + // Record one source's self-reported replication evidence off its status event. + // A server that predates the fields reports nothing, which stays UNKNOWN here + // rather than being read as healthy: `undefined` is not `false`. + _recordUpstreamStatus(sourceIndex, event){ + let stale = (typeof event.replica_stale === 'boolean') ? event.replica_stale : null; + let secondsBehind = (typeof event.replica_seconds_behind === 'number' + && Number.isFinite(event.replica_seconds_behind)) + ? event.replica_seconds_behind : null; + let sourceHeight = (typeof event.source_block_height === 'number' + && Number.isFinite(event.source_block_height)) + ? event.source_block_height : null; + this._upstreamStatus.set(sourceIndex, { sourceHeight, stale, secondsBehind }); + } + + // The upstream replication verdict this follower's own /status must carry, so + // lag_blocks is never read as a clean bill of health for a source whose database + // said otherwise. Separate from the transport-liveness signal isSourceHeightStale: + // that one says whether the server is still SPEAKING, this one says whether what it + // said certifies its data. + // + // stale is TRI-STATE, and the third state is the point: null means no connected + // source has reported the field at all (nothing heard yet, or an older server), which + // is unknown and must not read as fresh. true means at least one connected source + // reported its own DB stale; the follower applies from all of them, so any stale + // source qualifies the row. secondsBehind is the WORST reported lag and sourceHeight + // the HIGHEST reported upstream DB tip, which is what makes broadcaster-vs-source lag + // visible on the follower. + getUpstreamReplicaState(){ + let stale = null, secondsBehind = null, sourceHeight = null; + for(let [sourceIndex, seen] of this._upstreamStatus){ + if(this._evictedSources.has(sourceIndex)) continue; + if(seen.stale === true) stale = true; + else if(seen.stale === false && stale === null) stale = false; + if(seen.secondsBehind !== null && (secondsBehind === null || seen.secondsBehind > secondsBehind)) + secondsBehind = seen.secondsBehind; + if(seen.sourceHeight !== null && (sourceHeight === null || seen.sourceHeight > sourceHeight)) + sourceHeight = seen.sourceHeight; + } + return { stale, secondsBehind, sourceHeight }; + } + _safeParse(s){ try { return JSON.parse(s); } catch(e){ return s; } } // Durable-marker key for this client's truncation join floor. Namespaced by diff --git a/src/HubClient.js b/src/HubClient.js index cca5298..bf8f1cc 100644 --- a/src/HubClient.js +++ b/src/HubClient.js @@ -100,10 +100,15 @@ class HubClient { // getallconfigs sits in the hub's sensitive-read tier (its response carries DB // credentials) and 401s without it once the hub sets a key, while methods that do // not need it ignore it, so sending unconditionally is safe. + // HUB_CONFIG_SECRETS_API_KEY wins when set: the hub can split the credential + // tier (getallconfigs with include_secrets, the only way this client gets the + // replication sources' DB passwords) onto a key of its own, and one request + // carries one x-api-key header. Unset, the bulk key covers both tiers. async _call(data, timeout = 5000){ this.lastFailures = []; let headers = {}; - if(process.env.HUB_API_KEY) headers['x-api-key'] = process.env.HUB_API_KEY; + let hubKey = process.env.HUB_CONFIG_SECRETS_API_KEY || process.env.HUB_API_KEY; + if(hubKey) headers['x-api-key'] = hubKey; for(let i = 0; i < this.urls.length; i++){ let idx = (this._lastGoodIdx + i) % this.urls.length; let url = this.urls[idx]; @@ -126,6 +131,34 @@ class HubClient { return result !== null; } + // Params for every getallconfigs call this client makes. + // + // include_secrets is NOT optional for sync: _extractDbConfigs turns this tree + // into the replication sources' connection details (db_user/db_pass per + // coin/network), so a redacted response hands every source the literal + // "[redacted]" as its password. The hub redacts secret-bearing params by + // default and serves them only to a caller that asks and is authorized + // (HUB_CONFIG_SECRETS_API_KEY when the hub sets one, the bulk HUB_API_KEY + // otherwise). Older hubs ignore an unknown param and return the full tree, so + // this is safe to deploy ahead of the hub change (and must be: a sync without + // the flag against a redacting hub loses its DB passwords). + _configParams(cursor){ + return { since_updated_at: cursor, include_secrets: true }; + } + + // One warning, not one per poll: a redacted response means this service asked + // for credentials and is not authorized for them, so every DB pool built from + // the result will fail to authenticate several layers away from the cause. + _warnIfRedacted(result){ + if(!result || typeof result !== 'object' || result.secrets_redacted !== true) return; + if(this._warnedRedacted) return; + this._warnedRedacted = true; + console.error('Hub served a CREDENTIAL-REDACTED config tree (' + (result.redacted_params || 0) + + ' params withheld): this service asked for secrets but is not authorized for them. Set ' + + 'HUB_API_KEY (or the hub\'s HUB_CONFIG_SECRETS_API_KEY) to the value the hub expects; until ' + + 'then every replication source built from this config will fail to authenticate.'); + } + // Returns the full nested tree { coin: { network: { module: { param: value } } } } // whatever shape the hub answered in; see _applyConfigResult for the version // handling. Sync discovers DBs at startup, so this.lastSeq is tracked for @@ -138,7 +171,7 @@ class HubClient { method: 'getallconfigs', // Echo the high-water mark so the hub returns only rows changed since // our last poll; 0 requests the full tree (initial fetch / old hub). - params: { since_updated_at: this.lastWatermark }, + params: this._configParams(this.lastWatermark), id: 1 }, 10000); // _call returns null when every endpoint failed; preserve that signal so @@ -156,12 +189,13 @@ class HubClient { result = await this._call({ jsonrpc: '2.0', method: 'getallconfigs', - params: { since_updated_at: 0 }, + params: this._configParams(0), id: 1 }, 10000); if(result === null) return null; } + this._warnIfRedacted(result); this.configs = this._applyConfigResult(result); // Bind the (possibly advanced) cursor to the endpoint that answered. this._watermarkEndpointIdx = this._lastGoodIdx; diff --git a/src/ServerPoller.js b/src/ServerPoller.js index f93c570..d5c426e 100644 --- a/src/ServerPoller.js +++ b/src/ServerPoller.js @@ -437,9 +437,19 @@ class ServerPoller { console.log('Synced block ' + this.lastPolledBlock + ' for ' + this.chain + '/' + this.network + '/' + this.dbType); } - await this._updateStatus(streamTo); } + // Refresh status on EVERY poll, not only when blocks advanced. The replication + // verdict _updateStatus carries is the one field whose failure mode also stops + // block advancement: a native SQL replica that stops applying freezes the served + // tip, so a refresh gated on blocksProcessed > 0 never runs again and REST and the + // periodic WebSocket status keep republishing the last healthy replica_stale:false + // and a zero lag indefinitely, straight past SYNC_REPLICA_MAX_LAG_S. Idle polls + // cost no extra source read (streamTo is the tip this poll already read) and no + // extra WebSocket traffic (updateStatus only writes the broadcaster's map; the + // status push is api.js's own timer). + await this._updateStatus(streamTo); + return blocksProcessed; } @@ -750,9 +760,14 @@ class ServerPoller { let redriven = await collectRedrivenValidatorRewards(this.db, block_index, block_index, conn); if(redriven.length > 0){ let existing = payload.data['validator_rewards'] || []; - let seen = new Set(existing.map(r => r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference)); + // The dedup key is the FULL five-column identity. round_qualifier is the + // archive leg's snapshot_block, and its round_reference (MATCH_BATCH_SEQ) is + // a dense hub counter a rebase reissues, so two distinct archive rewards can + // share the four older columns; on the narrower key the second is treated as + // a duplicate and dropped from the payload before it ever reaches a replica. + let seen = new Set(existing.map(r => r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference + ':' + r.round_qualifier)); for(let r of redriven){ - let k = r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference; + let k = r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference + ':' + r.round_qualifier; if(!seen.has(k)){ seen.add(k); existing.push(r); } } payload.data['validator_rewards'] = existing; @@ -776,9 +791,12 @@ class ServerPoller { let derived = await collectDerivedAnchorRewards(this.db, block_index, block_index, conn); if(derived.length > 0){ let existing = payload.data['validator_rewards'] || []; - let seen = new Set(existing.map(r => r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference)); + // Five-column identity, same reason as the redriven merge above: the + // archive leg is exactly the channel that can present two distinct rewards + // differing only in round_qualifier. + let seen = new Set(existing.map(r => r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference + ':' + r.round_qualifier)); for(let r of derived){ - let k = r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference; + let k = r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference + ':' + r.round_qualifier; if(!seen.has(k)){ seen.add(k); existing.push(r); } } payload.data['validator_rewards'] = existing; diff --git a/src/SnapshotBuilder.js b/src/SnapshotBuilder.js index ef80ded..d862b7c 100644 --- a/src/SnapshotBuilder.js +++ b/src/SnapshotBuilder.js @@ -721,7 +721,11 @@ class SnapshotBuilder { } } else { if(indexerBlockScoped.has(table)){ - rows = await db.doQuery("SELECT * FROM `" + table + "` WHERE block_index >= ? ORDER BY block_index", [sinceBlock], conn); + // Scope by the registry's blockKey, never the literal: a table + // keyed by another column raises errno 1054, which this loop's + // catch tolerates as an older source schema and skips forever. + let key = tableLifecycle.blockKey(table); + rows = await db.doQuery("SELECT * FROM `" + table + "` WHERE " + key + " >= ? ORDER BY " + key, [sinceBlock], conn); } else if(indexerFullDump.has(table)){ if(skipLookups && lookupSet.has(table)) continue; rows = await db.doQuery("SELECT * FROM `" + table + "`", null, conn); @@ -829,9 +833,13 @@ class SnapshotBuilder { let redriven = await collectRedrivenValidatorRewards(db, sinceBlock, lastBlock, conn); if(redriven.length > 0){ rows = rows || []; - let seen = new Set(rows.map(r => r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference)); + // The dedup key is the FULL five-column identity: the archive leg's + // round_reference (MATCH_BATCH_SEQ) is a dense hub counter a rebase + // reissues, so two distinct archive rewards can share the four older + // columns and the narrower key silently drops one from the payload. + let seen = new Set(rows.map(r => r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference + ':' + r.round_qualifier)); for(let r of redriven){ - let k = r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference; + let k = r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference + ':' + r.round_qualifier; if(!seen.has(k)){ seen.add(k); rows.push(r); } } } @@ -852,9 +860,11 @@ class SnapshotBuilder { let derived = await collectDerivedAnchorRewards(db, sinceBlock, lastBlock, conn); if(derived.length > 0){ rows = rows || []; - let seen = new Set(rows.map(r => r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference)); + // Five-column identity, same reason as the redriven merge above; + // the derived-anchor channel is where the archive leg arrives. + let seen = new Set(rows.map(r => r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference + ':' + r.round_qualifier)); for(let r of derived){ - let k = r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference; + let k = r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference + ':' + r.round_qualifier; if(!seen.has(k)){ seen.add(k); rows.push(r); } } } diff --git a/src/SyncService.js b/src/SyncService.js index 678472b..2270ca8 100644 --- a/src/SyncService.js +++ b/src/SyncService.js @@ -32,6 +32,7 @@ const ClientRollback = require('./ClientRollback'); const HashVerifier = require('./HashVerifier'); const stateCommitment = require('./stateCommitment'); const { assertBootstrapDepthChains } = require('./config'); +const { assertPinnedEnvOverrides } = require('./pinnedValidators'); const Utility = require('./utility'); class SyncService { @@ -266,6 +267,13 @@ class SyncService { // both paths self-heal. Idempotent, so the double-call on the // direct-DB path is a cheap no-op. await db.ensureReplicaSecondaryIndexes(); + // Same again for the raw-wire-field charset widen: neither self-heal above + // retypes an existing column, so a replica bootstrapped before the + // 2026-09-02 indexer migration keeps utf8mb3 on contracts.code and the + // grammar-constrained fields and halts on the first 4-byte character the + // widened origin accepts. Idempotent, so the double-call on the direct-DB + // path is a cheap no-op. + await db.ensureReplicaUtf8mb4Columns(); // Fail closed on collation drift in the columns the stake-weight // snapshot orders on. The follower rebuilds stakes_root from the // byte-mirrored _cappedStakeWeightsSql, whose window caps truncate on @@ -308,6 +316,12 @@ class SyncService { if(this.config['SYNC_MODE'] !== 'server' && !this._bootstrapDepthChecked && this.databases.size > 0){ this._bootstrapDepthChecked = true; assertBootstrapDepthChains(this.config, this.getChains()); + // REFUSE a present-but-invalid CHECKPOINT_VALIDATORS_*/CHECKPOINT_SEED_* value on + // the same pass, and for the same reason: it is not inert either. It resolves to + // the null an ABSENT override resolves to, so _verifyCheckpointQuorum skips the + // anchor on a replica whose operator armed VERIFY_CHECKPOINT_QUORUM. Client mode + // only (a server reads no pinned set) and before any ClientSync is constructed. + assertPinnedEnvOverrides(); } if(newChains.length > 0){ @@ -376,7 +390,7 @@ class SyncService { if(this.clientSyncs.has(key)) return; let applier = new ClientApplier(db, this.util, cfg.coin, cfg.network); - let rollback = new ClientRollback(db, this.util, cfg.coin); + let rollback = new ClientRollback(db, this.util, cfg.coin, cfg.network); let sync = new ClientSync(cfg.coin, cfg.network, db, applier, rollback, this.hashVerifier, this.config, this.util); this.clientSyncs.set(key, sync); @@ -494,6 +508,11 @@ class SyncService { return { lastKnownServerBlock: sync ? sync.lastKnownServerBlock : null, sourceHeightStale: sync ? sync.isSourceHeightStale() : null, + // The upstream's OWN replication verdict, relayed on its status events. + // Tri-state stale: null is unknown, never fresh. + upstreamReplica: (sync && typeof sync.getUpstreamReplicaState === 'function') + ? sync.getUpstreamReplicaState() + : { stale: null, secondsBehind: null, sourceHeight: null }, halted: sync ? sync.isHalted() : false, haltInfo: (sync && sync.isHalted()) ? sync.getHaltInfo() : null, truncated: sync ? sync.isTruncated() : false, diff --git a/src/api.js b/src/api.js index 39e56f0..a6dc90c 100644 --- a/src/api.js +++ b/src/api.js @@ -315,6 +315,27 @@ async function buildStatusRow(syncService, db, dbType, chain, network){ // an operator the lag figure is computed against a source height we have not // heard confirmed recently (null = no live event seen yet, staleness unknown). row.source_height_stale = clientState.sourceHeightStale; + // The upstream server's OWN replication verdict, relayed on its status events and + // published under distinct names: `replica_stale` on a SERVER row is a claim about + // that node's own database, and reusing the name here would say something else + // under the same key. source_height_stale above is transport liveness (is the + // server still speaking); this is data authority (does what it said certify the + // heights). Both are needed: a server whose SQL replica stopped applying keeps + // heart-beating, so its follower catches up to the frozen tip and reports + // lag_blocks 0 with source_height_stale false. + // + // upstream_replica_stale is TRI-STATE. null means no connected source reported it + // (nothing heard yet, or a server older than the field) and must not be read as + // healthy; true means at least one connected source called its own database stale. + // upstream_source_height is the upstream's own DB tip, which is what makes its + // broadcaster-versus-source-database lag visible from here. + // Absent on a caller that predates the field (a test double, an older + // SyncService): unknown, which is the null tri-state, never a fresh verdict. + let upstream = clientState.upstreamReplica || {}; + row.upstream_replica_stale = (upstream.stale === true || upstream.stale === false) + ? upstream.stale : null; + row.upstream_replica_seconds_behind = (upstream.secondsBehind != null) ? upstream.secondsBehind : null; + row.upstream_source_height = (upstream.sourceHeight != null) ? upstream.sourceHeight : null; // Consensus-divergence halt: a halted client has STOPPED applying and // requires operator clearance. Surfaced so the dashboard monitor and // peers see a forked/Byzantine validator immediately. diff --git a/src/archive_rollback_author_scope_activation.js b/src/archive_rollback_author_scope_activation.js new file mode 100644 index 0000000..c8dba17 --- /dev/null +++ b/src/archive_rollback_author_scope_activation.js @@ -0,0 +1,105 @@ +/********************************************************************* + * + * Copyright © 2025-2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC - https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * + * Publisher-scoped archive rollback reset flag-day. + * + * THE PROBLEM. The reorg reset that clears a wedged 'invalid_archive' stamp off + * a surviving archive head self-joins the head to an orphaned v2 chunk on + * MATCH_BATCH_SEQ alone. That seq is not unique, and once archive batches are + * publisher-scoped (archive_batch_author_activation.js) two publishers can hold + * two live batches under one seq, each with its own head and its own chunks + * stored 'valid'. One publisher's orphaned chunk then resets the OTHER + * publisher's head, whose own batch is intact below the reorg and whose stamp a + * from-genesis replay still re-derives. + * + * THE RULE. Above the threshold the orphaned chunk must be authored by the same + * address as the head it resets, resolved through actions.source_id (the + * authoritative source for auth, never re-derived from the transaction). Inner + * joins throughout, so a row whose action linkage cannot be resolved is excluded + * and no reset fires: fail-closed by shape, matching the archive read path. + * + * ARMING PRECONDITION. Author equality is the exact batch key ONLY at or above + * that network's ARCHIVE_BATCH_AUTHOR height, where both stamping paths in + * anchor.js scope the chunk set to the head's own author. Below it the chunk set + * is scoped to the CANONICAL head's author, so a second head stamped by the + * head-side gate has a different author than the chunks that stamped it and the + * term would suppress a reset that is genuinely owed. Never arm a network here + * below its ARCHIVE_BATCH_AUTHOR height; a parity test pins the ordering. + * + * Residual the arming height must clear: a batch whose canonical head landed + * BELOW that network's ARCHIVE_BATCH_AUTHOR height keeps the legacy canonical + * head rule forever, so a squatting second head on such a batch still resolves + * to a foreign author. Arm at a height with no such batch still taking chunks. + * + * WHY GATED AT ALL. The reset is not a hash preimage (the class-6 anchor_invalid + * projection covers the stamp itself), and a from-genesis replay never runs + * rollback, so historical replay is byte-identical either way. The gate exists + * because the reset writes state the class-6 preimage READS, so two nodes that + * reorged under different rules answer a later recompute differently: the + * switchover wants one coordinated height per network, not a deploy race. + * + * SHIPS INERT ON EVERY NETWORK. Arming is a one-line edit here per network, and + * a replica arms only once SyncService hands ClientRollback its network: an + * omitted network reads as inactive, which is correct while every threshold is + * inert and wrong the moment one is not. A guard test in xchain-sync fails if a + * threshold is armed while that wiring is still optional. + * + * KEYED ON THE ROLLBACK'S OWN TARGET BLOCK, per network, on the DOGE scale the + * ANCHOR actions land on. The orphaned chunk always sits at or above that + * height, so the rule that judges it is the rule in force where it landed. + * + ********************************************************************/ + +'use strict'; + +// Per-network activation, interpreted against the block index a rollback targets. +// Every key is an INERT placeholder: no live network can reach it, so the reset +// keeps its deployed unscoped shape until an operator pins a real height. +const ARCHIVE_ROLLBACK_AUTHOR_SCOPE_ACTIVATION = { + mainnet: 9999999999, // INERT placeholder; pin on the coordinated activation train, never below ARCHIVE_BATCH_AUTHOR + testnet: 9999999999, // INERT placeholder; testnet arms ARCHIVE_BATCH_AUTHOR at 0, so any height here is legal once the wiring lands + regtest: 9999999999, // INERT placeholder +}; + +// The joins that bind an orphaned chunk to its own head's author. Spliced into +// the reset UPDATE by both the source indexer and the replica so the two cannot +// drift; `c` is the orphaned chunk and `p` the surviving head, as named there. +const ARCHIVE_AUTHOR_SCOPE_JOIN_SQL = + 'JOIN actions pact ON pact.action_index = p.action_index ' + + 'JOIN index_addresses padr ON padr.id = pact.source_id ' + + 'JOIN actions cact ON cact.action_index = c.action_index ' + + 'JOIN index_addresses cadr ON cadr.id = cact.source_id AND cadr.address = padr.address '; + +// Whether the reset is publisher-scoped for a rollback targeting `blockIndex` on +// `network`. A non-numeric height, an unknown network or an omitted one -> false +// (legacy unscoped reset, deployed behavior kept). +function isArchiveRollbackAuthorScopeActive(blockIndex, network){ + let b = parseInt(blockIndex); + if(!Number.isFinite(b)) return false; + let threshold = ARCHIVE_ROLLBACK_AUTHOR_SCOPE_ACTIVATION[network]; + if(threshold === undefined) return false; + return b >= threshold; +} + +// The join text to splice, or an empty string below the threshold. +function archiveAuthorScopeJoin(blockIndex, network){ + return isArchiveRollbackAuthorScopeActive(blockIndex, network) ? ARCHIVE_AUTHOR_SCOPE_JOIN_SQL : ''; +} + +module.exports = { + ARCHIVE_ROLLBACK_AUTHOR_SCOPE_ACTIVATION, + ARCHIVE_AUTHOR_SCOPE_JOIN_SQL, + isArchiveRollbackAuthorScopeActive, + archiveAuthorScopeJoin +}; diff --git a/src/checkpoint_commitment_activation.js b/src/checkpoint_commitment_activation.js index d7fb688..87dd765 100644 --- a/src/checkpoint_commitment_activation.js +++ b/src/checkpoint_commitment_activation.js @@ -25,8 +25,21 @@ * UNLIKE the Phase 1 state_commitment_activation (which gates on each chain's OWN * local block_index), this gates on the BTC-anchored `snapshot_block` carried by * every checkpoint canonical, exactly like stake_weighted_quorum / equivocation_ - * header, so the hub and the BTC/LTC/DOGE indexers all flip the SIGNED shape on the - * same anchor. + * header, so every peer that evaluates it (the hub signer, the SDK/explorer/sync + * verifiers) flips the SIGNED shape on the same anchor. + * + * WHY THIS MODULE HAS NO CALL SITE IN THE INDEXER. Nothing under xchain-indexer/src + * calls isCheckpointCommitmentActive, because the indexer's own checkpoint-section + * canonical (src/actions/anchor.js, FORMAT 0) appends the root suffix + * UNCONDITIONALLY, alone among the four builders, so there is no height for this side + * to test. Parity rests instead on the producer-side invariant recorded at that call + * site (no bundle carries a section whose own snapshot block is below this height), + * which is a deployment fact rather than a code property and is fail-closed when it + * breaks: a section the hub signed rootless fails every section signature and the + * whole bundle is refused. The file stays here as the indexer's REGISTRATION of the + * consensus parameter, pinned to the canonical map by + * test/unit/activationConstantsParity.test.js and inventoried by + * src/consensus_rules_digest.js. * * LOCAL COPY of the canonical map in xchain-documentation/protocol/constants.js, * kept byte-equal by the cross-service regression suite (a divergence forks the diff --git a/src/coins/BTC.js b/src/coins/BTC.js index f36c86f..c846405 100644 --- a/src/coins/BTC.js +++ b/src/coins/BTC.js @@ -284,6 +284,26 @@ module.exports = { OWNERSHIP_ESCROW: 50000, AIRDROP_PER_RECIPIENT: 100, DIVIDEND_PER_RECIPIENT: 100, + // SWEEP / CALLBACK, priced on the unified schedule from the + // UNIFIED_FEES_SWEEP_CALLBACK flag day. The legacy flat per-DB-hit fee prices a + // small action BELOW the dust threshold of a native-fee chain (LTC/DOGE, where a + // missing fee output is rejected outright rather than falling back to an XCHAIN + // balance debit), so the fee output cannot be created and the action cannot be + // submitted at all: a Litecoin SWEEP needs ~273 DB hits before it clears LTC's + // 5460-satoshi floor at LTC $100 / XCHAIN $2. + // + // The BASE keys are what close that: gas is what buys the output, so the SMALLEST + // possible SWEEP or CALLBACK has to buy an above-dust one on its own. The floor a + // chain demands is dust_sats * COIN_USD / (1000 * XCHAIN_USD) gas units, so 5000 gas + // (0.05 XCHAIN) clears Litecoin while COIN/XCHAIN stays under ~915 and Dogecoin + // while it stays under ~50, both far outside any plausible band. The PER_ITEM keys + // hold the marginal cost at AIRDROP/DIVIDEND per-recipient parity; a SWEEP item is + // one swept balance, one closed order/swap/dispenser escrow, or one transferred + // ownership. + SWEEP_BASE: 5000, + SWEEP_PER_ITEM: 100, + CALLBACK_BASE: 5000, + CALLBACK_PER_RECIPIENT: 100, // BET (parimutuel betting, spec decision F): feed creation is duration- // metered like ORDER/SWAP/DISPENSER expiration (same free window via // UNIFIED_EXPIRATION_FEE_FREE_DAYS) but under its OWN per-day key so the diff --git a/src/coins/DOGE.js b/src/coins/DOGE.js index 71bd5d5..0ec1a85 100644 --- a/src/coins/DOGE.js +++ b/src/coins/DOGE.js @@ -218,6 +218,26 @@ module.exports = { OWNERSHIP_ESCROW: 50000, AIRDROP_PER_RECIPIENT: 100, DIVIDEND_PER_RECIPIENT: 100, + // SWEEP / CALLBACK, priced on the unified schedule from the + // UNIFIED_FEES_SWEEP_CALLBACK flag day. The legacy flat per-DB-hit fee prices a + // small action BELOW the dust threshold of a native-fee chain (LTC/DOGE, where a + // missing fee output is rejected outright rather than falling back to an XCHAIN + // balance debit), so the fee output cannot be created and the action cannot be + // submitted at all: a Litecoin SWEEP needs ~273 DB hits before it clears LTC's + // 5460-satoshi floor at LTC $100 / XCHAIN $2. + // + // The BASE keys are what close that: gas is what buys the output, so the SMALLEST + // possible SWEEP or CALLBACK has to buy an above-dust one on its own. The floor a + // chain demands is dust_sats * COIN_USD / (1000 * XCHAIN_USD) gas units, so 5000 gas + // (0.05 XCHAIN) clears Litecoin while COIN/XCHAIN stays under ~915 and Dogecoin + // while it stays under ~50, both far outside any plausible band. The PER_ITEM keys + // hold the marginal cost at AIRDROP/DIVIDEND per-recipient parity; a SWEEP item is + // one swept balance, one closed order/swap/dispenser escrow, or one transferred + // ownership. + SWEEP_BASE: 5000, + SWEEP_PER_ITEM: 100, + CALLBACK_BASE: 5000, + CALLBACK_PER_RECIPIENT: 100, // BET (parimutuel betting, spec decision F): feed creation is duration- // metered like ORDER/SWAP/DISPENSER expiration (same free window via // UNIFIED_EXPIRATION_FEE_FREE_DAYS) but under its OWN per-day key so the diff --git a/src/coins/LTC.js b/src/coins/LTC.js index f02d1e3..3f26fe9 100644 --- a/src/coins/LTC.js +++ b/src/coins/LTC.js @@ -213,6 +213,26 @@ module.exports = { OWNERSHIP_ESCROW: 50000, AIRDROP_PER_RECIPIENT: 100, DIVIDEND_PER_RECIPIENT: 100, + // SWEEP / CALLBACK, priced on the unified schedule from the + // UNIFIED_FEES_SWEEP_CALLBACK flag day. The legacy flat per-DB-hit fee prices a + // small action BELOW the dust threshold of a native-fee chain (LTC/DOGE, where a + // missing fee output is rejected outright rather than falling back to an XCHAIN + // balance debit), so the fee output cannot be created and the action cannot be + // submitted at all: a Litecoin SWEEP needs ~273 DB hits before it clears LTC's + // 5460-satoshi floor at LTC $100 / XCHAIN $2. + // + // The BASE keys are what close that: gas is what buys the output, so the SMALLEST + // possible SWEEP or CALLBACK has to buy an above-dust one on its own. The floor a + // chain demands is dust_sats * COIN_USD / (1000 * XCHAIN_USD) gas units, so 5000 gas + // (0.05 XCHAIN) clears Litecoin while COIN/XCHAIN stays under ~915 and Dogecoin + // while it stays under ~50, both far outside any plausible band. The PER_ITEM keys + // hold the marginal cost at AIRDROP/DIVIDEND per-recipient parity; a SWEEP item is + // one swept balance, one closed order/swap/dispenser escrow, or one transferred + // ownership. + SWEEP_BASE: 5000, + SWEEP_PER_ITEM: 100, + CALLBACK_BASE: 5000, + CALLBACK_PER_RECIPIENT: 100, // BET (parimutuel betting, spec decision F): feed creation is duration- // metered like ORDER/SWAP/DISPENSER expiration (same free window via // UNIFIED_EXPIRATION_FEE_FREE_DAYS) but under its OWN per-day key so the diff --git a/src/coins/consensus_pin.js b/src/coins/consensus_pin.js index 50ce2d0..53a2113 100644 --- a/src/coins/consensus_pin.js +++ b/src/coins/consensus_pin.js @@ -66,16 +66,25 @@ module.exports = { // so the public testnet announces with zero pre-announcement test actions. // Same one-wave rule as every regeneration above. Regtest and mainnet are // untouched and were re-verified as unchanged by this edit. + // REGENERATED 2026-09-01: GAS_SCHEDULE gains SWEEP_BASE, + // SWEEP_PER_ITEM, CALLBACK_BASE and CALLBACK_PER_RECIPIENT, the unified prices + // SWEEP and CALLBACK move onto at the UNIFIED_FEES_SWEEP_CALLBACK flag day (both + // networks UNARMED; regtest genesis-active). GAS_SCHEDULE is hashed whole by + // consensusSubset(), so ADDING a key moves every hash even while the flag that + // reads it is unarmed, and the same one-wave rollout rule as every regeneration + // above applies in full: every service bundling these must ship the new values + // together, and a straggler fail-closes on verifyConsensusPin() at boot rather + // than forking. Mainnet stays null (Phase 6 arms it). testnet: { - BTC: 'f6589c6b88dc930db05998070ef0b73743f58623a0d23fbc30fdb158c49d1427', - LTC: '9faf066a1470be2486d8a2cd121548ca02de1397d0678a2ab8dc0e712ebfa8fd', - DOGE: '2991d7e7caf2b212de959dd5831ac1477e0b13da95ac1ed8c2b43e2704732439', + BTC: 'd3c66a4fb288b2666a2a4fad85200bbeac162bb36fed8a3eddcfc7b2d4d48070', + LTC: 'ae94a951a838e64f9c36e503b978d9b9ad5ea74f7b443465baaabca8f675ea0d', + DOGE: 'b90aec4381b0ad32caba078706c8fb244cbe267390e41668fa063d9e64fb60e6', }, regtest: { - BTC: '24e6a363e5a36285574dea357328a997fdee5762ef812d8947eacf69c51afc24', - LTC: '5ad03b383d873d309640e75dfefa2787a5806cb8a84ee46f4cc7fb25ca7f808b', - DOGE: '019220a461e34c99fcf5cbf107673f13d3f2a57d2a20e16a0323ed44c81edd11', + BTC: '29976bd33cad1842320c57acdc849250646adea765f70a0ae5dad3f201f7d5d7', + LTC: 'bca62db9f59a6f7566620b086380c10fffac08dabe99f00a4fcc7cd038e46146', + DOGE: '816632e9f6647e726042282c37789ae8d924e8d4a1b2995ddde8d6a54a0bba54', }, }, }; diff --git a/src/db.js b/src/db.js index 2c4fc80..d0cf331 100644 --- a/src/db.js +++ b/src/db.js @@ -34,6 +34,8 @@ const poolSizing = require('./poolSizing'); const swqCap = require('./swq_source_cap_activation'); const { isStateKeyBinCollationActive } = require('./state_key_collation_activation'); const stakeWeightCollation = require('./stake_weight_collation_activation'); +const utf8mb4Columns = require('./utf8mb4Columns'); +const lifecycle = require('./tableLifecycle'); // Guard for the few queries that must interpolate a table name into a // backtick-quoted identifier (COUNT(*), pagination, TRUNCATE). Parameter @@ -502,6 +504,11 @@ class Database { // tables are present (idempotent; safe on snapshot-bootstrapped replicas). await this.ensureReplicaSecondaryIndexes(); + // Neither of those retypes an existing column, so a replica built before the + // 2026-09-02 raw-wire-field widen keeps utf8mb3 on those columns and halts on the + // first 4-byte character the widened origin accepts. Converge them here. + await this.ensureReplicaUtf8mb4Columns(); + if(columnFailures.length){ let err = new Error('Schema replication into ' + this.dbName + ' left columns missing: ' + columnFailures.map(f => f.table + ' (errno ' + f.errno + ')').join(', ')); @@ -1008,6 +1015,76 @@ class Database { } } + // Widen the raw-wire-field columns to utf8mb4 on an already-existing replica. + // + // The source indexer converges through a dated migration + // (2026-09-02-utf8mb4-raw-wire-fields.sql and its NOT NULL pair). sync runs no + // migrations: a replica's tables are copied from the source's SHOW CREATE TABLE at + // bootstrap, and addMissingColumns only ever ADDs a column, never retypes one. So a + // replica built before that migration keeps utf8mb3 on these columns forever - and the + // moment the widened ORIGIN accepts a 4-byte character (a contract whose source carries + // an emoji, an EXECUTE method name, a VOTE quorum), every aged follower halts applying + // that block with errno 1366 while the source runs on. This is the replica half of that + // migration, and it has to land with it: an origin that can hold the bytes and a + // follower that cannot is a fleet-wide halt with no schema error anywhere upstream. + // + // src/utf8mb4Columns.js is the byte-identical twin of the indexer's copy, so the two + // sides cannot disagree about which columns are in the set or what shape they take. + // + // Idempotent and additive: a column already utf8mb4 is skipped (so a snapshot-bootstrapped + // replica pays one information_schema read per table and nothing else), an absent table or + // column is skipped, and the widen only ever grows the accepted byte domain - utf8mb3 is a + // strict subset of utf8mb4, so no stored value is rewritten and no row is lost. The + // per-table clauses ride ONE ALTER because each ALTER is a COPY rebuild under a metadata + // lock. indexer replicas only (decoder replicas hold none of these tables). + async ensureReplicaUtf8mb4Columns(){ + if(this.dbType !== 'indexer') return; + for(const [table, entries] of utf8mb4Columns.byTable()){ + let rows; + try { + // rethrow, not the fail-soft default: outside a transaction doQuery logs a + // driver fault and returns [], which the absent-table branch below would read + // as "this replica does not carry the table" and skip silently, leaving the + // follower narrow. A transient fault must look like a fault. + rows = await this.doQuery( + "SELECT COLUMN_NAME, CHARACTER_SET_NAME FROM information_schema.columns " + + "WHERE table_schema = ? AND table_name = ?", + [this.dbName, table], + null, + { rethrow: true } + ); + } catch(e){ + console.error('Failed to read the column charsets of ' + table + ' while widening to utf8mb4:', e); + continue; + } + if(!rows || rows.length === 0) continue; // table absent on this replica + + let live = new Map(); + for(const row of rows) + live.set(String(row.COLUMN_NAME || row.column_name || '').toLowerCase(), row); + + let pending = entries.filter(entry => { + let row = live.get(entry.column.toLowerCase()); + return row !== undefined && !utf8mb4Columns.isAlreadyUtf8mb4(row); + }); + if(pending.length === 0) continue; + + try { + await this.doQuery('ALTER TABLE `' + table + '` ' + + pending.map(utf8mb4Columns.modifyClause).join(', '), [], null, { rethrow: true }); + console.log('Widened ' + pending.length + ' raw-wire-field column(s) on ' + table + + ' to utf8mb4 in ' + this.dbName + ': ' + pending.map(e => e.column).join(', ')); + } catch(e){ + // Not fatal to startup: the replica is exactly as usable as it was a moment + // ago, and every other table still converges. But it stays wedge-capable on + // these columns, and nothing downstream would say so, so log it loudly. + console.error('Failed to widen ' + table + ' to utf8mb4 (errno ' + ((e && e.errno) || 'unknown') + + '); this replica still halts on a 4-byte character in ' + + pending.map(e => e.column).join(', '), e); + } + } + } + // Get a database connection (with exponential backoff + circuit breaker). // Returns the active shared transaction connection when one is open, so every // query on this Db instance funnels through that same transaction. @@ -1960,7 +2037,13 @@ class Database { // (recovery pre-seed, API read-path createAddress). Those are benign // source-local drift, excluded exactly as computeIndexMapChecksum // excludes them, so they can never raise a false alarm. - query = "SELECT * FROM `" + table + "` WHERE block_index IS NOT NULL AND block_index BETWEEN ? AND ?"; + // + // Window by the registry's scope column, never the literal: a close_block-keyed + // table raises errno 1054 on `block_index`, the caller's per-table catch drops + // it, and it stays reported as parity-covered while nothing ever checks it. + let key = lifecycle.blockKey(table); + assertValidIdentifier(key); + query = "SELECT * FROM `" + table + "` WHERE " + key + " IS NOT NULL AND " + key + " BETWEEN ? AND ?"; } return await this.doQueryStrict(query, [fromBlock, toBlock], conn); } @@ -1984,11 +2067,17 @@ class Database { "SELECT * FROM `" + table + "` WHERE id > ? AND id <= ?", [fromId, toId], conn); } - // Get all rows from a table for a given block (block_index-scoped tables). - // ORDER BY block_index, then by the first column for deterministic ordering + // Get all rows from a table for a given block (block-scoped tables). + // ORDER BY the scope column, then by the first column for deterministic ordering // across sources with differing insert histories (matches the snapshot path). + // + // Scope by the registry's blockKey, never the literal `block_index`: a table keyed + // by another column raises errno 1054, which ServerPoller classifies as an older + // source schema and drops from the payload with no log line and no delivery. async getBlockScopedRows(table, block_index, conn){ - let query = "SELECT * FROM `" + table + "` WHERE block_index = ? ORDER BY block_index ASC, 1 ASC"; + let key = lifecycle.blockKey(table); + assertValidIdentifier(key); + let query = "SELECT * FROM `" + table + "` WHERE " + key + " = ? ORDER BY " + key + " ASC, 1 ASC"; return await this.doQuery(query, [block_index], conn); } diff --git a/src/derivedRewards.js b/src/derivedRewards.js index 798c432..8a7528e 100644 --- a/src/derivedRewards.js +++ b/src/derivedRewards.js @@ -47,7 +47,7 @@ // window [fromBlock, toBlock] whose own block_index (earn-block E) is BELOW their // materialization block, so the block-keyed channels missed them. Returns raw rows for // the caller to merge into the `validator_rewards` array, deduped on the UNIQUE identity -// (source_id, signing_pubkey_id, reward_type, round_reference). `db` must be an +// (source_id, signing_pubkey_id, reward_type, round_reference, round_qualifier). `db` must be an // indexer-dbType Database (callers gate that), and passing `conn` lets a snapshot's // REPEATABLE READ view read these at the same height as the rest of its payload. async function collectDerivedAnchorRewards(db, fromBlock, toBlock, conn){ @@ -77,9 +77,14 @@ async function collectDerivedAnchorRewards(db, fromBlock, toBlock, conn){ "WHERE vr.derive_block_index BETWEEN ? AND ? " + " AND vr.block_index < vr.derive_block_index", [from, to], conn); + // round_qualifier closes the key. The archive leg's round_reference is + // MATCH_BATCH_SEQ, a dense hub counter a rebase reissues, so two genuinely distinct + // archive rewards can share the four older columns; on the four-column key the + // second overwrites the first in this Map and never reaches the replica at all. for(let r of (rows || [])){ if(r && r.source_id != null && r.signing_pubkey_id != null) - acc.set(r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference, r); + acc.set(r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference + + ':' + r.round_qualifier, r); } } catch(e){ // derive_block_index may not exist on an older source schema (pre-RB-ANCHOR); diff --git a/src/observability/logShipper.js b/src/observability/logShipper.js index 656141a..dae9542 100644 --- a/src/observability/logShipper.js +++ b/src/observability/logShipper.js @@ -187,6 +187,15 @@ function readLogEnv(env = process.env) { }; } +// fetch() settles on the response HEADERS, so the body is still an open stream +// holding its socket while the abort timer is cleared. Release it inside that +// window by cancel, not read; a cancel on an errored body is not a ship failure. +function releaseBody(res) { + const stream = res && res.body; + if (!stream || typeof stream.cancel !== 'function') return Promise.resolve(); + return Promise.resolve(stream.cancel()).catch(() => {}); +} + class LogShipper { /** * @param {object} opts @@ -331,9 +340,9 @@ class LogShipper { const headers = { 'Content-Type': 'application/x-ndjson' }; if (token) headers.Authorization = `Bearer ${token}`; return fetch(url, { method: 'POST', headers, body, signal: controller.signal }) - .then((res) => { + .then((res) => releaseBody(res).then(() => { if (!res.ok) throw new Error(`collector responded ${res.status}`); - }) + })) .finally(() => clearTimeout(timer)); } diff --git a/src/pinnedValidators.js b/src/pinnedValidators.js index b9d99d0..996e2a0 100644 --- a/src/pinnedValidators.js +++ b/src/pinnedValidators.js @@ -34,12 +34,23 @@ * CHECKPOINT_VALIDATORS_BTC_MAINNET='[{"pubkey":"..","weight":"..","source":".."}]') * which overrides the baked-in entry for that key. * + * ABSENT is not INVALID. An unset override is inert: no trust root exists, so + * ClientSync._verifyCheckpointQuorum skips the step, which is what the config.js + * VERIFY_CHECKPOINT_QUORUM contract means by "skipped, never bypassed". An override + * the operator DID supply but got wrong used to resolve to the same null, and with + * every baked-in key still null that silently switched checkpoint authentication OFF + * on a replica whose operator armed the flag believing it on. So the getters still + * return null (they are read on hot paths and must never throw), and + * `assertPinnedEnvOverrides` runs once at client startup to REFUSE to start on a + * present-but-invalid value, the same shape as config.js assertBootstrapDepthChains. + * * Validator ROTATION past the launch epoch: the launch set eventually stops * signing, so `getPinnedCheckpoint` below provides the out-of-band SEED checkpoint * (a committed state_root plus its block/snapshot height) from which a client rolls * its trust root FORWARD, proving each successor oracle_publish set against the * committed BTC stakes_root (spec §7.3). That registry is INERT too until launch - * values land. Env override: CHECKPOINT_SEED__ (JSON, fail-closed). + * values land. Env override: CHECKPOINT_SEED__ (JSON), under the same + * absent-versus-invalid rule as the validator override above. * * FILL AT LAUNCH: replace a key's null with the oracle_publish signer set that signs * the launch checkpoints, in the stake-weighted shape checkpoint.js verifyCheckpoint @@ -83,22 +94,37 @@ function _envKey(chain, network) { return 'CHECKPOINT_VALIDATORS_' + String(chain).toUpperCase() + '_' + String(network).toUpperCase(); } -// Parse + lightly validate an env-supplied set; returns null on any malformation -// so a bad override never weakens verification (fails closed: no set to verify -// against means the quorum step is skipped, not bypassed). +// Parse + lightly validate an env-supplied set into { set, error }: `set` when the +// value is usable, `error` naming WHY it is not. The reason is what separates an +// absent override from an explicitly supplied invalid one, which the getters cannot +// express in their null and assertPinnedEnvOverrides refuses to start on. +function _parseValidatorSetEnv(raw) { + let arr; + try { arr = JSON.parse(raw); } catch (e) { return { set: null, error: 'is not valid JSON (' + e.message + ')' }; } + if (!Array.isArray(arr)) return { set: null, error: 'is not a JSON array' }; + if (arr.length === 0) return { set: null, error: 'is an empty array (an empty set verifies nothing)' }; + for (let i = 0; i < arr.length; i++) { + const v = arr[i]; + if (!v || typeof v !== 'object' || Array.isArray(v)) return { set: null, error: 'entry ' + i + ' is not an object' }; + for (const f of ['pubkey', 'weight', 'source']) { + if (typeof v[f] !== 'string') return { set: null, error: 'entry ' + i + ' has no string `' + f + '`' }; + } + } + return { set: arr, error: null }; +} + +// Resolve the env-supplied set for (chain, network), or null when it is absent or +// unusable. Never throws: this is on the per-verify read path, and startup already +// refused an invalid explicit value. function _fromEnv(chain, network) { const raw = process.env[_envKey(chain, network)]; if (!raw) return null; - let arr; - try { arr = JSON.parse(raw); } catch (e) { - console.warn('[pinnedValidators] malformed ' + _envKey(chain, network) + ' override, falling back (fail-closed):', e.message); + const { set, error } = _parseValidatorSetEnv(raw); + if (error) { + console.warn('[pinnedValidators] ' + _envKey(chain, network) + ' override ' + error + '; no env trust root for this key'); return null; } - if (!Array.isArray(arr) || arr.length === 0) return null; - for (const v of arr) { - if (!v || typeof v.pubkey !== 'string' || typeof v.weight !== 'string' || typeof v.source !== 'string') return null; - } - return arr; + return set; } /** @@ -147,20 +173,32 @@ function _seedEnvKey(chain, network) { return 'CHECKPOINT_SEED_' + String(chain).toUpperCase() + '_' + String(network).toUpperCase(); } -// Parse + lightly validate an env-supplied seed checkpoint; returns null on any -// malformation so a bad override never weakens the trust root (fail-closed: no seed -// means forward-following is skipped, not bypassed). state_root is the field the -// walk anchors successor-set proofs to, so it is required and must be a string. +// Parse + lightly validate an env-supplied seed checkpoint into { seed, error }, the +// same absent-versus-invalid split as _parseValidatorSetEnv. state_root is the field +// the forward walk anchors successor-set proofs to, so it is required and a string. +function _parseSeedEnv(raw) { + let cp; + try { cp = JSON.parse(raw); } catch (e) { return { seed: null, error: 'is not valid JSON (' + e.message + ')' }; } + if (!cp || typeof cp !== 'object' || Array.isArray(cp)) return { seed: null, error: 'is not a JSON object' }; + if (typeof cp.state_root !== 'string' || !cp.state_root) return { seed: null, error: 'has no non-empty string `state_root`' }; + for (const f of ['block_index', 'snapshot_block']) { + if (typeof cp[f] !== 'number' || !Number.isFinite(cp[f]) || cp[f] < 0) + return { seed: null, error: 'has no finite non-negative number `' + f + '`' }; + } + return { seed: cp, error: null }; +} + +// Resolve the env-supplied seed for (chain, network), or null when it is absent or +// unusable. Never throws, for the same reason _fromEnv does not. function _seedFromEnv(chain, network) { const raw = process.env[_seedEnvKey(chain, network)]; if (!raw) return null; - let cp; - try { cp = JSON.parse(raw); } catch (e) { return null; } - if (!cp || typeof cp !== 'object' || Array.isArray(cp)) return null; - if (typeof cp.state_root !== 'string' || !cp.state_root) return null; - if (typeof cp.block_index !== 'number' || !Number.isFinite(cp.block_index) || cp.block_index < 0) return null; - if (typeof cp.snapshot_block !== 'number' || !Number.isFinite(cp.snapshot_block) || cp.snapshot_block < 0) return null; - return cp; + const { seed, error } = _parseSeedEnv(raw); + if (error) { + console.warn('[pinnedValidators] ' + _seedEnvKey(chain, network) + ' override ' + error + '; no env seed for this key'); + return null; + } + return seed; } /** @@ -178,4 +216,55 @@ function getPinnedCheckpoint(chain, network) { return entry || null; } -module.exports = { getPinnedValidators, getPinnedCheckpoint, PINNED, PINNED_CHECKPOINTS }; +// Env names the getters can actually read: the prefix plus a CHAIN_NETWORK suffix, +// both halves non-empty. Mirrors config.js bootstrapDepthEnvKey's shape rule. +const _OVERRIDE_PREFIXES = [ + { prefix: 'CHECKPOINT_VALIDATORS_', parse: _parseValidatorSetEnv, what: 'pinned validator set' }, + { prefix: 'CHECKPOINT_SEED_', parse: _parseSeedEnv, what: 'pinned seed checkpoint' }, +]; + +function _isChainNetworkShaped(prefix, envKey) { + if (envKey.indexOf(prefix) !== 0) return false; + const rest = envKey.slice(prefix.length); + const sep = rest.lastIndexOf('_'); + return sep > 0 && sep < rest.length - 1; +} + +/** + * REFUSE to start when an explicitly supplied CHECKPOINT_VALIDATORS_* or + * CHECKPOINT_SEED_* value is present but unusable. + * + * A malformed override is not inert. The getters answer null for it, exactly as they + * do for an override nobody set, and every baked-in pin still ships null, so + * ClientSync._verifyCheckpointQuorum's `if(!validators || !validators.length) return;` + * skips checkpoint authentication entirely on a replica whose operator turned + * VERIFY_CHECKPOINT_QUORUM on. An unset variable stays inert and is NOT an error; only + * a value the operator supplied and got wrong is. Same rationale, and the same + * "Refusing to start" shape, as config.js assertBootstrapDepthChains. + * + * @param {Record} [env] Environment to scan; defaults to process.env. + * @throws {Error} naming every offending variable and why it is unusable. + */ +function assertPinnedEnvOverrides(env) { + const source = env || process.env; + const bad = []; + for (const envKey of Object.keys(source)) { + for (const { prefix, parse, what } of _OVERRIDE_PREFIXES) { + if (!_isChainNetworkShaped(prefix, envKey)) continue; + const raw = source[envKey]; + if (raw === undefined || raw === null || raw === '') break; // absent: inert, not an error + const { error } = parse(raw); + if (error) bad.push(envKey + ' (' + what + ') ' + error); + break; + } + } + if (bad.length === 0) return; + throw new Error( + 'Invalid checkpoint pin override: ' + bad.join('; ') + + '. Refusing to start: an unusable override resolves to the same null as an ABSENT one, ' + + 'which silently skips checkpoint-quorum verification instead of anchoring it. ' + + 'Fix the value or unset the variable to run deliberately unanchored.' + ); +} + +module.exports = { getPinnedValidators, getPinnedCheckpoint, assertPinnedEnvOverrides, PINNED, PINNED_CHECKPOINTS }; diff --git a/src/recoveryRewards.js b/src/recoveryRewards.js index ed10ca9..acfb1c9 100644 --- a/src/recoveryRewards.js +++ b/src/recoveryRewards.js @@ -41,7 +41,8 @@ // inside the inclusive window [fromBlock, toBlock] whose own block_index (earn-block E) // is BELOW that window, so the normal block-scoped channels missed them. Returns raw // rows for the caller to merge into the `validator_rewards` array, deduped on the UNIQUE -// identity (source_id, signing_pubkey_id, reward_type, round_reference). `db` must be an +// identity (source_id, signing_pubkey_id, reward_type, round_reference, round_qualifier). +// `db` must be an // indexer-dbType Database (callers gate that), and passing `conn` lets a snapshot's // REPEATABLE READ view read these at the same height as the rest of its payload. async function collectRedrivenValidatorRewards(db, fromBlock, toBlock, conn){ @@ -77,9 +78,16 @@ async function collectRedrivenValidatorRewards(db, fromBlock, toBlock, conn){ " AND rpr.applied_block BETWEEN ? AND ? " + " AND vr.block_index < rpr.applied_block", [from, to], conn); + // round_qualifier closes the key, exactly as in derivedRewards.js: two distinct + // archive rewards can share the four older columns after a hub rebase reissues + // MATCH_BATCH_SEQ, and on the four-column key the second overwrites the first here + // and never reaches the replica. recovery_pending_rewards has no round_qualifier + // column (xchain-indexer/src/sql/recovery_pending_rewards.sql), so the JOIN above + // stays as it is; over-selecting there is harmless once this key is qualifier-aware. for(let r of (rows || [])){ if(r && r.source_id != null && r.signing_pubkey_id != null) - acc.set(r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference, r); + acc.set(r.source_id + ':' + r.signing_pubkey_id + ':' + r.reward_type + ':' + r.round_reference + + ':' + r.round_qualifier, r); } } catch(e){ // recovery_pending_rewards / applied_block may not exist on a non-recovery stack diff --git a/src/replicatedTables.js b/src/replicatedTables.js index 73e0c49..0a9beb0 100644 --- a/src/replicatedTables.js +++ b/src/replicatedTables.js @@ -53,8 +53,8 @@ * capability_snapshots, per-block stream. xchain-sync NEVER replicates them * state_checkpoints, on any channel, snapshots included. A serving node * price_snapshots, does not fall back to a local mirror either: the - * anchor_reward_attestations explorer reads the consensus-relevant ones from the - * MANDATORY co-located hub DB and fails loud without + * anchor_reward_attestations, explorer reads the consensus-relevant ones from the + * attestation_responses MANDATORY co-located hub DB and fails loud without * it, rather than serving stale local rows. The set is * every tableLifecycle entry with replication * 'hub-mirror'; that registry is the authority and diff --git a/src/tableLifecycle.js b/src/tableLifecycle.js index d1e741e..5596786 100644 --- a/src/tableLifecycle.js +++ b/src/tableLifecycle.js @@ -61,6 +61,15 @@ * carried by xchain-sync in any channel * 'local' never leaves the node (OPERATOR_LOCAL) * 'follower-derived' recomputed by the follower, not carried + * blockKey the column a 'stream:block' entry is really scoped by; absent + * means 'block_index'. Declared because the class name is not the + * column name (rollcalls/rollcall_absences key on close_block) and + * a reader assuming the default raises errno 1054, which every + * forward channel swallows as an older source schema: silent + * non-delivery, never an error. + * xchain-sync test/unit/streamScopeColumns.test.js binds this + * field to the owning DDL, so a scope column the schema does not + * have fails the build instead of shipping un-replicated. * rollback source-indexer reorg handling (src/rollback.js): * 'action' generic DELETE by action_index (dataTables) * 'block' generic DELETE by block_index (blockTables) @@ -228,7 +237,7 @@ const TABLES = [ note: 'The in-place invalid_archive stamp on a surviving v1 parent is covered by the state_hash anchor_invalid class. New rows are otherwise action-derived; status_id is deliberately in no block-hash projection.' } }, { table: 'attests', owner: 'indexer', replication: 'stream:action', rollback: 'action', replicaRollback: 'mirror', hashed: { classes: ['state_hash'], - note: 'The v0 request_status terminal flip is an in-place mutation on a surviving row; the state_hash request_status class covers it. New rows are otherwise action-derived.' } }, + note: 'The v0 request_status terminal flip (updateAttestationRequestStatus) is an in-place mutation on a surviving row; the state_hash request_status class covers it. Three more in-place writers exist. setAttestationResponseCallbackIndex and setAttestationResponseBatchIndex stamp action_index links (callback_execute_action_index, batch_action_index) on a surviving v1 row; both are display-only and deliberately in no hash projection (state_hash reads attests at version 0 only). setAttestBatchStatus re-stamps status_id on a surviving v5 batch head when the completing v6 continuation fails reassembly or quorum, and that stamp is a KNOWN GAP: it is in no state_hash class, not carried by the updated_rows forward channel, and not reset by either rollback. Closing it needs a flag-day-gated class in the anchor_invalid shape plus the forward carry and both reorg resets.' } }, { table: 'prices', owner: 'indexer', replication: 'stream:action', rollback: 'action', replicaRollback: 'mirror', hashed: DERIVED }, { table: 'pending_hub_pushes', owner: 'indexer', replication: 'local', rollback: 'action', replicaRollback: 'local', hashed: { classes: [], note: 'Indexer-local outbound hub-push queue; never replicated (OPERATOR_LOCAL) and meaningless on a replica.' } }, @@ -278,12 +287,12 @@ const TABLES = [ hashed: { classes: [], note: 'Oracle/attest rewards derive deterministically during block processing; anchor_* rounds arrive via hub push but are quorum-verified before persistence. Reward credits they mint are ledger-hashed.' }, note: 'TWO block-scoped rollback keys, not one. block_index is the EARN block; derive_block_index is the MATERIALIZATION block, non-NULL only for the BTC-side anchor/archive derivation, which earns at the checkpoint SNAPSHOT_BLOCK but writes the row while processing a later BTC block. rollback() deletes on BOTH, or a reorg into the gap between them leaves a COLLECT-spendable reward a from-genesis replay has not derived yet.' }, - { table: 'rollcalls', owner: 'indexer', replication: 'stream:block', rollback: 'special', replicaRollback: 'special', + { table: 'rollcalls', owner: 'indexer', replication: 'stream:block', blockKey: 'close_block', rollback: 'special', replicaRollback: 'special', hashed: DERIVED, - note: 'One row per epoch that reached its close block, INCLUDING unrolled ones. An unrolled epoch counts for nobody, but the K-streak must know which epochs to SKIP, and a missing row is indistinguishable from an epoch that has not closed yet. SPECIAL, not "block": the block-scoped key is close_block and there is no block_index column, so the generic DELETE ... WHERE block_index >= ? would throw 1054 and fail the whole rollback transaction. Carries the pinned responsible set, without which the K-streak cannot tell "present" from "was not in R".' }, - { table: 'rollcall_absences', owner: 'indexer', replication: 'stream:block', rollback: 'special', replicaRollback: 'special', + note: 'One row per epoch that reached its close block, INCLUDING unrolled ones. An unrolled epoch counts for nobody, but the K-streak must know which epochs to SKIP, and a missing row is indistinguishable from an epoch that has not closed yet. blockKey is close_block on BOTH dimensions: there is no block_index column, so rollback is SPECIAL (the generic DELETE ... WHERE block_index >= ? would throw 1054 and fail the whole rollback transaction) and the forward readers scope by the declared key (they used to assume block_index, raise 1054, and have it swallowed as an older source schema, so the table never replicated on any forward channel while the reorg delete still removed it). Carries the pinned responsible set, without which the K-streak cannot tell "present" from "was not in R".' }, + { table: 'rollcall_absences', owner: 'indexer', replication: 'stream:block', blockKey: 'close_block', rollback: 'special', replicaRollback: 'special', hashed: DERIVED, - note: 'One row per responsible SOURCE that did not sign at a rolled epoch, pinned at close and never re-derived (SLASH rewrites stakes.amount in place, so a later re-derivation can differ from the set the verdict was taken over). evicted = 1 is the rollback key the delegations repair clause keys on, because the eviction writes no DELEGATE-revoke row for the generic self-join repair to find. SPECIAL for the same reason as rollcalls: the block-scoped key is close_block, not block_index.' }, + note: 'One row per responsible SOURCE that did not sign at a rolled epoch, pinned at close and never re-derived (SLASH rewrites stakes.amount in place, so a later re-derivation can differ from the set the verdict was taken over). evicted = 1 is the rollback key the delegations repair clause keys on, because the eviction writes no DELEGATE-revoke row for the generic self-join repair to find. Same close_block blockKey and same SPECIAL rollback as rollcalls, for the same reason.' }, { table: 'contract_state', owner: 'indexer', replication: 'stream:block', rollback: 'block', replicaRollback: 'mirror', hashed: { classes: ['contracts'], note: 'Latest value per state key written in the block.' } }, { table: 'escrow_leaf_journal', owner: 'indexer', replication: 'stream:block', rollback: 'block', replicaRollback: 'mirror', @@ -372,6 +381,9 @@ const TABLES = [ { table: 'anchor_reward_attestations', owner: 'indexer', replication: 'hub-mirror', rollback: 'exempt', replicaRollback: 'exempt', hashed: { classes: [], note: 'Not hashed: transport for the XANCPUB quorum. The BTC indexer re-verifies the sigs and derives validator_rewards, which itself is not in the state-hash preimage (COLLECT-mediated only).' }, note: 'Hub-mirrored, append-only, never retracted: written only after the XANCPUB quorum resolves for a FINALIZED checkpoint, so there is no un-finalize to retract. The derived validator_rewards row (block_index = snapshot_block) rolls back normally as a dataTable and re-derives idempotently on replay; a DOGE reorg cannot un-quorum an already-attested publish.' }, + { table: 'attestation_responses', owner: 'indexer', replication: 'hub-mirror', rollback: 'exempt', replicaRollback: 'exempt', + hashed: { classes: [], note: 'Not hashed: transport for the finalized ATTEST response. The applier re-verifies the signatures against the responsible set resolved from its OWN request row and synthesizes an ATTEST v1 action; the APPLIED state lives in attests, and the v0 status flip it drives is covered through resolved_block.' }, + note: 'Hub-mirrored, insert-only (no column is ever updated after insert) and never retracted: the mirror row is INERT without a pending local request, so a reorg that removes the request removes every applied row with it (they sit at blocks above the request) and the re-parse finds no request to re-bind to, while a reorg that keeps the request re-binds at the same block on every node. Deleting the mirror row on a local reorg would instead lose a response no chain replay can regenerate. Natural-key mirror on (network, request_id) with the hub id stripped on apply, so it also re-pages from since_id 0 (hub_db_sync FULL_REPAGE_TABLES).' }, { table: 'state_tree_nodes', owner: 'indexer', replication: 'snapshot', rollback: 'exempt', replicaRollback: 'exempt', hashed: { classes: ['state_commitment'], note: 'Content-addressed SMT node store; nodes are keyed by their own hash.' }, note: 'Copy-on-write: a node surviving a reorg is harmless (re-apply INSERT-IGNOREs the same hashes) and the surviving fork-point root in state_tree_roots anchors the correct tree. Orphans are reclaimed by the indexer\'s opt-in mark-and-sweep pruner (retention.js computeReachable/reclaimOrphanNodes, off unless STATE_ROOT_RETENTION_BLOCKS is positive AND STATE_NODE_RECLAIM is set, and serialized against block processing via runExclusive so a forward insert cannot re-reference a node between the mark and the delete); per-block deletion is impossible (no block_index, nodes shared across blocks).' }, @@ -570,6 +582,16 @@ function streamTopology(){ }; } +// The column a block-scoped table is really scoped by: the live per-block +// payload, the incremental catch-up range and the content-parity window all +// read it. Defaults to 'block_index', so it is a no-op for every table the +// class name describes correctly. Declared because the wrong answer is SILENT +// (errno 1054, which every forward channel swallows as an older source schema). +function blockKey(table){ + let e = entry(table); + return (e && e.blockKey) ? e.blockKey : 'block_index'; +} + // Source-side coverage buckets for the rollback-coverage guard. function rollbackBuckets(){ let sweepTables = [...new Set(ORPHAN_SWEEPS.map(s => s.table))]; @@ -638,7 +660,7 @@ module.exports = { TABLES, ORPHAN_SWEEPS, CONTENT_PARITY_CARVE_OUTS, CONTENT_PARITY_EXCLUDED_COLUMNS, allTables, entry, tablesWhere, - rollbackTables, replicaRollbackTables, streamTopology, + rollbackTables, replicaRollbackTables, streamTopology, blockKey, rollbackBuckets, replicaRollbackBuckets, hashClassTables, contentParityCarveOut, contentParityMutableTables, contentParityExcludedColumns, contentParityLookupBound, diff --git a/src/utf8mb4Columns.js b/src/utf8mb4Columns.js new file mode 100644 index 0000000..6fd7b60 --- /dev/null +++ b/src/utf8mb4Columns.js @@ -0,0 +1,258 @@ +/********************************************************************* + * + * Copyright © 2025-2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC - https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * The utf8mb4 widen set for the columns that ingest RAW wire fields. + * + * WHY THIS EXISTS + * --------------- + * An action that fails validation is still persisted: every action family calls its + * create* writer AFTER computing the status string, with the wire fields exactly as they + * arrived. So a field the grammar constrains (a contract's source code, a method name, a + * quorum fraction, a COIN tag) reaches its column holding whatever bytes the sender put on + * chain. If that column is utf8mb3 (three bytes per character, the tables' legacy + * DEFAULT CHARSET=utf8), a 4-byte character fails the INSERT with errno 1366 under + * STRICT_TRANS_TABLES, the block loop retries the block forever, and every indexer on the + * chain halts at the same height. That is a liveness wedge any sender can arm for the + * price of one transaction. Probed against the real writers, the columns that carry a raw + * 4-byte character all the way to the INSERT today are contracts.code, + * deploy_chunks.code_part, messages.coin, contract_executions.method_name / input_params / + * error_message, polls.quorum / min_vote_balance / decide_threshold, votes.share / memo, + * gated_files.gate_ticker / gate_min_amount, and the ATTEST / XCALL payload and callback + * fields. + * + * THE NUMERIC FIELDS ARE HERE AS DEFENCE IN DEPTH, NOT AS LIVE WEDGES. + * db.normalizeDataValues nulls any NUMBER_FIELDS / LOCK_FIELDS entry that is not numeric + * before the INSERT, so an AMOUNT / VALUE / FEE of "" currently stores as NULL + * rather than halting. That list is the ONLY thing standing between those columns and the + * same errno 1366, it is hand-maintained, and it has already failed once in exactly this + * way: CONTRACT_ACTION_INDEX was missing from it until a `DEPOSIT|0|null|...` broadcast + * wedged the LTC-regtest venue on 2026-07-05 (see the entry's comment in src/config.js). + * A column that cannot hold the bytes is a schema property; a column that never receives + * them is a property of one list somebody has to remember to edit. Widen the columns and + * the wedge stops depending on the list. + * + * The 2026-08-19 utf8mb4 pass widened the FREE-FORM text columns and deliberately left two + * groups behind: contracts.code (in the contract_hash preimage) and the grammar-constrained + * raw fields. Both were still wedge-capable. This module is the widen set for those two + * groups, and the ONE definition the three paths share: + * + * DEFINITION src/sql/.sql declares the column CHARACTER SET utf8mb4 (fresh + * installs; xchain-indexer only). + * LEDGER a dated migration MODIFYs it (aged origin DBs; xchain-indexer only). + * REPLICA xchain-sync widens the same columns at startup, because sync runs no + * migrations: a replica's tables are copied from the source's SHOW CREATE + * TABLE at bootstrap and addMissingColumns only ever ADDs columns, never + * retypes one. Without the replica pass, the origin accepts the character + * and every aged follower halts applying that block. + * + * NOT CONSENSUS-VISIBLE. Of the tables here only contracts, contract_executions, deposits + * and withdrawals enter a block-hash preimage (db.getBlockHashes), and none of the columns + * below is SELECTed into one except deposits.amount / withdrawals.amount, which the + * preimage orders WITHOUT an explicit COLLATE. utf8mb4_general_ci orders every BMP + * character exactly as utf8_general_ci does, and a 4-byte character cannot be present in + * any existing row (it would have halted the indexer that wrote it), so the ordering over + * every row that can exist today is unchanged. contracts is hashed on code_hash, never on + * code. Widening rewrites no stored value: utf8mb3 is a strict subset of utf8mb4. + * + * STILL EXCLUDED, each for a reason that is not "we forgot": + * * index_addresses.address - a raw DESTINATION reaches it through createAddress, so it + * IS a halt vector, but the consensus preimages ORDER BY it with an explicit + * `COLLATE utf8_bin`, which becomes illegal (errno 1253) the moment the column is + * utf8mb4. Retiring that pin has to move in lockstep across the indexer and sync + * preimage queries and the stake-weight collation guard, so it is its own ruling and + * its own change, the way state_key already is. + * * contract_state.state_key / state_key_bin / state_value - the state_key collation is + * a height-gated consensus flag-day (src/state_key_collation_activation.js). + * * polls.callback_params - its ADD COLUMN migration (2026-07-05) is checksum-immutable + * and declares plain MEDIUMTEXT, so a charset on the definition would break the + * ADD-COLUMN parity gate with no legal way to converge the two paths. + * * index_statuses.status, index_actions.action - the indexer writes those strings + * itself (status text, action names); no wire byte reaches them. + * * the ledger amount columns (balances / credits / debits / escrows) and the derived + * projections (order_matches, dispenses, fees) - only a validated, canonical decimal + * string is ever written there, so they are not a halt vector. + * + * BYTE-ALIGNED TWIN: copied verbatim into xchain-sync/src/utf8mb4Columns.js (sync has no + * dependency on this package by design; same convention as tableLifecycle.js / + * stateHash.js). Edit here, then `cp` to the twin; the sync suite asserts byte-identity. + * + * Entry fields: + * table table name (src/sql/
.sql) + * column column name + * type the column type EXACTLY as the definition file declares it + * tail what follows the charset clause in the definition ('' | 'NULL' | 'NOT NULL') + * mode 'auto' - the widen ships in the mode=auto migration and applies unattended + * 'manual' - the column is NOT NULL, and a MODIFY carrying NOT NULL is + * indistinguishable from a narrowing to the auto-apply + * destructive-DDL classifier (db._destructiveAutoStatement), so it + * ships in the paired mode=manual file. Dropping NOT NULL is not an + * option: MODIFY restates the whole column, so it would silently + * relax the column and diverge the two schema paths. + ********************************************************************/ + +'use strict'; + +// The charset every entry below is widened to. utf8mb4_general_ci is the collation the +// 2026-08-19 pass used, and it orders BMP characters identically to utf8_general_ci. +const UTF8MB4_CHARSET_CLAUSE = 'CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci'; + +const UTF8MB4_RAW_FIELD_COLUMNS = [ + // ---- contract source code (DEPLOY / DEPLOY_CHUNK carry it verbatim) -------------- + { table: 'contracts', column: 'code', type: 'MEDIUMTEXT', tail: 'NOT NULL', mode: 'manual' }, + { table: 'deploy_chunks', column: 'code_part', type: 'MEDIUMTEXT', tail: 'NOT NULL', mode: 'manual' }, + + // ---- BROADCAST ------------------------------------------------------------------ + { table: 'broadcasts', column: 'value', type: 'VARCHAR(25)', tail: '', mode: 'auto' }, + { table: 'broadcasts', column: 'fee', type: 'VARCHAR(11)', tail: '', mode: 'auto' }, + + // ---- token movement ------------------------------------------------------------- + { table: 'sends', column: 'amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'mints', column: 'amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'destroys', column: 'amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'dividends', column: 'amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'callbacks', column: 'callback_amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'airdrops', column: 'amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + + // ---- ISSUE ---------------------------------------------------------------------- + { table: 'issues', column: 'max_supply', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'max_mint', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'decimals', type: 'VARCHAR(2)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'mint_supply', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'lock_max_supply', type: 'VARCHAR(1)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'lock_mint', type: 'VARCHAR(1)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'lock_mint_supply', type: 'VARCHAR(1)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'lock_max_mint', type: 'VARCHAR(1)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'lock_description', type: 'VARCHAR(1)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'lock_sleep', type: 'VARCHAR(1)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'lock_callback', type: 'VARCHAR(1)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'callback_block', type: 'VARCHAR(15)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'callback_amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'mint_address_max', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'mint_start_block', type: 'VARCHAR(15)', tail: '', mode: 'auto' }, + { table: 'issues', column: 'mint_stop_block', type: 'VARCHAR(15)', tail: '', mode: 'auto' }, + + // ---- markets (ORDER / SWAP / DISPENSER / BET) ------------------------------------ + { table: 'orders', column: 'give_amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'orders', column: 'get_amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'swaps', column: 'give_amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'swaps', column: 'get_amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'dispensers', column: 'give_amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'dispensers', column: 'give_escrow', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'dispensers', column: 'get_amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'dispensers', column: 'fiat_amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'dispenser_edits', column: 'give_escrow', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'bets', column: 'amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'bet_feeds', column: 'fee', type: 'VARCHAR(11)', tail: '', mode: 'auto' }, + { table: 'bet_feeds', column: 'min_amount', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + + // ---- SLEEP / LIST / MESSAGE envelope fields -------------------------------------- + { table: 'sleeps', column: 'resume_block', type: 'VARCHAR(25)', tail: '', mode: 'auto' }, + { table: 'lists', column: 'type', type: 'VARCHAR(1)', tail: '', mode: 'auto' }, + { table: 'lists', column: 'edit', type: 'VARCHAR(1)', tail: '', mode: 'auto' }, + { table: 'messages', column: 'coin', type: 'VARCHAR(4)', tail: '', mode: 'auto' }, + { table: 'messages', column: 'encryption_method', type: 'VARCHAR(1)', tail: '', mode: 'auto' }, + + // ---- VOTE ----------------------------------------------------------------------- + { table: 'votes', column: 'share', type: 'VARCHAR(60)', tail: '', mode: 'auto' }, + { table: 'votes', column: 'memo', type: 'MEDIUMTEXT', tail: '', mode: 'auto' }, + { table: 'polls', column: 'quorum', type: 'VARCHAR(60)', tail: '', mode: 'auto' }, + { table: 'polls', column: 'min_vote_balance', type: 'VARCHAR(60)', tail: '', mode: 'auto' }, + { table: 'polls', column: 'decide_threshold', type: 'VARCHAR(60)', tail: '', mode: 'auto' }, + { table: 'polls', column: 'deposit_amount', type: 'VARCHAR(60)', tail: '', mode: 'auto' }, + { table: 'polls', column: 'gas_escrow', type: 'VARCHAR(60)', tail: '', mode: 'auto' }, + { table: 'polls', column: 'callback_method', type: 'VARCHAR(64)', tail: '', mode: 'auto' }, + + // ---- EXECUTE / PRICE / FILE ------------------------------------------------------ + { table: 'contract_executions', column: 'method_name', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'contract_executions', column: 'input_params', type: 'TEXT', tail: '', mode: 'auto' }, + { table: 'contract_executions', column: 'error_message', type: 'TEXT', tail: '', mode: 'auto' }, + { table: 'prices', column: 'value', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'prices', column: 'fee', type: 'VARCHAR(250)', tail: '', mode: 'auto' }, + { table: 'gated_files', column: 'gate_min_amount', type: 'VARCHAR(40)', tail: 'NULL', mode: 'auto' }, + + // ---- ATTEST / XCALL -------------------------------------------------------------- + { table: 'attests', column: 'payload', type: 'MEDIUMTEXT', tail: '', mode: 'auto' }, + { table: 'attests', column: 'callback_method', type: 'VARCHAR(64)', tail: '', mode: 'auto' }, + { table: 'attests', column: 'callback_params_json', type: 'TEXT', tail: '', mode: 'auto' }, + { table: 'attests', column: 'gas_escrow', type: 'VARCHAR(60)', tail: '', mode: 'auto' }, + { table: 'attests', column: 'fee_amount', type: 'VARCHAR(60)', tail: '', mode: 'auto' }, + { table: 'attests', column: 'response_payload', type: 'MEDIUMTEXT', tail: '', mode: 'auto' }, + { table: 'attests', column: 'meta', type: 'VARCHAR(256)', tail: '', mode: 'auto' }, + { table: 'xcalls', column: 'method', type: 'VARCHAR(64)', tail: '', mode: 'auto' }, + { table: 'xcalls', column: 'params_json', type: 'TEXT', tail: '', mode: 'auto' }, + { table: 'xcalls', column: 'callback_method', type: 'VARCHAR(64)', tail: '', mode: 'auto' }, + { table: 'xcalls', column: 'callback_params_json', type: 'TEXT', tail: '', mode: 'auto' }, + + // ---- NOT NULL raw fields (paired mode=manual file) ------------------------------- + { table: 'deposits', column: 'amount', type: 'VARCHAR(250)', tail: 'NOT NULL', mode: 'manual' }, + { table: 'withdrawals', column: 'amount', type: 'VARCHAR(250)', tail: 'NOT NULL', mode: 'manual' }, + { table: 'stakes', column: 'amount', type: 'VARCHAR(250)', tail: 'NOT NULL', mode: 'manual' }, + { table: 'unstakes', column: 'amount', type: 'VARCHAR(250)', tail: 'NOT NULL', mode: 'manual' }, + { table: 'contract_stakes', column: 'amount', type: 'VARCHAR(250)', tail: 'NOT NULL', mode: 'manual' }, + { table: 'contract_unstakes', column: 'amount', type: 'VARCHAR(250)', tail: 'NOT NULL', mode: 'manual' }, + { table: 'reward_claims', column: 'amount', type: 'VARCHAR(250)', tail: 'NOT NULL', mode: 'manual' }, + { table: 'gated_files', column: 'gate_ticker', type: 'VARCHAR(250)', tail: 'NOT NULL', mode: 'manual' }, + { table: 'attests', column: 'provider_id', type: 'VARCHAR(32)', tail: 'NOT NULL', mode: 'manual' }, +]; + +// The column spec as both the definition file and the migration MODIFY must state it. +// One producer, so the two paths cannot word it differently (the schema-parity gate +// compares them normalized, and a reordered charset clause reads as a divergence). +function columnSpec(entry){ + return [entry.type, UTF8MB4_CHARSET_CLAUSE, entry.tail].filter(Boolean).join(' '); +} + +// `MODIFY \`col\` ` - the clause body of an ALTER TABLE, used by the migrations and +// by the xchain-sync replica widen. +function modifyClause(entry){ + return 'MODIFY `' + entry.column + '` ' + columnSpec(entry); +} + +// The whole statement for one entry, for callers that widen a single column at a time. +function alterStatement(entry){ + return 'ALTER TABLE `' + entry.table + '` ' + modifyClause(entry); +} + +// Entries grouped by table, preserving declaration order, so a migration or a replica +// pass can issue one ALTER per table instead of one per column (each ALTER is a COPY +// rebuild under a metadata lock, so batching matters on a large table). +function byTable(entries){ + const out = new Map(); + for(const entry of (entries || UTF8MB4_RAW_FIELD_COLUMNS)){ + if(!out.has(entry.table)) out.set(entry.table, []); + out.get(entry.table).push(entry); + } + return out; +} + +const forMode = (mode) => UTF8MB4_RAW_FIELD_COLUMNS.filter(e => e.mode === mode); + +// True when an information_schema.columns row already reports the column as utf8mb4, so +// the widen is a no-op. MariaDB 10.6 renamed utf8 to utf8mb3, so this reads the positive +// (already wide) case rather than comparing against a legacy charset spelling. +function isAlreadyUtf8mb4(row){ + if(!row) return false; + const charset = String(row.CHARACTER_SET_NAME || row.character_set_name || '').toLowerCase(); + return charset === 'utf8mb4'; +} + +module.exports = { + UTF8MB4_CHARSET_CLAUSE, + UTF8MB4_RAW_FIELD_COLUMNS, + columnSpec, + modifyClause, + alterStatement, + byTable, + forMode, + isAlreadyUtf8mb4, +}; diff --git a/test/unit/ClientApplier.test.js b/test/unit/ClientApplier.test.js index 2cf8ea8..11bf1dd 100644 --- a/test/unit/ClientApplier.test.js +++ b/test/unit/ClientApplier.test.js @@ -154,6 +154,13 @@ describe('ClientApplier', function(){ assert.ok(/d\.source_id = vr\.source_id AND d\.signing_pubkey_id = vr\.signing_pubkey_id/.test(del.args[0])); assert.ok(/d\.reward_type = vr\.reward_type AND d\.round_reference <=> vr\.round_reference/.test(del.args[0]), 'NULL-safe round_reference match (the UNIQUE key component is nullable)'); + // round_qualifier joined reward_unique in the 2026-08-24 indexer migration and + // anchor_reward_reconcile_log pre-images it. Without this predicate the keyed + // delete ALSO matches the other archive snapshot's reward whenever a hub rebase + // reissued the MATCH_BATCH_SEQ round_reference, destroying a row the source still + // holds. Both columns are NOT NULL DEFAULT 0, so the match is `=`, not `<=>`. + assert.ok(/AND d\.round_qualifier = vr\.round_qualifier/.test(del.args[0]), + 'the mirror delete must carry the full five-column reward identity'); assert.ok(/WHERE d\.block_index = \?$/.test(del.args[0]), 'scoped to THIS block\'s reconcile rows'); assert.deepStrictEqual(del.args[1], [961700]); let logInsert = calls.findIndex(c => /anchor_reward_reconcile_log/.test(c.args[0]) && /^INSERT/.test(c.args[0])); @@ -362,6 +369,34 @@ describe('ClientApplier', function(){ assert.strictEqual(genericWipe, false, 'state_tree_roots must not be whole-table wiped by the clear loop'); }); + it('binds the scoped state_tree_roots clear to the TICKER, not the full coin name @regression', async function(){ + // SyncService constructs ClientApplier with cfg.coin, the hub's full lowercase + // name ('bitcoin'), while every state_tree_roots writer is called with + // this.coinTicker, so the rows carry 'BTC'. Binding this.chain made the scoped + // clear match zero rows on every production chain: a permanent silent no-op that + // left future-dated orphan roots for the Explorer to serve as commitments. The + // enclosing suite's applier is built with NO chain, so both fields are null there + // and cannot tell the two apart; this case supplies the full name on purpose. + let localDb = createMockDb(); + let fullNameApplier = new ClientApplier(localDb, new Utility(), 'bitcoin', 'mainnet'); + assert.strictEqual(fullNameApplier.chain, 'bitcoin'); + assert.strictEqual(fullNameApplier.coinTicker, 'BTC', 'the two identities must actually differ here'); + + await fullNameApplier.applyFullSnapshot({ + schema_version: SCHEMA_VERSION.indexer, + block_height: 10, + tables: { blocks: [{ block_index: 1 }] } + }); + + let scoped = localDb.doQuery.getCalls().find(c => + /DELETE FROM state_tree_roots/.test(c.args[0]) && /block_index >= \?/.test(c.args[0])); + assert.ok(scoped, 'the scoped delete must still be issued'); + assert.strictEqual(scoped.args[1][0], 'BTC', + 'the chain predicate must bind the ticker the rows are written with'); + assert.notStrictEqual(scoped.args[1][0], 'bitcoin', + 'binding the full coin name makes the cleanup match zero rows'); + }); + it('ignores node-local tables (mempool_transactions) shipped by an older source', async function(){ let snapshot = { schema_version: SCHEMA_VERSION.indexer, @@ -541,6 +576,20 @@ describe('ClientApplier', function(){ assert.ok(!query.includes('ON DUPLICATE KEY UPDATE')); }); + // The two close_block-keyed roll-call tables ride the bootstrap full dump AND + // stream per block, so an overlapping window re-delivers a row already applied. + // Each is pinned at its close and never re-derived, so the repeat is identical: + // IGNORE is a no-op, while a plain INSERT aborts the whole apply transaction. + for(const table of ['rollcalls', 'rollcall_absences']){ + it('uses INSERT IGNORE for the re-deliverable ' + table, async function(){ + await applier._insertRows(table, [{ epoch_height: 1000, close_block: 1100 }]); + let query = db.doQuery.firstCall.args[0]; + assert.ok(query.startsWith('INSERT IGNORE'), table + ' must be INSERT IGNORE'); + assert.ok(!query.includes('ON DUPLICATE KEY UPDATE'), + table + ' is pinned at close and must never be overwritten by a re-delivery'); + }); + } + for(const table of ['markets', 'attest_validator_stats']){ it('upserts ' + table + ' with ON DUPLICATE KEY UPDATE covering every carried column', async function(){ await applier._insertRows(table, [{ id: 1, a: 'x', b: 'y' }]); diff --git a/test/unit/ClientRollback.test.js b/test/unit/ClientRollback.test.js index 2d1e0a9..104590d 100644 --- a/test/unit/ClientRollback.test.js +++ b/test/unit/ClientRollback.test.js @@ -355,6 +355,11 @@ describe('ClientRollback', function(){ assert.strictEqual(restores.length, 2, 'expected contract_stakes + contract_unstakes slash restores'); for(let r of restores){ let sql = r.args[0]; + // The pick follows the debit chain's own values (highest orphaned prev_amount = + // the amount before the first orphaned debit). The position columns invert under + // a re-entrant nested EXECUTE and serve only as the tiebreak for equal amounts. + assert.ok(/CAST\(e\.prev_amount AS DECIMAL\(60,18\)\)\s*>\s*CAST\(d\.prev_amount AS DECIMAL\(60,18\)\)/.test(sql), + 'restore must pick the highest orphaned prev_amount, not the lowest position key'); assert.ok(/e\.execution_index\s*<\s*d\.execution_index/.test(sql), 'restore must order by execution_index (deterministic, replay-stable)'); assert.ok(/e\.slash_position\s*<\s*d\.slash_position/.test(sql), @@ -407,6 +412,17 @@ describe('ClientRollback', function(){ 'restore must also require the loser materialization block to survive the reorg'); assert.ok(/derive_block_index\)/.test(restore.args[0]), 'restore must carry derive_block_index back onto the restored row'); + // round_qualifier is part of reward_unique (2026-08-24 indexer migration) and the + // pre-image log carries it, so both the column list and the projection must name + // it. Dropped, the loser comes back under the schema default 0, a DIFFERENT row + // from the one the reconcile deleted: INSERT IGNORE then either swallows it + // against a legacy qualifier-0 row or lands a wrong-identity duplicate, and the + // replica forks SUM(validator_rewards) either way. The source twin at + // xchain-indexer/src/rollback.js already carries both halves. + assert.ok(/round_reference, round_qualifier,/.test(restore.args[0]), + 'restore column list must name round_qualifier'); + assert.ok(/d\.round_qualifier/.test(restore.args[0]), + 'restore projection must select d.round_qualifier rather than fall back to the schema default 0'); assert.deepStrictEqual(restore.args[1], [100, 100, 100]); let restoreIdx = calls.indexOf(restore); let deleteIdx = calls.findIndex(c => c.args[0].includes('DELETE FROM `validator_rewards`')); diff --git a/test/unit/ClientSync.test.js b/test/unit/ClientSync.test.js index f51c4bb..c424dc4 100644 --- a/test/unit/ClientSync.test.js +++ b/test/unit/ClientSync.test.js @@ -372,6 +372,69 @@ describe('ClientSync', function(){ assert.strictEqual(sync._maybeVerifyCompleteness.calledOnce, true); assert.strictEqual(sync._maybeVerifyCompleteness.firstCall.args[0], 'http://source1:3006'); }); + + // The server publishes its own replication verdict on every status tick. Dropping + // it left this follower's lag_blocks certifying a server whose SQL replica had + // stopped applying: its heights freeze together, so we catch up to the frozen tip + // and the heartbeats keep source_height_stale false. + describe('upstream replication evidence', function(){ + beforeEach(function(){ + sync.lastAppliedBlock = 100; + sinon.stub(sync, '_maybeVerifyCompleteness').resolves(); + }); + + it('is unknown, not fresh, before any status event', function(){ + assert.deepStrictEqual(sync.getUpstreamReplicaState(), + { stale: null, secondsBehind: null, sourceHeight: null }); + }); + + it('keeps the source height, staleness verdict and lag from a status event', async function(){ + await sync._handleEvent({ + type: 'status', block_height: 100, source_block_height: 140, + replica_stale: true, replica_seconds_behind: 900 + }, 0); + assert.deepStrictEqual(sync.getUpstreamReplicaState(), + { stale: true, secondsBehind: 900, sourceHeight: 140 }); + }); + + it('re-reads the verdict on a status tick that does not advance the height', async function(){ + await sync._handleEvent({ + type: 'status', block_height: 100, source_block_height: 100, + replica_stale: false, replica_seconds_behind: 2 + }, 0); + assert.strictEqual(sync.getUpstreamReplicaState().stale, false); + + // The upstream replica stops applying: its height never moves again. + await sync._handleEvent({ + type: 'status', block_height: 100, source_block_height: 100, + replica_stale: true, replica_seconds_behind: null + }, 0); + assert.strictEqual(sync.getUpstreamReplicaState().stale, true); + }); + + it('reads a server older than the fields as unknown rather than fresh', async function(){ + await sync._handleEvent({ type: 'status', block_height: 100 }, 0); + assert.deepStrictEqual(sync.getUpstreamReplicaState(), + { stale: null, secondsBehind: null, sourceHeight: null }); + }); + + it('takes the worst verdict across sources and ignores an evicted one', async function(){ + await sync._handleEvent({ + type: 'status', block_height: 100, source_block_height: 100, + replica_stale: false, replica_seconds_behind: 1 + }, 0); + await sync._handleEvent({ + type: 'status', block_height: 100, source_block_height: 130, + replica_stale: true, replica_seconds_behind: 700 + }, 1); + assert.deepStrictEqual(sync.getUpstreamReplicaState(), + { stale: true, secondsBehind: 700, sourceHeight: 130 }); + + sync._evictedSources.add(1); + assert.deepStrictEqual(sync.getUpstreamReplicaState(), + { stale: false, secondsBehind: 1, sourceHeight: 100 }); + }); + }); }); describe('_maybeVerifyCompleteness', function(){ diff --git a/test/unit/HubClient.test.js b/test/unit/HubClient.test.js index 0bf512b..9743e67 100644 --- a/test/unit/HubClient.test.js +++ b/test/unit/HubClient.test.js @@ -357,4 +357,77 @@ describe('HubClient', function(){ assert.deepStrictEqual(HubClient.parseEndpoints({}), ['http://localhost:10000']); }); }); + + // The hub redacts secret-bearing config params (rpc/DB passwords) unless the + // caller asks with include_secrets. Sync is one of only two services that + // genuinely needs them: _extractDbConfigs turns this tree into the + // replication sources' db_user/db_pass, so a redacted response points every + // source at a password of "[redacted]". + describe('credential tier', function(){ + + const savedKeys = {}; + beforeEach(function(){ + for(const k of ['HUB_API_KEY', 'HUB_CONFIG_SECRETS_API_KEY']){ + savedKeys[k] = process.env[k]; + delete process.env[k]; + } + }); + afterEach(function(){ + for(const [k, v] of Object.entries(savedKeys)){ + if(v === undefined) delete process.env[k]; + else process.env[k] = v; + } + }); + + it('asks for secrets on the initial fetch', async function(){ + let post = sinon.stub(axios, 'post').resolves({ data: { result: {} } }); + await hub.getallconfigs(); + assert.deepStrictEqual(post.firstCall.args[1].params, + { since_updated_at: 0, include_secrets: true }); + }); + + it('keeps asking on the delta poll, cursor and all', async function(){ + let post = sinon.stub(axios, 'post') + .resolves({ data: { result: { configs: {}, seq: 1, watermark: 5000 } } }); + await hub.getallconfigs(); + await hub.getallconfigs(); + assert.deepStrictEqual(post.secondCall.args[1].params, + { since_updated_at: 5000, include_secrets: true }); + }); + + it('sends HUB_CONFIG_SECRETS_API_KEY when the hub splits the credential tier', async function(){ + process.env.HUB_API_KEY = 'bulk-key'; + process.env.HUB_CONFIG_SECRETS_API_KEY = 'secrets-key'; + let post = sinon.stub(axios, 'post').resolves({ data: { result: {} } }); + await hub.getallconfigs(); + assert.strictEqual(post.firstCall.args[2].headers['x-api-key'], 'secrets-key'); + }); + + it('falls back to the bulk key when the hub does not split the tier', async function(){ + process.env.HUB_API_KEY = 'bulk-key'; + let post = sinon.stub(axios, 'post').resolves({ data: { result: {} } }); + await hub.getallconfigs(); + assert.strictEqual(post.firstCall.args[2].headers['x-api-key'], 'bulk-key'); + }); + + it('names the cause once when the hub redacts anyway, not once per poll', async function(){ + sinon.stub(axios, 'post').resolves({ data: { result: { + configs: {}, seq: 1, watermark: 5000, secrets_redacted: true, redacted_params: 6 + } } }); + await hub.getallconfigs(); + await hub.getallconfigs(); + // console.error is stubbed by the outer beforeEach. + let hits = console.error.getCalls().filter((c) => /CREDENTIAL-REDACTED/.test(String(c.args[0]))); + assert.strictEqual(hits.length, 1); + assert.ok(hits[0].args[0].includes('6 params withheld'), hits[0].args[0]); + }); + + it('says nothing when the hub served the credentials', async function(){ + sinon.stub(axios, 'post').resolves({ data: { result: { + configs: {}, seq: 1, watermark: 5000, secrets_redacted: false, redacted_params: 0 + } } }); + await hub.getallconfigs(); + assert.ok(!console.error.getCalls().some((c) => /CREDENTIAL-REDACTED/.test(String(c.args[0])))); + }); + }); }); diff --git a/test/unit/ServerPoller.test.js b/test/unit/ServerPoller.test.js index a445fef..66eabff 100644 --- a/test/unit/ServerPoller.test.js +++ b/test/unit/ServerPoller.test.js @@ -97,13 +97,17 @@ describe('ServerPoller', function(){ // contract_stakes row through it, in both directions: forward via updatedRows, // backward via the ClientRollback key restore. assert.ok(poller.blockScopedTables.includes('contract_delegation_rotations')); - // ROLLCALL epoch closes and their pinned absences. These ARE block-scoped for - // REPLICATION even though their rollback is bespoke: the replica needs the rows - // streamed like any other per-block table, and only the reorg delete differs, - // because their block key is close_block rather than block_index. + // ROLLCALL epoch closes and their pinned absences. Block-scoped for + // REPLICATION, but keyed by close_block on BOTH dimensions, which is why the + // rollback is bespoke and why the registry declares blockKey. Membership is + // NOT delivery: while the reader assumed block_index these two raised errno + // 1054 on every poll and were dropped from the payload in silence, so the + // read the membership drives is asserted separately below. assert.ok(poller.blockScopedTables.includes('rollcalls')); assert.ok(poller.blockScopedTables.includes('rollcall_absences')); assert.strictEqual(poller.blockScopedTables.length, 13); + // The read those two names drive is asserted in db.test.js + // (Database.getBlockScopedRows), because that is where it can fail. }); it('has action-scoped tables', function(){ @@ -167,6 +171,34 @@ describe('ServerPoller', function(){ assert.strictEqual(log.recordBlock.called, false); }); + // The failure the replication verdict exists to catch is exactly the one that + // stops block advancement: a native SQL replica that stops applying freezes the + // served tip, so a refresh gated on blocksProcessed > 0 never runs again and the + // last healthy verdict is republished forever. + it('re-evaluates the replica verdict on idle polls', async function(){ + poller.lastPolledBlock = 100; + db.getLastBlock.resolves(100); + db.getBlockHashRow.resolves({ + block_index: 100, block_time: 1700000000, + ledger_hash: 'lh', actions_hash: 'ah', contract_hash: 'ch' + }); + db.getReplicaStatus = sinon.stub().resolves({ isReplica: true, running: true, secondsBehind: 5 }); + + await poller._poll(); + assert.strictEqual(broadcaster.updateStatus.callCount, 1); + assert.strictEqual(broadcaster.updateStatus.lastCall.args[2].replica_stale, false); + + // Replication stops applying. The served tip is frozen from here on, so no + // later poll ever processes a block. + db.getReplicaStatus.resolves({ isReplica: true, running: false, secondsBehind: null }); + + await poller._poll(); + assert.strictEqual(broadcaster.broadcast.called, false); + assert.strictEqual(broadcaster.updateStatus.callCount, 2); + assert.strictEqual(broadcaster.updateStatus.lastCall.args[2].replica_stale, true); + assert.strictEqual(broadcaster.updateStatus.lastCall.args[2].block_height, 100); + }); + it('limits to 100 blocks per poll', async function(){ poller.lastPolledBlock = 0; db.getLastBlock.resolves(200); diff --git a/test/unit/SyncService.test.js b/test/unit/SyncService.test.js index 32077b3..64f5f33 100644 --- a/test/unit/SyncService.test.js +++ b/test/unit/SyncService.test.js @@ -251,6 +251,45 @@ describe('SyncService', function(){ assert.strictEqual(startSync.callCount, 0, 'no ClientSync started on a refused config'); }); + // A malformed CHECKPOINT_VALIDATORS_* is not inert either: it resolves to the same + // null an ABSENT override does, so _verifyCheckpointQuorum skips the anchor on a + // replica whose operator armed VERIFY_CHECKPOINT_QUORUM believing it on. Discovery + // must refuse it on the same pass, before any ClientSync replicates a block. + it('client mode REFUSES a malformed checkpoint pin override, before starting any sync', async function(){ + const PINKEY = 'CHECKPOINT_VALIDATORS_BITCOIN_MAINNET'; + config.SYNC_MODE = 'client'; + config.VERIFY_CHECKPOINT_QUORUM = true; + process.env[PINKEY] = '[{"pubkey":"aa","weight":100}]'; // weight not a string, no source + service = new SyncService(config); + sinon.stub(service.hubClient, 'getIndexerConfigs').resolves([indexerCfg()]); + sinon.stub(service.hubClient, 'getDecoderConfigs').resolves([]); + stubDiscoveryDb(); + let startSync = sinon.stub(service, '_startClientSyncForChain'); + + try { + await assert.rejects(() => service._discoverChains(), new RegExp(PINKEY)); + assert.strictEqual(startSync.callCount, 0, 'no ClientSync started on a refused pin override'); + } finally { delete process.env[PINKEY]; } + }); + + it('client mode accepts a well-formed checkpoint pin override', async function(){ + const PINKEY = 'CHECKPOINT_VALIDATORS_BITCOIN_MAINNET'; + config.SYNC_MODE = 'client'; + config.VERIFY_CHECKPOINT_QUORUM = true; + process.env[PINKEY] = JSON.stringify([{ pubkey: 'ab'.repeat(32), weight: '100', source: 'S1' }]); + service = new SyncService(config); + sinon.stub(service.hubClient, 'getIndexerConfigs').resolves([indexerCfg()]); + sinon.stub(service.hubClient, 'getDecoderConfigs').resolves([]); + stubDiscoveryDb(); + let startSync = sinon.stub(service, '_startClientSyncForChain'); + + try { + let newChains = await service._discoverChains(); + assert.strictEqual(newChains.length, 1); + assert.strictEqual(startSync.callCount, 1, 'a valid override does not block startup'); + } finally { delete process.env[PINKEY]; } + }); + it('client mode accepts a bootstrap-depth key whose chain the hub published under its full name', async function(){ config.SYNC_MODE = 'client'; config.SYNC_BOOTSTRAP_DEPTH = { 'BTC:MAINNET': 50000 }; @@ -478,7 +517,9 @@ describe('SyncService', function(){ describe('getClientSyncState', function(){ it('returns nulls/false when no sync exists for the key', function(){ assert.deepStrictEqual(service.getClientSyncState('bitcoin', 'mainnet'), - { lastKnownServerBlock: null, sourceHeightStale: null, halted: false, + { lastKnownServerBlock: null, sourceHeightStale: null, + upstreamReplica: { stale: null, secondsBehind: null, sourceHeight: null }, + halted: false, haltInfo: null, truncated: false, bootstrapBase: null, sourceQuorum: null, sourcesConfigured: null, sourcesActive: null, sourcesAgreeing: null, sourcesEvicted: [] }); diff --git a/test/unit/db.test.js b/test/unit/db.test.js index 80051ce..9ecf752 100644 --- a/test/unit/db.test.js +++ b/test/unit/db.test.js @@ -1142,6 +1142,31 @@ describe('Database.getBlockScopedRows()', function () { assert.ok(sql.includes('blocks')); assert.ok(sql.includes('block_index')); }); + + it('scopes a close_block-keyed table by close_block, not the class default', async function () { + // rollcalls and rollcall_absences are declared stream:block and have no + // block_index column, so the old literal raised errno 1054, which + // ServerPoller classifies as an older source schema and drops without a log + // line: the tables were streamed by membership and delivered by nothing. + // ServerPoller.test.js asserts the membership; this asserts the read. + sinon.stub(db, 'doQuery').resolves([]); + await db.getBlockScopedRows('rollcalls', 900); + await db.getBlockScopedRows('rollcall_absences', 900); + assert.strictEqual(db.doQuery.firstCall.args[0], + 'SELECT * FROM `rollcalls` WHERE close_block = ? ORDER BY close_block ASC, 1 ASC'); + assert.strictEqual(db.doQuery.secondCall.args[0], + 'SELECT * FROM `rollcall_absences` WHERE close_block = ? ORDER BY close_block ASC, 1 ASC'); + }); + + it('leaves every other block-scoped table on block_index', async function () { + // The registry field defaults, so the eleven tables that really are + // block_index-scoped must produce byte-identical SQL to before the change. + sinon.stub(db, 'doQuery').resolves([]); + for(const table of ['blocks', 'transactions', 'slash_events', 'escrow_leaf_journal']) + await db.getBlockScopedRows(table, 5); + for(const call of db.doQuery.getCalls()) + assert.match(call.args[0], /WHERE block_index = \? ORDER BY block_index ASC, 1 ASC$/); + }); }); describe('Database.getActionScopedRows()', function () { diff --git a/test/unit/derivedRewards.test.js b/test/unit/derivedRewards.test.js index 8ea0bac..2834a0d 100644 --- a/test/unit/derivedRewards.test.js +++ b/test/unit/derivedRewards.test.js @@ -64,6 +64,23 @@ describe('collectDerivedAnchorRewards', function(){ assert.strictEqual(out[0].derive_block_index, 961700); }); + // The archive leg keys round_reference on MATCH_BATCH_SEQ, a dense hub counter a + // wipe-and-replay rebase reissues, so two genuinely distinct archive rewards can share + // source_id/signing_pubkey_id/reward_type/round_reference and differ only in + // round_qualifier (the snapshot_block). The 2026-08-24 migration put that column in + // reward_unique; a four-column dedup key here treats the second reward as a duplicate + // and it never reaches the replica at all. + it('keeps two archive rewards that differ only in round_qualifier (the five-column identity)', async function(){ + let db = { doQuery: async () => [ + row({ reward_type: 'anchor_archive', round_reference: 42, round_qualifier: 100 }), + row({ reward_type: 'anchor_archive', round_reference: 42, round_qualifier: 200 }) + ] }; + let out = await collectDerivedAnchorRewards(db, 961700, 961700); + assert.strictEqual(out.length, 2, + 'a reissued MATCH_BATCH_SEQ makes round_reference non-unique; round_qualifier is what separates them'); + assert.deepStrictEqual(out.map(r => r.round_qualifier).sort(), [100, 200]); + }); + it('swallows ONLY a schema gap (1054/1146); a transient fault propagates so the block is retried', async function(){ let gap = Object.assign(new Error('Unknown column derive_block_index'), { errno: 1054 }); assert.deepStrictEqual(await collectDerivedAnchorRewards({ doQuery: async () => { throw gap; } }, 1, 1), []); diff --git a/test/unit/observability.test.js b/test/unit/observability.test.js index e0d06c0..61e789d 100644 --- a/test/unit/observability.test.js +++ b/test/unit/observability.test.js @@ -16,11 +16,14 @@ // log shim that redacts credentials and never throws at a dead collector. // // Ported from the canonical suite at xchain-hub/test/unit/observability.test.js. -// src/observability/ here is a verbatim vendored copy (parity is gated by a -// check across the vendored copies in CI), so this file runs -// the same assertions against xchain-sync's own copy, express version and -// Node engine. Behaviour changes belong in the canonical suite first; re-port -// rather than hand-editing, or the two drift apart silently. +// src/observability/ here is a verbatim vendored copy, vendored and verified by +// xchain-hub/bin/sync-observability.sh. Parity is gated in the HUB, not here: +// the hub's pre-push gate (bin/ci-full.sh) and the drift-guards job of its +// ci.yml both run that script with --check against all six consumers, so a +// hand-edit to this copy reddens the hub. This file runs the same assertions +// against xchain-sync's own copy, express version and Node engine. +// Behaviour changes belong in the canonical suite first; re-port rather than +// hand-editing, or the two drift apart silently. const { expect } = require('chai'); const express = require('express'); @@ -372,6 +375,50 @@ describe('observability/logShipper', function () { await log.stop(); expect(log.timer).to.equal(null); }); + + // Exercises the real _post/fetch path. Every other test here injects a + // transport, which is why the unreleased response body below went unseen. + it('releases the response body so a stalled collector cannot pin the socket', async function () { + this.timeout(5000); + let closed = false; + const sockets = new Set(); + const server = http.createServer((req, res) => { + req.resume(); + // Answer with headers and a first chunk, then never end the body. + req.on('end', () => { res.writeHead(200); res.write('ack'); }); + res.socket.on('close', () => { closed = true; }); + }); + server.on('connection', (s) => { sockets.add(s); s.on('close', () => sockets.delete(s)); }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address(); + + const log = createLogShipper({ + service: 'svc', + env: { + LOG_SHIP_ENABLED: '1', + LOG_SHIP_URL: `http://127.0.0.1:${port}/logs`, + LOG_SHIP_BATCH_SIZE: '1', + LOG_SHIP_TIMEOUT_MS: '400' + }, + console: fakeConsole() + }); + + try { + log.info('one'); + await log.flush(); + // fetch() resolves on headers and the abort timer is cleared with it, + // so an unreleased body leaves nothing that will ever close this + // socket. Measured: released in under 2ms, unreleased still open at 3s. + for (let i = 0; i < 100 && !closed; i++) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(closed).to.equal(true); + } finally { + await log.stop(); + for (const s of sockets) s.destroy(); + await new Promise((resolve) => server.close(resolve)); + } + }); }); describe('observability/installObservability', function () { diff --git a/test/unit/pinnedValidators.test.js b/test/unit/pinnedValidators.test.js index e6a1948..161ddae 100644 --- a/test/unit/pinnedValidators.test.js +++ b/test/unit/pinnedValidators.test.js @@ -93,3 +93,82 @@ describe('pinnedValidators: rotation seed checkpoint @regression', function(){ assert.strictEqual(pinned.getPinnedCheckpoint('BTC', 'regtest'), null); }); }); + +// The getters answer null for an override nobody set AND for one the operator set and +// got wrong, and with every baked-in pin null those two states reach +// ClientSync._verifyCheckpointQuorum identically: `if(!validators||!validators.length) +// return;` skips checkpoint authentication on a replica whose operator armed +// VERIFY_CHECKPOINT_QUORUM. assertPinnedEnvOverrides is the startup refusal that keeps +// an explicit-but-invalid value from reading as "no pin configured". +describe('pinnedValidators: assertPinnedEnvOverrides @regression', function(){ + const VKEY = 'CHECKPOINT_VALIDATORS_BTC_REGTEST'; + const SKEY = 'CHECKPOINT_SEED_BTC_REGTEST'; + const goodSet = [{ pubkey: 'aa'.repeat(32), weight: '100', source: 'S1' }]; + const goodSeed = { + block_index: 1000, snapshot_block: 994, checkpoint_seq: 0, + state_root: 'ab'.repeat(32), state_root_version: 1, + block_merkle_root: 'cd'.repeat(32), block_merkle_version: 1 + }; + + function assertRefuses(env, needle){ + assert.throws( + () => pinned.assertPinnedEnvOverrides(env), + (e) => e instanceof Error + && /Refusing to start/.test(e.message) + && e.message.indexOf(needle) !== -1, + 'expected a refusal naming ' + needle + ' for ' + JSON.stringify(env) + ); + } + + it('does not throw when no override is present at all', function(){ + pinned.assertPinnedEnvOverrides({ SYNC_MODE: 'client' }); + }); + + it('treats an unset or empty-string override as absent, not invalid', function(){ + pinned.assertPinnedEnvOverrides({ [VKEY]: '', [SKEY]: '' }); + }); + + it('accepts well-formed validator and seed overrides', function(){ + pinned.assertPinnedEnvOverrides({ [VKEY]: JSON.stringify(goodSet), [SKEY]: JSON.stringify(goodSeed) }); + }); + + it('refuses a validator override that is not JSON, not an array, empty, or wrongly shaped', function(){ + assertRefuses({ [VKEY]: 'not json' }, VKEY); + assertRefuses({ [VKEY]: JSON.stringify({ pubkey: 'aa' }) }, 'is not a JSON array'); + assertRefuses({ [VKEY]: JSON.stringify([]) }, 'is an empty array'); + assertRefuses({ [VKEY]: JSON.stringify([{ pubkey: 'aa', weight: 100, source: 'S1' }]) }, 'string `weight`'); + assertRefuses({ [VKEY]: JSON.stringify([{ pubkey: 'aa', weight: '100' }]) }, 'string `source`'); + }); + + it('refuses a seed override that is not JSON, not an object, or missing a required field', function(){ + assertRefuses({ [SKEY]: 'not json' }, SKEY); + assertRefuses({ [SKEY]: JSON.stringify([goodSeed]) }, 'is not a JSON object'); + assertRefuses({ [SKEY]: JSON.stringify(Object.assign({}, goodSeed, { state_root: undefined })) }, '`state_root`'); + assertRefuses({ [SKEY]: JSON.stringify(Object.assign({}, goodSeed, { block_index: '1000' })) }, '`block_index`'); + }); + + it('names every offender, not just the first', function(){ + try { + pinned.assertPinnedEnvOverrides({ [VKEY]: 'not json', [SKEY]: 'not json' }); + assert.fail('expected a refusal'); + } catch(e){ + assert.ok(e.message.indexOf(VKEY) !== -1, 'names the validator override'); + assert.ok(e.message.indexOf(SKEY) !== -1, 'names the seed override'); + } + }); + + it('ignores a prefixed name with no CHAIN_NETWORK shape (the getters never read it)', function(){ + pinned.assertPinnedEnvOverrides({ CHECKPOINT_VALIDATORS_BADKEY: 'not json' }); + }); + + it('does not reject a chain/network pair that is absent from the baked-in map', function(){ + pinned.assertPinnedEnvOverrides({ CHECKPOINT_VALIDATORS_BITCOIN_MAINNET: JSON.stringify(goodSet) }); + }); + + it('defaults to process.env when called with no argument', function(){ + process.env[VKEY] = 'not json'; + try { + assert.throws(() => pinned.assertPinnedEnvOverrides(), /Refusing to start/); + } finally { delete process.env[VKEY]; } + }); +}); diff --git a/test/unit/recoveryRewards.test.js b/test/unit/recoveryRewards.test.js index 154647c..7804525 100644 --- a/test/unit/recoveryRewards.test.js +++ b/test/unit/recoveryRewards.test.js @@ -94,6 +94,21 @@ describe('collectRedrivenValidatorRewards', function(){ assert.strictEqual(out.length, 2); }); + // Same five-column identity the 2026-08-24 migration put in reward_unique: the archive + // leg's round_reference is MATCH_BATCH_SEQ, a dense hub counter a rebase reissues, so + // round_qualifier (snapshot_block) is the only column separating two real rewards. A + // four-column key collapses them and the second never reaches the replica. + it('distinguishes archive rows that differ only in round_qualifier', async function(){ + let db = { doQuery: async () => [ + row({ reward_type: 'anchor_archive', round_reference: 42, round_qualifier: 100 }), + row({ reward_type: 'anchor_archive', round_reference: 42, round_qualifier: 200 }) + ] }; + let out = await collectRedrivenValidatorRewards(db, 150, 150); + assert.strictEqual(out.length, 2, + 'round_qualifier is part of reward_unique; collapsing on it drops a real reward'); + assert.deepStrictEqual(out.map(r => r.round_qualifier).sort(), [100, 200]); + }); + it('swallows ONLY a schema gap (1146/1054); a transient fault propagates so the block is retried', async function(){ // Regression: a bare catch here made both callers' isSchemaGapError gates dead // code (ServerPoller freezes the cursor, SnapshotBuilder aborts the stream), so a diff --git a/test/unit/replicaFreshness.test.js b/test/unit/replicaFreshness.test.js index 312d239..341db87 100644 --- a/test/unit/replicaFreshness.test.js +++ b/test/unit/replicaFreshness.test.js @@ -17,6 +17,8 @@ // class; the server path fronting a replica had nothing. const assert = require('assert'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire'); const { applyReplicaFreshness } = require('../../src/api'); const config = require('../../src/config'); @@ -47,6 +49,69 @@ describe('/status replication freshness', function(){ assert.strictEqual(row.replica_stale, false, 'no signal is the pre-replica topology default'); }); + // A follower's own lag_blocks is computed against a height its SOURCE published. + // The source's status event says whether its own database was fit to publish that + // height; discarding it left the follower certifying an upstream whose SQL replica + // had stopped applying, because both of the server's heights freeze together and + // its heartbeats keep source_height_stale false. + describe('client row carries the upstream verdict', function(){ + function loadClientApi(){ + let prior = process.env.SYNC_MODE; + process.env.SYNC_MODE = 'client'; + let api = proxyquire('../../src/api', {}); + if(prior === undefined) delete process.env.SYNC_MODE; else process.env.SYNC_MODE = prior; + return api; + } + + function mockDb(){ + return { + dbName: 'replica_db', dbType: 'indexer', + getLastBlock: sinon.stub().resolves(100), + getBlockHashRow: sinon.stub().resolves({ block_index: 100, block_time: 1, + ledger_hash: 'a', actions_hash: 'b', contract_hash: 'c' }), + getTableCount: sinon.stub().resolves(0), + listExistingTables: sinon.stub().resolves(new Set()) + }; + } + + function mockService(upstreamReplica){ + return { + getClientSyncState: () => ({ + lastKnownServerBlock: 100, sourceHeightStale: false, upstreamReplica, + halted: false, haltInfo: null, truncated: false, bootstrapBase: null, + sourceQuorum: 1, sourcesConfigured: 1, sourcesActive: 1, + sourcesAgreeing: 1, sourcesEvicted: [] + }) + }; + } + + afterEach(function(){ sinon.restore(); }); + + it('publishes the upstream verdict beside a lag_blocks of 0', async function(){ + let { buildStatusRow } = loadClientApi(); + let row = await buildStatusRow( + mockService({ stale: true, secondsBehind: 900, sourceHeight: 140 }), + mockDb(), 'indexer', 'bitcoin', 'mainnet'); + + // The exact green shape the incident showed, now qualified. + assert.strictEqual(row.lag_blocks, 0); + assert.strictEqual(row.source_height_stale, false); + assert.strictEqual(row.upstream_replica_stale, true); + assert.strictEqual(row.upstream_replica_seconds_behind, 900); + assert.strictEqual(row.upstream_source_height, 140); + }); + + it('is unknown (null), never false, when no source reported the fields', async function(){ + let { buildStatusRow } = loadClientApi(); + let row = await buildStatusRow( + mockService({ stale: null, secondsBehind: null, sourceHeight: null }), + mockDb(), 'indexer', 'bitcoin', 'mainnet'); + assert.strictEqual(row.upstream_replica_stale, null); + assert.strictEqual(row.upstream_replica_seconds_behind, null); + assert.strictEqual(row.upstream_source_height, null); + }); + }); + it('SYNC_REPLICA_MAX_LAG_S is configurable and defaults to 120s', function(){ let saved = process.env.SYNC_REPLICA_MAX_LAG_S; try { diff --git a/test/unit/rollback-coverage.test.js b/test/unit/rollback-coverage.test.js index 0bcfe74..2a3a267 100644 --- a/test/unit/rollback-coverage.test.js +++ b/test/unit/rollback-coverage.test.js @@ -360,6 +360,35 @@ describe('Rollback coverage guard @regression', function(){ 'cross-chain mirror reorg delete SQL drifted between xchain-sync/ClientRollback.js and xchain-indexer/rollback.js; keep them identical'); }); + // Cross-repo drift guard for the contract slash reorg-restore. Both the source + // (xchain-indexer/src/rollback.js) and the replica (xchain-sync/src/ClientRollback.js) + // copy back the highest orphaned contract_slash_debits.prev_amount for a mutated stake + // row. A predicate that picks a different debit on one side restores a different active + // stake there, and active stake drives staker weighting and quorum eligibility, so the + // two nodes fork. Both files carry the statement between // + // markers; this concatenates its string literals (the indexer spells them as template + // literals, the replica as double-quoted concatenation, and the interpolated table name + // drops out of both) and asserts whitespace-normalised equality. + it('contract slash-restore SQL is identical across xchain-indexer and xchain-sync (cross-repo drift guard)', function(){ + const fs = require('fs'); + function slashRestoreSql(path){ + const src = fs.readFileSync(path, 'utf8'); + const m = src.match(/\/\/([\s\S]*?)\/\/<\/CONTRACT-SLASH-RESTORE-SQL>/); + assert.ok(m, `CONTRACT-SLASH-RESTORE-SQL markers not found in ${path}`); + const lits = m[1].match(/`[^`]*`|"(?:[^"\\]|\\.)*"/g) || []; + assert.ok(lits.length >= 2, `expected >=2 SQL literals in the marked block of ${path}, got ${lits.length}`); + return lits.map(l => l.slice(1, -1)).join('').replace(/\s+/g, ' ').trim(); + } + const syncPath = require('path').resolve(__dirname, '../../src/ClientRollback.js'); + const indexerPath = indexerFile('src/rollback.js'); + if(!requireSibling(this, indexerPath)) return; + const sql = slashRestoreSql(syncPath); + assert.ok(/CAST\(e\.prev_amount AS DECIMAL\(60,18\)\) > CAST\(d\.prev_amount AS DECIMAL\(60,18\)\)/.test(sql), + 'the restore must pick the highest orphaned prev_amount; the position columns invert under a nested EXECUTE'); + assert.strictEqual(sql, slashRestoreSql(indexerPath), + 'contract slash-restore SQL drifted between xchain-sync/ClientRollback.js and xchain-indexer/rollback.js; keep them identical'); + }); + // Cross-repo drift guard for the light-client stakes_root query (SPV spec sec.4.1). // The follower rebuilds the BTC stakes_root from db._stakeWeightsSql; it MUST stay // byte-identical to xchain-indexer/src/db.js _stakeWeightsSql, or the follower's @@ -607,8 +636,25 @@ describe('Rollback coverage guard @regression', function(){ `${f} does not call collectDerivedAnchorRewards; its replication channel drops derived anchor/archive rewards`); } const applier = norm(fs.readFileSync(pathMod.resolve(__dirname, '../../src/ClientApplier.js'), 'utf8')); - assert.ok(/DELETE vr FROM validator_rewards vr JOIN anchor_reward_reconcile_log d ON d\.source_id = vr\.source_id AND d\.signing_pubkey_id = vr\.signing_pubkey_id AND d\.reward_type = vr\.reward_type AND d\.round_reference <=> vr\.round_reference/.test(applier), - 'ClientApplier.js must mirror the reconcile DELETE from the replicated pre-image log (forward twin of the RB-ANCHOR restore)'); + assert.ok(/DELETE vr FROM validator_rewards vr JOIN anchor_reward_reconcile_log d ON d\.source_id = vr\.source_id AND d\.signing_pubkey_id = vr\.signing_pubkey_id AND d\.reward_type = vr\.reward_type AND d\.round_reference <=> vr\.round_reference AND d\.round_qualifier = vr\.round_qualifier/.test(applier), + 'ClientApplier.js must mirror the reconcile DELETE from the replicated pre-image log (forward twin of the RB-ANCHOR restore) on the FULL five-column reward identity; without round_qualifier the keyed delete also reaches the other archive snapshot\'s surviving reward'); + // RB-ANCHOR restore parity on that same identity. The source twin + // (xchain-indexer/src/rollback.js) names round_qualifier in BOTH the INSERT column + // list and the projection, so the replica must too: without it the restored loser + // lands under the schema default 0, a different row from the one the reconcile + // deleted, and INSERT IGNORE either swallows it or lands a wrong-identity duplicate. + assert.ok(/INSERT IGNORE INTO validator_rewards \(source_id, signing_pubkey_id, reward_type, round_reference, round_qualifier, amount, block_index, derive_block_index\) SELECT .*d\.round_qualifier/.test(rbSync), + 'ClientRollback.js RB-ANCHOR restore must carry round_qualifier in both the column list and the projection, mirroring xchain-indexer/src/rollback.js'); + // The four JS payload-merge dedup keys ride the same identity: a four-column key + // treats two distinct archive rewards as one and drops the second from the payload + // before it ever reaches a replica. + for(const f of ['../../src/ServerPoller.js', '../../src/SnapshotBuilder.js', + '../../src/derivedRewards.js', '../../src/recoveryRewards.js']){ + // Collapse whitespace only (the quotes around ':' are part of the key text). + const src = fs.readFileSync(pathMod.resolve(__dirname, f), 'utf8').replace(/\s+/g, ' '); + const stale = src.match(/r\.reward_type \+ ':' \+ r\.round_reference(?! \+ ':' \+ r\.round_qualifier)/g); + assert.ok(!stale, `${f} still builds a reward dedup key on the pre-migration four columns (${stale && stale.length} site(s)); reward_unique carries round_qualifier`); + } }); // Bespoke-logic parity: anchor invalid_archive to unverified reset. When the final v2 @@ -637,6 +683,11 @@ describe('Rollback coverage guard @regression', function(){ // (${ARCHIVE_HEAD_VERSIONS_SQL}) and the sync side's string concat. { name: 'archive-head version predicate (shared v1+v6 constant)', re: /WHERE p\.version (\$\{)?ARCHIVE_HEAD_VERSIONS_SQL\}? AND p\.action_index < \?/ }, + // The publisher-scope term is spliced from the shared activation module on both + // sides, between the chunk-status join and the reset join. A side that drops it + // resets under a different batch key than its twin the moment the flag day arms. + { name: 'publisher author-scope splice', + re: /cs\.status = valid (\$\{)?authorScope\}? JOIN index_statuses us/ }, ]; for(const [label, p] of [['ClientRollback.js (replica)', syncPath], ['rollback.js (source)', indexerPath]]){ const src = norm(fs.readFileSync(p, 'utf8')); @@ -646,6 +697,34 @@ describe('Rollback coverage guard @regression', function(){ } }); + // The publisher-scope flag day is one file, twinned. A per-network height that differs + // between source and replica is a fleet split at the reorg the gate governs. + it('archive_rollback_author_scope_activation.js is byte-identical across xchain-indexer and xchain-sync', function(){ + const fs = require('fs'), pathMod = require('path'); + const rel = 'src/archive_rollback_author_scope_activation.js'; + const indexerPath = indexerFile(rel); + if(!requireSibling(this, indexerPath)) return; + const syncPath = pathMod.resolve(__dirname, '../..', rel); + assert.strictEqual(fs.readFileSync(syncPath, 'utf8'), fs.readFileSync(indexerPath, 'utf8'), + 'the publisher-scope activation must be the same file on both sides; a divergent height ' + + 'makes source and replica reset a reorg under different batch keys'); + }); + + // An omitted network reads as inactive, which is only correct while every threshold is + // inert. Arming one without first making the network mandatory would leave every + // un-wired construction site quietly on the legacy unscoped rule. + it('cannot arm the publisher scope while ClientRollback still accepts an omitted network', function(){ + const { ARCHIVE_ROLLBACK_AUTHOR_SCOPE_ACTIVATION } = require('../../src/archive_rollback_author_scope_activation'); + const INERT = 9999999999; + const armed = Object.keys(ARCHIVE_ROLLBACK_AUTHOR_SCOPE_ACTIVATION) + .filter(n => ARCHIVE_ROLLBACK_AUTHOR_SCOPE_ACTIVATION[n] !== INERT); + if(!armed.length) return; + assert.throws(() => new ClientRollback({ dbType: 'indexer' }, new Utility(), 'DOGE'), + /network/i, + 'arming ' + armed.join(', ') + ' requires ClientRollback to demand a network: ' + + 'every construction site must be wired before a replica can run the scoped reset'); + }); + // The archive-head version set is defined ONCE (stateHash.js, twinned across // repos) and consumed by every parent-selecting predicate. Pin its value and the SQL // fragment shape, and pin the forward updatedRows class to the same constant so an @@ -729,6 +808,41 @@ describe('Rollback coverage guard @regression', function(){ 'updatedRows.js must also select polls by callback_due_block with a fired stamp (the deferred callback fire is an in-place UPDATE at the due block)'); }); + // Forward parity for DELEGATE v1 signing-key rotations: the materialization sweep + // rewrites signing_pubkey_id IN PLACE on surviving stake-ledger rows whose action_index + // sits below the window, so only the contract_delegation_rotations journal pins the + // rewrite to a block. Dropping a table or the journal join silently stops replicating + // the rotation and a follower hands contracts a stale staker set. Pin the class table + // list and its journal-window predicate in updatedRows.js. + it('updated_rows carries the DELEGATE v1 rotation rewrite keyed by the rotations journal window', function(){ + const { ROTATION_TABLES } = require('../../src/updatedRows'); + assert.deepStrictEqual(ROTATION_TABLES, ['contract_stakes', 'contract_unstakes'], + 'updated_rows must track the rotation rewrite on both contract stake tables'); + const fs = require('fs'), pathMod = require('path'); + const src = fs.readFileSync(pathMod.resolve(__dirname, '../../src/updatedRows.js'), 'utf8') + .replace(/[`"']/g, ' ').replace(/\s+\+\s+/g, ' ').replace(/\s+/g, ' '); + assert.ok(/JOIN contract_delegation_rotations r ON r\.stake_action_index = t\.action_index WHERE r\.target_table = \? AND r\.block_index BETWEEN \? AND \?/.test(src), + 'updatedRows.js must select rotated stake rows through the contract_delegation_rotations journal keyed by target_table and block_index window (the same journal ClientRollback restores from)'); + }); + + // Forward parity for the BET in-place flips: the closed latch, the terminal flip and + // settlement each mutate a surviving bet_feeds / bets row the action-scoped stream + // cannot reach, stamping a block column. Dropping a stamp silently stops replicating + // that flip and a follower keeps a stale feed or bet status. Pin the class spec list + // and the per-stamp window predicate it drives in updatedRows.js. + it('updated_rows carries the BET status flips keyed by their stamp columns', function(){ + const { BET_STATUS_SPECS } = require('../../src/updatedRows'); + assert.deepStrictEqual(BET_STATUS_SPECS, [ + { table: 'bet_feeds', stamps: ['closed_block', 'terminal_block'] }, + { table: 'bets', stamps: ['settled_block'] } + ], 'updated_rows must track the feed closed/terminal stamps and the bet settlement stamp'); + const fs = require('fs'), pathMod = require('path'); + const src = fs.readFileSync(pathMod.resolve(__dirname, '../../src/updatedRows.js'), 'utf8') + .replace(/[`"']/g, ' ').replace(/\s+\+\s+/g, ' ').replace(/\s+/g, ' '); + assert.ok(/for\(let spec of BET_STATUS_SPECS\)\{ try \{ let where = spec\.stamps\.map\(col => col BETWEEN \? AND \? \)\.join\( OR \);/.test(src), + 'updatedRows.js must select each BET class by every stamp column landing in the window, OR-joined so a feed that latches and goes terminal in one window is still carried'); + }); + // The state_hash (replication-integrity 4th hash) is computed on BOTH sides from // src/stateHash.js: the indexer stores it at index-time, the follower recomputes it // apply-time and halts on mismatch. The two copies are a byte-aligned twin (separate @@ -792,7 +906,12 @@ describe('Rollback coverage guard @regression', function(){ // different stakes_root at an armed height - the same class of fork the source-cap // twin above guards. It also carries the schema-drift contract both services' // startup checks read, so one definition of "undrifted" serves both fleets. - for(const twin of ['merkle.js', 'state_commitment_activation.js', 'swq_source_cap_activation.js', 'state_key_collation_activation.js', 'stake_weight_collation_activation.js', 'state_subtree_activation.js', 'contractStateSubtree.js', 'escrowLeafSubtree.js', 'tableLifecycle.js']){ + // utf8mb4Columns.js is the widen set for the columns that ingest raw wire fields + // (contracts.code and the grammar-constrained fields). The source converges through a + // dated migration and the follower through ensureReplicaUtf8mb4Columns, so a drifted + // copy means an origin that accepts a 4-byte character and a replica that halts on it + // with errno 1366 - a fleet-wide follower halt with no schema error upstream. + for(const twin of ['merkle.js', 'state_commitment_activation.js', 'swq_source_cap_activation.js', 'state_key_collation_activation.js', 'stake_weight_collation_activation.js', 'state_subtree_activation.js', 'contractStateSubtree.js', 'escrowLeafSubtree.js', 'tableLifecycle.js', 'utf8mb4Columns.js']){ it(twin + ' is byte-identical across xchain-sync and xchain-indexer (cross-repo twin)', function(){ const fs = require('fs'), pathMod = require('path'); const syncPath = pathMod.resolve(__dirname, '../../src/' + twin); @@ -803,6 +922,30 @@ describe('Rollback coverage guard @regression', function(){ }); } + // Lockstep, read from the other end: the replica widen and the source MIGRATION must + // land the same column shape. Byte-identity of the module alone does not prove that - + // the two copies could agree with each other while the indexer's dated migration says + // something else, leaving the origin on one charset and every follower on another. + // Every entry's MODIFY clause must therefore appear verbatim in one of the sibling's + // dated migration files, which is exactly what ensureReplicaUtf8mb4Columns issues. + it('every utf8mb4 widen entry is carried by a dated xchain-indexer migration (source/replica lockstep)', function(){ + const fs = require('fs'), pathMod = require('path'); + const widenSet = require('../../src/utf8mb4Columns'); + const migDir = indexerFile(pathMod.join('src', 'sql', 'migrations')); + if(!requireSibling(this, migDir)) return; + const ledger = fs.readdirSync(migDir).filter(f => f.endsWith('.sql')) + .map(f => fs.readFileSync(pathMod.join(migDir, f), 'utf8')).join('\n'); + const missing = []; + for(const entry of widenSet.UTF8MB4_RAW_FIELD_COLUMNS){ + if(!ledger.includes(widenSet.modifyClause(entry))) + missing.push(' ' + entry.table + '.' + entry.column + ': ' + widenSet.modifyClause(entry)); + } + assert.deepStrictEqual(missing, [], + 'These columns are widened on the replica but no dated xchain-indexer migration MODIFYs them to ' + + 'the same shape, so the source and its followers converge on DIFFERENT column charsets:\n' + + missing.join('\n')); + }); + // The gate's own conformance suite is a twin too: it is what proves the carrier // is inert (identical state_root to the two-sub-root v1 assembly) and that the // slot list matches merkle.STATE_SUBTREES. Both repos must assert the same diff --git a/test/unit/security/configuration/dependency-advisories.test.js b/test/unit/security/configuration/dependency-advisories.test.js index 1c5cee8..9d2903a 100644 --- a/test/unit/security/configuration/dependency-advisories.test.js +++ b/test/unit/security/configuration/dependency-advisories.test.js @@ -89,6 +89,23 @@ describe('Security: remediated dependency advisories @regression @tier4', functi // reach here, not dev-only: express-rate-limit and geoip-lite both parse the // client request IP, so a padded or mapped form could key a different // rate-limit bucket than its canonical address. + + // mariadb <=3.5.2 is the production DB driver every service here opens its + // pool with, not a dev-only reach. GHSA-cqhc-2h57-wpxf (HIGH) sends the + // password in the clear to a man in the middle even when the pool asked for + // `ssl: true`, because the connector falls back to a plaintext handshake + // rather than failing closed; GHSA-42r5-vhpq-m858 is the same + // cleartext-credential exposure stated as its own advisory; and + // GHSA-g5xc-5w98-jfvm is SQL injection through Buffer parameter escaping + // under the big5, gbk, sjis, cp932 and gb18030 client charsets. Measured + // exposure in this topology is nil today (no repo passes `ssl:` to a pool, + // every DB connection is host-local, and none selects one of those + // charsets), which is why the fix rode the ordinary fleet path instead of a + // hotfix. The guard is what stops a lockfile refresh, or the first service + // that does dial a remote database over TLS, from landing back inside the + // range. 3.5.3 is the patch, and it also moves the driver's own lru-cache + // onto the 11.x line, so a splice that bumps mariadb and leaves lru-cache + // at 10.4.3 has not actually installed the fixed driver. const advisories = [ { name: 'fast-uri', minSafe: [3, 1, 5], majorSeries: 3 }, { name: 'brace-expansion', minSafe: [5, 0, 9], majorSeries: 5 }, @@ -103,7 +120,8 @@ describe('Security: remediated dependency advisories @regression @tier4', functi { name: 'shell-quote', minSafe: [1, 9, 0], majorSeries: 1 }, { name: 'form-data', minSafe: [4, 0, 6], majorSeries: 4 }, { name: 'tmp', minSafe: [0, 2, 6], majorSeries: 0 }, - { name: 'ip-address', minSafe: [10, 3, 1], majorSeries: 10 } + { name: 'ip-address', minSafe: [10, 3, 1], majorSeries: 10 }, + { name: 'mariadb', minSafe: [3, 5, 3], majorSeries: 3 } ]; // Compares dotted numeric version triples without pulling in semver. @@ -201,4 +219,18 @@ describe('Security: remediated dependency advisories @regression @tier4', functi assert.ok(cmp(parse(axios.VERSION), [1, 18, 0]) >= 0, `installed axios is ${axios.VERSION}, inside the vulnerable range (fixed in 1.18.0)`); }); + + // Same reasoning as ADV-5, for the one entry in this list every service + // opens a socket with. mariadb's `exports` block hides its own + // package.json from require() and the module exports no version constant, + // so read the installed manifest off disk instead. + it('ADV-10: the installed mariadb reports a patched runtime version', function () { + if (!lockEntries('mariadb').length) return this.skip(); + const manifest = path.join(root, 'node_modules', 'mariadb', 'package.json'); + if (!fs.existsSync(manifest)) return this.skip(); + + const installed = JSON.parse(fs.readFileSync(manifest, 'utf8')).version; + assert.ok(cmp(parse(installed), [3, 5, 3]) >= 0, + `installed mariadb is ${installed}, inside the vulnerable range (fixed in 3.5.3)`); + }); }); diff --git a/test/unit/sibling-coverage.test.js b/test/unit/sibling-coverage.test.js index 80bc7c7..a0e92b3 100644 --- a/test/unit/sibling-coverage.test.js +++ b/test/unit/sibling-coverage.test.js @@ -63,7 +63,7 @@ const SIBLINGS = [ { repo: 'xchain-indexer', envs: ['XCHAIN_INDEXER_DIR'], marker: 'src', altEnvs: ['XCHAIN_INDEXER_SQL_PATH'], - guards: 'rollback coverage, the block-hash and protocol-address twins, and the validator-set parity' }, + guards: 'rollback coverage, the block-hash and protocol-address twins, the validator-set parity, and the stream scope-column binding' }, { repo: 'xchain-hub', envs: ['XCHAIN_HUB_DIR'], marker: path.join('src', 'coins'), guards: 'vendored coins-registry byte-identity (BTC/LTC/DOGE/index/consensus_pin)' }, diff --git a/test/unit/streamScopeColumns.test.js b/test/unit/streamScopeColumns.test.js new file mode 100644 index 0000000..a7ae96c --- /dev/null +++ b/test/unit/streamScopeColumns.test.js @@ -0,0 +1,170 @@ +/******************************************************************** + * + * Copyright © 2025-2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC - https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ******************************************************************** + * test/unit/streamScopeColumns.test.js + * + * Binds every streaming declaration in src/tableLifecycle.js to the DDL of the + * table it declares. A 'stream:block' entry must own its blockKey column, a + * 'stream:action' entry must own action_index, and a 'stream:index' entry must + * own the id cursor replicatedTables.lookupCursorColumn pages by. + * + * Why a guard rather than a runtime check: the readers build the scope column + * into their SQL, so a column the registry names and the schema lacks raises + * MariaDB errno 1054, and EVERY forward channel classifies 1054 as "the source + * runs an older schema" and drops the table from the payload without a log line + * (ServerPoller.isSchemaGapError, SnapshotBuilder's per-table catch, + * BlockHasher.computeTableContentChecksums). The table then stays in the + * /status completeness count, in the content-parity plan and in both rollback + * sets while none of its rows ever move. That is what happened to rollcalls and + * rollcall_absences, which are keyed by close_block: declared, counted, deleted + * on reorg, and never delivered on any forward channel. + * + * This guard is what makes that class fail at build time instead. + */ + +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); + +const lifecycle = require('../../src/tableLifecycle'); +const replicatedTables = require('../../src/replicatedTables'); + +// The registry declares indexer-schema tables plus the handful xchain-sync owns, +// so those are the two DDL trees to scan. The decoder topology is declared +// literally in replicatedTables.js and is not generated from this registry. +// +// The indexer half is a sibling checkout, resolved the way generatedColumns.test.js +// resolves it: absent in a standalone checkout, and XCHAIN_REQUIRE_SIBLINGS=1 turns +// green-by-skip into a failure. +const INDEXER_SQL = process.env.XCHAIN_INDEXER_SQL_PATH + || path.resolve(__dirname, '..', '..', '..', 'xchain-indexer', 'src', 'sql'); +const SYNC_SQL = path.resolve(__dirname, '..', '..', 'src', 'sql'); +const SIBLING_REQUIRED = process.env.XCHAIN_REQUIRE_SIBLINGS === '1'; + +// The scope column each replication class reads by. null means the class carries +// no scope column of its own (stream:special rides the completeness count only). +function scopeColumnFor(entry){ + if(entry.replication === 'stream:block') return lifecycle.blockKey(entry.table); + if(entry.replication === 'stream:action') return 'action_index'; + if(entry.replication === 'stream:index') return replicatedTables.lookupCursorColumn(entry.table); + return null; +} + +// Column names of every CREATE TABLE in a DDL directory, keyed by table. +// +// Line comments are stripped first: the DDL documents its keys in prose and a +// comment naming a column must not be read as declaring one. The body is cut at +// the closing paren of the column list, and index clauses (PRIMARY KEY, KEY, ...) +// are dropped so a table that merely INDEXES a column is not credited with having it. +const INDEX_CLAUSE = /^(PRIMARY|UNIQUE|KEY|INDEX|CONSTRAINT|FOREIGN|FULLTEXT|SPATIAL|CHECK)$/i; + +function deriveColumns(sqlDir){ + const found = {}; + for(const file of fs.readdirSync(sqlDir).filter(f => f.endsWith('.sql'))){ + const body = fs.readFileSync(path.join(sqlDir, file), 'utf8').replace(/^\s*--.*$/gm, ''); + const m = body.match(/CREATE TABLE(?:\s+IF NOT EXISTS)?\s+`?(\w+)`?\s*\(([\s\S]*?)\n\)/i); + if(!m) continue; + if(found[m[1]]) continue; // first definition wins, as the loader sees it + const columns = []; + for(const line of m[2].split('\n')){ + const c = line.match(/^\s+`?([a-z_][a-z0-9_]*)`?\s+[A-Za-z]/); + if(c && !INDEX_CLAUSE.test(c[1])) columns.push(c[1]); + } + found[m[1]] = columns; + } + return found; +} + +describe('streamScopeColumns: every streamed table owns the column it is scoped by @regression', function(){ + + it('declares a scope column the DDL actually has', function(){ + if(!fs.existsSync(INDEXER_SQL)){ + if(SIBLING_REQUIRED) + throw new Error('XCHAIN_REQUIRE_SIBLINGS=1 but the indexer schema is absent: ' + INDEXER_SQL); + this.skip(); + return; + } + const columns = Object.assign({}, deriveColumns(INDEXER_SQL), deriveColumns(SYNC_SQL)); + + // The scan must actually find tables, or a CREATE TABLE regex that stopped + // matching would make every assertion below vacuous and this test green. + assert.ok(Object.keys(columns).length > 50, + 'derived only ' + Object.keys(columns).length + ' tables from ' + INDEXER_SQL + + ' and ' + SYNC_SQL + '; the DDL scan is broken'); + + let checked = 0; + for(const entry of lifecycle.TABLES){ + const need = scopeColumnFor(entry); + if(need === null) continue; + const cols = columns[entry.table]; + assert.ok(cols, entry.table + ' is declared ' + entry.replication + + ' but no CREATE TABLE for it was found in the indexer or sync DDL'); + assert.ok(cols.indexOf(need) !== -1, + entry.table + ' is declared ' + entry.replication + ' and scoped by `' + need + + '`, which its DDL does not declare (columns: ' + cols.join(', ') + '). ' + + 'The reader would raise errno 1054 and every forward channel swallows that as an ' + + 'older source schema, so the table would ship un-replicated while still counted complete.'); + checked++; + } + + // Pin the coverage too: if the registry stopped classifying tables as streamed, + // the loop above would assert nothing and still pass. + assert.ok(checked > 80, 'only ' + checked + ' streamed entries were checked; expected the whole registry'); + }); + + it('resolves the block scope column from the registry, defaulting to block_index', function(){ + // The default is what makes the field a no-op for the other block-scoped + // tables, so it is pinned here rather than left implied. + assert.strictEqual(lifecycle.blockKey('blocks'), 'block_index'); + assert.strictEqual(lifecycle.blockKey('contract_state'), 'block_index'); + assert.strictEqual(lifecycle.blockKey('sync_meta'), 'block_index'); + assert.strictEqual(lifecycle.blockKey('not_a_table'), 'block_index'); + // The two the class name lied about. + assert.strictEqual(lifecycle.blockKey('rollcalls'), 'close_block'); + assert.strictEqual(lifecycle.blockKey('rollcall_absences'), 'close_block'); + }); + + it('reads the two close_block tables by close_block on every forward channel', function(){ + // Source-text guard, in the tableContentParity.test.js style: these three + // readers cannot be exercised without a live DB, and each one hard-coding + // `block_index` is exactly how the tables shipped un-replicated. + const read = (p) => fs.readFileSync(path.resolve(__dirname, '..', '..', p), 'utf8'); + + const db = read('src/db.js'); + const blockScoped = db.slice(db.indexOf('async getBlockScopedRows('), db.indexOf('async getActionScopedRows(')); + assert.ok(/lifecycle\.blockKey\(table\)/.test(blockScoped), + 'getBlockScopedRows must scope by the registry column; the literal block_index ' + + 'raised 1054 on rollcalls/rollcall_absences and ServerPoller swallowed it'); + assert.ok(!/WHERE block_index = \?/.test(blockScoped), + 'getBlockScopedRows still carries the hard-coded block_index predicate'); + + const window = db.slice(db.indexOf('async getContentWindowRows('), db.indexOf('async getMaxRowId(')); + assert.ok(/lifecycle\.blockKey\(table\)/.test(window), + 'the content-parity block window must use the registry scope column, or the two ' + + 'tables it cannot read stay reported as covered'); + + // Only the INDEXER branch: the decoder topology is declared literally in + // replicatedTables.js, not generated from this registry, so its block_index + // range is correct as written and must not be swept up by this assertion. + const snapshot = read('src/SnapshotBuilder.js'); + const start = snapshot.indexOf('if(indexerBlockScoped.has(table)){'); + assert.ok(start !== -1, 'the incremental catch-up indexer branch moved; this guard cannot see it'); + const branch = snapshot.slice(start, snapshot.indexOf('indexerFullDump.has(table)', start)); + assert.ok(/tableLifecycle\.blockKey\(table\)/.test(branch), + 'the incremental catch-up branch must use the registry scope column'); + assert.ok(!/WHERE block_index >= \?/.test(branch), + 'the incremental catch-up branch still carries the hard-coded block_index range'); + }); +}); diff --git a/test/unit/tableContentParity.test.js b/test/unit/tableContentParity.test.js index 3e68dd8..9461eea 100644 --- a/test/unit/tableContentParity.test.js +++ b/test/unit/tableContentParity.test.js @@ -160,7 +160,13 @@ describe('Advisory table-content parity', function(){ // would make an honest replica fail forever. const src = require('fs').readFileSync(require('path').join(__dirname, '../../src/db.js'), 'utf8'); const body = src.slice(src.indexOf('async getContentWindowRows('), src.indexOf('async getMaxRowId(')); - assert.ok(/block_index IS NOT NULL AND block_index BETWEEN/.test(body), + // The scope column is now the registry's (lifecycle.blockKey), so the guard + // pins the PROPERTY (the same column, NULL-excluded, then range-bounded) + // rather than the literal 'block_index' a hard-coded query would assume. + assert.ok(/lifecycle\.blockKey\(table\)/.test(body), + 'getContentWindowRows must window by the registry scope column; the literal block_index ' + + 'raised errno 1054 on the close_block-keyed tables and the caller swallowed it'); + assert.ok(/" \+ key \+ " IS NOT NULL AND " \+ key \+ " BETWEEN/.test(body), 'getContentWindowRows lost its NULL-block exclusion; benign out-of-band rows would false-alarm'); assert.ok(/doQueryStrict/.test(body), 'getContentWindowRows must read strictly: a fail-soft [] would read as "table has no rows" on both sides'); diff --git a/test/unit/utf8mb4-replica-widen.test.js b/test/unit/utf8mb4-replica-widen.test.js new file mode 100644 index 0000000..1e702d4 --- /dev/null +++ b/test/unit/utf8mb4-replica-widen.test.js @@ -0,0 +1,176 @@ +// Copyright © 2025-2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC - https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. +// +// The replica half of the raw-wire-field utf8mb4 widen. +// +// The source indexer converges through a dated migration. sync runs none: a replica's +// tables are copied from the source's SHOW CREATE TABLE at bootstrap and addMissingColumns +// only ever ADDs a column, never retypes one. So the moment the widened ORIGIN accepts a +// 4-byte character (a contract whose source carries an emoji, an EXECUTE method name, a +// VOTE quorum), every replica built before the migration halts applying that block with +// errno 1366 while the source runs on. ensureReplicaUtf8mb4Columns is what closes that, +// and these are its guards: it must widen what is narrow, skip what is already wide, skip +// what is absent, and never take startup down when the DB says no. + +const assert = require('assert'); +const sinon = require('sinon'); + +const Database = require('../../src/db'); +const utf8mb4Columns = require('../../src/utf8mb4Columns'); + +function makeDb(dbType){ + const util = { isNull: (v) => v === null || v === undefined, logError: () => {} }; + return new Database('localhost', 3306, 'replica_db', 'u', 'p', util, dbType || 'indexer'); +} + +// information_schema rows for `table`, reporting every column in the widen set as +// utf8mb3 (narrow) or utf8mb4 (already converged). +function schemaRows(table, charset){ + return utf8mb4Columns.UTF8MB4_RAW_FIELD_COLUMNS + .filter(e => e.table === table) + .map(e => ({ COLUMN_NAME: e.column, CHARACTER_SET_NAME: charset })); +} + +const altersIn = (calls) => calls.filter(sql => /^ALTER\s+TABLE/i.test(sql)); + +describe('Database.ensureReplicaUtf8mb4Columns', function(){ + + let db; + + beforeEach(function(){ + db = makeDb(); + sinon.stub(console, 'log'); + sinon.stub(console, 'error'); + sinon.stub(console, 'warn'); + }); + + afterEach(async function(){ + sinon.restore(); + await db.close(); + }); + + it('widens every narrow column, one ALTER per table, with the twin module\'s exact clause', async function(){ + const calls = []; + sinon.stub(db, 'doQuery').callsFake(async (sql, args) => { + calls.push(String(sql)); + if(/information_schema\.columns/i.test(sql)) return schemaRows(args[1], 'utf8mb3'); + return []; + }); + + await db.ensureReplicaUtf8mb4Columns(); + + const alters = altersIn(calls); + const tables = [...utf8mb4Columns.byTable().keys()]; + assert.strictEqual(alters.length, tables.length, + 'expected exactly one ALTER per table in the widen set'); + + // Every entry's MODIFY must appear verbatim: the replica and the origin have to + // land on the same column shape, not merely on "some utf8mb4". + const joined = alters.join('\n'); + for(const entry of utf8mb4Columns.UTF8MB4_RAW_FIELD_COLUMNS){ + assert.ok(joined.includes(utf8mb4Columns.modifyClause(entry)), + 'the replica widen never issues ' + utf8mb4Columns.modifyClause(entry) + + ' for ' + entry.table + '.' + entry.column); + } + // contracts.code is the named half of the ledger item; pin it by name so a list + // edit cannot quietly drop the column the operator ruling called out. + assert.ok(/ALTER TABLE `contracts` MODIFY `code` MEDIUMTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL/ + .test(joined), 'contracts.code is not widened on the replica'); + }); + + it('is a no-op on a replica whose columns are already utf8mb4', async function(){ + const calls = []; + sinon.stub(db, 'doQuery').callsFake(async (sql, args) => { + calls.push(String(sql)); + if(/information_schema\.columns/i.test(sql)) return schemaRows(args[1], 'utf8mb4'); + return []; + }); + + await db.ensureReplicaUtf8mb4Columns(); + + assert.deepStrictEqual(altersIn(calls), [], + 'a converged replica must pay one information_schema read per table and nothing else'); + }); + + it('skips a table this replica does not carry', async function(){ + const calls = []; + sinon.stub(db, 'doQuery').callsFake(async (sql) => { + calls.push(String(sql)); + return []; // no rows = table absent + }); + + await db.ensureReplicaUtf8mb4Columns(); + + assert.deepStrictEqual(altersIn(calls), []); + }); + + it('widens only the columns the replica actually has, leaving the rest of the ALTER intact', async function(){ + // A replica mid-upgrade can carry the table but not every column of it (the column + // self-heal runs earlier and may have failed). Naming an absent column makes the + // whole ALTER errno 1054 and NOTHING in that table converges, so the pass must + // filter to the columns information_schema actually reports. + const calls = []; + sinon.stub(db, 'doQuery').callsFake(async (sql, args) => { + calls.push(String(sql)); + if(!/information_schema\.columns/i.test(sql)) return []; + if(args[1] !== 'polls') return schemaRows(args[1], 'utf8mb4'); + return [{ COLUMN_NAME: 'quorum', CHARACTER_SET_NAME: 'utf8mb3' }]; + }); + + await db.ensureReplicaUtf8mb4Columns(); + + const alters = altersIn(calls); + assert.strictEqual(alters.length, 1); + assert.ok(alters[0].includes('MODIFY `quorum`')); + assert.ok(!alters[0].includes('MODIFY `decide_threshold`'), + 'the ALTER names a column this replica does not have, so the whole statement fails errno 1054'); + }); + + it('does not run on a decoder replica (none of these tables exist there)', async function(){ + const decoder = makeDb('decoder'); + const doQuery = sinon.stub(decoder, 'doQuery').resolves([]); + await decoder.ensureReplicaUtf8mb4Columns(); + assert.strictEqual(doQuery.callCount, 0); + await decoder.close(); + }); + + it('logs and carries on when a table cannot be read or cannot be altered', async function(){ + // A transient driver fault must not be read as "table absent" and must not take + // startup down: the replica is exactly as usable as it was a moment ago, and every + // other table still converges. + const calls = []; + sinon.stub(db, 'doQuery').callsFake(async (sql, args) => { + calls.push(String(sql)); + if(/information_schema\.columns/i.test(sql)){ + if(args[1] === 'contracts') throw Object.assign(new Error('read timeout'), { errno: 2013 }); + return schemaRows(args[1], 'utf8mb3'); + } + if(/ALTER TABLE `deploy_chunks`/.test(sql)) + throw Object.assign(new Error('Row size too large'), { errno: 1118 }); + return []; + }); + + await db.ensureReplicaUtf8mb4Columns(); // must not reject + + assert.ok(console.error.called, 'a failed read / failed ALTER must be reported, not swallowed'); + // The tables after the two failures still converge. + assert.ok(altersIn(calls).some(sql => /ALTER TABLE `broadcasts`/.test(sql)), + 'one table failing must not abandon the rest of the widen set'); + }); + + it('reads the charset case-insensitively and under either information_schema column casing', function(){ + assert.strictEqual(utf8mb4Columns.isAlreadyUtf8mb4({ CHARACTER_SET_NAME: 'utf8mb4' }), true); + assert.strictEqual(utf8mb4Columns.isAlreadyUtf8mb4({ character_set_name: 'UTF8MB4' }), true); + // MariaDB 10.6 renamed utf8 to utf8mb3, so both legacy spellings must read narrow. + assert.strictEqual(utf8mb4Columns.isAlreadyUtf8mb4({ CHARACTER_SET_NAME: 'utf8mb3' }), false); + assert.strictEqual(utf8mb4Columns.isAlreadyUtf8mb4({ CHARACTER_SET_NAME: 'utf8' }), false); + assert.strictEqual(utf8mb4Columns.isAlreadyUtf8mb4(null), false); + }); +});