diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ba2e9d..188b498 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.15.0] - 2026-09-07 + +### Changed +- The vendored coin registry is resynced from the hub. +- mariadb moved off the cleartext-credential advisory range with the floor pinned in the dependency gate. + ## [0.12.0] - 2026-08-30 ### Added diff --git a/README.md b/README.md index 73c98ca..0871276 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ # XChain Platform Decoder

- Version - Tests + Version + Tests Node License

@@ -114,7 +114,7 @@ defaults hold on an unconfigured box: | `npm run migrate` | Apply pending database migrations (auto + manual; `--file ` scopes to specific migration(s)) | | `npm run ci` | The full no-external-services gate: unit, security, smoke, regression, chaos, and a 100-iteration fuzz pass (about a minute) | | `npm run test:smoke` | Smoke tests (58 tests, no external services) | -| `npm run test:unit` | Unit tests (1,447 tests, no external services) | +| `npm run test:unit` | Unit tests (1,450 tests, no external services) | | `npm run test:security` | Security tests (83 tests, no external services) | | `npm run test:integration` | Integration tests (30 tests; brings up its own throwaway regtest node and MariaDB, requires Docker) | | `npm run test:e2e` | End-to-end tests (72 tests; brings up its own throwaway regtest node and MariaDB on separate ports, requires Docker) | diff --git a/package-lock.json b/package-lock.json index 20107f1..eaf29cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "xchain-decoder", - "version": "0.12.0", + "version": "0.15.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "xchain-decoder", - "version": "0.12.0", + "version": "0.15.0", "license": "AGPL-3.0-or-later", "dependencies": { "axios": "^1.18.1", @@ -17,15 +17,11 @@ "cors": "^2.8.5", "dotenv": "^16.4.5", "ecpair": "2.1.0", - "encoding-down": "^7.1.0", "express": "^5.2.1", "express-json-rpc-router": "^1.4.0", "express-rate-limit": "^8.5.2", "helmet": "^8.2.0", - "leveldown": "^6.1.1", - "levelup": "^5.1.1", "mariadb": "3.5.3", - "memdown": "^6.1.1", "tiny-secp256k1": "2.2.4" }, "devDependencies": { @@ -1300,24 +1296,6 @@ "uuid": "bin/uuid" } }, - "node_modules/abstract-leveldown": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-7.2.0.tgz", - "integrity": "sha512-DnhQwcFEaYsvYDnACLZhMmCWd3rkOeEvglpa4q5i/5Jlm3UIsWaxVzuXvDLFCSCWRO3yy2/+V/G7FusFgejnfQ==", - "deprecated": "Superseded by abstract-level (https://github.com/Level/community#faq)", - "license": "MIT", - "dependencies": { - "buffer": "^6.0.3", - "catering": "^2.0.0", - "is-buffer": "^2.0.5", - "level-concat-iterator": "^3.0.0", - "level-supports": "^2.0.1", - "queue-microtask": "^1.2.3" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -1514,26 +1492,6 @@ "integrity": "sha512-uAZ8x6r6S3aUM9rbHGVOIsR15U/ZSc82b3ymnCPsT45Gk1DDvhDPdIgB5MrhirZWt+5K0EEPQH985kNqZgNPFw==", "license": "MIT" }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/baseline-browser-mapping": { "version": "2.11.5", "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.5.tgz", @@ -1762,30 +1720,6 @@ "bs58": "^5.0.0" } }, - "node_modules/buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, "node_modules/bunyan": { "version": "1.8.15", "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.15.tgz", @@ -1935,15 +1869,6 @@ "dev": true, "license": "Apache-2.0" }, - "node_modules/catering": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/catering/-/catering-2.1.1.tgz", - "integrity": "sha512-K7Qy8O9p76sL3/3m7/zLKbRkyOlSZAgzEaLhyj2mXS8PsCud2Eo4hAb8aLtZqHh0QGqLcb9dlJSu6lHRVENm1w==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/chalk": { "version": "5.6.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", @@ -2273,20 +2198,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/deferred-leveldown": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-7.0.0.tgz", - "integrity": "sha512-QKN8NtuS3BC6m0B8vAnBls44tX1WXAFATUsJlruyAYbZpysWV3siH6o/i3g9DCHauzodksO60bdj5NazNbjCmg==", - "deprecated": "Superseded by abstract-level (https://github.com/Level/community#faq)", - "license": "MIT", - "dependencies": { - "abstract-leveldown": "^7.2.0", - "inherits": "^2.0.3" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/define-data-property": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", @@ -2461,22 +2372,6 @@ "node": ">= 0.8" } }, - "node_modules/encoding-down": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-7.1.0.tgz", - "integrity": "sha512-ky47X5jP84ryk5EQmvedQzELwVJPjCgXDQZGeb9F6r4PdChByCGHTBrVcF3h8ynKVJ1wVbkxTsDC8zBROPypgQ==", - "deprecated": "Superseded by abstract-level (https://github.com/Level/community#faq)", - "license": "MIT", - "dependencies": { - "abstract-leveldown": "^7.2.0", - "inherits": "^2.0.3", - "level-codec": "^10.0.0", - "level-errors": "^3.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -2926,12 +2821,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/functional-red-black-tree": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", - "integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==", - "license": "MIT" - }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -3314,26 +3203,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "BSD-3-Clause" - }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -3371,29 +3240,6 @@ "node": ">= 0.10" } }, - "node_modules/is-buffer": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", - "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -3701,98 +3547,6 @@ "node": ">=0.6.0" } }, - "node_modules/level-codec": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-10.0.0.tgz", - "integrity": "sha512-QW3VteVNAp6c/LuV6nDjg7XDXx9XHK4abmQarxZmlRSDyXYk20UdaJTSX6yzVvQ4i0JyWSB7jert0DsyD/kk6g==", - "deprecated": "Superseded by level-transcoder (https://github.com/Level/community#faq)", - "license": "MIT", - "dependencies": { - "buffer": "^6.0.3" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/level-concat-iterator": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-3.1.0.tgz", - "integrity": "sha512-BWRCMHBxbIqPxJ8vHOvKUsaO0v1sLYZtjN3K2iZJsRBYtp+ONsY6Jfi6hy9K3+zolgQRryhIn2NRZjZnWJ9NmQ==", - "deprecated": "Superseded by abstract-level (https://github.com/Level/community#faq)", - "license": "MIT", - "dependencies": { - "catering": "^2.1.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/level-errors": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-3.0.1.tgz", - "integrity": "sha512-tqTL2DxzPDzpwl0iV5+rBCv65HWbHp6eutluHNcVIftKZlQN//b6GEnZDM2CvGZvzGYMwyPtYppYnydBQd2SMQ==", - "deprecated": "Superseded by abstract-level (https://github.com/Level/community#faq)", - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/level-iterator-stream": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-5.0.0.tgz", - "integrity": "sha512-wnb1+o+CVFUDdiSMR/ZymE2prPs3cjVLlXuDeSq9Zb8o032XrabGEXcTCsBxprAtseO3qvFeGzh6406z9sOTRA==", - "license": "MIT", - "dependencies": { - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/level-supports": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-2.1.0.tgz", - "integrity": "sha512-E486g1NCjW5cF78KGPrMDRBYzPuueMZ6VBXHT6gC7A8UYWGiM14fGgp+s/L1oFfDWSPV/+SFkYCmZ0SiESkRKA==", - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/leveldown": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-6.1.1.tgz", - "integrity": "sha512-88c+E+Eizn4CkQOBHwqlCJaTNEjGpaEIikn1S+cINc5E9HEvJ77bqY4JY/HxT5u0caWqsc3P3DcFIKBI1vHt+A==", - "deprecated": "Superseded by classic-level (https://github.com/Level/community#faq)", - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "abstract-leveldown": "^7.2.0", - "napi-macros": "~2.0.0", - "node-gyp-build": "^4.3.0" - }, - "engines": { - "node": ">=10.12.0" - } - }, - "node_modules/levelup": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/levelup/-/levelup-5.1.1.tgz", - "integrity": "sha512-0mFCcHcEebOwsQuk00WJwjLI6oCjbBuEYdh/RaRqhjnyVlzqf41T1NnDtCedumZ56qyIh8euLFDqV1KfzTAVhg==", - "deprecated": "Superseded by abstract-level (https://github.com/Level/community#faq)", - "license": "MIT", - "dependencies": { - "catering": "^2.0.0", - "deferred-leveldown": "^7.0.0", - "level-errors": "^3.0.1", - "level-iterator-stream": "^5.0.0", - "level-supports": "^2.0.1", - "queue-microtask": "^1.2.3" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -3893,12 +3647,6 @@ "yallist": "^3.0.2" } }, - "node_modules/ltgt": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", - "integrity": "sha512-AI2r85+4MquTw9ZYqabu4nMwy9Oftlfa/e/52t9IjtfG+mGBbTNdAoZ3RQKLHR6r0wQnwZnPIEh/Ya6XTWAKNA==", - "license": "MIT" - }, "node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", @@ -3981,23 +3729,6 @@ "node": ">= 0.8" } }, - "node_modules/memdown": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/memdown/-/memdown-6.1.1.tgz", - "integrity": "sha512-vh2RiuVrn6Vv73088C1KzLwy9+hhRwoZsgddYqIoVuFFrcoc2Rt+lq/KrmkFn6ulko7AtQ0AvqtYid35exb38A==", - "deprecated": "Superseded by memory-level (https://github.com/Level/community#faq)", - "license": "MIT", - "dependencies": { - "abstract-leveldown": "^7.2.0", - "buffer": "^6.0.3", - "functional-red-black-tree": "^1.0.1", - "inherits": "^2.0.1", - "ltgt": "^2.2.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/merge-descriptors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", @@ -4214,12 +3945,6 @@ "license": "MIT", "optional": true }, - "node_modules/napi-macros": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", - "integrity": "sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==", - "license": "MIT" - }, "node_modules/ncp": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", @@ -4240,17 +3965,6 @@ "node": ">= 0.6" } }, - "node_modules/node-gyp-build": { - "version": "4.8.4", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", - "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", - "license": "MIT", - "bin": { - "node-gyp-build": "bin.js", - "node-gyp-build-optional": "optional.js", - "node-gyp-build-test": "build-test.js" - } - }, "node_modules/node-releases": { "version": "2.0.51", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.51.tgz", @@ -4571,26 +4285,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -4624,20 +4318,6 @@ "node": ">= 0.10" } }, - "node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "license": "MIT", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/readdirp": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", @@ -5135,15 +4815,6 @@ "node": ">= 0.8" } }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, "node_modules/string-width": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", diff --git a/package.json b/package.json index 1246606..709ced7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "xchain-decoder", "description": "xchain-decoder decodes XChain platform transactions from a given blockchain and populates a database with the decoded data.", - "version": "0.12.0", + "version": "0.15.0", "license": "AGPL-3.0-or-later", "repository": { "type": "git", @@ -16,15 +16,11 @@ "cors": "^2.8.5", "dotenv": "^16.4.5", "ecpair": "2.1.0", - "encoding-down": "^7.1.0", "express": "^5.2.1", "express-json-rpc-router": "^1.4.0", "express-rate-limit": "^8.5.2", "helmet": "^8.2.0", - "leveldown": "^6.1.1", - "levelup": "^5.1.1", "mariadb": "3.5.3", - "memdown": "^6.1.1", "tiny-secp256k1": "2.2.4" }, "scripts": { diff --git a/src/BlockchainConnector.js b/src/BlockchainConnector.js index 50ae228..877ef6b 100644 --- a/src/BlockchainConnector.js +++ b/src/BlockchainConnector.js @@ -521,7 +521,26 @@ class BlockchainConnector { } return headerHex + encodeVarintHex(txHexes.length) + txHexes.join('') } catch (err) { - throw new Error("There were problems reassembling a block without auxpow. " + err.message) + // Carry the fault's identity out with the message. The three RPC fetches above + // sit INSIDE this try, so a transport fault (an ECONNRESET from a saturated + // Dogecoin 1.14 RPC queue, an ECONNABORTED timeout, a node restart) lands here + // beside a genuine content fault, and only error.code and the rpcCode/rpcMessage + // sanitizeRpcError attaches separate the two. _auxPowParseErrorCount never + // decays, so once a height has escalated to this path every later failure at + // that height arrives through this catch, which is precisely where an operator + // has to tell an unreachable node from a block whose bytes are unusable. + // Mirrors the cause attachment getBlockWithoutAuxPow makes above. + // + // Deliberately NOT tagged auxPowParseFailure: that flag is the only signal + // fetchBlockHex escalates on, and aiming a per-tx fan-out at a node that is + // merely unreachable is the failure the comment above getBlockWithoutAuxPow + // describes. Errors leaving these RPC helpers have already passed through + // sanitizeRpcError, which scrubs config.auth, the Authorization header and + // error.request in place, so attaching one as cause carries no credential. + const reassembleErr = new Error("There were problems reassembling a block without auxpow. " + err.message) + reassembleErr.cause = err + if (err && err.code !== undefined) reassembleErr.code = err.code + throw reassembleErr } } diff --git a/src/XChainDecoder.js b/src/XChainDecoder.js index 4854829..808ec5f 100644 --- a/src/XChainDecoder.js +++ b/src/XChainDecoder.js @@ -30,6 +30,7 @@ const CryptoNetworks = require('./CryptoNetworks') const XChainBlockDecoder = require('./XChainBlockDecoder') const { isOracleFeeCaptureActive, isOracleFeeSetCaptureActive, oracleAddressFromCreate, isCompactedOracleAddress, V0_GIVE_COIN_INDEX, V0_GET_COIN_INDEX, V0_GET_ADDRESS_INDEX, V0_REQUIRED_FIELD_COUNT, ORACLE_ADDRESS_INDEX, V0_EXPIRATION_INDEX, V2_EXPIRATION_INDEX } = require('./oracleFeeOutput') const { isDispenserExpiryRealignActive } = require('./dispenserExpiryRealign') +const { cancelGraceFloor } = require('./dispenserCancelGrace') const { captureCommands, collapseDispenserRegistrations, isBatchSubCommandCaptureActive } = require('./batchSubCommandCapture') const { chainTierMismatch, chainFieldMissing, chainGenesisMismatch, chainGenesisUnpinned } = require('./chainIdentity') // REORG_HALT rides getLogger() rather than this.logError, because a patched @@ -102,15 +103,16 @@ const SYNCED_THRESHOLD = 3 //Maximum blocks behind to be synced // reorg-recovery window, or a row is deleted before a legal in-window reorg can // restore it (deleteBlockByIndex then matches zero rows), permanently losing a // money-bearing dispenser on the reorged node. The platform's deepest window is -// DOGE = 120 (xchain-utxo-tracker DEFAULT_UNDO_BLOCKS: BTC 12 / LTC 48 / DOGE 120); -// the previous flat 100 sat BELOW DOGE's window. Invariant: SAFE_DEPTH >= +// 120, and TWO chains now sit on it (xchain-utxo-tracker DEFAULT_UNDO_BLOCKS: +// BTC 12 / LTC 120 / DOGE 120; LTC was 48 until a 2026-09-01 testnet fork walked +// past it); the previous flat 100 sat BELOW that window. Invariant: SAFE_DEPTH >= // deepest undo window + margin. The +6 margin means a small undo-window re-tune // cannot land exactly at the purge threshold; dispenserSafeDepth.test.js // enforces the invariant with a conformance read of undo-blocks.js, so raising // any chain's window past the margin fails the suite until this is bumped. // Purging deeper is the conservative direction (rows are merely retained longer // before hard-purge; expiry semantics and action evaluation are unchanged). -const DISPENSER_EXPIRE_SAFE_DEPTH = 126 // 120 (DOGE undo window) + 6 margin +const DISPENSER_EXPIRE_SAFE_DEPTH = 126 // 120 (deepest undo window, LTC and DOGE) + 6 margin // There is deliberately no DISPENSER_CLOSE_DELAY twin of the indexer's here: the decoder // does not mirror dispenser cancels, so it never needs to close a row at the height the // indexer's DISPENSER_CLOSE fires. Reintroducing a closing mirror would need that pinned @@ -417,6 +419,12 @@ class XChainDecoder { this.reorgHaltReason = null this.reorgHaltAt = null this.reorgHaltCheckedAt = 0 + // Whether a REORG_HALT row is known to be READABLE, as distinct from + // whether this decoder is halted. null = no halt has been raised or read + // yet; false = a halt exists in memory whose durable write could not be + // confirmed, which is the one state where a restart silently resumes the + // rollback and the bootstrap gate finds nothing to refuse on. + this.reorgHaltMarkerPersisted = null this._reorgHaltProbeInFlight = null } @@ -632,6 +640,11 @@ class XChainDecoder { this.reorgHaltReason = (marker && marker.reason) || null this.reorgHaltAt = (marker && marker.at) || null this.reorgHaltCheckedAt = now + // A marker this probe just READ is durable by observation, whatever the + // write that produced it reported. Raised here and never cleared here: + // finding no row is exactly the state an unconfirmed in-process halt + // leaves behind, so clearing on absence would erase the one signal. + if (this.reorgHalted) this.reorgHaltMarkerPersisted = true if (this.reorgHalted && !wasHalted){ console.error('XChainDecoder: LATENT REORG_HALT MARKER PRESENT - this decoder carries a durable ' + 'REORG_HALT row from an aborted rollback. It will keep parsing forward and look healthy, but ' + @@ -655,13 +668,17 @@ class XChainDecoder { // Cached view of the halt marker for health surfaces. `checked_at` is null until // the first successful probe, so a consumer can tell "not halted" apart from - // "never looked". + // "never looked". `marker_persisted` splits the halt from its evidence: null when + // no halt has been raised or seen, false when this process halted and could not + // confirm the durable row, true when a row is known readable. getReorgHaltStatus(){ return { halted: !!this.reorgHalted, reason: this.reorgHaltReason || null, at: this.reorgHaltAt || null, - checked_at: this.reorgHaltCheckedAt || null + checked_at: this.reorgHaltCheckedAt || null, + marker_persisted: (this.reorgHaltMarkerPersisted === null || this.reorgHaltMarkerPersisted === undefined) + ? null : !!this.reorgHaltMarkerPersisted } } @@ -1455,9 +1472,37 @@ class XChainDecoder { // P2WSH chunk carrier: same shape as P2SH, chunks in the witness. } else if (dataWithoutObfuscation.subarray(MAGIC_WORD.length).equals(P2WSH_BUFFER)){ p2shFundingTxId = firstInputTxId // commit tx carrying any native-coin fee output + // A chain that declares no segwit has no witness carrier, so refuse + // to read payload out of a witness stack there instead of trusting + // upstream node validation to keep one from ever arriving. Same + // per-chain capability gate the taproot envelope lane already carries + // (envelopeRecognitionHeight), which this older lane never got. + // + // `=== false`, never a falsy test: supportsSegwit is declared only on + // the non-segwit coin (src/coins/DOGE.js), so it is undefined on + // BTC/LTC and `!this.network.supportsSegwit` would disable the whole + // P2WSH lane on the chains that DO use it and change how already + // indexed history decodes. + // + // Placed inside the branch body rather than in the `else if` + // condition, and after p2shFundingTxId is set, on purpose. Folding it + // into the condition would fall through to the trailing `else`, which + // appends the marker remainder as raw payload; clearing the funding + // txid would drop the commit's native-fee attribution. Both are + // behaviour changes on a live chain, and this is a capability gate. + // Against chain-realistic input it is a strict no-op: a non-segwit + // transaction carries no witness stack, so every input already failed + // the shape check below and nextDataBuffer already stayed empty. for (let txInputIndex=0;txInputIndex < transaction.ins.length;txInputIndex++){ let nextInput = transaction.ins[txInputIndex] try { + // Per-chain capability gate (see above). `continue`, not + // `break`: this branch sits inside the enclosing OUTPUT loop, + // so breaking here would stop scanning the transaction's + // remaining outputs. Same idiom and same meaning as the + // witness-shape check on the next line: this input carries no + // payload for us. + if (this.network.supportsSegwit === false) continue if (!nextInput["witness"] || nextInput["witness"].length < 3 || !Buffer.isBuffer(nextInput["witness"][2])) continue let decodedRedeemScript = bitcoin.script.decompile(nextInput["witness"][2]) if (!decodedRedeemScript || decodedRedeemScript.length < 1 || !Buffer.isBuffer(decodedRedeemScript[0])) continue @@ -1752,8 +1797,50 @@ class XChainDecoder { throw new Error(msg) } - // Persist the durable halt marker before an abort throws (best-effort: swallow - // write errors so a marker failure never masks the loud abort). Feature-detected. + // Depth already rolled back and not yet re-synced, carried across restarts. + // + // The guard above depends on a marker written on the ABORT path, which is + // exactly when the database may be the thing failing: markReorgHalted is + // best-effort, so two failed writes leave the halt recorded nowhere and this + // entry guard sees a clean database. The counter below does not have that + // hole, because deleteBlockByIndex commits each block's REORG marker inside + // the same transaction as the delete: whatever else fails, the evidence of a + // completed delete is durable. Counting the marked heights above the current + // tip therefore reconstructs the depth of an interrupted rollback, and the + // ceiling holds across a restart with no successful abort-time write. + // + // Fail-closed: an unreadable count is retried, and a persistent fault throws + // out of verifyReorg BEFORE any delete. Deliberately NOT a haltReorg - like + // the walk's read-fault catch below, a read fault is infrastructure, and a + // durable REORG_HALT would block every later reorg until an operator cleared + // it. Feature-detected so the minimal-mock verifyReorg tests stay unaffected. + let priorDepth = 0 + if (typeof this.db.countReorgDeletesAboveTip === 'function'){ + let seedErr = null + for (let attempt = 1; attempt <= 3; attempt++){ + try { + priorDepth = await this.db.countReorgDeletesAboveTip() + seedErr = null + break + } catch (err){ + seedErr = err + console.error(`reorg: could not read the prior rollback depth (attempt ${attempt}/3)`, err) + if (attempt < 3) await this.sleep(3000) + } + } + if (seedErr){ + const msg = 'verifyReorg: the prior rollback depth could not be read, so the dispenser ' + + 'safe-depth ceiling cannot be enforced across a restart. Refusing to delete any block: ' + + (seedErr.message || String(seedErr)) + console.error(msg) + throw new Error(msg) + } + } + + // Persist the durable halt marker before an abort throws. Feature-detected, and + // non-throwing so a marker failure never masks the loud abort, but NOT silent: + // the outcome is honoured, published on the health surface and logged, because + // an unrecorded halt is the one state where a restart resumes the rollback. const haltReorg = async (reason) => { // Set the in-memory health state first: the durable write is best-effort, // but this decoder is halted either way and every health surface must say @@ -1776,9 +1863,12 @@ class XChainDecoder { network: this.consensusNetwork, reason: reason, depth: blocksDeleted.length, - marker_persisted: canPersist, + // 'attempting', not 'persisted': this record is emitted BEFORE the + // write, so it cannot know the outcome and must not claim one. The + // REORG_HALT_MARKER record below carries the real answer. + marker_write: canPersist ? 'attempting' : 'unavailable', // Spelled out rather than left for the reader to infer from the - // boolean: this is the one halt that /status and /live cannot + // field: this is the one halt that /status and /live cannot // report, because the marker they read is never written. detail: canPersist ? undefined : 'db.markReorgHalted is unavailable: the durable halt marker cannot be persisted, ' @@ -1787,11 +1877,60 @@ class XChainDecoder { }) } catch (_) { /* a diagnostic must never mask the abort it describes */ } - if (!canPersist) return + if (!canPersist) { + this.reorgHaltMarkerPersisted = false + return + } + + // Honour the write result. markReorgHalted confirms the row by read-back + // and returns false when it cannot; the catch below only ever fires for a + // connection or SELECT fault, because insertEvent eats the INSERT error. + // Retried ONCE and without a sleep: a failed insertEvent rolls the open + // block transaction back (db.js insertEvent -> endTransaction), so the + // second attempt runs on a freshly leased pooled connection, which is a + // materially different attempt rather than the same one repeated. No + // backoff, because this sits directly in front of the abort throw and a + // marker write must never delay the fault it is describing. + let persisted = false + let lastError = null + let attempts = 0 + while (attempts < 2 && !persisted){ + attempts++ + try { + persisted = (await this.db.markReorgHalted(reason)) === true + } catch (e) { + lastError = e + } + } + this.reorgHaltMarkerPersisted = persisted + + // The outcome record. Separate from the one above because the two answer + // different questions ("what halted, and why" vs "did the evidence land"), + // and because collapsing them would put the reason behind the write that + // may be the thing failing. try { - await this.db.markReorgHalted(reason) - } catch (e) { - console.error('verifyReorg: failed to persist REORG_HALT marker:', e) + getLogger().error('REORG_HALT_MARKER', { + coin: this.coinTick, + network: this.consensusNetwork, + marker_persisted: persisted, + attempts: attempts, + err: lastError ? (lastError.message || String(lastError)) : undefined + }) + } catch (_) { /* a diagnostic must never mask the abort it describes */ } + + if (!persisted){ + // The incident shape the bootstrap gate exists to stop: the process is + // about to exit, the restart policy recycles the container, the entry + // guard reads a marker that was never written, the decoder finishes the + // over-deep rollback, and the gate counts zero markers and publishes + // this database as known-good. Nothing durable records it, so this line + // is the only evidence and it has to name the required action. + console.error('verifyReorg: the durable REORG_HALT marker could NOT be persisted after ' + + attempts + ' attempt(s)' + + (lastError ? ' (' + (lastError.message || String(lastError)) + ')' : '') + + '. This database is NOT a valid bootstrap source: a restart will re-enter verifyReorg ' + + 'with a zeroed depth counter and silently resume the over-deep rollback. ' + + 'REQUIRED OPERATOR ACTION: full resync from a known-good snapshot.') } } @@ -1805,11 +1944,19 @@ class XChainDecoder { // strictly safer than a silently corrupt DB: stop and require an // operator-driven resync. Called BEFORE each delete attempt (outside // the per-block retry try/catch, so the throw is not retried away). + // + // The ceiling is measured over priorDepth + this run's deletes, because the + // dispenser purge window is a property of the DATABASE, not of one process: + // 100 blocks deleted before a restart and 100 after are 200 blocks past the + // tip either way, and counting only the current invocation is what let a + // restart finish an aborted over-deep rollback. const assertWithinSafeDepth = async (lastBlockIndex) => { - if (blocksDeleted.length >= DISPENSER_EXPIRE_SAFE_DEPTH){ + if (priorDepth + blocksDeleted.length >= DISPENSER_EXPIRE_SAFE_DEPTH){ const msg = "verifyReorg: reorg depth exceeds the dispenser safe-depth window " + "(DISPENSER_EXPIRE_SAFE_DEPTH=" + DISPENSER_EXPIRE_SAFE_DEPTH + "). Already rolled back " - + blocksDeleted.length + " blocks; soft-expired dispenser rows for block height " + + (priorDepth + blocksDeleted.length) + " blocks (" + blocksDeleted.length + + " in this run, resumed from " + priorDepth + " already deleted above the tip); " + + "soft-expired dispenser rows for block height " + lastBlockIndex + " and below have already been hard-purged, so continuing would " + "silently lose money-bearing dispenser state. Aborting. Recovery: perform a full " + "resync from a known-good snapshot." @@ -2625,7 +2772,16 @@ class XChainDecoder { // null signals the query failed: decoding the block against an empty set // would silently drop every dispense output on this instance only, so // retry the block instead. - let openDispenserAddresses = await this.db.getAllOpenDispenserAddresses() + // + // CANCELLATION GRACE (at/above DISPENSER_CANCEL_GRACE_ACTIVATION): the floor + // widens the set by dispensers whose expiration is inside the indexer's + // cancellation grace period, which the indexer keeps fillable for an hour past + // a cancel while the decoder's soft-expire knows nothing about cancels. Below + // the gate the floor is null and the set is the unwidened one, so a + // from-genesis re-decode reproduces what the fleet wrote. The floor derives + // only from this block's header time, so every honest node loads the same set. + let openDispenserAddresses = await this.db.getAllOpenDispenserAddresses( + cancelGraceFloor(this.consensusNetwork, block.timestamp)) if (openDispenserAddresses == null){ console.error(`Could not load open dispenser addresses for block ${nextBlockHeight}; retrying block`) await this.db.endTransaction() @@ -3325,9 +3481,24 @@ class XChainDecoder { let mempoolStartTime = Date.now() this.mempoolBusy = true let rawMempool = [] + // Mempool size as the node reported it, held separately because + // deleteAndCompareTxsNotInList below empties and refills rawMempool in place. + let nodeMempoolCount = 0 try { let rawMempoolUnordered = await this.connector.getRawMempool() + // getrawmempool answers with an array of txids; rpcResult only guarantees the + // result member is present, never its type. Reject any other shape HERE, at the + // boundary, and let the catch below skip the poll: a malformed-but-iterable + // answer (a bare string from an RPC proxy or a trimmed body) dedups into + // per-character "txids", and deleteAndCompareTxsNotInList then anti-joins the + // stored table against that snapshot and deletes every pending row, blanking + // the published feed until a healthy poll refills it. Mirrors the shape check + // the verbose-block consumer makes in BlockchainConnector.getBlockReassembled. + if (!Array.isArray(rawMempoolUnordered)) { + throw new Error('getrawmempool did not return an array') + } + // Dedup + single O(n log n) sort. The old per-txid binary-insert // (bs + splice) was O(n^2) in mempool size every poll cycle, a CPU // hazard under a mempool flood. What the consumer needs is the DEDUP: @@ -3342,7 +3513,8 @@ class XChainDecoder { // Snapshot the node's total mempool size for the API's getmempool // method (deduped count, matching what this cycle actually processes). - this.nodeMempoolTxCount = rawMempool.length + nodeMempoolCount = rawMempool.length + this.nodeMempoolTxCount = nodeMempoolCount this.nodeMempoolUpdatedAt = Date.now() } catch (error) { @@ -3361,7 +3533,10 @@ class XChainDecoder { let deletedInfo = await this.mempoolDb.deleteAndCompareTxsNotInList(rawMempool) let deletedTransactionsCount = deletedInfo.transactionsDeleted - + // Read the length before the batch loop, while it still means "new arrivals": + // the call above truncated rawMempool down to the txids this node has not stored. + let newArrivalsCount = rawMempool.length + let i = 0 while (i < rawMempool.length) { let nextRawMempoolChunk = rawMempool.slice(i, i + MEMPOOL_BATCH_SIZE) @@ -3464,8 +3639,10 @@ class XChainDecoder { let mempoolEndTime = Date.now() let timeString = this.millisecondsToTimeString(mempoolEndTime - mempoolStartTime) + // nodeMempoolCount, not rawMempool.length: the db diff empties and refills + // rawMempool in place, so by here its length is the new-arrival count. console.log("Mempool updated!" - + " Transactions (" + rawMempool.length + " in mempool, " + validTransactionsCount + " valid, " + deletedTransactionsCount + " less) [" + timeString + "]") + + " Transactions (" + nodeMempoolCount + " in mempool, " + newArrivalsCount + " new, " + validTransactionsCount + " valid, " + deletedTransactionsCount + " less) [" + timeString + "]") } finally { // Always clear the busy flag, even if a DB or parse operation above threw. // Otherwise a single transient failure would leave mempool tracking frozen diff --git a/src/api.js b/src/api.js index b4179f6..37d7b66 100644 --- a/src/api.js +++ b/src/api.js @@ -515,7 +515,15 @@ async function startApi(){ running: decoderRunning, reorg_halted: reorgHalt.halted, reorg_halt_reason: reorgHalt.reason, - reorg_halted_at: reorgHalt.at + reorg_halted_at: reorgHalt.at, + // Ships beside the boolean, never without it. "Not halted" is only an answer + // if something looked, and the probe is fail-soft: its state starts at + // not-halted with checked_at null, so a decoder that has NEVER completed a + // probe publishes exactly what a clean one publishes. Consumers that gate on + // this body (xchain-node's BootstrapHealthGate falls back to GET /status when + // the JSON-RPC health surface is unavailable) can only tell those two apart + // if this route carries the timestamp the health method already carries. + reorg_halt_checked_at: reorgHalt.checked_at }) }) diff --git a/src/batchSubCommandCapture.js b/src/batchSubCommandCapture.js index c76bcce..b469199 100644 --- a/src/batchSubCommandCapture.js +++ b/src/batchSubCommandCapture.js @@ -51,7 +51,10 @@ const ACTION_ALIASES = require('./actionAliases.js') const { COMMAND_LIMIT, ACTION_LIMITS, GATED_ACTION_LIMITS, - CHILD_ISSUE_KEY } = require('./protocol/indexerBatchLimits.js') + CHILD_ISSUE_KEY, + WEIGHT_BUDGET, + COMMAND_WEIGHTS, + COST_WEIGHTING_ACTIVATION } = require('./protocol/indexerBatchLimits.js') // The BATCH FORMAT versions the indexer registers (xchain-indexer/src/actions/batch.js // `this.formats`, which today holds only 0 = 'VERSION|COMMAND'). A BATCH whose FORMAT is @@ -174,10 +177,11 @@ function subCommandActionName(command){ // verdict this file can reach on its own evidence. // // The rest of the class is now closed as far as it is provable, in hasProvablyRejectedBatch -// below: the nested BATCH, the per-ACTION caps and the 250-command cap, against the indexer's -// cap tables vendored canonically in src/protocol/indexerBatchLimits.js. The UNKNOWN NAME is -// still the one cause left open, and deliberately, for the reason this paragraph gives: a -// vendored name LIST is not closed under registry growth, so a stale one under-captures. +// below: the nested BATCH, the per-ACTION caps, the 250-command cap and the +// BATCH_COST_WEIGHTING weight budget, against the indexer's tables vendored canonically in +// src/protocol/indexerBatchLimits.js. The UNKNOWN NAME is still the one cause left open, and +// deliberately, for the reason this paragraph gives: a vendored name LIST is not closed under +// registry growth, so a stale one under-captures. // // A '' name is reachable two ways and both are covered, because both are what // `split('|')[0]` yields: an EMPTY element (a trailing ';', a ';;', or the whole command @@ -399,10 +403,18 @@ function maxIdenticalMintTicks(ticks){ // * THE AGGREGATE GAS PRE-CHECK ('invalid: GAS (insufficient)'). Same reason: it reads the // SOURCE's balances and the token set from the indexer database. // +// A FOURTH cause, the BATCH_COST_WEIGHTING weight budget, IS mirrored, and unlike the caps +// above it is gated on its own vendored instant rather than assumed on (see +// isBatchCostWeightingActive). Its one deliberate under-estimate, the DEPLOY discount, is +// argued at subCommandCostWeight. +// // Order of the checks is irrelevant to the verdict (any one of them means "rejected"), so // this does NOT reproduce the indexer's error precedence, which decides only WHICH string a // rejected batch reports. -function hasProvablyRejectedBatch(subCommands, aliases){ +// +// `consensusNetwork` and `blockTime` are OPTIONAL and default to "the weight budget is not +// provably active", so every caller written before the budget existed keeps today's verdicts. +function hasProvablyRejectedBatch(subCommands, aliases, consensusNetwork, blockTime){ if (!Array.isArray(subCommands)) return false // The global command cap, counted over the raw ';'-split list with empty elements // included - the same list, and the same counting rule, the indexer caps. @@ -427,9 +439,74 @@ function hasProvablyRejectedBatch(subCommands, aliases){ : (tally.get(action) || 0) if (count > caps[action]) return true } + + // The weighted budget, which the indexer applies INSTEAD of the flat count at/after + // BATCH_COST_WEIGHTING. The count cap above stays a sound pre-filter either way, because + // every weight is >= 1 and the budget is the same number. + if (isBatchCostWeightingActive(consensusNetwork, blockTime) && + batchCostWeight(subCommands, aliases) > WEIGHT_BUDGET) return true + return false } +// Is the indexer's BATCH_COST_WEIGHTING weight budget in force at this block? +// +// Its own vendored per-network instant, NOT the ordering argument the caps lean on. That +// argument is specific to BATCH_ISSUANCE_LIMITS, whose instant the decoder's capture gate is +// required to sit at or after; the weighting flag has no such relationship and today it is +// the counter-example, with mainnet capture ARMED and the weighting instant still on the +// house sentinel. An absent or DISARMED (null) entry is inactive at every block time, which +// leaves today's over-capture in place rather than inventing a suppression rule. +function isBatchCostWeightingActive(consensusNetwork, blockTime){ + const activation = COST_WEIGHTING_ACTIVATION[consensusNetwork] + if (typeof activation !== 'number') return false + const t = Number(blockTime) + if (!Number.isFinite(t)) return false + return t >= activation +} + +// The cost weight of ONE sub-command: a strict LOWER BOUND on the indexer's subCommandWeight. +// +// The bound is the whole design, because the directions are not symmetric. Charging MORE than +// the indexer pushes the sum over the budget for a batch the indexer really dispatches, which +// suppresses capture and loses a settlement output; charging LESS only leaves today's +// over-capture open for that shape. +// +// So DEPLOY is deliberately UNDER-charged at the default 1 rather than its table weight of +// 30: the indexer discounts a format-4 chunk carrier back to 1, and this module does not read +// FORMAT versions. DEPLOY is capped at 1 per BATCH by GATED_ACTION_LIMITS, so the whole +// under-estimate is bounded at 29 of the 250 budget. Every other weighted ACTION +// (AIRDROP/DIVIDEND/EXECUTE/XEXEC) is charged unconditionally by the indexer, so the table +// value is exact there. +// +// Alias expansion matches the indexer's, which normalizes the name before weighing; the +// module header's ordering argument covers that BATCH_SUBACTION_NORMALIZATION is on wherever +// capture runs. A name the table does not carry weighs 1, and hasOwnProperty keeps +// `constructor`/`__proto__` off the prototype chain. +function subCommandCostWeight(command, aliases){ + const rawName = subCommandActionName(command) + if (rawName === null) return 1 + const action = expandAliasName(rawName, aliases) + if (action === 'DEPLOY') return 1 + if (!Object.prototype.hasOwnProperty.call(COMMAND_WEIGHTS, action)) return 1 + const weight = COMMAND_WEIGHTS[action] + return (Number.isInteger(weight) && weight >= 1) ? weight : 1 +} + +// Total cost weight of a BATCH: the sum of subCommandCostWeight over the raw ';'-split list, +// empty elements included, exactly the list the indexer weighs. Never throws (a crash here +// would take down block decoding); an unweighable list falls back to 0, which is "not +// provably rejected". +function batchCostWeight(subCommands, aliases){ + try { + let total = 0 + for (const command of subCommands) total += subCommandCostWeight(command, aliases) + return total + } catch (e) { + return 0 + } +} + // The list of action strings the output-capture decision should be taken over. // // Below the gate, and for every transaction that is not a BATCH, this is exactly @@ -465,7 +542,7 @@ function captureCommands(decodedData, consensusNetwork, blockTime){ return [decodedData] const subCommands = batchSubCommands(decodedData) if (subCommands === null) return [decodedData] - if (hasProvablyRejectedBatch(subCommands, ACTION_ALIASES)) return [] + if (hasProvablyRejectedBatch(subCommands, ACTION_ALIASES, consensusNetwork, blockTime)) return [] return subCommands.map(command => expandSubCommandAlias(command, ACTION_ALIASES)) } @@ -532,9 +609,10 @@ module.exports = { subCommandActionName, hasProvablyRejectedSubCommand, hasProvablyRejectedBatch, + isBatchCostWeightingActive, + subCommandCostWeight, + batchCostWeight, expandSubCommandAlias, - expandAliasName, - isNumeric, isLegacyActionFormat, subCommandTick, subCommandLimitKey, @@ -545,4 +623,7 @@ module.exports = { ACTION_LIMITS, GATED_ACTION_LIMITS, CHILD_ISSUE_KEY, + WEIGHT_BUDGET, + COMMAND_WEIGHTS, + COST_WEIGHTING_ACTIVATION, } diff --git a/src/coins/BTC.js b/src/coins/BTC.js index f36c86f..c846405 100644 --- a/src/coins/BTC.js +++ b/src/coins/BTC.js @@ -284,6 +284,26 @@ module.exports = { OWNERSHIP_ESCROW: 50000, AIRDROP_PER_RECIPIENT: 100, DIVIDEND_PER_RECIPIENT: 100, + // SWEEP / CALLBACK, priced on the unified schedule from the + // UNIFIED_FEES_SWEEP_CALLBACK flag day. The legacy flat per-DB-hit fee prices a + // small action BELOW the dust threshold of a native-fee chain (LTC/DOGE, where a + // missing fee output is rejected outright rather than falling back to an XCHAIN + // balance debit), so the fee output cannot be created and the action cannot be + // submitted at all: a Litecoin SWEEP needs ~273 DB hits before it clears LTC's + // 5460-satoshi floor at LTC $100 / XCHAIN $2. + // + // The BASE keys are what close that: gas is what buys the output, so the SMALLEST + // possible SWEEP or CALLBACK has to buy an above-dust one on its own. The floor a + // chain demands is dust_sats * COIN_USD / (1000 * XCHAIN_USD) gas units, so 5000 gas + // (0.05 XCHAIN) clears Litecoin while COIN/XCHAIN stays under ~915 and Dogecoin + // while it stays under ~50, both far outside any plausible band. The PER_ITEM keys + // hold the marginal cost at AIRDROP/DIVIDEND per-recipient parity; a SWEEP item is + // one swept balance, one closed order/swap/dispenser escrow, or one transferred + // ownership. + SWEEP_BASE: 5000, + SWEEP_PER_ITEM: 100, + CALLBACK_BASE: 5000, + CALLBACK_PER_RECIPIENT: 100, // BET (parimutuel betting, spec decision F): feed creation is duration- // metered like ORDER/SWAP/DISPENSER expiration (same free window via // UNIFIED_EXPIRATION_FEE_FREE_DAYS) but under its OWN per-day key so the diff --git a/src/coins/DOGE.js b/src/coins/DOGE.js index 71bd5d5..0ec1a85 100644 --- a/src/coins/DOGE.js +++ b/src/coins/DOGE.js @@ -218,6 +218,26 @@ module.exports = { OWNERSHIP_ESCROW: 50000, AIRDROP_PER_RECIPIENT: 100, DIVIDEND_PER_RECIPIENT: 100, + // SWEEP / CALLBACK, priced on the unified schedule from the + // UNIFIED_FEES_SWEEP_CALLBACK flag day. The legacy flat per-DB-hit fee prices a + // small action BELOW the dust threshold of a native-fee chain (LTC/DOGE, where a + // missing fee output is rejected outright rather than falling back to an XCHAIN + // balance debit), so the fee output cannot be created and the action cannot be + // submitted at all: a Litecoin SWEEP needs ~273 DB hits before it clears LTC's + // 5460-satoshi floor at LTC $100 / XCHAIN $2. + // + // The BASE keys are what close that: gas is what buys the output, so the SMALLEST + // possible SWEEP or CALLBACK has to buy an above-dust one on its own. The floor a + // chain demands is dust_sats * COIN_USD / (1000 * XCHAIN_USD) gas units, so 5000 gas + // (0.05 XCHAIN) clears Litecoin while COIN/XCHAIN stays under ~915 and Dogecoin + // while it stays under ~50, both far outside any plausible band. The PER_ITEM keys + // hold the marginal cost at AIRDROP/DIVIDEND per-recipient parity; a SWEEP item is + // one swept balance, one closed order/swap/dispenser escrow, or one transferred + // ownership. + SWEEP_BASE: 5000, + SWEEP_PER_ITEM: 100, + CALLBACK_BASE: 5000, + CALLBACK_PER_RECIPIENT: 100, // BET (parimutuel betting, spec decision F): feed creation is duration- // metered like ORDER/SWAP/DISPENSER expiration (same free window via // UNIFIED_EXPIRATION_FEE_FREE_DAYS) but under its OWN per-day key so the diff --git a/src/coins/LTC.js b/src/coins/LTC.js index f02d1e3..3f26fe9 100644 --- a/src/coins/LTC.js +++ b/src/coins/LTC.js @@ -213,6 +213,26 @@ module.exports = { OWNERSHIP_ESCROW: 50000, AIRDROP_PER_RECIPIENT: 100, DIVIDEND_PER_RECIPIENT: 100, + // SWEEP / CALLBACK, priced on the unified schedule from the + // UNIFIED_FEES_SWEEP_CALLBACK flag day. The legacy flat per-DB-hit fee prices a + // small action BELOW the dust threshold of a native-fee chain (LTC/DOGE, where a + // missing fee output is rejected outright rather than falling back to an XCHAIN + // balance debit), so the fee output cannot be created and the action cannot be + // submitted at all: a Litecoin SWEEP needs ~273 DB hits before it clears LTC's + // 5460-satoshi floor at LTC $100 / XCHAIN $2. + // + // The BASE keys are what close that: gas is what buys the output, so the SMALLEST + // possible SWEEP or CALLBACK has to buy an above-dust one on its own. The floor a + // chain demands is dust_sats * COIN_USD / (1000 * XCHAIN_USD) gas units, so 5000 gas + // (0.05 XCHAIN) clears Litecoin while COIN/XCHAIN stays under ~915 and Dogecoin + // while it stays under ~50, both far outside any plausible band. The PER_ITEM keys + // hold the marginal cost at AIRDROP/DIVIDEND per-recipient parity; a SWEEP item is + // one swept balance, one closed order/swap/dispenser escrow, or one transferred + // ownership. + SWEEP_BASE: 5000, + SWEEP_PER_ITEM: 100, + CALLBACK_BASE: 5000, + CALLBACK_PER_RECIPIENT: 100, // BET (parimutuel betting, spec decision F): feed creation is duration- // metered like ORDER/SWAP/DISPENSER expiration (same free window via // UNIFIED_EXPIRATION_FEE_FREE_DAYS) but under its OWN per-day key so the diff --git a/src/coins/consensus_pin.js b/src/coins/consensus_pin.js index 50ce2d0..53a2113 100644 --- a/src/coins/consensus_pin.js +++ b/src/coins/consensus_pin.js @@ -66,16 +66,25 @@ module.exports = { // so the public testnet announces with zero pre-announcement test actions. // Same one-wave rule as every regeneration above. Regtest and mainnet are // untouched and were re-verified as unchanged by this edit. + // REGENERATED 2026-09-01: GAS_SCHEDULE gains SWEEP_BASE, + // SWEEP_PER_ITEM, CALLBACK_BASE and CALLBACK_PER_RECIPIENT, the unified prices + // SWEEP and CALLBACK move onto at the UNIFIED_FEES_SWEEP_CALLBACK flag day (both + // networks UNARMED; regtest genesis-active). GAS_SCHEDULE is hashed whole by + // consensusSubset(), so ADDING a key moves every hash even while the flag that + // reads it is unarmed, and the same one-wave rollout rule as every regeneration + // above applies in full: every service bundling these must ship the new values + // together, and a straggler fail-closes on verifyConsensusPin() at boot rather + // than forking. Mainnet stays null (Phase 6 arms it). testnet: { - BTC: 'f6589c6b88dc930db05998070ef0b73743f58623a0d23fbc30fdb158c49d1427', - LTC: '9faf066a1470be2486d8a2cd121548ca02de1397d0678a2ab8dc0e712ebfa8fd', - DOGE: '2991d7e7caf2b212de959dd5831ac1477e0b13da95ac1ed8c2b43e2704732439', + BTC: 'd3c66a4fb288b2666a2a4fad85200bbeac162bb36fed8a3eddcfc7b2d4d48070', + LTC: 'ae94a951a838e64f9c36e503b978d9b9ad5ea74f7b443465baaabca8f675ea0d', + DOGE: 'b90aec4381b0ad32caba078706c8fb244cbe267390e41668fa063d9e64fb60e6', }, regtest: { - BTC: '24e6a363e5a36285574dea357328a997fdee5762ef812d8947eacf69c51afc24', - LTC: '5ad03b383d873d309640e75dfefa2787a5806cb8a84ee46f4cc7fb25ca7f808b', - DOGE: '019220a461e34c99fcf5cbf107673f13d3f2a57d2a20e16a0323ed44c81edd11', + BTC: '29976bd33cad1842320c57acdc849250646adea765f70a0ae5dad3f201f7d5d7', + LTC: 'bca62db9f59a6f7566620b086380c10fffac08dabe99f00a4fcc7cd038e46146', + DOGE: '816632e9f6647e726042282c37789ae8d924e8d4a1b2995ddde8d6a54a0bba54', }, }, }; diff --git a/src/db.js b/src/db.js index 2ca87c5..3a582a9 100644 --- a/src/db.js +++ b/src/db.js @@ -2540,16 +2540,39 @@ class Database { // distinguishable, because decoding a block against a silently-empty set would // drop every dispense output on this instance only (instance-dependent block // contents). The block loop retries the block on null. - async getAllOpenDispenserAddresses(){ + // + // CANCELLATION GRACE. `graceFloor` is the oldest expiration still eligible for capture, + // computed by dispenserCancelGrace.cancelGraceFloor from the block's own header time, and + // null below DISPENSER_CANCEL_GRACE_ACTIVATION. A finite floor admits rows the soft-expire + // has already stamped whose expiration is no older than it, which is how the decoder keeps + // capturing payments to a dispenser the indexer holds fillable through its cancellation + // grace period. It widens THIS query and nothing else: the expiry mark, the extend mirror, + // the oracle-address resolution and the hard purge keep their timing, so the divergence + // stays in the over-capture direction the advisory contract above calls safe. Rationale and + // the reason the MARK must not move instead: src/dispenserCancelGrace.js. + async getAllOpenDispenserAddresses(graceFloor){ let db = await this.getConnection(); - let query = - `SELECT ia.address AS address + // Strict number test, not Number(): `Number(null)` is 0, which would arm a floor of + // 1970 on the null cancelGraceFloor returns below the gate and widen the capture set + // on an unarmed network. Fail closed on anything that is not already a finite number. + const floor = graceFloor + const graceActive = (typeof floor === 'number') && Number.isFinite(floor) + // Two literal statements rather than one composed string: the below-gate query must + // stay exactly the text the fleet has been running, so a re-decode of pre-flag-day + // history cannot drift on a formatting edit. + let query = graceActive + ? `SELECT ia.address AS address + FROM dispensers op + LEFT JOIN index_addresses ia ON ia.id = op.address_id + WHERE op.expired_block_index IS NULL + OR op.expiration >= ?` + : `SELECT ia.address AS address FROM dispensers op LEFT JOIN index_addresses ia ON ia.id = op.address_id WHERE op.expired_block_index IS NULL` let addresses = new Set() try { - let rows = await db.query(query); + let rows = graceActive ? await db.query(query, [floor]) : await db.query(query); for (let row of rows){ if (row["address"] != null) addresses.add(row["address"]) @@ -2663,6 +2686,73 @@ class Database { } } + // How many distinct block heights above the current tip have already been + // rolled back and not yet re-synced. + // + // This is the restart-durable half of the safe-depth ceiling. The REORG_HALT + // marker above is best-effort by construction: markReorgHalted runs on the + // abort path, so a DB fault at exactly that moment leaves the halt recorded + // nowhere, and a restarted decoder re-entered verifyReorg with a zeroed depth + // counter and finished the over-deep rollback. The evidence this method reads + // cannot be lost that way, because deleteBlockByIndex commits the REORG marker + // INSIDE the same transaction as the block delete: a deleted block and its + // marker are atomic, so the marker rows above the tip ARE the rollback depth. + // + // Distinct heights, not a row count: a height deleted, re-synced and deleted + // again writes two markers and is one block of depth. Bounded scan: the ceiling + // is 126, so the newest few thousand REORG rows cover every reachable depth, and + // (code, id) is indexed (src/sql/events.sql). THROWS on an unreadable or + // unparseable result - "we could not tell" must never reach the caller as "no + // prior rollback", which is the exact collapse this whole guard exists to stop. + async countReorgDeletesAboveTip(scanLimit = 5000){ + // Throws (after its own retries) rather than returning a sentinel, so an + // unknown tip cannot silently become "everything is above it" or "nothing is". + const tip = await this.getLastBlockIndex() + // Interpolated, not bound: LIMIT placeholders are not used anywhere else in + // this file, so the bound is range-checked here instead and the SQL stays the + // plain shape the rest of the module uses. The value is internal, never + // operator input, and the guard is what makes that literal safe. + const limit = Number(scanLimit) + if (!Number.isInteger(limit) || limit < 1 || limit > 1000000) + throw new Error('countReorgDeletesAboveTip: refusing an out-of-range scan limit: ' + scanLimit) + const query = `SELECT id, data FROM events WHERE code = 'REORG' ORDER BY id DESC LIMIT ${limit};` + let connection = await this.getConnection() + const ownLease = (this.transactionConnection == null) + try { + const rows = await connection.query(query) + if (!Array.isArray(rows)) + throw new Error('countReorgDeletesAboveTip: the REORG marker scan returned no readable rows') + const heightsAboveTip = new Set() + for (const row of rows){ + let payload + try { + payload = (typeof row.data === 'string') ? JSON.parse(row.data) : row.data + } catch (err){ + throw new Error('countReorgDeletesAboveTip: REORG marker id ' + row.id + + ' has an unreadable payload, so the rollback depth cannot be bounded: ' + err.message) + } + // Both marker shapes are arrays of {block_index, block_hash} (one entry + // per row since M-12, several on older rows); anything else means this + // is not the marker whose depth we are counting. + if (!Array.isArray(payload)) + throw new Error('countReorgDeletesAboveTip: REORG marker id ' + row.id + + ' is not the expected array payload, so the rollback depth cannot be bounded') + for (const entry of payload){ + const height = Number(entry && entry.block_index) + if (!Number.isFinite(height)) + throw new Error('countReorgDeletesAboveTip: REORG marker id ' + row.id + + ' carries a non-numeric block_index, so the rollback depth cannot be bounded') + if (height > tip) heightsAboveTip.add(height) + } + } + return heightsAboveTip.size + } finally { + if (ownLease){ + await connection.release() + } + } + } + // Read the durable halt marker WITH its detail. isReorgHalted() above // answers the one question verifyReorg asks (may I roll back?) and deliberately // stays a bare existence probe on the hot reorg path. Operator-facing surfaces @@ -2702,9 +2792,29 @@ class Database { // Called on every verifyReorg abort path BEFORE the throw, so a restart cannot // resume the over-deep rollback. Best-effort by design; the caller swallows any // error so a marker-write failure never masks the original loud abort. + // + // Returns TRUE only when a REORG_HALT row is readable afterwards, never merely + // "the INSERT reported no error". insertEvent swallows every write error and + // returns false, so the boolean it hands back is the only failure signal that + // exists here, and a caller that trusts it without a read-back is trusting a + // driver's ack for a row nobody has seen. That distinction is the whole point: + // this marker is the only thing standing between a restarted decoder and a + // silently resumed over-deep rollback, and every consumer of it (the entry + // guard, the health surfaces, the bootstrap gate) reads the ROW, not the ack. async markReorgHalted(reason){ if (await this.isReorgHalted()) return true - return this.insertEvent('REORG_HALT', { reason: reason, at: new Date().toISOString() }) + const written = await this.insertEvent('REORG_HALT', { reason: reason, at: new Date().toISOString() }) + // Anything other than a clean insert is a failure. DUPLICATED_TRANSACTION + // is truthy and would otherwise read as success, so the read-back below + // decides that case on the row rather than on the errno. + if (written === false) return false + try { + return await this.isReorgHalted() + } catch (_) { + // The write may well have landed, but nothing here can say so, and an + // unconfirmed marker must never report as a confirmed one. + return false + } } } diff --git a/src/decoderMetrics.js b/src/decoderMetrics.js index 4d70e55..967df24 100644 --- a/src/decoderMetrics.js +++ b/src/decoderMetrics.js @@ -124,4 +124,4 @@ function registerDecoderMetrics(registry, decoder) { return { gauges, counters, collector }; } -module.exports = { registerDecoderMetrics, DECODER_GAUGES, DECODER_COUNTERS }; +module.exports = { registerDecoderMetrics }; diff --git a/src/dispenserCancelGrace.js b/src/dispenserCancelGrace.js new file mode 100644 index 0000000..a80d307 --- /dev/null +++ b/src/dispenserCancelGrace.js @@ -0,0 +1,100 @@ +/********************************************************************* + * + * Copyright © 2025-2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC - https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * + * XChain Decoder - dispenser cancellation grace window on payment capture + * + * The indexer keeps a CANCELLED dispenser fillable past its own expiration. Its expiration + * pass skips `cancelling` rows (xchain-indexer/src/db.js getExpiredItems, `s2.status='open'`), + * findMatchingDispensers still matches `status IN ('open','cancelling')`, and DISPENSER_CLOSE + * fires only at the cancel's block time plus DISPENSER_CLOSE_DELAY (3600s). The decoder + * mirrors no cancel at all, deliberately, so it soft-expires that dispenser at its raw + * expiration and drops the address from the block loop's payment-capture set. + * + * Cancel a funded dispenser shortly before its expiration and the two lifecycles diverge in + * the money-bearing direction: the indexer still settles fills, the decoder captures no + * output, and because the indexer only ever sees outputs the decoder persisted to + * transaction_outputs, the buyer's native coin reaches the seller with no DISPENSE record and + * no inventory release. + * + * A blanket grace window closes it by construction. A valid cancel always PRECEDES the + * dispenser's own expiration, so `expiration + grace` always covers `cancel_time + close + * delay`, with no cancel parsing, no BATCH sub-command gate dependency, and no cancel-target + * resolution (the guess the advisory contract in db.js retired). + * + * WHAT THE GRACE MOVES, AND WHAT IT MUST NOT. The widening applies to the CAPTURE SET only: + * getAllOpenDispenserAddresses admits a row whose expiration is no older than the floor this + * module computes. The soft-expire predicate, the expiry MARK, the extend mirror, the + * oracle-address resolution and the hard purge keep their current timing. That confines the + * change to the over-capture direction the advisory contract calls safe, because capture is a + * Set membership test that the indexer arbitrates afterwards. Delaying the MARK instead would + * reach getOpenDispenserOracleAddressBySource, whose below-gate `ORDER BY ... LIMIT 1` would + * then rank a dead row first; oracle-fee capture is a single-address EQUALITY test, so a wrong + * pick captures NOTHING. That is the under-capture direction, a second funds-loss path rather + * than a fix. Widen the capture set, never the mark. + * + * Changing which outputs are persisted is consensus-affecting, so it rides a flag-day in the + * same shape as dispenserExpiryRealign: this module holds the one pure decision. + * + ********************************************************************/ + +'use strict'; + +const { DISPENSER_CANCEL_GRACE_ACTIVATION } = require('./protocol/constants.js') + +// Seconds a soft-expired dispenser stays an eligible payment destination at/above the gate. +// +// Pinned to the indexer's DISPENSER_CLOSE_DELAY (xchain-indexer/src/config.js). The invariant +// is GRACE >= CLOSE_DELAY: the indexer stops matching a cancelled dispenser at cancel time +// plus its close delay, and the cancel precedes the expiration, so a grace of at least the +// close delay covers every block in which the indexer can still settle a fill. Equal, not +// larger, because every extra second is capture the indexer discards. dispenserCancelGrace +// tests read the indexer's value directly, so retuning it there fails this suite until this +// constant follows. +const DISPENSER_CANCEL_GRACE_SECONDS = 3600 + +// Is the grace window in force for a block at `blockTime` on this network? +// +// Fails CLOSED twice over, since either failure mode would widen the persisted output set on +// a chain whose fleet has not armed the change (a fork): +// * an unrecognized network name reads as "no grace", not "no gate"; +// * a null (DISARMED) entry means the network's maintainers have not ratified an instant +// yet, and stays inactive at every block time rather than defaulting to genesis-on. +// +// Comparison is `blockTime >= activation`, the same >= semantics the indexer's +// protocol_changes gates and the sibling dispenser gates use. +function isDispenserCancelGraceActive(consensusNetwork, blockTime){ + const activation = DISPENSER_CANCEL_GRACE_ACTIVATION[consensusNetwork] + if (typeof activation !== 'number') return false + const t = Number(blockTime) + if (!Number.isFinite(t)) return false + return t >= activation +} + +// The capture floor for a block: the oldest expiration still eligible for payment capture. +// +// Returns null below the gate, which getAllOpenDispenserAddresses reads as "keep the +// unwidened set", so a from-genesis re-decode of pre-flag-day history reproduces the output +// set the fleet wrote live, byte for byte. Above it the floor is a pure function of the +// block's own header time, so two honest nodes load the identical capture set. +function cancelGraceFloor(consensusNetwork, blockTime){ + if (!isDispenserCancelGraceActive(consensusNetwork, blockTime)) return null + return Number(blockTime) - DISPENSER_CANCEL_GRACE_SECONDS +} + +module.exports = { + DISPENSER_CANCEL_GRACE_ACTIVATION, + DISPENSER_CANCEL_GRACE_SECONDS, + isDispenserCancelGraceActive, + cancelGraceFloor, +} diff --git a/src/observability/logShipper.js b/src/observability/logShipper.js index 656141a..dae9542 100644 --- a/src/observability/logShipper.js +++ b/src/observability/logShipper.js @@ -187,6 +187,15 @@ function readLogEnv(env = process.env) { }; } +// fetch() settles on the response HEADERS, so the body is still an open stream +// holding its socket while the abort timer is cleared. Release it inside that +// window by cancel, not read; a cancel on an errored body is not a ship failure. +function releaseBody(res) { + const stream = res && res.body; + if (!stream || typeof stream.cancel !== 'function') return Promise.resolve(); + return Promise.resolve(stream.cancel()).catch(() => {}); +} + class LogShipper { /** * @param {object} opts @@ -331,9 +340,9 @@ class LogShipper { const headers = { 'Content-Type': 'application/x-ndjson' }; if (token) headers.Authorization = `Bearer ${token}`; return fetch(url, { method: 'POST', headers, body, signal: controller.signal }) - .then((res) => { + .then((res) => releaseBody(res).then(() => { if (!res.ok) throw new Error(`collector responded ${res.status}`); - }) + })) .finally(() => clearTimeout(timer)); } diff --git a/src/protocol/constants.js b/src/protocol/constants.js index 5c8d842..e2a5d26 100644 --- a/src/protocol/constants.js +++ b/src/protocol/constants.js @@ -427,6 +427,59 @@ const DISPENSER_EXPIRY_REALIGN_ACTIVATION = { regtest: 0, }; +// DISPENSER_CANCEL_GRACE_ACTIVATION (dispenser cancellation grace capture): the flag-day +// at/above which the DECODER keeps a just-expired dispenser in the block loop's payment +// CAPTURE SET for a grace window past its expiration. Keyed on BLOCK TIME with the same >= +// semantics as DISPENSER_EXPIRY_REALIGN_ACTIVATION, because dispensers settle on BTC, LTC +// and DOGE, whose heights diverge. +// +// WHY IT EXISTS: the indexer keeps a CANCELLED dispenser fillable past its own expiration. +// It excludes `cancelling` rows from its expiration pass (xchain-indexer/src/db.js +// getExpiredItems, `s2.status='open'`), keeps them matchable through +// `status IN ('open','cancelling')` in findMatchingDispensers, and closes only at the +// cancel's block time plus DISPENSER_CLOSE_DELAY (3600s). The decoder mirrors no cancel at +// all, by design, so it soft-expires that dispenser at its raw expiration and drops the +// address from the capture set. Cancel a funded dispenser shortly before its expiration and +// a window opens: the indexer still settles fills, the decoder captures no output, and the +// buyer's native coin reaches the seller with no DISPENSE record and no inventory release. +// +// At/above the gate the CAPTURE SET alone widens: a row whose expiration is no older than +// the grace window stays an eligible payment destination even once the soft-expire has +// stamped it. The soft-expire itself, the expiry MARK, the extend mirror, the oracle-address +// resolution and the hard purge all keep their current timing, which confines the change to +// the over-capture direction the decoder's advisory contract (xchain-decoder/src/db.js, +// above extendOpenDispenserExpirationBySource) calls safe. Delaying the MARK instead reaches +// the legacy single-pick oracle resolution, whose ORDER BY ... LIMIT 1 then ranks a dead row +// first and captures nothing at all: the under-capture direction, a second money-bearing +// defect rather than a fix. Widen the capture set, never the mark. +// +// CONSENSUS-AFFECTING: it changes the set of outputs persisted to transaction_outputs, so an +// ungated widening breaks from-genesis byte-identity and forks validators. The unwidened +// capture set therefore stays live BELOW the gate, and a re-decode of pre-flag-day history +// reproduces exactly what the fleet wrote. +// +// null means DISARMED (never active), the fail-closed default: mainnet keeps the unwidened +// capture set until that network's maintainers ratify an instant, chosen with the fleet's +// upgrade state in hand, because arming it too early forks the chain and arming it in the +// past rewrites agreed history. +// +// DEPLOY DEADLINE, once an instant is armed: EVERY decoder on that network MUST be running +// the armed value before the instant, or the fleet splits on the first block whose header +// time passes a cancelled dispenser's expiration. +// +// Vendored byte-equal into xchain-decoder/src/protocol/constants.js; the conformance suite +// keeps the two copies in lockstep. +const DISPENSER_CANCEL_GRACE_ACTIVATION = { + mainnet: null, // DISARMED: awaiting the operator's ratified per-network instant + // ARMED AT GENESIS (instant 0 = always in force), matching the sibling + // DISPENSER_EXPIRY_REALIGN_ACTIVATION under the pre-launch ruling that every feature must + // be ACTIVE on testnet. This gate closes a defect that spends a payer's native coin and + // gives nothing back, so a public testnet WILL hit it. Safe at 0 because testnet + // decoder/indexer state is REBUILT from the chain before launch. + testnet: 0, + regtest: 0, +}; + // BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION (output capture AND open-dispenser registration // through a BATCH): the flag-day at/above which the DECODER reads a BATCH's SUB-COMMANDS // instead of only its top-level ACTION name, both when deciding which native-coin outputs to @@ -619,6 +672,7 @@ module.exports = { ORACLE_FEE_OUTPUT_ACTIVATION, ORACLE_FEE_SET_CAPTURE_ACTIVATION, DISPENSER_EXPIRY_REALIGN_ACTIVATION, + DISPENSER_CANCEL_GRACE_ACTIVATION, BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION, ENVELOPE_MAX_PAYLOAD, ENVELOPE_RECOGNITION_ACTIVATION, diff --git a/src/protocol/indexerBatchLimits.js b/src/protocol/indexerBatchLimits.js index 4752b3f..59730c4 100644 --- a/src/protocol/indexerBatchLimits.js +++ b/src/protocol/indexerBatchLimits.js @@ -57,9 +57,37 @@ const GATED_ACTION_LIMITS = { // BATCH_ISSUANCE_LIMITS, so child issuance is exempt from the top-level ISSUE cap. const CHILD_ISSUE_KEY = "ISSUE.CHILD"; +// Weighted per-BATCH cost budget (indexer: this.weightBudget), which REPLACES the flat +// command cap at/after BATCH_COST_WEIGHTING. Breached => 'invalid: COMMAND (limit)', the +// same string, whole batch. +const WEIGHT_BUDGET = 250; + +// Per-ACTION cost weights (indexer: this.commandWeights). An ACTION absent from this table +// weighs the DEFAULT of 1, which is every ordinary action. +const COMMAND_WEIGHTS = { + "AIRDROP": 25, + "DEPLOY": 30, + "DIVIDEND": 25, + "EXECUTE": 30, + "XEXEC": 30, +}; + +// Per-network BATCH_COST_WEIGHTING activation instants (block TIME, >=), read off the +// sibling's protocol-change registry. Unlike BATCH_ISSUANCE_LIMITS this flag is NOT provably +// on wherever the decoder's capture gate is: mainnet capture is armed while this instant is +// still the house sentinel. null means DISARMED, which is inactive at every block time. +const COST_WEIGHTING_ACTIVATION = { + "mainnet": 9999999999, + "testnet": 0, + "regtest": 0, +}; + module.exports = { COMMAND_LIMIT, ACTION_LIMITS, GATED_ACTION_LIMITS, CHILD_ISSUE_KEY, + WEIGHT_BUDGET, + COMMAND_WEIGHTS, + COST_WEIGHTING_ACTIVATION, }; diff --git a/src/sql/mempool_transactions.sql b/src/sql/mempool_transactions.sql index 9260aa0..97723cc 100644 --- a/src/sql/mempool_transactions.sql +++ b/src/sql/mempool_transactions.sql @@ -16,9 +16,15 @@ DROP TABLE IF EXISTS mempool_transactions; CREATE TABLE mempool_transactions ( tx_hash VARCHAR(250), -- raw transaction hash (NOT an index_transactions id) source VARCHAR(120), -- raw source address (NOT an index_addresses id) - destination VARCHAR(120), -- raw destination address (NOT an index_addresses id) - amount BIGINT, -- BTC amount sent - fee BIGINT, -- BTC Fee paid (miners fee) + -- These three mirror the confirmed twin's shape (transactions.destination_id/amount/fee) + -- so a pending row and its confirmed row line up column for column, and like that twin + -- the decoder is not their authority. The single mempool writer + -- (XChainDecoder.updateMempool -> Database.insertMempoolTransaction) binds + -- parseTransaction's result, whose only success return hardcodes destination:null and + -- carries no amount key, plus a literal fee of 0. + destination VARCHAR(120), -- Not authoritative: always NULL (parseTransaction hardcodes destination:null). Typed to hold a raw address (NOT an index_addresses id) like source. A pending tx's destinations live inside the decoded ACTION string in data, which callers parse. + amount BIGINT, -- Not authoritative: always NULL (parseTransaction emits no amount, so the writer binds undefined). The indexer derives COIN_AMOUNT from transaction_outputs at confirmation. + fee BIGINT, -- Not authoritative: the writer binds a literal 0 (miner fee is not tracked here), mirroring transactions.fee. -- utf8mb4 per column, mirroring transactions.data: a pending row must accept exactly -- what its confirmed twin accepts, or a non-BMP ACTION fails the mempool INSERT with -- errno 1366 and the tx is skipped on every poll. The table default stays utf8mb3 so @@ -44,4 +50,9 @@ CREATE TABLE mempool_transactions ( -- block-confirmation processing; this transient table keeps the raw values verbatim. CREATE UNIQUE INDEX mempool_tx_hash ON mempool_transactions (tx_hash); CREATE INDEX mempool_source ON mempool_transactions (source); +-- mempool_destination covers a column the writer always binds NULL, so it holds exactly one +-- distinct value and selects nothing. It is kept so this table's index set stays a mirror of +-- the confirmed twin's; dropping it needs a dated mode=manual migration plus an operator +-- migrate run per node, since reconcileTableIndexes re-adds any index declared here. Consumers +-- must not filter on destination expecting rows: see xchain-explorer getDecoderMempoolRows. CREATE INDEX mempool_destination ON mempool_transactions (destination); diff --git a/test/e2e/fixtures/docker-compose.test.yml b/test/e2e/fixtures/docker-compose.test.yml index ffb20ca..738f272 100644 --- a/test/e2e/fixtures/docker-compose.test.yml +++ b/test/e2e/fixtures/docker-compose.test.yml @@ -28,7 +28,7 @@ name: xchain-decoder-e2e services: bitcoind-test: - image: bitcoin/bitcoin:28.1 + image: bitcoin/bitcoin:31.1 command: - bitcoind - -regtest diff --git a/test/fixtures/action-manifest.json b/test/fixtures/action-manifest.json index 93e02ab..ad21bd2 100644 --- a/test/fixtures/action-manifest.json +++ b/test/fixtures/action-manifest.json @@ -11,7 +11,7 @@ }, "categories": { "wire-user": "user-encodable on-chain action: decoded + indexed + SDK-encodable", - "validator": "validator-broadcast on-chain action: decoded + indexed but NOT user-encodable (ANCHOR/ATTEST/NODEPROOF/SLASH)", + "validator": "validator-broadcast on-chain action: decoded + indexed but NOT user-encodable (ANCHOR/ATTEST/NODEPROOF/ROLLCALL/SLASH)", "mirror-injected": "indexer-injected from the hub mirror, NOT chain-decoded (XCALL/XEXEC/CROSS_SETTLE)", "lifecycle": "system-generated sub-action, never a decoded wire tx (matches/expiries/dispense)", "explorer-legacy-render": "render-only in the explorer (legacy order/dispenser cancel+edit views); no decoder/indexer twin" diff --git a/test/integration/fixtures/docker-compose.test.yml b/test/integration/fixtures/docker-compose.test.yml index 1d03725..bed92c6 100644 --- a/test/integration/fixtures/docker-compose.test.yml +++ b/test/integration/fixtures/docker-compose.test.yml @@ -23,7 +23,7 @@ name: xchain-decoder-it services: bitcoind-test: - image: bitcoin/bitcoin:28.1 + image: bitcoin/bitcoin:31.1 command: - bitcoind - -regtest diff --git a/test/tools/sync-batch-limits.js b/test/tools/sync-batch-limits.js index fabef80..abf1845 100644 --- a/test/tools/sync-batch-limits.js +++ b/test/tools/sync-batch-limits.js @@ -43,6 +43,15 @@ * answered; the ordering, its block-index gates and its consensus-version gate are asserted * in batchLimitsVendoring.test.js instead. * + * WHAT IT DOES VENDOR, AND WHY THE PARAGRAPH ABOVE DOES NOT COVER IT: the + * BATCH_COST_WEIGHTING activation INSTANT. That ordering invariant is specific to + * BATCH_ISSUANCE_LIMITS and it does NOT hold for the weighting flag: mainnet capture is + * ARMED at 1786838400 while the weighting instant is still the 9999999999 house sentinel, + * so mainnet is a live block time where this module's rules run and the weight budget does + * NOT apply. Applying the budget there would suppress capture for batches the indexer + * dispatches, which is the money-bearing under-capture direction. The instant is therefore + * carried per network and the rule is gated on it, rather than assumed on. + * * Lives under test/ rather than bin/ because it is a maintenance tool for the conformance * suite that consumes it: batchLimitsVendoring.test.js requires deriveFromSibling() and * renderModule() from HERE, so the check and the fix can never implement two different ideas @@ -84,9 +93,44 @@ function deriveFromSibling(){ ACTION_LIMITS: sortedCopy(batch.actionLimits), GATED_ACTION_LIMITS: sortedCopy(batch.gatedActionLimits), CHILD_ISSUE_KEY: batch.childIssueKey, + WEIGHT_BUDGET: batch.weightBudget, + COMMAND_WEIGHTS: sortedCopy(batch.commandWeights), + COST_WEIGHTING_ACTIVATION: costWeightingActivation(), + }; +} + +// The BATCH_COST_WEIGHTING per-network activation instants, read off a REAL ProtocolChanges +// instance. The registration carries all three networks in one object, so one read gives the +// whole map whatever NETWORK the instance was built for. Unlike BATCH_ISSUANCE_LIMITS this +// one IS written into the vendored module: see the header for why no ordering invariant +// excuses it. +function costWeightingChange(){ + const ProtocolChanges = require(INDEXER_CHANGES); + const changes = new ProtocolChanges({ + config: { NETWORK: 'regtest' }, + decoderDb: { getBlockTime: async () => 0 }, + }); + return { change: changes.changes['BATCH_COST_WEIGHTING'], consensusVersion: changes.version }; +} + +// The per-network instant map the vendored module carries. +function costWeightingActivation(){ + const { change } = costWeightingChange(); + // A flag the sibling does not register at all reads as DISARMED everywhere, which leaves + // today's over-capture in place rather than inventing a suppression rule. + if (!change) return { mainnet: null, testnet: null, regtest: null }; + return { + mainnet: instantOf(change.mainnet_time), + testnet: instantOf(change.testnet_time), + regtest: instantOf(change.regtest_time), }; } +// A registered instant, or null when the field is not a finite number (DISARMED). +function instantOf(value){ + return Number.isFinite(Number(value)) ? Number(value) : null; +} + // Key order is normalized so a reordering in the sibling cannot show up as drift, and so the // rendered file is stable across regenerations. function sortedCopy(table){ @@ -171,11 +215,29 @@ const GATED_ACTION_LIMITS = ${renderTable(derived.GATED_ACTION_LIMITS, 0)}; // BATCH_ISSUANCE_LIMITS, so child issuance is exempt from the top-level ISSUE cap. const CHILD_ISSUE_KEY = ${literal(derived.CHILD_ISSUE_KEY)}; +// Weighted per-BATCH cost budget (indexer: this.weightBudget), which REPLACES the flat +// command cap at/after BATCH_COST_WEIGHTING. Breached => 'invalid: COMMAND (limit)', the +// same string, whole batch. +const WEIGHT_BUDGET = ${literal(derived.WEIGHT_BUDGET)}; + +// Per-ACTION cost weights (indexer: this.commandWeights). An ACTION absent from this table +// weighs the DEFAULT of 1, which is every ordinary action. +const COMMAND_WEIGHTS = ${renderTable(derived.COMMAND_WEIGHTS, 0)}; + +// Per-network BATCH_COST_WEIGHTING activation instants (block TIME, >=), read off the +// sibling's protocol-change registry. Unlike BATCH_ISSUANCE_LIMITS this flag is NOT provably +// on wherever the decoder's capture gate is: mainnet capture is armed while this instant is +// still the house sentinel. null means DISARMED, which is inactive at every block time. +const COST_WEIGHTING_ACTIVATION = ${renderTable(derived.COST_WEIGHTING_ACTIVATION, 0)}; + module.exports = { COMMAND_LIMIT, ACTION_LIMITS, GATED_ACTION_LIMITS, CHILD_ISSUE_KEY, + WEIGHT_BUDGET, + COMMAND_WEIGHTS, + COST_WEIGHTING_ACTIVATION, }; `; } @@ -211,5 +273,6 @@ module.exports = { VENDORED, deriveFromSibling, issuanceLimitsChange, + costWeightingChange, renderModule, }; diff --git a/test/unit/auxpowReassembly.test.js b/test/unit/auxpowReassembly.test.js index 9c1aae1..43ee03f 100644 --- a/test/unit/auxpowReassembly.test.js +++ b/test/unit/auxpowReassembly.test.js @@ -97,6 +97,49 @@ describe('malformed-AuxPoW block reassembly fallback', function () { }) await assert.rejects(() => connector.getBlockReassembled('hash'), /no raw tx for in-block txid/) }) + + // The three RPC fetches sit inside the try, so a transport fault is wrapped by the + // same catch that wraps a content fault. Once _auxPowParseErrorCount has escalated a + // height into this path it never decays, so every later failure at that height comes + // through here, and error.code is the only thing separating "the node is unreachable" + // from "this block's bytes are unusable" in the operator log. + it('preserves error.code and the original error as cause on a transport fault', async function () { + const transportErr = new Error('socket hang up') + transportErr.code = 'ECONNRESET' + const connector = makeConnector({ + getBlockHeader: async () => { throw transportErr }, + }) + await assert.rejects( + () => connector.getBlockReassembled('hash'), + (err) => { + assert.strictEqual(err.code, 'ECONNRESET', 'error.code must survive the wrap') + assert.strictEqual(err.cause, transportErr, 'the original error must travel as cause') + assert.match(err.message, /^There were problems reassembling a block without auxpow\. /, + 'the message prefix must stay byte-identical for existing log greps') + assert.strictEqual(err.auxPowParseFailure, undefined, + 'a transport fault must never carry the escalation tag') + return true + } + ) + }) + + // A content fault raised inside the try has no .code, so the wrapper must not invent + // one, and must still stay untagged: escalation is getBlockWithoutAuxPow's to signal. + it('wraps a content fault with a cause and no invented code', async function () { + const connector = makeConnector({ + getBlockHeader: async () => HEADER_HEX, + getBlockVerbose: async () => ({ tx: 'not-an-array' }), + }) + await assert.rejects( + () => connector.getBlockReassembled('hash'), + (err) => { + assert.strictEqual(err.code, undefined, 'no code exists to copy, so none may be set') + assert.match(err.cause.message, /verbose getblock returned no tx array/) + assert.strictEqual(err.auxPowParseFailure, undefined) + return true + } + ) + }) }) describe('BlockchainConnector.probeTxIndex', function () { diff --git a/test/unit/batchLimitsVendoring.test.js b/test/unit/batchLimitsVendoring.test.js index c1ab878..cee8889 100644 --- a/test/unit/batchLimitsVendoring.test.js +++ b/test/unit/batchLimitsVendoring.test.js @@ -42,6 +42,9 @@ const path = require('path'); const VENDORED_MODULE = require('../../src/protocol/indexerBatchLimits.js'); const { BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION, hasProvablyRejectedBatch, + captureCommands, + batchCostWeight, + subCommandCostWeight, subCommandLimitKey, subCommandTick, CHILD_ISSUE_KEY } = require('../../src/batchSubCommandCapture.js'); @@ -84,10 +87,13 @@ function realBatch(opts) { util.addAddressTicker = () => {}; util.detectFeePaymentMode = () => 'native'; + // Network and block time are overridable so the weight-budget cases can drive the SAME + // handler where BATCH_COST_WEIGHTING is armed and where it is not. + const blockTime = (opts.blockTime === undefined) ? T0 : opts.blockTime; const changes = new ProtocolChanges({ - config: { NETWORK: 'regtest' }, + config: { NETWORK: opts.network || 'regtest' }, util: util, - decoderDb: { getBlockTime: async () => T0 }, + decoderDb: { getBlockTime: async () => blockTime }, }); const ids = opts.tickIds || new Map(); @@ -195,6 +201,31 @@ describe('BATCH limit vendoring and cross-repo conformance', function () { assert.deepStrictEqual(VENDORED_MODULE.GATED_ACTION_LIMITS, derived.GATED_ACTION_LIMITS); }); + it('carries the live weight budget, weight table and activation instants', function () { + // A retune of either number in the sibling moves indexer verdicts, and before this + // pin the decoder had no copy of them at all, so the retune was invisible here. + if (!siblingOrSkip(this, sync.INDEXER_BATCH)) return; + const derived = sync.deriveFromSibling(); + assert.strictEqual(VENDORED_MODULE.WEIGHT_BUDGET, derived.WEIGHT_BUDGET); + assert.deepStrictEqual(VENDORED_MODULE.COMMAND_WEIGHTS, derived.COMMAND_WEIGHTS); + assert.deepStrictEqual(VENDORED_MODULE.COST_WEIGHTING_ACTIVATION, + derived.COST_WEIGHTING_ACTIVATION); + }); + + it('keeps every weight an integer >= 1, which is what makes the count cap a sound pre-filter', function () { + // The decoder still checks the raw count first. That is exact rather than + // conservative only while no weight can be below 1. + assert.ok(Number.isInteger(VENDORED_MODULE.WEIGHT_BUDGET) && + VENDORED_MODULE.WEIGHT_BUDGET > 0); + for (const action of Object.keys(VENDORED_MODULE.COMMAND_WEIGHTS)) { + const weight = VENDORED_MODULE.COMMAND_WEIGHTS[action]; + assert.ok(Number.isInteger(weight) && weight >= 1, + action + ' weighs ' + weight + '; a weight below 1 would let a batch whose ' + + 'raw count exceeds the budget still weigh in under it, and the count ' + + 'pre-filter would start rejecting batches the indexer runs'); + } + }); + it('keeps the ungated and gated caps in the tables they came from', function () { // Placement is not cosmetic: everything in ACTION_LIMITS binds in BOTH flag // states, so mirroring it needs no flag reasoning at all, while everything in @@ -263,11 +294,143 @@ describe('BATCH limit vendoring and cross-repo conformance', function () { 'cap and the DEPLOY cap would be enforced here and nowhere else'); } }); + + it('registers BATCH_COST_WEIGHTING with no block-index threshold, so a TIME mirror is sound', function () { + // The decoder mirrors this flag on block TIME alone. isEnabled ANDs a block-index + // leg onto that, so a non-zero threshold could hold the flag off past its instant + // while the decoder already applied the budget: suppression where the indexer + // dispatches, the money-bearing direction. + if (!siblingOrSkip(this, sync.INDEXER_CHANGES)) return; + const { change } = sync.costWeightingChange(); + assert.ok(change, 'BATCH_COST_WEIGHTING must be registered in the sibling'); + for (const network of ['mainnet', 'testnet', 'regtest']) + assert.strictEqual(change[network + '_block'], 0, + network + ' BATCH_COST_WEIGHTING grew a block-index threshold that the ' + + 'vendored instant map cannot express'); + }); + + it('registers it at or below the indexer compiled consensus version', function () { + // Same AND: the version leg could hold the flag off after its instant. + if (!siblingOrSkip(this, sync.INDEXER_CHANGES)) return; + const { change, consensusVersion } = sync.costWeightingChange(); + const current = consensusVersion.split('.').map(Number); + const at = [change.version_major, change.version_minor, change.version_revision]; + const ordered = (at[0] !== current[0]) ? at[0] < current[0] + : (at[1] !== current[1]) ? at[1] < current[1] + : at[2] <= current[2]; + assert.ok(ordered, + 'BATCH_COST_WEIGHTING is registered at ' + at.join('.') + ' but the indexer ' + + 'compiles ' + consensusVersion + ': the version leg of isEnabled would hold ' + + 'the flag off while the decoder applied the budget'); + }); + + it('carries the weighting instants the sibling registers, per network', function () { + if (!siblingOrSkip(this, sync.INDEXER_CHANGES)) return; + const { change } = sync.costWeightingChange(); + for (const network of ['mainnet', 'testnet', 'regtest']) + assert.strictEqual(VENDORED_MODULE.COST_WEIGHTING_ACTIVATION[network], + change[network + '_time'], + network + ': the vendored weighting instant drifted from the sibling. ' + + 'EARLIER here than there means the decoder suppresses capture for batches ' + + 'the indexer still dispatches'); + }); + + it('does NOT assume the weighting flag is on wherever capture is, which is why it is gated', function () { + // The counter-example that killed the ordering shortcut, pinned so it stays a + // counter-example: mainnet capture is armed and mainnet weighting is not. An + // ungated budget would suppress mainnet capture today. + const captureGate = BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION.mainnet; + const weightGate = VENDORED_MODULE.COST_WEIGHTING_ACTIVATION.mainnet; + if (captureGate === null || typeof weightGate !== 'number') return; + assert.ok(captureGate < weightGate, + 'mainnet capture (' + captureGate + ') is no longer earlier than mainnet ' + + 'weighting (' + weightGate + '); re-read isBatchCostWeightingActive before ' + + 'relying on the gate, and re-derive whether the budget may now be unconditional'); + }); }); // ------------------------------------------------------------------------------------- describe('tier 3: driven against the REAL indexer Batch handler', function () { + // The weight budget, driven on BOTH sides of its own flag. Every wire here is under + // the 250-COUNT cap, so nothing in the pre-weighting rule set can explain a rejection: + // the only thing that moves is the summed weight. + const WEIGHT_VECTORS = [ + { name: '9x EXECUTE + SEND', weight: 271, + wire: 'BATCH|0|SEND|0|BTC|TICK|1|addr;' + + Array.from({ length: 9 }, () => 'EXECUTE|0|1|a').join(';') }, + { name: '11x AIRDROP', weight: 275, + wire: 'BATCH|0|' + + Array.from({ length: 11 }, () => 'AIRDROP|0|BTC|TICK|1|a').join(';') }, + ]; + const UNDER_BUDGET = [ + { name: '8x EXECUTE + SEND', weight: 241, + wire: 'BATCH|0|SEND|0|BTC|TICK|1|addr;' + + Array.from({ length: 8 }, () => 'EXECUTE|0|1|a').join(';') }, + { name: '10x AIRDROP', weight: 250, + wire: 'BATCH|0|' + + Array.from({ length: 10 }, () => 'AIRDROP|0|BTC|TICK|1|a').join(';') }, + ]; + const MAINNET_LIVE = 1800000000; // above mainnet capture, below the weighting sentinel + + it('suppresses an over-budget batch on regtest, where the handler rejects it whole', async function () { + if (!siblingOrSkip(this, sync.INDEXER_BATCH)) return; + for (const vector of WEIGHT_VECTORS) { + assert.strictEqual(subCommandsOf(vector.wire).length <= VENDORED_MODULE.COMMAND_LIMIT, + true, vector.name + ' must stay under the COUNT cap or it proves nothing'); + assert.strictEqual( + batchCostWeight(subCommandsOf(vector.wire), ACTION_ALIASES), vector.weight); + const status = await indexerStatus(vector.wire, { network: 'regtest', blockTime: 0 }); + assert.strictEqual(status, 'invalid: COMMAND (limit)', + vector.name + ': premise wrong, the real handler said ' + status); + assert.deepStrictEqual(captureCommands(vector.wire, 'regtest', 0), [], + vector.name + ' still captures on regtest; the weight budget is not mirrored'); + } + }); + + it('still captures an over-budget batch on MAINNET, where the flag is unarmed', async function () { + // The under-capture control, and the reason the rule is gated instead of + // unconditional. Pre-gate reasoning would have suppressed these. + if (!siblingOrSkip(this, sync.INDEXER_BATCH)) return; + for (const vector of WEIGHT_VECTORS) { + const status = await indexerStatus(vector.wire, + { network: 'mainnet', blockTime: MAINNET_LIVE }); + assert.strictEqual(status, 'valid', + vector.name + ': premise wrong, mainnet handler said ' + status); + const view = captureCommands(vector.wire, 'mainnet', MAINNET_LIVE); + assert.strictEqual(view.length, subCommandsOf(vector.wire).length, + 'UNDER-CAPTURE on mainnet: the mirror suppressed ' + vector.name + + ', which the real handler dispatches in full'); + } + }); + + it('leaves a batch AT the budget alone on both networks', async function () { + if (!siblingOrSkip(this, sync.INDEXER_BATCH)) return; + for (const vector of UNDER_BUDGET) { + assert.strictEqual( + batchCostWeight(subCommandsOf(vector.wire), ACTION_ALIASES), vector.weight); + assert.strictEqual(await indexerStatus(vector.wire, { network: 'regtest', blockTime: 0 }), + 'valid', vector.name + ': premise wrong on regtest'); + assert.strictEqual(captureCommands(vector.wire, 'regtest', 0).length, + subCommandsOf(vector.wire).length, + 'UNDER-CAPTURE: ' + vector.name + ' weighs exactly the budget and is valid'); + } + }); + + it('under-charges DEPLOY rather than guessing its format, which is the safe direction', async function () { + // The indexer charges DEPLOY 30 and discounts a format-4 chunk carrier to 1. This + // module reads no FORMAT, so it charges 1 for both: an UNDER-estimate bounded at 29 + // by the one-DEPLOY-per-batch cap. Charging 30 would suppress a batch carrying a + // chunk carrier the indexer runs. + if (!siblingOrSkip(this, sync.INDEXER_BATCH)) return; + assert.strictEqual(VENDORED_MODULE.COMMAND_WEIGHTS.DEPLOY, 30, + 'the sibling stopped weighting DEPLOY at 30; re-derive the discount argument'); + assert.strictEqual(subCommandCostWeight('DEPLOY|0|code', ACTION_ALIASES), 1); + assert.strictEqual(subCommandCostWeight('DEPLOY|4|chunk', ACTION_ALIASES), 1); + assert.strictEqual(VENDORED_MODULE.GATED_ACTION_LIMITS.DEPLOY, 1, + 'the per-batch DEPLOY cap is what bounds the under-estimate at 29'); + }); + it('agrees with it on every vector, and never suppresses a batch it accepts', async function () { if (!siblingOrSkip(this, sync.INDEXER_BATCH)) return; let mirrored = 0; diff --git a/test/unit/decoderHaltDiagnostics.test.js b/test/unit/decoderHaltDiagnostics.test.js index f691faf..385bf1e 100644 --- a/test/unit/decoderHaltDiagnostics.test.js +++ b/test/unit/decoderHaltDiagnostics.test.js @@ -48,8 +48,11 @@ function installSink() { }) } +// Match the EVENT, not a substring of it. formatTextLine renders +// ` [] k=v ...`, so REORG_HALT_MARKER contains +// REORG_HALT and a bare includes() would fold the two records into one count. function linesFor(event) { - return sink.lines.filter((l) => l.includes(event)) + return sink.lines.filter((l) => l.includes('] ' + event + ' ') || l.endsWith('] ' + event)) } function makeDecoder() { @@ -96,21 +99,89 @@ describe('REORG_HALT: a halt the marker cannot record still leaves a record', fu 'the record must carry the depth it was about to persist: ' + line) assert.ok(/reason="[^"]*safe-depth[^"]*"/.test(line), 'the record must carry the reason it was about to persist: ' + line) - assert.ok(line.includes('marker_persisted=false'), + assert.ok(line.includes('marker_write=unavailable'), 'the record must say the marker could not be written: ' + line) assert.ok(line.includes('/status') && line.includes('/live'), 'the record must say which surfaces will NOT report the halt: ' + line) + // Nothing was attempted, so nothing may report an outcome. + assert.strictEqual(linesFor('REORG_HALT_MARKER').length, 0) + assert.strictEqual(decoder.getReorgHaltStatus().marker_persisted, false) }) it('still emits REORG_HALT on the normal path, and says the marker was written', async function () { let marked = null - const decoder = haltingDecoder({ markReorgHalted: async (r) => { marked = r } }) + // The db contract markReorgHalted answers on: TRUE only once a REORG_HALT row + // is readable. A stub returning undefined would be a stub asserting a write it + // never confirmed, which is the exact defect these cases exist for. + const decoder = haltingDecoder({ markReorgHalted: async (r) => { marked = r; return true } }) await assert.rejects(() => decoder.verifyReorg(NODE_TIP), /safe-depth/) const halts = linesFor('REORG_HALT') assert.strictEqual(halts.length, 1) - assert.ok(halts[0].includes('marker_persisted=true'), halts[0]) + assert.ok(halts[0].includes('marker_write=attempting'), halts[0]) + // The pre-write record cannot know the outcome, so it must not claim one. + assert.ok(!halts[0].includes('marker_persisted='), + 'the pre-write record must not assert persistence: ' + halts[0]) + + const outcome = linesFor('REORG_HALT_MARKER') + assert.strictEqual(outcome.length, 1, 'the write outcome must produce exactly one record') + assert.ok(outcome[0].includes('marker_persisted=true'), outcome[0]) + assert.ok(outcome[0].includes('attempts=1'), outcome[0]) assert.ok(marked && /safe-depth/.test(marked), 'the durable marker is still written') + assert.strictEqual(decoder.getReorgHaltStatus().marker_persisted, true) + }) + + // The failure the bootstrap gate exists to stop: the marker write fails, the + // process exits, the restart policy recycles the container, the entry guard reads + // a row that was never written, and the gate counts zero markers and publishes the + // database as known-good. Before this, insertEvent swallowed the write error and + // returned false, markReorgHalted handed that straight back, haltReorg discarded + // it, and the one structured record said marker_persisted=true regardless. + it('reports marker_persisted=false when the durable write is refused, and still aborts', async function () { + const errors = [] + const realError = console.error + console.error = (...a) => { errors.push(a.map(String).join(' ')) } + let attempts = 0 + try { + const decoder = haltingDecoder({ markReorgHalted: async () => { attempts++; return false } }) + await assert.rejects(() => decoder.verifyReorg(NODE_TIP), /safe-depth/, + 'a marker failure must never mask or replace the abort') + + const outcome = linesFor('REORG_HALT_MARKER') + assert.strictEqual(outcome.length, 1) + assert.ok(outcome[0].includes('marker_persisted=false'), + 'a refused write must never report as persisted: ' + outcome[0]) + assert.strictEqual(attempts, 2, 'a refused write is retried once on a fresh connection') + assert.ok(outcome[0].includes('attempts=2'), outcome[0]) + assert.strictEqual(decoder.getReorgHaltStatus().marker_persisted, false) + assert.strictEqual(decoder.getReorgHaltStatus().halted, true) + } finally { + console.error = realError + } + const critical = errors.filter((l) => l.includes('could NOT be persisted')) + assert.strictEqual(critical.length, 1, + 'the only live evidence of an unrecorded halt must be logged: ' + JSON.stringify(errors)) + assert.ok(/full resync/i.test(critical[0]), + 'the line must name the required operator action: ' + critical[0]) + assert.ok(/not a valid bootstrap source/i.test(critical[0]), critical[0]) + }) + + it('carries the cause when the marker write throws rather than returning false', async function () { + const realError = console.error + console.error = () => {} + try { + const decoder = haltingDecoder({ + markReorgHalted: async () => { throw new Error('lost connection to server') } + }) + await assert.rejects(() => decoder.verifyReorg(NODE_TIP), /safe-depth/) + const outcome = linesFor('REORG_HALT_MARKER') + assert.strictEqual(outcome.length, 1) + assert.ok(outcome[0].includes('marker_persisted=false'), outcome[0]) + assert.ok(outcome[0].includes('lost connection to server'), + 'the cause must ride the record: ' + outcome[0]) + } finally { + console.error = realError + } }) it('reports the halt in memory even when nothing durable can be written', async function () { @@ -288,3 +359,43 @@ describe('db: a failed temp-table drop stops being silent', function () { assert.ok(warned[0].includes('lost connection to server'), warned[0]) }) }) + +// api.js registers GET /status inside startApi(), which builds a real decoder and +// opens a listening socket, so the route is not reachable from a unit test. The +// contract is pinned at the source instead, the same way this repo pins the +// getmempool JSON-RPC method (test/unit/mempoolApiSurface.test.js). +// +// What is pinned: reorg_halted never ships without reorg_halt_checked_at. +// xchain-node's BootstrapHealthGate refuses a payload that owns the boolean and +// carries a null/absent timestamp, because a decoder whose fail-soft marker probe +// has NEVER completed publishes exactly what a clean one publishes. That gate probes +// the JSON-RPC health surface first and falls back to GET /status, so the boolean +// must never appear alone in either body. +describe('api.js GET /status halt surface (source pin)', function () { + const fs = require('fs') + const path = require('path') + const src = fs.readFileSync(path.join(__dirname, '../../src/api.js'), 'utf8') + + function statusRouteBody() { + const at = src.indexOf("app.get('/status'") + assert.ok(at > -1, 'GET /status route missing from api.js') + return src.slice(at, at + 3000) + } + + it('publishes reorg_halt_checked_at beside reorg_halted', function () { + const body = statusRouteBody() + assert.ok(/reorg_halted:\s+reorgHalt\.halted/.test(body), + 'GET /status no longer publishes reorg_halted') + assert.ok(/reorg_halt_checked_at:\s*reorgHalt\.checked_at/.test(body), + 'GET /status publishes reorg_halted without the probe timestamp its consumers ' + + 'need to tell "clean" apart from "never looked"') + }) + + it('spells the key exactly as the JSON-RPC health surface does', function () { + // The gate reads one key name across both surfaces; a near-miss spelling on + // the fallback body reads as an absent timestamp and refuses every decoder. + const occurrences = src.match(/reorg_halt_checked_at/g) || [] + assert.ok(occurrences.length >= 2, + 'reorg_halt_checked_at must appear on both the health result and GET /status') + }) +}) diff --git a/test/unit/dispenserCancelGrace.test.js b/test/unit/dispenserCancelGrace.test.js new file mode 100644 index 0000000..8aca4e1 --- /dev/null +++ b/test/unit/dispenserCancelGrace.test.js @@ -0,0 +1,338 @@ +'use strict'; + +// Copyright © 2025-2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC - https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// Dispenser CANCELLATION GRACE: payment capture must outlast the indexer's fill window. +// +// The two services disagree about when a CANCELLED dispenser stops taking money, and the +// disagreement runs in the money-bearing direction. The indexer excludes `cancelling` rows +// from its expiration pass, keeps matching `status IN ('open','cancelling')`, and closes only +// at cancel time + DISPENSER_CLOSE_DELAY (3600s). The decoder mirrors no cancel at all, so it +// soft-expires the row at its raw expiration and drops the address from the block loop's +// capture set. Cancel a funded dispenser shortly before its expiration and the indexer keeps +// settling fills while the decoder captures nothing, so the buyer's native coin reaches the +// seller with no DISPENSE record and no inventory release. +// +// The invariant this suite drives, over a sweep of block times rather than one lucky point: +// for every block in which the INDEXER would still settle a fill, +// the DECODER's capture set contains the dispenser's address. +// +// SENSITIVITY: the sweep and the single-block case FAIL against a decoder whose capture set +// is `expired_block_index IS NULL` alone, which is the behavior below the flag-day and the +// behavior this suite exists to change. Below-the-gate assertions pin that older behavior in +// place, so the green side is a gate flip and not a rewritten expectation. + +const assert = require('assert') +const sinon = require('sinon') + +const XChainDecoder = require('../../src/XChainDecoder') +const Database = require('../../src/db.js') +const { DISPENSER_CANCEL_GRACE_SECONDS, + cancelGraceFloor } = require('../../src/dispenserCancelGrace') + +const PREV_WIRE = Buffer.from( + '00112233445566778899aabbccddeeff0123456789abcdeffedcba9876543210', + 'hex' +) +// The reorg check compares a block's prevHash, byte-reversed to display order, against the +// stored hash of the block below it. +const PREV_HASH = Buffer.from(PREV_WIRE).reverse().toString('hex') + +const ADDR = 'bcrt1qgracedispenser' +const EXPIRATION = 1700000000 // the dispenser's own expiry instant +const CANCEL_AT = EXPIRATION - 600 // cancelled 10 minutes before it +// The indexer's cancel close-delay, named here to express "a block the indexer would still +// settle a fill in". The decoder carries its own copy as DISPENSER_CANCEL_GRACE_SECONDS, and +// dispenserCancelGraceActivation.test.js pins the two together against the indexer's config. +const INDEXER_CLOSE_DELAY = 3600 + +// The indexer's rule for a CANCELLED dispenser: expiration does not close it (getExpiredItems +// skips `cancelling` rows), findMatchingDispensers still matches it, and DISPENSER_CLOSE fires +// at cancel time + close delay. +function indexerStillSettlesFill(blockTime){ + return blockTime < CANCEL_AT + INDEXER_CLOSE_DELAY +} + +// A faithful in-memory model of the decoder `dispensers` table, mirroring the db.js SQL for +// the two methods this fix touches. Every capture load is recorded with the floor it was +// given, so a test can assert on the exact set the block loop received. +class DispenserModel { + constructor(){ + this.rows = [] + this.captureLoads = [] + } + async insertDispenser(){ return true } + async extendOpenDispenserExpirationBySource(){ return true } + async getOpenDispenserOracleAddressBySource(){ return null } + async getOpenDispenserOracleAddressesBySource(){ return [] } + async purgeExpiredDispensers(){ return true } + // Mirrors deleteOpenDispensers: stamp open rows whose expiration < minExpiration. + async deleteOpenDispensers(blockIndex, minExpiration){ + for (const r of this.rows) + if (r.expiredBlockIndex === null && r.expiration < Number(minExpiration)) + r.expiredBlockIndex = blockIndex + return true + } + // Mirrors getAllOpenDispenserAddresses: + // WHERE expired_block_index IS NULL (no floor) + // WHERE expired_block_index IS NULL OR expiration >= ? (floor bound) + async getAllOpenDispenserAddresses(graceFloor){ + // Same strict number test as db.js: `Number(null)` is 0, which would silently arm a + // 1970 floor below the gate. + const floor = graceFloor + const graceActive = (typeof floor === 'number') && Number.isFinite(floor) + const set = new Set(this.rows + .filter(r => r.expiredBlockIndex === null || (graceActive && r.expiration >= floor)) + .map(r => r.address)) + this.captureLoads.push({ floor: graceActive ? floor : null, set }) + return set + } +} + +function fakeTx(id){ + return { getId: () => id, outs: [] } +} + +// An inert parseTransaction result: this suite exercises the block loop's CAPTURE-SET +// plumbing, not the decode path. +function inertParseResult(){ + return { + data: Buffer.alloc(0), source: null, destination: null, amount: 0, + dispenseOutputs: [], paymentOutputs: [], compiledDataLength: 0, rawData: null, + } +} + +// Drive the real block loop over two blocks on `consensusNetwork`: +// block 0 at `expireAt` - the decoder's own soft-expire stamps the dispenser here; +// block 1 at `payAt` - the payment block whose capture set the test asserts on. +// Nothing is pre-stamped by hand: the stamp under test is written by the production +// deleteOpenDispensers call site. +function runTwoBlocks(consensusNetwork, expireAt, payAt, model){ + const decoder = new XChainDecoder( + 'bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null + ) + // The gate reads consensusNetwork, so set it directly rather than routing a mainnet name + // through the chain-identity machinery this suite does not exercise. + decoder.consensusNetwork = consensusNetwork + decoder.startBlockIndex = 0 + decoder.sleep = async () => {} + + const timesByHeight = { 0: expireAt, 1: payAt } + const setsSeenByParse = [] + decoder.parseTransaction = async (tx, openDispenserAddresses) => { + setsSeenByParse.push(openDispenserAddresses) + return inertParseResult() + } + + decoder.connector = { + getBlockchainInfo: async () => ({ verificationprogress: 1, blocks: 1 }), + getBlockHash: async (height) => 'height:' + height, + getBlock: async (hash) => hash, + } + + let commits = 0 + decoder.db = { + createDatabase: async () => true, + verifyDatabase: async () => true, + verifyTables: async () => true, + runMigrations: async () => ({ applied: [], pending: [] }), + getLastBlockIndex: async () => -1, + getLastTxIndex: async () => 0, + // Block 1 runs the reorg check against block 0's stored hash. Both blocks carry the + // same PREV_WIRE, so answering with that value keeps the chain contiguous and the loop + // out of its reorg branch, which this suite does not exercise. + getBlockByIndex: async () => ({ block_hash: PREV_HASH }), + beginTransaction: async () => {}, + endTransaction: async () => {}, + commitTransaction: async () => { if (++commits >= 2) decoder.stopFlag = true; return true }, + insertBlock: async () => true, + insertEvent: async () => true, + insertTransaction: async () => true, + insertTransactionOutput: async () => true, + POISON_ROW: 2, + DUPLICATED_TRANSACTION: 1, + insertDispenser: (d) => model.insertDispenser(d), + extendOpenDispenserExpirationBySource: (s, e, b) => model.extendOpenDispenserExpirationBySource(s, e, b), + deleteOpenDispensers: (b, m) => model.deleteOpenDispensers(b, m), + purgeExpiredDispensers: (h) => model.purgeExpiredDispensers(h), + getAllOpenDispenserAddresses: (f) => model.getAllOpenDispenserAddresses(f), + getOpenDispenserOracleAddressBySource: (s) => model.getOpenDispenserOracleAddressBySource(s), + getOpenDispenserOracleAddressesBySource: (s) => model.getOpenDispenserOracleAddressesBySource(s), + } + + decoder.xchainBlockDecoder = { + blockFromHex: (hex) => ({ + prevHash: Buffer.from(PREV_WIRE), + timestamp: timesByHeight[Number(String(hex).split(':')[1])], + transactions: [fakeTx('tx-at-' + String(hex))], + }) + } + + return decoder.start().then(() => ({ setsSeenByParse })) +} + +// One funded dispenser, cancelled shortly before its expiration, as the decoder holds it: +// the decoder mirrors no cancel, so the row carries only its own expiration. +function fundedCancelledDispenser(){ + const model = new DispenserModel() + model.rows.push({ address: ADDR, expiration: EXPIRATION, expiredBlockIndex: null }) + return model +} + +describe('dispenser cancellation grace: decoder capture outlasts the indexer fill window', function () { + this.timeout(0) + + it('captures a payment made after expiry while the indexer still settles fills', async () => { + // The finding's named failure mode, driven end to end. The dispenser is cancelled at + // EXPIRATION - 600, so the indexer keeps settling until EXPIRATION + 3000. A payment + // 30 minutes past the expiration lands squarely inside that window. + const payAt = EXPIRATION + 1800 + assert.ok(indexerStillSettlesFill(payAt), + 'the probe block must be one the indexer would still settle a fill in') + + const model = fundedCancelledDispenser() + const { setsSeenByParse } = await runTwoBlocks('regtest', EXPIRATION + 1, payAt, model) + + // The production soft-expire really did stamp the row on the earlier block, so the + // grace clause is what carries it, not an unexpired row. + assert.strictEqual(model.rows[0].expiredBlockIndex, 0, + 'block 0 must have soft-expired the dispenser, or this test proves nothing') + + assert.strictEqual(model.captureLoads.length, 2) + const payLoad = model.captureLoads[1] + assert.strictEqual(payLoad.floor, payAt - DISPENSER_CANCEL_GRACE_SECONDS, + 'the block loop must pass the grace floor derived from this block header time') + assert.ok(payLoad.set.has(ADDR), + 'a payment inside the indexer fill window must still be captured by the decoder') + + // The set the loop handed parseTransaction is the same object, so capture really runs + // against the widened set rather than a copy made for the assertion. + assert.strictEqual(setsSeenByParse[1], payLoad.set) + }) + + it('keeps the unwidened capture set below the flag-day (the other side of the gate)', async () => { + // Same blocks, same model, DISARMED network. This is the behavior the fleet runs today + // and the behavior a from-genesis re-decode of pre-flag-day history must reproduce. + const payAt = EXPIRATION + 1800 + const model = fundedCancelledDispenser() + await runTwoBlocks('mainnet', EXPIRATION + 1, payAt, model) + + assert.strictEqual(model.rows[0].expiredBlockIndex, 0) + const payLoad = model.captureLoads[1] + assert.strictEqual(payLoad.floor, null, + 'below the gate the block loop must pass no floor at all') + assert.ok(!payLoad.set.has(ADDR), + 'below the gate the expired dispenser stays out of the capture set') + }) + + it('closes capture once the indexer can no longer settle a fill', async () => { + // The grace is a window, not an amnesty: past expiration + grace the address leaves the + // capture set, and by then the indexer stopped matching the dispenser long ago. + const payAt = EXPIRATION + DISPENSER_CANCEL_GRACE_SECONDS + 1 + assert.ok(!indexerStillSettlesFill(payAt), + 'the probe block must be one the indexer has already closed') + + const model = fundedCancelledDispenser() + await runTwoBlocks('regtest', EXPIRATION + 1, payAt, model) + + assert.ok(!model.captureLoads[1].set.has(ADDR), + 'past the grace window the dispenser leaves the capture set') + }) + + it('covers every block of the indexer fill window, swept at five-minute steps', async () => { + // The invariant, not a lucky point. Walk the payment block from the expiration out past + // the grace and assert the implication in both directions at each step. + let insideWindowBlocks = 0 + for (let payAt = EXPIRATION + 1; payAt <= EXPIRATION + 4500; payAt += 300){ + const model = fundedCancelledDispenser() + await runTwoBlocks('regtest', EXPIRATION + 1, payAt, model) + const captured = model.captureLoads[model.captureLoads.length - 1].set.has(ADDR) + + if (indexerStillSettlesFill(payAt)){ + insideWindowBlocks++ + assert.ok(captured, + `block time ${payAt}: the indexer still settles fills here, so the decoder ` + + 'must still capture payments to the dispenser') + } + // Outside the indexer's window capture is merely allowed to continue to the end of + // the grace: over-capture is the direction the advisory contract calls safe, and + // the indexer drops the surplus. + if (payAt > EXPIRATION + DISPENSER_CANCEL_GRACE_SECONDS) + assert.ok(!captured, `block time ${payAt}: capture must end with the grace window`) + } + // Guard against a vacuous sweep: an arithmetic slip that made the window empty would + // otherwise pass every assertion above. + assert.ok(insideWindowBlocks >= 8, + `the sweep must cross at least 8 blocks inside the indexer fill window, saw ${insideWindowBlocks}`) + }) +}) + +describe('Database#getAllOpenDispenserAddresses() grace floor', function () { + afterEach(() => sinon.restore()) + + function makeDb(){ return new Database('127.0.0.1', 3306, 'xchain_btc_regtest', 'u', 'p') } + function withConn(queryStub){ + const conn = { + query: queryStub, release: sinon.stub().resolves(), + beginTransaction: sinon.stub().resolves(), commit: sinon.stub().resolves(), + rollback: sinon.stub().resolves(), + } + return { pool: { getConnection: sinon.stub().resolves(conn) } } + } + + it('runs the unwidened predicate and binds nothing when no floor is given', async () => { + const db = makeDb() + const q = sinon.stub().resolves([{ address: ADDR }]) + db.pool = withConn(q).pool + await db.getAllOpenDispenserAddresses() + const [sql, params] = q.firstCall.args + assert.ok(/expired_block_index IS NULL/.test(sql)) + assert.ok(!/expiration >= \?/.test(sql), + 'the below-gate query must not carry the grace clause') + assert.strictEqual(params, undefined, 'the below-gate query must bind no parameter') + }) + + it('adds the grace clause and binds the floor when one is given', async () => { + const db = makeDb() + const q = sinon.stub().resolves([{ address: ADDR }]) + db.pool = withConn(q).pool + const floor = cancelGraceFloor('regtest', EXPIRATION + 1800) + await db.getAllOpenDispenserAddresses(floor) + const [sql, params] = q.firstCall.args + assert.ok(/expired_block_index IS NULL\s*\n\s*OR op\.expiration >= \?/.test(sql), + 'the above-gate query must admit rows whose expiration is no older than the floor') + assert.deepStrictEqual(params, [EXPIRATION + 1800 - DISPENSER_CANCEL_GRACE_SECONDS]) + }) + + it('treats a null or non-finite floor as no grace at all', async () => { + // cancelGraceFloor returns null below the gate, so this is the fail-closed path that + // keeps an unarmed network on the legacy capture set. + for (const floor of [null, undefined, NaN, 'soon']){ + const db = makeDb() + const q = sinon.stub().resolves([]) + db.pool = withConn(q).pool + await db.getAllOpenDispenserAddresses(floor) + const [sql, params] = q.firstCall.args + assert.ok(!/expiration >= \?/.test(sql), `floor ${String(floor)} must not widen the query`) + assert.strictEqual(params, undefined) + } + }) + + it('still returns null on a query fault, with or without a floor', async () => { + // A failed read and an empty set must stay distinguishable; the grace path must not + // quietly become an empty-set success. + for (const floor of [null, EXPIRATION]){ + const db = makeDb() + db.pool = withConn(sinon.stub().rejects(new Error('fail'))).pool + assert.strictEqual(await db.getAllOpenDispenserAddresses(floor), null) + } + }) +}) diff --git a/test/unit/dispenserCancelGraceActivation.test.js b/test/unit/dispenserCancelGraceActivation.test.js new file mode 100644 index 0000000..600a503 --- /dev/null +++ b/test/unit/dispenserCancelGraceActivation.test.js @@ -0,0 +1,196 @@ +'use strict'; + +// Copyright © 2025-2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC - https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// DISPENSER_CANCEL_GRACE_ACTIVATION drift guard, plus the two cross-repo invariants the +// grace constant depends on. +// +// The gate widens the block loop's payment-capture address set by a grace window past a +// dispenser's expiration, which changes the set of outputs persisted to transaction_outputs. +// That is consensus-affecting in both directions: +// * arming it in the PAST rewrites agreed history, so a from-genesis re-decode stops +// matching what the fleet wrote live; +// * arming it on a network whose decoders are not all running the value forks the fleet at +// the first block that passes a cancelled dispenser's expiration. +// So mainnet is DISARMED (null) until the operator ratifies an instant, and the helper fails +// closed on anything that is not a number. +// +// Two tiers, so a one-sided edit fails somewhere no matter which checkout is present: +// 1. PIN - the vendored map has the disarmed/genesis-on shape, in this repo alone. +// 2. DOCS - it is value-identical to the canonical map in +// xchain-documentation/protocol/constants.js. +// Tier 2 skips when the sibling checkout is absent (standalone deploy); set +// XCHAIN_REQUIRE_SIBLINGS=1 in CI so a missing sibling hard-fails instead of green-by-skip. + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); + +const { DISPENSER_CANCEL_GRACE_ACTIVATION, + DISPENSER_CANCEL_GRACE_SECONDS, + isDispenserCancelGraceActive, + cancelGraceFloor } = require('../../src/dispenserCancelGrace.js'); +const XChainDecoder = require('../../src/XChainDecoder.js'); + +const DOCS_CONSTANTS = process.env.XCHAIN_DOCS_DIR + ? path.join(process.env.XCHAIN_DOCS_DIR, 'protocol', 'constants.js') + : path.join(__dirname, '..', '..', '..', 'xchain-documentation', 'protocol', 'constants.js'); +const INDEXER_DIR = process.env.XCHAIN_INDEXER_DIR || + path.join(__dirname, '..', '..', '..', 'xchain-indexer'); +const INDEXER_CONFIG = path.join(INDEXER_DIR, 'src', 'config.js'); +const REQUIRE_SIBLINGS = process.env.XCHAIN_REQUIRE_SIBLINGS === '1'; + +function siblingOrSkip(ctx, file){ + if (fs.existsSync(file)) return true; + if (REQUIRE_SIBLINGS) + throw new Error('XCHAIN_REQUIRE_SIBLINGS=1 but sibling not found: ' + file); + ctx.skip(); + return false; +} + +// The indexer pins DISPENSER_CLOSE_DELAY as a literal in a config builder that wants a live +// environment, so read the assignment out of the source rather than executing the module. +function indexerCloseDelay(){ + const src = fs.readFileSync(INDEXER_CONFIG, 'utf8'); + const m = src.match(/config\['DISPENSER_CLOSE_DELAY'\]\s*=\s*(\d+)\s*;/); + assert.ok(m, 'xchain-indexer/src/config.js must assign a numeric DISPENSER_CLOSE_DELAY'); + return Number(m[1]); +} + +describe('DISPENSER_CANCEL_GRACE_ACTIVATION conformance', function () { + + it('keeps mainnet DISARMED, with testnet and regtest genesis-on', function () { + // Teeth for the ratification requirement: a number on MAINNET means someone armed a + // consensus boundary without the operator's ratified instant. + assert.strictEqual(DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet, null); + assert.strictEqual(DISPENSER_CANCEL_GRACE_ACTIVATION.testnet, 0); + assert.strictEqual(DISPENSER_CANCEL_GRACE_ACTIVATION.regtest, 0); + }); + + it('is value-identical to the canonical map in xchain-documentation', function () { + if (!siblingOrSkip(this, DOCS_CONSTANTS)) return; + const canon = require(DOCS_CONSTANTS).DISPENSER_CANCEL_GRACE_ACTIVATION; + assert.ok(canon && typeof canon === 'object', + 'xchain-documentation/protocol/constants.js must export DISPENSER_CANCEL_GRACE_ACTIVATION'); + assert.deepStrictEqual( + { mainnet: DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet, + testnet: DISPENSER_CANCEL_GRACE_ACTIVATION.testnet, + regtest: DISPENSER_CANCEL_GRACE_ACTIVATION.regtest }, + { mainnet: canon.mainnet, testnet: canon.testnet, regtest: canon.regtest }, + 'the vendored map drifted from the canonical one; a one-sided flag-day edit forks ' + + 'the decoder fleet at the first block that passes a cancelled dispenser expiration'); + }); + + it('a DISARMED network is inactive at every block time, including absurd ones', function () { + // A `time >= null` coercion would read 0 and arm mainnet from genesis, which is the + // failure this pins. + assert.strictEqual(isDispenserCancelGraceActive('mainnet', 0), false); + assert.strictEqual(isDispenserCancelGraceActive('mainnet', 1786060800), false); + assert.strictEqual(isDispenserCancelGraceActive('mainnet', 4000000000), false); + assert.strictEqual(cancelGraceFloor('mainnet', 4000000000), null); + }); + + it('testnet and regtest are active from genesis', function () { + assert.strictEqual(isDispenserCancelGraceActive('testnet', 0), true); + assert.strictEqual(isDispenserCancelGraceActive('regtest', 1700000000), true); + }); + + it('fails closed on an unrecognized network name', function () { + // An unknown network must read as "no grace", never as "no gate": the latter would + // widen the persisted output set on an unarmed chain. + assert.strictEqual(isDispenserCancelGraceActive('signet', 4000000000), false); + assert.strictEqual(isDispenserCancelGraceActive(undefined, 4000000000), false); + assert.strictEqual(isDispenserCancelGraceActive('', 4000000000), false); + assert.strictEqual(cancelGraceFloor('signet', 4000000000), null); + }); + + it('fails closed on a non-finite block time', function () { + assert.strictEqual(isDispenserCancelGraceActive('regtest', NaN), false); + assert.strictEqual(isDispenserCancelGraceActive('regtest', undefined), false); + assert.strictEqual(isDispenserCancelGraceActive('regtest', 'not-a-time'), false); + assert.strictEqual(cancelGraceFloor('regtest', NaN), null); + }); + + it('flips exactly at the armed instant once a network IS armed (>= semantics)', function () { + // The map is disarmed today, so arm a network in place for the length of this test and + // drive the REAL helper (the module reads the map per call, so the mutation is + // visible). This pins the boundary the operator will ratify onto: >=, so the block AT + // the instant already carries the grace, matching every protocol_changes gate. + const ARMED = 1789430400; + const saved = DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet; + DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet = ARMED; + try { + assert.strictEqual(isDispenserCancelGraceActive('mainnet', ARMED - 1), false, + 'the block below the instant keeps the unwidened capture set'); + assert.strictEqual(cancelGraceFloor('mainnet', ARMED - 1), null); + assert.strictEqual(isDispenserCancelGraceActive('mainnet', ARMED), true, + 'the block AT the instant already carries the grace'); + assert.strictEqual(cancelGraceFloor('mainnet', ARMED), + ARMED - DISPENSER_CANCEL_GRACE_SECONDS); + assert.strictEqual(isDispenserCancelGraceActive('mainnet', ARMED + 1), true); + } finally { + DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet = saved; + } + assert.strictEqual(isDispenserCancelGraceActive('mainnet', ARMED), false, + 'the map must be back to DISARMED after the probe'); + }); + + it('the floor is exactly one grace window below the block time', function () { + // The floor is the whole consensus decision: two nodes reading the same header time + // must load the same capture set, so it is a pure subtraction and nothing else. + assert.strictEqual(cancelGraceFloor('regtest', 1700000000), + 1700000000 - DISPENSER_CANCEL_GRACE_SECONDS); + assert.strictEqual(cancelGraceFloor('regtest', 0), -DISPENSER_CANCEL_GRACE_SECONDS); + }); +}); + +describe('DISPENSER_CANCEL_GRACE_SECONDS cross-repo invariants', function () { + + it('covers the indexer cancellation grace period', function () { + // The invariant the whole fix rests on. The indexer stops matching a cancelled + // dispenser at cancel time + DISPENSER_CLOSE_DELAY, and a valid cancel always precedes + // the dispenser's own expiration, so a grace of at least the close delay covers every + // block in which the indexer can still settle a fill. A shorter grace reopens the + // funds-loss window silently, because the arithmetic keeps working. + if (!siblingOrSkip(this, INDEXER_CONFIG)) return; + const closeDelay = indexerCloseDelay(); + assert.ok( + DISPENSER_CANCEL_GRACE_SECONDS >= closeDelay, + `DISPENSER_CANCEL_GRACE_SECONDS (${DISPENSER_CANCEL_GRACE_SECONDS}) must be >= the ` + + `indexer DISPENSER_CLOSE_DELAY (${closeDelay}); the indexer was retuned without ` + + 'following it in src/dispenserCancelGrace.js' + ); + }); + + it('holds the hand-pinned close delay the constant is set from', function () { + // Baseline that fires even without the sibling checkout: the constant is pinned EQUAL + // to the indexer's delay, not merely above it, because every extra second is capture + // the indexer discards. + assert.strictEqual(DISPENSER_CANCEL_GRACE_SECONDS, 3600); + }); + + it('the hard purge cannot reclaim a row that is still inside the grace window', function () { + // purgeExpiredDispensers hard-deletes a row 126 blocks (DISPENSER_EXPIRE_SAFE_DEPTH) + // after the block that stamped it. A row purged while still inside its grace window + // would drop out of the widened capture set early, so the depth has to outlast the + // grace on the FASTEST chain the platform decodes. Purge is keyed on block HEIGHT, so + // even a burst that outran this stays deterministic across nodes; the margin is what + // keeps the grace from being cosmetic on DOGE. + const FASTEST_TARGET_SPACING = 60; // DOGE (BTC 600 / LTC 150 / DOGE 60) + const purgeSpan = XChainDecoder.DISPENSER_EXPIRE_SAFE_DEPTH * FASTEST_TARGET_SPACING; + assert.ok( + purgeSpan > DISPENSER_CANCEL_GRACE_SECONDS, + `the purge span (${purgeSpan}s at target spacing) must exceed the grace window ` + + `(${DISPENSER_CANCEL_GRACE_SECONDS}s), or a cancelled dispenser is hard-deleted ` + + 'before the indexer stops matching it' + ); + }); +}); diff --git a/test/unit/dispenserSafeDepth.test.js b/test/unit/dispenserSafeDepth.test.js index 74847cf..4e54493 100644 --- a/test/unit/dispenserSafeDepth.test.js +++ b/test/unit/dispenserSafeDepth.test.js @@ -32,7 +32,7 @@ const XChainDecoder = require('../../src/XChainDecoder.js'); // Baseline floor (always asserted, even without the sibling checkout). // Mirrors xchain-utxo-tracker/src/undo-blocks.js DEFAULT_UNDO_BLOCKS. -const DEEPEST_UNDO_WINDOW = 120; // DOGE (BTC 12 / LTC 48 / DOGE 120) +const DEEPEST_UNDO_WINDOW = 120; // LTC and DOGE (BTC 12 / LTC 120 / DOGE 120) // Headroom above the deepest window so a small undo-window re-tune can never // land exactly at the purge threshold. Matches the margin baked into @@ -40,7 +40,7 @@ const DEEPEST_UNDO_WINDOW = 120; // DOGE (BTC 12 / LTC 48 / DOGE 120) const SAFETY_MARGIN = 6; describe('DISPENSER_EXPIRE_SAFE_DEPTH', function () { - it('is at least as deep as the deepest per-chain reorg window (DOGE = 120) + margin', function () { + it('is at least as deep as the deepest per-chain reorg window (LTC and DOGE = 120) + margin', function () { assert.ok( XChainDecoder.DISPENSER_EXPIRE_SAFE_DEPTH >= DEEPEST_UNDO_WINDOW + SAFETY_MARGIN, `SAFE_DEPTH (${XChainDecoder.DISPENSER_EXPIRE_SAFE_DEPTH}) must be >= ${DEEPEST_UNDO_WINDOW + SAFETY_MARGIN} ` + diff --git a/test/unit/mempoolIsolation.test.js b/test/unit/mempoolIsolation.test.js index 613a330..98f34fa 100644 --- a/test/unit/mempoolIsolation.test.js +++ b/test/unit/mempoolIsolation.test.js @@ -100,4 +100,56 @@ describe('updateMempool DB isolation', function () { assert.ok(!calls.db.includes('endTransaction'), 'mempool failure must not roll back the block db') assert.strictEqual(decoder.mempoolBusy, false, 'the busy flag must be cleared after the cycle') }) + + // getrawmempool answers with an array of txids, and the JSON-RPC unwrap only checks that a + // result member is present. A malformed-but-iterable answer is the destructive one: a bare + // string dedups into per-character "txids", and deleteAndCompareTxsNotInList anti-joins the + // stored table against that snapshot and deletes every pending row. + it('a non-array getrawmempool answer skips the poll instead of reaching the delete', async () => { + const { decoder, calls } = buildDecoder(true) + decoder.connector.getRawMempool = async () => 'aabbcc' + await decoder.updateMempool() + assert.deepStrictEqual(calls.mempoolDb, [], 'a shape fault must not reach the mempool DB at all') + assert.deepStrictEqual(calls.db, []) + assert.strictEqual(decoder.mempoolBusy, false, 'the busy flag must be cleared so the next poll runs') + }) + + it('an empty node mempool still reaches the delete, so departed txs are pruned', async () => { + // [] is a legitimate answer, not a shape fault: the anti-join is exactly what prunes the + // stored rows when the node's mempool has drained. + const { decoder } = buildDecoder(true) + let received = null + decoder.mempoolDb.deleteAndCompareTxsNotInList = async (list) => { + received = list.slice(); return { transactionsDeleted: 0 } + } + decoder.connector.getRawMempool = async () => [] + await decoder.updateMempool() + assert.deepStrictEqual(received, [], 'an empty snapshot must still be handed to the diff') + }) + + // deleteAndCompareTxsNotInList empties and refills the caller's array in place, leaving it + // holding only the new arrivals, so the cycle summary must not read that array for the + // mempool size: in steady state that reads 0 against a full table, which an operator takes + // for "mempool tracking has stopped". + it('the cycle summary reports the node mempool size, not the post-diff new arrivals', async () => { + const { decoder } = buildDecoder(true) + decoder.connector.getRawMempool = async () => ['ccc', 'bbb', 'aaa'] + decoder.connector.getRawTransactions = async () => [] + // Steady state: every txid the node reported is already stored, so the diff empties the list. + decoder.mempoolDb.deleteAndCompareTxsNotInList = async (list) => { + list.length = 0; return { transactionsDeleted: 0 } + } + const lines = [] + const realLog = console.log + console.log = (...args) => { lines.push(args.join(' ')) } + try { + await decoder.updateMempool() + } finally { + console.log = realLog + } + const summary = lines.find((line) => line.startsWith('Mempool updated!')) + assert.ok(summary, 'the cycle must emit its summary line') + assert.match(summary, /3 in mempool/, 'the summary must report the size the node reported') + assert.match(summary, /0 new/, 'the post-diff length is the new-arrival count and must be labelled so') + }) }) diff --git a/test/unit/parseTransaction.test.js b/test/unit/parseTransaction.test.js index 847f6b6..c857f8b 100644 --- a/test/unit/parseTransaction.test.js +++ b/test/unit/parseTransaction.test.js @@ -629,3 +629,86 @@ describe('XChainDecoder#getSourceFromOutput()', () => { assert.ok(decoder.connector.getRawTransaction.calledTwice) }) }) + +// Per-chain capability gate on the P2WSH witness carrier. +// +// The branch recognized the XCHNp2wsh marker and read payload out of +// transaction.ins[i].witness[2] on any chain, consulting only the witness stack's +// SHAPE. On a chain that declares no segwit the lane relied entirely on upstream +// node validation to keep witness data from ever arriving, while the sibling +// taproot envelope lane has carried an explicit per-chain gate all along. +// +// The BTC case is the control that makes the DOGE case mean something: the same +// bytes, the same stubs, the same helper, and the only difference is the chain. +// Without it a blanket disable of the whole P2WSH lane would look identical. +describe('XChainDecoder#parseTransaction() P2WSH per-chain segwit gate', () => { + + const WITNESS_PAYLOAD = Buffer.from('witness-carrier-payload') + + function decoderFor(network) { + const decoder = new XChainDecoder( + network, null, null, null, null, null, + '127.0.0.1', 18443, 'rpc', 'rpc', false + ) + decoder.db = { isThereADispenserForAddress: sinon.stub().resolves(false) } + decoder.connector = { getRawTransaction: sinon.stub().rejects(new Error('mocked')) } + decoder.getSourceFromOutput = sinon.stub().resolves(null) + // The chunk lanes set p2shFundingTxId, which drives a commit fetch this + // test is not about. Stubbed on BOTH decoders so the only difference + // between them stays the chain. + sinon.stub(decoder, 'findFundingFeeOutputs').resolves([]) + sinon.stub(decoder, 'removeObfuscation').resolves( + Buffer.concat([Buffer.from('XCHN'), Buffer.from('p2wsh')]) + ) + return decoder + } + + // One well-formed witness carrier: a 3-element stack whose third element + // decompiles to a single payload push, which is exactly what the extraction + // path reads. Byte-identical for both chains. + function witnessCarrierTx() { + const tx = new bitcoin.Transaction() + tx.version = 2 + tx.addInput(PREV_HASH, 1) + tx.ins[0].witness = [ + Buffer.alloc(72, 0x30), + Buffer.alloc(33, 0x02), + // The extraction reads decompile(witness[2])[0] as this input's chunk, + // and the reassembled chunks are themselves decompiled as the action + // stream, so the carried chunk is a COMPILED push inside one more push. + bitcoin.script.compile([bitcoin.script.compile([WITNESS_PAYLOAD])]) + ] + tx.addOutput(bitcoin.script.compile([bitcoin.opcodes.OP_RETURN, Buffer.alloc(8, 0xAB)]), 0) + addP2PKHOutput(tx) + return tx + } + + afterEach(() => { + sinon.restore() + }) + + it('a segwit chain still extracts the witness payload (control)', async () => { + const decoder = decoderFor('bitcoin-regtest') + const result = await decoder.parseTransaction(witnessCarrierTx()) + + assert.ok(result, 'the witness carrier must still produce an action on a segwit chain') + assert.strictEqual(result.data.toString('utf-8'), WITNESS_PAYLOAD.toString('utf-8')) + }) + + it('a non-segwit chain extracts nothing from the same bytes and does not throw', async () => { + const decoder = decoderFor('dogecoin-regtest') + const result = await decoder.parseTransaction(witnessCarrierTx()) + + const extracted = result && result.data ? result.data : Buffer.alloc(0) + assert.strictEqual(extracted.length, 0, + 'a chain declaring supportsSegwit:false must read no payload out of a witness stack') + }) + + it('the gate is chain capability, not a parse error: the non-segwit chain records none', async () => { + const decoder = decoderFor('dogecoin-regtest') + const before = decoder.parseErrors + await decoder.parseTransaction(witnessCarrierTx()) + assert.strictEqual(decoder.parseErrors, before, + 'skipping an impossible carrier is not a malformed-transaction event') + }) +}) diff --git a/test/unit/reorgDepthAcrossRestart.test.js b/test/unit/reorgDepthAcrossRestart.test.js new file mode 100644 index 0000000..0a89734 --- /dev/null +++ b/test/unit/reorgDepthAcrossRestart.test.js @@ -0,0 +1,229 @@ +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// The safe-depth ceiling must survive a restart that lost the halt marker. +// +// verifyReorg's abort writes a durable REORG_HALT row, and its entry guard reads +// it. That marker is written on the ABORT path, which is exactly the moment the +// database may be the thing failing: markReorgHalted gets two attempts and then +// gives up, recording the halt only in memory and in logs. A restarted decoder +// then found a clean entry guard, started blocksDeleted empty, and finished the +// over-deep rollback past the dispenser purge window. +// +// These pin the second, restart-durable leg: the depth is reconstructed from the +// REORG markers deleteBlockByIndex commits inside each delete's own transaction, +// which no abort-time write failure can lose. + +'use strict' + +const assert = require('assert') +const sinon = require('sinon') +const XChainDecoder = require('../../src/XChainDecoder') +const Database = require('../../src/db.js') + +const SAFE_DEPTH = 126 +const NODE_TIP = 100 + +function makeDecoder() { + return new XChainDecoder( + 'bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null + ) +} + +// A decoder holding blocks far above the node tip, so verifyReorg takes its +// above-tip delete branch and rolls back one block per pass. `db` overrides let +// each case state only the restart evidence it is about. +function restartedDecoder(db) { + const decoder = makeDecoder() + let height = 300 + const deleted = [] + decoder.db = Object.assign({ + getLastBlockIndex: async () => height, + getBlockByIndex: async (i) => (i < 0 ? null : { block_index: i, block_hash: 'aa'.repeat(32) }), + deleteBlockByIndex: async (i) => { deleted.push(i); height -= 1; return true }, + // A restart that LOST the halt marker: the entry guard sees a clean database, + // which is the whole premise of the hazard. + isReorgHalted: async () => false, + markReorgHalted: async () => true + }, db) + decoder.connector = { rpcErrors: 0 } + // The seed read retries with a 3s sleep; no test may pay for that in wall time. + decoder.sleep = async () => {} + return { decoder, deleted } +} + +describe('verifyReorg: the safe-depth ceiling survives a lost halt marker', function () { + + it('refuses the FIRST delete when the committed REORG markers already reach the ceiling', async function () { + const { decoder, deleted } = restartedDecoder({ + countReorgDeletesAboveTip: async () => SAFE_DEPTH + }) + + await assert.rejects(() => decoder.verifyReorg(NODE_TIP), /safe-depth/) + + assert.strictEqual(deleted.length, 0, + 'a rollback already at the ceiling must not delete one more block after a restart') + assert.strictEqual(decoder.getReorgHaltStatus().halted, true, + 'the decoder must publish the halt on its health surface') + }) + + it('spends only the depth that is left, then aborts', async function () { + const { decoder, deleted } = restartedDecoder({ + countReorgDeletesAboveTip: async () => SAFE_DEPTH - 3 + }) + + await assert.rejects(() => decoder.verifyReorg(NODE_TIP), /safe-depth/) + + assert.strictEqual(deleted.length, 3, + 'the remaining budget is the ceiling minus what was already deleted above the tip') + }) + + it('names both halves of the depth in the abort message, so the operator sees the resume', async function () { + const { decoder } = restartedDecoder({ + countReorgDeletesAboveTip: async () => SAFE_DEPTH - 2 + }) + + await assert.rejects(() => decoder.verifyReorg(NODE_TIP), (err) => { + assert.match(err.message, /Already rolled back 126 blocks/) + assert.match(err.message, /2 in this run/) + assert.match(err.message, /resumed from 124 already deleted above the tip/) + return true + }) + }) + + // The pre-fix behaviour, kept explicit: with the marker lost and no durable + // depth evidence, the run would delete 126 more blocks on top of whatever the + // aborted one had already taken. + it('without the durable count, a restart spends a whole fresh budget', async function () { + const { decoder, deleted } = restartedDecoder({}) // no countReorgDeletesAboveTip + + await assert.rejects(() => decoder.verifyReorg(NODE_TIP), /safe-depth/) + + assert.strictEqual(deleted.length, SAFE_DEPTH, + 'this is the resume the durable count exists to stop') + }) + + it('deletes nothing at all when the prior depth cannot be read', async function () { + let attempts = 0 + const { decoder, deleted } = restartedDecoder({ + countReorgDeletesAboveTip: async () => { attempts++; throw new Error('connection lost') } + }) + + await assert.rejects(() => decoder.verifyReorg(NODE_TIP), + /prior rollback depth could not be read/) + + assert.strictEqual(attempts, 3, 'a transient read fault is retried before the refusal') + assert.strictEqual(deleted.length, 0, + 'an unknown depth must never be treated as a zero depth') + // Deliberately NOT a durable halt: a read fault is infrastructure, and a + // REORG_HALT row would block every later reorg until an operator cleared it. + assert.strictEqual(decoder.getReorgHaltStatus().halted, false) + }) + + it('recovers when the read fault is transient', async function () { + let attempts = 0 + const { decoder, deleted } = restartedDecoder({ + countReorgDeletesAboveTip: async () => { + if (++attempts < 3) throw new Error('connection lost') + return SAFE_DEPTH - 1 + } + }) + + await assert.rejects(() => decoder.verifyReorg(NODE_TIP), /safe-depth/) + assert.strictEqual(deleted.length, 1) + }) + + it('still refuses on the halt marker when one DID survive, before counting anything', async function () { + let counted = false + const { decoder, deleted } = restartedDecoder({ + isReorgHalted: async () => true, + countReorgDeletesAboveTip: async () => { counted = true; return 0 } + }) + + await assert.rejects(() => decoder.verifyReorg(NODE_TIP), /HALTED from a prior over-deep reorg abort/) + assert.strictEqual(counted, false, 'the cheap durable guard still runs first') + assert.strictEqual(deleted.length, 0) + }) +}) + +describe('Database#countReorgDeletesAboveTip()', function () { + + afterEach(() => sinon.restore()) + + // Answers the two queries the method makes, in order: the tip, then the scan. + function dbWith(tipRows, eventRows) { + const db = new Database('127.0.0.1', 3306, 'xchain_btc_mainnet', 'u', 'p') + const query = sinon.stub().callsFake(async (sql) => { + if (/MAX\(block_index\)/.test(sql)) return tipRows + if (/code = 'REORG'/.test(sql)) return eventRows + throw new Error('unexpected query: ' + sql) + }) + db.pool = { getConnection: sinon.stub().resolves({ query, release: sinon.stub().resolves() }) } + return { db, query } + } + + const marker = (height) => ({ id: height, data: JSON.stringify([{ block_index: height, block_hash: 'bb' }]) }) + + it('counts only the marked heights above the current tip', async function () { + const { db } = dbWith([{ max_height: 200n }], [marker(203), marker(202), marker(201), marker(199)]) + assert.strictEqual(await db.countReorgDeletesAboveTip(), 3) + }) + + it('counts a height once even when it was deleted, re-synced and deleted again', async function () { + const { db } = dbWith([{ max_height: 200n }], [marker(201), marker(201), marker(202)]) + assert.strictEqual(await db.countReorgDeletesAboveTip(), 2) + }) + + it('returns zero on a database with no REORG markers at all', async function () { + const { db } = dbWith([{ max_height: 200n }], []) + assert.strictEqual(await db.countReorgDeletesAboveTip(), 0) + }) + + it('handles the pre-M-12 multi-entry payload shape', async function () { + const rows = [{ id: 9, data: JSON.stringify([{ block_index: 201 }, { block_index: 202 }, { block_index: 199 }]) }] + const { db } = dbWith([{ max_height: 200n }], rows) + assert.strictEqual(await db.countReorgDeletesAboveTip(), 2) + }) + + // "We could not tell" must never arrive at verifyReorg as "no prior rollback". + it('THROWS on an unparseable marker payload', async function () { + const { db } = dbWith([{ max_height: 200n }], [{ id: 7, data: '{not json' }]) + await assert.rejects(() => db.countReorgDeletesAboveTip(), /unreadable payload/) + }) + + it('THROWS on a marker payload that is not the expected array', async function () { + const { db } = dbWith([{ max_height: 200n }], [{ id: 7, data: JSON.stringify({ block_index: 201 }) }]) + await assert.rejects(() => db.countReorgDeletesAboveTip(), /not the expected array payload/) + }) + + it('THROWS on a non-numeric block_index', async function () { + const { db } = dbWith([{ max_height: 200n }], [{ id: 7, data: JSON.stringify([{ block_index: 'tip' }]) }]) + await assert.rejects(() => db.countReorgDeletesAboveTip(), /non-numeric block_index/) + }) + + it('bounds the scan, and refuses a nonsense bound rather than emitting it as SQL', async function () { + const { db, query } = dbWith([{ max_height: 200n }], []) + await db.countReorgDeletesAboveTip(250) + const scan = query.getCalls().map(c => String(c.args[0])).find(s => /code = 'REORG'/.test(s)) + assert.match(scan, /ORDER BY id DESC LIMIT 250;/) + + await assert.rejects(() => db.countReorgDeletesAboveTip('5; DROP TABLE blocks'), + /out-of-range scan limit/) + await assert.rejects(() => db.countReorgDeletesAboveTip(0), /out-of-range scan limit/) + }) + + it('propagates a tip read that could not be answered, rather than counting against a guess', async function () { + const db = new Database('127.0.0.1', 3306, 'xchain_btc_mainnet', 'u', 'p') + db.sleep = async () => {} + const query = sinon.stub().rejects(new Error('connection lost')) + db.pool = { getConnection: sinon.stub().resolves({ query, release: sinon.stub().resolves() }) } + await assert.rejects(() => db.countReorgDeletesAboveTip(), /getLastBlockIndex failed/) + }) +}) diff --git a/test/unit/reorgHaltSurface.test.js b/test/unit/reorgHaltSurface.test.js index 9e82fde..ba21e51 100644 --- a/test/unit/reorgHaltSurface.test.js +++ b/test/unit/reorgHaltSurface.test.js @@ -213,3 +213,57 @@ describe('Database.getReorgHaltMarker', function () { assert.strictEqual(marker.at, '2026-07-26 03:30:31') }) }) + +// The marker is the only thing standing between a restarted decoder and a silently +// resumed over-deep rollback, and every consumer of it reads the ROW: the entry +// guard, the health surfaces, and xchain-node's BootstrapHealthGate, which counts +// `events WHERE code='REORG_HALT'` before publishing a database as a bootstrap +// source. insertEvent swallows every write error and returns false, so a caller that +// trusts its ack without reading the row back certifies a marker nobody has seen. +describe('Database.markReorgHalted reports the row, not the ack', function () { + + function stubMarkerDb({ alreadyHalted = false, insertResult = true, readBack = true, readBackThrows = false } = {}) { + const db = new Database('127.0.0.1', 3306, 'test_db', 'u', 'p') + const calls = { insert: 0, probes: 0 } + db.isReorgHalted = async () => { + calls.probes++ + if (calls.probes === 1) return alreadyHalted + if (readBackThrows) throw new Error('connection lost during read-back') + return readBack + } + db.insertEvent = async () => { calls.insert++; return insertResult } + return { db, calls } + } + + it('returns true when the row is readable after the insert', async function () { + const { db, calls } = stubMarkerDb() + assert.strictEqual(await db.markReorgHalted('over-deep'), true) + assert.strictEqual(calls.insert, 1) + assert.strictEqual(calls.probes, 2, 'the write must be confirmed by a read-back') + }) + + it('returns false when insertEvent swallowed the write error', async function () { + const { db } = stubMarkerDb({ insertResult: false }) + assert.strictEqual(await db.markReorgHalted('over-deep'), false) + }) + + // The case the old contract got wrong: insertEvent answers truthy and no row + // exists (the transaction was rolled back under it, or the errno was 1062 and + // DUPLICATED_TRANSACTION came back, which is truthy and is not a written row). + it('returns false when the insert claims success but no row is readable', async function () { + const { db } = stubMarkerDb({ readBack: false }) + assert.strictEqual(await db.markReorgHalted('over-deep'), false, + 'an unconfirmed marker must never report as a confirmed one') + }) + + it('returns false rather than throwing when the read-back itself fails', async function () { + const { db } = stubMarkerDb({ readBackThrows: true }) + assert.strictEqual(await db.markReorgHalted('over-deep'), false) + }) + + it('stays idempotent: an existing marker is true without a second write', async function () { + const { db, calls } = stubMarkerDb({ alreadyHalted: true }) + assert.strictEqual(await db.markReorgHalted('over-deep'), true) + assert.strictEqual(calls.insert, 0) + }) +}) diff --git a/test/unit/security/configuration/dependency-advisories.test.js b/test/unit/security/configuration/dependency-advisories.test.js index 1c5cee8..9d2903a 100644 --- a/test/unit/security/configuration/dependency-advisories.test.js +++ b/test/unit/security/configuration/dependency-advisories.test.js @@ -89,6 +89,23 @@ describe('Security: remediated dependency advisories @regression @tier4', functi // reach here, not dev-only: express-rate-limit and geoip-lite both parse the // client request IP, so a padded or mapped form could key a different // rate-limit bucket than its canonical address. + + // mariadb <=3.5.2 is the production DB driver every service here opens its + // pool with, not a dev-only reach. GHSA-cqhc-2h57-wpxf (HIGH) sends the + // password in the clear to a man in the middle even when the pool asked for + // `ssl: true`, because the connector falls back to a plaintext handshake + // rather than failing closed; GHSA-42r5-vhpq-m858 is the same + // cleartext-credential exposure stated as its own advisory; and + // GHSA-g5xc-5w98-jfvm is SQL injection through Buffer parameter escaping + // under the big5, gbk, sjis, cp932 and gb18030 client charsets. Measured + // exposure in this topology is nil today (no repo passes `ssl:` to a pool, + // every DB connection is host-local, and none selects one of those + // charsets), which is why the fix rode the ordinary fleet path instead of a + // hotfix. The guard is what stops a lockfile refresh, or the first service + // that does dial a remote database over TLS, from landing back inside the + // range. 3.5.3 is the patch, and it also moves the driver's own lru-cache + // onto the 11.x line, so a splice that bumps mariadb and leaves lru-cache + // at 10.4.3 has not actually installed the fixed driver. const advisories = [ { name: 'fast-uri', minSafe: [3, 1, 5], majorSeries: 3 }, { name: 'brace-expansion', minSafe: [5, 0, 9], majorSeries: 5 }, @@ -103,7 +120,8 @@ describe('Security: remediated dependency advisories @regression @tier4', functi { name: 'shell-quote', minSafe: [1, 9, 0], majorSeries: 1 }, { name: 'form-data', minSafe: [4, 0, 6], majorSeries: 4 }, { name: 'tmp', minSafe: [0, 2, 6], majorSeries: 0 }, - { name: 'ip-address', minSafe: [10, 3, 1], majorSeries: 10 } + { name: 'ip-address', minSafe: [10, 3, 1], majorSeries: 10 }, + { name: 'mariadb', minSafe: [3, 5, 3], majorSeries: 3 } ]; // Compares dotted numeric version triples without pulling in semver. @@ -201,4 +219,18 @@ describe('Security: remediated dependency advisories @regression @tier4', functi assert.ok(cmp(parse(axios.VERSION), [1, 18, 0]) >= 0, `installed axios is ${axios.VERSION}, inside the vulnerable range (fixed in 1.18.0)`); }); + + // Same reasoning as ADV-5, for the one entry in this list every service + // opens a socket with. mariadb's `exports` block hides its own + // package.json from require() and the module exports no version constant, + // so read the installed manifest off disk instead. + it('ADV-10: the installed mariadb reports a patched runtime version', function () { + if (!lockEntries('mariadb').length) return this.skip(); + const manifest = path.join(root, 'node_modules', 'mariadb', 'package.json'); + if (!fs.existsSync(manifest)) return this.skip(); + + const installed = JSON.parse(fs.readFileSync(manifest, 'utf8')).version; + assert.ok(cmp(parse(installed), [3, 5, 3]) >= 0, + `installed mariadb is ${installed}, inside the vulnerable range (fixed in 3.5.3)`); + }); }); diff --git a/test/unit/verifyReorgRetry.test.js b/test/unit/verifyReorgRetry.test.js index bea3e9a..56c8212 100644 --- a/test/unit/verifyReorgRetry.test.js +++ b/test/unit/verifyReorgRetry.test.js @@ -203,7 +203,10 @@ describe('XChainDecoder.verifyReorg durable halt (restart-mid-reorg)', function deleteBlockByIndex: async (h) => { deleted.push(h); dbState.top = h - 1 }, insertEvent: async () => true, isReorgHalted: async () => store.halted, - markReorgHalted: async () => { store.halted = true } + // Answers the db contract (true = a REORG_HALT row is now readable) rather + // than shrugging with undefined: haltReorg honours this value, and a stub + // that shrugs models a decoder whose marker write silently failed. + markReorgHalted: async () => { store.halted = true; return true } } return { decoder, deleted } }