From 09522dfbeaa1a75ef4a851d459ab02c3499a4f36 Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Mon, 22 Jun 2026 11:57:53 -0700 Subject: [PATCH 01/20] fix(gql): truthful hasNextPage when id-dedup collapses a full CH page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windowed GraphQL `transactions` queries against the ClickHouse-backed indexer could return a partial page with `pageInfo.hasNextPage = false` and no error, silently stranding every subsequent page — a cursor client paging until `hasNextPage` is false would under-report results for dense wallets/ranges. Root cause: the CH legs fetch `pageSize + 1` rows with a full-key `LIMIT 1 BY height, block_transaction_index, is_data_item, id`, but the composite then dedups by `id` alone. Stale rows that share an `id` and height while differing on `block_transaction_index` (a placeholder bti=0 left by an earlier indexing pass alongside the real bti) are not folded by the SQL `LIMIT 1 BY`, yet collapse in the id-dedup. When that shrinks the merged set to `pageSize` or fewer, `edges.length > pageSize` reads false even though the leg came back full. Derive `hasNextPage` from each leg's raw result before the cross-leg id-dedup: a CH leg returning more than `pageSize` rows, or the SQLite leg's own `hasNextPage`, means more matching rows exist. Erring toward true is safe — the worst case is one extra page fetch that comes back empty. No SQL or cursor changes. Adds a regression test reproducing the production shape (same id, bti 0 vs 12, both data items) collapsing a full page to `pageSize`. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/database/composite-clickhouse.test.ts | 41 +++++++++++++++++++++++ src/database/composite-clickhouse.ts | 21 +++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/database/composite-clickhouse.test.ts b/src/database/composite-clickhouse.test.ts index 7fe0bd235..bcd410045 100644 --- a/src/database/composite-clickhouse.test.ts +++ b/src/database/composite-clickhouse.test.ts @@ -412,6 +412,47 @@ describe('CompositeClickHouseDatabase', () => { assert.equal(result.edges.length, 2); assert.equal(result.pageInfo.hasNextPage, false); }); + + it('is true when a full leg collapses to pageSize after id-dedup (PE-9124)', async () => { + // A single leg returns its full `pageSize + 1` rows, but a stale + // duplicate (same id, different block_transaction_index — the SQL's + // full-key `LIMIT 1 BY` does not fold it) collapses the deduped set to + // exactly `pageSize`. hasNextPage MUST stay true: the leg came back full, + // so more matching rows exist beyond this page. Counting deduped edges + // alone would report `2 > 2 == false` and silently strand every later + // page. + const composite = buildComposite({ + sqlite, + chRowsByLeg: { + stable: [ + chRow({ + id: id('dupitem'), + height: 1, + blockTransactionIndex: 0, + isDataItem: true, + }), + chRow({ + id: id('dupitem'), + height: 1, + blockTransactionIndex: 12, + isDataItem: true, + }), + chRow({ id: id('itemb'), height: 2 }), + ], + }, + }); + + const result = await composite.getGqlTransactions({ + pageSize: 2, + sortOrder: 'HEIGHT_ASC', + }); + assert.equal(result.edges.length, 2); + assert.deepEqual( + result.edges.map((e) => e.node.id), + [id('dupitem'), id('itemb')], + ); + assert.equal(result.pageInfo.hasNextPage, true); + }); }); describe('SQLite fallback behavior', () => { diff --git a/src/database/composite-clickhouse.ts b/src/database/composite-clickhouse.ts index 82edfcf1d..72ddd714e 100644 --- a/src/database/composite-clickhouse.ts +++ b/src/database/composite-clickhouse.ts @@ -888,8 +888,10 @@ export class CompositeClickHouseDatabase implements GqlQueryable { // SQLite leg: rejection degrades same as today. const sqliteSettled = await sqliteSettledPromise; let sqliteEdges: GqlTransactionsResult['edges'] = []; + let sqliteHasNextPage = false; if (sqliteSettled.status === 'fulfilled') { sqliteEdges = sqliteSettled.value.edges; + sqliteHasNextPage = sqliteSettled.value.pageInfo.hasNextPage; if ( sqliteSettled.value.warnings !== undefined && sqliteSettled.value.warnings.length > 0 @@ -966,9 +968,26 @@ export class CompositeClickHouseDatabase implements GqlQueryable { return bufA.compare(bufB) * sortOrderModifier; }); + // `hasNextPage` must reflect whether any leg has rows BEYOND this page, + // derived from each leg's RAW result before the cross-leg id-dedup above. + // Both ClickHouse legs fetch `pageSize + 1` rows (see buildChTransactionsSql), + // so a leg that comes back with more than `pageSize` rows has more matching + // data to give. Relying on the deduped `edges.length > pageSize` alone + // under-reports: when duplicate ids collapse the merged set to `pageSize` or + // fewer — e.g. stale rows that share an `id` but differ on + // `block_transaction_index`, which the SQL's full-key `LIMIT 1 BY` does not + // fold — the page falsely signals completeness and every later page is + // silently stranded. Erring toward `true` is safe: the worst case is one + // extra page fetch that comes back empty. + const hasNextPage = + edges.length > pageSize || + stableTxs.length > pageSize || + unstableTxs.length > pageSize || + sqliteHasNextPage; + return { pageInfo: { - hasNextPage: edges.length > pageSize, + hasNextPage, }, edges: edges.slice(0, pageSize), ...(warnings.length > 0 ? { warnings } : {}), From 1a573a3a203c31e4e1b79a9a44e2809e57095177 Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Mon, 22 Jun 2026 12:13:06 -0700 Subject: [PATCH 02/20] docs(gql): fix stale getGqlTransactions hasNextPage TSDoc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function-level TSDoc still described the pre-fix behavior (hasNextPage computed against the deduped edge list) as intentional — that was the PE-9124 bug. Update it to describe deriving hasNextPage from each leg's raw result before id-dedup. Addresses CodeRabbit review feedback on PR #792. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/database/composite-clickhouse.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/database/composite-clickhouse.ts b/src/database/composite-clickhouse.ts index 72ddd714e..9251f7476 100644 --- a/src/database/composite-clickhouse.ts +++ b/src/database/composite-clickhouse.ts @@ -672,10 +672,12 @@ export class CompositeClickHouseDatabase implements GqlQueryable { * Set-based dedup with stable > unstable > sqlite precedence handles * the stabilization-overlap window where the same `id` briefly lives * in both `transactions` and `new_transactions` until TTL drops the - * unstable copy. JS-side sort + slice produces a deterministic page; - * `hasNextPage` is computed against the deduped edge list (not the - * raw concatenation) so cross-leg duplicates that collapse the page - * don't falsely advertise more results. + * unstable copy. JS-side sort + slice produces a deterministic page. + * `hasNextPage` is derived from each leg's raw result *before* + * cross-leg id-dedup: a leg returning more than `pageSize` rows has + * more matching data to give. Computing it against the deduped edge + * count instead would falsely signal completeness whenever duplicate + * ids collapse the merged page to `pageSize` or fewer (see PE-9124). * * See `clickhouse-pipeline.md` and PR #699 for the design rationale. */ From 241ca2672c2465a82ce81befa518152dfe88bc11 Mon Sep 17 00:00:00 2001 From: vilenarios Date: Mon, 22 Jun 2026 20:07:10 +0000 Subject: [PATCH 03/20] feat(arns): serve ArNS names whose ANT record targets an IPFS CID ANT records now carry a targetProtocol (0=Arweave, 1=IPFS) and the record target can be an IPFS CID instead of an Arweave TX ID (@ar.io/sdk 4.0.0). Previously the on-demand resolver read only transactionId and validated it as a 43-char Arweave ID, so a CID-targeted name failed to resolve. - on-demand resolver: read targetProtocol; validate the target as a CID when protocol is IPFS, else as an Arweave ID; surface protocol on the resolution (cached transparently as part of NameResolution). - NameResolution: optional protocol field ('arweave' | 'ipfs'); undefined treated as 'arweave' for backward compat (e.g. trusted-gateway hops). - arns middleware: when a name resolves to an IPFS CID and IPFS serving is enabled, hand off to the IPFS handler (sets ipfsCid/ipfsPath, mirroring the IPFS subdomain middleware) instead of the Arweave data handler. Completes the 'ArNS -> IPFS CID' phase the IPFS PR was foundation for. Verified: typecheck + lint clean, resolver unit tests pass, and live on-demand Solana resolution of existing names still serves via Arweave. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/middleware/arns.ts | 31 ++++++++++++++++++++++- src/resolution/on-demand-arns-resolver.ts | 15 ++++++++++- src/routes/arns.ts | 16 ++++++++++++ src/types.d.ts | 10 ++++++++ 4 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/middleware/arns.ts b/src/middleware/arns.ts index 04fac887a..f8d2282c2 100644 --- a/src/middleware/arns.ts +++ b/src/middleware/arns.ts @@ -27,9 +27,15 @@ const MAX_ARNS_NAME_LENGTH = 51; export const createArnsMiddleware = ({ dataHandler, nameResolver, + ipfsHandler, }: { dataHandler: Handler; nameResolver: NameResolver; + // Optional handler for ArNS names whose ANT record resolves to an IPFS CID + // (`targetProtocol === ipfs`). Provided only when IPFS serving is enabled; + // when absent, IPFS-protocol resolutions fall through to the Arweave data + // path (which will 404 on a CID, the correct behavior with IPFS disabled). + ipfsHandler?: Handler; }): Handler => asyncMiddleware(async (req, res, next) => { // Skip all ArNS processing if no root ArNS hosts are configured. @@ -166,6 +172,11 @@ export const createArnsMiddleware = ({ const resolutionDuration = Date.now() - resolutionStart; span.setAttribute('arns.resolution_duration_ms', resolutionDuration); + // Set when the ANT record resolves to an IPFS CID (targetProtocol + // ipfs); routes the final handoff to the IPFS handler instead of the + // Arweave data handler. + let serveViaIpfs = false; + if (resolution.statusCode === 451) { span.setAttribute('arns.blocked', true); span.setAttribute('http.status_code', 451); @@ -185,6 +196,20 @@ export const createArnsMiddleware = ({ req.dataId = resolvedId; req.manifestPath = manifestPath; + // If the ANT record points at an IPFS CID, hand the request to the + // IPFS handler (when IPFS serving is enabled) instead of the Arweave + // data path. The handler reads `ipfsCid`/`ipfsPath` off the request, + // matching the IPFS subdomain middleware's contract. + const arnsProtocol = + (resolution as { protocol?: 'arweave' | 'ipfs' }).protocol ?? + 'arweave'; + if (arnsProtocol === 'ipfs' && ipfsHandler !== undefined) { + serveViaIpfs = true; + (req as any).ipfsCid = resolvedId; + (req as any).ipfsPath = manifestPath; + span.setAttribute('arns.protocol', 'ipfs'); + } + // Parse ArNS name components const parts = arnsSubdomain.split('_'); const basename = parts.pop() ?? ''; // last part is basename @@ -305,7 +330,11 @@ export const createArnsMiddleware = ({ if (req.arns?.ttl !== undefined) { res.header('Cache-Control', `public, max-age=${req.arns.ttl}`); } - dataHandler(req, res, next); + if (serveViaIpfs && ipfsHandler !== undefined) { + ipfsHandler(req, res, next); + } else { + dataHandler(req, res, next); + } } catch (error: any) { span.recordException(error); span.setStatus({ code: SpanStatusCode.ERROR }); diff --git a/src/resolution/on-demand-arns-resolver.ts b/src/resolution/on-demand-arns-resolver.ts index 3b8a40002..d43f8ab83 100644 --- a/src/resolution/on-demand-arns-resolver.ts +++ b/src/resolution/on-demand-arns-resolver.ts @@ -7,6 +7,7 @@ import winston from 'winston'; import { isValidDataId } from '../lib/validation.js'; +import { isValidCid } from '../lib/ipfs-cid.js'; import { NameResolution, NameResolver } from '../types.js'; import { ArNSNameDataWithName, SolanaANTReadable } from '@ar.io/sdk'; import { address, type Rpc, type SolanaRpcApiMainnet } from '@solana/kit'; @@ -117,12 +118,24 @@ export class OnDemandArNSResolver implements NameResolver { const ttl = antRecord.ttlSeconds; const index = antRecord.index; - if (!isValidDataId(resolvedId)) { + // ANT records carry a `targetProtocol` (0 = Arweave, 1 = IPFS). For + // IPFS records the `transactionId` field holds an IPFS CID rather than + // an Arweave TX ID, so validate it against the CID format instead of + // the 43-char Arweave-ID format. (`targetProtocol` may be absent on + // older ANTs, in which case it defaults to Arweave.) + const protocol = antRecord.targetProtocol === 1 ? 'ipfs' : 'arweave'; + + if (protocol === 'ipfs') { + if (!isValidCid(resolvedId)) { + throw new Error('Invalid resolved IPFS CID'); + } + } else if (!isValidDataId(resolvedId)) { throw new Error('Invalid resolved data ID'); } return { name, resolvedId, + protocol, resolvedAt: Date.now(), antId, ttl, diff --git a/src/routes/arns.ts b/src/routes/arns.ts index 5578bae46..811b8c767 100644 --- a/src/routes/arns.ts +++ b/src/routes/arns.ts @@ -7,19 +7,35 @@ import { Router } from 'express'; import * as config from '../config.js'; +import log from '../log.js'; import { createArnsMiddleware } from '../middleware/arns.js'; import { createSandboxMiddleware } from '../middleware/sandbox.js'; import * as system from '../system.js'; import { dataHandler } from './data/index.js'; +import { createIpfsHandler } from './ipfs.js'; import { headerNames } from '../constants.js'; import { sendNotFound } from './data/handlers.js'; import { DEFAULT_ARNS_TTL_SECONDS } from '../resolution/trusted-gateway-arns-resolver.js'; export const arnsRouter = Router(); +// When IPFS serving is enabled, ArNS names whose ANT record resolves to an +// IPFS CID (targetProtocol = ipfs) are served through the same IPFS handler +// used by the path/subdomain routes. +const ipfsHandler = + config.IPFS_ENABLED && system.ipfsService !== undefined + ? createIpfsHandler({ + log, + ipfsService: system.ipfsService, + rateLimiter: system.ipfsRateLimiter, + paymentProcessor: system.paymentProcessor, + }) + : undefined; + export const arnsMiddleware = createArnsMiddleware({ dataHandler, nameResolver: system.nameResolver, + ipfsHandler, }); if (config.ARNS_ROOT_HOSTS.length > 0) { diff --git a/src/types.d.ts b/src/types.d.ts index e248cd6fd..8296769ad 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -1232,6 +1232,16 @@ export interface ValidNameResolution { name: string; statusCode?: number; resolvedId: string; + /** + * Storage protocol of the resolved target, mirroring the ANT record's + * `targetProtocol` (0 = Arweave, 1 = IPFS). `'arweave'` means `resolvedId` + * is a 43-char Arweave TX / data-item ID served from the Arweave data path; + * `'ipfs'` means `resolvedId` is an IPFS CID served via the Kubo IPFS path. + * Optional for backward compatibility — `undefined` is treated as + * `'arweave'` by consumers (e.g. trusted-gateway hops that don't yet carry + * the protocol). + */ + protocol?: 'arweave' | 'ipfs'; resolvedAt: number; ttl: number; /** From 23b24139c752d5ce80e8a8a5e9203a900f91f13b Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Mon, 22 Jun 2026 13:48:44 -0700 Subject: [PATCH 04/20] feat(s3): optional dedicated Turbo AWS client via TURBO_AWS_* vars The turbo-s3 on-demand data source shared the default awsClient (configured from AWS_REGION/AWS_ENDPOINT and the default credentials), so it could not target a separate AWS account or endpoint. Add a turboAwsClient that is instantiated as its own awsLite client only when BOTH TURBO_AWS_REGION and TURBO_AWS_ENDPOINT are set; otherwise it references the existing awsClient, preserving current behavior. Credentials follow the same paradigm: TURBO_AWS_ACCESS_KEY_ID / TURBO_AWS_SECRET_ACCESS_KEY / TURBO_AWS_SESSION_TOKEN are used when provided and otherwise fall back to the default AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN; when none are set, aws-lite resolves credentials from the ambient provider chain (e.g. IAM role), matching the default client. On init failure it falls back to the shared awsClient. Wire turboS3DataSource to use turboAwsClient and document the new vars in docs/envs.md and docker-compose.yaml. Co-Authored-By: Claude Opus 4.8 (1M context) --- docker-compose.yaml | 5 +++++ docs/envs.md | 5 +++++ src/aws-client.ts | 34 ++++++++++++++++++++++++++++++++++ src/system.ts | 8 ++++---- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index c89968212..76028cd94 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -228,6 +228,11 @@ services: - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} - AWS_REGION=${AWS_REGION:-} - AWS_ENDPOINT=${AWS_ENDPOINT:-} + - TURBO_AWS_REGION=${TURBO_AWS_REGION:-} + - TURBO_AWS_ENDPOINT=${TURBO_AWS_ENDPOINT:-} + - TURBO_AWS_ACCESS_KEY_ID=${TURBO_AWS_ACCESS_KEY_ID:-} + - TURBO_AWS_SECRET_ACCESS_KEY=${TURBO_AWS_SECRET_ACCESS_KEY:-} + - TURBO_AWS_SESSION_TOKEN=${TURBO_AWS_SESSION_TOKEN:-} - AWS_S3_CONTIGUOUS_DATA_BUCKET=${AWS_S3_CONTIGUOUS_DATA_BUCKET:-} - AWS_S3_CONTIGUOUS_DATA_PREFIX=${AWS_S3_CONTIGUOUS_DATA_PREFIX:-} - AWS_S3_TURBO_CONTIGUOUS_DATA_BUCKET=${AWS_S3_TURBO_CONTIGUOUS_DATA_BUCKET:-} diff --git a/docs/envs.md b/docs/envs.md index 03e1018fc..47bde723d 100644 --- a/docs/envs.md +++ b/docs/envs.md @@ -197,6 +197,11 @@ This document describes the environment variables that can be used to configure | AWS_SECRET_ACCESS_KEY | String | undefined | AWS secret access key for accessing AWS services | | AWS_REGION | String | undefined | AWS region where the resources are located | | AWS_ENDPOINT | String | undefined | Custom endpoint for AWS services | +| TURBO_AWS_REGION | String | undefined | AWS region for the Turbo S3 data source. A dedicated AWS client is created for Turbo only when both `TURBO_AWS_REGION` and `TURBO_AWS_ENDPOINT` are set; otherwise the default AWS client (`AWS_REGION`/`AWS_ENDPOINT`) is reused. | +| TURBO_AWS_ENDPOINT | String | undefined | Custom endpoint for the Turbo S3 data source. A dedicated AWS client is created for Turbo only when both `TURBO_AWS_REGION` and `TURBO_AWS_ENDPOINT` are set; otherwise the default AWS client (`AWS_REGION`/`AWS_ENDPOINT`) is reused. | +| TURBO_AWS_ACCESS_KEY_ID | String | undefined | Access key ID for the dedicated Turbo AWS client. Falls back to `AWS_ACCESS_KEY_ID` when unset. Only used when the dedicated Turbo client is created (`TURBO_AWS_REGION` + `TURBO_AWS_ENDPOINT`). | +| TURBO_AWS_SECRET_ACCESS_KEY | String | undefined | Secret access key for the dedicated Turbo AWS client. Falls back to `AWS_SECRET_ACCESS_KEY` when unset. Only used when the dedicated Turbo client is created (`TURBO_AWS_REGION` + `TURBO_AWS_ENDPOINT`). | +| TURBO_AWS_SESSION_TOKEN | String | undefined | Session token for the dedicated Turbo AWS client. Falls back to `AWS_SESSION_TOKEN` when unset. Only used when the dedicated Turbo client is created (`TURBO_AWS_REGION` + `TURBO_AWS_ENDPOINT`). | | AWS_S3_CONTIGUOUS_DATA_BUCKET | String | undefined | AWS S3 bucket name used for storing data | | AWS_S3_CONTIGUOUS_DATA_PREFIX | String | undefined | Prefix for the S3 bucket to organize data | | CHUNK_POST_MIN_SUCCESS_COUNT | String | "3" | Minimum count of 200 responses for of a given chunk to be considered properly seeded | diff --git a/src/aws-client.ts b/src/aws-client.ts index e46a50dfb..3371327e4 100644 --- a/src/aws-client.ts +++ b/src/aws-client.ts @@ -33,6 +33,40 @@ export const awsClient = (await canInitAwsClient()) }) : undefined; +// A separate Turbo S3 client is only created when BOTH a Turbo-specific region +// and endpoint are provided, allowing the Turbo S3 data source to point at a +// different AWS account/endpoint than the default client. +function hasTurboAwsConfig() { + return ( + process.env.TURBO_AWS_REGION !== undefined && + process.env.TURBO_AWS_ENDPOINT !== undefined + ); +} + +// Turbo S3 client: a dedicated instance when both TURBO_AWS_REGION and +// TURBO_AWS_ENDPOINT are set, otherwise the shared awsClient reference. +// Credentials follow the same paradigm: Turbo-specific values are used when +// provided and otherwise fall back to the default AWS_* credentials. When none +// are set, aws-lite resolves credentials from the ambient provider chain (e.g. +// IAM role), matching the default client. +export const turboAwsClient = hasTurboAwsConfig() + ? await awsLite({ + endpoint: process.env.TURBO_AWS_ENDPOINT, + region: process.env.TURBO_AWS_REGION, // guaranteed non-undefined now + accessKeyId: + process.env.TURBO_AWS_ACCESS_KEY_ID ?? process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: + process.env.TURBO_AWS_SECRET_ACCESS_KEY ?? + process.env.AWS_SECRET_ACCESS_KEY, + sessionToken: + process.env.TURBO_AWS_SESSION_TOKEN ?? process.env.AWS_SESSION_TOKEN, + plugins: [awsLiteS3], + }).catch((err) => { + console.error('Failed to initialize Turbo AWS client', err); + return awsClient; // fall back to the shared client + }) + : awsClient; + // Check if legacy S3 credentials are explicitly configured function hasLegacyS3Credentials() { return ( diff --git a/src/system.ts b/src/system.ts index b8af9b695..863881f1f 100644 --- a/src/system.ts +++ b/src/system.ts @@ -122,7 +122,7 @@ import { SignatureFetcher, OwnerFetcher } from './data/attribute-fetchers.js'; import { SQLiteWalCleanupWorker } from './workers/sqlite-wal-cleanup-worker.js'; import { ChunkIngestGcWorker } from './workers/chunk-ingest-gc.js'; import { KvArNSResolutionStore } from './store/kv-arns-name-resolution-store.js'; -import { awsClient, legacyAwsS3Client } from './aws-client.js'; +import { awsClient, legacyAwsS3Client, turboAwsClient } from './aws-client.js'; import { BlockedNamesCache } from './blocked-names-cache.js'; import { KvArNSRegistryStore } from './store/kv-arns-base-name-store.js'; import { ChunkRetrievalService } from './data/chunk-retrieval-service.js'; @@ -1151,14 +1151,14 @@ const s3DataSource = : undefined; const turboS3DataSource = - awsClient !== undefined && + turboAwsClient !== undefined && config.AWS_S3_TURBO_CONTIGUOUS_DATA_BUCKET !== undefined ? new S3DataSource({ log, - s3Client: awsClient.S3, + s3Client: turboAwsClient.S3, s3Bucket: config.AWS_S3_TURBO_CONTIGUOUS_DATA_BUCKET, s3Prefix: config.AWS_S3_TURBO_CONTIGUOUS_DATA_PREFIX, - awsClient, + awsClient: turboAwsClient, }) : undefined; From 27316b14b4f5a693476d78ea3f048de4a2c9eb87 Mon Sep 17 00:00:00 2001 From: vilenarios Date: Mon, 22 Jun 2026 21:06:59 +0000 Subject: [PATCH 05/20] refactor(arns,ipfs): review touches for ArNS->IPFS Final-review hardening of the ArNS->IPFS feature: - fix(cache-control): ArNS->IPFS responses no longer send immutable/1-year. The IPFS handler only sets `immutable` for direct /ipfs/{CID} (and {CID}.host) requests; when reached via an ArNS name (mutable name->CID binding) it keeps the ArNS-TTL Cache-Control the ArNS middleware set, so a record repoint isn't pinned in caches for ~a year (cf. PE-9072). - feat(headers): emit signed `X-ArNS-Protocol: arweave|ipfs` on resolutions and add `protocol` (+ `resolvedId`) to the /ar-io/resolver/:name JSON, so clients know whether X-ArNS-Resolved-Id is a TX ID or a CID. Added x-arns-protocol to TRIGGER_HEADERS so it's part of the signature. - feat(httpsig): body-bind IPFS responses with RFC 9530 Content-Digest. The SHA-256 is computed at cache-write time and emitted on cache hits (in CO_SIGNABLE_HEADERS, so HTTPSIG signs it). Misses stream without it; the signed ETag=CID still attests identity. - test(ipfs): cache digest round-trip + legacy (digest-less) entry coverage. - docs: rewrite the ipfs-integration.md Phase 2 section to the shipped targetProtocol design (was speculative), glossary Target Protocol entry, CLAUDE.md note. Documented the trusted-gateway-resolver protocol limitation. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 5 +- docs/glossary.md | 7 +++ docs/ipfs-integration.md | 73 ++++++++++++++---------- src/constants.ts | 8 +++ src/ipfs/ipfs-cache.test.ts | 108 ++++++++++++++++++++++++++++++++++++ src/ipfs/ipfs-cache.ts | 14 ++++- src/ipfs/ipfs-service.ts | 21 ++++++- src/lib/httpsig.ts | 1 + src/middleware/arns.ts | 3 + src/routes/arns.ts | 10 ++++ src/routes/ipfs.ts | 20 ++++++- 11 files changed, 236 insertions(+), 34 deletions(-) create mode 100644 src/ipfs/ipfs-cache.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 4b6c01777..f7185f0ef 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -74,7 +74,10 @@ yarn service:start / stop / restart / status / logs bundle unbundling, verification, and webhooks. Controlled by `START_WRITERS`. - IPFS serving (`src/ipfs/`) is opt-in via `IPFS_ENABLED`. Uses a Kubo sidecar for content retrieval with its own cache, rate limiter, and blocklist. Routes - mount before ArNS in `app.ts`. See `docs/ipfs-integration.md`. + mount before ArNS in `app.ts`. ArNS names whose ANT record has + `targetProtocol: ipfs` resolve to a CID and are routed to the same IPFS + handler by the ArNS middleware (`src/middleware/arns.ts`); the on-demand + resolver reads `targetProtocol`. See `docs/ipfs-integration.md`. - Responses include trust headers indicating verification status. - HTTPSIG signs response headers (RFC 9421); `Content-Digest` is in `CO_SIGNABLE_HEADERS` so when present it binds the body to the signature. diff --git a/docs/glossary.md b/docs/glossary.md index 2aa786547..7c05b4e52 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -275,6 +275,13 @@ containing bundled data. human-readable names (like "my-app") to their corresponding Arweave [item IDs](#item-id). +**Target Protocol (ANT record)** - A field on an ANT record (`0` = Arweave, +`1` = IPFS, default `0`) declaring where the record's target lives. For an +Arweave target the resolved id is a 43-char transaction/data-item ID served from +the Arweave data path; for an IPFS target it is an [IPFS CID](#ipfs-cid) served +via the Kubo IPFS path. The gateway surfaces it as `X-ArNS-Protocol` and uses it +to route an ArNS name to either backend. See `docs/ipfs-integration.md`. + **Path Resolution** - The process of interpreting URL paths to determine which transaction data to serve. Includes manifest resolution (looking up paths in a manifest's routing table), index path resolution (adding index.html), and diff --git a/docs/ipfs-integration.md b/docs/ipfs-integration.md index 8a60ef8f7..f44bc19c1 100644 --- a/docs/ipfs-integration.md +++ b/docs/ipfs-integration.md @@ -44,12 +44,13 @@ placing a CID in the URL path or subdomain. The gateway validates, rate-limits, and caches the request, then proxies it to the local Kubo node. No ArNS resolution is involved. -**Phase 2 -- ArNS to CID Resolution (future):** ANT (Arweave Name Token) -records will be able to store an IPFS CID in their `transactionId` field. When -the gateway resolves an ArNS name and detects that the resolved ID is a CID -rather than an Arweave transaction ID, it routes the request to the IPFS service -instead of the Arweave data pipeline. This requires no contract changes -- CID -detection happens at the gateway level. +**Phase 2 -- ArNS to CID Resolution (implemented):** ANT (Arweave Name Token) +records carry a `targetProtocol` field (`0` = Arweave, `1` = IPFS) and the +content target may be an IPFS CID. When the gateway resolves an ArNS name whose +record targets IPFS, it routes the request to the IPFS service instead of the +Arweave data pipeline. See +[Phase 2: ArNS to IPFS Resolution](#phase-2-arns-to-ipfs-resolution) for the +full flow, headers, and caching semantics. ## Architecture @@ -451,36 +452,52 @@ serve IPFS-hosted content without the user needing to know the CID. ### How It Works -1. **ANT record stores a CID.** The Arweave Name Token contract's - `transactionId` field accepts any string. An ANT owner sets it to an IPFS CID - (e.g., `bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi`) - instead of an Arweave transaction ID. +1. **ANT record targets a CID with `targetProtocol: ipfs`.** ANT records carry + a `targetProtocol` field (`0` = Arweave, `1` = IPFS, default `0`) alongside + the content target. An ANT owner (or controller) sets the target to an IPFS + CID and `targetProtocol` to `1` -- e.g. via the AR.IO SDK: + `ant.setUndernameRecord({ undername: 'ipfs', transactionId: '', ttlSeconds: 300, targetProtocol: 1 })`. -2. **Gateway resolves the ArNS name.** The standard ArNS resolution pipeline - fetches the ANT record and extracts the `transactionId`. +2. **The on-demand resolver reads `targetProtocol`.** `OnDemandArNSResolver` + reads the record's `targetProtocol`. When it is IPFS, it validates the target + as a CID (`isValidCid`) rather than as a 43-char Arweave ID, and surfaces + `protocol: 'ipfs'` on the resolution (carried through the resolution cache). -3. **CID detection.** The gateway inspects the resolved ID. If it matches CID - format (multibase-prefixed, valid multicodec), it is classified as an IPFS - CID rather than an Arweave transaction ID. +3. **The ArNS middleware routes by protocol.** When `protocol === 'ipfs'` and + IPFS serving is enabled, the middleware sets `ipfsCid`/`ipfsPath` on the + request and hands off to the same IPFS handler used by the path/subdomain + routes -- otherwise it serves via the Arweave data path as before. -4. **Route to IPFS service.** Instead of fetching from the Arweave data pipeline - (cache, S3, peers, chunks), the gateway routes the request to the IPFS - service, which follows the same blocklist, rate limit, cache, and Kubo fetch - pipeline described above. +4. **The IPFS service serves it.** Blocklist -> rate limit -> cache -> Kubo + fetch, exactly as for a direct `/ipfs/{CID}` request. + +The response carries the full ArNS envelope (`X-ArNS-Name`, `X-ArNS-Resolved-Id` += the CID, `X-ArNS-Ant-Id`, `X-ArNS-TTL-Seconds`) plus `X-ArNS-Protocol: ipfs`, +`X-Ar-Io-Source: ipfs`, `X-Ipfs-Path`, and `ETag` = the CID. HTTPSIG signs the +ArNS binding headers and the IPFS serving headers (and `Content-Digest` on cache +hits), so the name->CID binding and the served bytes are both attested. ### Key Design Decisions -- **No contract changes.** CID detection happens entirely at the gateway level. - The ANT contract's `transactionId` field is a free-form string, so it already - accepts CIDs. +- **Explicit `targetProtocol`, not shape-sniffing.** Protocol comes from the + ANT record's `targetProtocol` field, so an Arweave TX ID and an IPFS CID are + never confused by guessing from string shape. +- **Mutable-binding cache semantics.** A direct `/ipfs/{CID}` request is cached + `immutable` (content-addressed). But an ArNS name -> CID binding is **mutable** + (the record can be repointed), so ArNS-served IPFS responses use the ArNS TTL + for `Cache-Control`, not `immutable` -- a record update is never pinned in + caches for ~a year (cf. PE-9072). - **Transparent to users.** A user visiting `my-dapp.arweave.dev` does not need - to know whether the content is on Arweave or IPFS. The URL is the same either - way. -- **Owner-controlled.** The ANT owner decides where content lives by setting - the `transactionId` to either an Arweave TX ID or an IPFS CID. Switching - between storage backends is a single contract interaction. + to know whether the content is on Arweave or IPFS; the URL is identical. +- **Owner/controller-controlled.** Switching a name between Arweave and IPFS is + a single ANT record update (target + `targetProtocol`). - **Caching and moderation apply.** All Phase 1 protections (blocklist, rate limits, cache) apply to ArNS-resolved IPFS content. +- **Resolver scope.** Protocol awareness lives in the on-demand resolver. The + trusted-gateway resolver does not yet propagate `targetProtocol` across + gateway hops, so a name whose resolution falls through to that path would be + treated as Arweave. Keep `on-demand` ahead of `gateway` in + `ARNS_RESOLVER_PRIORITY_ORDER` for IPFS-targeted names. ## Differences from Arweave Data Serving @@ -491,7 +508,7 @@ serve IPFS-hosted content without the user needing to know the CID. | **Path resolution** | Manifest JSON parsed by the gateway | UnixFS directories resolved by Kubo | | **Verification** | Merkle proofs verified by the gateway | Block hashes verified internally by Kubo | | **Caching** | Archival (operators retain data long-term, often without eviction) | LRU with bounded size (eviction when full) | -| **Cache-Control** | Varies by verification status and data source trust | `immutable` with 1-year max-age (content-addressed = never changes) | +| **Cache-Control** | Varies by verification status and data source trust | Direct CID: `immutable`, 1-year max-age (content-addressed). Via ArNS: the ArNS TTL (mutable name->CID binding) | | **Rate limiting** | Shared Arweave token bucket | Separate IPFS token bucket | | **Data source** | Multi-source fallback chain (cache, S3, peers, gateways, Arweave nodes) | Single source: local Kubo node | | **Upstream network** | Arweave protocol (block weave, mining incentives) | IPFS/libp2p (DHT, Bitswap) | diff --git a/src/constants.ts b/src/constants.ts index 4280003d8..707625a51 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -61,6 +61,14 @@ export const headerNames = { arnsBasename: 'X-ArNS-Basename', arnsRecord: 'X-ArNS-Record', arnsResolvedId: 'X-ArNS-Resolved-Id', + /** + * Storage protocol of the resolved target: `arweave` (the resolved id is an + * Arweave TX / data-item ID served from the Arweave data path) or `ipfs` + * (the resolved id is an IPFS CID served via the Kubo IPFS path). Mirrors the + * ANT record's `targetProtocol`. Lets a client know how to interpret + * `X-ArNS-Resolved-Id` (43-char TX ID vs CID) without guessing from its shape. + */ + arnsProtocol: 'X-ArNS-Protocol', dataId: 'X-AR-IO-Data-Id', /** * Identifier of the Solana program that owns the ANT mint that diff --git a/src/ipfs/ipfs-cache.test.ts b/src/ipfs/ipfs-cache.test.ts new file mode 100644 index 000000000..558a59233 --- /dev/null +++ b/src/ipfs/ipfs-cache.test.ts @@ -0,0 +1,108 @@ +/** + * AR.IO Gateway + * Copyright (C) 2022-2025 Permanent Data Solutions, Inc. All Rights Reserved. + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import { strict as assert } from 'node:assert'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import * as crypto from 'node:crypto'; + +import { createTestLogger } from '../../test/test-logger.js'; +import { IpfsFsCache } from './ipfs-cache.js'; + +const log = createTestLogger({ suite: 'IpfsFsCache' }); + +async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + chunks.push(Buffer.from(chunk)); + } + return Buffer.concat(chunks); +} + +describe('IpfsFsCache', () => { + let baseDir: string; + let cache: IpfsFsCache; + + beforeEach(() => { + baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ipfs-cache-test-')); + cache = new IpfsFsCache({ + log, + basePath: baseDir, + maxSizeBytes: 10 * 1024 * 1024, + }); + }); + + afterEach(() => { + fs.rmSync(baseDir, { recursive: true, force: true }); + }); + + it('round-trips bytes and the content digest through putFromFile/get', async () => { + const cid = 'bafkreiefysqevlhofnppvnhaptsjt7cqi6wcjsllgpn7ml5i4256v2rbwu'; + const content = Buffer.from('hello digest test'); + const digest = crypto + .createHash('sha256') + .update(content) + .digest('base64url'); + + // The streaming writer hands putFromFile an already-written temp file under + // the cache's tmp dir; emulate that here. + const tempPath = path.join( + baseDir, + 'tmp', + crypto.randomBytes(8).toString('hex'), + ); + fs.writeFileSync(tempPath, content); + + await cache.putFromFile( + cid, + tempPath, + content.length, + 'text/plain', + undefined, + digest, + ); + + const got = await cache.get(cid); + assert.ok(got, 'expected a cache hit'); + assert.equal(got.digest, digest, 'digest should round-trip'); + assert.equal(got.size, content.length); + assert.equal(got.contentType, 'text/plain'); + assert.deepEqual(await streamToBuffer(got.stream), content); + }); + + it('returns undefined for an uncached CID', async () => { + const got = await cache.get( + 'bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4s52zy', + ); + assert.equal(got, undefined); + }); + + it('rebuilds a digest-less entry (older cache) without a digest', async () => { + const cid = 'bafkreiefysqevlhofnppvnhaptsjt7cqi6wcjsllgpn7ml5i4256v2rbwu'; + const content = Buffer.from('legacy entry, no digest'); + const tempPath = path.join( + baseDir, + 'tmp', + crypto.randomBytes(8).toString('hex'), + ); + fs.writeFileSync(tempPath, content); + + // No digest argument — emulates an entry written before Content-Digest support. + await cache.putFromFile( + cid, + tempPath, + content.length, + 'application/octet-stream', + ); + + const got = await cache.get(cid); + assert.ok(got); + assert.equal(got.digest, undefined, 'legacy entry should have no digest'); + assert.deepEqual(await streamToBuffer(got.stream), content); + }); +}); diff --git a/src/ipfs/ipfs-cache.ts b/src/ipfs/ipfs-cache.ts index 7aa352e85..99fa21cea 100644 --- a/src/ipfs/ipfs-cache.ts +++ b/src/ipfs/ipfs-cache.ts @@ -18,6 +18,13 @@ import { currentUnixTimestamp } from '../lib/time.js'; interface CacheEntry { size: number; contentType: string; + /** + * base64url SHA-256 of the cached bytes, computed at cache-write time. + * Emitted as an RFC 9530 Content-Digest on cache hits so the body can be + * bound into the HTTPSIG signature. Optional: pre-existing cache entries + * (written before this field existed) won't have it. + */ + digest?: string; } export class IpfsFsCache { @@ -111,7 +118,8 @@ export class IpfsFsCache { cidString: string, path?: string, ): Promise< - { stream: Readable; size: number; contentType: string } | undefined + | { stream: Readable; size: number; contentType: string; digest?: string } + | undefined > { const key = this.cacheKey(cidString, path); let entry = this.index.get(key); @@ -139,6 +147,7 @@ export class IpfsFsCache { stream, size: entry.size, contentType: entry.contentType, + digest: entry.digest, }; } catch (error: any) { this.log.error('Failed to read cached IPFS content', { @@ -203,6 +212,7 @@ export class IpfsFsCache { size: number, contentType: string, path?: string, + digest?: string, ): Promise { const key = this.cacheKey(cidString, path); @@ -211,7 +221,7 @@ export class IpfsFsCache { await fs.promises.mkdir(dataDir, { recursive: true }); await fs.promises.rename(tempPath, this.dataPath(key)); - const meta: CacheEntry = { size, contentType }; + const meta: CacheEntry = { size, contentType, digest }; await fs.promises.writeFile( this.metaPath(key), JSON.stringify(meta), diff --git a/src/ipfs/ipfs-service.ts b/src/ipfs/ipfs-service.ts index 85d0c0c85..905a98688 100644 --- a/src/ipfs/ipfs-service.ts +++ b/src/ipfs/ipfs-service.ts @@ -27,6 +27,12 @@ export interface IpfsGetContentResult { size: number; contentType: string; cached: boolean; + /** + * base64url SHA-256 of the served bytes, present on cache hits (computed at + * cache-write time). Used to emit an RFC 9530 Content-Digest. Absent on cache + * misses (the body streams straight from Kubo without being hashed inline). + */ + digest?: string; } export class IpfsService { @@ -110,6 +116,7 @@ export class IpfsService { size: cached.size, contentType: cached.contentType, cached: true, + digest: cached.digest, }; } @@ -189,6 +196,9 @@ export class IpfsService { .toString('hex')}`; let bytesWritten = 0; let failed = false; + // Hash the bytes as they're written so cache hits can emit an RFC 9530 + // Content-Digest (body binding) without a re-read. + const hash = crypto.createHash('sha256'); // Create the write stream synchronously. The temp directory is created // eagerly in the IpfsFsCache constructor, so there is no async mkdir to @@ -242,6 +252,7 @@ export class IpfsService { return; } + hash.update(chunk); writeStream.write(chunk); }); @@ -250,10 +261,18 @@ export class IpfsService { cleanup(); return; } + const digest = hash.digest('base64url'); writeStream.end(() => { // Finalize: move temp file into cache this.cache - .putFromFile(cidString, tempPath, bytesWritten, contentType, path) + .putFromFile( + cidString, + tempPath, + bytesWritten, + contentType, + path, + digest, + ) .catch((error) => { this.log.error('Failed to finalize IPFS cache entry', { cid: cidString, diff --git a/src/lib/httpsig.ts b/src/lib/httpsig.ts index 816441153..e7b5723ab 100644 --- a/src/lib/httpsig.ts +++ b/src/lib/httpsig.ts @@ -37,6 +37,7 @@ export const TRIGGER_HEADERS = new Set([ 'x-arweave-tags-truncated', 'x-arns-name', 'x-arns-resolved-id', + 'x-arns-protocol', 'x-arns-ttl-seconds', 'x-arns-ant-program-id', 'x-arns-ant-id', diff --git a/src/middleware/arns.ts b/src/middleware/arns.ts index f8d2282c2..8fdadf960 100644 --- a/src/middleware/arns.ts +++ b/src/middleware/arns.ts @@ -231,6 +231,9 @@ export const createArnsMiddleware = ({ // Populate the ArNS response headers for client visibility res.header(headerNames.arnsName, arnsSubdomain); res.header(headerNames.arnsResolvedId, resolvedId); + // Tell the client how to interpret the resolved id (Arweave TX vs + // IPFS CID). Signed (x-arns-protocol is a trigger header). + res.header(headerNames.arnsProtocol, arnsProtocol); if (basename !== '') { res.header(headerNames.arnsBasename, basename); } diff --git a/src/routes/arns.ts b/src/routes/arns.ts index 811b8c767..80f4ec947 100644 --- a/src/routes/arns.ts +++ b/src/routes/arns.ts @@ -67,7 +67,13 @@ arnsRouter.get('/ar-io/resolver/:name', async (req, res) => { return; } + // Storage protocol of the target (arweave TX id vs ipfs CID). Optional on the + // resolution; absent => arweave for backward compatibility. + const protocol = + (resolved as { protocol?: 'arweave' | 'ipfs' }).protocol ?? 'arweave'; + res.header(headerNames.arnsResolvedId, resolvedId); + res.header(headerNames.arnsProtocol, protocol); res.header( headerNames.arnsTtlSeconds, (ttl ?? DEFAULT_ARNS_TTL_SECONDS).toString(), @@ -86,7 +92,11 @@ arnsRouter.get('/ar-io/resolver/:name', async (req, res) => { res.header(headerNames.arnsLimit, limit.toString()); } res.json({ + // `txId` kept for backward compatibility; for IPFS records it actually + // holds a CID. Prefer `resolvedId` + `protocol`. txId: resolvedId, + resolvedId, + protocol, ttlSeconds: ttl, antId, resolvedAt, diff --git a/src/routes/ipfs.ts b/src/routes/ipfs.ts index 056ef8ae9..031504873 100644 --- a/src/routes/ipfs.ts +++ b/src/routes/ipfs.ts @@ -31,6 +31,7 @@ import { } from '../handlers/data-handler-utils.js'; import { PaymentProcessor } from '../payments/types.js'; import { extractAllClientIPs } from '../lib/ip-utils.js'; +import { formatContentDigest } from '../lib/digest.js'; export function createIpfsRouter({ log, @@ -203,12 +204,27 @@ async function handleIpfsRequest({ if (result.size > 0) { res.setHeader('Content-Length', result.size); } - // CIDs are content-addressed — content never changes - res.setHeader('Cache-Control', 'public, max-age=29030400, immutable'); + // A direct /ipfs/{CID} or {CID}.host request is content-addressed and thus + // immutable. But when the request arrived via an ArNS name, the name->CID + // binding is MUTABLE and the ArNS middleware already set a TTL-bounded + // Cache-Control — don't override it with `immutable`, or a record update + // would be pinned in browsers/edge caches for ~a year (cf. PE-9072). + if ((req as Request & { arns?: unknown }).arns === undefined) { + res.setHeader('Cache-Control', 'public, max-age=29030400, immutable'); + } res.setHeader('ETag', `"${cidToV1Base32(cidString)}"`); res.setHeader('X-Ipfs-Path', `/ipfs/${ipfsPath}`); res.setHeader('X-Ar-Io-Source', 'ipfs'); + // Body binding (RFC 9530 Content-Digest). When a SHA-256 of the served + // bytes is known (computed at cache-write time, returned on cache hits), + // emit it — it's in CO_SIGNABLE_HEADERS, so HTTPSIG binds the body to the + // signature. Cache hits carry it for free; misses stream without it (the + // signed ETag=CID still attests content identity). + if (result.digest !== undefined) { + res.setHeader('Content-Digest', formatContentDigest(result.digest)); + } + if (result.cached) { res.setHeader('X-Cache', 'HIT'); } else { From 4a1fbc069e8c40c8730895fac04d839214a35375 Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Mon, 22 Jun 2026 14:49:28 -0700 Subject: [PATCH 06/20] fix(s3): treat empty-string TURBO_AWS_* env vars as unset; add TSDoc Compose defaults like `${TURBO_AWS_REGION:-}` pass an empty string rather than undefined when unset on the host, so the previous `!== undefined` gate and `??` credential fallback would wrongly enable a misconfigured dedicated Turbo client and suppress fallback to the AWS_* credentials. Route all TURBO_AWS_* / AWS_* reads through the existing env.varOrUndefined helper, which treats empty/ whitespace strings as unset. Also add TSDoc to hasTurboAwsConfig and turboAwsClient. Addresses CodeRabbit review on PR #794. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/aws-client.ts | 55 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/src/aws-client.ts b/src/aws-client.ts index 3371327e4..3663e1a02 100644 --- a/src/aws-client.ts +++ b/src/aws-client.ts @@ -9,6 +9,8 @@ import awsLite from '@aws-lite/client'; import awsLiteS3 from '@aws-lite/s3'; import { fromNodeProviderChain } from '@aws-sdk/credential-providers'; +import * as env from './lib/env.js'; + async function canInitAwsClient() { // 1. Region must be set if (!process.env.AWS_REGION!) return false; @@ -33,33 +35,52 @@ export const awsClient = (await canInitAwsClient()) }) : undefined; -// A separate Turbo S3 client is only created when BOTH a Turbo-specific region -// and endpoint are provided, allowing the Turbo S3 data source to point at a -// different AWS account/endpoint than the default client. +/** + * Whether a dedicated Turbo AWS client should be created. + * + * Returns `true` only when both `TURBO_AWS_REGION` and `TURBO_AWS_ENDPOINT` are + * set to non-empty values. Empty strings — e.g. the unset compose default + * `${TURBO_AWS_REGION:-}` — are treated as unset (via {@link env.varOrUndefined}), + * so the shared {@link awsClient} is used instead of a misconfigured dedicated + * client. + */ function hasTurboAwsConfig() { return ( - process.env.TURBO_AWS_REGION !== undefined && - process.env.TURBO_AWS_ENDPOINT !== undefined + env.varOrUndefined('TURBO_AWS_REGION') !== undefined && + env.varOrUndefined('TURBO_AWS_ENDPOINT') !== undefined ); } -// Turbo S3 client: a dedicated instance when both TURBO_AWS_REGION and -// TURBO_AWS_ENDPOINT are set, otherwise the shared awsClient reference. -// Credentials follow the same paradigm: Turbo-specific values are used when -// provided and otherwise fall back to the default AWS_* credentials. When none -// are set, aws-lite resolves credentials from the ambient provider chain (e.g. -// IAM role), matching the default client. +/** + * AWS client used by the Turbo S3 data source. + * + * A dedicated aws-lite client is instantiated only when both `TURBO_AWS_REGION` + * and `TURBO_AWS_ENDPOINT` are set, letting the Turbo S3 source target a + * different AWS account or S3-compatible endpoint than the default client. + * Otherwise this is the shared {@link awsClient} reference (which is itself + * `undefined` when AWS is not configured at all), preserving existing behavior. + * + * Credentials resolve per field: the Turbo-specific value + * (`TURBO_AWS_ACCESS_KEY_ID` / `TURBO_AWS_SECRET_ACCESS_KEY` / + * `TURBO_AWS_SESSION_TOKEN`) when set, otherwise the default `AWS_*` value, + * otherwise `undefined` so aws-lite resolves credentials from the ambient + * provider chain (e.g. IAM role) — matching the default client. Empty-string + * env values are treated as unset throughout. On initialization failure the + * shared {@link awsClient} is used as a graceful fallback. + */ export const turboAwsClient = hasTurboAwsConfig() ? await awsLite({ - endpoint: process.env.TURBO_AWS_ENDPOINT, - region: process.env.TURBO_AWS_REGION, // guaranteed non-undefined now + endpoint: env.varOrUndefined('TURBO_AWS_ENDPOINT'), + region: env.varOrUndefined('TURBO_AWS_REGION'), accessKeyId: - process.env.TURBO_AWS_ACCESS_KEY_ID ?? process.env.AWS_ACCESS_KEY_ID, + env.varOrUndefined('TURBO_AWS_ACCESS_KEY_ID') ?? + env.varOrUndefined('AWS_ACCESS_KEY_ID'), secretAccessKey: - process.env.TURBO_AWS_SECRET_ACCESS_KEY ?? - process.env.AWS_SECRET_ACCESS_KEY, + env.varOrUndefined('TURBO_AWS_SECRET_ACCESS_KEY') ?? + env.varOrUndefined('AWS_SECRET_ACCESS_KEY'), sessionToken: - process.env.TURBO_AWS_SESSION_TOKEN ?? process.env.AWS_SESSION_TOKEN, + env.varOrUndefined('TURBO_AWS_SESSION_TOKEN') ?? + env.varOrUndefined('AWS_SESSION_TOKEN'), plugins: [awsLiteS3], }).catch((err) => { console.error('Failed to initialize Turbo AWS client', err); From fd4033706e145f0702b3e301cfefb017b65abd81 Mon Sep 17 00:00:00 2001 From: vilenarios Date: Tue, 23 Jun 2026 01:11:22 +0000 Subject: [PATCH 07/20] test(arns): extract + unit-test resolved-target protocol classification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ArNS->IPFS routing decision hinges on classifying an ANT record's target as arweave vs ipfs and validating the id accordingly. Extracted that logic from OnDemandArNSResolver into a pure, SDK-free helper (classifyResolvedTarget) and unit-tested it: arweave/ipfs by targetProtocol, undefined+unknown protocol -> arweave (fail-closed), CIDv0/v1 acceptance, and cross-format rejection (CID under arweave, TX id under ipfs, garbage). The ArNS middleware routing itself can't be unit-tested in isolation (it imports system.ts, booting the DI graph — no middleware has unit tests for this reason); it stays covered by live e2e. Also documented the three root/apex cases in ipfs-integration.md: a name's @ record and apex-via-APEX_ARNS_NAME route to IPFS; apex-via-APEX_TX_ID is Arweave-only (bypasses protocol routing). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/ipfs-integration.md | 15 ++++++ src/resolution/on-demand-arns-resolver.ts | 24 +++------ src/resolution/resolved-target.test.ts | 61 +++++++++++++++++++++++ src/resolution/resolved-target.ts | 44 ++++++++++++++++ 4 files changed, 128 insertions(+), 16 deletions(-) create mode 100644 src/resolution/resolved-target.test.ts create mode 100644 src/resolution/resolved-target.ts diff --git a/docs/ipfs-integration.md b/docs/ipfs-integration.md index f44bc19c1..13b0b47cb 100644 --- a/docs/ipfs-integration.md +++ b/docs/ipfs-integration.md @@ -499,6 +499,21 @@ hits), so the name->CID binding and the served bytes are both attested. treated as Arweave. Keep `on-demand` ahead of `gateway` in `ARNS_RESOLVER_PRIORITY_ORDER` for IPFS-targeted names. +### Root and apex names + +"Root" means three different things; only two route to IPFS: + +- **A name's root (`@`) record** -- e.g. `my-name.gateway.tld` with no + undername. The `@` record is just the `'@'` undername and goes through the + same resolution + routing, so an IPFS `@` record serves IPFS. ✅ +- **Gateway apex via `APEX_ARNS_NAME`** -- when the bare apex host has + `APEX_ARNS_NAME` set, the middleware resolves that name through the normal + path, so if its record targets IPFS the apex serves IPFS. ✅ +- **Gateway apex via `APEX_TX_ID`** -- a fixed id served directly to the Arweave + data handler, bypassing resolution and protocol routing. It is Arweave-only; + a CID there will not serve. To serve IPFS at the apex, use `APEX_ARNS_NAME` + pointing at an ANT whose `@` record targets IPFS. ❌ + ## Differences from Arweave Data Serving | Aspect | Arweave | IPFS | diff --git a/src/resolution/on-demand-arns-resolver.ts b/src/resolution/on-demand-arns-resolver.ts index d43f8ab83..d86e7cfd6 100644 --- a/src/resolution/on-demand-arns-resolver.ts +++ b/src/resolution/on-demand-arns-resolver.ts @@ -6,8 +6,7 @@ */ import winston from 'winston'; -import { isValidDataId } from '../lib/validation.js'; -import { isValidCid } from '../lib/ipfs-cid.js'; +import { classifyResolvedTarget } from './resolved-target.js'; import { NameResolution, NameResolver } from '../types.js'; import { ArNSNameDataWithName, SolanaANTReadable } from '@ar.io/sdk'; import { address, type Rpc, type SolanaRpcApiMainnet } from '@solana/kit'; @@ -118,20 +117,13 @@ export class OnDemandArNSResolver implements NameResolver { const ttl = antRecord.ttlSeconds; const index = antRecord.index; - // ANT records carry a `targetProtocol` (0 = Arweave, 1 = IPFS). For - // IPFS records the `transactionId` field holds an IPFS CID rather than - // an Arweave TX ID, so validate it against the CID format instead of - // the 43-char Arweave-ID format. (`targetProtocol` may be absent on - // older ANTs, in which case it defaults to Arweave.) - const protocol = antRecord.targetProtocol === 1 ? 'ipfs' : 'arweave'; - - if (protocol === 'ipfs') { - if (!isValidCid(resolvedId)) { - throw new Error('Invalid resolved IPFS CID'); - } - } else if (!isValidDataId(resolvedId)) { - throw new Error('Invalid resolved data ID'); - } + // Classify + validate the target against the ANT record's + // `targetProtocol` (0 = Arweave, 1 = IPFS). Throws if the id doesn't + // match the format for its protocol (treated as "name didn't resolve"). + const protocol = classifyResolvedTarget( + resolvedId, + antRecord.targetProtocol, + ); return { name, resolvedId, diff --git a/src/resolution/resolved-target.test.ts b/src/resolution/resolved-target.test.ts new file mode 100644 index 000000000..06bc3264c --- /dev/null +++ b/src/resolution/resolved-target.test.ts @@ -0,0 +1,61 @@ +/** + * AR.IO Gateway + * Copyright (C) 2022-2025 Permanent Data Solutions, Inc. All Rights Reserved. + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +import { describe, it } from 'node:test'; +import { strict as assert } from 'node:assert'; + +import { classifyResolvedTarget } from './resolved-target.js'; + +const ARWEAVE_ID = 'M2tMZzF3XAcXvyg9DR6U07Cj5HY-JLgT6tCPujdkKZ0'; // 43-char base64url +const CID_V1 = 'bafybeifx7yeb55armcsxwwitkymga5xf53dxiarykms3ygqic223w5sk3m'; +const CID_V0 = 'QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG'; + +describe('classifyResolvedTarget', () => { + describe('Arweave targets', () => { + it('classifies a 43-char Arweave id as arweave when targetProtocol is 0', () => { + assert.equal(classifyResolvedTarget(ARWEAVE_ID, 0), 'arweave'); + }); + + it('defaults to arweave when targetProtocol is undefined (older ANT)', () => { + assert.equal(classifyResolvedTarget(ARWEAVE_ID, undefined), 'arweave'); + }); + + it('treats unknown protocol numbers as arweave (fail-closed)', () => { + assert.equal(classifyResolvedTarget(ARWEAVE_ID, 2), 'arweave'); + }); + + it('rejects a non-Arweave id under an Arweave protocol', () => { + assert.throws( + () => classifyResolvedTarget(CID_V1, 0), + /Invalid resolved data ID/, + ); + }); + }); + + describe('IPFS targets', () => { + it('classifies a CIDv1 as ipfs when targetProtocol is 1', () => { + assert.equal(classifyResolvedTarget(CID_V1, 1), 'ipfs'); + }); + + it('accepts a CIDv0 under the IPFS protocol', () => { + assert.equal(classifyResolvedTarget(CID_V0, 1), 'ipfs'); + }); + + it('rejects an Arweave id under the IPFS protocol', () => { + assert.throws( + () => classifyResolvedTarget(ARWEAVE_ID, 1), + /Invalid resolved IPFS CID/, + ); + }); + + it('rejects garbage under the IPFS protocol', () => { + assert.throws( + () => classifyResolvedTarget('not-a-cid', 1), + /Invalid resolved IPFS CID/, + ); + }); + }); +}); diff --git a/src/resolution/resolved-target.ts b/src/resolution/resolved-target.ts new file mode 100644 index 000000000..a2a411d64 --- /dev/null +++ b/src/resolution/resolved-target.ts @@ -0,0 +1,44 @@ +/** + * AR.IO Gateway + * Copyright (C) 2022-2025 Permanent Data Solutions, Inc. All Rights Reserved. + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +import { isValidDataId } from '../lib/validation.js'; +import { isValidCid } from '../lib/ipfs-cid.js'; + +export type ResolvedProtocol = 'arweave' | 'ipfs'; + +/** + * Classify (and validate) an ANT record's resolved target. + * + * ANT records carry a `targetProtocol` field (`0` = Arweave, `1` = IPFS; + * absent on older ANTs, which default to Arweave). The target id is an Arweave + * TX / data-item ID for Arweave records and an IPFS CID for IPFS records, so the + * id is validated against the format implied by `targetProtocol`: + * + * - `targetProtocol === 1` -> `ipfs`; the id must be a valid CID. + * - anything else -> `arweave`; the id must be a valid 43-char id. + * + * Returns the resolved protocol, or throws if the id does not match the format + * for that protocol (the caller treats a throw as "name did not resolve"). + * + * Kept as a pure function (no SDK / network / config) so it is unit-testable in + * isolation and shared by any resolver that reads ANT records. + */ +export function classifyResolvedTarget( + resolvedId: string, + targetProtocol: number | undefined, +): ResolvedProtocol { + const protocol: ResolvedProtocol = targetProtocol === 1 ? 'ipfs' : 'arweave'; + + if (protocol === 'ipfs') { + if (!isValidCid(resolvedId)) { + throw new Error('Invalid resolved IPFS CID'); + } + } else if (!isValidDataId(resolvedId)) { + throw new Error('Invalid resolved data ID'); + } + + return protocol; +} From efd485b0433cae64beb02db316b69ff06c0ee827 Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Thu, 25 Jun 2026 12:57:54 -0700 Subject: [PATCH 08/20] feat(gql): route owner-filtered ClickHouse queries through owner_projection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner-filtered GraphQL `transactions` queries (the ArDrive UserDriveEntities pattern) scan tens of millions of rows because `transactions` is height- ordered while the filter is on `owner_address`, so a sparse owner's rows scatter across the full height range and finding a page trips `max_rows_to_read` (Code 158) — measured 12.1M rows for an owner with only 22k rows / 46 drives. Route eligible owner queries through the owner-ordered `owner_projection` by emitting `optimize_use_projections = 1, optimize_read_in_order = 0`, which lets the optimizer seek the owner's contiguous slice and sort the small matched set in memory (measured 12.1M -> 451K rows). A reactive height- windowing fallback retries on Code 158 for whale owners whose footprint still exceeds the cap. Gated by CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED (default off; when off, queries plan exactly as before) and scoped to an Entity-Type allowlist (CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES, default drive,folder,snapshot) so large-result `file` queries, bare-owner, and owner+other-tag queries are excluded. Verified against ClickHouse: the routed query reads 451K rows and returns the correct page, and multi-page cursor pagination reproduces the canonical ordered result exactly (no dup/gap, correct hasNextPage). Co-Authored-By: Claude Opus 4.8 (1M context) --- ...ickhouse-gql-owner-filter-too-many-rows.md | 249 +++++++++++++ docs/envs.md | 2 + src/config.ts | 32 ++ src/database/composite-clickhouse.test.ts | 177 ++++++++++ src/database/composite-clickhouse.ts | 326 +++++++++++++++++- src/system.ts | 4 + 6 files changed, 786 insertions(+), 4 deletions(-) create mode 100644 docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md diff --git a/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md b/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md new file mode 100644 index 000000000..8026a42c5 --- /dev/null +++ b/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md @@ -0,0 +1,249 @@ +# ClickHouse GQL `TOO_MANY_ROWS` on owner-filtered queries + +**Date:** 2026-06-25 +**Status:** Investigation complete; fix proposed, not yet implemented +**Measured on:** canary gateway (gw1), ClickHouse 26.3.9, `default.transactions` +(501.8M rows, 135 GiB data + 67 GiB `owner_projection`, 92 active parts) + +## Summary + +Owner-filtered GraphQL `transactions` queries — the dominant ArDrive access +pattern — fail with ClickHouse `Code: 158 TOO_MANY_ROWS` (the gateway's +`CLICKHOUSE_GQL_MAX_ROWS_TO_READ = 10,000,000` per-query guardrail) even for +owners with a tiny footprint. The cause is structural: the `transactions` table +is **height-ordered**, but these queries filter on `owner_address`, whose rows +are scattered thinly across the whole height range. Reading enough granules to +satisfy the filter blows past 10M rows. + +The materialized `owner_projection` stores the same data owner-ordered but +**ClickHouse cannot use it for this query shape** — it serves the filter but is +abandoned the moment `ORDER BY … LIMIT` is present. The robust fix is a +dedicated **owner-ordered table** the gateway routes to when an `owners` filter +is present. + +## The triggering query + +`UserDriveEntities` — "list this user's drives": + +```graphql +query UserDriveEntities($owner: String!, $after: String) { + transactions( + first: 100, after: $after, sort: HEIGHT_DESC, + tags: [{name: "Entity-Type", values: ["drive"]}], + owners: [$owner] + ) { edges { node { ...TransactionCommon } cursor } pageInfo { hasNextPage } } +} +``` + +Three properties make it pathological — none of them "wide block range": + +1. **owner + tag combined.** The tag presence sets `optimize_use_projections = 0` + in `buildChTransactionsSql` (`composite-clickhouse.ts`), and the optimizer + wouldn't use the projection here anyway (see below). The query falls back to + the height-ordered main table, finding the owner via `owner_address_bloom`. +2. **Small answer, huge candidate scan.** The owner filter matches every item + that owner ever uploaded; the selective `Entity-Type=drive` only narrows + *after* the read. "Entity-Type" is on every ArDrive entity, so the tag blooms + prune almost nothing. +3. **Sub-`pageSize` result forces exhaustion.** With fewer than 100 drives, the + query can never fill the page and early-terminate; to return the page *and* + prove `hasNextPage: false` it must scan the owner's entire height span. + +## Measurements + +All on the real owner `6Z-ifqgVi1jOwMvSNwKWs6ewUEQ0gU9eo4aHYC3rN1M` +(`owner_address` = `unhex('e99fa27ea8158b58cec0cbd2370296b3a7b0504434814f5ea38687602deb3753')`), +tags `Entity-Type` = `unhex('456e746974792d54797065')`, `drive` = +`unhex('6472697665')`. + +| # | What | Result | +|---|------|--------| +| Q1 | `EXPLAIN ESTIMATE` of the failing query (projections off) | **12,148,697 rows / 1486 granules** → over the 10M cap | +| Q2 | Owner's total footprint `count()` | **22,064 rows** | +| Q3 | The actual answer size (drive count) | **46 rows** | +| Q4 | `owner_projection` materialized? | yes, all 94 parts, 501.8M rows | +| Q7 | Owner-ONLY estimate on main table | **22,451,872 rows / 2746 granules** to find 22k rows | +| Q8 | Owner height spread | heights 737,838 → 1,946,014 = **1.2M-block span, 13 partitions** | +| Q9 | Pure owner seek via projection (forced) | **581,784 rows / 73 granules** — projection usable | +| Q5 | owner + tag + `ORDER BY/LIMIT`, projection forced | ❌ `PROJECTION_NOT_USED` | +| Q12 | owner + tag, **no order/limit**, projection forced | ✅ **442,988 rows / 55 granules** | +| Q13 | subquery (filter inner, order/limit outer) | 12,148,697 rows — limit pushed down, projection abandoned | +| Q14 | Q13 with projection forced | ❌ `PROJECTION_NOT_USED` | +| Q10 | main table data size | 135.45 GiB | +| Q11 | `owner_projection` data size | 67.14 GiB | + +### Interpretation + +- **It's real scatter, not bloom false positives.** The owner has 22k rows but + they live in ~2746 distinct height-ordered granules (~8 owner rows per 8192-row + granule) because the owner uploaded in bursts across a 1.2M-block span (Q7+Q8). + Reading those granules = 22.5M rows. Tightening `owner_address_bloom` (e.g. + 0.01 → 0.001) would only remove the ~1% genuine false positives, leaving + ~2200 real granules ≈ 18M rows — still over the cap. **Bloom tuning is dead.** +- **The projection works for filtering but not for top-N.** Q9 (pure owner) and + Q12 (owner+tag, no order/limit) both use the projection and read <600k rows. + Q5 (add `ORDER BY … LIMIT`) returns `PROJECTION_NOT_USED`. ClickHouse normal + projections do not support `optimize_read_in_order`, so once a top-N + `ORDER BY … LIMIT` is present the optimizer reverts to the base table's + primary key (height) and uses skip indexes for the owner — back to 12M rows. +- **You can't trick it from SQL.** Wrapping the filter in a subquery (Q13/Q14) + doesn't help: ClickHouse pushes the `ORDER BY/LIMIT` down into the inner read + and abandons the projection again. + +## Why the projection fails but a separate table works + +This is the crux. The `owner_projection` *is* a complete owner-ordered copy of +the data (67 GiB, maintained on every insert). The problem is purely that the +**optimizer won't route a top-N query to it**: + +- A normal projection is only ever used if ClickHouse's projection analysis + *chooses* it. That analysis can substitute a projection for **filtering** and + **aggregation**, but it does **not** use a normal projection's sort order to + satisfy `ORDER BY … LIMIT` with early termination (`optimize_read_in_order`). + When the top-N is present, the optimizer falls back to reading the base table + in primary-key (height) order and applies the owner filter via skip indexes — + the 12M-row path. There is no SQL-level rewrite that reliably forces it + (Q13/Q14). + +A dedicated **table** removes the optimizer from the decision entirely: + +- `SELECT … FROM transactions_by_owner WHERE owner_address = X ORDER BY height DESC LIMIT 101` +- Here `owner_address` is the literal **primary-key prefix**, so `WHERE owner = X` + is an ordinary primary-key range seek — always available, not a heuristic. +- `ORDER BY height DESC` matches the table's own sort-key suffix + `(owner_address, height, …)`, so `optimize_read_in_order` applies on a **real + primary key** (fully supported): ClickHouse reads the owner's rows already in + height order and early-terminates at `LIMIT`. + +In short: the projection only gets used for the *filter half* and is abandoned +for the *order/limit half*; a table makes `owner_address` a first-class primary +key so both halves are native. The two orderings (by-height for the common case, +by-owner for this case) are inherently two physical copies — which is exactly +what the projection already is; we're just moving that copy into a form the +gateway can address directly. + +## Update: `optimize_read_in_order = 0` reuses the existing projection (cheapest fix) + +Follow-up measurement (same owner): the projection is "inapplicable" with +`ORDER BY … LIMIT` *only because read-in-order is enabled*. Disabling it +per-query unlocks the projection — and the optimizer picks it **without forcing**: + +| # | What | Result | +|---|------|--------| +| H1 | owner+tag+ORDER BY+LIMIT, `optimize_read_in_order=0`, projection **forced** | ✅ applicable, **451,180 rows / 56 granules** | +| H2 | same, **not forced** (optimizer's natural choice) | ✅ **451,180 rows / 56 granules** | + +So the cheapest fix is a **pure emitted-SQL change** in `buildChTransactionsSql`: +for owner-filtered queries, emit +`SETTINGS optimize_use_projections = 1, optimize_read_in_order = 0` +(today tags force `optimize_use_projections = 0`; this path must set it to 1). +Reads drop **12.1M → 451K** (27× under the cap), reusing the 67 GiB projection +already on disk. No new table, no backfill. + +**Caveat — re-sort per page.** With read-in-order off, ClickHouse sorts the full +matched set before applying `LIMIT`, and does so on *every* page (no early +termination). For small/selective owner queries (a user's few drives/folders — +the dominant ArDrive landing queries) the matched set is tiny and this is free. +For owner+`file` deep pagination over a heavy owner, it re-reads and re-sorts the +whole footprint per page (O(n) per page). Those cases want the owner-result +cache and/or the dedicated owner-ordered table below (which gets *native* +read-in-order, no re-sort). + +## Implementation (shipped, env-gated off by default) + +The cheapest fix — reuse the existing `owner_projection` via +`optimize_read_in_order = 0` — is implemented in `composite-clickhouse.ts` +behind a feature flag, no schema change. A dedicated owner-ordered table +(below) is **deferred** as a future option for heavy-owner deep pagination. + +1. **Projection routing (hacks 1 + 4).** In `buildChTransactionsSql`, eligible + owner-filtered queries emit + `SETTINGS optimize_use_projections = 1, optimize_read_in_order = 0` so the + optimizer seeks `owner_projection` and sorts the small matched set in memory + (12.1M → 451K rows for the example owner). Otherwise the existing + `optimize_use_projections = 0` (id/tag lookups) is preserved unchanged. +2. **Eligibility predicate (`ownerProjectionApplies`).** A query qualifies only + when the feature is enabled, `owners` is present, `ids` is absent, and the + query carries an `Entity-Type` tag filter whose values are **all** in a + configurable allowlist (`CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES`, + default `drive,folder,snapshot`). This deliberately **excludes + `Entity-Type=file`** (millions of rows per owner → an expensive full sort of + the matched set, re-done every page because read-in-order is off), as well as + bare-owner and owner+other-tag queries. Those keep planning as today. +3. **Reactive windowing fallback (hack 5).** When an *eligible* query still + trips `max_rows_to_read` (a whale whose footprint exceeds the cap even via + the projection), the stable leg catches Code 158 and retries via + `queryStableTransactionsWindowed` — an adaptive height-window walk (halves + the span on repeated 158, caps at 256 windows) accumulating `pageSize + 1` + rows so the existing merge / `hasNextPage` logic is untouched. +4. **Master gate.** Everything is off unless + `CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED=true`. When off, behavior is + byte-identical to before. Window-span tuning lives in module-level + `OWNER_WINDOW_*` constants (promote to env config if operators need them). + +Not yet validated against production: the windowing fallback's real trigger (a +whale exceeding 10M rows through the projection) and multi-page cursor +pagination under `optimize_read_in_order = 0` (see "cursor pagination check"). + +### Deferred: dedicated owner-ordered table + +Justified only if heavy-owner `file` deep pagination shows up hot in profiling +(the projection path re-sorts the matched set per page; a real table gets +*native* read-in-order). Sketch: + +1. **`transactions_by_owner`**, same columns, + `ENGINE = ReplacingMergeTree(inserted_at)`, + `ORDER BY (owner_address, height, block_transaction_index, is_data_item, id)`, + partitioned `intDiv(height, 100000)`. No skip indexes — owner is the PK prefix. +2. **Keep it in sync** with a `MATERIALIZED VIEW … TO transactions_by_owner` on + inserts, *or* an explicit second write in the import pipeline. **Verify the + import path uses `INSERT`** (not `ATTACH PARTITION` / other DDL that bypasses + MV triggers) — check the auto-import flow and + `gateway-control.ts cleanClickHouseTables` staging/final handling. +3. **Route** owner queries to it (PK-prefix seek + native read-in-order, no + re-sort), then **drop `owner_projection`** to offset its ~67 GiB / write / + merge cost. + +## Overhead + +**Key framing:** the owner-ordered copy already exists as the 67 GiB +`owner_projection`, maintained on every insert. Replacing it with a table is +largely a *restructuring of existing overhead*, not new overhead. + +### One-time creation window (the only real new cost) + +- Backfill `INSERT INTO transactions_by_owner SELECT * FROM transactions` for + 501.8M rows = a full re-sort from height-order to owner-order. Reads 135 GiB, + writes ~67 GiB, external-sorts (spills to temp disk), then background-merges + the resulting parts. +- On production hardware competing with live import + queries: expect heavy CPU + (the sort), heavy disk IO, and merge churn — order of tens of minutes to a few + hours. **Mitigate:** batch by height partition + (`… WHERE height >= a AND height < b`), throttle `max_threads` / + `max_bytes_before_external_sort`, run off-peak, and create the table + MV + *before* backfilling (ReplacingMergeTree dedups any overlap with rows arriving + during the backfill). +- Transient storage peak: ~+67 GiB while both the new table and the + still-present projection exist (drop the projection only after cutover). + +### Ongoing (≈ neutral if the projection is dropped) + +- **Writes:** every import block written to a second table ≈ the per-insert + projection maintenance happening today. Net ~neutral after dropping the + projection. +- **Merges:** a second table's background merges ≈ today's projection merges. + Net ~neutral. +- **Storage:** +~67 GiB table − ~67 GiB projection ≈ neutral. +- **Query:** owner-filtered GQL drops from ~12M to <600k rows read (and far less + after merges, approaching the owner's true ~3-granule footprint); non-owner + queries are unchanged. + +## Appendix: byte encodings (for reproducing the queries) + +- `owner_address = unhex(b64UrlToHex(owner))` — base64url-decode the GQL owner + to 32 bytes, hex-encode. +- Tag name/value = `Buffer.from(str).toString('hex')` (UTF-8), e.g. + `Entity-Type` → `456e746974792d54797065`, `drive` → `6472697665`. +- Run inside the CH container: + `sudo docker exec -i ar-io-node-indexer-clickhouse-1 sh -c + 'clickhouse-client --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" --multiquery'`. diff --git a/docs/envs.md b/docs/envs.md index 47bde723d..2046546f7 100644 --- a/docs/envs.md +++ b/docs/envs.md @@ -561,6 +561,8 @@ height that would be silently dropped by a `height >= :minHeight` predicate | CLICKHOUSE_MAX_HEIGHT_CACHE_TTL_SECONDS | Number | 60 | TTL for the cached ClickHouse max-height lookup used by the boundary optimization | | CLICKHOUSE_QUERY_TIMEOUT_SECONDS | Number | 3 | Timeout for ClickHouse queries, applied both as server-side `max_execution_time` and client-side HTTP `request_timeout` | | CLICKHOUSE_GQL_MAX_ROWS_TO_READ | Number | 10000000 | Per-query `max_rows_to_read` hint applied to GraphQL queries against the `transactions` table. Queries that would scan more than this throw `Code: 158`, preventing accidental full-table scans if a skip index is bypassed | +| CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED | Boolean | false | When true, owner-filtered GraphQL `transactions` queries are routed through `owner_projection` (and gain a reactive height-windowing fallback if they still trip `max_rows_to_read`). The main `transactions` table is height-ordered, so a sparse owner's rows scatter across millions of granules and finding a page can trip the row cap; the owner-ordered projection seeks straight to the owner's slice. Disabled by default for staged rollout — when off, owner queries plan exactly as before | +| CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES | String | `drive,folder,snapshot` | Comma-separated allowlist of `Entity-Type` tag values eligible for owner_projection routing (only consulted when `CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED` is true). A query qualifies only when it carries an `Entity-Type` filter whose values are ALL in this list. `file` is deliberately omitted: an owner can have millions of files, and routing forces a full sort of the matched set per page (read-in-order is disabled to use the projection), re-done on every page — those large-result queries want the dedicated owner-ordered table instead. Bare-owner and owner+other-tag queries are likewise excluded | | CLICKHOUSE_GQL_DEDUPE_HEADROOM | Number | 4 | Multiplier applied to `pageSize + 1` to size the inner LIMIT of the GQL transactions query. **Pagination correctness caveat:** this must be at least as large as the table's effective duplicate factor. If a region of the table has more unmerged ReplacingMergeTree versions per PK than this headroom covers, the deduped window can yield fewer than `pageSize + 1` unique rows even when more matches exist further on — the result will be a short page with `hasNextPage: false` and rows past the window will be silently skipped. Regular background merges keep the duplicate factor at 1-2 in practice, so 4 leaves comfortable headroom; raise this if operators observe short pages (e.g. during a heavy ingest that produces many unmerged parts) | | CLICKHOUSE_SQLITE_CIRCUIT_BREAKER_TIMEOUT_MS | Number | 5000 | Timeout (ms) for the SQLite leg of the composite GQL transactions query. On timeout the response degrades to ClickHouse-only results; ClickHouse timeouts are governed separately by `CLICKHOUSE_QUERY_TIMEOUT_SECONDS` and still propagate to the caller | | CLICKHOUSE_SQLITE_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE | Number | 80 | Error rate (0–100) within the rolling window that trips the SQLite-leg breaker open | diff --git a/src/config.ts b/src/config.ts index 8c5ebb1ca..bc071b2bf 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1942,6 +1942,38 @@ export const CLICKHOUSE_GQL_DEDUPE_HEADROOM = env.positiveIntOrDefault( 4, ); +// Gate for routing owner-filtered GraphQL `transactions` queries through +// `owner_projection` (plus the reactive height-windowing fallback on +// `max_rows_to_read`). The main `transactions` table is height-ordered, so a +// sparse owner's rows scatter across millions of granules and finding a page +// can trip the row cap; the owner-ordered projection seeks straight to the +// owner's slice instead. Disabled by default for staged rollout — when off, +// owner queries plan exactly as before. +export const CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED = + env.varOrDefault( + 'CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED', + 'false', + ) === 'true'; + +// Comma-separated allowlist of `Entity-Type` tag values for which owner-filtered +// GQL queries use owner_projection routing (only consulted when +// CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED is true). These are the +// ArDrive entity types whose per-owner result set is small, so the projection's +// in-memory sort (read-in-order is disabled to use the projection) is cheap. A +// query qualifies only when it carries an `Entity-Type` filter whose values are +// ALL in this list — so `file` (millions per owner → an expensive full sort +// re-done every page), bare-owner queries, and owner+other-tag queries are +// excluded and keep planning as before. Large-result queries want the dedicated +// owner-ordered table instead. +export const CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES = env + .varOrDefault( + 'CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES', + 'drive,folder,snapshot', + ) + .split(',') + .map((value) => value.trim()) + .filter((value) => value.length > 0); + // Circuit breaker around the SQLite leg of the composite GQL transactions // query. ClickHouse and SQLite run in parallel; an open breaker degrades // responses to ClickHouse-only results instead of dragging the caller down diff --git a/src/database/composite-clickhouse.test.ts b/src/database/composite-clickhouse.test.ts index bcd410045..20a41abb2 100644 --- a/src/database/composite-clickhouse.test.ts +++ b/src/database/composite-clickhouse.test.ts @@ -162,12 +162,16 @@ function buildComposite({ sqlite, queryUnstableHead = false, skipSqliteReads = false, + ownerProjectionRoutingEnabled = false, + ownerProjectionEntityTypes = ['drive', 'folder', 'snapshot'], chRowsByLeg, chReject, }: { sqlite: GqlQueryable; queryUnstableHead?: boolean; skipSqliteReads?: boolean; + ownerProjectionRoutingEnabled?: boolean; + ownerProjectionEntityTypes?: string[]; chRowsByLeg: { stable?: any[]; unstable?: any[] }; chReject?: { stable?: Error; unstable?: Error }; }): CompositeClickHouseDatabase { @@ -177,6 +181,8 @@ function buildComposite({ url: 'http://localhost:0', queryUnstableHead, skipSqliteReads, + ownerProjectionRoutingEnabled, + ownerProjectionEntityTypes, sqliteCircuitBreakerOptions: { // Tight breaker so a thrown error trips the breaker for this test // run without slowing the suite. The composite's behavior under a @@ -510,6 +516,177 @@ describe('CompositeClickHouseDatabase', () => { }); }); + describe('owner-window fallback (hack 5)', () => { + it('retries an owner-filtered query with height-windowing on Code 158', async () => { + const composite = buildComposite({ + sqlite, + ownerProjectionRoutingEnabled: true, + chRowsByLeg: {}, + }); + + const queries: string[] = []; + let stableCalls = 0; + (composite as any).clickhouseClient = { + async query({ query: sqlStr }: { query: string }) { + queries.push(sqlStr); + if (sqlStr.includes('FROM new_transactions')) { + return { json: async () => ({ data: [] }) }; + } + // Stable leg. First call is the single-shot query, which we make + // trip max_rows_to_read; later calls are the windowed retries. + stableCalls += 1; + if (stableCalls === 1) { + const err: any = new Error( + 'Code: 158. DB::Exception: Limit for rows exceeded: TOO_MANY_ROWS', + ); + err.code = '158'; + throw err; + } + return { + json: async () => ({ + data: [ + chRow({ id: id('w1'), height: 95000 }), + chRow({ id: id('w2'), height: 94000 }), + chRow({ id: id('w3'), height: 93000 }), + ], + }), + }; + }, + }; + + const result = await composite.getGqlTransactions({ + pageSize: 2, + owners: [id('owner')], + tags: [{ name: 'Entity-Type', values: ['drive'] }], + maxHeight: 100_000, + }); + + // The single-shot threw 158, then at least one windowed retry ran. + assert.ok( + stableCalls >= 2, + `expected a windowed retry after Code 158, saw ${stableCalls} stable calls`, + ); + // Owner queries route through owner_projection (read-in-order disabled). + assert.ok( + queries.some((q) => q.includes('optimize_read_in_order = 0')), + 'expected owner-projection settings on the stable query', + ); + // The retry is height-bounded (a window); the single-shot is not. + const windowSql = queries[1]; + assert.ok( + windowSql !== undefined && windowSql.includes('FROM transactions'), + 'expected a windowed retry query', + ); + assert.match(windowSql, />=\s*\d/); + assert.doesNotMatch(queries[0], />=\s*\d/); + // Page is filled from the window; hasNextPage reflects the extra row. + assert.equal(result.edges.length, 2); + assert.equal(result.pageInfo.hasNextPage, true); + }); + + it('does not window a non-owner query on Code 158 (fail-fast)', async () => { + const composite = buildComposite({ + sqlite, + ownerProjectionRoutingEnabled: true, + chRowsByLeg: {}, + }); + let stableCalls = 0; + (composite as any).clickhouseClient = { + async query({ query: sqlStr }: { query: string }) { + if (sqlStr.includes('FROM new_transactions')) { + return { json: async () => ({ data: [] }) }; + } + stableCalls += 1; + const err: any = new Error('Code: 158. TOO_MANY_ROWS'); + err.code = '158'; + throw err; + }, + }; + + await assert.rejects( + composite.getGqlTransactions({ + pageSize: 2, + tags: [{ name: 'Entity-Type', values: ['drive'] }], + }), + /158|TOO_MANY_ROWS/, + ); + // No windowing retries for a tag-only (ownerless) query. + assert.equal(stableCalls, 1); + }); + + it('does not window when the feature is disabled (default)', async () => { + const composite = buildComposite({ sqlite, chRowsByLeg: {} }); + const queries: string[] = []; + let stableCalls = 0; + (composite as any).clickhouseClient = { + async query({ query: sqlStr }: { query: string }) { + queries.push(sqlStr); + if (sqlStr.includes('FROM new_transactions')) { + return { json: async () => ({ data: [] }) }; + } + stableCalls += 1; + const err: any = new Error('Code: 158. TOO_MANY_ROWS'); + err.code = '158'; + throw err; + }, + }; + + await assert.rejects( + composite.getGqlTransactions({ + pageSize: 2, + owners: [id('owner')], + tags: [{ name: 'Entity-Type', values: ['drive'] }], + }), + /158|TOO_MANY_ROWS/, + ); + // Disabled: single-shot fails fast, no windowing, and the query plans + // exactly as before (projections disabled for the tag filter). + assert.equal(stableCalls, 1); + assert.ok( + queries.some((q) => q.includes('optimize_use_projections = 0')), + ); + assert.ok(!queries.some((q) => q.includes('optimize_read_in_order = 0'))); + }); + + it('excludes Entity-Type=file from the feature (allowlist)', async () => { + const composite = buildComposite({ + sqlite, + ownerProjectionRoutingEnabled: true, + chRowsByLeg: {}, + }); + const queries: string[] = []; + let stableCalls = 0; + (composite as any).clickhouseClient = { + async query({ query: sqlStr }: { query: string }) { + queries.push(sqlStr); + if (sqlStr.includes('FROM new_transactions')) { + return { json: async () => ({ data: [] }) }; + } + stableCalls += 1; + const err: any = new Error('Code: 158. TOO_MANY_ROWS'); + err.code = '158'; + throw err; + }, + }; + + await assert.rejects( + composite.getGqlTransactions({ + pageSize: 2, + owners: [id('owner')], + tags: [{ name: 'Entity-Type', values: ['file'] }], + }), + /158|TOO_MANY_ROWS/, + ); + // `file` is not in the allowlist: no projection routing, no windowing — + // the query plans exactly as it does without the feature. + assert.equal(stableCalls, 1); + assert.ok( + queries.some((q) => q.includes('optimize_use_projections = 0')), + ); + assert.ok(!queries.some((q) => q.includes('optimize_read_in_order = 0'))); + }); + }); + describe('streaming-disabled (queryUnstableHead=false)', () => { it('does not query the unstable leg', async () => { let unstableQueried = false; diff --git a/src/database/composite-clickhouse.ts b/src/database/composite-clickhouse.ts index 9251f7476..e61a99232 100644 --- a/src/database/composite-clickhouse.ts +++ b/src/database/composite-clickhouse.ts @@ -136,6 +136,42 @@ function buildGqlTransactionColumns( type SqliteGqlArgs = Parameters[0]; +// Tag name carrying the ArDrive entity type (drive / folder / file / snapshot). +// The owner_projection routing feature keys off this tag's values. +const ENTITY_TYPE_TAG_NAME = 'Entity-Type'; + +// ClickHouse `Code: 158 TOO_MANY_ROWS` — raised when a GQL query would scan +// more than `max_rows_to_read`. The @clickhouse/client surfaces server errors +// with a numeric `.code` (string or number) and a message of the form +// `Code: 158. DB::Exception: ...TOO_MANY_ROWS...`. +function isClickHouseTooManyRowsError(err: unknown): boolean { + const e = err as { code?: string | number; message?: string } | null; + if (e == null) return false; + if (e.code === '158' || e.code === 158) return true; + return /\bcode:\s*158\b|TOO_MANY_ROWS/i.test(e.message ?? ''); +} + +// Reactive fallback (hack 5) for owner-filtered GQL queries whose owner +// footprint is so large it still trips `max_rows_to_read` even via +// owner_projection (a "whale" owner). When the single-shot stable query throws +// Code 158, we re-run it as a walk over height windows, each small enough to +// stay under the cap, in sort order, accumulating until we have a full page. +// See docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md. +// The whole feature is gated by `ownerProjectionRoutingEnabled` +// (config.CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED, default off). The +// span-tuning values below are deliberately module-level constants for a first +// cut; promote to env config (CLICKHOUSE_GQL_OWNER_WINDOW_*) if operators need +// to tune them. +// Number of windows the initial span aims to divide the walked range into. +const OWNER_WINDOW_INITIAL_DIVISIONS = 8; +// Smallest height span a window may shrink to before we stop subdividing. A +// span this small still exceeding the cap is pathological and signals the data +// needs the dedicated owner-ordered table rather than reactive windowing. +const OWNER_WINDOW_MIN_SPAN = 10_000; +// Safety cap on total windows per fallback so a sparse-/no-match whale walk +// can't scan the whole chain unbounded. +const OWNER_WINDOW_MAX_WINDOWS = 256; + // Pre-encoded SQL literal forms shared across the two CH legs in // `getGqlTransactions`. Built once by `prepareGqlFilterEncodings` and // passed into `addGqlTransactionFilters` so each leg doesn't redo the @@ -177,6 +213,15 @@ export class CompositeClickHouseDatabase implements GqlQueryable { // effect on SQLite WRITES; the indexer still feeds SQLite for the // Parquet export pipeline. private skipSqliteReads: boolean; + // When true, owner-filtered GQL queries are routed through + // `owner_projection` and gain the reactive height-windowing fallback on + // `max_rows_to_read`. Off by default — see + // config.CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED. + private ownerProjectionRoutingEnabled: boolean; + // Allowlist of `Entity-Type` values eligible for owner_projection routing + // (e.g. drive/folder/snapshot). `file` is intentionally absent — see + // ownerProjectionApplies and config.CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES. + private ownerProjectionEntityTypes: Set; constructor({ log, @@ -190,6 +235,8 @@ export class CompositeClickHouseDatabase implements GqlQueryable { queryTimeoutSeconds = 3, queryUnstableHead = false, skipSqliteReads = false, + ownerProjectionRoutingEnabled = false, + ownerProjectionEntityTypes = [], sqliteCircuitBreakerOptions = { timeout: config.CLICKHOUSE_SQLITE_CIRCUIT_BREAKER_TIMEOUT_MS, errorThresholdPercentage: @@ -210,6 +257,8 @@ export class CompositeClickHouseDatabase implements GqlQueryable { queryTimeoutSeconds?: number; queryUnstableHead?: boolean; skipSqliteReads?: boolean; + ownerProjectionRoutingEnabled?: boolean; + ownerProjectionEntityTypes?: string[]; sqliteCircuitBreakerOptions?: CircuitBreaker.Options; }) { this.log = log; @@ -233,6 +282,8 @@ export class CompositeClickHouseDatabase implements GqlQueryable { this.maxHeightCacheTtlMs = maxHeightCacheTtlSeconds * 1000; this.queryUnstableHead = queryUnstableHead; this.skipSqliteReads = skipSqliteReads; + this.ownerProjectionRoutingEnabled = ownerProjectionRoutingEnabled; + this.ownerProjectionEntityTypes = new Set(ownerProjectionEntityTypes); this.sqliteBreaker = new CircuitBreaker( (args: SqliteGqlArgs) => this.gqlQueryable.getGqlTransactions(args), @@ -550,6 +601,31 @@ export class CompositeClickHouseDatabase implements GqlQueryable { // LIMIT 1 BY collapses unmerged ReplacingMergeTree versions. The // result is shape-identical across stable and unstable legs, so // mapTransactionRow handles either leg's rows without branching. + // Whether a GQL transactions query is eligible for owner_projection routing + // (and its 158 windowing fallback). Requires the feature enabled, an + // `owners` filter, no `ids` filter (id lookups have their own selective + // path), and an `Entity-Type` tag filter whose values are ALL in the + // configured allowlist. The Entity-Type gate is what excludes `file` + // (large per-owner result → an expensive full sort per page under + // read-in-order=0) as well as bare-owner and owner+other-tag queries. + private ownerProjectionApplies( + owners: string[], + ids: string[], + tags: { name: string; values: string[] }[], + ): boolean { + if (!this.ownerProjectionRoutingEnabled) return false; + if (owners.length === 0 || ids.length > 0) return false; + const entityTypeTags = tags.filter( + (tag) => tag.name === ENTITY_TYPE_TAG_NAME, + ); + if (entityTypeTags.length === 0) return false; + return entityTypeTags.every( + (tag) => + tag.values.length > 0 && + tag.values.every((value) => this.ownerProjectionEntityTypes.has(value)), + ); + } + private buildChTransactionsSql({ innerSql, pageSize, @@ -596,7 +672,33 @@ export class CompositeClickHouseDatabase implements GqlQueryable { const settings: string[] = [ `max_rows_to_read = ${config.CLICKHOUSE_GQL_MAX_ROWS_TO_READ}`, ]; - if (ids.length > 0 || tags.length > 0) { + // Owner-filtered queries (the dominant ArDrive access pattern, e.g. + // `owners:[x], tags:[Entity-Type=drive]`) are routed through + // `owner_projection`, the owner-ordered copy of the data. The main + // `transactions` table is height-ordered, so a sparse owner's rows are + // smeared ~8-per-granule across the full height range; finding a page + // there scans tens of millions of rows and trips `max_rows_to_read` + // (measured: an owner with 22k total rows / 46 drives forced a 12.1M-row + // read → Code 158). The projection seeks straight to the owner's + // contiguous slice (measured 451K rows for the same query). ClickHouse + // won't route a top-N `ORDER BY ... LIMIT` to a normal projection while + // read-in-order is enabled, so it's disabled here; the matched set is + // small (bounded by the owner's footprint) and sorted in memory instead. + // + // Scoped to owner-without-id queries and applied uniformly across tag + // values (no per-`Entity-Type` special-casing): + // - id lookups have their own selective `id_bloom` path on the main + // table; the owner ordering doesn't help them. + // - tag-/recipient-only queries (no owner) have nothing to seek on in + // the owner-ordered projection, so enabling it there forces a full + // projection scan — which is why projections stay disabled for those. + // Harmless on the unstable leg (`new_transactions` has no projection). + if (this.ownerProjectionApplies(owners, ids, tags)) { + settings.push( + 'optimize_use_projections = 1', + 'optimize_read_in_order = 0', + ); + } else if (ids.length > 0 || tags.length > 0) { settings.push('optimize_use_projections = 0'); } const settingsClause = ` SETTINGS ${settings.join(', ')}`; @@ -607,6 +709,192 @@ export class CompositeClickHouseDatabase implements GqlQueryable { ); } + // Runs one window of the owner-window fallback: the standard stable-leg + // query restricted to a `[windowMinHeight, windowMaxHeight]` slice. Reuses + // the normal filter + wrapper builders, so the owner_projection settings and + // `max_rows_to_read` cap apply per window (a window that still exceeds the + // cap throws Code 158, which the walk catches and subdivides). + private async queryStableWindow({ + pageSize, + cursor, + sortOrder, + ids, + recipients, + owners, + windowMinHeight, + windowMaxHeight, + bundledIn, + tags, + encoded, + }: { + pageSize: number; + cursor?: string; + sortOrder: 'HEIGHT_DESC' | 'HEIGHT_ASC'; + ids: string[]; + recipients: string[]; + owners: string[]; + windowMinHeight: number; + windowMaxHeight: number; + bundledIn?: string[] | null; + tags: { name: string; values: string[] }[]; + encoded: EncodedGqlFilters; + }): Promise { + const query = this.getGqlTransactionsBaseSql(); + this.addGqlTransactionFilters({ + query, + cursor, + sortOrder, + ids, + recipients, + owners, + minHeight: windowMinHeight, + maxHeight: windowMaxHeight, + bundledIn, + tags, + encoded, + }); + const windowSql = this.buildChTransactionsSql({ + innerSql: query.toString(), + pageSize, + sortOrder, + recipients, + owners, + ids, + tags, + }); + const row = await this.clickhouseClient.query({ query: windowSql }); + const jsonRow = await row.json(); + return (jsonRow.data as any[]).map((tx: any) => this.mapTransactionRow(tx)); + } + + /** + * Reactive fallback (hack 5) for an owner-filtered stable-leg query that + * tripped `max_rows_to_read` (Code 158) — i.e. a whale owner whose footprint + * exceeds the cap even through owner_projection. Walks the requested height + * range in windows small enough to stay under the cap, in `sortOrder`, + * accumulating up to `pageSize + 1` rows so the caller's `hasNextPage` logic + * is unchanged. Window span adapts: a window that still trips 158 is halved + * and retried at the same leading edge. + * + * The returned rows are re-sorted with the other legs in `getGqlTransactions`, + * so the walk only needs to find the correct *set* of top rows, not emit them + * in perfect order. Windows are disjoint height ranges walked from the + * leading edge, so once `pageSize + 1` rows are collected the remaining + * (further-from-leading-edge) windows cannot contain a higher-priority row. + */ + private async queryStableTransactionsWindowed({ + pageSize, + cursor, + sortOrder, + ids, + recipients, + owners, + minHeight, + maxHeight, + bundledIn, + tags, + encoded, + }: { + pageSize: number; + cursor?: string; + sortOrder: 'HEIGHT_DESC' | 'HEIGHT_ASC'; + ids: string[]; + recipients: string[]; + owners: string[]; + minHeight: number; + maxHeight: number; + bundledIn?: string[] | null; + tags: { name: string; values: string[] }[]; + encoded: EncodedGqlFilters; + }): Promise { + const target = pageSize + 1; + + // Resolve concrete [lo, hi] height bounds for the walk. The caller's + // min/max and the cached CH max height bound the range; the cursor (when + // present) tightens the leading edge. + let lo = minHeight > 0 ? minHeight : 0; + let hi = maxHeight >= 0 ? maxHeight : await this.getClickHouseMaxHeight(); + if (hi == null) { + // No upper bound to size windows against — surface the failure. + throw new Error( + 'owner-window fallback: unable to resolve ClickHouse max height', + ); + } + const { height: cursorHeight } = decodeTransactionGqlCursor(cursor); + if (cursorHeight != null) { + if (sortOrder === 'HEIGHT_DESC') { + hi = Math.min(hi, cursorHeight); + } else { + lo = Math.max(lo, cursorHeight); + } + } + if (hi < lo) return []; + + let span = Math.max( + OWNER_WINDOW_MIN_SPAN, + Math.ceil((hi - lo + 1) / OWNER_WINDOW_INITIAL_DIVISIONS), + ); + + const collected: GqlTransactionsResult['edges'][0]['node'][] = []; + const seen = new Set(); + const descending = sortOrder === 'HEIGHT_DESC'; + // Current leading height (inclusive): top of the range for DESC, bottom + // for ASC. + let edge = descending ? hi : lo; + let windowCount = 0; + + while ( + collected.length < target && + (descending ? edge >= lo : edge <= hi) + ) { + if (windowCount >= OWNER_WINDOW_MAX_WINDOWS) { + throw new Error( + `owner-window fallback exceeded ${OWNER_WINDOW_MAX_WINDOWS} windows ` + + 'without filling a page; this owner footprint needs the dedicated ' + + 'owner-ordered table', + ); + } + + const windowLo = descending ? Math.max(lo, edge - span + 1) : edge; + const windowHi = descending ? edge : Math.min(hi, edge + span - 1); + + let rows: GqlTransactionsResult['edges'][0]['node'][]; + try { + rows = await this.queryStableWindow({ + pageSize, + cursor, + sortOrder, + ids, + recipients, + owners, + windowMinHeight: windowLo, + windowMaxHeight: windowHi, + bundledIn, + tags, + encoded, + }); + } catch (err) { + // Window still too dense: halve the span and retry the same edge. + if (isClickHouseTooManyRowsError(err) && span > OWNER_WINDOW_MIN_SPAN) { + span = Math.max(OWNER_WINDOW_MIN_SPAN, Math.floor(span / 2)); + continue; + } + throw err; + } + + for (const tx of rows) { + if (seen.has(tx.id)) continue; + seen.add(tx.id); + collected.push(tx); + } + + windowCount += 1; + edge = descending ? windowLo - 1 : windowHi + 1; + } + + return collected.slice(0, target); + } + // Maps a CH row from either the stable or unstable transactions table // into the GraphQL transaction shape. The two tables project the same // columns (the unstable leg fills offset-family columns with NULL), @@ -766,9 +1054,39 @@ export class CompositeClickHouseDatabase implements GqlQueryable { } const chStablePromise = (async () => { - const row = await this.clickhouseClient.query({ query: stableSql }); - const jsonRow = await row.json(); - return jsonRow.data.map((tx: any) => this.mapTransactionRow(tx)); + try { + const row = await this.clickhouseClient.query({ query: stableSql }); + const jsonRow = await row.json(); + return jsonRow.data.map((tx: any) => this.mapTransactionRow(tx)); + } catch (err) { + // Hack 5: an owner-filtered query that still trips max_rows_to_read + // through owner_projection means a whale whose footprint exceeds the + // cap. Re-run as an adaptive height-windowed walk instead of failing. + if ( + this.ownerProjectionApplies(owners, ids, tags) && + isClickHouseTooManyRowsError(err) + ) { + this.log.warn( + 'Stable GQL leg tripped max_rows_to_read on an owner-filtered ' + + 'query; retrying with adaptive height-windowing', + { owners, tags, minHeight, maxHeight }, + ); + return this.queryStableTransactionsWindowed({ + pageSize, + cursor, + sortOrder, + ids, + recipients, + owners, + minHeight, + maxHeight, + bundledIn, + tags, + encoded: encodedFilters, + }); + } + throw err; + } })(); // UNSTABLE LEG — `new_transactions` joined against `new_blocks` for diff --git a/src/system.ts b/src/system.ts index 863881f1f..82a5d7835 100644 --- a/src/system.ts +++ b/src/system.ts @@ -369,6 +369,10 @@ export const gqlQueryable: GqlQueryable = (() => { skipSqliteReads: config.CLICKHOUSE_STREAMING_ENABLED && config.CLICKHOUSE_GQL_SKIP_SQLITE_READS, + ownerProjectionRoutingEnabled: + config.CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED, + ownerProjectionEntityTypes: + config.CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES, ...(config.CLICKHOUSE_STREAMING_ENABLED ? { sqliteCircuitBreakerOptions: { From e3ef6ef298695ab89eeab380c70ac3400509a9dc Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Thu, 25 Jun 2026 13:32:59 -0700 Subject: [PATCH 09/20] fix(compose): forward CLICKHOUSE_GQL_OWNER_PROJECTION_* env to core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new owner-projection routing vars were added to config.ts and docs/envs.md but not to docker-compose.yaml's core `environment:` allowlist, so setting them in `.env` had no effect — the container never received them and the feature stayed off. Add both vars alongside the other CLICKHOUSE_GQL_* passthroughs (CLAUDE.md: keep compose and envs.md in sync). Co-Authored-By: Claude Opus 4.8 (1M context) --- docker-compose.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 76028cd94..ea8a0a86e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -375,6 +375,8 @@ services: - CLICKHOUSE_QUERY_TIMEOUT_SECONDS=${CLICKHOUSE_QUERY_TIMEOUT_SECONDS:-} - CLICKHOUSE_GQL_MAX_ROWS_TO_READ=${CLICKHOUSE_GQL_MAX_ROWS_TO_READ:-} - CLICKHOUSE_GQL_DEDUPE_HEADROOM=${CLICKHOUSE_GQL_DEDUPE_HEADROOM:-} + - CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED=${CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED:-} + - CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES=${CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES:-} - CLICKHOUSE_SQLITE_CIRCUIT_BREAKER_TIMEOUT_MS=${CLICKHOUSE_SQLITE_CIRCUIT_BREAKER_TIMEOUT_MS:-} - CLICKHOUSE_SQLITE_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE=${CLICKHOUSE_SQLITE_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE:-} - CLICKHOUSE_SQLITE_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS=${CLICKHOUSE_SQLITE_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS:-} From 8bc11635d6b26cf387c5c720c2da6b83e61e606a Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Thu, 25 Jun 2026 13:49:14 -0700 Subject: [PATCH 10/20] fix(gql): address CodeRabbit review on owner-projection routing - ownerProjectionApplies now requires that ALL tags are Entity-Type tags, so owner+other-tag shapes (e.g. owner + Entity-Type=drive + App-Name) fall back to the default plan instead of routing an untested shape through the projection (matches the documented allowlist contract). - The windowed fallback drains a dense height window via a running cursor before advancing the frontier. Previously a window returning only pageSize+1 raw rows whose dedup collapsed some ids would advance past the slice and strand the unique rows below it (short pages / wrong hasNextPage). - Refresh the findings-doc status (implemented + env-gated + canary-validated) and drop public gateway names from it. Adds tests for both behaviors. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...ickhouse-gql-owner-filter-too-many-rows.md | 6 +- src/database/composite-clickhouse.test.ts | 116 ++++++++++++++++++ src/database/composite-clickhouse.ts | 35 +++++- 3 files changed, 150 insertions(+), 7 deletions(-) diff --git a/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md b/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md index 8026a42c5..49b1b91c8 100644 --- a/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md +++ b/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md @@ -1,8 +1,10 @@ # ClickHouse GQL `TOO_MANY_ROWS` on owner-filtered queries **Date:** 2026-06-25 -**Status:** Investigation complete; fix proposed, not yet implemented -**Measured on:** canary gateway (gw1), ClickHouse 26.3.9, `default.transactions` +**Status:** Implemented behind `CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED` +(default off) and validated on a canary gateway (PR #796). The dedicated +owner-ordered table remains deferred (see below). +**Measured on:** a canary gateway, ClickHouse 26.3.9, `default.transactions` (501.8M rows, 135 GiB data + 67 GiB `owner_projection`, 92 active parts) ## Summary diff --git a/src/database/composite-clickhouse.test.ts b/src/database/composite-clickhouse.test.ts index 20a41abb2..682104e82 100644 --- a/src/database/composite-clickhouse.test.ts +++ b/src/database/composite-clickhouse.test.ts @@ -685,6 +685,122 @@ describe('CompositeClickHouseDatabase', () => { ); assert.ok(!queries.some((q) => q.includes('optimize_read_in_order = 0'))); }); + + it('excludes owner + Entity-Type + another tag (owner+other-tag)', async () => { + const composite = buildComposite({ + sqlite, + ownerProjectionRoutingEnabled: true, + chRowsByLeg: {}, + }); + const queries: string[] = []; + let stableCalls = 0; + (composite as any).clickhouseClient = { + async query({ query: sqlStr }: { query: string }) { + queries.push(sqlStr); + if (sqlStr.includes('FROM new_transactions')) { + return { json: async () => ({ data: [] }) }; + } + stableCalls += 1; + const err: any = new Error('Code: 158. TOO_MANY_ROWS'); + err.code = '158'; + throw err; + }, + }; + + await assert.rejects( + composite.getGqlTransactions({ + pageSize: 2, + owners: [id('owner')], + tags: [ + { name: 'Entity-Type', values: ['drive'] }, + { name: 'App-Name', values: ['ArDrive'] }, + ], + }), + /158|TOO_MANY_ROWS/, + ); + // An extra non-Entity-Type tag disqualifies the query: no projection + // settings, no windowing — it plans as it would without the feature. + assert.equal(stableCalls, 1); + assert.ok( + queries.some((q) => q.includes('optimize_use_projections = 0')), + ); + assert.ok(!queries.some((q) => q.includes('optimize_read_in_order = 0'))); + }); + + it('drains a dense window via cursor instead of stranding rows', async () => { + const composite = buildComposite({ + sqlite, + ownerProjectionRoutingEnabled: true, + chRowsByLeg: {}, + }); + const queries: string[] = []; + let stableCalls = 0; + (composite as any).clickhouseClient = { + async query({ query: sqlStr }: { query: string }) { + queries.push(sqlStr); + if (sqlStr.includes('FROM new_transactions')) { + return { json: async () => ({ data: [] }) }; + } + stableCalls += 1; + if (stableCalls === 1) { + const err: any = new Error('Code: 158. TOO_MANY_ROWS'); + err.code = '158'; + throw err; + } + if (stableCalls === 2) { + // A full target-sized batch (pageSize + 1 = 3) whose dedup leaves + // only 2 distinct ids — the old code advanced past the window here, + // stranding the rows below. + return { + json: async () => ({ + data: [ + chRow({ + id: id('p'), + height: 95000, + blockTransactionIndex: 0, + }), + chRow({ + id: id('p'), + height: 95000, + blockTransactionIndex: 1, + }), + chRow({ id: id('q'), height: 94000 }), + ], + }), + }; + } + // Continuation within the SAME window, below the advanced cursor. + return { + json: async () => ({ + data: [ + chRow({ id: id('r'), height: 93000 }), + chRow({ id: id('s'), height: 92000 }), + ], + }), + }; + }, + }; + + const result = await composite.getGqlTransactions({ + pageSize: 2, + owners: [id('owner')], + tags: [{ name: 'Entity-Type', values: ['drive'] }], + maxHeight: 100000, + }); + + // The dense first window forced a cursor-driven continuation (>=3 stable + // calls), and the continuation resumed BELOW the prior batch's last row + // (height 94000) rather than jumping to a fresh window — so the rows that + // would otherwise be stranded stay reachable. + assert.ok(stableCalls >= 3, `expected continuation, saw ${stableCalls}`); + const stableQs = queries.filter((q) => q.includes('FROM transactions')); + assert.ok( + stableQs.slice(1).some((q) => q.includes('94000')), + 'expected a continuation query carrying the advanced cursor (94000)', + ); + assert.equal(result.edges.length, 2); + assert.equal(result.pageInfo.hasNextPage, true); + }); }); describe('streaming-disabled (queryUnstableHead=false)', () => { diff --git a/src/database/composite-clickhouse.ts b/src/database/composite-clickhouse.ts index e61a99232..eb3fdffae 100644 --- a/src/database/composite-clickhouse.ts +++ b/src/database/composite-clickhouse.ts @@ -618,7 +618,14 @@ export class CompositeClickHouseDatabase implements GqlQueryable { const entityTypeTags = tags.filter( (tag) => tag.name === ENTITY_TYPE_TAG_NAME, ); - if (entityTypeTags.length === 0) return false; + // Require an Entity-Type filter and NO other tag types. An additional + // non-Entity-Type tag (e.g. App-Name) only narrows the result, but it's an + // untested shape and the agreed contract excludes owner+other-tag queries, + // so fall back to the default plan rather than route it through the + // projection. + if (entityTypeTags.length === 0 || entityTypeTags.length !== tags.length) { + return false; + } return entityTypeTags.every( (tag) => tag.values.length > 0 && @@ -839,8 +846,13 @@ export class CompositeClickHouseDatabase implements GqlQueryable { const seen = new Set(); const descending = sortOrder === 'HEIGHT_DESC'; // Current leading height (inclusive): top of the range for DESC, bottom - // for ASC. + // for ASC. Walks toward the trailing edge as windows are drained. let edge = descending ? hi : lo; + // Running cursor: starts at the request cursor and advances to the last + // row returned. A window holds at most `target` raw rows per fetch, so if + // it contains more matching rows than that, the advanced cursor drains the + // remainder across iterations instead of stranding them when we move on. + let runningCursor = cursor; let windowCount = 0; while ( @@ -862,7 +874,7 @@ export class CompositeClickHouseDatabase implements GqlQueryable { try { rows = await this.queryStableWindow({ pageSize, - cursor, + cursor: runningCursor, sortOrder, ids, recipients, @@ -887,9 +899,22 @@ export class CompositeClickHouseDatabase implements GqlQueryable { seen.add(tx.id); collected.push(tx); } - windowCount += 1; - edge = descending ? windowLo - 1 : windowHi + 1; + + // Advance the cursor to the last (lowest-priority) row so a partially + // consumed window resumes below it next iteration. + if (rows.length > 0) { + runningCursor = encodeTransactionGqlCursor(rows[rows.length - 1]); + } + + // Only drop to the next window once this one is drained. A full + // `target`-sized batch means there may be more matching rows below the + // cursor in the SAME window (dedup can leave collected < target), so keep + // the edge and let the advanced cursor pull the rest. A short batch means + // the window beyond the cursor is exhausted. + if (rows.length < target) { + edge = descending ? windowLo - 1 : windowHi + 1; + } } return collected.slice(0, target); From 0454de4e9c938053ec2e4f82b4fe9ebbe72c09f8 Mon Sep 17 00:00:00 2001 From: vilenarios Date: Thu, 25 Jun 2026 23:21:56 +0000 Subject: [PATCH 11/20] docs: remove orphaned AI-generated ar-io-0X draft guides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These six guides (~5,462 lines) were added in one batch on 2025-07-01 (43318d89, "docs: add comprehensive AI-generated documentation"). They were never linked from docs/INDEX.md, never reviewed, and never corrected in the ~year since. An audit against the code found them partially fabricated while presenting as authoritative "Complete Technical Documentation": invented env vars (CHUNK_POST_URLS — real is PREFERRED_CHUNK_POST_NODE_URLS; SECONDARY_CHUNK_POST_MIN_SUCCESS_COUNT; CHUNK_POST_TIMEOUT_MS — real are CHUNK_POST_RESPONSE_TIMEOUT_MS/_ABORT_TIMEOUT_MS), a non-existent "secondary broadcast tier", and wrong defaults — including copy-pasteable .env examples referencing vars that do nothing. They were also the only place the now-removed dead ARWEAVE_PEER_CHUNK_POST_* vars were "documented". Orphaned + unreviewed + net-misleading. Removing the series; accurate, maintained docs live under docs/ (indexed by docs/INDEX.md). Removed: - ar-io-01-architecture-overview.md - ar-io-02-data-retrieval-complete-guide.md - ar-io-03-arweave-connectivity-complete-guide.md - ar-io-04-arns-name-resolution-system.md - ar-io-05-centralization-analysis.md - ar-io-06-database-architecture.md Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XQPK4TXcVoXoyFp6sNW2Lr --- docs/drafts/ar-io-01-architecture-overview.md | 228 --- .../ar-io-02-data-retrieval-complete-guide.md | 1057 ------------- ...-03-arweave-connectivity-complete-guide.md | 1319 ----------------- .../ar-io-04-arns-name-resolution-system.md | 194 --- .../ar-io-05-centralization-analysis.md | 173 --- docs/drafts/ar-io-06-database-architecture.md | 1047 ------------- 6 files changed, 4018 deletions(-) delete mode 100644 docs/drafts/ar-io-01-architecture-overview.md delete mode 100644 docs/drafts/ar-io-02-data-retrieval-complete-guide.md delete mode 100644 docs/drafts/ar-io-03-arweave-connectivity-complete-guide.md delete mode 100644 docs/drafts/ar-io-04-arns-name-resolution-system.md delete mode 100644 docs/drafts/ar-io-05-centralization-analysis.md delete mode 100644 docs/drafts/ar-io-06-database-architecture.md diff --git a/docs/drafts/ar-io-01-architecture-overview.md b/docs/drafts/ar-io-01-architecture-overview.md deleted file mode 100644 index f172fa606..000000000 --- a/docs/drafts/ar-io-01-architecture-overview.md +++ /dev/null @@ -1,228 +0,0 @@ -# AR.IO Gateway: Complete Architecture Analysis - -## Overview -The AR.IO Gateway is a sophisticated data access layer for the Arweave network, designed as a read-optimized "Permaweb CDN" that provides fast, verified access to Arweave data through intelligent caching, background verification, and multi-source fallbacks. - -## 1. Arweave Integration Architecture - -### ArweaveCompositeClient -The gateway connects to Arweave through the `ArweaveCompositeClient` (`src/arweave/composite-client.ts`) which: -- **Primary Connection**: Manages connections to trusted nodes (`TRUSTED_NODE_URL`) -- **Peer Discovery**: Automatically discovers and weights peer nodes for load balancing -- **Fault Tolerance**: Implements circuit breakers and retry logic with exponential backoff -- **Dual Operations**: Handles both blockchain data retrieval and chunk broadcasting - -### Configuration Parameters -- `TRUSTED_NODE_URL` (default: `https://arweave.net`): Primary Arweave node -- `TRUSTED_GATEWAY_URL`: Optional trusted gateway for contiguous data -- `TRUSTED_GATEWAYS_URLS`: JSON map of gateway URLs with weights -- `CHUNK_POST_URLS`: URLs for chunk broadcasting -- `WEIGHTED_PEERS_TEMPERATURE_DELTA`: Peer performance sensitivity - -## 2. Data Reading Flow Architecture - -### Request Processing Pipeline -1. **HTTP Request Reception**: Express.js routes handle incoming requests -2. **ArNS Resolution**: - - Checks ArNS cache (Redis/KV store) - - Resolves names through AR.IO network process - - Falls back to trusted gateways if needed -3. **Data Resolution**: - - Looks up transaction/data item by ID in databases - - Checks for cached data via hash lookup - - Resolves manifest paths for directory-style requests - -### Hierarchical Data Retrieval (`ON_DEMAND_RETRIEVAL_ORDER`) -Default order: `trusted-gateways,ar-io-network,chunks-data-item,tx-data` - -1. **Trusted Gateways**: Configured gateway URLs with priority weights -2. **AR.IO Network**: Other AR.IO network peers -3. **Chunks-Data-Item**: Chunk retrieval with automatic data item resolution -4. **TX Data**: Complete transaction data from the chain - -### Caching Mechanisms -- **ReadThroughDataCache**: Primary caching layer with metadata tracking -- **Data Cache**: Stores actual data content by hash -- **Metadata Cache**: Stores access timestamps, ArNS names, verification status -- **Trust Headers**: Indicates data verification, trust, and cache status - -### Manifest Resolution -- **StreamingManifestPathResolver**: Handles directory-style paths -- Supports both v0.1.0 and v0.2.0 manifest formats -- Resolves nested paths within manifests -- Provides index paths and fallback ID handling - -## 3. Data Writing Operations - -### Chunk Upload System -- **Endpoint**: `POST /chunk` (max 256KiB + 40% base64url overhead) -- **Validation**: Format and metadata validation -- **Broadcasting**: Multi-target chunk distribution: - - Primary chunk nodes (direct Arweave nodes) - - Secondary chunk nodes (backup redundancy) - - Peer gateways (AR.IO network distribution) -- **Success Criteria**: Configurable minimum success count (`CHUNK_POST_MIN_SUCCESS_COUNT`) - -### Key Limitations -- **No transaction creation**: Gateway doesn't create or sign transactions -- **Chunk-only uploads**: Only accepts chunk data, not full transactions -- **Broadcast relay model**: Acts as relay to Arweave nodes -- **Admin-only data ingestion**: Direct data queuing requires authentication - -## 4. Data Verification System - -### DataVerificationWorker (`src/workers/data-verification.ts`) -- **Background Processing**: Runs periodically to verify data integrity -- **Queue Management**: Uses `fastq` library for concurrent processing -- **Verification Process**: - 1. Queries for unverified data IDs - 2. Maps data IDs to root transaction IDs - 3. Computes Merkle data roots using `DataRootComputer` - 4. Compares against indexed data roots - -### DataRootComputer (`src/lib/data-root.ts`) -- **Worker Threads**: Uses Node.js worker threads for CPU-intensive operations -- **Stream Processing**: Handles large data streams efficiently -- **Merkle Computation**: - - Chunks data per Arweave algorithm (256KB max, configurable min) - - Computes SHA-256 for each chunk - - Builds Merkle tree using Arweave's algorithm - - Returns base64URL-encoded root - -### Verification Database Schema -```sql --- Verification tracking with retry intelligence -verification_retry_count INTEGER, -verification_priority INTEGER, -first_verification_attempted_at INTEGER, -last_verification_attempted_at INTEGER, -verified BOOLEAN DEFAULT FALSE -``` - -### Self-Healing Architecture -- **Verification Failure**: Triggers data re-import from network -- **Missing Data Root**: Triggers bundle unbundling -- **Priority System**: Failed items get higher priority for retry -- **Retry Intelligence**: Exponential backoff prevents infinite loops - -## 5. Database Architecture - -### Four SQLite Database System -1. **Core Database** (`data/sqlite/core.db`): - - Blocks, transactions, tags, wallets - - Block/transaction indexing and relationships - - Missing transaction tracking - -2. **Data Database** (`data/sqlite/data.db`): - - Contiguous data storage metadata - - Hash-to-content mapping - - Data verification tracking with retry logic - - Parent-child data relationships - -3. **Bundles Database** (`data/sqlite/bundles.db`): - - ANS-104 bundle processing - - Data item indexing and relationships - - Bundle processing state tracking - -4. **Moderation Database** (`data/sqlite/moderation.db`): - - Content blocking and filtering - - Name blocklists - -### Storage Systems -- **Filesystem Storage**: - - `data/contiguous/` - Cached contiguous data - - `data/chunks/` - Transaction chunks - - `data/headers/` - Block/transaction headers -- **S3 Storage**: Alternative cloud storage backend -- **Redis Cache**: ArNS resolution and registry caching - -## 6. Background Worker Architecture - -### Block Synchronization -- **BlockImporter**: Syncs blocks, handles forks, manages missing transactions - -### Bundle Processing -- **Ans104Unbundler**: Processes ANS-104 bundles with configurable filters -- **DataItemIndexer**: Indexes individual data items -- **Ans104DataIndexer**: Handles nested bundle indexing - -### Data Management -- **DataImporter**: Downloads data for verification with priority queues -- **DataVerificationWorker**: Verifies integrity with retry intelligence - -### Maintenance Workers -- **TransactionFetcher**: Retrieves missing transactions -- **TransactionRepairWorker**: Repairs incomplete transaction data -- **BundleRepairWorker**: Repairs bundle processing failures -- **FsCleanupWorker**: Manages cache cleanup based on access patterns -- **SQLiteWalCleanupWorker**: Maintains database performance - -### Monitoring and Integration -- **MempoolWatcher**: Monitors mempool for pending transactions -- **WebhookEmitter**: Sends notifications for indexed data -- **ParquetExporter**: Exports data for analytics - -## 7. Key Architectural Patterns - -### Event-Driven Processing -- **Loose Coupling**: Components communicate through EventEmitter -- **Processing Pipeline**: `BLOCK_TX_INDEXED` → `TX_INDEXED` → Bundle processing -- **Flexible Architecture**: Easy addition of new functionality - -### Hierarchical Fallback System -- **Multi-Source Retrieval**: Cascades through sources until successful -- **Performance Optimization**: Fastest sources tried first -- **Resilience**: Multiple fallbacks ensure high availability - -### Retry and Recovery Systems -- **Comprehensive Retry Logic**: Exponential backoff with priority systems -- **Self-Healing**: Automatic data repair through re-fetching -- **State Preservation**: Maintains retry history for intelligence - -### Configurable Filtering -- **ANS104_UNBUNDLE_FILTER**: Controls which bundles are processed -- **ANS104_INDEX_FILTER**: Controls which bundles are indexed -- **WEBHOOK_INDEX_FILTER**: Controls webhook emissions -- **Resource Optimization**: Operators control processing scope - -## 8. Trust and Security Model - -### Trust Hierarchy -1. **Verified + Cached**: Highest trust - cryptographically verified local data -2. **Cached (Trusted)**: Data from trusted sources stored locally -3. **Network Stream**: Lower trust - data streamed from remote sources - -### Verification Headers -- `X-AR-IO-Verified`: Data cryptographically verified -- `X-AR-IO-Trusted`: Data from trusted source -- `X-Cache`: Cache hit/miss status -- `X-AR-IO-Stable`: Data before fork depth - -### Security Features -- **Hash Validation**: All cached data validated against content hash -- **Merkle Proof Validation**: Chunks validated using Arweave proofs -- **Circuit Breakers**: Prevent cascade failures -- **Admin API Protection**: Requires authentication for sensitive operations - -## 9. Performance Optimizations - -### Caching Strategy -- **Multi-Level Caching**: Memory, filesystem, and S3 storage -- **Intelligent Prefetching**: Based on access patterns and ArNS preferences -- **Cache Cleanup**: Automatic cleanup based on access frequency - -### Concurrent Processing -- **Worker Pools**: Configurable concurrency for all background operations -- **Queue Management**: Backpressure handling and priority processing -- **Resource Management**: Memory limits and worker count optimization - -### Network Optimization -- **Weighted Peer Selection**: Performance-based peer weighting -- **Request Queuing**: Rate limiting to prevent node overwhelming -- **Streaming Data**: Avoids memory overhead for large files - -## Conclusion - -The AR.IO Gateway represents a sophisticated architecture that transforms the Arweave network into a performant, reliable data access layer. Its design prioritizes read performance through intelligent caching while maintaining data integrity through comprehensive verification systems. The event-driven, worker-based architecture provides excellent scalability and maintainability, making it well-suited for production deployment as a "Permaweb CDN." - -The gateway's strength lies in its ability to provide fast access to Arweave data while maintaining cryptographic guarantees of data integrity, making it an ideal bridge between the decentralized Arweave network and traditional web applications requiring performant data access. \ No newline at end of file diff --git a/docs/drafts/ar-io-02-data-retrieval-complete-guide.md b/docs/drafts/ar-io-02-data-retrieval-complete-guide.md deleted file mode 100644 index 9d2828c8d..000000000 --- a/docs/drafts/ar-io-02-data-retrieval-complete-guide.md +++ /dev/null @@ -1,1057 +0,0 @@ -# AR.IO Gateway Data Retrieval: Complete Technical Guide - -## Table of Contents - -1. [Executive Summary](#executive-summary) -2. [Multi-Tier Data Fetching Architecture](#multi-tier-architecture) -3. [Header Fetching Mechanisms](#header-fetching) -4. [Chunk Retrieval and Validation Systems](#chunk-systems) -5. [Contiguous Data Fetching and Assembly](#contiguous-data) -6. [Peer Discovery and Trust Management](#peer-management) -7. [Data Source Prioritization and Fallback](#source-prioritization) -8. [Performance Optimizations](#performance-optimizations) -9. [Error Handling and Recovery](#error-handling) -10. [Data Integrity Guarantees](#data-integrity) -11. [Metrics and Monitoring](#metrics-monitoring) -12. [Configuration Reference](#configuration-reference) -13. [Implementation Details](#implementation-details) -14. [Certainty Assessment](#certainty-assessment) - ---- - -## Executive Summary {#executive-summary} - -The ar.io gateway implements a sophisticated multi-tier data fetching architecture that transforms the decentralized Arweave network into a performant, reliable data access layer. The system employs intelligent fallback mechanisms, cryptographic validation, and adaptive peer management to ensure high availability while maintaining data integrity. - -### Key Achievements - -- **Multi-layered fallback systems** ensuring high availability -- **Intelligent caching strategies** optimizing speed and storage costs -- **Dynamic peer management** adapting to network conditions -- **Comprehensive validation** maintaining data integrity -- **Circuit breaker patterns** preventing cascading failures - -**Certainty**: 100% - Based on comprehensive code analysis - ---- - -## Multi-Tier Data Fetching Architecture {#multi-tier-architecture} - -### Data Source Hierarchy - -The gateway implements two distinct retrieval orders configured via environment variables: - -#### On-Demand Retrieval (User-Facing) -- **Configuration**: `ON_DEMAND_RETRIEVAL_ORDER` (src/config.ts:296) -- **Default Order**: `'trusted-gateways,ar-io-network,chunks-data-item,tx-data'` -- **Implementation**: `SequentialDataSource` (src/data/sequential-data-source.ts) -- **Purpose**: Optimized for latency and user experience -- **Certainty**: 100% - Directly observed in code - -#### Background Retrieval (Verification/Import) -- **Configuration**: `BACKGROUND_RETRIEVAL_ORDER` (src/config.ts:304) -- **Default Order**: `'chunks'` -- **Purpose**: Used by DataImporter and verification workers -- **Focus**: Data integrity over speed -- **Certainty**: 100% - Explicitly documented in code - -### Sequential Fallback Pattern - -```typescript -// src/data/sequential-data-source.ts:35-51 -async getData({ id, dataAttributes, requestAttributes, region }) { - for (const dataSource of this.dataSources) { - try { - const data = await dataSource.getData({ - id, - dataAttributes, - requestAttributes, - region, - }); - return data; // Success - return immediately - } catch (error: any) { - this.log.warn('Unable to fetch data from data source', { - id, - message: error.message, - }); - } - } - throw new Error('Unable to fetch data from any data source'); -} -``` -**Certainty**: 100% - Core implementation pattern - -### Data Source Types - -1. **S3DataSource** - Cloud storage integration - - Direct S3 bucket access for pre-cached data - - Supports range requests via native S3 APIs - - No verification needed (pre-verified) - - **Certainty**: 100% - -2. **GatewaysDataSource** - Trusted gateway network - - Priority-based gateway selection - - Load distribution within tiers - - HTTP range request support - - **Certainty**: 100% - -3. **TxChunksDataSource** - Chunk assembly - - Direct reconstruction from Arweave chunks - - Most reliable but higher latency - - Always produces verified data - - **Certainty**: 100% - -4. **ArweaveCompositeClient** - Direct node access - - Transaction data from chain - - Falls back to peer network - - **Certainty**: 100% - -5. **ArIODataSource** - AR.IO peer network - - Dynamic peer discovery - - Performance-based selection - - Hash validation required - - **Certainty**: 100% - ---- - -## Header Fetching Mechanisms {#header-fetching} - -### ArweaveCompositeClient Architecture - -The `ArweaveCompositeClient` is the central orchestrator for all header fetching operations, managing a sophisticated multi-tier fetching strategy. - -#### Core Components (src/arweave/composite-client.ts) -- **Request Queue**: `fastq.promise` with configurable concurrency (line 199) - - Default: 100 concurrent requests - - **Certainty**: 100% -- **Rate Limiter**: Token bucket algorithm (lines 1004-1015) - - Default: 5 requests/second - - Fills continuously at configured rate - - **Certainty**: 100% -- **Promise Cache**: NodeCache for in-flight deduplication (lines 142-149) - - Block TTL: 30 seconds - - Transaction TTL: 60 seconds - - **Certainty**: 100% - -#### Request Management -```typescript -// Token bucket implementation for rate limiting -private trustedNodeRequestBucket = 0; -// Fills at maxRequestsPerSecond rate -setInterval(() => { - if (this.trustedNodeRequestBucket <= maxRequestsPerSecond * 300) { - this.trustedNodeRequestBucket += maxRequestsPerSecond; - } -}, 1000); -``` -**Certainty**: 100% - Rate limiting implementation - -### Header Caching Architecture - -Multi-layer caching strategy with three distinct levels: - -#### 1. In-Memory Promise Cache -- TTL: 30 seconds for blocks, 60 seconds for transactions -- Prevents duplicate in-flight requests -- Weak references to prevent memory leaks -- **Certainty**: 100% - -#### 2. Filesystem Cache (FsBlockStore) -- **Directory Structure**: `hash/{first2chars}/{next2chars}/{hash}.msgpack` -- **Height Symlinks**: `height/{height%1000}/{height}.msgpack` -- **Encoding**: MessagePack for efficiency -- **Atomic Writes**: Uses temp file + rename pattern -- **Certainty**: 100% - Implementation verified - -#### 3. KV Store Cache (KvBlockStore) -- **Key Patterns**: - - Blocks: `B#|{hash}` - - Height mappings: `BH|{height}` -- **Certainty**: 100% - Clear implementation - -### Trusted vs Untrusted Source Fetching - -#### Trusted Node Fetching -- Always uses the configured trusted node URL -- Automatic retry with exponential backoff (default: 5 retries) -- 429 rate limit responses decrease token bucket exponentially -- All requests go through the central request queue -- **Certainty**: 100% - -#### Peer Fetching -- **Peer Selection**: Weighted random selection based on success/failure history -- **Weight Adjustment**: - ```typescript - // Success: increase weight by WEIGHTED_PEERS_TEMPERATURE_DELTA (default: 2) - weightedPeer.weight = Math.min(weight + delta, 100); - // Failure: decrease weight - weightedPeer.weight = Math.max(weight - delta, 1); - ``` -- **Parallel Attempts**: For transactions, tries 3 peers in parallel using `Promise.any()` -- **Validation**: Peer-fetched transactions are verified using Arweave SDK -- **Certainty**: 100% - Direct code observation - -### Block Header Synchronization - -The `BlockImporter` handles sequential block processing with sophisticated fork detection. - -#### Fork Detection Algorithm (src/workers/block-importer.ts:121-139) -```typescript -const previousDbBlockHash = await chainIndex.getBlockHashByHeight(height - 1); - -if (previousDbBlockHash === undefined) { - // Gap detected - chainIndex.resetToHeight(previousHeight - 1); - return getBlockOrForkedBlock(previousHeight, forkDepth + 1); -} else if (block.previous_block !== previousDbBlockHash) { - // Fork detected - metrics.forksCounter.inc(); - chainIndex.resetToHeight(previousHeight - 1); - return getBlockOrForkedBlock(previousHeight, forkDepth + 1); -} -``` -- **Maximum Fork Depth**: 18 blocks (MAX_FORK_DEPTH) -- **Certainty**: 100% - Well-documented pattern - -### Transaction Header Fetching - -#### Prefetch Strategy -1. Transactions are prefetched when their containing block is fetched -2. Configurable prefetch depth (default: 1 block of transactions) -3. Failed prefetches return `undefined` rather than throwing -4. **Certainty**: 100% - -#### Caching Mechanism -- Same multi-layer approach as blocks -- Filesystem: `{first2chars}/{next2chars}/{txid}.msgpack` -- Data payload stripped to minimize memory usage -- ECDSA public key recovery for empty owner fields -- **Certainty**: 100% - -### Header Validation - -#### Block Validation (`sanityCheckBlock`) -- Validates `indep_hash` format (64-char base64url) -- Ensures `height` is a number -- Checks `previous_block` exists for non-genesis blocks -- **Certainty**: 100% - -#### Transaction Validation (`sanityCheckTx`) -- Validates transaction ID format (43-char base64url) -- Ensures required fields exist -- Additional Arweave SDK verification for peer-fetched txs -- **Certainty**: 100% - ---- - -## Chunk Retrieval and Validation Systems {#chunk-systems} - -### Chunk Fetching Strategies and Algorithms - -The ArweaveCompositeClient implements sophisticated chunk fetching with multiple fallback strategies. - -#### Primary Strategy: Trusted Node First (src/arweave/composite-client.ts:944-991) -```typescript -async getChunkByAny({ - txSize, - absoluteOffset, - dataRoot, - relativeOffset, -}: ChunkDataByAnySourceParams): Promise -``` - -The algorithm follows this priority order: -1. **Trusted Node Request** - Uses a queue-based request system with rate limiting -2. **WeakMap Cache Check** - In-memory cache with 5-second TTL -3. **Peer Network Fallback** - Weighted peer selection with dynamic scoring -- **Certainty**: 100% - Sequential try-catch pattern observed - -#### Request Queue Management -- Uses `fastq` for concurrent request management -- Implements token bucket rate limiting (5 requests/second default) -- Maximum concurrent requests: 100 (configurable) -- Request timeout: 15 seconds default -- Retry count: 5 attempts with exponential backoff -- **Certainty**: 100% - -#### Peer Selection Algorithm -```typescript -// Weighted random selection -randomWeightedChoices({ - table: this[peerListName], - count: peerCount, -}); -``` -- Peers maintain weights from 1-100 -- Success increases weight by `WEIGHTED_PEERS_TEMPERATURE_DELTA` (default: 2) -- Failure decreases weight by same delta -- Minimum weight: 1 (never fully excluded) -- **Certainty**: 100% - -### Chunk Validation Using Merkle Proofs - -The validation process is rigorous and multi-layered. - -#### Hash Validation (src/lib/validation.ts:79-95) -```typescript -const chunkHash = crypto.createHash('sha256').update(chunk.chunk).digest(); -if (!chunkHash.equals(chunk.data_path.slice(-64, -32))) { - throw new Error('Invalid chunk: hash does not match data_path'); -} -``` - -#### Merkle Path Validation -```typescript -const validChunk = await validatePath( - dataRoot, // Transaction's data root - relativeOffset, // Position within transaction - 0, // Start offset - txSize, // Total transaction size - chunk.data_path // Merkle proof path -); -``` -- **Certainty**: 100% - Critical validation logic - -The validation ensures: -- Chunk hash matches the leaf node in Merkle tree -- Merkle path proves chunk belongs to the transaction -- Data root verification against transaction metadata - -### Chunk Assembly Process - -The TxChunksDataSource implements streaming chunk assembly. - -#### Sequential Streaming Assembly (src/data/tx-chunks-data-source.ts:83-109) -```typescript -const stream = new Readable({ - read: async function () { - const chunkData = await chunkDataPromise; - this.push(chunkData.chunk); - bytes += chunkData.chunk.length; - - if (bytes < size) { - chunkDataPromise = getChunkDataByAny( - startOffset + bytes, - txDataRoot, - bytes, - ); - } - } -}); -``` - -Key aspects: -- Chunks are fetched sequentially as needed -- Stream-based approach minimizes memory usage -- Automatic prefetching of next chunk while current streams -- **Certainty**: 100% - Core data retrieval mechanism - -### Circuit Breaker Protection - -Configuration (src/arweave/composite-client.ts:240-265): - -```typescript -// Primary endpoints -new CircuitBreaker(postChunk, { - errorThresholdPercentage: 50, - resetTimeout: 5000, - capacity: 100 -}); -``` - -Circuit breakers prevent cascading failures: -- **Primary Endpoints**: 50% error threshold, 5s reset -- **Secondary Endpoints**: 50% error threshold, 10s reset -- **Peer Endpoints**: 50% error threshold, 10s reset -- Opens after error threshold reached -- **Certainty**: 100% - Opossum circuit breaker configurations - -### Chunk Caching Strategies - -Multi-layered caching approach: - -#### In-Memory WeakMap Cache -```typescript -private chunkCache: WeakMap< - { absoluteOffset: number }, - { cachedAt: number; chunk: Chunk } ->; -``` -- 5-second TTL for hot chunks -- Automatic garbage collection -- No memory pressure concerns -- **Certainty**: 100% - -#### File System Cache -Storage structure: -``` -data/ -├── by-dataroot/ -│ └── {prefix}/{dataRoot}/{relativeOffset} -└── by-hash/ - └── {prefix}/{hash} -``` -- Stores chunks by data root and offset -- Uses symlinks to deduplicate identical chunks -- **Certainty**: 100% - -#### Read-Through Cache Pattern -- Check cache first -- Fetch from source on miss -- Populate cache asynchronously -- Handles concurrent requests for same chunk -- **Certainty**: 100% - -### Chunk Broadcasting - -Multi-tier broadcast strategy: - -```typescript -// Broadcast order: -1. Primary nodes (required: CHUNK_POST_MIN_SUCCESS_COUNT) -2. Secondary nodes (optional: SECONDARY_CHUNK_POST_MIN_SUCCESS_COUNT) -3. Peer network (background: ARWEAVE_PEER_CHUNK_POST_MIN_SUCCESS_COUNT) -``` - -#### Headers Propagation -```typescript -// Origin tracking -{ - 'X-AR-IO-Hops': req.headers['x-ar-io-hops'], - 'X-AR-IO-Origin': req.headers['x-ar-io-origin'] -} -``` - -#### Success Criteria -- Primary: Minimum 3 successful posts (configurable) -- Returns immediately after threshold -- Background propagation continues -- **Certainty**: 100% - ---- - -## Contiguous Data Fetching and Assembly {#contiguous-data} - -### ContiguousDataSource Hierarchy and Orchestration - -The AR.IO node implements a sophisticated hierarchy of data sources through the `ContiguousDataSource` interface: - -```typescript -export interface ContiguousDataSource { - getData({ - id, - dataAttributes, - requestAttributes, - region, - }: { - id: string; - dataAttributes?: ContiguousDataAttributes; - requestAttributes?: RequestAttributes; - region?: Region; - }): Promise; -} -``` - -The hierarchy consists of: -- **ReadThroughDataCache** - Top-level caching layer -- **SequentialDataSource** - Fallback orchestrator -- **TxChunksDataSource** - Chunk-based assembly from Arweave -- **S3DataSource** - Cloud storage integration -- **GatewaysDataSource** - Trusted gateway fetching -- **ArIODataSource** - AR.IO peer network fetching - -**Certainty**: 100% - Interface definitions - -### ReadThroughDataCache Caching Strategies - -The `ReadThroughDataCache` implements sophisticated caching: - -#### Cache Strategy (src/data/read-through-data-cache.ts) -1. Direct hash lookup in ContiguousDataIndex (line 229) -2. Parent data traversal for nested items (lines 255-280) -3. Falls back to underlying data sources (line 319) -- **Certainty**: 100% - Step-by-step implementation - -#### Cache Population Rules -- Only caches trusted or hash-verified data (lines 371-380) -- Streams through SHA-256 hasher during write (line 352) -- Atomic operations with cleanup on failure -- **Certainty**: 100% - Explicit conditions in code - -#### Metadata Tracking -- Tracks MRU (Most Recently Used) ArNS names -- Records access timestamps -- Maintains verification priorities for preferred ArNS names -- **Certainty**: 100% - -### Contiguous Data Assembly - -Data can be assembled in two primary ways: - -#### A. Chunk-based Assembly (TxChunksDataSource) -```typescript -const stream = new Readable({ - read: async function () { - const chunkData = await chunkDataPromise; - this.push(chunkData.chunk); - bytes += chunkData.chunk.length; - - if (bytes < size) { - chunkDataPromise = getChunkDataByAny( - startOffset + bytes, - txDataRoot, - bytes, - ); - } - } -}); -``` - -#### B. Direct Fetching -- From S3 with metadata-aware range requests -- From trusted gateways with priority-based failover -- From AR.IO peers with weighted selection -- **Certainty**: 100% - -### S3DataSource Cloud Integration - -The S3DataSource implements sophisticated cloud storage handling. - -#### Metadata Handling (src/data/s3-data-source.ts:89-124) -- `x-amz-meta-payload-data-start`: Data offset in file -- `x-amz-meta-payload-content-type`: Content type override -- Supports native S3 range requests -- **Certainty**: 100% - S3 metadata tags documented - -#### Range Request Logic -```typescript -let range = 'bytes=0-'; -if (region) { - range = `bytes=${region.offset}-${region.offset + region.size - 1}`; -} else if (head.headers?.[payloadDataStartS3MetaDataTag] !== undefined) { - range = `bytes=${head.headers[payloadDataStartS3MetaDataTag]}-`; -} -``` -**Certainty**: 100% - -### GatewaysDataSource - -The `GatewaysDataSource` implements: - -#### Priority-Based Selection (src/data/gateways-data-source.ts:58-85) -- Groups gateways by priority tier -- Shuffles within tier for load distribution -- Falls back to lower priority on failure -- **Certainty**: 100% - Clear implementation pattern - -#### Response Validation -- Verifies status codes (200 for full, 206 for range requests) -- Tracks request attributes through hop counting -- Maintains request timeout controls -- **Certainty**: 100% - -### Large File Streaming and Range Requests - -#### ByteRangeTransform Stream (src/lib/byte-range-transform-stream.ts) -```typescript -_transform(chunk: Buffer, _, callback: TransformCallback): void { - const chunkStart = Math.max(0, this.offset - this.bytesRead); - const chunkEnd = Math.min( - chunk.length, - this.offset + this.size - this.bytesRead, - ); - - if (chunkStart < chunkEnd) { - const slicedChunk = chunk.slice(chunkStart, chunkEnd); - this.bytesWritten += slicedChunk.length; - this.push(slicedChunk); - } -} -``` - -#### Stream Management -- All streams use `setMaxListeners(Infinity)` to prevent warnings -- Implements proper error handling with metric tracking -- Supports partial content delivery through range transformations -- **Certainty**: 100% - Stream implementation verified - ---- - -## Peer Discovery and Trust Management {#peer-management} - -### The Peer Discovery Process - -ar.io nodes discover peers through two distinct mechanisms: - -#### a) Arweave Network Peers (ArweaveCompositeClient) -Discovery details (src/arweave/composite-client.ts:697-741): -- Discovered via the trusted node's `/peers` endpoint -- Refreshed every 10 minutes via `setInterval(() => this.refreshPeers(), 10 * 60 * 1000)` -- Returns peer hosts as `string[]` in format `host:port` -- Each peer is probed at `/info` endpoint with a 5-second timeout -- Peer info includes: `blocks`, `height`, `lastSeen` timestamp -- Peers in `ARWEAVE_NODE_IGNORE_URLS` environment variable are filtered out -- **Certainty**: 100% - Timer and implementation verified - -#### b) ar.io Gateway Peers (ArIODataSource) -Discovery details (src/data/ar-io-data-source.ts:247-283): -- Discovered from the ar.io network process on Solana via `@ar.io/sdk` -- Uses paginated `getGateways()` API with 1000 items per page -- Refreshed every hour by default -- Filters out the node's own wallet address -- Constructs peer URLs as `${gateway.settings.protocol}://${gateway.settings.fqdn}` -- **Certainty**: 100% - Implementation matches description - -### The Weighted Peer Selection Algorithm - -The system uses a sophisticated weighted random selection algorithm with temperature-based adjustments. - -```typescript -type WeightedElement = { - id: T; - weight: number; // 1-100, default 50 -}; -``` - -#### Weight Calculation Process -1. Initial weight: 50 (neutral) -2. Temperature delta: ±2 points per success/failure (`WEIGHTED_PEERS_TEMPERATURE_DELTA`) -3. Weight bounds: [1, 100] -4. Temperature parameter adjusts selection probability distribution - -#### Weight Adjustment (src/arweave/composite-client.ts:495-502) -```typescript -// Success -weightedPeer.weight = Math.min( - weight + config.WEIGHTED_PEERS_TEMPERATURE_DELTA, - 100 -); -// Failure -weightedPeer.weight = Math.max( - weight - config.WEIGHTED_PEERS_TEMPERATURE_DELTA, - 1 -); -``` -- Default delta: 2 points (configurable) -- Weight range: [1, 100] -- **Certainty**: 100% - Direct code observation - -#### The Algorithm (randomWeightedChoices) -- Converts temperature (0-100) to T (-1 to +1) -- Calculates urgency for each peer: `urgency = weight + T * influence * (avg - weight)` -- Higher temperature favors lower-weighted peers (exploration) -- Lower temperature favors higher-weighted peers (exploitation) -- Uses cumulative urgency sum for roulette wheel selection -- **Certainty**: 100% - -### Trust Relationship Establishment - -Trust is established through different mechanisms: - -#### For Arweave Peers -- Three separate weighted peer lists: - - `weightedChainPeers` - for transaction/block data - - `weightedGetChunkPeers` - for chunk retrieval - - `weightedPostChunkPeers` - for chunk broadcasting -- Trust validation for transactions: `arweave.transactions.verify(tx)` -- Chunk validation: `validateChunk()` with merkle proof verification -- **Certainty**: 100% - -#### For ar.io Peers -Trust headers (src/routes/data/handlers.ts:162-168): -- `x-ar-io-verified: true/false` - Cryptographic verification status -- `x-ar-io-trusted: true/false` - Source trust status -- `x-ar-io-digest` - Expected hash for validation -- **Certainty**: 100% - Header names in constants - -Validation Requirements: -- AR.IO peers must indicate verified OR trusted (src/data/ar-io-data-source.ts:183-189) -- Digest must match if provided (line 192-198) -- **Certainty**: 100% - Explicit validation logic - -### Difference Between ar.io Peers and Regular Arweave Nodes - -#### ar.io Peers -- Registered in the ar.io network contract -- Have FQDN and protocol settings -- Support ar.io-specific headers (verified, trusted, digest) -- Accessed via `/raw/{id}` endpoints -- Performance tracked with TTFB and download rate - -#### Regular Arweave Nodes -- Discovered from Arweave network peers list -- Support standard Arweave APIs (/chunk, /tx, /info) -- No ar.io-specific features -- Used primarily for chunk and transaction data - -**Certainty**: 100% - -### Peer Performance Measurement - -#### Metrics Tracked -- **Success/failure counts** via Prometheus counters -- **Time to First Byte (TTFB)** for ar.io peers -- **Download rate (Kbps)** for ar.io peers -- **Request duration** and error rates - -#### Performance Windows -- ar.io peers: sliding window of 20 requests (`GATEWAY_PEERS_REQUEST_WINDOW_COUNT`) -- Weight adjustments based on comparison to average TTFB and Kbps - -#### AR.IO Peer Performance Tracking (src/data/ar-io-data-source.ts:373-391) -- Tracks TTFB and download rate -- Sliding window: 20 requests -- Additional weight for better-than-average performance -- **Certainty**: 100% - Performance calculation verified - -### Circuit Breaker Implementations - -Three types of circuit breakers protect peer operations: - -#### Primary Chunk POST Circuit Breakers -```typescript -{ - name: `primaryBroadcastChunk-${url}`, - capacity: 100, - resetTimeout: 5000, - errorThresholdPercentage: 50 -} -``` - -#### Secondary Chunk POST Circuit Breakers -```typescript -{ - name: `secondaryBroadcastChunk-${url}`, - capacity: 10, - resetTimeout: 10000, - errorThresholdPercentage: 50 -} -``` - -#### ar.io Gateway Circuit Breaker (src/data/ar-io-data-source.ts:86-92) -```typescript -{ - timeout: 60000, // 60 seconds - errorThresholdPercentage: 30, - rollingCountTimeout: 600000, // 10 minutes - resetTimeout: 1200000 // 20 minutes -} -``` -**Certainty**: 100% - Exact configuration values - ---- - -## Data Source Prioritization and Fallback {#source-prioritization} - -### Complete Hierarchy of Data Sources and Priority Orders - -The ar.io node implements a sophisticated multi-tier data source hierarchy with two distinct retrieval contexts. - -#### How ON_DEMAND_RETRIEVAL_ORDER and BACKGROUND_RETRIEVAL_ORDER Work - -The system uses **SequentialDataSource** as the orchestrator: - -```typescript -// In system.ts -const onDemandDataSources: ContiguousDataSource[] = []; -for (const sourceName of config.ON_DEMAND_RETRIEVAL_ORDER) { - const dataSource = getDataSource(sourceName); - if (dataSource !== undefined) { - onDemandDataSources.push(dataSource); - } -} - -// SequentialDataSource implementation -async getData({ id, dataAttributes, requestAttributes, region }) { - for (const dataSource of this.dataSources) { - try { - const data = await dataSource.getData({ - id, - dataAttributes, - requestAttributes, - region, - }); - return data; // Success - return immediately - } catch (error: any) { - // Log and continue to next source - this.log.warn('Unable to fetch data from data source', { - id, - message: error.message, - stack: error.stack, - }); - } - } - throw new Error('Unable to fetch data from any data source'); -} -``` -**Certainty**: 100% - -### Decision-Making Process for Source Selection - -The selection process follows a strict waterfall pattern with intelligent optimizations: - -#### Phase 1: Cache Check (ReadThroughDataCache) -1. Check local cache via hash lookup in ContiguousDataIndex -2. If hash exists, attempt retrieval from local storage (FS or S3) -3. Check for parent data relationships (nested data items) -4. Update MRU (Most Recently Used) ArNS name tracking - -#### Phase 2: Sequential Source Attempts -1. Each source is tried in configured order -2. First successful response terminates the sequence -3. Failures are logged but don't halt the process -4. Final failure throws after all sources exhausted - -#### Phase 3: Response Processing -- Trusted sources: Data cached immediately -- Untrusted sources: Hash validation required -- Verification priority calculated based on ArNS preferences - -**Certainty**: 100% - -### Cost/Performance Optimization Strategies - -#### S3 First Strategy -- S3 positioned first in on-demand order -- Lowest latency for cached content -- Predictable costs and performance - -#### Weighted Peer Selection -- Performance metrics tracked: - - TTFB (Time To First Byte) - - Download rate (Kbps) -- Weight adjustments: - ```typescript - const additionalWeightFromTtfb = - ttfb > currentAverageTtfb ? 0 : config.WEIGHTED_PEERS_TEMPERATURE_DELTA; - const additionalWeightFromKbps = - kbps <= currentAverageKbps ? 0 : config.WEIGHTED_PEERS_TEMPERATURE_DELTA; - ``` - -#### Memoized Peer Selection -- Random weighted choices cached for 5 seconds -- Reduces computation overhead -- Balances load distribution with performance - -**Certainty**: 100% - -### Balancing Speed, Cost, and Reliability - -#### Speed Optimizations -1. **Parallel Chunk Fetching**: Chunks pre-fetched for streaming -2. **Connection Reuse**: Axios instances per gateway -3. **Caching**: Multi-level cache hierarchy -4. **Memoization**: Peer selection results cached - -#### Cost Management -1. **S3 First**: Leverages pre-cached data -2. **Peer Rotation**: Distributes load across network -3. **Circuit Breakers**: Prevents wasteful retries -4. **Selective Caching**: Only verified/trusted data - -#### Reliability Features -1. **Multiple Fallback Layers**: 5+ sources in chain -2. **Health-Based Selection**: Dynamic weight adjustments -3. **Hash Verification**: Ensures data integrity -4. **Parent Data Resolution**: Handles nested data items - -**Certainty**: 100% - ---- - -## Performance Optimizations {#performance-optimizations} - -### Request Management - -#### Queue Configuration (src/arweave/composite-client.ts:199) -- Maximum concurrent requests: 100 (configurable) -- Request timeout: 15 seconds default -- Retry count: 5 with exponential backoff -- **Certainty**: 100% - Configuration values verified - -### Caching Strategies - -#### Multi-Level Caching -1. **In-Memory**: Promise cache for in-flight requests -2. **WeakMap**: 5-second TTL for hot chunks -3. **Filesystem**: MessagePack encoded headers/chunks -4. **S3**: Optional cloud storage backend -- **Certainty**: 100% - All caching layers identified - -### Stream Processing - -#### Memory Efficiency (src/lib/byte-range-transform-stream.ts) -- Transforms streams for range requests -- No full data buffering -- `setMaxListeners(Infinity)` to prevent warnings -- **Certainty**: 100% - Stream implementation verified - ---- - -## Error Handling and Recovery {#error-handling} - -### Circuit Breaker Configurations - -Circuit breakers prevent cascade failures with configurable thresholds: -- **Timeout**: Request timeout before circuit opens -- **Error Threshold**: Percentage of errors triggering open state -- **Reset Timeout**: Time before attempting recovery -- **Capacity**: Maximum requests in rolling window - -**Certainty**: 100% - -### Retry Mechanisms - -#### Verification Retry System (database schema) -- `verification_retry_count`: Tracks attempts -- `verification_priority`: Higher = sooner retry -- `first/last_verification_attempted_at`: Timestamps -- Ordering: priority DESC, retry_count ASC, id ASC -- **Certainty**: 100% - Database schema and index verified - ---- - -## Data Integrity Guarantees {#data-integrity} - -### Hash Validation Flow - -1. **Expected hash** passed in dataAttributes -2. **Streaming SHA-256** during retrieval (src/data/read-through-data-cache.ts:352) -3. **Post-retrieval validation** before caching -4. **Cache population** only on match -- **Certainty**: 100% - Complete flow traced - -### Verification System - -#### DataVerificationWorker (src/workers/data-verification.ts) -- Computes Merkle roots using worker threads -- Compares against indexed data roots -- Triggers re-import on mismatch -- **Certainty**: 100% - Worker implementation verified - ---- - -## Metrics and Monitoring {#metrics-monitoring} - -### Performance Metrics - -#### Tracked Metrics (src/metrics.ts) -- `arweave_peer_info_errors_total` -- `arweave_peer_refresh_errors_total` -- `get_data_stream_successes_total` (labeled by source) -- `get_data_stream_errors_total` (labeled by source) -- **Certainty**: 100% - Prometheus metrics verified - -### Request Tracking - -#### Request Attributes (src/types.d.ts:RequestAttributes) -- `hops`: Gateway hop count -- `origin`: Original gateway -- `arnsName`: ArNS resolution tracking -- **Certainty**: 100% - Interface definition confirmed - ---- - -## Configuration Reference {#configuration-reference} - -### Key Environment Variables - -#### Data Retrieval -| Variable | Default | Description | -|----------|---------|-------------| -| `ON_DEMAND_RETRIEVAL_ORDER` | `trusted-gateways,ar-io-network,chunks-data-item,tx-data` | User request sources | -| `BACKGROUND_RETRIEVAL_ORDER` | `chunks` | Background processing sources | -| `TRUSTED_NODE_URL` | `https://arweave.net` | Primary Arweave node | -| `TRUSTED_GATEWAYS_URLS` | - | JSON gateway map | - -#### Performance Tuning -| Variable | Default | Description | -|----------|---------|-------------| -| `WEIGHTED_PEERS_TEMPERATURE_DELTA` | `2` | Weight adjustment rate | -| `CHUNK_POST_MIN_SUCCESS_COUNT` | `3` | Minimum broadcasts | -| `GATEWAY_PEERS_REQUEST_WINDOW_COUNT` | `20` | Performance window | - -**Certainty**: 100% - All from envs.md and config.ts - ---- - -## Implementation Details {#implementation-details} - -### Request Processing Flow - -``` -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ Client │────▶│ Gateway │────▶│ Cache │────▶│ Sources │ -└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ - │ │ │ │ - │ 1. Request Data │ │ │ - │───────────────────▶│ │ │ - │ │ 2. Check Cache │ │ - │ │────────────────────▶│ │ - │ │◀ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┤ │ - │ │ (cache miss) │ │ - │ │ │ │ - │ │ 3. Sequential Fetch │ │ - │ │─────────────────────┼───────────────────▶│ - │ │ │ │ - │ │ │ ┌─────────────┐ │ - │ │ │ │ S3 │ │ - │ │ │ │ Gateways │ │ - │ │ │ │ Chunks │ │ - │ │ │ │ Peers │ │ - │ │ │ └─────────────┘ │ - │ │ │ │ - │ │◀────────────────────┼────────────────────┤ - │ │ 4. Update Cache │ │ - │ │────────────────────▶│ │ - │◀───────────────────│ │ │ - │ 5. Return Data │ │ │ - ▼ ▼ ▼ ▼ -``` - ---- - -## Certainty Assessment {#certainty-assessment} - -### Assessment Summary - -- **100% Certain**: All core mechanisms, implementations, and configurations -- **No Uncertainties**: All insights are backed by direct code observation -- **Code References**: Every claim has specific file and line references -- **Verification Method**: Direct code analysis and cross-referencing - -### Areas Verified - -1. **Source Files Analyzed**: - - All files in `src/arweave/` - - All files in `src/data/` - - All files in `src/lib/` - - Worker implementations in `src/workers/` - - Configuration in `src/config.ts` - -2. **Implementation Verified**: - - Complete data flow paths - - All caching mechanisms - - Error handling strategies - - Performance optimizations - -3. **Configuration Confirmed**: - - All environment variables - - Default values - - Integration points - ---- - -## Conclusion - -The ar.io gateway's data retrieval system represents a masterclass in distributed systems engineering. Through its sophisticated multi-tier architecture, intelligent caching strategies, and comprehensive error handling, it successfully transforms the decentralized Arweave network into an enterprise-ready data access layer. - -### Key Achievements - -1. **Enterprise-Grade Reliability**: Multi-tier fallbacks and circuit breakers ensure consistent availability -2. **Performance Excellence**: Intelligent caching and stream processing minimize latency -3. **Data Integrity**: Cryptographic validation at every layer maintains trust -4. **Operational Maturity**: Comprehensive metrics and monitoring enable proactive management -5. **Scalability**: Configurable concurrency and distributed architecture support growth - -This architecture demonstrates how thoughtful engineering can bridge the gap between decentralized storage principles and production application requirements, making the Arweave permaweb accessible at scale. \ No newline at end of file diff --git a/docs/drafts/ar-io-03-arweave-connectivity-complete-guide.md b/docs/drafts/ar-io-03-arweave-connectivity-complete-guide.md deleted file mode 100644 index 27d1f4c2e..000000000 --- a/docs/drafts/ar-io-03-arweave-connectivity-complete-guide.md +++ /dev/null @@ -1,1319 +0,0 @@ -# AR.IO Gateway - Arweave Network Connectivity: Complete Technical Documentation - -## Table of Contents - -1. [Executive Summary](#executive-summary) -2. [Core Integration Architecture](#core-integration-architecture) -3. [Network Communication Protocol](#network-communication-protocol) -4. [Blockchain Synchronization](#blockchain-synchronization) -5. [Chunk Management](#chunk-management) -6. [Peer Network Management](#peer-network-management) -7. [Data Format Compatibility](#data-format-compatibility) -8. [Error Handling and Resilience](#error-handling-resilience) -9. [Performance Optimizations](#performance-optimizations) -10. [Monitoring and Metrics](#monitoring-metrics) -11. [Protocol Evolution Support](#protocol-evolution) -12. [Technical Implementation Details](#technical-implementation) -13. [Configuration Reference](#configuration-reference) -14. [Conclusion](#conclusion) - ---- - -## Executive Summary {#executive-summary} - -The ar.io gateway integrates with the Arweave network through a sophisticated client architecture that manages blockchain synchronization, data retrieval, and chunk broadcasting. The integration is built around the `ArweaveCompositeClient` class, which orchestrates connections to both trusted nodes and a dynamic peer network while maintaining high availability through intelligent fallback mechanisms. - -### Key Integration Points - -- **Primary Interface**: ArweaveCompositeClient manages all Arweave network interactions -- **Protocol**: HTTP/HTTPS with automatic retry and rate limiting -- **Data Formats**: JSON with base64url encoding for binary data -- **Validation**: Cryptographic verification at every layer -- **Resilience**: Circuit breakers, fallback chains, and fork recovery - ---- - -## Core Integration Architecture {#core-integration-architecture} - -### ArweaveCompositeClient Overview - -The `ArweaveCompositeClient` (src/arweave/composite-client.ts) serves as the primary integration point between the ar.io gateway and the Arweave network. It manages: - -- **Trusted Node Connection**: A single, reliable Arweave node for primary operations -- **Peer Network**: Dynamically discovered and weighted peer nodes -- **Request Management**: Rate-limited queue system for network requests -- **Circuit Breakers**: Fault tolerance for network operations -- **Caching Layers**: Multiple cache levels for performance optimization - -### System Initialization - -**Code Reference** (src/system.ts:171-195): -```typescript -export const arweaveClient = new ArweaveCompositeClient({ - log, - arweave, - trustedNodeUrl: config.TRUSTED_NODE_URL, - chunkPostUrls: config.CHUNK_POST_URLS, - skipCache: config.SKIP_CACHE, - maxRequestsPerSecond: 5, - maxConcurrentRequests: 100, - requestTimeout: 15000, - requestRetryCount: 5, - blockPrefetchCount: 50, - blockStore: makeBlockStore({ - log, - type: config.CHAIN_CACHE_TYPE, - }), - txStore: makeTxStore({ - log, - type: config.CHAIN_CACHE_TYPE, - }), - failureSimulator: new UniformFailureSimulator({ - failureRate: config.SIMULATED_REQUEST_FAILURE_RATE, - }), -}); -``` - -### Component Responsibilities - -1. **Request Queue Management** - - Token bucket rate limiting (5 requests/second default) - - Concurrent request control (100 max default) - - Automatic retry with exponential backoff - -2. **Peer Network Management** - - Dynamic peer discovery and health monitoring - - Weighted selection based on performance - - Automatic blacklist filtering - -3. **Data Caching** - - In-memory promise cache for deduplication - - Persistent block/transaction storage - - Weak reference chunk caching - -4. **Circuit Breaking** - - Prevents cascade failures - - Configurable error thresholds - - Automatic recovery timers - ---- - -## Network Communication Protocol {#network-communication-protocol} - -### HTTP Client Configuration - -The gateway uses axios with retry-axios for reliable HTTP communication: - -**Trusted Node Client Configuration** (src/arweave/composite-client.ts:181-192): -```typescript -this.trustedNodeAxios = axios.create({ - baseURL: this.trustedNodeUrl, - timeout: this.requestTimeout, -}); - -axiosRetry(this.trustedNodeAxios, { - retries: this.requestRetryCount, - retryDelay: axiosRetry.exponentialDelay, - retryCondition: (error) => { - if (error?.response?.status === 429) { - // Rate limit response - reduce token bucket - this.trustedNodeRequestBucket = Math.floor( - this.trustedNodeRequestBucket * 0.99 - ); - } - return axiosRetry.isRetryableError(error); - }, -}); -``` - -### Arweave HTTP API Endpoints - -#### Blockchain Operations - -| Endpoint | Method | Purpose | Response Format | -|----------|---------|---------|-----------------| -| `/height` | GET | Current blockchain height | Number | -| `/block/height/{height}` | GET | Fetch block by height | JSON Block | -| `/block/current` | GET | Latest block information | JSON Block | -| `/tx/{txId}` | GET | Transaction details | JSON Transaction | -| `/tx/{txId}/offset` | GET | Transaction offset in weave | JSON Offset | -| `/tx/{txId}/status` | GET | Transaction confirmation status | JSON Status | -| `/tx/{txId}/field/{field}` | GET | Specific transaction field | Field Value | -| `/tx/{txId}/data` | GET | Full transaction data | Binary Data | -| `/tx/pending` | GET | Pending transaction IDs | Array of IDs | -| `/unconfirmed_tx/{txId}` | GET | Unconfirmed transaction details | JSON Transaction | - -#### Chunk Operations - -| Endpoint | Method | Purpose | Request/Response | -|----------|---------|---------|------------------| -| `/chunk/{offset}` | GET | Retrieve chunk by absolute offset | JSON Chunk | -| `/chunk` | POST | Broadcast chunk to network | JSON Chunk → Status | - -#### Network Operations - -| Endpoint | Method | Purpose | Response Format | -|----------|---------|---------|-----------------| -| `/peers` | GET | Discover network peers | Array of peer URLs | -| `/info` | GET | Node information and status | JSON Info | - -### Request Queue Management - -**Rate Limiting Implementation** (src/arweave/composite-client.ts:1004-1015): -```typescript -private trustedNodeRequestBucket = 0; - -// Token bucket refill mechanism -setInterval(() => { - const availableTokens = maxRequestsPerSecond * 300; // 5-minute capacity - if (this.trustedNodeRequestBucket <= availableTokens) { - this.trustedNodeRequestBucket += maxRequestsPerSecond; - } -}, 1000); - -// Request execution with rate limiting -private async trustedNodeRequest(request: QueuedRequest): Promise { - while (this.trustedNodeRequestBucket < 1) { - await wait(100); // Wait for token availability - } - this.trustedNodeRequestBucket--; - - const response = await this.trustedNodeAxios.request({ - method: request.method, - url: request.url, - data: request.data, - }); - - return response.data; -} -``` - -### Request Queuing Strategy - -1. **Queue Implementation**: Uses `fastq.promise` for efficient promise-based queuing -2. **Concurrency Control**: Maximum 100 concurrent requests (configurable) -3. **Priority**: FIFO with no explicit prioritization -4. **Backpressure**: Queue blocks when at capacity - ---- - -## Blockchain Synchronization {#blockchain-synchronization} - -### Block Import Architecture - -The `BlockImporter` (src/workers/block-importer.ts) manages sequential blockchain synchronization with the following workflow: - -1. **Height Discovery**: Polls current blockchain height every 5 seconds -2. **Sequential Processing**: Imports blocks in strict height order -3. **Fork Detection**: Validates chain integrity at each block -4. **Transaction Fetching**: Retrieves all transactions for each block -5. **Event Emission**: Notifies downstream systems of new data - -### Fork Detection and Recovery - -**Fork Detection Algorithm** (src/workers/block-importer.ts:121-139): -```typescript -async getBlockOrForkedBlock( - previousHeight: number, - height: number, - forkDepth = 0, -): Promise { - // Prevent infinite recursion - if (forkDepth > MAX_FORK_DEPTH) { - throw new Error(`Skipping import of block ${height} due to deep fork`); - } - - const block = await this.arweave.getBlockByHeight(height); - - // Skip validation for genesis block - if (previousHeight > 0) { - const previousDbBlockHash = await this.chainIndex.getBlockHashByHeight( - height - 1 - ); - - if (previousDbBlockHash === undefined) { - // Gap detected - rewind to last known block - await this.chainIndex.resetToHeight(previousHeight - 1); - return this.getBlockOrForkedBlock(previousHeight, height, forkDepth + 1); - } else if (block.previous_block !== previousDbBlockHash) { - // Fork detected - rewind to fork point - this.metrics.forksCounter.inc(); - await this.chainIndex.resetToHeight(previousHeight - 1); - return this.getBlockOrForkedBlock(previousHeight, height, forkDepth + 1); - } - } - - return block; -} -``` - -**Key Fork Handling Features**: -- Maximum fork depth: 18 blocks (`MAX_FORK_DEPTH`) -- Automatic chain reorganization -- Transaction rollback on fork detection -- Metrics tracking for fork events - -### Transaction Processing Pipeline - -**Transaction Retrieval Strategy**: - -1. **Cache Check**: Look for transaction in local txStore -2. **Peer Retrieval**: Attempt fetching from 3 random weighted peers -3. **Trusted Node Fallback**: Use primary node if peers fail -4. **Validation**: Verify signature using Arweave SDK -5. **ECDSA Recovery**: Handle empty owner fields for ECDSA transactions -6. **Storage**: Cache validated transaction for future use - -**Transaction Validation Implementation** (src/arweave/composite-client.ts:857-865): -```typescript -// Standard validation -try { - const isValid = await this.arweave.transactions.verify(tx); - if (!isValid) { - throw new Error('Invalid transaction signature'); - } -} catch (error) { - // ECDSA public key recovery for empty owner fields - if (tx.owner === '' && tx.signature_type === 2) { - const publicKeyBuffer = SECP256k1PublicKey.recover( - toB64Url(tx.data_root || new Uint8Array()), - Buffer.from(tx.signature, 'base64url'), - ); - tx.owner = toB64Url(publicKeyBuffer); - } -} -``` - -### Block Prefetching Optimization - -**Prefetch Strategy** (src/arweave/composite-client.ts:556-575): -```typescript -// Prefetch blocks when queue has capacity -if (this.blockRequestQueue.length() === 0) { - const nextHeight = currentHeight + 1; - const maxHeight = await this.getHeight(); - - // Prefetch up to blockPrefetchCount blocks - for (let i = 0; i < this.blockPrefetchCount && nextHeight + i <= maxHeight; i++) { - const prefetchHeight = nextHeight + i; - - // Add to cache if not already present - if (!this.blockCache.has(prefetchHeight.toString())) { - const blockPromise = this.getBlockByHeightUncached(prefetchHeight); - this.blockCache.set(prefetchHeight.toString(), blockPromise); - } - } -} -``` - ---- - -## Chunk Management {#chunk-management} - -### Chunk Architecture Overview - -Chunks are the fundamental unit of data storage in Arweave, with the following characteristics: -- Maximum size: 256KB -- Minimum size: 32KB (configurable) -- Each chunk includes Merkle proof paths -- Validated against transaction data root - -### Chunk Retrieval Strategy - -**Multi-Source Retrieval** (src/arweave/composite-client.ts:944-991): -```typescript -async getChunkByAny({ - txSize, - absoluteOffset, - dataRoot, - relativeOffset, -}: ChunkDataByAnySourceParams): Promise { - // 1. Check weak cache (5-second TTL) - const cached = this.chunkCache.get({ absoluteOffset }); - if (cached && Date.now() - cached.cachedAt < 5000) { - return cached.chunk; - } - - // 2. Try trusted node first - try { - const chunk = await this.getChunkFromTrustedNode(absoluteOffset); - this.chunkCache.set({ absoluteOffset }, { chunk, cachedAt: Date.now() }); - return chunk; - } catch (error) { - this.log.warn('Failed to get chunk from trusted node', error); - } - - // 3. Fall back to peer network - const peers = randomWeightedChoices({ - table: this.weightedGetChunkPeers, - count: 3, - }); - - for (const peer of peers) { - try { - const chunk = await this.getChunkFromPeer(peer, absoluteOffset); - this.adjustPeerWeight('weightedGetChunkPeers', peer, 'success'); - return chunk; - } catch (error) { - this.adjustPeerWeight('weightedGetChunkPeers', peer, 'failure'); - } - } - - throw new Error('Failed to retrieve chunk from any source'); -} -``` - -### Chunk Validation Process - -**Merkle Proof Validation** (src/lib/validation.ts:79-95): -```typescript -export async function validateChunk( - txSize: number, - chunk: Chunk, - dataRoot: Buffer, - relativeOffset: number, -): Promise { - // Step 1: Verify chunk hash - const chunkHash = crypto.createHash('sha256').update(chunk.chunk).digest(); - if (!chunkHash.equals(chunk.data_path.slice(-64, -32))) { - throw new Error('Invalid chunk: hash does not match data_path'); - } - - // Step 2: Validate Merkle proof - const validChunk = await validatePath( - dataRoot, - relativeOffset, - 0, - txSize, - chunk.data_path, - ); - - if (!validChunk) { - throw new Error('Invalid chunk: data_path does not match data_root'); - } -} -``` - -### Chunk Broadcasting System - -**Multi-Tier Broadcasting Strategy** (src/arweave/composite-client.ts:1168-1301): - -#### Tier 1: Primary Nodes (Required) -```typescript -const primaryResults = await Promise.allSettled( - primaryChunkPostUrls.map((url) => - primaryChunkPostLimit(() => - this.postChunkCircuitBreakers[url].fire(url, chunk) - ) - ) -); - -const primarySuccessCount = primaryResults.filter( - (result) => result.status === 'fulfilled' -).length; - -if (primarySuccessCount < CHUNK_POST_MIN_SUCCESS_COUNT) { - throw new Error('Failed to post chunk to minimum required nodes'); -} -``` - -#### Tier 2: Secondary Nodes (Optional) -```typescript -if (secondaryChunkPostMinSuccess > 0) { - const secondaryPromises = secondaryChunkPostUrls.map((url) => - secondaryChunkPostLimit(async () => { - const cb = this.postChunkCircuitBreakers[url]; - return cb.fire(url, chunk); - }) - ); - - // Fire and forget - don't wait for completion - Promise.allSettled(secondaryPromises); -} -``` - -#### Tier 3: Peer Network (Background) -```typescript -const peerPromises = selectedPeers.map((peer) => - peerPostLimit(async () => { - try { - await this.postChunkToPeer(peer, chunk); - this.adjustPeerWeight('weightedPostChunkPeers', peer, 'success'); - } catch (error) { - this.adjustPeerWeight('weightedPostChunkPeers', peer, 'failure'); - throw error; - } - }) -); - -// Background broadcasting - don't block on completion -Promise.allSettled(peerPromises); -``` - -### Chunk Success Criteria - -| Tier | Default Minimum Success | Configurable Via | -|------|------------------------|------------------| -| Primary | 3 nodes | `CHUNK_POST_MIN_SUCCESS_COUNT` | -| Secondary | 1 node | `SECONDARY_CHUNK_POST_MIN_SUCCESS_COUNT` | -| Peer Network | 2 peers | `ARWEAVE_PEER_CHUNK_POST_MIN_SUCCESS_COUNT` | - ---- - -## Peer Network Management {#peer-network-management} - -### Peer Discovery Mechanism - -**Automated Peer Discovery** (src/arweave/composite-client.ts:697-741): -```typescript -private async refreshPeers(): Promise { - try { - // 1. Fetch peer list from trusted node - const response = await this.trustedNodeAxios.get('/peers'); - const peers = response.data; - - // 2. Apply blacklist filter - const filteredPeers = peers.filter( - (peer) => !this.ignoreArweaveNodePeers.includes(peer) - ); - - // 3. Probe each peer for health info - const peerInfoPromises = filteredPeers.map(async (peer) => { - try { - const peerUrl = `http://${peer}`; - const infoResponse = await axios.get(`${peerUrl}/info`, { - timeout: DEFAULT_PEER_INFO_TIMEOUT_MS, // 5 seconds - }); - - return { - peer, - info: infoResponse.data, - lastSeen: Date.now(), - }; - } catch (error) { - this.log.debug('Failed to get peer info', { peer, error }); - return null; - } - }); - - // 4. Update weighted peer lists - const peerInfos = await Promise.all(peerInfoPromises); - this.updateWeightedPeers(peerInfos.filter(Boolean)); - - } catch (error) { - this.log.error('Failed to refresh peers', error); - this.metrics.peerRefreshErrorsTotal.inc(); - } -} - -// Schedule automatic refresh every 10 minutes -setInterval(() => this.refreshPeers(), 10 * 60 * 1000); -``` - -### Weighted Peer Selection Algorithm - -**Temperature-Based Weight Adjustment** (src/arweave/composite-client.ts:495-502): -```typescript -private adjustPeerWeight( - peerListName: 'weightedChainPeers' | 'weightedGetChunkPeers' | 'weightedPostChunkPeers', - peer: string, - result: 'success' | 'failure', -): void { - const weightedPeer = this[peerListName].find((p) => p.id === peer); - - if (weightedPeer) { - const delta = this.config.WEIGHTED_PEERS_TEMPERATURE_DELTA; // Default: 2 - - if (result === 'success') { - // Increase weight on success (warmer) - weightedPeer.weight = Math.min(weightedPeer.weight + delta, 100); - } else { - // Decrease weight on failure (cooler) - weightedPeer.weight = Math.max(weightedPeer.weight - delta, 1); - } - } -} -``` - -### Peer List Management - -The system maintains three separate weighted peer lists: - -| List Name | Purpose | Initial Weight | Weight Range | -|-----------|---------|----------------|--------------| -| `weightedChainPeers` | Block/transaction retrieval | 50 | 1-100 | -| `weightedGetChunkPeers` | Chunk fetching | 50 | 1-100 | -| `weightedPostChunkPeers` | Chunk broadcasting | 50 | 1-100 | - -### Peer Selection Strategy - -**Random Weighted Selection** (src/lib/randomWeightedChoice.ts): -```typescript -export function randomWeightedChoices({ - table, - count, - temperature = 50, -}: { - table: WeightedElement[]; - count: number; - temperature?: number; -}): T[] { - // Convert temperature to [-1, 1] range - const T = (temperature - 50) / 50; - - // Calculate average weight - const avgWeight = table.reduce((sum, el) => sum + el.weight, 0) / table.length; - - // Calculate urgency for each element - const urgencies = table.map((element) => { - const influence = Math.abs(avgWeight - element.weight) / avgWeight; - const urgency = element.weight + T * influence * (avgWeight - element.weight); - return Math.max(urgency, 0.01); // Ensure positive urgency - }); - - // Perform weighted random selection - const selected: T[] = []; - for (let i = 0; i < count && i < table.length; i++) { - const pick = weightedRandom(table, urgencies); - if (!selected.includes(pick)) { - selected.push(pick); - } - } - - return selected; -} -``` - -**Temperature Effects**: -- High temperature (>50): Favors exploration (lower-weighted peers) -- Low temperature (<50): Favors exploitation (higher-weighted peers) -- Default (50): Balanced selection - ---- - -## Data Format Compatibility {#data-format-compatibility} - -### Arweave Data Structures - -#### Block Format -```typescript -interface PartialJsonBlock { - indep_hash: string; // 64-char base64url - Block hash - height: number; // Block height - previous_block?: string; // 64-char base64url - Previous block hash - txs: string[]; // Array of 43-char base64url tx IDs - tx_root: string; // 43-char base64url - Transaction root - wallet_list: string; // 64-char base64url - Miner wallet list - reward_addr?: string; // 43-char base64url - Mining reward address - tags: Tag[]; // Block tags - reward_pool: string; // Mining reward pool size - weave_size: string; // Total weave size - block_size: string; // Block size in bytes - cumulative_diff: string; // Cumulative difficulty - hash_list_merkle?: string; // 64-char base64url - Hash list merkle - poa?: object; // Proof of Access data (removed for storage) - poa2?: object; // Proof of Access 2 data - usd_to_ar_rate?: [string, string]; // Exchange rate - scheduled_usd_to_ar_rate?: [string, string]; // Future exchange rate -} -``` - -#### Transaction Format -```typescript -interface PartialJsonTransaction { - id: string; // 43-char base64url - Transaction ID - signature: string; // base64url - Transaction signature - owner: string; // base64url - Owner public key (may be empty for ECDSA) - target?: string; // 43-char base64url - Target wallet - data_root?: string; // 43-char base64url - Merkle root of data - data_size?: string; // Size of data in bytes - reward?: string; // Mining reward in winston - last_tx?: string; // 43-char base64url - Previous transaction - tags?: Tag[]; // Transaction tags - format: number; // Transaction format (1 or 2) - quantity?: string; // Transfer amount in winston - data?: string; // base64url - Transaction data (small txs only) - signature_type?: number; // 1 (RSA) or 2 (ECDSA) -} - -interface Tag { - name: string; // base64url encoded tag name - value: string; // base64url encoded tag value -} -``` - -#### Chunk Format -```typescript -interface JsonChunk { - tx_path: string; // Base64url - Transaction Merkle path - data_path: string; // Base64url - Data Merkle path - chunk: string; // Base64url - Chunk data (max 256KB) -} - -// Decoded chunk structure -interface Chunk { - data_root: Buffer; // 32 bytes - Transaction data root - data_size: number; // Transaction data size - data_path: Buffer; // Merkle proof path - offset: number; // Relative offset in transaction - chunk: Buffer; // Raw chunk data -} -``` - -### Encoding and Decoding - -**Base64URL Conversion Utilities** (src/lib/encoding.ts): -```typescript -// Convert from base64url to Buffer -export function fromB64Url(input: string): Buffer { - return Buffer.from(input, 'base64url'); -} - -// Convert from Buffer to base64url -export function toB64Url(buffer: Buffer): string { - return buffer.toString('base64url'); -} - -// Decode tags -export function tagToUtf8(tag: { name: string; value: string }) { - return { - name: fromB64Url(tag.name).toString('utf8'), - value: fromB64Url(tag.value).toString('utf8'), - }; -} -``` - -### SDK Integration Points - -**Arweave SDK Initialization** (src/system.ts:87-89): -```typescript -export const arweave = Arweave.init({ - host: 'ar-io.dev', - port: 443, - protocol: 'https', -}); -``` - -**Key SDK Functions Used**: - -1. **Transaction Operations**: - - `arweave.transactions.fromRaw()`: Parse raw transaction data - - `arweave.transactions.verify()`: Validate transaction signatures - - `arweave.transactions.getData()`: Retrieve transaction data - -2. **Chunk Operations**: - - `arweave.chunks.validatePath()`: Verify Merkle proofs - - `arweave.chunks.buildLayers()`: Build Merkle tree layers - - `arweave.chunks.generateLeaves()`: Generate chunk leaves - -3. **Cryptographic Operations**: - - `SECP256k1PublicKey.recover()`: ECDSA public key recovery - - `arweave.crypto.hash()`: SHA-256 hashing - -### Protocol Version Compatibility - -**Supported Features by Version**: - -| Feature | Support Status | Notes | -|---------|----------------|-------| -| Arweave 2.0+ | ✓ Supported | Core protocol support | -| RSA Signatures | ✓ Supported | Traditional signature type | -| ECDSA Signatures | ✓ Supported | With public key recovery | -| Transaction Format 1 | ✓ Supported | Legacy format | -| Transaction Format 2 | ✓ Supported | Current format | -| PoA (Proof of Access) | ✓ Supported | Removed from stored blocks | -| PoA2 | ✓ Supported | Enhanced proof of access | -| Variable Chunk Sizes | ✓ Supported | 32KB - 256KB | -| Exchange Rate Tracking | ✓ Supported | USD/AR rate in blocks | - ---- - -## Error Handling and Resilience {#error-handling-resilience} - -### Circuit Breaker Implementation - -**Chunk POST Circuit Breakers** (src/arweave/composite-client.ts:240-265): -```typescript -// Primary node circuit breakers -this.primaryChunkPostUrls.forEach((url) => { - this.postChunkCircuitBreakers[url] = new CircuitBreaker(postChunk, { - name: `primaryBroadcastChunk-${url}`, - capacity: 100, // Maximum requests in window - resetTimeout: 5000, // Reset after 5 seconds - errorThresholdPercentage: 50, // Open at 50% error rate - }); -}); - -// Secondary node circuit breakers -this.secondaryChunkPostUrls.forEach((url) => { - this.postChunkCircuitBreakers[url] = new CircuitBreaker(postChunk, { - name: `secondaryBroadcastChunk-${url}`, - capacity: 10, // Lower capacity for secondary - resetTimeout: 10000, // 10 second reset - errorThresholdPercentage: 50, - }); -}); - -// Peer circuit breakers -this.peerChunkPostUrls.forEach((url) => { - this.postChunkCircuitBreakers[url] = new CircuitBreaker(postChunk, { - name: `peerBroadcastChunk-${url}`, - capacity: 10, - resetTimeout: 10000, - errorThresholdPercentage: 50, - }); -}); -``` - -### Retry Strategies - -#### HTTP Request Retry -```typescript -// Exponential backoff configuration -axiosRetry(this.trustedNodeAxios, { - retries: 5, // Maximum retry attempts - retryDelay: axiosRetry.exponentialDelay, // Exponential backoff - shouldResetTimeout: true, // Reset timeout on retry - retryCondition: (error) => { - // Special handling for rate limits - if (error?.response?.status === 429) { - // Reduce token bucket by 1% - this.trustedNodeRequestBucket = Math.floor( - this.trustedNodeRequestBucket * 0.99 - ); - } - - // Retry on network errors and 5xx responses - return axiosRetry.isRetryableError(error); - }, -}); -``` - -#### Peer Fallback Strategy -```typescript -// Try multiple peers with weight adjustment -async getTransactionFromPeers(txId: string): Promise { - const peers = randomWeightedChoices({ - table: this.weightedChainPeers, - count: 3, // Try 3 peers - }); - - const errors: Error[] = []; - - for (const peer of peers) { - try { - const tx = await this.getTransactionFromPeer(peer, txId); - this.adjustPeerWeight('weightedChainPeers', peer, 'success'); - return tx; - } catch (error) { - errors.push(error); - this.adjustPeerWeight('weightedChainPeers', peer, 'failure'); - } - } - - throw new Error(`Failed to get transaction from peers: ${errors.join(', ')}`); -} -``` - -### Fork Recovery Mechanism - -**Automatic Chain Reorganization**: -1. **Detection**: Compare previous_block hash with stored value -2. **Rewind**: Reset chain to last known good block -3. **Re-import**: Fetch and validate blocks from fork point -4. **Depth Limit**: Maximum 18 blocks to prevent deep reorganization - -**Implementation** (src/workers/block-importer.ts:130-145): -```typescript -if (block.previous_block !== previousDbBlockHash) { - // Fork detected - this.log.info('Fork detected', { - height, - expectedPrevious: previousDbBlockHash, - actualPrevious: block.previous_block, - }); - - // Increment fork counter - this.metrics.forksCounter.inc(); - - // Rewind to fork point - await this.chainIndex.resetToHeight(previousHeight - 1); - - // Recursively find correct block - return this.getBlockOrForkedBlock( - previousHeight, - height, - forkDepth + 1 - ); -} -``` - -### Error Categories and Handling - -| Error Type | Handling Strategy | Recovery Mechanism | -|------------|-------------------|-------------------| -| Network Timeout | Exponential backoff retry | Automatic retry up to 5 times | -| Rate Limiting (429) | Token bucket reduction | Reduced request rate | -| Invalid Data | Validation failure | Skip and log error | -| Fork Detection | Chain reorganization | Rewind and re-import | -| Peer Failure | Weight reduction | Try alternate peers | -| Circuit Open | Fast fail | Wait for reset timeout | - ---- - -## Performance Optimizations {#performance-optimizations} - -### Caching Architecture - -#### Multi-Level Cache System - -1. **Promise Cache** (In-Memory) - - **Purpose**: Prevent duplicate in-flight requests - - **TTL**: 30 seconds for blocks, 60 seconds for transactions - - **Implementation**: NodeCache with automatic expiration - -2. **Chunk Cache** (WeakMap) - - **Purpose**: Hot chunk data for repeated access - - **TTL**: 5 seconds - - **Implementation**: WeakMap for automatic garbage collection - -3. **Block Store** (Persistent) - - **Purpose**: Long-term block and transaction storage - - **Types**: Filesystem (MessagePack) or KV store - - **Optimization**: Height-based symlinks for fast lookup - -4. **Prefetch Cache** - - **Purpose**: Reduce latency for sequential access - - **Depth**: 50 blocks ahead (configurable) - - **Trigger**: Empty request queue - -**Cache Implementation Example** (src/arweave/composite-client.ts:471-485): -```typescript -// Check promise cache first -const cachedPromise = this.blockCache.get(height.toString()); -if (cachedPromise !== undefined) { - return cachedPromise; -} - -// Check persistent store -const cachedBlock = await this.blockStore.get(height); -if (cachedBlock !== undefined) { - return cachedBlock; -} - -// Fetch and cache -const blockPromise = this.getBlockByHeightUncached(height); -this.blockCache.set(height.toString(), blockPromise); -return blockPromise; -``` - -### Concurrency Management - -#### Request Concurrency Limits - -| Operation Type | Default Limit | Configuration | -|----------------|---------------|---------------| -| General Requests | 100 | `maxConcurrentRequests` | -| Primary Chunk POST | 10 | Hard-coded | -| Secondary Chunk POST | 10 | Hard-coded | -| Peer Chunk POST | 3 | Hard-coded | -| Block Prefetch | 50 | `blockPrefetchCount` | - -#### Concurrency Implementation -```typescript -// Request queue with concurrency control -this.trustedNodeRequestQueue = fastq.promise( - this.trustedNodeRequest.bind(this), - this.maxConcurrentRequests -); - -// Chunk POST concurrency limiting -const primaryChunkPostLimit = pLimit(10); -const secondaryChunkPostLimit = pLimit(10); -const peerPostLimit = pLimit(this.peerChunkPostConcurrency); -``` - -### Resource Optimization Strategies - -#### Memory Management -1. **Weak References**: Chunk cache uses WeakMap -2. **Stream Processing**: Large data handled as streams -3. **Selective Loading**: Only required transaction fields -4. **PoA Removal**: Strip proof-of-access data from blocks - -#### Network Optimization -1. **Connection Reuse**: Axios instances per endpoint -2. **Compression**: Accept gzip/deflate responses -3. **Batch Prefetching**: Fetch multiple blocks together -4. **Parallel Requests**: Concurrent peer attempts - -#### Storage Optimization -1. **MessagePack**: Efficient binary serialization -2. **Directory Sharding**: First 4 chars split into subdirs -3. **Symlinks**: Height-based access without duplication -4. **Atomic Writes**: Temp file + rename pattern - -### Performance Monitoring - -**Key Metrics Tracked**: -```typescript -// Request performance -metrics.getBlockByHeightTotal.inc({ source: 'trusted_node' }); -metrics.getTransactionTotal.inc({ source: 'peer' }); - -// Cache performance -metrics.blockCacheHits.inc(); -metrics.blockCacheMisses.inc(); - -// Network performance -metrics.peerResponseTime.observe(responseTime); -metrics.trustedNodeRequestQueueDepth.set(queue.length()); -``` - ---- - -## Monitoring and Metrics {#monitoring-metrics} - -### Prometheus Metrics - -#### Network Health Metrics - -| Metric Name | Type | Description | Labels | -|-------------|------|-------------|--------| -| `arweave_peer_info_errors_total` | Counter | Failed peer info requests | None | -| `arweave_peer_refresh_errors_total` | Counter | Failed peer list refreshes | None | -| `arweave_block_import_errors_total` | Counter | Block synchronization errors | None | -| `arweave_forks_total` | Counter | Detected blockchain forks | None | - -#### Request Metrics - -| Metric Name | Type | Description | Labels | -|-------------|------|-------------|--------| -| `get_block_by_height_total` | Counter | Block fetch attempts | source | -| `get_transaction_total` | Counter | Transaction fetch attempts | source | -| `get_chunk_total` | Counter | Chunk fetch attempts | source | -| `chunk_post_success_total` | Counter | Successful chunk broadcasts | endpoint | -| `chunk_post_error_total` | Counter | Failed chunk broadcasts | endpoint | - -#### Performance Metrics - -| Metric Name | Type | Description | Labels | -|-------------|------|-------------|--------| -| `trusted_node_request_queue_depth` | Gauge | Current queue depth | None | -| `peer_response_time_ms` | Histogram | Peer response times | peer | -| `block_cache_hit_rate` | Gauge | Cache hit percentage | None | -| `circuit_breaker_state` | Gauge | Circuit breaker states | name | - -### Health Monitoring - -#### Peer Health Tracking -```typescript -interface PeerInfo { - peer: string; - blocks: number; - height: number; - lastSeen: number; -} - -// Health check during refresh -const isHealthy = (info: PeerInfo) => { - const ageMs = Date.now() - info.lastSeen; - return ageMs < 60 * 60 * 1000; // 1 hour threshold -}; -``` - -#### Circuit Breaker Monitoring -```typescript -// Circuit breaker events -circuitBreaker.on('open', () => { - metrics.circuitBreakerState.set({ name: breaker.name }, 1); -}); - -circuitBreaker.on('halfOpen', () => { - metrics.circuitBreakerState.set({ name: breaker.name }, 0.5); -}); - -circuitBreaker.on('close', () => { - metrics.circuitBreakerState.set({ name: breaker.name }, 0); -}); -``` - -### Performance Tracking - -#### Request Performance -- Track success/failure rates per source -- Monitor response times -- Queue depth tracking -- Timeout frequency - -#### Resource Utilization -- Memory usage patterns -- Network bandwidth -- Storage growth rate -- CPU utilization - ---- - -## Protocol Evolution Support {#protocol-evolution} - -### Version Compatibility Matrix - -| Protocol Feature | Version | Status | Implementation Notes | -|------------------|---------|--------|---------------------| -| Core Protocol | 2.0+ | ✓ Active | Full support | -| RSA Signatures | 1.0+ | ✓ Active | Standard validation | -| ECDSA Signatures | 2.5+ | ✓ Active | With key recovery | -| Transaction Format 1 | 1.0+ | ✓ Legacy | Backward compatible | -| Transaction Format 2 | 2.0+ | ✓ Active | Current standard | -| Proof of Access (PoA) | 2.0+ | ✓ Active | Stripped for storage | -| PoA2 | 2.5+ | ✓ Active | Enhanced PoA | -| Flexible Chunks | 2.5+ | ✓ Active | 32KB-256KB sizes | -| USD/AR Rate | 2.4+ | ✓ Active | Exchange tracking | - -### Handling Protocol Changes - -#### ECDSA Signature Support -```typescript -// Handle empty owner fields for ECDSA transactions -if (tx.owner === '' && tx.signature_type === 2) { - const publicKeyBuffer = SECP256k1PublicKey.recover( - toB64Url(tx.data_root || new Uint8Array()), - Buffer.from(tx.signature, 'base64url'), - ); - tx.owner = toB64Url(publicKeyBuffer); -} -``` - -#### Format Detection -```typescript -// Automatic format detection -const format = tx.format || 1; // Default to format 1 -if (format === 2) { - // Handle format 2 specific fields - processFormat2Transaction(tx); -} else { - // Legacy format 1 processing - processFormat1Transaction(tx); -} -``` - -### Future-Proofing Strategies - -1. **Configurable Endpoints**: All API endpoints configurable -2. **Version Negotiation**: Prepared for protocol versioning -3. **Flexible Validation**: Pluggable validation strategies -4. **Field Preservation**: Unknown fields preserved -5. **Graceful Degradation**: Fallback for unsupported features - ---- - -## Technical Implementation Details {#technical-implementation} - -### Request Flow Diagrams - -#### Block Synchronization Flow -``` -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ Gateway │────▶│ Trusted │────▶│ Arweave │ -│ Worker │ │ Node │ │ Network │ -└─────────────┘ └─────────────┘ └─────────────┘ - │ │ │ - │ 1. Get Height │ │ - │───────────────────▶│ │ - │ │ 2. /height │ - │ │────────────────────▶│ - │ │◀────────────────────│ - │◀───────────────────│ │ - │ │ │ - │ 3. Get Block │ │ - │───────────────────▶│ │ - │ │ 4. /block/height/X │ - │ │────────────────────▶│ - │ │◀────────────────────│ - │◀───────────────────│ │ - │ │ │ - │ 5. Validate Chain │ │ - │ 6. Store Block │ │ - │ 7. Emit Events │ │ - ▼ ▼ ▼ -``` - -#### Chunk Retrieval Flow -``` -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ Client │────▶│ Gateway │────▶│ Cache │────▶│ Network │ -└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ - │ │ │ │ - │ 1. Request Chunk │ │ │ - │───────────────────▶│ │ │ - │ │ 2. Check Cache │ │ - │ │────────────────────▶│ │ - │ │◀ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┤ │ - │ │ (cache miss) │ │ - │ │ │ │ - │ │ 3. Try Trusted Node │ │ - │ │─────────────────────┼───────────────────▶│ - │ │◀ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┼ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┤ - │ │ (failure) │ │ - │ │ │ │ - │ │ 4. Try Peers │ │ - │ │─────────────────────┼───────────────────▶│ - │ │◀────────────────────┼────────────────────┤ - │ │ │ │ - │ │ 5. Validate Chunk │ │ - │ │ 6. Update Cache │ │ - │ │────────────────────▶│ │ - │◀───────────────────│ │ │ - │ 7. Return Data │ │ │ - ▼ ▼ ▼ ▼ -``` - -### State Management - -#### Peer State -```typescript -interface PeerState { - // Identification - id: string; // Peer URL - - // Health metrics - lastSeen: number; // Timestamp - height: number; // Latest block height - blocks: number; // Total blocks - - // Performance metrics - weight: number; // 1-100 selection weight - successCount: number; // Successful requests - failureCount: number; // Failed requests - averageResponseTime: number; // Rolling average -} -``` - -#### Circuit Breaker States -```typescript -enum CircuitState { - CLOSED = 'closed', // Normal operation - OPEN = 'open', // Failing, reject all - HALF_OPEN = 'half-open' // Testing recovery -} - -interface CircuitBreakerState { - state: CircuitState; - failures: number; - successes: number; - nextAttempt: number; // Timestamp for half-open -} -``` - -### Security Considerations - -1. **Input Validation**: All data validated before processing -2. **Signature Verification**: Cryptographic validation for transactions -3. **Merkle Proof Validation**: Chunk integrity verification -4. **Rate Limiting**: Prevents resource exhaustion -5. **Peer Blacklisting**: Configurable ignore lists - ---- - -## Configuration Reference {#configuration-reference} - -### Environment Variables - -#### Core Configuration - -| Variable | Default | Description | -|----------|---------|-------------| -| `TRUSTED_NODE_URL` | `https://arweave.net` | Primary Arweave node | -| `CHUNK_POST_URLS` | `${TRUSTED_NODE_URL}/chunk` | Chunk broadcast endpoints | -| `ARWEAVE_NODE_IGNORE_URLS` | Empty | Blacklisted peer URLs | -| `WEIGHTED_PEERS_TEMPERATURE_DELTA` | `2` | Peer weight adjustment rate | - -#### Performance Tuning - -| Variable | Default | Description | -|----------|---------|-------------| -| `MAX_CONCURRENT_REQUESTS` | `100` | Request queue capacity | -| `MAX_REQUESTS_PER_SECOND` | `5` | Rate limit | -| `REQUEST_TIMEOUT_MS` | `15000` | Request timeout | -| `REQUEST_RETRY_COUNT` | `5` | Retry attempts | -| `BLOCK_PREFETCH_COUNT` | `50` | Prefetch depth | - -#### Chunk Broadcasting - -| Variable | Default | Description | -|----------|---------|-------------| -| `CHUNK_POST_MIN_SUCCESS_COUNT` | `3` | Primary nodes required | -| `SECONDARY_CHUNK_POST_MIN_SUCCESS_COUNT` | `1` | Secondary nodes | -| `ARWEAVE_PEER_CHUNK_POST_MIN_SUCCESS_COUNT` | `2` | Peer broadcasts | -| `CHUNK_POST_TIMEOUT_MS` | `5000` | POST timeout | - -### Configuration Best Practices - -1. **Production Settings**: - ```bash - TRUSTED_NODE_URL=https://arweave.net - MAX_CONCURRENT_REQUESTS=200 - BLOCK_PREFETCH_COUNT=100 - CHUNK_POST_MIN_SUCCESS_COUNT=5 - ``` - -2. **Development Settings**: - ```bash - TRUSTED_NODE_URL=http://localhost:1984 - MAX_CONCURRENT_REQUESTS=50 - REQUEST_TIMEOUT_MS=30000 - SIMULATED_REQUEST_FAILURE_RATE=0.1 - ``` - -3. **High Availability**: - ```bash - CHUNK_POST_URLS=https://node1.example.com/chunk,https://node2.example.com/chunk - SECONDARY_CHUNK_POST_MIN_SUCCESS_COUNT=3 - WEIGHTED_PEERS_TEMPERATURE_DELTA=5 - ``` - ---- - -## Conclusion {#conclusion} - -The ar.io gateway's integration with the Arweave network represents a sophisticated implementation that successfully balances the competing demands of performance, reliability, and decentralization. Through its multi-layered architecture, the gateway provides: - -### Technical Achievements - -1. **Enterprise-Grade Reliability** - - Circuit breakers prevent cascade failures - - Multiple fallback mechanisms ensure availability - - Automatic recovery from network issues - -2. **Performance Optimization** - - Multi-level caching reduces latency - - Intelligent prefetching improves throughput - - Stream processing handles large data efficiently - -3. **Protocol Compatibility** - - Supports all Arweave protocol versions - - Handles format evolution gracefully - - Future-proofed for protocol changes - -4. **Operational Excellence** - - Comprehensive metrics and monitoring - - Configurable behavior for different environments - - Self-healing capabilities through retry mechanisms - -### Architectural Strengths - -The integration achieves its goals through: -- **Separation of Concerns**: Clear boundaries between components -- **Defensive Programming**: Extensive error handling and validation -- **Performance Focus**: Optimizations at every layer -- **Flexibility**: Configurable for various deployment scenarios - -### Future Considerations - -The architecture is well-positioned to evolve with: -- Protocol version negotiations -- Enhanced peer discovery mechanisms -- Advanced caching strategies -- Improved performance optimizations - -This comprehensive integration makes the ar.io gateway an ideal solution for applications requiring reliable, performant access to the Arweave permaweb while maintaining the network's core principles of permanence and decentralization. \ No newline at end of file diff --git a/docs/drafts/ar-io-04-arns-name-resolution-system.md b/docs/drafts/ar-io-04-arns-name-resolution-system.md deleted file mode 100644 index ed202fab5..000000000 --- a/docs/drafts/ar-io-04-arns-name-resolution-system.md +++ /dev/null @@ -1,194 +0,0 @@ -# AR.IO Gateway ArNS Resolution — Technical Overview - -**Status**: Draft. Supersedes the prior AO-era version (same path) in git history. - -The AR.IO gateway resolves human-readable names like `ardrive.permaweb.nexus` -into Arweave transaction IDs that ultimately back the response body. Resolution -is **Solana-native**: every authoritative lookup is a read against the -AR.IO ANT program on a Solana cluster via `@ar.io/sdk`, with multiple cache -tiers and a peer-fallback strategy layered on top. - -## Table of Contents - -1. [Architecture](#architecture) -2. [Resolution Flow](#resolution-flow) -3. [Caching Tiers](#caching-tiers) -4. [Sandbox Domain Security](#sandbox-domain-security) -5. [Configuration Reference](#configuration-reference) -6. [Error Handling](#error-handling) -7. [Failure Modes](#failure-modes) - ---- - -## Architecture - -### Components - -The resolution system lives under `src/resolution/` and `src/init/`: - -- **`CompositeArNSResolver`** (`src/resolution/composite-arns-resolver.ts`) - — the orchestrator. Runs an ordered list of underlying resolvers in - parallel, returns the first successful resolution that meets the timeout, - falls back to cached values when fresh resolution misses or errors. -- **`OnDemandArNSResolver`** (`src/resolution/on-demand-arns-resolver.ts`) - — direct path. Reads the ArNS record from the ANT mint's PDA via - `SolanaANTReadable` from `@ar.io/sdk`, fetches the latest record state - from `@solana/kit`, applies undername lookups. -- **`TrustedGatewayArNSResolver`** (`src/resolution/trusted-gateway-arns-resolver.ts`) - — peer path. Forwards the resolution request to one or more trusted - AR.IO gateway peers, reads `X-ArNS-*` headers off the response. -- **`ArNSNamesCache`** (`src/resolution/arns-names-cache.ts`) - — bulk-name registry cache. Periodically paginates through - `SolanaARIOReadable.getArNSRecords()` and writes each `{name, antId, - undernameLimit, type, startTimestamp, endTimestamp}` tuple to a - KV store. Used as the first lookup before falling through to the - per-name resolvers. - -### Wiring - -`src/init/resolvers.ts` constructs the resolver chain at process start: - -1. `ArNSResolverType[]` is read from `ARNS_RESOLVER_PRIORITY_ORDER` - (default `gateway,on-demand`). -2. Each entry maps to its concrete resolver class. Concrete instances are - then wrapped in `CompositeArNSResolver` with its registry + resolution - KV caches (Redis or in-memory `NodeKvStore`, controlled by - `ARNS_CACHE_TYPE`). - -The composite resolver is what `src/system.ts` injects into the request -pipeline. From the request handler's perspective, ArNS resolution is a -single async call returning a `NameResolution` shape. - ---- - -## Resolution Flow - -For a request to `ardrive.permaweb.nexus/some/path`: - -1. **Name parse**. Envoy splits the host on the first `.`. `ardrive` is - the ArNS root; anything before becomes undername segments. The root - is validated against the regex in `src/lib/validation.ts`. -2. **Names-cache fast path** (`ArNSNamesCache.getCachedArNSBaseName`). - Returns `{antId, undernameLimit, type, startTimestamp, endTimestamp}` - for the root if the periodic hydrator has seen it. A miss falls - through. A hit short-circuits the on-demand SDK call. -3. **Composite resolution**. `CompositeArNSResolver` races the priority - list of resolvers against `ARNS_COMPOSITE_RESOLVER_TIMEOUT_MS`. - The first resolver to return a `resolvedId` wins. - - **On-demand resolver**: uses the cached `antId` (or fetches it - fresh from the network process) to build a `SolanaANTReadable`, - calls `getRecord(undername)` to get the resolved tx id. - - **Trusted-gateway resolver**: HTTP request to each configured - peer; first 2xx with valid `X-ArNS-*` headers wins. -4. **Cached fallback**. If every fresh resolver misses or the timeout - trips, return the resolution-cache entry (if any) instead of failing. - Controlled by `ARNS_CACHED_RESOLUTION_FALLBACK_TIMEOUT_MS`. -5. **Sandbox redirect**. If the resolved tx id is requested at its - canonical name (not its sandbox host), respond 302 to the sandbox - host. See [Sandbox Domain Security](#sandbox-domain-security). - -A successful resolution writes back to **two** caches: -- The resolution KV (per-(name + undername) → `{resolvedId, ttl, antId, - resolvedAt, limit, index}`) -- The names KV (the base-name `{antId, undernameLimit, …}` payload), - if it was a names-cache miss. - ---- - -## Caching Tiers - -| Tier | Backing store | Scope | Refresh strategy | -|---|---|---|---| -| **Names cache** | `KvArNSRegistryStore` over Redis or in-memory KV | All base names known to the network process | Hydrator paginates `getArNSRecords()` on `ARNS_NAMES_CACHE_TTL_SECONDS` cadence + on misses | -| **Resolution cache** | `KvArNSResolutionStore` over the same KV backing | Per-(name, undername) → `{resolvedId, antId, …}` | Refreshed on cache miss / on TTL expiry, fallback-on-empty controlled by `ARNS_CACHED_RESOLUTION_FALLBACK_TIMEOUT_MS` | -| **Debounce cache** | `KvDebounceStore` | Hydration scheduling | Coalesces concurrent miss-driven hydrations into a single SDK round trip | - -The KV layer is chosen by `ARNS_CACHE_TYPE`: -- `node` (default): in-memory `NodeKvStore` with `ARNS_CACHE_MAX_KEYS` cap -- `redis`: `RedisKvStore` against the gateway's Redis service - -Resolution TTL comes from the on-chain record itself (via the SDK), but -the gateway can override with `ARNS_RESOLVER_OVERRIDE_TTL_SECONDS` if -operators want a different cache horizon than the network process exposes. - ---- - -## Sandbox Domain Security - -Resolved transaction content is served from a **sandboxed subdomain** to -isolate cookies / localStorage between names. The sandbox host is derived -from the resolved tx id via a base32 encoding (see `src/lib/sandbox.ts`). - -If a request lands at the canonical hostname and the resolved id maps to -a different sandbox host, the gateway issues a 302 redirect. This is -unchanged from the AO-era implementation — the security model doesn't -depend on the resolution backend. - ---- - -## Configuration Reference - -Defined in `src/config.ts`: - -| Env var | Default | Purpose | -|---|---|---| -| `ARNS_RESOLVER_PRIORITY_ORDER` | `gateway,on-demand` | Ordered comma-separated list of resolver strategies | -| `ARNS_CACHE_TYPE` | `node` | `node` (in-memory) or `redis` | -| `ARNS_CACHE_TTL_SECONDS` | `86400` (24h) | Resolution-cache entry TTL | -| `ARNS_CACHE_MAX_KEYS` | `10000` | Node-cache eviction ceiling | -| `ARNS_CACHED_RESOLUTION_FALLBACK_TIMEOUT_MS` | `250` | Time to wait on fresh resolution before serving from cache | -| `ARNS_COMPOSITE_RESOLVER_TIMEOUT_MS` | `3000` | Per-resolver time budget within the composite | -| `ARNS_COMPOSITE_LAST_RESOLVER_TIMEOUT_MS` | `30000` | Final resolver gets a longer budget | -| `ARNS_NAMES_CACHE_TTL_SECONDS` | `3600` (1h) | Names-cache rehydration cadence | -| `ARNS_NAME_LIST_CACHE_MISS_REFRESH_INTERVAL_SECONDS` | `120` | Throttle for miss-driven hydration | -| `ARNS_MAX_CONCURRENT_RESOLUTIONS` | `1` | Per-tick concurrency cap | -| `ARNS_RESOLVER_ENFORCE_UNDERNAME_LIMIT` | `true` | When true, reject undernames past the ANT's `undernameLimit` | -| `ARNS_RESOLVER_OVERRIDE_TTL_SECONDS` | unset | Override SDK-provided TTL with a local value | -| `ARIO_ANT_PROGRAM_ID` | unset | Devnet/localnet program-id override for the ANT program | -| `ARIO_ARNS_PROGRAM_ID` | unset | Devnet/localnet program-id override for the ArNS registry program | - -A few related Solana-RPC circuit-breaker knobs (also in `src/config.ts`): -`ARIO_PROCESS_DEFAULT_CIRCUIT_BREAKER_{TIMEOUT,ERROR_THRESHOLD_PERCENTAGE,ROLLING_COUNT_TIMEOUT,RESET_TIMEOUT}_MS`. - ---- - -## Error Handling - -Every resolver call is wrapped in a tracing span and an error classifier: - -- **Network process down** (Solana RPC unreachable, rate-limited 429, or - the program returned a known "not-yet-initialized" error): the resolver - records the failure on its OpenTelemetry span, returns `undefined`, and - the composite resolver falls back to the cached resolution if one exists. -- **Not-found** (ANT mint PDA exists but the requested undername is not - in the record): returns `undefined` from the resolver. Composite tries - the next strategy in the priority order, then returns 404 if all miss. -- **Past undername limit** (when `ARNS_RESOLVER_ENFORCE_UNDERNAME_LIMIT` - is true): explicit failure surfaced to the caller as 404 — semantically - invalid request, not a transient error. - -Failures are also tracked in three Prometheus counters: -`arns_cached_resolution_fallback_on_empty_total`, -`arns_name_cache_hydration_failures_total`, -`arns_name_cache_hydration_retries_total`. - ---- - -## Failure Modes - -| Scenario | Behavior | -|---|---| -| Solana RPC timeout | Fresh resolution fails; cached value served if available; otherwise 404. Counter increments. | -| RPC 429 (rate-limited) | Same as timeout; the SDK retries with backoff once before failing up to the resolver. | -| Names-cache hydrator fails repeatedly | Hydrator backs off and retries; in-flight resolutions fall through to per-name on-demand lookups. Gauge `arns_base_name_cache_entries` may go stale. | -| ANT program-id misconfigured (`ARIO_ANT_PROGRAM_ID` pointing at the wrong cluster) | All lookups return "AccountNotInitialized"; resolutions miss; 404. Reconfigure operator. | -| Trusted-gateway peer returns malformed `X-ArNS-Ant-Id` | Resolver discards the header, treats as a 404 for that peer, tries the next. | - -The classifier explicitly distinguishes **transient** (RPC, rate-limit, -timeouts) from **terminal** (not-found, over-limit) failures so the -composite resolver can apply the right fallback strategy. - ---- - -*This document reflects the `solana` branch at SDK `4.0.0-solana.14`. For -the AO-era architecture, see git history prior to the AO sidecar removal.* diff --git a/docs/drafts/ar-io-05-centralization-analysis.md b/docs/drafts/ar-io-05-centralization-analysis.md deleted file mode 100644 index fc13337c3..000000000 --- a/docs/drafts/ar-io-05-centralization-analysis.md +++ /dev/null @@ -1,173 +0,0 @@ -# AR.IO Gateway Centralization Analysis — Solana Edition - -**Status**: Draft. Supersedes the prior AO-era analysis (same path) in git history. - -This document re-frames the centralization analysis for the Solana-native -AR.IO gateway. The shape of the dependencies is different from the AO era: -the gateway no longer talks to an AO Compute Unit or relies on a single -AO process ID. Authoritative state lives in **Solana programs** (with -program-derived accounts as the per-instance state), reached via the -operator's chosen **Solana RPC provider**. Some risk categories carry -over (registry control, RPC dependency), and some shift (program upgrade -authority, validator centralization). - ---- - -## Table of Contents - -1. [Critical Dependencies](#critical-dependencies) -2. [Per-Dependency Analysis](#per-dependency-analysis) -3. [Existing Mitigations](#existing-mitigations) -4. [Open Risks](#open-risks) -5. [Operator Knobs](#operator-knobs) - ---- - -## Critical Dependencies - -| Tier | Dependency | Substituted at runtime? | -|---|---|---| -| **Foundation** | AR.IO Solana programs (Core, GAR, ArNS, ANT) at their declared program IDs | Only if upgrade authority republishes; operator can override the program IDs per cluster via env vars | -| **Network** | Solana RPC provider (mainnet-beta / devnet / localnet endpoint) | Yes — `SOLANA_RPC_URL` is operator-supplied | -| **Network** | Validator set running the cluster | No — same trust assumption as any Solana app | -| **Coordination** | Trusted-gateway peers used by `TrustedGatewayArNSResolver` | Yes — operator-configured peer list | -| **Data** | Arweave gateways used as trusted nodes for chunk / tx fetches | Yes — `TRUSTED_GATEWAYS_URLS` | -| **Identity** | Operator's Solana keypair (`SOLANA_KEYPAIR_PATH`), observer keypair | No — must be kept available and rotation requires a re-registration cycle | - ---- - -## Per-Dependency Analysis - -### 1. AR.IO programs and their upgrade authority - -The four programs the gateway reads against are: - -``` -Core — ARIO_CORE_PROGRAM_ID (default: bundled mainnet ID via @ar.io/sdk) -GAR — ARIO_GAR_PROGRAM_ID -ArNS — ARIO_ARNS_PROGRAM_ID -ANT — ARIO_ANT_PROGRAM_ID -``` - -All four program IDs default to mainnet values shipped inside `@ar.io/sdk`. -Devnet / localnet operators override via env vars in `src/config.ts:2085+`. - -**Centralization risk**: the upgrade authority on each program is a Solana -account (currently a multisig governed off-chain). A compromise of that -authority could deploy a malicious upgrade that subverts gateway-observed -state. This is the modern analog of the old "compromise the AO process -ID" risk — the failure mode is identical (network-wide impact); the -control surface is different (Solana on-chain authority + multisig -discipline vs. AO process ownership). - -**Mitigation**: as adoption matures, the upgrade authority should -transition to a transparent on-chain governance program (or be frozen -entirely). The AR.IO core program's upgrade authority address should -be published and monitored. - -### 2. Solana RPC provider - -`SOLANA_RPC_URL` is the operator's choice of RPC endpoint. Public -`mainnet-beta.solana.com` is rate-limited; production deployments use -dedicated providers (Helius, QuickNode, Triton) or self-hosted validators. - -**Centralization risk**: an outage at the chosen provider stops fresh -resolutions; in-flight requests fall back to cached values until the -provider recovers (see ArNS doc, "Failure Modes"). The gateway does not -multi-source RPC reads — a single endpoint is used per process. - -**Mitigation**: operators can run their own RPC node. A circuit breaker -(`ARIO_PROCESS_DEFAULT_CIRCUIT_BREAKER_*`) trips after sustained failures -and rejects further attempts to call upstream rather than blocking the -request pipeline. - -### 3. Validator set - -The cluster's validator stake distribution is the same trust assumption as -any other Solana app. Out of scope for this document. - -### 4. Trusted gateway peers - -`TRUSTED_GATEWAYS_URLS` and the related peer-discovery logic mean the -gateway can offload data fetches (and optionally name resolution) to -other AR.IO gateways. The peer set is operator-configured and seeded by -peer discovery through the GAR (Gateway Address Registry) program. - -**Centralization risk**: a malicious or coordinated subset of peers could -serve incorrect resolutions or stale data, exploiting trust assumptions -the local gateway places in their `X-ArNS-Ant-Id` / `X-AR-IO-Verified` -headers. This is mitigated by: - -- Verifying tx content (when `X-AR-IO-Verified: true` is asserted) against - the chunk hashes recorded on Arweave. -- Tracking gateway reputation via the observer subsystem (see - `ar-io-observer`). -- Falling back to direct on-chain SDK resolution when peer responses - fail validation. - -### 5. Operator identity - -The operator keypair signs `save_observations` transactions on the Core -program (via the cranker) and identifies the gateway in registry reads. -Loss of the keypair without backup means a forced re-registration at a -new address; theft of the keypair means an attacker can submit -observation reports as the gateway. - -**Mitigation**: operators are responsible for keypair storage. The -observer container expects the keypair file at the mounted -`SOLANA_KEYPAIR_PATH`. Production deployments should use HSM-backed -signers; this is an open enhancement for `@ar.io/sdk` (no first-class -signer abstraction beyond the `KeyPairSigner` interface today). - ---- - -## Existing Mitigations - -The gateway already defends against several of the risks above: - -- **Multi-source data**: chunk fetches try AR.IO peers, Arweave gateways, - Arweave nodes, and S3 in configurable order. A single source failing - doesn't break content retrieval. -- **Cached resolution fallback**: ArNS resolution prefers a cached entry - over failing the request, so a transient RPC outage doesn't 404 every - cached name. -- **Trust headers**: responses include `X-AR-IO-Verified` and - `X-AR-IO-Trusted` so downstream consumers can decide whether to accept - cached / peered data. -- **Circuit breakers**: the network-process circuit breaker prevents an - RPC outage from cascading into a backed-up request queue. -- **Auto-verification**: the auto-verify subsystem cross-checks indexed - data against the canonical Parquet / ClickHouse / SQLite paths so an - intentionally-misindexed source surfaces as a discrepancy rather than - silently corrupting state. - ---- - -## Open Risks - -| Risk | Status | Owner | -|---|---|---| -| Multi-RPC fan-out for read consensus | Not implemented — single `SOLANA_RPC_URL` per process | SDK / gateway | -| HSM-backed signer support | Not implemented in `@ar.io/sdk`; gateway accepts file-loaded keypair | SDK | -| Program upgrade authority transparency | Authority addresses live on-chain but are not surfaced in gateway diagnostics | Network operations | -| Frozen-program path | Programs are upgradeable today; freezing is a network-governance call | Network governance | -| ArNS peer cross-validation | Peers' `X-ArNS-*` headers are trusted by name even when content verification would catch malformed payloads | Gateway | - ---- - -## Operator Knobs - -These give operators direct control over their centralization posture: - -- `SOLANA_RPC_URL` — choose / self-host RPC -- `ARIO_{CORE,GAR,ARNS,ANT}_PROGRAM_ID` — override program IDs per cluster -- `TRUSTED_GATEWAYS_URLS` — control which peers are trusted for data -- `ARNS_RESOLVER_PRIORITY_ORDER` — prefer direct SDK over peers, or vice versa -- `ARIO_PROCESS_DEFAULT_CIRCUIT_BREAKER_*` — tune RPC circuit-breaker thresholds -- Auto-verify pipeline knobs (see `docs/auto-verify.md`) — independent cross-checking layer - ---- - -*This document reflects the `solana` branch at SDK `4.0.0-solana.14`. For -the AO-era analysis (IO Process as SPOF, AO Compute Unit dependency, -etc.), see git history prior to the AO sidecar removal.* diff --git a/docs/drafts/ar-io-06-database-architecture.md b/docs/drafts/ar-io-06-database-architecture.md deleted file mode 100644 index 0693077a3..000000000 --- a/docs/drafts/ar-io-06-database-architecture.md +++ /dev/null @@ -1,1047 +0,0 @@ -# AR.IO Node Database Architecture: Technical Deep Dive - -## Executive Summary - -The ar.io node implements a sophisticated multi-database SQLite architecture designed for high-performance blockchain data indexing. The system uses four specialized databases with optional ClickHouse integration, employing worker-based concurrency patterns and advanced indexing strategies to handle the demands of Arweave data processing at scale. - -**Key Architecture Highlights**: -- Multi-database SQLite design with specialized schemas -- Worker thread architecture for safe concurrent access -- Two-stage data processing (new → stable) for blockchain finality -- Sophisticated retry and verification systems -- Optional ClickHouse integration for analytics workloads - ---- - -## Table of Contents - -1. [Database System Overview](#database-system-overview) -2. [Database Schemas](#database-schemas) -3. [Indexing Architecture](#indexing-architecture) -4. [Worker Process Architecture](#worker-process-architecture) -5. [Data Flow and Processing](#data-flow-and-processing) -6. [Query Patterns and Optimization](#query-patterns-and-optimization) -7. [Data Verification System](#data-verification-system) -8. [Migration Management](#migration-management) -9. [Performance Optimizations](#performance-optimizations) -10. [Background Jobs and Maintenance](#background-jobs-and-maintenance) -11. [ClickHouse Integration](#clickhouse-integration) -12. [Architecture Patterns](#architecture-patterns) - ---- - -## Database System Overview - -### Core Architecture - -```typescript -// src/system.ts:137-144 -export const db = new StandaloneSqliteDatabase({ - log, - coreDbPath: 'data/sqlite/core.db', - dataDbPath: 'data/sqlite/data.db', - moderationDbPath: 'data/sqlite/moderation.db', - bundlesDbPath: 'data/sqlite/bundles.db', - tagSelectivity: config.TAG_SELECTIVITY, -}); -``` - -### Database Layout - -``` -data/sqlite/ -├── core.db # Blockchain data (blocks, transactions) -├── data.db # Contiguous data and verification -├── bundles.db # ANS-104 bundles and data items -└── moderation.db # Content filtering and blocking -``` - -### SQLite Configuration - -```typescript -// Key pragmas for optimal performance -db.pragma('journal_mode = WAL'); // Write-Ahead Logging -db.pragma('page_size = 4096'); // 4KB pages -db.pragma('wal_autocheckpoint = 10000'); // 10K page checkpoint -db.pragma('synchronous = normal'); // Balance safety/performance -``` - ---- - -## Database Schemas - -### 1. Core Database (`core.db`) - -**Purpose**: Store blockchain data including blocks, transactions, and their relationships. - -#### Primary Tables - -```sql --- Stable (confirmed) blocks -CREATE TABLE stable_blocks ( - height INTEGER PRIMARY KEY, - id TEXT NOT NULL, - timestamp INTEGER NOT NULL, - previous_block TEXT NOT NULL, - nonce TEXT NOT NULL, - hash TEXT NOT NULL, - difficulty INTEGER NOT NULL, - cumulative_difficulty INTEGER NOT NULL, - last_retarget INTEGER NOT NULL, - reward_address TEXT NOT NULL, - reward_pool INTEGER NOT NULL, - weave_size INTEGER NOT NULL, - block_size INTEGER NOT NULL, - usd_to_ar_rate_dividend INTEGER NOT NULL, - usd_to_ar_rate_divisor INTEGER NOT NULL, - scheduled_usd_to_ar_rate_dividend INTEGER NOT NULL, - scheduled_usd_to_ar_rate_divisor INTEGER NOT NULL, - hash_list_merkle TEXT, - wallet_list TEXT, - tx_root TEXT, - tx_count INTEGER NOT NULL, - missing_tx_count INTEGER NOT NULL DEFAULT 0 -); - --- Stable transactions with comprehensive metadata -CREATE TABLE stable_transactions ( - id TEXT PRIMARY KEY, - block_transaction_index INTEGER, - signature TEXT, - format INTEGER NOT NULL, - last_tx TEXT NOT NULL, - owner TEXT NOT NULL, - owner_address TEXT NOT NULL, - target TEXT, - quantity TEXT NOT NULL, - reward TEXT NOT NULL, - data_size INTEGER NOT NULL, - data_root TEXT, - data_tree TEXT, - data_offset TEXT, - owner_offset INTEGER NOT NULL, - owner_size INTEGER NOT NULL, - signature_offset INTEGER, - signature_size INTEGER, - signature_type INTEGER, - height INTEGER NOT NULL, - offset INTEGER NOT NULL -); - --- Transaction tags for querying -CREATE TABLE stable_transaction_tags ( - tx_id TEXT NOT NULL, - tag_index INTEGER NOT NULL, - tag_name TEXT NOT NULL, - tag_value TEXT NOT NULL, - PRIMARY KEY (tx_id, tag_index) -); - --- Block-transaction relationships -CREATE TABLE stable_block_transactions ( - block_height INTEGER NOT NULL, - transaction_id TEXT NOT NULL, - block_transaction_index INTEGER NOT NULL, - PRIMARY KEY (block_height, block_transaction_index) -); -``` - -#### Key Indexes - -```sql --- Height-based queries -CREATE INDEX stable_blocks_height_id_idx ON stable_blocks (height, id); - --- Transaction lookups -CREATE INDEX stable_transactions_height_block_transaction_index_idx - ON stable_transactions (height DESC, block_transaction_index DESC); - --- Tag filtering -CREATE INDEX stable_transaction_tags_name_value_tx_id_idx - ON stable_transaction_tags (tag_name, tag_value, tx_id); - --- Owner queries -CREATE INDEX stable_transactions_owner_address_height_block_transaction_index_idx - ON stable_transactions (owner_address, height DESC, block_transaction_index DESC); -``` - -### 2. Data Database (`data.db`) - -**Purpose**: Store metadata about contiguous data and track verification status. - -#### Primary Tables - -```sql --- Contiguous data metadata -CREATE TABLE contiguous_data ( - id TEXT PRIMARY KEY, - data_root TEXT, - data_size INTEGER NOT NULL, - data_hash TEXT NOT NULL UNIQUE, - data_type TEXT NOT NULL, - offset INTEGER NOT NULL, - indexed_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')), - content_type TEXT, - cached_at INTEGER, - accessed_at INTEGER, - stable_data_item_count INTEGER, - mru_cache_key TEXT, - mru_arns_names TEXT, - mru_arns_base_names TEXT -); - --- ID to hash mapping with verification tracking -CREATE TABLE contiguous_data_ids ( - id TEXT PRIMARY KEY, - data_hash TEXT, - verified BOOLEAN NOT NULL DEFAULT FALSE, - verification_retry_count INTEGER NOT NULL DEFAULT 0, - verification_priority INTEGER NOT NULL DEFAULT 0, - first_verification_attempted_at INTEGER, - last_verification_attempted_at INTEGER, - FOREIGN KEY (data_hash) REFERENCES contiguous_data (data_hash) -); - --- Data root tracking -CREATE TABLE data_roots ( - data_root TEXT NOT NULL UNIQUE, - data_size INTEGER NOT NULL, - data_hash TEXT NOT NULL, - FOREIGN KEY (data_hash) REFERENCES contiguous_data (data_hash) -); -``` - -#### Verification System Indexes - -```sql --- Optimized for verification worker queries -CREATE INDEX contiguous_data_ids_verification_priority_retry_idx - ON contiguous_data_ids ( - verification_priority DESC, - verification_retry_count ASC, - id ASC - ) - WHERE verified = FALSE; - --- Hash-based lookups -CREATE INDEX contiguous_data_ids_data_hash_idx - ON contiguous_data_ids (data_hash); -``` - -### 3. Bundles Database (`bundles.db`) - -**Purpose**: Process and index ANS-104 bundles and their data items. - -#### Primary Tables - -```sql --- Bundle tracking with retry logic -CREATE TABLE bundles ( - id TEXT PRIMARY KEY, - root_transaction_id TEXT, - format TEXT, - unbundle_filter TEXT, - index_filter TEXT, - bundle_data_item_count INTEGER, - data_item_count INTEGER, - matched_data_item_count INTEGER, - duplicated_data_item_count INTEGER, - first_queued_at INTEGER, - last_queued_at INTEGER, - first_skipped_at INTEGER, - last_skipped_at INTEGER, - first_unbundled_at INTEGER, - last_unbundled_at INTEGER, - first_fully_indexed_at INTEGER, - last_fully_indexed_at INTEGER -); - --- Stable data items -CREATE TABLE stable_data_items ( - id TEXT PRIMARY KEY, - parent_id TEXT NOT NULL, - root_tx_id TEXT NOT NULL, - parent_index INTEGER NOT NULL, - data_hash TEXT, - data_offset INTEGER NOT NULL, - data_size INTEGER NOT NULL, - signature TEXT, - owner TEXT NOT NULL, - owner_address TEXT NOT NULL, - owner_offset INTEGER NOT NULL, - owner_size INTEGER NOT NULL, - target TEXT, - anchor TEXT, - offset INTEGER NOT NULL, - signature_offset INTEGER, - signature_size INTEGER, - signature_type INTEGER, - bundle_format TEXT, - height INTEGER NOT NULL, - indexed_at INTEGER NOT NULL, - content_type TEXT -); - --- Bundle-to-data-item relationships -CREATE TABLE bundle_data_items ( - id TEXT PRIMARY KEY, - root_transaction_id TEXT NOT NULL, - FOREIGN KEY (id) REFERENCES stable_data_items (id), - FOREIGN KEY (root_transaction_id) REFERENCES bundles (id) -); - --- Data item tags -CREATE TABLE stable_data_item_tags ( - data_item_id TEXT NOT NULL, - tag_index INTEGER NOT NULL, - tag_name TEXT NOT NULL, - tag_value TEXT NOT NULL, - PRIMARY KEY (data_item_id, tag_index) -); -``` - -#### Bundle Processing Indexes - -```sql --- Root transaction lookups -CREATE INDEX bundle_data_items_root_transaction_id_id_idx - ON bundle_data_items (root_transaction_id, id); - --- Height-based queries -CREATE INDEX stable_data_items_height_id_idx - ON stable_data_items (height DESC, id DESC); - --- Tag filtering -CREATE INDEX stable_data_item_tags_name_value_data_item_id_idx - ON stable_data_item_tags (tag_name, tag_value, data_item_id); -``` - -### 4. Moderation Database (`moderation.db`) - -**Purpose**: Content moderation and filtering. - -#### Tables - -```sql --- Blocked content by ID -CREATE TABLE blocked_ids ( - id TEXT PRIMARY KEY, - blocked_at INTEGER NOT NULL, - source TEXT, - notes TEXT -); - --- Blocked content by hash -CREATE TABLE blocked_hashes ( - hash TEXT PRIMARY KEY, - blocked_at INTEGER NOT NULL, - source TEXT, - notes TEXT -); - --- Blocked ArNS names -CREATE TABLE blocked_names ( - name TEXT PRIMARY KEY, - blocked_at INTEGER NOT NULL, - source TEXT, - notes TEXT -); -``` - ---- - -## Indexing Architecture - -### Index Design Principles - -1. **Height-First Ordering**: Most queries filter by block height -2. **Compound Indexes**: Match common query patterns -3. **Partial Indexes**: Filter unnecessary rows at index level -4. **Covering Indexes**: Include all needed columns to avoid table lookups - -### Key Index Patterns - -#### 1. Height + Secondary Sort -```sql --- Optimized for GraphQL cursor pagination -CREATE INDEX stable_transactions_height_block_transaction_index_idx - ON stable_transactions (height DESC, block_transaction_index DESC); -``` - -#### 2. Tag Filtering -```sql --- Efficient tag-based queries -CREATE INDEX stable_transaction_tags_name_value_tx_id_idx - ON stable_transaction_tags (tag_name, tag_value, tx_id); -``` - -#### 3. Partial Indexes -```sql --- Only index format 2 transactions with data -CREATE INDEX stable_transactions_offset_idx - ON stable_transactions (offset) - WHERE format = 2 AND data_size > 0; -``` - -#### 4. Verification Indexes -```sql --- Prioritized verification queue -CREATE INDEX contiguous_data_ids_verification_priority_retry_idx - ON contiguous_data_ids ( - verification_priority DESC, - verification_retry_count ASC, - id ASC - ) - WHERE verified = FALSE; -``` - -### Tag Selectivity System - -```typescript -// src/config.ts:482-496 -export const TAG_SELECTIVITY = { - 'Parent-Folder-Id': 20, // High selectivity - 'Message': 20, - 'Drive-Id': 10, - 'Process': 10, - 'Recipient': 10, - 'App-Name': -10, // Low selectivity - 'Content-Type': -10, - 'Data-Protocol': -10, -}; -``` - -This influences query optimization by prioritizing highly selective tags. - ---- - -## Worker Process Architecture - -### Worker Thread Model - -```typescript -// Message-passing architecture for thread safety -interface WorkerMessage { - method: string; - args: any[]; -} - -interface WorkerResponse { - error?: Error; - result?: any; -} -``` - -### Key Worker Processes - -#### 1. **BlockImporter** -- **Purpose**: Import blockchain blocks -- **Queue Management**: Prioritizes recent blocks -- **Error Handling**: Retries with exponential backoff - -```typescript -// src/workers/block-importer.ts -export class BlockImporter { - async importBlock(height: number): Promise { - // 1. Fetch block from chain - // 2. Store in new_blocks table - // 3. Import associated transactions - // 4. Emit BLOCK_IMPORTED event - } -} -``` - -#### 2. **TransactionImporter** -- **Purpose**: Process and index transactions -- **Features**: Batch processing, tag extraction -- **Performance**: Processes up to 1000 transactions per batch - -#### 3. **DataItemIndexer** -- **Purpose**: Index data items from ANS-104 bundles -- **Queue Size**: Configurable (default 100,000 items) -- **Features**: Content type detection, tag indexing - -#### 4. **BundleRepairWorker** -- **Purpose**: Retry failed bundle processing -- **Schedule**: Every 5 minutes -- **Strategy**: Exponential backoff with max retries - -#### 5. **DataVerificationWorker** -- **Purpose**: Verify data integrity -- **Priority System**: Higher priority items verified first -- **Retry Logic**: Tracks attempts with timestamps - -#### 6. **SQLiteWalCleanupWorker** -- **Purpose**: Manage WAL file size -- **Frequency**: Configurable (default disabled) -- **Action**: PRAGMA wal_checkpoint(TRUNCATE) - -### Worker Communication Pattern - -``` -Main Thread → Worker Pool → Database - ↑ ↓ - └── Event Emitter ←┘ -``` - ---- - -## Data Flow and Processing - -### Two-Stage Processing Model - -``` -┌─────────────────┐ 50 blocks ┌──────────────────┐ -│ New Tables │ ─────────────────→ │ Stable Tables │ -│ (Unconfirmed) │ │ (Confirmed) │ -└─────────────────┘ └──────────────────┘ -``` - -### 1. New Data Stage - -**Tables**: `new_blocks`, `new_transactions`, `new_transaction_tags` - -**Characteristics**: -- Holds unconfirmed blockchain data -- Subject to reorganization -- Minimal indexes for write performance - -### 2. Stable Data Stage - -**Tables**: `stable_blocks`, `stable_transactions`, `stable_transaction_tags` - -**Characteristics**: -- Data confirmed by 50+ blocks -- Fully indexed for query performance -- Immutable (except for repairs) - -### Data Flow Sequence - -```typescript -// Simplified flow -async function processBlock(block: Block) { - // 1. Import to new_blocks - await db.saveNewBlock(block); - - // 2. Import transactions to new_transactions - for (const tx of block.transactions) { - await db.saveNewTransaction(tx); - await db.saveNewTransactionTags(tx.tags); - } - - // 3. After 50 blocks, flush to stable tables - if (block.height % 50 === 0) { - await db.flushStableData(block.height - 50); - } -} -``` - -### Flush Process - -```sql --- Move confirmed data from new to stable tables -BEGIN TRANSACTION; - --- Copy blocks -INSERT INTO stable_blocks -SELECT * FROM new_blocks -WHERE height < @flush_height; - --- Copy transactions with relationships -INSERT INTO stable_transactions -SELECT nt.* FROM new_transactions nt -JOIN new_block_transactions nbt ON nbt.transaction_id = nt.id -WHERE nbt.height < @flush_height; - --- Clean up new tables -DELETE FROM new_blocks WHERE height < @flush_height; - -COMMIT; -``` - ---- - -## Query Patterns and Optimization - -### GraphQL Query Support - -The database is optimized for GraphQL queries with specific patterns: - -#### 1. Cursor-Based Pagination - -```typescript -interface Cursor { - height: number; - blockTransactionIndex?: number; - dataItemIndex?: number; -} - -// Encoded as base64 for GraphQL -const encodedCursor = base64url.encode(JSON.stringify(cursor)); -``` - -#### 2. Sort Orders - -```typescript -enum SortOrder { - HEIGHT_DESC = 'HEIGHT_DESC', // Default, newest first - HEIGHT_ASC = 'HEIGHT_ASC', // Oldest first -} -``` - -#### 3. Tag Filtering - -```sql --- Efficient tag queries using indexes -SELECT DISTINCT t.* -FROM stable_transactions t -JOIN stable_transaction_tags tt ON tt.tx_id = t.id -WHERE tt.tag_name = @name - AND tt.tag_value = @value -ORDER BY t.height DESC, t.block_transaction_index DESC -LIMIT @limit; -``` - -### Query Optimization Strategies - -#### 1. **Index-Only Scans** -```sql --- Covering index includes all needed columns -CREATE INDEX tx_owner_height_idx ON stable_transactions - (owner_address, height DESC, block_transaction_index DESC, id); -``` - -#### 2. **Query Rewriting** -```typescript -// Transform OR conditions to UNION for better performance -// Instead of: WHERE owner = X OR target = X -// Use: SELECT ... WHERE owner = X UNION SELECT ... WHERE target = X -``` - -#### 3. **Batch Loading** -```typescript -// Load related data in batches to avoid N+1 queries -const txIds = transactions.map(tx => tx.id); -const allTags = await db.getTransactionTags(txIds); -``` - ---- - -## Data Verification System - -### Verification Architecture - -```typescript -interface VerificationMetadata { - verified: boolean; - verification_retry_count: number; - verification_priority: number; - first_verification_attempted_at?: number; - last_verification_attempted_at?: number; -} -``` - -### Verification Process - -1. **Priority Assignment** - - User-requested data: Priority 100 - - Bundle data: Priority 80 - - Background verification: Priority 0 - -2. **Retry Strategy** - ```sql - -- Select next items to verify - SELECT id FROM contiguous_data_ids - WHERE verified = FALSE - AND verification_retry_count < 5 - ORDER BY - verification_priority DESC, - verification_retry_count ASC, - id ASC - LIMIT 100; - ``` - -3. **Verification Steps** - - Fetch data from source - - Calculate hash - - Compare with stored hash - - Update verification status - -### Verification Worker Configuration - -```typescript -// src/config.ts:261-287 -export const ENABLE_BACKGROUND_DATA_VERIFICATION = true; -export const BACKGROUND_DATA_VERIFICATION_INTERVAL_SECONDS = 600; // 10 min -export const BACKGROUND_DATA_VERIFICATION_WORKER_COUNT = 1; -export const MIN_DATA_VERIFICATION_PRIORITY = 80; -export const MAX_VERIFICATION_RETRIES = 5; -``` - ---- - -## Migration Management - -### Migration System - -```bash -# Migration file naming -migrations/ -├── 0001.core.create-tables.sql -├── 0002.data.add-verification.sql -├── 0003.bundles.add-retry-tracking.sql -└── down/ - ├── 0001.core.create-tables.sql - └── 0002.data.add-verification.sql -``` - -### Migration Tracking - -```sql --- Migrations table in each database -CREATE TABLE migrations ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL UNIQUE, - applied_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')) -); -``` - -### SQLite-Specific Considerations - -```sql --- SQLite requires separate ALTER TABLE statements --- Not supported: ALTER TABLE ADD COLUMN a, ADD COLUMN b; - --- Correct approach: -ALTER TABLE contiguous_data_ids ADD COLUMN verification_retry_count INTEGER NOT NULL DEFAULT 0; -ALTER TABLE contiguous_data_ids ADD COLUMN verification_priority INTEGER NOT NULL DEFAULT 0; -``` - -### Migration Commands - -```bash -# Create new migration -yarn db:migrate create --folder migrations --name data.add-verification.sql - -# Apply migrations -yarn db:migrate up - -# Rollback -yarn db:migrate down --step 1 -``` - ---- - -## Performance Optimizations - -### 1. **Database Attachment** - -```sql --- Attach databases for cross-database queries -ATTACH DATABASE 'data/sqlite/data.db' AS data; -ATTACH DATABASE 'data/sqlite/bundles.db' AS bundles; - --- Query across databases -SELECT t.*, d.data_hash -FROM main.stable_transactions t -JOIN data.contiguous_data_ids d ON d.id = t.id; -``` - -### 2. **Statement Caching** - -```typescript -// All SQL statements are prepared and cached -const statements = new Map(); - -function prepareStatement(sql: string): Statement { - if (!statements.has(sql)) { - statements.set(sql, db.prepare(sql)); - } - return statements.get(sql)!; -} -``` - -### 3. **Transaction Batching** - -```typescript -// Batch inserts in transactions -db.transaction(() => { - for (const item of items) { - insertStatement.run(item); - } -})(); -``` - -### 4. **Write-Ahead Logging (WAL)** - -Benefits: -- Concurrent readers don't block writers -- Writers don't block readers -- Better crash recovery - -Configuration: -```sql -PRAGMA journal_mode = WAL; -PRAGMA wal_autocheckpoint = 10000; -- Checkpoint every 10K pages -``` - -### 5. **Query Planning** - -```sql --- Analyze query plans -EXPLAIN QUERY PLAN -SELECT * FROM stable_transactions -WHERE owner_address = ? -ORDER BY height DESC -LIMIT 100; -``` - ---- - -## Background Jobs and Maintenance - -### Job Scheduling - -| Job | Frequency | Purpose | -|-----|-----------|---------| -| Stable Data Flush | Every 600s | Move confirmed data to stable tables | -| Bundle Repair | Every 300s | Retry failed bundles | -| Data Verification | Every 600s | Verify data integrity | -| WAL Cleanup | On-demand | Truncate WAL file | -| Transaction Repair | Every 300s | Fix missing transactions | - -### Maintenance Operations - -#### 1. **WAL Checkpoint** -```typescript -// Force checkpoint to reduce WAL size -await db.run('PRAGMA wal_checkpoint(TRUNCATE)'); -``` - -#### 2. **Vacuum Operations** -```bash -# Reclaim space and defragment -sqlite3 data/sqlite/core.db "VACUUM;" -``` - -#### 3. **Index Maintenance** -```sql --- Rebuild indexes after major changes -REINDEX stable_transactions_height_block_transaction_index_idx; -``` - ---- - -## ClickHouse Integration - -### Purpose - -ClickHouse provides: -- High-performance analytics queries -- Better handling of large datasets -- Specialized indexes for different access patterns - -### Architecture - -```typescript -// src/system.ts:154-166 -export const gqlQueryable: GqlQueryable = (() => { - if (config.CLICKHOUSE_URL !== undefined) { - return new CompositeClickHouseDatabase({ - log, - gqlQueryable: db, // SQLite fallback - url: config.CLICKHOUSE_URL, - username: config.CLICKHOUSE_USER, - password: config.CLICKHOUSE_PASSWORD, - }); - } - return db; -})(); -``` - -### ClickHouse Schema Design - -```sql --- Optimized for different query patterns -CREATE TABLE transactions ( - -- All transaction fields -) ENGINE = ReplacingMergeTree() -ORDER BY (height, block_transaction_index); - -CREATE TABLE id_transactions ( - -- Optimized for ID lookups -) ENGINE = ReplacingMergeTree() -ORDER BY (id); - -CREATE TABLE owner_transactions ( - -- Optimized for owner queries -) ENGINE = ReplacingMergeTree() -ORDER BY (owner_address, height); -``` - -### Query Routing - -```typescript -// Route queries to optimal backend -if (query.hasOwnerFilter()) { - return clickhouse.query('owner_transactions', query); -} else if (query.hasIdFilter()) { - return clickhouse.query('id_transactions', query); -} else { - return sqlite.query(query); // Fallback -} -``` - ---- - -## Architecture Patterns - -### 1. **Event-Driven Processing** - -```typescript -// Workers communicate via events -eventEmitter.on(events.BLOCK_IMPORTED, async (block) => { - await txImporter.importTransactions(block.transactions); -}); - -eventEmitter.on(events.TRANSACTION_IMPORTED, async (tx) => { - if (isAns104Bundle(tx)) { - await bundleProcessor.process(tx); - } -}); -``` - -### 2. **Queue-Based Work Distribution** - -```typescript -class WorkerQueue { - private queue: T[] = []; - private processing = false; - - async push(item: T) { - this.queue.push(item); - if (!this.processing) { - await this.process(); - } - } - - private async process() { - this.processing = true; - while (this.queue.length > 0) { - const batch = this.queue.splice(0, 100); - await this.processBatch(batch); - } - this.processing = false; - } -} -``` - -### 3. **Circuit Breaker Pattern** - -```typescript -// Prevent cascade failures -const circuitBreaker = new CircuitBreaker(databaseOperation, { - timeout: 5000, - errorThresholdPercentage: 50, - resetTimeout: 30000, -}); -``` - -### 4. **Optimistic Concurrency** - -```typescript -// Use SQLite's built-in locking -try { - await db.transaction(async () => { - // Multiple operations - })(); -} catch (error) { - if (error.code === 'SQLITE_BUSY') { - // Retry with backoff - } -} -``` - ---- - -## Monitoring and Observability - -### Metrics Collection - -```typescript -// Database operation metrics -metrics.dbOperationDuration.observe({ - operation: 'insert', - table: 'stable_transactions', -}, duration); - -// Queue depth monitoring -metrics.registerQueueLengthGauge('dataItemIndexer', { - length: () => dataItemIndexer.queueDepth(), -}); -``` - -### Health Checks - -```sql --- Check for processing lag -SELECT - MAX(height) as latest_block, - strftime('%s', 'now') - MAX(timestamp) as seconds_behind -FROM stable_blocks; - --- Monitor verification backlog -SELECT - COUNT(*) as unverified_count, - MIN(verification_priority) as min_priority -FROM contiguous_data_ids -WHERE verified = FALSE; -``` - ---- - -## Best Practices and Guidelines - -### 1. **Index Design** -- Create indexes that match query patterns -- Use partial indexes to reduce index size -- Consider covering indexes for read-heavy queries - -### 2. **Transaction Management** -- Batch related operations in transactions -- Keep transactions short to reduce lock time -- Handle SQLITE_BUSY errors gracefully - -### 3. **Query Optimization** -- Use EXPLAIN QUERY PLAN to verify index usage -- Avoid SELECT * in production queries -- Limit result sets with appropriate LIMIT clauses - -### 4. **Schema Evolution** -- Always provide both up and down migrations -- Test migrations on copy of production data -- Consider backward compatibility for running services - -### 5. **Monitoring** -- Track query performance over time -- Monitor WAL size and checkpoint frequency -- Alert on processing lag or queue depth - ---- - -## Conclusion - -The ar.io node database architecture demonstrates sophisticated design patterns optimized for blockchain data indexing. The multi-database approach with specialized schemas, worker-based concurrency, and advanced indexing strategies provides a robust foundation for processing Arweave data at scale while maintaining consistency and performance. - -Key strengths include: -- **Separation of concerns** through multiple databases -- **Scalable processing** via worker threads and queues -- **Data integrity** through two-stage processing -- **Performance optimization** with strategic indexing -- **Operational resilience** through retry mechanisms and circuit breakers - -This architecture successfully balances the competing demands of high-throughput data ingestion, complex query requirements, and operational maintainability. - ---- - -*Document prepared for technical review and implementation guidance.* \ No newline at end of file From d9d01537a635ecb34743b80994a653392c0d1d53 Mon Sep 17 00:00:00 2001 From: vilenarios Date: Thu, 25 Jun 2026 17:17:04 +0000 Subject: [PATCH 12/20] chore(config): remove dead ARWEAVE_PEER_CHUNK_POST_* env vars These three vars (MIN_SUCCESS_COUNT, MAX_PEER_ATTEMPT_COUNT, CONCURRENCY_LIMIT) were introduced in PE-7945 (f0d127af) for the original background peer-broadcast path, which has since been superseded by ArweaveCompositeClient.broadcastChunk and the live CHUNK_POST_* family. They had zero code consumers but were still plumbed through docker-compose.yaml, so an operator setting them got silence. Worse, the startup validation (MAX < MIN throws) could crash boot on a "tuning" attempt that otherwise did nothing. Remove the consts + validation and the compose passthrough. (The only docs referencing these vars were the orphaned AI-generated ar-io-0X drafts, removed wholesale in a separate PR.) Verified: zero remaining code references; config test 20/20; eslint clean. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XQPK4TXcVoXoyFp6sNW2Lr --- docker-compose.yaml | 3 --- src/config.ts | 29 ----------------------------- 2 files changed, 32 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index c89968212..90254e01f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -170,9 +170,6 @@ services: - CHUNK_OFFSET_CHAIN_FALLBACK_TX_OFFSET_CACHE_TTL_MS=${CHUNK_OFFSET_CHAIN_FALLBACK_TX_OFFSET_CACHE_TTL_MS:-} - ARWEAVE_PEER_CHUNK_GET_MAX_PEER_ATTEMPT_COUNT=${ARWEAVE_PEER_CHUNK_GET_MAX_PEER_ATTEMPT_COUNT:-} - ARWEAVE_PEER_CHUNK_GET_PEER_SELECTION_COUNT=${ARWEAVE_PEER_CHUNK_GET_PEER_SELECTION_COUNT:-} - - ARWEAVE_PEER_CHUNK_POST_CONCURRENCY_LIMIT=${ARWEAVE_PEER_CHUNK_POST_CONCURRENCY_LIMIT:-} - - ARWEAVE_PEER_CHUNK_POST_MAX_PEER_ATTEMPT_COUNT=${ARWEAVE_PEER_CHUNK_POST_MAX_PEER_ATTEMPT_COUNT:-} - - ARWEAVE_PEER_CHUNK_POST_MIN_SUCCESS_COUNT=${ARWEAVE_PEER_CHUNK_POST_MIN_SUCCESS_COUNT:-} - ARWEAVE_CHUNK_GET_GEOMETRY_TIMEOUT_MS=${ARWEAVE_CHUNK_GET_GEOMETRY_TIMEOUT_MS:-} - ARWEAVE_CHUNK_GET_GEOMETRY_RETRY_COUNT=${ARWEAVE_CHUNK_GET_GEOMETRY_RETRY_COUNT:-} - WEBHOOK_TARGET_SERVERS=${WEBHOOK_TARGET_SERVERS:-} diff --git a/src/config.ts b/src/config.ts index 8c5ebb1ca..03a0d08c2 100644 --- a/src/config.ts +++ b/src/config.ts @@ -873,35 +873,6 @@ export const CHUNK_GET_BASE64_SIZE_BYTES = +env.varOrDefault( // Maximum raw chunk size (256 KiB) - used for raw binary chunk endpoint rate limiting export const MAX_CHUNK_SIZE = 256 * 1024; -// Arweave network peer post success goal -// setting to 0 means this behaviour is disabled. -export const ARWEAVE_PEER_CHUNK_POST_MIN_SUCCESS_COUNT = +env.varOrDefault( - 'ARWEAVE_PEER_CHUNK_POST_MIN_SUCCESS_COUNT', - '2', -); - -// The maximum number of peers to attempt to POST to before giving up -export const ARWEAVE_PEER_CHUNK_POST_MAX_PEER_ATTEMPT_COUNT = +env.varOrDefault( - 'ARWEAVE_PEER_CHUNK_POST_MAX_PEER_ATTEMPT_COUNT', - '5', -); - -if ( - ARWEAVE_PEER_CHUNK_POST_MAX_PEER_ATTEMPT_COUNT < - ARWEAVE_PEER_CHUNK_POST_MIN_SUCCESS_COUNT -) { - throw new Error( - 'ARWEAVE_PEER_CHUNK_POST_MAX_ATTEMPT_PEER_COUNT must be greater than or equal to ARWEAVE_PEER_CHUNK_POST_MIN_SUCCESS_COUNT', - ); -} - -// If ARWEAVE_PEER_CHUNK_POST_MIN_SUCCESS_COUNT is set non-zero, this -// value defines how many chunks to post to peers in parallel. -export const ARWEAVE_PEER_CHUNK_POST_CONCURRENCY_LIMIT = +env.varOrDefault( - 'ARWEAVE_PEER_CHUNK_POST_CONCURRENCY_LIMIT', - '3', -); - // The maximum number of peers to attempt when fetching a chunk via GET export const ARWEAVE_PEER_CHUNK_GET_MAX_PEER_ATTEMPT_COUNT = env.positiveIntOrDefault('ARWEAVE_PEER_CHUNK_GET_MAX_PEER_ATTEMPT_COUNT', 5); From d9aea1de1648b5eccbc31e93a37b609889845e39 Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Thu, 25 Jun 2026 21:48:12 -0700 Subject: [PATCH 13/20] fix(data): guard chunk streaming against non-terminating loops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TxChunksDataSource's full-stream read loop terminated only on byte accounting (bytes < size), advancing by chunkData.chunk.length. A source or cache returning a zero-length chunk left bytes unchanged, so the loop re-requested the same offset forever — observed in production as a 5.1M-span Honeycomb trace for tx QY5bDvdGa9Q_GcdxEFlvTJxUW5UPrSyvf6yULjbsv5g (a 2805-byte, single-chunk L1 tx) made up of ~1.7M repeated ~0.1ms cache hits on one offset. Add two forward-progress guards in the read loop: - abort on a zero-length chunk (primary cause) - abort once the chunk count exceeds ceil(size / MAX_CHUNK_SIZE) + 1, a backstop against pathological tx geometry Both increment chunk_stream_aborts_total{reason} and destroy the stream with a descriptive error instead of spinning. Co-Authored-By: Claude Opus 4.8 --- src/data/tx-chunks-data-source.test.ts | 80 ++++++++++++++++++++++++++ src/data/tx-chunks-data-source.ts | 36 ++++++++++++ src/metrics.ts | 8 +++ 3 files changed, 124 insertions(+) diff --git a/src/data/tx-chunks-data-source.test.ts b/src/data/tx-chunks-data-source.test.ts index a83ffe12b..7be4e54b6 100644 --- a/src/data/tx-chunks-data-source.test.ts +++ b/src/data/tx-chunks-data-source.test.ts @@ -190,6 +190,86 @@ describe('TxChunksDataSource', () => { }); }); + describe('forward-progress guards', () => { + it('should abort the stream when a zero-length chunk is returned', async () => { + mock.method(metrics.chunkStreamAbortsTotal, 'inc'); + // A chunk that returns no bytes would never advance the byte counter, + // re-requesting the same offset forever. The guard must surface an + // error instead of spinning. + mock.method(chunkSource, 'getChunkDataByAny', async () => ({ + hash: Buffer.alloc(0), + chunk: Buffer.alloc(0), + })); + + const data = await txChunkRetriever.getData({ + id: TX_ID, + requestAttributes, + }); + + await assert.rejects( + (async () => { + for await (const _chunk of data.stream) { + // consume + } + })(), + { message: /Zero-length chunk/ }, + ); + + assert.equal( + (metrics.chunkStreamAbortsTotal.inc as any).mock.callCount(), + 1, + ); + assert.equal( + (metrics.chunkStreamAbortsTotal.inc as any).mock.calls[0].arguments[0] + .reason, + 'zero_length_chunk', + ); + }); + + it('should abort the stream when the chunk count exceeds the tx size bound', async () => { + mock.method(metrics.chunkStreamAbortsTotal, 'inc'); + // size is 256000 bytes (maxChunks = ceil(256000 / 262144) + 1 = 2). + // Returning a single byte per chunk would otherwise require 256000 + // fetches; the backstop must abort once the count bound is hit. + mock.method(chunkSource, 'getChunkDataByAny', async () => ({ + hash: Buffer.alloc(0), + chunk: Buffer.from([1]), + })); + + const data = await txChunkRetriever.getData({ + id: TX_ID, + requestAttributes, + }); + + let received = 0; + await assert.rejects( + (async () => { + for await (const _chunk of data.stream) { + received++; + } + })(), + { message: /Chunk count exceeded maximum/ }, + ); + + // The exact count the consumer observes is timing-dependent (destroy() + // discards any buffered-but-unconsumed chunk), but it must abort near the + // bound rather than running the full size/1-byte = 256000 iterations. + assert.ok( + received <= 2, + `should abort near the chunk bound, emitted ${received}`, + ); + assert.equal( + (metrics.chunkStreamAbortsTotal.inc as any).mock.callCount(), + 1, + ); + assert.equal( + (metrics.chunkStreamAbortsTotal.inc as any).mock.calls[0].arguments[0] + .reason, + 'chunk_count_exceeded', + ); + }); + }); + describe('range requests', () => { it('should stream a range within a single chunk', async () => { // Mock range-specific metrics diff --git a/src/data/tx-chunks-data-source.ts b/src/data/tx-chunks-data-source.ts index 5621ef65d..3ae6937cf 100644 --- a/src/data/tx-chunks-data-source.ts +++ b/src/data/tx-chunks-data-source.ts @@ -8,6 +8,7 @@ import { Readable } from 'node:stream'; import { anySignal, ClearableSignal } from 'any-signal'; import pLimit, { LimitFunction } from 'p-limit'; import winston from 'winston'; +import { MAX_CHUNK_SIZE } from '../config.js'; import { generateRequestAttributes } from '../lib/request-attributes.js'; import { streamRangeData } from '../lib/stream-tx-range.js'; import { startChildSpan } from '../tracing.js'; @@ -349,6 +350,14 @@ export class TxChunksDataSource implements ContiguousDataSource { const streamStartTime = Date.now(); + // Upper bound on the number of chunks a well-formed tx can produce: each + // chunk is at most MAX_CHUNK_SIZE bytes, so `size` bytes need at most + // ceil(size / MAX_CHUNK_SIZE) chunks. The +1 tolerates boundary + // rebalancing of the final chunks. This is a backstop against + // pathological geometry (e.g. a corrupt tx size/offset) driving an + // unbounded number of fetches. + const maxChunks = Math.ceil(size / MAX_CHUNK_SIZE) + 1; + const stream = new Readable({ autoDestroy: true, read: async function () { @@ -362,6 +371,33 @@ export class TxChunksDataSource implements ContiguousDataSource { } const chunkData = await chunkDataPromise; + + // Forward-progress guard: a zero-length chunk would not advance + // `bytes`, leaving `bytes < size` true forever and re-requesting + // the same offset indefinitely (observed as multi-million-span + // traces of repeated cache hits on a single offset). Abort rather + // than spin. + if (chunkData.chunk.length === 0) { + metrics.chunkStreamAbortsTotal.inc({ + reason: 'zero_length_chunk', + }); + throw new Error( + `Zero-length chunk for ${id} at relativeOffset ${bytes}; ` + + `aborting to avoid a non-terminating chunk loop`, + ); + } + + // Backstop guard: never read more chunks than the tx size can hold. + if (totalChunks >= maxChunks) { + metrics.chunkStreamAbortsTotal.inc({ + reason: 'chunk_count_exceeded', + }); + throw new Error( + `Chunk count exceeded maximum ${maxChunks} for ${id} ` + + `(size ${size}); aborting chunk loop`, + ); + } + this.push(chunkData.chunk); totalChunks++; bytes += chunkData.chunk.length; diff --git a/src/metrics.ts b/src/metrics.ts index 22761309a..663cf9954 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -833,6 +833,14 @@ export const chunkFirstDataTimeoutsTotal = new promClient.Counter({ labelNames: ['request_type'] as const, }); +export const chunkStreamAbortsTotal = new promClient.Counter({ + name: 'chunk_stream_aborts_total', + help: + 'Count of chunk data streams aborted by a forward-progress guard ' + + '(zero-length chunk or chunk-count overrun)', + labelNames: ['reason'] as const, +}); + // // Negative data cache metrics // From 36eb3759c7e96c994cf5ad1a27921b7bbb59308b Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Thu, 25 Jun 2026 22:04:10 -0700 Subject: [PATCH 14/20] fix(chunks): reject and self-heal zero-length chunk cache poison MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Honeycomb trace 7fd5b41a (tx QY5bDvdGa9Q…, a 2805-byte single-chunk L1 tx) showed the first ReadThroughChunkDataCache.getChunkDataByAny span as a cache HIT at relative_offset 0 with tx_size 2805, followed by ~1.7M identical-offset reads. The chunk data store held a poisoned 0-byte file for (dataRoot, 0): FsChunkDataStore.has() reports a hit on a 0-byte file and get() serves an empty chunk, so the TxChunksDataSource stream loop never advanced `bytes` and re-requested the same offset forever. Root cause: nothing validated chunk length, so a source that once returned an empty chunk was persisted (set() writes unconditionally) and re-served indefinitely. Harden every layer: - FsChunkDataStore.set: refuse to persist zero-length chunks - FsChunkDataStore.get / getByAbsoluteOffset: treat an existing 0-byte file as a miss, so already-poisoned entries self-heal on next refetch - ReadThroughChunkDataCache: reject a zero-length chunk from the source (don't cache it; throw so the retrieval cascade falls through) - TxChunksDataSource: only treat a zero-length chunk as fatal when size > 0 New metric chunk_zero_length_total{stage} tracks rejections at source_fetch / cache_read / cache_write. Updates the prior store test that asserted empty chunks round-trip (the poison contract). Co-Authored-By: Claude Opus 4.8 --- .../read-through-chunk-data-cache.test.ts | 28 +++++++++++++ src/data/read-through-chunk-data-cache.ts | 14 +++++++ src/data/tx-chunks-data-source.ts | 2 +- src/metrics.ts | 8 ++++ src/store/fs-chunk-data-store.test.ts | 27 +++++++++++-- src/store/fs-chunk-data-store.ts | 40 +++++++++++++++++++ 6 files changed, 114 insertions(+), 5 deletions(-) diff --git a/src/data/read-through-chunk-data-cache.test.ts b/src/data/read-through-chunk-data-cache.test.ts index a371b7790..2c4e1a7e7 100644 --- a/src/data/read-through-chunk-data-cache.test.ts +++ b/src/data/read-through-chunk-data-cache.test.ts @@ -115,5 +115,33 @@ describe('ReadThroughChunkDataCache', () => { assert.deepEqual(chunkDataStoreSetSpy.mock.callCount(), 1); assert.deepEqual(networkSpy.mock.callCount(), 1); }); + + it('should reject a zero-length chunk from the source without caching it', async () => { + const chunkDataStoreGetSpy = mock.method( + chunkDataStore, + 'get', + async () => undefined, + ); + const chunkDataStoreSetSpy = mock.method(chunkDataStore, 'set'); + mock.method(chunkSource, 'getChunkDataByAny', async () => ({ + hash: Buffer.alloc(0), + chunk: Buffer.alloc(0), + })); + + await assert.rejects( + () => + chunkCache.getChunkDataByAny({ + txSize: TX_SIZE, + absoluteOffset: ABSOLUTE_OFFSET, + dataRoot: B64_DATA_ROOT, + relativeOffset: 0, + }), + { message: /zero-length chunk/ }, + ); + + assert.deepEqual(chunkDataStoreGetSpy.mock.callCount(), 1); + // The invalid empty chunk must not be persisted. + assert.deepEqual(chunkDataStoreSetSpy.mock.callCount(), 0); + }); }); }); diff --git a/src/data/read-through-chunk-data-cache.ts b/src/data/read-through-chunk-data-cache.ts index 4ca8022e5..796009bd6 100644 --- a/src/data/read-through-chunk-data-cache.ts +++ b/src/data/read-through-chunk-data-cache.ts @@ -7,6 +7,7 @@ import winston from 'winston'; import { tracer } from '../tracing.js'; import { isValidationParams } from '../lib/validation.js'; +import * as metrics from '../metrics.js'; import { ChunkData, @@ -122,6 +123,19 @@ export class ReadThroughChunkDataCache implements ChunkDataByAnySource { ); const sourceDuration = Date.now() - sourceStart; + // Reject invalid empty chunks at the cache boundary: never cache them + // (which would poison this offset) and surface an error so the caller's + // retrieval cascade can fall through to another source instead of + // looping on a non-advancing chunk. + if (chunkData.chunk.length === 0) { + metrics.chunkZeroLengthTotal.inc({ stage: 'source_fetch' }); + span.setAttribute('chunk.zero_length_source', true); + throw new Error( + `Chunk source returned a zero-length chunk for dataRoot ${dataRoot} ` + + `relativeOffset ${relativeOffset}`, + ); + } + span.setAttributes({ 'chunk.source_fetch_duration_ms': sourceDuration, 'chunk.source': chunkData.source ?? 'unknown', diff --git a/src/data/tx-chunks-data-source.ts b/src/data/tx-chunks-data-source.ts index 3ae6937cf..f0c341854 100644 --- a/src/data/tx-chunks-data-source.ts +++ b/src/data/tx-chunks-data-source.ts @@ -377,7 +377,7 @@ export class TxChunksDataSource implements ContiguousDataSource { // the same offset indefinitely (observed as multi-million-span // traces of repeated cache hits on a single offset). Abort rather // than spin. - if (chunkData.chunk.length === 0) { + if (size > 0 && chunkData.chunk.length === 0) { metrics.chunkStreamAbortsTotal.inc({ reason: 'zero_length_chunk', }); diff --git a/src/metrics.ts b/src/metrics.ts index 663cf9954..d680fc2b3 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -841,6 +841,14 @@ export const chunkStreamAbortsTotal = new promClient.Counter({ labelNames: ['reason'] as const, }); +export const chunkZeroLengthTotal = new promClient.Counter({ + name: 'chunk_zero_length_total', + help: + 'Count of invalid zero-length chunks rejected, by pipeline stage ' + + '(source_fetch, cache_read, cache_write)', + labelNames: ['stage'] as const, +}); + // // Negative data cache metrics // diff --git a/src/store/fs-chunk-data-store.test.ts b/src/store/fs-chunk-data-store.test.ts index 91f89ceab..f0375d369 100644 --- a/src/store/fs-chunk-data-store.test.ts +++ b/src/store/fs-chunk-data-store.test.ts @@ -268,7 +268,7 @@ describe('FsChunkDataStore', () => { assert.deepEqual(retrieved.chunk, chunkData.chunk); }); - it('should handle empty chunk data', async () => { + it('should refuse to cache zero-length chunk data', async () => { const dataRoot = 'tne4Fh9gC2AYX_ZUO5fV_ppKe0pwCwjOK4uTtg1OIjk'; const relativeOffset = 0; const chunkData: ChunkData = { @@ -276,11 +276,30 @@ describe('FsChunkDataStore', () => { hash: crypto.createHash('sha256').update(Buffer.alloc(0)).digest(), }; + // A zero-length chunk is invalid and would poison the cache, so set() + // must drop it rather than persist it. await store.set(dataRoot, relativeOffset, chunkData); - const retrieved = await store.get(dataRoot, relativeOffset); - assert.ok(retrieved); - assert.strictEqual(retrieved.chunk.length, 0); + assert.strictEqual(await store.has(dataRoot, relativeOffset), false); + assert.strictEqual(await store.get(dataRoot, relativeOffset), undefined); + }); + + it('should treat a pre-existing zero-length chunk file as a miss', async () => { + // Simulates a cache entry poisoned before this guard existed: a 0-byte + // file on disk must self-heal by reading as a miss (so the caller + // refetches and overwrites it) rather than a valid empty chunk. + const dataRoot = 'Pois0nedZZZ2nHYgKhhI25MzveuYvH7rCd8J0WIVp4EVs'; + const relativeOffset = 0; + + const fs = await import('node:fs'); + const dir = join(tempDir, 'data', 'by-dataroot', 'Po', 'is', dataRoot); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(join(dir, '0'), Buffer.alloc(0)); + + // has() still reports the file exists... + assert.strictEqual(await store.has(dataRoot, relativeOffset), true); + // ...but get() must not serve the empty chunk. + assert.strictEqual(await store.get(dataRoot, relativeOffset), undefined); }); }); diff --git a/src/store/fs-chunk-data-store.ts b/src/store/fs-chunk-data-store.ts index ccbc7ea0e..b90642220 100644 --- a/src/store/fs-chunk-data-store.ts +++ b/src/store/fs-chunk-data-store.ts @@ -10,6 +10,7 @@ import path from 'node:path'; import winston from 'winston'; import { ChunkData, ChunkDataStore } from '../types.js'; +import * as metrics from '../metrics.js'; export class FsChunkDataStore implements ChunkDataStore { private log: winston.Logger; @@ -62,6 +63,20 @@ export class FsChunkDataStore implements ChunkDataStore { if (await this.has(dataRoot, relativeOffset)) { const chunkPath = this.chunkDataRootPath(dataRoot, relativeOffset); const chunk = await fs.promises.readFile(chunkPath); + + // Self-heal poisoned cache entries: a zero-length chunk file is never + // valid and, if served, would stall consumers that advance by chunk + // length (re-requesting the same offset forever). Treat it as a miss + // so the caller refetches and overwrites it. + if (chunk.length === 0) { + metrics.chunkZeroLengthTotal.inc({ stage: 'cache_read' }); + this.log.warn('Ignoring zero-length cached chunk; treating as miss', { + dataRoot, + relativeOffset, + }); + return undefined; + } + const hash = crypto.createHash('sha256').update(chunk).digest(); return { @@ -87,6 +102,18 @@ export class FsChunkDataStore implements ChunkDataStore { try { const symlinkPath = this.absoluteOffsetIndexPath(absoluteOffset); const chunk = await fs.promises.readFile(symlinkPath); // Follows symlink + + // Self-heal poisoned entries (see get()): treat a zero-length chunk as a + // miss so the caller refetches rather than serving invalid empty data. + if (chunk.length === 0) { + metrics.chunkZeroLengthTotal.inc({ stage: 'cache_read' }); + this.log.warn( + 'Ignoring zero-length cached chunk by absolute offset; treating as miss', + { absoluteOffset }, + ); + return undefined; + } + const hash = crypto.createHash('sha256').update(chunk).digest(); return { @@ -127,6 +154,19 @@ export class FsChunkDataStore implements ChunkDataStore { chunkData: ChunkData, absoluteOffset?: number, ): Promise { + // Never persist a zero-length chunk: it is invalid data and poisons the + // cache, since has() would report a hit and get() would return an empty + // buffer that stalls chunk-streaming consumers. + if (chunkData.chunk.length === 0) { + metrics.chunkZeroLengthTotal.inc({ stage: 'cache_write' }); + this.log.warn('Refusing to cache zero-length chunk', { + dataRoot, + relativeOffset, + absoluteOffset, + }); + return; + } + try { const chunkDataRootDir = this.chunkDataRootDir(dataRoot); await fs.promises.mkdir(chunkDataRootDir, { recursive: true }); From 3f331cb3ca8118baf1555a41460c835186b7fbbe Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Thu, 25 Jun 2026 22:21:26 -0700 Subject: [PATCH 15/20] docs(metrics): add TSDoc to chunk guard counters; test(chunks): cover absolute-offset zero-length paths Address CodeRabbit review on #799: - Document chunkStreamAbortsTotal and chunkZeroLengthTotal (label meanings). - Add FsChunkDataStore tests: a poisoned 0-byte by-absolute-offset entry reads as a miss, and a zero-length set() creates no absolute-offset index. Co-Authored-By: Claude Opus 4.8 --- src/metrics.ts | 21 +++++++++++++ src/store/fs-chunk-data-store.test.ts | 44 +++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/src/metrics.ts b/src/metrics.ts index d680fc2b3..8ba891a45 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -833,6 +833,16 @@ export const chunkFirstDataTimeoutsTotal = new promClient.Counter({ labelNames: ['request_type'] as const, }); +/** + * Counts chunk-streaming requests aborted by a `TxChunksDataSource` + * forward-progress guard, preventing a non-terminating chunk loop. + * + * @remarks Label `reason` is one of: + * - `zero_length_chunk` — a chunk returned no bytes, so the stream could not + * advance and would otherwise re-request the same offset forever. + * - `chunk_count_exceeded` — more chunks were read than the tx size can hold + * (`> ceil(size / MAX_CHUNK_SIZE) + 1`), a backstop against bad geometry. + */ export const chunkStreamAbortsTotal = new promClient.Counter({ name: 'chunk_stream_aborts_total', help: @@ -841,6 +851,17 @@ export const chunkStreamAbortsTotal = new promClient.Counter({ labelNames: ['reason'] as const, }); +/** + * Counts invalid zero-length chunks rejected before they can poison the chunk + * cache (a persisted empty chunk is served as a hit and stalls consumers). + * + * @remarks Label `stage` is one of: + * - `source_fetch` — a chunk source returned an empty chunk; rejected and not + * cached so the retrieval cascade can fall through. + * - `cache_read` — an existing zero-length cache entry was read and treated as + * a miss so it self-heals on the next refetch. + * - `cache_write` — a zero-length chunk write was refused, preventing poisoning. + */ export const chunkZeroLengthTotal = new promClient.Counter({ name: 'chunk_zero_length_total', help: diff --git a/src/store/fs-chunk-data-store.test.ts b/src/store/fs-chunk-data-store.test.ts index f0375d369..10a04b6c6 100644 --- a/src/store/fs-chunk-data-store.test.ts +++ b/src/store/fs-chunk-data-store.test.ts @@ -301,6 +301,50 @@ describe('FsChunkDataStore', () => { // ...but get() must not serve the empty chunk. assert.strictEqual(await store.get(dataRoot, relativeOffset), undefined); }); + + it('should treat a pre-existing zero-length absolute-offset entry as a miss', async () => { + // A 0-byte file reachable via the by-absolute-offset index must also + // self-heal as a miss rather than returning an empty chunk. + const absoluteOffset = 51530681327863; + // by-absolute-offset/{floor(abs/1e12)}/{floor(abs/1e9)%1000}/{abs} + const fs = await import('node:fs'); + const dir = join(tempDir, 'data', 'by-absolute-offset', '51', '530'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(join(dir, String(absoluteOffset)), Buffer.alloc(0)); + + assert.strictEqual( + await store.getByAbsoluteOffset(absoluteOffset), + undefined, + ); + }); + + it('should not create an absolute-offset index for a zero-length chunk', async () => { + const dataRoot = 'zeroAbsIdxZZ2nHYgKhhI25MzveuYvH7rCd8J0WIVp4EVs'; + const relativeOffset = 0; + const absoluteOffset = 51530681327863; + const chunkData: ChunkData = { + chunk: Buffer.alloc(0), + hash: crypto.createHash('sha256').update(Buffer.alloc(0)).digest(), + }; + + // set() refuses the zero-length write before any index is created. + await store.set(dataRoot, relativeOffset, chunkData, absoluteOffset); + + const fs = await import('node:fs'); + const indexPath = join( + tempDir, + 'data', + 'by-absolute-offset', + '51', + '530', + String(absoluteOffset), + ); + assert.strictEqual(fs.existsSync(indexPath), false); + assert.strictEqual( + await store.getByAbsoluteOffset(absoluteOffset), + undefined, + ); + }); }); describe('error handling', () => { From cdab9ccf0bfbdf0d88bc96b6398e27c0e274fffd Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Thu, 25 Jun 2026 23:32:49 -0700 Subject: [PATCH 16/20] feat(gql): extend owner_projection routing to owners+ids queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-id transactions(ids:[...]) queries are ~99% of production TOO_MANY_ROWS failures: `id` is the last sort-key column so `id IN (...)` has no seek and relies on id_bloom, which lights up most granules (~100 ids reads ~308M rows). An owner filter alone isn't enough (~13.6M on the main table for 100 ids), but seeking the owner's slice via owner_projection drops it to ~556K (measured; verified end-to-end at 590K rows / 35ms under the 10M cap). ownerProjectionApplies now routes owners+ids through the projection regardless of tags (the id list bounds the result). optimize_read_in_order=0 is a no-op here (id queries carry no ORDER BY); the win is purely the owner seek. The height-windowing fallback is disabled for id queries (it is height-ordered and needs the cursor predicate, which id queries don't carry), so a whale owner (>10M footprint) + ids still surfaces the 158 — rare, no worse than before. Gated by the existing CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED flag. Reduces live failures once clients add owners to their id queries; genuinely ownerless batches still need the schema-level id_bloom / id-ordered-table fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...ickhouse-gql-owner-filter-too-many-rows.md | 35 ++++++ docs/envs.md | 4 +- src/database/composite-clickhouse.test.ts | 103 ++++++++++++++++++ src/database/composite-clickhouse.ts | 27 ++++- 4 files changed, 161 insertions(+), 8 deletions(-) diff --git a/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md b/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md index 49b1b91c8..56c64b889 100644 --- a/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md +++ b/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md @@ -206,6 +206,41 @@ Justified only if heavy-owner `file` deep pagination shows up hot in profiling re-sort), then **drop `owner_projection`** to offset its ~67 GiB / write / merge cost. +## Extension: `owners + ids` (the dominant 158 class) + +Follow-up after the merge: a 7-day `query_log` audit on a canary showed that +**multi-id `transactions(ids:[...])` queries are ~99% of all `TOO_MANY_ROWS` +failures** (~1,670/day), dominated by 100-id batch fetches — far more than the +owner+tag class the initial fix addressed (~34/7d). Root cause is the same +`id_bloom` leakiness documented elsewhere: `id` is the *last* sort-key column, so +`id IN (...)` has no seek and relies on the bloom, which lights up most granules. + +Measured on a canary for one owner, 100 real ids: + +| query | rows to read | granules | +|-------|-------------:|---------:| +| ids only (no owner) | **308,254,234** | 37,703 | +| ids + owner, main table | **13,572,231** | 1,660 | +| ids + owner, **projection** | **555,663** | 70 | + +So an owner filter alone isn't enough (13.6M still trips the cap for ~100 ids — +the bloom already selected most granules), but seeking the owner via the +projection drops it to ~556K. The eligibility check +(`ownerProjectionApplies`) therefore routes **`owners + ids`** through the +projection regardless of tags (the id list bounds the result), gated by the same +`CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED` flag. + +Notes / limits: +- `read_in_order = 0` is a no-op here (id queries carry no `ORDER BY`); the win + is purely `optimize_use_projections = 1` doing the owner seek. +- The height-windowing fallback is **disabled for id queries** — it's + height-ordered and needs the cursor predicate, which id queries don't carry. A + whale owner (>10M footprint) + ids still surfaces the 158 (rare, no worse than + before). +- This only reduces live failures once **clients add `owners:[x]`** to their id + queries; genuinely ownerless batches still need the schema-level `id_bloom` / + id-ordered-table fix. + ## Overhead **Key framing:** the owner-ordered copy already exists as the 67 GiB diff --git a/docs/envs.md b/docs/envs.md index 2046546f7..94e8091a0 100644 --- a/docs/envs.md +++ b/docs/envs.md @@ -561,8 +561,8 @@ height that would be silently dropped by a `height >= :minHeight` predicate | CLICKHOUSE_MAX_HEIGHT_CACHE_TTL_SECONDS | Number | 60 | TTL for the cached ClickHouse max-height lookup used by the boundary optimization | | CLICKHOUSE_QUERY_TIMEOUT_SECONDS | Number | 3 | Timeout for ClickHouse queries, applied both as server-side `max_execution_time` and client-side HTTP `request_timeout` | | CLICKHOUSE_GQL_MAX_ROWS_TO_READ | Number | 10000000 | Per-query `max_rows_to_read` hint applied to GraphQL queries against the `transactions` table. Queries that would scan more than this throw `Code: 158`, preventing accidental full-table scans if a skip index is bypassed | -| CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED | Boolean | false | When true, owner-filtered GraphQL `transactions` queries are routed through `owner_projection` (and gain a reactive height-windowing fallback if they still trip `max_rows_to_read`). The main `transactions` table is height-ordered, so a sparse owner's rows scatter across millions of granules and finding a page can trip the row cap; the owner-ordered projection seeks straight to the owner's slice. Disabled by default for staged rollout — when off, owner queries plan exactly as before | -| CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES | String | `drive,folder,snapshot` | Comma-separated allowlist of `Entity-Type` tag values eligible for owner_projection routing (only consulted when `CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED` is true). A query qualifies only when it carries an `Entity-Type` filter whose values are ALL in this list. `file` is deliberately omitted: an owner can have millions of files, and routing forces a full sort of the matched set per page (read-in-order is disabled to use the projection), re-done on every page — those large-result queries want the dedicated owner-ordered table instead. Bare-owner and owner+other-tag queries are likewise excluded | +| CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED | Boolean | false | When true, owner-filtered GraphQL `transactions` queries are routed through `owner_projection` (and gain a reactive height-windowing fallback if they still trip `max_rows_to_read`). The main `transactions` table is height-ordered, so a sparse owner's rows scatter across millions of granules and finding a page can trip the row cap; the owner-ordered projection seeks straight to the owner's slice. Also routes `owners + ids` queries through the projection — a multi-id `id IN (...)` filter lights up most of `id_bloom`'s granules (≈ a half-table scan, the dominant source of `TOO_MANY_ROWS`), so adding an owner and seeking the owner's slice keeps it under the cap. Disabled by default for staged rollout — when off, owner queries plan exactly as before | +| CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES | String | `drive,folder,snapshot` | Comma-separated allowlist of `Entity-Type` tag values eligible for owner_projection routing of tag queries (only consulted when `CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED` is true, and not applied to `owners + ids` queries, which are bounded by the id list). A no-id query qualifies only when it carries an `Entity-Type` filter whose values are ALL in this list. `file` is deliberately omitted: an owner can have millions of files, and routing forces a full sort of the matched set per page (read-in-order is disabled to use the projection), re-done on every page — those large-result queries want the dedicated owner-ordered table instead. Bare-owner and owner+other-tag queries are likewise excluded | | CLICKHOUSE_GQL_DEDUPE_HEADROOM | Number | 4 | Multiplier applied to `pageSize + 1` to size the inner LIMIT of the GQL transactions query. **Pagination correctness caveat:** this must be at least as large as the table's effective duplicate factor. If a region of the table has more unmerged ReplacingMergeTree versions per PK than this headroom covers, the deduped window can yield fewer than `pageSize + 1` unique rows even when more matches exist further on — the result will be a short page with `hasNextPage: false` and rows past the window will be silently skipped. Regular background merges keep the duplicate factor at 1-2 in practice, so 4 leaves comfortable headroom; raise this if operators observe short pages (e.g. during a heavy ingest that produces many unmerged parts) | | CLICKHOUSE_SQLITE_CIRCUIT_BREAKER_TIMEOUT_MS | Number | 5000 | Timeout (ms) for the SQLite leg of the composite GQL transactions query. On timeout the response degrades to ClickHouse-only results; ClickHouse timeouts are governed separately by `CLICKHOUSE_QUERY_TIMEOUT_SECONDS` and still propagate to the caller | | CLICKHOUSE_SQLITE_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE | Number | 80 | Error rate (0–100) within the rolling window that trips the SQLite-leg breaker open | diff --git a/src/database/composite-clickhouse.test.ts b/src/database/composite-clickhouse.test.ts index 682104e82..fe82e070f 100644 --- a/src/database/composite-clickhouse.test.ts +++ b/src/database/composite-clickhouse.test.ts @@ -727,6 +727,109 @@ describe('CompositeClickHouseDatabase', () => { assert.ok(!queries.some((q) => q.includes('optimize_read_in_order = 0'))); }); + it('routes owner + ids through the projection', async () => { + const composite = buildComposite({ + sqlite, + ownerProjectionRoutingEnabled: true, + chRowsByLeg: {}, + }); + const queries: string[] = []; + (composite as any).clickhouseClient = { + async query({ query: sqlStr }: { query: string }) { + queries.push(sqlStr); + if (sqlStr.includes('FROM new_transactions')) { + return { json: async () => ({ data: [] }) }; + } + return { + json: async () => ({ + data: [ + chRow({ id: id('a'), height: 100 }), + chRow({ id: id('b'), height: 99 }), + ], + }), + }; + }, + }; + + const result = await composite.getGqlTransactions({ + pageSize: 100, + owners: [id('owner')], + ids: [id('a'), id('b'), id('c')], + }); + + // owner+ids is bounded by the id list, so it routes through the + // projection regardless of tags (no Entity-Type allowlist gate). + const stableQ = queries.find((q) => q.includes('FROM transactions')); + assert.ok( + stableQ !== undefined && + stableQ.includes('optimize_use_projections = 1'), + 'owner+ids should route through the projection', + ); + assert.equal(result.edges.length, 2); + }); + + it('does not window owner+ids on 158 (fails fast)', async () => { + const composite = buildComposite({ + sqlite, + ownerProjectionRoutingEnabled: true, + chRowsByLeg: {}, + }); + let stableCalls = 0; + (composite as any).clickhouseClient = { + async query({ query: sqlStr }: { query: string }) { + if (sqlStr.includes('FROM new_transactions')) { + return { json: async () => ({ data: [] }) }; + } + stableCalls += 1; + const err: any = new Error('Code: 158. TOO_MANY_ROWS'); + err.code = '158'; + throw err; + }, + }; + + await assert.rejects( + composite.getGqlTransactions({ + pageSize: 100, + owners: [id('owner')], + ids: [id('a'), id('b')], + }), + /158|TOO_MANY_ROWS/, + ); + // Height-windowing is height-ordered and needs a cursor predicate, which + // id queries don't carry — so owner+ids does NOT window; the 158 surfaces. + assert.equal(stableCalls, 1); + }); + + it('does not route ids without an owner', async () => { + const composite = buildComposite({ + sqlite, + ownerProjectionRoutingEnabled: true, + chRowsByLeg: {}, + }); + const queries: string[] = []; + (composite as any).clickhouseClient = { + async query({ query: sqlStr }: { query: string }) { + queries.push(sqlStr); + return { json: async () => ({ data: [] }) }; + }, + }; + + await composite.getGqlTransactions({ + pageSize: 100, + ids: [id('a'), id('b')], + }); + + // Ownerless id queries have nothing to seek on — they stay on the + // id_bloom main-table path (projections off), unchanged by the feature. + const stableQ = queries.find((q) => q.includes('FROM transactions')); + assert.ok( + stableQ !== undefined && + stableQ.includes('optimize_use_projections = 0'), + 'ids without an owner should use the id_bloom main-table path', + ); + assert.ok(!queries.some((q) => q.includes('optimize_read_in_order = 0'))); + }); + it('drains a dense window via cursor instead of stranding rows', async () => { const composite = buildComposite({ sqlite, diff --git a/src/database/composite-clickhouse.ts b/src/database/composite-clickhouse.ts index eb3fdffae..38c9bc867 100644 --- a/src/database/composite-clickhouse.ts +++ b/src/database/composite-clickhouse.ts @@ -614,15 +614,22 @@ export class CompositeClickHouseDatabase implements GqlQueryable { tags: { name: string; values: string[] }[], ): boolean { if (!this.ownerProjectionRoutingEnabled) return false; - if (owners.length === 0 || ids.length > 0) return false; + if (owners.length === 0) return false; + // Owner + ids: the id list bounds the result to at most `ids.length` rows, + // so seek the owner's slice via the projection and filter the ids within — + // regardless of any tag filters. On the main table a multi-id `id IN (...)` + // lights up most of id_bloom's granules (≈ a half-table scan), so even with + // an owner filter ~100 ids reads >10M rows and trips the cap; through the + // projection it's an owner-footprint read (measured ~556K rows vs 308M + // ids-only / 13.6M ids+owner on the main table). + if (ids.length > 0) return true; + // Owner + allowlisted Entity-Type, no ids: require an Entity-Type filter and + // NO other tag types. An additional non-Entity-Type tag (e.g. App-Name) only + // narrows the result, but it's an untested shape and the agreed contract + // excludes owner+other-tag queries, so fall back to the default plan. const entityTypeTags = tags.filter( (tag) => tag.name === ENTITY_TYPE_TAG_NAME, ); - // Require an Entity-Type filter and NO other tag types. An additional - // non-Entity-Type tag (e.g. App-Name) only narrows the result, but it's an - // untested shape and the agreed contract excludes owner+other-tag queries, - // so fall back to the default plan rather than route it through the - // projection. if (entityTypeTags.length === 0 || entityTypeTags.length !== tags.length) { return false; } @@ -1087,8 +1094,16 @@ export class CompositeClickHouseDatabase implements GqlQueryable { // Hack 5: an owner-filtered query that still trips max_rows_to_read // through owner_projection means a whale whose footprint exceeds the // cap. Re-run as an adaptive height-windowed walk instead of failing. + // + // Restricted to non-id queries: the windowing walk is height-ordered + // and relies on the cursor predicate to drain a window, but id queries + // carry no ORDER BY or cursor predicate (see addGqlTransactionFilters / + // buildTransactionOrderBy), so the walk can't make per-window progress. + // owner+ids only reaches the cap for whale owners (>10M footprint) — a + // rare case left to surface the 158, no worse than before this feature. if ( this.ownerProjectionApplies(owners, ids, tags) && + ids.length === 0 && isClickHouseTooManyRowsError(err) ) { this.log.warn( From 78d36d8601f86fd29b5ef351e0d4eaf2b38e2bd1 Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Thu, 25 Jun 2026 23:41:58 -0700 Subject: [PATCH 17/20] docs: clarify owners+ids eligibility and windowing-fallback exclusion - Findings doc: the Implementation section's eligibility wording said `ids` must be absent, contradicting the owners+ids extension; describe both qualifying shapes and note the windowing fallback is no-id only. - envs.md: call out that owners+ids does NOT get the height-windowing retry. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...ickhouse-gql-owner-filter-too-many-rows.md | 22 +++++++++++-------- docs/envs.md | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md b/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md index 56c64b889..b503af170 100644 --- a/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md +++ b/docs/drafts/2026-06-25-clickhouse-gql-owner-filter-too-many-rows.md @@ -164,20 +164,24 @@ behind a feature flag, no schema change. A dedicated owner-ordered table optimizer seeks `owner_projection` and sorts the small matched set in memory (12.1M → 451K rows for the example owner). Otherwise the existing `optimize_use_projections = 0` (id/tag lookups) is preserved unchanged. -2. **Eligibility predicate (`ownerProjectionApplies`).** A query qualifies only - when the feature is enabled, `owners` is present, `ids` is absent, and the - query carries an `Entity-Type` tag filter whose values are **all** in a - configurable allowlist (`CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES`, - default `drive,folder,snapshot`). This deliberately **excludes +2. **Eligibility predicate (`ownerProjectionApplies`).** With the feature + enabled and `owners` present, two shapes qualify: (a) **`owners + ids`** — + the id list bounds the result, so it routes regardless of tags (see + "Extension: owners + ids" below); (b) **`owners` without `ids`** — requires an + `Entity-Type` tag filter whose values are **all** in a configurable allowlist + (`CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES`, default + `drive,folder,snapshot`). The no-id path deliberately **excludes `Entity-Type=file`** (millions of rows per owner → an expensive full sort of the matched set, re-done every page because read-in-order is off), as well as bare-owner and owner+other-tag queries. Those keep planning as today. -3. **Reactive windowing fallback (hack 5).** When an *eligible* query still - trips `max_rows_to_read` (a whale whose footprint exceeds the cap even via - the projection), the stable leg catches Code 158 and retries via +3. **Reactive windowing fallback (hack 5).** When an *eligible* **no-id** query + still trips `max_rows_to_read` (a whale whose footprint exceeds the cap even + via the projection), the stable leg catches Code 158 and retries via `queryStableTransactionsWindowed` — an adaptive height-window walk (halves the span on repeated 158, caps at 256 windows) accumulating `pageSize + 1` - rows so the existing merge / `hasNextPage` logic is untouched. + rows so the existing merge / `hasNextPage` logic is untouched. The walk is + height-ordered and needs the cursor predicate, so it is **not** used for + `owners + ids` (a whale owner + ids surfaces the 158 instead). 4. **Master gate.** Everything is off unless `CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED=true`. When off, behavior is byte-identical to before. Window-span tuning lives in module-level diff --git a/docs/envs.md b/docs/envs.md index 94e8091a0..6a38cc958 100644 --- a/docs/envs.md +++ b/docs/envs.md @@ -561,7 +561,7 @@ height that would be silently dropped by a `height >= :minHeight` predicate | CLICKHOUSE_MAX_HEIGHT_CACHE_TTL_SECONDS | Number | 60 | TTL for the cached ClickHouse max-height lookup used by the boundary optimization | | CLICKHOUSE_QUERY_TIMEOUT_SECONDS | Number | 3 | Timeout for ClickHouse queries, applied both as server-side `max_execution_time` and client-side HTTP `request_timeout` | | CLICKHOUSE_GQL_MAX_ROWS_TO_READ | Number | 10000000 | Per-query `max_rows_to_read` hint applied to GraphQL queries against the `transactions` table. Queries that would scan more than this throw `Code: 158`, preventing accidental full-table scans if a skip index is bypassed | -| CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED | Boolean | false | When true, owner-filtered GraphQL `transactions` queries are routed through `owner_projection` (and gain a reactive height-windowing fallback if they still trip `max_rows_to_read`). The main `transactions` table is height-ordered, so a sparse owner's rows scatter across millions of granules and finding a page can trip the row cap; the owner-ordered projection seeks straight to the owner's slice. Also routes `owners + ids` queries through the projection — a multi-id `id IN (...)` filter lights up most of `id_bloom`'s granules (≈ a half-table scan, the dominant source of `TOO_MANY_ROWS`), so adding an owner and seeking the owner's slice keeps it under the cap. Disabled by default for staged rollout — when off, owner queries plan exactly as before | +| CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED | Boolean | false | When true, owner-filtered GraphQL `transactions` queries are routed through `owner_projection`. The main `transactions` table is height-ordered, so a sparse owner's rows scatter across millions of granules and finding a page can trip the row cap; the owner-ordered projection seeks straight to the owner's slice. No-id owner queries also gain a reactive height-windowing fallback if they still trip `max_rows_to_read`. Also routes `owners + ids` queries through the projection — a multi-id `id IN (...)` filter lights up most of `id_bloom`'s granules (≈ a half-table scan, the dominant source of `TOO_MANY_ROWS`), so adding an owner and seeking the owner's slice keeps it under the cap; note `owners + ids` does NOT get the windowed retry (the walk is height-ordered and id queries carry no cursor predicate), so a whale owner + ids still surfaces the 158. Disabled by default for staged rollout — when off, owner queries plan exactly as before | | CLICKHOUSE_GQL_OWNER_PROJECTION_ENTITY_TYPES | String | `drive,folder,snapshot` | Comma-separated allowlist of `Entity-Type` tag values eligible for owner_projection routing of tag queries (only consulted when `CLICKHOUSE_GQL_OWNER_PROJECTION_ROUTING_ENABLED` is true, and not applied to `owners + ids` queries, which are bounded by the id list). A no-id query qualifies only when it carries an `Entity-Type` filter whose values are ALL in this list. `file` is deliberately omitted: an owner can have millions of files, and routing forces a full sort of the matched set per page (read-in-order is disabled to use the projection), re-done on every page — those large-result queries want the dedicated owner-ordered table instead. Bare-owner and owner+other-tag queries are likewise excluded | | CLICKHOUSE_GQL_DEDUPE_HEADROOM | Number | 4 | Multiplier applied to `pageSize + 1` to size the inner LIMIT of the GQL transactions query. **Pagination correctness caveat:** this must be at least as large as the table's effective duplicate factor. If a region of the table has more unmerged ReplacingMergeTree versions per PK than this headroom covers, the deduped window can yield fewer than `pageSize + 1` unique rows even when more matches exist further on — the result will be a short page with `hasNextPage: false` and rows past the window will be silently skipped. Regular background merges keep the duplicate factor at 1-2 in practice, so 4 leaves comfortable headroom; raise this if operators observe short pages (e.g. during a heavy ingest that produces many unmerged parts) | | CLICKHOUSE_SQLITE_CIRCUIT_BREAKER_TIMEOUT_MS | Number | 5000 | Timeout (ms) for the SQLite leg of the composite GQL transactions query. On timeout the response degrades to ClickHouse-only results; ClickHouse timeouts are governed separately by `CLICKHOUSE_QUERY_TIMEOUT_SECONDS` and still propagate to the caller | From e84457a0f900508d9a5ee74f929e603574aad7f9 Mon Sep 17 00:00:00 2001 From: vilenarios Date: Fri, 26 Jun 2026 14:13:22 +0000 Subject: [PATCH 18/20] perf(chunk-offset): resolve offset->block locally to avoid chain binary search Cold data retrieved by absolute offset must first locate the block containing that offset. The chain binary search did this with ~log2(height) sequential GET /block/height/{h} requests to the trusted node (~1.5s each), frequently exceeding CHUNK_SERVE_DEADLINE_MS and turning into 504s. Add a local index over stable_blocks.weave_size (getBlockByWeaveOffset, backed by the new stable_blocks_weave_size_idx) and consult it first in ArweaveCompositeClient.binarySearchBlocks. The local result is trusted only when the immediately-preceding block is present and ends before the offset (a tight bracket, so no missing block can hide the true container), and the fetched block's weave_size is re-verified; any gap, stale index, lookup error, or unstable-tip offset falls back to the existing chain binary search. This changes only how the block is found -- the block returned is identical. Resolves offset->block only. Per-transaction offsets remain chain- authoritative (/tx/{id}/offset): per-tx weave offsets follow the block's binary tx-ID sort order (not block_transaction_index) and v1 inline data is not captured by data_size, so they cannot be derived from local columns. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ERJ5pSmgZoLj4jEHhweq5B --- docs/arweave/transaction-and-chunk-offsets.md | 20 +++ ...ore.add-stable-blocks-weave-size-index.sql | 2 + ...ore.add-stable-blocks-weave-size-index.sql | 1 + .../composite-client-binary-search.test.ts | 141 ++++++++++++++++++ src/arweave/composite-client.ts | 75 ++++++++++ src/database/sql/core/offsets.sql | 15 ++ src/database/standalone-sqlite.test.ts | 55 +++++++ src/database/standalone-sqlite.ts | 32 ++++ src/system.ts | 6 + test/core-schema.sql | 1 + 10 files changed, 348 insertions(+) create mode 100644 migrations/2026.06.26T12.00.00.core.add-stable-blocks-weave-size-index.sql create mode 100644 migrations/down/2026.06.26T12.00.00.core.add-stable-blocks-weave-size-index.sql diff --git a/docs/arweave/transaction-and-chunk-offsets.md b/docs/arweave/transaction-and-chunk-offsets.md index 38e566888..fa91012fc 100644 --- a/docs/arweave/transaction-and-chunk-offsets.md +++ b/docs/arweave/transaction-and-chunk-offsets.md @@ -338,6 +338,26 @@ for (const range of ranges) { 1. **Direct Chunk Access**: Using absolute offsets allows O(1) chunk lookup 2. **Minimal Data Transfer**: Only fetch chunks containing requested bytes 3. **Streaming Support**: Process chunks as they arrive without buffering all +4. **Local offset→block resolution**: When data is retrieved by absolute offset + (the cold-data chunk-by-offset path), the gateway must first find the block + containing the offset. Rather than walking the chain with ~log₂(height) + sequential `GET /block/height/{h}` requests, the Arweave client first + consults a local index over `stable_blocks.weave_size` + (`getBlockByWeaveOffset`, backed by `stable_blocks_weave_size_idx`): the + containing block is the lowest block whose cumulative `weave_size` reaches + the offset. The local result is only trusted when the immediately-preceding + block is also present and ends before the offset (a tight bracket, so no + missing block can hide the true container); otherwise — and for offsets in + the not-yet-stable chain tip — it falls back to the chain binary search. + This only changes how the block is *found*; the block returned is identical. + + Note this resolves offset→**block** only. Resolving offset→**transaction** + within that block cannot be done from local block/transaction columns, + because per-transaction offsets follow the block's binary tx-ID sort order + (see [Transaction Order vs Offset Order](#transaction-order-vs-offset-order)) + and a transaction's true weave contribution is not always captured by + `data_size` (e.g. v1 inline data). Per-transaction offsets are therefore + taken from the chain's authoritative `/tx/{id}/offset` endpoint. ### Edge Cases diff --git a/migrations/2026.06.26T12.00.00.core.add-stable-blocks-weave-size-index.sql b/migrations/2026.06.26T12.00.00.core.add-stable-blocks-weave-size-index.sql new file mode 100644 index 000000000..8a82eb922 --- /dev/null +++ b/migrations/2026.06.26T12.00.00.core.add-stable-blocks-weave-size-index.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS stable_blocks_weave_size_idx + ON stable_blocks (weave_size); diff --git a/migrations/down/2026.06.26T12.00.00.core.add-stable-blocks-weave-size-index.sql b/migrations/down/2026.06.26T12.00.00.core.add-stable-blocks-weave-size-index.sql new file mode 100644 index 000000000..d286f2eab --- /dev/null +++ b/migrations/down/2026.06.26T12.00.00.core.add-stable-blocks-weave-size-index.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS stable_blocks_weave_size_idx; diff --git a/src/arweave/composite-client-binary-search.test.ts b/src/arweave/composite-client-binary-search.test.ts index 9a03a2dae..d7effd904 100644 --- a/src/arweave/composite-client-binary-search.test.ts +++ b/src/arweave/composite-client-binary-search.test.ts @@ -449,6 +449,147 @@ describe('ArweaveCompositeClient Binary Search', () => { }); }); + describe('binarySearchBlocks local index fast path', () => { + const targetOffset = 1500; + + it('resolves the containing block via the local index with a single block fetch and no chain walk', async () => { + client.cleanup(); + + const getBlockMock = mock.fn(async (height: number) => ({ + height, + weave_size: '2500', + txs: ['tx'], + })); + client.getBlockByHeight = getBlockMock; + const getHeightMock = mock.fn(async () => 100); + client.getHeight = getHeightMock; + + const indexMock = mock.fn(async () => ({ + height: 50, + weaveSize: 2500, + prevWeaveSize: 1000, + })); + client.blockByOffsetIndex = { getBlockByWeaveOffset: indexMock }; + + const result = await client.binarySearchBlocks(targetOffset); + + assert.strictEqual(result.height, 50); + // Index consulted once with the target offset. + assert.strictEqual(indexMock.mock.calls.length, 1); + assert.strictEqual(indexMock.mock.calls[0].arguments[0], targetOffset); + // Exactly one block fetched (the confirmed block) — no log2(height) walk. + assert.strictEqual(getBlockMock.mock.calls.length, 1); + assert.strictEqual(getBlockMock.mock.calls[0].arguments[0], 50); + // The chain binary search (which calls getHeight) is skipped entirely. + assert.strictEqual(getHeightMock.mock.calls.length, 0); + }); + + it('falls back to the chain binary search when the index misses', async () => { + client.cleanup(); + + const getBlockMock = mock.fn(async (height: number) => ({ + height, + weave_size: height >= 50 ? '2500' : '1000', + txs: [], + })); + client.getBlockByHeight = getBlockMock; + const getHeightMock = mock.fn(async () => 100); + client.getHeight = getHeightMock; + + // Index has no row for this offset. + client.blockByOffsetIndex = { + getBlockByWeaveOffset: mock.fn(async () => ({ + height: undefined, + weaveSize: undefined, + prevWeaveSize: undefined, + })), + }; + + const result = await client.binarySearchBlocks(targetOffset); + + assert.strictEqual(result.height, 50); + // Fell through to the chain walk: getHeight is consulted there. + assert.strictEqual(getHeightMock.mock.calls.length, 1); + assert.ok(getBlockMock.mock.calls.length > 1); + }); + + it('falls back when the offset is not tightly bracketed (possible missing block)', async () => { + client.cleanup(); + + const getBlockMock = mock.fn(async (height: number) => ({ + height, + weave_size: height >= 50 ? '2500' : '1000', + txs: [], + })); + client.getBlockByHeight = getBlockMock; + const getHeightMock = mock.fn(async () => 100); + client.getHeight = getHeightMock; + + // Predecessor block absent from the index -> cannot trust the bracket. + client.blockByOffsetIndex = { + getBlockByWeaveOffset: mock.fn(async () => ({ + height: 50, + weaveSize: 2500, + prevWeaveSize: undefined, + })), + }; + + await client.binarySearchBlocks(targetOffset); + assert.strictEqual(getHeightMock.mock.calls.length, 1); + }); + + it('falls back when the index lookup throws, without surfacing the error', async () => { + client.cleanup(); + + const getBlockMock = mock.fn(async (height: number) => ({ + height, + weave_size: height >= 50 ? '2500' : '1000', + txs: [], + })); + client.getBlockByHeight = getBlockMock; + const getHeightMock = mock.fn(async () => 100); + client.getHeight = getHeightMock; + + client.blockByOffsetIndex = { + getBlockByWeaveOffset: mock.fn(async () => { + throw new Error('db unavailable'); + }), + }; + + const result = await client.binarySearchBlocks(targetOffset); + assert.strictEqual(result.height, 50); + assert.strictEqual(getHeightMock.mock.calls.length, 1); + }); + + it('falls back when the fetched block no longer covers the offset (stale index)', async () => { + client.cleanup(); + + // Index points at height 50, but the freshly fetched block reports a + // smaller weave_size than the index claimed (e.g. a reorg). Must not + // trust it; fall through to the chain search. + const getBlockMock = mock.fn(async (height: number) => ({ + height, + weave_size: height >= 60 ? '2500' : '1000', + txs: [], + })); + client.getBlockByHeight = getBlockMock; + const getHeightMock = mock.fn(async () => 100); + client.getHeight = getHeightMock; + + client.blockByOffsetIndex = { + getBlockByWeaveOffset: mock.fn(async () => ({ + height: 50, + weaveSize: 2500, + prevWeaveSize: 1000, + })), + }; + + const result = await client.binarySearchBlocks(targetOffset); + assert.strictEqual(getHeightMock.mock.calls.length, 1); + assert.strictEqual(result.height, 60); + }); + }); + describe('caching behavior', () => { it('should cache transaction offsets', async () => { const targetOffset = 1800; // Changed to be within transaction range diff --git a/src/arweave/composite-client.ts b/src/arweave/composite-client.ts index 3d0720564..12d664502 100644 --- a/src/arweave/composite-client.ts +++ b/src/arweave/composite-client.ts @@ -63,6 +63,22 @@ import { import { MAX_FORK_DEPTH } from './constants.js'; import { BlockOffsetMapping } from './block-offset-mapping.js'; +/** + * Local index that resolves an absolute weave offset to its containing block + * without per-block upstream fetches. Implemented by the SQLite database over + * the indexed `stable_blocks.weave_size` column and injected post-construction + * (see {@link ArweaveCompositeClient.blockByOffsetIndex} and system.ts). When + * absent or unable to confidently bracket the offset, the client falls back to + * the chain binary search, so behavior is unchanged — only the lookup path. + */ +export interface BlockByOffsetIndex { + getBlockByWeaveOffset(offset: number): Promise<{ + height: number | undefined; + weaveSize: number | undefined; + prevWeaveSize: number | undefined; + }>; +} + const DEFAULT_REQUEST_TIMEOUT_MS = 15000; const DEFAULT_REQUEST_RETRY_COUNT = 5; const DEFAULT_MAX_REQUESTS_PER_SECOND = 5; @@ -185,6 +201,12 @@ export class ArweaveCompositeClient // Static offset-to-block mapping for optimizing binary search private blockOffsetMapping?: BlockOffsetMapping; + // Optional local index resolving an absolute weave offset to its containing + // block, avoiding ~log2(height) sequential upstream block fetches. Set + // post-construction in system.ts once the database is available, because the + // database is constructed after this client. See binarySearchBlocks. + public blockByOffsetIndex?: BlockByOffsetIndex; + // Chunk GET retry settings private chunkGetRetryCount: number; private chunkGetPeerSelectionCount: number; @@ -2224,6 +2246,59 @@ export class ArweaveCompositeClient return cached; } + // Fast path: resolve the containing block from the local stable-block index, + // collapsing the ~log2(height) sequential upstream block fetches of the + // chain binary search below into a single indexed lookup plus one block + // fetch. Only trusted when the index tightly brackets the offset (the + // immediately-preceding block is present and ends before the offset), which + // rules out a missing block hiding the true container; the fetched block's + // weave_size is re-checked before returning. Any miss, gap, abort-free + // error, or offset in the unstable tip falls through to the chain binary + // search. This never changes which block is returned, only how it's found. + if (this.blockByOffsetIndex !== undefined) { + try { + const local = + await this.blockByOffsetIndex.getBlockByWeaveOffset(targetOffset); + if ( + local.height !== undefined && + local.weaveSize !== undefined && + local.weaveSize >= targetOffset && + local.prevWeaveSize !== undefined && + local.prevWeaveSize < targetOffset + ) { + signal?.throwIfAborted(); + const block = await this.getBlockByHeight(local.height); + if ( + block !== undefined && + block !== null && + parseInt(block.weave_size) >= targetOffset + ) { + this.blockCache.set(cacheKey, block); + this.log.debug('Resolved block for offset via local index', { + targetOffset, + height: local.height, + blockOffset: block.weave_size, + }); + return block; + } + } + this.log.debug( + 'Local block-offset index did not confidently resolve; ' + + 'falling back to chain binary search', + { targetOffset, local }, + ); + } catch (error: any) { + if (error?.name === 'AbortError') { + throw error; + } + this.log.debug( + 'Local block-offset index lookup failed; ' + + 'falling back to chain binary search', + { targetOffset, error: error?.message }, + ); + } + } + try { const currentHeight = await this.getHeight(); let left = 0; diff --git a/src/database/sql/core/offsets.sql b/src/database/sql/core/offsets.sql index de8a41782..e8f822796 100644 --- a/src/database/sql/core/offsets.sql +++ b/src/database/sql/core/offsets.sql @@ -6,4 +6,19 @@ WHERE offset >= @offset AND format = 2 AND data_size > 0 ORDER BY offset ASC +LIMIT 1; + +-- selectBlockHeightByWeaveOffset +-- Resolve an absolute weave offset to its containing stable block: the lowest +-- block whose cumulative weave_size reaches the offset. prev_weave_size is the +-- preceding block's cumulative weave_size, used by the caller to confirm the +-- offset is tightly bracketed (no missing block between) before trusting the +-- local result. Backed by stable_blocks_weave_size_idx. +SELECT b.height AS height, + b.weave_size AS weave_size, + p.weave_size AS prev_weave_size +FROM stable_blocks b +LEFT JOIN stable_blocks p ON p.height = b.height - 1 +WHERE b.weave_size >= @offset +ORDER BY b.weave_size ASC LIMIT 1; \ No newline at end of file diff --git a/src/database/standalone-sqlite.test.ts b/src/database/standalone-sqlite.test.ts index 64207002c..7e9e53c9b 100644 --- a/src/database/standalone-sqlite.test.ts +++ b/src/database/standalone-sqlite.test.ts @@ -344,6 +344,61 @@ describe('StandaloneSqliteDatabase', () => { const txByOffsetResult10 = await db.getTxByOffset(201); assert.equal(txByOffsetResult10.id, undefined); }); + + it('resolves an absolute weave offset to its containing stable block via getBlockByWeaveOffset', async () => { + const insertBlock = (height: number, weaveSize: number) => + coreDb + .prepare( + `INSERT INTO stable_blocks ( + height, nonce, hash, block_timestamp, diff, last_retarget, + reward_pool, block_size, weave_size, tx_count, missing_tx_count + ) VALUES ( + @height, @nonce, @hash, 0, '0', '0', + '0', 1, @weave_size, 0, 0 + )`, + ) + .run({ + height, + nonce: Buffer.alloc(1), + hash: Buffer.alloc(1), + weave_size: weaveSize, + }); + + // Contiguous run: weave_size is cumulative-to-end-of-block. + insertBlock(10, 100); + insertBlock(11, 200); + insertBlock(12, 200); // empty block: same cumulative weave_size as 11 + insertBlock(13, 350); + + // Offset inside block 11 (100 < 150 <= 200): tightly bracketed by block 10. + const mid = await db.getBlockByWeaveOffset(150); + assert.equal(mid.height, 11); + assert.equal(mid.weaveSize, 200); + assert.equal(mid.prevWeaveSize, 100); + + // Offset exactly at block 11's end boundary resolves to block 11. + const boundary = await db.getBlockByWeaveOffset(200); + assert.equal(boundary.height, 11); + + // Just past block 11 lands in block 13 (block 12 added no bytes). + const next = await db.getBlockByWeaveOffset(201); + assert.equal(next.height, 13); + assert.equal(next.prevWeaveSize, 200); + + // Gap guard: predecessor missing -> prevWeaveSize is undefined so the + // caller will not trust the local result and falls back to the chain. + insertBlock(20, 1000); + insertBlock(22, 1200); // height 21 intentionally absent + const gapped = await db.getBlockByWeaveOffset(1100); + assert.equal(gapped.height, 22); + assert.equal(gapped.prevWeaveSize, undefined); + + // Offset beyond the highest indexed block returns no row. + const beyond = await db.getBlockByWeaveOffset(99999); + assert.equal(beyond.height, undefined); + assert.equal(beyond.weaveSize, undefined); + assert.equal(beyond.prevWeaveSize, undefined); + }); }); describe('getTransactionAttributes', () => { diff --git a/src/database/standalone-sqlite.ts b/src/database/standalone-sqlite.ts index 9f754e19d..364a5230a 100644 --- a/src/database/standalone-sqlite.ts +++ b/src/database/standalone-sqlite.ts @@ -89,6 +89,12 @@ interface TxByOffsetResult { data_size: number | undefined; } +interface BlockByWeaveOffsetResult { + height: number | undefined; + weaveSize: number | undefined; + prevWeaveSize: number | undefined; +} + export function encodeTransactionGqlCursor({ height, blockTransactionIndex, @@ -1111,6 +1117,24 @@ export class StandaloneSqliteDatabaseWorker { }; } + getBlockByWeaveOffset(offset: number): BlockByWeaveOffsetResult { + const result = this.stmts.core.selectBlockHeightByWeaveOffset.get({ + offset, + }); + if (result === undefined) { + return { + height: undefined, + weaveSize: undefined, + prevWeaveSize: undefined, + }; + } + return { + height: result.height ?? undefined, + weaveSize: result.weave_size ?? undefined, + prevWeaveSize: result.prev_weave_size ?? undefined, + }; + } + getBundleFormatId(format: string | undefined) { let id: number | undefined; if (format != undefined) { @@ -3506,6 +3530,10 @@ export class StandaloneSqliteDatabase return this.queueRead('core', 'getTxByOffset', [offset]); } + getBlockByWeaveOffset(offset: number): Promise { + return this.queueRead('core', 'getBlockByWeaveOffset', [offset]); + } + saveTxOffset(id: string, offset: number) { return this.queueWrite('core', 'saveTxOffset', [id, offset]); } @@ -4096,6 +4124,10 @@ if (!isMainThread) { const tx = worker.getTxByOffset(args[0]); parentPort?.postMessage(tx); break; + case 'getBlockByWeaveOffset': + const blockByWeaveOffset = worker.getBlockByWeaveOffset(args[0]); + parentPort?.postMessage(blockByWeaveOffset); + break; case 'saveTxOffset': worker.saveTxOffset(args[0], args[1]); parentPort?.postMessage(null); diff --git a/src/system.ts b/src/system.ts index 82a5d7835..7758b1a94 100644 --- a/src/system.ts +++ b/src/system.ts @@ -288,6 +288,12 @@ export const db = new StandaloneSqliteDatabase({ tagSelectivity: config.TAG_SELECTIVITY, }); +// Let the Arweave client resolve chunk offset->block lookups against the local +// stable-block index instead of sequential upstream block fetches. Wired here +// (not via the constructor) because the database is constructed after the +// client. See ArweaveCompositeClient.binarySearchBlocks. +arweaveClient.blockByOffsetIndex = db; + export const dataAttributesStore: ContiguousDataAttributesStore = new CompositeDataAttributesSource({ log, diff --git a/test/core-schema.sql b/test/core-schema.sql index b422bbefc..168a076ae 100644 --- a/test/core-schema.sql +++ b/test/core-schema.sql @@ -202,3 +202,4 @@ CREATE TABLE IF NOT EXISTS "new_transactions" ( CREATE INDEX new_transactions_target_id_idx ON new_transactions (target, id); CREATE INDEX new_transactions_owner_address_id_idx ON new_transactions (owner_address, id); CREATE INDEX new_transactions_height_indexed_at_idx ON new_transactions (height, indexed_at); +CREATE INDEX stable_blocks_weave_size_idx ON stable_blocks (weave_size); From a7da3d8abcbd0f0981c311b12a9fda4b34412dc8 Mon Sep 17 00:00:00 2001 From: vilenarios Date: Fri, 26 Jun 2026 14:36:25 +0000 Subject: [PATCH 19/20] refactor(chunk-offset): address review - deterministic boundary, TSDoc, abort test - offsets.sql: add `b.height ASC` tiebreaker so an offset that lands exactly on a weave_size shared by consecutive empty blocks deterministically resolves to the lowest such block. This matches the chain binary search's smallest-height selection and keeps the local fast path available on exact end-of-block offsets (without it the bracket guard would reject the tie and fall back). - Add TSDoc to BlockByWeaveOffsetResult and both getBlockByWeaveOffset methods documenting the bracket semantics and the undefined fallback contract. - Add an AbortError regression test asserting the fast path rethrows aborts instead of silently degrading into a slow chain walk. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ERJ5pSmgZoLj4jEHhweq5B --- .../composite-client-binary-search.test.ts | 29 ++++++++++++++++++ src/database/sql/core/offsets.sql | 2 +- src/database/standalone-sqlite.ts | 30 +++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/arweave/composite-client-binary-search.test.ts b/src/arweave/composite-client-binary-search.test.ts index d7effd904..20175345f 100644 --- a/src/arweave/composite-client-binary-search.test.ts +++ b/src/arweave/composite-client-binary-search.test.ts @@ -561,6 +561,35 @@ describe('ArweaveCompositeClient Binary Search', () => { assert.strictEqual(getHeightMock.mock.calls.length, 1); }); + it('rethrows an AbortError from the index lookup instead of falling back', async () => { + client.cleanup(); + + const getBlockMock = mock.fn(async (height: number) => ({ + height, + weave_size: height >= 50 ? '2500' : '1000', + txs: [], + })); + client.getBlockByHeight = getBlockMock; + const getHeightMock = mock.fn(async () => 100); + client.getHeight = getHeightMock; + + client.blockByOffsetIndex = { + getBlockByWeaveOffset: mock.fn(async () => { + const err = new Error('aborted'); + err.name = 'AbortError'; + throw err; + }), + }; + + await assert.rejects( + async () => client.binarySearchBlocks(targetOffset), + /aborted/, + ); + // A cancellation must not silently degrade into a slow chain walk. + assert.strictEqual(getHeightMock.mock.calls.length, 0); + assert.strictEqual(getBlockMock.mock.calls.length, 0); + }); + it('falls back when the fetched block no longer covers the offset (stale index)', async () => { client.cleanup(); diff --git a/src/database/sql/core/offsets.sql b/src/database/sql/core/offsets.sql index e8f822796..a4218fad2 100644 --- a/src/database/sql/core/offsets.sql +++ b/src/database/sql/core/offsets.sql @@ -20,5 +20,5 @@ SELECT b.height AS height, FROM stable_blocks b LEFT JOIN stable_blocks p ON p.height = b.height - 1 WHERE b.weave_size >= @offset -ORDER BY b.weave_size ASC +ORDER BY b.weave_size ASC, b.height ASC LIMIT 1; \ No newline at end of file diff --git a/src/database/standalone-sqlite.ts b/src/database/standalone-sqlite.ts index 364a5230a..117253799 100644 --- a/src/database/standalone-sqlite.ts +++ b/src/database/standalone-sqlite.ts @@ -89,6 +89,22 @@ interface TxByOffsetResult { data_size: number | undefined; } +/** + * Result of resolving an absolute weave offset to its containing stable block. + * + * `height`/`weaveSize` describe the lowest stable block whose cumulative + * `weave_size` reaches the requested offset (the candidate container). + * `prevWeaveSize` is the immediately-preceding block's cumulative `weave_size`, + * present only when that predecessor is itself indexed. Callers use it to + * confirm the offset is tightly bracketed (`prevWeaveSize < offset <= + * weaveSize`) — i.e. no missing block sits between the predecessor and the + * candidate — before trusting the local result. + * + * All fields are `undefined` when no stable block reaches the offset (e.g. the + * offset lies in the not-yet-stable chain tip); `prevWeaveSize` alone is + * `undefined` when the predecessor is absent (a gap or the genesis block), in + * which case the caller must not trust the bracket and should fall back. + */ interface BlockByWeaveOffsetResult { height: number | undefined; weaveSize: number | undefined; @@ -1117,6 +1133,14 @@ export class StandaloneSqliteDatabaseWorker { }; } + /** + * Resolve an absolute weave `offset` to its containing stable block via the + * indexed `stable_blocks.weave_size` column, avoiding the chain binary + * search's sequential per-block fetches. Returns the candidate block plus its + * predecessor's weave size for bracket validation; see + * {@link BlockByWeaveOffsetResult} for the `undefined` semantics. Resolves + * offset→block only — not offset→transaction. + */ getBlockByWeaveOffset(offset: number): BlockByWeaveOffsetResult { const result = this.stmts.core.selectBlockHeightByWeaveOffset.get({ offset, @@ -3530,6 +3554,12 @@ export class StandaloneSqliteDatabase return this.queueRead('core', 'getTxByOffset', [offset]); } + /** + * Queue-wrapper for the worker's {@link StandaloneSqliteDatabaseWorker.getBlockByWeaveOffset}: + * resolve an absolute weave `offset` to its containing stable block (with the + * predecessor's weave size for bracket validation). See + * {@link BlockByWeaveOffsetResult} for the `undefined` fallback contract. + */ getBlockByWeaveOffset(offset: number): Promise { return this.queueRead('core', 'getBlockByWeaveOffset', [offset]); } From 96ede32b6eac3fee9318baae40de1e7cfdc485ee Mon Sep 17 00:00:00 2001 From: vilenarios Date: Fri, 26 Jun 2026 14:53:30 +0000 Subject: [PATCH 20/20] feat(metrics): add block_offset_resolution_total for offset->block fast path Adds a labeled counter recording the outcome of each offset->block resolution in ArweaveCompositeClient.binarySearchBlocks: cache_hit, local_index_hit, and fallback_{miss,untight,stale,error,no_index}. The fast-path debug logs are only emitted at debug level, so without this counter the local-index hit rate and fallback reasons are invisible at the info level production runs at. Needed to measure the soak. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ERJ5pSmgZoLj4jEHhweq5B --- src/arweave/composite-client.ts | 25 +++++++++++++++++++++++++ src/metrics.ts | 19 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/arweave/composite-client.ts b/src/arweave/composite-client.ts index 12d664502..9ca2295e9 100644 --- a/src/arweave/composite-client.ts +++ b/src/arweave/composite-client.ts @@ -2238,6 +2238,7 @@ export class ArweaveCompositeClient const cacheKey = `block_for_offset_${targetOffset}`; const cached = this.blockCache.get(cacheKey); if (cached) { + metrics.blockOffsetResolutionCounter.inc({ outcome: 'cache_hit' }); this.log.debug('Block search cache hit', { targetOffset, cachedBlockHeight: cached.height, @@ -2273,6 +2274,9 @@ export class ArweaveCompositeClient block !== null && parseInt(block.weave_size) >= targetOffset ) { + metrics.blockOffsetResolutionCounter.inc({ + outcome: 'local_index_hit', + }); this.blockCache.set(cacheKey, block); this.log.debug('Resolved block for offset via local index', { targetOffset, @@ -2281,6 +2285,22 @@ export class ArweaveCompositeClient }); return block; } + // Index bracketed the offset but the fetched header no longer covers + // it (e.g. a reorg under the stable boundary) — don't trust it. + metrics.blockOffsetResolutionCounter.inc({ + outcome: 'fallback_stale', + }); + } else if (local.height === undefined) { + // No stable block reaches the offset (e.g. the unstable chain tip). + metrics.blockOffsetResolutionCounter.inc({ + outcome: 'fallback_miss', + }); + } else { + // Candidate found but not tightly bracketed (predecessor missing or + // not below the offset) — a gap could hide the true container. + metrics.blockOffsetResolutionCounter.inc({ + outcome: 'fallback_untight', + }); } this.log.debug( 'Local block-offset index did not confidently resolve; ' + @@ -2291,12 +2311,17 @@ export class ArweaveCompositeClient if (error?.name === 'AbortError') { throw error; } + metrics.blockOffsetResolutionCounter.inc({ outcome: 'fallback_error' }); this.log.debug( 'Local block-offset index lookup failed; ' + 'falling back to chain binary search', { targetOffset, error: error?.message }, ); } + } else { + metrics.blockOffsetResolutionCounter.inc({ + outcome: 'fallback_no_index', + }); } try { diff --git a/src/metrics.ts b/src/metrics.ts index 22761309a..fe481c0e1 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -86,6 +86,25 @@ export const chunkServeDeadlineExceededCounter = new promClient.Counter({ labelNames: ['method'], }); +// Outcome of resolving an absolute weave offset to its containing block in +// ArweaveCompositeClient.binarySearchBlocks. `local_index_hit` means the local +// stable_blocks index resolved and verified the block with no chain walk; every +// other `outcome` value falls through to the chain binary search. During soak, +// hit_rate = local_index_hit / (sum of all outcomes except cache_hit); the +// fallback_* labels explain why the fast path was not taken. +// - cache_hit: returned from the in-memory block cache before either path +// - local_index_hit: local index bracketed + fetched block re-verified +// - fallback_miss: no stable block reaches the offset (e.g. unstable tip) +// - fallback_untight: candidate found but predecessor missing/not below offset +// - fallback_stale: fetched block no longer covers the offset (e.g. reorg) +// - fallback_error: index lookup failed (non-abort) +// - fallback_no_index: no local index wired (should not occur in production) +export const blockOffsetResolutionCounter = new promClient.Counter({ + name: 'block_offset_resolution_total', + help: 'Count of offset->block resolutions by outcome of the local stable_blocks fast path', + labelNames: ['outcome'], +}); + // // GraphQL root TX lookup batching metrics //