From 9b6f441f70c1520cb372a675ec1b344fc4fce54d Mon Sep 17 00:00:00 2001 From: chmwang Date: Fri, 3 Jul 2026 10:24:18 +0800 Subject: [PATCH 1/2] fix --- index.d.ts | 33 +++- jest.config.node.cjs | 3 + package.json | 3 +- src/browser/HttpSealedFileStream.js | 127 +++++++------ src/browser/Unsealer.js | 15 ++ src/browser/blob_download.js | 36 +++- src/browser/downloadUnsealed.js | 43 +++-- src/browser/stream_download.js | 60 ++++-- src/common/locale.js | 4 +- src/common/progress.js | 20 +- src/common/unsealer_core.js | 20 +- src/common/watchdog.js | 27 +++ src/index.browser.js | 12 +- src/index.node.js | 8 +- src/locales/en.js | 56 ++++++ src/locales/en.json | 3 + src/locales/zh-CN.js | 56 ++++++ src/locales/zh-CN.json | 3 + src/node/DataProvider.js | 4 + src/node/Recoverable.js | 87 +++++++-- src/node/Sealer.js | 3 - src/node/Unsealer.js | 36 +++- src/node/index.js | 8 +- test/downloadFunctions.spec.mjs | 14 +- test/progress.spec.mjs | 4 +- test/regression_fixes.spec.js | 280 ++++++++++++++++++++++++++++ 26 files changed, 825 insertions(+), 140 deletions(-) create mode 100644 src/common/watchdog.js create mode 100644 src/locales/en.js create mode 100644 src/locales/zh-CN.js create mode 100644 test/regression_fixes.spec.js diff --git a/index.d.ts b/index.d.ts index 19f7aac..f262d62 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,5 +1,16 @@ import { Transform, Readable, Writable } from 'stream'; +export class MetaEncryptorError extends Error { + code: string; + detail?: any; + readonly localizedMessage: string; + constructor(code: string, options?: { detail?: any; cause?: any }); + toJSON(): { name: string; code: string; message: string; detail?: any; causeMessage?: string }; +} + +export function configureLocale(options?: { messages?: Record | null }): void; +export function detectLocale(): string; + export class ToString extends Transform { constructor(options?: any, schema?: any); } @@ -63,8 +74,24 @@ export class DataProviderClass { } export const DataProvider: typeof DataProviderClass; -export const checkSealedData: any; -export const unsealData: any; + +// Sealed format constants / helpers +export const HeaderSize: number; +export const BlockInfoSize: number; +export const MaxItemSize: number; +export function validateHeader(headerBytes: Uint8Array): { itemNumber: number; blockNumber: number }; +export class UnsealerCore { + constructor(opts: any); + processChunk(chunk: Uint8Array): Promise; + readonly finished: boolean; + readonly headerReady: boolean; + readonly totalItems: number; + readonly readItemCount: number; + remaining: Uint8Array; + readonly processedBytes: number; + readonly writeBytes: number; +} +export function createInactivityWatchdog(ms: number, onStall: () => void): { kick(): void; stop(): void }; export const YPCNtObject: any; export const YPCCrypto: any; @@ -87,8 +114,6 @@ export default { forwardSkey, calculateSealedHash, DataProvider, - checkSealedData, - unsealData, YPCNtObject, YPCCrypto }; diff --git a/jest.config.node.cjs b/jest.config.node.cjs index d6a05ea..659e7ad 100644 --- a/jest.config.node.cjs +++ b/jest.config.node.cjs @@ -1,6 +1,9 @@ module.exports = { testEnvironment: 'node', + // Several suites seal/unseal 100–500MB files; jest's default 5s timeout + // fails them before they get a chance to run. + testTimeout: 300000, testMatch: [ "**/test/*.spec.js" ], diff --git a/package.json b/package.json index 54422d9..7652ac5 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,13 @@ { "name": "@yeez-tech/meta-encryptor", - "version": "5.0.4", + "version": "5.0.5", "description": "Data Seal/Unseal for Fidelius", "main": "build/commonjs/index.node.cjs", "module": "build/es/index.node.js", "types": "index.d.ts", "exports": { ".": { + "types": "./index.d.ts", "browser": { "import": "./build/es/index.browser.js" }, diff --git a/src/browser/HttpSealedFileStream.js b/src/browser/HttpSealedFileStream.js index d7a6b91..8147fab 100644 --- a/src/browser/HttpSealedFileStream.js +++ b/src/browser/HttpSealedFileStream.js @@ -3,6 +3,10 @@ * and streams its raw content (header + data, skipping block-info bytes). * * Analogous to Node's SealedFileStream but works over HTTP with Range requests. + * Pull-based: one Range request per pull(), so downstream backpressure limits + * how much data is buffered (the old implementation fetched the whole file + * inside start(), buffering it unboundedly when the consumer was slow or + * never ready). * * Usage: * const stream = new HttpSealedFileStream('https://example.com/file.sealed'); @@ -23,94 +27,97 @@ export class HttpSealedFileStream extends ReadableStream { * @param {object} [options] * @param {number} [options.chunkSize] - bytes per Range request (default 1 MB) * @param {Function} [options.fetch] - fetch impl (defaults to globalThis.fetch) + * @param {AbortSignal} [options.signal] - aborts in-flight requests */ - constructor(url, { chunkSize = DEFAULT_CHUNK, fetch: fetchFn } = {}) { + constructor(url, { chunkSize = DEFAULT_CHUNK, fetch: fetchFn, signal } = {}) { const _fetch = fetchFn || fetch.bind(globalThis); const state = { url, + fetchUrl: url, totalSize: 0, blockNumber: 0, contentSize: 0, + pos: 0, }; const CHUNK = chunkSize; super({ start: async (controller) => { - let headResp; try { - headResp = await _fetch(url, { method: 'HEAD' }); - } catch (e) { - controller.error(new MetaEncryptorError('ERR_HEAD_REQUEST_FAILED', { detail: { status: e.message }, cause: e })); - return; - } - if (!headResp.ok) { - controller.error(new MetaEncryptorError('ERR_HEAD_REQUEST_FAILED', { detail: { status: headResp.status } })); - return; - } - const totalSize = parseInt(headResp.headers.get('Content-Length') || '0', 10); - if (totalSize < HeaderSize) { - controller.error(new MetaEncryptorError('ERR_FILE_TOO_SMALL')); - return; - } - state.totalSize = totalSize; - const fetchUrl = resolvedFetchUrl(headResp, url); + let headResp; + try { + headResp = await _fetch(url, { method: 'HEAD', signal }); + } catch (e) { + throw new MetaEncryptorError('ERR_HEAD_REQUEST_FAILED', { detail: { status: e.message }, cause: e }); + } + if (!headResp.ok) { + throw new MetaEncryptorError('ERR_HEAD_REQUEST_FAILED', { detail: { status: headResp.status } }); + } + const totalSize = parseInt(headResp.headers.get('Content-Length') || '0', 10); + if (totalSize < HeaderSize) { + throw new MetaEncryptorError('ERR_FILE_TOO_SMALL'); + } + state.totalSize = totalSize; + state.fetchUrl = resolvedFetchUrl(headResp, url); - const tailStart = totalSize - HeaderSize; - let tailResp; - try { - const result = await fetchRange(fetchUrl, { start: tailStart, end: totalSize - 1, fetch: _fetch }); - tailResp = result.response; - } catch (e) { - controller.error(e); - return; - } - const headerBuf = new Uint8Array(await tailResp.arrayBuffer()); - if (headerBuf.length !== HeaderSize) { - controller.error(new MetaEncryptorError('ERR_HEADER_INCOMPLETE', { detail: { expected: HeaderSize, actual: headerBuf.length } })); - return; - } + const tailStart = totalSize - HeaderSize; + const { response: tailResp } = await fetchRange(state.fetchUrl, { + start: tailStart, end: totalSize - 1, fetch: _fetch, signal + }); + const headerBuf = new Uint8Array(await tailResp.arrayBuffer()); + if (headerBuf.length !== HeaderSize) { + throw new MetaEncryptorError('ERR_HEADER_INCOMPLETE', { + detail: { expected: HeaderSize, actual: headerBuf.length } + }); + } - const dv = new DataView(headerBuf.buffer, headerBuf.byteOffset, headerBuf.byteLength); - const lo = dv.getUint32(16, true); - const hi = dv.getUint32(20, true); - state.blockNumber = hi * 0x100000000 + lo; + const { blockNumber } = validateHeader(headerBuf); + state.blockNumber = blockNumber; - controller.enqueue(headerBuf); + state.contentSize = totalSize - HeaderSize - BlockInfoSize * state.blockNumber; + if (state.contentSize <= 0) { + throw new MetaEncryptorError('ERR_EMPTY_CONTENT'); + } - state.contentSize = totalSize - HeaderSize - BlockInfoSize * state.blockNumber; - if (state.contentSize <= 0) { - controller.error(new MetaEncryptorError('ERR_EMPTY_CONTENT')); - return; + controller.enqueue(headerBuf); + } catch (e) { + controller.error(e); } + }, - let pos = 0; - while (pos < state.contentSize) { - const chunkEnd = Math.min(pos + CHUNK, state.contentSize); - let resp; - try { - const result = await fetchRange(url, { start: pos, end: chunkEnd - 1, fetch: _fetch }); - resp = result.response; - } catch (e) { - controller.error(e); + pull: async (controller) => { + try { + if (state.pos >= state.contentSize) { + controller.close(); return; } + const chunkEnd = Math.min(state.pos + CHUNK, state.contentSize); + const { response: resp } = await fetchRange(state.fetchUrl, { + start: state.pos, end: chunkEnd - 1, fetch: _fetch, signal + }); const buf = new Uint8Array(await resp.arrayBuffer()); - if (buf.length > 0) { - controller.enqueue(buf); + const expected = chunkEnd - state.pos; + if (buf.length !== expected) { + throw new MetaEncryptorError('ERR_UNEXPECTED_EOF', { + detail: { expected, actual: buf.length, pos: state.pos } + }); } - pos = chunkEnd; + controller.enqueue(buf); + state.pos = chunkEnd; + } catch (e) { + controller.error(e); } - - controller.close(); } }); - // expose state via public getters - this.url = state.url || url; - this.totalSize = state.totalSize; - this.blockNumber = state.blockNumber; - this.contentSize = state.contentSize; + // expose live state (start() is async, so plain copies would stay 0) + this.url = url; + Object.defineProperties(this, { + totalSize: { get: () => state.totalSize }, + blockNumber: { get: () => state.blockNumber }, + contentSize: { get: () => state.contentSize }, + }); } } diff --git a/src/browser/Unsealer.js b/src/browser/Unsealer.js index 5628058..85a99f4 100644 --- a/src/browser/Unsealer.js +++ b/src/browser/Unsealer.js @@ -10,6 +10,7 @@ import { UnsealerCore } from '../common/unsealer_core.js'; import { BrowserCrypto } from './ypccrypto.browser.js'; +import { MetaEncryptorError } from '../common/errors.js'; export class Unsealer extends TransformStream { /** @type {UnsealerCore} */ @@ -25,6 +26,20 @@ export class Unsealer extends TransformStream { transform: async (chunk, controller) => { core.onPlain = (plain) => controller.enqueue(plain); await core.processChunk(chunk); + }, + flush: async () => { + // Upstream closed: if not every declared item was decrypted the sealed + // input was truncated — fail instead of finishing with shorter output. + // headerReady with totalItems === 0 is a legitimately empty stream. + if (!core.headerReady || (core.totalItems > 0 && !core.finished)) { + throw new MetaEncryptorError('ERR_TRUNCATED_INPUT', { + detail: { + headerReady: core.headerReady, + readItemCount: core.readItemCount, + totalItems: core.totalItems, + } + }); + } } }); diff --git a/src/browser/blob_download.js b/src/browser/blob_download.js index a6b1d6d..440b06c 100644 --- a/src/browser/blob_download.js +++ b/src/browser/blob_download.js @@ -1,27 +1,46 @@ import { Unsealer } from './Unsealer.js' import { HttpSealedFileStream } from './HttpSealedFileStream.js' +import { MetaEncryptorError } from '../common/errors.js'; import { createProgressTransformer, createDownloadReadyTransformer } from '../common/progress.js'; +import { createInactivityWatchdog } from '../common/watchdog.js'; +import { DEFAULT_STALL_MS } from './stream_download.js'; /** * @param {string} url * @param {string} privateKeyHex * @param {string} filename - * @param {{ log?: Function, onProgress?: Function, fetch?: Function, size?: number, onDownloadReady?: Function }} [opts] + * @param {{ log?: Function, onProgress?: Function, fetch?: Function, size?: number, onDownloadReady?: Function, timeoutMs?: number }} [opts] */ -export async function blobDownloadAndDecrypt(url, privateKeyHex, filename, { log, onProgress, fetch: _fetch, size, onDownloadReady } = {}) { +export async function blobDownloadAndDecrypt(url, privateKeyHex, filename, { log, onProgress, fetch: _fetch, size, onDownloadReady, timeoutMs } = {}) { log = log || (() => {}) + + const stallMs = timeoutMs === undefined ? DEFAULT_STALL_MS : timeoutMs; + const abort = new AbortController(); + const watchdog = createInactivityWatchdog(stallMs, () => { + log(`Blob download stalled: no data for ${stallMs}ms, aborting`); + abort.abort(new MetaEncryptorError('ERR_STREAM_STALLED', { detail: { timeoutMs: stallMs } })); + }); + try { const chunks = [] - const stream = new HttpSealedFileStream(url, { fetch: _fetch }) + const stream = new HttpSealedFileStream(url, { fetch: _fetch, signal: abort.signal }) const unsealer = new Unsealer({ privateKeyHex: privateKeyHex.trim(), progressHandler: (total, processed, readBytes, writeBytes) => { if (onProgress) onProgress(total, processed, readBytes, writeBytes) } }) + const watchdogTap = new TransformStream({ + transform(chunk, controller) { + watchdog.kick(); + controller.enqueue(chunk); + } + }) + watchdog.kick(); await stream + .pipeThrough(watchdogTap) .pipeThrough(createDownloadReadyTransformer(onDownloadReady)) .pipeThrough(unsealer) .pipeThrough(createProgressTransformer(size, onProgress)) @@ -29,7 +48,7 @@ export async function blobDownloadAndDecrypt(url, privateKeyHex, filename, { log write(plain) { chunks.push(new Uint8Array(plain)) } - })) + }), { signal: abort.signal }) const blob = new Blob(chunks, { type: 'application/octet-stream' }) const urlObj = URL.createObjectURL(blob) @@ -43,7 +62,10 @@ export async function blobDownloadAndDecrypt(url, privateKeyHex, filename, { log log('Download complete (client-side Blob decrypt)'); return { ok: true } } catch (e) { - log('Blob download failed: ' + e.message); - throw e + const err = (abort.signal.aborted && abort.signal.reason instanceof Error) ? abort.signal.reason : e; + log('Blob download failed: ' + err.message); + throw err + } finally { + watchdog.stop(); } -} \ No newline at end of file +} diff --git a/src/browser/downloadUnsealed.js b/src/browser/downloadUnsealed.js index 9352a89..68e574f 100644 --- a/src/browser/downloadUnsealed.js +++ b/src/browser/downloadUnsealed.js @@ -22,7 +22,7 @@ function makeLogger(onLog) { }; } -async function inspectSealed(url, log) { +export async function inspectSealed(url, log) { log('HEAD ' + url); let headResp; try { @@ -69,10 +69,22 @@ async function inspectSealed(url, log) { const sealedContentSize = totalSize - HeaderSize - blockNumber * BlockInfoSize; if (sealedContentSize <= 0) throw new MetaEncryptorError('ERR_EMPTY_CONTENT'); - // per-item overhead: 8 (len prefix) + 12 (IV) + 64 (public key) + 16 (GCM tag) = 100 - const plaintextSize = sealedContentSize - itemNumber * 100; - - return { totalSize, blockNumber, itemNumber, sealedContentSize, plaintextSize }; + // Per-item overhead is at least 100 bytes (8 len prefix + 12 IV + 64 public + // key + 16 GCM tag), but each encrypted item also contains nt-package + // framing (12B per item + 20B per nt-input) that cannot be derived from the + // header alone. This value is therefore an OVER-estimate of the plaintext + // size — never use it as an exact Content-Length. + const plaintextSizeEstimate = sealedContentSize - itemNumber * 100; + + return { + totalSize, + blockNumber, + itemNumber, + sealedContentSize, + plaintextSizeEstimate, + // legacy alias, kept for compatibility + plaintextSize: plaintextSizeEstimate, + }; } @@ -88,6 +100,7 @@ async function inspectSealed(url, log) { * @param {Function} [options.onDownloadReady] - HTTP 流首个数据块进入管道时触发（可关蒙层） * @param {Function} [options.onSuccess] * @param {Function} [options.onError] + * @param {number} [options.timeoutMs] - inactivity watchdog (ms); 0 disables * @returns {Promise} */ export async function downloadUnsealed({ @@ -98,7 +111,8 @@ export async function downloadUnsealed({ onProgress, onDownloadReady, onSuccess, - onError + onError, + timeoutMs }) { const log = makeLogger(onLog); const key = privateKey.trim() @@ -113,7 +127,7 @@ export async function downloadUnsealed({ log('Checking file url=' + url + ' filename=' + filename); const meta = await inspectSealed(url, log); log('inspect file succ'); - log(`Plaintext size=${meta.plaintextSize} bytes, sealed=${meta.sealedContentSize} bytes, totalSize=${meta.totalSize}`); + log(`Plaintext size(est)=${meta.plaintextSizeEstimate} bytes, sealed=${meta.sealedContentSize} bytes, totalSize=${meta.totalSize}`); const mobile = isMobile() const limit = mobile ? MOBILE_LIMIT : DESKTOP_LIMIT @@ -121,9 +135,9 @@ export async function downloadUnsealed({ const ua = typeof navigator !== 'undefined' ? navigator.userAgent : 'n/a'; log(`Detected ${mode} (ua=${ua}), limit ${(limit / 1024 / 1024).toFixed(0)} MB`); - if (meta.plaintextSize > limit) { + if (meta.plaintextSizeEstimate > limit) { throw new MetaEncryptorError('ERR_FILE_TOO_LARGE', { - detail: { size: (meta.plaintextSize / 1024 / 1024).toFixed(0), mode, limit: (limit / 1024 / 1024).toFixed(0) } + detail: { size: (meta.plaintextSizeEstimate / 1024 / 1024).toFixed(0), mode, limit: (limit / 1024 / 1024).toFixed(0) } }) } @@ -133,8 +147,9 @@ export async function downloadUnsealed({ await streamDownloadAndDecrypt(url, key, filename, { log, onProgress, - size: meta.plaintextSize, + size: meta.plaintextSizeEstimate, onDownloadReady, + timeoutMs, }) if (onSuccess) onSuccess({ filename }) return @@ -144,7 +159,13 @@ export async function downloadUnsealed({ } log('Using Blob download...') - await blobDownloadAndDecrypt(url, key, filename, { log, onProgress, onDownloadReady }) + await blobDownloadAndDecrypt(url, key, filename, { + log, + onProgress, + size: meta.plaintextSizeEstimate, + onDownloadReady, + timeoutMs, + }) if (onSuccess) onSuccess({ filename }) } catch (error) { log('Download failed: ' + error.message) diff --git a/src/browser/stream_download.js b/src/browser/stream_download.js index 94eb67c..995865f 100644 --- a/src/browser/stream_download.js +++ b/src/browser/stream_download.js @@ -2,6 +2,11 @@ import { Unsealer } from './Unsealer.js' import { HttpSealedFileStream } from './HttpSealedFileStream.js' import { MetaEncryptorError } from '../common/errors.js'; import { createProgressTransformer, createDownloadReadyTransformer } from '../common/progress.js'; +import { createInactivityWatchdog } from '../common/watchdog.js'; + +export const DEFAULT_STALL_MS = 60 * 1000; + +const STREAMSAVER_LOAD_TIMEOUT_MS = 4000; async function ensureStreamSaver(log) { if (typeof window === 'undefined') return null; @@ -9,11 +14,18 @@ async function ensureStreamSaver(log) { try { log?.('Loading StreamSaver...'); + // The CDN may be unreachable (blocked networks) and the