diff --git a/.changeset/five-mice-clap.md b/.changeset/five-mice-clap.md new file mode 100644 index 0000000..c2f4028 --- /dev/null +++ b/.changeset/five-mice-clap.md @@ -0,0 +1,35 @@ +--- +"@yeez-tech/meta-encryptor": patch +--- + +修复下载/解密卡死与静默截断问题: + +- **浏览器流式下载卡 99%(主因)**:`inspectSealed` 的明文大小是高估值(漏算 nt-package 封装开销), + 之前作为 `Content-Length` 传给 StreamSaver,导致浏览器下载永远到不了声明大小。现在不再向 + `createWriteStream` 声明 size;该值更名为 `plaintextSizeEstimate`(保留 `plaintextSize` 别名), + 仅用于大小限制与进度估算。进度 transformer 中间值钳制在 99% 以下,流真正完成时在 `flush()` + 补发最终 `onProgress(total, total)`。 +- **解密失败静默跳过**:`UnsealerCore` 解密结果为空/过短时原来 `continue`(不计数、不报错),导致 + `finished` 永远为假、输出被静默截断。现在抛出 `ERR_DECRYPT_FAILED`。 +- **Node Unsealer 流终止**:不再在 `_transform` 内 `push(null)`(消除 push-after-EOF 风险); + `_flush` 检测截断输入(新会话中 `totalItems>0 && !finished` → `ERR_TRUNCATED_INPUT`;续传会话宽松处理)。 + 浏览器 `Unsealer` 的 `flush()` 同样检测截断。 +- **Recoverable 流挂起**:`RecoverableReadStream` 在输入结束于任意状态时都能正确终止(含 header + 阶段 EOF 报错、防止 `once('readable')` 叠加);`RecoverableWriteStream._final` 在内部流已 + finish/destroy 时不再永久等待,并补充 `_destroy`;已提交 item 数现在持久化到 context + (`readItemCount`),续传时正确恢复。 +- **HttpSealedFileStream 背压**:重写为 pull-based(每次 pull 拉取一个 Range 分片),所有 await + 进入 try/catch,支持 `signal`(AbortSignal),Range 使用重定向后的 URL,分片长度校验。 +- **停滞看门狗**:`downloadUnsealed`/`streamDownloadAndDecrypt`/`blobDownloadAndDecrypt` 新增 + 不活动看门狗(默认 60s 无数据中止,`timeoutMs` 可配置,0 禁用),流式尝试挂起时能真正触发 + Blob 降级而不是永久挂起;StreamSaver CDN 脚本加载增加 4s 超时(CDN 被墙时不再挂死)。 +- **空输入密封产物非法**:`DataProvider` 构造时未设置 magic number,空输入(0 item)密封出的 + 文件永远无法校验/解封。现在构造时即设置。 +- **死导出清理**:移除始终为 `undefined` 的 `checkSealedData`/`unsealData` 导出。 +- **新增导出**:`HeaderSize`、`BlockInfoSize`、`MaxItemSize`、`validateHeader`、`UnsealerCore`、 + `createInactivityWatchdog`;浏览器入口另导出 `inspectSealed`、`blobDownloadAndDecrypt`、 + `getBestWritable`、progress transformers。 +- locale JSON 转为 JS 模块(裸 Node ESM 可直接 import,修复 `gen:fixtures` 崩溃)。 + +⚠️ 行为变更:以往"静默截断也算成功"的输入(密钥错误、数据不完整)现在会报错 +(`ERR_DECRYPT_FAILED` / `ERR_TRUNCATED_INPUT`)——这是预期行为,损坏数据不应被当作成功。 diff --git a/index.d.ts b/index.d.ts index 19f7aac..f262d62 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,5 +1,16 @@ import { Transform, Readable, Writable } from 'stream'; +export class MetaEncryptorError extends Error { + code: string; + detail?: any; + readonly localizedMessage: string; + constructor(code: string, options?: { detail?: any; cause?: any }); + toJSON(): { name: string; code: string; message: string; detail?: any; causeMessage?: string }; +} + +export function configureLocale(options?: { messages?: Record | null }): void; +export function detectLocale(): string; + export class ToString extends Transform { constructor(options?: any, schema?: any); } @@ -63,8 +74,24 @@ export class DataProviderClass { } export const DataProvider: typeof DataProviderClass; -export const checkSealedData: any; -export const unsealData: any; + +// Sealed format constants / helpers +export const HeaderSize: number; +export const BlockInfoSize: number; +export const MaxItemSize: number; +export function validateHeader(headerBytes: Uint8Array): { itemNumber: number; blockNumber: number }; +export class UnsealerCore { + constructor(opts: any); + processChunk(chunk: Uint8Array): Promise; + readonly finished: boolean; + readonly headerReady: boolean; + readonly totalItems: number; + readonly readItemCount: number; + remaining: Uint8Array; + readonly processedBytes: number; + readonly writeBytes: number; +} +export function createInactivityWatchdog(ms: number, onStall: () => void): { kick(): void; stop(): void }; export const YPCNtObject: any; export const YPCCrypto: any; @@ -87,8 +114,6 @@ export default { forwardSkey, calculateSealedHash, DataProvider, - checkSealedData, - unsealData, YPCNtObject, YPCCrypto }; diff --git a/jest.config.node.cjs b/jest.config.node.cjs index d6a05ea..659e7ad 100644 --- a/jest.config.node.cjs +++ b/jest.config.node.cjs @@ -1,6 +1,9 @@ module.exports = { testEnvironment: 'node', + // Several suites seal/unseal 100–500MB files; jest's default 5s timeout + // fails them before they get a chance to run. + testTimeout: 300000, testMatch: [ "**/test/*.spec.js" ], diff --git a/package.json b/package.json index 54422d9..7652ac5 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,13 @@ { "name": "@yeez-tech/meta-encryptor", - "version": "5.0.4", + "version": "5.0.5", "description": "Data Seal/Unseal for Fidelius", "main": "build/commonjs/index.node.cjs", "module": "build/es/index.node.js", "types": "index.d.ts", "exports": { ".": { + "types": "./index.d.ts", "browser": { "import": "./build/es/index.browser.js" }, diff --git a/src/browser/HttpSealedFileStream.js b/src/browser/HttpSealedFileStream.js index d7a6b91..8147fab 100644 --- a/src/browser/HttpSealedFileStream.js +++ b/src/browser/HttpSealedFileStream.js @@ -3,6 +3,10 @@ * and streams its raw content (header + data, skipping block-info bytes). * * Analogous to Node's SealedFileStream but works over HTTP with Range requests. + * Pull-based: one Range request per pull(), so downstream backpressure limits + * how much data is buffered (the old implementation fetched the whole file + * inside start(), buffering it unboundedly when the consumer was slow or + * never ready). * * Usage: * const stream = new HttpSealedFileStream('https://example.com/file.sealed'); @@ -23,94 +27,97 @@ export class HttpSealedFileStream extends ReadableStream { * @param {object} [options] * @param {number} [options.chunkSize] - bytes per Range request (default 1 MB) * @param {Function} [options.fetch] - fetch impl (defaults to globalThis.fetch) + * @param {AbortSignal} [options.signal] - aborts in-flight requests */ - constructor(url, { chunkSize = DEFAULT_CHUNK, fetch: fetchFn } = {}) { + constructor(url, { chunkSize = DEFAULT_CHUNK, fetch: fetchFn, signal } = {}) { const _fetch = fetchFn || fetch.bind(globalThis); const state = { url, + fetchUrl: url, totalSize: 0, blockNumber: 0, contentSize: 0, + pos: 0, }; const CHUNK = chunkSize; super({ start: async (controller) => { - let headResp; try { - headResp = await _fetch(url, { method: 'HEAD' }); - } catch (e) { - controller.error(new MetaEncryptorError('ERR_HEAD_REQUEST_FAILED', { detail: { status: e.message }, cause: e })); - return; - } - if (!headResp.ok) { - controller.error(new MetaEncryptorError('ERR_HEAD_REQUEST_FAILED', { detail: { status: headResp.status } })); - return; - } - const totalSize = parseInt(headResp.headers.get('Content-Length') || '0', 10); - if (totalSize < HeaderSize) { - controller.error(new MetaEncryptorError('ERR_FILE_TOO_SMALL')); - return; - } - state.totalSize = totalSize; - const fetchUrl = resolvedFetchUrl(headResp, url); + let headResp; + try { + headResp = await _fetch(url, { method: 'HEAD', signal }); + } catch (e) { + throw new MetaEncryptorError('ERR_HEAD_REQUEST_FAILED', { detail: { status: e.message }, cause: e }); + } + if (!headResp.ok) { + throw new MetaEncryptorError('ERR_HEAD_REQUEST_FAILED', { detail: { status: headResp.status } }); + } + const totalSize = parseInt(headResp.headers.get('Content-Length') || '0', 10); + if (totalSize < HeaderSize) { + throw new MetaEncryptorError('ERR_FILE_TOO_SMALL'); + } + state.totalSize = totalSize; + state.fetchUrl = resolvedFetchUrl(headResp, url); - const tailStart = totalSize - HeaderSize; - let tailResp; - try { - const result = await fetchRange(fetchUrl, { start: tailStart, end: totalSize - 1, fetch: _fetch }); - tailResp = result.response; - } catch (e) { - controller.error(e); - return; - } - const headerBuf = new Uint8Array(await tailResp.arrayBuffer()); - if (headerBuf.length !== HeaderSize) { - controller.error(new MetaEncryptorError('ERR_HEADER_INCOMPLETE', { detail: { expected: HeaderSize, actual: headerBuf.length } })); - return; - } + const tailStart = totalSize - HeaderSize; + const { response: tailResp } = await fetchRange(state.fetchUrl, { + start: tailStart, end: totalSize - 1, fetch: _fetch, signal + }); + const headerBuf = new Uint8Array(await tailResp.arrayBuffer()); + if (headerBuf.length !== HeaderSize) { + throw new MetaEncryptorError('ERR_HEADER_INCOMPLETE', { + detail: { expected: HeaderSize, actual: headerBuf.length } + }); + } - const dv = new DataView(headerBuf.buffer, headerBuf.byteOffset, headerBuf.byteLength); - const lo = dv.getUint32(16, true); - const hi = dv.getUint32(20, true); - state.blockNumber = hi * 0x100000000 + lo; + const { blockNumber } = validateHeader(headerBuf); + state.blockNumber = blockNumber; - controller.enqueue(headerBuf); + state.contentSize = totalSize - HeaderSize - BlockInfoSize * state.blockNumber; + if (state.contentSize <= 0) { + throw new MetaEncryptorError('ERR_EMPTY_CONTENT'); + } - state.contentSize = totalSize - HeaderSize - BlockInfoSize * state.blockNumber; - if (state.contentSize <= 0) { - controller.error(new MetaEncryptorError('ERR_EMPTY_CONTENT')); - return; + controller.enqueue(headerBuf); + } catch (e) { + controller.error(e); } + }, - let pos = 0; - while (pos < state.contentSize) { - const chunkEnd = Math.min(pos + CHUNK, state.contentSize); - let resp; - try { - const result = await fetchRange(url, { start: pos, end: chunkEnd - 1, fetch: _fetch }); - resp = result.response; - } catch (e) { - controller.error(e); + pull: async (controller) => { + try { + if (state.pos >= state.contentSize) { + controller.close(); return; } + const chunkEnd = Math.min(state.pos + CHUNK, state.contentSize); + const { response: resp } = await fetchRange(state.fetchUrl, { + start: state.pos, end: chunkEnd - 1, fetch: _fetch, signal + }); const buf = new Uint8Array(await resp.arrayBuffer()); - if (buf.length > 0) { - controller.enqueue(buf); + const expected = chunkEnd - state.pos; + if (buf.length !== expected) { + throw new MetaEncryptorError('ERR_UNEXPECTED_EOF', { + detail: { expected, actual: buf.length, pos: state.pos } + }); } - pos = chunkEnd; + controller.enqueue(buf); + state.pos = chunkEnd; + } catch (e) { + controller.error(e); } - - controller.close(); } }); - // expose state via public getters - this.url = state.url || url; - this.totalSize = state.totalSize; - this.blockNumber = state.blockNumber; - this.contentSize = state.contentSize; + // expose live state (start() is async, so plain copies would stay 0) + this.url = url; + Object.defineProperties(this, { + totalSize: { get: () => state.totalSize }, + blockNumber: { get: () => state.blockNumber }, + contentSize: { get: () => state.contentSize }, + }); } } diff --git a/src/browser/Unsealer.js b/src/browser/Unsealer.js index 5628058..85a99f4 100644 --- a/src/browser/Unsealer.js +++ b/src/browser/Unsealer.js @@ -10,6 +10,7 @@ import { UnsealerCore } from '../common/unsealer_core.js'; import { BrowserCrypto } from './ypccrypto.browser.js'; +import { MetaEncryptorError } from '../common/errors.js'; export class Unsealer extends TransformStream { /** @type {UnsealerCore} */ @@ -25,6 +26,20 @@ export class Unsealer extends TransformStream { transform: async (chunk, controller) => { core.onPlain = (plain) => controller.enqueue(plain); await core.processChunk(chunk); + }, + flush: async () => { + // Upstream closed: if not every declared item was decrypted the sealed + // input was truncated — fail instead of finishing with shorter output. + // headerReady with totalItems === 0 is a legitimately empty stream. + if (!core.headerReady || (core.totalItems > 0 && !core.finished)) { + throw new MetaEncryptorError('ERR_TRUNCATED_INPUT', { + detail: { + headerReady: core.headerReady, + readItemCount: core.readItemCount, + totalItems: core.totalItems, + } + }); + } } }); diff --git a/src/browser/blob_download.js b/src/browser/blob_download.js index a6b1d6d..440b06c 100644 --- a/src/browser/blob_download.js +++ b/src/browser/blob_download.js @@ -1,27 +1,46 @@ import { Unsealer } from './Unsealer.js' import { HttpSealedFileStream } from './HttpSealedFileStream.js' +import { MetaEncryptorError } from '../common/errors.js'; import { createProgressTransformer, createDownloadReadyTransformer } from '../common/progress.js'; +import { createInactivityWatchdog } from '../common/watchdog.js'; +import { DEFAULT_STALL_MS } from './stream_download.js'; /** * @param {string} url * @param {string} privateKeyHex * @param {string} filename - * @param {{ log?: Function, onProgress?: Function, fetch?: Function, size?: number, onDownloadReady?: Function }} [opts] + * @param {{ log?: Function, onProgress?: Function, fetch?: Function, size?: number, onDownloadReady?: Function, timeoutMs?: number }} [opts] */ -export async function blobDownloadAndDecrypt(url, privateKeyHex, filename, { log, onProgress, fetch: _fetch, size, onDownloadReady } = {}) { +export async function blobDownloadAndDecrypt(url, privateKeyHex, filename, { log, onProgress, fetch: _fetch, size, onDownloadReady, timeoutMs } = {}) { log = log || (() => {}) + + const stallMs = timeoutMs === undefined ? DEFAULT_STALL_MS : timeoutMs; + const abort = new AbortController(); + const watchdog = createInactivityWatchdog(stallMs, () => { + log(`Blob download stalled: no data for ${stallMs}ms, aborting`); + abort.abort(new MetaEncryptorError('ERR_STREAM_STALLED', { detail: { timeoutMs: stallMs } })); + }); + try { const chunks = [] - const stream = new HttpSealedFileStream(url, { fetch: _fetch }) + const stream = new HttpSealedFileStream(url, { fetch: _fetch, signal: abort.signal }) const unsealer = new Unsealer({ privateKeyHex: privateKeyHex.trim(), progressHandler: (total, processed, readBytes, writeBytes) => { if (onProgress) onProgress(total, processed, readBytes, writeBytes) } }) + const watchdogTap = new TransformStream({ + transform(chunk, controller) { + watchdog.kick(); + controller.enqueue(chunk); + } + }) + watchdog.kick(); await stream + .pipeThrough(watchdogTap) .pipeThrough(createDownloadReadyTransformer(onDownloadReady)) .pipeThrough(unsealer) .pipeThrough(createProgressTransformer(size, onProgress)) @@ -29,7 +48,7 @@ export async function blobDownloadAndDecrypt(url, privateKeyHex, filename, { log write(plain) { chunks.push(new Uint8Array(plain)) } - })) + }), { signal: abort.signal }) const blob = new Blob(chunks, { type: 'application/octet-stream' }) const urlObj = URL.createObjectURL(blob) @@ -43,7 +62,10 @@ export async function blobDownloadAndDecrypt(url, privateKeyHex, filename, { log log('Download complete (client-side Blob decrypt)'); return { ok: true } } catch (e) { - log('Blob download failed: ' + e.message); - throw e + const err = (abort.signal.aborted && abort.signal.reason instanceof Error) ? abort.signal.reason : e; + log('Blob download failed: ' + err.message); + throw err + } finally { + watchdog.stop(); } -} \ No newline at end of file +} diff --git a/src/browser/downloadUnsealed.js b/src/browser/downloadUnsealed.js index 9352a89..68e574f 100644 --- a/src/browser/downloadUnsealed.js +++ b/src/browser/downloadUnsealed.js @@ -22,7 +22,7 @@ function makeLogger(onLog) { }; } -async function inspectSealed(url, log) { +export async function inspectSealed(url, log) { log('HEAD ' + url); let headResp; try { @@ -69,10 +69,22 @@ async function inspectSealed(url, log) { const sealedContentSize = totalSize - HeaderSize - blockNumber * BlockInfoSize; if (sealedContentSize <= 0) throw new MetaEncryptorError('ERR_EMPTY_CONTENT'); - // per-item overhead: 8 (len prefix) + 12 (IV) + 64 (public key) + 16 (GCM tag) = 100 - const plaintextSize = sealedContentSize - itemNumber * 100; - - return { totalSize, blockNumber, itemNumber, sealedContentSize, plaintextSize }; + // Per-item overhead is at least 100 bytes (8 len prefix + 12 IV + 64 public + // key + 16 GCM tag), but each encrypted item also contains nt-package + // framing (12B per item + 20B per nt-input) that cannot be derived from the + // header alone. This value is therefore an OVER-estimate of the plaintext + // size — never use it as an exact Content-Length. + const plaintextSizeEstimate = sealedContentSize - itemNumber * 100; + + return { + totalSize, + blockNumber, + itemNumber, + sealedContentSize, + plaintextSizeEstimate, + // legacy alias, kept for compatibility + plaintextSize: plaintextSizeEstimate, + }; } @@ -88,6 +100,7 @@ async function inspectSealed(url, log) { * @param {Function} [options.onDownloadReady] - HTTP 流首个数据块进入管道时触发(可关蒙层) * @param {Function} [options.onSuccess] * @param {Function} [options.onError] + * @param {number} [options.timeoutMs] - inactivity watchdog (ms); 0 disables * @returns {Promise} */ export async function downloadUnsealed({ @@ -98,7 +111,8 @@ export async function downloadUnsealed({ onProgress, onDownloadReady, onSuccess, - onError + onError, + timeoutMs }) { const log = makeLogger(onLog); const key = privateKey.trim() @@ -113,7 +127,7 @@ export async function downloadUnsealed({ log('Checking file url=' + url + ' filename=' + filename); const meta = await inspectSealed(url, log); log('inspect file succ'); - log(`Plaintext size=${meta.plaintextSize} bytes, sealed=${meta.sealedContentSize} bytes, totalSize=${meta.totalSize}`); + log(`Plaintext size(est)=${meta.plaintextSizeEstimate} bytes, sealed=${meta.sealedContentSize} bytes, totalSize=${meta.totalSize}`); const mobile = isMobile() const limit = mobile ? MOBILE_LIMIT : DESKTOP_LIMIT @@ -121,9 +135,9 @@ export async function downloadUnsealed({ const ua = typeof navigator !== 'undefined' ? navigator.userAgent : 'n/a'; log(`Detected ${mode} (ua=${ua}), limit ${(limit / 1024 / 1024).toFixed(0)} MB`); - if (meta.plaintextSize > limit) { + if (meta.plaintextSizeEstimate > limit) { throw new MetaEncryptorError('ERR_FILE_TOO_LARGE', { - detail: { size: (meta.plaintextSize / 1024 / 1024).toFixed(0), mode, limit: (limit / 1024 / 1024).toFixed(0) } + detail: { size: (meta.plaintextSizeEstimate / 1024 / 1024).toFixed(0), mode, limit: (limit / 1024 / 1024).toFixed(0) } }) } @@ -133,8 +147,9 @@ export async function downloadUnsealed({ await streamDownloadAndDecrypt(url, key, filename, { log, onProgress, - size: meta.plaintextSize, + size: meta.plaintextSizeEstimate, onDownloadReady, + timeoutMs, }) if (onSuccess) onSuccess({ filename }) return @@ -144,7 +159,13 @@ export async function downloadUnsealed({ } log('Using Blob download...') - await blobDownloadAndDecrypt(url, key, filename, { log, onProgress, onDownloadReady }) + await blobDownloadAndDecrypt(url, key, filename, { + log, + onProgress, + size: meta.plaintextSizeEstimate, + onDownloadReady, + timeoutMs, + }) if (onSuccess) onSuccess({ filename }) } catch (error) { log('Download failed: ' + error.message) diff --git a/src/browser/stream_download.js b/src/browser/stream_download.js index 94eb67c..995865f 100644 --- a/src/browser/stream_download.js +++ b/src/browser/stream_download.js @@ -2,6 +2,11 @@ import { Unsealer } from './Unsealer.js' import { HttpSealedFileStream } from './HttpSealedFileStream.js' import { MetaEncryptorError } from '../common/errors.js'; import { createProgressTransformer, createDownloadReadyTransformer } from '../common/progress.js'; +import { createInactivityWatchdog } from '../common/watchdog.js'; + +export const DEFAULT_STALL_MS = 60 * 1000; + +const STREAMSAVER_LOAD_TIMEOUT_MS = 4000; async function ensureStreamSaver(log) { if (typeof window === 'undefined') return null; @@ -9,11 +14,18 @@ async function ensureStreamSaver(log) { try { log?.('Loading StreamSaver...'); + // The CDN may be unreachable (blocked networks) and the