From dca3c9673be8125c567fdd2fbdc3f36f53aeb77e Mon Sep 17 00:00:00 2001 From: Cayman Date: Mon, 11 May 2026 11:14:37 -0400 Subject: [PATCH 1/5] feat: add progressive types --- packages/ssz/package.json | 2 +- packages/ssz/src/index.ts | 4 + packages/ssz/src/type/bitList.ts | 3 + packages/ssz/src/type/compatibleUnion.ts | 642 +++++++++++++ packages/ssz/src/type/composite.ts | 45 +- packages/ssz/src/type/progressive.ts | 166 ++++ packages/ssz/src/type/progressiveBitList.ts | 161 ++++ packages/ssz/src/type/progressiveContainer.ts | 757 ++++++++++++++++ packages/ssz/src/type/progressiveList.ts | 852 ++++++++++++++++++ packages/ssz/test/spec/generic/index.test.ts | 111 ++- packages/ssz/test/spec/generic/types.ts | 198 ++++ packages/ssz/test/spec/runValidTest.ts | 8 +- packages/ssz/test/specTestVersioning.ts | 4 +- .../byType/compatibleUnion/invalid.test.ts | 27 + .../unit/byType/compatibleUnion/valid.test.ts | 100 ++ .../unit/byType/progressive/valid.test.ts | 213 +++++ 16 files changed, 3226 insertions(+), 67 deletions(-) create mode 100644 packages/ssz/src/type/compatibleUnion.ts create mode 100644 packages/ssz/src/type/progressive.ts create mode 100644 packages/ssz/src/type/progressiveBitList.ts create mode 100644 packages/ssz/src/type/progressiveContainer.ts create mode 100644 packages/ssz/src/type/progressiveList.ts create mode 100644 packages/ssz/test/unit/byType/compatibleUnion/invalid.test.ts create mode 100644 packages/ssz/test/unit/byType/compatibleUnion/valid.test.ts create mode 100644 packages/ssz/test/unit/byType/progressive/valid.test.ts diff --git a/packages/ssz/package.json b/packages/ssz/package.json index 67a92f1d..2201b592 100644 --- a/packages/ssz/package.json +++ b/packages/ssz/package.json @@ -34,7 +34,7 @@ "benchmark:local": "pnpm benchmark --local", "test:unit": "vitest run --dir test/unit", "test:spec": "pnpm test:spec-generic && pnpm test:spec-static test:spec-eip-4881", - "test:spec-generic": "vitest run --dir test/spec/generic", + "test:spec-generic": "vitest run --coverage.enabled=false --dir test/spec/generic", "test:spec-static": "pnpm test:spec-static-minimal && pnpm test:spec-static-mainnet", "test:spec-static-minimal": "LODESTAR_PRESET=minimal vitest run --dir test/spec/ test/spec/ssz_static.test.ts", "test:spec-static-mainnet": "LODESTAR_PRESET=mainnet vitest run --dir test/spec/ test/spec/ssz_static.test.ts", diff --git a/packages/ssz/src/index.ts b/packages/ssz/src/index.ts index 5d1c543a..90a6279d 100644 --- a/packages/ssz/src/index.ts +++ b/packages/ssz/src/index.ts @@ -4,10 +4,14 @@ export {BitVectorType} from "./type/bitVector.ts"; export {BooleanType} from "./type/boolean.ts"; export {ByteListType} from "./type/byteList.ts"; export {ByteVectorType} from "./type/byteVector.ts"; +export {CompatibleUnionType} from "./type/compatibleUnion.ts"; export {ContainerType} from "./type/container.ts"; export {ContainerNodeStructType} from "./type/containerNodeStruct.ts"; export {ListBasicType} from "./type/listBasic.ts"; export {ListCompositeType} from "./type/listComposite.ts"; +export {ProgressiveBitListType} from "./type/progressiveBitList.ts"; +export {ProgressiveContainerType} from "./type/progressiveContainer.ts"; +export {ProgressiveListBasicType, ProgressiveListCompositeType} from "./type/progressiveList.ts"; export {PartialListCompositeType} from "./type/partialListComposite.ts"; export {NoneType} from "./type/none.ts"; export {UintBigintType, UintNumberType} from "./type/uint.ts"; diff --git a/packages/ssz/src/type/bitList.ts b/packages/ssz/src/type/bitList.ts index 46553943..b7632480 100644 --- a/packages/ssz/src/type/bitList.ts +++ b/packages/ssz/src/type/bitList.ts @@ -161,6 +161,9 @@ export function deserializeUint8ArrayBitListFromBytes( if (end > data.length) { throw Error(`BitList attempting to read byte ${end} of data length ${data.length}`); } + if (end <= start) { + throw Error("BitList requires a padding bit"); + } const lastByte = data[end - 1]; const size = end - start; diff --git a/packages/ssz/src/type/compatibleUnion.ts b/packages/ssz/src/type/compatibleUnion.ts new file mode 100644 index 00000000..c5f28aaf --- /dev/null +++ b/packages/ssz/src/type/compatibleUnion.ts @@ -0,0 +1,642 @@ +import {allocUnsafe} from "@chainsafe/as-sha256"; +import { + Gindex, + HashComputationLevel, + Node, + Proof, + ProofType, + Tree, + concatGindices, + createProof, + getHashComputations, + getNode, + merkleizeBlocksBytes, +} from "@chainsafe/persistent-merkle-tree"; +import {byteArrayEquals} from "../util/byteArray.ts"; +import {namedClass} from "../util/named.ts"; +import {Require} from "../util/types.ts"; +import {TreeView} from "../view/abstract.ts"; +import {TreeViewDU} from "../viewDU/abstract.ts"; +import {ByteViews, JsonPath, Type} from "./abstract.ts"; +import {addLengthNode, getLengthFromRootNode} from "./arrayBasic.ts"; +import {BasicType, isBasicType} from "./basic.ts"; +import {BitListType} from "./bitList.ts"; +import {BitVectorType} from "./bitVector.ts"; +import {BooleanType} from "./boolean.ts"; +import {ByteListType} from "./byteList.ts"; +import {ByteVectorType} from "./byteVector.ts"; +import {CompositeType, isCompositeType} from "./composite.ts"; +import {ContainerType} from "./container.ts"; +import {ListBasicType} from "./listBasic.ts"; +import {ListCompositeType} from "./listComposite.ts"; +import {ProgressiveBitListType} from "./progressiveBitList.ts"; +import {ProgressiveContainerType} from "./progressiveContainer.ts"; +import {ProgressiveListBasicType, ProgressiveListCompositeType} from "./progressiveList.ts"; +import {UintBigintType, UintNumberType} from "./uint.ts"; +import {VectorBasicType} from "./vectorBasic.ts"; +import {VectorCompositeType} from "./vectorComposite.ts"; + +export type CompatibleUnion = { + readonly selector: number; + data: T; +}; + +export type CompatibleUnionOpts = { + typeName?: string; +}; + +const VALUE_GINDEX = BigInt(2); +const SELECTOR_GINDEX = BigInt(3); + +/** + * CompatibleUnion: union type containing one of the given subtypes with compatible Merkleization. + * - Notation: CompatibleUnion({selector: type}), e.g. CompatibleUnion({1: Square, 2: Circle}) + */ +export class CompatibleUnionType>> extends CompositeType< + CompatibleUnion, + CompatibleUnionTreeView, + CompatibleUnionTreeViewDU +> { + readonly typeName: string; + readonly depth = 1; + readonly maxChunkCount = 1; + readonly fixedSize = null; + readonly minSize: number; + readonly maxSize: number; + readonly isList = true; + readonly isViewMutable = true; + readonly mixInSelectorBlockBytes = new Uint8Array(64); + readonly mixInSelectorBuffer = Buffer.from( + this.mixInSelectorBlockBytes.buffer, + this.mixInSelectorBlockBytes.byteOffset, + this.mixInSelectorBlockBytes.byteLength + ); + + readonly selectors: number[]; + private readonly selectorToType: Record>; + private readonly representativeType: Type; + private readonly selectorType = new UintNumberType(1); + + constructor(options: Types, opts?: CompatibleUnionOpts) { + super(); + + const selectorToType: Record> = {}; + const selectors = Object.keys(options) + .map((selector) => Number(selector)) + .sort((a, b) => a - b); + + if (selectors.length === 0) { + throw Error("CompatibleUnion must have at least one type option"); + } + + for (const selector of selectors) { + if (!Number.isSafeInteger(selector) || selector < 1 || selector > 127) { + throw Error(`CompatibleUnion selector ${selector} must be in range 1..127`); + } + selectorToType[selector] = options[selector]; + } + + for (let i = 0; i < selectors.length; i++) { + for (let j = i + 1; j < selectors.length; j++) { + const selectorA = selectors[i]; + const selectorB = selectors[j]; + if (!areTypesCompatible(selectorToType[selectorA], selectorToType[selectorB])) { + throw Error(`CompatibleUnion options ${selectorA} and ${selectorB} are not compatible`); + } + } + } + + this.selectorToType = selectorToType; + this.selectors = selectors; + this.representativeType = selectorToType[selectors[0]]; + this.typeName = + opts?.typeName ?? + `CompatibleUnion({${selectors.map((selector) => `${selector}: ${selectorToType[selector].typeName}`).join(",")}})`; + this.minSize = 1 + Math.min(...selectors.map((selector) => selectorToType[selector].minSize)); + this.maxSize = 1 + Math.max(...selectors.map((selector) => selectorToType[selector].maxSize)); + this.blocksBuffer = new Uint8Array(32); + } + + static named>>( + options: Types, + opts: Require + ): CompatibleUnionType { + return new (namedClass(CompatibleUnionType, opts.typeName))(options, opts); + } + + defaultValue(): CompatibleUnion { + throw Error("CompatibleUnion does not define a default value"); + } + + getView(tree: Tree): CompatibleUnionTreeView { + return new CompatibleUnionTreeView(this, tree); + } + + getViewDU(node: Node): CompatibleUnionTreeViewDU { + return new CompatibleUnionTreeViewDU(this, node); + } + + cacheOfViewDU(): unknown { + return; + } + + commitView(view: CompatibleUnionTreeView): Node { + return view.node; + } + + commitViewDU( + view: CompatibleUnionTreeViewDU, + hcOffset = 0, + hcByLevel: HashComputationLevel[] | null = null + ): Node { + view.commit(hcOffset, hcByLevel); + return view.node; + } + + value_serializedSize(value: CompatibleUnion): number { + return 1 + this.getType(value.selector).value_serializedSize(value.data); + } + + value_serializeToBytes(output: ByteViews, offset: number, value: CompatibleUnion): number { + output.uint8Array[offset] = value.selector; + return this.getType(value.selector).value_serializeToBytes(output, offset + 1, value.data); + } + + value_deserializeFromBytes( + data: ByteViews, + start: number, + end: number, + reuseBytes?: boolean + ): CompatibleUnion { + if (end <= start) { + throw Error("CompatibleUnion requires a selector byte"); + } + + const selector = data.uint8Array[start]; + const type = this.getType(selector); + return { + selector, + data: type.value_deserializeFromBytes(data, start + 1, end, reuseBytes), + }; + } + + tree_serializedSize(node: Node): number { + const selector = getLengthFromRootNode(node); + return 1 + this.getType(selector).tree_serializedSize(node.left); + } + + tree_serializeToBytes(output: ByteViews, offset: number, node: Node): number { + const selector = getLengthFromRootNode(node); + output.uint8Array[offset] = selector; + return this.getType(selector).tree_serializeToBytes(output, offset + 1, node.left); + } + + tree_deserializeFromBytes(data: ByteViews, start: number, end: number): Node { + if (end <= start) { + throw Error("CompatibleUnion requires a selector byte"); + } + + const selector = data.uint8Array[start]; + const valueNode = this.getType(selector).tree_deserializeFromBytes(data, start + 1, end); + return addLengthNode(valueNode, selector); + } + + hashTreeRoot(value: CompatibleUnion): Uint8Array { + const root = allocUnsafe(32); + this.hashTreeRootInto(value, root, 0); + return root; + } + + hashTreeRootInto(value: CompatibleUnion, output: Uint8Array, offset: number): void { + const type = this.getType(value.selector); + type.hashTreeRootInto(value.data, this.mixInSelectorBlockBytes, 0); + this.mixInSelectorBlockBytes.subarray(32).fill(0); + this.mixInSelectorBuffer.writeUIntLE(value.selector, 32, 6); + merkleizeBlocksBytes(this.mixInSelectorBlockBytes, 2, output, offset); + } + + protected getBlocksBytes(value: CompatibleUnion): Uint8Array { + this.getType(value.selector).hashTreeRootInto(value.data, this.blocksBuffer, 0); + return this.blocksBuffer; + } + + getPropertyGindex(prop: string): Gindex { + switch (prop) { + case "data": + return VALUE_GINDEX; + case "selector": + return SELECTOR_GINDEX; + default: + throw new Error(`Invalid CompatibleUnion type property ${prop}`); + } + } + + createFromProof(proof: Proof, root?: Uint8Array): CompatibleUnionTreeView { + const rootNode = Tree.createFromProof(proof).rootNode; + if (root !== undefined && !byteArrayEquals(rootNode.root, root)) { + throw new Error("Proof does not match trusted root"); + } + + return this.getView(new Tree(rootNode)); + } + + tree_createProof(node: Node, jsonPaths: JsonPath[]): Proof { + const gindices = this.tree_createProofGindexes(node, jsonPaths); + return createProof(node, { + type: ProofType.treeOffset, + gindices, + }); + } + + tree_createProofGindexes(node: Node, jsonPaths: JsonPath[]): Gindex[] { + const gindices: Gindex[] = []; + const selector = getLengthFromRootNode(node); + const selectedType = this.getType(selector); + const dataNode = getNode(node, VALUE_GINDEX); + + for (const jsonPath of jsonPaths) { + const [prop, ...remainingPath] = jsonPath; + + if (prop === undefined) { + gindices.push(...this.tree_getLeafGindices(BigInt(1), node)); + continue; + } + + if (prop === "selector") { + if (remainingPath.length > 0) { + throw Error("Invalid path: cannot navigate beyond CompatibleUnion selector"); + } + gindices.push(SELECTOR_GINDEX); + continue; + } + + if (prop !== "data") { + throw Error(`Invalid CompatibleUnion type property ${String(prop)}`); + } + + if (remainingPath.length === 0) { + if (isCompositeType(selectedType)) { + gindices.push( + ...selectedType.tree_getLeafGindices(VALUE_GINDEX, selectedType.fixedSize === null ? dataNode : undefined) + ); + } else { + gindices.push(VALUE_GINDEX); + } + continue; + } + + if (!isCompositeType(selectedType)) { + throw Error("Invalid path: cannot navigate beyond CompatibleUnion basic data"); + } + + const childGindices = selectedType.tree_createProofGindexes(dataNode, [remainingPath]); + for (const childGindex of childGindices) { + gindices.push(concatGindices([VALUE_GINDEX, childGindex])); + } + } + + return gindices; + } + + getPropertyType(prop: string): Type { + switch (prop) { + case "selector": + return this.selectorType; + case "data": + return this.representativeType; + default: + throw new Error(`Invalid CompatibleUnion type property ${prop}`); + } + } + + getIndexProperty(index: number): string { + if (index === 0) return "data"; + if (index === 1) return "selector"; + throw Error("CompatibleUnion index of out bounds"); + } + + tree_getLeafGindices(rootGindex: Gindex, rootNode?: Node): Gindex[] { + if (!rootNode) { + throw Error("rootNode required"); + } + + const selector = getLengthFromRootNode(rootNode); + const type = this.getType(selector); + const valueGindex = concatGindices([rootGindex, VALUE_GINDEX]); + const gindices: Gindex[] = []; + if (isCompositeType(type)) { + gindices.push(...type.tree_getLeafGindices(valueGindex, getNode(rootNode, VALUE_GINDEX))); + } else { + gindices.push(valueGindex); + } + gindices.push(concatGindices([rootGindex, SELECTOR_GINDEX])); + return gindices; + } + + tree_fromProofNode(node: Node): {node: Node; done: boolean} { + return {node, done: true}; + } + + fromJson(json: unknown): CompatibleUnion { + if (typeof json !== "object" || json === null) { + throw new Error("JSON must be of type object"); + } + + const union = json as {selector?: unknown; data?: unknown}; + const selector = parseSelector(union.selector); + const type = this.getType(selector); + if (!("data" in union)) { + throw new Error("JSON CompatibleUnion missing data"); + } + + return { + selector, + data: type.fromJson(union.data), + }; + } + + toJson(value: CompatibleUnion): Record { + return { + selector: value.selector.toString(10), + data: this.getType(value.selector).toJson(value.data), + }; + } + + clone(value: CompatibleUnion): CompatibleUnion { + return { + selector: value.selector, + data: this.getType(value.selector).clone(value.data), + }; + } + + equals(a: CompatibleUnion, b: CompatibleUnion): boolean { + if (a.selector !== b.selector) { + return false; + } + + return this.getType(a.selector).equals(a.data, b.data); + } + + getType(selector: number): Type { + const type = this.selectorToType[selector]; + if (type === undefined) { + throw Error(`Invalid CompatibleUnion selector ${selector}`); + } + return type; + } +} + +export class CompatibleUnionTreeView>> extends TreeView< + CompatibleUnionType +> { + constructor( + readonly type: CompatibleUnionType, + protected tree: Tree + ) { + super(); + } + + get node(): Node { + return this.tree.rootNode; + } + + get selector(): number { + return getLengthFromRootNode(this.tree.rootNode); + } + + get data(): unknown { + const selector = this.selector; + return this.type.getType(selector).tree_toValue(this.tree.rootNode.left); + } +} + +export class CompatibleUnionTreeViewDU>> extends TreeViewDU< + CompatibleUnionType +> { + constructor( + readonly type: CompatibleUnionType, + protected _rootNode: Node + ) { + super(); + } + + get node(): Node { + return this._rootNode; + } + + get cache(): unknown { + return undefined; + } + + get selector(): number { + return getLengthFromRootNode(this._rootNode); + } + + get data(): unknown { + const selector = this.selector; + return this.type.getType(selector).tree_toValue(this._rootNode.left); + } + + commit(hcOffset = 0, hcByLevel: HashComputationLevel[] | null = null): void { + if (hcByLevel !== null && this._rootNode.h0 === null) { + getHashComputations(this._rootNode, hcOffset, hcByLevel); + } + } + + protected clearCache(): void { + return; + } +} + +export function areTypesCompatible(a: Type, b: Type): boolean { + if (a === b) { + return true; + } + + if (a instanceof CompatibleUnionType && b instanceof CompatibleUnionType) { + return a.selectors.every((selectorA) => + b.selectors.every((selectorB) => areTypesCompatible(a.getType(selectorA), b.getType(selectorB))) + ); + } + + if (isBasicType(a) && isBasicType(b)) { + return areBasicTypesCompatible(a, b); + } + + if (a instanceof BitListType && b instanceof BitListType) { + return a.limitBits === b.limitBits; + } + if (a instanceof BitVectorType && b instanceof BitVectorType) { + return a.lengthBits === b.lengthBits; + } + if (a instanceof ProgressiveBitListType && b instanceof ProgressiveBitListType) { + return true; + } + + if (isByteListCompatibleType(a) && isByteListCompatibleType(b)) { + return getByteListCompatibleLimit(a) === getByteListCompatibleLimit(b); + } + if (isByteVectorCompatibleType(a) && isByteVectorCompatibleType(b)) { + return getByteVectorCompatibleLength(a) === getByteVectorCompatibleLength(b); + } + + if (isLimitedListType(a) && isLimitedListType(b)) { + return a.limit === b.limit && areTypesCompatible(a.elementType, b.elementType); + } + if (isVectorType(a) && isVectorType(b)) { + return a.length === b.length && areTypesCompatible(a.elementType, b.elementType); + } + + if (isProgressiveListType(a) && isProgressiveListType(b)) { + return areTypesCompatible(a.elementType, b.elementType); + } + + if (a instanceof ContainerType && b instanceof ContainerType) { + return areContainersCompatible(a, b); + } + if (a instanceof ProgressiveContainerType && b instanceof ProgressiveContainerType) { + return areProgressiveContainersCompatible(a, b); + } + + return false; +} + +function parseSelector(selector: unknown): number { + if (typeof selector === "number") { + if (Number.isSafeInteger(selector)) { + return selector; + } + } else if (typeof selector === "bigint") { + if (selector >= BigInt(Number.MIN_SAFE_INTEGER) && selector <= BigInt(Number.MAX_SAFE_INTEGER)) { + return Number(selector); + } + } else if (typeof selector === "string") { + const parsed = Number(selector); + if (Number.isSafeInteger(parsed) && parsed.toString(10) === selector) { + return parsed; + } + } + + throw Error("Invalid JSON CompatibleUnion selector"); +} + +function areBasicTypesCompatible(a: BasicType, b: BasicType): boolean { + if (a instanceof BooleanType || b instanceof BooleanType) { + return a instanceof BooleanType && b instanceof BooleanType; + } + + if (isUintType(a) && isUintType(b)) { + return a.byteLength === b.byteLength; + } + + return a.constructor === b.constructor && a.byteLength === b.byteLength; +} + +function isUintType(type: Type): type is UintNumberType | UintBigintType { + return type instanceof UintNumberType || type instanceof UintBigintType; +} + +function isByteBasicType(type: Type): boolean { + return isUintType(type) && type.byteLength === 1; +} + +function isByteListCompatibleType(type: Type): type is ByteListType | ListBasicType> { + return type instanceof ByteListType || (type instanceof ListBasicType && isByteBasicType(type.elementType)); +} + +function getByteListCompatibleLimit(type: ByteListType | ListBasicType>): number { + return type instanceof ByteListType ? type.limitBytes : type.limit; +} + +function isByteVectorCompatibleType(type: Type): type is ByteVectorType | VectorBasicType> { + return type instanceof ByteVectorType || (type instanceof VectorBasicType && isByteBasicType(type.elementType)); +} + +function getByteVectorCompatibleLength(type: ByteVectorType | VectorBasicType>): number { + return type instanceof ByteVectorType ? type.lengthBytes : type.length; +} + +function isLimitedListType( + type: Type +): type is ListBasicType> | ListCompositeType> { + return type instanceof ListBasicType || type instanceof ListCompositeType; +} + +function isVectorType( + type: Type +): type is VectorBasicType> | VectorCompositeType> { + return type instanceof VectorBasicType || type instanceof VectorCompositeType; +} + +function isProgressiveListType( + type: Type +): type is + | ProgressiveListBasicType> + | ProgressiveListCompositeType> { + return type instanceof ProgressiveListBasicType || type instanceof ProgressiveListCompositeType; +} + +function areContainersCompatible( + a: ContainerType>>, + b: ContainerType>> +): boolean { + if (a.fieldsEntries.length !== b.fieldsEntries.length) { + return false; + } + + for (let i = 0; i < a.fieldsEntries.length; i++) { + const fieldA = a.fieldsEntries[i]; + const fieldB = b.fieldsEntries[i]; + if (fieldA.fieldName !== fieldB.fieldName || !areTypesCompatible(fieldA.fieldType, fieldB.fieldType)) { + return false; + } + } + return true; +} + +function areProgressiveContainersCompatible( + a: ProgressiveContainerType>>, + b: ProgressiveContainerType>> +): boolean { + const fieldsA = progressiveContainerFieldMap(a); + const fieldsB = progressiveContainerFieldMap(b); + const maxActiveFields = Math.max(a.activeFields.bitLen, b.activeFields.bitLen); + + for (let i = 0; i < maxActiveFields; i++) { + if (a.activeFields.get(i) && b.activeFields.get(i)) { + const fieldA = a.getIndexProperty(i); + const fieldB = b.getIndexProperty(i); + if (fieldA === null || fieldB === null || fieldA !== fieldB) { + return false; + } + const fieldInfoA = fieldsA.get(fieldA); + const fieldInfoB = fieldsB.get(fieldB); + if ( + fieldInfoA === undefined || + fieldInfoB === undefined || + !areTypesCompatible(fieldInfoA.type, fieldInfoB.type) + ) { + return false; + } + } + } + + for (const [fieldName, fieldA] of fieldsA) { + const fieldB = fieldsB.get(fieldName); + if (fieldB !== undefined && fieldA.chunkIndex !== fieldB.chunkIndex) { + return false; + } + } + + return true; +} + +function progressiveContainerFieldMap( + type: ProgressiveContainerType>> +): Map}> { + const fields = new Map}>(); + for (const entry of type.fieldsEntries) { + fields.set(entry.fieldName as string, {chunkIndex: entry.chunkIndex, type: entry.fieldType}); + } + return fields; +} diff --git a/packages/ssz/src/type/composite.ts b/packages/ssz/src/type/composite.ts index 0de0d129..4001f6c6 100644 --- a/packages/ssz/src/type/composite.ts +++ b/packages/ssz/src/type/composite.ts @@ -291,24 +291,43 @@ export abstract class CompositeType extends Type { const gindexes: Gindex[] = []; for (const jsonPath of jsonPaths) { - const {type, gindex} = this.getPathInfo(jsonPath); - if (!isCompositeType(type)) { - gindexes.push(gindex); - } else { - // if the path subtype is composite, include the gindices of all the leaves - const leafGindexes = type.tree_getLeafGindices( - gindex, - type.fixedSize === null ? getNode(node, gindex) : undefined - ); - for (const gindex of leafGindexes) { - gindexes.push(gindex); - } - } + gindexes.push(...this.tree_createProofGindexesForPath(node, jsonPath)); } return gindexes; } + private tree_createProofGindexesForPath(node: Node, jsonPath: JsonPath): Gindex[] { + const [prop, ...remainingPath] = jsonPath; + + if (prop === undefined) { + return this.tree_getLeafGindices(BigInt(1), node); + } + + const childGindex = this.getPropertyGindex(prop); + if (childGindex === null) { + return this.tree_getLeafGindices(BigInt(1), node); + } + + const childType = this.getPropertyType(prop); + + if (!isCompositeType(childType)) { + if (remainingPath.length > 0) { + throw new Error("Invalid path: cannot navigate beyond a basic type"); + } + return [childGindex]; + } + + const childNode = getNode(node, childGindex); + if (remainingPath.length === 0) { + return childType.tree_getLeafGindices(childGindex, childNode); + } + + return childType + .tree_createProofGindexes(childNode, [remainingPath]) + .map((gindex) => concatGindices([childGindex, gindex])); + } + /** * Navigate to a subtype & gindex using a path */ diff --git a/packages/ssz/src/type/progressive.ts b/packages/ssz/src/type/progressive.ts new file mode 100644 index 00000000..d1925cda --- /dev/null +++ b/packages/ssz/src/type/progressive.ts @@ -0,0 +1,166 @@ +import { + BranchNode, + Gindex, + Node, + concatGindices, + digest64Into, + getNodesAtDepth, + merkleizeBlocksBytes, + subtreeFillToContents, + toGindex, + zeroNode, +} from "@chainsafe/persistent-merkle-tree"; + +export const PROGRESSIVE_LIST_MAX_SIZE = 0xffffffff; + +const BASE_CHUNK_COUNT = 1; +const SCALING_FACTOR = 4; +const ZERO_ROOT = new Uint8Array(32); + +/** + * Return the generalized index of a chunk inside a progressive Merkle tree. + * + * The progressive tree is a right-recursive, zero-terminated sequence of + * subtrees with capacities 1, 4, 16, ... + */ +export function progressiveChunkGindex(chunkIndex: number): Gindex { + if (chunkIndex < 0 || !Number.isSafeInteger(chunkIndex)) { + throw Error(`Invalid progressive chunk index ${chunkIndex}`); + } + + let subtreeIndex = 0; + let subtreeStart = 0; + let subtreeLength = BASE_CHUNK_COUNT; + while (chunkIndex >= subtreeStart + subtreeLength) { + subtreeStart += subtreeLength; + subtreeLength *= SCALING_FACTOR; + subtreeIndex++; + } + + const parts: Gindex[] = []; + for (let i = 0; i < subtreeIndex; i++) { + // Navigate to the successor subtree. + parts.push(BigInt(3)); + } + // Navigate to the subtree containing this chunk. + parts.push(BigInt(2)); + + const subtreeDepth = progressiveSubtreeDepth(subtreeIndex); + if (subtreeDepth > 0) { + parts.push(toGindex(subtreeDepth, BigInt(chunkIndex - subtreeStart))); + } + + return concatGindices(parts); +} + +export function progressiveChunkGindexFromRoot(rootGindex: Gindex, chunkIndex: number): Gindex { + return concatGindices([rootGindex, progressiveChunkGindex(chunkIndex)]); +} + +export function progressiveSubtreeCount(chunkCount: number): number { + let remaining = chunkCount; + let subtreeLength = BASE_CHUNK_COUNT; + let count = 0; + while (remaining > 0) { + remaining -= Math.min(remaining, subtreeLength); + subtreeLength *= SCALING_FACTOR; + count++; + } + return count; +} + +export function progressiveSubtreeDepth(subtreeIndex: number): number { + return subtreeIndex * 2; +} + +export function merkleizeProgressiveBytes( + chunksBytes: Uint8Array, + chunkCount: number, + output: Uint8Array, + offset: number +): void { + if (chunkCount === 0) { + output.set(ZERO_ROOT, offset); + return; + } + if (chunksBytes.length < chunkCount * 32) { + throw Error(`chunksBytes length ${chunksBytes.length} is less than chunkCount ${chunkCount}`); + } + + const subtreeRoots = new Array(progressiveSubtreeCount(chunkCount)); + let chunkOffset = 0; + let subtreeLength = BASE_CHUNK_COUNT; + + for (let subtreeIndex = 0; chunkOffset < chunkCount; subtreeIndex++) { + const subtreeChunkCount = Math.min(subtreeLength, chunkCount - chunkOffset); + const subtreeRoot = new Uint8Array(32); + const start = chunkOffset * 32; + const end = start + subtreeChunkCount * 32; + + let subtreeBytes: Uint8Array; + if (subtreeChunkCount === subtreeLength && (subtreeLength === 1 || subtreeChunkCount % 2 === 0)) { + subtreeBytes = chunksBytes.subarray(start, end); + } else { + subtreeBytes = new Uint8Array(Math.max(32, Math.ceil(subtreeChunkCount / 2) * 64)); + subtreeBytes.set(chunksBytes.subarray(start, end)); + } + + merkleizeBlocksBytes(subtreeBytes, subtreeLength, subtreeRoot, 0); + subtreeRoots[subtreeIndex] = subtreeRoot; + + chunkOffset += subtreeChunkCount; + subtreeLength *= SCALING_FACTOR; + } + + const root = new Uint8Array(32); + for (let i = subtreeRoots.length - 1; i >= 0; i--) { + digest64Into(subtreeRoots[i], root, root); + } + output.set(root, offset); +} + +export function progressiveSubtreeFillToContents(nodes: Node[]): Node { + if (nodes.length === 0) { + return zeroNode(0); + } + + const subtreeRoots: Node[] = []; + let nodeOffset = 0; + let subtreeLength = BASE_CHUNK_COUNT; + + for (let subtreeIndex = 0; nodeOffset < nodes.length; subtreeIndex++) { + const subtreeNodeCount = Math.min(subtreeLength, nodes.length - nodeOffset); + const subtreeDepth = progressiveSubtreeDepth(subtreeIndex); + subtreeRoots.push(subtreeFillToContents(nodes.slice(nodeOffset, nodeOffset + subtreeNodeCount), subtreeDepth)); + nodeOffset += subtreeNodeCount; + subtreeLength *= SCALING_FACTOR; + } + + let root = zeroNode(0); + for (let i = subtreeRoots.length - 1; i >= 0; i--) { + root = new BranchNode(subtreeRoots[i], root); + } + return root; +} + +export function getNodesAtProgressiveDepth(rootNode: Node, count: number): Node[] { + const nodes: Node[] = []; + let node = rootNode; + let remaining = count; + let subtreeLength = BASE_CHUNK_COUNT; + + for (let subtreeIndex = 0; remaining > 0; subtreeIndex++) { + if (node.isLeaf()) { + throw Error("Invalid progressive tree: missing subtree branch"); + } + + const subtreeNodeCount = Math.min(subtreeLength, remaining); + const subtreeDepth = progressiveSubtreeDepth(subtreeIndex); + nodes.push(...getNodesAtDepth(node.left, subtreeDepth, 0, subtreeNodeCount)); + node = node.right; + remaining -= subtreeNodeCount; + subtreeLength *= SCALING_FACTOR; + } + + return nodes; +} diff --git a/packages/ssz/src/type/progressiveBitList.ts b/packages/ssz/src/type/progressiveBitList.ts new file mode 100644 index 00000000..33aec49c --- /dev/null +++ b/packages/ssz/src/type/progressiveBitList.ts @@ -0,0 +1,161 @@ +import {allocUnsafe} from "@chainsafe/as-sha256"; +import { + Gindex, + Node, + Proof, + Tree, + concatGindices, + merkleizeBlocksBytes, + packedNodeRootsToBytes, + packedRootsBytesToLeafNodes, +} from "@chainsafe/persistent-merkle-tree"; +import {byteArrayEquals} from "../util/byteArray.ts"; +import {namedClass} from "../util/named.ts"; +import {Require} from "../util/types.ts"; +import {BitArray} from "../value/bitArray.ts"; +import {addLengthNode, getChunksNodeFromRootNode, getLengthFromRootNode} from "./arrayBasic.ts"; +import {BitArrayType} from "./bitArray.ts"; +import {deserializeUint8ArrayBitListFromBytes} from "./bitList.ts"; +import {ByteViews} from "./composite.ts"; +import { + PROGRESSIVE_LIST_MAX_SIZE, + getNodesAtProgressiveDepth, + merkleizeProgressiveBytes, + progressiveChunkGindex, + progressiveSubtreeFillToContents, +} from "./progressive.ts"; + +export interface ProgressiveBitListOptions { + typeName?: string; +} + +const CHUNKS_GINDEX = BigInt(2); +const LENGTH_GINDEX = BigInt(3); + +/** + * ProgressiveBitList: variable-length collection of boolean values without a limit. + * - Serialization is identical to BitList, including the padding bit. + * - Merkleization uses EIP-7916 progressive merkleization. + */ +export class ProgressiveBitListType extends BitArrayType { + readonly typeName: string; + readonly depth = 1; + readonly chunkDepth = 0; + readonly fixedSize = null; + readonly minSize = 1; + readonly maxSize = PROGRESSIVE_LIST_MAX_SIZE; + readonly maxChunkCount = Number.MAX_SAFE_INTEGER; + readonly isList = true; + readonly mixInLengthBlockBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength + ); + + constructor(opts?: ProgressiveBitListOptions) { + super(); + this.typeName = opts?.typeName ?? "ProgressiveBitList"; + } + + static named(opts: Require): ProgressiveBitListType { + return new (namedClass(ProgressiveBitListType, opts.typeName))(opts); + } + + defaultValue(): BitArray { + return BitArray.fromBitLen(0); + } + + createFromProof(proof: Proof, root?: Uint8Array): ReturnType { + const rootNode = Tree.createFromProof(proof).rootNode; + if (root !== undefined && !byteArrayEquals(rootNode.root, root)) { + throw new Error("Proof does not match trusted root"); + } + return this.getView(new Tree(rootNode)); + } + + value_serializedSize(value: BitArray): number { + return bitLenToSerializedLength(value.bitLen); + } + + value_serializeToBytes(output: ByteViews, offset: number, value: BitArray): number { + output.uint8Array.set(value.uint8Array, offset); + return applyPaddingBit(output.uint8Array, offset, value.bitLen); + } + + value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): BitArray { + const {uint8Array, bitLen} = deserializeUint8ArrayBitListFromBytes(data.uint8Array, start, end, reuseBytes); + return new BitArray(uint8Array, bitLen); + } + + tree_serializedSize(node: Node): number { + return bitLenToSerializedLength(getLengthFromRootNode(node)); + } + + tree_serializeToBytes(output: ByteViews, offset: number, node: Node): number { + const chunksNode = getChunksNodeFromRootNode(node); + const bitLen = getLengthFromRootNode(node); + + const byteLen = Math.ceil(bitLen / 8); + const chunkLen = Math.ceil(byteLen / 32); + const nodes = getNodesAtProgressiveDepth(chunksNode, chunkLen); + packedNodeRootsToBytes(output.dataView, offset, byteLen, nodes); + + return applyPaddingBit(output.uint8Array, offset, bitLen); + } + + tree_deserializeFromBytes(data: ByteViews, start: number, end: number): Node { + const {uint8Array, bitLen} = deserializeUint8ArrayBitListFromBytes(data.uint8Array, start, end); + const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); + const nodes = packedRootsBytesToLeafNodes(dataView, 0, uint8Array.length); + return addLengthNode(progressiveSubtreeFillToContents(nodes), bitLen); + } + + tree_getByteLen(node?: Node): number { + if (!node) throw new Error("ProgressiveBitListType requires a node to get leaves"); + return Math.ceil(getLengthFromRootNode(node) / 8); + } + + hashTreeRoot(value: BitArray): Uint8Array { + const root = allocUnsafe(32); + this.hashTreeRootInto(value, root, 0); + return root; + } + + hashTreeRootInto(value: BitArray, output: Uint8Array, offset: number): void { + const byteLen = value.uint8Array.length; + const chunkCount = Math.ceil(byteLen / 32); + const blockBytes = this.getBlocksBytes(value); + merkleizeProgressiveBytes(blockBytes, chunkCount, this.mixInLengthBlockBytes, 0); + this.mixInLengthBuffer.writeUIntLE(value.bitLen, 32, 6); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, 2, output, offset); + } + + tree_getLeafGindices(rootGindex: Gindex, rootNode?: Node): Gindex[] { + const byteLen = this.tree_getByteLen(rootNode); + const chunkCount = Math.ceil(byteLen / 32); + const gindices = new Array(chunkCount); + for (let i = 0; i < chunkCount; i++) { + gindices[i] = concatGindices([rootGindex, CHUNKS_GINDEX, progressiveChunkGindex(i)]); + } + gindices.push(concatGindices([rootGindex, LENGTH_GINDEX])); + return gindices; + } +} + +function bitLenToSerializedLength(bitLen: number): number { + const bytes = Math.ceil(bitLen / 8); + return bitLen % 8 === 0 ? bytes + 1 : bytes; +} + +function applyPaddingBit(output: Uint8Array, offset: number, bitLen: number): number { + const byteLen = Math.ceil(bitLen / 8); + const newOffset = offset + byteLen; + if (bitLen % 8 === 0) { + output[newOffset] = 1; + return newOffset + 1; + } + + output[newOffset - 1] |= 1 << (bitLen % 8); + return newOffset; +} diff --git a/packages/ssz/src/type/progressiveContainer.ts b/packages/ssz/src/type/progressiveContainer.ts new file mode 100644 index 00000000..bfe007be --- /dev/null +++ b/packages/ssz/src/type/progressiveContainer.ts @@ -0,0 +1,757 @@ +import { + BranchNode, + Gindex, + HashComputationLevel, + LeafNode, + Node, + Proof, + Tree, + concatGindices, + getHashComputations, + getNode, + setNode, + zeroNode, +} from "@chainsafe/persistent-merkle-tree"; +import {byteArrayEquals} from "../util/byteArray.ts"; +import {ValueWithCachedPermanentRoot, cacheRoot, symbolCachedPermanentRoot} from "../util/merkleize.ts"; +import {namedClass} from "../util/named.ts"; +import {Case} from "../util/strings.ts"; +import {Require} from "../util/types.ts"; +import {BitArray} from "../value/bitArray.ts"; +import {TreeView} from "../view/abstract.ts"; +import {TreeViewDU} from "../viewDU/abstract.ts"; +import {Type, ValueOf} from "./abstract.ts"; +import {BasicType, isBasicType} from "./basic.ts"; +import { + ByteViews, + CompositeType, + CompositeTypeAny, + CompositeView, + CompositeViewDU, + isCompositeType, +} from "./composite.ts"; +import {merkleizeProgressiveBytes, progressiveChunkGindex, progressiveSubtreeFillToContents} from "./progressive.ts"; +import {mixInActiveFields} from "./stableContainer.ts"; + +type BytesRange = {start: number; end: number}; + +export type ProgressiveContainerOptions> = { + typeName?: string; + jsonCase?: KeyCase; + casingMap?: CasingMap; + cachePermanentRootStruct?: boolean; +}; + +export type KeyCase = "eth2" | "snake" | "constant" | "camel" | "header" | "pascal"; + +type CasingMap> = Partial<{[K in keyof Fields]: string}>; + +export type ValueOfFields>> = {[K in keyof Fields]: ValueOf}; + +type FieldsView>> = { + [K in keyof Fields]: Fields[K] extends CompositeType + ? TV + : Fields[K] extends BasicType + ? V + : never; +}; + +type FieldsViewDU>> = { + [K in keyof Fields]: Fields[K] extends CompositeType + ? TVDU + : Fields[K] extends BasicType + ? V + : never; +}; + +export type ProgressiveContainerTreeViewType>> = FieldsView & + TreeView>; +export type ProgressiveContainerTreeViewDUType>> = FieldsViewDU & + TreeViewDU>; + +type ProgressiveContainerTreeViewTypeConstructor>> = new ( + type: ProgressiveContainerType, + tree: Tree +) => ProgressiveContainerTreeViewType; + +type ProgressiveContainerTreeViewDUTypeConstructor>> = new ( + type: ProgressiveContainerType, + node: Node, + cache?: unknown +) => ProgressiveContainerTreeViewDUType; + +type FieldEntry>> = { + fieldName: keyof Fields; + fieldType: Fields[keyof Fields]; + jsonKey: string; + gindex: Gindex; + chunkIndex: number; +}; + +const FIELDS_GINDEX = BigInt(2); + +/** + * ProgressiveContainer: ordered heterogeneous collection with EIP-7916 progressive merkleization. + * - Serialization is identical to Container over the active fields. + * - Hash tree root mixes in the active-fields bitvector from the type definition. + */ +export class ProgressiveContainerType>> extends CompositeType< + ValueOfFields, + ProgressiveContainerTreeViewType, + ProgressiveContainerTreeViewDUType +> { + readonly typeName: string; + readonly depth = 2; + readonly maxChunkCount: number; + readonly fixedSize: number | null; + readonly minSize: number; + readonly maxSize: number; + readonly isList = false; + readonly isViewMutable = true; + readonly fieldsEntries: FieldEntry[]; + readonly activeFields: BitArray; + protected readonly fieldsGindex: Record; + protected readonly jsonKeyToFieldName: Record; + protected readonly isFixedLen: boolean[]; + protected readonly fieldRangesFixedLen: BytesRange[]; + protected readonly variableOffsetsPosition: number[]; + readonly fixedEnd: number; + protected readonly TreeView: ProgressiveContainerTreeViewTypeConstructor; + protected readonly TreeViewDU: ProgressiveContainerTreeViewDUTypeConstructor; + private readonly tempRoot = new Uint8Array(32); + + constructor( + readonly fields: Fields, + activeFields: BitArray | boolean[], + readonly opts?: ProgressiveContainerOptions + ) { + super(opts?.cachePermanentRootStruct); + + this.activeFields = Array.isArray(activeFields) ? BitArray.fromBoolArray(activeFields) : activeFields.clone(); + validateActiveFields(this.activeFields, Object.keys(fields).length); + + this.typeName = opts?.typeName ?? renderProgressiveContainerTypeName(fields); + this.maxChunkCount = this.activeFields.bitLen; + + const activeFieldIndexes = this.activeFields.getTrueBitIndexes(); + const fieldNames = Object.keys(fields) as (keyof Fields)[]; + this.fieldsEntries = []; + for (let i = 0; i < fieldNames.length; i++) { + const fieldName = fieldNames[i]; + const chunkIndex = activeFieldIndexes[i]; + this.fieldsEntries.push({ + fieldName, + fieldType: fields[fieldName], + jsonKey: precomputeJsonKey(fieldName, opts?.casingMap, opts?.jsonCase), + gindex: concatGindices([FIELDS_GINDEX, progressiveChunkGindex(chunkIndex)]), + chunkIndex, + }); + } + + this.fieldsGindex = {} as Record; + for (const {fieldName, gindex} of this.fieldsEntries) { + this.fieldsGindex[fieldName] = gindex; + } + + this.jsonKeyToFieldName = {}; + for (const {fieldName, jsonKey} of this.fieldsEntries) { + this.jsonKeyToFieldName[jsonKey] = fieldName; + } + + const {minLen, maxLen, fixedSize} = precomputeSizes(fields); + this.minSize = minLen; + this.maxSize = maxLen; + this.fixedSize = fixedSize; + + const {isFixedLen, fieldRangesFixedLen, variableOffsetsPosition, fixedEnd} = precomputeSerdesData(fields); + this.isFixedLen = isFixedLen; + this.fieldRangesFixedLen = fieldRangesFixedLen; + this.variableOffsetsPosition = variableOffsetsPosition; + this.fixedEnd = fixedEnd; + + this.TreeView = getProgressiveContainerTreeViewClass(this); + this.TreeViewDU = getProgressiveContainerTreeViewDUClass(this); + + const fieldBytes = this.activeFields.bitLen * 32; + this.blocksBuffer = new Uint8Array(Math.ceil(fieldBytes / 64) * 64); + } + + static named>>( + fields: Fields, + activeFields: BitArray | boolean[], + opts: Require, "typeName"> + ): ProgressiveContainerType { + return new (namedClass(ProgressiveContainerType, opts.typeName))(fields, activeFields, opts); + } + + defaultValue(): ValueOfFields { + const value = {} as ValueOfFields; + for (const {fieldName, fieldType} of this.fieldsEntries) { + value[fieldName] = fieldType.defaultValue() as ValueOf; + } + return value; + } + + getView(tree: Tree): ProgressiveContainerTreeViewType { + return new this.TreeView(this, tree); + } + + getViewDU(node: Node, cache?: unknown): ProgressiveContainerTreeViewDUType { + return new this.TreeViewDU(this, node, cache); + } + + cacheOfViewDU(view: ProgressiveContainerTreeViewDUType): unknown { + return view.cache; + } + + commitView(view: ProgressiveContainerTreeViewType): Node { + return view.node; + } + + commitViewDU( + view: ProgressiveContainerTreeViewDUType, + hcOffset = 0, + hcByLevel: HashComputationLevel[] | null = null + ): Node { + view.commit(hcOffset, hcByLevel); + return view.node; + } + + createFromProof(proof: Proof, root?: Uint8Array): ProgressiveContainerTreeViewType { + const rootNode = Tree.createFromProof(proof).rootNode; + if (root !== undefined && !byteArrayEquals(rootNode.root, root)) { + throw new Error("Proof does not match trusted root"); + } + return this.getView(new Tree(rootNode)); + } + + value_serializedSize(value: ValueOfFields): number { + let totalSize = 0; + for (const {fieldName, fieldType} of this.fieldsEntries) { + totalSize += + fieldType.fixedSize === null ? 4 + fieldType.value_serializedSize(value[fieldName]) : fieldType.fixedSize; + } + return totalSize; + } + + value_serializeToBytes(output: ByteViews, offset: number, value: ValueOfFields): number { + let fixedIndex = offset; + let variableIndex = offset + this.fixedEnd; + + for (const {fieldName, fieldType} of this.fieldsEntries) { + if (fieldType.fixedSize === null) { + output.dataView.setUint32(fixedIndex, variableIndex - offset, true); + fixedIndex += 4; + variableIndex = fieldType.value_serializeToBytes(output, variableIndex, value[fieldName]); + } else { + fixedIndex = fieldType.value_serializeToBytes(output, fixedIndex, value[fieldName]); + } + } + return variableIndex; + } + + value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): ValueOfFields { + const fieldRanges = this.getFieldRanges(data.dataView, start, end); + const value = {} as {[K in keyof Fields]: unknown}; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType} = this.fieldsEntries[i]; + const fieldRange = fieldRanges[i]; + value[fieldName] = fieldType.value_deserializeFromBytes( + data, + start + fieldRange.start, + start + fieldRange.end, + reuseBytes + ); + } + + return value as ValueOfFields; + } + + tree_serializedSize(node: Node): number { + let totalSize = 0; + for (const {fieldType, gindex} of this.fieldsEntries) { + const fieldNode = getNode(node, gindex); + totalSize += fieldType.fixedSize === null ? 4 + fieldType.tree_serializedSize(fieldNode) : fieldType.fixedSize; + } + return totalSize; + } + + tree_serializeToBytes(output: ByteViews, offset: number, node: Node): number { + let fixedIndex = offset; + let variableIndex = offset + this.fixedEnd; + + for (const {fieldType, gindex} of this.fieldsEntries) { + const fieldNode = getNode(node, gindex); + if (fieldType.fixedSize === null) { + output.dataView.setUint32(fixedIndex, variableIndex - offset, true); + fixedIndex += 4; + variableIndex = fieldType.tree_serializeToBytes(output, variableIndex, fieldNode); + } else { + fixedIndex = fieldType.tree_serializeToBytes(output, fixedIndex, fieldNode); + } + } + return variableIndex; + } + + tree_deserializeFromBytes(data: ByteViews, start: number, end: number): Node { + const fieldRanges = this.getFieldRanges(data.dataView, start, end); + const nodes = new Array(this.activeFields.bitLen).fill(zeroNode(0)); + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldType, chunkIndex} = this.fieldsEntries[i]; + const fieldRange = fieldRanges[i]; + nodes[chunkIndex] = fieldType.tree_deserializeFromBytes(data, start + fieldRange.start, start + fieldRange.end); + } + + return new BranchNode(progressiveSubtreeFillToContents(nodes), activeFieldsToNode(this.activeFields)); + } + + hashTreeRootInto(value: ValueOfFields, output: Uint8Array, offset: number, safeCache = false): void { + if (this.cachePermanentRootStruct) { + const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; + if (cachedRoot) { + output.set(cachedRoot, offset); + return; + } + } + + const blocksBytes = this.getBlocksBytes(value); + merkleizeProgressiveBytes(blocksBytes, this.activeFields.bitLen, this.tempRoot, 0); + mixInActiveFields(this.tempRoot, this.activeFields, output, offset); + + if (this.cachePermanentRootStruct) { + cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); + } + } + + protected getBlocksBytes(struct: ValueOfFields): Uint8Array { + this.blocksBuffer.fill(0); + for (const {fieldName, fieldType, chunkIndex} of this.fieldsEntries) { + fieldType.hashTreeRootInto(struct[fieldName], this.blocksBuffer, chunkIndex * 32); + } + return this.blocksBuffer; + } + + getPropertyGindex(prop: string): Gindex | null { + const gindex = this.fieldsGindex[prop] ?? this.fieldsGindex[this.jsonKeyToFieldName[prop]]; + if (gindex === undefined) throw Error(`Unknown container property ${prop}`); + return gindex; + } + + getPropertyType(prop: string): Type { + const fieldName = this.fields[prop] ? prop : this.jsonKeyToFieldName[prop]; + const type = this.fields[fieldName]; + if (type === undefined) throw Error(`Unknown container property ${prop}`); + return type; + } + + getIndexProperty(index: number): string | null { + const entry = this.fieldsEntries.find((entry) => entry.chunkIndex === index); + return entry ? (entry.fieldName as string) : null; + } + + tree_getLeafGindices(rootGindex: Gindex, rootNode?: Node): Gindex[] { + const gindices: Gindex[] = []; + for (const {fieldName, fieldType} of this.fieldsEntries) { + const fieldGindex = this.fieldsGindex[fieldName]; + const fieldGindexFromRoot = concatGindices([rootGindex, fieldGindex]); + + if (fieldType.isBasic) { + gindices.push(fieldGindexFromRoot); + } else { + const compositeType = fieldType as unknown as CompositeTypeAny; + if (fieldType.fixedSize === null) { + if (!rootNode) { + throw new Error("variable type requires tree argument to get leaves"); + } + gindices.push(...compositeType.tree_getLeafGindices(fieldGindexFromRoot, getNode(rootNode, fieldGindex))); + } else { + gindices.push(...compositeType.tree_getLeafGindices(fieldGindexFromRoot)); + } + } + } + return gindices; + } + + tree_fromProofNode(node: Node): {node: Node; done: boolean} { + return {node, done: true}; + } + + fromJson(json: unknown): ValueOfFields { + if (typeof json !== "object") throw Error("JSON must be of type object"); + if (json === null) throw Error("JSON must not be null"); + + const value = {} as ValueOfFields; + for (const {fieldName, fieldType, jsonKey} of this.fieldsEntries) { + const jsonValue = (json as Record)[jsonKey]; + if (jsonValue === undefined) throw Error(`JSON expected key ${jsonKey} is undefined`); + value[fieldName] = fieldType.fromJson(jsonValue) as ValueOf; + } + return value; + } + + toJson(value: ValueOfFields): Record { + const json: Record = {}; + for (const {fieldName, fieldType, jsonKey} of this.fieldsEntries) { + json[jsonKey] = fieldType.toJson(value[fieldName]); + } + return json; + } + + clone(value: ValueOfFields): ValueOfFields { + const newValue = {} as ValueOfFields; + for (const {fieldName, fieldType} of this.fieldsEntries) { + newValue[fieldName] = fieldType.clone(value[fieldName]) as ValueOf; + } + return newValue; + } + + equals(a: ValueOfFields, b: ValueOfFields): boolean { + for (const {fieldName, fieldType} of this.fieldsEntries) { + if (!fieldType.equals(a[fieldName], b[fieldName])) { + return false; + } + } + return true; + } + + getFieldRanges(data: DataView, start: number, end: number): BytesRange[] { + if (this.variableOffsetsPosition.length === 0) { + const size = end - start; + if (size !== this.fixedEnd) { + throw Error(`${this.typeName} size ${size} not equal fixed size ${this.fixedEnd}`); + } + return this.fieldRangesFixedLen; + } + + const offsets = readVariableOffsets(data, start, end, this.fixedEnd, this.variableOffsetsPosition); + offsets.push(end - start); + + let variableIdx = 0; + let fixedIdx = 0; + const fieldRanges = new Array(this.isFixedLen.length); + for (let i = 0; i < this.isFixedLen.length; i++) { + if (this.isFixedLen[i]) { + fieldRanges[i] = this.fieldRangesFixedLen[fixedIdx++]; + } else { + fieldRanges[i] = {start: offsets[variableIdx], end: offsets[variableIdx + 1]}; + variableIdx++; + } + } + return fieldRanges; + } +} + +class ProgressiveContainerTreeView>> extends TreeView< + ProgressiveContainerType +> { + constructor( + readonly type: ProgressiveContainerType, + readonly tree: Tree + ) { + super(); + } + + get node(): Node { + return this.tree.rootNode; + } +} + +class ProgressiveContainerTreeViewDU>> extends TreeViewDU< + ProgressiveContainerType +> { + protected nodes: Node[] = []; + protected caches: unknown[] = []; + protected readonly nodesChanged = new Set(); + protected readonly viewsChanged = new Map(); + + constructor( + readonly type: ProgressiveContainerType, + protected _rootNode: Node, + cache?: {nodes: Node[]; caches: unknown[]} + ) { + super(); + if (cache) { + this.nodes = cache.nodes; + this.caches = cache.caches; + } + } + + get node(): Node { + return this._rootNode; + } + + get cache(): {nodes: Node[]; caches: unknown[]} { + return {nodes: this.nodes, caches: this.caches}; + } + + commit(hcOffset = 0, hcByLevel: HashComputationLevel[] | null = null): void { + for (const [index, view] of this.viewsChanged) { + const {fieldType, gindex} = this.type.fieldsEntries[index]; + const compositeType = fieldType as unknown as CompositeTypeAny; + const node = compositeType.commitViewDU(view); + this.nodes[index] = node; + this.caches[index] = compositeType.cacheOfViewDU(view); + this._rootNode = setNode(this._rootNode, gindex, node); + } + + for (const index of this.nodesChanged) { + this._rootNode = setNode(this._rootNode, this.type.fieldsEntries[index].gindex, this.nodes[index]); + } + + this.nodesChanged.clear(); + this.viewsChanged.clear(); + + if (hcByLevel !== null && this._rootNode.h0 === null) { + getHashComputations(this._rootNode, hcOffset, hcByLevel); + } + } + + protected clearCache(): void { + this.nodes = []; + this.caches = []; + this.nodesChanged.clear(); + this.viewsChanged.clear(); + } +} + +function getProgressiveContainerTreeViewClass>>( + containerType: ProgressiveContainerType +): {new (type: ProgressiveContainerType, tree: Tree): ProgressiveContainerTreeViewType} { + class CustomProgressiveContainerTreeView extends ProgressiveContainerTreeView {} + + for (const [index, {fieldName, fieldType, gindex}] of containerType.fieldsEntries.entries()) { + if (isBasicType(fieldType)) { + Object.defineProperty(CustomProgressiveContainerTreeView.prototype, fieldName, { + configurable: false, + enumerable: true, + get: function (this: CustomProgressiveContainerTreeView) { + return fieldType.tree_getFromNode(this.tree.getNode(gindex) as LeafNode); + }, + set: function (this: CustomProgressiveContainerTreeView, value) { + const leafNode = (this.tree.getNode(gindex) as LeafNode).clone(); + fieldType.tree_setToNode(leafNode, value); + this.tree.setNode(gindex, leafNode); + }, + }); + } else if (isCompositeType(fieldType)) { + Object.defineProperty(CustomProgressiveContainerTreeView.prototype, fieldName, { + configurable: false, + enumerable: true, + get: function (this: CustomProgressiveContainerTreeView) { + return fieldType.getView(this.tree.getSubtree(gindex)); + }, + set: function (this: CustomProgressiveContainerTreeView, view: CompositeView) { + this.tree.setNode(gindex, fieldType.commitView(view)); + }, + }); + } else { + throw Error(`Unknown fieldType ${fieldType.typeName} at index ${index}`); + } + } + + Object.defineProperty(CustomProgressiveContainerTreeView, "name", {value: containerType.typeName, writable: false}); + return CustomProgressiveContainerTreeView as unknown as { + new (type: ProgressiveContainerType, tree: Tree): ProgressiveContainerTreeViewType; + }; +} + +function getProgressiveContainerTreeViewDUClass>>( + containerType: ProgressiveContainerType +): { + new (type: ProgressiveContainerType, node: Node, cache?: unknown): ProgressiveContainerTreeViewDUType; +} { + class CustomProgressiveContainerTreeViewDU extends ProgressiveContainerTreeViewDU {} + + for (const [index, {fieldName, fieldType, gindex}] of containerType.fieldsEntries.entries()) { + if (isBasicType(fieldType)) { + Object.defineProperty(CustomProgressiveContainerTreeViewDU.prototype, fieldName, { + configurable: false, + enumerable: true, + get: function (this: CustomProgressiveContainerTreeViewDU) { + let node = this.nodes[index]; + if (node === undefined) { + node = getNode(this._rootNode, gindex); + this.nodes[index] = node; + } + return fieldType.tree_getFromNode(node as LeafNode); + }, + set: function (this: CustomProgressiveContainerTreeViewDU, value) { + let nodeChanged: LeafNode; + if (this.nodesChanged.has(index)) { + nodeChanged = this.nodes[index] as LeafNode; + } else { + nodeChanged = ((this.nodes[index] ?? getNode(this._rootNode, gindex)) as LeafNode).clone(); + this.nodes[index] = nodeChanged; + this.nodesChanged.add(index); + } + fieldType.tree_setToNode(nodeChanged, value); + }, + }); + } else if (isCompositeType(fieldType)) { + Object.defineProperty(CustomProgressiveContainerTreeViewDU.prototype, fieldName, { + configurable: false, + enumerable: true, + get: function (this: CustomProgressiveContainerTreeViewDU) { + const viewChanged = this.viewsChanged.get(index); + if (viewChanged) { + return viewChanged; + } + + let node = this.nodes[index]; + if (node === undefined) { + node = getNode(this._rootNode, gindex); + this.nodes[index] = node; + } + + const view = fieldType.getViewDU(node, this.caches[index]); + if (fieldType.isViewMutable) { + this.viewsChanged.set(index, view); + } + return view; + }, + set: function (this: CustomProgressiveContainerTreeViewDU, view: CompositeViewDU) { + this.viewsChanged.set(index, view); + }, + }); + } else { + throw Error(`Unknown fieldType ${fieldType.typeName} at index ${index}`); + } + } + + Object.defineProperty(CustomProgressiveContainerTreeViewDU, "name", {value: containerType.typeName, writable: false}); + return CustomProgressiveContainerTreeViewDU as unknown as { + new ( + type: ProgressiveContainerType, + node: Node, + cache?: unknown + ): ProgressiveContainerTreeViewDUType; + }; +} + +function validateActiveFields(activeFields: BitArray, fieldCount: number): void { + if (activeFields.bitLen === 0) { + throw Error("ProgressiveContainer activeFields must not be empty"); + } + if (activeFields.bitLen > 256) { + throw Error("ProgressiveContainer activeFields bit length must be <= 256"); + } + if (!activeFields.get(activeFields.bitLen - 1)) { + throw Error("ProgressiveContainer activeFields must not end in false"); + } + if (activeFields.getTrueBitIndexes().length !== fieldCount) { + throw Error("ProgressiveContainer activeFields true bit count must equal field count"); + } + if (fieldCount === 0) { + throw Error("ProgressiveContainer must have > 0 fields"); + } +} + +function activeFieldsToNode(activeFields: BitArray): Node { + const root = new Uint8Array(32); + root.set(activeFields.uint8Array); + return LeafNode.fromRoot(root); +} + +function readVariableOffsets( + data: DataView, + start: number, + end: number, + fixedEnd: number, + variableOffsetsPosition: number[] +): number[] { + const size = end - start; + const offsets = new Array(variableOffsetsPosition.length); + for (let i = 0; i < variableOffsetsPosition.length; i++) { + const offset = data.getUint32(start + variableOffsetsPosition[i], true); + + if (offset > size) { + throw new Error(`Offset out of bounds ${offset} > ${size}`); + } + if (i === 0) { + if (offset !== fixedEnd) { + throw new Error(`First offset must equal to fixedEnd ${offset} != ${fixedEnd}`); + } + } else if (offset < offsets[i - 1]) { + throw new Error(`Offsets must be increasing ${offset} < ${offsets[i - 1]}`); + } + + offsets[i] = offset; + } + + return offsets; +} + +function precomputeSerdesData(fields: Record>): { + isFixedLen: boolean[]; + fieldRangesFixedLen: BytesRange[]; + variableOffsetsPosition: number[]; + fixedEnd: number; +} { + const isFixedLen: boolean[] = []; + const fieldRangesFixedLen: BytesRange[] = []; + const variableOffsetsPosition: number[] = []; + let pointerFixed = 0; + + for (const fieldType of Object.values(fields)) { + isFixedLen.push(fieldType.fixedSize !== null); + if (fieldType.fixedSize === null) { + variableOffsetsPosition.push(pointerFixed); + pointerFixed += 4; + } else { + fieldRangesFixedLen.push({start: pointerFixed, end: pointerFixed + fieldType.fixedSize}); + pointerFixed += fieldType.fixedSize; + } + } + + return {isFixedLen, fieldRangesFixedLen, variableOffsetsPosition, fixedEnd: pointerFixed}; +} + +function precomputeSizes(fields: Record>): { + minLen: number; + maxLen: number; + fixedSize: number | null; +} { + let minLen = 0; + let maxLen = 0; + let fixedSize: number | null = 0; + + for (const fieldType of Object.values(fields)) { + minLen += fieldType.minSize; + maxLen += fieldType.maxSize; + + if (fieldType.fixedSize === null) { + minLen += 4; + maxLen += 4; + fixedSize = null; + } else if (fixedSize !== null) { + fixedSize += fieldType.fixedSize; + } + } + return {minLen, maxLen, fixedSize}; +} + +function precomputeJsonKey>( + fieldName: keyof Fields, + casingMap?: CasingMap, + jsonCase?: KeyCase +): string { + if (casingMap) { + const keyFromCaseMap = casingMap[fieldName]; + if (keyFromCaseMap === undefined) { + throw Error(`casingMap[${String(fieldName as symbol)}] not defined`); + } + return keyFromCaseMap as string; + } + if (jsonCase) return Case[jsonCase](fieldName as string); + return fieldName as string; +} + +function renderProgressiveContainerTypeName>>(fields: Fields): string { + const fieldNames = Object.keys(fields) as (keyof Fields)[]; + const fieldTypeNames = fieldNames + .map((fieldName) => `${String(fieldName as symbol)}: ${fields[fieldName].typeName}`) + .join(", "); + return `ProgressiveContainer({${fieldTypeNames}})`; +} diff --git a/packages/ssz/src/type/progressiveList.ts b/packages/ssz/src/type/progressiveList.ts new file mode 100644 index 00000000..15cdc9f5 --- /dev/null +++ b/packages/ssz/src/type/progressiveList.ts @@ -0,0 +1,852 @@ +import {allocUnsafe} from "@chainsafe/as-sha256"; +import { + Gindex, + HashComputationLevel, + LeafNode, + Node, + Proof, + Tree, + concatGindices, + getHashComputations, + getNode, + merkleizeBlocksBytes, + packedNodeRootsToBytes, + packedRootsBytesToLeafNodes, + setNode, +} from "@chainsafe/persistent-merkle-tree"; +import {byteArrayEquals} from "../util/byteArray.ts"; +import {ValueWithCachedPermanentRoot, cacheRoot, symbolCachedPermanentRoot} from "../util/merkleize.ts"; +import {namedClass} from "../util/named.ts"; +import {Require} from "../util/types.ts"; +import {TreeView} from "../view/abstract.ts"; +import {TreeViewDU} from "../viewDU/abstract.ts"; +import {ValueOf} from "./abstract.ts"; +import {ArrayType} from "./array.ts"; +import { + addLengthNode, + assertValidArrayLength, + getLengthFromRootNode, + setChunksNode, + value_deserializeFromBytesArrayBasic, + value_serializeToBytesArrayBasic, +} from "./arrayBasic.ts"; +import { + minSizeArrayComposite, + value_deserializeFromBytesArrayComposite, + value_serializeToBytesArrayComposite, + value_serializedSizeArrayComposite, +} from "./arrayComposite.ts"; +import {BasicType} from "./basic.ts"; +import {ByteViews} from "./composite.ts"; +import {CompositeType, CompositeView, CompositeViewDU} from "./composite.ts"; +import { + PROGRESSIVE_LIST_MAX_SIZE, + getNodesAtProgressiveDepth, + merkleizeProgressiveBytes, + progressiveChunkGindex, + progressiveSubtreeFillToContents, +} from "./progressive.ts"; + +export interface ProgressiveListOpts { + typeName?: string; + cachePermanentRootStruct?: boolean; +} + +const CHUNKS_GINDEX = BigInt(2); +const LENGTH_GINDEX = BigInt(3); + +export class ProgressiveListBasicType> extends ArrayType< + ElementType, + ProgressiveListBasicTreeView, + ProgressiveListBasicTreeViewDU +> { + readonly typeName: string; + readonly itemsPerChunk: number; + readonly depth = 1; + readonly chunkDepth = 0; + readonly maxChunkCount = Number.MAX_SAFE_INTEGER; + readonly fixedSize = null; + readonly minSize = 0; + readonly maxSize = PROGRESSIVE_LIST_MAX_SIZE; + readonly isList = true; + readonly isViewMutable = true; + readonly limit = Number.MAX_SAFE_INTEGER; + readonly mixInLengthBlockBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength + ); + protected readonly defaultLen = 0; + + constructor( + readonly elementType: ElementType, + opts?: ProgressiveListOpts + ) { + super(elementType, opts?.cachePermanentRootStruct); + + if (!elementType.isBasic) throw Error("elementType must be basic"); + + this.typeName = opts?.typeName ?? `ProgressiveList[${elementType.typeName}]`; + this.itemsPerChunk = 32 / elementType.byteLength; + } + + static named>( + elementType: ElementType, + opts: Require + ): ProgressiveListBasicType { + return new (namedClass(ProgressiveListBasicType, opts.typeName))(elementType, opts); + } + + getView(tree: Tree): ProgressiveListBasicTreeView { + return new ProgressiveListBasicTreeView(this, tree); + } + + getViewDU(node: Node): ProgressiveListBasicTreeViewDU { + return new ProgressiveListBasicTreeViewDU(this, node); + } + + commitView(view: ProgressiveListBasicTreeView): Node { + return view.node; + } + + commitViewDU( + view: ProgressiveListBasicTreeViewDU, + hcOffset = 0, + hcByLevel: HashComputationLevel[] | null = null + ): Node { + view.commit(hcOffset, hcByLevel); + return view.node; + } + + cacheOfViewDU(): unknown { + return; + } + + createFromProof(proof: Proof, root?: Uint8Array): ProgressiveListBasicTreeView { + const rootNode = Tree.createFromProof(proof).rootNode; + if (root !== undefined && !byteArrayEquals(rootNode.root, root)) { + throw new Error("Proof does not match trusted root"); + } + return this.getView(new Tree(rootNode)); + } + + value_serializedSize(value: ValueOf[]): number { + return value.length * this.elementType.byteLength; + } + + value_serializeToBytes(output: ByteViews, offset: number, value: ValueOf[]): number { + return value_serializeToBytesArrayBasic(this.elementType, value.length, output, offset, value); + } + + value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOf[] { + return value_deserializeFromBytesArrayBasic(this.elementType, data, start, end, this); + } + + tree_serializedSize(node: Node): number { + return this.tree_getLength(node) * this.elementType.byteLength; + } + + tree_serializeToBytes(output: ByteViews, offset: number, node: Node): number { + const chunksNode = this.tree_getChunksNode(node); + const length = this.tree_getLength(node); + const size = length * this.elementType.byteLength; + const chunkCount = Math.ceil(size / 32); + const nodes = getNodesAtProgressiveDepth(chunksNode, chunkCount); + packedNodeRootsToBytes(output.dataView, offset, size, nodes); + return offset + size; + } + + tree_deserializeFromBytes(data: ByteViews, start: number, end: number): Node { + const length = (end - start) / this.elementType.byteLength; + assertValidArrayLength(length, this, true); + const nodes = packedRootsBytesToLeafNodes(data.dataView, start, end); + return addLengthNode(progressiveSubtreeFillToContents(nodes), length); + } + + tree_getLength(node: Node): number { + return getLengthFromRootNode(node); + } + + tree_setLength(tree: Tree, length: number): void { + tree.rootNode = addLengthNode(tree.rootNode.left, length); + } + + tree_getChunksNode(node: Node): Node { + return node.left; + } + + tree_chunksNodeOffset(): number { + return 1; + } + + tree_setChunksNode( + rootNode: Node, + chunksNode: Node, + newLength: number | null, + hcOffset = 0, + hcByLevel: HashComputationLevel[] | null = null + ): Node { + return setChunksNode(rootNode, chunksNode, newLength, hcOffset, hcByLevel); + } + + hashTreeRoot(value: ValueOf[]): Uint8Array { + if (this.cachePermanentRootStruct) { + const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; + if (cachedRoot) { + return cachedRoot; + } + } + + const root = allocUnsafe(32); + this.hashTreeRootInto(value, root, 0, true); + return root; + } + + hashTreeRootInto(value: ValueOf[], output: Uint8Array, offset: number, safeCache = false): void { + if (this.cachePermanentRootStruct) { + const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; + if (cachedRoot) { + output.set(cachedRoot, offset); + return; + } + } + + const blockBytes = this.getBlocksBytes(value); + const chunkCount = Math.ceil(this.value_serializedSize(value) / 32); + + merkleizeProgressiveBytes(blockBytes, chunkCount, this.mixInLengthBlockBytes, 0); + this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, 2, output, offset); + + if (this.cachePermanentRootStruct) { + cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); + } + } + + protected getBlocksBytes(value: ValueOf[]): Uint8Array { + const byteLen = this.value_serializedSize(value); + const blockByteLen = Math.ceil(byteLen / 64) * 64; + if (blockByteLen > this.blocksBuffer.length) { + this.blocksBuffer = new Uint8Array(blockByteLen); + } + const blockBytes = this.blocksBuffer.subarray(0, blockByteLen); + const uint8Array = blockBytes.subarray(0, byteLen); + const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); + value_serializeToBytesArrayBasic(this.elementType, value.length, {uint8Array, dataView}, 0, value); + blockBytes.subarray(byteLen).fill(0); + return blockBytes; + } + + getPropertyGindex(prop: string | number): Gindex { + if (typeof prop !== "number") { + throw Error(`Invalid array index: ${prop}`); + } + + const chunkIndex = Math.floor(prop / this.itemsPerChunk); + return concatGindices([CHUNKS_GINDEX, progressiveChunkGindex(chunkIndex)]); + } + + tree_getLeafGindices(rootGindex: Gindex, rootNode?: Node): Gindex[] { + if (!rootNode) { + throw new Error("ProgressiveList type requires tree argument to get leaves"); + } + + const length = this.tree_getLength(rootNode); + const chunkCount = Math.ceil(length / this.itemsPerChunk); + const gindices = new Array(chunkCount); + for (let i = 0; i < chunkCount; i++) { + gindices[i] = concatGindices([rootGindex, CHUNKS_GINDEX, progressiveChunkGindex(i)]); + } + gindices.push(concatGindices([rootGindex, LENGTH_GINDEX])); + return gindices; + } + + tree_fromProofNode(node: Node): {node: Node; done: boolean} { + return {node, done: true}; + } +} + +export class ProgressiveListCompositeType< + ElementType extends CompositeType, CompositeView, CompositeViewDU>, +> extends ArrayType< + ElementType, + ProgressiveListCompositeTreeView, + ProgressiveListCompositeTreeViewDU +> { + readonly typeName: string; + readonly itemsPerChunk = 1; + readonly depth = 1; + readonly chunkDepth = 0; + readonly maxChunkCount = Number.MAX_SAFE_INTEGER; + readonly fixedSize = null; + readonly minSize: number; + readonly maxSize: number; + readonly isList = true; + readonly isViewMutable = true; + readonly limit = Number.MAX_SAFE_INTEGER; + readonly mixInLengthBlockBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength + ); + protected readonly defaultLen = 0; + + constructor( + readonly elementType: ElementType, + opts?: ProgressiveListOpts + ) { + super(elementType, opts?.cachePermanentRootStruct); + + if (elementType.isBasic) throw Error("elementType must not be basic"); + + this.typeName = opts?.typeName ?? `ProgressiveList[${elementType.typeName}]`; + this.minSize = minSizeArrayComposite(elementType, 0); + this.maxSize = PROGRESSIVE_LIST_MAX_SIZE; + } + + static named< + ElementType extends CompositeType, CompositeView, CompositeViewDU>, + >( + elementType: ElementType, + opts: Require + ): ProgressiveListCompositeType { + return new (namedClass(ProgressiveListCompositeType, opts.typeName))(elementType, opts); + } + + getView(tree: Tree): ProgressiveListCompositeTreeView { + return new ProgressiveListCompositeTreeView(this, tree); + } + + getViewDU(node: Node, cache?: unknown): ProgressiveListCompositeTreeViewDU { + return new ProgressiveListCompositeTreeViewDU(this, node, cache as ProgressiveListCompositeTreeViewDUCache); + } + + commitView(view: ProgressiveListCompositeTreeView): Node { + return view.node; + } + + commitViewDU( + view: ProgressiveListCompositeTreeViewDU, + hcOffset = 0, + hcByLevel: HashComputationLevel[] | null = null + ): Node { + view.commit(hcOffset, hcByLevel); + return view.node; + } + + cacheOfViewDU(): unknown { + return; + } + + createFromProof(proof: Proof, root?: Uint8Array): ProgressiveListCompositeTreeView { + const rootNode = Tree.createFromProof(proof).rootNode; + if (root !== undefined && !byteArrayEquals(rootNode.root, root)) { + throw new Error("Proof does not match trusted root"); + } + return this.getView(new Tree(rootNode)); + } + + value_serializedSize(value: ValueOf[]): number { + return value_serializedSizeArrayComposite(this.elementType, value.length, value); + } + + value_serializeToBytes(output: ByteViews, offset: number, value: ValueOf[]): number { + return value_serializeToBytesArrayComposite(this.elementType, value.length, output, offset, value); + } + + value_deserializeFromBytes( + data: ByteViews, + start: number, + end: number, + reuseBytes?: boolean + ): ValueOf[] { + return value_deserializeFromBytesArrayComposite(this.elementType, data, start, end, this, reuseBytes); + } + + tree_serializedSize(node: Node): number { + const chunksNode = this.tree_getChunksNode(node); + const length = this.tree_getLength(node); + const nodes = getNodesAtProgressiveDepth(chunksNode, length); + + if (this.elementType.fixedSize === null) { + let size = 0; + for (const node of nodes) { + size += 4 + this.elementType.tree_serializedSize(node); + } + return size; + } + + return length * this.elementType.fixedSize; + } + + tree_serializeToBytes(output: ByteViews, offset: number, node: Node): number { + const chunksNode = this.tree_getChunksNode(node); + const length = this.tree_getLength(node); + const nodes = getNodesAtProgressiveDepth(chunksNode, length); + + if (this.elementType.fixedSize === null) { + let variableIndex = offset + length * 4; + for (let i = 0; i < nodes.length; i++) { + output.dataView.setUint32(offset + i * 4, variableIndex - offset, true); + variableIndex = this.elementType.tree_serializeToBytes(output, variableIndex, nodes[i]); + } + return variableIndex; + } + + for (const node of nodes) { + offset = this.elementType.tree_serializeToBytes(output, offset, node); + } + return offset; + } + + tree_deserializeFromBytes(data: ByteViews, start: number, end: number): Node { + const offsets = readOffsetsProgressiveListComposite(this.elementType.fixedSize, data.dataView, start, end, this); + const length = offsets.length; + const nodes = new Array(length); + + for (let i = 0; i < length; i++) { + const startEl = start + offsets[i]; + const endEl = i === length - 1 ? end : start + offsets[i + 1]; + nodes[i] = this.elementType.tree_deserializeFromBytes(data, startEl, endEl); + } + + return addLengthNode(progressiveSubtreeFillToContents(nodes), length); + } + + tree_getLength(node: Node): number { + return getLengthFromRootNode(node); + } + + tree_setLength(tree: Tree, length: number): void { + tree.rootNode = addLengthNode(tree.rootNode.left, length); + } + + tree_getChunksNode(node: Node): Node { + return node.left; + } + + tree_chunksNodeOffset(): number { + return 1; + } + + tree_setChunksNode( + rootNode: Node, + chunksNode: Node, + newLength: number | null, + hcOffset = 0, + hcByLevel: HashComputationLevel[] | null = null + ): Node { + return setChunksNode(rootNode, chunksNode, newLength, hcOffset, hcByLevel); + } + + hashTreeRoot(value: ValueOf[]): Uint8Array { + if (this.cachePermanentRootStruct) { + const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; + if (cachedRoot) { + return cachedRoot; + } + } + + const root = allocUnsafe(32); + this.hashTreeRootInto(value, root, 0, true); + return root; + } + + hashTreeRootInto(value: ValueOf[], output: Uint8Array, offset: number, safeCache = false): void { + if (this.cachePermanentRootStruct) { + const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; + if (cachedRoot) { + output.set(cachedRoot, offset); + return; + } + } + + const blockBytes = this.getBlocksBytes(value); + + merkleizeProgressiveBytes(blockBytes, value.length, this.mixInLengthBlockBytes, 0); + this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, 2, output, offset); + + if (this.cachePermanentRootStruct) { + cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); + } + } + + protected getBlocksBytes(value: ValueOf[]): Uint8Array { + const blockBytesLen = Math.ceil(value.length / 2) * 64; + if (blockBytesLen > this.blocksBuffer.length) { + this.blocksBuffer = new Uint8Array(blockBytesLen); + } + const blockBytes = this.blocksBuffer.subarray(0, blockBytesLen); + for (let i = 0; i < value.length; i++) { + this.elementType.hashTreeRootInto(value[i], blockBytes, i * 32); + } + if (value.length % 2 === 1) { + blockBytes.subarray(value.length * 32, value.length * 32 + 32).fill(0); + } + return blockBytes; + } + + getPropertyGindex(prop: string | number): Gindex { + if (typeof prop !== "number") { + throw Error(`Invalid array index: ${prop}`); + } + + return concatGindices([CHUNKS_GINDEX, progressiveChunkGindex(prop)]); + } + + tree_getLeafGindices(rootGindex: Gindex, rootNode?: Node): Gindex[] { + if (!rootNode) { + throw new Error("ProgressiveList type requires tree argument to get leaves"); + } + + const length = this.tree_getLength(rootNode); + const gindices: Gindex[] = []; + for (let i = 0; i < length; i++) { + const elementGindexFromListRoot = concatGindices([CHUNKS_GINDEX, progressiveChunkGindex(i)]); + const elementGindex = concatGindices([rootGindex, elementGindexFromListRoot]); + if (this.elementType.isBasic) { + gindices.push(elementGindex); + } else if (this.elementType.fixedSize === null) { + gindices.push( + ...this.elementType.tree_getLeafGindices(elementGindex, getNode(rootNode, elementGindexFromListRoot)) + ); + } else { + gindices.push(...this.elementType.tree_getLeafGindices(elementGindex)); + } + } + gindices.push(concatGindices([rootGindex, LENGTH_GINDEX])); + return gindices; + } + + tree_fromProofNode(node: Node): {node: Node; done: boolean} { + return {node, done: true}; + } +} + +export class ProgressiveListBasicTreeView> extends TreeView< + ProgressiveListBasicType +> { + constructor( + readonly type: ProgressiveListBasicType, + protected tree: Tree + ) { + super(); + } + + get length(): number { + return this.type.tree_getLength(this.tree.rootNode); + } + + get node(): Node { + return this.tree.rootNode; + } + + get(index: number): ValueOf { + const chunkIndex = Math.floor(index / this.type.itemsPerChunk); + const leafNode = this.tree.getNode(chunkGindexFromListRoot(chunkIndex)) as LeafNode; + return this.type.elementType.tree_getFromPackedNode(leafNode, index) as ValueOf; + } + + set(index: number, value: ValueOf): void { + const length = this.length; + if (index >= length) { + throw Error(`Error setting index over length ${index} > ${length}`); + } + + const chunkIndex = Math.floor(index / this.type.itemsPerChunk); + const gindex = chunkGindexFromListRoot(chunkIndex); + const leafNode = (this.tree.getNode(gindex) as LeafNode).clone(); + this.type.elementType.tree_setToPackedNode(leafNode, index, value); + this.tree.setNode(gindex, leafNode); + } + + getAll(values?: ValueOf[]): ValueOf[] { + const length = this.length; + if (values && values.length !== length) { + throw Error(`Expected ${length} values, got ${values.length}`); + } + + const chunksNode = this.type.tree_getChunksNode(this.node); + const chunkCount = Math.ceil(length / this.type.itemsPerChunk); + const leafNodes = getNodesAtProgressiveDepth(chunksNode, chunkCount) as LeafNode[]; + values = values ?? new Array>(length); + + for (let i = 0; i < length; i++) { + values[i] = this.type.elementType.tree_getFromPackedNode( + leafNodes[Math.floor(i / this.type.itemsPerChunk)], + i + ) as ValueOf; + } + return values; + } +} + +export class ProgressiveListBasicTreeViewDU> extends TreeViewDU< + ProgressiveListBasicType +> { + constructor( + readonly type: ProgressiveListBasicType, + protected _rootNode: Node + ) { + super(); + } + + get length(): number { + return this.type.tree_getLength(this._rootNode); + } + + get node(): Node { + return this._rootNode; + } + + get cache(): unknown { + return undefined; + } + + commit(hcOffset = 0, hcByLevel: HashComputationLevel[] | null = null): void { + if (hcByLevel !== null && this._rootNode.h0 === null) { + getHashComputations(this._rootNode, hcOffset, hcByLevel); + } + } + + get(index: number): ValueOf { + const chunkIndex = Math.floor(index / this.type.itemsPerChunk); + const leafNode = getNode(this._rootNode, chunkGindexFromListRoot(chunkIndex)) as LeafNode; + return this.type.elementType.tree_getFromPackedNode(leafNode, index) as ValueOf; + } + + set(index: number, value: ValueOf): void { + const length = this.length; + if (index >= length) { + throw Error(`Error setting index over length ${index} > ${length}`); + } + + const chunkIndex = Math.floor(index / this.type.itemsPerChunk); + const gindex = chunkGindexFromListRoot(chunkIndex); + const leafNode = (getNode(this._rootNode, gindex) as LeafNode).clone(); + this.type.elementType.tree_setToPackedNode(leafNode, index, value); + this._rootNode = setNode(this._rootNode, gindex, leafNode); + } + + push(value: ValueOf): void { + const values = this.type.tree_toValue(this._rootNode); + values.push(value); + this._rootNode = this.type.value_toTree(values); + } + + getAll(values?: ValueOf[]): ValueOf[] { + const view = new ProgressiveListBasicTreeView(this.type, new Tree(this._rootNode)); + return view.getAll(values); + } + + protected clearCache(): void { + // No cached data to clear. + } +} + +export class ProgressiveListCompositeTreeView< + ElementType extends CompositeType, CompositeView, CompositeViewDU>, +> extends TreeView> { + constructor( + readonly type: ProgressiveListCompositeType, + protected tree: Tree + ) { + super(); + } + + get length(): number { + return this.type.tree_getLength(this.tree.rootNode); + } + + get node(): Node { + return this.tree.rootNode; + } + + get(index: number): CompositeView { + const subtree = this.tree.getSubtree(chunkGindexFromListRoot(index)); + return this.type.elementType.getView(subtree); + } + + set(index: number, view: CompositeView): void { + const length = this.length; + if (index >= length) { + throw Error(`Error setting index over length ${index} > ${length}`); + } + + this.tree.setNode(chunkGindexFromListRoot(index), this.type.elementType.commitView(view)); + } + + getAllReadonlyValues(values?: ValueOf[]): ValueOf[] { + const length = this.length; + if (values && values.length !== length) { + throw Error(`Expected ${length} values, got ${values.length}`); + } + + const chunksNode = this.type.tree_getChunksNode(this.node); + const nodes = getNodesAtProgressiveDepth(chunksNode, length); + values = values ?? new Array>(length); + for (let i = 0; i < length; i++) { + values[i] = this.type.elementType.tree_toValue(nodes[i]); + } + return values; + } +} + +export class ProgressiveListCompositeTreeViewDU< + ElementType extends CompositeType, CompositeView, CompositeViewDU>, +> extends TreeViewDU> { + private caches: unknown[] = []; + private readonly viewsChanged = new Map>(); + + constructor( + readonly type: ProgressiveListCompositeType, + protected _rootNode: Node, + cache?: ProgressiveListCompositeTreeViewDUCache + ) { + super(); + if (cache) { + this.caches = cache.caches; + } + } + + get length(): number { + return this.type.tree_getLength(this._rootNode); + } + + get node(): Node { + return this._rootNode; + } + + get cache(): ProgressiveListCompositeTreeViewDUCache { + return {caches: this.caches}; + } + + commit(hcOffset = 0, hcByLevel: HashComputationLevel[] | null = null): void { + for (const [index, view] of this.viewsChanged) { + const gindex = chunkGindexFromListRoot(index); + this._rootNode = setNode(this._rootNode, gindex, this.type.elementType.commitViewDU(view)); + this.caches[index] = this.type.elementType.cacheOfViewDU(view); + } + this.viewsChanged.clear(); + + if (hcByLevel !== null && this._rootNode.h0 === null) { + getHashComputations(this._rootNode, hcOffset, hcByLevel); + } + } + + get(index: number): CompositeViewDU { + const changed = this.viewsChanged.get(index); + if (changed) { + return changed; + } + + const view = this.type.elementType.getViewDU( + getNode(this._rootNode, chunkGindexFromListRoot(index)), + this.caches[index] + ); + if (this.type.elementType.isViewMutable) { + this.viewsChanged.set(index, view); + } + return view; + } + + set(index: number, view: CompositeViewDU): void { + const length = this.length; + if (index >= length) { + throw Error(`Error setting index over length ${index} > ${length}`); + } + + this.viewsChanged.set(index, view); + } + + push(view: CompositeViewDU): void { + const values = this.type.tree_toValue(this._rootNode); + values.push(this.type.elementType.toValueFromViewDU(view)); + this._rootNode = this.type.value_toTree(values); + } + + protected clearCache(): void { + this.caches = []; + this.viewsChanged.clear(); + } +} + +type ProgressiveListCompositeTreeViewDUCache = { + caches: unknown[]; +}; + +function chunkGindexFromListRoot(chunkIndex: number): Gindex { + return concatGindices([CHUNKS_GINDEX, progressiveChunkGindex(chunkIndex)]); +} + +function readOffsetsProgressiveListComposite( + elementFixedSize: null | number, + data: DataView, + start: number, + end: number, + arrayProps: {isList: true; limit: number} +): Uint32Array { + const size = end - start; + + if (elementFixedSize === null) { + const offsets = readVariableOffsetsProgressiveList(data, start, size); + assertValidArrayLength(offsets.length, arrayProps); + return offsets; + } + + if (elementFixedSize === 0) { + throw Error("element fixed length is 0"); + } + if (size % elementFixedSize !== 0) { + throw Error(`size ${size} is not multiple of element fixedSize ${elementFixedSize}`); + } + + const length = size / elementFixedSize; + assertValidArrayLength(length, arrayProps); + const offsets = new Uint32Array(length); + for (let i = 0; i < length; i++) { + offsets[i] = i * elementFixedSize; + } + return offsets; +} + +function readVariableOffsetsProgressiveList(data: DataView, start: number, size: number): Uint32Array { + if (size === 0) { + return new Uint32Array(0); + } + if (size < 4) { + throw Error(`Variable length list data is too short ${size}`); + } + + const firstOffset = data.getUint32(start, true); + if (firstOffset % 4 !== 0) { + throw Error(`First offset must be a multiple of 4, got ${firstOffset}`); + } + if (firstOffset > size) { + throw Error(`First offset out of bounds ${firstOffset} > ${size}`); + } + + const length = firstOffset / 4; + if (length === 0) { + throw Error("Variable length list first offset must be greater than zero"); + } + + const offsets = new Uint32Array(length); + offsets[0] = firstOffset; + + for (let i = 1; i < length; i++) { + const offset = data.getUint32(start + i * 4, true); + if (offset < offsets[i - 1]) { + throw Error(`Offsets must be increasing ${offset} < ${offsets[i - 1]}`); + } + if (offset > size) { + throw Error(`Offset out of bounds ${offset} > ${size}`); + } + offsets[i] = offset; + } + + return offsets; +} diff --git a/packages/ssz/test/spec/generic/index.test.ts b/packages/ssz/test/spec/generic/index.test.ts index 430948a6..bc1cf067 100644 --- a/packages/ssz/test/spec/generic/index.test.ts +++ b/packages/ssz/test/spec/generic/index.test.ts @@ -14,64 +14,77 @@ const rootGenericSszPath = path.join( "ssz_generic" ); +const GENERIC_TEST_TIMEOUT_MS = 120_000; + for (const testType of fs.readdirSync(rootGenericSszPath)) { const testTypePath = path.join(rootGenericSszPath, testType); - describe(`${testType} invalid`, () => { - const invalidCasesPath = path.join(testTypePath, "invalid"); - for (const invalidCase of fs.readdirSync(invalidCasesPath)) { - const onlyId = process.env.ONLY_ID; - if (onlyId && !invalidCase.includes(onlyId)) { - continue; - } - - it(invalidCase, () => { - // TODO: Strong type errors and assert that the entire it() throws known errors - if (invalidCase.endsWith("_0")) { - expect(() => getTestType(testType, invalidCase), "Must throw constructing type").toThrow(); - return; - } + const onlyId = process.env.ONLY_ID; - const type = getTestType(testType, invalidCase); - const testData = parseSszGenericInvalidTestcase(path.join(invalidCasesPath, invalidCase)); + const invalidCasesPath = path.join(testTypePath, "invalid"); + const invalidCases = fs + .readdirSync(invalidCasesPath) + .filter((invalidCase) => onlyId === undefined || invalidCase.includes(onlyId)); + if (invalidCases.length > 0) { + describe(`${testType} invalid`, () => { + for (const invalidCase of invalidCases) { + it( + invalidCase, + () => { + // TODO: Strong type errors and assert that the entire it() throws known errors + if ((testType === "basic_vector" || testType === "bitvector") && invalidCase.endsWith("_0")) { + expect(() => getTestType(testType, invalidCase), "Must throw constructing type").toThrow(); + return; + } - if (process.env.DEBUG) { - console.log({serialized: Buffer.from(testData.serialized).toString("hex")}); - } - - // Unlike the valid suite, invalid encodings do not have any value or hash tree root. The serialized data - // should simply not be decoded without raising an error. - // Note that for some type declarations in the invalid suite, the type itself may technically be invalid. - // This is a valid way of detecting invalid data too. E.g. a 0-length basic vector. - expect(() => type.deserialize(testData.serialized), "Must throw on deserialize").toThrow(); - }); - } - }); + const type = getTestType(testType, invalidCase); + const testData = parseSszGenericInvalidTestcase(path.join(invalidCasesPath, invalidCase)); - describe(`${testType} valid`, () => { - const validCasesPath = path.join(testTypePath, "valid"); - for (const validCase of fs.readdirSync(validCasesPath)) { - // NOTE: ComplexTestStruct tests are not correctly generated. - // where deserialized .d value is D: '0x00'. However the tests guide mark that field as D: Bytes[256]. - // Those test won't be fixed since most implementations staticly compile types. - if (validCase.startsWith("ComplexTestStruct")) { - continue; - } + if (process.env.DEBUG) { + console.log({serialized: Buffer.from(testData.serialized).toString("hex")}); + } - const onlyId = process.env.ONLY_ID; - if (onlyId && !validCase.includes(onlyId)) { - continue; + // Unlike the valid suite, invalid encodings do not have any value or hash tree root. The serialized data + // should simply not be decoded without raising an error. + // Note that for some type declarations in the invalid suite, the type itself may technically be invalid. + // This is a valid way of detecting invalid data too. E.g. a 0-length basic vector. + expect(() => type.deserialize(testData.serialized), "Must throw on deserialize").toThrow(); + }, + GENERIC_TEST_TIMEOUT_MS + ); } + }); + } - it(validCase, () => { - const type = getTestType(testType, validCase); - const testData = parseSszGenericValidTestcase(path.join(validCasesPath, validCase)); - runValidSszTest(type, { - root: testData.root, - serialized: testData.serialized, - jsonValue: testData.jsonValue, - }); - }); + const validCasesPath = path.join(testTypePath, "valid"); + const validCases = fs.readdirSync(validCasesPath).filter((validCase) => { + // NOTE: ComplexTestStruct tests are not correctly generated. + // where deserialized .d value is D: '0x00'. However the tests guide mark that field as D: Bytes[256]. + // Those test won't be fixed since most implementations staticly compile types. + if (validCase.startsWith("ComplexTestStruct")) { + return false; } + + return onlyId === undefined || validCase.includes(onlyId); }); + + if (validCases.length > 0) { + describe(`${testType} valid`, () => { + for (const validCase of validCases) { + it( + validCase, + () => { + const type = getTestType(testType, validCase); + const testData = parseSszGenericValidTestcase(path.join(validCasesPath, validCase)); + runValidSszTest(type, { + root: testData.root, + serialized: testData.serialized, + jsonValue: testData.jsonValue, + }); + }, + GENERIC_TEST_TIMEOUT_MS + ); + } + }); + } } diff --git a/packages/ssz/test/spec/generic/types.ts b/packages/ssz/test/spec/generic/types.ts index 63bed5df..4054e9f3 100644 --- a/packages/ssz/test/spec/generic/types.ts +++ b/packages/ssz/test/spec/generic/types.ts @@ -2,8 +2,14 @@ import { BitListType, BitVectorType, BooleanType, + CompatibleUnionType, ContainerType, ListBasicType, + ListCompositeType, + ProgressiveBitListType, + ProgressiveContainerType, + ProgressiveListBasicType, + ProgressiveListCompositeType, Type, UintBigintType, UintNumberType, @@ -55,6 +61,19 @@ const VarTestStruct = new ContainerType({ C: uint8, }); +// class ProgressiveTestStruct(Container): +// A: ProgressiveList[byte] +// B: ProgressiveList[uint64] +// C: ProgressiveList[SmallTestStruct] +// D: ProgressiveList[ProgressiveList[VarTestStruct]] +const ProgressiveListVarTestStruct = new ProgressiveListCompositeType(VarTestStruct); +const ProgressiveTestStruct = new ContainerType({ + A: new ProgressiveListBasicType(byte), + B: new ProgressiveListBasicType(uint64), + C: new ProgressiveListCompositeType(SmallTestStruct), + D: new ProgressiveListCompositeType(ProgressiveListVarTestStruct), +}); + // class ComplexTestStruct(Container): // A: uint16 // B: List[uint16, 128] @@ -87,13 +106,157 @@ const BitsStruct = new ContainerType({ E: new BitVectorType(8), }); +// class ProgressiveBitsStruct(Container): +// A: Bitvector[256] +// B: Bitlist[256] +// C: ProgressiveBitlist +// D: Bitvector[257] +// E: Bitlist[257] +// F: ProgressiveBitlist +// G: Bitvector[1280] +// H: Bitlist[1280] +// I: ProgressiveBitlist +// J: Bitvector[1281] +// K: Bitlist[1281] +// L: ProgressiveBitlist +const ProgressiveBitsStruct = new ContainerType({ + A: new BitVectorType(256), + B: new BitListType(256), + C: new ProgressiveBitListType(), + D: new BitVectorType(257), + E: new BitListType(257), + F: new ProgressiveBitListType(), + G: new BitVectorType(1280), + H: new BitListType(1280), + I: new ProgressiveBitListType(), + J: new BitVectorType(1281), + K: new BitListType(1281), + L: new ProgressiveBitListType(), +}); + +// class ProgressiveSingleFieldContainerTestStruct(ProgressiveContainer(active_fields=[1])): +// A: byte +const ProgressiveSingleFieldContainerTestStruct = new ProgressiveContainerType({A: byte}, [true]); + +// class ProgressiveSingleListContainerTestStruct(ProgressiveContainer(active_fields=[0, 0, 0, 0, 1])): +// C: ProgressiveBitlist +const ProgressiveSingleListContainerTestStruct = new ProgressiveContainerType({C: new ProgressiveBitListType()}, [ + false, + false, + false, + false, + true, +]); + +// class ProgressiveVarTestStruct(ProgressiveContainer(active_fields=[1, 0, 1, 0, 1])): +// A: byte +// B: List[uint16, 123] +// C: ProgressiveBitlist +const ProgressiveVarTestStruct = new ProgressiveContainerType( + { + A: byte, + B: new ListBasicType(uint16, 123), + C: new ProgressiveBitListType(), + }, + [true, false, true, false, true] +); + +// class ProgressiveComplexTestStruct( +// ProgressiveContainer( +// active_fields=[1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1] +// ) +// ): +// A: byte +// B: List[uint16, 123] +// C: ProgressiveBitlist +// D: ProgressiveList[uint64] +// E: ProgressiveList[SmallTestStruct] +// F: ProgressiveList[ProgressiveList[VarTestStruct]] +// G: List[ProgressiveSingleFieldContainerTestStruct, 10] +// H: ProgressiveList[ProgressiveVarTestStruct] +const ProgressiveComplexTestStruct = new ProgressiveContainerType( + { + A: byte, + B: new ListBasicType(uint16, 123), + C: new ProgressiveBitListType(), + D: new ProgressiveListBasicType(uint64), + E: new ProgressiveListCompositeType(SmallTestStruct), + F: new ProgressiveListCompositeType(ProgressiveListVarTestStruct), + G: new ListCompositeType(ProgressiveSingleFieldContainerTestStruct, 10), + H: new ProgressiveListCompositeType(ProgressiveVarTestStruct), + }, + [ + true, + false, + true, + false, + true, + false, + false, + false, + true, + false, + false, + false, + true, + true, + false, + false, + false, + false, + false, + false, + true, + true, + ] +); + const containerTypes = { SingleFieldTestStruct, SmallTestStruct, FixedTestStruct, VarTestStruct, ComplexTestStruct, + ProgressiveTestStruct, BitsStruct, + ProgressiveBitsStruct, +}; + +const progressiveContainerTypes = { + ProgressiveSingleFieldContainerTestStruct, + ProgressiveSingleListContainerTestStruct, + ProgressiveVarTestStruct, + ProgressiveComplexTestStruct, +}; + +// CompatibleUnion({1: ProgressiveSingleFieldContainerTestStruct}) +const CompatibleUnionA = new CompatibleUnionType({ + 1: ProgressiveSingleFieldContainerTestStruct, +}); + +// CompatibleUnion({2: ProgressiveSingleListContainerTestStruct, 3: ProgressiveVarTestStruct}) +const CompatibleUnionBC = new CompatibleUnionType({ + 2: ProgressiveSingleListContainerTestStruct, + 3: ProgressiveVarTestStruct, +}); + +// CompatibleUnion({ +// 1: ProgressiveSingleFieldContainerTestStruct, +// 2: ProgressiveSingleListContainerTestStruct, +// 3: ProgressiveVarTestStruct, +// 4: ProgressiveSingleFieldContainerTestStruct, +// }) +const CompatibleUnionABCA = new CompatibleUnionType({ + 1: ProgressiveSingleFieldContainerTestStruct, + 2: ProgressiveSingleListContainerTestStruct, + 3: ProgressiveVarTestStruct, + 4: ProgressiveSingleFieldContainerTestStruct, +}); + +const compatibleUnionTypes = { + CompatibleUnionA, + CompatibleUnionBC, + CompatibleUnionABCA, }; const vecElementTypes = { @@ -137,6 +300,21 @@ export function getTestType(testType: string, testCase: string): Type { return new BitVectorType(parseSecondNum(testCase, "length")); } + // `proglist_{element type}_{mode}_{length}` + // {element type}: bool, uint8, uint16, uint32, uint64, uint128, uint256 + case "basic_progressive_list": { + const match = testCase.match(/proglist_([^\W_]+)_/); + const [, elementTypeStr] = match || []; + const elementType = vecElementTypes[elementTypeStr as keyof typeof vecElementTypes]; + if (elementType === undefined) + throw Error(`No progressive list elementType for ${elementTypeStr}: '${testCase}'`); + return new ProgressiveListBasicType(elementType); + } + + // A progressive bitlist has no limit variations. + case "progressive_bitlist": + return new ProgressiveBitListType(); + // A boolean has no type variations. Instead, file names just plainly describe the contents for debugging. case "boolean": return bool; @@ -151,6 +329,26 @@ export function getTestType(testType: string, testCase: string): Type { return containerType; } + // {container name} + // {container name}: Any of the progressive container names listed above. + case "progressive_containers": { + const match = testCase.match(/([^\W_]+)/); + const containerName = (match || [])[1]; + const containerType = progressiveContainerTypes[containerName as keyof typeof progressiveContainerTypes]; + if (containerType === undefined) throw Error(`No progressiveContainerType for ${containerName}`); + return containerType; + } + + // {compatible union name} + // {compatible union name}: Any of the compatible union names listed above. + case "compatible_unions": { + const match = testCase.match(/([^\W_]+)/); + const compatibleUnionName = (match || [])[1]; + const compatibleUnionType = compatibleUnionTypes[compatibleUnionName as keyof typeof compatibleUnionTypes]; + if (compatibleUnionType === undefined) throw Error(`No compatibleUnionType for ${compatibleUnionName}`); + return compatibleUnionType; + } + // `uint_{size}` // {size}: the uint size: 8, 16, 32, 64, 128 or 256. case "uints": { diff --git a/packages/ssz/test/spec/runValidTest.ts b/packages/ssz/test/spec/runValidTest.ts index 648c9e77..4c12eb21 100644 --- a/packages/ssz/test/spec/runValidTest.ts +++ b/packages/ssz/test/spec/runValidTest.ts @@ -61,14 +61,18 @@ export function runValidSszTest(type: Type, testData: ValidTestCaseData expect(isEqual).to.equal(true, "Value is not equal to itself"); } - { - // value - not equals + // value - not equals + try { const defaultValue = type.defaultValue(); const defaultSerialized = wrapErr(() => type.serialize(defaultValue), "serialize default"); if (toHexString(defaultSerialized) !== testDataSerializedHex) { const isEqual = wrapErr(() => type.equals(testDataValue, defaultValue), "type.equals()"); expect(isEqual).to.equal(false, "Value is equal to default value"); } + } catch (e) { + if (!String(e).includes("does not define a default value")) { + throw e; + } } { diff --git a/packages/ssz/test/specTestVersioning.ts b/packages/ssz/test/specTestVersioning.ts index 9ba02883..1f4d775c 100644 --- a/packages/ssz/test/specTestVersioning.ts +++ b/packages/ssz/test/specTestVersioning.ts @@ -10,8 +10,8 @@ import {DownloadTestsOptions} from "@lodestar/spec-test-util/downloadTests"; // The contents of this file MUST include the URL, version and target path, and nothing else. export const ethereumConsensusSpecsTests: DownloadTestsOptions = { - specVersion: "v1.4.0-beta.1", + specVersion: "v1.7.0-alpha.5", outputDir: path.join(path.dirname(url.fileURLToPath(import.meta.url)), "../spec-tests"), - specTestsRepoUrl: "https://github.com/ethereum/consensus-spec-tests", + specTestsRepoUrl: "https://github.com/ethereum/consensus-specs", testsToDownload: ["general", "mainnet", "minimal"], }; diff --git a/packages/ssz/test/unit/byType/compatibleUnion/invalid.test.ts b/packages/ssz/test/unit/byType/compatibleUnion/invalid.test.ts new file mode 100644 index 00000000..f9890f88 --- /dev/null +++ b/packages/ssz/test/unit/byType/compatibleUnion/invalid.test.ts @@ -0,0 +1,27 @@ +import {describe, expect, it} from "vitest"; +import {CompatibleUnionType, ProgressiveContainerType, UintNumberType} from "../../../../src/index.ts"; + +const byte = new UintNumberType(1); +const FieldA = new ProgressiveContainerType({A: byte}, [true]); + +describe("Invalid CompatibleUnionType", () => { + it("rejects empty options", () => { + expect(() => new CompatibleUnionType({})).toThrow("at least one"); + }); + + it("rejects reserved selectors", () => { + expect(() => new CompatibleUnionType({0: FieldA})).toThrow("1..127"); + expect(() => new CompatibleUnionType({128: FieldA})).toThrow("1..127"); + }); + + it("rejects incompatible options", () => { + const FieldB = new ProgressiveContainerType({B: byte}, [true]); + expect(() => new CompatibleUnionType({1: FieldA, 2: FieldB})).toThrow("not compatible"); + }); + + it("rejects missing and unknown selectors", () => { + const type = new CompatibleUnionType({1: FieldA}); + expect(() => type.deserialize(Uint8Array.from([]))).toThrow("selector byte"); + expect(() => type.deserialize(Uint8Array.from([2, 0]))).toThrow("Invalid CompatibleUnion selector 2"); + }); +}); diff --git a/packages/ssz/test/unit/byType/compatibleUnion/valid.test.ts b/packages/ssz/test/unit/byType/compatibleUnion/valid.test.ts new file mode 100644 index 00000000..38e985ad --- /dev/null +++ b/packages/ssz/test/unit/byType/compatibleUnion/valid.test.ts @@ -0,0 +1,100 @@ +import {describe, expect, it} from "vitest"; +import { + BitArray, + ByteListType, + ByteVectorType, + CompatibleUnionType, + ContainerType, + ListBasicType, + ProgressiveBitListType, + ProgressiveContainerType, + UintNumberType, + VectorBasicType, + hash64, + toHexString, +} from "../../../../src/index.ts"; + +const byte = new UintNumberType(1); + +describe("CompatibleUnionType", () => { + const Single = new ProgressiveContainerType({A: byte}, [true]); + const WithGap = new ProgressiveContainerType({A: byte, C: new ProgressiveBitListType()}, [true, false, true]); + const type = new CompatibleUnionType({ + 1: Single, + 3: WithGap, + }); + + it("serializes selector plus selected data", () => { + const value = {selector: 1, data: {A: 7}}; + expect(toHexString(type.serialize(value))).to.equal("0x0107"); + expect(type.deserialize(type.serialize(value))).to.deep.equal(value); + }); + + it("uses selector-object JSON with data", () => { + const value = type.fromJson({selector: "1", data: {A: "7"}}); + expect(value).to.deep.equal({selector: 1, data: {A: 7}}); + expect(type.toJson(value)).to.deep.equal({selector: "1", data: {A: "7"}}); + }); + + it("mixes the selector into the selected value root", () => { + const value = {selector: 1, data: {A: 7}}; + const selectorChunk = new Uint8Array(32); + selectorChunk[0] = 1; + expect(toHexString(type.hashTreeRoot(value))).to.equal( + toHexString(hash64(Single.hashTreeRoot(value.data), selectorChunk)) + ); + }); + + it("has no default value", () => { + expect(() => type.defaultValue()).toThrow("does not define a default value"); + }); + + it("creates and restores proofs for selector and selected data paths", () => { + const value = {selector: 1, data: {A: 7}}; + const root = type.hashTreeRoot(value); + const view = type.toView(value); + + expect(view.selector).to.equal(value.selector); + expect(view.data).to.deep.equal(value.data); + expect(type.toValueFromView(view)).to.deep.equal(value); + + for (const path of [[["selector"]], [["data"]], [["data", "A"]]]) { + const proof = view.createProof(path); + const restored = type.createFromProof(proof, root); + expect(toHexString(restored.hashTreeRoot())).to.equal(toHexString(root)); + } + }); + + it("creates and restores selected data proofs when nested in a container", () => { + const wrapperType = new ContainerType({shape: type}); + const value = {shape: {selector: 1, data: {A: 7}}}; + const root = wrapperType.hashTreeRoot(value); + const proof = wrapperType.toView(value).createProof([["shape", "data", "A"]]); + const restored = wrapperType.createFromProof(proof, root); + + expect(toHexString(restored.hashTreeRoot())).to.equal(toHexString(root)); + }); + + it("creates nested selected data proofs using the active selector type", () => { + const wrapperType = new ContainerType({shape: type}); + const value = { + shape: { + selector: 3, + data: { + A: 7, + C: BitArray.fromBoolArray([true, false, true]), + }, + }, + }; + const root = wrapperType.hashTreeRoot(value); + const proof = wrapperType.toView(value).createProof([["shape", "data", "C"]]); + const restored = wrapperType.createFromProof(proof, root); + + expect(toHexString(restored.hashTreeRoot())).to.equal(toHexString(root)); + }); + + it("allows byte list and vector aliases to match uint8 arrays", () => { + expect(() => new CompatibleUnionType({1: new ByteListType(4), 2: new ListBasicType(byte, 4)})).not.toThrow(); + expect(() => new CompatibleUnionType({1: new ByteVectorType(4), 2: new VectorBasicType(byte, 4)})).not.toThrow(); + }); +}); diff --git a/packages/ssz/test/unit/byType/progressive/valid.test.ts b/packages/ssz/test/unit/byType/progressive/valid.test.ts new file mode 100644 index 00000000..1a078831 --- /dev/null +++ b/packages/ssz/test/unit/byType/progressive/valid.test.ts @@ -0,0 +1,213 @@ +import {describe, expect, it} from "vitest"; +import { + BitArray, + ContainerType, + ProgressiveBitListType, + ProgressiveContainerType, + ProgressiveListBasicType, + ProgressiveListCompositeType, + UintNumberType, + hash64, + toHexString, +} from "../../../../src/index.ts"; +import {Type} from "../../../../src/type/abstract.ts"; +import {merkleize, mixInLength} from "../../../../src/util/merkleize.ts"; + +const uint8 = new UintNumberType(1); +const uint16 = new UintNumberType(2); + +describe("ProgressiveListBasicType", () => { + const type = new ProgressiveListBasicType(uint8); + + it("round-trips serialization and computes progressive roots", () => { + const value = [1, 2, 3, 4, 5]; + const serialized = type.serialize(value); + expect(toHexString(serialized)).to.equal("0x0102030405"); + expect(type.deserialize(serialized)).to.deep.equal(value); + expect(toHexString(type.hashTreeRoot(value))).to.equal(toHexString(progressiveListBasicRoot(uint8, value))); + }); + + it("supports empty lists", () => { + expect(toHexString(type.serialize([]))).to.equal("0x"); + expect(toHexString(type.hashTreeRoot([]))).to.equal(toHexString(mixInLength(new Uint8Array(32), 0))); + }); + + it("supports TreeViewDU mutation beyond the first progressive subtree", () => { + const value = Array.from({length: 33}, (_, i) => i); + const view = type.toViewDU(value); + expect(view.get(32)).to.equal(32); + view.set(32, 99); + view.push(100); + + const expected = [...value]; + expected[32] = 99; + expected.push(100); + expect(toHexString(view.hashTreeRoot())).to.equal(toHexString(type.hashTreeRoot(expected))); + }); + + it("creates and restores proofs for variable-depth chunks", () => { + const value = Array.from({length: 33}, (_, i) => i); + const view = type.toView(value); + const proof = view.createProof([[32]]); + const restored = type.createFromProof(proof, type.hashTreeRoot(value)); + expect(toHexString(restored.hashTreeRoot())).to.equal(toHexString(type.hashTreeRoot(value))); + }); +}); + +describe("ProgressiveListCompositeType", () => { + const elementType = new ContainerType({a: uint8, b: uint16}); + const type = new ProgressiveListCompositeType(elementType); + + it("round-trips fixed composite elements and computes progressive roots", () => { + const value = [ + {a: 1, b: 2}, + {a: 3, b: 4}, + {a: 5, b: 6}, + {a: 7, b: 8}, + {a: 9, b: 10}, + ]; + const serialized = type.serialize(value); + expect(type.deserialize(serialized)).to.deep.equal(value); + expect(toHexString(type.hashTreeRoot(value))).to.equal( + toHexString(progressiveListCompositeRoot(elementType, value)) + ); + }); + + it("creates nested proofs for variable-size composite elements", () => { + const variableElementType = new ContainerType({a: uint8, bits: new ProgressiveBitListType()}); + const variableListType = new ProgressiveListCompositeType(variableElementType); + const wrapperType = new ContainerType({list: variableListType}); + const value = { + list: [ + {a: 1, bits: BitArray.fromBoolArray([true])}, + {a: 2, bits: BitArray.fromBoolArray([false, true])}, + ], + }; + const root = wrapperType.hashTreeRoot(value); + const proof = wrapperType.toView(value).createProof([["list"]]); + const restored = wrapperType.createFromProof(proof, root); + + expect(toHexString(restored.hashTreeRoot())).to.equal(toHexString(root)); + }); +}); + +describe("ProgressiveBitListType", () => { + const type = new ProgressiveBitListType(); + + it("round-trips bitlist serialization and computes progressive roots", () => { + const value = BitArray.fromBoolArray([true, false, true, true, false, false, true, false, true]); + const serialized = type.serialize(value); + expect(toHexString(serialized)).to.equal("0x4d03"); + const deserialized = type.deserialize(serialized); + expect(deserialized.toBoolArray()).to.deep.equal(value.toBoolArray()); + expect(toHexString(type.hashTreeRoot(value))).to.equal(toHexString(progressiveBitlistRoot(value))); + }); +}); + +describe("ProgressiveContainerType", () => { + const type = new ProgressiveContainerType( + { + side: uint16, + color: uint8, + radius: uint16, + }, + [true, false, true, true] + ); + + it("serializes like a container over active fields and computes progressive roots", () => { + const value = {side: 0x42, color: 1, radius: 0x42}; + const serialized = type.serialize(value); + expect(toHexString(serialized)).to.equal("0x4200014200"); + expect(type.deserialize(serialized)).to.deep.equal(value); + expect(toHexString(type.hashTreeRoot(value))).to.equal( + toHexString( + progressiveContainerRoot(type, [uint16.hashTreeRoot(0x42), uint8.hashTreeRoot(1), uint16.hashTreeRoot(0x42)]) + ) + ); + }); + + it("supports TreeViewDU mutation through inactive-field gaps", () => { + const view = type.toViewDU({side: 0x42, color: 1, radius: 0x42}); + view.color = 7; + view.radius = 0x99; + + expect(toHexString(view.hashTreeRoot())).to.equal( + toHexString(type.hashTreeRoot({side: 0x42, color: 7, radius: 0x99})) + ); + }); + + it("reuses generated view classes", () => { + const value = {side: 0x42, color: 1, radius: 0x42}; + expect(type.toView(value).constructor).to.equal(type.toView(value).constructor); + expect(type.toViewDU(value).constructor).to.equal(type.toViewDU(value).constructor); + }); + + it("creates and restores proofs through inactive-field gaps", () => { + const value = {side: 0x42, color: 1, radius: 0x42}; + const view = type.toView(value); + const proof = view.createProof([["color"]]); + const restored = type.createFromProof(proof, type.hashTreeRoot(value)); + expect(toHexString(restored.hashTreeRoot())).to.equal(toHexString(type.hashTreeRoot(value))); + }); +}); + +function progressiveListBasicRoot(elementType: UintNumberType, value: T[]): Uint8Array { + const serialized = new Uint8Array(value.length * elementType.byteLength); + const dataView = new DataView(serialized.buffer, serialized.byteOffset, serialized.byteLength); + for (let i = 0; i < value.length; i++) { + elementType.value_serializeToBytes( + {uint8Array: serialized, dataView}, + i * elementType.byteLength, + value[i] as number + ); + } + return mixInLength(progressiveRoot(chunkify(serialized)), value.length); +} + +function progressiveListCompositeRoot(elementType: Type, value: T[]): Uint8Array { + return mixInLength(progressiveRoot(value.map((element) => elementType.hashTreeRoot(element))), value.length); +} + +function progressiveBitlistRoot(value: BitArray): Uint8Array { + return mixInLength(progressiveRoot(chunkify(value.uint8Array)), value.bitLen); +} + +function progressiveContainerRoot(type: {activeFields: BitArray}, fieldRoots: Uint8Array[]): Uint8Array { + const chunks = [fieldRoots[0], new Uint8Array(32), fieldRoots[1], fieldRoots[2]]; + const activeFieldsChunk = new Uint8Array(32); + activeFieldsChunk.set(type.activeFields.uint8Array); + return hash64(progressiveRoot(chunks), activeFieldsChunk); +} + +function progressiveRoot(chunks: Uint8Array[]): Uint8Array { + if (chunks.length === 0) { + return new Uint8Array(32); + } + + const subtreeRoots: Uint8Array[] = []; + let offset = 0; + let subtreeLength = 1; + while (offset < chunks.length) { + const subtreeChunks = chunks.slice(offset, offset + subtreeLength).map((chunk) => new Uint8Array(chunk)); + subtreeRoots.push(merkleize(subtreeChunks, subtreeLength)); + offset += subtreeLength; + subtreeLength *= 4; + } + + const root = new Uint8Array(32); + for (let i = subtreeRoots.length - 1; i >= 0; i--) { + root.set(hash64(subtreeRoots[i], root)); + } + return root; +} + +function chunkify(bytes: Uint8Array): Uint8Array[] { + const chunkCount = Math.ceil(bytes.length / 32); + const chunks: Uint8Array[] = []; + for (let i = 0; i < chunkCount; i++) { + const chunk = new Uint8Array(32); + chunk.set(bytes.subarray(i * 32, (i + 1) * 32)); + chunks.push(chunk); + } + return chunks; +} From e8e4f528ae8617ca16773dfde042d76de2478065 Mon Sep 17 00:00:00 2001 From: Cayman Date: Mon, 11 May 2026 14:16:26 -0400 Subject: [PATCH 2/5] chore: tweak tests --- .../persistent-merkle-tree/test/unit/proof/index.test.ts | 6 ++---- .../persistent-merkle-tree/test/unit/tree/getNodes.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/persistent-merkle-tree/test/unit/proof/index.test.ts b/packages/persistent-merkle-tree/test/unit/proof/index.test.ts index 119f6efe..4d82131c 100644 --- a/packages/persistent-merkle-tree/test/unit/proof/index.test.ts +++ b/packages/persistent-merkle-tree/test/unit/proof/index.test.ts @@ -1,4 +1,4 @@ -import {describe, expect, it, vi} from "vitest"; +import {describe, expect, it} from "vitest"; import { ProofType, computeDescriptor, @@ -28,8 +28,6 @@ describe("proof equivalence", () => { } }); it("should compute the same root from different proof types - multiple leaves", () => { - vi.setConfig({testTimeout: 10_000}); - const depth = 6; const maxIndex = 2 ** depth; const node = createTree(depth); @@ -60,7 +58,7 @@ describe("proof equivalence", () => { } } } - }); + }, 30_000); }); describe("proof serialize/deserialize", () => { diff --git a/packages/persistent-merkle-tree/test/unit/tree/getNodes.test.ts b/packages/persistent-merkle-tree/test/unit/tree/getNodes.test.ts index 4f443b67..9f446200 100644 --- a/packages/persistent-merkle-tree/test/unit/tree/getNodes.test.ts +++ b/packages/persistent-merkle-tree/test/unit/tree/getNodes.test.ts @@ -24,11 +24,11 @@ describe("tree / getNodes", () => { it("getNodesAtDepth", () => { const nodes = getNodesAtDepth(tree.rootNode, depth, 0, length); assertValidNodes(nodes, expectedNodes); - }); + }, 60_000); function assertValidNodes(nodes: Node[], expectedNodes: Node[]): void { for (let i = 0; i < expectedNodes.length; i++) { - expect(nodes[i]).toEqualWithMessage(expectedNodes[i], `Wrong node index ${i}`); + expect(nodes[i]).toBeWithMessage(expectedNodes[i], `Wrong node index ${i}`); } } }); From d2ca294de6e44c3471510f379fa4e4bc61bef1c3 Mon Sep 17 00:00:00 2001 From: Cayman Date: Fri, 15 May 2026 09:53:53 -0400 Subject: [PATCH 3/5] feat: add ProgressiveByteListType --- packages/ssz/src/index.ts | 1 + packages/ssz/src/type/compatibleUnion.ts | 14 ++ packages/ssz/src/type/progressiveByteList.ts | 134 ++++++++++++++++++ .../unit/byType/compatibleUnion/valid.test.ts | 5 + .../unit/byType/progressive/valid.test.ts | 37 +++++ 5 files changed, 191 insertions(+) create mode 100644 packages/ssz/src/type/progressiveByteList.ts diff --git a/packages/ssz/src/index.ts b/packages/ssz/src/index.ts index 90a6279d..68e0f505 100644 --- a/packages/ssz/src/index.ts +++ b/packages/ssz/src/index.ts @@ -10,6 +10,7 @@ export {ContainerNodeStructType} from "./type/containerNodeStruct.ts"; export {ListBasicType} from "./type/listBasic.ts"; export {ListCompositeType} from "./type/listComposite.ts"; export {ProgressiveBitListType} from "./type/progressiveBitList.ts"; +export {ProgressiveByteListType} from "./type/progressiveByteList.ts"; export {ProgressiveContainerType} from "./type/progressiveContainer.ts"; export {ProgressiveListBasicType, ProgressiveListCompositeType} from "./type/progressiveList.ts"; export {PartialListCompositeType} from "./type/partialListComposite.ts"; diff --git a/packages/ssz/src/type/compatibleUnion.ts b/packages/ssz/src/type/compatibleUnion.ts index c5f28aaf..963e54fb 100644 --- a/packages/ssz/src/type/compatibleUnion.ts +++ b/packages/ssz/src/type/compatibleUnion.ts @@ -30,6 +30,7 @@ import {ContainerType} from "./container.ts"; import {ListBasicType} from "./listBasic.ts"; import {ListCompositeType} from "./listComposite.ts"; import {ProgressiveBitListType} from "./progressiveBitList.ts"; +import {ProgressiveByteListType} from "./progressiveByteList.ts"; import {ProgressiveContainerType} from "./progressiveContainer.ts"; import {ProgressiveListBasicType, ProgressiveListCompositeType} from "./progressiveList.ts"; import {UintBigintType, UintNumberType} from "./uint.ts"; @@ -480,6 +481,10 @@ export function areTypesCompatible(a: Type, b: Type): boolean return getByteVectorCompatibleLength(a) === getByteVectorCompatibleLength(b); } + if (isProgressiveByteListCompatibleType(a) && isProgressiveByteListCompatibleType(b)) { + return true; + } + if (isLimitedListType(a) && isLimitedListType(b)) { return a.limit === b.limit && areTypesCompatible(a.elementType, b.elementType); } @@ -556,6 +561,15 @@ function getByteVectorCompatibleLength(type: ByteVectorType | VectorBasicType +): type is ProgressiveByteListType | ProgressiveListBasicType> { + return ( + type instanceof ProgressiveByteListType || + (type instanceof ProgressiveListBasicType && isByteBasicType(type.elementType)) + ); +} + function isLimitedListType( type: Type ): type is ListBasicType> | ListCompositeType> { diff --git a/packages/ssz/src/type/progressiveByteList.ts b/packages/ssz/src/type/progressiveByteList.ts new file mode 100644 index 00000000..fabb2b26 --- /dev/null +++ b/packages/ssz/src/type/progressiveByteList.ts @@ -0,0 +1,134 @@ +import {allocUnsafe} from "@chainsafe/as-sha256"; +import { + Gindex, + Node, + Proof, + Tree, + concatGindices, + merkleizeBlocksBytes, + packedNodeRootsToBytes, + packedRootsBytesToLeafNodes, +} from "@chainsafe/persistent-merkle-tree"; +import {byteArrayEquals} from "../util/byteArray.ts"; +import {namedClass} from "../util/named.ts"; +import {Require} from "../util/types.ts"; +import {addLengthNode, getChunksNodeFromRootNode, getLengthFromRootNode} from "./arrayBasic.ts"; +import {ByteArray, ByteArrayType} from "./byteArray.ts"; +import {ByteViews} from "./composite.ts"; +import { + PROGRESSIVE_LIST_MAX_SIZE, + getNodesAtProgressiveDepth, + merkleizeProgressiveBytes, + progressiveChunkGindex, + progressiveSubtreeFillToContents, +} from "./progressive.ts"; + +export interface ProgressiveByteListOptions { + typeName?: string; +} + +const CHUNKS_GINDEX = BigInt(2); +const LENGTH_GINDEX = BigInt(3); + +/** + * ProgressiveByteList: Immutable alias of ProgressiveList[byte] + * - Value: `Uint8Array` + * - View: `Uint8Array` + * - ViewDU: `Uint8Array` + * + * ProgressiveByteList is an immutable value represented by a Uint8Array for memory efficiency and performance. + * Note: Consumers of this type MUST never mutate the `Uint8Array` representation of a ProgressiveByteList. + */ +export class ProgressiveByteListType extends ByteArrayType { + readonly typeName: string; + readonly depth = 1; + readonly chunkDepth = 0; + readonly fixedSize = null; + readonly minSize = 0; + readonly maxSize = PROGRESSIVE_LIST_MAX_SIZE; + readonly maxChunkCount = Number.MAX_SAFE_INTEGER; + readonly isList = true; + readonly mixInLengthBlockBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength + ); + + constructor(opts?: ProgressiveByteListOptions) { + super(); + this.typeName = opts?.typeName ?? "ProgressiveByteList"; + } + + static named(opts: Require): ProgressiveByteListType { + return new (namedClass(ProgressiveByteListType, opts.typeName))(opts); + } + + createFromProof(proof: Proof, root?: Uint8Array): ByteArray { + const rootNode = Tree.createFromProof(proof).rootNode; + if (root !== undefined && !byteArrayEquals(rootNode.root, root)) { + throw new Error("Proof does not match trusted root"); + } + return this.getView(new Tree(rootNode)); + } + + value_serializedSize(value: Uint8Array): number { + return value.length; + } + + tree_serializedSize(node: Node): number { + return getLengthFromRootNode(node); + } + + tree_serializeToBytes(output: ByteViews, offset: number, node: Node): number { + const chunksNode = getChunksNodeFromRootNode(node); + const byteLen = getLengthFromRootNode(node); + const chunkLen = Math.ceil(byteLen / 32); + const nodes = getNodesAtProgressiveDepth(chunksNode, chunkLen); + packedNodeRootsToBytes(output.dataView, offset, byteLen, nodes); + return offset + byteLen; + } + + tree_deserializeFromBytes(data: ByteViews, start: number, end: number): Node { + this.assertValidSize(end - start); + const nodes = packedRootsBytesToLeafNodes(data.dataView, start, end); + return addLengthNode(progressiveSubtreeFillToContents(nodes), end - start); + } + + tree_getByteLen(node?: Node): number { + if (!node) throw new Error("ProgressiveByteListType requires a node to get leaves"); + return getLengthFromRootNode(node); + } + + hashTreeRoot(value: ByteArray): Uint8Array { + const root = allocUnsafe(32); + this.hashTreeRootInto(value, root, 0); + return root; + } + + hashTreeRootInto(value: Uint8Array, output: Uint8Array, offset: number): void { + const blockBytes = this.getBlocksBytes(value); + const chunkCount = Math.ceil(value.length / 32); + + merkleizeProgressiveBytes(blockBytes, chunkCount, this.mixInLengthBlockBytes, 0); + this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, 2, output, offset); + } + + tree_getLeafGindices(rootGindex: Gindex, rootNode?: Node): Gindex[] { + const byteLen = this.tree_getByteLen(rootNode); + const chunkCount = Math.ceil(byteLen / 32); + const gindices = new Array(chunkCount); + for (let i = 0; i < chunkCount; i++) { + gindices[i] = concatGindices([rootGindex, CHUNKS_GINDEX, progressiveChunkGindex(i)]); + } + gindices.push(concatGindices([rootGindex, LENGTH_GINDEX])); + return gindices; + } + + protected assertValidSize(size: number): void { + if (size > PROGRESSIVE_LIST_MAX_SIZE) { + throw Error(`ProgressiveByteList invalid size ${size} max ${PROGRESSIVE_LIST_MAX_SIZE}`); + } + } +} diff --git a/packages/ssz/test/unit/byType/compatibleUnion/valid.test.ts b/packages/ssz/test/unit/byType/compatibleUnion/valid.test.ts index 38e985ad..f845cf73 100644 --- a/packages/ssz/test/unit/byType/compatibleUnion/valid.test.ts +++ b/packages/ssz/test/unit/byType/compatibleUnion/valid.test.ts @@ -7,7 +7,9 @@ import { ContainerType, ListBasicType, ProgressiveBitListType, + ProgressiveByteListType, ProgressiveContainerType, + ProgressiveListBasicType, UintNumberType, VectorBasicType, hash64, @@ -96,5 +98,8 @@ describe("CompatibleUnionType", () => { it("allows byte list and vector aliases to match uint8 arrays", () => { expect(() => new CompatibleUnionType({1: new ByteListType(4), 2: new ListBasicType(byte, 4)})).not.toThrow(); expect(() => new CompatibleUnionType({1: new ByteVectorType(4), 2: new VectorBasicType(byte, 4)})).not.toThrow(); + expect( + () => new CompatibleUnionType({1: new ProgressiveByteListType(), 2: new ProgressiveListBasicType(byte)}) + ).not.toThrow(); }); }); diff --git a/packages/ssz/test/unit/byType/progressive/valid.test.ts b/packages/ssz/test/unit/byType/progressive/valid.test.ts index 1a078831..c14cba06 100644 --- a/packages/ssz/test/unit/byType/progressive/valid.test.ts +++ b/packages/ssz/test/unit/byType/progressive/valid.test.ts @@ -3,6 +3,7 @@ import { BitArray, ContainerType, ProgressiveBitListType, + ProgressiveByteListType, ProgressiveContainerType, ProgressiveListBasicType, ProgressiveListCompositeType, @@ -104,6 +105,38 @@ describe("ProgressiveBitListType", () => { }); }); +describe("ProgressiveByteListType", () => { + const type = new ProgressiveByteListType(); + + it("round-trips byte list serialization and computes progressive roots", () => { + const value = Uint8Array.from({length: 32 * 5 + 7}, (_, i) => i % 256); + const equivalentList = new ProgressiveListBasicType(uint8); + const serialized = type.serialize(value); + expect(toHexString(serialized)).to.equal(toHexString(value)); + expect(toHexString(serialized)).to.equal(toHexString(equivalentList.serialize(Array.from(value)))); + expect(type.deserialize(serialized)).to.deep.equal(value); + expect(type.toJson(value)).to.equal(toHexString(value)); + expect(type.fromJson(toHexString(value))).to.deep.equal(value); + expect(toHexString(type.hashTreeRoot(value))).to.equal(toHexString(progressiveByteListRoot(value))); + expect(toHexString(type.hashTreeRoot(value))).to.equal(toHexString(equivalentList.hashTreeRoot(Array.from(value)))); + }); + + it("supports empty byte lists", () => { + expect(toHexString(type.serialize(new Uint8Array(0)))).to.equal("0x"); + expect(toHexString(type.hashTreeRoot(new Uint8Array(0)))).to.equal(toHexString(mixInLength(new Uint8Array(32), 0))); + }); + + it("creates and restores proofs for progressive byte chunks", () => { + const wrapperType = new ContainerType({bytes: type}); + const value = {bytes: Uint8Array.from({length: 32 * 5 + 1}, (_, i) => i % 256)}; + const root = wrapperType.hashTreeRoot(value); + const proof = wrapperType.toView(value).createProof([["bytes"]]); + const restored = wrapperType.createFromProof(proof, root); + + expect(toHexString(restored.hashTreeRoot())).to.equal(toHexString(root)); + }); +}); + describe("ProgressiveContainerType", () => { const type = new ProgressiveContainerType( { @@ -172,6 +205,10 @@ function progressiveBitlistRoot(value: BitArray): Uint8Array { return mixInLength(progressiveRoot(chunkify(value.uint8Array)), value.bitLen); } +function progressiveByteListRoot(value: Uint8Array): Uint8Array { + return mixInLength(progressiveRoot(chunkify(value)), value.length); +} + function progressiveContainerRoot(type: {activeFields: BitArray}, fieldRoots: Uint8Array[]): Uint8Array { const chunks = [fieldRoots[0], new Uint8Array(32), fieldRoots[1], fieldRoots[2]]; const activeFieldsChunk = new Uint8Array(32); From 8a02728a80e84bfff86c95aa4f5dbba9430b30e9 Mon Sep 17 00:00:00 2001 From: Cayman Date: Fri, 15 May 2026 09:59:56 -0400 Subject: [PATCH 4/5] fix: StableContainer compatible union proofs --- packages/ssz/src/type/stableContainer.ts | 40 ++++++++++++------- .../unit/byType/compatibleUnion/valid.test.ts | 28 +++++++++++++ 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/packages/ssz/src/type/stableContainer.ts b/packages/ssz/src/type/stableContainer.ts index 5c8b66cb..28500719 100644 --- a/packages/ssz/src/type/stableContainer.ts +++ b/packages/ssz/src/type/stableContainer.ts @@ -418,32 +418,42 @@ export class StableContainerType>> e const activeFields = this.tree_getActiveFields(node); for (const jsonPath of jsonPaths) { - const prop = jsonPath[0]; + const [prop, ...remainingPath] = jsonPath; if (prop == null) { continue; } - const fieldIndex = this.fieldsEntries.findIndex((entry) => entry.fieldName === prop); - if (fieldIndex === -1) throw Error(`Unknown container property ${prop}`); + if (typeof prop !== "string") { + throw Error(`Unknown container property ${String(prop)}`); + } + const fieldName = this.fields[prop] ? prop : this.jsonKeyToFieldName[prop]; + const fieldIndex = this.fieldsEntries.findIndex((entry) => entry.fieldName === fieldName); + if (fieldIndex === -1) throw Error(`Unknown container property ${String(prop)}`); const entry = this.fieldsEntries[fieldIndex]; if (entry.optional && !activeFields.get(fieldIndex)) { // field is inactive and doesn't count as a leaf continue; } - // same to Composite - const {type, gindex} = this.getPathInfo(jsonPath); - if (!isCompositeType(type)) { - gindexes.push(gindex); - } else { - // if the path subtype is composite, include the gindices of all the leaves - const leafGindexes = type.tree_getLeafGindices( - gindex, - type.fixedSize === null ? getNode(node, gindex) : undefined - ); - for (const gindex of leafGindexes) { - gindexes.push(gindex); + const {fieldType, gindex} = entry; + if (!isCompositeType(fieldType)) { + if (remainingPath.length > 0) { + throw new Error("Invalid path: cannot navigate beyond a basic type"); } + gindexes.push(gindex); + continue; } + + const childNode = getNode(node, gindex); + if (remainingPath.length === 0) { + gindexes.push(...fieldType.tree_getLeafGindices(gindex, childNode)); + continue; + } + + gindexes.push( + ...fieldType + .tree_createProofGindexes(childNode, [remainingPath]) + .map((childGindex) => concatGindices([gindex, childGindex])) + ); } return gindexes; diff --git a/packages/ssz/test/unit/byType/compatibleUnion/valid.test.ts b/packages/ssz/test/unit/byType/compatibleUnion/valid.test.ts index f845cf73..5f4acca8 100644 --- a/packages/ssz/test/unit/byType/compatibleUnion/valid.test.ts +++ b/packages/ssz/test/unit/byType/compatibleUnion/valid.test.ts @@ -10,6 +10,7 @@ import { ProgressiveByteListType, ProgressiveContainerType, ProgressiveListBasicType, + StableContainerType, UintNumberType, VectorBasicType, hash64, @@ -95,6 +96,33 @@ describe("CompatibleUnionType", () => { expect(toHexString(restored.hashTreeRoot())).to.equal(toHexString(root)); }); + it("creates selected data proofs when nested in a StableContainer", () => { + const CommonOnly = new ProgressiveContainerType({common: byte}, [true]); + const WithExtra = new ProgressiveContainerType({common: byte, fieldOnlyInB: byte}, [true, true]); + const unionType = new CompatibleUnionType({ + 1: CommonOnly, + 2: WithExtra, + }); + const wrapperType = new StableContainerType({shape: unionType}, 4); + const value = { + shape: { + selector: 2, + data: { + common: 7, + fieldOnlyInB: 9, + }, + }, + }; + const root = wrapperType.hashTreeRoot(value); + + for (const path of [[["shape", "data", "common"]], [["shape", "data", "fieldOnlyInB"]]]) { + const proof = wrapperType.toView(value).createProof(path); + const restored = wrapperType.createFromProof(proof, root); + + expect(toHexString(restored.hashTreeRoot())).to.equal(toHexString(root)); + } + }); + it("allows byte list and vector aliases to match uint8 arrays", () => { expect(() => new CompatibleUnionType({1: new ByteListType(4), 2: new ListBasicType(byte, 4)})).not.toThrow(); expect(() => new CompatibleUnionType({1: new ByteVectorType(4), 2: new VectorBasicType(byte, 4)})).not.toThrow(); From 2d485301ecc3508ced21a246e548ff032ebe4bd2 Mon Sep 17 00:00:00 2001 From: Cayman Date: Fri, 15 May 2026 10:38:51 -0400 Subject: [PATCH 5/5] chore: address progressive review follow-ups --- packages/ssz/src/type/compatibleUnion.ts | 2 + packages/ssz/src/type/progressiveList.ts | 71 +++++++++++++++++-- .../test/unit/byType/bitList/invalid.test.ts | 5 ++ .../unit/byType/progressive/valid.test.ts | 24 +++++++ 4 files changed, 96 insertions(+), 6 deletions(-) diff --git a/packages/ssz/src/type/compatibleUnion.ts b/packages/ssz/src/type/compatibleUnion.ts index 963e54fb..377c4829 100644 --- a/packages/ssz/src/type/compatibleUnion.ts +++ b/packages/ssz/src/type/compatibleUnion.ts @@ -52,6 +52,8 @@ const SELECTOR_GINDEX = BigInt(3); /** * CompatibleUnion: union type containing one of the given subtypes with compatible Merkleization. * - Notation: CompatibleUnion({selector: type}), e.g. CompatibleUnion({1: Square, 2: Circle}) + * + * The right leaf encodes the selector and reuses existing list length-node helpers for tree operations. */ export class CompatibleUnionType>> extends CompositeType< CompatibleUnion, diff --git a/packages/ssz/src/type/progressiveList.ts b/packages/ssz/src/type/progressiveList.ts index 15cdc9f5..352e4488 100644 --- a/packages/ssz/src/type/progressiveList.ts +++ b/packages/ssz/src/type/progressiveList.ts @@ -1,5 +1,6 @@ import {allocUnsafe} from "@chainsafe/as-sha256"; import { + BranchNode, Gindex, HashComputationLevel, LeafNode, @@ -13,6 +14,8 @@ import { packedNodeRootsToBytes, packedRootsBytesToLeafNodes, setNode, + subtreeFillToContents, + zeroNode, } from "@chainsafe/persistent-merkle-tree"; import {byteArrayEquals} from "../util/byteArray.ts"; import {ValueWithCachedPermanentRoot, cacheRoot, symbolCachedPermanentRoot} from "../util/merkleize.ts"; @@ -44,6 +47,7 @@ import { getNodesAtProgressiveDepth, merkleizeProgressiveBytes, progressiveChunkGindex, + progressiveSubtreeDepth, progressiveSubtreeFillToContents, } from "./progressive.ts"; @@ -632,9 +636,22 @@ export class ProgressiveListBasicTreeViewDU): void { - const values = this.type.tree_toValue(this._rootNode); - values.push(value); - this._rootNode = this.type.value_toTree(values); + const length = this.length; + if (length >= this.type.limit) { + throw Error("Error pushing over limit"); + } + + const chunkIndex = Math.floor(length / this.type.itemsPerChunk); + const gindex = chunkGindexFromListRoot(chunkIndex); + const leafNode = + length % this.type.itemsPerChunk === 0 + ? LeafNode.fromZero() + : (getNode(this._rootNode, gindex) as LeafNode).clone(); + + this.type.elementType.tree_setToPackedNode(leafNode, length, value); + + const chunksNode = appendProgressiveChunk(this.type.tree_getChunksNode(this._rootNode), chunkIndex, leafNode); + this._rootNode = this.type.tree_setChunksNode(this._rootNode, chunksNode, length + 1); } getAll(values?: ValueOf[]): ValueOf[] { @@ -763,9 +780,20 @@ export class ProgressiveListCompositeTreeViewDU< } push(view: CompositeViewDU): void { - const values = this.type.tree_toValue(this._rootNode); - values.push(this.type.elementType.toValueFromViewDU(view)); - this._rootNode = this.type.value_toTree(values); + this.commit(); + + const length = this.length; + if (length >= this.type.limit) { + throw Error("Error pushing over limit"); + } + + const node = this.type.elementType.commitViewDU(view); + const chunksNode = appendProgressiveChunk(this.type.tree_getChunksNode(this._rootNode), length, node); + this._rootNode = this.type.tree_setChunksNode(this._rootNode, chunksNode, length + 1); + this.caches[length] = this.type.elementType.cacheOfViewDU(view); + if (this.type.elementType.isViewMutable) { + this.viewsChanged.set(length, view); + } } protected clearCache(): void { @@ -782,6 +810,37 @@ function chunkGindexFromListRoot(chunkIndex: number): Gindex { return concatGindices([CHUNKS_GINDEX, progressiveChunkGindex(chunkIndex)]); } +function appendProgressiveChunk(chunksNode: Node, chunkIndex: number, chunkNode: Node): Node { + const {subtreeIndex, subtreeStart} = progressiveSubtreeIndexAndStart(chunkIndex); + if (chunkIndex !== subtreeStart) { + return setNode(chunksNode, progressiveChunkGindex(chunkIndex), chunkNode); + } + + const subtreeNode = subtreeFillToContents([chunkNode], progressiveSubtreeDepth(subtreeIndex)); + const subtreeBranch = new BranchNode(subtreeNode, zeroNode(0)); + if (subtreeIndex === 0) { + return subtreeBranch; + } + + return setNode(chunksNode, progressiveSubtreeBranchGindex(subtreeIndex), subtreeBranch); +} + +function progressiveSubtreeIndexAndStart(chunkIndex: number): {subtreeIndex: number; subtreeStart: number} { + let subtreeIndex = 0; + let subtreeStart = 0; + let subtreeLength = 1; + while (chunkIndex >= subtreeStart + subtreeLength) { + subtreeStart += subtreeLength; + subtreeLength *= 4; + subtreeIndex++; + } + return {subtreeIndex, subtreeStart}; +} + +function progressiveSubtreeBranchGindex(subtreeIndex: number): Gindex { + return concatGindices(Array.from({length: subtreeIndex}, () => BigInt(3))); +} + function readOffsetsProgressiveListComposite( elementFixedSize: null | number, data: DataView, diff --git a/packages/ssz/test/unit/byType/bitList/invalid.test.ts b/packages/ssz/test/unit/byType/bitList/invalid.test.ts index ed885b0d..de06bb86 100644 --- a/packages/ssz/test/unit/byType/bitList/invalid.test.ts +++ b/packages/ssz/test/unit/byType/bitList/invalid.test.ts @@ -26,6 +26,11 @@ describe("BitListType constructor errors", () => { }); describe("Extra error cases", () => { + it("No padding byte - empty", () => { + const bitListType = new BitListType(8 * 8); + expect(() => bitListType.deserialize(new Uint8Array(0))).toThrow("padding bit"); + }); + it("Wrong range over bytes end", () => { const bitListType = new BitListType(8 * 8); const uint8Array = new Uint8Array(0); diff --git a/packages/ssz/test/unit/byType/progressive/valid.test.ts b/packages/ssz/test/unit/byType/progressive/valid.test.ts index c14cba06..cd715760 100644 --- a/packages/ssz/test/unit/byType/progressive/valid.test.ts +++ b/packages/ssz/test/unit/byType/progressive/valid.test.ts @@ -46,6 +46,19 @@ describe("ProgressiveListBasicType", () => { expect(toHexString(view.hashTreeRoot())).to.equal(toHexString(type.hashTreeRoot(expected))); }); + it("supports incremental TreeViewDU pushes across progressive subtrees", () => { + const value = Array.from({length: 32 * 5 + 1}, (_, i) => i % 256); + const view = type.defaultViewDU(); + for (const element of value) { + view.push(element); + } + + expect(view.get(32)).to.equal(value[32]); + expect(view.get(32 * 5)).to.equal(value[32 * 5]); + expect(view.getAll()).to.deep.equal(value); + expect(toHexString(view.hashTreeRoot())).to.equal(toHexString(type.hashTreeRoot(value))); + }); + it("creates and restores proofs for variable-depth chunks", () => { const value = Array.from({length: 33}, (_, i) => i); const view = type.toView(value); @@ -90,6 +103,17 @@ describe("ProgressiveListCompositeType", () => { expect(toHexString(restored.hashTreeRoot())).to.equal(toHexString(root)); }); + + it("supports incremental TreeViewDU pushes across progressive subtrees", () => { + const value = Array.from({length: 6}, (_, i) => ({a: i, b: i + 1})); + const view = type.defaultViewDU(); + for (const element of value) { + view.push(elementType.toViewDU(element)); + } + + expect(type.toValueFromViewDU(view)).to.deep.equal(value); + expect(toHexString(view.hashTreeRoot())).to.equal(toHexString(type.hashTreeRoot(value))); + }); }); describe("ProgressiveBitListType", () => {