diff --git a/yarn-project/foundation/src/curves/bn254/field.ts b/yarn-project/foundation/src/curves/bn254/field.ts index 54048344f24f..715d6816b715 100644 --- a/yarn-project/foundation/src/curves/bn254/field.ts +++ b/yarn-project/foundation/src/curves/bn254/field.ts @@ -6,6 +6,7 @@ import { toBigIntBE, toBufferBE } from '../../bigint-buffer/index.js'; import { randomBytes } from '../../crypto/random/index.js'; import { hexSchemaFor } from '../../schemas/utils.js'; import { BufferReader } from '../../serialize/buffer_reader.js'; +import type { BufferSink } from '../../serialize/buffer_sink.js'; /** * Represents a field derived from BaseField. @@ -64,10 +65,16 @@ abstract class BaseField { protected abstract modulus(): bigint; /** - * Converts the bigint to a Buffer. + * Converts the bigint to a Buffer. With a sink, streams the 32 big-endian bytes straight in (no allocation) + * and returns undefined; without one, returns a freshly allocated buffer. */ - toBuffer(): Buffer { - return toBufferBE(this.asBigInt, 32); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return toBufferBE(this.asBigInt, BaseField.SIZE_IN_BYTES); + } + sink.writeField(this.asBigInt); } toString(): `0x${string}` { diff --git a/yarn-project/foundation/src/serialize/buffer_sink.test.ts b/yarn-project/foundation/src/serialize/buffer_sink.test.ts new file mode 100644 index 000000000000..3367dff593e3 --- /dev/null +++ b/yarn-project/foundation/src/serialize/buffer_sink.test.ts @@ -0,0 +1,249 @@ +import { Fq, Fr } from '../curves/bn254/field.js'; +import { BufferReader } from './buffer_reader.js'; +import { BufferSink, serializeArrayToSink, serializeToSink } from './buffer_sink.js'; +import { bigintToUInt64BE, bigintToUInt128BE, numToUInt32BE } from './free_funcs.js'; +import { boolToBuffer, serializeArrayOfBufferableToVector, serializeBigInt, serializeToBuffer } from './serialize.js'; + +// These bigints exercise the full 32-byte width: zero (hits the legacy zero fast-path), small values, +// odd limb boundaries, and the largest value that still fits 32 bytes. +const FIELD_VALUES = [ + 0n, + 1n, + 0xffn, + 0x1234567890abcdefn, + (1n << 64n) - 1n, + (1n << 64n) + 1n, + (1n << 200n) + 123n, + (1n << 256n) - 1n, +]; + +describe('BufferSink', () => { + describe('writeField / writeBigInt(32) match serializeBigInt byte-for-byte', () => { + it.each(FIELD_VALUES)('writeField(%s)', value => { + const sink = new BufferSink(); + sink.writeField(value); + expect(sink.toBuffer()).toEqual(serializeBigInt(value, 32)); + }); + + it.each(FIELD_VALUES)('writeBigInt(%s) defaults to 32 bytes', value => { + const sink = new BufferSink(); + sink.writeBigInt(value); + expect(sink.toBuffer()).toEqual(serializeBigInt(value, 32)); + }); + }); + + describe('writeBigInt with non-32 width (per-byte branch) matches serializeBigInt', () => { + // Exercises both the pure-limb path (widths that are multiples of 8) and the limb + leftover-head-byte + // path (widths that are not), at zero, small, mid, and max-for-width values. + const cases: Array<[bigint, number]> = [ + [0n, 16], + [1n, 16], + [(1n << 128n) - 1n, 16], + [0xabcdn, 2], + [255n, 1], + [(1n << 64n) - 1n, 8], + [42n, 20], + [(1n << 160n) - 1n, 20], + [(1n << 136n) - 1n, 17], + [(1n << 192n) - 1n, 24], + [(1n << 248n) - 1n, 31], + [(1n << 384n) - 1n, 48], + [0x23456789abcdefn, 7], + ]; + it.each(cases)('writeBigInt(%s, %i)', (value, width) => { + const sink = new BufferSink(); + sink.writeBigInt(value, width); + expect(sink.toBuffer()).toEqual(serializeBigInt(value, width)); + }); + + it('matches bigintToUInt128BE for 16-byte writes', () => { + const value = 0x0123456789abcdef0123456789abcdefn; + const sink = new BufferSink(); + sink.writeBigInt(value, 16); + expect(sink.toBuffer()).toEqual(bigintToUInt128BE(value)); + }); + }); + + describe('rejects out-of-range bigints', () => { + it('throws when writeField overflows 32 bytes', () => { + expect(() => new BufferSink().writeField(1n << 256n)).toThrow(/does not fit into 32 bytes/); + }); + it('throws when writeBigInt overflows the default 32 bytes', () => { + expect(() => new BufferSink().writeBigInt(1n << 256n)).toThrow(/does not fit into 32 bytes/); + }); + it('throws when writeBigInt overflows a narrow width', () => { + expect(() => new BufferSink().writeBigInt(1n << 128n, 16)).toThrow(/does not fit into 16 bytes/); + }); + it('throws on negative values', () => { + expect(() => new BufferSink().writeField(-1n)).toThrow(/negative/); + expect(() => new BufferSink().writeBigInt(-5n)).toThrow(/negative/); + }); + }); + + describe('primitive writers match the legacy free functions', () => { + const write = (fn: (sink: BufferSink) => void): Buffer => { + const sink = new BufferSink(); + fn(sink); + return sink.toBuffer(); + }; + + it('writeBool', () => { + expect(write(s => s.writeBool(true))).toEqual(boolToBuffer(true)); + expect(write(s => s.writeBool(false))).toEqual(boolToBuffer(false)); + }); + it('writeNumber matches numToUInt32BE', () => { + expect(write(s => s.writeNumber(0xdeadbeef))).toEqual(numToUInt32BE(0xdeadbeef)); + }); + it('writeUInt64 matches bigintToUInt64BE', () => { + const v = 0x1122334455667788n; + expect(write(s => s.writeUInt64(v))).toEqual(bigintToUInt64BE(v)); + }); + it('writeUInt16 / writeUInt8 are big-endian', () => { + expect(write(s => s.writeUInt16(0xbeef))).toEqual(Buffer.from([0xbe, 0xef])); + expect(write(s => s.writeUInt8(0x7f))).toEqual(Buffer.from([0x7f])); + }); + it('writeString is a 4-byte length prefix followed by UTF-8 bytes', () => { + const value = 'héllo'; + const utf8 = Buffer.from(value); + expect(write(s => s.writeString(value))).toEqual(Buffer.concat([numToUInt32BE(utf8.length), utf8])); + }); + }); + + describe('writeFields fast path', () => { + it('matches a sequence of individual writeField calls', () => { + const fields = FIELD_VALUES.filter(v => v <= (1n << 256n) - 1n).map(v => new Fr(v % Fr.MODULUS)); + const sink = new BufferSink(); + sink.writeFields(fields); + + const expected = serializeToBuffer(fields); + expect(sink.toBuffer()).toEqual(expected); + }); + + it('throws on a negative element', () => { + expect(() => new BufferSink().writeFields([{ toBigInt: () => -1n }])).toThrow(/negative/); + }); + }); + + describe('capacity growth', () => { + it('grows from a tiny initial capacity without corrupting data', () => { + const sink = new BufferSink(1); + const values = Array.from({ length: 300 }, (_, i) => BigInt(i) * 7n); + for (const v of values) { + sink.writeField(v); + } + expect(sink.length).toBe(300 * 32); + expect(sink.toBuffer()).toEqual(Buffer.concat(values.map(v => serializeBigInt(v, 32)))); + }); + }); + + describe('reset reuses the backing buffer', () => { + it('drops the cursor while leaving previously returned buffers intact', () => { + const sink = new BufferSink(); + sink.writeField(123n); + const first = sink.toBuffer(); + + sink.reset(); + expect(sink.length).toBe(0); + + sink.writeField(456n); + expect(sink.toBuffer()).toEqual(serializeBigInt(456n, 32)); + // The earlier toBuffer() is a copy, so it survives the reset + reuse. + expect(first).toEqual(serializeBigInt(123n, 32)); + }); + }); + + describe('asUint8Array', () => { + it('returns a zero-copy view of the written region', () => { + const sink = new BufferSink(64); + sink.writeField(7n); + const view = sink.asUint8Array(); + expect(view.length).toBe(32); + expect(view[31]).toBe(7); + }); + }); + + describe('serializeToSink dispatch matches serializeToBuffer', () => { + it('handles mixed primitives, buffers, and nested arrays', () => { + const sink = new BufferSink(); + serializeToSink(sink, true, 5, 9000000000n, 'hi', Buffer.from([1, 2, 3]), [false, 7]); + + const expected = serializeToBuffer(true, 5, 9000000000n, 'hi', Buffer.from([1, 2, 3]), [false, 7]); + expect(sink.toBuffer()).toEqual(expected); + }); + + it('streams a migrated node (writes into the sink, returns void)', () => { + const migrated = { toBuffer: (sink: BufferSink) => sink.writeUInt8(0xcc) }; + const sink = new BufferSink(); + serializeToSink(sink, migrated); + expect(sink.toBuffer()).toEqual(Buffer.from([0xcc])); + }); + + it('folds in a not-yet-migrated node via the return-value fallback', () => { + const legacy = { toBuffer: () => Buffer.from([0xaa, 0xbb]) }; + const sink = new BufferSink(); + serializeToSink(sink, legacy); + expect(sink.toBuffer()).toEqual(Buffer.from([0xaa, 0xbb])); + }); + + it('throws on an unsupported value', () => { + expect(() => serializeToSink(new BufferSink(), { not: 'serializable' } as any)).toThrow(/Cannot serialize/); + }); + }); + + describe('serializeArrayToSink matches serializeArrayOfBufferableToVector', () => { + it('writes a 4-byte count prefix, flattening nested arrays', () => { + const sink = new BufferSink(); + serializeArrayToSink(sink, [1, [2, 3], 4], 4); + expect(sink.toBuffer()).toEqual(serializeArrayOfBufferableToVector([1, [2, 3], 4], 4)); + }); + + it('writes a 1-byte count prefix', () => { + const sink = new BufferSink(); + serializeArrayToSink(sink, [1, 2, 3], 1); + expect(sink.toBuffer()).toEqual(serializeArrayOfBufferableToVector([1, 2, 3], 1)); + }); + + it('rejects an unsupported prefix length', () => { + expect(() => serializeArrayToSink(new BufferSink(), [1], 2)).toThrow(/prefix length/); + }); + }); + + describe('BufferSink.serialize', () => { + it('serializes a migrated object into a single buffer', () => { + const obj = { toBuffer: (sink: BufferSink) => serializeToSink(sink, 1n, 2, true) }; + expect(BufferSink.serialize(obj)).toEqual(serializeToBuffer(1n, 2, true)); + }); + }); + + // alexg's suggestion: write into a sink, then read everything back out with a BufferReader. + describe('round-trips through BufferReader', () => { + it('reads back exactly what was written', () => { + const fr = Fr.random(); + const fq = Fq.random(); + const sink = new BufferSink(); + sink.writeBool(true); + sink.writeNumber(0x01020304); + sink.writeUInt64(0xdeadbeefcafebaben); + sink.writeField(fr.toBigInt()); + sink.writeBigInt(fq.toBigInt()); + sink.writeString('blob-sink'); + + const reader = BufferReader.asReader(sink.toBuffer()); + expect(reader.readBoolean()).toBe(true); + expect(reader.readNumber()).toBe(0x01020304); + expect(reader.readUInt64()).toBe(0xdeadbeefcafebaben); + expect(reader.readObject(Fr)).toEqual(fr); + expect(reader.readObject(Fq)).toEqual(fq); + expect(reader.readString()).toBe('blob-sink'); + }); + + it('round-trips a field vector written via serializeArrayToSink', () => { + const fields = Array.from({ length: 5 }, () => Fr.random()); + const sink = new BufferSink(); + serializeArrayToSink(sink, fields, 4); + + const reader = BufferReader.asReader(sink.toBuffer()); + expect(reader.readVector(Fr)).toEqual(fields); + }); + }); +}); diff --git a/yarn-project/foundation/src/serialize/buffer_sink.ts b/yarn-project/foundation/src/serialize/buffer_sink.ts new file mode 100644 index 000000000000..636fa44d1fc2 --- /dev/null +++ b/yarn-project/foundation/src/serialize/buffer_sink.ts @@ -0,0 +1,350 @@ +/** + * A value that can be appended to a {@link BufferSink}. Mirrors `Bufferable`, but the object arm allows the + * optional-sink overload of `toBuffer` so a legacy `toBuffer(): Buffer` and a migrated `toBuffer(sink?)` both fit. + */ +export type Sinkable = + | boolean + | Buffer + | Uint8Array + | number + | bigint + | string + | { toBuffer(sink?: BufferSink): Buffer | void } + | Sinkable[]; + +const DEFAULT_INITIAL_CAPACITY = 1024; +const MASK_64 = (1n << 64n) - 1n; + +/** + * A growable, append-only binary sink backed by a single `ArrayBuffer`. + * + * It exists to replace the recursive `toBuffer()` chain — which allocates a `Buffer` at every node and + * `Buffer.concat`s at every level — with one buffer that the whole object graph streams into and that is sliced + * exactly once at the root. Encodings are big-endian, matching `serializeToBuffer` byte-for-byte. + * + * The migration contract is the optional-sink `toBuffer`: + * + * ```ts + * toBuffer(): Buffer; + * toBuffer(sink: BufferSink): void; + * toBuffer(sink?: BufferSink): Buffer | void { + * if (!sink) return BufferSink.serialize(this); // own a sink, fill it, slice once + * serializeToSink(sink, this.a, this.b, this.c); // stream into the caller's sink + * } + * ``` + * + * When a sink is passed it writes and returns `undefined`; otherwise it returns its own buffer. Nodes that have + * not been migrated keep their plain `toBuffer(): Buffer` and are folded in via the return-value fallback in + * {@link serializeToSink}, so the tree stays valid at every step of an incremental migration. + * + * Performance note: bigints are written via `DataView.setBigUint64` limbs ({@link writeBigInt}, + * {@link writeField}), NOT a per-byte shift loop. The limb form is ~16x faster than the legacy hex round-trip + * on field-heavy structures, and faster than the legacy path at every width — a naive per-byte shift loop is + * actually the slowest option, since each byte allocates a fresh BigInt. See the spike benchmark. + */ +export class BufferSink { + private buffer: Uint8Array; + private view: DataView; + private offset = 0; + + constructor(initialCapacity: number = DEFAULT_INITIAL_CAPACITY) { + const ab = new ArrayBuffer(initialCapacity); + this.buffer = new Uint8Array(ab); + this.view = new DataView(ab); + } + + /** Number of bytes written so far. */ + public get length(): number { + return this.offset; + } + + /** + * Serialize a migrated value into a fresh sink and return the single resulting buffer. This is the no-sink + * branch of every migrated `toBuffer`: one allocation, one slice, only at the root. + * + * @param obj - The value to serialize via its sink-aware `toBuffer`. + * @param initialCapacity - Optional size hint (e.g. from `getSize()`) to skip reallocations on hot paths. + */ + public static serialize(obj: { toBuffer(sink: BufferSink): void }, initialCapacity?: number): Buffer { + const sink = new BufferSink(initialCapacity); + obj.toBuffer(sink); + return sink.toBuffer(); + } + + private ensure(extra: number): void { + const required = this.offset + extra; + if (required <= this.buffer.length) { + return; + } + let next = this.buffer.length * 2; + while (next < required) { + next *= 2; + } + const grown = new Uint8Array(next); + grown.set(this.buffer.subarray(0, this.offset)); + this.buffer = grown; + this.view = new DataView(grown.buffer); + } + + /** Append a single byte (0/1) for a boolean, matching `boolToBuffer`. */ + public writeBool(value: boolean): void { + this.ensure(1); + this.buffer[this.offset++] = value ? 1 : 0; + } + + /** Append an unsigned 8-bit integer. */ + public writeUInt8(value: number): void { + this.ensure(1); + this.buffer[this.offset++] = value & 0xff; + } + + /** Append a big-endian unsigned 16-bit integer. */ + public writeUInt16(value: number): void { + this.ensure(2); + this.view.setUint16(this.offset, value, false); + this.offset += 2; + } + + /** Append a big-endian unsigned 32-bit integer, matching `numToUInt32BE`. */ + public writeNumber(value: number): void { + this.ensure(4); + this.view.setUint32(this.offset, value, false); + this.offset += 4; + } + + /** Append a big-endian unsigned 64-bit integer, matching `bigintToUInt64BE`. */ + public writeUInt64(value: bigint): void { + this.ensure(8); + this.view.setBigUint64(this.offset, value, false); + this.offset += 8; + } + + /** + * Append a non-negative bigint as a big-endian unsigned integer of `width` bytes (default 32), matching + * `serializeBigInt` / `toBufferBE`. Every width is written limb-wise via `setBigUint64` rather than the + * legacy per-byte BigInt shift loop — that loop allocated a fresh BigInt per byte and benchmarked as the + * *slowest* option (slower even than the legacy hex round-trip); the limb form is faster at every width. + * + * The bulk of the value is written as 64-bit big-endian limbs from the least-significant tail upward, and + * a final `width % 8` (at most 7) high bytes that don't fill a whole limb are written individually. The + * common widths (8/16/32) are exact multiples of 8, so they take the pure-limb path with no remainder. + * `width === 32` keeps its own straight-line four-limb form (shared with {@link writeField}) since it is + * the `Fr`/`Fq`/Chonk-proof hot path and the unrolled body is marginally faster than the loop. + * + * @param value - The non-negative bigint to serialize. + * @param width - Output width in bytes. + */ + public writeBigInt(value: bigint, width = 32): void { + if (value < 0n) { + throw new Error(`Cannot serialize negative bigint ${value}`); + } + this.ensure(width); + const o = this.offset; + if (width === 32) { + let x = value; + this.view.setBigUint64(o + 24, x & MASK_64, false); + x >>= 64n; + this.view.setBigUint64(o + 16, x & MASK_64, false); + x >>= 64n; + this.view.setBigUint64(o + 8, x & MASK_64, false); + x >>= 64n; + this.view.setBigUint64(o, x & MASK_64, false); + x >>= 64n; + if (x !== 0n) { + throw new Error(`BigInt ${value} does not fit into 32 bytes`); + } + this.offset += 32; + return; + } + let x = value; + let pos = o + width; + while (pos - 8 >= o) { + pos -= 8; + this.view.setBigUint64(pos, x & MASK_64, false); + x >>= 64n; + } + for (let i = pos - 1; i >= o; i--) { + this.buffer[i] = Number(x & 0xffn); + x >>= 8n; + } + if (x !== 0n) { + throw new Error(`BigInt ${value} does not fit into ${width} bytes`); + } + this.offset += width; + } + + /** + * Append a 32-byte (256-bit) big-endian non-negative bigint. Specialized form of {@link writeBigInt} + * with no width parameter and no width-branch, so V8 can inline the four `setBigUint64` limb writes + * without the wider branchy form's instability. This is the Fr/Fq leaf hot path; prefer it over + * `writeBigInt(value, 32)` in callers that always serialize 32-byte fields. + */ + public writeField(value: bigint): void { + if (value < 0n) { + throw new Error(`Cannot serialize negative bigint ${value}`); + } + this.ensure(32); + const o = this.offset; + let x = value; + this.view.setBigUint64(o + 24, x & MASK_64, false); + x >>= 64n; + this.view.setBigUint64(o + 16, x & MASK_64, false); + x >>= 64n; + this.view.setBigUint64(o + 8, x & MASK_64, false); + x >>= 64n; + this.view.setBigUint64(o, x & MASK_64, false); + if (x >> 64n !== 0n) { + throw new Error(`BigInt ${value} does not fit into 32 bytes`); + } + this.offset = o + 32; + } + + /** + * Append an array of field elements (or any objects exposing `toBigInt(): bigint`) as a contiguous + * sequence of 32-byte big-endian limbs. Iterates inline — no per-element `toBuffer` dispatch and no + * per-element function-call indirection through the generic sinkable dispatcher. Use this in classes + * that hold large flat field arrays (e.g. the 1632-element ChonkProof field list). + */ + public writeFields(fields: ReadonlyArray<{ toBigInt(): bigint }>): void { + this.ensure(32 * fields.length); + for (let i = 0, n = fields.length; i < n; i++) { + const value = fields[i].toBigInt(); + if (value < 0n) { + throw new Error(`Cannot serialize negative bigint ${value}`); + } + const o = this.offset; + let x = value; + this.view.setBigUint64(o + 24, x & MASK_64, false); + x >>= 64n; + this.view.setBigUint64(o + 16, x & MASK_64, false); + x >>= 64n; + this.view.setBigUint64(o + 8, x & MASK_64, false); + x >>= 64n; + this.view.setBigUint64(o, x & MASK_64, false); + if (x >> 64n !== 0n) { + throw new Error(`BigInt ${value} does not fit into 32 bytes`); + } + this.offset = o + 32; + } + } + + /** Append raw bytes verbatim (no length prefix), matching how a `Buffer`/`Uint8Array` is serialized inline. */ + public writeBytes(bytes: Uint8Array): void { + this.ensure(bytes.length); + this.buffer.set(bytes, this.offset); + this.offset += bytes.length; + } + + /** Append a string as a 4-byte big-endian length prefix followed by its UTF-8 bytes, matching the string case. */ + public writeString(value: string): void { + const bytes = Buffer.from(value); + this.writeNumber(bytes.length); + this.writeBytes(bytes); + } + + /** + * Reset the write cursor without releasing the backing buffer. Lets the same sink be reused across many + * serializations without reallocating — the per-call cost drops to the bytes written (no `new ArrayBuffer`). + * Any `Buffer` previously returned by {@link toBuffer} is a fresh copy and remains valid; any view from + * {@link asUint8Array} aliases the buffer and is invalidated. + */ + public reset(): void { + this.offset = 0; + } + + /** Copy the written region into a freshly allocated `Buffer`. The single allocation of the whole serialization. */ + public toBuffer(): Buffer { + return Buffer.from(this.buffer.subarray(0, this.offset)); + } + + /** A zero-copy view of the written region. Valid only until the next write (which may reallocate). */ + public asUint8Array(): Uint8Array { + return this.buffer.subarray(0, this.offset); + } +} + +/** + * Streaming counterpart to `serializeToBufferArray`: walk a list of {@link Sinkable}s and append each to the sink. + * Dispatch matches `serializeToBufferArray` byte-for-byte. An object is serialized via `obj.toBuffer(sink)`; a + * migrated node writes into the sink and returns `undefined`, while a not-yet-migrated node ignores the argument + * and returns its `Buffer`, which we copy in. That return-value fallback is what makes the migration incremental. + * + * Dispatch is hot-pathed for the common case of objects exposing `toBuffer` (Fr/Fq and every other migrated leaf), + * and arrays recurse via an inner-array helper to avoid the per-call rest-args allocation that a spread-recurse + * would force on large flat arrays like the 1632-element ChonkProof field list. + * + * @param sink - The destination sink. + * @param objs - The values to serialize, in order. + */ +export function serializeToSink(sink: BufferSink, ...objs: Sinkable[]): void { + serializeArrayToSinkInner(sink, objs); +} + +function serializeArrayToSinkInner(sink: BufferSink, objs: Sinkable[]): void { + for (let i = 0, n = objs.length; i < n; i++) { + serializeOneToSink(sink, objs[i]); + } +} + +function serializeOneToSink(sink: BufferSink, obj: Sinkable): void { + if (typeof obj === 'object' && obj !== null) { + // Hot path: migrated nodes (anything implementing toBuffer, including Fr/Fq via BaseField). This branch + // is taken for the vast majority of recursive serialization, so it is checked before the typeof primitives. + if ((obj as any).toBuffer !== undefined) { + const ret = (obj as { toBuffer(sink: BufferSink): Buffer | void }).toBuffer(sink); + if (ret !== undefined) { + sink.writeBytes(ret); + } + return; + } + if (Array.isArray(obj)) { + serializeArrayToSinkInner(sink, obj); + return; + } + if (Buffer.isBuffer(obj) || obj instanceof Uint8Array) { + sink.writeBytes(obj); + return; + } + throw new Error(`Cannot serialize input to sink: ${typeof obj} ${(obj as any).constructor?.name}`); + } + if (typeof obj === 'boolean') { + sink.writeBool(obj); + } else if (typeof obj === 'bigint') { + sink.writeBigInt(obj); + } else if (typeof obj === 'number') { + sink.writeNumber(obj); + } else if (typeof obj === 'string') { + sink.writeString(obj); + } else { + throw new Error(`Cannot serialize input to sink: ${typeof obj}`); + } +} + +/** + * Streaming counterpart to `serializeArrayOfBufferableToVector`: write a length prefix (1 or 4 bytes) giving the + * number of serialized elements, then each element. Byte-identical to the buffer version. + * + * @param sink - The destination sink. + * @param objs - The vector elements. + * @param prefixLength - Width of the count prefix in bytes (1 or 4). + */ +export function serializeArrayToSink(sink: BufferSink, objs: Sinkable[], prefixLength = 4): void { + const count = countSinkables(objs); + if (prefixLength === 1) { + sink.writeUInt8(count); + } else if (prefixLength === 4) { + sink.writeNumber(count); + } else { + throw new Error(`Unsupported prefix length. Got ${prefixLength}, expected 1 or 4`); + } + serializeArrayToSinkInner(sink, objs); +} + +/** Count how many top-level buffers `serializeToBufferArray` would emit, flattening nested arrays as it does. */ +function countSinkables(objs: Sinkable[]): number { + let count = 0; + for (const obj of objs) { + count += Array.isArray(obj) ? countSinkables(obj) : 1; + } + return count; +} diff --git a/yarn-project/foundation/src/serialize/index.ts b/yarn-project/foundation/src/serialize/index.ts index 33cb9a5bb4d8..083377cb0f65 100644 --- a/yarn-project/foundation/src/serialize/index.ts +++ b/yarn-project/foundation/src/serialize/index.ts @@ -1,5 +1,6 @@ export * from './free_funcs.js'; export * from './buffer_reader.js'; +export * from './buffer_sink.js'; export * from './field_reader.js'; export * from './types.js'; export * from './serialize.js'; diff --git a/yarn-project/stdlib/src/aztec-address/index.ts b/yarn-project/stdlib/src/aztec-address/index.ts index 372fe2be7c48..e821d3997020 100644 --- a/yarn-project/stdlib/src/aztec-address/index.ts +++ b/yarn-project/stdlib/src/aztec-address/index.ts @@ -2,7 +2,7 @@ import { NULL_MSG_SENDER_CONTRACT_ADDRESS } from '@aztec/constants'; import { Fr, fromBuffer } from '@aztec/foundation/curves/bn254'; import { Point } from '@aztec/foundation/curves/grumpkin'; import { type ZodFor, bufferSchemaFor, hexSchemaFor } from '@aztec/foundation/schemas'; -import { type BufferReader, FieldReader } from '@aztec/foundation/serialize'; +import { type BufferReader, type BufferSink, FieldReader } from '@aztec/foundation/serialize'; import { hexToBuffer } from '@aztec/foundation/string'; import { inspect } from 'util'; @@ -137,8 +137,13 @@ export class AztecAddress { return Point.fromXAndSign(this.xCoord, true); } - toBuffer() { - return this.xCoord.toBuffer(); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return this.xCoord.toBuffer(); + } + this.xCoord.toBuffer(sink); } toBigInt() { diff --git a/yarn-project/stdlib/src/gas/gas.ts b/yarn-project/stdlib/src/gas/gas.ts index f404609a55bf..099d235f700c 100644 --- a/yarn-project/stdlib/src/gas/gas.ts +++ b/yarn-project/stdlib/src/gas/gas.ts @@ -1,6 +1,6 @@ import { Fr } from '@aztec/foundation/curves/bn254'; import { schemas } from '@aztec/foundation/schemas'; -import { BufferReader, FieldReader, serializeToBuffer, serializeToFields } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToFields, serializeToSink } from '@aztec/foundation/serialize'; import type { FieldsOf } from '@aztec/foundation/types'; import { inspect } from 'util'; @@ -79,8 +79,13 @@ export class Gas { return new Gas(reader.readNumber(), reader.readNumber()); } - toBuffer() { - return serializeToBuffer(this.daGas, this.l2Gas); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.daGas, this.l2Gas); } [inspect.custom]() { diff --git a/yarn-project/stdlib/src/gas/gas_fees.ts b/yarn-project/stdlib/src/gas/gas_fees.ts index 975de3af77eb..b8c5ecd725da 100644 --- a/yarn-project/stdlib/src/gas/gas_fees.ts +++ b/yarn-project/stdlib/src/gas/gas_fees.ts @@ -1,12 +1,6 @@ import { Fr } from '@aztec/foundation/curves/bn254'; import { schemas } from '@aztec/foundation/schemas'; -import { - BufferReader, - FieldReader, - bigintToUInt128BE, - serializeToBuffer, - serializeToFields, -} from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToFields } from '@aztec/foundation/serialize'; import type { FieldsOf } from '@aztec/foundation/types'; import { inspect } from 'util'; @@ -110,8 +104,14 @@ export class GasFees { return new GasFees(reader.readUInt128(), reader.readUInt128()); } - toBuffer() { - return serializeToBuffer(bigintToUInt128BE(this.feePerDaGas), bigintToUInt128BE(this.feePerL2Gas)); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + sink.writeBigInt(this.feePerDaGas, 16); + sink.writeBigInt(this.feePerL2Gas, 16); } static fromFields(fields: Fr[] | FieldReader) { diff --git a/yarn-project/stdlib/src/gas/gas_settings.ts b/yarn-project/stdlib/src/gas/gas_settings.ts index 2f768a1c920b..f1b472ef69f0 100644 --- a/yarn-project/stdlib/src/gas/gas_settings.ts +++ b/yarn-project/stdlib/src/gas/gas_settings.ts @@ -1,6 +1,6 @@ import { GAS_SETTINGS_LENGTH, MAX_PROCESSABLE_DA_GAS_PER_CHECKPOINT, MAX_PROCESSABLE_L2_GAS } from '@aztec/constants'; import { Fr } from '@aztec/foundation/curves/bn254'; -import { BufferReader, FieldReader, serializeToBuffer, serializeToFields } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToFields, serializeToSink } from '@aztec/foundation/serialize'; import type { FieldsOf } from '@aztec/foundation/types'; import { z } from 'zod'; @@ -194,8 +194,13 @@ export class GasSettings { ); } - toBuffer() { - return serializeToBuffer(...GasSettings.getFields(this)); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, ...GasSettings.getFields(this)); } static fromFields(fields: Fr[] | FieldReader): GasSettings { diff --git a/yarn-project/stdlib/src/kernel/log_hash.ts b/yarn-project/stdlib/src/kernel/log_hash.ts index 5b5082c33533..aafa1ceded48 100644 --- a/yarn-project/stdlib/src/kernel/log_hash.ts +++ b/yarn-project/stdlib/src/kernel/log_hash.ts @@ -1,5 +1,5 @@ import { Fr } from '@aztec/foundation/curves/bn254'; -import { BufferReader, FieldReader, serializeToBuffer } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToSink } from '@aztec/foundation/serialize'; import type { FieldsOf } from '@aztec/foundation/types'; import { inspect } from 'util'; @@ -33,8 +33,13 @@ export class LogHash { return new LogHash(Fr.zero(), 0); } - toBuffer(): Buffer { - return serializeToBuffer(this.value, this.length); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.value, this.length); } static fromBuffer(buffer: Buffer | BufferReader) { @@ -78,8 +83,13 @@ export class CountedLogHash { return new CountedLogHash(LogHash.empty(), 0); } - toBuffer(): Buffer { - return serializeToBuffer(this.logHash, this.counter); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.logHash, this.counter); } static fromBuffer(buffer: Buffer | BufferReader) { @@ -115,8 +125,13 @@ export class ScopedLogHash { return new ScopedLogHash(LogHash.empty(), AztecAddress.ZERO); } - toBuffer(): Buffer { - return serializeToBuffer(this.logHash, this.contractAddress); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.logHash, this.contractAddress); } static fromBuffer(buffer: Buffer | BufferReader) { @@ -144,8 +159,13 @@ export class ScopedCountedLogHash { return new ScopedCountedLogHash(reader.readObject(CountedLogHash), AztecAddress.fromField(reader.readField())); } - toBuffer(): Buffer { - return serializeToBuffer(this.inner, this.contractAddress); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.inner, this.contractAddress); } static fromBuffer(buffer: Buffer | BufferReader) { diff --git a/yarn-project/stdlib/src/kernel/private_kernel_tail_circuit_public_inputs.ts b/yarn-project/stdlib/src/kernel/private_kernel_tail_circuit_public_inputs.ts index 3bf2c6787830..2aeba3158368 100644 --- a/yarn-project/stdlib/src/kernel/private_kernel_tail_circuit_public_inputs.ts +++ b/yarn-project/stdlib/src/kernel/private_kernel_tail_circuit_public_inputs.ts @@ -1,6 +1,6 @@ import { Fr } from '@aztec/foundation/curves/bn254'; import { bufferSchemaFor } from '@aztec/foundation/schemas'; -import { BufferReader, bigintToUInt64BE, serializeToBuffer } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, serializeToSink } from '@aztec/foundation/serialize'; import { AztecAddress } from '../aztec-address/index.js'; import { Gas } from '../gas/gas.js'; @@ -60,8 +60,14 @@ export class PartialPrivateTailPublicInputsForPublic { ); } - toBuffer() { - return serializeToBuffer( + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink( + sink, this.nonRevertibleAccumulatedData, this.revertibleAccumulatedData, this.publicTeardownCallRequest, @@ -89,8 +95,13 @@ export class PartialPrivateTailPublicInputsForRollup { return this.end.getSize(); } - toBuffer() { - return serializeToBuffer(this.end); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.end); } static empty() { @@ -300,16 +311,20 @@ export class PrivateKernelTailCircuitPublicInputs { ); } - toBuffer() { + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } const isForPublic = !!this.forPublic; - return serializeToBuffer( - isForPublic, - this.constants, - this.gasUsed, - this.feePayer, - bigintToUInt64BE(this.expirationTimestamp), - isForPublic ? this.forPublic!.toBuffer() : this.forRollup!.toBuffer(), - ); + serializeToSink(sink, isForPublic, this.constants, this.gasUsed, this.feePayer); + sink.writeUInt64(this.expirationTimestamp); + if (isForPublic) { + this.forPublic!.toBuffer(sink); + } else { + this.forRollup!.toBuffer(sink); + } } static empty() { diff --git a/yarn-project/stdlib/src/kernel/private_to_avm_accumulated_data.ts b/yarn-project/stdlib/src/kernel/private_to_avm_accumulated_data.ts index 9442439a039a..02cee6a6423a 100644 --- a/yarn-project/stdlib/src/kernel/private_to_avm_accumulated_data.ts +++ b/yarn-project/stdlib/src/kernel/private_to_avm_accumulated_data.ts @@ -10,11 +10,12 @@ import { Fr } from '@aztec/foundation/curves/bn254'; import { schemas } from '@aztec/foundation/schemas'; import { BufferReader, + BufferSink, FieldReader, type Tuple, assertLength, - serializeToBuffer, serializeToFields, + serializeToSink, } from '@aztec/foundation/serialize'; import { inspect } from 'util'; @@ -100,8 +101,13 @@ export class PrivateToAvmAccumulatedData { ); } - toBuffer() { - return serializeToBuffer(...PrivateToAvmAccumulatedData.getFields(this)); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, ...PrivateToAvmAccumulatedData.getFields(this)); } static empty() { @@ -200,8 +206,13 @@ export class PrivateToAvmAccumulatedDataArrayLengths { return new PrivateToAvmAccumulatedDataArrayLengths(reader.readNumber(), reader.readNumber(), reader.readNumber()); } - toBuffer() { - return serializeToBuffer(...PrivateToAvmAccumulatedDataArrayLengths.getFields(this)); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, ...PrivateToAvmAccumulatedDataArrayLengths.getFields(this)); } static empty() { diff --git a/yarn-project/stdlib/src/kernel/private_to_public_accumulated_data.ts b/yarn-project/stdlib/src/kernel/private_to_public_accumulated_data.ts index ab1b79190672..dca430dfc31f 100644 --- a/yarn-project/stdlib/src/kernel/private_to_public_accumulated_data.ts +++ b/yarn-project/stdlib/src/kernel/private_to_public_accumulated_data.ts @@ -12,10 +12,11 @@ import { arraySerializedSizeOfNonEmpty } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { BufferReader, + BufferSink, FieldReader, type Tuple, - serializeToBuffer, serializeToFields, + serializeToSink, } from '@aztec/foundation/serialize'; import { inspect } from 'util'; @@ -85,8 +86,13 @@ export class PrivateToPublicAccumulatedData { ); } - toBuffer() { - return serializeToBuffer(...PrivateToPublicAccumulatedData.getFields(this)); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, ...PrivateToPublicAccumulatedData.getFields(this)); } toFields(): Fr[] { diff --git a/yarn-project/stdlib/src/kernel/private_to_public_kernel_circuit_public_inputs.ts b/yarn-project/stdlib/src/kernel/private_to_public_kernel_circuit_public_inputs.ts index 5a9e6a08ff61..c108011b6ee9 100644 --- a/yarn-project/stdlib/src/kernel/private_to_public_kernel_circuit_public_inputs.ts +++ b/yarn-project/stdlib/src/kernel/private_to_public_kernel_circuit_public_inputs.ts @@ -2,7 +2,7 @@ import { DomainSeparator, PRIVATE_TO_PUBLIC_KERNEL_CIRCUIT_PUBLIC_INPUTS_LENGTH import { poseidon2HashWithSeparator } from '@aztec/foundation/crypto/poseidon'; import type { Fr } from '@aztec/foundation/curves/bn254'; import { bufferSchemaFor } from '@aztec/foundation/schemas'; -import { BufferReader, bigintToUInt64BE, serializeToBuffer, serializeToFields } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, serializeToFields, serializeToSink } from '@aztec/foundation/serialize'; import { bufferToHex, hexToBuffer } from '@aztec/foundation/string'; import type { FieldsOf } from '@aztec/foundation/types'; @@ -24,16 +24,22 @@ export class PrivateToPublicKernelCircuitPublicInputs { public expirationTimestamp: UInt64, ) {} - toBuffer() { - return serializeToBuffer( + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink( + sink, this.constants, this.nonRevertibleAccumulatedData, this.revertibleAccumulatedData, this.publicTeardownCallRequest, this.gasUsed, this.feePayer, - bigintToUInt64BE(this.expirationTimestamp), ); + sink.writeUInt64(this.expirationTimestamp); } static getFields(fields: FieldsOf) { diff --git a/yarn-project/stdlib/src/kernel/private_to_rollup_accumulated_data.ts b/yarn-project/stdlib/src/kernel/private_to_rollup_accumulated_data.ts index 7807f6ad9b97..c79302cb115c 100644 --- a/yarn-project/stdlib/src/kernel/private_to_rollup_accumulated_data.ts +++ b/yarn-project/stdlib/src/kernel/private_to_rollup_accumulated_data.ts @@ -10,7 +10,7 @@ import { type FieldsOf, makeTuple } from '@aztec/foundation/array'; import { arraySerializedSizeOfNonEmpty } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { bufferSchemaFor } from '@aztec/foundation/schemas'; -import { BufferReader, type Tuple, serializeToBuffer, serializeToFields } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, type Tuple, serializeToFields, serializeToSink } from '@aztec/foundation/serialize'; import { bufferToHex, hexToBuffer } from '@aztec/foundation/string'; import { inspect } from 'util'; @@ -79,8 +79,13 @@ export class PrivateToRollupAccumulatedData { return this.toBuffer(); } - toBuffer() { - return serializeToBuffer(...PrivateToRollupAccumulatedData.getFields(this)); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, ...PrivateToRollupAccumulatedData.getFields(this)); } toString() { diff --git a/yarn-project/stdlib/src/kernel/private_to_rollup_kernel_circuit_public_inputs.ts b/yarn-project/stdlib/src/kernel/private_to_rollup_kernel_circuit_public_inputs.ts index a78d01380a16..1498f7863817 100644 --- a/yarn-project/stdlib/src/kernel/private_to_rollup_kernel_circuit_public_inputs.ts +++ b/yarn-project/stdlib/src/kernel/private_to_rollup_kernel_circuit_public_inputs.ts @@ -2,7 +2,7 @@ import { DomainSeparator, PRIVATE_TO_ROLLUP_KERNEL_CIRCUIT_PUBLIC_INPUTS_LENGTH import { poseidon2HashWithSeparator } from '@aztec/foundation/crypto/poseidon'; import type { Fr } from '@aztec/foundation/curves/bn254'; import { bufferSchemaFor } from '@aztec/foundation/schemas'; -import { BufferReader, bigintToUInt64BE, serializeToBuffer, serializeToFields } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, serializeToFields, serializeToSink } from '@aztec/foundation/serialize'; import { bufferToHex, hexToBuffer } from '@aztec/foundation/string'; import type { FieldsOf } from '@aztec/foundation/types'; @@ -44,14 +44,14 @@ export class PrivateToRollupKernelCircuitPublicInputs { return this.end.nullifiers.filter(n => !n.isZero()); } - toBuffer() { - return serializeToBuffer( - this.constants, - this.end, - this.gasUsed, - this.feePayer, - bigintToUInt64BE(this.expirationTimestamp), - ); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.constants, this.end, this.gasUsed, this.feePayer); + sink.writeUInt64(this.expirationTimestamp); } /** diff --git a/yarn-project/stdlib/src/kernel/public_call_request.ts b/yarn-project/stdlib/src/kernel/public_call_request.ts index 452765203be6..e9442d95d697 100644 --- a/yarn-project/stdlib/src/kernel/public_call_request.ts +++ b/yarn-project/stdlib/src/kernel/public_call_request.ts @@ -1,7 +1,7 @@ import { COUNTED_PUBLIC_CALL_REQUEST_LENGTH, PUBLIC_CALL_REQUEST_LENGTH } from '@aztec/constants'; import { Fr } from '@aztec/foundation/curves/bn254'; import { schemas } from '@aztec/foundation/schemas'; -import { BufferReader, FieldReader, serializeToBuffer, serializeToFields } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToFields, serializeToSink } from '@aztec/foundation/serialize'; import type { FieldsOf } from '@aztec/foundation/types'; import { inspect } from 'util'; @@ -89,8 +89,13 @@ export class PublicCallRequest { ); } - toBuffer() { - return serializeToBuffer(...PublicCallRequest.getFields(this)); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, ...PublicCallRequest.getFields(this)); } static empty() { @@ -188,8 +193,13 @@ export class PublicCallRequestArrayLengths { return new PublicCallRequestArrayLengths(reader.readNumber(), reader.readNumber(), reader.readBoolean()); } - toBuffer() { - return serializeToBuffer(...PublicCallRequestArrayLengths.getFields(this)); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, ...PublicCallRequestArrayLengths.getFields(this)); } static empty() { @@ -250,8 +260,13 @@ export class CountedPublicCallRequest { return new CountedPublicCallRequest(reader.readObject(PublicCallRequest), reader.readNumber()); } - toBuffer() { - return serializeToBuffer(...CountedPublicCallRequest.getFields(this)); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, ...CountedPublicCallRequest.getFields(this)); } static empty() { diff --git a/yarn-project/stdlib/src/logs/contract_class_log.ts b/yarn-project/stdlib/src/logs/contract_class_log.ts index aa6aad262f99..fd4f43a33e6e 100644 --- a/yarn-project/stdlib/src/logs/contract_class_log.ts +++ b/yarn-project/stdlib/src/logs/contract_class_log.ts @@ -2,7 +2,7 @@ import { CONTRACT_CLASS_LOG_LENGTH, CONTRACT_CLASS_LOG_SIZE_IN_FIELDS } from '@a import { poseidon2Hash } from '@aztec/foundation/crypto/poseidon'; import { Fr } from '@aztec/foundation/curves/bn254'; import { schemas } from '@aztec/foundation/schemas'; -import { BufferReader, FieldReader, serializeToBuffer, serializeToFields } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToFields, serializeToSink } from '@aztec/foundation/serialize'; import type { FieldsOf } from '@aztec/foundation/types'; import { inspect } from 'util'; @@ -56,8 +56,13 @@ export class ContractClassLogFields { ); } - toBuffer() { - return serializeToBuffer(this.fields); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.fields); } static fromBuffer(buffer: Buffer | BufferReader) { @@ -169,8 +174,13 @@ export class ContractClassLog { return new ContractClassLog(AztecAddress.ZERO, ContractClassLogFields.empty(), 0); } - toBuffer(): Buffer { - return serializeToBuffer([this.contractAddress, this.fields, this.emittedLength]); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.contractAddress, this.fields, this.emittedLength); } static fromBuffer(buffer: Buffer | BufferReader) { diff --git a/yarn-project/stdlib/src/logs/private_log.ts b/yarn-project/stdlib/src/logs/private_log.ts index 10f915a1e6bd..c5f34795c6a6 100644 --- a/yarn-project/stdlib/src/logs/private_log.ts +++ b/yarn-project/stdlib/src/logs/private_log.ts @@ -5,10 +5,11 @@ import { Fr } from '@aztec/foundation/curves/bn254'; import { schemas } from '@aztec/foundation/schemas'; import { BufferReader, + BufferSink, FieldReader, type Tuple, - serializeToBuffer, serializeToFields, + serializeToSink, } from '@aztec/foundation/serialize'; import { inspect } from 'util'; @@ -67,8 +68,13 @@ export class PrivateLog { return new PrivateLog(makeTuple(PRIVATE_LOG_SIZE_IN_FIELDS, Fr.zero), 0); } - toBuffer(): Buffer { - return serializeToBuffer(this.fields, this.emittedLength); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.fields, this.emittedLength); } static fromBuffer(buffer: Buffer | BufferReader) { diff --git a/yarn-project/stdlib/src/messaging/l2_to_l1_message.ts b/yarn-project/stdlib/src/messaging/l2_to_l1_message.ts index e75698991a66..a285f50a796a 100644 --- a/yarn-project/stdlib/src/messaging/l2_to_l1_message.ts +++ b/yarn-project/stdlib/src/messaging/l2_to_l1_message.ts @@ -1,7 +1,7 @@ import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; import { schemas } from '@aztec/foundation/schemas'; -import { BufferReader, FieldReader, serializeToBuffer, serializeToFields } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToFields, serializeToSink } from '@aztec/foundation/serialize'; import { z } from 'zod'; @@ -54,8 +54,13 @@ export class L2ToL1Message { * Serialize this as a buffer. * @returns The buffer. */ - toBuffer(): Buffer { - return serializeToBuffer(this.recipient, this.content); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.recipient, this.content); } /** @@ -127,8 +132,13 @@ export class CountedL2ToL1Message { return new CountedL2ToL1Message(reader.readObject(L2ToL1Message), reader.readNumber()); } - toBuffer(): Buffer { - return serializeToBuffer(this.message, this.counter); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.message, this.counter); } static fromFields(fields: Fr[] | FieldReader) { @@ -187,8 +197,13 @@ export class ScopedL2ToL1Message { return new ScopedL2ToL1Message(reader.readObject(L2ToL1Message), reader.readObject(AztecAddress)); } - toBuffer(): Buffer { - return serializeToBuffer(this.message, this.contractAddress); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.message, this.contractAddress); } static fromFields(fields: Fr[] | FieldReader) { @@ -229,8 +244,13 @@ export class ScopedCountedL2ToL1Message { return new ScopedCountedL2ToL1Message(reader.readObject(CountedL2ToL1Message), reader.readObject(AztecAddress)); } - toBuffer(): Buffer { - return serializeToBuffer(this.inner, this.contractAddress); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.inner, this.contractAddress); } static fromFields(fields: Fr[] | FieldReader) { diff --git a/yarn-project/stdlib/src/proofs/chonk_proof.ts b/yarn-project/stdlib/src/proofs/chonk_proof.ts index 0669cd276a09..5efcab51e904 100644 --- a/yarn-project/stdlib/src/proofs/chonk_proof.ts +++ b/yarn-project/stdlib/src/proofs/chonk_proof.ts @@ -4,7 +4,7 @@ import { times } from '@aztec/foundation/collection'; import { randomBytes } from '@aztec/foundation/crypto/random'; import { Fr } from '@aztec/foundation/curves/bn254'; import { bufferSchemaFor } from '@aztec/foundation/schemas'; -import { BufferReader, numToUInt32BE, serializeToBuffer } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink } from '@aztec/foundation/serialize'; /** * Serialization format detection for ChonkProof is size-based: @@ -133,13 +133,22 @@ export class ChonkProof { * If compressed bytes are available, uses the compressed format (~1.7x smaller). * Otherwise falls back to legacy field element format. */ - public toBuffer() { + public toBuffer(): Buffer; + public toBuffer(sink: BufferSink): void; + public toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } if (this.compressedProof) { // Compressed format: [compressed_byte_count: uint32] [compressed_bytes] - return Buffer.concat([numToUInt32BE(this.compressedProof.length), this.compressedProof]); + sink.writeNumber(this.compressedProof.length); + sink.writeBytes(this.compressedProof); + return; } // Legacy format: [field_count=1632: uint32] [fields...] - return serializeToBuffer(this.fields.length, this.fields); + // Use the specialized Fr-array fast path to skip per-element sinkable dispatch on the 1632 leaves. + sink.writeNumber(this.fields.length); + sink.writeFields(this.fields); } } @@ -196,8 +205,14 @@ export class ChonkProofWithPublicInputs { return new ChonkProofWithPublicInputs(proof); } - public toBuffer() { - return serializeToBuffer(this.fieldsWithPublicInputs.length, this.fieldsWithPublicInputs); + public toBuffer(): Buffer; + public toBuffer(sink: BufferSink): void; + public toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + sink.writeNumber(this.fieldsWithPublicInputs.length); + sink.writeFields(this.fieldsWithPublicInputs); } // Called when constructing from bb proving results. diff --git a/yarn-project/stdlib/src/trees/append_only_tree_snapshot.ts b/yarn-project/stdlib/src/trees/append_only_tree_snapshot.ts index dbe699dd0df3..3fd875f03e3d 100644 --- a/yarn-project/stdlib/src/trees/append_only_tree_snapshot.ts +++ b/yarn-project/stdlib/src/trees/append_only_tree_snapshot.ts @@ -1,6 +1,6 @@ import { Fr } from '@aztec/foundation/curves/bn254'; import { schemas } from '@aztec/foundation/schemas'; -import { BufferReader, FieldReader, serializeToBuffer } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToSink } from '@aztec/foundation/serialize'; import { bufferToHex, hexToBuffer } from '@aztec/foundation/string'; import { inspect } from 'util'; @@ -44,8 +44,13 @@ export class AppendOnlyTreeSnapshot { return this.root.size + 4; } - toBuffer() { - return serializeToBuffer(this.root, this.nextAvailableLeafIndex); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.root, this.nextAvailableLeafIndex); } toFields(): Fr[] { diff --git a/yarn-project/stdlib/src/tx/block_header.ts b/yarn-project/stdlib/src/tx/block_header.ts index 8a32161027b5..605537678886 100644 --- a/yarn-project/stdlib/src/tx/block_header.ts +++ b/yarn-project/stdlib/src/tx/block_header.ts @@ -4,7 +4,7 @@ import { poseidon2HashWithSeparator } from '@aztec/foundation/crypto/poseidon'; import { randomInt } from '@aztec/foundation/crypto/random'; import { Fr } from '@aztec/foundation/curves/bn254'; import { type ZodFor, schemas } from '@aztec/foundation/schemas'; -import { BufferReader, FieldReader, serializeToBuffer, serializeToFields } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToFields, serializeToSink } from '@aztec/foundation/serialize'; import { bufferToHex, hexToBuffer } from '@aztec/foundation/string'; import type { FieldsOf } from '@aztec/foundation/types'; @@ -85,8 +85,13 @@ export class BlockHeader { ); } - toBuffer() { - return serializeToBuffer(...BlockHeader.getFields(this)); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, ...BlockHeader.getFields(this)); } toFields(): Fr[] { diff --git a/yarn-project/stdlib/src/tx/global_variables.ts b/yarn-project/stdlib/src/tx/global_variables.ts index a5b5ff521ab8..c346332232ea 100644 --- a/yarn-project/stdlib/src/tx/global_variables.ts +++ b/yarn-project/stdlib/src/tx/global_variables.ts @@ -4,13 +4,7 @@ import { randomInt } from '@aztec/foundation/crypto/random'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; import { jsonStringify } from '@aztec/foundation/json-rpc'; -import { - BufferReader, - FieldReader, - bigintToUInt64BE, - serializeToBuffer, - serializeToFields, -} from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToFields, serializeToSink } from '@aztec/foundation/serialize'; import type { FieldsOf } from '@aztec/foundation/types'; import { inspect } from 'util'; @@ -149,17 +143,15 @@ export class GlobalVariables { ] as const; } - toBuffer() { - return serializeToBuffer([ - this.chainId, - this.version, - this.blockNumber, - this.slotNumber, - bigintToUInt64BE(this.timestamp), - this.coinbase, - this.feeRecipient, - this.gasFees, - ]); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.chainId, this.version, this.blockNumber, this.slotNumber); + sink.writeUInt64(this.timestamp); + serializeToSink(sink, this.coinbase, this.feeRecipient, this.gasFees); } toFields() { diff --git a/yarn-project/stdlib/src/tx/hashed_values.ts b/yarn-project/stdlib/src/tx/hashed_values.ts index 8b59f759c0af..5a4fef151852 100644 --- a/yarn-project/stdlib/src/tx/hashed_values.ts +++ b/yarn-project/stdlib/src/tx/hashed_values.ts @@ -1,5 +1,5 @@ import { Fr } from '@aztec/foundation/curves/bn254'; -import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, serializeToSink } from '@aztec/foundation/serialize'; import type { FieldsOf } from '@aztec/foundation/types'; import { z } from 'zod'; @@ -48,8 +48,13 @@ export class HashedValues { return new HashedValues([Fr.random(), Fr.random()], Fr.random()); } - toBuffer() { - return serializeToBuffer(new Vector(this.values), this.hash); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, new Vector(this.values), this.hash); } static fromBuffer(buffer: Buffer | BufferReader): HashedValues { diff --git a/yarn-project/stdlib/src/tx/partial_state_reference.ts b/yarn-project/stdlib/src/tx/partial_state_reference.ts index 1330904430eb..ed2f4bc5f2b1 100644 --- a/yarn-project/stdlib/src/tx/partial_state_reference.ts +++ b/yarn-project/stdlib/src/tx/partial_state_reference.ts @@ -1,6 +1,6 @@ import { PARTIAL_STATE_REFERENCE_LENGTH } from '@aztec/constants'; import type { Fr } from '@aztec/foundation/curves/bn254'; -import { BufferReader, FieldReader, serializeToBuffer } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToSink } from '@aztec/foundation/serialize'; import type { FieldsOf } from '@aztec/foundation/types'; import { z } from 'zod'; @@ -88,8 +88,13 @@ export class PartialStateReference { ); } - toBuffer() { - return serializeToBuffer(this.noteHashTree, this.nullifierTree, this.publicDataTree); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.noteHashTree, this.nullifierTree, this.publicDataTree); } toFields() { diff --git a/yarn-project/stdlib/src/tx/public_call_request_with_calldata.ts b/yarn-project/stdlib/src/tx/public_call_request_with_calldata.ts index 9489e131f38f..92932e7daec8 100644 --- a/yarn-project/stdlib/src/tx/public_call_request_with_calldata.ts +++ b/yarn-project/stdlib/src/tx/public_call_request_with_calldata.ts @@ -1,5 +1,5 @@ import { Fr } from '@aztec/foundation/curves/bn254'; -import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, serializeToSink } from '@aztec/foundation/serialize'; import { inspect } from 'util'; import { z } from 'zod'; @@ -63,8 +63,13 @@ export class PublicCallRequestWithCalldata { ); } - toBuffer() { - return serializeToBuffer(this.request, new Vector(this.calldata)); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.request, new Vector(this.calldata)); } static fromBuffer(buffer: Buffer | BufferReader) { diff --git a/yarn-project/stdlib/src/tx/state_reference.ts b/yarn-project/stdlib/src/tx/state_reference.ts index a785d8b696b1..79a59204c7ac 100644 --- a/yarn-project/stdlib/src/tx/state_reference.ts +++ b/yarn-project/stdlib/src/tx/state_reference.ts @@ -5,7 +5,7 @@ import { STATE_REFERENCE_LENGTH, } from '@aztec/constants'; import type { Fr } from '@aztec/foundation/curves/bn254'; -import { BufferReader, FieldReader, serializeToBuffer } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToSink } from '@aztec/foundation/serialize'; import type { FieldsOf } from '@aztec/foundation/types'; import { inspect } from 'util'; @@ -46,9 +46,14 @@ export class StateReference { return new StateReference(...StateReference.getFields(fields)); } - toBuffer() { - // Note: The order here must match the order in the ProposedHeaderLib solidity library. - return serializeToBuffer(this.l1ToL2MessageTree, this.partial); + // Note: The order here must match the order in the ProposedHeaderLib solidity library. + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.l1ToL2MessageTree, this.partial); } toFields(): Fr[] { diff --git a/yarn-project/stdlib/src/tx/tree_snapshots.ts b/yarn-project/stdlib/src/tx/tree_snapshots.ts index c843ecc1e979..9d42d8970fb8 100644 --- a/yarn-project/stdlib/src/tx/tree_snapshots.ts +++ b/yarn-project/stdlib/src/tx/tree_snapshots.ts @@ -1,6 +1,6 @@ import { TREE_SNAPSHOTS_LENGTH } from '@aztec/constants'; import type { Fr } from '@aztec/foundation/curves/bn254'; -import { BufferReader, FieldReader, serializeToBuffer } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToSink } from '@aztec/foundation/serialize'; import { inspect } from 'util'; import { z } from 'zod'; @@ -51,9 +51,14 @@ export class TreeSnapshots { ); } - toBuffer() { - // Note: The order here must match the order in the ProposedHeaderLib solidity library. - return serializeToBuffer(this.l1ToL2MessageTree, this.noteHashTree, this.nullifierTree, this.publicDataTree); + // Note: The order here must match the order in the ProposedHeaderLib solidity library. + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.l1ToL2MessageTree, this.noteHashTree, this.nullifierTree, this.publicDataTree); } static fromFields(fields: Fr[] | FieldReader) { diff --git a/yarn-project/stdlib/src/tx/tx.ts b/yarn-project/stdlib/src/tx/tx.ts index 3a70c3c9f5dd..787aafd375b4 100644 --- a/yarn-project/stdlib/src/tx/tx.ts +++ b/yarn-project/stdlib/src/tx/tx.ts @@ -2,7 +2,13 @@ import { DA_GAS_PER_FIELD, TX_DA_GAS_OVERHEAD } from '@aztec/constants'; import { type BaseBuffer32, Buffer32 } from '@aztec/foundation/buffer'; import { Fr } from '@aztec/foundation/curves/bn254'; import type { ZodFor } from '@aztec/foundation/schemas'; -import { BufferReader, serializeArrayOfBufferableToVector, serializeToBuffer } from '@aztec/foundation/serialize'; +import { + BufferReader, + BufferSink, + serializeArrayOfBufferableToVector, + serializeArrayToSink, + serializeToSink, +} from '@aztec/foundation/serialize'; import { type FieldsOf, unfreeze } from '@aztec/foundation/types'; import { z } from 'zod'; @@ -19,6 +25,14 @@ import { HashedValues } from './hashed_values.js'; import { PublicCallRequestWithCalldata } from './public_call_request_with_calldata.js'; import { TxHash } from './tx_hash.js'; +// Static presize hint for the BufferSink the no-sink Tx.toBuffer() path allocates. Empirically a +// public-with-enqueued-calls Tx serialized by the bootstrapped bench (yarn-project/stdlib/src/tx/ +// tx_bench.test.ts) is ~129128 bytes; a private-only Tx is ~81763 bytes. 128 KiB covers both shapes +// without a single doubling-growth `ensure()` resize on the cold path. Real-world Txs that happen to +// exceed this still serialize correctly — the sink falls back to its standard doubling growth — just +// with the existing cost. +const TX_SINK_PRESIZE_BYTES = 131072; + /** * The interface of an L2 transaction. */ @@ -125,14 +139,15 @@ export class Tx extends Gossipable { * Serializes the Tx object into a Buffer. * @returns Buffer representation of the Tx object. */ - toBuffer() { - return serializeToBuffer([ - this.txHash, - this.data, - this.chonkProof, - serializeArrayOfBufferableToVector(this.contractClassLogFields, 1), - serializeArrayOfBufferableToVector(this.publicFunctionCalldata, 1), - ]); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this, TX_SINK_PRESIZE_BYTES); + } + serializeToSink(sink, this.txHash, this.data, this.chonkProof); + serializeArrayToSink(sink, this.contractClassLogFields, 1); + serializeArrayToSink(sink, this.publicFunctionCalldata, 1); } static get schema(): ZodFor { @@ -334,7 +349,12 @@ export class TxArray extends Array { } } - public toBuffer(): Buffer { - return serializeArrayOfBufferableToVector(this); + public toBuffer(): Buffer; + public toBuffer(sink: BufferSink): void; + public toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return serializeArrayOfBufferableToVector(this); + } + serializeArrayToSink(sink, [...this]); } } diff --git a/yarn-project/stdlib/src/tx/tx_bench.test.ts b/yarn-project/stdlib/src/tx/tx_bench.test.ts index d8d0133809a3..948db87957d0 100644 --- a/yarn-project/stdlib/src/tx/tx_bench.test.ts +++ b/yarn-project/stdlib/src/tx/tx_bench.test.ts @@ -1,3 +1,4 @@ +import { BufferSink } from '@aztec/foundation/serialize'; import { Timer } from '@aztec/foundation/timer'; import { webcrypto } from 'node:crypto'; @@ -9,17 +10,33 @@ import { mockTx, mockTxForRollup } from '../tests/mocks.js'; import { Tx } from './tx.js'; const RUNS = 100; +const TO_BUFFER_ITERS_PER_SAMPLE = 20; + describe('Tx', () => { let privateTxHistogram: RecordableHistogram; let publicTxHistogram: RecordableHistogram; let privateSha256Histogram: RecordableHistogram; let publicSha256Histogram: RecordableHistogram; + let privateToBufferUsHistogram: RecordableHistogram; + let publicToBufferUsHistogram: RecordableHistogram; + let privateToSinkUsHistogram: RecordableHistogram; + let publicToSinkUsHistogram: RecordableHistogram; + let privateColdToBufferUsHistogram: RecordableHistogram; + let publicColdToBufferUsHistogram: RecordableHistogram; + let privateToBufferBytes = 0; + let publicToBufferBytes = 0; beforeAll(() => { privateTxHistogram = createHistogram(); publicTxHistogram = createHistogram(); privateSha256Histogram = createHistogram(); publicSha256Histogram = createHistogram(); + privateToBufferUsHistogram = createHistogram(); + publicToBufferUsHistogram = createHistogram(); + privateToSinkUsHistogram = createHistogram(); + publicToSinkUsHistogram = createHistogram(); + privateColdToBufferUsHistogram = createHistogram(); + publicColdToBufferUsHistogram = createHistogram(); }); afterAll(async () => { @@ -30,12 +47,28 @@ describe('Tx', () => { data.push({ name: `${name}/p50`, value: histogram.percentile(50), unit: 'ms' }); data.push({ name: `${name}/p95`, value: histogram.percentile(95), unit: 'ms' }); }; + const recordUsHistogram = (name: string, histogram: RecordableHistogram) => { + data.push({ name: `${name}/avg`, value: histogram.mean, unit: 'us' }); + data.push({ name: `${name}/p50`, value: histogram.percentile(50), unit: 'us' }); + data.push({ name: `${name}/p95`, value: histogram.percentile(95), unit: 'us' }); + }; recordHistogram('Tx/private/getTxHash', privateTxHistogram); recordHistogram('Tx/public/getTxHash', publicTxHistogram); recordHistogram('Tx/private/sha256', privateSha256Histogram); recordHistogram('Tx/public/sha256', publicSha256Histogram); + recordUsHistogram('Tx/private/toBuffer', privateToBufferUsHistogram); + data.push({ name: `Tx/private/toBuffer/bytes`, value: privateToBufferBytes, unit: 'bytes' }); + recordUsHistogram('Tx/public/toBuffer', publicToBufferUsHistogram); + data.push({ name: `Tx/public/toBuffer/bytes`, value: publicToBufferBytes, unit: 'bytes' }); + + recordUsHistogram('Tx/private/toBufferReusedSink', privateToSinkUsHistogram); + recordUsHistogram('Tx/public/toBufferReusedSink', publicToSinkUsHistogram); + + recordUsHistogram('Tx/private/toBufferCold', privateColdToBufferUsHistogram); + recordUsHistogram('Tx/public/toBufferCold', publicColdToBufferUsHistogram); + await fs.mkdir(path.dirname(process.env.BENCH_OUTPUT), { recursive: true }); await fs.writeFile(process.env.BENCH_OUTPUT, JSON.stringify(data, null, 2)); } else if (process.env.BENCH_OUTPUT_MD) { @@ -52,6 +85,25 @@ describe('Tx', () => { await writeRow('PUB', publicTxHistogram); await writeRow('PRV-SHA256', privateSha256Histogram); await writeRow('PUB-SHA256', publicSha256Histogram); + + await f.write('\n|toBuffer (us/op)|MIN|AVG|P50|P90|MAX|BYTES|\n'); + await f.write('|----|---|---|---|---|---|---|\n'); + await f.write( + `|PRV|${privateToBufferUsHistogram.min}|${privateToBufferUsHistogram.mean}|${privateToBufferUsHistogram.percentile(50)}|${privateToBufferUsHistogram.percentile(90)}|${privateToBufferUsHistogram.max}|${privateToBufferBytes}|\n`, + ); + await f.write( + `|PUB|${publicToBufferUsHistogram.min}|${publicToBufferUsHistogram.mean}|${publicToBufferUsHistogram.percentile(50)}|${publicToBufferUsHistogram.percentile(90)}|${publicToBufferUsHistogram.max}|${publicToBufferBytes}|\n`, + ); + + await f.write('\n|toBuffer(sink) reused-sink (us/op)|MIN|AVG|P50|P90|MAX|\n'); + await f.write('|----|---|---|---|---|---|\n'); + await writeRow('PRV', privateToSinkUsHistogram); + await writeRow('PUB', publicToSinkUsHistogram); + + await f.write('\n|toBuffer cold per-tx (us/op)|MIN|AVG|P50|P90|MAX|\n'); + await f.write('|----|---|---|---|---|---|\n'); + await writeRow('PRV', privateColdToBufferUsHistogram); + await writeRow('PUB', publicColdToBufferUsHistogram); } }); @@ -90,4 +142,97 @@ describe('Tx', () => { publicSha256Histogram.record(Math.max(1, Math.ceil(timer.ms()))); } }); + + it('serializes a private-only tx to buffer', async () => { + const tx = await mockTxForRollup(42); + privateToBufferBytes = tx.toBuffer().length; + for (let i = 0; i < RUNS / 2; i++) { + tx.toBuffer(); + } + for (let i = 0; i < RUNS; i++) { + const start = process.hrtime.bigint(); + for (let j = 0; j < TO_BUFFER_ITERS_PER_SAMPLE; j++) { + tx.toBuffer(); + } + const ns = Number(process.hrtime.bigint() - start); + privateToBufferUsHistogram.record(Math.max(1, Math.round(ns / TO_BUFFER_ITERS_PER_SAMPLE / 1000))); + } + }); + + it('serializes a tx with enqueued public calls to buffer', async () => { + const tx = await mockTx(42); + publicToBufferBytes = tx.toBuffer().length; + for (let i = 0; i < RUNS / 2; i++) { + tx.toBuffer(); + } + for (let i = 0; i < RUNS; i++) { + const start = process.hrtime.bigint(); + for (let j = 0; j < TO_BUFFER_ITERS_PER_SAMPLE; j++) { + tx.toBuffer(); + } + const ns = Number(process.hrtime.bigint() - start); + publicToBufferUsHistogram.record(Math.max(1, Math.round(ns / TO_BUFFER_ITERS_PER_SAMPLE / 1000))); + } + }); + + it('serializes a private-only tx into a reused sink', async () => { + const tx = await mockTxForRollup(42); + const sink = new BufferSink(privateToBufferBytes || tx.toBuffer().length); + for (let i = 0; i < RUNS / 2; i++) { + tx.toBuffer(sink); + sink.reset(); + } + for (let i = 0; i < RUNS; i++) { + const start = process.hrtime.bigint(); + for (let j = 0; j < TO_BUFFER_ITERS_PER_SAMPLE; j++) { + tx.toBuffer(sink); + sink.reset(); + } + const ns = Number(process.hrtime.bigint() - start); + privateToSinkUsHistogram.record(Math.max(1, Math.round(ns / TO_BUFFER_ITERS_PER_SAMPLE / 1000))); + } + }); + + it('serializes a tx with enqueued public calls into a reused sink', async () => { + const tx = await mockTx(42); + const sink = new BufferSink(publicToBufferBytes || tx.toBuffer().length); + for (let i = 0; i < RUNS / 2; i++) { + tx.toBuffer(sink); + sink.reset(); + } + for (let i = 0; i < RUNS; i++) { + const start = process.hrtime.bigint(); + for (let j = 0; j < TO_BUFFER_ITERS_PER_SAMPLE; j++) { + tx.toBuffer(sink); + sink.reset(); + } + const ns = Number(process.hrtime.bigint() - start); + publicToSinkUsHistogram.record(Math.max(1, Math.round(ns / TO_BUFFER_ITERS_PER_SAMPLE / 1000))); + } + }); + + // Cold-start benchmarks: one toBuffer per Tx, with each Tx never previously serialized. This is the + // realistic per-tx cost (cold sink-size hint, no per-instance state warmed), the inverse of the + // steady-state cases above which reuse one Tx across thousands of calls. + it('serializes 100 freshly constructed private-only txs (cold)', async () => { + const pool = await Promise.all(Array.from({ length: RUNS }, (_, i) => mockTxForRollup(1000 + i))); + for (let i = 0; i < RUNS; i++) { + const tx = pool[i]; + const start = process.hrtime.bigint(); + tx.toBuffer(); + const ns = Number(process.hrtime.bigint() - start); + privateColdToBufferUsHistogram.record(Math.max(1, Math.round(ns / 1000))); + } + }); + + it('serializes 100 freshly constructed public txs (cold)', async () => { + const pool = await Promise.all(Array.from({ length: RUNS }, (_, i) => mockTx(1000 + i))); + for (let i = 0; i < RUNS; i++) { + const tx = pool[i]; + const start = process.hrtime.bigint(); + tx.toBuffer(); + const ns = Number(process.hrtime.bigint() - start); + publicColdToBufferUsHistogram.record(Math.max(1, Math.round(ns / 1000))); + } + }); }); diff --git a/yarn-project/stdlib/src/tx/tx_constant_data.ts b/yarn-project/stdlib/src/tx/tx_constant_data.ts index 8f66fcb08866..1a590e4d1516 100644 --- a/yarn-project/stdlib/src/tx/tx_constant_data.ts +++ b/yarn-project/stdlib/src/tx/tx_constant_data.ts @@ -1,6 +1,6 @@ import { TX_CONSTANT_DATA_LENGTH } from '@aztec/constants'; import { Fr } from '@aztec/foundation/curves/bn254'; -import { BufferReader, FieldReader, serializeToBuffer, serializeToFields } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToFields, serializeToSink } from '@aztec/foundation/serialize'; import type { FieldsOf } from '@aztec/foundation/types'; import { BlockHeader } from './block_header.js'; @@ -57,8 +57,13 @@ export class TxConstantData { ); } - toBuffer() { - return serializeToBuffer(...TxConstantData.getFields(this)); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, ...TxConstantData.getFields(this)); } static empty() { diff --git a/yarn-project/stdlib/src/tx/tx_context.ts b/yarn-project/stdlib/src/tx/tx_context.ts index ad32b6d3713e..607498b66e95 100644 --- a/yarn-project/stdlib/src/tx/tx_context.ts +++ b/yarn-project/stdlib/src/tx/tx_context.ts @@ -1,7 +1,7 @@ import { TX_CONTEXT_LENGTH } from '@aztec/constants'; import { Fr } from '@aztec/foundation/curves/bn254'; import { schemas } from '@aztec/foundation/schemas'; -import { BufferReader, FieldReader, serializeToBuffer, serializeToFields } from '@aztec/foundation/serialize'; +import { BufferReader, BufferSink, FieldReader, serializeToFields, serializeToSink } from '@aztec/foundation/serialize'; import type { FieldsOf } from '@aztec/foundation/types'; import { z } from 'zod'; @@ -49,8 +49,13 @@ export class TxContext { * Serialize as a buffer. * @returns The buffer. */ - toBuffer() { - return serializeToBuffer(...TxContext.getFields(this)); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, ...TxContext.getFields(this)); } static fromFields(fields: Fr[] | FieldReader): TxContext { diff --git a/yarn-project/stdlib/src/tx/tx_hash.ts b/yarn-project/stdlib/src/tx/tx_hash.ts index fd4788405c2e..c7b99c1a6e72 100644 --- a/yarn-project/stdlib/src/tx/tx_hash.ts +++ b/yarn-project/stdlib/src/tx/tx_hash.ts @@ -1,5 +1,10 @@ import { Fr } from '@aztec/foundation/curves/bn254'; -import { BufferReader, serializeArrayOfBufferableToVector } from '@aztec/foundation/serialize'; +import { + BufferReader, + type BufferSink, + serializeArrayOfBufferableToVector, + serializeArrayToSink, +} from '@aztec/foundation/serialize'; import { schemas } from '../schemas/index.js'; @@ -34,8 +39,13 @@ export class TxHash { return new TxHash(value); } - public toBuffer() { - return this.hash.toBuffer(); + public toBuffer(): Buffer; + public toBuffer(sink: BufferSink): void; + public toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return this.hash.toBuffer(); + } + this.hash.toBuffer(sink); } public toString() { @@ -82,7 +92,12 @@ export class TxHashArray extends Array { } } - public toBuffer(): Buffer { - return serializeArrayOfBufferableToVector([...this]); + public toBuffer(): Buffer; + public toBuffer(sink: BufferSink): void; + public toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return serializeArrayOfBufferableToVector([...this]); + } + serializeArrayToSink(sink, [...this]); } } diff --git a/yarn-project/stdlib/src/types/shared.ts b/yarn-project/stdlib/src/types/shared.ts index 0489e1405985..90a74d98f3de 100644 --- a/yarn-project/stdlib/src/types/shared.ts +++ b/yarn-project/stdlib/src/types/shared.ts @@ -1,4 +1,4 @@ -import { type Bufferable, serializeToBuffer } from '@aztec/foundation/serialize'; +import { BufferSink, type Bufferable, type Sinkable, serializeToSink } from '@aztec/foundation/serialize'; /** * Implementation of a vector. Matches how we are serializing and deserializing vectors in cpp (length in the first position, followed by the items). @@ -11,8 +11,13 @@ export class Vector { public items: T[], ) {} - toBuffer() { - return serializeToBuffer(this.items.length, this.items); + toBuffer(): Buffer; + toBuffer(sink: BufferSink): void; + toBuffer(sink?: BufferSink): Buffer | void { + if (!sink) { + return BufferSink.serialize(this); + } + serializeToSink(sink, this.items.length, this.items as Sinkable[]); } toFriendlyJSON() {