From 2c18206165f618fb444034659fe813b1ea6c80f8 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 21 Jul 2026 10:35:38 +0800 Subject: [PATCH 1/2] fix: reduce loro-js snapshot memory --- .changeset/lazy-snapshots-rest.md | 7 + context/loro-js-performance.md | 44 +- loro-js/benchmarks/snapshot-memory.mjs | 89 ++++ loro-js/package.json | 1 + loro-js/src/codec/bytes.ts | 4 +- loro-js/src/codec/document.ts | 72 ++- loro-js/src/codec/sstable.ts | 325 +++++++++-- loro-js/src/codec/state-snapshot.ts | 88 +++ loro-js/src/runtime/containers.ts | 47 ++ loro-js/src/runtime/document.ts | 711 +++++++++++++++++-------- loro-js/tests/runtime.test.ts | 40 ++ loro-js/tests/sstable.test.ts | 39 +- 12 files changed, 1169 insertions(+), 298 deletions(-) create mode 100644 .changeset/lazy-snapshots-rest.md create mode 100644 loro-js/benchmarks/snapshot-memory.mjs diff --git a/.changeset/lazy-snapshots-rest.md b/.changeset/lazy-snapshots-rest.md new file mode 100644 index 000000000..9ab2ad18c --- /dev/null +++ b/.changeset/lazy-snapshots-rest.md @@ -0,0 +1,7 @@ +--- +"loro-js": patch +--- + +Keep large latest-state snapshots encoded and hydrate containers on demand. +Local edits and later updates now use a small history overlay, while snapshot +export rewrites only dirty SSTable blocks and avoids redundant output buffers. diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 06da61d6a..15792b0d2 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -1,6 +1,6 @@ # loro-js Performance Architecture -Verified against code 2026-07-18. +Verified against code 2026-07-21. The pure TypeScript runtime lives in `loro-js/src/runtime`. Its performance target is the asymptotic behavior of the Rust runtime, while accepting a larger @@ -102,12 +102,20 @@ JavaScript constant factor. - Snapshot SSTables choose interoperable LZ4 blocks when they reduce size. DeltaRLE state columns encode and decode as streams rather than allocating million-item BigInt intermediates, and LZ4 decode writes into typed storage. - Importing an initial latest-state snapshot hydrates current containers and - validates its frontier blocks immediately, while retaining the other owned - history blocks in encoded form. A history query, edit, checkout, export, or - later update builds and validates all history indexes once on a staging - document before installing them; current reads, version, frontiers, and - operation count do not force that work. + Importing an initial latest-state snapshot validates every state entry and + frontier block immediately, but retains current state as an owned encoded + SSTable. Root containers are hydrated at import; referenced descendants are + decoded one SSTable block at a time when first accessed. Untouched blocks are + copied directly during snapshot export, while dirty container entries are + locally rewritten. The encoded history remains a read-only base and later + local or imported changes use a small materialized overlay. Local edits, full + update export, latest snapshot export, current reads, version, frontiers, and + operation count therefore do not build the complete history DAG. Historical + queries, checkout, partial-range export, and other APIs requiring arbitrary + dependency traversal still build and validate all history indexes once on a + staging document before installing them. Import subscribers retain eager + state hydration because their import event must describe every changed + container. When an element's deleted flag, tree parent/position, or map visibility changes, mutate it through its owning index helper. Direct mutation leaves subtree or @@ -130,6 +138,28 @@ with: pnpm --dir loro-js bench:complexity -- 1000,2000,4000,8000 ``` +Measure a real latest-state snapshot through import, a local Map edit, a remote +update import, full update export, and snapshot export with: + +```sh +pnpm --dir loro-js bench:snapshot-memory -- /path/to/document.snapshot +``` + +For the 11,387,982-byte ProCloud document with 423,797 operations and 115,147 +containers, Node 26.4.0 reports 70.92 MiB RSS after loading the input and a +160.70 MiB process peak after snapshot export: an 89.78 MiB incremental peak. +Used JS heap peaks at 8.58 MiB. Snapshot import takes about 0.90 seconds, the +local commit about 1.9 ms, full update export about 6.6 ms, and snapshot export +about 57 ms on the measured Apple M5 Pro. Before lazy state and history-overlay +integration, the same workflow retained roughly 703 MiB heap immediately after +import, exceeded 860 MiB after the first local edit, and reached roughly 1.66 GB +RSS during snapshot export. + +The ordinary fully materialized snapshot path remains neutral in a same-machine +A/B check. After three warmups, two 15-sample B4 snapshot-export runs measured +110.5/107.0 ms medians at the parent revision and 107.7/107.7 ms with lazy +snapshots. Both revisions emitted the same 309,780-byte snapshot. + On an Apple M5 Pro with Node 26.4.0, the complete 259,778-action B4 trace now applies in a 353.5 ms three-sample median (351.8–354.9 ms samples) and finishes at 104,852 UTF-16 code units. The resulting process reported 107.1 MB of used JS diff --git a/loro-js/benchmarks/snapshot-memory.mjs b/loro-js/benchmarks/snapshot-memory.mjs new file mode 100644 index 000000000..da28f646f --- /dev/null +++ b/loro-js/benchmarks/snapshot-memory.mjs @@ -0,0 +1,89 @@ +/* eslint-disable no-console */ + +import { readFileSync } from "node:fs"; +import process from "node:process"; + +import { LoroDoc } from "../dist/index.js"; + +const positionalArguments = process.argv.slice(2).filter((argument) => argument !== "--"); +const snapshotPath = positionalArguments[0]; +const rootName = positionalArguments[1] ?? "root"; +if (snapshotPath === undefined) { + throw new TypeError( + "usage: pnpm bench:snapshot-memory -- [root-map-name]", + ); +} + +const mib = 1024 * 1024; +const phases = []; +function measure(phase, extra = {}) { + globalThis.gc?.(); + const memory = process.memoryUsage(); + const sample = { + phase, + heapUsedMiB: memory.heapUsed / mib, + externalMiB: memory.external / mib, + rssMiB: memory.rss / mib, + maxRssMiB: process.resourceUsage().maxRSS / 1024, + ...extra, + }; + phases.push(sample); + console.log(JSON.stringify(sample)); +} + +function timed(callback) { + const start = performance.now(); + const value = callback(); + return { value, milliseconds: performance.now() - start }; +} + +const input = readFileSync(snapshotPath); +measure("input-loaded", { bytes: input.length }); +const baselineRss = phases[0].rssMiB; + +const doc = new LoroDoc(); +let result = timed(() => doc.import(input)); +measure("snapshot-imported", { milliseconds: result.milliseconds }); + +result = timed(() => { + doc.getMap(rootName).set("__loro_js_memory_bench_local", 1); + doc.commit(); +}); +measure("local-change-committed", { milliseconds: result.milliseconds }); + +const remote = new LoroDoc(); +remote.setPeerId(0xffff_ffffn); +remote.getMap(rootName).set("__loro_js_memory_bench_remote", 2); +remote.commit(); +const remoteUpdate = remote.export({ mode: "update" }); +result = timed(() => doc.import(remoteUpdate)); +measure("update-imported", { + milliseconds: result.milliseconds, + bytes: remoteUpdate.length, +}); + +result = timed(() => doc.export({ mode: "update" })); +const update = result.value; +measure("update-exported", { + milliseconds: result.milliseconds, + bytes: update.length, +}); + +result = timed(() => doc.export({ mode: "snapshot" })); +const snapshot = result.value; +measure("snapshot-exported", { + milliseconds: result.milliseconds, + bytes: snapshot.length, +}); + +const peakRss = process.resourceUsage().maxRSS / 1024; +console.log( + JSON.stringify({ + phase: "summary", + baselineRssMiB: baselineRss, + peakRssMiB: peakRss, + incrementalPeakRssMiB: peakRss - baselineRss, + peakHeapUsedMiB: Math.max(...phases.map(({ heapUsedMiB }) => heapUsedMiB)), + under100MiB: peakRss - baselineRss < 100, + }), +); diff --git a/loro-js/package.json b/loro-js/package.json index 12ae6541f..de53dc272 100644 --- a/loro-js/package.json +++ b/loro-js/package.json @@ -46,6 +46,7 @@ "scripts": { "bench:b4": "pnpm build && node --expose-gc benchmarks/b4.mjs", "bench:complexity": "pnpm build && node --expose-gc benchmarks/complexity.mjs", + "bench:snapshot-memory": "pnpm build && node --expose-gc benchmarks/snapshot-memory.mjs", "bench:subscribers": "pnpm build && node --expose-gc benchmarks/container-subscriber.mjs", "build": "vp pack", "check": "pnpm format:check && pnpm lint && pnpm typecheck && pnpm test && pnpm build", diff --git a/loro-js/src/codec/bytes.ts b/loro-js/src/codec/bytes.ts index 0dc8b3ad3..46f75e99e 100644 --- a/loro-js/src/codec/bytes.ts +++ b/loro-js/src/codec/bytes.ts @@ -182,7 +182,9 @@ export class ByteWriter { } toUint8Array(): Uint8Array { - return this.#buffer.slice(0, this.#length); + return this.#length === this.#buffer.length + ? this.#buffer + : this.#buffer.slice(0, this.#length); } private ensureCapacity(extra: number): void { diff --git a/loro-js/src/codec/document.ts b/loro-js/src/codec/document.ts index 1bd42d6cf..f147dd37a 100644 --- a/loro-js/src/codec/document.ts +++ b/loro-js/src/codec/document.ts @@ -63,16 +63,17 @@ export function encodeDocument(mode: EncodeMode, body: Uint8Array): Uint8Array { if (mode !== EncodeMode.FastSnapshot && mode !== EncodeMode.FastUpdates) { throw new LoroEncodeError(`unsupported document mode ${mode as number}`); } - const checksumInput = new ByteWriter(2 + body.length); - checksumInput.writeU16BE(mode); - checksumInput.writeBytes(body); - const checksumBytes = checksumInput.toUint8Array(); - const writer = new ByteWriter(DOCUMENT_HEADER_LENGTH + body.length); - writer.writeBytes(DOCUMENT_MAGIC); - writer.writeBytes(new Uint8Array(12)); - writer.writeU32LE(xxhash32(checksumBytes, LORO_XXHASH_SEED)); - writer.writeBytes(checksumBytes); - return writer.toUint8Array(); + const output = new Uint8Array(DOCUMENT_HEADER_LENGTH + body.length); + output.set(DOCUMENT_MAGIC, 0); + output[20] = mode >>> 8; + output[21] = mode & 0xff; + output.set(body, DOCUMENT_HEADER_LENGTH); + new DataView(output.buffer, output.byteOffset, output.byteLength).setUint32( + 16, + xxhash32(output.subarray(20), LORO_XXHASH_SEED), + true, + ); + return output; } export function decodeFastSnapshotBody(body: Uint8Array): FastSnapshotBody { @@ -85,13 +86,27 @@ export function decodeFastSnapshotBody(body: Uint8Array): FastSnapshotBody { } export function encodeFastSnapshotBody(snapshot: FastSnapshotBody): Uint8Array { - const writer = new ByteWriter( + for (const [value, label] of [ + [snapshot.oplog, "oplog"], + [snapshot.state, "state"], + [snapshot.shallowRootState, "shallow root state"], + ] as const) { + if (value.length > 0xffff_ffff) { + throw new LoroEncodeError(`${label} is too large`); + } + } + const output = new Uint8Array( 12 + snapshot.oplog.length + snapshot.state.length + snapshot.shallowRootState.length, ); - writeU32LengthPrefixed(writer, snapshot.oplog, "oplog"); - writeU32LengthPrefixed(writer, snapshot.state, "state"); - writeU32LengthPrefixed(writer, snapshot.shallowRootState, "shallow root state"); - return writer.toUint8Array(); + const view = new DataView(output.buffer, output.byteOffset, output.byteLength); + let offset = 0; + for (const value of [snapshot.oplog, snapshot.state, snapshot.shallowRootState]) { + view.setUint32(offset, value.length, true); + offset += 4; + output.set(value, offset); + offset += value.length; + } + return output; } export function decodeFastUpdatesBody(body: Uint8Array): Uint8Array[] { @@ -105,7 +120,11 @@ export function decodeFastUpdatesBody(body: Uint8Array): Uint8Array[] { } export function encodeFastUpdatesBody(blocks: readonly Uint8Array[]): Uint8Array { - const writer = new ByteWriter(); + const length = blocks.reduce( + (sum, block) => sum + ulebByteLength(block.length) + block.length, + 0, + ); + const writer = new ByteWriter(length); for (const block of blocks) { writeUleb128(writer, block.length); writer.writeBytes(block); @@ -113,6 +132,15 @@ export function encodeFastUpdatesBody(blocks: readonly Uint8Array[]): Uint8Array return writer.toUint8Array(); } +function ulebByteLength(value: number): number { + let length = 1; + while (value >= 0x80) { + value = Math.floor(value / 0x80); + length += 1; + } + return length; +} + export function decodeFastSnapshot( bytes: Uint8Array, options?: DecodeDocumentOptions, @@ -155,15 +183,3 @@ function readU32LengthPrefixed(reader: ByteReader, label: string): Uint8Array { } return reader.readBytes(length); } - -function writeU32LengthPrefixed( - writer: ByteWriter, - value: Uint8Array, - label: string, -): void { - if (value.length > 0xffff_ffff) { - throw new LoroEncodeError(`${label} is too large`); - } - writer.writeU32LE(value.length); - writer.writeBytes(value); -} diff --git a/loro-js/src/codec/sstable.ts b/loro-js/src/codec/sstable.ts index fe814fb6a..40cb96892 100644 --- a/loro-js/src/codec/sstable.ts +++ b/loro-js/src/codec/sstable.ts @@ -23,6 +23,11 @@ export interface EncodeSstableOptions { readonly compression?: SstableCompression; } +export interface SstableReplacement { + readonly key: Uint8Array; + readonly value: Uint8Array | undefined; +} + interface BlockMetadata { readonly offset: number; readonly large: boolean; @@ -43,58 +48,243 @@ export function decodeSstable( bytes: Uint8Array, options: DecodeSstableOptions = {}, ): SstableEntry[] { - if (bytes.length === 0) { - return []; - } - decodeAssert(bytes.length >= 17, "SSTable is too short", 0); - decodeAssert( - bytesEqual(bytes.subarray(0, 4), SSTABLE_MAGIC), - "invalid SSTable magic", - 0, - ); - decodeAssert(bytes[4] === SSTABLE_SCHEMA, "unsupported SSTable schema", 4); - const footer = new ByteReader(bytes, bytes.length - 4, 4); - const metadataOffset = footer.readU32LE(); - decodeAssert( - metadataOffset >= 5 && metadataOffset < bytes.length - 4, - "invalid SSTable metadata offset", - bytes.length - 4, - ); - const metadataBytes = bytes.subarray(metadataOffset, bytes.length - 4); - const metadata = decodeMetadata(metadataBytes, options.checkChecksum !== false); - const entries: SstableEntry[] = []; - for (let index = 0; index < metadata.length; index += 1) { - const current = metadata[index]!; - const end = - index + 1 < metadata.length ? metadata[index + 1]!.offset : metadataOffset; - decodeAssert(current.offset >= 5, "invalid SSTable block offset", current.offset); + if (bytes.length === 0) return []; + return [...new SstableReader(bytes, options).entries()]; +} + +/** + * A validated, low-retention view over an SSTable. + * + * Unlike `decodeSstable`, this keeps the encoded table and only retains one + * decompressed block while iterating or looking up an entry. This matters for + * latest-state snapshots containing hundreds of thousands of small containers. + */ +export class SstableReader { + readonly bytes: Uint8Array; + readonly #options: DecodeSstableOptions; + readonly #metadataOffset: number; + readonly #metadata: readonly BlockMetadata[]; + #validated = false; + + constructor(bytes: Uint8Array, options: DecodeSstableOptions = {}) { + decodeAssert(bytes.length >= 17, "SSTable is too short", 0); decodeAssert( - end > current.offset && end <= metadataOffset, - "invalid SSTable block range", - current.offset, + bytesEqual(bytes.subarray(0, 4), SSTABLE_MAGIC), + "invalid SSTable magic", + 0, ); - const stored = bytes.subarray(current.offset, end); - decodeAssert(stored.length >= 4, "SSTable block lacks checksum", current.offset); + decodeAssert(bytes[4] === SSTABLE_SCHEMA, "unsupported SSTable schema", 4); + const footer = new ByteReader(bytes, bytes.length - 4, 4); + const metadataOffset = footer.readU32LE(); + decodeAssert( + metadataOffset >= 5 && metadataOffset < bytes.length - 4, + "invalid SSTable metadata offset", + bytes.length - 4, + ); + const metadata = decodeMetadata( + bytes.subarray(metadataOffset, bytes.length - 4), + options.checkChecksum !== false, + ); + for (let index = 0; index < metadata.length; index += 1) { + const current = metadata[index]!; + const end = + index + 1 < metadata.length ? metadata[index + 1]!.offset : metadataOffset; + decodeAssert(current.offset >= 5, "invalid SSTable block offset", current.offset); + decodeAssert( + end > current.offset && end <= metadataOffset, + "invalid SSTable block range", + current.offset, + ); + } + this.bytes = bytes; + this.#options = options; + this.#metadataOffset = metadataOffset; + this.#metadata = metadata; + } + + validate(): void { + if (this.#validated) return; + for (const _entry of this.entries()) { + // Iteration validates checksums, compression, entry encoding, and order. + } + this.#validated = true; + } + + *entries(): IterableIterator { + let previous: Uint8Array | undefined; + for (let index = 0; index < this.#metadata.length; index += 1) { + for (const entry of this.#decodeBlock(index)) { + if (previous !== undefined) { + decodeAssert( + compareBytes(previous, entry.key) < 0, + "SSTable keys are not strictly increasing", + ); + } + previous = entry.key; + yield entry; + } + } + this.#validated = true; + } + + get(key: Uint8Array): Uint8Array | undefined { + this.validate(); + let low = 0; + let high = this.#metadata.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (compareBytes(this.#metadata[middle]!.firstKey, key) <= 0) low = middle + 1; + else high = middle; + } + const index = low - 1; + if (index < 0) return undefined; + const metadata = this.#metadata[index]!; + if (metadata.large) { + return bytesEqual(metadata.firstKey, key) + ? this.#decodeBlock(index)[0]!.value + : undefined; + } + if (metadata.lastKey === undefined || compareBytes(key, metadata.lastKey) > 0) { + return undefined; + } + const entries = this.#decodeBlock(index); + let entryLow = 0; + let entryHigh = entries.length; + while (entryLow < entryHigh) { + const middle = (entryLow + entryHigh) >>> 1; + if (compareBytes(entries[middle]!.key, key) < 0) entryLow = middle + 1; + else entryHigh = middle; + } + const entry = entries[entryLow]; + return entry !== undefined && bytesEqual(entry.key, key) ? entry.value : undefined; + } + + rewrite( + input: readonly SstableReplacement[], + options: EncodeSstableOptions = {}, + ): Uint8Array { + this.validate(); + const replacements = input.map(({ key, value }) => ({ + key: key.slice(), + value: value?.slice(), + })); + replacements.sort((left, right) => compareBytes(left.key, right.key)); + validateReplacementOrder(replacements); + if (replacements.length === 0) return this.bytes.slice(); + + const blockSize = checkedBlockSize(options.blockSize); + const compression = options.compression ?? "auto"; + const blocks: EncodedBlock[] = []; + let replacementIndex = 0; + const encodeReplacementRange = (end: number): void => { + const entries: SstableEntry[] = []; + while (replacementIndex < end) { + const replacement = replacements[replacementIndex++]!; + if (replacement.value !== undefined) { + entries.push({ key: replacement.key, value: replacement.value }); + } + } + blocks.push(...encodeBlocks(entries, blockSize, compression)); + }; + + for (let blockIndex = 0; blockIndex < this.#metadata.length; blockIndex += 1) { + const metadata = this.#metadata[blockIndex]!; + let beforeEnd = replacementIndex; + while ( + beforeEnd < replacements.length && + compareBytes(replacements[beforeEnd]!.key, metadata.firstKey) < 0 + ) { + beforeEnd += 1; + } + encodeReplacementRange(beforeEnd); + + const upperKey = metadata.large ? metadata.firstKey : metadata.lastKey!; + let rangeEnd = replacementIndex; + while ( + rangeEnd < replacements.length && + compareBytes(replacements[rangeEnd]!.key, upperKey) <= 0 + ) { + rangeEnd += 1; + } + if (rangeEnd === replacementIndex) { + blocks.push(this.#storedBlock(blockIndex)); + continue; + } + + const source = this.#decodeBlock(blockIndex); + const merged: SstableEntry[] = []; + let sourceIndex = 0; + while (sourceIndex < source.length || replacementIndex < rangeEnd) { + const sourceEntry = source[sourceIndex]; + const replacement = replacements[replacementIndex]; + const order = + sourceEntry === undefined + ? 1 + : replacement === undefined || replacementIndex >= rangeEnd + ? -1 + : compareBytes(sourceEntry.key, replacement.key); + if (order < 0) { + merged.push(sourceEntry!); + sourceIndex += 1; + } else if (order > 0) { + if (replacement!.value !== undefined) { + merged.push({ key: replacement!.key, value: replacement!.value }); + } + replacementIndex += 1; + } else { + if (replacement!.value !== undefined) { + merged.push({ key: replacement!.key, value: replacement!.value }); + } + sourceIndex += 1; + replacementIndex += 1; + } + } + blocks.push(...encodeBlocks(merged, blockSize, compression)); + } + encodeReplacementRange(replacements.length); + return blocks.length === 0 ? new Uint8Array() : encodeTable(blocks); + } + + #decodeBlock(index: number): SstableEntry[] { + const metadata = this.#metadata[index]!; + const stored = this.#storedBytes(index); const payload = stored.subarray(0, stored.length - 4); const checksum = new ByteReader(stored, stored.length - 4, 4).readU32LE(); - if (options.checkChecksum !== false) { + if (this.#options.checkChecksum !== false) { decodeAssert( checksum === xxhash32(payload, LORO_XXHASH_SEED), "SSTable block checksum mismatch", - end - 4, + metadata.offset + stored.length - 4, ); } const decoded = - current.compression === 0 ? payload : decodeLz4Frame(payload, options); - if (current.large) { - entries.push({ key: current.firstKey, value: decoded }); - continue; - } - const blockEntries = decodeNormalBlock(decoded, current); - entries.push(...blockEntries); + metadata.compression === 0 ? payload : decodeLz4Frame(payload, this.#options); + return metadata.large + ? [{ key: metadata.firstKey, value: decoded }] + : decodeNormalBlock(decoded, metadata); + } + + #storedBytes(index: number): Uint8Array { + const metadata = this.#metadata[index]!; + const end = + index + 1 < this.#metadata.length + ? this.#metadata[index + 1]!.offset + : this.#metadataOffset; + const stored = this.bytes.subarray(metadata.offset, end); + decodeAssert(stored.length >= 4, "SSTable block lacks checksum", metadata.offset); + return stored; + } + + #storedBlock(index: number): EncodedBlock { + const metadata = this.#metadata[index]!; + return { + bytes: this.#storedBytes(index), + large: metadata.large, + compression: metadata.compression, + firstKey: metadata.firstKey, + lastKey: metadata.lastKey, + }; } - validateEntryOrder(entries); - return entries; } export function encodeSstable( @@ -104,17 +294,22 @@ export function encodeSstable( if (input.length === 0) { return new Uint8Array(); } - const blockSize = options.blockSize ?? 4096; + const blockSize = checkedBlockSize(options.blockSize); const compression = options.compression ?? "auto"; - if (!Number.isSafeInteger(blockSize) || blockSize <= 0 || blockSize > 0xffff) { - throw new LoroEncodeError(`invalid SSTable block size ${blockSize}`); - } const entries = input.map(({ key, value }) => ({ - key: key.slice(), - value: value.slice(), + key, + value, })); entries.sort((left, right) => compareBytes(left.key, right.key)); validateEntriesForEncoding(entries); + return encodeTable(encodeBlocks(entries, blockSize, compression)); +} + +function encodeBlocks( + entries: readonly SstableEntry[], + blockSize: number, + compression: SstableCompression, +): EncodedBlock[] { const blocks: EncodedBlock[] = []; for (let index = 0; index < entries.length; ) { const first = entries[index]!; @@ -164,7 +359,15 @@ export function encodeSstable( encodeStoredBlock(body.toUint8Array(), false, firstKey, lastKey, compression), ); } - return encodeTable(blocks); + return blocks; +} + +function checkedBlockSize(value: number | undefined): number { + const blockSize = value ?? 4096; + if (!Number.isSafeInteger(blockSize) || blockSize <= 0 || blockSize > 0xffff) { + throw new LoroEncodeError(`invalid SSTable block size ${blockSize}`); + } + return blockSize; } function decodeMetadata(bytes: Uint8Array, checkChecksum: boolean): BlockMetadata[] { @@ -298,7 +501,14 @@ function encodeTable(blocks: readonly EncodedBlock[]): Uint8Array { encodeAssert(offset <= 0xffff_ffff, "SSTable exceeds u32 offsets"); } const metadataOffset = offset; - const metadataWriter = new ByteWriter(); + const metadataLength = + 8 + + metadata.reduce( + (length, item) => + length + 7 + item.firstKey.length + (item.large ? 0 : 2 + item.lastKey!.length), + 0, + ); + const metadataWriter = new ByteWriter(metadataLength); metadataWriter.writeU32LE(metadata.length); for (const item of metadata) { encodeAssert(item.firstKey.length <= 0xffff, "SSTable key exceeds u16 length"); @@ -317,7 +527,7 @@ function encodeTable(blocks: readonly EncodedBlock[]): Uint8Array { metadataWriter.writeU32LE( xxhash32(metadataWithoutChecksum.subarray(4), LORO_XXHASH_SEED), ); - const writer = new ByteWriter(); + const writer = new ByteWriter(metadataOffset + metadataLength + 4); writer.writeBytes(SSTABLE_MAGIC); writer.writeU8(SSTABLE_SCHEMA); for (const block of blocks) { @@ -352,6 +562,21 @@ function validateEntriesForEncoding(entries: readonly SstableEntry[]): void { } } +function validateReplacementOrder(replacements: readonly SstableReplacement[]): void { + for (let index = 0; index < replacements.length; index += 1) { + const current = replacements[index]!; + if (current.key.length === 0) { + throw new LoroEncodeError("SSTable keys cannot be empty"); + } + if (current.key.length > 0xffff) { + throw new LoroEncodeError("SSTable key exceeds u16 length"); + } + if (index > 0 && compareBytes(replacements[index - 1]!.key, current.key) >= 0) { + throw new LoroEncodeError("SSTable replacement keys must be unique"); + } + } +} + function validateEntryOrder(entries: readonly SstableEntry[]): void { for (let index = 1; index < entries.length; index += 1) { if (compareBytes(entries[index - 1]!.key, entries[index]!.key) >= 0) { diff --git a/loro-js/src/codec/state-snapshot.ts b/loro-js/src/codec/state-snapshot.ts index 6b9add504..5ade21265 100644 --- a/loro-js/src/codec/state-snapshot.ts +++ b/loro-js/src/codec/state-snapshot.ts @@ -28,8 +28,10 @@ import { import { decodeSstable, encodeSstable, + SstableReader, type DecodeSstableOptions, type EncodeSstableOptions, + type SstableReplacement, } from "./sstable"; import { ContainerType, @@ -183,6 +185,21 @@ export type StateSnapshotStore = readonly containers: readonly StateSnapshotContainerEntry[]; }; +export type LazyStateSnapshotStore = + | { readonly kind: "absent" } + | { readonly kind: "empty" } + | { + readonly kind: "sstable"; + readonly frontiers: Frontiers | undefined; + readonly roots: readonly ContainerId[]; + readonly table: SstableReader; + }; + +export interface LazyStateSnapshotReplacement { + readonly id: ContainerId; + readonly wrapper: ContainerStateWrapper | undefined; +} + export function decodeMapStateSnapshot(bytes: Uint8Array): MapStateSnapshot { const reader = new PostcardReader(bytes); const values = readPostcardValueMap(reader); @@ -665,6 +682,77 @@ export function decodeStateSnapshotStore( return { kind: "sstable", frontiers, containers }; } +/** + * Validates a state snapshot without retaining every decoded container. + * Individual container states can then be decoded by key as they are accessed. + */ +export function decodeLazyStateSnapshotStore( + bytes: Uint8Array, + options?: DecodeSstableOptions, +): LazyStateSnapshotStore { + if (bytes.length === 0) return { kind: "absent" }; + if (bytesEqual(bytes, EMPTY_STATE_SENTINEL)) return { kind: "empty" }; + + const table = new SstableReader(bytes, options); + const roots: ContainerId[] = []; + let frontiers: Frontiers | undefined; + for (const entry of table.entries()) { + if (bytesEqual(entry.key, FRONTIERS_KEY)) { + decodeAssert(frontiers === undefined, "duplicate state frontiers entry"); + frontiers = decodePostcardFrontiers(entry.value); + continue; + } + const id = decodeContainerId(entry.key); + const wrapper = decodeContainerStateWrapper(entry.value); + decodeAssert( + sameContainerType(id.containerType, wrapper.containerType), + "state container key and wrapper types differ", + ); + if (id.kind === "root" && wrapper.parent === undefined) roots.push(id); + } + table.validate(); + return { kind: "sstable", frontiers, roots, table }; +} + +export function getLazyStateSnapshotContainer( + store: LazyStateSnapshotStore, + id: ContainerId, +): StateSnapshotContainerEntry | undefined { + if (store.kind !== "sstable") return undefined; + const bytes = store.table.get(encodeContainerId(id)); + if (bytes === undefined) return undefined; + const wrapper = decodeContainerStateWrapper(bytes); + decodeAssert( + sameContainerType(id.containerType, wrapper.containerType), + "state container key and wrapper types differ", + ); + return { id, wrapper }; +} + +export function rewriteLazyStateSnapshotStore( + store: LazyStateSnapshotStore, + replacements: readonly LazyStateSnapshotReplacement[], + options?: EncodeSstableOptions, +): Uint8Array { + if (store.kind !== "sstable") { + const containers = replacements.flatMap(({ id, wrapper }) => + wrapper === undefined ? [] : [{ id, wrapper }], + ); + return encodeStateSnapshotStore( + containers.length === 0 + ? store + : { kind: "sstable", frontiers: undefined, containers }, + options, + ); + } + const encoded: SstableReplacement[] = replacements.map(({ id, wrapper }) => ({ + key: encodeContainerId(id), + value: wrapper === undefined ? undefined : encodeContainerStateWrapper(wrapper), + })); + const rewritten = store.table.rewrite(encoded, options); + return rewritten.length === 0 ? EMPTY_STATE_SENTINEL.slice() : rewritten; +} + export function encodeStateSnapshotStore( store: StateSnapshotStore, options?: EncodeSstableOptions, diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index afff30f04..9e0bd2860 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -136,6 +136,10 @@ export abstract class LoroContainer { ): void { this._parentLink = { container: parent, binding }; } + + _ensureHydrated(): void { + this._doc?._ensureContainerHydrated(this); + } } export interface MapRecord { @@ -165,6 +169,7 @@ export class LoroMap< get(key: Key): T[Key] | undefined; get(key: string): unknown; get(key: string): unknown { + this._ensureHydrated(); const record = this._entries.get(key); return record === undefined || record.deleted ? undefined @@ -174,6 +179,7 @@ export class LoroMap< set(key: Key, value: T[Key]): void; set(key: string, value: unknown): void; set(key: string, value: unknown): void { + this._ensureHydrated(); if (isContainer(value)) { throw new TypeError("use setContainer() to attach a child container"); } @@ -189,6 +195,7 @@ export class LoroMap< } delete(key: string): void { + this._ensureHydrated(); if (this._doc === undefined) { if (this._entries.has(key)) this._removeVisibleKey(key); this._entries.delete(key); @@ -202,6 +209,7 @@ export class LoroMap< } keys(): string[] { + this._ensureHydrated(); return this._keyIndex.values().map(({ key }) => key); } @@ -214,10 +222,12 @@ export class LoroMap< } get size(): number { + this._ensureHydrated(); return this._keyIndex.size; } setContainer(key: string, child: C): C { + this._ensureHydrated(); if (this._doc === undefined) { this._applyValue(key, child, { peer: 0n, lamport: 0 }); return child; @@ -261,6 +271,7 @@ export class LoroMap< } getLastEditor(key: string): string | undefined { + this._ensureHydrated(); return this._entries.get(key)?.writer.peer.toString(); } @@ -343,6 +354,7 @@ export class LoroMap< } private _ensureMergeable(key: string, type: ContainerType): Container { + this._ensureHydrated(); if (this._doc === undefined) { throw new Error("cannot ensure a mergeable child on a detached map"); } @@ -355,6 +367,7 @@ export class LoroList extends LoroContainer { _detachedCounter = 0; get _elements(): SequenceElement[] { + this._ensureHydrated(); return this._sequence.all(); } @@ -363,10 +376,12 @@ export class LoroList extends LoroContainer { } get length(): number { + this._ensureHydrated(); return this._sequence.visibleLength; } get(index: number): T | undefined { + this._ensureHydrated(); return cloneRuntimeValue(this._sequence.atVisible(index)?.value) as T | undefined; } @@ -460,20 +475,24 @@ export class LoroList extends LoroContainer { } _visibleElements(): SequenceElement[] { + this._ensureHydrated(); return this._sequence.visible(); } _visibleElementsRange(start: number, end: number): SequenceElement[] { + this._ensureHydrated(); return this._sequence.visibleRange(start, end); } _valuesRange(start: number, end: number): unknown[] { + this._ensureHydrated(); return this._sequence .visibleRange(start, end) .map((element) => cloneRuntimeValue(element.value)); } _visibleElementAt(position: number): SequenceElement | undefined { + this._ensureHydrated(); return this._sequence.atVisible(position); } @@ -1198,6 +1217,7 @@ export class LoroText extends LoroContainer { >(); get _elements(): TextElement[] { + this._ensureHydrated(); return this._sequence.all(); } @@ -1206,14 +1226,17 @@ export class LoroText extends LoroContainer { } get length(): number { + this._ensureHydrated(); return this._sequence.visibleUtf16Length; } toString(): string { + this._ensureHydrated(); return this._stringRange(0, this._sequence.visibleLength); } _stringRange(start: number, end: number): string { + this._ensureHydrated(); const chunks: string[] = []; let chunk = ""; let chunkLength = 0; @@ -1231,6 +1254,7 @@ export class LoroText extends LoroContainer { } iter(callback: (chunk: string) => boolean | void | null): void { + this._ensureHydrated(); let chunk: string[] = []; let previous: TextElement | undefined; let stopped = false; @@ -1387,6 +1411,7 @@ export class LoroText extends LoroContainer { } sliceDelta(start: number, end: number): Delta[] { + this._ensureHydrated(); validateRange(start, end - start, this.length); const unicodeStart = this._unicodePosition(start); const unicodeEnd = this._unicodePosition(end); @@ -1397,6 +1422,7 @@ export class LoroText extends LoroContainer { } sliceDeltaUtf8(start: number, end: number): Delta[] { + this._ensureHydrated(); const utf16Start = this.convertPos(start, "utf8", "utf16"); const utf16End = this.convertPos(end, "utf8", "utf16"); if (utf16Start === undefined || utf16End === undefined) { @@ -1406,6 +1432,7 @@ export class LoroText extends LoroContainer { } convertPos(index: number, from: TextPosType, to: TextPosType): number | undefined { + this._ensureHydrated(); if (!isTextPosType(from) || !isTextPosType(to)) return undefined; const visibleLength = this._sequence.visibleLength; const directMetric = @@ -1467,14 +1494,17 @@ export class LoroText extends LoroContainer { } _visibleElements(): TextElement[] { + this._ensureHydrated(); return this._sequence.visible(); } _visibleElementsRange(start: number, end: number): TextElement[] { + this._ensureHydrated(); return this._sequence.visibleRange(start, end); } _visibleElementAt(position: number): TextElement | undefined { + this._ensureHydrated(); return this._sequence.atVisible(position); } @@ -1668,6 +1698,7 @@ export class LoroText extends LoroContainer { } _validateInsertPosition(position: number): number { + this._ensureHydrated(); const unicodePosition = this.convertPos(position, "utf16", "unicode"); if (unicodePosition === undefined) { throw new RangeError(`text position ${position} is out of range`); @@ -1676,6 +1707,7 @@ export class LoroText extends LoroContainer { } _unicodePosition(position: number): number { + this._ensureHydrated(); const unicodePosition = this.convertPos(position, "utf16", "unicode"); if (unicodePosition === undefined) { throw new RangeError(`text position ${position} is not on a UTF-16 boundary`); @@ -1865,6 +1897,7 @@ export class LoroCounter extends LoroContainer { } increment(value: number): void { + this._ensureHydrated(); if (!Number.isFinite(value)) throw new TypeError("counter increment must be finite"); if (this._doc === undefined) { this._value += value; @@ -1878,14 +1911,17 @@ export class LoroCounter extends LoroContainer { } get value(): number { + this._ensureHydrated(); return this._value; } getValue(): number { + this._ensureHydrated(); return this._value; } toJSON(): number { + this._ensureHydrated(); return this._value; } @@ -1929,23 +1965,27 @@ export class LoroTree< } createNode(parent?: TreeID, index?: number): LoroTreeNode { + this._ensureHydrated(); if (this._doc === undefined) throw new Error("tree nodes can only be created on an attached tree"); return this._doc._treeCreate(this, parent, index); } move(target: TreeID, parent?: TreeID, index?: number): void { + this._ensureHydrated(); if (this._doc === undefined) throw new Error("tree nodes can only be moved on an attached tree"); this._doc._treeMove(this, target, parent, index); } delete(target: TreeID): void { + this._ensureHydrated(); if (this._doc === undefined) return; this._doc._treeDelete(this, target); } has(target: TreeID): boolean { + this._ensureHydrated(); return this._nodes.has(target); } @@ -1954,6 +1994,7 @@ export class LoroTree< } isNodeDeleted(target: TreeID): boolean { + this._ensureHydrated(); return this._nodes.get(target)?.deleted ?? false; } @@ -1973,11 +2014,13 @@ export class LoroTree< } getNodeByID(target: TreeID): LoroTreeNode | undefined { + this._ensureHydrated(); const record = this._nodes.get(target); return record === undefined ? undefined : new LoroTreeNode(this, record.id); } getNodes(options: { withDeleted?: boolean } = {}): LoroTreeNode[] { + this._ensureHydrated(); return [...this._nodes.values()] .filter((record) => options.withDeleted === true || !record.deleted) .map((record) => new LoroTreeNode(this, record.id)); @@ -1988,18 +2031,21 @@ export class LoroTree< } roots(): LoroTreeNode[] { + this._ensureHydrated(); return this._childrenOf(undefined).map( (record) => new LoroTreeNode(this, record.id), ); } toArray(): TreeNodeValue[] { + this._ensureHydrated(); return this._childrenOf(undefined).map( (record, index) => this._recordToNodeValue(record, index) as TreeNodeValue, ); } toJSON(): TreeJsonValue[] { + this._ensureHydrated(); return this._childrenOf(undefined).map( (record, index) => this._recordToValue(record, index) as TreeJsonValue, ); @@ -2012,6 +2058,7 @@ export class LoroTree< } _childrenOf(parent: CodecId | undefined): TreeNodeRecord[] { + this._ensureHydrated(); return this._children.get(treeParentKey(parent))?.values() ?? []; } diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index 93a19dec8..b9ad35699 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -24,9 +24,13 @@ import { import { decodeChangeBlockKey, encodeChangeBlockKey } from "../codec/id"; import { decodeSstable, encodeSstable, type SstableEntry } from "../codec/sstable"; import { + decodeLazyStateSnapshotStore, decodeStateSnapshotStore, encodeStateSnapshotStore, + getLazyStateSnapshotContainer, + rewriteLazyStateSnapshotStore, type ContainerStateSnapshot, + type LazyStateSnapshotStore, type MapStateMetadata, type StateSnapshotContainerEntry, type StateSnapshotStore, @@ -158,12 +162,16 @@ interface DecodedImportData { interface DeferredSnapshotHistory { readonly entries: readonly SstableEntry[]; - readonly validatedBlocks: ReadonlyMap; + readonly validatedBlocks: Map; readonly endVersion: VersionVector; readonly frontiers: readonly CodecId[]; readonly operationCount: number; } +interface DeferredSnapshotState { + readonly store: Extract; +} + interface IndexedHistoryOperation { readonly record: HistoryRecord; readonly operation: DecodedOperation; @@ -246,6 +254,12 @@ export class LoroDoc = Record(); #pendingHistory = new Map(); #deferredSnapshotHistory: DeferredSnapshotHistory | undefined; + #deferredSnapshotState: DeferredSnapshotState | undefined; + #hydratedSnapshotContainers = new Set(); + #hydratingSnapshotContainers = new Set(); + #snapshotContainerDepths = new Map(); + #dirtySnapshotContainers = new Set(); + #deletedSnapshotContainers = new Map(); #containers = new Map(); #roots = new Map(); #pending: PendingChange | undefined; @@ -376,7 +390,16 @@ export class LoroDoc = Record = Record = Record = Record = Record = Record = Record formatContainerId(id)), - ); - if (this.#hasEventSubscribers()) { - beforeValues = this.#captureContainerEventValues(deferredChanged); + if (lazyStateStore?.kind === "sstable") { + this.#deferredSnapshotState = { store: lazyStateStore }; + deferredChanged = new Set(lazyStateStore.roots.map(formatContainerId)); + for (const root of lazyStateStore.roots) this.#getOrCreateContainer(root); + } else { + if (hydratedStore?.kind !== "sstable") { + throw new Error("deferred snapshot state must be an SSTable"); + } + deferredChanged = new Set( + hydratedStore.containers.map(({ id }) => formatContainerId(id)), + ); + if (this.#hasEventSubscribers()) { + beforeValues = this.#captureContainerEventValues(deferredChanged); + } + this.#hydrateState(hydratedStore); } - this.#hydrateState(hydratedStore); this.#deferredSnapshotHistory = { entries: oplogEntries, validatedBlocks, @@ -753,6 +811,9 @@ export class LoroDoc = Record [idKey(id), { ...id }] as const), + ); for (const { peer } of endVersion._codecEntriesUnsorted()) { this.#seenCommittedPeers.add(peer); } @@ -778,10 +839,10 @@ export class LoroDoc = Record 0) { const recording = this.#hasEventSubscribers() @@ -1352,10 +1413,9 @@ export class LoroDoc = Record (version.get(peer) ?? 0)) version.set(peer, end); } @@ -1367,11 +1427,6 @@ export class LoroDoc = Record = Record { @@ -1581,15 +1638,17 @@ export class LoroDoc = Record = Record = Record = Record = Record = Record = Record = Record = Record = Record = Record = Record = Record { + ): Map { const changeEntriesByPeer = new Map< bigint, { readonly entry: SstableEntry; readonly start: CodecId }[] @@ -4737,9 +4849,51 @@ export class LoroDoc = Record= (deferred.endVersion.get(id.peer) ?? 0) + ) { + return undefined; + } + let candidate: SstableEntry | undefined; + let candidateStart = -1; + for (const entry of deferred.entries) { + if (entry.key.length !== 12) continue; + const start = decodeChangeBlockKey(entry.key); + if ( + start.peer === id.peer && + start.counter <= id.counter && + start.counter > candidateStart + ) { + candidate = entry; + candidateStart = start.counter; + } + } + if (candidate === undefined) return undefined; + let records = deferred.validatedBlocks.get(candidate); + if (records === undefined) { + records = this.#readChangeBlock(candidate.value); + const expected = decodeChangeBlockKey(candidate.key); + if (records[0] !== undefined && !idsEqual(records[0].change.id, expected)) { + throw new Error("snapshot change key does not match its block"); + } + deferred.validatedBlocks.set(candidate, records); + } + return records.find( + ({ change }) => + change.id.peer === id.peer && + change.id.counter <= id.counter && + id.counter < change.id.counter + changeLength(change), + ); + } + #materializeDeferredHistory(): void { const deferred = this.#deferredSnapshotHistory; if (deferred === undefined) return; + const overlay = this.#historyOrder.values().map(cloneHistoryRecord); const records: HistoryRecord[] = []; for (const entry of deferred.entries) { @@ -4775,6 +4929,11 @@ export class LoroDoc = Record 0) { + throw new Error("snapshot overlay contains changes with missing dependencies"); + } + // State was already hydrated from the latest-state SSTable. Installing these // structures restores only history/DAG indexes; applying records would // duplicate counters and sequence content. @@ -4915,8 +5074,28 @@ export class LoroDoc = Record { + if ( + this.#deferredSnapshotHistory !== undefined && + frontierSetsEqual( + frontiers.map(formatOpId), + [...this.#historyFrontiers.values()].map(formatOpId), + ) + ) { + return new Map( + this.#historyVersion() + ._codecEntriesUnsorted() + .map(({ peer, counter }) => [peer, counter] as const), + ); + } const version = new Map( - this.#shallowStartVersion + (this.#deferredSnapshotHistory !== undefined && + frontierSetsEqual( + frontiers.map(formatOpId), + this.#deferredSnapshotHistory.frontiers.map(formatOpId), + ) + ? this.#deferredSnapshotHistory.endVersion + : this.#shallowStartVersion + ) ._codecEntriesUnsorted() .map(({ peer, counter }) => [peer, counter] as const), ); @@ -4935,11 +5114,24 @@ export class LoroDoc = Record { const cached = this.#dependencyVersionCache.get(change); if (cached !== undefined) return cached; + const deferredBase = + this.#deferredSnapshotHistory !== undefined && + frontierSetsEqual( + change.dependencies.map(formatOpId), + this.#deferredSnapshotHistory.frontiers.map(formatOpId), + ); const version = new Map( - this.#shallowStartVersion + (deferredBase + ? this.#deferredSnapshotHistory!.endVersion + : this.#shallowStartVersion + ) ._codecEntriesUnsorted() .map(({ peer, counter }) => [peer, counter] as const), ); + if (deferredBase) { + this.#dependencyVersionCache.set(change, version); + return version; + } for (const dependency of change.dependencies) { const dependencyRecord = this.#recordContaining(dependency); if (dependencyRecord !== undefined && dependencyRecord.change !== change) { @@ -4991,10 +5183,6 @@ export class LoroDoc = Record = Record entry.key.length === 12) + .map((entry) => entry.value); + for (const record of this.#historyOrder.values()) { + blocks.push( + encodeChangeBlock({ + peers: [record.change.id.peer], + keys: record.keys, + containers: [], + positions: [], + changes: [record.change], + }), + ); + } + return encodeDocument(EncodeMode.FastUpdates, encodeFastUpdatesBody(blocks)); + } + #encodeSnapshot(): Uint8Array { if (this.isShallow()) { return this.#encodeShallowSnapshot(this.shallowSinceFrontiers()); } + if ( + this.#deferredSnapshotHistory !== undefined && + this.#deferredSnapshotState !== undefined + ) { + const historyEntries = this.#deferredSnapshotHistory.entries + .filter( + (entry) => + !bytesEqual(entry.key, VERSION_KEY) && !bytesEqual(entry.key, FRONTIERS_KEY), + ) + .map(({ key, value }) => ({ key, value })); + for (const record of this.#historyOrder.values()) { + historyEntries.push({ + key: encodeChangeBlockKey(record.change.id), + value: encodeChangeBlock({ + peers: [record.change.id.peer], + keys: record.keys, + containers: [], + positions: [], + changes: [record.change], + }), + }); + } + historyEntries.push( + { + key: VERSION_KEY, + value: encodePostcardVersionVector(this.version().codecEntries()), + }, + { + key: FRONTIERS_KEY, + value: encodePostcardFrontiers(this.#frontiersCodec()), + }, + ); + const body = encodeFastSnapshotBody({ + oplog: encodeSstable(historyEntries, { compression: "auto" }), + state: this.#encodeDeferredSnapshotState(), + shallowRootState: new Uint8Array(), + }); + return encodeDocument(EncodeMode.FastSnapshot, body); + } const historyEntries = this.#sortedHistory().map((record) => ({ key: encodeChangeBlockKey(record.change.id), value: encodeChangeBlock({ @@ -5062,6 +5309,49 @@ export class LoroDoc = Record = Record [item.key, item])); - for (const [key, value] of state.values) { - const item = metadata.get(key)!; - const rawValue = this.#decodeSnapshotValue(value, container); - container._applyValue( - key, - this.#materializeMapValue(container, key, rawValue), - { - peer: state.peers[Number(item.peerIndex)]!, - lamport: Number(item.lamport), - }, - rawValue, - ); - } - for (const key of state.deletedKeys) { - const item = metadata.get(key)!; - container._applyDelete(key, { + const container = this.#getOrCreateContainer(id, undefined, false); + this.#hydrateContainerState(container, wrapper.state); + } + } + + #hydrateContainerState(container: LoroContainer, state: ContainerStateSnapshot): void { + if (container instanceof LoroMap && state.kind === CodecContainerType.Map) { + const metadata = new Map(state.metadata.map((item) => [item.key, item])); + for (const [key, value] of state.values) { + const item = metadata.get(key)!; + const rawValue = this.#decodeSnapshotValue(value, container); + container._applyValue( + key, + this.#materializeMapValue(container, key, rawValue, false), + { peer: state.peers[Number(item.peerIndex)]!, lamport: Number(item.lamport), - }); + }, + rawValue, + ); + } + for (const key of state.deletedKeys) { + const item = metadata.get(key)!; + container._applyDelete(key, { + peer: state.peers[Number(item.peerIndex)]!, + lamport: Number(item.lamport), + }); + } + } else if (container instanceof LoroText && state.kind === CodecContainerType.Text) { + const characters = Array.from(state.text); + const ids: CodecId[] = []; + const lamports: number[] = []; + const styleRuns: { + readonly run: { readonly start: CodecId; readonly length: number }; + readonly key: string; + readonly meta: TextStyleMeta; + }[] = []; + const stylesById = new Map(); + const active = new Map(); + let characterIndex = 0; + let markIndex = 0; + for (const span of state.spans) { + const peer = state.peers[Number(span.peerIndex)]!; + if (span.length === 0) { + const mark = state.marks[markIndex++]!; + const key = state.keys[mark.keyIndex]!; + const value = this.#decodeSnapshotValue(mark.value); + const meta: TextStyleMeta = { + startId: { peer, counter: span.counter }, + lamport: span.counter + span.lamportSub, + info: mark.info, + value, + }; + stylesById.set(idKey(meta.startId), { key, meta }); + const stack = active.get(key) ?? []; + stack.push(meta); + active.set(key, stack); + continue; } - } else if ( - container instanceof LoroText && - state.kind === CodecContainerType.Text - ) { - const characters = Array.from(state.text); - const ids: CodecId[] = []; - const lamports: number[] = []; - const styleRuns: { - readonly run: { readonly start: CodecId; readonly length: number }; - readonly key: string; - readonly meta: TextStyleMeta; - }[] = []; - const stylesById = new Map(); - const active = new Map(); - let characterIndex = 0; - let markIndex = 0; - for (const span of state.spans) { - const peer = state.peers[Number(span.peerIndex)]!; - if (span.length === 0) { - const mark = state.marks[markIndex++]!; - const key = state.keys[mark.keyIndex]!; - const value = this.#decodeSnapshotValue(mark.value); - const meta: TextStyleMeta = { - startId: { peer, counter: span.counter }, - lamport: span.counter + span.lamportSub, - info: mark.info, - value, - }; - stylesById.set(idKey(meta.startId), { key, meta }); - const stack = active.get(key) ?? []; - stack.push(meta); - active.set(key, stack); - continue; - } - if (span.length === -1) { - const style = stylesById.get(idKey({ peer, counter: span.counter - 1 })); - if (style !== undefined) { - const stack = active.get(style.key); - if (stack !== undefined) { - const index = stack.lastIndexOf(style.meta); - if (index >= 0) stack.splice(index, 1); - if (stack.length === 0) active.delete(style.key); - } + if (span.length === -1) { + const style = stylesById.get(idKey({ peer, counter: span.counter - 1 })); + if (style !== undefined) { + const stack = active.get(style.key); + if (stack !== undefined) { + const index = stack.lastIndexOf(style.meta); + if (index >= 0) stack.splice(index, 1); + if (stack.length === 0) active.delete(style.key); } - continue; } - if (span.length < -1) continue; - for (const [key, stack] of active) { - const meta = stack.at(-1); - if (meta !== undefined) { - styleRuns.push({ - run: { start: { peer, counter: span.counter }, length: span.length }, - key, - meta, - }); - } - } - for (let offset = 0; offset < span.length; offset += 1) { - ids.push({ peer, counter: span.counter + offset }); - lamports.push(span.counter + offset + span.lamportSub); - characterIndex += 1; - } - } - container._insertVisible(0, characters.slice(0, characterIndex), ids, lamports); - for (const { run, key, meta } of styleRuns) { - container._styleIndex.add([run], key, meta); - } - container._attributeHistoryComplete = false; - } else if ( - container instanceof LoroTree && - state.kind === CodecContainerType.Tree - ) { - const records: TreeNodeRecord[] = []; - for (let index = 0; index < state.nodes.length; index += 1) { - const node = state.nodes[index]!; - const nodeId = { - peer: state.peers[Number(node.peerIndex)]!, - counter: node.counter, - }; - const dataId: CodecContainerId = { - kind: "normal", - ...nodeId, - containerType: CodecContainerType.Map, - }; - const record: TreeNodeRecord = { - id: nodeId, - parent: undefined, - position: state.positions[node.fractionalIndexIndex]!.slice(), - deleted: node.parentIndexPlusTwo === 1n, - writer: { - peer: state.peers[Number(node.lastSetPeerIndex)]!, - lamport: node.lastSetCounter + node.lastSetLamportSub, - }, - lastMoveId: { - peer: state.peers[Number(node.lastSetPeerIndex)]!, - counter: node.lastSetCounter, - }, - data: this.#getOrCreateContainer(dataId, container) as LoroMap, - }; - records.push(record); + continue; } - for (let index = 0; index < state.nodes.length; index += 1) { - const parent = state.nodes[index]!.parentIndexPlusTwo; - if (parent >= 2n) records[index]!.parent = records[Number(parent - 2n)]!.id; + if (span.length < -1) continue; + for (const [key, stack] of active) { + const meta = stack.at(-1); + if (meta !== undefined) { + styleRuns.push({ + run: { start: { peer, counter: span.counter }, length: span.length }, + key, + meta, + }); + } } - for (const record of records) container._setRecord(record); - } else if ( - container instanceof LoroCounter && - state.kind === CodecContainerType.Counter - ) { - const bytes = new Uint8Array(8); - let bits = state.bits; - for (let index = 0; index < 8; index += 1) { - bytes[index] = Number(bits & 0xffn); - bits >>= 8n; + for (let offset = 0; offset < span.length; offset += 1) { + ids.push({ peer, counter: span.counter + offset }); + lamports.push(span.counter + offset + span.lamportSub); + characterIndex += 1; } - container._value = new DataView(bytes.buffer).getFloat64(0, true); - } else if ( - container instanceof LoroMovableList && - state.kind === CodecContainerType.MovableList - ) { - const ids = state.listItemIds.slice(0, state.values.length); - container._insertVisible( - 0, - state.values.map((value) => this.#decodeSnapshotValue(value, container)), - ids.map((item) => ({ - peer: state.peers[Number(item.peerIndex)]!, - counter: item.counter, - })), - ids.map((item) => item.counter + item.lamportSub), - ); - container._valueHistoryComplete = false; - container._moveHistoryComplete = false; - } else if ( - container instanceof LoroList && - state.kind === CodecContainerType.List - ) { - container._insertVisible( - 0, - state.values.map((value) => this.#decodeSnapshotValue(value, container)), - state.ids.map((item) => ({ - peer: state.peers[Number(item.peerIndex)]!, - counter: item.counter, - })), - state.ids.map((item) => item.counter + item.lamportSub), - ); } + container._insertVisible(0, characters.slice(0, characterIndex), ids, lamports); + for (const { run, key, meta } of styleRuns) { + container._styleIndex.add([run], key, meta); + } + container._attributeHistoryComplete = false; + } else if (container instanceof LoroTree && state.kind === CodecContainerType.Tree) { + const records: TreeNodeRecord[] = []; + for (let index = 0; index < state.nodes.length; index += 1) { + const node = state.nodes[index]!; + const nodeId = { + peer: state.peers[Number(node.peerIndex)]!, + counter: node.counter, + }; + const dataId: CodecContainerId = { + kind: "normal", + ...nodeId, + containerType: CodecContainerType.Map, + }; + const record: TreeNodeRecord = { + id: nodeId, + parent: undefined, + position: state.positions[node.fractionalIndexIndex]!.slice(), + deleted: node.parentIndexPlusTwo === 1n, + writer: { + peer: state.peers[Number(node.lastSetPeerIndex)]!, + lamport: node.lastSetCounter + node.lastSetLamportSub, + }, + lastMoveId: { + peer: state.peers[Number(node.lastSetPeerIndex)]!, + counter: node.lastSetCounter, + }, + data: this.#getOrCreateContainer(dataId, container, false) as LoroMap, + }; + records.push(record); + } + for (let index = 0; index < state.nodes.length; index += 1) { + const parent = state.nodes[index]!.parentIndexPlusTwo; + if (parent >= 2n) records[index]!.parent = records[Number(parent - 2n)]!.id; + } + for (const record of records) container._setRecord(record); + } else if ( + container instanceof LoroCounter && + state.kind === CodecContainerType.Counter + ) { + const bytes = new Uint8Array(8); + let bits = state.bits; + for (let index = 0; index < 8; index += 1) { + bytes[index] = Number(bits & 0xffn); + bits >>= 8n; + } + container._value = new DataView(bytes.buffer).getFloat64(0, true); + } else if ( + container instanceof LoroMovableList && + state.kind === CodecContainerType.MovableList + ) { + const ids = state.listItemIds.slice(0, state.values.length); + container._insertVisible( + 0, + state.values.map((value) => this.#decodeSnapshotValue(value, container)), + ids.map((item) => ({ + peer: state.peers[Number(item.peerIndex)]!, + counter: item.counter, + })), + ids.map((item) => item.counter + item.lamportSub), + ); + container._valueHistoryComplete = false; + container._moveHistoryComplete = false; + } else if (container instanceof LoroList && state.kind === CodecContainerType.List) { + container._insertVisible( + 0, + state.values.map((value) => this.#decodeSnapshotValue(value, container)), + state.ids.map((item) => ({ + peer: state.peers[Number(item.peerIndex)]!, + counter: item.counter, + })), + state.ids.map((item) => item.counter + item.lamportSub), + ); } } @@ -5669,7 +5955,7 @@ export class LoroDoc = Record [key, this.#decodeSnapshotValue(item)]), ); case "container": - return this.#getOrCreateContainer(value.value, parent); + return this.#getOrCreateContainer(value.value, parent, false); } } @@ -5677,6 +5963,7 @@ export class LoroDoc = Record = Record { expect(target.changeCount()).toBe(source.changeCount()); }); + test("edits and re-exports lazy snapshot state without hydrating siblings", () => { + const source = new LoroDoc(); + source.setPeerId(7); + const root = source.getMap("root"); + for (let index = 0; index < 200; index += 1) { + const child = root.setContainer(`child-${index}`, new LoroMap()); + child.set("value", index); + } + source.commit(); + const snapshotVersion = source.version(); + const snapshot = source.export({ mode: "snapshot" }); + + const target = LoroDoc.fromSnapshot(snapshot); + snapshot.fill(0); + const child = target.getMap("root").get("child-123"); + expect(child).toBeInstanceOf(LoroMap); + (child as LoroMap).set("local", true); + target.commit(); + + source.getMap("root").set("remote", true); + source.commit(); + target.import(source.export({ mode: "update", from: snapshotVersion })); + + const restoredFromUpdate = new LoroDoc(); + restoredFromUpdate.import(target.export({ mode: "update" })); + expect(restoredFromUpdate.getMap("root").get("remote")).toBe(true); + expect( + (restoredFromUpdate.getMap("root").get("child-123") as LoroMap).toJSON(), + ).toEqual({ local: true, value: 123 }); + + const restored = LoroDoc.fromSnapshot(target.export({ mode: "snapshot" })); + expect(restored.getMap("root").get("remote")).toBe(true); + expect((restored.getMap("root").get("child-123") as LoroMap).toJSON()).toEqual({ + local: true, + value: 123, + }); + expect(restored.getMap("root").get("child-122")).toBeInstanceOf(LoroMap); + expect(target.getAllChanges().size).toBeGreaterThan(0); + }); + test("materializes a snapshot before local edits without repeating first-peer events", () => { const source = new LoroDoc(); source.setPeerId(7); diff --git a/loro-js/tests/sstable.test.ts b/loro-js/tests/sstable.test.ts index c91e6c11c..26b2ed824 100644 --- a/loro-js/tests/sstable.test.ts +++ b/loro-js/tests/sstable.test.ts @@ -1,6 +1,11 @@ import { describe, expect, test } from "vitest"; -import { LoroDecodeError, decodeSstable, encodeSstable } from "../src/codec/index"; +import { + LoroDecodeError, + SstableReader, + decodeSstable, + encodeSstable, +} from "../src/codec/index"; describe("SSTable", () => { test("uses zero bytes for an empty KV store", () => { @@ -40,4 +45,36 @@ describe("SSTable", () => { corrupted[6] = corrupted[6]! ^ 1; expect(() => decodeSstable(corrupted)).toThrow(LoroDecodeError); }); + + test("looks up and rewrites entries one block at a time", () => { + const source = Array.from({ length: 12 }, (_, index) => ({ + key: Uint8Array.of(index + 1), + value: new Uint8Array(12).fill(index + 1), + })); + const table = new SstableReader( + encodeSstable(source, { blockSize: 32, compression: "lz4" }), + ); + + expect(table.get(Uint8Array.of(7))).toEqual(new Uint8Array(12).fill(7)); + expect(table.get(Uint8Array.of(99))).toBeUndefined(); + + const rewritten = table.rewrite( + [ + { key: Uint8Array.of(0), value: Uint8Array.of(100) }, + { key: Uint8Array.of(2), value: Uint8Array.of(20) }, + { key: Uint8Array.of(3), value: undefined }, + { key: Uint8Array.of(13), value: Uint8Array.of(130) }, + ], + { blockSize: 32, compression: "auto" }, + ); + expect( + decodeSstable(rewritten).map(({ key, value }) => [key[0], [...value]]), + ).toEqual([ + [0, [100]], + [1, Array(12).fill(1)], + [2, [20]], + ...source.slice(3).map(({ key, value }) => [key[0], [...value]]), + [13, [130]], + ]); + }); }); From f335d28a948ffc1554d31771302e69768627a65f Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 21 Jul 2026 10:39:09 +0800 Subject: [PATCH 2/2] docs: anonymize snapshot benchmark --- context/loro-js-performance.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 15792b0d2..ca74598a9 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -145,7 +145,7 @@ update import, full update export, and snapshot export with: pnpm --dir loro-js bench:snapshot-memory -- /path/to/document.snapshot ``` -For the 11,387,982-byte ProCloud document with 423,797 operations and 115,147 +For the 11,387,982-byte test document with 423,797 operations and 115,147 containers, Node 26.4.0 reports 70.92 MiB RSS after loading the input and a 160.70 MiB process peak after snapshot export: an 89.78 MiB incremental peak. Used JS heap peaks at 8.58 MiB. Snapshot import takes about 0.90 seconds, the