diff --git a/.changeset/critical-version-fallback.md b/.changeset/critical-version-fallback.md new file mode 100644 index 000000000..b0eb8f7e6 --- /dev/null +++ b/.changeset/critical-version-fallback.md @@ -0,0 +1,10 @@ +--- +"loro-crdt": patch +--- + +Retreat the conservative replay base to the latest single-head critical +version of the two versions' combined history instead of the beginning of +history. When an import or checkout involves a genuinely concurrent branch, +the causal replay now starts at the most recent point that no concurrency +crosses (in the sense of Eg-walker's critical versions), skipping the +fully-synced common prefix that the old empty-version fallback replayed. diff --git a/.changeset/igu-entry-check.md b/.changeset/igu-entry-check.md new file mode 100644 index 000000000..92fd73f88 --- /dev/null +++ b/.changeset/igu-entry-check.md @@ -0,0 +1,13 @@ +--- +"loro-crdt": patch +--- + +Fix a convergence bug where a movable tree's incrementally maintained state +could diverge from a full replay of its own oplog. When newly imported +operations were concurrent with part of the receiving peer's multi-head +frontier, the diff mode was misclassified as concurrency-free and the tree +fast path applied the new moves without adjudicating them against the +existing concurrent branch. The classifier now verifies that every entry +point of the imported region causally covers the whole current frontier, +and otherwise retreats the replay base to the latest critical version so +the competing branches are replayed together. diff --git a/.changeset/quiet-imports-move.md b/.changeset/quiet-imports-move.md new file mode 100644 index 000000000..0b043682d --- /dev/null +++ b/.changeset/quiet-imports-move.md @@ -0,0 +1,9 @@ +--- +"loro-crdt": patch +--- + +Correct LCA unmatched-branch detection when an explicit dependency already +covers the same peer's implicit predecessor. Keep those causally newer imports +on the current replay base, and avoid rebuilding unchanged list-like containers +when a genuinely concurrent update needs a conservative base containing a large +common history. diff --git a/.gitignore b/.gitignore index 38723c384..72bf5deee 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ loom_test.json .env sponsorkit/.cache.json .claude/ +.gstack/ diff --git a/context/internal-encoding.md b/context/internal-encoding.md index dc8533e47..01270e5a2 100644 --- a/context/internal-encoding.md +++ b/context/internal-encoding.md @@ -1,6 +1,6 @@ # Internal Encoding Context -Verified against code 2026-07-21. +Verified against code 2026-07-31. Loro has one binary blob envelope, two current binary body formats, two recognized-but-unsupported legacy top-level modes, and a separate JSON updates @@ -149,6 +149,32 @@ Because the set is only ever conservative, a rollback needs no invalidation — stale names just force the general diff path. Decode failures or exceeding the name-byte cap permanently disable the optimization for that store. +The general diff path may choose a replay base older than the current state +(the latest single-head critical version, Eg-walker §3.5) so list-like trackers +have enough position context. When that happens, +`DiffCalculator::calc_diff_internal` still walks the common causal history, but +routes it only to containers that have operations in the version-vector +difference between `before` and `after`. Do not treat every container seen since +the conservative base as changed: the List/Text/MovableList safety fallback can +otherwise replay the full history once per unchanged container. + +The replay-base walk expands both explicit change dependencies and the implicit previous +counter of the same peer. A change from an existing peer can therefore produce +two paths: an explicit relay dependency and an implicit same-peer predecessor. +The relay may already contain that predecessor. In that case the second path can +reach the end of the queue without meeting the other side even though it is not +concurrent. + +To distinguish those cases, `_find_common_ancestor_new` carries the dependency +tip where each path split. When a path remains unmatched, it checks only that tip +against the ancestors of the candidate common frontiers. A covered tip is a +redundant route and does not lower the replay base; an uncovered tip is a real +concurrent branch and keeps the conservative fallback. This is a targeted DAG +reachability check with visited-node and Lamport pruning. Do not replace it with +a complete version-vector containment check for every new peer range: that work +scales with both the update's peer count and the size of the current version +vector, and it duplicates the causal decision the walk is already making. + For a large snapshot regression check, first build the Node package, then run: ```sh diff --git a/crates/loro-internal/AGENTS.md b/crates/loro-internal/AGENTS.md index a4c7af4f0..84beddf49 100644 --- a/crates/loro-internal/AGENTS.md +++ b/crates/loro-internal/AGENTS.md @@ -28,6 +28,9 @@ over graceful degradation. `MapHandler::ensure_mergeable_*`. - `src/diff_calc/`: diff calculation when moving between versions. - `docs/diff_calc.md`: design notes for diff calculation. +- `docs/critical-version-spec.md`: specification and proof skeleton for + replay-base selection (Eg-walker-aligned terminology; defines critical + version, the entry check, and the fallback sweep). - `docs/mergeable-container-id.md`: current mergeable container id encoding. - `tests/mergeable_container/` and `tests/mergeable_cid_encoding.rs`: focused mergeable container regression tests. @@ -50,6 +53,16 @@ coverage under `crates/fuzz` and ask before running long fuzz targets. ## Working Rules +- Replay-base selection uses Eg-walker terminology (arXiv:2409.14252 §3.5): + a version V is **critical** when every event outside `Events(V)` happened + after all of `Events(V)` — no concurrency crosses the cut. Non-`Checkout` + diff modes and the tree calculator's lamport windows are only sound when + the base satisfies this; `dag.rs` enforces it via the + `ImportGreaterUpdates` entry check and the + `latest_single_head_critical_version` fallback. Do not use "LCA" in new + code or docs: the meet of two versions is generally NOT a safe replay + base. Read `docs/critical-version-spec.md` before touching + `find_common_ancestor`, diff modes, or `diff_calc/tree.rs` windows. - Internal invariant violation should fail fast. Invalid external bytes or JSON should return `Err`. - Do not silently skip ops, containers, state entries, diffs, or pending changes. diff --git a/crates/loro-internal/docs/critical-version-spec.md b/crates/loro-internal/docs/critical-version-spec.md new file mode 100644 index 000000000..6f3cfa51d --- /dev/null +++ b/crates/loro-internal/docs/critical-version-spec.md @@ -0,0 +1,608 @@ +# 找共同祖先:Loro 历史图遍历算法的规格与证明骨架 + +状态:草稿 v2,2026-08-01。对应 `crates/loro-internal/src/dag.rs` 中的 +`_find_common_ancestor_new`(含 PR #1058 与 tips 去重修复之后的版本)。 + +这份文档是自包含的:读者不需要了解 Loro、CRDT 或本仓库的代码。 +需要的全部预备知识是:集合、有向图、数学归纳法,以及"堆/优先队列"这个数据结构。 +文档先讲问题从哪里来(第 1 章),再给出数学模型(第 2 章)、算法(第 3 章)、 +我们承诺的性质即"规格"(第 4 章)、证明骨架(第 5 章),最后是待定问题(第 6 章) +和词汇表(第 7 章)。写作顺序都是**先直观解释、再数学定义**。 + +--- + +## 1. 问题从哪里来 + +### 1.1 多人协作编辑与操作历史 + +Loro 是一个协作编辑库。它要解决的场景是:几个人(或几台设备)同时编辑同一份 +文档——比如一份购物清单——而且允许离线编辑,之后再同步。同步之后, +所有人看到的结果必须完全一致。实现这一点的数学工具叫 CRDT, +但本文档不需要你懂 CRDT,只需要懂下面这个更基本的东西:**操作历史**。 + +每个参与者(叫一个 **peer**,可以理解为"一台设备上的一个副本") +做的每一次修改都是一个**操作(op)**。操作不会被覆盖或删除,只会不断追加, +所以整个文档的历史就是所有人产生的所有操作的集合。 + +每个操作有一个全局唯一的编号:**(p, c)**,读作"第 p 个 peer 的第 c 个操作", +文本形式记作 **c@p**(counter 在前,与 Loro 代码输出一致;见第 2 章记法约定)。 +同一个 peer 的 c 从 0 开始,每做一个操作加一。 + +操作之间有一种天然的先后关系:**做某个操作时,作者已经看到了哪些别的操作**。 +比如 Bob 在看到 Alice 写下"牛奶"之后才补了一句"鸡蛋",那么"鸡蛋"这个操作 +就**依赖**"牛奶"那个操作。反过来,如果两个人离线时各改各的, +互相都没看见对方的修改,这两个操作就是**并发的(concurrent)**——谁也不依赖谁。 + +把每个操作画成一个点、依赖画成有向边,整个历史就是一张**有向无环图(DAG)**。 + +一个具体的小例子(A = Alice,B = Bob;箭头为 **happened-before** 方向, +由早指向晚——与 Eg-walker 论文一致;注意存储时每个事件记录的是反向的 +"父事件"引用): + +``` +A0 "加入苹果" ──→ A1 "加入牛奶" ──→ A2 "加入面包" + │ + └────────────→ B0 "加入鸡蛋" +``` + +- A1 依赖 A0(同一个人的下一个操作,天然依赖上一个); +- B0 是 Bob 看到 A1 之后做的,所以 B0 依赖 A1; +- A2 是 Alice 在**没看到** B0 的情况下做的,所以 A2 只依赖 A1。 +- A2 和 B0 互相都不依赖:它们是并发的。 + +### 1.2 版本与前沿(frontier) + +"文档在某一时刻的版本"是什么?直观上:**一个操作集合**——这一时刻已经发生 +(且被这个副本看到)的所有操作。这种集合有个重要特点:**因果闭合**—— +你看到了某个操作,就必然看到了它依赖的一切操作(否则你根本无法理解它)。 + +因果闭合的集合不需要逐个元素列出来,只要报出它的"最新端点"就够了: +比如上图中"Bob 视角的版本"是 {A0, A1, B0},只要说"我最新到 B0"即可, +因为 B0 的全部祖先自动包含在内。这组最新端点就叫**前沿(frontier)**。 +一个前沿里不会有谁是谁的祖先(否则那个祖先是多余的), +这种"两两互不为祖先"的集合在数学上叫**反链(antichain)**。 + +前沿可以有多个端点:上图中 Alice 同步了 B0 之后,她的版本是全部 5 个操作, +前沿是 {A2, B0}——两个并发的端点,谁也代表不了谁。 + +### 1.3 要解决的问题:给两个版本找"重放起点" + +Loro 里有两个高频动作: + +- **import(导入)**:收到别人发来的更新,把新操作合并进本地历史; +- **checkout(检出/时间旅行)**:把文档切换到历史上任意一个版本。 + +两者的核心都是同一个计算:**给定旧版本 L 和新版本 R(各用一个前沿表示), +算出文档状态从 L 到 R 的"差量"(diff)**。计算差量的通用办法是**重放**: +从两个版本**共同拥有**的历史中选一个起点,把起点之后的操作按因果顺序重演一遍, +一边重演一边算出状态差。 + +于是问题变成:**重放起点选哪里?** + +- 起点选得**太老**:要白白重演大量双方本来就共有的历史。这不是理论担忧—— + 本仓库最近修的一个真实 bug(PR #1058)就是起点被错误地退到了历史开头, + 导致导入一个 124 字节的小更新要重演全部历史、耗时是正常情况的几百倍。 +- 起点选**错**(比如漏掉某条并发分支的分岔点):差量会算错, + 文档内容错乱。这比慢严重得多。 + +直觉上理想的起点是两个版本公共操作中"最靠上"的那些——本文档记作 +meet(版本格的最大下界前沿,第 2 章 D8 给精确定义;它是一个反链, +不一定是单点)。但 meet 只是**候选**:起点还必须满足一个"切得开"的 +条件(critical version,D8b),meet 不满足时要向更早处回退—— +这正是场景④"保守回退"的由来,严格定义见 4.1 节。 + +除了起点,调用方还需要知道第二件事:**R 是否完全包含 L**? +如果是(典型情况:导入的更新完全建立在本地版本之上),差量计算可以走快路径 +(不需要重建复杂的中间结构);如果否(存在并发),必须走慢而稳的路径。 +算法把这个判断以"模式"(mode)的形式一并返回: + +- `Linear`:R 包含 L,且新增部分是一条无分叉的直线; +- `ImportGreaterUpdates`(下文简称 IGU):R 包含 L; +- `Checkout`:其余所有情况(存在并发、或时间倒流),走保守路径。 + +### 1.4 为什么这个问题不平凡:三个陷阱 + +**陷阱一:隐式依赖造出"冗余路径"。** +同一个 peer 的第 c 个操作天然依赖第 c−1 个,这条边**不会被显式存储** +(存了就太浪费了),遍历时要自己补上。这会造出"一个祖先、两条到达路径"的局面: + +``` +0@1 ──→ 0@2(中转) ──→ 1@1 + │ ↑ + └──────(隐式:同 peer 前驱)─┘ +``` + +peer 1 做完 0@1 后,peer 2 基于它做了 0@2;然后 peer 1 看到 0@2 又做了 1@1。 +1@1 显式依赖 0@2,同时隐式依赖自己的前驱 0@1。从 1@1 出发往下走会分出两条路: +显式路经 0@2 到 0@1,隐式路直达 0@1。第二条路是**冗余**的——它到达的一切 +都能从第一条路到达。但一个只看"这条路走到死也没碰到对方"的简单算法 +会把冗余路误判成**并发分支**,进而把重放起点错误地退到历史开头。 +这正是 PR #1058 修复的 bug。区分"冗余路"和"真并发分支"是本算法最微妙的部分。 + +**陷阱二:操作是按"段"存储的。** +为了效率,同一个 peer 连续的一串操作打包成一个**节点**(内部叫 Change)存储, +遍历时弹出的是一整段。而别的路径的依赖可能恰好指向这一段的**中间**某个操作。 +算法必须保证"整段前进"不会大步跨过别人指向的中间点(3.3 节的"对齐"机制)。 + +**陷阱三:性能约束。** +历史可以有百万级操作。遍历的开销必须只和"两个版本附近的区域"成正比, +不能做全图搜索。手段是给每个操作配一个 **lamport 时间戳**(2.1 节), +永远从"时间戳最大"的未处理条目开始处理——像水面从高处向低处退去, +两个版本的探索会在共同祖先附近自然会合,不必触碰更深的历史。 + +--- + +## 2. 数学模型 + +本章把上面的直观图景变成精确定义。每个定义前先用一句话回顾直观含义。 + +**D1(操作 id)** 操作的编号是二元组 (p, c) ∈ P × ℕ,P 是 peer 的集合。 + +**记法约定** 具体 id 的文本形式一律写作 **c@p**(counter 在前、peer 在后), +与代码中 `ID` 的 Display 输出一致(`loro-common/src/id.rs`)。数学元组仍写 +(p, c),读作"peer p 的第 c 个操作";若行文需要 peer 在前的紧凑形式,写 +P⟨p⟩_C⟨c⟩,**绝不写 p@c**,以免与标准文本形混淆。叙事中的字母形 +(A0 = Alice 的第 0 个操作)仅用于故事。 + +**D2(节点划分;公理 A2)** *——"连续操作打包存储"。* +每个 peer 的操作序列被划分为若干互不重叠的连续区间,每个区间叫一个**节点** +N = [s, e)(含 s 不含 e)。每个操作恰好属于一个节点。节点携带一个 +**显式依赖集** deps(N) ⊆ Ids,语义上挂在节点的第一个操作 (p, s) 上。 + +**D3(父事件 parents 与事件图;Eg-walker §2.2)** +*——"做这个操作前刚看到的最新操作们"。* +每个事件 e 携带**父事件集** parents(e)(论文记 e.parents;Loro 代码称 deps): +- 隐式父:(p, c−1) ∈ parents((p, c)),对一切 c > 0; +- 显式父:deps(N) ⊆ parents((p, s)),s 是节点 N 的起点。 +事件与父引用构成**事件图(event graph)**:一张 DAG,每个节点是一个事件 +(一次操作 + 唯一 id + 父事件 id 集,论文 §2.2)。 + +**D4(happened-before → 与并发 ∥;Eg-walker §2.2)** +*——"由早指向晚的因果链"。* +a **happened before** b(记 **a → b**)当且仅当沿父引用存在从 b 回到 a 的 +有向路径:a → b ⟺ a ∈ parents(b) ∨ ∃e: a → e ∧ e ∈ parents(b)。 +箭头方向沿用论文与 Lamport:**由早指向晚**(父引用存储方向与之相反)。 +**并发**:a ∥ b ⟺ a ≠ b ∧ a ↛ b ∧ b ↛ a(论文 §2.2)。 +引理中常用的自反版本 **≤**:v ≤ u ⟺ v = u ∨ v → u。 +祖先闭包沿用论文记法 **Events(·)**(§2.3): +Events(V) = V ∪ { e₁ ∣ ∃e₂ ∈ V: e₁ → e₂ }。 +为行文简短,本文把 Events({u}) 记作 ancestry(u)、Events(F) 记作 ancestry(F)。 + +**D5(lamport 时间戳;公理 A1)** *——"逻辑时钟:你依赖的东西一定比你早"。* +函数 lamport : Ids → ℕ 满足: +- 沿父引用严格递减:a ∈ parents(b) ⟹ lamport(a) < lamport(b); +- 节点内逐操作加一:lamport(p, c+1) = lamport(p, c) + 1。 + +由传递性立即得到本文档最常用的推论: + +> **L0**:v → u(v happened before u) ⟹ lamport(v) < lamport(u)。 + +注意反过来**不成立**:lamport 小不代表 happened-before(并发操作的 lamport 也可比较大小)。 + +**D6(前沿 / version;Eg-walker §2.3)** 有限反链(成员两两并发)。 +论文把因果闭子图 G′ 的前沿称为它的 **version**: +Version(G′) = { e₁ ∈ G′ ∣ ∄e₂ ∈ G′: e₁ → e₂ }(没有后继的事件全体)。 +前沿与因果闭集互逆:Events(Version(G′)) = G′,Version(Events(F)) = F。 +算法输入是两个前沿 L 和 R。 + +**D7(版本与版本向量)** *——"版本 = 因果闭合的操作集 = 每个 peer 报一个前缀长度"。* +称操作集 S 是**因果闭**的,若 u ∈ S 且 v ≤ u 则 v ∈ S。ancestry(F) 总是因果闭的。 +由隐式边,因果闭集在每个 peer 上必是前缀 {(p,0), …, (p,k−1)}, +所以一个因果闭集可以用"每个 peer 一个数字 k(p)"紧凑表示——这就是**版本向量**。 +两个版本的包含关系等价于逐 peer 比较数字: + +> vv(R) ⊇ vv(L) ⟺ ancestry(L) ⊆ ancestry(R)。 + +**D8(公共历史与 meet 前沿)** C(L, R) = ancestry(L) ∩ ancestry(R)。 +两个因果闭集的交仍是因果闭的。定义 **meet(L, R) := Max(C(L,R))**, +即 C 中的极大元全体——版本格(因果闭集按 ⊆ 构成的格)中两版本最大下界 +的前沿,这是一个反链。测试里的"oracle"(暴力对照实现)算的就是这个对象。 + +命名说明:本文档与代码曾把它借称为 "LCA",2026-08-01 起已全面弃用: +代码中 `iter_from_lca_causally` 改名 `iter_from_replay_base_causally`、 +`lca_vv` 改名 `replay_base_vv`、回退函数命名为 +`latest_single_head_critical_version`,文档改名 +`critical-version-spec.md`。理由:经典 LCA 是图上两个**节点**的最深公共 +祖先(单点),而这里的对象是两个**版本**在格上的最大下界;更重要的是, +meet 只是算法的**候选**,不是算法真正要交付的东西(见 D8b 与 4.1 节)。 +`find_common_ancestor` 这个名字保留——它返回的确实是公共祖先版本。 + +1.1 节例子中:L = {A2},R = {B0},C = {A0, A1},meet = {A1}。 + +**D8b(critical version;Eg-walker,arXiv:2409.14252 §3.5)** +*——"能把历史一刀两断的版本"。* +版本 V 在事件图 G 中是 **critical** 的,当且仅当它把 G 划分为 +G₁ = Events(V)(V 的祖先闭包)与 G₂ = G − G₁,使得 G₁ 中每个事件都 +happened-before G₂ 中每个事件:∀e₁∈G₁, ∀e₂∈G₂: e₁ → e₂。 +等价说法:切口两侧没有任何一对并发事件。空版本 ∅ 平凡地 critical +(G₁ 为空,条件真空成立)。 + +重放起点的**合法性判据**正是它:以 B 为起点重放区域 G − Events(B) 时 +tracker 无需 B 以下的任何位置上下文 ⟺ B 在联合图 +ancestry(L) ∪ ancestry(R) 中 critical。Eg-walker 合并新事件时取 +"发生在双方之前的**最晚** critical version"为起点(§3.6 的 V_crit)。 + +**D9(段 / OrdIdSpan)** *——"节点的一个前缀切片"。* +⟨N, e⟩ 表示节点 N 内的操作段 [s .. e](含两端)。记 +id_last = (p, e),lamport_last = lamport(p, e)。 +from_dag_node(u) := 包含 u 的节点在 u 处截断的段。 +**段的 deps 恒等于所属节点的 deps**(挂在节点起点上),与截断位置无关—— +这一点很重要:截掉段的尾部不会丢失任何依赖边。 + +**D10(浅历史 / trimmed;公理 A5)** *——"太老的历史可能已被裁剪掉"。* +可用操作集 D ⊆ Ids;查询函数 get 在 D 之外返回"不存在"。 +输入前沿的成员都在 D 内,但节点的 deps 可以指向 D 外(裁剪边界)。 + +--- + +## 3. 算法 + +### 3.1 直观图景:两支探险队下山 + +把事件图想成一座山:lamport 时间戳是海拔,事件越晚海拔越高 +(A1:happened-before 箭头由山下指向山上);队员沿**父引用**向山下走。 + +- 给左版本 L 派一支**红队**,从 L 的每个前沿端点出发; + 给右版本 R 派一支**蓝队**,从 R 的端点出发。 +- 所有队员放进同一个**优先队列**,永远先处理**海拔最高**的那个队员。 + 队员每一步沿依赖边往山下走(一个队员分出几条依赖就分裂成几个队员)。 +- **红蓝两队在同一个操作上相遇 → 该点染紫**:它是公共祖先,收进候选集 ans, + 并且**不再继续往下走**(紫点以下全是更老的公共历史,无需探索—— + 这是性能的关键,也是后面一切微妙性的来源)。 +- 某个队员**走到了死路**——走到没有依赖的根,或者队列里已经没有别人了—— + 却始终没碰到对方颜色,说明什么? + +最后一问正是陷阱一。有两种可能: + +1. 这个队员代表一条**真正的并发分支**(对方版本里根本没有这段历史)—— + 此时必须放弃候选起点、保守回退(把起点退到最老,模式 Checkout); +2. 这个队员只是一条**冗余路**(1.4 节的隐式前驱路径):它要到达的地方, + 别的队友早就经由紫点到达了。它"死"只是因为紫点不再展开、没人来接应它。 + 此时**不应该**回退。 + +区分办法(**tips 机制**):每个队员身上带一块牌子(tips), +写着"我是从哪个岔路口分出来的"。队员死亡时,查它的牌子: + +> 岔路口 ∈ 已找到紫点的祖先集? +> 是 → 整条死路都在公共历史里,冗余,虚惊一场; +> 否 → 真并发分支,保守回退。 + +这个检查本身也是一次山上的小规模走查(从紫点出发往下、按海拔剪枝), +它只在有死路时发生。 + +### 3.2 精确描述 + +堆条目为三元组 e = (span, type, tips):span 是一个段(D9), +type ∈ {A(红,来自 L), B(蓝,来自 R), Shared(紫)},tips ⊆ Ids。 +堆按 (lamport_last, peer, 更短的段优先) 的字典序弹出最大者。 + +**初始化**:L 的每个成员 u 以 (from_dag_node(u), A, tips₀) 入堆;R 同理入堆为 B。 +其中 tips₀:若该侧前沿有多个端点,则为 {u}(每个端点自成一个"岔路口"); +单端点时为空集(这条路还没分过岔)。 + +**主循环**——弹出 e = (n, t, tips),依次执行: + +1. **聚合**:只要堆顶条目与 n 的段相等、或 id_last 相同:把它也弹出并合并进 e; + 两者 type 不同 ⟹ t := Shared;tips 取并集(**去重**——不去重会指数爆炸, + 这是本次 review 修复的性能 bug)。 +2. **会合**:若 t = Shared:ans 收进 id_last(n);continue(不展开)。 +3. **死亡(队空)**:若堆已空:记录 unmatched(tips 非空记 tips, + 否则记 id_last(n) 自身);break。 +4. **偏序线索**:若 t = A(红队员在堆非空时被单独弹出,说明左侧有右侧 + 尚未包含的操作):is_right_greater := false。 +5. **对齐**(设堆顶为 o): + a. 若 n 包含 id_last(o) 且 t ≠ type(o):把 n 截断为以 id_last(o) 结尾, + 重新入堆;continue。 *(对方指向我的中间:先切到对齐,下一轮聚合。)* + b. 否则若 len(n) > 1:把 n 收缩到 lamport_last 不超过 o 的 lamport_last + (且至少缩短 1,即取 min(按海拔对齐的长度, len − 1)),重新入堆;continue。 + *(长段不许大步跨过任何海拔更高的旁人。"len − 1"上限处理海拔并列: + 并列时按海拔对齐算出的长度等于自身,若原样重入堆即死循环; + 上限强制严格进展,这正是 L10 终止度量在此分支下降的原因。 + 丢弃的段尾是安全的:依赖边挂在节点起点上(D9),截尾不丢边; + 且以段尾为终点的条目会先被聚合分支合并,未来也不再有指向该海拔的 + 新条目(L1),故段尾不可能是任何会合点。)* +6. **展开**:parents* := 显式 deps(N) ∪ {隐式前驱(若未被显式依赖覆盖)} + (即 e.parents,D3;代码中称 deps)。 + - 可解析且非空:每个 d ∈ parents* 以 (from_dag_node(d), t, tips_d) 入堆; + 其中 tips_d:若 tips = ∅ 且 |parents*| > 1(**第一次分岔**), + 则 tips_d = {id_last(d)}(在岔路口发牌子);否则继承 tips。 + is_linear := false。 + - 不可解析(依赖被裁剪,D10):unresolved := true;continue。 + - parents* = ∅(走到根)且堆非空:记录 unmatched(同步骤 3);continue。 + +**收尾**: +- ans := ans 的极大元反链(去掉互为祖先的冗余成员); +- uncovered := unresolved ∨ ¬( unmatched 中每个 tip ∈ ancestry(ans) )。 + 覆盖判定用**单次**多源走查:从 ans 的所有成员出发向下, + 按"尚未证实的 tips 的最小 lamport"剪枝(L0 保证剪枝安全); +- 若 uncovered 且 ans 的依赖未触及裁剪边界: + ans := 并集图上最晚的单头 critical version(L11 扫描;不存在则 ∅); +- 若 uncovered:is_right_greater := false; + 否则若 ans = L:is_right_greater := true; +- mode := Checkout(若 ¬is_right_greater);否则 Linear(若 is_linear 仍为真); + 否则 IGU。 + +此外实现里还有三个**快速路径**(R 为空、L 为空、L 与 R 都是单点且同 peer), +它们绕过主循环直接返回。每个都需要独立的小证明(见 Q6)。 + +### 3.3 三个机制再各给一个直观注解 + +**聚合**:站在同一位置的队员合并成一人;红 + 蓝 = 紫。合并同时合并牌子(去重)。 + +**对齐(回应陷阱二)**:一个段是"一列纵队"。规则 5b 说:纵队每轮最多下行到 +与堆中次高者平齐的海拔,绝不越过;规则 5a 说:若对方明确指着我纵队中间的 +某个人,我先在那个人处断开。两条规则合起来保证: +**任何指向段中间的会合都不会被跳过**。为什么"迟到的指针"不存在? +因为指向 (p, c) 的条目是由它的某个 depender 展开时入堆的, +而 depender 的海拔严格高于 (p, c)(L0);再加上"堆的最高海拔单调下降"(L1), +指针必然在覆盖 (p, c) 的段消亡之前就已入堆。这就是第 5 章的 L3。 + +**tips 与覆盖检查(回应陷阱一)**:牌子只在"第一次分岔"时发放, +之后一路继承;两条路合流时牌子取并。于是不变式是: +**一条路径死亡时,它走过的每个操作都是它某块牌子的祖先**(L6)。 +所以"牌子 ∈ ancestry(紫点)"就足以证明整条死路都泡在公共历史里(L7)—— +检查牌子(少数几个点)而不必检查整条路径(可能很长)。 + +### 3.4 模式判定的含义 + +- is_right_greater 想回答"ancestry(L) ⊆ ancestry(R) 且起点可以就用 L"。 + 它有两个信息来源:走查过程中红队员是否曾被单独弹出(步骤 4), + 以及收尾时死路是否全部被覆盖、ans 是否恰好等于 L。 +- is_linear 想回答"新增历史是一条无分叉直线":任何一次依赖展开都会把它清 false。 +- 三种模式对下游意味着:Linear/IGU → 从 L 直接重放、走快路径; + Checkout → 从 ans(可能为 ∅ = 历史开头)重放、走保守路径。 + +--- + +## 4. 规格:我们向调用方承诺什么 + +| 编号 | 陈述 | 直观含义 | +|---|---|---| +| S1 | ans ⊆ C(L,R) | 给出的起点确实是双方共有的历史 | +| S2 | ans 是反链 | 起点集合里没有冗余成员 | +| S3 | mode ∈ {Linear, IGU} ⟹ ans = L ∧ ancestry(L) ⊆ ancestry(R) | 宣布"快路径可用"时,R 确实完整包含 L,且起点就是 L | +| S3L | mode = Linear ⟹ 上述之外还有 \|ans\| ≤ 1,且新区 ancestry(R)∖ancestry(L) 是 ≤-全序链 | 宣布"直线"时新历史真的无分叉(表述待定,见 Q2) | +| S4 | mode ≠ Checkout ⟹ 新区每个事件 ≥ L 的全部头(已强化,见下) | 由入场检查(L12)保证 | +| S5a | mode = Checkout ⟹ 只承诺 S1 ∧ S2(ans = ∅ 合法) | 保守模式下起点允许偏老,偏老只影响速度不影响正确性 | +| S5b | 猜想 C1:走查无死路 ⟹ ans = meet(L,R)(精确) | 常规情形下起点不多退一步(是否纳入规格待定,见 Q1) | +| T | 算法总终止 | | + +**S4(已强化,2026-08-01)**:mode ≠ Checkout ⟹ 新区每个事件都 +因果晚于 L 的**全部**头(即 L 对并集图 critical)。 + +历史背景:本条曾是"非目标声明"——旧版只承诺版本级包含 +(vv(R) ⊇ vv(L)),把逐操作并发交给下游守卫兜底。两个反例都会通过: + +``` +反例一(同 peer 段截断):L = {A5, L2},R = {r},deps(r) = {A9, L2}, + A6..A9 与 L2 并发; +反例二(movable tree 线上 issue):L = {B6, A0},新区 A1..A3 只依赖 + A0、与 B6 并发。 +``` + +后者证明了 Tree 计算器的 IGU 快速路径**没有**守卫(不同于 text/list +的 `mark_source_not_in_op_context`):A1 未与 B4..B6 打擂台就被采纳, +增量状态与全量重放分叉。现由**入场检查**(L12)在多头 L 时强制验证, +两个反例都会被降级为 Checkout + L11 安全基准;单头 L 由 L9 的覆盖 +论证自动满足本条。 + +### 4.1 算法真正在找什么(严格版,按 Eg-walker 的语言) + +算法的候选是 meet(L,R)(D8),但合法重放起点的严格判据是 +**critical version**(D8b):起点 B 合法 ⟺ B 在联合图 +ancestry(L) ∪ ancestry(R) 中 critical ⟺ 重放区域里没有任何事件与 +ancestry(B) 中的事件并发。Eg-walker 的理论最优是"发生在双方之前的 +最晚 critical version"(§3.6 的 V_crit)。据此把算法的行为分三类: + +1. **meet 恰好 critical**(场景①③的常规形状):候选即答案, + "找 meet"与"找合法起点"重合。 +2. **meet 不 critical,且走查探测到证据**(uncovered 死路,场景④): + 回退到**并集图上最晚的单头 critical version**(已实现,2026-08-01): + 复用主循环的聚合/对齐机件,从 L ∪ R 做一次单色下山扫描; + 待处理堆首次收窄为单个段的时刻,其 id_last 即所求(引理 L11)。 + 若某条链在堆非空时死于根或裁剪边界,其端点与其下一切事件并发, + 之后不再可能有合法单头切口——立即放弃并返回 ∅(即旧行为; + 场景④的双根正是此情形)。扫描只在回退真正发生时执行, + 代价不超过随后重放区域的一次遍历。 + 与 Eg-walker 的 V_crit 相比仍有两处已知取舍:只找**单头**切口 + (多头 critical version 无法用"堆宽 = 1"判据探测); + 找不到单头切口时不再细分、直接 ∅。 +3. **meet 不 critical,但走查未探测到**(2026-08-01 起大幅收窄): + IGU 候选(ans = L 且无死路)现在必须通过**入场检查**(L12): + 新区每个入场 change 的因果父集必须版本覆盖 L 的全部头, + 不过则降级并调用 L11 扫描取安全基准。这消灭了原第 3 类中影响 + IGU 的全部已知构造(S4 的两个反例)。仍然遗留的是 **Checkout + 模式下 ans = meet 非 critical** 的情形(双根 x ∥ y 菱形:四条路 + 全部会合、无死路,meet = {x,y},但区域内有事件与 y 并发): + text/list 依赖下游守卫兜底,Tree 的 Checkout 路径是否暴露 + 尚未查证——见 Q7。 + +因此严格的安全陈述是**双层契约**:「(ans, mode) + 下游守卫」合起来 +保证收敛;单看走查,无条件承诺的只有 S1/S2/S3/T。用论文语言可以把 +守卫公理写准(这就是 Q3 要拍板的边界): + +> **守卫公理(接口)**:对任意被重放的 change c,若 vv(c) ⊉ vv(起点), +> 则 tracker 以完整上下文重建后再应用 c,其效果与从任一合法 +> critical version 重放一致。 + +--- + +## 5. 引理骨架(未来 Lean 里的定理清单) + +每条引理:直观一句话 → 精确陈述 → 证明思路。 + +**L0(海拔与 happened-before)** *越晚的事件海拔严格更高。* +v → u ⟹ lamport(v) < lamport(u)。 +证明:沿父引用逐条严格递减(A1),传递。 + +**L1(无迟到)** *水位只降不升;新入堆者总在水位之下。* +堆中最大 lamport_last 随时间单调不增;且任何一次 push 的条目, +其 lamport_last 严格小于 push 发生时刻的堆最大值。 +证明:三类 push——展开(新条目是被弹条目的父事件,L0 给出严格更小)、 +对齐重入(收缩后 ≤ 原值)、聚合不 push。弹出只移除最大者。归纳。 +推论:**水位一旦降到 λ 之下,海拔 ≥ λ 的条目永远不会再出现。** + +**L2(锁步)** *长纵队不越人。* +len > 1 的段只能通过聚合被吸收或通过对齐被收缩; +只有收缩到 len = 1(即只剩节点起点)时才会展开依赖; +且届时堆中不存在 lamport 更高的未处理条目。 +证明:直接读主循环的分支结构——步骤 5b 拦截一切 len > 1 且未聚合的段。 + +**L3(会合对齐定理)** *该相遇的一定会相遇,哪怕会合点在段中间。* +若操作 u 既 ∈ 红队探索范围又 ∈ 蓝队探索范围, +则两侧条目必在 u 处(经对齐与聚合)合并为 Shared。 +证明思路:指向 u 的条目由某 depender 展开产生,lamport(depender) > lamport(u) +(L0);由 L1,该 push 发生时水位 > lamport(u),而覆盖 u 的段 +按 L2 要到水位 = lamport(节点起点) ≤ lamport(u) 时才消亡—— +所以两个条目必有同时在堆的时刻;此后对齐规则 5a/5b 使两者 id_last 相等, +聚合规则将其合并。(分侧讨论 A/B/双向、以及等海拔并列的情形。) + +**L4(颜色不变式)** *红队只踩左山,蓝队只踩右山,紫点必是公共的。* +任意时刻:A 型条目的段 ⊆ ancestry(L);B 型 ⊆ ancestry(R); +Shared 仅由 A 与 B 条目聚合产生。 +证明:对"初始化、截断、收缩、展开"四种状态变迁归纳; +段是节点前缀 + 段的 deps 属于节点起点(D9)保证展开不越界。 +**⟹ S1**。 + +**L5(极大化正确性)** 收尾的 shrink 恰好输出输入集合的极大元反链。**⟹ S2**。 + +**L6(牌子上界)** *死路走过的每一步都是某块牌子的祖先。* +一条路径自最近一次牌子发放(初始化多端点、或第一次分岔)之后 +访问的每个操作 u,满足 ∃ t ∈ tips,u ≤ t。 +证明:对"继承、合并取并、截断"归纳;发牌时刻 tips = {路径当前最新端}成立。 + +**L7(覆盖 ⟹ 冗余)** *牌子泡在紫水里,整条路都泡在紫水里。* +死路径的每个 tip ∈ ancestry(ans) ⟹ 该路径访问过的全部操作 ∈ ancestry(ans) ⊆ C。 +证明:L6 + ancestry 的传递性(u ≤ t ≤ ans 成员)。 + +**L8(覆盖走查正确性)** 收尾的多源剪枝走查返回真 ⟺ tips ⊆ ancestry(ans)。 +证明:完备性——走查是从 ans 出发沿父引用(逆 → 方向)的可达性搜索; +剪枝安全性——被剪节点的一切祖先 lamport 低于剩余 tips 的最小 lamport(L0), +不可能"包含"任何剩余 tip。 + +**L9(模式一致性)** 收尾后 is_right_greater = true ⟺ ans = L ∧ 无 uncovered。 +关键子引理:**红条目一旦在堆非空时被单独弹出,ans ≠ L**—— +因为该 L 成员随后要么被截短(id_last 变小)、要么被展开消耗, +再也不会以完整 id 进入 ans。 +另一方向:ans = L ∧ 全覆盖 ⟹ L 的每个成员都聚合成了紫点 +⟹ L ⊆ ancestry(R) ⟹(D7)vv(R) ⊇ vv(L)。**结合 L3、L6–L8 ⟹ S3**。 + +**L10(终止)** 度量:堆条目按 (lamport_last, len) 取字典序, +整个堆构成一个多重集,用 Dershowitz–Manna 多重集序比较。 +每轮循环:聚合/弹出移除元素;对齐以严格更小的元素替换; +展开以有限个严格更小(lamport_last 更小,L0)的元素替换被弹元素。 +多重集序良基 ⟹ 终止。**⟹ T**。(mathlib 已有 Dershowitz–Manna 序。) + +**L11(回退扫描的正确性)** *堆是未探索区域的切口; +切口收窄成单个段的瞬间,段尾就是最晚的单头 critical version。* +设从 L ∪ R 出发做单色下山扫描(聚合与对齐机件同主循环), +且此前没有链在堆非空时死于根或裁剪边界。若某次弹出并聚合后堆恰为空, +记被弹段的 id_last = v,则 {v} 在并集图 Events(L) ∪ Events(R) 中 +critical,且 v 是最晚的单头 critical version。证明要点: +(a) **切口不变式**:任意时刻,每个已发现未处理的事件都 ≤ 某个堆中条目 +(同 L1 的推入论证);堆空 ⟹ 未发现事件全部 ≤ v。 +(b) **已处理事件 ≥ v**:每个已处理事件的每条向下路径都经由对齐/聚合 +汇入唯一幸存的段;截断分支保证段尾取所有汇入点的最小 counter, +故所有已处理事件 happened-after (p, e) = v。 +(c) **根死亡毒化**:某链在堆非空时死于根 r,则对其后任何候选 v: +v ≤ r 不可能(r 无祖先),r ≤ v 不可能(v 在 r 之后弹出 ⟹ +lamport(v) ≤ lamport(r),而 r → v 要求严格更大,L0),故 r ∥ v, +{v} 不 critical——必须放弃。裁剪死亡同理(链的延续未知,保守放弃)。 +最晚性:扫描按 lamport 降序推进,首个满足条件的时刻即最高的切口。 + +**L12(入场检查的正确性)** *新区经由"入场 change"挂到旧历史上; +入场者看全了 L,其一切后代自动看全。* +设 IGU 候选成立(ans = L、无 uncovered)。定义入场 change 为新区中 +因果父集(显式 deps ∪ 隐式同 peer 前驱)全部落在 Events(L) 内的 +change。断言:新区每个事件 ≥ L 的全部头 ⟺ 每个入场 change 的父集 +版本覆盖 L。 +证明要点:(⇐)新区任意事件 e 沿因果链下行必经某入场 change c, +e ≥ c ≥ Events(父集) ⊇ L;(⇒)入场 change 自身是新区事件。 +跨节点straddle 情形(节点前半旧、后半新):多头 L 下节点中部的新 op +只有隐式父,其父若覆盖 L 全部头则该父 ≥ 每个头、又 ≤ 某头, +与反链性矛盾——故按节点起点父集判定与逐 op 判定等价。 +快路径:父集与 L 集合相等 ⟹ 覆盖(O(|L|),日常导入的主流形状)。 + +**依赖图**: +S1 ← L4;S2 ← L5;S3 ← L9 ← {L3, L6, L7, L8};T ← L10 ← L0; +S5b(C1) ← L3 + "无死路 ⟹ Max(C) 的每个成员被双侧到达"(未证,猜想)。 + +**引理与现有测试的对应**(代码一致性的桥): +四套随机 oracle 测试 ≈ S1/S2/S3 的随机检验; +三个定向单测(trimmed、左多头、右多头)分别钉 L9 的三个分支; +criss-cross ladder 测试钉"tips 去重后复杂度线性"(性能声明,不进证明范围)。 + +--- + +## 6. 待定问题(需要维护者拍板) + +- **Q1** 猜想 C1(无死路 ⟹ ans = 精确 meet)要不要写进规格? + 现有 oracle 在 Checkout 模式下不检查精确性,纳入则需补测试。 +- **Q2** Linear 的外延表述("新区是 ≤-全序链且因果晚于 L") + 与差量计算器的实际假设是否一致? +- **Q3**(已部分解决,2026-08-01)movable tree issue 证明守卫公理对 + Tree 不成立,故不再把守卫当作 IGU 正确性的依据:入场检查(L12)使 + S3/S4 直接成立。守卫仍是 text/list 在 Checkout 模式下的兜底,其公理 + 化留给 Q7。 +- **Q7**(已核查,2026-08-01:安全)Checkout 模式下 meet 非 critical 时 + Tree 的 checkout_diff 是否一致?机制审计发现 Tree 的 Checkout 路径是 + **相对**计算(retreat/forward 都按 lca 前沿的 change 起点 lamport 开窗, + 窗外 op 静默跳过)——窗口隐含 critical 假设。但可证明修复后恒安全: + 区域内经会合进入公共历史的 op 必 ≥ 某 meet 头(在窗口内);与 meet 头 + 并发的 op 必产生 uncovered 死路 → L11 扫描 → critical 基准 → 窗口从 + 基准起点覆盖全区域。实证:非 critical meet 菱形(tree/movable list/ + text)与低 lamport 并发分支两组 checkout 探针全部 canonical,已钉为 + 回归测试 `checkout_across_non_critical_meet_stays_canonical` 与 + `checkout_with_low_lamport_concurrent_branch_stays_canonical` + (后者显式保护"扫描 ↔ Tree 窗口"的耦合:削弱扫描会静默破坏它)。 +- **Q8**(已解决,2026-08-01:澄清而非改行为)深入推演后确认这是 + **刻意的双模式设计**:顶层返回值是**方向模式**(origin),其唯一消费者 + `DocState::apply_diff` 只用它做方向敏感的死容器缓存策略(Checkout 可能 + 倒退 → 全清;其余模式必为前进 → 只清 alive 标记),语义正确;逐容器的 + `InternalContainerDiff::diff_mode` 才是**计算模式**(各计算器如实上报, + Persist 降级后亦然),`need_check`/`need_compare` 消费的是它,配套一致。 + 风险只在"易混淆",已修:`calc_diff_internal` 内部局部量改名 + `calc_mode`,两处消费点各加不变式注释。 +- **Q9**(新,维护性)`find_path` 使用独立的旧版 `_find_common_ancestor` + 实现(仅服务诊断 API `find_id_spans_between`,不参与状态变更; + relay 形状实证输出精确)。双实现存在漂移风险,建议择机合并或改为 + vv 差集直接计算。 +- **Q4** trimmed 情形:uncovered 但 ans 触及裁剪边界时保留非空 ans, + 其安全性依赖 oplog 层把重放起点钳制到浅历史根部。 + 规格按"走查 + 钳制成对成立"写,还是要求走查单独成立? +- **Q5** Lean 先证**模型层**(本文档的数学对象), + Rust 代码一致性由穷举小规模 oracle 测试桥接——可接受, + 还是希望直接做 Rust 代码级验证(Verus/Creusot,成本高得多)? +- **Q6** 三个快速路径:纳入证明范围,还是从代码中删除以换取更小的可信基? + 其中"单-单同 peer"窥孔的证明要用到 A2 划分公理,是三者中最绕的。 + +--- + +## 7. 词汇表 + +| 术语 | 含义 | +|---|---| +| CRDT | 一类允许离线并发编辑、合并后自动一致的数据结构;本文档不依赖其细节 | +| peer | 一个参与编辑的副本/设备 | +| op(操作) | 一次不可再分的修改,编号 (p, c) | +| Change / 节点 | 同一 peer 连续操作的存储打包单位 | +| span / 段 | 节点的一个前缀切片,遍历的基本单位 | +| lamport | 逻辑时间戳:沿 happened-before 严格递增(a → b ⟹ lamport(a) < lamport(b)) | +| parents / deps | 事件的父事件集(论文记 e.parents;Loro 代码称 deps) | +| happened-before → | a → b:a 在 b 的因果过去(由早指向晚,Lamport 方向;Eg-walker §2.2) | +| ∥(并发) | a ∥ b ⟺ a ≠ b ∧ a ↛ b ∧ b ↛ a | +| Events(V) / Version(G) | 论文记法(§2.3):版本的祖先闭包 / 因果闭子图的前沿,两者互逆 | +| V_crit | Eg-walker §3.6:合并的理论最优起点——发生在双方之前的最晚 critical version(Loro 已实现其单头近似,见 L11) | +| frontier / 前沿 | 用"最新端点"反链表示一个版本 | +| 反链 | 集合中两两互不为祖先 | +| 因果闭集 | 包含成员的所有祖先的操作集;与合法版本一一对应 | +| 版本向量 | 因果闭集的紧凑表示:每 peer 一个前缀长度 | +| C(L,R) | 两版本的公共历史 ancestry(L) ∩ ancestry(R) | +| meet 前沿 | C 的极大元反链——版本格最大下界;算法的候选起点(旧文借称 "LCA",不严格) | +| critical version | 能把事件图一刀两断的版本(Eg-walker §3.5);重放起点的合法性判据,∅ 平凡成立 | +| import / checkout | 导入远端更新 / 切换到任意历史版本 | +| 重放(replay) | 从起点按因果序重演操作以计算状态差 | +| tips / 牌子 | 路径携带的分岔点标记,用于死路的冗余性判定 | +| unmatched / 死路 | 未与对方会合而终止的路径 | +| uncovered | 存在牌子不在 ans 祖先集内的死路 ⟹ 真并发 ⟹ 保守回退 | +| trimmed / 浅历史 | 早期历史被裁剪,依赖可能指向不可用区 | diff --git a/crates/loro-internal/docs/diff_calc.md b/crates/loro-internal/docs/diff_calc.md index 22e7d78b6..769358da9 100644 --- a/crates/loro-internal/docs/diff_calc.md +++ b/crates/loro-internal/docs/diff_calc.md @@ -1,23 +1,49 @@ -# Internal of Diff Calculation - -Diff calculation is the core of the `diff` command. It is responsible for calculating the difference between two versions of a container. - -# Three modes of diff calculation - -## 1. Checkout Mode - -This is the most general mode of diff calculation. It can be used whenever a user want to switch to a different version. -But it is also the slowest mode. It relies on the `ContainerHistoryCache`, which is expensive to build and maintain in memory. - -## 2. Import Mode - -This mode is used when the user imports new updates. It is faster than the checkout mode, but it is still slower than the linear mode. - -- The difference between the import mode and the checkout mode: in import mode, target version > current version. - So when calculating the `DiffCalculator` doesn't need to rely on `ContainerHistoryCache`, except for the Tree container. -- The difference between the import mode and the linear mode: in linear mode, all the imported updates are ordered, no concurrent update exists. - so there is no need to build CRDTs for the calculation - -## 3. Linear Mode - -This mode is used when we don't need to build CRDTs to calculate the difference. It is the fastest mode. +# Internal Diff Calculation + +Diff calculation produces the state patch between two versions. Checkout, +forking, import, and revert all use this path. + +## Replay base and changed containers + +`OpLog::iter_from_replay_base_causally` first chooses a replay base. The safe +base is a **critical version** in the Eg-walker sense (arXiv:2409.14252 §3.5): +a version that no concurrency crosses. When a concurrent branch invalidates +the candidate, the DAG retreats the base to the latest single-head critical +version, which can be older than the meet of the two versions. + +An old base is not evidence that every container in the replay range changed. +`DiffCalculator::calc_diff_internal` derives the changed container set from the +version-vector difference and routes common history only to those calculators. +Otherwise each unchanged List/Text/MovableList can trigger its own full-history +tracker rebuild. + +The DAG walk follows explicit dependencies plus the implicit previous counter +of the same peer. An implicit path may be redundant when an explicit relay +dependency already contains that predecessor. The walk remembers the dependency +tip where an unmatched path split and performs a targeted ancestor lookup from +the candidate common frontiers: + +- a covered tip is another route into already-common history, so `from` remains + the replay base; +- an uncovered tip is a real concurrent branch, so the conservative base is + retained. + +This lookup runs only for unmatched branch tips and prunes by visited DAG nodes +and Lamport time. It does not calculate a causal version for every new peer and +does not compare each one with the complete `from` version vector. + +## Diff modes + +- `Checkout` is the general and slowest mode. It can move in either direction + and may use `ContainerHistoryCache`. +- `Import` requires `to > from`, but some imported operations may be concurrent + with `from`. +- `ImportGreaterUpdates` additionally guarantees that every imported operation + is causally after `from`, so the replay base is `from`. +- `Linear` additionally guarantees that imported operations are ordered, so + diff calculation does not need to build CRDT trackers. + +List, Text, and MovableList still rebuild their trackers from CRDT IDs when +retreating, when their source context is incomplete, or when shallow history +requires it. That fallback is a correctness requirement, not a replay-base +optimization. diff --git a/crates/loro-internal/src/dag.rs b/crates/loro-internal/src/dag.rs index 9b14434ef..9332f53b6 100644 --- a/crates/loro-internal/src/dag.rs +++ b/crates/loro-internal/src/dag.rs @@ -484,6 +484,24 @@ where ans } +/// Finds the replay base and diff mode for the transition `left -> right`. +/// +/// Contract (see `docs/critical-version-spec.md` for definitions and proofs): +/// - S1: every id in the returned frontier is a common ancestor of both sides; +/// - S2: the returned frontier is an antichain; +/// - S3: a non-`Checkout` mode additionally guarantees that the base equals +/// `left` and that EVERY newly imported event is causally after every head +/// of `left` (i.e. `left` is a critical version of the union graph in the +/// Eg-walker sense, arXiv:2409.14252 §3.5) — enforced for multi-head `left` +/// by `new_region_after_all_left_heads`; +/// - when a genuinely concurrent branch invalidates the candidate base, the +/// base retreats to the latest single-head critical version +/// (`latest_single_head_critical_version`), or to the empty version when no +/// such cut exists. +/// +/// Downstream consumers rely on these guarantees to skip CRDT adjudication +/// (tree/map fast paths) and to bound their replay windows (tree checkout); +/// see the comment in `diff_calc/tree.rs::checkout_diff`. fn _find_common_ancestor_new<'a, F, D>( get: &'a F, left: &Frontiers, @@ -547,7 +565,8 @@ where let mut is_linear = left.len() <= 1 && right.len() == 1; let mut is_right_greater = true; - let mut has_unmatched_branch = false; + let mut unmatched_branches = FxHashSet::default(); + let mut has_unresolved_unmatched_branch = false; let mut ans: Frontiers = Default::default(); fn ids_to_ord_id_spans<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( @@ -623,6 +642,227 @@ where }) } + /// Whether every tip in `tips` is contained in the ancestors of `frontiers` + /// (the tip itself counts). One walk answers all tips: it starts from every + /// frontier with a shared visited set and prunes by the lowest lamport among + /// the still-unproven tips, so the ancestor region is traversed at most once + /// per call instead of once per (tip, frontier) pair. + fn all_tips_covered_by_ancestors<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + get: &'a F, + frontiers: &Frontiers, + tips: &FxHashSet, + ) -> bool { + let mut remaining = Vec::with_capacity(tips.len()); + for id in tips.iter() { + let Some(span) = OrdIdSpan::from_dag_node(*id, get) else { + return false; + }; + remaining.push(span); + } + + if remaining.is_empty() { + return true; + } + + let mut min_lamport = remaining + .iter() + .map(|tip| tip.lamport_last()) + .min() + .unwrap(); + let mut visited = FxHashSet::default(); + let mut pending = Vec::new(); + for frontier in frontiers.iter() { + if let Some(node) = OrdIdSpan::from_dag_node(frontier, get) { + pending.push(node); + } + } + + while let Some(node) = pending.pop() { + let len_before = remaining.len(); + remaining.retain(|tip| !node.contains_id(tip.id_last())); + if remaining.is_empty() { + return true; + } + + if remaining.len() != len_before { + min_lamport = remaining + .iter() + .map(|tip| tip.lamport_last()) + .min() + .unwrap(); + } + + // Ancestors of `node` all have lamport strictly below the node's + // start, so nothing below `min_lamport` can contain a remaining tip. + if node.lamport_last() < min_lamport { + continue; + } + + if !visited.insert(node.id_start()) { + continue; + } + + if let Some(deps) = deps_to_ord_id_spans(&node, get) { + for dep in deps { + pending.push(dep); + } + } + } + + false + } + + /// The latest single-head critical version (Eg-walker §3.5/§3.6) of the + /// union graph `ancestry(left) ∪ ancestry(right)`: the newest event `v` + /// such that every other event in the union is an ancestor of `v` or + /// causally after `v`. Replaying from `{v}` is safe because no concurrency + /// crosses it, and it skips the fully-synced prefix of common history that + /// the old `∅` fallback would replay. + /// + /// Method: a one-colour descent from both frontiers using the same + /// coalesce/align machinery as the main walk. The pending heap is always a + /// cut of the unexplored region, so the first moment it narrows to a + /// single span, that span's `id_last` is the latest such `v`. If a chain + /// dies at a root or at trimmed history while other chains remain, its + /// endpoint is concurrent with everything below the current level, so no + /// later singleton can be valid — bail out to the empty version, which is + /// the old behaviour. + fn latest_single_head_critical_version<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + get: &'a F, + left: &Frontiers, + right: &Frontiers, + ) -> Frontiers { + let mut queue: BinaryHeap = BinaryHeap::new(); + let Some(spans) = ids_to_ord_id_spans(left, get) else { + return Default::default(); + }; + queue.extend(spans); + let Some(spans) = ids_to_ord_id_spans(right, get) else { + return Default::default(); + }; + queue.extend(spans); + + while let Some(mut node) = queue.pop() { + while let Some(other) = queue.peek() { + if node == *other || node.id_last() == other.id_last() { + queue.pop(); + } else { + break; + } + } + + if queue.is_empty() { + return node.id_last().into(); + } + + if let Some(other) = queue.peek() { + if node.contains_id(other.id_last()) { + node.len = (other.id_last().counter - node.id.counter + 1) as usize; + queue.push(node); + continue; + } + + if node.len > 1 { + node.len = if other.lamport_last() >= node.lamport { + (other.lamport_last() - node.lamport + 1).min(node.len as u32 - 1) as usize + } else { + 1 + }; + queue.push(node); + continue; + } + } + + match deps_to_ord_id_spans(&node, get) { + Some(deps) if !deps.is_empty() => { + for dep in deps { + queue.push(dep); + } + } + _ => return Default::default(), + } + } + + Default::default() + } + + /// Verifies the `ImportGreaterUpdates` contract for a multi-head `left`: + /// every event in `Events(right) − Events(left)` must be causally after + /// every member of `left` (i.e. `left` must be a critical version of the + /// union graph). The main walk only proves version inclusion, which is + /// weaker: a new branch can enter old history through one left head while + /// staying concurrent with another, and the tree calculator's fast path + /// applies such updates without adjudication (see + /// `docs/critical-version-spec.md` §4.1). + /// + /// Method: descend from `right` over new changes only. A change whose + /// causal parents (explicit deps plus the implicit same-peer predecessor) + /// are all old is an entry point of the new region; its parents must + /// causally cover every left head. Non-entry changes inherit coverage + /// through their new parents. The common healthy shape — a change whose + /// deps equal the left frontier — passes with a set comparison and no + /// graph walk. Trimmed history fails conservatively. + fn new_region_after_all_left_heads<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + get: &'a F, + left: &Frontiers, + right: &Frontiers, + ) -> bool { + let left_ids: FxHashSet = left.iter().collect(); + let is_old = |id: ID| -> bool { + if left_ids.contains(&id) { + return true; + } + let mut single = FxHashSet::default(); + single.insert(id); + all_tips_covered_by_ancestors(get, left, &single) + }; + + let mut visited: FxHashSet = FxHashSet::default(); + let mut stack: Vec = Vec::new(); + for id in right.iter() { + if is_old(id) { + continue; + } + let Some(span) = OrdIdSpan::from_dag_node(id, get) else { + return false; + }; + stack.push(span); + } + + while let Some(span) = stack.pop() { + if !visited.insert(span.id_start()) { + continue; + } + + let Some(parents) = deps_to_ord_id_spans(&span, get) else { + return false; + }; + let mut entry_point = true; + let mut old_parents = Frontiers::default(); + for parent in parents { + let pid = parent.id_last(); + if is_old(pid) { + old_parents.push(pid); + } else { + entry_point = false; + stack.push(parent); + } + } + + if entry_point { + if &old_parents == left { + continue; + } + + if !all_tips_covered_by_ancestors(get, &old_parents, &left_ids) { + return false; + } + } + } + + true + } + fn contains_in_ancestors<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( get: &'a F, frontier: ID, @@ -657,23 +897,44 @@ where false } - let mut queue: BinaryHeap<(OrdIdSpan, NodeType)> = BinaryHeap::new(); + // The third tuple item carries the dependency tips at which this path first + // split from its sibling paths. If the path dies without meeting the other + // side, those tips tell us whether it was a real concurrent branch or merely + // a redundant route into an ancestor we already found. + let mut queue: BinaryHeap<(OrdIdSpan, NodeType, Vec)> = BinaryHeap::new(); for span in ids_to_ord_id_spans(left, get).unwrap() { - queue.push((span, NodeType::A)); + let branch_tips = (left.len() > 1) + .then(|| vec![span.id_last()]) + .unwrap_or_default(); + queue.push((span, NodeType::A, branch_tips)); } for span in ids_to_ord_id_spans(right, get).unwrap() { - queue.push((span, NodeType::B)); + let branch_tips = (right.len() > 1) + .then(|| vec![span.id_last()]) + .unwrap_or_default(); + queue.push((span, NodeType::B, branch_tips)); } - while let Some((mut node, mut node_type)) = queue.pop() { - while let Some((other_node, other_type)) = queue.peek() { + while let Some((mut node, mut node_type, mut branch_tips)) = queue.pop() { + while let Some((other_node, other_type, _)) = queue.peek() { if node == *other_node || node.id_last() == other_node.id_last() { if node_type != *other_type { node_type = NodeType::Shared; } - queue.pop(); + let (_, _, other_branch_tips) = queue.pop().unwrap(); + // Union, not concatenation. On criss-cross histories the same + // tips arrive from both parents at every re-merge; concatenating + // doubles the vec per sync round, and the per-dep clone below + // then makes the walk O(2^rounds). Distinct tips stay few: they + // are only minted for multi-head initial frontiers and at each + // side's first split. + for tip in other_branch_tips { + if !branch_tips.contains(&tip) { + branch_tips.push(tip); + } + } } else { break; } @@ -685,8 +946,11 @@ where } if queue.is_empty() { - has_unmatched_branch = true; - is_right_greater = false; + if branch_tips.is_empty() { + unmatched_branches.insert(node.id_last()); + } else { + unmatched_branches.extend(branch_tips); + } break; } @@ -698,25 +962,35 @@ where if let Some(other) = queue.peek() { if node.contains_id(other.0.id_last()) && node_type != other.1 { node.len = (other.0.id_last().counter - node.id.counter + 1) as usize; - queue.push((node, node_type)); + queue.push((node, node_type, branch_tips)); continue; } if node.len > 1 { + // The `min(..., len - 1)` cap guarantees strict progress when + // lamports tie: aligning to the other's lamport alone would + // re-push the span unchanged and loop forever. It is also what + // makes the termination measure decrease on this branch. node.len = if other.0.lamport_last() >= node.lamport { (other.0.lamport_last() - node.lamport + 1).min(node.len as u32 - 1) as usize } else { 1 }; - queue.push((node, node_type)); + queue.push((node, node_type, branch_tips)); continue; } } if let Some(deps) = deps_to_ord_id_spans(&node, get) { if !deps.is_empty() { + let starts_new_branches = branch_tips.is_empty() && deps.len() > 1; for dep in deps { - queue.push((dep, node_type)); + let child_branch_tips = if starts_new_branches { + vec![dep.id_last()] + } else { + branch_tips.clone() + }; + queue.push((dep, node_type, child_branch_tips)); } is_linear = false; continue; @@ -724,9 +998,9 @@ where } else { // The dependency is on trimmed shallow history. The exact ancestor is // not representable in the current DAG, so fall back to a conservative - // checkout base. - has_unmatched_branch = true; - is_right_greater = false; + // checkout base. No tip needs recording: this flag short-circuits the + // coverage check entirely. + has_unresolved_unmatched_branch = true; continue; } @@ -734,16 +1008,53 @@ where // Some checkout calculators still require replaying from a base that // includes every branch whose operation positions may affect the diff. // In non-linear checkout mode, an earlier common ancestor is a valid - // conservative base even when it is not the mathematical LCA. - has_unmatched_branch = true; - is_right_greater = false; + // conservative base even when it is not the meet of the two versions. + if branch_tips.is_empty() { + unmatched_branches.insert(node.id_last()); + } else { + unmatched_branches.extend(branch_tips); + } continue; } } ans = shrink_ancestor_frontiers(&ans, get); - if has_unmatched_branch && !has_trimmed_history_deps(&ans, get) { - ans = Default::default(); + // A branch can look unmatched merely because a shared node was found first + // and was therefore not expanded. In that case another queued path may still + // walk into one of the shared node's ancestors. That path is redundant, not a + // concurrent branch. Only fall back when an unmatched tip is not causally + // covered by the common ancestors we found. + let has_uncovered_unmatched_branch = has_unresolved_unmatched_branch + || !all_tips_covered_by_ancestors(get, &ans, &unmatched_branches); + if has_uncovered_unmatched_branch && !has_trimmed_history_deps(&ans, get) { + // A genuine concurrent branch invalidates the meet as a replay base. + // Instead of always retreating to the beginning of history, retreat to + // the latest single-head critical version below both sides (empty when + // no such cut exists, which matches the old behaviour). + ans = latest_single_head_critical_version(get, left, right); + } + + if has_uncovered_unmatched_branch { + is_right_greater = false; + } else if &ans == left { + // An A-only path may also have been a redundant path into `ans`. Once all + // such paths are proved covered, equality with `left` is the exact proof + // that `right` causally includes `left`. + is_right_greater = true; + } + + if is_right_greater && left.len() > 1 && !new_region_after_all_left_heads(get, left, right) { + // Version inclusion holds, but part of the new region entered old + // history through a strict subset of the left heads and is therefore + // concurrent with the others. `ImportGreaterUpdates` promises "no + // update is concurrent to the current version", so claiming it here + // lets fast paths (notably the tree calculator's) apply the new ops + // without adjudicating against the concurrent existing branch, and the + // materialized state diverges from a full replay. Demote to the + // conservative mode and retreat the base to a safe cut so the + // competing branches are replayed together. + is_right_greater = false; + ans = latest_single_head_critical_version(get, left, right); } let mode = if is_right_greater { @@ -975,7 +1286,7 @@ mod tests { for id in actual.iter() { assert!( left_ancestors.contains(&id) && right_ancestors.contains(&id), - "actual LCA id {id} must be common: left={left:?} right={right:?} actual={actual:?} expected={expected:?} mode={mode:?}\ndag={dag:?}", + "every replay-base id {id} must be common: left={left:?} right={right:?} actual={actual:?} expected={expected:?} mode={mode:?}\ndag={dag:?}", ); } @@ -984,7 +1295,7 @@ mod tests { if a != b { assert!( !is_ancestor(dag, a, b), - "actual LCA must be a minimal frontier set: left={left:?} right={right:?} actual={actual:?} expected={expected:?} mode={mode:?}\ndag={dag:?}", + "the replay base must be a minimal frontier set: left={left:?} right={right:?} actual={actual:?} expected={expected:?} mode={mode:?}\ndag={dag:?}", ); } } @@ -997,7 +1308,7 @@ mod tests { ); assert_eq!( &actual, left, - "non-checkout mode must use left as LCA: left={left:?} right={right:?} mode={mode:?}" + "non-checkout mode must use left as the replay base: left={left:?} right={right:?} mode={mode:?}" ); for id in left.iter() { assert!( @@ -1211,9 +1522,66 @@ mod tests { assert_eq!(mode, DiffMode::ImportGreaterUpdates); } + #[test] + fn common_ancestor_ignores_implicit_predecessor_covered_by_explicit_dependency() { + let peer_one_previous = node(1, 0, 1, 0, Frontiers::default()); + let relay = node(2, 0, 1, 1, peer_one_previous.id.into()); + // This is peer 1's next change, so it implicitly depends on + // `peer_one_previous`. Its explicit dependency on `relay` already covers + // that predecessor, though, so the two paths are not concurrent. + let peer_one_next = node(1, 1, 1, 2, relay.id.into()); + let dag = TestDag::new( + vec![peer_one_previous, relay.clone(), peer_one_next.clone()], + peer_one_next.id.into(), + ); + + let (ancestor, mode) = dag.find_common_ancestor(&relay.id.into(), &peer_one_next.id.into()); + assert_eq!(ancestor, relay.id.into()); + assert_eq!(mode, DiffMode::ImportGreaterUpdates); + } + + #[test] + fn common_ancestor_ignores_implicit_predecessor_covered_transitively() { + let peer_one_previous = node(1, 0, 1, 0, Frontiers::default()); + let first_relay = node(2, 0, 1, 1, peer_one_previous.id.into()); + let current_frontier = node(3, 0, 1, 2, first_relay.id.into()); + let peer_one_next = node(1, 1, 1, 3, current_frontier.id.into()); + let dag = TestDag::new( + vec![ + peer_one_previous, + first_relay, + current_frontier.clone(), + peer_one_next.clone(), + ], + peer_one_next.id.into(), + ); + + let (ancestor, mode) = + dag.find_common_ancestor(¤t_frontier.id.into(), &peer_one_next.id.into()); + assert_eq!(ancestor, current_frontier.id.into()); + assert_eq!(mode, DiffMode::ImportGreaterUpdates); + } + + #[test] + fn common_ancestor_keeps_uncovered_implicit_predecessor_conservative() { + let peer_one_previous = node(1, 0, 1, 0, Frontiers::default()); + let concurrent = node(2, 0, 1, 1, Frontiers::default()); + let peer_one_next = node(1, 1, 1, 2, concurrent.id.into()); + let dag = TestDag::new( + vec![peer_one_previous, concurrent.clone(), peer_one_next.clone()], + peer_one_next.id.into(), + ); + + let (ancestor, mode) = + dag.find_common_ancestor(&concurrent.id.into(), &peer_one_next.id.into()); + assert_eq!(ancestor, Frontiers::default()); + assert_eq!(mode, DiffMode::Checkout); + } + #[test] fn common_ancestor_falls_back_when_right_adds_concurrent_branch_from_shared_root() { let root = node(1, 0, 1, 0, Frontiers::default()); + let root_id = root.id; let left = node(2, 0, 1, 1, root.id.into()); let concurrent = node(3, 0, 1, 2, root.id.into()); let merge = node(4, 0, 1, 3, Frontiers::from([left.id, concurrent.id])); @@ -1223,10 +1591,144 @@ mod tests { ); let (ancestor, mode) = dag.find_common_ancestor(&left.id.into(), &merge.id.into()); + // The conservative base retreats to the latest single-head critical + // version below both sides — here the shared root — instead of the + // beginning of history. + assert_eq!(ancestor, root_id.into()); + assert_eq!(mode, DiffMode::Checkout); + } + + #[test] + fn common_ancestor_falls_back_when_right_frontiers_add_concurrent_branch() { + let root = node(1, 0, 1, 0, Frontiers::default()); + let left = node(2, 0, 1, 1, root.id.into()); + let concurrent = node(3, 0, 1, 2, root.id.into()); + let right = Frontiers::from([left.id, concurrent.id]); + let dag = TestDag::new(vec![root, left.clone(), concurrent], right.clone()); + + let (ancestor, mode) = dag.find_common_ancestor(&left.id.into(), &right); + // Conservative base = the shared root (latest single-head critical + // version), not the beginning of history. + assert_eq!(ancestor, ID::new(1, 0).into()); + assert_eq!(mode, DiffMode::Checkout); + } + + #[test] + fn common_ancestor_falls_back_when_left_frontiers_add_concurrent_branch() { + let root = node(1, 0, 1, 0, Frontiers::default()); + let shared = node(2, 0, 1, 1, root.id.into()); + let concurrent = node(3, 0, 1, 2, root.id.into()); + let left = Frontiers::from([shared.id, concurrent.id]); + let dag = TestDag::new(vec![root, shared.clone(), concurrent], left.clone()); + + // The concurrent left head walks below `shared` without ever meeting the + // right side. Only the tip seeded for the multi-element left frontier can + // prove it uncovered; the deep node it dies at is itself covered. + let (ancestor, mode) = dag.find_common_ancestor(&left, &shared.id.into()); + // Conservative base = the shared root (latest single-head critical + // version), not the beginning of history. + assert_eq!(ancestor, ID::new(1, 0).into()); + assert_eq!(mode, DiffMode::Checkout); + } + + #[test] + fn common_ancestor_falls_back_when_walk_reaches_trimmed_history() { + // Peer 9's changes are not present in the DAG, as after shallow trimming. + // Both sides die at the missing dependency inside the main walk, which + // must force the conservative empty base regardless of tip coverage. + let a = node(1, 0, 1, 5, ID::new(9, 5).into()); + let b = node(2, 0, 1, 6, ID::new(9, 7).into()); + let dag = TestDag::new(vec![a.clone(), b.clone()], Frontiers::from([a.id, b.id])); + + let (ancestor, mode) = dag.find_common_ancestor(&a.id.into(), &b.id.into()); assert_eq!(ancestor, Frontiers::default()); assert_eq!(mode, DiffMode::Checkout); } + #[test] + fn import_greater_updates_requires_new_ops_after_all_left_heads() { + // Movable-tree staleness issue shape: B3 → B4..B6 and B3 → A0 → A1..A3. + // from = {A0, B6}, to = {A3, B6}. Version inclusion holds + // (vv(to) ⊇ vv(from)), yet A1..A3 are concurrent with the left head B6, + // so ImportGreaterUpdates must not be claimed: the tree calculator's + // fast path would apply A1..A3 without adjudicating against B4..B6. + // The base must retreat to the latest critical version {B3}. + let b_creates = node(2, 0, 4, 0, Frontiers::default()); + let b_moves = node(2, 4, 3, 4, ID::new(2, 3).into()); + let a0 = node(1, 0, 1, 4, ID::new(2, 3).into()); + let a_moves = node(1, 1, 3, 5, ID::new(1, 0).into()); + let dag = TestDag::new( + vec![b_creates, b_moves, a0, a_moves], + Frontiers::from([ID::new(1, 3), ID::new(2, 6)]), + ); + + let left = Frontiers::from([ID::new(1, 0), ID::new(2, 6)]); + let right = Frontiers::from([ID::new(1, 3), ID::new(2, 6)]); + let (ancestor, mode) = dag.find_common_ancestor(&left, &right); + assert_eq!(ancestor, ID::new(2, 3).into()); + assert_eq!(mode, DiffMode::Checkout); + } + + #[test] + fn import_greater_updates_kept_when_new_ops_depend_on_all_left_heads() { + // Healthy multi-head import: the new change's deps equal the whole + // left frontier, so every new op is causally after all of it and the + // fast path must be preserved (entry check passes by set equality). + let root = node(1, 0, 1, 0, Frontiers::default()); + let x = node(2, 0, 1, 1, ID::new(1, 0).into()); + let y = node(3, 0, 1, 1, ID::new(1, 0).into()); + let merge = node( + 4, + 0, + 1, + 2, + Frontiers::from([ID::new(2, 0), ID::new(3, 0)]), + ); + let dag = TestDag::new(vec![root, x, y, merge], ID::new(4, 0).into()); + + let left = Frontiers::from([ID::new(2, 0), ID::new(3, 0)]); + let (ancestor, mode) = dag.find_common_ancestor(&left, &ID::new(4, 0).into()); + assert_eq!(ancestor, left); + assert_eq!(mode, DiffMode::ImportGreaterUpdates); + } + + #[test] + fn common_ancestor_criss_cross_ladder_stays_linear() { + // Two peers that each merge both previous heads every round — the shape + // ordinary bidirectional sync produces. Branch tips re-merge at every + // rung; without deduplication in the tip union the walk is O(2^rounds) + // (~minutes at this depth), with it the walk is linear (~microseconds). + const ROUNDS: usize = 26; + let root = node(3, 0, 1, 0, Frontiers::default()); + let mut a_head = node(1, 0, 1, 1, root.id.into()); + let mut b_head = node(2, 0, 1, 1, root.id.into()); + let mut nodes = vec![root.clone(), a_head.clone(), b_head.clone()]; + for k in 1..=ROUNDS { + let deps = Frontiers::from([a_head.id_last(), b_head.id_last()]); + let lamport = (k as Lamport) * 2; + let next_a = node(1, k as Counter, 1, lamport, deps.clone()); + let next_b = node(2, k as Counter, 1, lamport, deps); + nodes.push(next_a.clone()); + nodes.push(next_b.clone()); + a_head = next_a; + b_head = next_b; + } + let frontier = Frontiers::from([a_head.id_last(), b_head.id_last()]); + let dag = TestDag::new(nodes, frontier.clone()); + + let start = std::time::Instant::now(); + let (ancestor, mode) = dag.find_common_ancestor(&root.id.into(), &frontier); + assert_eq!(ancestor, root.id.into()); + assert_eq!(mode, DiffMode::ImportGreaterUpdates); + // Generous 4-orders-of-magnitude margin over the fixed cost; the broken + // exponential path needs minutes here even in release mode. + assert!( + start.elapsed() < std::time::Duration::from_secs(10), + "criss-cross walk took {:?}; branch-tip growth is no longer linear", + start.elapsed() + ); + } + #[test] fn common_ancestor_keeps_target_when_checking_out_to_ancestor_with_extra_branch() { let root = node(1, 0, 1, 0, Frontiers::default()); diff --git a/crates/loro-internal/src/diff_calc.rs b/crates/loro-internal/src/diff_calc.rs index 1af8ccd60..7e7df462c 100644 --- a/crates/loro-internal/src/diff_calc.rs +++ b/crates/loro-internal/src/diff_calc.rs @@ -92,7 +92,7 @@ pub(crate) enum DiffMode { /// /// It has stricter requirements than `Import`. /// - All the updates are greater than the current version. No update is concurrent to the current version. - /// - So LCA is always the `from` version + /// - So the replay base is always the `from` version ImportGreaterUpdates, /// This mode is used when we don't need to build CRDTs to calculate the difference. It is the fastest mode. /// @@ -108,7 +108,43 @@ pub(crate) struct DiffCalcVersionInfo<'a> { to_vv: &'a VersionVector, from_frontiers: &'a Frontiers, to_frontiers: &'a Frontiers, - lca_vv: &'a VersionVector, + replay_base_vv: &'a VersionVector, +} + +fn changed_containers_between( + oplog: &OpLog, + before: &VersionVector, + after: &VersionVector, +) -> FxHashSet { + let (retreat, forward) = before.diff_iter(after); + let mut containers = FxHashSet::default(); + // Only `op.container` is needed, so scan the borrowed changes directly + // instead of materializing a cloned RichOp per op via `oplog.iter_ops`. + let mut last_container = None; + for span in retreat.chain(forward) { + for change in oplog.change_store().iter_changes(span) { + let start_counter = span.counter.min().max(change.id.counter); + let end_counter = span.counter.norm_end(); + let start = change + .ops + .binary_search_by(|op| op.ctr_last().cmp(&start_counter)) + .unwrap_or_else(|e| e); + for op in &change.ops.vec()[start..] { + if op.counter >= end_counter { + break; + } + + // Consecutive ops overwhelmingly share a container; skip the + // hash probe when it hasn't changed. + if last_container != Some(op.container) { + containers.insert(op.container); + last_container = Some(op.container); + } + } + } + } + + containers } impl DiffCalculator { @@ -156,9 +192,27 @@ impl DiffCalculator { let mut merged = before.clone(); merged.merge(after); - let (lca, origin_diff_mode, iter) = - oplog.iter_from_lca_causally(before, before_frontiers, after, after_frontiers); - let mut diff_mode = origin_diff_mode; + let (replay_base, origin_diff_mode, iter) = + oplog.iter_from_replay_base_causally(before, before_frontiers, after, after_frontiers); + // A conservative replay base may be much older than `before`. The causal replay + // still needs that common history as position context, but containers + // whose ops are present on both sides cannot contribute to the diff. + // Without this filter, every such List/Text/MovableList can trigger its + // own full-history safety rebuild below. + let changed_containers = + (&replay_base != before).then(|| changed_containers_between(oplog, before, after)); + // Two distinct mode values live in this function — do not conflate them: + // - `origin_diff_mode` describes the DIRECTION of the transition + // (Checkout can go backwards; the other modes imply `after ⊇ before`). + // It is what this function returns, and its consumer + // (`DocState::apply_diff`) uses it only for direction-sensitive + // policies such as the dead-containers cache. + // - `calc_mode` is the mode the calculators actually COMPUTE with. A + // persistent calculator always computes in Checkout mode, and each + // calculator reports its own effective mode in the per-container + // `InternalContainerDiff::diff_mode`, which is what the state layer's + // per-container logic (`need_check` / `need_compare`) consumes. + let mut calc_mode = origin_diff_mode; match &mut self.retain_mode { DiffCalculatorRetainMode::Once { used } => { if *used { @@ -166,12 +220,12 @@ impl DiffCalculator { } } DiffCalculatorRetainMode::Persist => { - diff_mode = DiffMode::Checkout; + calc_mode = DiffMode::Checkout; } } let affected_set = { - loro_common::debug!("LCA: {:?} mode={:?}", &lca, diff_mode); + loro_common::debug!("replay_base: {:?} mode={:?}", &replay_base, calc_mode); let mut started_set = FxHashSet::default(); for (change, (start_counter, end_counter), vv) in iter { let iter_start = change @@ -185,6 +239,13 @@ impl DiffCalculator { } let idx = op.container; + if changed_containers + .as_ref() + .is_some_and(|containers| !containers.contains(&idx)) + { + continue; + } + if let Some(filter) = container_filter { if !filter(idx) { continue; @@ -219,7 +280,7 @@ impl DiffCalculator { if !started_set.contains(&op.container) { started_set.insert(container); - calculator.start_tracking(oplog, &lca, diff_mode); + calculator.start_tracking(oplog, &replay_base, calc_mode); } if !vv.includes_vv(before) { @@ -267,7 +328,7 @@ impl DiffCalculator { to_vv: after, from_frontiers: before_frontiers, to_frontiers: after_frontiers, - lca_vv: &lca, + replay_base_vv: &replay_base, }; while !all.is_empty() { // sort by depth and lamport, ensure we iterate from top to bottom @@ -452,7 +513,7 @@ fn replay_container_ops_from_empty( let empty_frontiers = Frontiers::default(); let target_frontiers = oplog.dag.vv_to_frontiers(vv); let (_, _, iter) = - oplog.iter_from_lca_causally(&empty_vv, &empty_frontiers, vv, &target_frontiers); + oplog.iter_from_replay_base_causally(&empty_vv, &empty_frontiers, vv, &target_frontiers); for (change, (start_counter, end_counter), vv) in iter { let iter_start = change @@ -736,7 +797,7 @@ impl DiffCalculatorTrait for ListDiffCalculator { let (mut retreat, _) = info.from_vv.diff_iter(info.to_vv); let has_retreat = retreat.next().is_some(); let should_rebuild = matches!(idx.get_type(), crate::ContainerType::List) - && (has_retreat || info.lca_vv != info.from_vv || self.source_not_in_op_context); + && (has_retreat || info.replay_base_vv != info.from_vv || self.source_not_in_op_context); let diff_items = if should_rebuild { let mut merged = info.from_vv.clone(); merged.merge(info.to_vv); @@ -1615,12 +1676,13 @@ impl DiffCalculatorTrait for RichtextDiffCalculator { let (mut retreat, _) = info.from_vv.diff_iter(info.to_vv); let has_retreat = retreat.next().is_some(); let should_rebuild = has_retreat - || info.lca_vv != info.from_vv + || info.replay_base_vv != info.from_vv || *source_not_in_op_context || !oplog.shallow_since_vv().is_empty(); if should_rebuild { - // Richtext diffs can start from a tracker that only knows the LCA state as - // unknown spans. Expressing a rollback or an import from `lca != from` as local + // Richtext diffs can start from a tracker that only knows the replay-base + // state as unknown spans. Expressing a rollback or an import from a base + // older than `from` as local // edits can target the wrong visible text when the source state contains // concurrent inserts or sliced ops. The same risk exists when an op is replayed // from a dependency version that does not include the visible source state. @@ -1806,7 +1868,7 @@ impl DiffCalculatorTrait for MovableListDiffCalculator { ) -> (InternalDiff, DiffMode) { let (mut retreat, _) = info.from_vv.diff_iter(info.to_vv); let has_retreat = retreat.next().is_some(); - if has_retreat || info.lca_vv != info.from_vv || self.list.source_not_in_op_context { + if has_retreat || info.replay_base_vv != info.from_vv || self.list.source_not_in_op_context { let mut merged = info.from_vv.clone(); merged.merge(info.to_vv); self.rebuild_full_tracker(idx, oplog, &merged); @@ -2034,3 +2096,219 @@ fn test_size() { let size = std::mem::size_of_val(&calc); assert!(size < 50, "ContainerDiffCalculator size: {}", size); } + +/// A doc with a movable list "items" holding a nested map (`value: 0`) plus +/// unrelated list/text containers, committed as peer 1. +#[cfg(test)] +fn movable_list_fixture() -> crate::LoroDoc { + use crate::{LoroDoc, MapHandler}; + + let base = LoroDoc::new_auto_commit(); + base.set_peer_id(1).unwrap(); + let nested = base + .get_movable_list("items") + .push_container(MapHandler::new_detached()) + .unwrap(); + nested.insert("value", 0).unwrap(); + base.get_list("unrelated-list").push("a").unwrap(); + base.get_text("unrelated-text") + .insert(0, "a", crate::cursor::PosType::Unicode) + .unwrap(); + base.commit_then_renew(); + base +} + +#[cfg(test)] +fn nested_map(doc: &crate::LoroDoc) -> crate::MapHandler { + use crate::handler::ValueOrHandler; + + match doc.get_movable_list("items").get_(0).unwrap() { + ValueOrHandler::Handler(handler) => handler.into_map().unwrap(), + ValueOrHandler::Value(value) => panic!("expected nested map, got {value:?}"), + } +} + +/// Asserts the diff round produced exactly one internal map diff, for +/// `expected_idx`, whose `key` entry resolves to `expected`. +#[cfg(test)] +fn assert_single_map_value_diff( + diffs: &[InternalContainerDiff], + expected_idx: ContainerIdx, + key: &str, + expected: LoroValue, +) { + assert_eq!(diffs.len(), 1); + let diff = &diffs[0]; + assert_eq!(diff.idx, expected_idx); + let DiffVariant::Internal(InternalDiff::Map(map_diff)) = &diff.diff else { + panic!("expected an internal map diff, got {:?}", diff.diff); + }; + let value = map_diff + .updated + .get(&InternalString::from(key)) + .unwrap_or_else(|| panic!("map diff has no entry for {key:?}: {map_diff:?}")) + .as_ref() + .and_then(|map_value| map_value.value.clone()); + assert_eq!(value, Some(expected)); +} + +#[test] +fn causal_existing_peer_import_uses_current_version_as_replay_base() { + use crate::{handler::HandlerTrait, loro::ExportMode}; + + let base = movable_list_fixture(); + let target = base.fork(); + let source = base.fork(); + let relay = base.fork(); + // Continue an existing peer after another peer has become the current + // frontier. This gives the new change both an explicit dependency on the + // relay and an implicit dependency on its own previous counter. + source.set_peer_id(1).unwrap(); + relay.set_peer_id(2).unwrap(); + relay.get_map("relay").insert("ready", true).unwrap(); + relay.commit_then_renew(); + let relay_updates = relay.export(ExportMode::updates(&base.oplog_vv())).unwrap(); + target.import(&relay_updates).unwrap(); + source.import(&relay_updates).unwrap(); + + let before = target.oplog_vv(); + let before_frontiers = target.oplog_frontiers(); + nested_map(&source).insert("value", 1).unwrap(); + source.commit_then_renew(); + let nested_update = source.export(ExportMode::updates(&before)).unwrap(); + + // Import only into the oplog so this test can inspect the exact diff round. + target.detach(); + target.import(&nested_update).unwrap(); + let after = target.oplog_vv(); + let after_frontiers = target.oplog_frontiers(); + let expected_idx = nested_map(&target).idx(); + + let oplog = target.oplog().lock(); + let (replay_base, mode, _) = + oplog.iter_from_replay_base_causally(&before, &before_frontiers, &after, &after_frontiers); + assert_eq!(replay_base, before); + assert_eq!(mode, DiffMode::ImportGreaterUpdates); + assert_eq!( + changed_containers_between(&oplog, &before, &after), + [expected_idx].into_iter().collect() + ); + + let mut calculator = DiffCalculator::new(false); + let (diffs, _) = calculator.calc_diff_internal( + &oplog, + &before, + &before_frontiers, + &after, + &after_frontiers, + None, + ); + assert_eq!(calculator.calculators.len(), 1); + assert!(calculator.get_calc(expected_idx).is_some()); + assert_single_map_value_diff(&diffs, expected_idx, "value", 1.into()); +} + +#[test] +fn conservative_replay_only_builds_calculators_for_changed_containers() { + use crate::{handler::HandlerTrait, loro::ExportMode}; + + let base = movable_list_fixture(); + let target = base.fork(); + target.set_peer_id(2).unwrap(); + target.get_map("relay").insert("ready", true).unwrap(); + target.commit_then_renew(); + + let source = base.fork(); + source.set_peer_id(3).unwrap(); + nested_map(&source).insert("value", 1).unwrap(); + source.commit_then_renew(); + + let before = target.oplog_vv(); + let before_frontiers = target.oplog_frontiers(); + let nested_update = source.export(ExportMode::updates(&before)).unwrap(); + + // Import only into the oplog so this test can inspect the exact diff round. + target.detach(); + target.import(&nested_update).unwrap(); + let after = target.oplog_vv(); + let after_frontiers = target.oplog_frontiers(); + let expected_idx = nested_map(&target).idx(); + + let oplog = target.oplog().lock(); + let (replay_base, mode, _) = + oplog.iter_from_replay_base_causally(&before, &before_frontiers, &after, &after_frontiers); + assert_ne!(replay_base, before, "the fixture must exercise conservative replay"); + assert_eq!(mode, DiffMode::Import); + assert_eq!( + changed_containers_between(&oplog, &before, &after), + [expected_idx].into_iter().collect() + ); + + let mut calculator = DiffCalculator::new(false); + let (diffs, _) = calculator.calc_diff_internal( + &oplog, + &before, + &before_frontiers, + &after, + &after_frontiers, + None, + ); + assert_eq!(calculator.calculators.len(), 1); + assert!(calculator.get_calc(expected_idx).is_some()); + assert_single_map_value_diff(&diffs, expected_idx, "value", 1.into()); +} + +#[test] +fn conservative_checkout_replay_filters_to_retreat_changed_containers() { + use crate::{handler::HandlerTrait, loro::ExportMode}; + + let base = movable_list_fixture(); + let target = base.fork(); + target.set_peer_id(2).unwrap(); + target.get_map("relay").insert("ready", true).unwrap(); + target.commit_then_renew(); + + let source = base.fork(); + source.set_peer_id(3).unwrap(); + nested_map(&source).insert("value", 1).unwrap(); + source.commit_then_renew(); + + // `after` is the older version; the concurrent nested-map edit exists only + // on the `before` side, so the changed set comes from the retreat spans. + let after = target.oplog_vv(); + let after_frontiers = target.oplog_frontiers(); + let nested_update = source + .export(ExportMode::updates(&base.oplog_vv())) + .unwrap(); + + // Import only into the oplog so this test can inspect the exact diff round. + target.detach(); + target.import(&nested_update).unwrap(); + let before = target.oplog_vv(); + let before_frontiers = target.oplog_frontiers(); + let expected_idx = nested_map(&target).idx(); + + let oplog = target.oplog().lock(); + let (replay_base, mode, _) = + oplog.iter_from_replay_base_causally(&before, &before_frontiers, &after, &after_frontiers); + assert_ne!(replay_base, before, "the fixture must exercise conservative replay"); + assert_eq!(mode, DiffMode::Checkout); + assert_eq!( + changed_containers_between(&oplog, &before, &after), + [expected_idx].into_iter().collect() + ); + + let mut calculator = DiffCalculator::new(false); + let (diffs, _) = calculator.calc_diff_internal( + &oplog, + &before, + &before_frontiers, + &after, + &after_frontiers, + None, + ); + assert_eq!(calculator.calculators.len(), 1); + assert!(calculator.get_calc(expected_idx).is_some()); + // Checking out backward removes the concurrent edit again. + assert_single_map_value_diff(&diffs, expected_idx, "value", 0.into()); +} diff --git a/crates/loro-internal/src/diff_calc/tree.rs b/crates/loro-internal/src/diff_calc/tree.rs index 58c410316..181c5ae80 100644 --- a/crates/loro-internal/src/diff_calc/tree.rs +++ b/crates/loro-internal/src/diff_calc/tree.rs @@ -240,22 +240,42 @@ impl TreeDiffCalculator { let from_frontiers = info.from_frontiers; let (common_ancestors, _mode) = oplog.dag.find_common_ancestor(from_frontiers, to_frontiers); - let lca_vv = oplog.dag.frontiers_to_vv(&common_ancestors).unwrap(); - let lca_frontiers = common_ancestors; + let base_vv = oplog.dag.frontiers_to_vv(&common_ancestors).unwrap(); + let base_frontiers = common_ancestors; let to_max_lamport = self.get_max_lamport_by_frontiers(to_frontiers, oplog); - let lca_min_lamport = self.get_min_lamport_by_frontiers(&lca_frontiers, oplog); + // CORRECTNESS: the retreat/forward passes below only look at ops + // with lamport >= `base_min_lamport` (the minimum change-start + // lamport of the base frontiers). Ops outside the window are + // silently skipped, which is only sound if no op of the diff + // region sits below the window. That holds because of how + // `find_common_ancestor` picks the base: + // - a region op that reaches the base through causal edges is a + // descendant of some base head, so its lamport is strictly + // greater than that head's change-start lamport, which is >= the + // window minimum; + // - a region op CONCURRENT with a base head forces the walk to + // detect an uncovered branch and retreat the base to a critical + // version (latest_single_head_critical_version), below which no + // concurrency crosses — the window then starts at that base. + // See docs/critical-version-spec.md (Q7) and the regression tests + // `checkout_across_non_critical_meet_stays_canonical` and + // `checkout_with_low_lamport_concurrent_branch_stays_canonical` + // in crates/loro/tests/issue.rs. Weakening either the critical + // version fallback or the ImportGreaterUpdates entry check in + // dag.rs breaks this invariant. + let base_min_lamport = self.get_min_lamport_by_frontiers(&base_frontiers, oplog); // retreat for diff let mut diffs = vec![]; - if !(tree_cache.current_vv == lca_vv && &lca_vv == info.from_vv) { + if !(tree_cache.current_vv == base_vv && &base_vv == info.from_vv) { let mut retreat_ops = vec![]; for (_target, ops) in tree_cache.tree.iter() { for op in ops.iter().rev() { - if op.id.lamport < lca_min_lamport { + if op.id.lamport < base_min_lamport { break; } - if !lca_vv.includes_id(op.id.id()) { + if !base_vv.includes_id(op.id.id()) { retreat_ops.push(op.clone()); } } @@ -316,7 +336,7 @@ impl TreeDiffCalculator { } } } - tree_cache.current_vv = lca_vv; + tree_cache.current_vv = base_vv; // forward let group = h .get_importing_cache(&self.container, mark) @@ -325,7 +345,7 @@ impl TreeDiffCalculator { .unwrap(); for (idlp, op) in group.ops().range( IdLp { - lamport: lca_min_lamport, + lamport: base_min_lamport, peer: 0, }..=IdLp { lamport: to_max_lamport, diff --git a/crates/loro-internal/src/encoding/shallow_snapshot.rs b/crates/loro-internal/src/encoding/shallow_snapshot.rs index 2bcaf732b..f6bdf9ceb 100644 --- a/crates/loro-internal/src/encoding/shallow_snapshot.rs +++ b/crates/loro-internal/src/encoding/shallow_snapshot.rs @@ -313,10 +313,10 @@ fn restore_export_doc_state( /// Calculates optimal starting version for the shallow doc /// -/// It should be the LCA of the user given version and the latest version. +/// It should be a common ancestor version of the user-given version and the latest version. /// Otherwise, users cannot replay the history from the initial version till the latest version. fn calc_shallow_doc_start(oplog: &crate::OpLog, frontiers: &Frontiers) -> Frontiers { - // Find the LCA of the given frontiers by iteratively pairwise GCA. + // Find a common ancestor version of the given frontiers by iterative pairwise reduction. // This converges to a single frontier or empty if there is no common ancestor. let mut current = frontiers.clone(); while current.len() > 1 { diff --git a/crates/loro-internal/src/oplog.rs b/crates/loro-internal/src/oplog.rs index 27146cdeb..0bf69c507 100644 --- a/crates/loro-internal/src/oplog.rs +++ b/crates/loro-internal/src/oplog.rs @@ -40,7 +40,7 @@ pub use change_store::{BlockChangeRef, ChangeStore}; /// So you can derive different versions of the state from the [OpLog]. /// It allows us to build a version control system. /// -/// The causal graph should always be a DAG and complete. So we can always find the LCA. +/// The causal graph should always be a DAG and complete. So we can always find a common ancestor version. /// If deps are missing, we can't import the change. It will be put into the `pending_changes`. pub struct OpLog { pub(crate) dag: AppDag, @@ -577,7 +577,10 @@ impl OpLog { decode_oplog(self, data) } - /// iterates over all changes between LCA(common ancestors) to the merged version of (`from` and `to`) causally + /// Iterates causally over all changes between the replay base (a common + /// ancestor version chosen by `find_common_ancestor`; ideally the latest + /// critical version in the Eg-walker sense, see + /// `docs/critical-version-spec.md`) and the merged version of `from`/`to`. /// /// Tht iterator will include a version vector when the change is applied /// @@ -588,7 +591,7 @@ impl OpLog { /// /// If frontiers are provided, it will be faster (because we don't need to calculate it from version vector #[allow(clippy::type_complexity)] - pub(crate) fn iter_from_lca_causally( + pub(crate) fn iter_from_replay_base_causally( &self, from: &VersionVector, from_frontiers: &Frontiers, @@ -608,31 +611,31 @@ impl OpLog { let mut merged_vv = from.clone(); merged_vv.merge(to); loro_common::debug!("to_frontiers={:?} vv={:?}", &to_frontiers, to); - let (mut common_ancestors, mut diff_mode) = + let (mut replay_base_frontiers, mut diff_mode) = self.dag.find_common_ancestor(from_frontiers, to_frontiers); if diff_mode == DiffMode::Checkout && to > from { diff_mode = DiffMode::Import; } - let mut common_ancestors_vv = self.dag.frontiers_to_vv(&common_ancestors).unwrap(); + let mut replay_base_vv = self.dag.frontiers_to_vv(&replay_base_frontiers).unwrap(); let shallow_since_vv = self.dag.shallow_since_vv().to_vv(); - if !common_ancestors_vv.includes_vv(&shallow_since_vv) { + if !replay_base_vv.includes_vv(&shallow_since_vv) { // The replay base cannot point before shallow history because those // ops are no longer available to the causal iterator. - common_ancestors = self.dag.shallow_since_frontiers().clone(); - common_ancestors_vv = self + replay_base_frontiers = self.dag.shallow_since_frontiers().clone(); + replay_base_vv = self .dag - .frontiers_to_vv(&common_ancestors) + .frontiers_to_vv(&replay_base_frontiers) .unwrap_or(shallow_since_vv); } - // go from lca to merged_vv - let diff = common_ancestors_vv.diff(&merged_vv).forward; - let mut iter = self.dag.iter_causal(common_ancestors, diff); + // go from the replay base to merged_vv + let diff = replay_base_vv.diff(&merged_vv).forward; + let mut iter = self.dag.iter_causal(replay_base_frontiers, diff); let mut node = iter.next(); let mut cur_cnt = 0; let vv = Rc::new(RefCell::new(VersionVector::default())); ( - common_ancestors_vv.clone(), + replay_base_vv.clone(), diff_mode, std::iter::from_fn(move || { if let Some(inner) = &node { @@ -646,7 +649,7 @@ impl OpLog { .data .cnt .max(cur_cnt) - .max(common_ancestors_vv.get(&peer).copied().unwrap_or(0)); + .max(replay_base_vv.get(&peer).copied().unwrap_or(0)); let dag_node_end = (inner.data.cnt + inner.data.len as Counter) .min(merged_vv.get(&peer).copied().unwrap_or(0)); let change = self.change_store.get_change(ID::new(peer, cnt)).unwrap(); diff --git a/crates/loro-internal/src/oplog/loro_dag.rs b/crates/loro-internal/src/oplog/loro_dag.rs index 3772d9299..bafc28d3e 100644 --- a/crates/loro-internal/src/oplog/loro_dag.rs +++ b/crates/loro-internal/src/oplog/loro_dag.rs @@ -21,7 +21,7 @@ use super::change_store::BatchDecodeInfo; use super::ChangeStore; /// [AppDag] maintains the causal graph of the app. -/// It's faster to answer the question like what's the LCA version +/// It's faster to answer questions like what the common ancestor version is #[derive(Debug)] pub struct AppDag { change_store: ChangeStore, diff --git a/crates/loro-internal/src/state.rs b/crates/loro-internal/src/state.rs index cf4430907..a6dfeaaba 100644 --- a/crates/loro-internal/src/state.rs +++ b/crates/loro-internal/src/state.rs @@ -653,6 +653,12 @@ impl DocState { return Err(LoroError::internal("state apply failpoint")); } } + // `diff_mode` here is the DIRECTION mode (`origin_diff_mode` from + // `calc_diff_internal`), not the mode the calculators computed with: + // Checkout means the transition may go backwards, so any cached + // dead/alive knowledge can be invalidated; every other mode implies a + // forward transition, where alive-markers may change but dead + // containers stay dead unless a diff revives them. match diff_mode { DiffMode::Checkout => { self.dead_containers_cache.clear(); diff --git a/crates/loro/tests/issue.rs b/crates/loro/tests/issue.rs index e09308a0b..3cfb9a152 100644 --- a/crates/loro/tests/issue.rs +++ b/crates/loro/tests/issue.rs @@ -499,3 +499,222 @@ fn get_unknown_cursor_position_but_its_in_pending() { assert!(!doc_1.has_container(&text.id())); assert_eq!(doc_1.get_path_to_container(&text.id()), None); } + +/// Concurrent movable-tree moves must keep the incrementally maintained +/// `DocState` identical to a fresh replay of the full oplog. +/// +/// History shape: B creates four roots (B0..B3), then B reorders them +/// (B4..B6) while A — knowing only B0..B3 — creates one root (A0) and +/// reorders everything (A1..A3). A1..A3 are concurrent with B4..B6, but the +/// receiving peer's frontier {B6, A0} is version-included in the merged +/// version, which used to be misclassified as ImportGreaterUpdates: the tree +/// fast path applied A1..A3 without adjudicating against B4..B6 and the +/// materialized state diverged from replay. Run both import orders so either +/// peer can be the one receiving the concurrent branch incrementally. +#[test] +fn tree_concurrent_moves_incremental_state_matches_replay() { + fn canonical(doc: &LoroDoc) -> LoroDoc { + let replay = LoroDoc::new(); + replay.get_tree("tree").enable_fractional_index(0); + replay + .import(&doc.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + replay.checkout_to_latest(); + replay + } + + fn run(first_into_b: bool) { + let a = LoroDoc::new(); + a.set_peer_id(1).unwrap(); + let b = LoroDoc::new(); + b.set_peer_id(2).unwrap(); + a.get_tree("tree").enable_fractional_index(0); + b.get_tree("tree").enable_fractional_index(0); + let ta = a.get_tree("tree"); + let tb = b.get_tree("tree"); + + let n0 = tb.create(None).unwrap(); + let n1 = tb.create(None).unwrap(); + let n2 = tb.create(None).unwrap(); + let n3 = tb.create(None).unwrap(); + b.commit(); + a.import(&b.export(ExportMode::updates(&a.oplog_vv())).unwrap()) + .unwrap(); + + tb.mov_to(n1, None, 0).unwrap(); + tb.mov_to(n2, None, 1).unwrap(); + tb.mov_to(n3, None, 2).unwrap(); + tb.mov_to(n0, None, 3).unwrap(); + b.commit(); + + let na = ta.create(None).unwrap(); + a.commit(); + b.import(&a.export(ExportMode::updates(&b.oplog_vv())).unwrap()) + .unwrap(); + + ta.mov_to(n3, None, 0).unwrap(); + ta.mov_to(n1, None, 1).unwrap(); + ta.mov_to(n0, None, 2).unwrap(); + ta.mov_to(na, None, 3).unwrap(); + ta.mov_to(n2, None, 4).unwrap(); + a.commit(); + + let a_updates = a.export(ExportMode::all_updates()).unwrap(); + let b_updates = b.export(ExportMode::all_updates()).unwrap(); + if first_into_b { + b.import(&a_updates).unwrap(); + a.import(&b_updates).unwrap(); + } else { + a.import(&b_updates).unwrap(); + b.import(&a_updates).unwrap(); + } + a.checkout_to_latest(); + b.checkout_to_latest(); + + let replay_a = canonical(&a); + let replay_b = canonical(&b); + assert_eq!(a.oplog_vv(), b.oplog_vv()); + assert_eq!(a.get_deep_value(), b.get_deep_value()); + assert_eq!(replay_a.get_deep_value(), replay_b.get_deep_value()); + assert_eq!(a.get_deep_value(), replay_a.get_deep_value()); + assert_eq!(b.get_deep_value(), replay_b.get_deep_value()); + } + + run(true); + run(false); +} + +/// Checkout between divergent versions whose meet is NOT a critical version +/// (diamond: the region op `1@1` is concurrent with the meet head `0@2`). +/// The tree/movable-list/text calculators replay relatively around the base; +/// this pins that such checkouts stay canonical. +#[test] +fn checkout_across_non_critical_meet_stays_canonical() { + use loro::{Frontiers, TreeID, ID}; + let s = LoroDoc::new(); + s.set_peer_id(9).unwrap(); + s.get_tree("tree").enable_fractional_index(0); + let ts = s.get_tree("tree"); + let _n = ts.create(None).unwrap(); + let _sib = ts.create(None).unwrap(); + s.commit(); + let prefix = s.export(ExportMode::all_updates()).unwrap(); + + let p1 = LoroDoc::new(); + p1.set_peer_id(1).unwrap(); + p1.get_tree("tree").enable_fractional_index(0); + p1.import(&prefix).unwrap(); + let t1 = p1.get_tree("tree"); + let _m1 = t1.create(None).unwrap(); + t1.mov_to(TreeID::new(9, 0), None, 2).unwrap(); + p1.commit(); + let b1 = p1.export(ExportMode::updates(&s.oplog_vv())).unwrap(); + + let p2 = LoroDoc::new(); + p2.set_peer_id(2).unwrap(); + p2.get_tree("tree").enable_fractional_index(0); + p2.import(&prefix).unwrap(); + let t2 = p2.get_tree("tree"); + t2.mov_to(TreeID::new(9, 0), None, 1).unwrap(); + let _m2 = t2.create(None).unwrap(); + p2.commit(); + let b2 = p2.export(ExportMode::updates(&s.oplog_vv())).unwrap(); + + let v1 = Frontiers::from(vec![ID::new(1, 1), ID::new(2, 0)]); + let v2 = Frontiers::from(vec![ID::new(2, 1), ID::new(1, 0)]); + let make_full = || { + let d = LoroDoc::new(); + d.get_tree("tree").enable_fractional_index(0); + d.import(&prefix).unwrap(); + d.import(&b1).unwrap(); + d.import(&b2).unwrap(); + d + }; + let ref1 = make_full(); + ref1.checkout(&v1).unwrap(); + let ref2 = make_full(); + ref2.checkout(&v2).unwrap(); + + let d = make_full(); + d.checkout(&v1).unwrap(); + assert_eq!(d.get_deep_value(), ref1.get_deep_value()); + d.checkout(&v2).unwrap(); + assert_eq!(d.get_deep_value(), ref2.get_deep_value()); + d.checkout(&v1).unwrap(); + assert_eq!(d.get_deep_value(), ref1.get_deep_value()); +} + +/// Checkout where the diff region contains a LOW-lamport concurrent branch +/// (below the meet frontier's lamport window). The tree calculator's +/// retreat/forward windows skip ops below `lca_min_lamport`, so this is only +/// safe because the walk retreats the base to a critical version (the +/// latest-singleton-cut sweep) whenever such a branch exists. Pins that +/// interplay: weakening the sweep would silently break this. +#[test] +fn checkout_with_low_lamport_concurrent_branch_stays_canonical() { + use loro::{Frontiers, TreeID, ID}; + let p = LoroDoc::new(); + p.set_peer_id(9).unwrap(); + p.get_tree("tree").enable_fractional_index(0); + let tp = p.get_tree("tree"); + let _n = tp.create(None).unwrap(); + p.commit(); + let blob_first = p.export(ExportMode::all_updates()).unwrap(); + for _ in 0..4 { + tp.create(None).unwrap(); + } + p.commit(); + let prefix = p.export(ExportMode::all_updates()).unwrap(); + + let p3 = LoroDoc::new(); + p3.set_peer_id(3).unwrap(); + p3.get_tree("tree").enable_fractional_index(0); + p3.import(&blob_first).unwrap(); + let _x = p3.get_tree("tree").create(None).unwrap(); // 0@3, lamport 1 + p3.commit(); + let blob_o = p3.export(ExportMode::updates(&p.oplog_vv())).unwrap(); + + let p1 = LoroDoc::new(); + p1.set_peer_id(1).unwrap(); + p1.get_tree("tree").enable_fractional_index(0); + p1.import(&prefix).unwrap(); + let t1 = p1.get_tree("tree"); + let _m1 = t1.create(None).unwrap(); + t1.mov_to(TreeID::new(9, 0), None, 3).unwrap(); + p1.commit(); + let bx = p1.export(ExportMode::updates(&p.oplog_vv())).unwrap(); + + let p2 = LoroDoc::new(); + p2.set_peer_id(2).unwrap(); + p2.get_tree("tree").enable_fractional_index(0); + p2.import(&prefix).unwrap(); + let t2 = p2.get_tree("tree"); + t2.mov_to(TreeID::new(9, 0), None, 1).unwrap(); + let _m2 = t2.create(None).unwrap(); + p2.commit(); + let by = p2.export(ExportMode::updates(&p.oplog_vv())).unwrap(); + + let make_full = || { + let d = LoroDoc::new(); + d.get_tree("tree").enable_fractional_index(0); + d.import(&prefix).unwrap(); + d.import(&blob_o).unwrap(); + d.import(&bx).unwrap(); + d.import(&by).unwrap(); + d + }; + let v1 = Frontiers::from(vec![ID::new(1, 1), ID::new(2, 0), ID::new(3, 0)]); + let v2 = Frontiers::from(vec![ID::new(2, 1), ID::new(1, 0)]); + let ref1 = make_full(); + ref1.checkout(&v1).unwrap(); + let ref2 = make_full(); + ref2.checkout(&v2).unwrap(); + + let d = make_full(); + d.checkout(&v2).unwrap(); + assert_eq!(d.get_deep_value(), ref2.get_deep_value()); + d.checkout(&v1).unwrap(); // forward the lamport-1 branch across the window + assert_eq!(d.get_deep_value(), ref1.get_deep_value()); + d.checkout(&v2).unwrap(); // and retreat it again + assert_eq!(d.get_deep_value(), ref2.get_deep_value()); +} diff --git a/crates/loro/tests/moon_transcode.rs b/crates/loro/tests/moon_transcode.rs index e768d4cca..c7d0f50c6 100644 --- a/crates/loro/tests/moon_transcode.rs +++ b/crates/loro/tests/moon_transcode.rs @@ -5,6 +5,16 @@ use std::process::Command; use std::sync::OnceLock; use std::time::{SystemTime, UNIX_EPOCH}; +/// Tmp dirs used to be keyed by (pid, nanos) alone; parallel tests in the +/// same process can hit the same nanosecond bucket, sharing a dir and +/// overwriting each other's in.blob (which surfaces as another test's peers +/// showing up in the decoded output). A process-wide counter makes every call +/// unique. +fn next_tmp_id() -> u64 { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) +} + use loro::{ ExpandType, ExportMode, Frontiers, LoroDoc, LoroValue, StyleConfig, StyleConfigMap, Timestamp, ToJson, TreeParentId, VersionVector, @@ -81,7 +91,7 @@ fn run_transcode(node_bin: &str, cli_js: &Path, input: &[u8]) -> anyhow::Result< .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); - let tmp = std::env::temp_dir().join(format!("loro-moon-transcode-{}-{ts}", std::process::id())); + let tmp = std::env::temp_dir().join(format!("loro-moon-transcode-{}-{ts}-{}", std::process::id(), next_tmp_id())); std::fs::create_dir_all(&tmp)?; let in_path = tmp.join("in.blob"); let out_path = tmp.join("out.blob"); @@ -107,8 +117,9 @@ fn run_decode_updates(node_bin: &str, cli_js: &Path, input: &[u8]) -> anyhow::Re .unwrap() .as_nanos(); let tmp = std::env::temp_dir().join(format!( - "loro-moon-decode-updates-{}-{ts}", - std::process::id() + "loro-moon-decode-updates-{}-{ts}-{}", + std::process::id(), + next_tmp_id() )); std::fs::create_dir_all(&tmp)?; let in_path = tmp.join("in.blob"); @@ -128,8 +139,9 @@ fn run_export_jsonschema(node_bin: &str, cli_js: &Path, input: &[u8]) -> anyhow: .unwrap() .as_nanos(); let tmp = std::env::temp_dir().join(format!( - "loro-moon-export-jsonschema-{}-{ts}", - std::process::id() + "loro-moon-export-jsonschema-{}-{ts}-{}", + std::process::id(), + next_tmp_id() )); std::fs::create_dir_all(&tmp)?; let in_path = tmp.join("in.blob"); @@ -155,8 +167,9 @@ fn run_export_deep_json(node_bin: &str, cli_js: &Path, input: &[u8]) -> anyhow:: .unwrap() .as_nanos(); let tmp = std::env::temp_dir().join(format!( - "loro-moon-export-deep-json-{}-{ts}", - std::process::id() + "loro-moon-export-deep-json-{}-{ts}-{}", + std::process::id(), + next_tmp_id() )); std::fs::create_dir_all(&tmp)?; let in_path = tmp.join("in.blob"); @@ -180,8 +193,9 @@ fn run_encode_jsonschema( .unwrap() .as_nanos(); let tmp = std::env::temp_dir().join(format!( - "loro-moon-encode-jsonschema-{}-{ts}", - std::process::id() + "loro-moon-encode-jsonschema-{}-{ts}-{}", + std::process::id(), + next_tmp_id() )); std::fs::create_dir_all(&tmp)?; let in_path = tmp.join("in.json"); @@ -212,8 +226,9 @@ fn run_transcode_output( .unwrap() .as_nanos(); let tmp = std::env::temp_dir().join(format!( - "loro-moon-transcode-raw-{}-{ts}", - std::process::id() + "loro-moon-transcode-raw-{}-{ts}-{}", + std::process::id(), + next_tmp_id() )); std::fs::create_dir_all(&tmp)?; let in_path = tmp.join("in.blob"); @@ -240,8 +255,9 @@ fn run_decode_updates_output( .unwrap() .as_nanos(); let tmp = std::env::temp_dir().join(format!( - "loro-moon-decode-updates-raw-{}-{ts}", - std::process::id() + "loro-moon-decode-updates-raw-{}-{ts}-{}", + std::process::id(), + next_tmp_id() )); std::fs::create_dir_all(&tmp)?; let in_path = tmp.join("in.blob"); @@ -263,8 +279,9 @@ fn run_export_jsonschema_output( .unwrap() .as_nanos(); let tmp = std::env::temp_dir().join(format!( - "loro-moon-export-jsonschema-raw-{}-{ts}", - std::process::id() + "loro-moon-export-jsonschema-raw-{}-{ts}-{}", + std::process::id(), + next_tmp_id() )); std::fs::create_dir_all(&tmp)?; let in_path = tmp.join("in.blob"); @@ -286,8 +303,9 @@ fn run_export_deep_json_output( .unwrap() .as_nanos(); let tmp = std::env::temp_dir().join(format!( - "loro-moon-export-deep-json-raw-{}-{ts}", - std::process::id() + "loro-moon-export-deep-json-raw-{}-{ts}-{}", + std::process::id(), + next_tmp_id() )); std::fs::create_dir_all(&tmp)?; let in_path = tmp.join("in.blob"); @@ -309,8 +327,9 @@ fn run_encode_jsonschema_output( .unwrap() .as_nanos(); let tmp = std::env::temp_dir().join(format!( - "loro-moon-encode-jsonschema-raw-{}-{ts}", - std::process::id() + "loro-moon-encode-jsonschema-raw-{}-{ts}-{}", + std::process::id(), + next_tmp_id() )); std::fs::create_dir_all(&tmp)?; let in_path = tmp.join("in.json");