diff --git a/TreeDB/docs/spec/command-wal-nativewire-alignment.json b/TreeDB/docs/spec/command-wal-nativewire-alignment.json index 2564dd4775..d590ee4e2d 100644 --- a/TreeDB/docs/spec/command-wal-nativewire-alignment.json +++ b/TreeDB/docs/spec/command-wal-nativewire-alignment.json @@ -2,6 +2,7 @@ "version": 1, "owner": "TreeDB/docs/spec/user-command-wal.md", "tracker": "https://github.com/snissn/gomap/issues/1529", + "raft_r3a_tracker": "https://github.com/snissn/gomap/issues/1654", "relationship_values": [ "lowered_equivalent_v1", "lowered_kind_only_v1", @@ -9,6 +10,11 @@ "local_only_rejected_v1", "local_only_barrier_v1" ], + "raft_r3a_harness_values": [ + "eligible_v1", + "rejected_until_wal_supported_v1", + "local_only_not_replicated_v1" + ], "ack_recoverability": { "visible": "process-visible after local command-WAL recoverability and normal-executor install; does not require root publication or AppliedLSN advancement", "flushed": "backend roots and AppliedLSN are selected in the same backend commit", @@ -25,6 +31,7 @@ "local_fixture": "TreeDB/internal/commitlog/testdata/command_wal_v1_catalog_create_collection.hex", "local_fixture_sha256": "4782f9c9cca647e33b44f1143de7e17158a7f80c3302399611596641626d807e", "support_matrix_status": "WAL-supported", + "raft_r3a_harness": "eligible_v1", "relationship": "lowered_kind_only_v1", "notes": "Kind-only alignment: native-wire deterministic-entry fixtures and local command-WAL fixtures pin create-collection support and acknowledgement semantics, but they encode collection metadata with different wire payload shapes. A future equivalence fixture must use matched metadata payloads before claiming lowered_equivalent_v1." }, @@ -37,6 +44,7 @@ "local_fixture": "", "local_fixture_sha256": "", "support_matrix_status": "WAL-rejected", + "raft_r3a_harness": "rejected_until_wal_supported_v1", "relationship": "future_rejected_v1", "notes": "Native-wire deterministic bytes are pinned, but local command WAL must reject index DDL until catalog index command payloads and recovery tests land." }, @@ -49,6 +57,7 @@ "local_fixture": "", "local_fixture_sha256": "", "support_matrix_status": "WAL-rejected", + "raft_r3a_harness": "rejected_until_wal_supported_v1", "relationship": "future_rejected_v1", "notes": "Native-wire deterministic bytes are pinned, but local command WAL must reject index DDL until catalog index command payloads and recovery tests land." }, @@ -61,6 +70,7 @@ "local_fixture": "", "local_fixture_sha256": "", "support_matrix_status": "WAL-rejected", + "raft_r3a_harness": "rejected_until_wal_supported_v1", "relationship": "local_only_rejected_v1", "notes": "CommandDropCollection is a local-only native-wire mutation today and has no deterministic-entry fixture. Local command WAL rejects it until collection-drop catalog payloads and recovery tests land." }, @@ -73,6 +83,7 @@ "local_fixture": "TreeDB/internal/commitlog/testdata/command_wal_v1_collection_insert_by_id.hex", "local_fixture_sha256": "122bc8ee2e535d011ff956ca145b4c8a06ed4503e05cb480b40c49b10366d7b3", "support_matrix_status": "WAL-supported", + "raft_r3a_harness": "eligible_v1", "relationship": "lowered_kind_only_v1", "notes": "Kind-only alignment: native-wire deterministic-entry fixtures and local command-WAL fixtures pin the same command kind and supported surface, but they are not byte-equivalent and currently use different logical fixture inputs. A future equivalence fixture must use matched collection/id/document inputs before claiming lowered_equivalent_v1." }, @@ -85,6 +96,7 @@ "local_fixture": "TreeDB/internal/commitlog/testdata/command_wal_v1_collection_update_by_id.hex", "local_fixture_sha256": "3f3c82673c27f3952aa8683febfa6a96061562421b53ab05958d65998fac396d", "support_matrix_status": "WAL-supported", + "raft_r3a_harness": "eligible_v1", "relationship": "lowered_kind_only_v1", "notes": "Kind-only alignment: native-wire deterministic-entry fixtures and local command-WAL fixtures pin the same command kind and supported surface, but they are not byte-equivalent and currently use different logical fixture inputs. A future equivalence fixture must use matched collection/id/document inputs before claiming lowered_equivalent_v1." }, @@ -97,6 +109,7 @@ "local_fixture": "TreeDB/internal/commitlog/testdata/command_wal_v1_collection_delete_by_id.hex", "local_fixture_sha256": "1e76994102339bf932e4bc4abde4122a744210dcf32376f58f516f79588577d7", "support_matrix_status": "WAL-supported", + "raft_r3a_harness": "eligible_v1", "relationship": "lowered_kind_only_v1", "notes": "Kind-only alignment: native-wire deterministic-entry fixtures and local command-WAL fixtures pin the same command kind and supported surface, but they are not byte-equivalent and currently use different logical fixture inputs. A future equivalence fixture must use matched collection/id/document inputs before claiming lowered_equivalent_v1." }, @@ -109,6 +122,7 @@ "local_fixture": "", "local_fixture_sha256": "", "support_matrix_status": "WAL-supported", + "raft_r3a_harness": "local_only_not_replicated_v1", "relationship": "local_only_barrier_v1", "notes": "CommandFlushCollection is a local-only durability barrier, not replicated command identity. It must observe command-WAL root and AppliedLSN boundaries but does not add a replayable user command payload." }, @@ -121,6 +135,7 @@ "local_fixture": "", "local_fixture_sha256": "", "support_matrix_status": "WAL-supported", + "raft_r3a_harness": "local_only_not_replicated_v1", "relationship": "local_only_barrier_v1", "notes": "CommandFlushAll is a local-only durability barrier, not replicated command identity. It must observe command-WAL root and AppliedLSN boundaries but does not add a replayable user command payload." }, @@ -133,6 +148,7 @@ "local_fixture": "", "local_fixture_sha256": "", "support_matrix_status": "WAL-supported", + "raft_r3a_harness": "local_only_not_replicated_v1", "relationship": "local_only_barrier_v1", "notes": "CommandCheckpoint is a local-only durability barrier, not replicated command identity. It must observe command-WAL root and AppliedLSN boundaries but does not add a replayable user command payload." } diff --git a/TreeDB/docs/spec/native-query-raft-roadmap.md b/TreeDB/docs/spec/native-query-raft-roadmap.md index 28201d882d..6d89cc27fb 100644 --- a/TreeDB/docs/spec/native-query-raft-roadmap.md +++ b/TreeDB/docs/spec/native-query-raft-roadmap.md @@ -197,18 +197,59 @@ Acceptance: - rejection tests for non-deterministic sections, duplicate singleton sections, unsupported command versions, local handles, and missing guards. +### R3a. Deterministic Apply Harness + +Before wiring a Raft library, implement the state-machine boundary tracked in +https://github.com/snissn/gomap/issues/1654. + +R3a accepts deterministic replicated command-entry bytes, decodes them without +reconstructing native-wire requests, lowers supported entries to local +user-command WAL payloads, and applies them through the normal TreeDB executor. +It is an in-process harness, not a networked Raft group. + +The initial R3a allowlist is limited to command kinds that are both deterministic +and `WAL-supported` in the command-WAL support matrix: + +- create collection; +- insert batch by explicit document ID; +- replace/update-as-final-replacement batch by explicit document ID; +- delete batch by explicit document ID. + +R3a must reject unsupported command kinds before local WAL append or visible +mutation. In particular, create/drop index, drop collection, query-wide +update/delete, flush/checkpoint barriers, column-store file publish, and +physical maintenance remain outside the replicated command-entry surface until +their command-WAL semantics and deterministic guards are explicit. + +Acceptance: + +- the same deterministic entry sequence applied to two fresh DBs produces the + same logical state digest; +- request metadata such as request ID, acknowledgement policy, deadlines, + tracing, compression, and response shaping does not affect deterministic + command-entry bytes or the logical digest; +- unsupported replicated command kinds fail before local command WAL append, + `AppliedLSN` advancement, or visible state mutation; +- the apply path uses the existing local command-WAL/recovery discipline instead + of a Raft-specific mutation bypass; +- the harness exposes the future applied-index/idempotency metadata boundary and + documents that those records must not advance past local recoverability. + ### R3. Raft MVP for Writes Add Raft around the deterministic write set only. Replicate: -- collection create/drop metadata, -- index create/drop metadata, +- collection create metadata, - insert/replace/delete batches, - deterministic schema/catalog guards, - idempotency records. +Replicate collection drop and index create/drop only after those command kinds +become `WAL-supported` and have deterministic guard/recovery tests. Until then, +they are request-layer rejections in cluster write mode. + Do not replicate: - ordinary reads, @@ -522,6 +563,8 @@ affect routing, snapshots, or catch-up behavior. ### Raft TODO +- Land the #1654 R3a deterministic apply harness before choosing a Raft library + or committing to a log-store boundary. - Choose Raft library or implementation boundary. - Define `CommandEntryV1` bytes. - Define idempotency record storage. diff --git a/TreeDB/docs/spec/user-command-wal.md b/TreeDB/docs/spec/user-command-wal.md index 7942017cc5..b14d82e603 100644 --- a/TreeDB/docs/spec/user-command-wal.md +++ b/TreeDB/docs/spec/user-command-wal.md @@ -7,6 +7,7 @@ collection-specific physical/root-delta WAL target in `collection-wal-durability-plan.md` for future implementation work. Tracker: https://github.com/snissn/gomap/issues/1529 +Raft R3a tracker: https://github.com/snissn/gomap/issues/1654 TreeDB is pre-alpha. On-disk formats and public APIs may change. This freedom must not create fail-open recovery behavior: once a directory advertises a @@ -171,6 +172,11 @@ Entries marked `future_rejected_v1` have pinned native-wire deterministic bytes but remain explicitly rejected by local command WAL until the matching command kind and recovery tests land. +The same manifest also records the R3a Raft-apply harness eligibility for each +native-wire mutation. R3a may use only entries that are deterministic and +currently `WAL-supported`; rejected or local-only barrier commands must fail +before local command WAL append when presented as replicated command entries. + This is a compatibility-breaking WAL format transition. TreeDB is pre-alpha, so the command WAL implementation may require old directories to be cleanly checkpointed with the previous binary or rebuilt. Once a directory advertises @@ -590,6 +596,14 @@ its local apply/durability rule is satisfied. Local page IDs/root IDs are not a portable consensus state root; a future cluster state digest must be defined separately if consensus requires byte-independent state equality. +The first Raft-facing implementation slice is the R3a deterministic apply +harness in https://github.com/snissn/gomap/issues/1654. That harness is not a +Raft log/store implementation. It should decode deterministic command-entry +bytes, lower supported entries to local command WAL payloads, apply through the +normal executor, and prove logical convergence across fresh DBs. It must not +replicate local command WAL frames, physical root IDs, page IDs, checkpoint +barriers, or reconstructed native-wire requests. + Native-wire deterministic command entries are the canonical schema source for wire-exposed mutations. The local command WAL should reuse those deterministic payload schemas and golden fixtures wherever the same command exists. If a