Implement milestone #6 - Manifest files & startup recovery

.github/workflows/ci.yml

@@ -2,10 +2,14 @@ name: CI
 
 on:
   push:
-    branches: [main, master, "milestone/*"]
+    branches: [main, master]
   pull_request:
     branches: [main, master]
 
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build-and-test:
     runs-on: ubuntu-latest
@@ -18,7 +22,7 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: "1.26"
-      
+
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v8
         with:
@@ -33,3 +37,21 @@ jobs:
 
       - name: Test
         run: make test
+
+      - name: Examples
+        run: make examples
+
+  crash:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.26"
+
+      - name: Crash harness (ci profile)
+        run: make crash-check PROFILE=ci

.gitignore

@@ -24,3 +24,4 @@ profile.cov
 
 # Internal stuff
 docs-internal/
+scratchpad/

.golangci.yml

@@ -164,6 +164,10 @@ linters:
     # Use some standard exclusion presets
     presets:
       - std-error-handling
+    # Fully skip the scratchpad: it's a throwaway feature playground, not
+    # production code, and is gitignored.
+    paths:
+      - scratchpad/
     rules:
       # Exclude examples from all linters (they're for demonstration, not production)
       - path: examples/

Makefile

@@ -1,7 +1,22 @@
-.PHONY: all build test coverage lint fmt-check fmt clean check examples crash-check help
+.PHONY: all build test coverage lint fmt-check fmt clean check examples crash-check bench fuzz help
 
 CYCLES ?= 100
 
+# Crash harness profile: `full` (uses CYCLES) or `ci` (fast deterministic preset).
+PROFILE ?= full
+
+# Bench knobs. Override on the CLI: `make bench PKG=./internal/wal BENCH=BenchmarkX BENCHTIME=3s`
+PKG ?= ./...
+BENCH ?= .
+BENCHTIME ?= 1s
+
+# Fuzz knobs. By default `make fuzz` discovers every Fuzz* function across the
+# tree and runs each for FUZZTIME. -fuzz only accepts one target at a time
+# (and one package), so the target loops rather than passes `./...`.
+FUZZPKG ?= ./...
+FUZZ ?= ^Fuzz
+FUZZTIME ?= 30s
+
 # Default target
 all: build
 
@@ -22,7 +37,11 @@ test:
 examples:
 	@echo "Running examples..."
 	@for file in $$(find examples -name "*.go" -type f | sort); do \
-		echo "\n=== Running $$file ==="; \
+		header="=== Running $$file ==="; \
+		sep=$$(printf '%*s' $${#header} '' | tr ' ' '='); \
+		echo "\n$$sep"; \
+		echo "$$header"; \
+		echo "$$sep"; \
 		go run $$file || exit 1; \
 	done
 	@echo "\n✓ All examples completed successfully"
@@ -48,15 +67,31 @@ fmt:
 ## check: Runs fmt-check, lint and test
 check: fmt-check lint test
 
-## crash-check: Run the controller/worker crash harness ($(CYCLES) cycles) with a temporary workspace
+## bench: Run benchmarks project-wide. Override with `make bench PKG=./internal/wal BENCH=BenchmarkX BENCHTIME=3s`
+bench:
+	go test -run=^$$ -bench=$(BENCH) -benchmem -benchtime=$(BENCHTIME) $(PKG)
+
+## fuzz: Run every fuzz target FUZZTIME each across FUZZPKG (default ./..., 30s each). `make fuzz FUZZTIME=1m`
+fuzz:
+	@set -eu; \
+	for pkg in $$(go list $(FUZZPKG)); do \
+		targets=$$(go test -list '$(FUZZ)' $$pkg 2>/dev/null | grep -E '^Fuzz' || true); \
+		for t in $$targets; do \
+			echo "==> $$pkg :: $$t ($(FUZZTIME))"; \
+			go test -run=^$$ -fuzz="^$$t$$" -fuzztime=$(FUZZTIME) $$pkg; \
+		done; \
+	done
+
+## crash-check: Run the controller/worker crash harness with a temporary workspace. `make crash-check PROFILE=ci`
 crash-check:
 	@set -eu; \
 	tmpdir=$$(mktemp -d /tmp/beachdb-crash.XXXXXX); \
 	dbdir="$$tmpdir/db"; \
 	artdir="$$tmpdir/artifacts"; \
-	echo "Running crash harness ($(CYCLES) cycles) in $$dbdir"; \
+	echo "Running crash harness (profile=$(PROFILE), cycles=$(CYCLES)) in $$dbdir"; \
 	echo ""; \
 	go run ./cmd/crash run \
+		--profile=$(PROFILE) \
 		--dbdir="$$dbdir" \
 		--artifact-dir="$$artdir" \
 		--cycles=$(CYCLES) \

README.md

@@ -31,9 +31,9 @@ BeachDB is my attempt to re-learn the fundamentals by building them from scratch
 - [x] **Memtable v1**: sorted structure + tombstones, see: [memtable blog post](https://aalhour.com/posts/beachdb-memtable-v1/)
 - [x] **Reference-model randomized tests** (model vs implementation)
 - [x] **SSTables v1**: immutable sorted files + `sst_dump`, see: [sstables blog post](https://aalhour.com/posts/beachdb-sstables-v1/)
-- [x] **Crash-loop harness**: kill mid-write, reopen, validate invariants, see: [crash-testing, part 1](https://aalhour.com/posts/beachdb-crash-testing-part1/) blog post
+- [x] **Crash-loop harness**: kill mid-write, reopen, validate invariants, see: [crash-testing, part 1 blog post](https://aalhour.com/posts/beachdb-crash-testing-part1/)
+- [x] **Manifest v1**: durable SSTable catalog (`CURRENT` + `VersionEdit` log) for startup reconstruction + `manifest_dump`, see: [manifest blog post](https://aalhour.com/posts/beachdb-manifest-v1/)
 - [ ] **Merge iterators** (memtable + SSTs) + **snapshot reads** (seqno-based)
-- [ ] **Manifest/versioning** + `manifest_dump` (startup reconstruction)
 - [ ] **Read path acceleration**: block index + bloom filters + benchmark evidence
 - [ ] **Compaction v1**: one strategy, minimal knobs + amplification measurements
 - [ ] **Adversarial testing**: fault injection + fuzzing (WAL/SST decode paths)

cmd/manifest_dump/README.md

@@ -0,0 +1,103 @@
+# manifest_dump
+
+CLI tool for inspecting BeachDB's MANIFEST files without running the database.
+It reads `CURRENT`, replays the manifest it points at, prints every
+`VersionEdit` in order, and then prints the reconstructed `Version`.
+
+## Build
+
+```sh
+make build
+# or
+go build -o bin/manifest_dump ./cmd/manifest_dump
+```
+
+## Usage
+
+```
+manifest_dump <db-dir>
+```
+
+The argument is the database directory (the one containing `CURRENT` and the
+`MANIFEST-NNNNNN` files), not a manifest file path.
+
+## Examples
+
+### Normal database
+
+```sh
+$ manifest_dump /tmp/mydb
+```
+
+```
+Manifest: MANIFEST-000001
+Path:     /tmp/mydb/MANIFEST-000001
+
+Edit #0:
+  next_file_id:  1
+  last_sequence: 0
+  log_number:    0
+Edit #1:
+  next_file_id:  2
+  last_sequence: 3
+  add_file:    level=0 id=1 size=170 smallest="apple/2/Put" largest="cherry/3/Put"
+Edit #2:
+  next_file_id:  3
+  last_sequence: 5
+  add_file:    level=0 id=2 size=126 smallest="apple/5/Delete" largest="kiwi/4/Put"
+Current Version:
+  Level 0: 2 files (296 bytes total)
+    [1] apple..cherry (170 bytes)
+    [2] apple..kiwi (126 bytes)
+```
+
+Each edit prints only the fields it sets. `add_file` keys are shown as
+`<userkey>/<seqno>/<kind>` (the same convention as `sst_dump`), where kind is
+`Put` or `Delete`.
+
+### Fresh database
+
+A directory with no `CURRENT` file is a fresh database — there is nothing to
+dump, and the tool exits 0:
+
+```sh
+$ manifest_dump /tmp/empty-dir
+No CURRENT file in /tmp/empty-dir — fresh database, nothing to dump.
+```
+
+### Truncated manifest
+
+A partial trailing record (a crash mid-append) is recoverable. The tool notes
+it, prints the `Version` reconstructed from the complete edits, and exits 0:
+
+```
+Edit #2: incomplete trailing record (crash mid-append, recoverable)
+
+Current Version:
+  Level 0: 1 files (170 bytes total)
+    [1] apple..cherry (170 bytes)
+```
+
+### Corrupt manifest
+
+A checksum mismatch or undecodable edit is fatal. The tool prints the `Version`
+reconstructed up to the failure, reports the last valid edit, and exits
+non-zero:
+
+```sh
+$ manifest_dump /tmp/mydb
+...
+Current Version:
+  Level 0: 1 files (170 bytes total)
+    [1] apple..cherry (170 bytes)
+
+Last valid edit: #1
+Error: corrupt manifest at edit #2: beachdb/record: checksum mismatch
+```
+
+## Exit codes
+
+| Code | Meaning                                                        |
+|------|----------------------------------------------------------------|
+| `0`  | Manifest dumped (or fresh database, or recoverable truncation) |
+| `1`  | Bad arguments, unreadable directory, or a corrupt manifest     |