diff --git a/TreeDB/caching/db.go b/TreeDB/caching/db.go index 7739c15200..0de6383987 100644 --- a/TreeDB/caching/db.go +++ b/TreeDB/caching/db.go @@ -19428,7 +19428,7 @@ func (db *DB) Set(key, value []byte) error { } db.waitForCheckpoint() unlock := db.lockUpdateKey(key) - defer unlock() + defer unlock.Unlock() return db.set(key, value, false) } @@ -19441,7 +19441,7 @@ func (db *DB) SetSync(key, value []byte) error { } db.waitForCheckpoint() unlock := db.lockUpdateKey(key) - defer unlock() + defer unlock.Unlock() return db.set(key, value, true) } @@ -19469,7 +19469,7 @@ func (db *DB) update(key []byte, fn backenddb.UpdateFunc, syncWrite bool) error for { unlock := db.lockUpdateKey(key) old, err := db.getForUpdate(key) - unlock() + unlock.Unlock() if err != nil { return err } @@ -19489,28 +19489,28 @@ func (db *DB) update(key []byte, fn backenddb.UpdateFunc, syncWrite bool) error unlock = db.lockUpdateKey(key) latest, err := db.getForUpdate(key) if err != nil { - unlock() + unlock.Unlock() return err } if !sameUpdateValue(observed, latest) { - unlock() + unlock.Unlock() continue } switch result.Op { case backenddb.UpdateNoop: - unlock() + unlock.Unlock() return nil case backenddb.UpdateSet: err = db.set(key, result.Value, syncWrite) - unlock() + unlock.Unlock() return err case backenddb.UpdateDelete: err = db.delete(key, syncWrite) - unlock() + unlock.Unlock() return err default: - unlock() + unlock.Unlock() return fmt.Errorf("treedb: unknown update op %d", result.Op) } } @@ -19781,7 +19781,7 @@ func (db *DB) Delete(key []byte) error { } db.waitForCheckpoint() unlock := db.lockUpdateKey(key) - defer unlock() + defer unlock.Unlock() return db.delete(key, false) } @@ -20395,13 +20395,13 @@ func (db *DB) DeleteSync(key []byte) error { } db.waitForCheckpoint() unlock := db.lockUpdateKey(key) - defer unlock() + defer unlock.Unlock() return db.delete(key, true) } -func (db *DB) lockUpdateKey(key []byte) func() { +func (db *DB) lockUpdateKey(key []byte) keyupdate.Unlocker { if db == nil { - return func() {} + return keyupdate.Unlocker{} } return db.updateLocks.Lock(key) } @@ -22792,14 +22792,42 @@ type getManyProbeRef struct { shard int } -func copyGetManyValueToRefs(out [][]byte, refs []getManyProbeRef, val []byte) { +const ( + getManyValueGuessBytes = 128 + getManyMaxArenaInitialCapBytes = 1 << 20 +) + +var getManyEmptyValue = []byte{} + +type getManyValueCopyArena struct { + buf []byte +} + +func newGetManyValueCopyArena(n int) getManyValueCopyArena { + arenaCap := n * getManyValueGuessBytes + if arenaCap < 0 { + arenaCap = 0 + } + if arenaCap > getManyMaxArenaInitialCapBytes { + arenaCap = getManyMaxArenaInitialCapBytes + } + return getManyValueCopyArena{buf: make([]byte, 0, arenaCap)} +} + +func (arena *getManyValueCopyArena) copyToRefs(out [][]byte, refs []getManyProbeRef, val []byte) { if val == nil { return } + if len(val) == 0 { + for _, ref := range refs { + out[ref.idx] = getManyEmptyValue + } + return + } for _, ref := range refs { - cpy := make([]byte, len(val)) - copy(cpy, val) - out[ref.idx] = cpy + start := len(arena.buf) + arena.buf = append(arena.buf, val...) + out[ref.idx] = arena.buf[start:len(arena.buf):len(arena.buf)] } } @@ -22834,6 +22862,7 @@ func (db *DB) getManyFromPublishedRootPointShards(view *memtableView, keys [][]b db.noteRootDomainGetManyNative(len(keys), len(unique)) results := make([]rootDomainProbeResult, len(unique)) + arena := newGetManyValueCopyArena(len(unique)) start := 0 for start < len(unique) { end := start + 1 @@ -22849,8 +22878,8 @@ func (db *DB) getManyFromPublishedRootPointShards(view *memtableView, keys [][]b start = end } - backendIdx := make([]int, 0, len(unique)) - backendKeys := make([][]byte, 0, len(unique)) + var backendIdx []int + var backendKeys [][]byte for i, res := range results { groupEnd := len(refs) if i+1 < len(groupStarts) { @@ -22859,23 +22888,28 @@ func (db *DB) getManyFromPublishedRootPointShards(view *memtableView, keys [][]b groupRefs := refs[groupStarts[i]:groupEnd] switch { case !res.found: + if backendIdx == nil { + backendCap := len(unique) - i + backendIdx = make([]int, 0, backendCap) + backendKeys = make([][]byte, 0, backendCap) + } backendIdx = append(backendIdx, i) backendKeys = append(backendKeys, unique[i].key) case res.flags&node.FlagTombstone != 0: case res.flags&node.FlagPointer != 0: if res.val != nil { - copyGetManyValueToRefs(out, groupRefs, res.val) + arena.copyToRefs(out, groupRefs, res.val) break } readVal, err := db.readValueLog(unique[i].key, res.ptr) if err != nil { return nil, err } - copyGetManyValueToRefs(out, groupRefs, readVal) + arena.copyToRefs(out, groupRefs, readVal) case res.val == nil: - copyGetManyValueToRefs(out, groupRefs, []byte{}) + arena.copyToRefs(out, groupRefs, getManyEmptyValue) default: - copyGetManyValueToRefs(out, groupRefs, res.val) + arena.copyToRefs(out, groupRefs, res.val) } } if len(backendKeys) > 0 { @@ -22892,7 +22926,7 @@ func (db *DB) getManyFromPublishedRootPointShards(view *memtableView, keys [][]b if uniqueIdx+1 < len(groupStarts) { groupEnd = groupStarts[uniqueIdx+1] } - copyGetManyValueToRefs(out, refs[groupStarts[uniqueIdx]:groupEnd], backendVals[i]) + arena.copyToRefs(out, refs[groupStarts[uniqueIdx]:groupEnd], backendVals[i]) } } return out, nil @@ -23149,25 +23183,13 @@ func (db *DB) GetMany(keys [][]byte) ([][]byte, error) { // // The cache layer may need to resolve value-log pointers for memtable hits; // by using the append path, those decodes can write directly into this arena - // instead of allocating per key. The limit below bounds only the initial - // arena capacity; subsequent appends may still grow the backing array, so - // multiple underlying allocations may be retained. - const ( - getManyValueGuessBytes = 128 - getManyMaxArenaInitialCapBytes = 1 << 20 - ) - arenaCap := len(keys) * getManyValueGuessBytes - if arenaCap < 0 { - arenaCap = 0 - } - if arenaCap > getManyMaxArenaInitialCapBytes { - arenaCap = getManyMaxArenaInitialCapBytes - } - arena := make([]byte, 0, arenaCap) - emptyValue := []byte{} + // instead of allocating per key. newGetManyValueCopyArena caps only the + // initial arena capacity; subsequent appends may still grow the backing + // array, so multiple underlying allocations may be retained. + arena := newGetManyValueCopyArena(len(keys)) for i, key := range keys { - start := len(arena) - nextArena, found, err := db.getMemtableAppend(key, arena) + start := len(arena.buf) + nextArena, found, err := db.getMemtableAppend(key, arena.buf) if err != nil { if err == tree.ErrKeyNotFound { // Tombstone in cache layers: treat as a missing key and do not fall @@ -23182,12 +23204,12 @@ func (db *DB) GetMany(keys [][]byte) ([][]byte, error) { } } if found { - arena = nextArena - if len(arena) == start { - out[i] = emptyValue + arena.buf = nextArena + if len(arena.buf) == start { + out[i] = getManyEmptyValue continue } - out[i] = arena[start:len(arena):len(arena)] + out[i] = arena.buf[start:len(arena.buf):len(arena.buf)] continue } backendIdx = append(backendIdx, i) @@ -25836,19 +25858,75 @@ func (db *DB) Iterator(start, end []byte) (merging.Iterator, error) { type debugIterator struct { merging.Iterator - queueLen int - sourcesUsed int + queueLen int + sourcesUsed int + keyScratch []byte + valueScratch []byte +} + +type unsafeIteratorView interface { + // UnsafeKey and UnsafeValue return views owned by the iterator and valid + // only until the iterator moves or closes. Wrappers may return safe Key/Value + // copies when the wrapped iterator does not expose unsafe views. + UnsafeKey() []byte + UnsafeValue() []byte +} + +func unsafeIteratorViewKey(it merging.Iterator, scratch *[]byte) []byte { + if it == nil { + return nil + } + if u, ok := it.(unsafeIteratorView); ok { + return u.UnsafeKey() + } + if scratch == nil { + return it.Key() + } + *scratch = it.KeyCopy((*scratch)[:0]) + return *scratch +} + +func unsafeIteratorViewValue(it merging.Iterator, scratch *[]byte) []byte { + if it == nil { + return nil + } + if u, ok := it.(unsafeIteratorView); ok { + return u.UnsafeValue() + } + if scratch == nil { + return it.Value() + } + *scratch = it.ValueCopy((*scratch)[:0]) + return *scratch } func (it *debugIterator) DebugStats() (queueLen int, sourcesUsed int) { return it.queueLen, it.sourcesUsed } +func (it *debugIterator) UnsafeKey() []byte { + return unsafeIteratorViewKey(it.Iterator, &it.keyScratch) +} + +func (it *debugIterator) UnsafeValue() []byte { + return unsafeIteratorViewValue(it.Iterator, &it.valueScratch) +} + type leasedMergingIterator struct { merging.Iterator - closeOnce sync.Once - closeErr error - release func() + closeOnce sync.Once + closeErr error + release func() + keyScratch []byte + valueScratch []byte +} + +func (it *leasedMergingIterator) UnsafeKey() []byte { + return unsafeIteratorViewKey(it.Iterator, &it.keyScratch) +} + +func (it *leasedMergingIterator) UnsafeValue() []byte { + return unsafeIteratorViewValue(it.Iterator, &it.valueScratch) } func (it *leasedMergingIterator) Close() error { @@ -25863,9 +25941,19 @@ func (it *leasedMergingIterator) Close() error { type foregroundTrackedIterator struct { merging.Iterator - db *DB - closeOnce sync.Once - closeErr error + db *DB + closeOnce sync.Once + closeErr error + keyScratch []byte + valueScratch []byte +} + +func (it *foregroundTrackedIterator) UnsafeKey() []byte { + return unsafeIteratorViewKey(it.Iterator, &it.keyScratch) +} + +func (it *foregroundTrackedIterator) UnsafeValue() []byte { + return unsafeIteratorViewValue(it.Iterator, &it.valueScratch) } func (it *foregroundTrackedIterator) Close() error { @@ -25962,6 +26050,20 @@ func (it *concatUnsafeIterator) Value() []byte { return it.cur.Value() } +func (it *concatUnsafeIterator) UnsafeKey() []byte { + if !it.valid { + panic("iterator invalid") + } + return it.cur.UnsafeKey() +} + +func (it *concatUnsafeIterator) UnsafeValue() []byte { + if !it.valid { + panic("iterator invalid") + } + return it.cur.UnsafeValue() +} + func (it *concatUnsafeIterator) KeyCopy(dst []byte) []byte { if !it.valid { panic("iterator invalid") diff --git a/TreeDB/caching/iterator_unsafe_forward_test.go b/TreeDB/caching/iterator_unsafe_forward_test.go new file mode 100644 index 0000000000..47e607bf0c --- /dev/null +++ b/TreeDB/caching/iterator_unsafe_forward_test.go @@ -0,0 +1,205 @@ +package caching + +import "testing" + +type unsafeForwardTestIterator struct { + key []byte + value []byte + valid bool + + keyCalls int + valueCalls int + keyCopyCalls int + valueCopyCalls int +} + +func (it *unsafeForwardTestIterator) Next() { + it.valid = false +} + +func (it *unsafeForwardTestIterator) Valid() bool { return it.valid } + +func (it *unsafeForwardTestIterator) Key() []byte { + it.keyCalls++ + return append([]byte(nil), it.key...) +} + +func (it *unsafeForwardTestIterator) Value() []byte { + it.valueCalls++ + return append([]byte(nil), it.value...) +} + +func (it *unsafeForwardTestIterator) KeyCopy(dst []byte) []byte { + it.keyCopyCalls++ + return append(dst[:0], it.key...) +} + +func (it *unsafeForwardTestIterator) ValueCopy(dst []byte) []byte { + it.valueCopyCalls++ + return append(dst[:0], it.value...) +} + +func (it *unsafeForwardTestIterator) Close() error { return nil } + +func (it *unsafeForwardTestIterator) Error() error { return nil } + +func (it *unsafeForwardTestIterator) Domain() ([]byte, []byte) { return nil, nil } + +func (it *unsafeForwardTestIterator) UnsafeKey() []byte { return it.key } + +func (it *unsafeForwardTestIterator) UnsafeValue() []byte { return it.value } + +type safeFallbackTestIterator struct { + key []byte + value []byte + valid bool + + keyCalls int + valueCalls int + keyCopyCalls int + valueCopyCalls int +} + +func (it *safeFallbackTestIterator) Next() { + it.valid = false +} + +func (it *safeFallbackTestIterator) Valid() bool { return it.valid } + +func (it *safeFallbackTestIterator) Key() []byte { + it.keyCalls++ + return append([]byte(nil), it.key...) +} + +func (it *safeFallbackTestIterator) Value() []byte { + it.valueCalls++ + return append([]byte(nil), it.value...) +} + +func (it *safeFallbackTestIterator) KeyCopy(dst []byte) []byte { + it.keyCopyCalls++ + return append(dst[:0], it.key...) +} + +func (it *safeFallbackTestIterator) ValueCopy(dst []byte) []byte { + it.valueCopyCalls++ + return append(dst[:0], it.value...) +} + +func (it *safeFallbackTestIterator) Close() error { return nil } + +func (it *safeFallbackTestIterator) Error() error { return nil } + +func (it *safeFallbackTestIterator) Domain() ([]byte, []byte) { return nil, nil } + +func TestIteratorWrappersForwardUnsafeViews(t *testing.T) { + base := &unsafeForwardTestIterator{ + key: []byte("key"), + value: []byte("value"), + valid: true, + } + foreground, ok := (&DB{}).wrapForegroundIterator(base).(unsafeIteratorView) + if !ok { + t.Fatalf("foreground iterator type=%T does not implement unsafeIteratorView", foreground) + } + + for _, tc := range []struct { + name string + view unsafeIteratorView + }{ + {name: "debug", view: &debugIterator{Iterator: base}}, + {name: "leased", view: &leasedMergingIterator{Iterator: base}}, + {name: "foreground", view: foreground}, + } { + key := tc.view.UnsafeKey() + if len(key) == 0 || &key[0] != &base.key[0] { + t.Fatalf("%s UnsafeKey did not forward the backing key view", tc.name) + } + value := tc.view.UnsafeValue() + if len(value) == 0 || &value[0] != &base.value[0] { + t.Fatalf("%s UnsafeValue did not forward the backing value view", tc.name) + } + } + + if base.keyCalls != 0 || base.valueCalls != 0 || base.keyCopyCalls != 0 || base.valueCopyCalls != 0 { + t.Fatalf( + "safe iterator fallback called: key=%d value=%d keyCopy=%d valueCopy=%d", + base.keyCalls, + base.valueCalls, + base.keyCopyCalls, + base.valueCopyCalls, + ) + } +} + +func TestIteratorWrappersFallbackToSafeCopiesWithoutUnsafeViews(t *testing.T) { + base := &safeFallbackTestIterator{ + key: []byte("key"), + value: []byte("value"), + valid: true, + } + foreground, ok := (&DB{}).wrapForegroundIterator(base).(unsafeIteratorView) + if !ok { + t.Fatalf("foreground iterator type=%T does not implement unsafeIteratorView", foreground) + } + + for _, tc := range []struct { + name string + view unsafeIteratorView + }{ + {name: "debug", view: &debugIterator{Iterator: base}}, + {name: "leased", view: &leasedMergingIterator{Iterator: base}}, + {name: "foreground", view: foreground}, + } { + key := tc.view.UnsafeKey() + if string(key) != "key" { + t.Fatalf("%s UnsafeKey fallback=%q want key", tc.name, key) + } + if len(key) != 0 && &key[0] == &base.key[0] { + t.Fatalf("%s UnsafeKey fallback returned backing key view", tc.name) + } + value := tc.view.UnsafeValue() + if string(value) != "value" { + t.Fatalf("%s UnsafeValue fallback=%q want value", tc.name, value) + } + if len(value) != 0 && &value[0] == &base.value[0] { + t.Fatalf("%s UnsafeValue fallback returned backing value view", tc.name) + } + } + + if base.keyCalls != 0 || base.valueCalls != 0 { + t.Fatalf( + "safe iterator fallback used allocating accessors: key=%d value=%d keyCopy=%d valueCopy=%d", + base.keyCalls, + base.valueCalls, + base.keyCopyCalls, + base.valueCopyCalls, + ) + } + if base.keyCopyCalls == 0 || base.valueCopyCalls == 0 { + t.Fatalf( + "safe iterator fallback did not use copy accessors: key=%d value=%d keyCopy=%d valueCopy=%d", + base.keyCalls, + base.valueCalls, + base.keyCopyCalls, + base.valueCopyCalls, + ) + } +} + +func TestConcatUnsafeIteratorPanicsWhenInvalid(t *testing.T) { + it := &concatUnsafeIterator{} + for name, fn := range map[string]func(){ + "UnsafeKey": func() { _ = it.UnsafeKey() }, + "UnsafeValue": func() { _ = it.UnsafeValue() }, + } { + t.Run(name, func(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatal("expected panic") + } + }() + fn() + }) + } +} diff --git a/TreeDB/caching/point_read_shard_queue_test.go b/TreeDB/caching/point_read_shard_queue_test.go index 1eb954473f..cdb21a7d8f 100644 --- a/TreeDB/caching/point_read_shard_queue_test.go +++ b/TreeDB/caching/point_read_shard_queue_test.go @@ -1,6 +1,7 @@ package caching import ( + "bytes" "encoding/binary" "fmt" "testing" @@ -576,12 +577,52 @@ func TestGetMany_DuplicateHitsReuseSingleProbeWithoutResultAliasing(t *testing.T if ct.iterCalls != 1 { t.Fatalf("expected one iterator probe, got %d", ct.iterCalls) } + + beforeAppend := append([]byte(nil), got[1]...) + _ = append(got[0], []byte("-suffix")...) + if !bytes.Equal(got[1], beforeAppend) { + t.Fatalf("append to duplicate output corrupted another output: got=%q want=%q", got[1], beforeAppend) + } got[0][0] = 'X' if string(got[1]) != "value" || string(got[2]) != "value" { t.Fatalf("duplicate outputs must not alias: %#v", got) } } +func TestGetMany_PublishedRootPointShardsPreservesEmptyValue(t *testing.T) { + db := &DB{ + backend: panicBackend{}, + mutableShards: make([]memShard, 1), + mutableShardMask: 0, + } + + mt, err := memtable.NewWithCapacityMode(0, memtable.ModeHashSorted) + if err != nil { + t.Fatalf("new memtable: %v", err) + } + ct := &countingTable{inner: mt} + key := []byte("empty") + ct.SetEntry(key, []byte{}, page.ValuePtr{}, node.FlagInline) + db.memtables.Store(&memtableView{ + rootPointShards: []rootDomainSnapshot{ + {immutables: []memtable.Table{ct}}, + }, + }) + + got, err := db.GetMany([][]byte{key, key}) + if err != nil { + t.Fatalf("GetMany: %v", err) + } + if len(got) != 2 { + t.Fatalf("len(GetMany)=%d want 2", len(got)) + } + for i := range got { + if got[i] == nil || len(got[i]) != 0 { + t.Fatalf("got[%d]=%q want empty non-nil value", i, got[i]) + } + } +} + func TestGetMany_DuplicateMissesCollapseToSingleBackendKey(t *testing.T) { backend := &countingBackend{} db := &DB{ @@ -615,6 +656,16 @@ func TestGetMany_DuplicateMissesCollapseToSingleBackendKey(t *testing.T) { if len(backend.lastGetMany) != 1 || string(backend.lastGetMany[0]) != "missing" { t.Fatalf("expected one deduped backend key, got %#v", backend.lastGetMany) } + + beforeAppend := append([]byte(nil), got[1]...) + _ = append(got[0], []byte("-suffix")...) + if !bytes.Equal(got[1], beforeAppend) { + t.Fatalf("append to duplicate backend output corrupted another output: got=%q want=%q", got[1], beforeAppend) + } + got[0][0] = 'X' + if string(got[1]) != "backend" || string(got[2]) != "backend" { + t.Fatalf("duplicate backend outputs must not alias: %#v", got) + } } func BenchmarkGetMany_PublishedRootPointShards(b *testing.B) { diff --git a/TreeDB/caching/root_domain.go b/TreeDB/caching/root_domain.go index a2a1eb70ad..51439bcefc 100644 --- a/TreeDB/caching/root_domain.go +++ b/TreeDB/caching/root_domain.go @@ -19,6 +19,10 @@ type rootDomainLookup interface { GetEntry(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool) } +type rootDomainLookupWithError interface { + GetEntryWithError(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool, err error) +} + type rootDomainIteratorFactory interface { Iterator(start, end []byte) (iterator.UnsafeIterator, error) } @@ -692,18 +696,17 @@ type backendSnapshotLookup struct { rootID uint64 } -func (l backendSnapshotLookup) GetEntry(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool) { +func (l backendSnapshotLookup) GetEntryWithError(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool, err error) { if l.snapshot == nil { - return nil, page.ValuePtr{}, 0, false + return nil, page.ValuePtr{}, 0, false, backenddb.ErrClosed } if l.db != nil { if err := l.db.flushValueLogForBackendRead(); err != nil { - return nil, page.ValuePtr{}, 0, false + return nil, page.ValuePtr{}, 0, false, err } } var ( entry node.LeafEntry - err error ) if l.rootID != 0 { entry, err = l.snapshot.GetEntryAtRoot(l.rootID, key) @@ -712,11 +715,16 @@ func (l backendSnapshotLookup) GetEntry(key []byte) (val []byte, ptr page.ValueP } if err != nil { if errors.Is(err, tree.ErrKeyNotFound) { - return nil, page.ValuePtr{}, 0, false + return nil, page.ValuePtr{}, 0, false, nil } - return nil, page.ValuePtr{}, 0, false + return nil, page.ValuePtr{}, 0, false, err } - return entry.Value, entry.ValuePtr, entry.Flags, true + return entry.Value, entry.ValuePtr, entry.Flags, true, nil +} + +func (l backendSnapshotLookup) GetEntry(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool) { + val, ptr, flags, found, _ = l.GetEntryWithError(key) + return val, ptr, flags, found } func (l backendSnapshotLookup) GetValueAppend(key, dst []byte) ([]byte, error) { @@ -889,12 +897,142 @@ func rootDomainSystemSnapshotFromCachedSnapshot(s *Snapshot) rootDomainSnapshot snap.publishedRootID = rootID } if rootID != 0 { - snap.published = backendSnapshotLookup{db: s.db, snapshot: s.backend, rootID: rootID} + snap.published = s.backendSnapshotLookupForRoot(rootID) } } return snap } +func (s *Snapshot) staticBackendSnapshotLookupForRoot(rootID uint64) *backendSnapshotLookup { + if s == nil || s.backend == nil { + return nil + } + if s.backendRootOK && (s.backendRoot.rootID == rootID || rootID == 0) { + return &s.backendRoot + } + if s.backendSystemOK && s.backendSystem.rootID == rootID { + return &s.backendSystem + } + return nil +} + +func (s *Snapshot) backendSnapshotLookupForRoot(rootID uint64) rootDomainLookup { + if s == nil || s.backend == nil { + return nil + } + if lookup := s.staticBackendSnapshotLookupForRoot(rootID); lookup != nil { + return lookup + } + return backendSnapshotLookup{db: s.db, snapshot: s.backend, rootID: rootID} +} + +func (s *Snapshot) installBackendPublishedRootLookups(publishedRootsOwned bool) { + if s == nil || s.backend == nil { + return + } + needed := 0 + directPointNeedsInstall := false + countRef := func(ref publishedRootRef) { + if ref.lookup == nil && ref.rootID != 0 && s.staticBackendSnapshotLookupForRoot(ref.rootID) == nil { + needed++ + } + } + countSnapshot := func(snap rootDomainSnapshot) bool { + if snap.published != nil || snap.publishedRootID == 0 { + return false + } + if s.staticBackendSnapshotLookupForRoot(snap.publishedRootID) == nil { + needed++ + } + return true + } + if s.publishedRoots != nil { + for _, ref := range s.publishedRoots.pointShards { + countRef(ref) + } + countRef(s.publishedRoots.system) + countRef(s.publishedRoots.iterator) + } + for _, snap := range s.rootPointShards { + if countSnapshot(snap) { + directPointNeedsInstall = true + } + } + countSnapshot(s.rootSystem) + countSnapshot(s.rootIterator) + if needed == 0 { + return + } + + cloned := s.publishedRoots + if cloned != nil && !publishedRootsOwned { + cloned = clonePublishedRootSet(s.publishedRoots) + } + s.backendPublishedLookups = make([]backendSnapshotLookup, needed) + next := 0 + installLookup := func(rootID uint64) rootDomainLookup { + if rootID == 0 { + return nil + } + if lookup := s.staticBackendSnapshotLookupForRoot(rootID); lookup != nil { + return lookup + } + s.backendPublishedLookups[next] = backendSnapshotLookup{db: s.db, snapshot: s.backend, rootID: rootID} + lookup := &s.backendPublishedLookups[next] + next++ + return lookup + } + installRef := func(ref *publishedRootRef) { + if ref == nil || ref.lookup != nil || ref.rootID == 0 { + return + } + ref.lookup = installLookup(ref.rootID) + } + installSnapshot := func(snap *rootDomainSnapshot) { + if snap == nil || snap.published != nil || snap.publishedRootID == 0 { + return + } + snap.published = installLookup(snap.publishedRootID) + } + if cloned != nil { + for i := range cloned.pointShards { + installRef(&cloned.pointShards[i]) + } + installRef(&cloned.system) + installRef(&cloned.iterator) + } + if directPointNeedsInstall { + s.rootPointShards = append([]rootDomainSnapshot(nil), s.rootPointShards...) + } + for i := range s.rootPointShards { + installSnapshot(&s.rootPointShards[i]) + } + installSnapshot(&s.rootSystem) + installSnapshot(&s.rootIterator) + s.backendPublishedLookups = s.backendPublishedLookups[:next] + s.publishedRoots = cloned +} + +func backendSnapshotLookupFromRootDomainLookup(lookup rootDomainLookup) (backendSnapshotLookup, bool) { + switch l := lookup.(type) { + case backendSnapshotLookup: + return l, true + case *backendSnapshotLookup: + if l != nil { + return *l, true + } + } + return backendSnapshotLookup{}, false +} + +func (s *Snapshot) publishedLookupBackedByBackendSnapshot(snap rootDomainSnapshot) bool { + if s == nil || s.backend == nil { + return false + } + lookup, ok := backendSnapshotLookupFromRootDomainLookup(snap.published) + return ok && lookup.snapshot == s.backend +} + func rootDomainSnapshotBackendRootID(s *Snapshot, fallback uint64) uint64 { if fallback != 0 { return fallback @@ -934,7 +1072,7 @@ func rootDomainApplyPublishedRef(s *Snapshot, snap *rootDomainSnapshot, ref publ return true } if s != nil && s.backend != nil && snap.publishedRootID != 0 { - snap.published = backendSnapshotLookup{db: s.db, snapshot: s.backend, rootID: snap.publishedRootID} + snap.published = s.backendSnapshotLookupForRoot(snap.publishedRootID) return true } return false @@ -948,7 +1086,7 @@ func rootDomainApplyBackendFallback(s *Snapshot, snap *rootDomainSnapshot) { if rootID != 0 { snap.publishedRootID = rootID } - snap.published = backendSnapshotLookup{db: s.db, snapshot: s.backend, rootID: rootID} + snap.published = s.backendSnapshotLookupForRoot(rootID) } func rootDomainIteratorPublishedRef(set *publishedRootSet) publishedRootRef { @@ -1120,7 +1258,7 @@ func (s rootDomainSnapshot) getEntry(key []byte) (val []byte, ptr page.ValuePtr, return val, ptr, flags, found } -func (s rootDomainSnapshot) getEntryWithSource(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool, source rootDomainEntrySource) { +func (s rootDomainSnapshot) getCachedEntryWithSource(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool, source rootDomainEntrySource) { if s.mutable != nil { if val, ptr, flags, found = s.mutable.GetEntry(key); found { return val, ptr, flags, true, rootDomainEntrySourceCached @@ -1131,13 +1269,45 @@ func (s rootDomainSnapshot) getEntryWithSource(key []byte) (val []byte, ptr page return val, ptr, flags, true, rootDomainEntrySourceCached } } - if s.published != nil { - val, ptr, flags, found = s.published.GetEntry(key) + return nil, page.ValuePtr{}, 0, false, rootDomainEntrySourceNone +} + +func (s rootDomainSnapshot) getPublishedEntryWithSource(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool, source rootDomainEntrySource) { + val, ptr, flags, found, source, _ = s.getPublishedEntryWithSourceError(key) + return val, ptr, flags, found, source +} + +func (s rootDomainSnapshot) getPublishedEntryWithSourceError(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool, source rootDomainEntrySource, err error) { + if s.published == nil { + return nil, page.ValuePtr{}, 0, false, rootDomainEntrySourceNone, nil + } + if lookup, ok := s.published.(rootDomainLookupWithError); ok { + val, ptr, flags, found, err = lookup.GetEntryWithError(key) + if err != nil { + return nil, page.ValuePtr{}, 0, false, rootDomainEntrySourceNone, err + } if found { - return val, ptr, flags, true, rootDomainEntrySourcePublished + return val, ptr, flags, true, rootDomainEntrySourcePublished, nil } + return nil, page.ValuePtr{}, 0, false, rootDomainEntrySourceNone, nil } - return nil, page.ValuePtr{}, 0, false, rootDomainEntrySourceNone + val, ptr, flags, found = s.published.GetEntry(key) + if found { + return val, ptr, flags, true, rootDomainEntrySourcePublished, nil + } + return nil, page.ValuePtr{}, 0, false, rootDomainEntrySourceNone, nil +} + +func (s rootDomainSnapshot) getEntryWithSource(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool, source rootDomainEntrySource) { + val, ptr, flags, found, source, _ = s.getEntryWithSourceError(key) + return val, ptr, flags, found, source +} + +func (s rootDomainSnapshot) getEntryWithSourceError(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool, source rootDomainEntrySource, err error) { + if val, ptr, flags, found, source = s.getCachedEntryWithSource(key); found { + return val, ptr, flags, true, source, nil + } + return s.getPublishedEntryWithSourceError(key) } func (s rootDomainSnapshot) visibleValue(key []byte) ([]byte, bool) { diff --git a/TreeDB/caching/root_group_snapshot_test.go b/TreeDB/caching/root_group_snapshot_test.go index 6903e23e37..8d36051cb8 100644 --- a/TreeDB/caching/root_group_snapshot_test.go +++ b/TreeDB/caching/root_group_snapshot_test.go @@ -508,6 +508,13 @@ func TestAcquireSnapshot_FallsBackToBackendPublishedSetWithoutInstalledGroup(t * if rootSnap.publishedRootID == 0 { t.Fatal("expected backend published root id") } + lookup, ok := rootSnap.published.(*backendSnapshotLookup) + if !ok { + t.Fatalf("published lookup type=%T, want *backendSnapshotLookup", rootSnap.published) + } + if lookup != &snap.backendRoot { + t.Fatal("expected point fallback to reuse snapshot backendRoot lookup") + } if stats := db.rootDomainPublishStatsSnapshot(); stats.backendFallbacks != 1 { t.Fatalf("backendFallbacks=%d want 1", stats.backendFallbacks) } @@ -536,4 +543,258 @@ func TestAcquireSnapshot_BackendFallbackPinsSystemRootPageID(t *testing.T) { if got, want := systemSnap.publishedRootID, backend.State().SystemRootPageID; got != want { t.Fatalf("system published root id=%d want %d", got, want) } + lookup, ok := systemSnap.published.(*backendSnapshotLookup) + if !ok { + t.Fatalf("system published lookup type=%T, want *backendSnapshotLookup", systemSnap.published) + } + if lookup != &snap.backendSystem { + t.Fatal("expected system fallback to reuse snapshot backendSystem lookup") + } +} + +func TestAcquireSnapshot_InstallsBackendLookupForPublishedPointRoots(t *testing.T) { + dir := t.TempDir() + backend, err := backenddb.Open(backenddb.Options{Dir: dir}) + if err != nil { + t.Fatalf("open backend: %v", err) + } + defer backend.Close() + + pointTable := newRootDomainTestTable(t, rootDomainTestOp{key: "published/k", value: "published-v"}) + pointRootID, err := backend.PublishOrderedRootIterator(0, pointTable.NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish point root: %v", err) + } + if pointRootID == backend.State().RootPageID { + t.Fatalf("test point root unexpectedly matches default root %d", pointRootID) + } + + db := &DB{ + backend: backend, + mutableShards: make([]memShard, 1), + mutableShardMask: 0, + } + view := &memtableView{ + rootSnapshotShards: []rootDomainSnapshot{{}}, + publishedRoots: &publishedRootSet{ + pointShards: []publishedRootRef{{rootID: pointRootID}}, + }, + } + view.refs.Store(1) + db.memtables.Store(view) + + snap := db.AcquireSnapshot() + if snap == nil { + t.Fatal("expected snapshot") + } + defer snap.Close() + + if snap.publishedRoots == view.publishedRoots { + t.Fatal("expected snapshot to clone published root set before installing lookups") + } + if got := len(snap.backendPublishedLookups); got != 1 { + t.Fatalf("backendPublishedLookups len=%d want 1", got) + } + + rootSnap := rootDomainSnapshotFromCachedSnapshot(snap, []byte("published/k")) + lookup, ok := rootSnap.published.(*backendSnapshotLookup) + if !ok { + t.Fatalf("published lookup type=%T, want *backendSnapshotLookup", rootSnap.published) + } + if lookup != &snap.backendPublishedLookups[0] { + t.Fatal("expected point published ref to use snapshot-owned backend lookup") + } + if lookup.rootID != pointRootID { + t.Fatalf("lookup rootID=%d want %d", lookup.rootID, pointRootID) + } + if lookup.snapshot != snap.backend { + t.Fatal("expected published lookup to use acquired backend snapshot") + } + if lookup.db != db { + t.Fatal("expected published lookup to retain parent db") + } +} + +func TestAcquireSnapshot_InstallsBackendLookupForPublishedSystemAndIteratorRoots(t *testing.T) { + dir := t.TempDir() + backend, err := backenddb.Open(backenddb.Options{Dir: dir}) + if err != nil { + t.Fatalf("open backend: %v", err) + } + defer backend.Close() + + systemTable := newRootDomainTestTable(t, rootDomainTestOp{key: "system/k", value: "system-v"}) + systemRootID, err := backend.PublishOrderedRootIterator(0, systemTable.NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish system root: %v", err) + } + iteratorTable := newRootDomainTestTable(t, rootDomainTestOp{key: "iterator/k", value: "iterator-v"}) + iteratorRootID, err := backend.PublishOrderedRootIterator(0, iteratorTable.NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish iterator root: %v", err) + } + if state := backend.State(); state != nil { + if systemRootID == state.RootPageID || systemRootID == state.SystemRootPageID { + t.Fatalf("test system root unexpectedly matches static root state: %d", systemRootID) + } + if iteratorRootID == state.RootPageID || iteratorRootID == state.SystemRootPageID { + t.Fatalf("test iterator root unexpectedly matches static root state: %d", iteratorRootID) + } + } + + db := &DB{ + backend: backend, + mutableShards: make([]memShard, 1), + mutableShardMask: 0, + } + view := &memtableView{ + publishedRoots: &publishedRootSet{ + system: publishedRootRef{rootID: systemRootID}, + iterator: publishedRootRef{rootID: iteratorRootID}, + }, + } + view.refs.Store(1) + db.memtables.Store(view) + + snap := db.AcquireSnapshot() + if snap == nil { + t.Fatal("expected snapshot") + } + defer snap.Close() + + if got := len(snap.backendPublishedLookups); got != 2 { + t.Fatalf("backendPublishedLookups len=%d want 2", got) + } + systemLookup, ok := snap.publishedRoots.system.lookup.(*backendSnapshotLookup) + if !ok { + t.Fatalf("system lookup type=%T, want *backendSnapshotLookup", snap.publishedRoots.system.lookup) + } + iteratorLookup, ok := snap.publishedRoots.iterator.lookup.(*backendSnapshotLookup) + if !ok { + t.Fatalf("iterator lookup type=%T, want *backendSnapshotLookup", snap.publishedRoots.iterator.lookup) + } + if systemLookup.rootID != systemRootID { + t.Fatalf("system lookup rootID=%d want %d", systemLookup.rootID, systemRootID) + } + if iteratorLookup.rootID != iteratorRootID { + t.Fatalf("iterator lookup rootID=%d want %d", iteratorLookup.rootID, iteratorRootID) + } + if systemLookup == iteratorLookup { + t.Fatal("expected distinct system and iterator lookup slots") + } +} + +func TestAcquireSnapshot_InstallsBackendLookupForDirectPublishedPointRootID(t *testing.T) { + dir := t.TempDir() + backend, err := backenddb.Open(backenddb.Options{Dir: dir}) + if err != nil { + t.Fatalf("open backend: %v", err) + } + defer backend.Close() + + pointTable := newRootDomainTestTable(t, rootDomainTestOp{key: "published/k", value: "published-v"}) + pointRootID, err := backend.PublishOrderedRootIterator(0, pointTable.NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish point root: %v", err) + } + if pointRootID == backend.State().RootPageID { + t.Fatalf("test point root unexpectedly matches default root %d", pointRootID) + } + + db := &DB{ + backend: backend, + mutableShards: make([]memShard, 1), + mutableShardMask: 0, + } + view := &memtableView{ + queue: []memtable.Table{newRootDomainTestTable(t, rootDomainTestOp{key: "queued/k", value: "queued-v"})}, + rootSnapshotShards: []rootDomainSnapshot{{publishedRootID: pointRootID}}, + } + view.refs.Store(1) + db.memtables.Store(view) + + snap := db.AcquireSnapshot() + if snap == nil { + t.Fatal("expected snapshot") + } + defer snap.Close() + + if got := len(snap.backendPublishedLookups); got != 1 { + t.Fatalf("backendPublishedLookups len=%d want 1", got) + } + if snap.publishedRoots != nil { + t.Fatal("expected direct published root ID to avoid synthesizing a published root set") + } + if view.rootSnapshotShards[0].published != nil { + t.Fatal("expected snapshot acquisition not to mutate retained memtable view root snapshot") + } + lookup, ok := snap.rootPointShards[0].published.(*backendSnapshotLookup) + if !ok { + t.Fatalf("published lookup type=%T, want *backendSnapshotLookup", snap.rootPointShards[0].published) + } + if lookup != &snap.backendPublishedLookups[0] { + t.Fatal("expected direct point root to use snapshot-owned backend lookup") + } + if lookup.rootID != pointRootID { + t.Fatalf("lookup rootID=%d want %d", lookup.rootID, pointRootID) + } +} + +func TestAcquireSnapshot_InstalledPublishedPointRootAllocsBounded(t *testing.T) { + if testRaceEnabled { + t.Skip("AllocsPerRun is not stable under -race") + } + dir := t.TempDir() + backend, err := backenddb.Open(backenddb.Options{Dir: dir}) + if err != nil { + t.Fatalf("open backend: %v", err) + } + defer backend.Close() + + pointTable := newRootDomainTestTable(t, rootDomainTestOp{key: "published/k", value: "published-v"}) + pointRootID, err := backend.PublishOrderedRootIterator(0, pointTable.NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish point root: %v", err) + } + if pointRootID == backend.State().RootPageID { + t.Fatalf("test point root unexpectedly matches default root %d", pointRootID) + } + + db := &DB{ + backend: backend, + mutableShards: make([]memShard, 1), + mutableShardMask: 0, + } + view := &memtableView{ + rootSnapshotShards: []rootDomainSnapshot{{}}, + publishedRoots: &publishedRootSet{ + pointShards: []publishedRootRef{{rootID: pointRootID}}, + }, + } + view.refs.Store(1) + db.memtables.Store(view) + + warm := db.AcquireSnapshot() + if warm == nil { + t.Fatal("warm AcquireSnapshot=nil") + } + if err := warm.Close(); err != nil { + t.Fatalf("warm Close: %v", err) + } + + allocs := testing.AllocsPerRun(1000, func() { + snap := db.AcquireSnapshot() + if snap == nil { + t.Fatal("AcquireSnapshot=nil") + } + if len(snap.backendPublishedLookups) != 1 { + t.Fatalf("backendPublishedLookups len=%d want 1", len(snap.backendPublishedLookups)) + } + if err := snap.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + }) + if allocs > 5.1 { + t.Fatalf("AcquireSnapshot allocs/run=%f, want <= 5.1 with installed published point root", allocs) + } } diff --git a/TreeDB/caching/snapshot.go b/TreeDB/caching/snapshot.go index ac52e76dbd..41ead5c043 100644 --- a/TreeDB/caching/snapshot.go +++ b/TreeDB/caching/snapshot.go @@ -30,14 +30,19 @@ import ( // Snapshot pointers are single-use: after Close returns, callers must discard the // pointer and treat further use as invalid. type Snapshot struct { - db *DB - view *memtableView - backend *backenddb.Snapshot - rootVersion uint64 - rootPointShards []rootDomainSnapshot // snapshot point roots; mutable runs are intentionally excluded - rootSystem rootDomainSnapshot - rootIterator rootDomainSnapshot - publishedRoots *publishedRootSet + db *DB + view *memtableView + backend *backenddb.Snapshot + rootVersion uint64 + rootPointShards []rootDomainSnapshot // snapshot point roots; mutable runs are intentionally excluded + rootSystem rootDomainSnapshot + rootIterator rootDomainSnapshot + publishedRoots *publishedRootSet + backendRoot backendSnapshotLookup + backendRootOK bool + backendSystem backendSnapshotLookup + backendSystemOK bool + backendPublishedLookups []backendSnapshotLookup closed atomic.Bool } @@ -125,6 +130,7 @@ func (db *DB) AcquireSnapshot() *Snapshot { viewRootSystem rootDomainSnapshot viewRootIterator rootDomainSnapshot viewPublishedRoots *publishedRootSet + publishedRootsOwned bool ) if view != nil { viewRootVersion = view.rootVersion @@ -141,6 +147,7 @@ func (db *DB) AcquireSnapshot() *Snapshot { } else { viewRootPointShards = append([]rootDomainSnapshot(nil), view.rootSnapshotShards...) viewPublishedRoots = clonePublishedRootSet(view.publishedRoots) + publishedRootsOwned = true } db.releaseMemtableView(view) view = nil @@ -163,11 +170,21 @@ func (db *DB) AcquireSnapshot() *Snapshot { } snap := &Snapshot{db: db, view: view, backend: backendSnap} + snap.backendRoot = backendSnapshotLookup{db: db, snapshot: backendSnap} + snap.backendRootOK = true + if state := backendSnap.State(); state != nil { + snap.backendRoot.rootID = state.RootPageID + if state.SystemRootPageID != 0 { + snap.backendSystem = backendSnapshotLookup{db: db, snapshot: backendSnap, rootID: state.SystemRootPageID} + snap.backendSystemOK = true + } + } snap.rootVersion = viewRootVersion snap.rootPointShards = viewRootPointShards snap.rootSystem = viewRootSystem snap.rootIterator = viewRootIterator snap.publishedRoots = viewPublishedRoots + snap.installBackendPublishedRootLookups(publishedRootsOwned) if snap.publishedRoots == nil { db.rootPublishStats.backendFallbacks.Add(1) } @@ -209,42 +226,44 @@ func (s *Snapshot) Close() error { s.rootSystem = rootDomainSnapshot{} s.rootIterator = rootDomainSnapshot{} s.publishedRoots = nil + s.backendRoot = backendSnapshotLookup{} + s.backendRootOK = false + s.backendSystem = backendSnapshotLookup{} + s.backendSystemOK = false + s.backendPublishedLookups = nil s.rootVersion = 0 s.db = nil return err } -func (s *Snapshot) lookupQueueEntry(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool) { - val, ptr, flags, found, _ = s.lookupRootDomainEntry(key) - return val, ptr, flags, found -} - -func (s *Snapshot) lookupRootDomainSnapshotEntry(key []byte) (snap rootDomainSnapshot, val []byte, ptr page.ValuePtr, flags byte, found bool, source rootDomainEntrySource) { +func (s *Snapshot) lookupRootDomainSnapshotEntryWithError(key []byte) (snap rootDomainSnapshot, val []byte, ptr page.ValuePtr, flags byte, found bool, source rootDomainEntrySource, err error) { if s == nil { - return rootDomainSnapshot{}, nil, page.ValuePtr{}, 0, false, rootDomainEntrySourceNone + return rootDomainSnapshot{}, nil, page.ValuePtr{}, 0, false, rootDomainEntrySourceNone, nil } snap = rootDomainSnapshotFromCachedSnapshot(s, key) - val, ptr, flags, found, source = snap.getEntryWithSource(key) - return snap, val, ptr, flags, found, source -} - -func (s *Snapshot) lookupRootDomainEntry(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool, source rootDomainEntrySource) { - _, val, ptr, flags, found, source = s.lookupRootDomainSnapshotEntry(key) - return val, ptr, flags, found, source + val, ptr, flags, found, source, err = snap.getEntryWithSourceError(key) + return snap, val, ptr, flags, found, source, err } -func (s *Snapshot) lookupCachedRootDomainEntry(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool) { +func (s *Snapshot) cachedRootDomainSnapshot(key []byte) rootDomainSnapshot { if s == nil || len(s.rootPointShards) == 0 { - return nil, page.ValuePtr{}, 0, false + return rootDomainSnapshot{} } shardIdx := 0 if s.db != nil { shardIdx = s.db.shardIndex(key) } if shardIdx < 0 || shardIdx >= len(s.rootPointShards) { + return rootDomainSnapshot{} + } + return s.rootPointShards[shardIdx] +} + +func (s *Snapshot) lookupCachedRootDomainEntry(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool) { + snap := s.cachedRootDomainSnapshot(key) + if !rootDomainSnapshotHasInMemoryState(snap) { return nil, page.ValuePtr{}, 0, false } - snap := s.rootPointShards[shardIdx] snap.published = nil snap.publishedRootID = 0 return snap.getEntry(key) @@ -300,6 +319,36 @@ func rootDomainPublishedGetUnsafe(snap rootDomainSnapshot, key []byte) ([]byte, return out, true, err } +func (s *Snapshot) appendRootDomainEntryValue( + key, dst []byte, + val []byte, + ptr page.ValuePtr, + flags byte, + source rootDomainEntrySource, + oldLen int, +) ([]byte, error) { + if flags&node.FlagTombstone != 0 { + return dst, tree.ErrKeyNotFound + } + if flags&node.FlagPointer != 0 { + if s == nil || s.db == nil { + return dst, errors.New("caching snapshot: value-log reader unavailable") + } + out, err := s.db.readValueLogAppend(key, ptr, dst) + if err != nil { + return dst, err + } + recordSnapshotRootDomainRead(source, true, len(out)-oldLen) + return out, nil + } + if val == nil { + recordSnapshotRootDomainRead(source, false, 0) + return dst, nil + } + recordSnapshotRootDomainRead(source, false, len(val)) + return append(dst, val...), nil +} + func (s *Snapshot) iteratorSources(start, end []byte, reverse bool) ([]merging.IteratorSource, error) { if s == nil || s.backend == nil { return nil, backenddb.ErrClosed @@ -376,40 +425,32 @@ func (s *Snapshot) ReverseIterator(start, end []byte) (merging.Iterator, error) } func (s *Snapshot) GetAppend(key, dst []byte) ([]byte, error) { - snap, val, ptr, flags, found, source := s.lookupRootDomainSnapshotEntry(key) + if s == nil { + return dst, tree.ErrKeyNotFound + } + snap := rootDomainSnapshotFromCachedSnapshot(s, key) + val, ptr, flags, found, _ := snap.getCachedEntryWithSource(key) if found { - if flags&node.FlagTombstone != 0 { - return dst, tree.ErrKeyNotFound - } - if flags&node.FlagPointer != 0 { - if source == rootDomainEntrySourcePublished { - oldLen := len(dst) - out, ok, err := rootDomainPublishedGetAppend(snap, key, dst) - if ok { - if err != nil { - return dst, err - } - recordSnapshotRootDomainRead(source, true, len(out)-oldLen) - return out, nil - } - } - if s.db == nil { - return dst, errors.New("caching snapshot: value-log reader unavailable") - } - oldLen := len(dst) - out, err := s.db.readValueLogAppend(key, ptr, dst) - if err != nil { - return dst, err - } - recordSnapshotRootDomainRead(source, true, len(out)-oldLen) + return s.appendRootDomainEntryValue(key, dst, val, ptr, flags, rootDomainEntrySourceCached, len(dst)) + } + + oldLen := len(dst) + out, ok, err := rootDomainPublishedGetAppend(snap, key, dst) + if ok { + if err == nil { + recordSnapshotRootDomainRead(rootDomainEntrySourcePublished, true, len(out)-oldLen) return out, nil } - if val == nil { - recordSnapshotRootDomainRead(source, false, 0) - return dst, nil + if !errors.Is(err, tree.ErrKeyNotFound) { + return dst, err } - recordSnapshotRootDomainRead(source, false, len(val)) - return append(dst, val...), nil + if s.publishedLookupBackedByBackendSnapshot(snap) { + return dst, tree.ErrKeyNotFound + } + } + val, ptr, flags, found, source := snap.getPublishedEntryWithSource(key) + if found { + return s.appendRootDomainEntryValue(key, dst, val, ptr, flags, source, oldLen) } if s == nil || s.backend == nil || s.db == nil { @@ -418,8 +459,7 @@ func (s *Snapshot) GetAppend(key, dst []byte) ([]byte, error) { if err := s.db.flushValueLogForBackendRead(); err != nil { return dst, err } - oldLen := len(dst) - out, err := s.backend.GetAppend(key, dst) + out, err = s.backend.GetAppend(key, dst) if err != nil { return dst, err } @@ -431,7 +471,10 @@ func (s *Snapshot) GetAppend(key, dst []byte) ([]byte, error) { } func (s *Snapshot) Get(key []byte) ([]byte, error) { - snap, val, ptr, flags, found, source := s.lookupRootDomainSnapshotEntry(key) + snap, val, ptr, flags, found, source, err := s.lookupRootDomainSnapshotEntryWithError(key) + if err != nil { + return nil, err + } if found { if flags&node.FlagTombstone != 0 { return nil, tree.ErrKeyNotFound @@ -480,6 +523,9 @@ func (s *Snapshot) Get(key []byte) ([]byte, error) { copy(owned, val) return owned, nil } + if s.publishedLookupBackedByBackendSnapshot(snap) { + return nil, tree.ErrKeyNotFound + } if s == nil || s.backend == nil || s.db == nil { return nil, tree.ErrKeyNotFound @@ -501,7 +547,10 @@ func (s *Snapshot) Get(key []byte) ([]byte, error) { } func (s *Snapshot) GetUnsafe(key []byte) ([]byte, error) { - snap, val, ptr, flags, found, source := s.lookupRootDomainSnapshotEntry(key) + snap, val, ptr, flags, found, source, err := s.lookupRootDomainSnapshotEntryWithError(key) + if err != nil { + return nil, err + } if found { if flags&node.FlagTombstone != 0 { return nil, tree.ErrKeyNotFound @@ -523,6 +572,9 @@ func (s *Snapshot) GetUnsafe(key []byte) ([]byte, error) { } return val, nil } + if s.publishedLookupBackedByBackendSnapshot(snap) { + return nil, tree.ErrKeyNotFound + } if s == nil || s.backend == nil || s.db == nil { return nil, tree.ErrKeyNotFound @@ -534,15 +586,28 @@ func (s *Snapshot) GetUnsafe(key []byte) ([]byte, error) { } func (s *Snapshot) Has(key []byte) (bool, error) { - _, _, flags, found := s.lookupCachedRootDomainEntry(key) - if found { - return flags&node.FlagTombstone == 0, nil + if s == nil { + return false, nil + } + cachedSnap := s.cachedRootDomainSnapshot(key) + if s.publishedRoots == nil && !rootDomainSnapshotHasPublishedState(cachedSnap) { + _, _, flags, found := s.lookupCachedRootDomainEntry(key) + if found { + return flags&node.FlagTombstone == 0, nil + } + if s.backend == nil { + return false, nil + } + return s.backend.Has(key) + } + snap, _, _, flags, found, _, err := s.lookupRootDomainSnapshotEntryWithError(key) + if err != nil { + return false, err } - _, _, flags, found = s.lookupQueueEntry(key) if found { return flags&node.FlagTombstone == 0, nil } - if s == nil || s.backend == nil { + if s.publishedLookupBackedByBackendSnapshot(snap) || s.backend == nil { return false, nil } return s.backend.Has(key) @@ -722,7 +787,10 @@ func (s *Snapshot) HasPrefixes(prefixes [][]byte) ([]bool, error) { } func (s *Snapshot) GetEntry(key []byte) (node.LeafEntry, error) { - val, ptr, flags, found := s.lookupQueueEntry(key) + snap, val, ptr, flags, found, _, err := s.lookupRootDomainSnapshotEntryWithError(key) + if err != nil { + return node.LeafEntry{}, err + } if found { keyCopy := append([]byte(nil), key...) return node.LeafEntry{ @@ -732,6 +800,9 @@ func (s *Snapshot) GetEntry(key []byte) (node.LeafEntry, error) { Flags: flags, }, nil } + if s.publishedLookupBackedByBackendSnapshot(snap) { + return node.LeafEntry{}, tree.ErrKeyNotFound + } if s == nil || s.backend == nil || s.db == nil { return node.LeafEntry{}, tree.ErrKeyNotFound @@ -743,7 +814,10 @@ func (s *Snapshot) GetEntry(key []byte) (node.LeafEntry, error) { } func (s *Snapshot) GetEntryExact(key []byte) (node.LeafEntry, error) { - val, ptr, flags, found := s.lookupQueueEntry(key) + snap, val, ptr, flags, found, _, err := s.lookupRootDomainSnapshotEntryWithError(key) + if err != nil { + return node.LeafEntry{}, err + } if found { keyCopy := append([]byte(nil), key...) return node.LeafEntry{ @@ -753,6 +827,9 @@ func (s *Snapshot) GetEntryExact(key []byte) (node.LeafEntry, error) { Flags: flags, }, nil } + if s.publishedLookupBackedByBackendSnapshot(snap) { + return node.LeafEntry{}, tree.ErrKeyNotFound + } if s == nil || s.backend == nil || s.db == nil { return node.LeafEntry{}, tree.ErrKeyNotFound diff --git a/TreeDB/caching/snapshot_getappend_test.go b/TreeDB/caching/snapshot_getappend_test.go new file mode 100644 index 0000000000..3ad7b6e2bb --- /dev/null +++ b/TreeDB/caching/snapshot_getappend_test.go @@ -0,0 +1,403 @@ +package caching + +import ( + "errors" + "testing" + + backenddb "github.com/snissn/gomap/TreeDB/db" + "github.com/snissn/gomap/TreeDB/node" + "github.com/snissn/gomap/TreeDB/page" + "github.com/snissn/gomap/TreeDB/tree" +) + +type snapshotPublishedValueLookup struct { + value []byte + + getEntryCalls int + getValueAppendCalls int + getValueUnsafeCalls int +} + +func snapshotGetAppendTestKey(key []byte) bool { + return len(key) == 1 && key[0] == 'k' +} + +func (l *snapshotPublishedValueLookup) GetEntry(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool) { + l.getEntryCalls++ + if !snapshotGetAppendTestKey(key) { + return nil, page.ValuePtr{}, 0, false + } + return l.value, page.ValuePtr{}, node.FlagInline, true +} + +func (l *snapshotPublishedValueLookup) GetValueAppend(key, dst []byte) ([]byte, error) { + l.getValueAppendCalls++ + if !snapshotGetAppendTestKey(key) { + return dst, tree.ErrKeyNotFound + } + return append(dst, l.value...), nil +} + +func (l *snapshotPublishedValueLookup) GetValueUnsafe(key []byte) ([]byte, error) { + l.getValueUnsafeCalls++ + if !snapshotGetAppendTestKey(key) { + return nil, tree.ErrKeyNotFound + } + return l.value, nil +} + +type snapshotPublishedEntryOnlyLookup struct { + value []byte + flags byte + getEntryCalls int +} + +func (l *snapshotPublishedEntryOnlyLookup) GetEntry(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool) { + l.getEntryCalls++ + if !snapshotGetAppendTestKey(key) { + return nil, page.ValuePtr{}, 0, false + } + flags = l.flags + if flags == 0 { + flags = node.FlagInline + } + return l.value, page.ValuePtr{}, flags, true +} + +type snapshotPublishedAppendMissLookup struct { + value []byte + flags byte + + getEntryCalls int + getValueAppendCalls int + getValueUnsafeCalls int +} + +func (l *snapshotPublishedAppendMissLookup) GetEntry(key []byte) (val []byte, ptr page.ValuePtr, flags byte, found bool) { + l.getEntryCalls++ + if !snapshotGetAppendTestKey(key) { + return nil, page.ValuePtr{}, 0, false + } + flags = l.flags + if flags == 0 { + flags = node.FlagInline + } + return l.value, page.ValuePtr{}, flags, true +} + +func (l *snapshotPublishedAppendMissLookup) GetValueAppend(_ []byte, dst []byte) ([]byte, error) { + l.getValueAppendCalls++ + return dst, tree.ErrKeyNotFound +} + +func (l *snapshotPublishedAppendMissLookup) GetValueUnsafe(_ []byte) ([]byte, error) { + l.getValueUnsafeCalls++ + return nil, tree.ErrKeyNotFound +} + +func TestSnapshotGetAppendPublishedUsesValueAppendDirectly(t *testing.T) { + lookup := &snapshotPublishedValueLookup{value: []byte("published")} + snap := &Snapshot{ + rootPointShards: []rootDomainSnapshot{{ + published: lookup, + publishedRootID: 1, + }}, + } + + got, err := snap.GetAppend([]byte("k"), []byte("p:")) + if err != nil { + t.Fatalf("GetAppend: %v", err) + } + if string(got) != "p:published" { + t.Fatalf("value=%q, want p:published", got) + } + if lookup.getValueAppendCalls != 1 { + t.Fatalf("GetValueAppend calls=%d, want 1", lookup.getValueAppendCalls) + } + if lookup.getEntryCalls != 0 { + t.Fatalf("GetEntry calls=%d, want 0", lookup.getEntryCalls) + } + if lookup.getValueUnsafeCalls != 0 { + t.Fatalf("GetValueUnsafe calls=%d, want 0", lookup.getValueUnsafeCalls) + } +} + +func TestSnapshotGetAppendPublishedValueAppendAllocs(t *testing.T) { + if testRaceEnabled { + t.Skip("AllocsPerRun is not stable under -race") + } + lookup := &snapshotPublishedValueLookup{value: []byte("published")} + snap := &Snapshot{ + rootPointShards: []rootDomainSnapshot{{ + published: lookup, + publishedRootID: 1, + }}, + } + key := []byte("k") + prefix := []byte("p:") + wantLen := len(prefix) + len(lookup.value) + buf := make([]byte, len(prefix), wantLen) + copy(buf, prefix) + + allocs := testing.AllocsPerRun(1000, func() { + dst := buf[:len(prefix)] + got, err := snap.GetAppend(key, dst) + if err != nil { + t.Fatalf("GetAppend: %v", err) + } + if len(got) != wantLen || got[0] != 'p' || got[1] != ':' || got[2] != 'p' { + t.Fatalf("unexpected GetAppend value %q", got) + } + }) + if allocs != 0 { + t.Fatalf("GetAppend allocs/run=%f, want 0", allocs) + } + if lookup.getEntryCalls != 0 { + t.Fatalf("GetEntry calls=%d, want 0", lookup.getEntryCalls) + } + if lookup.getValueUnsafeCalls != 0 { + t.Fatalf("GetValueUnsafe calls=%d, want 0", lookup.getValueUnsafeCalls) + } +} + +func TestSnapshotGetAppendPublishedFallsBackToEntryLookup(t *testing.T) { + lookup := &snapshotPublishedEntryOnlyLookup{value: []byte("published")} + snap := &Snapshot{ + rootPointShards: []rootDomainSnapshot{{ + published: lookup, + publishedRootID: 1, + }}, + } + + got, err := snap.GetAppend([]byte("k"), []byte("p:")) + if err != nil { + t.Fatalf("GetAppend: %v", err) + } + if string(got) != "p:published" { + t.Fatalf("value=%q, want p:published", got) + } + if lookup.getEntryCalls != 1 { + t.Fatalf("GetEntry calls=%d, want 1", lookup.getEntryCalls) + } +} + +func TestSnapshotGetAppendPublishedAppendMissFallsBackToEntryLookup(t *testing.T) { + lookup := &snapshotPublishedAppendMissLookup{value: []byte("published")} + snap := &Snapshot{ + rootPointShards: []rootDomainSnapshot{{ + published: lookup, + publishedRootID: 1, + }}, + } + + got, err := snap.GetAppend([]byte("k"), []byte("p:")) + if err != nil { + t.Fatalf("GetAppend: %v", err) + } + if string(got) != "p:published" { + t.Fatalf("value=%q, want p:published", got) + } + if lookup.getValueAppendCalls != 1 { + t.Fatalf("GetValueAppend calls=%d, want 1", lookup.getValueAppendCalls) + } + if lookup.getEntryCalls != 1 { + t.Fatalf("GetEntry calls=%d, want 1", lookup.getEntryCalls) + } + if lookup.getValueUnsafeCalls != 0 { + t.Fatalf("GetValueUnsafe calls=%d, want 0", lookup.getValueUnsafeCalls) + } +} + +func TestSnapshotGetAppendPublishedAppendMissPreservesTombstone(t *testing.T) { + lookup := &snapshotPublishedAppendMissLookup{flags: node.FlagTombstone} + snap := &Snapshot{ + rootPointShards: []rootDomainSnapshot{{ + published: lookup, + publishedRootID: 1, + }}, + } + + got, err := snap.GetAppend([]byte("k"), []byte("p:")) + if !errors.Is(err, tree.ErrKeyNotFound) { + t.Fatalf("GetAppend err=%v, want ErrKeyNotFound", err) + } + if string(got) != "p:" { + t.Fatalf("value=%q, want unchanged prefix", got) + } + if lookup.getValueAppendCalls != 1 { + t.Fatalf("GetValueAppend calls=%d, want 1", lookup.getValueAppendCalls) + } + if lookup.getEntryCalls != 1 { + t.Fatalf("GetEntry calls=%d, want 1", lookup.getEntryCalls) + } + if lookup.getValueUnsafeCalls != 0 { + t.Fatalf("GetValueUnsafe calls=%d, want 0", lookup.getValueUnsafeCalls) + } +} + +func TestSnapshotGetAppendBackendPublishedMissDoesNotFallBackToDefaultRoot(t *testing.T) { + snap := newSnapshotWithBackendPublishedPointRootMissingKey(t) + + got, err := snap.GetAppend([]byte("k"), []byte("p:")) + if !errors.Is(err, tree.ErrKeyNotFound) { + t.Fatalf("GetAppend err=%v, want ErrKeyNotFound", err) + } + if string(got) != "p:" { + t.Fatalf("value=%q, want unchanged prefix", got) + } +} + +func TestSnapshotBackendPublishedMissConsistentAcrossReadAPIs(t *testing.T) { + snap := newSnapshotWithBackendPublishedPointRootMissingKey(t) + + if _, err := snap.Get([]byte("k")); !errors.Is(err, tree.ErrKeyNotFound) { + t.Fatalf("Get err=%v, want ErrKeyNotFound", err) + } + if _, err := snap.GetUnsafe([]byte("k")); !errors.Is(err, tree.ErrKeyNotFound) { + t.Fatalf("GetUnsafe err=%v, want ErrKeyNotFound", err) + } + ok, err := snap.Has([]byte("k")) + if err != nil { + t.Fatalf("Has: %v", err) + } + if ok { + t.Fatal("Has=true, want false") + } + if _, err := snap.GetEntry([]byte("k")); !errors.Is(err, tree.ErrKeyNotFound) { + t.Fatalf("GetEntry err=%v, want ErrKeyNotFound", err) + } + if _, err := snap.GetEntryExact([]byte("k")); !errors.Is(err, tree.ErrKeyNotFound) { + t.Fatalf("GetEntryExact err=%v, want ErrKeyNotFound", err) + } +} + +func TestSnapshotBackendPublishedReadErrorsPropagate(t *testing.T) { + snap := &Snapshot{ + rootPointShards: []rootDomainSnapshot{{ + published: backendSnapshotLookup{}, + publishedRootID: 1, + }}, + } + if _, err := snap.Get([]byte("k")); !errors.Is(err, backenddb.ErrClosed) { + t.Fatalf("Get err=%v, want ErrClosed", err) + } + if _, err := snap.GetUnsafe([]byte("k")); !errors.Is(err, backenddb.ErrClosed) { + t.Fatalf("GetUnsafe err=%v, want ErrClosed", err) + } + if _, err := snap.GetAppend([]byte("k"), nil); !errors.Is(err, backenddb.ErrClosed) { + t.Fatalf("GetAppend err=%v, want ErrClosed", err) + } + if _, err := snap.Has([]byte("k")); !errors.Is(err, backenddb.ErrClosed) { + t.Fatalf("Has err=%v, want ErrClosed", err) + } + if _, err := snap.GetEntry([]byte("k")); !errors.Is(err, backenddb.ErrClosed) { + t.Fatalf("GetEntry err=%v, want ErrClosed", err) + } + if _, err := snap.GetEntryExact([]byte("k")); !errors.Is(err, backenddb.ErrClosed) { + t.Fatalf("GetEntryExact err=%v, want ErrClosed", err) + } +} + +func TestSnapshotGetAppendBackendPublishedHitViaInstalledLookup(t *testing.T) { + dir := t.TempDir() + backend, err := backenddb.Open(backenddb.Options{Dir: dir}) + if err != nil { + t.Fatalf("open backend: %v", err) + } + t.Cleanup(func() { _ = backend.Close() }) + + if err := backend.SetSync([]byte("k"), []byte("default-root")); err != nil { + t.Fatalf("backend set: %v", err) + } + + // Publish a backend root that contains the key we want to read. + pubTable := newRootDomainTestTable(t, rootDomainTestOp{key: "k", value: "from-published-root"}) + pointRootID, err := backend.PublishOrderedRootIterator(0, pubTable.NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish root: %v", err) + } + if pointRootID == backend.State().RootPageID { + t.Fatalf("test point root unexpectedly matches default root %d", pointRootID) + } + + db := &DB{ + backend: backend, + mutableShards: make([]memShard, 1), + mutableShardMask: 0, + } + + // Install a published root set with only a rootID (no lookup object). + // AcquireSnapshot must wire the backend lookup via installBackendPublishedRootLookups. + db.mu.Lock() + db.installPublishedRootSetLocked(&publishedRootSet{ + generation: 1, + pointShards: []publishedRootRef{{rootID: pointRootID}}, + }) + db.mu.Unlock() + + snap := db.AcquireSnapshot() + if snap == nil { + t.Fatal("expected snapshot") + } + t.Cleanup(func() { _ = snap.Close() }) + + // Verify the backend lookup was wired by installBackendPublishedRootLookups. + if len(snap.backendPublishedLookups) == 0 { + t.Fatal("expected backendPublishedLookups to be populated") + } + + // GetAppend must read through the wired backend lookup without falling through + // to the default-root backend fallback. + got, err := snap.GetAppend([]byte("k"), []byte("prefix:")) + if err != nil { + t.Fatalf("GetAppend: %v", err) + } + if string(got) != "prefix:from-published-root" { + t.Fatalf("GetAppend value=%q, want prefix:from-published-root", got) + } + entry, err := snap.GetEntry([]byte("k")) + if err != nil { + t.Fatalf("GetEntry: %v", err) + } + if string(entry.Value) != "from-published-root" { + t.Fatalf("GetEntry value=%q, want from-published-root", entry.Value) + } +} + +func newSnapshotWithBackendPublishedPointRootMissingKey(t *testing.T) *Snapshot { + t.Helper() + + dir := t.TempDir() + backend, err := backenddb.Open(backenddb.Options{Dir: dir}) + if err != nil { + t.Fatalf("open backend: %v", err) + } + t.Cleanup(func() { _ = backend.Close() }) + + if err := backend.SetSync([]byte("k"), []byte("default")); err != nil { + t.Fatalf("backend set: %v", err) + } + otherRoot := newRootDomainTestTable(t, rootDomainTestOp{key: "other", value: "published"}) + pointRootID, err := backend.PublishOrderedRootIterator(0, otherRoot.NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish point root: %v", err) + } + if pointRootID == backend.State().RootPageID { + t.Fatalf("test point root unexpectedly matches default root %d", pointRootID) + } + + backendSnap := backend.AcquireSnapshot() + if backendSnap == nil { + t.Fatal("expected backend snapshot") + } + t.Cleanup(func() { _ = backendSnap.Close() }) + + db := &DB{backend: backend, mutableShards: make([]memShard, 1)} + return &Snapshot{ + db: db, + backend: backendSnap, + rootPointShards: []rootDomainSnapshot{{publishedRootID: pointRootID}}, + backendRoot: backendSnapshotLookup{db: db, snapshot: backendSnap, rootID: backendSnap.State().RootPageID}, + backendRootOK: true, + } +} diff --git a/TreeDB/collections/api.go b/TreeDB/collections/api.go index f9c0d34711..cf73a125d3 100644 --- a/TreeDB/collections/api.go +++ b/TreeDB/collections/api.go @@ -15,6 +15,7 @@ import ( "sync/atomic" "time" "unicode/utf8" + "unsafe" "github.com/cespare/xxhash/v2" "github.com/snissn/gomap/TreeDB/batch" @@ -24,6 +25,7 @@ import ( "github.com/snissn/gomap/TreeDB/node" "github.com/snissn/gomap/TreeDB/page" "github.com/snissn/gomap/TreeDB/tree" + "github.com/snissn/gomap/TreeDB/zipper" "go.mongodb.org/mongo-driver/v2/bson" ) @@ -54,6 +56,12 @@ const ( // well-amortized InsertBatch calls on the immediate publish path. Smaller // batches use the indexed write-domain memtable path by default. DefaultIndexedWriteMemtableDirectBatchDocuments = 16000 + // DefaultIndexedWriteMemtableLowFanoutDirectBatchDocuments keeps moderately + // large low-index InsertBatch calls on the immediate publish path. With only + // one or two secondary indexes, root apply is already cheap enough that the + // indexed write-domain pending-visibility indexes can cost more than they + // save. + DefaultIndexedWriteMemtableLowFanoutDirectBatchDocuments = 4096 // DefaultIndexedWriteMemtableAsyncFlushMaxQueuedUnits bounds opt-in // background indexed flush work. When the queue reaches this many immutable // flush units, the triggering writer publishes synchronously to cap memory @@ -320,15 +328,23 @@ type CollectionUpdateStats struct { Indexes int Runs int BufferedBatches int + BatchValidate time.Duration + BatchClone time.Duration + PlanSetup time.Duration + BufferedReadSnapshot time.Duration CurrentRead time.Duration + BSONIDValidation time.Duration Callback time.Duration + ReplacementStage time.Duration PrepareDocuments time.Duration IndexStateExtraction time.Duration + IndexStateCompare time.Duration UniqueIndexPreflight time.Duration TemplateRunBuild time.Duration PrimaryRunBuild time.Duration IndexStateRunBuild time.Duration SecondaryRunBuild time.Duration + SemanticRecordBuild time.Duration BufferStage time.Duration // Buffer-stage subphase timings are populated only when // CollectionManager.SetUpdateBatchDetailedStatsEnabled(true) is enabled. @@ -340,19 +356,21 @@ type CollectionUpdateStats struct { // relock contention after an async flush wait. It does not include async // flush completion waits performed after releasing the mutex for // backpressure. - BufferStageLockWait time.Duration - BufferStageLockHold time.Duration - BufferStageValidation time.Duration - BufferStageRootScan time.Duration - BufferStageDomainPrepare time.Duration - BufferStagePrimaryIdx time.Duration - BufferStageUniqueIdx time.Duration - BufferStageRootAppend time.Duration + BufferStageLockWait time.Duration + BufferStageLockHold time.Duration + BufferStageValidation time.Duration + BufferStageRootScan time.Duration + BufferStageDomainPrepare time.Duration + BufferStagePrimaryIdx time.Duration + BufferStageUniqueIdx time.Duration + BufferStageRootAppend time.Duration + BufferStageSemanticAppend time.Duration // BufferStageFlush measures local threshold-flush schedule/publish work // performed while staging an indexed buffered update batch. It excludes // waits for an already-running async flush that leave no local schedule or // publish work for the current batch. BufferStageFlush time.Duration + PlanClose time.Duration Publish time.Duration SecondaryDeleteEntries int SecondarySetEntries int @@ -403,94 +421,152 @@ type CollectionUpdateIndexStats struct { // CollectionManager. The counters are process-local observability; they are // not persisted with collection metadata. type CollectionManagerStats struct { - Domains int - PendingDocuments int - PendingBytes int64 - PendingRootRuns int - PendingIndexedFlushUnits int - OverlayMutableDocuments int - OverlayQueuedIndexedFlushUnits int - OverlayActiveIndexedFlushUnits int - OverlayVisibleDepth int - IndexedAsyncFlushRunning int - MutationLockCalls uint64 - MutationLockWait time.Duration - MutationLockHold time.Duration - IndexedStageBatches uint64 - IndexedStageDocs uint64 - IndexedStageBytes uint64 - IndexedStageRootRuns uint64 - IndexedAutoFlushes uint64 - IndexedAsyncFlushScheduled uint64 - IndexedAsyncFlushBackpressure uint64 - IndexedAsyncFlushWait time.Duration - IndexedAsyncFlushErrors uint64 - IndexedFlushCalls uint64 - IndexedFlushErrors uint64 - IndexedFlushForcedDrains uint64 - IndexedFlushUnits uint64 - IndexedFlushDocs uint64 - IndexedFlushBytes uint64 - IndexedFlushRootRuns uint64 - IndexedFlushRoots uint64 - IndexedFlushDuration time.Duration - IndexedFlushMaterialize time.Duration - IndexedFlushPublish time.Duration - RootDeltaPlanPrimaryRoots uint64 - RootDeltaPlanTemplateRoots uint64 - RootDeltaPlanIndexStateRoots uint64 - RootDeltaPlanSecondaryRoots uint64 - RootDeltaPlanEntries uint64 - RootDeltaPlanKeyBytes uint64 - RootDeltaPlanValueBytes uint64 - RootDeltaPlanTombstones uint64 - PrimaryOnlyUpdateCalls uint64 - PrimaryOnlyMatched uint64 - PrimaryOnlyModified uint64 - PrimaryOnlyBufferedCalls uint64 - PrimaryOnlyRootPublishes uint64 - PrimaryOnlyRootDeltaEntries uint64 - PrimaryOnlyRootDeltaKeyBytes uint64 - PrimaryOnlyRootDeltaValueBytes uint64 - PrimaryOnlyCoalescedDocs uint64 - UpdateCombineRequests uint64 - UpdateCombineBatches uint64 - UpdateCombineBatchedRequests uint64 - UpdateCombineFallbackRequests uint64 - UpdateCombineQueueDepthMax uint64 - UpdateBatchCalls uint64 - UpdateBatchItems uint64 - UpdateBatchMatched uint64 - UpdateBatchModified uint64 - UpdateBatchRuns uint64 - UpdateBatchBufferedBatches uint64 - UpdateBatchCurrentRead time.Duration - UpdateBatchCallback time.Duration - UpdateBatchPrepareDocuments time.Duration - UpdateBatchIndexStateExtract time.Duration - UpdateBatchUniquePreflight time.Duration - UpdateBatchTemplateRunBuild time.Duration - UpdateBatchPrimaryRunBuild time.Duration - UpdateBatchIndexStateRunBuild time.Duration - UpdateBatchSecondaryRunBuild time.Duration - UpdateBatchBufferStage time.Duration + Domains int + PendingDocuments int + PendingBytes int64 + PendingRootRuns int + PendingIndexedFlushUnits int + PendingIndexedSemanticRecords int + OverlayMutableDocuments int + OverlayQueuedIndexedFlushUnits int + OverlayActiveIndexedFlushUnits int + OverlayVisibleDepth int + IndexedAsyncFlushRunning int + MutationLockCalls uint64 + MutationLockWait time.Duration + MutationLockHold time.Duration + IndexedStageBatches uint64 + IndexedStageDocs uint64 + IndexedStageBytes uint64 + IndexedStageRootRuns uint64 + IndexedSemanticRawRecords uint64 + IndexedSemanticRawIndexDeltas uint64 + IndexedSemanticFallbackRecords uint64 + IndexedSemanticEffectiveRecords uint64 + IndexedSemanticSkippedSecondaryRoots uint64 + IndexedAutoFlushes uint64 + IndexedAsyncFlushScheduled uint64 + IndexedAsyncFlushBackpressure uint64 + IndexedAsyncFlushWait time.Duration + IndexedAsyncFlushErrors uint64 + IndexedFlushCalls uint64 + IndexedFlushErrors uint64 + IndexedFlushForcedDrains uint64 + IndexedFlushUnits uint64 + IndexedFlushDocs uint64 + IndexedFlushBytes uint64 + IndexedFlushRootRuns uint64 + IndexedFlushRoots uint64 + IndexedFlushPreflight time.Duration + IndexedFlushRotate time.Duration + IndexedFlushMerge time.Duration + IndexedFlushDuration time.Duration + IndexedFlushMaterialize time.Duration + IndexedFlushSemanticPlan time.Duration + IndexedFlushBuildInputs time.Duration + IndexedFlushPlanStats time.Duration + IndexedFlushPublish time.Duration + CoalescedFlushBatches uint64 + CoalescedFlushBatchUnits uint64 + CoalescedFlushBatchDocs uint64 + CoalescedFlushBatchBytes uint64 + CoalescedFlushNetZeroBatches uint64 + RootDeltaPlanPrimaryRoots uint64 + RootDeltaPlanTemplateRoots uint64 + RootDeltaPlanIndexStateRoots uint64 + RootDeltaPlanSecondaryRoots uint64 + RootDeltaPlanEntries uint64 + RootDeltaPlanKeyBytes uint64 + RootDeltaPlanValueBytes uint64 + RootDeltaPlanTombstones uint64 + RootDeltaPlanRawUnitPrimaryEntries uint64 + RootDeltaPlanRawUnitPrimaryBytes uint64 + RootDeltaPlanRawUnitPrimaryTombstones uint64 + RootDeltaPlanRawUnitTemplateEntries uint64 + RootDeltaPlanRawUnitTemplateBytes uint64 + RootDeltaPlanRawUnitTemplateTombstones uint64 + RootDeltaPlanRawUnitIndexStateEntries uint64 + RootDeltaPlanRawUnitIndexStateBytes uint64 + RootDeltaPlanRawUnitIndexStateTombstones uint64 + RootDeltaPlanRawUnitSecondaryEntries uint64 + RootDeltaPlanRawUnitSecondaryBytes uint64 + RootDeltaPlanRawUnitSecondaryTombstones uint64 + RootDeltaPlanFinalPrimaryEntries uint64 + RootDeltaPlanFinalPrimaryBytes uint64 + RootDeltaPlanFinalPrimaryTombstones uint64 + RootDeltaPlanFinalTemplateEntries uint64 + RootDeltaPlanFinalTemplateBytes uint64 + RootDeltaPlanFinalTemplateTombstones uint64 + RootDeltaPlanFinalIndexStateEntries uint64 + RootDeltaPlanFinalIndexStateBytes uint64 + RootDeltaPlanFinalIndexStateTombstones uint64 + RootDeltaPlanFinalSecondaryEntries uint64 + RootDeltaPlanFinalSecondaryBytes uint64 + RootDeltaPlanFinalSecondaryTombstones uint64 + RootDeltaPlanSquashedEntries uint64 + RootDeltaPlanNetZeroPlans uint64 + PrimaryOnlyUpdateCalls uint64 + PrimaryOnlyMatched uint64 + PrimaryOnlyModified uint64 + PrimaryOnlyBufferedCalls uint64 + PrimaryOnlyRootPublishes uint64 + PrimaryOnlyRootDeltaEntries uint64 + PrimaryOnlyRootDeltaKeyBytes uint64 + PrimaryOnlyRootDeltaValueBytes uint64 + PrimaryOnlyCoalescedDocs uint64 + PrimaryOnlyDuplicateIDsCoalesced uint64 + PrimaryOnlyDrainCalls uint64 + PrimaryOnlyDrainDocs uint64 + PrimaryOnlyDrainBytes uint64 + PrimaryOnlyDrainDuration time.Duration + UpdateCombineRequests uint64 + UpdateCombineBatches uint64 + UpdateCombineBatchedRequests uint64 + UpdateCombineFallbackRequests uint64 + UpdateCombineQueueDepthMax uint64 + UpdateBatchCalls uint64 + UpdateBatchItems uint64 + UpdateBatchMatched uint64 + UpdateBatchModified uint64 + UpdateBatchRuns uint64 + UpdateBatchBufferedBatches uint64 + UpdateBatchValidate time.Duration + UpdateBatchClone time.Duration + UpdateBatchPlanSetup time.Duration + UpdateBatchBufferedReadSnapshot time.Duration + UpdateBatchCurrentRead time.Duration + UpdateBatchBSONIDValidation time.Duration + UpdateBatchCallback time.Duration + UpdateBatchReplacementStage time.Duration + UpdateBatchPrepareDocuments time.Duration + UpdateBatchIndexStateExtract time.Duration + UpdateBatchIndexStateCompare time.Duration + UpdateBatchUniquePreflight time.Duration + UpdateBatchTemplateRunBuild time.Duration + UpdateBatchPrimaryRunBuild time.Duration + UpdateBatchIndexStateRunBuild time.Duration + UpdateBatchSecondaryRunBuild time.Duration + UpdateBatchSemanticRecordBuild time.Duration + UpdateBatchBufferStage time.Duration // Detailed buffer-stage aggregate timings are populated only when // CollectionManager.SetUpdateBatchDetailedStatsEnabled(true) is enabled. // UpdateBatchBufferLockHold is an enclosing domain mutex hold-time metric // and overlaps the validation/root/index/root-append subphases; it is not // additive with those child counters. - UpdateBatchBufferPrecheck time.Duration - UpdateBatchBufferLockWait time.Duration - UpdateBatchBufferLockHold time.Duration - UpdateBatchBufferValidation time.Duration - UpdateBatchBufferRootScan time.Duration - UpdateBatchBufferDomainPrepare time.Duration - UpdateBatchBufferPrimaryIdx time.Duration - UpdateBatchBufferUniqueIdx time.Duration - UpdateBatchBufferRootAppend time.Duration + UpdateBatchBufferPrecheck time.Duration + UpdateBatchBufferLockWait time.Duration + UpdateBatchBufferLockHold time.Duration + UpdateBatchBufferValidation time.Duration + UpdateBatchBufferRootScan time.Duration + UpdateBatchBufferDomainPrepare time.Duration + UpdateBatchBufferPrimaryIdx time.Duration + UpdateBatchBufferUniqueIdx time.Duration + UpdateBatchBufferRootAppend time.Duration + UpdateBatchBufferSemanticAppend time.Duration // UpdateBatchBufferFlush measures only threshold-flush work that was // actually scheduled/executed while staging indexed buffered update batches. UpdateBatchBufferFlush time.Duration + UpdateBatchPlanClose time.Duration UpdateBatchPublish time.Duration UpdateBatchSecondaryDeletes uint64 UpdateBatchSecondarySets uint64 @@ -581,6 +657,14 @@ type CollectionOptions struct { // Maintenance can later compact overlay roots into base roots; reads merge // overlay roots over base roots while they are pending. BufferedIndexedOverlayRoots bool `json:"buffered_indexed_overlay_roots,omitempty"` + // BufferedIndexedReadOnlyPrepare asks indexed write-domain flush publish to + // run the DB read-only leaf-span preparation pass. This is + // observability/planning only and does not change publish output. + BufferedIndexedReadOnlyPrepare bool `json:"buffered_indexed_read_only_prepare,omitempty"` + // BufferedIndexedReadOnlyPrepareWorkerCount records deterministic leaf-span + // worker-range summaries for this target worker count when read-only prepare + // is enabled. Zero disables worker-range summaries. + BufferedIndexedReadOnlyPrepareWorkerCount int `json:"buffered_indexed_read_only_prepare_worker_count,omitempty"` // BufferedIndexedAsyncFlushMaxQueuedUnits bounds immutable indexed flush // units queued for the background publisher. Zero uses the native default // when async flush is enabled on an indexed schema. @@ -649,34 +733,92 @@ type noIndexBatchEntry struct { document []byte } +type indexedSemanticRecordKind uint8 + +const ( + indexedSemanticRecordUnknown indexedSemanticRecordKind = iota + indexedSemanticRecordUpdate +) + +type indexedSemanticFallbackReason uint8 + +const ( + indexedSemanticFallbackNone indexedSemanticFallbackReason = iota + indexedSemanticFallbackRawOnly +) + +type indexedSemanticRecord struct { + kind indexedSemanticRecordKind + documentID []byte + indexDeltas []indexedSemanticIndexDelta + fallback indexedSemanticFallbackReason +} + +type indexedSemanticIndexDelta struct { + indexName string + rootName string + runtimeIdx int + unique bool + oldValues [][]byte + newValues [][]byte +} + type indexedFlushUnit struct { rootRuns map[string][]memtable.Table rootPolicies map[string]backenddb.OrderedRootStoragePolicy rootBaseIDs map[string]uint64 uniqueValueRuns map[string][]memtable.Table + semanticRecords []indexedSemanticRecord arenaRefs [][]byte docCount int byteCount int64 rootRunCount int + rootDeltaStats collectionRootDeltaPlanStats } -type indexedFlushPublishWork struct { - pin *backenddb.Snapshot - meta CollectionMeta - catalog *collectionCatalog - baseSystemRoot uint64 - baseCommitSeq uint64 - units []indexedFlushUnit - flushUnit indexedFlushUnit +type coalescedFlushBatchState uint8 + +const ( + coalescedFlushBatchQueued coalescedFlushBatchState = iota + coalescedFlushBatchActive + coalescedFlushBatchMaterializing + coalescedFlushBatchPublishing + coalescedFlushBatchPublished + coalescedFlushBatchRequeued + coalescedFlushBatchLostOwnership +) + +type coalescedFlushBatch struct { + state coalescedFlushBatchState + + // Original immutable units, in FIFO order. The merged unit is only the + // mechanical publish view for the current DB ordered-root API. + units []indexedFlushUnit + mergedUnit indexedFlushUnit + semanticRecords []indexedSemanticRecord + rootNames []string rootBaseIDs map[string]uint64 rootOverlays map[string][]uint64 rootOverlayFilters map[string]collectionRootOverlayFilter - docCount int - byteCount int64 - rootRunCount int - rootCount int - rootDeltaStats collectionRootDeltaPlanStats + + docCount int + byteCount int64 + rootRunCount int + rootCount int + rootDeltaStats collectionRootDeltaPlanStats + rawRootDeltaStats collectionRootDeltaPlanStats + rawRootDeltaReady bool + effectiveRecords int +} + +type indexedFlushPublishWork struct { + pin *backenddb.Snapshot + meta CollectionMeta + catalog *collectionCatalog + baseSystemRoot uint64 + baseCommitSeq uint64 + batch coalescedFlushBatch } type bufferedIndexedCheckpoint struct { @@ -696,11 +838,13 @@ type bufferedIndexedCheckpoint struct { rootPolicies map[string]backenddb.OrderedRootStoragePolicy rootBaseIDs map[string]uint64 rootValueArenas [][]byte + indexedSemanticRecords []indexedSemanticRecord indexedPublishingUnits []indexedFlushUnit indexedFlushUnits []indexedFlushUnit primaryRunIndexActive bool uniqueValueRuns map[string][]memtable.Table rootRunCount int + rootDeltaStats collectionRootDeltaPlanStats } type bufferedUniqueValueIndex struct { @@ -710,9 +854,8 @@ type bufferedUniqueValueIndex struct { } type bufferedPrimaryRunIndex struct { - values map[uint64]bufferedPrimaryRunRef - collisions map[uint64][]bufferedPrimaryRunRef - arenas [][]byte + values map[string]memtable.Table + arenas [][]byte } type bufferedPrimaryRunRef struct { @@ -751,6 +894,8 @@ type collectionWriteDomain struct { rootPolicies map[string]backenddb.OrderedRootStoragePolicy rootBaseIDs map[string]uint64 rootValueArenas [][]byte + indexedSemanticRecords []indexedSemanticRecord + rootDeltaStats collectionRootDeltaPlanStats primaryIDIndex *bufferedUniqueValueIndex // Built lazily by readers so write-only indexed buffering does not pay for // an auxiliary lookup structure it never uses. @@ -764,99 +909,156 @@ type collectionWriteDomain struct { rootRunCount int writeGeneration uint64 - mutationLockCalls atomic.Uint64 - mutationLockWaitTotalNs atomic.Uint64 - mutationLockHoldTotalNs atomic.Uint64 - indexedStageBatches atomic.Uint64 - indexedStageDocs atomic.Uint64 - indexedStageBytes atomic.Uint64 - indexedStageRootRuns atomic.Uint64 - indexedAutoFlushes atomic.Uint64 - indexedAsyncFlushScheduled atomic.Uint64 - indexedAsyncFlushBackpressure atomic.Uint64 - indexedAsyncFlushWaitTotalNs atomic.Uint64 - indexedAsyncFlushErrors atomic.Uint64 - indexedFlushCalls atomic.Uint64 - indexedFlushErrors atomic.Uint64 - indexedFlushForcedDrains atomic.Uint64 - indexedFlushUnitsTotal atomic.Uint64 - indexedFlushRequeues atomic.Uint64 - indexedFlushRequeuedUnits atomic.Uint64 - indexedFlushLostOwnership atomic.Uint64 - indexedFlushRootBaseMismatches atomic.Uint64 - indexedFlushDocs atomic.Uint64 - indexedFlushBytes atomic.Uint64 - indexedFlushRootRuns atomic.Uint64 - indexedFlushRoots atomic.Uint64 - indexedFlushDurationTotalNs atomic.Uint64 - indexedFlushMaterializeTotalNs atomic.Uint64 - indexedFlushPublishTotalNs atomic.Uint64 - rootDeltaPlanPrimaryRoots atomic.Uint64 - rootDeltaPlanTemplateRoots atomic.Uint64 - rootDeltaPlanIndexStateRoots atomic.Uint64 - rootDeltaPlanSecondaryRoots atomic.Uint64 - rootDeltaPlanEntries atomic.Uint64 - rootDeltaPlanKeyBytes atomic.Uint64 - rootDeltaPlanValueBytes atomic.Uint64 - rootDeltaPlanTombstones atomic.Uint64 - primaryOnlyUpdateCalls atomic.Uint64 - primaryOnlyMatched atomic.Uint64 - primaryOnlyModified atomic.Uint64 - primaryOnlyBufferedCalls atomic.Uint64 - primaryOnlyRootPublishes atomic.Uint64 - primaryOnlyRootDeltaEntries atomic.Uint64 - primaryOnlyRootDeltaKeyBytes atomic.Uint64 - primaryOnlyRootDeltaValueBytes atomic.Uint64 - primaryOnlyCoalescedDocs atomic.Uint64 - updateCombineRequests atomic.Uint64 - updateCombineBatches atomic.Uint64 - updateCombineBatchedRequests atomic.Uint64 - updateCombineFallbackRequests atomic.Uint64 - updateCombineQueueDepthMax atomic.Uint64 - updateBatchCalls atomic.Uint64 - updateBatchItems atomic.Uint64 - updateBatchMatched atomic.Uint64 - updateBatchModified atomic.Uint64 - updateBatchRuns atomic.Uint64 - updateBatchBufferedBatches atomic.Uint64 - updateBatchCurrentReadNs atomic.Uint64 - updateBatchCallbackNs atomic.Uint64 - updateBatchPrepareNs atomic.Uint64 - updateBatchIndexStateNs atomic.Uint64 - updateBatchUniquePreflightNs atomic.Uint64 - updateBatchTemplateRunNs atomic.Uint64 - updateBatchPrimaryRunNs atomic.Uint64 - updateBatchIndexStateRunNs atomic.Uint64 - updateBatchSecondaryRunNs atomic.Uint64 - updateBatchBufferStageNs atomic.Uint64 - updateBatchBufferPrecheckNs atomic.Uint64 - updateBatchBufferLockWaitNs atomic.Uint64 - updateBatchBufferLockHoldNs atomic.Uint64 - updateBatchBufferValidationNs atomic.Uint64 - updateBatchBufferRootScanNs atomic.Uint64 - updateBatchBufferDomainPrepareNs atomic.Uint64 - updateBatchBufferPrimaryIdxNs atomic.Uint64 - updateBatchBufferUniqueIdxNs atomic.Uint64 - updateBatchBufferRootAppendNs atomic.Uint64 - updateBatchBufferFlushNs atomic.Uint64 - updateBatchPublishNs atomic.Uint64 - updateBatchSecondaryDeletes atomic.Uint64 - updateBatchSecondarySets atomic.Uint64 - updateBatchSecondaryKeyBytes atomic.Uint64 - updateBatchIndexValueChanges atomic.Uint64 - updateBatchIndexValueUnchanged atomic.Uint64 - updateBatchMaskFallbacks atomic.Uint64 - updateBatchUniqueChecks atomic.Uint64 - updateBatchUniqueCheckSkips atomic.Uint64 - updateBatchDetailedStats atomic.Bool - updateBatchIndexChanged [maxCollectionUpdateInlineIndexStats]atomic.Uint64 - updateBatchIndexUnchanged [maxCollectionUpdateInlineIndexStats]atomic.Uint64 - updateBatchIndexUniqueChecks [maxCollectionUpdateInlineIndexStats]atomic.Uint64 - updateBatchIndexUniqueSkips [maxCollectionUpdateInlineIndexStats]atomic.Uint64 - updateBatchIndexSecondaryRuns [maxCollectionUpdateInlineIndexStats]atomic.Uint64 - updateBatchIndexSecondaryDeletes [maxCollectionUpdateInlineIndexStats]atomic.Uint64 - updateBatchIndexSecondarySets [maxCollectionUpdateInlineIndexStats]atomic.Uint64 - updateBatchIndexSecondaryBytes [maxCollectionUpdateInlineIndexStats]atomic.Uint64 + mutationLockCalls atomic.Uint64 + mutationLockWaitTotalNs atomic.Uint64 + mutationLockHoldTotalNs atomic.Uint64 + indexedStageBatches atomic.Uint64 + indexedStageDocs atomic.Uint64 + indexedStageBytes atomic.Uint64 + indexedStageRootRuns atomic.Uint64 + indexedSemanticRawRecords atomic.Uint64 + indexedSemanticRawIndexDeltas atomic.Uint64 + indexedSemanticFallbackRecords atomic.Uint64 + indexedSemanticEffectiveRecords atomic.Uint64 + indexedSemanticSkippedSecondaryRoots atomic.Uint64 + indexedAutoFlushes atomic.Uint64 + indexedAsyncFlushScheduled atomic.Uint64 + indexedAsyncFlushBackpressure atomic.Uint64 + indexedAsyncFlushWaitTotalNs atomic.Uint64 + indexedAsyncFlushErrors atomic.Uint64 + indexedFlushCalls atomic.Uint64 + indexedFlushErrors atomic.Uint64 + indexedFlushForcedDrains atomic.Uint64 + indexedFlushUnitsTotal atomic.Uint64 + indexedFlushRequeues atomic.Uint64 + indexedFlushRequeuedUnits atomic.Uint64 + indexedFlushLostOwnership atomic.Uint64 + indexedFlushRootBaseMismatches atomic.Uint64 + indexedFlushDocs atomic.Uint64 + indexedFlushBytes atomic.Uint64 + indexedFlushRootRuns atomic.Uint64 + indexedFlushRoots atomic.Uint64 + indexedFlushPreflightTotalNs atomic.Uint64 + indexedFlushRotateTotalNs atomic.Uint64 + indexedFlushMergeTotalNs atomic.Uint64 + indexedFlushDurationTotalNs atomic.Uint64 + indexedFlushMaterializeTotalNs atomic.Uint64 + indexedFlushSemanticPlanTotalNs atomic.Uint64 + indexedFlushBuildInputsTotalNs atomic.Uint64 + indexedFlushPlanStatsTotalNs atomic.Uint64 + indexedFlushPublishTotalNs atomic.Uint64 + coalescedFlushBatches atomic.Uint64 + coalescedFlushBatchUnits atomic.Uint64 + coalescedFlushBatchDocs atomic.Uint64 + coalescedFlushBatchBytes atomic.Uint64 + coalescedFlushNetZeroBatches atomic.Uint64 + rootDeltaPlanPrimaryRoots atomic.Uint64 + rootDeltaPlanTemplateRoots atomic.Uint64 + rootDeltaPlanIndexStateRoots atomic.Uint64 + rootDeltaPlanSecondaryRoots atomic.Uint64 + rootDeltaPlanEntries atomic.Uint64 + rootDeltaPlanKeyBytes atomic.Uint64 + rootDeltaPlanValueBytes atomic.Uint64 + rootDeltaPlanTombstones atomic.Uint64 + rootDeltaPlanRawUnitPrimaryEntries atomic.Uint64 + rootDeltaPlanRawUnitPrimaryBytes atomic.Uint64 + rootDeltaPlanRawUnitPrimaryTombstones atomic.Uint64 + rootDeltaPlanRawUnitTemplateEntries atomic.Uint64 + rootDeltaPlanRawUnitTemplateBytes atomic.Uint64 + rootDeltaPlanRawUnitTemplateTombstones atomic.Uint64 + rootDeltaPlanRawUnitIndexStateEntries atomic.Uint64 + rootDeltaPlanRawUnitIndexStateBytes atomic.Uint64 + rootDeltaPlanRawUnitIndexStateTombstones atomic.Uint64 + rootDeltaPlanRawUnitSecondaryEntries atomic.Uint64 + rootDeltaPlanRawUnitSecondaryBytes atomic.Uint64 + rootDeltaPlanRawUnitSecondaryTombstones atomic.Uint64 + rootDeltaPlanFinalPrimaryEntries atomic.Uint64 + rootDeltaPlanFinalPrimaryBytes atomic.Uint64 + rootDeltaPlanFinalPrimaryTombstones atomic.Uint64 + rootDeltaPlanFinalTemplateEntries atomic.Uint64 + rootDeltaPlanFinalTemplateBytes atomic.Uint64 + rootDeltaPlanFinalTemplateTombstones atomic.Uint64 + rootDeltaPlanFinalIndexStateEntries atomic.Uint64 + rootDeltaPlanFinalIndexStateBytes atomic.Uint64 + rootDeltaPlanFinalIndexStateTombstones atomic.Uint64 + rootDeltaPlanFinalSecondaryEntries atomic.Uint64 + rootDeltaPlanFinalSecondaryBytes atomic.Uint64 + rootDeltaPlanFinalSecondaryTombstones atomic.Uint64 + rootDeltaPlanSquashedEntries atomic.Uint64 + rootDeltaPlanNetZeroPlans atomic.Uint64 + primaryOnlyUpdateCalls atomic.Uint64 + primaryOnlyMatched atomic.Uint64 + primaryOnlyModified atomic.Uint64 + primaryOnlyBufferedCalls atomic.Uint64 + primaryOnlyRootPublishes atomic.Uint64 + primaryOnlyRootDeltaEntries atomic.Uint64 + primaryOnlyRootDeltaKeyBytes atomic.Uint64 + primaryOnlyRootDeltaValueBytes atomic.Uint64 + primaryOnlyCoalescedDocs atomic.Uint64 + primaryOnlyDuplicateIDsCoalesced atomic.Uint64 + primaryOnlyDrainCalls atomic.Uint64 + primaryOnlyDrainDocs atomic.Uint64 + primaryOnlyDrainBytes atomic.Uint64 + primaryOnlyDrainDurationTotalNs atomic.Uint64 + updateCombineRequests atomic.Uint64 + updateCombineBatches atomic.Uint64 + updateCombineBatchedRequests atomic.Uint64 + updateCombineFallbackRequests atomic.Uint64 + updateCombineQueueDepthMax atomic.Uint64 + updateBatchCalls atomic.Uint64 + updateBatchItems atomic.Uint64 + updateBatchMatched atomic.Uint64 + updateBatchModified atomic.Uint64 + updateBatchRuns atomic.Uint64 + updateBatchBufferedBatches atomic.Uint64 + updateBatchValidateNs atomic.Uint64 + updateBatchCloneNs atomic.Uint64 + updateBatchPlanSetupNs atomic.Uint64 + updateBatchBufferedReadSnapshotNs atomic.Uint64 + updateBatchCurrentReadNs atomic.Uint64 + updateBatchBSONIDValidationNs atomic.Uint64 + updateBatchCallbackNs atomic.Uint64 + updateBatchReplacementStageNs atomic.Uint64 + updateBatchPrepareNs atomic.Uint64 + updateBatchIndexStateNs atomic.Uint64 + updateBatchIndexStateCompareNs atomic.Uint64 + updateBatchUniquePreflightNs atomic.Uint64 + updateBatchTemplateRunNs atomic.Uint64 + updateBatchPrimaryRunNs atomic.Uint64 + updateBatchIndexStateRunNs atomic.Uint64 + updateBatchSecondaryRunNs atomic.Uint64 + updateBatchSemanticRecordNs atomic.Uint64 + updateBatchBufferStageNs atomic.Uint64 + updateBatchBufferPrecheckNs atomic.Uint64 + updateBatchBufferLockWaitNs atomic.Uint64 + updateBatchBufferLockHoldNs atomic.Uint64 + updateBatchBufferValidationNs atomic.Uint64 + updateBatchBufferRootScanNs atomic.Uint64 + updateBatchBufferDomainPrepareNs atomic.Uint64 + updateBatchBufferPrimaryIdxNs atomic.Uint64 + updateBatchBufferUniqueIdxNs atomic.Uint64 + updateBatchBufferRootAppendNs atomic.Uint64 + updateBatchBufferSemanticAppendNs atomic.Uint64 + updateBatchBufferFlushNs atomic.Uint64 + updateBatchPlanCloseNs atomic.Uint64 + updateBatchPublishNs atomic.Uint64 + updateBatchSecondaryDeletes atomic.Uint64 + updateBatchSecondarySets atomic.Uint64 + updateBatchSecondaryKeyBytes atomic.Uint64 + updateBatchIndexValueChanges atomic.Uint64 + updateBatchIndexValueUnchanged atomic.Uint64 + updateBatchMaskFallbacks atomic.Uint64 + updateBatchUniqueChecks atomic.Uint64 + updateBatchUniqueCheckSkips atomic.Uint64 + updateBatchDetailedStats atomic.Bool + updateBatchIndexChanged [maxCollectionUpdateInlineIndexStats]atomic.Uint64 + updateBatchIndexUnchanged [maxCollectionUpdateInlineIndexStats]atomic.Uint64 + updateBatchIndexUniqueChecks [maxCollectionUpdateInlineIndexStats]atomic.Uint64 + updateBatchIndexUniqueSkips [maxCollectionUpdateInlineIndexStats]atomic.Uint64 + updateBatchIndexSecondaryRuns [maxCollectionUpdateInlineIndexStats]atomic.Uint64 + updateBatchIndexSecondaryDeletes [maxCollectionUpdateInlineIndexStats]atomic.Uint64 + updateBatchIndexSecondarySets [maxCollectionUpdateInlineIndexStats]atomic.Uint64 + updateBatchIndexSecondaryBytes [maxCollectionUpdateInlineIndexStats]atomic.Uint64 } func NewCollectionManager(database *backenddb.DB) *CollectionManager { @@ -1027,6 +1229,7 @@ func (m *CollectionManager) Stats() map[string]string { out["treedb.collections.write_domain.pending_bytes"] = fmt.Sprintf("%d", stats.PendingBytes) out["treedb.collections.write_domain.pending_root_runs"] = fmt.Sprintf("%d", stats.PendingRootRuns) out["treedb.collections.write_domain.pending_indexed_flush_units"] = fmt.Sprintf("%d", stats.PendingIndexedFlushUnits) + out["treedb.collections.write_domain.pending_indexed_semantic_records"] = fmt.Sprintf("%d", stats.PendingIndexedSemanticRecords) out["treedb.collections.write_domain.overlay.mutable_docs"] = fmt.Sprintf("%d", stats.OverlayMutableDocuments) out["treedb.collections.write_domain.overlay.queued_indexed_flush_units"] = fmt.Sprintf("%d", stats.OverlayQueuedIndexedFlushUnits) out["treedb.collections.write_domain.overlay.active_indexed_flush_units"] = fmt.Sprintf("%d", stats.OverlayActiveIndexedFlushUnits) @@ -1042,6 +1245,11 @@ func (m *CollectionManager) Stats() map[string]string { out["treedb.collections.write_domain.indexed_stage.docs_total"] = fmt.Sprintf("%d", stats.IndexedStageDocs) out["treedb.collections.write_domain.indexed_stage.bytes_total"] = fmt.Sprintf("%d", stats.IndexedStageBytes) out["treedb.collections.write_domain.indexed_stage.root_runs_total"] = fmt.Sprintf("%d", stats.IndexedStageRootRuns) + out["treedb.collections.write_domain.indexed_semantic.raw_records_total"] = fmt.Sprintf("%d", stats.IndexedSemanticRawRecords) + out["treedb.collections.write_domain.indexed_semantic.raw_index_deltas_total"] = fmt.Sprintf("%d", stats.IndexedSemanticRawIndexDeltas) + out["treedb.collections.write_domain.indexed_semantic.fallback_records_total"] = fmt.Sprintf("%d", stats.IndexedSemanticFallbackRecords) + out["treedb.collections.write_domain.indexed_semantic.effective_records_total"] = fmt.Sprintf("%d", stats.IndexedSemanticEffectiveRecords) + out["treedb.collections.write_domain.indexed_semantic.skipped_secondary_roots_total"] = fmt.Sprintf("%d", stats.IndexedSemanticSkippedSecondaryRoots) out["treedb.collections.write_domain.indexed_stage.auto_flushes_total"] = fmt.Sprintf("%d", stats.IndexedAutoFlushes) out["treedb.collections.write_domain.indexed_async_flush.scheduled_total"] = fmt.Sprintf("%d", stats.IndexedAsyncFlushScheduled) out["treedb.collections.write_domain.indexed_async_flush.backpressure_sync_total"] = fmt.Sprintf("%d", stats.IndexedAsyncFlushBackpressure) @@ -1059,9 +1267,20 @@ func (m *CollectionManager) Stats() map[string]string { out["treedb.collections.write_domain.indexed_flush.bytes_total"] = fmt.Sprintf("%d", stats.IndexedFlushBytes) out["treedb.collections.write_domain.indexed_flush.root_runs_total"] = fmt.Sprintf("%d", stats.IndexedFlushRootRuns) out["treedb.collections.write_domain.indexed_flush.roots_total"] = fmt.Sprintf("%d", stats.IndexedFlushRoots) + out["treedb.collections.write_domain.indexed_flush.preflight_ns_total"] = fmt.Sprintf("%d", stats.IndexedFlushPreflight.Nanoseconds()) + out["treedb.collections.write_domain.indexed_flush.rotate_ns_total"] = fmt.Sprintf("%d", stats.IndexedFlushRotate.Nanoseconds()) + out["treedb.collections.write_domain.indexed_flush.merge_ns_total"] = fmt.Sprintf("%d", stats.IndexedFlushMerge.Nanoseconds()) out["treedb.collections.write_domain.indexed_flush.duration_ns_total"] = fmt.Sprintf("%d", stats.IndexedFlushDuration.Nanoseconds()) out["treedb.collections.write_domain.indexed_flush.materialize_ns_total"] = fmt.Sprintf("%d", stats.IndexedFlushMaterialize.Nanoseconds()) + out["treedb.collections.write_domain.indexed_flush.materialize_semantic_plan_ns_total"] = fmt.Sprintf("%d", stats.IndexedFlushSemanticPlan.Nanoseconds()) + out["treedb.collections.write_domain.indexed_flush.materialize_build_inputs_ns_total"] = fmt.Sprintf("%d", stats.IndexedFlushBuildInputs.Nanoseconds()) + out["treedb.collections.write_domain.indexed_flush.materialize_plan_stats_ns_total"] = fmt.Sprintf("%d", stats.IndexedFlushPlanStats.Nanoseconds()) out["treedb.collections.write_domain.indexed_flush.publish_ns_total"] = fmt.Sprintf("%d", stats.IndexedFlushPublish.Nanoseconds()) + out["treedb.collections.write_domain.coalesced_flush_batch.batches_total"] = fmt.Sprintf("%d", stats.CoalescedFlushBatches) + out["treedb.collections.write_domain.coalesced_flush_batch.units_total"] = fmt.Sprintf("%d", stats.CoalescedFlushBatchUnits) + out["treedb.collections.write_domain.coalesced_flush_batch.docs_total"] = fmt.Sprintf("%d", stats.CoalescedFlushBatchDocs) + out["treedb.collections.write_domain.coalesced_flush_batch.bytes_total"] = fmt.Sprintf("%d", stats.CoalescedFlushBatchBytes) + out["treedb.collections.write_domain.coalesced_flush_batch.net_zero_batches_total"] = fmt.Sprintf("%d", stats.CoalescedFlushNetZeroBatches) out["treedb.collections.write_domain.root_delta_plan.roots.primary_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanPrimaryRoots) out["treedb.collections.write_domain.root_delta_plan.roots.template_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanTemplateRoots) out["treedb.collections.write_domain.root_delta_plan.roots.index_state_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanIndexStateRoots) @@ -1070,6 +1289,32 @@ func (m *CollectionManager) Stats() map[string]string { out["treedb.collections.write_domain.root_delta_plan.key_bytes_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanKeyBytes) out["treedb.collections.write_domain.root_delta_plan.value_bytes_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanValueBytes) out["treedb.collections.write_domain.root_delta_plan.tombstones_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanTombstones) + out["treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanRawUnitPrimaryEntries) + out["treedb.collections.write_domain.root_delta_plan.raw_unit.primary.bytes_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanRawUnitPrimaryBytes) + out["treedb.collections.write_domain.root_delta_plan.raw_unit.primary.tombstones_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanRawUnitPrimaryTombstones) + out["treedb.collections.write_domain.root_delta_plan.raw_unit.template.entries_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanRawUnitTemplateEntries) + out["treedb.collections.write_domain.root_delta_plan.raw_unit.template.bytes_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanRawUnitTemplateBytes) + out["treedb.collections.write_domain.root_delta_plan.raw_unit.template.tombstones_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanRawUnitTemplateTombstones) + out["treedb.collections.write_domain.root_delta_plan.raw_unit.index_state.entries_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanRawUnitIndexStateEntries) + out["treedb.collections.write_domain.root_delta_plan.raw_unit.index_state.bytes_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanRawUnitIndexStateBytes) + out["treedb.collections.write_domain.root_delta_plan.raw_unit.index_state.tombstones_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanRawUnitIndexStateTombstones) + out["treedb.collections.write_domain.root_delta_plan.raw_unit.secondary.entries_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanRawUnitSecondaryEntries) + out["treedb.collections.write_domain.root_delta_plan.raw_unit.secondary.bytes_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanRawUnitSecondaryBytes) + out["treedb.collections.write_domain.root_delta_plan.raw_unit.secondary.tombstones_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanRawUnitSecondaryTombstones) + out["treedb.collections.write_domain.root_delta_plan.final.primary.entries_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanFinalPrimaryEntries) + out["treedb.collections.write_domain.root_delta_plan.final.primary.bytes_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanFinalPrimaryBytes) + out["treedb.collections.write_domain.root_delta_plan.final.primary.tombstones_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanFinalPrimaryTombstones) + out["treedb.collections.write_domain.root_delta_plan.final.template.entries_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanFinalTemplateEntries) + out["treedb.collections.write_domain.root_delta_plan.final.template.bytes_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanFinalTemplateBytes) + out["treedb.collections.write_domain.root_delta_plan.final.template.tombstones_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanFinalTemplateTombstones) + out["treedb.collections.write_domain.root_delta_plan.final.index_state.entries_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanFinalIndexStateEntries) + out["treedb.collections.write_domain.root_delta_plan.final.index_state.bytes_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanFinalIndexStateBytes) + out["treedb.collections.write_domain.root_delta_plan.final.index_state.tombstones_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanFinalIndexStateTombstones) + out["treedb.collections.write_domain.root_delta_plan.final.secondary.entries_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanFinalSecondaryEntries) + out["treedb.collections.write_domain.root_delta_plan.final.secondary.bytes_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanFinalSecondaryBytes) + out["treedb.collections.write_domain.root_delta_plan.final.secondary.tombstones_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanFinalSecondaryTombstones) + out["treedb.collections.write_domain.root_delta_plan.squashed_entries_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanSquashedEntries) + out["treedb.collections.write_domain.root_delta_plan.net_zero_plans_total"] = fmt.Sprintf("%d", stats.RootDeltaPlanNetZeroPlans) out["treedb.collections.write_domain.primary_only.update_calls_total"] = fmt.Sprintf("%d", stats.PrimaryOnlyUpdateCalls) out["treedb.collections.write_domain.primary_only.matched_total"] = fmt.Sprintf("%d", stats.PrimaryOnlyMatched) out["treedb.collections.write_domain.primary_only.modified_total"] = fmt.Sprintf("%d", stats.PrimaryOnlyModified) @@ -1079,6 +1324,11 @@ func (m *CollectionManager) Stats() map[string]string { out["treedb.collections.write_domain.primary_only.root_delta_key_bytes_total"] = fmt.Sprintf("%d", stats.PrimaryOnlyRootDeltaKeyBytes) out["treedb.collections.write_domain.primary_only.root_delta_value_bytes_total"] = fmt.Sprintf("%d", stats.PrimaryOnlyRootDeltaValueBytes) out["treedb.collections.write_domain.primary_only.coalesced_docs_total"] = fmt.Sprintf("%d", stats.PrimaryOnlyCoalescedDocs) + out["treedb.collections.write_domain.primary_only.duplicate_ids_coalesced_total"] = fmt.Sprintf("%d", stats.PrimaryOnlyDuplicateIDsCoalesced) + out["treedb.collections.write_domain.primary_only.drains_total"] = fmt.Sprintf("%d", stats.PrimaryOnlyDrainCalls) + out["treedb.collections.write_domain.primary_only.drain_docs_total"] = fmt.Sprintf("%d", stats.PrimaryOnlyDrainDocs) + out["treedb.collections.write_domain.primary_only.drain_bytes_total"] = fmt.Sprintf("%d", stats.PrimaryOnlyDrainBytes) + out["treedb.collections.write_domain.primary_only.drain_ns_total"] = fmt.Sprintf("%d", stats.PrimaryOnlyDrainDuration.Nanoseconds()) out["treedb.collections.write_domain.update_combine.requests_total"] = fmt.Sprintf("%d", stats.UpdateCombineRequests) out["treedb.collections.write_domain.update_combine.batches_total"] = fmt.Sprintf("%d", stats.UpdateCombineBatches) out["treedb.collections.write_domain.update_combine.batched_requests_total"] = fmt.Sprintf("%d", stats.UpdateCombineBatchedRequests) @@ -1090,15 +1340,23 @@ func (m *CollectionManager) Stats() map[string]string { out["treedb.collections.write_domain.update_batch.modified_total"] = fmt.Sprintf("%d", stats.UpdateBatchModified) out["treedb.collections.write_domain.update_batch.root_runs_total"] = fmt.Sprintf("%d", stats.UpdateBatchRuns) out["treedb.collections.write_domain.update_batch.buffered_batches_total"] = fmt.Sprintf("%d", stats.UpdateBatchBufferedBatches) + out["treedb.collections.write_domain.update_batch.validate_items_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchValidate.Nanoseconds()) + out["treedb.collections.write_domain.update_batch.clone_items_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchClone.Nanoseconds()) + out["treedb.collections.write_domain.update_batch.plan_setup_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchPlanSetup.Nanoseconds()) + out["treedb.collections.write_domain.update_batch.buffered_read_snapshot_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchBufferedReadSnapshot.Nanoseconds()) out["treedb.collections.write_domain.update_batch.current_read_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchCurrentRead.Nanoseconds()) + out["treedb.collections.write_domain.update_batch.bson_id_validation_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchBSONIDValidation.Nanoseconds()) out["treedb.collections.write_domain.update_batch.callback_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchCallback.Nanoseconds()) + out["treedb.collections.write_domain.update_batch.replacement_stage_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchReplacementStage.Nanoseconds()) out["treedb.collections.write_domain.update_batch.prepare_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchPrepareDocuments.Nanoseconds()) out["treedb.collections.write_domain.update_batch.index_state_extract_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchIndexStateExtract.Nanoseconds()) + out["treedb.collections.write_domain.update_batch.index_state_compare_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchIndexStateCompare.Nanoseconds()) out["treedb.collections.write_domain.update_batch.unique_preflight_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchUniquePreflight.Nanoseconds()) out["treedb.collections.write_domain.update_batch.template_run_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchTemplateRunBuild.Nanoseconds()) out["treedb.collections.write_domain.update_batch.primary_run_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchPrimaryRunBuild.Nanoseconds()) out["treedb.collections.write_domain.update_batch.index_state_run_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchIndexStateRunBuild.Nanoseconds()) out["treedb.collections.write_domain.update_batch.secondary_runs_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchSecondaryRunBuild.Nanoseconds()) + out["treedb.collections.write_domain.update_batch.semantic_record_build_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchSemanticRecordBuild.Nanoseconds()) out["treedb.collections.write_domain.update_batch.buffer_stage_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchBufferStage.Nanoseconds()) out["treedb.collections.write_domain.update_batch.buffer_stage_precheck_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchBufferPrecheck.Nanoseconds()) out["treedb.collections.write_domain.update_batch.buffer_stage_lock_wait_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchBufferLockWait.Nanoseconds()) @@ -1109,7 +1367,9 @@ func (m *CollectionManager) Stats() map[string]string { out["treedb.collections.write_domain.update_batch.buffer_stage_primary_index_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchBufferPrimaryIdx.Nanoseconds()) out["treedb.collections.write_domain.update_batch.buffer_stage_unique_index_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchBufferUniqueIdx.Nanoseconds()) out["treedb.collections.write_domain.update_batch.buffer_stage_root_append_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchBufferRootAppend.Nanoseconds()) + out["treedb.collections.write_domain.update_batch.buffer_stage_semantic_append_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchBufferSemanticAppend.Nanoseconds()) out["treedb.collections.write_domain.update_batch.buffer_stage_flush_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchBufferFlush.Nanoseconds()) + out["treedb.collections.write_domain.update_batch.plan_close_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchPlanClose.Nanoseconds()) out["treedb.collections.write_domain.update_batch.publish_ns_total"] = fmt.Sprintf("%d", stats.UpdateBatchPublish.Nanoseconds()) out["treedb.collections.write_domain.update_batch.secondary_deletes_total"] = fmt.Sprintf("%d", stats.UpdateBatchSecondaryDeletes) out["treedb.collections.write_domain.update_batch.secondary_sets_total"] = fmt.Sprintf("%d", stats.UpdateBatchSecondarySets) @@ -1230,6 +1490,7 @@ func (s *CollectionManagerStats) add(other CollectionManagerStats) { s.PendingBytes = saturatingAddNonNegativeInt64(s.PendingBytes, other.PendingBytes) s.PendingRootRuns = saturatingAddNonNegativeInt(s.PendingRootRuns, other.PendingRootRuns) s.PendingIndexedFlushUnits = saturatingAddNonNegativeInt(s.PendingIndexedFlushUnits, other.PendingIndexedFlushUnits) + s.PendingIndexedSemanticRecords = saturatingAddNonNegativeInt(s.PendingIndexedSemanticRecords, other.PendingIndexedSemanticRecords) s.OverlayMutableDocuments = saturatingAddNonNegativeInt(s.OverlayMutableDocuments, other.OverlayMutableDocuments) s.OverlayQueuedIndexedFlushUnits = saturatingAddNonNegativeInt(s.OverlayQueuedIndexedFlushUnits, other.OverlayQueuedIndexedFlushUnits) s.OverlayActiveIndexedFlushUnits = saturatingAddNonNegativeInt(s.OverlayActiveIndexedFlushUnits, other.OverlayActiveIndexedFlushUnits) @@ -1242,6 +1503,11 @@ func (s *CollectionManagerStats) add(other CollectionManagerStats) { s.IndexedStageDocs += other.IndexedStageDocs s.IndexedStageBytes += other.IndexedStageBytes s.IndexedStageRootRuns += other.IndexedStageRootRuns + s.IndexedSemanticRawRecords += other.IndexedSemanticRawRecords + s.IndexedSemanticRawIndexDeltas += other.IndexedSemanticRawIndexDeltas + s.IndexedSemanticFallbackRecords += other.IndexedSemanticFallbackRecords + s.IndexedSemanticEffectiveRecords += other.IndexedSemanticEffectiveRecords + s.IndexedSemanticSkippedSecondaryRoots += other.IndexedSemanticSkippedSecondaryRoots s.IndexedAutoFlushes += other.IndexedAutoFlushes s.IndexedAsyncFlushScheduled += other.IndexedAsyncFlushScheduled s.IndexedAsyncFlushBackpressure += other.IndexedAsyncFlushBackpressure @@ -1259,9 +1525,20 @@ func (s *CollectionManagerStats) add(other CollectionManagerStats) { s.IndexedFlushBytes += other.IndexedFlushBytes s.IndexedFlushRootRuns += other.IndexedFlushRootRuns s.IndexedFlushRoots += other.IndexedFlushRoots + s.IndexedFlushPreflight += other.IndexedFlushPreflight + s.IndexedFlushRotate += other.IndexedFlushRotate + s.IndexedFlushMerge += other.IndexedFlushMerge s.IndexedFlushDuration += other.IndexedFlushDuration s.IndexedFlushMaterialize += other.IndexedFlushMaterialize + s.IndexedFlushSemanticPlan += other.IndexedFlushSemanticPlan + s.IndexedFlushBuildInputs += other.IndexedFlushBuildInputs + s.IndexedFlushPlanStats += other.IndexedFlushPlanStats s.IndexedFlushPublish += other.IndexedFlushPublish + s.CoalescedFlushBatches += other.CoalescedFlushBatches + s.CoalescedFlushBatchUnits += other.CoalescedFlushBatchUnits + s.CoalescedFlushBatchDocs += other.CoalescedFlushBatchDocs + s.CoalescedFlushBatchBytes += other.CoalescedFlushBatchBytes + s.CoalescedFlushNetZeroBatches += other.CoalescedFlushNetZeroBatches s.RootDeltaPlanPrimaryRoots += other.RootDeltaPlanPrimaryRoots s.RootDeltaPlanTemplateRoots += other.RootDeltaPlanTemplateRoots s.RootDeltaPlanIndexStateRoots += other.RootDeltaPlanIndexStateRoots @@ -1270,6 +1547,32 @@ func (s *CollectionManagerStats) add(other CollectionManagerStats) { s.RootDeltaPlanKeyBytes += other.RootDeltaPlanKeyBytes s.RootDeltaPlanValueBytes += other.RootDeltaPlanValueBytes s.RootDeltaPlanTombstones += other.RootDeltaPlanTombstones + s.RootDeltaPlanRawUnitPrimaryEntries += other.RootDeltaPlanRawUnitPrimaryEntries + s.RootDeltaPlanRawUnitPrimaryBytes += other.RootDeltaPlanRawUnitPrimaryBytes + s.RootDeltaPlanRawUnitPrimaryTombstones += other.RootDeltaPlanRawUnitPrimaryTombstones + s.RootDeltaPlanRawUnitTemplateEntries += other.RootDeltaPlanRawUnitTemplateEntries + s.RootDeltaPlanRawUnitTemplateBytes += other.RootDeltaPlanRawUnitTemplateBytes + s.RootDeltaPlanRawUnitTemplateTombstones += other.RootDeltaPlanRawUnitTemplateTombstones + s.RootDeltaPlanRawUnitIndexStateEntries += other.RootDeltaPlanRawUnitIndexStateEntries + s.RootDeltaPlanRawUnitIndexStateBytes += other.RootDeltaPlanRawUnitIndexStateBytes + s.RootDeltaPlanRawUnitIndexStateTombstones += other.RootDeltaPlanRawUnitIndexStateTombstones + s.RootDeltaPlanRawUnitSecondaryEntries += other.RootDeltaPlanRawUnitSecondaryEntries + s.RootDeltaPlanRawUnitSecondaryBytes += other.RootDeltaPlanRawUnitSecondaryBytes + s.RootDeltaPlanRawUnitSecondaryTombstones += other.RootDeltaPlanRawUnitSecondaryTombstones + s.RootDeltaPlanFinalPrimaryEntries += other.RootDeltaPlanFinalPrimaryEntries + s.RootDeltaPlanFinalPrimaryBytes += other.RootDeltaPlanFinalPrimaryBytes + s.RootDeltaPlanFinalPrimaryTombstones += other.RootDeltaPlanFinalPrimaryTombstones + s.RootDeltaPlanFinalTemplateEntries += other.RootDeltaPlanFinalTemplateEntries + s.RootDeltaPlanFinalTemplateBytes += other.RootDeltaPlanFinalTemplateBytes + s.RootDeltaPlanFinalTemplateTombstones += other.RootDeltaPlanFinalTemplateTombstones + s.RootDeltaPlanFinalIndexStateEntries += other.RootDeltaPlanFinalIndexStateEntries + s.RootDeltaPlanFinalIndexStateBytes += other.RootDeltaPlanFinalIndexStateBytes + s.RootDeltaPlanFinalIndexStateTombstones += other.RootDeltaPlanFinalIndexStateTombstones + s.RootDeltaPlanFinalSecondaryEntries += other.RootDeltaPlanFinalSecondaryEntries + s.RootDeltaPlanFinalSecondaryBytes += other.RootDeltaPlanFinalSecondaryBytes + s.RootDeltaPlanFinalSecondaryTombstones += other.RootDeltaPlanFinalSecondaryTombstones + s.RootDeltaPlanSquashedEntries += other.RootDeltaPlanSquashedEntries + s.RootDeltaPlanNetZeroPlans += other.RootDeltaPlanNetZeroPlans s.PrimaryOnlyUpdateCalls += other.PrimaryOnlyUpdateCalls s.PrimaryOnlyMatched += other.PrimaryOnlyMatched s.PrimaryOnlyModified += other.PrimaryOnlyModified @@ -1279,6 +1582,11 @@ func (s *CollectionManagerStats) add(other CollectionManagerStats) { s.PrimaryOnlyRootDeltaKeyBytes += other.PrimaryOnlyRootDeltaKeyBytes s.PrimaryOnlyRootDeltaValueBytes += other.PrimaryOnlyRootDeltaValueBytes s.PrimaryOnlyCoalescedDocs += other.PrimaryOnlyCoalescedDocs + s.PrimaryOnlyDuplicateIDsCoalesced += other.PrimaryOnlyDuplicateIDsCoalesced + s.PrimaryOnlyDrainCalls += other.PrimaryOnlyDrainCalls + s.PrimaryOnlyDrainDocs += other.PrimaryOnlyDrainDocs + s.PrimaryOnlyDrainBytes += other.PrimaryOnlyDrainBytes + s.PrimaryOnlyDrainDuration += other.PrimaryOnlyDrainDuration s.UpdateCombineRequests += other.UpdateCombineRequests s.UpdateCombineBatches += other.UpdateCombineBatches s.UpdateCombineBatchedRequests += other.UpdateCombineBatchedRequests @@ -1292,15 +1600,23 @@ func (s *CollectionManagerStats) add(other CollectionManagerStats) { s.UpdateBatchModified += other.UpdateBatchModified s.UpdateBatchRuns += other.UpdateBatchRuns s.UpdateBatchBufferedBatches += other.UpdateBatchBufferedBatches + s.UpdateBatchValidate += other.UpdateBatchValidate + s.UpdateBatchClone += other.UpdateBatchClone + s.UpdateBatchPlanSetup += other.UpdateBatchPlanSetup + s.UpdateBatchBufferedReadSnapshot += other.UpdateBatchBufferedReadSnapshot s.UpdateBatchCurrentRead += other.UpdateBatchCurrentRead + s.UpdateBatchBSONIDValidation += other.UpdateBatchBSONIDValidation s.UpdateBatchCallback += other.UpdateBatchCallback + s.UpdateBatchReplacementStage += other.UpdateBatchReplacementStage s.UpdateBatchPrepareDocuments += other.UpdateBatchPrepareDocuments s.UpdateBatchIndexStateExtract += other.UpdateBatchIndexStateExtract + s.UpdateBatchIndexStateCompare += other.UpdateBatchIndexStateCompare s.UpdateBatchUniquePreflight += other.UpdateBatchUniquePreflight s.UpdateBatchTemplateRunBuild += other.UpdateBatchTemplateRunBuild s.UpdateBatchPrimaryRunBuild += other.UpdateBatchPrimaryRunBuild s.UpdateBatchIndexStateRunBuild += other.UpdateBatchIndexStateRunBuild s.UpdateBatchSecondaryRunBuild += other.UpdateBatchSecondaryRunBuild + s.UpdateBatchSemanticRecordBuild += other.UpdateBatchSemanticRecordBuild s.UpdateBatchBufferStage += other.UpdateBatchBufferStage s.UpdateBatchBufferPrecheck += other.UpdateBatchBufferPrecheck s.UpdateBatchBufferLockWait += other.UpdateBatchBufferLockWait @@ -1311,7 +1627,9 @@ func (s *CollectionManagerStats) add(other CollectionManagerStats) { s.UpdateBatchBufferPrimaryIdx += other.UpdateBatchBufferPrimaryIdx s.UpdateBatchBufferUniqueIdx += other.UpdateBatchBufferUniqueIdx s.UpdateBatchBufferRootAppend += other.UpdateBatchBufferRootAppend + s.UpdateBatchBufferSemanticAppend += other.UpdateBatchBufferSemanticAppend s.UpdateBatchBufferFlush += other.UpdateBatchBufferFlush + s.UpdateBatchPlanClose += other.UpdateBatchPlanClose s.UpdateBatchPublish += other.UpdateBatchPublish s.UpdateBatchSecondaryDeletes += other.UpdateBatchSecondaryDeletes s.UpdateBatchSecondarySets += other.UpdateBatchSecondarySets @@ -1337,6 +1655,7 @@ func (domain *collectionWriteDomain) statsSnapshot() CollectionManagerStats { pendingRootRuns := bufferedIndexedRootRunCount(domain) stats.PendingRootRuns = pendingRootRuns stats.PendingIndexedFlushUnits = len(domain.indexedPublishingUnits) + len(domain.indexedFlushUnits) + stats.PendingIndexedSemanticRecords = pendingIndexedSemanticRecordCountLocked(domain) stats.OverlayMutableDocuments = domain.mutableCount stats.OverlayQueuedIndexedFlushUnits = len(domain.indexedFlushUnits) stats.OverlayActiveIndexedFlushUnits = len(domain.indexedPublishingUnits) @@ -1363,6 +1682,11 @@ func (domain *collectionWriteDomain) statsSnapshot() CollectionManagerStats { stats.IndexedStageDocs = domain.indexedStageDocs.Load() stats.IndexedStageBytes = domain.indexedStageBytes.Load() stats.IndexedStageRootRuns = domain.indexedStageRootRuns.Load() + stats.IndexedSemanticRawRecords = domain.indexedSemanticRawRecords.Load() + stats.IndexedSemanticRawIndexDeltas = domain.indexedSemanticRawIndexDeltas.Load() + stats.IndexedSemanticFallbackRecords = domain.indexedSemanticFallbackRecords.Load() + stats.IndexedSemanticEffectiveRecords = domain.indexedSemanticEffectiveRecords.Load() + stats.IndexedSemanticSkippedSecondaryRoots = domain.indexedSemanticSkippedSecondaryRoots.Load() stats.IndexedAutoFlushes = domain.indexedAutoFlushes.Load() stats.IndexedAsyncFlushScheduled = domain.indexedAsyncFlushScheduled.Load() stats.IndexedAsyncFlushBackpressure = domain.indexedAsyncFlushBackpressure.Load() @@ -1380,9 +1704,20 @@ func (domain *collectionWriteDomain) statsSnapshot() CollectionManagerStats { stats.IndexedFlushBytes = domain.indexedFlushBytes.Load() stats.IndexedFlushRootRuns = domain.indexedFlushRootRuns.Load() stats.IndexedFlushRoots = domain.indexedFlushRoots.Load() + stats.IndexedFlushPreflight = durationFromAtomicNs(domain.indexedFlushPreflightTotalNs.Load()) + stats.IndexedFlushRotate = durationFromAtomicNs(domain.indexedFlushRotateTotalNs.Load()) + stats.IndexedFlushMerge = durationFromAtomicNs(domain.indexedFlushMergeTotalNs.Load()) stats.IndexedFlushDuration = durationFromAtomicNs(domain.indexedFlushDurationTotalNs.Load()) stats.IndexedFlushMaterialize = durationFromAtomicNs(domain.indexedFlushMaterializeTotalNs.Load()) + stats.IndexedFlushSemanticPlan = durationFromAtomicNs(domain.indexedFlushSemanticPlanTotalNs.Load()) + stats.IndexedFlushBuildInputs = durationFromAtomicNs(domain.indexedFlushBuildInputsTotalNs.Load()) + stats.IndexedFlushPlanStats = durationFromAtomicNs(domain.indexedFlushPlanStatsTotalNs.Load()) stats.IndexedFlushPublish = durationFromAtomicNs(domain.indexedFlushPublishTotalNs.Load()) + stats.CoalescedFlushBatches = domain.coalescedFlushBatches.Load() + stats.CoalescedFlushBatchUnits = domain.coalescedFlushBatchUnits.Load() + stats.CoalescedFlushBatchDocs = domain.coalescedFlushBatchDocs.Load() + stats.CoalescedFlushBatchBytes = domain.coalescedFlushBatchBytes.Load() + stats.CoalescedFlushNetZeroBatches = domain.coalescedFlushNetZeroBatches.Load() stats.RootDeltaPlanPrimaryRoots = domain.rootDeltaPlanPrimaryRoots.Load() stats.RootDeltaPlanTemplateRoots = domain.rootDeltaPlanTemplateRoots.Load() stats.RootDeltaPlanIndexStateRoots = domain.rootDeltaPlanIndexStateRoots.Load() @@ -1391,6 +1726,32 @@ func (domain *collectionWriteDomain) statsSnapshot() CollectionManagerStats { stats.RootDeltaPlanKeyBytes = domain.rootDeltaPlanKeyBytes.Load() stats.RootDeltaPlanValueBytes = domain.rootDeltaPlanValueBytes.Load() stats.RootDeltaPlanTombstones = domain.rootDeltaPlanTombstones.Load() + stats.RootDeltaPlanRawUnitPrimaryEntries = domain.rootDeltaPlanRawUnitPrimaryEntries.Load() + stats.RootDeltaPlanRawUnitPrimaryBytes = domain.rootDeltaPlanRawUnitPrimaryBytes.Load() + stats.RootDeltaPlanRawUnitPrimaryTombstones = domain.rootDeltaPlanRawUnitPrimaryTombstones.Load() + stats.RootDeltaPlanRawUnitTemplateEntries = domain.rootDeltaPlanRawUnitTemplateEntries.Load() + stats.RootDeltaPlanRawUnitTemplateBytes = domain.rootDeltaPlanRawUnitTemplateBytes.Load() + stats.RootDeltaPlanRawUnitTemplateTombstones = domain.rootDeltaPlanRawUnitTemplateTombstones.Load() + stats.RootDeltaPlanRawUnitIndexStateEntries = domain.rootDeltaPlanRawUnitIndexStateEntries.Load() + stats.RootDeltaPlanRawUnitIndexStateBytes = domain.rootDeltaPlanRawUnitIndexStateBytes.Load() + stats.RootDeltaPlanRawUnitIndexStateTombstones = domain.rootDeltaPlanRawUnitIndexStateTombstones.Load() + stats.RootDeltaPlanRawUnitSecondaryEntries = domain.rootDeltaPlanRawUnitSecondaryEntries.Load() + stats.RootDeltaPlanRawUnitSecondaryBytes = domain.rootDeltaPlanRawUnitSecondaryBytes.Load() + stats.RootDeltaPlanRawUnitSecondaryTombstones = domain.rootDeltaPlanRawUnitSecondaryTombstones.Load() + stats.RootDeltaPlanFinalPrimaryEntries = domain.rootDeltaPlanFinalPrimaryEntries.Load() + stats.RootDeltaPlanFinalPrimaryBytes = domain.rootDeltaPlanFinalPrimaryBytes.Load() + stats.RootDeltaPlanFinalPrimaryTombstones = domain.rootDeltaPlanFinalPrimaryTombstones.Load() + stats.RootDeltaPlanFinalTemplateEntries = domain.rootDeltaPlanFinalTemplateEntries.Load() + stats.RootDeltaPlanFinalTemplateBytes = domain.rootDeltaPlanFinalTemplateBytes.Load() + stats.RootDeltaPlanFinalTemplateTombstones = domain.rootDeltaPlanFinalTemplateTombstones.Load() + stats.RootDeltaPlanFinalIndexStateEntries = domain.rootDeltaPlanFinalIndexStateEntries.Load() + stats.RootDeltaPlanFinalIndexStateBytes = domain.rootDeltaPlanFinalIndexStateBytes.Load() + stats.RootDeltaPlanFinalIndexStateTombstones = domain.rootDeltaPlanFinalIndexStateTombstones.Load() + stats.RootDeltaPlanFinalSecondaryEntries = domain.rootDeltaPlanFinalSecondaryEntries.Load() + stats.RootDeltaPlanFinalSecondaryBytes = domain.rootDeltaPlanFinalSecondaryBytes.Load() + stats.RootDeltaPlanFinalSecondaryTombstones = domain.rootDeltaPlanFinalSecondaryTombstones.Load() + stats.RootDeltaPlanSquashedEntries = domain.rootDeltaPlanSquashedEntries.Load() + stats.RootDeltaPlanNetZeroPlans = domain.rootDeltaPlanNetZeroPlans.Load() stats.PrimaryOnlyUpdateCalls = domain.primaryOnlyUpdateCalls.Load() stats.PrimaryOnlyMatched = domain.primaryOnlyMatched.Load() stats.PrimaryOnlyModified = domain.primaryOnlyModified.Load() @@ -1400,6 +1761,11 @@ func (domain *collectionWriteDomain) statsSnapshot() CollectionManagerStats { stats.PrimaryOnlyRootDeltaKeyBytes = domain.primaryOnlyRootDeltaKeyBytes.Load() stats.PrimaryOnlyRootDeltaValueBytes = domain.primaryOnlyRootDeltaValueBytes.Load() stats.PrimaryOnlyCoalescedDocs = domain.primaryOnlyCoalescedDocs.Load() + stats.PrimaryOnlyDuplicateIDsCoalesced = domain.primaryOnlyDuplicateIDsCoalesced.Load() + stats.PrimaryOnlyDrainCalls = domain.primaryOnlyDrainCalls.Load() + stats.PrimaryOnlyDrainDocs = domain.primaryOnlyDrainDocs.Load() + stats.PrimaryOnlyDrainBytes = domain.primaryOnlyDrainBytes.Load() + stats.PrimaryOnlyDrainDuration = durationFromAtomicNs(domain.primaryOnlyDrainDurationTotalNs.Load()) stats.UpdateCombineRequests = domain.updateCombineRequests.Load() stats.UpdateCombineBatches = domain.updateCombineBatches.Load() stats.UpdateCombineBatchedRequests = domain.updateCombineBatchedRequests.Load() @@ -1411,15 +1777,23 @@ func (domain *collectionWriteDomain) statsSnapshot() CollectionManagerStats { stats.UpdateBatchModified = domain.updateBatchModified.Load() stats.UpdateBatchRuns = domain.updateBatchRuns.Load() stats.UpdateBatchBufferedBatches = domain.updateBatchBufferedBatches.Load() + stats.UpdateBatchValidate = durationFromAtomicNs(domain.updateBatchValidateNs.Load()) + stats.UpdateBatchClone = durationFromAtomicNs(domain.updateBatchCloneNs.Load()) + stats.UpdateBatchPlanSetup = durationFromAtomicNs(domain.updateBatchPlanSetupNs.Load()) + stats.UpdateBatchBufferedReadSnapshot = durationFromAtomicNs(domain.updateBatchBufferedReadSnapshotNs.Load()) stats.UpdateBatchCurrentRead = durationFromAtomicNs(domain.updateBatchCurrentReadNs.Load()) + stats.UpdateBatchBSONIDValidation = durationFromAtomicNs(domain.updateBatchBSONIDValidationNs.Load()) stats.UpdateBatchCallback = durationFromAtomicNs(domain.updateBatchCallbackNs.Load()) + stats.UpdateBatchReplacementStage = durationFromAtomicNs(domain.updateBatchReplacementStageNs.Load()) stats.UpdateBatchPrepareDocuments = durationFromAtomicNs(domain.updateBatchPrepareNs.Load()) stats.UpdateBatchIndexStateExtract = durationFromAtomicNs(domain.updateBatchIndexStateNs.Load()) + stats.UpdateBatchIndexStateCompare = durationFromAtomicNs(domain.updateBatchIndexStateCompareNs.Load()) stats.UpdateBatchUniquePreflight = durationFromAtomicNs(domain.updateBatchUniquePreflightNs.Load()) stats.UpdateBatchTemplateRunBuild = durationFromAtomicNs(domain.updateBatchTemplateRunNs.Load()) stats.UpdateBatchPrimaryRunBuild = durationFromAtomicNs(domain.updateBatchPrimaryRunNs.Load()) stats.UpdateBatchIndexStateRunBuild = durationFromAtomicNs(domain.updateBatchIndexStateRunNs.Load()) stats.UpdateBatchSecondaryRunBuild = durationFromAtomicNs(domain.updateBatchSecondaryRunNs.Load()) + stats.UpdateBatchSemanticRecordBuild = durationFromAtomicNs(domain.updateBatchSemanticRecordNs.Load()) stats.UpdateBatchBufferStage = durationFromAtomicNs(domain.updateBatchBufferStageNs.Load()) stats.UpdateBatchBufferPrecheck = durationFromAtomicNs(domain.updateBatchBufferPrecheckNs.Load()) stats.UpdateBatchBufferLockWait = durationFromAtomicNs(domain.updateBatchBufferLockWaitNs.Load()) @@ -1430,7 +1804,9 @@ func (domain *collectionWriteDomain) statsSnapshot() CollectionManagerStats { stats.UpdateBatchBufferPrimaryIdx = durationFromAtomicNs(domain.updateBatchBufferPrimaryIdxNs.Load()) stats.UpdateBatchBufferUniqueIdx = durationFromAtomicNs(domain.updateBatchBufferUniqueIdxNs.Load()) stats.UpdateBatchBufferRootAppend = durationFromAtomicNs(domain.updateBatchBufferRootAppendNs.Load()) + stats.UpdateBatchBufferSemanticAppend = durationFromAtomicNs(domain.updateBatchBufferSemanticAppendNs.Load()) stats.UpdateBatchBufferFlush = durationFromAtomicNs(domain.updateBatchBufferFlushNs.Load()) + stats.UpdateBatchPlanClose = durationFromAtomicNs(domain.updateBatchPlanCloseNs.Load()) stats.UpdateBatchPublish = durationFromAtomicNs(domain.updateBatchPublishNs.Load()) stats.UpdateBatchSecondaryDeletes = domain.updateBatchSecondaryDeletes.Load() stats.UpdateBatchSecondarySets = domain.updateBatchSecondarySets.Load() @@ -1464,6 +1840,38 @@ func collectionWriteDomainVisibleDepthLocked(domain *collectionWriteDomain) int return depth } +func pendingIndexedSemanticRecordCountLocked(domain *collectionWriteDomain) int { + if domain == nil { + return 0 + } + total := len(domain.indexedSemanticRecords) + for _, unit := range domain.indexedPublishingUnits { + total = saturatingAddNonNegativeInt(total, len(unit.semanticRecords)) + } + for _, unit := range domain.indexedFlushUnits { + total = saturatingAddNonNegativeInt(total, len(unit.semanticRecords)) + } + return total +} + +func indexedSemanticRecordIndexDeltaCount(records []indexedSemanticRecord) int { + total := 0 + for _, record := range records { + total = saturatingAddNonNegativeInt(total, len(record.indexDeltas)) + } + return total +} + +func indexedSemanticRecordFallbackCount(records []indexedSemanticRecord) int { + total := 0 + for _, record := range records { + if record.fallback != indexedSemanticFallbackNone { + total = saturatingAddNonNegativeInt(total, 1) + } + } + return total +} + func collectionStatsUint64ToInt(v uint64) int { if v > uint64(maxCollectionInt) { return maxCollectionInt @@ -1527,15 +1935,23 @@ func (domain *collectionWriteDomain) observeUpdateBatchStats(stats CollectionUpd if stats.BufferedBatches > 0 { domain.updateBatchBufferedBatches.Add(uint64(stats.BufferedBatches)) } + domain.updateBatchValidateNs.Add(durationToAtomicNs(stats.BatchValidate)) + domain.updateBatchCloneNs.Add(durationToAtomicNs(stats.BatchClone)) + domain.updateBatchPlanSetupNs.Add(durationToAtomicNs(stats.PlanSetup)) + domain.updateBatchBufferedReadSnapshotNs.Add(durationToAtomicNs(stats.BufferedReadSnapshot)) domain.updateBatchCurrentReadNs.Add(durationToAtomicNs(stats.CurrentRead)) + domain.updateBatchBSONIDValidationNs.Add(durationToAtomicNs(stats.BSONIDValidation)) domain.updateBatchCallbackNs.Add(durationToAtomicNs(stats.Callback)) + domain.updateBatchReplacementStageNs.Add(durationToAtomicNs(stats.ReplacementStage)) domain.updateBatchPrepareNs.Add(durationToAtomicNs(stats.PrepareDocuments)) domain.updateBatchIndexStateNs.Add(durationToAtomicNs(stats.IndexStateExtraction)) + domain.updateBatchIndexStateCompareNs.Add(durationToAtomicNs(stats.IndexStateCompare)) domain.updateBatchUniquePreflightNs.Add(durationToAtomicNs(stats.UniqueIndexPreflight)) domain.updateBatchTemplateRunNs.Add(durationToAtomicNs(stats.TemplateRunBuild)) domain.updateBatchPrimaryRunNs.Add(durationToAtomicNs(stats.PrimaryRunBuild)) domain.updateBatchIndexStateRunNs.Add(durationToAtomicNs(stats.IndexStateRunBuild)) domain.updateBatchSecondaryRunNs.Add(durationToAtomicNs(stats.SecondaryRunBuild)) + domain.updateBatchSemanticRecordNs.Add(durationToAtomicNs(stats.SemanticRecordBuild)) domain.updateBatchBufferStageNs.Add(durationToAtomicNs(stats.BufferStage)) if collectionUpdateStatsHasBufferStageBreakdown(stats) { domain.updateBatchBufferPrecheckNs.Add(durationToAtomicNs(stats.BufferStagePrecheck)) @@ -1547,8 +1963,10 @@ func (domain *collectionWriteDomain) observeUpdateBatchStats(stats CollectionUpd domain.updateBatchBufferPrimaryIdxNs.Add(durationToAtomicNs(stats.BufferStagePrimaryIdx)) domain.updateBatchBufferUniqueIdxNs.Add(durationToAtomicNs(stats.BufferStageUniqueIdx)) domain.updateBatchBufferRootAppendNs.Add(durationToAtomicNs(stats.BufferStageRootAppend)) + domain.updateBatchBufferSemanticAppendNs.Add(durationToAtomicNs(stats.BufferStageSemanticAppend)) domain.updateBatchBufferFlushNs.Add(durationToAtomicNs(stats.BufferStageFlush)) } + domain.updateBatchPlanCloseNs.Add(durationToAtomicNs(stats.PlanClose)) domain.updateBatchPublishNs.Add(durationToAtomicNs(stats.Publish)) if stats.SecondaryDeleteEntries > 0 { domain.updateBatchSecondaryDeletes.Add(uint64(stats.SecondaryDeleteEntries)) @@ -1637,6 +2055,7 @@ func collectionUpdateStatsHasBufferStageBreakdown(stats CollectionUpdateStats) b stats.BufferStagePrimaryIdx != 0 || stats.BufferStageUniqueIdx != 0 || stats.BufferStageRootAppend != 0 || + stats.BufferStageSemanticAppend != 0 || stats.BufferStageFlush != 0 } @@ -1656,6 +2075,19 @@ func (domain *collectionWriteDomain) observeIndexedStage(docs int, bytes int64, } } +func (domain *collectionWriteDomain) observeIndexedSemanticRawRecords(records []indexedSemanticRecord) { + if domain == nil || len(records) == 0 { + return + } + domain.indexedSemanticRawRecords.Add(uint64(len(records))) + if deltas := indexedSemanticRecordIndexDeltaCount(records); deltas > 0 { + domain.indexedSemanticRawIndexDeltas.Add(uint64(deltas)) + } + if fallbacks := indexedSemanticRecordFallbackCount(records); fallbacks > 0 { + domain.indexedSemanticFallbackRecords.Add(uint64(fallbacks)) + } +} + func (domain *collectionWriteDomain) beginIndexedAsyncFlush() bool { if domain == nil { return false @@ -1708,7 +2140,7 @@ func (domain *collectionWriteDomain) waitIndexedAsyncFlush() { } domain.indexedAsyncMu.Unlock() if !waitStart.IsZero() { - domain.indexedAsyncFlushWaitTotalNs.Add(durationToAtomicNs(time.Since(waitStart))) + domain.indexedAsyncFlushWaitTotalNs.Add(durationToAtomicNs(collectionObservedElapsedSince(waitStart))) } } @@ -1796,6 +2228,24 @@ func (domain *collectionWriteDomain) observeIndexedFlush(units, docs int, bytes domain.indexedFlushPublishTotalNs.Add(durationToAtomicNs(publish)) } +func (domain *collectionWriteDomain) observeIndexedFlushPrepare(preflight, rotate, merge time.Duration) { + if domain == nil { + return + } + domain.indexedFlushPreflightTotalNs.Add(durationToAtomicNs(preflight)) + domain.indexedFlushRotateTotalNs.Add(durationToAtomicNs(rotate)) + domain.indexedFlushMergeTotalNs.Add(durationToAtomicNs(merge)) +} + +func (domain *collectionWriteDomain) observeIndexedFlushMaterializeBreakdown(semanticPlan, buildInputs, planStats time.Duration) { + if domain == nil { + return + } + domain.indexedFlushSemanticPlanTotalNs.Add(durationToAtomicNs(semanticPlan)) + domain.indexedFlushBuildInputsTotalNs.Add(durationToAtomicNs(buildInputs)) + domain.indexedFlushPlanStatsTotalNs.Add(durationToAtomicNs(planStats)) +} + func (domain *collectionWriteDomain) observeIndexedFlushForcedDrain() { if domain == nil { return @@ -1803,19 +2253,30 @@ func (domain *collectionWriteDomain) observeIndexedFlushForcedDrain() { domain.indexedFlushForcedDrains.Add(1) } +type collectionRootDeltaKindStats struct { + entries uint64 + bytes uint64 + tombstones uint64 +} + type collectionRootDeltaPlanStats struct { - primaryRoots uint64 - templateRoots uint64 - indexStateRoots uint64 - secondaryRoots uint64 - entries uint64 - keyBytes uint64 - valueBytes uint64 - tombstones uint64 - primaryEntries uint64 - primaryKeyBytes uint64 - primaryValueBytes uint64 - primaryTombstones uint64 + primaryRoots uint64 + templateRoots uint64 + indexStateRoots uint64 + secondaryRoots uint64 + secondaryUniqueRoots uint64 + entries uint64 + keyBytes uint64 + valueBytes uint64 + tombstones uint64 + primaryEntries uint64 + primaryKeyBytes uint64 + primaryValueBytes uint64 + primaryTombstones uint64 + primaryDetail collectionRootDeltaKindStats + templateDetail collectionRootDeltaKindStats + indexStateDetail collectionRootDeltaKindStats + secondaryDetail collectionRootDeltaKindStats } func (domain *collectionWriteDomain) observeRootDeltaPlan(stats collectionRootDeltaPlanStats) { @@ -1832,6 +2293,133 @@ func (domain *collectionWriteDomain) observeRootDeltaPlan(stats collectionRootDe domain.rootDeltaPlanTombstones.Add(stats.tombstones) } +func (domain *collectionWriteDomain) observeCoalescedFlushBatch(units, docs int, bytes int64, netZero bool) { + if domain == nil { + return + } + domain.coalescedFlushBatches.Add(1) + if units > 0 { + domain.coalescedFlushBatchUnits.Add(uint64(units)) + } + if docs > 0 { + domain.coalescedFlushBatchDocs.Add(uint64(docs)) + } + if bytes > 0 { + domain.coalescedFlushBatchBytes.Add(uint64(bytes)) + } + if netZero { + domain.coalescedFlushNetZeroBatches.Add(1) + } +} + +func (domain *collectionWriteDomain) observeRootDeltaPlanRawUnit(stats collectionRootDeltaPlanStats) { + if domain == nil || stats == (collectionRootDeltaPlanStats{}) { + return + } + domain.rootDeltaPlanRawUnitPrimaryEntries.Add(stats.primaryDetail.entries) + domain.rootDeltaPlanRawUnitPrimaryBytes.Add(stats.primaryDetail.bytes) + domain.rootDeltaPlanRawUnitPrimaryTombstones.Add(stats.primaryDetail.tombstones) + domain.rootDeltaPlanRawUnitTemplateEntries.Add(stats.templateDetail.entries) + domain.rootDeltaPlanRawUnitTemplateBytes.Add(stats.templateDetail.bytes) + domain.rootDeltaPlanRawUnitTemplateTombstones.Add(stats.templateDetail.tombstones) + domain.rootDeltaPlanRawUnitIndexStateEntries.Add(stats.indexStateDetail.entries) + domain.rootDeltaPlanRawUnitIndexStateBytes.Add(stats.indexStateDetail.bytes) + domain.rootDeltaPlanRawUnitIndexStateTombstones.Add(stats.indexStateDetail.tombstones) + domain.rootDeltaPlanRawUnitSecondaryEntries.Add(stats.secondaryDetail.entries) + domain.rootDeltaPlanRawUnitSecondaryBytes.Add(stats.secondaryDetail.bytes) + domain.rootDeltaPlanRawUnitSecondaryTombstones.Add(stats.secondaryDetail.tombstones) +} + +func (domain *collectionWriteDomain) observeRootDeltaPlanFinal(stats collectionRootDeltaPlanStats) { + if domain == nil || stats == (collectionRootDeltaPlanStats{}) { + return + } + domain.rootDeltaPlanFinalPrimaryEntries.Add(stats.primaryDetail.entries) + domain.rootDeltaPlanFinalPrimaryBytes.Add(stats.primaryDetail.bytes) + domain.rootDeltaPlanFinalPrimaryTombstones.Add(stats.primaryDetail.tombstones) + domain.rootDeltaPlanFinalTemplateEntries.Add(stats.templateDetail.entries) + domain.rootDeltaPlanFinalTemplateBytes.Add(stats.templateDetail.bytes) + domain.rootDeltaPlanFinalTemplateTombstones.Add(stats.templateDetail.tombstones) + domain.rootDeltaPlanFinalIndexStateEntries.Add(stats.indexStateDetail.entries) + domain.rootDeltaPlanFinalIndexStateBytes.Add(stats.indexStateDetail.bytes) + domain.rootDeltaPlanFinalIndexStateTombstones.Add(stats.indexStateDetail.tombstones) + domain.rootDeltaPlanFinalSecondaryEntries.Add(stats.secondaryDetail.entries) + domain.rootDeltaPlanFinalSecondaryBytes.Add(stats.secondaryDetail.bytes) + domain.rootDeltaPlanFinalSecondaryTombstones.Add(stats.secondaryDetail.tombstones) +} + +func (domain *collectionWriteDomain) observeRootDeltaPlanCoalescing(rawStats, finalStats collectionRootDeltaPlanStats) { + if domain == nil { + return + } + rawSecondaryRoots := rawStats.secondaryUniqueRoots + if rawSecondaryRoots == 0 && rawStats.secondaryRoots > 0 { + rawSecondaryRoots = rawStats.secondaryRoots + } + finalSecondaryRoots := finalStats.secondaryUniqueRoots + if finalSecondaryRoots == 0 && finalStats.secondaryRoots > 0 { + finalSecondaryRoots = finalStats.secondaryRoots + } + if rawStats.entries > finalStats.entries { + domain.rootDeltaPlanSquashedEntries.Add(rawStats.entries - finalStats.entries) + } + if rawSecondaryRoots > finalSecondaryRoots { + domain.indexedSemanticSkippedSecondaryRoots.Add(rawSecondaryRoots - finalSecondaryRoots) + } + if rawStats.entries > 0 && finalStats.entries == 0 { + domain.rootDeltaPlanNetZeroPlans.Add(1) + } +} + +func coalescedFlushBatchRawRootDeltaStats(batch coalescedFlushBatch) collectionRootDeltaPlanStats { + if batch.rawRootDeltaReady { + return batch.rawRootDeltaStats + } + return batch.rootDeltaStats +} + +func (domain *collectionWriteDomain) observeIndexedSemanticEffectiveRecords(records int) { + if domain == nil || records <= 0 { + return + } + domain.indexedSemanticEffectiveRecords.Add(uint64(records)) +} + +func ensureCoalescedFlushBatchRawRootDeltaStats(collectionName string, batch *coalescedFlushBatch) error { + if batch == nil || batch.rawRootDeltaReady { + return nil + } + if len(batch.units) <= 1 { + batch.rawRootDeltaStats = batch.rootDeltaStats + batch.rawRootDeltaReady = true + return nil + } + stats, err := collectionRootDeltaPlanStatsFromIndexedFlushUnits(collectionName, batch.units) + if err != nil { + return err + } + batch.rawRootDeltaStats = stats + batch.rawRootDeltaReady = true + return nil +} + +func (domain *collectionWriteDomain) observePrimaryOnlyDrain(docs int, bytes int64, uniqueDocs int, duration time.Duration) { + if domain == nil { + return + } + domain.primaryOnlyDrainCalls.Add(1) + if docs > 0 { + domain.primaryOnlyDrainDocs.Add(uint64(docs)) + } + if bytes > 0 { + domain.primaryOnlyDrainBytes.Add(uint64(bytes)) + } + if uniqueDocs >= 0 && docs > uniqueDocs { + domain.primaryOnlyDuplicateIDsCoalesced.Add(uint64(docs - uniqueDocs)) + } + domain.primaryOnlyDrainDurationTotalNs.Add(durationToAtomicNs(duration)) +} + func (domain *collectionWriteDomain) observePrimaryOnlyUpdate(matched, modified, published bool, deltaStats collectionRootDeltaPlanStats) { items := 1 matchedCount := 0 @@ -2958,7 +3546,21 @@ func (c *Collection) flushBufferedNoIndexLocked(domain *collectionWriteDomain) e baseCommitSeq := snapshotCommitSeq(pin) baseRootIDs := map[string]uint64{rootName: baseRoot} table := domain.table + drainStart := time.Now() + drainDocs := domain.count + drainBytes := domain.bufferedBytes + if drainBytes == 0 && table != nil { + drainBytes = table.Size() + } + drainUniqueDocs := -1 + if table != nil { + drainUniqueDocs = table.Len() + } + var deltaStats collectionRootDeltaPlanStats iter := table.NewIterator(nil, nil) + if c.writeDomain != nil { + iter = newCollectionRootDeltaStatsIterator(meta.Name, rootName, iter, &deltaStats) + } newSystemRoot, rootIDs, err := c.db.PublishOrderedRootDeltaGroupWithSystemDeltaBuilder([]backenddb.OrderedRootDeltaPublishInput{{ BaseRoot: baseRoot, @@ -2981,6 +3583,9 @@ func (c *Collection) flushBufferedNoIndexLocked(domain *collectionWriteDomain) e domain.baseCommitSeq = c.commitSeqForSystemRoot(newSystemRoot) domain.baseSystemRoot = newSystemRoot domain.primaryRoot = rootIDs[0] + domain.observePrimaryOnlyDrain(drainDocs, drainBytes, drainUniqueDocs, collectionObservedElapsedSince(drainStart)) + domain.observeRootDeltaPlanFinal(deltaStats) + domain.observeRootDeltaPlan(deltaStats) domain.table = newCollectionRunTable(0) domain.count = 0 domain.mutableCount = 0 @@ -3003,6 +3608,11 @@ func (c *Collection) shouldBufferIndexedInsertBatch(meta CollectionMeta, documen isDefaultIndexedWriteMemtableMaxDocuments(meta.Options) { return false } + if len(meta.Indexes) <= 2 && + documentCount >= DefaultIndexedWriteMemtableLowFanoutDirectBatchDocuments && + isDefaultIndexedWriteMemtableMaxDocuments(meta.Options) { + return false + } return true } @@ -3019,11 +3629,12 @@ func (c *Collection) bufferIndexedInsertPlanLocked(catalog *collectionCatalog, b if domain == nil { return 0, errors.New("collections: missing write domain") } - domain.mu.Lock() - defer domain.mu.Unlock() if catalog == nil { return 0, errCollectionNotFound } + rootDeltaStats := plan.stats.rootDeltaStats + domain.mu.Lock() + defer domain.mu.Unlock() if len(catalog.meta.Indexes) == 0 { return 0, errors.New("collections: indexed write buffer requires an indexed schema") } @@ -3145,6 +3756,7 @@ func (c *Collection) bufferIndexedInsertPlanLocked(catalog *collectionCatalog, b domain.mutableBytes = saturatingAddNonNegativeInt64(domain.mutableBytes, stagedBytes) domain.writeGeneration++ domain.observeIndexedStage(len(plan.resultIDs), stagedBytes, stagedRootRuns) + domain.rootDeltaStats.add(rootDeltaStats) c.meta = catalog.meta compactedObsolete, err := maybeCompactBufferedIndexedMutableRunsLocked(domain, catalog.meta.Options) if err != nil { @@ -3180,7 +3792,9 @@ func (c *Collection) initializeWriteDomainFromCatalogLocked(domain *collectionWr domain.rootPolicies = nil domain.rootBaseIDs = nil domain.rootValueArenas = nil + domain.indexedSemanticRecords = nil domain.rootRunCount = 0 + domain.rootDeltaStats = collectionRootDeltaPlanStats{} domain.mutableCount = 0 domain.mutableBytes = 0 domain.primaryIDIndex = nil @@ -3761,27 +4375,31 @@ func checkpointBufferedIndexedDomain(domain *collectionWriteDomain) bufferedInde return bufferedIndexedCheckpoint{} } return bufferedIndexedCheckpoint{ - loaded: domain.loaded, - meta: domain.meta, - catalog: domain.catalog, - baseCommitSeq: domain.baseCommitSeq, - baseSystemRoot: domain.baseSystemRoot, - primaryRoot: domain.primaryRoot, - count: domain.count, - bufferedBytes: domain.bufferedBytes, - mutableCount: domain.mutableCount, - mutableBytes: domain.mutableBytes, - writeGeneration: domain.writeGeneration, - rootRuns: cloneTableRunMap(domain.rootRuns), - rootMutableRuns: cloneMutableRunMap(domain.rootMutableRuns), - rootPolicies: cloneRootPolicyMap(domain.rootPolicies), - rootBaseIDs: cloneUint64Map(domain.rootBaseIDs), - rootValueArenas: cloneArenaRefs(domain.rootValueArenas), + loaded: domain.loaded, + meta: domain.meta, + catalog: domain.catalog, + baseCommitSeq: domain.baseCommitSeq, + baseSystemRoot: domain.baseSystemRoot, + primaryRoot: domain.primaryRoot, + count: domain.count, + bufferedBytes: domain.bufferedBytes, + mutableCount: domain.mutableCount, + mutableBytes: domain.mutableBytes, + writeGeneration: domain.writeGeneration, + rootRuns: cloneTableRunMap(domain.rootRuns), + rootMutableRuns: cloneMutableRunMap(domain.rootMutableRuns), + rootPolicies: cloneRootPolicyMap(domain.rootPolicies), + rootBaseIDs: cloneUint64Map(domain.rootBaseIDs), + rootValueArenas: cloneArenaRefs(domain.rootValueArenas), + // Semantic records are immutable after staging. Rollback only needs to + // restore the previous mutable slice view, not deep-clone every record. + indexedSemanticRecords: domain.indexedSemanticRecords, indexedPublishingUnits: cloneIndexedFlushUnits(domain.indexedPublishingUnits), indexedFlushUnits: cloneIndexedFlushUnits(domain.indexedFlushUnits), primaryRunIndexActive: domain.primaryRunIndex != nil, uniqueValueRuns: cloneTableRunMap(domain.uniqueValueRuns), rootRunCount: domain.rootRunCount, + rootDeltaStats: domain.rootDeltaStats, } } @@ -3811,7 +4429,12 @@ func rollbackBufferedIndexedDomain(domain *collectionWriteDomain, checkpoint buf domain.rootPolicies = checkpoint.rootPolicies domain.rootBaseIDs = checkpoint.rootBaseIDs domain.rootValueArenas = checkpoint.rootValueArenas + if len(domain.indexedSemanticRecords) > len(checkpoint.indexedSemanticRecords) { + clear(domain.indexedSemanticRecords[len(checkpoint.indexedSemanticRecords):]) + } + domain.indexedSemanticRecords = checkpoint.indexedSemanticRecords domain.rootRunCount = checkpoint.rootRunCount + domain.rootDeltaStats = checkpoint.rootDeltaStats pendingRuns := indexedFlushUnitPendingRootRunMap(indexedFlushUnitsWithPublishing(checkpoint.indexedPublishingUnits, checkpoint.indexedFlushUnits), checkpoint.rootRuns) domain.primaryIDIndex = rebuildBufferedPrimaryIDIndex(checkpoint.meta.Name, pendingRuns) if checkpoint.primaryRunIndexActive { @@ -3840,15 +4463,62 @@ func cloneIndexedFlushUnits(in []indexedFlushUnit) []indexedFlushUnit { rootPolicies: cloneRootPolicyMap(unit.rootPolicies), rootBaseIDs: cloneUint64Map(unit.rootBaseIDs), uniqueValueRuns: cloneTableRunMap(unit.uniqueValueRuns), + semanticRecords: cloneIndexedSemanticRecords(unit.semanticRecords), arenaRefs: cloneArenaRefs(unit.arenaRefs), docCount: unit.docCount, byteCount: unit.byteCount, rootRunCount: unit.rootRunCount, + rootDeltaStats: unit.rootDeltaStats, + } + } + return out +} + +func cloneIndexedSemanticRecords(in []indexedSemanticRecord) []indexedSemanticRecord { + if len(in) == 0 { + return nil + } + out := make([]indexedSemanticRecord, len(in)) + for i, record := range in { + out[i] = indexedSemanticRecord{ + kind: record.kind, + documentID: bytes.Clone(record.documentID), + fallback: record.fallback, + indexDeltas: cloneIndexedSemanticIndexDeltas(record.indexDeltas), } } return out } +func cloneIndexedSemanticIndexDeltas(in []indexedSemanticIndexDelta) []indexedSemanticIndexDelta { + if len(in) == 0 { + return nil + } + out := make([]indexedSemanticIndexDelta, len(in)) + for i, delta := range in { + out[i] = indexedSemanticIndexDelta{ + indexName: delta.indexName, + rootName: delta.rootName, + runtimeIdx: delta.runtimeIdx, + unique: delta.unique, + oldValues: cloneIndexedSemanticValueSet(delta.oldValues), + newValues: cloneIndexedSemanticValueSet(delta.newValues), + } + } + return out +} + +func cloneIndexedSemanticValueSet(in [][]byte) [][]byte { + if len(in) == 0 { + return nil + } + out := make([][]byte, len(in)) + for i, value := range in { + out[i] = bytes.Clone(value) + } + return out +} + func cloneArenaRefs(in [][]byte) [][]byte { if len(in) == 0 { return nil @@ -4220,50 +4890,22 @@ func newBufferedPrimaryRunIndex(capacity int) *bufferedPrimaryRunIndex { if capacity < 0 { capacity = 0 } - return &bufferedPrimaryRunIndex{values: make(map[uint64]bufferedPrimaryRunRef, capacity)} + return &bufferedPrimaryRunIndex{values: make(map[string]memtable.Table, capacity)} } -func (index *bufferedPrimaryRunIndex) addRef(hash uint64, ref bufferedPrimaryRunRef) { - if index == nil || ref.table == nil { - return - } - if existing, ok := index.values[hash]; !ok { - index.values[hash] = ref - return - } else if bytes.Equal(existing.key, ref.key) { - index.values[hash] = ref +func (index *bufferedPrimaryRunIndex) addRef(ref bufferedPrimaryRunRef) { + if index == nil || ref.table == nil || len(ref.key) == 0 { return } - collisions := index.collisions - if collisions == nil { - collisions = make(map[uint64][]bufferedPrimaryRunRef) - index.collisions = collisions - } - bucket := collisions[hash] - for i := len(bucket) - 1; i >= 0; i-- { - if bytes.Equal(bucket[i].key, ref.key) { - bucket[i] = ref - collisions[hash] = bucket - return - } - } - collisions[hash] = append(bucket, ref) + index.values[unsafe.String(&ref.key[0], len(ref.key))] = ref.table } func (index *bufferedPrimaryRunIndex) lookup(key []byte) (memtable.Table, bool) { - if index == nil { + if index == nil || len(key) == 0 { return nil, false } - hash := xxhash.Sum64(key) - if ref, ok := index.values[hash]; ok && bytes.Equal(ref.key, key) { - return ref.table, true - } - for _, ref := range index.collisions[hash] { - if bytes.Equal(ref.key, key) { - return ref.table, true - } - } - return nil, false + table, ok := index.values[unsafe.String(&key[0], len(key))] + return table, ok } func addBufferedPrimaryRunIndexEntries(index *bufferedPrimaryRunIndex, batchPrimary memtable.Table) error { @@ -4279,29 +4921,27 @@ func addBufferedPrimaryRunIndexEntries(index *bufferedPrimaryRunIndex, batchPrim if stableKeys { for it.Valid() { key := it.UnsafeKey() - index.addRef(xxhash.Sum64(key), bufferedPrimaryRunRef{key: key, table: batchPrimary}) + index.addRef(bufferedPrimaryRunRef{key: key, table: batchPrimary}) it.Next() } return it.Error() } arena := make([]byte, 0, bufferedPrimaryIDArenaCap(batchPrimary.Len())) refs := make([]bufferedPrimaryRunRef, 0, batchPrimary.Len()) - hashes := make([]uint64, 0, batchPrimary.Len()) for it.Valid() { key := it.UnsafeKey() refKey := key start := len(arena) arena = append(arena, key...) refKey = arena[start:len(arena)] - hashes = append(hashes, xxhash.Sum64(key)) refs = append(refs, bufferedPrimaryRunRef{key: refKey, table: batchPrimary}) it.Next() } if err := it.Error(); err != nil { return err } - for i, ref := range refs { - index.addRef(hashes[i], ref) + for _, ref := range refs { + index.addRef(ref) } if len(arena) > 0 { index.arenas = append(index.arenas, arena) @@ -4309,22 +4949,19 @@ func addBufferedPrimaryRunIndexEntries(index *bufferedPrimaryRunIndex, batchPrim return nil } -func addBufferedPrimaryRunIndexKeys(index *bufferedPrimaryRunIndex, keys [][]byte, table memtable.Table) { - if index == nil || table == nil || len(keys) == 0 { +func addBufferedPrimaryRunIndexDirectEntries(index *bufferedPrimaryRunIndex, entries []directBufferedRootEntry, table memtable.Table) { + if index == nil || table == nil || len(entries) == 0 { return } - arena := make([]byte, 0, bufferedPrimaryIDArenaCap(len(keys))) - for _, key := range keys { + for _, entry := range entries { + key := entry.key if len(key) == 0 { continue } - start := len(arena) - arena = append(arena, key...) - refKey := arena[start:len(arena)] - index.addRef(xxhash.Sum64(key), bufferedPrimaryRunRef{key: refKey, table: table}) - } - if len(arena) > 0 { - index.arenas = append(index.arenas, arena) + // Direct buffered primary entries own cloned document IDs and the table + // stores the same stable key slices, so the lookup index can reference + // them without an additional key arena. + index.addRef(bufferedPrimaryRunRef{key: key, table: table}) } } @@ -4922,6 +5559,9 @@ func (c *Collection) prepareIndexedAsyncPublishLocked(domain *collectionWriteDom if c == nil || c.db == nil || domain == nil || domain.count == 0 || !hasBufferedIndexedRootRuns(domain) { return nil, nil } + if len(domain.indexedPublishingUnits) != 0 { + return nil, errors.New("collections: indexed async publish still in flight") + } if len(domain.indexedFlushUnits) == 0 && len(domain.rootRuns) == 0 { return nil, nil } @@ -4974,44 +5614,34 @@ func (c *Collection) prepareIndexedAsyncPublishLocked(domain *collectionWriteDom rotateIndexedMutableToFlushUnitLocked(domain) units := domain.indexedFlushUnits - flushUnit := mergedIndexedFlushUnits(units) - rootNames := orderedBufferedRootNames(meta, flushUnit.rootRuns) - if len(rootNames) == 0 { + batch, err := buildCoalescedFlushBatchFromUnits(meta, catalog, units) + if err != nil { + return nil, err + } + if len(batch.rootNames) == 0 { _ = pin.Close() work.pin = nil + rawRootDeltaStats := coalescedFlushBatchRawRootDeltaStats(batch) + domain.observeCoalescedFlushBatch(len(batch.units), batch.docCount, batch.byteCount, true) + domain.observeRootDeltaPlanRawUnit(rawRootDeltaStats) + domain.observeRootDeltaPlanCoalescing(rawRootDeltaStats, collectionRootDeltaPlanStats{}) domain.indexedFlushUnits = nil domain.rootMutableRuns = nil domain.rootValueArenas = nil + domain.indexedSemanticRecords = nil + domain.rootDeltaStats = collectionRootDeltaPlanStats{} domain.count = 0 domain.bufferedBytes = 0 domain.mutableCount = 0 domain.mutableBytes = 0 return nil, nil } - rootBaseIDs := make(map[string]uint64, len(rootNames)) - rootOverlays := make(map[string][]uint64, len(rootNames)) - for _, rootName := range rootNames { - baseRoot, ok := flushUnit.rootBaseIDs[rootName] - if !ok { - err = fmt.Errorf("collections: buffered indexed flush missing base root for %q", rootName) - return nil, err - } - rootBaseIDs[rootName] = baseRoot - rootOverlays[rootName] = append([]uint64(nil), catalog.overlayRootIDs(rootName)...) - } + batch.state = coalescedFlushBatchActive work.baseSystemRoot = snapshotSystemRoot(pin) work.baseCommitSeq = snapshotCommitSeq(pin) - work.units = units - work.flushUnit = flushUnit - work.rootNames = rootNames - work.rootBaseIDs = rootBaseIDs - work.rootOverlays = rootOverlays - work.docCount = flushUnit.docCount - work.byteCount = flushUnit.byteCount - work.rootRunCount = indexedFlushUnitRootRunCount(flushUnit) - work.rootCount = len(rootNames) - - domain.indexedPublishingUnits = append(domain.indexedPublishingUnits, units...) + work.batch = batch + + domain.indexedPublishingUnits = append([]indexedFlushUnit(nil), units...) domain.indexedFlushUnits = nil domain.writeGeneration++ return work, nil @@ -5029,27 +5659,37 @@ func (c *Collection) publishPreparedIndexedFlush(work *indexedFlushPublishWork) }() if collectionMetaUsesIndexedOverlayRoots(work.meta) { materializeStart := time.Now() - rootOverlayFilters, err := buildCollectionRootOverlayFilters(work.rootNames, work.flushUnit.rootRuns, work.rootOverlays, work.catalog.rootOverlayFilters) + work.batch.state = coalescedFlushBatchMaterializing + rootOverlayFilters, err := buildCollectionRootOverlayFilters(work.batch.rootNames, work.batch.mergedUnit.rootRuns, work.batch.rootOverlays, work.catalog.rootOverlayFilters) if err != nil { materializeElapsed := collectionObservedElapsedSince(materializeStart) return c.completePreparedIndexedFlush(work, 0, nil, err, materializeElapsed, materializeElapsed, 0) } - work.rootOverlayFilters = rootOverlayFilters - ordered, cleanupDeltas, err := buildBufferedRootOverlayDeltaBatchPublishInputs(work.rootNames, work.flushUnit.rootRuns, work.flushUnit.rootPolicies, work.rootOverlays) + work.batch.rootOverlayFilters = rootOverlayFilters + ordered, cleanupDeltas, err := buildBufferedRootOverlayDeltaBatchPublishInputs(work.batch.rootNames, work.batch.mergedUnit.rootRuns, work.batch.mergedUnit.rootPolicies, work.batch.rootOverlays, work.meta.Options.BufferedIndexedReadOnlyPrepare, work.meta.Options.BufferedIndexedReadOnlyPrepareWorkerCount) if err != nil { materializeElapsed := collectionObservedElapsedSince(materializeStart) return c.completePreparedIndexedFlush(work, 0, nil, err, materializeElapsed, materializeElapsed, 0) } - work.rootDeltaStats = collectionRootDeltaPlanStatsFromOrdered(work.meta.Name, work.rootNames, ordered) + work.batch.rootDeltaStats = collectionRootDeltaPlanStatsFromOrdered(work.meta.Name, work.batch.rootNames, ordered) + if err := ensureCoalescedFlushBatchRawRootDeltaStats(work.meta.Name, &work.batch); err != nil { + cleanupDeltas() + materializeElapsed := collectionObservedElapsedSince(materializeStart) + return c.completePreparedIndexedFlush(work, 0, nil, err, materializeElapsed, materializeElapsed, 0) + } materializeElapsed := collectionObservedElapsedSince(materializeStart) publishStart := time.Now() - newSystemRoot, rootIDs, publishErr := c.db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder(ordered, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { - return c.buildRootOverlayDescriptorSystemDeltaIteratorForMeta(work.meta, work.baseCommitSeq, work.baseSystemRoot, work.rootNames, work.rootBaseIDs, work.rootOverlays, rootIDs) + work.batch.state = coalescedFlushBatchPublishing + preflight := func() error { + return c.validateRootOverlayDescriptorSystemDeltaForMeta(work.meta, work.baseCommitSeq, work.baseSystemRoot, work.batch.rootNames, work.batch.rootBaseIDs, work.batch.rootOverlays) + } + newSystemRoot, rootIDs, publishErr := c.db.PublishOrderedRootDeltaBatchGroupWithPreflightAndSystemDeltaBuilder(ordered, preflight, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return c.buildRootOverlayDescriptorSystemDeltaIteratorForMeta(work.meta, work.baseCommitSeq, work.baseSystemRoot, work.batch.rootNames, work.batch.rootBaseIDs, work.batch.rootOverlays, rootIDs) }) publishElapsed := collectionObservedElapsedSince(publishStart) cleanupDeltas() - if publishErr == nil && len(rootIDs) != len(work.rootNames) { - publishErr = unexpectedOrderedRootCountError(work.meta.Name, len(work.rootNames), len(rootIDs)) + if publishErr == nil && len(rootIDs) != len(work.batch.rootNames) { + publishErr = unexpectedOrderedRootCountError(work.meta.Name, len(work.batch.rootNames), len(rootIDs)) } completeErr := c.completePreparedIndexedFlush(work, newSystemRoot, rootIDs, publishErr, materializeElapsed+publishElapsed, materializeElapsed, publishElapsed) if completeErr != nil { @@ -5058,21 +5698,77 @@ func (c *Collection) publishPreparedIndexedFlush(work *indexedFlushPublishWork) return publishErr } materializeStart := time.Now() - ordered, cleanupDeltas, err := buildBufferedRootDeltaBatchPublishInputs(work.rootNames, work.flushUnit.rootRuns, work.rootBaseIDs, work.flushUnit.rootPolicies) + work.batch.state = coalescedFlushBatchMaterializing + preflightRootNames := append([]string(nil), work.batch.rootNames...) + preflightRootBaseIDs := cloneUint64Map(work.batch.rootBaseIDs) + semanticPlanStart := time.Now() + view, err := buildIndexedSemanticPublishView(work.meta, work.batch.mergedUnit, work.batch.rootNames, work.batch.rootBaseIDs) + semanticPlanElapsed := collectionObservedElapsedSince(semanticPlanStart) + if err != nil { + materializeElapsed := collectionObservedElapsedSince(materializeStart) + if c.writeDomain != nil { + c.writeDomain.observeIndexedFlushMaterializeBreakdown(semanticPlanElapsed, 0, 0) + } + return c.completePreparedIndexedFlush(work, 0, nil, err, materializeElapsed, materializeElapsed, 0) + } + defer resetIndexedSemanticPublishView(view) + work.batch.rootNames = view.rootNames + work.batch.rootBaseIDs = view.rootBaseIDs + work.batch.rootCount = len(view.rootNames) + work.batch.effectiveRecords = view.effectiveRecords + buildInputsStart := time.Now() + ordered, cleanupDeltas, err := buildBufferedRootDeltaBatchPublishInputs(view.rootNames, view.rootRuns, view.rootBaseIDs, view.rootPolicies, work.meta.Options.BufferedIndexedReadOnlyPrepare, work.meta.Options.BufferedIndexedReadOnlyPrepareWorkerCount) + buildInputsElapsed := collectionObservedElapsedSince(buildInputsStart) if err != nil { materializeElapsed := collectionObservedElapsedSince(materializeStart) + if c.writeDomain != nil { + c.writeDomain.observeIndexedFlushMaterializeBreakdown(semanticPlanElapsed, buildInputsElapsed, 0) + } return c.completePreparedIndexedFlush(work, 0, nil, err, materializeElapsed, materializeElapsed, 0) } - work.rootDeltaStats = collectionRootDeltaPlanStatsFromOrdered(work.meta.Name, work.rootNames, ordered) + planStatsStart := time.Now() + work.batch.rootDeltaStats = collectionRootDeltaPlanStatsFromOrdered(work.meta.Name, view.rootNames, ordered) + if !work.batch.rawRootDeltaReady { + if view.semanticApplied { + rawStats, err := collectionRootDeltaPlanStatsFromIndexedFlushUnits(work.meta.Name, work.batch.units) + if err != nil { + cleanupDeltas() + materializeElapsed := collectionObservedElapsedSince(materializeStart) + if c.writeDomain != nil { + c.writeDomain.observeIndexedFlushMaterializeBreakdown(semanticPlanElapsed, buildInputsElapsed, collectionObservedElapsedSince(planStatsStart)) + } + return c.completePreparedIndexedFlush(work, 0, nil, err, materializeElapsed, materializeElapsed, 0) + } + work.batch.rawRootDeltaStats = rawStats + work.batch.rawRootDeltaReady = true + } else { + if err := ensureCoalescedFlushBatchRawRootDeltaStats(work.meta.Name, &work.batch); err != nil { + cleanupDeltas() + materializeElapsed := collectionObservedElapsedSince(materializeStart) + if c.writeDomain != nil { + c.writeDomain.observeIndexedFlushMaterializeBreakdown(semanticPlanElapsed, buildInputsElapsed, collectionObservedElapsedSince(planStatsStart)) + } + return c.completePreparedIndexedFlush(work, 0, nil, err, materializeElapsed, materializeElapsed, 0) + } + } + } + planStatsElapsed := collectionObservedElapsedSince(planStatsStart) + if c.writeDomain != nil { + c.writeDomain.observeIndexedFlushMaterializeBreakdown(semanticPlanElapsed, buildInputsElapsed, planStatsElapsed) + } materializeElapsed := collectionObservedElapsedSince(materializeStart) publishStart := time.Now() - newSystemRoot, rootIDs, publishErr := c.db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder(ordered, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { - return c.buildRootDescriptorSystemDeltaIteratorForMeta(work.meta, work.baseCommitSeq, work.baseSystemRoot, work.rootNames, work.rootBaseIDs, rootIDs) + work.batch.state = coalescedFlushBatchPublishing + preflight := func() error { + return c.validateRootDescriptorSystemDeltaForMeta(work.meta, work.baseCommitSeq, work.baseSystemRoot, preflightRootNames, preflightRootBaseIDs) + } + newSystemRoot, rootIDs, publishErr := c.db.PublishOrderedRootDeltaBatchGroupWithPreflightAndSystemDeltaBuilder(ordered, preflight, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return c.buildRootDescriptorSystemDeltaIteratorForMeta(work.meta, work.baseCommitSeq, work.baseSystemRoot, view.rootNames, view.rootBaseIDs, rootIDs) }) publishElapsed := collectionObservedElapsedSince(publishStart) cleanupDeltas() - if publishErr == nil && len(rootIDs) != len(work.rootNames) { - publishErr = unexpectedOrderedRootCountError(work.meta.Name, len(work.rootNames), len(rootIDs)) + if publishErr == nil && len(rootIDs) != len(view.rootNames) { + publishErr = unexpectedOrderedRootCountError(work.meta.Name, len(view.rootNames), len(rootIDs)) } completeErr := c.completePreparedIndexedFlush(work, newSystemRoot, rootIDs, publishErr, materializeElapsed+publishElapsed, materializeElapsed, publishElapsed) if completeErr != nil { @@ -5081,15 +5777,17 @@ func (c *Collection) publishPreparedIndexedFlush(work *indexedFlushPublishWork) return publishErr } -func buildBufferedRootOverlayDeltaBatchPublishInputs(rootNames []string, rootRuns map[string][]memtable.Table, rootPolicies map[string]backenddb.OrderedRootStoragePolicy, rootOverlays map[string][]uint64) ([]backenddb.OrderedRootDeltaBatchPublishInput, func(), error) { +func buildBufferedRootOverlayDeltaBatchPublishInputs(rootNames []string, rootRuns map[string][]memtable.Table, rootPolicies map[string]backenddb.OrderedRootStoragePolicy, rootOverlays map[string][]uint64, prepareReadOnly bool, readOnlyPrepareWorkerCount int) ([]backenddb.OrderedRootDeltaBatchPublishInput, func(), error) { specs := make([]bufferedRootDeltaBatchSpec, len(rootNames)) for i, rootName := range rootNames { specs[i] = bufferedRootDeltaBatchSpec{ - rootName: rootName, - baseRoot: overlayDeltaBaseRoot(rootOverlays[rootName]), - storagePolicy: rootPolicies[rootName], - includeDeletedOnColdBuild: true, - parallelApply: true, + rootName: rootName, + baseRoot: overlayDeltaBaseRoot(rootOverlays[rootName]), + storagePolicy: rootPolicies[rootName], + includeDeletedOnColdBuild: true, + parallelApply: true, + prepareReadOnly: prepareReadOnly, + readOnlyPrepareWorkerCount: readOnlyPrepareWorkerCount, } } return buildBufferedRootDeltaBatchPublishInputsFromSpecs(specs, rootRuns) @@ -5102,7 +5800,7 @@ func overlayDeltaBaseRoot(overlays []uint64) uint64 { return 0 } -func buildBufferedRootDeltaBatchPublishInputs(rootNames []string, rootRuns map[string][]memtable.Table, rootBaseIDs map[string]uint64, rootPolicies map[string]backenddb.OrderedRootStoragePolicy) ([]backenddb.OrderedRootDeltaBatchPublishInput, func(), error) { +func buildBufferedRootDeltaBatchPublishInputs(rootNames []string, rootRuns map[string][]memtable.Table, rootBaseIDs map[string]uint64, rootPolicies map[string]backenddb.OrderedRootStoragePolicy, prepareReadOnly bool, readOnlyPrepareWorkerCount int) ([]backenddb.OrderedRootDeltaBatchPublishInput, func(), error) { specs := make([]bufferedRootDeltaBatchSpec, 0, len(rootNames)) for _, rootName := range rootNames { baseRoot, ok := rootBaseIDs[rootName] @@ -5110,26 +5808,31 @@ func buildBufferedRootDeltaBatchPublishInputs(rootNames []string, rootRuns map[s return nil, func() {}, fmt.Errorf("collections: buffered indexed flush missing base root for %q", rootName) } specs = append(specs, bufferedRootDeltaBatchSpec{ - rootName: rootName, - baseRoot: baseRoot, - storagePolicy: rootPolicies[rootName], - parallelApply: true, + rootName: rootName, + baseRoot: baseRoot, + storagePolicy: rootPolicies[rootName], + parallelApply: true, + prepareReadOnly: prepareReadOnly, + readOnlyPrepareWorkerCount: readOnlyPrepareWorkerCount, }) } return buildBufferedRootDeltaBatchPublishInputsFromSpecs(specs, rootRuns) } type bufferedRootDeltaBatchSpec struct { - rootName string - baseRoot uint64 - storagePolicy backenddb.OrderedRootStoragePolicy - includeDeletedOnColdBuild bool - parallelApply bool + rootName string + baseRoot uint64 + storagePolicy backenddb.OrderedRootStoragePolicy + includeDeletedOnColdBuild bool + parallelApply bool + prepareReadOnly bool + readOnlyPrepareWorkerCount int } func buildBufferedRootDeltaBatchPublishInputsFromSpecs(specs []bufferedRootDeltaBatchSpec, rootRuns map[string][]memtable.Table) ([]backenddb.OrderedRootDeltaBatchPublishInput, func(), error) { ordered := make([]backenddb.OrderedRootDeltaBatchPublishInput, len(specs)) iterators := make([]iterator.UnsafeIterator, len(specs)) + readOnlyPrepareResults := bufferedRootDeltaReadOnlyPrepareResults(specs) cleanup := func() { for idx := range ordered { if ordered[idx].Delta != nil { @@ -5150,6 +5853,7 @@ func buildBufferedRootDeltaBatchPublishInputsFromSpecs(specs []bufferedRootDelta return nil, func() {}, err } } + attachBufferedRootDeltaReadOnlyPrepareResults(specs, ordered, readOnlyPrepareResults) return ordered, cleanup, nil } @@ -5167,6 +5871,7 @@ func buildBufferedRootDeltaBatchPublishInputsFromSpecs(specs []bufferedRootDelta return nil, func() {}, err } } + attachBufferedRootDeltaReadOnlyPrepareResults(specs, ordered, readOnlyPrepareResults) return ordered, cleanup, nil } errs := make([]error, len(specs)) @@ -5192,9 +5897,30 @@ func buildBufferedRootDeltaBatchPublishInputsFromSpecs(specs []bufferedRootDelta return nil, func() {}, err } } + attachBufferedRootDeltaReadOnlyPrepareResults(specs, ordered, readOnlyPrepareResults) return ordered, cleanup, nil } +func bufferedRootDeltaReadOnlyPrepareResults(specs []bufferedRootDeltaBatchSpec) []zipper.ReadOnlyPrepareResult { + for i := range specs { + if specs[i].prepareReadOnly { + return make([]zipper.ReadOnlyPrepareResult, len(specs)) + } + } + return nil +} + +func attachBufferedRootDeltaReadOnlyPrepareResults(specs []bufferedRootDeltaBatchSpec, ordered []backenddb.OrderedRootDeltaBatchPublishInput, results []zipper.ReadOnlyPrepareResult) { + if len(results) == 0 { + return + } + for i := range specs { + if specs[i].prepareReadOnly { + ordered[i].ReadOnlyPrepareResult = &results[i] + } + } +} + func buildBufferedRootDeltaBatchPublishInput(spec bufferedRootDeltaBatchSpec, rootRuns map[string][]memtable.Table, ordered *backenddb.OrderedRootDeltaBatchPublishInput, iterOut *iterator.UnsafeIterator) error { iter := newBufferedRootRunsIteratorWithDeleted(rootRuns[spec.rootName], nil, nil, true) if iterOut != nil { @@ -5205,11 +5931,13 @@ func buildBufferedRootDeltaBatchPublishInput(spec bufferedRootDeltaBatchSpec, ro return err } *ordered = backenddb.OrderedRootDeltaBatchPublishInput{ - BaseRoot: spec.baseRoot, - Delta: delta, - StoragePolicy: spec.storagePolicy, - IncludeDeletedOnColdBuild: spec.includeDeletedOnColdBuild, - ParallelApply: spec.parallelApply, + BaseRoot: spec.baseRoot, + Delta: delta, + StoragePolicy: spec.storagePolicy, + IncludeDeletedOnColdBuild: spec.includeDeletedOnColdBuild, + ParallelApply: spec.parallelApply, + PrepareReadOnly: spec.prepareReadOnly, + ReadOnlyPrepareWorkerCount: spec.readOnlyPrepareWorkerCount, } return nil } @@ -5250,77 +5978,727 @@ func buildRootDeltaBatchPublishInputsFromTables(collectionName string, rootNames StoragePolicy: policies[i], }) } - return ordered, cleanup, nil + return ordered, cleanup, nil +} + +func collectionRootDeltaPlanStatsFromOrdered(collectionName string, rootNames []string, ordered []backenddb.OrderedRootDeltaBatchPublishInput) collectionRootDeltaPlanStats { + var stats collectionRootDeltaPlanStats + for i, rootName := range rootNames { + kind := stats.addRoot(collectionName, rootName) + if i < len(ordered) { + stats.addBatch(kind, ordered[i].Delta) + } + } + return stats +} + +func collectionRootDeltaPlanStatsFromIndexedFlushUnits(collectionName string, units []indexedFlushUnit) (collectionRootDeltaPlanStats, error) { + var stats collectionRootDeltaPlanStats + secondaryRootNames := make(map[string]struct{}) + for _, unit := range units { + unitStats, err := collectionRootDeltaPlanStatsFromRootRuns(collectionName, unit.rootRuns) + if err != nil { + return stats, err + } + stats.add(unitStats) + for rootName, runs := range unit.rootRuns { + if len(runs) > 0 && strings.HasPrefix(rootName, collectionName+"/index/") { + secondaryRootNames[rootName] = struct{}{} + } + } + } + stats.secondaryUniqueRoots = uint64(len(secondaryRootNames)) + return stats, nil +} + +func collectionRootDeltaPlanStatsFromRootRuns(collectionName string, rootRuns map[string][]memtable.Table) (collectionRootDeltaPlanStats, error) { + var stats collectionRootDeltaPlanStats + if len(rootRuns) == 0 { + return stats, nil + } + rootNames := make([]string, 0, len(rootRuns)) + for rootName, runs := range rootRuns { + if len(runs) > 0 { + rootNames = append(rootNames, rootName) + } + } + sort.Strings(rootNames) + for _, rootName := range rootNames { + kind := stats.addRoot(collectionName, rootName) + iter := newBufferedRootRunsIteratorWithDeleted(rootRuns[rootName], nil, nil, true) + stats.addIterator(kind, iter) + err := iter.Error() + closeErr := iter.Close() + if err != nil { + return stats, err + } + if closeErr != nil { + return stats, closeErr + } + } + return stats, nil +} + +type indexedSemanticPublishView struct { + rootNames []string + rootRuns map[string][]memtable.Table + rootPolicies map[string]backenddb.OrderedRootStoragePolicy + rootBaseIDs map[string]uint64 + ownedTables []memtable.Table + semanticApplied bool + effectiveRecords int +} + +func buildIndexedSemanticPublishView(meta CollectionMeta, unit indexedFlushUnit, rootNames []string, rootBaseIDs map[string]uint64) (indexedSemanticPublishView, error) { + view := indexedSemanticPublishView{ + rootNames: rootNames, + rootRuns: unit.rootRuns, + rootPolicies: unit.rootPolicies, + rootBaseIDs: rootBaseIDs, + } + if normalizedDocumentFormat(meta.Options.DocumentFormat) == DocumentFormatTemplateV1 || + len(unit.semanticRecords) == 0 || + unit.docCount != len(unit.semanticRecords) || + len(unit.uniqueValueRuns) != 0 || + (len(unit.semanticRecords) > 1 && !indexedSemanticRecordsHaveRepeatedDocumentID(unit.semanticRecords)) { + return view, nil + } + effectiveRuns, effectiveRecords, ok, err := buildIndexedSemanticEffectiveSecondaryRuns(unit.semanticRecords) + if err != nil || !ok || len(effectiveRuns) == 0 { + return view, err + } + + rootRuns := cloneTableRunMap(unit.rootRuns) + rootPolicies := cloneRootPolicyMap(unit.rootPolicies) + baseIDs := cloneUint64Map(rootBaseIDs) + for rootName, table := range effectiveRuns { + if table == nil || table.Len() == 0 { + resetCollectionRunTable(table) + delete(rootRuns, rootName) + delete(rootPolicies, rootName) + continue + } + rootRuns[rootName] = []memtable.Table{table} + view.ownedTables = append(view.ownedTables, table) + } + view.rootRuns = rootRuns + view.rootPolicies = rootPolicies + view.rootBaseIDs = baseIDs + view.rootNames = orderedBufferedRootNames(meta, rootRuns) + view.semanticApplied = true + if len(view.rootNames) == 0 { + resetIndexedSemanticPublishView(view) + return indexedSemanticPublishView{ + rootNames: rootNames, + rootRuns: unit.rootRuns, + rootPolicies: unit.rootPolicies, + rootBaseIDs: rootBaseIDs, + }, nil + } + view.effectiveRecords = effectiveRecords + return view, nil +} + +func resetIndexedSemanticPublishView(view indexedSemanticPublishView) { + for _, table := range view.ownedTables { + resetCollectionRunTable(table) + } +} + +func indexedSemanticRecordsHaveRepeatedDocumentID(records []indexedSemanticRecord) bool { + if len(records) < 2 { + return false + } + seen := make(map[uint64][]byte, len(records)) + for _, record := range records { + hash := xxhash.Sum64(record.documentID) + if prior, ok := seen[hash]; ok { + if bytes.Equal(prior, record.documentID) { + return true + } + // Hash collisions are vanishingly rare and only make us try the + // semantic planner conservatively; the planner still validates chains. + return true + } + seen[hash] = record.documentID + } + return false +} + +type indexedSemanticDocumentRootState struct { + documentID []byte + baseValues [][]byte + finalValues [][]byte +} + +func buildIndexedSemanticEffectiveSecondaryRuns(records []indexedSemanticRecord) (map[string]memtable.Table, int, bool, error) { + rootStates := make(map[string]map[string]*indexedSemanticDocumentRootState) + for _, record := range records { + if record.kind != indexedSemanticRecordUpdate { + return nil, 0, false, nil + } + var documentKey string + if len(record.indexDeltas) > 0 { + documentKey = string(record.documentID) + } + for _, delta := range record.indexDeltas { + if delta.unique { + return nil, 0, false, nil + } + if delta.rootName == "" { + return nil, 0, false, nil + } + states := rootStates[delta.rootName] + if states == nil { + states = make(map[string]*indexedSemanticDocumentRootState) + rootStates[delta.rootName] = states + } + state := states[documentKey] + if state == nil { + // Unit semantic records are immutable during publish planning; keep + // transient references instead of cloning value sets again. + states[documentKey] = &indexedSemanticDocumentRootState{ + documentID: record.documentID, + baseValues: delta.oldValues, + finalValues: delta.newValues, + } + continue + } + if !indexedSemanticValueSetsEqual(state.finalValues, delta.oldValues) { + return nil, 0, false, nil + } + state.finalValues = delta.newValues + } + } + if len(rootStates) == 0 { + return nil, 0, false, nil + } + + rootNames := make([]string, 0, len(rootStates)) + for rootName := range rootStates { + rootNames = append(rootNames, rootName) + } + sort.Strings(rootNames) + + rootTables := make(map[string]memtable.Table, len(rootNames)) + effectiveDocuments := make(map[string]struct{}) + for _, rootName := range rootNames { + states := rootStates[rootName] + table := newCollectionRunTable(0) + documentKeys := make([]string, 0, len(states)) + for documentKey := range states { + documentKeys = append(documentKeys, documentKey) + } + sort.Strings(documentKeys) + for _, documentKey := range documentKeys { + state := states[documentKey] + changed, err := applyIndexedSemanticValueSetDiff(table, state) + if err != nil { + resetCollectionRunTable(table) + for _, existing := range rootTables { + resetCollectionRunTable(existing) + } + return nil, 0, false, err + } + if changed { + effectiveDocuments[documentKey] = struct{}{} + } + } + table.Freeze() + rootTables[rootName] = table + } + return rootTables, len(effectiveDocuments), true, nil +} + +func applyIndexedSemanticValueSetDiff(table memtable.Table, state *indexedSemanticDocumentRootState) (bool, error) { + if state == nil { + return false, nil + } + base, final := state.baseValues, state.finalValues + if len(base) == 0 { + if len(final) == 0 { + return false, nil + } + for _, encoded := range final { + if _, err := setCollectionSecondaryIndexEntry(table, encoded, state.documentID); err != nil { + return false, err + } + } + return true, nil + } + if len(final) == 0 { + for _, encoded := range base { + if _, err := deleteCollectionSecondaryIndexEntry(table, encoded, state.documentID); err != nil { + return false, err + } + } + return true, nil + } + if len(base) == 1 && len(final) == 1 { + if bytes.Equal(base[0], final[0]) { + return false, nil + } + if _, err := deleteCollectionSecondaryIndexEntry(table, base[0], state.documentID); err != nil { + return false, err + } + if _, err := setCollectionSecondaryIndexEntry(table, final[0], state.documentID); err != nil { + return false, err + } + return true, nil + } + deletes, sets := indexedSemanticValueSetDiff(base, final) + if len(deletes) == 0 && len(sets) == 0 { + return false, nil + } + for _, encoded := range deletes { + if _, err := deleteCollectionSecondaryIndexEntry(table, encoded, state.documentID); err != nil { + return false, err + } + } + for _, encoded := range sets { + if _, err := setCollectionSecondaryIndexEntry(table, encoded, state.documentID); err != nil { + return false, err + } + } + return true, nil +} + +func indexedSemanticValueSetsEqual(left, right [][]byte) bool { + if len(left) != len(right) { + return false + } + switch len(left) { + case 0: + return true + case 1: + return bytes.Equal(left[0], right[0]) + } + seen := make(map[string]int, len(left)) + for _, value := range left { + seen[string(value)]++ + } + for _, value := range right { + key := string(value) + if seen[key] == 0 { + return false + } + seen[key]-- + } + return true +} + +func indexedSemanticValueSetDiff(base, final [][]byte) (deletes, sets [][]byte) { + if len(base) == 0 { + if len(final) == 0 { + return nil, nil + } + return nil, cloneIndexedSemanticValueSetRefs(final) + } + if len(final) == 0 { + return cloneIndexedSemanticValueSetRefs(base), nil + } + if len(base) == 1 && len(final) == 1 { + if bytes.Equal(base[0], final[0]) { + return nil, nil + } + return cloneIndexedSemanticValueSetRefs(base), cloneIndexedSemanticValueSetRefs(final) + } + baseCounts := make(map[string]int, len(base)) + for _, value := range base { + baseCounts[string(value)]++ + } + finalCounts := make(map[string]int, len(final)) + for _, value := range final { + finalCounts[string(value)]++ + } + for _, value := range base { + key := string(value) + if finalCounts[key] > 0 { + finalCounts[key]-- + continue + } + deletes = append(deletes, value) + } + for _, value := range final { + key := string(value) + if baseCounts[key] > 0 { + baseCounts[key]-- + continue + } + sets = append(sets, value) + } + return deletes, sets +} + +func cloneIndexedSemanticValueSetRefs(values [][]byte) [][]byte { + if len(values) == 0 { + return nil + } + out := make([][]byte, len(values)) + copy(out, values) + return out +} + +func collectionRootDeltaPlanStatsFromCollectionRootRuns(collectionName string, runs []collectionRootRun) (collectionRootDeltaPlanStats, error) { + var stats collectionRootDeltaPlanStats + for _, run := range runs { + if run.table == nil { + continue + } + kind := stats.addRoot(collectionName, run.name) + iter := run.table.NewIterator(nil, nil) + stats.addIterator(kind, iter) + err := iter.Error() + closeErr := iter.Close() + if err != nil { + return stats, err + } + if closeErr != nil { + return stats, closeErr + } + } + return stats, nil +} + +func collectionRootDeltaPlanStatsFromRootNameTables(collectionName string, rootNames []string, tables []memtable.Table) (collectionRootDeltaPlanStats, error) { + var stats collectionRootDeltaPlanStats + for i, rootName := range rootNames { + if i >= len(tables) || tables[i] == nil || tables[i].Len() == 0 { + continue + } + kind := stats.addRoot(collectionName, rootName) + iter := tables[i].NewIterator(nil, nil) + stats.addIterator(kind, iter) + err := iter.Error() + closeErr := iter.Close() + if err != nil { + return stats, err + } + if closeErr != nil { + return stats, closeErr + } + } + return stats, nil +} + +func collectionRootDeltaPlanStatsFromSystemTargetEntries(collectionName, rootName string, entries []systemTargetEntry) collectionRootDeltaPlanStats { + iter := &systemTargetIterator{entries: entries} + var stats collectionRootDeltaPlanStats + kind := stats.addRoot(collectionName, rootName) + stats.addIterator(kind, iter) + return stats +} + +type collectionRootDeltaPlanKind uint8 + +const ( + collectionRootDeltaPlanUnknown collectionRootDeltaPlanKind = iota + collectionRootDeltaPlanPrimary + collectionRootDeltaPlanTemplate + collectionRootDeltaPlanIndexState + collectionRootDeltaPlanSecondary +) + +func (stats *collectionRootDeltaPlanStats) add(other collectionRootDeltaPlanStats) { + if stats == nil { + return + } + stats.primaryRoots += other.primaryRoots + stats.templateRoots += other.templateRoots + stats.indexStateRoots += other.indexStateRoots + stats.secondaryRoots += other.secondaryRoots + stats.secondaryUniqueRoots += other.secondaryUniqueRoots + stats.entries += other.entries + stats.keyBytes += other.keyBytes + stats.valueBytes += other.valueBytes + stats.tombstones += other.tombstones + stats.primaryEntries += other.primaryEntries + stats.primaryKeyBytes += other.primaryKeyBytes + stats.primaryValueBytes += other.primaryValueBytes + stats.primaryTombstones += other.primaryTombstones + stats.primaryDetail.add(other.primaryDetail) + stats.templateDetail.add(other.templateDetail) + stats.indexStateDetail.add(other.indexStateDetail) + stats.secondaryDetail.add(other.secondaryDetail) +} + +func (stats *collectionRootDeltaKindStats) add(other collectionRootDeltaKindStats) { + if stats == nil { + return + } + stats.entries += other.entries + stats.bytes += other.bytes + stats.tombstones += other.tombstones +} + +func (stats *collectionRootDeltaPlanStats) addRoot(collectionName, rootName string) collectionRootDeltaPlanKind { + if stats == nil || rootName == "" { + return collectionRootDeltaPlanUnknown + } + switch { + case rootName == collectionPrimaryRootName(collectionName): + stats.primaryRoots++ + return collectionRootDeltaPlanPrimary + case rootName == collectionTemplateRootName(collectionName): + stats.templateRoots++ + return collectionRootDeltaPlanTemplate + case rootName == collectionIndexStateRootName(collectionName): + stats.indexStateRoots++ + return collectionRootDeltaPlanIndexState + case strings.HasPrefix(rootName, collectionName+"/index/"): + stats.secondaryRoots++ + stats.secondaryUniqueRoots++ + return collectionRootDeltaPlanSecondary + } + return collectionRootDeltaPlanUnknown +} + +func (stats *collectionRootDeltaPlanStats) addBatch(kind collectionRootDeltaPlanKind, delta *batch.Batch) { + if stats == nil || delta == nil { + return + } + for _, entry := range delta.SortedEntries() { + keyBytes := uint64(len(entry.Key)) + valueBytes := uint64(0) + tombstone := entry.Type == batch.OpDelete + if !tombstone { + valueBytes = uint64(len(entry.Value)) + if entry.IsPtr { + valueBytes += page.ValuePtrSize + } + } + stats.addEntry(kind, keyBytes, valueBytes, tombstone) + } +} + +func (stats *collectionRootDeltaPlanStats) addIterator(kind collectionRootDeltaPlanKind, iter iterator.UnsafeIterator) { + if stats == nil || iter == nil { + return + } + for ; iter.Valid(); iter.Next() { + key := iter.Key() + value, _, flags := iter.UnsafeEntry() + keyBytes := uint64(len(key)) + valueBytes := uint64(0) + tombstone := flags&node.FlagTombstone != 0 || iter.IsDeleted() + if !tombstone { + valueBytes = uint64(len(value)) + if flags&node.FlagPointer != 0 { + valueBytes += page.ValuePtrSize + } + } + stats.addEntry(kind, keyBytes, valueBytes, tombstone) + } +} + +func (stats *collectionRootDeltaPlanStats) addEntry(kind collectionRootDeltaPlanKind, keyBytes, valueBytes uint64, tombstone bool) { + if stats == nil { + return + } + tombstones := uint64(0) + if tombstone { + tombstones = 1 + } + stats.addEntries(kind, 1, keyBytes, valueBytes, tombstones) +} + +func (stats *collectionRootDeltaPlanStats) addEntries(kind collectionRootDeltaPlanKind, entries, keyBytes, valueBytes, tombstones uint64) { + if stats == nil || entries == 0 { + return + } + stats.entries += entries + stats.keyBytes += keyBytes + stats.valueBytes += valueBytes + stats.tombstones += tombstones + if detail := stats.detailForKind(kind); detail != nil { + detail.entries += entries + detail.bytes += keyBytes + valueBytes + detail.tombstones += tombstones + } + if kind == collectionRootDeltaPlanPrimary { + stats.primaryEntries += entries + stats.primaryKeyBytes += keyBytes + stats.primaryValueBytes += valueBytes + stats.primaryTombstones += tombstones + } +} + +type collectionRootDeltaStatsIterator struct { + inner iterator.UnsafeIterator + kind collectionRootDeltaPlanKind + stats *collectionRootDeltaPlanStats + observed bool +} + +func newCollectionRootDeltaStatsIterator(collectionName, rootName string, inner iterator.UnsafeIterator, stats *collectionRootDeltaPlanStats) iterator.UnsafeIterator { + if inner == nil || stats == nil { + return inner + } + return &collectionRootDeltaStatsIterator{ + inner: inner, + kind: stats.addRoot(collectionName, rootName), + stats: stats, + } +} + +func (it *collectionRootDeltaStatsIterator) observeCurrent() { + if it == nil || it.inner == nil || it.observed || !it.inner.Valid() { + return + } + key := it.inner.UnsafeKey() + value, _, flags := it.inner.UnsafeEntry() + keyBytes := uint64(len(key)) + valueBytes := uint64(0) + tombstone := flags&node.FlagTombstone != 0 || it.inner.IsDeleted() + if !tombstone { + valueBytes = uint64(len(value)) + if flags&node.FlagPointer != 0 { + valueBytes += page.ValuePtrSize + } + } + it.stats.addEntry(it.kind, keyBytes, valueBytes, tombstone) + it.observed = true +} + +func (it *collectionRootDeltaStatsIterator) Valid() bool { + if it == nil || it.inner == nil { + return false + } + ok := it.inner.Valid() + if ok { + it.observeCurrent() + } + return ok +} + +func (it *collectionRootDeltaStatsIterator) Next() { + if it == nil || it.inner == nil { + return + } + it.inner.Next() + it.observed = false +} + +func (it *collectionRootDeltaStatsIterator) Seek(key []byte) { + if it == nil || it.inner == nil { + return + } + it.inner.Seek(key) + it.observed = false +} + +func (it *collectionRootDeltaStatsIterator) UnsafeKey() []byte { + it.observeCurrent() + return it.inner.UnsafeKey() +} + +func (it *collectionRootDeltaStatsIterator) UnsafeValue() []byte { + it.observeCurrent() + return it.inner.UnsafeValue() +} + +func (it *collectionRootDeltaStatsIterator) UnsafeEntry() ([]byte, page.ValuePtr, byte) { + it.observeCurrent() + return it.inner.UnsafeEntry() +} + +func (it *collectionRootDeltaStatsIterator) Key() []byte { + it.observeCurrent() + return it.inner.Key() +} + +func (it *collectionRootDeltaStatsIterator) Value() []byte { + it.observeCurrent() + return it.inner.Value() +} + +func (it *collectionRootDeltaStatsIterator) KeyCopy(dst []byte) []byte { + it.observeCurrent() + return it.inner.KeyCopy(dst) +} + +func (it *collectionRootDeltaStatsIterator) ValueCopy(dst []byte) []byte { + it.observeCurrent() + return it.inner.ValueCopy(dst) +} + +func (it *collectionRootDeltaStatsIterator) IsDeleted() bool { + it.observeCurrent() + return it.inner.IsDeleted() +} + +func (it *collectionRootDeltaStatsIterator) Error() error { + if it == nil || it.inner == nil { + return nil + } + return it.inner.Error() } -func collectionRootDeltaPlanStatsFromOrdered(collectionName string, rootNames []string, ordered []backenddb.OrderedRootDeltaBatchPublishInput) collectionRootDeltaPlanStats { - var stats collectionRootDeltaPlanStats - for i, rootName := range rootNames { - kind := stats.addRoot(collectionName, rootName) - if i < len(ordered) { - stats.addBatch(kind, ordered[i].Delta) - } +func (it *collectionRootDeltaStatsIterator) Close() error { + if it == nil || it.inner == nil { + return nil } - return stats + return it.inner.Close() } -type collectionRootDeltaPlanKind uint8 +func (it *collectionRootDeltaStatsIterator) Domain() (start, end []byte) { + if it == nil || it.inner == nil { + return nil, nil + } + return it.inner.Domain() +} -const ( - collectionRootDeltaPlanUnknown collectionRootDeltaPlanKind = iota - collectionRootDeltaPlanPrimary - collectionRootDeltaPlanTemplate - collectionRootDeltaPlanIndexState - collectionRootDeltaPlanSecondary -) +func (it *collectionRootDeltaStatsIterator) StableUnsafeIteratorSlices() bool { + if it == nil || it.inner == nil { + return false + } + type stableUnsafeIterator interface { + StableUnsafeIteratorSlices() bool + } + stable, ok := it.inner.(stableUnsafeIterator) + return ok && stable.StableUnsafeIteratorSlices() +} -func (stats *collectionRootDeltaPlanStats) addRoot(collectionName, rootName string) collectionRootDeltaPlanKind { - if stats == nil || rootName == "" { - return collectionRootDeltaPlanUnknown +func (it *collectionRootDeltaStatsIterator) OrderedUniqueUnsafeIterator() bool { + if it == nil || it.inner == nil { + return false } - switch { - case rootName == collectionPrimaryRootName(collectionName): - stats.primaryRoots++ - return collectionRootDeltaPlanPrimary - case rootName == collectionTemplateRootName(collectionName): - stats.templateRoots++ - return collectionRootDeltaPlanTemplate - case rootName == collectionIndexStateRootName(collectionName): - stats.indexStateRoots++ - return collectionRootDeltaPlanIndexState - case strings.HasPrefix(rootName, collectionName+"/index/"): - stats.secondaryRoots++ - return collectionRootDeltaPlanSecondary + type orderedUniqueIterator interface { + OrderedUniqueUnsafeIterator() bool } - return collectionRootDeltaPlanUnknown + trusted, ok := it.inner.(orderedUniqueIterator) + return ok && trusted.OrderedUniqueUnsafeIterator() } -func (stats *collectionRootDeltaPlanStats) addBatch(kind collectionRootDeltaPlanKind, delta *batch.Batch) { - if stats == nil || delta == nil { - return +func (it *collectionRootDeltaStatsIterator) Len() int { + if it == nil || it.inner == nil || !it.inner.Valid() { + return 0 } - for _, entry := range delta.SortedEntries() { - stats.entries++ - stats.keyBytes += uint64(len(entry.Key)) - if kind == collectionRootDeltaPlanPrimary { - stats.primaryEntries++ - stats.primaryKeyBytes += uint64(len(entry.Key)) - } - if entry.Type == batch.OpDelete { - stats.tombstones++ - if kind == collectionRootDeltaPlanPrimary { - stats.primaryTombstones++ - } - continue - } - valueBytes := uint64(len(entry.Value)) - if entry.IsPtr { - valueBytes += page.ValuePtrSize - } - stats.valueBytes += valueBytes - if kind == collectionRootDeltaPlanPrimary { - stats.primaryValueBytes += valueBytes - } + type lenHintIterator interface { + Len() int + } + hint, ok := it.inner.(lenHintIterator) + if !ok { + return 0 + } + return hint.Len() +} + +func (stats *collectionRootDeltaPlanStats) detailForKind(kind collectionRootDeltaPlanKind) *collectionRootDeltaKindStats { + if stats == nil { + return nil + } + switch kind { + case collectionRootDeltaPlanPrimary: + return &stats.primaryDetail + case collectionRootDeltaPlanTemplate: + return &stats.templateDetail + case collectionRootDeltaPlanIndexState: + return &stats.indexStateDetail + case collectionRootDeltaPlanSecondary: + return &stats.secondaryDetail + default: + return nil } } @@ -5340,20 +6718,22 @@ func (c *Collection) completePreparedIndexedFlush(work *indexedFlushPublishWork, if errors.Is(publishErr, ErrConcurrentMutation) { domain.indexedFlushRootBaseMismatches.Add(1) } - removed, owned := removeIndexedPublishingWorkUnitsLocked(domain, work.units) + removed, owned := removeIndexedPublishingWorkUnitsLocked(domain, work.batch.units) if !owned { err := errors.Join(errIndexedFlushLostOwnership, publishErr) + work.batch.state = coalescedFlushBatchLostOwnership domain.indexedFlushLostOwnership.Add(1) - domain.observeIndexedFlush(len(work.units), work.docCount, work.byteCount, work.rootRunCount, work.rootCount, observedElapsed(), materializeElapsed, publishElapsed, err) + domain.observeIndexedFlush(len(work.batch.units), work.batch.docCount, work.batch.byteCount, work.batch.rootRunCount, work.batch.rootCount, observedElapsed(), materializeElapsed, publishElapsed, err) return err } if len(removed) > 0 { domain.indexedFlushRequeues.Add(1) domain.indexedFlushRequeuedUnits.Add(uint64(len(removed))) } + work.batch.state = coalescedFlushBatchRequeued domain.indexedFlushUnits = append(removed, domain.indexedFlushUnits...) rebuildBufferedPendingIndexesLocked(domain, work.meta.Name, preservePrimaryRunIndex) - domain.observeIndexedFlush(len(work.units), work.docCount, work.byteCount, work.rootRunCount, work.rootCount, observedElapsed(), materializeElapsed, publishElapsed, publishErr) + domain.observeIndexedFlush(len(work.batch.units), work.batch.docCount, work.batch.byteCount, work.batch.rootRunCount, work.batch.rootCount, observedElapsed(), materializeElapsed, publishElapsed, publishErr) return publishErr } baseCatalog := domain.catalog @@ -5361,35 +6741,43 @@ func (c *Collection) completePreparedIndexedFlush(work *indexedFlushPublishWork, baseCatalog = work.catalog } overlayPublish := collectionMetaUsesIndexedOverlayRoots(work.meta) - nextCatalog := cloneCatalogWithRootUpdates(baseCatalog, work.meta, work.rootNames, rootIDs) + nextCatalog := cloneCatalogWithRootUpdates(baseCatalog, work.meta, work.batch.rootNames, rootIDs) if overlayPublish { - nextCatalog = cloneCatalogWithRootOverlays(baseCatalog, work.meta, work.rootNames, rootIDs) - nextCatalog = cloneCatalogWithRootOverlayFilters(nextCatalog, work.rootNames, rootIDs, work.rootOverlayFilters) + nextCatalog = cloneCatalogWithRootOverlays(baseCatalog, work.meta, work.batch.rootNames, rootIDs) + nextCatalog = cloneCatalogWithRootOverlayFilters(nextCatalog, work.batch.rootNames, rootIDs, work.batch.rootOverlayFilters) } - oldPublishing, owned := removeIndexedPublishingWorkUnitsLocked(domain, work.units) + oldPublishing, owned := removeIndexedPublishingWorkUnitsLocked(domain, work.batch.units) if !owned { + work.batch.state = coalescedFlushBatchLostOwnership domain.indexedFlushLostOwnership.Add(1) - domain.observeIndexedFlush(len(work.units), work.docCount, work.byteCount, work.rootRunCount, work.rootCount, observedElapsed(), materializeElapsed, publishElapsed, errIndexedFlushLostOwnership) + domain.observeIndexedFlush(len(work.batch.units), work.batch.docCount, work.batch.byteCount, work.batch.rootRunCount, work.batch.rootCount, observedElapsed(), materializeElapsed, publishElapsed, errIndexedFlushLostOwnership) return errIndexedFlushLostOwnership } + work.batch.state = coalescedFlushBatchPublished domain.loaded = true domain.meta = work.meta domain.catalog = nextCatalog domain.baseCommitSeq = c.commitSeqForSystemRoot(newSystemRoot) domain.baseSystemRoot = newSystemRoot domain.primaryRoot = nextCatalog.rootID(collectionPrimaryRootName(work.meta.Name)) - domain.count = subtractNonNegativeInt(domain.count, work.docCount) - domain.bufferedBytes = subtractNonNegativeInt64(domain.bufferedBytes, work.byteCount) + domain.count = subtractNonNegativeInt(domain.count, work.batch.docCount) + domain.bufferedBytes = subtractNonNegativeInt64(domain.bufferedBytes, work.batch.byteCount) domain.clearIndexedAsyncFlushError() if !overlayPublish { - retargetPendingIndexedRootBaseIDsLocked(domain, work.rootNames, work.rootBaseIDs, rootIDs) + retargetPendingIndexedRootBaseIDsLocked(domain, work.batch.rootNames, work.batch.rootBaseIDs, rootIDs) } rebuildBufferedPendingIndexesLocked(domain, work.meta.Name, preservePrimaryRunIndex) c.meta = work.meta c.rememberCatalogAtSystemRoot(newSystemRoot, nextCatalog) resetIndexedFlushUnits(oldPublishing) - domain.observeIndexedFlush(len(work.units), work.docCount, work.byteCount, work.rootRunCount, work.rootCount, observedElapsed(), materializeElapsed, publishElapsed, nil) - domain.observeRootDeltaPlan(work.rootDeltaStats) + rawRootDeltaStats := coalescedFlushBatchRawRootDeltaStats(work.batch) + domain.observeIndexedFlush(len(work.batch.units), work.batch.docCount, work.batch.byteCount, work.batch.rootRunCount, work.batch.rootCount, observedElapsed(), materializeElapsed, publishElapsed, nil) + domain.observeCoalescedFlushBatch(len(work.batch.units), work.batch.docCount, work.batch.byteCount, work.batch.rootDeltaStats.entries == 0) + domain.observeRootDeltaPlanRawUnit(rawRootDeltaStats) + domain.observeRootDeltaPlanFinal(work.batch.rootDeltaStats) + domain.observeRootDeltaPlanCoalescing(rawRootDeltaStats, work.batch.rootDeltaStats) + domain.observeRootDeltaPlan(work.batch.rootDeltaStats) + domain.observeIndexedSemanticEffectiveRecords(work.batch.effectiveRecords) return nil } @@ -5403,6 +6791,7 @@ func (c *Collection) flushBufferedIndexedLocked(domain *collectionWriteDomain) ( if domain.catalog == nil { return errCollectionNotFound } + preflightStart := time.Now() currentCommitSeq, currentSystemRoot := dbCommitSeqAndSystemRoot(c.db) catalog, err := c.revalidateBufferedWriteDomainLocked(domain, currentCommitSeq, currentSystemRoot) if err != nil { @@ -5435,14 +6824,25 @@ func (c *Collection) flushBufferedIndexedLocked(domain *collectionWriteDomain) ( }); err != nil { return err } + preflightElapsed := collectionObservedElapsedSince(preflightStart) + rotateStart := time.Now() rotateIndexedMutableToFlushUnitLocked(domain) + rotateElapsed := collectionObservedElapsedSince(rotateStart) + mergeStart := time.Now() flushUnit := mergedIndexedFlushUnitLocked(domain) + flushUnits := len(domain.indexedFlushUnits) rootNames := orderedBufferedRootNames(meta, flushUnit.rootRuns) + mergeElapsed := collectionObservedElapsedSince(mergeStart) + domain.observeIndexedFlushPrepare(preflightElapsed, rotateElapsed, mergeElapsed) if len(rootNames) == 0 { + domain.observeCoalescedFlushBatch(len(domain.indexedFlushUnits), domain.count, domain.bufferedBytes, true) + domain.observeRootDeltaPlanCoalescing(collectionRootDeltaPlanStats{}, collectionRootDeltaPlanStats{}) domain.indexedFlushUnits = nil domain.rootMutableRuns = nil domain.rootValueArenas = nil + domain.indexedSemanticRecords = nil + domain.rootDeltaStats = collectionRootDeltaPlanStats{} domain.count = 0 domain.bufferedBytes = 0 domain.mutableCount = 0 @@ -5451,7 +6851,6 @@ func (c *Collection) flushBufferedIndexedLocked(domain *collectionWriteDomain) ( } flushDocs := domain.count flushBytes := domain.bufferedBytes - flushUnits := len(domain.indexedFlushUnits) flushRootRuns := bufferedIndexedRootRunCount(domain) flushRoots := len(rootNames) flushStart := time.Now() @@ -5482,39 +6881,92 @@ func (c *Collection) flushBufferedIndexedLocked(domain *collectionWriteDomain) ( materializeElapsed = collectionObservedElapsedSince(materializeStart) return err } - ordered, cleanupDeltas, err := buildBufferedRootOverlayDeltaBatchPublishInputs(rootNames, flushUnit.rootRuns, flushUnit.rootPolicies, rootOverlays) + ordered, cleanupDeltas, err := buildBufferedRootOverlayDeltaBatchPublishInputs(rootNames, flushUnit.rootRuns, flushUnit.rootPolicies, rootOverlays, meta.Options.BufferedIndexedReadOnlyPrepare, meta.Options.BufferedIndexedReadOnlyPrepareWorkerCount) if err != nil { materializeElapsed = collectionObservedElapsedSince(materializeStart) return err } rootDeltaStats := collectionRootDeltaPlanStatsFromOrdered(meta.Name, rootNames, ordered) + rawRootDeltaStats := flushUnit.rootDeltaStats + if rawRootDeltaStats == (collectionRootDeltaPlanStats{}) { + rawRootDeltaStats = rootDeltaStats + } materializeElapsed = collectionObservedElapsedSince(materializeStart) publishStart := time.Now() - newSystemRoot, rootIDs, err = c.db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder(ordered, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + preflight := func() error { + return c.validateRootOverlayDescriptorSystemDeltaForMeta(meta, baseCommitSeq, baseSystemRoot, rootNames, baseRootIDs, rootOverlays) + } + newSystemRoot, rootIDs, err = c.db.PublishOrderedRootDeltaBatchGroupWithPreflightAndSystemDeltaBuilder(ordered, preflight, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { return c.buildRootOverlayDescriptorSystemDeltaIteratorForMeta(meta, baseCommitSeq, baseSystemRoot, rootNames, baseRootIDs, rootOverlays, rootIDs) }) publishElapsed = collectionObservedElapsedSince(publishStart) cleanupDeltas() if err == nil { + domain.observeCoalescedFlushBatch(flushUnits, flushDocs, flushBytes, rootDeltaStats.entries == 0) + domain.observeRootDeltaPlanRawUnit(rawRootDeltaStats) + domain.observeRootDeltaPlanFinal(rootDeltaStats) + domain.observeRootDeltaPlanCoalescing(rawRootDeltaStats, rootDeltaStats) domain.observeRootDeltaPlan(rootDeltaStats) } } else { materializeStart := time.Now() - ordered, cleanupDeltas, err := buildBufferedRootDeltaBatchPublishInputs(rootNames, flushUnit.rootRuns, flushUnit.rootBaseIDs, flushUnit.rootPolicies) + semanticPlanStart := time.Now() + view, err := buildIndexedSemanticPublishView(meta, flushUnit, rootNames, baseRootIDs) + semanticPlanElapsed := collectionObservedElapsedSince(semanticPlanStart) if err != nil { materializeElapsed = collectionObservedElapsedSince(materializeStart) + domain.observeIndexedFlushMaterializeBreakdown(semanticPlanElapsed, 0, 0) return err } - rootDeltaStats := collectionRootDeltaPlanStatsFromOrdered(meta.Name, rootNames, ordered) + defer resetIndexedSemanticPublishView(view) + buildInputsStart := time.Now() + ordered, cleanupDeltas, err := buildBufferedRootDeltaBatchPublishInputs(view.rootNames, view.rootRuns, view.rootBaseIDs, view.rootPolicies, meta.Options.BufferedIndexedReadOnlyPrepare, meta.Options.BufferedIndexedReadOnlyPrepareWorkerCount) + buildInputsElapsed := collectionObservedElapsedSince(buildInputsStart) + if err != nil { + materializeElapsed = collectionObservedElapsedSince(materializeStart) + domain.observeIndexedFlushMaterializeBreakdown(semanticPlanElapsed, buildInputsElapsed, 0) + return err + } + planStatsStart := time.Now() + rootDeltaStats := collectionRootDeltaPlanStatsFromOrdered(meta.Name, view.rootNames, ordered) + rawRootDeltaStats := flushUnit.rootDeltaStats + if rawRootDeltaStats == (collectionRootDeltaPlanStats{}) && view.semanticApplied { + rawStats, err := collectionRootDeltaPlanStatsFromIndexedFlushUnits(meta.Name, domain.indexedFlushUnits) + if err != nil { + cleanupDeltas() + materializeElapsed = collectionObservedElapsedSince(materializeStart) + domain.observeIndexedFlushMaterializeBreakdown(semanticPlanElapsed, buildInputsElapsed, collectionObservedElapsedSince(planStatsStart)) + return err + } + rawRootDeltaStats = rawStats + } + if rawRootDeltaStats == (collectionRootDeltaPlanStats{}) { + rawRootDeltaStats = rootDeltaStats + } + planStatsElapsed := collectionObservedElapsedSince(planStatsStart) + domain.observeIndexedFlushMaterializeBreakdown(semanticPlanElapsed, buildInputsElapsed, planStatsElapsed) materializeElapsed = collectionObservedElapsedSince(materializeStart) publishStart := time.Now() - newSystemRoot, rootIDs, err = c.db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder(ordered, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { - return c.buildRootDescriptorSystemDeltaIterator(baseCommitSeq, baseSystemRoot, rootNames, baseRootIDs, rootIDs) + preflightRootNames := append([]string(nil), rootNames...) + preflightRootBaseIDs := cloneUint64Map(baseRootIDs) + preflight := func() error { + return c.validateRootDescriptorSystemDeltaForMeta(meta, baseCommitSeq, baseSystemRoot, preflightRootNames, preflightRootBaseIDs) + } + newSystemRoot, rootIDs, err = c.db.PublishOrderedRootDeltaBatchGroupWithPreflightAndSystemDeltaBuilder(ordered, preflight, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return c.buildRootDescriptorSystemDeltaIterator(baseCommitSeq, baseSystemRoot, view.rootNames, view.rootBaseIDs, rootIDs) }) publishElapsed = collectionObservedElapsedSince(publishStart) cleanupDeltas() if err == nil { + rootNames = view.rootNames + baseRootIDs = view.rootBaseIDs + flushRoots = len(view.rootNames) + domain.observeCoalescedFlushBatch(flushUnits, flushDocs, flushBytes, rootDeltaStats.entries == 0) + domain.observeRootDeltaPlanRawUnit(rawRootDeltaStats) + domain.observeRootDeltaPlanFinal(rootDeltaStats) + domain.observeRootDeltaPlanCoalescing(rawRootDeltaStats, rootDeltaStats) domain.observeRootDeltaPlan(rootDeltaStats) + domain.observeIndexedSemanticEffectiveRecords(view.effectiveRecords) } } if err != nil { @@ -5542,7 +6994,9 @@ func (c *Collection) flushBufferedIndexedLocked(domain *collectionWriteDomain) ( domain.rootPolicies = nil domain.rootBaseIDs = nil domain.rootValueArenas = nil + domain.indexedSemanticRecords = nil domain.rootRunCount = 0 + domain.rootDeltaStats = collectionRootDeltaPlanStats{} domain.primaryIDIndex = nil domain.primaryRunIndex = nil oldUniqueValueRuns := domain.uniqueValueRuns @@ -5574,10 +7028,12 @@ func rotateIndexedMutableToFlushUnitLocked(domain *collectionWriteDomain) bool { rootPolicies: domain.rootPolicies, rootBaseIDs: domain.rootBaseIDs, uniqueValueRuns: domain.uniqueValueRuns, + semanticRecords: domain.indexedSemanticRecords, arenaRefs: domain.rootValueArenas, docCount: domain.mutableCount, byteCount: domain.mutableBytes, rootRunCount: domain.rootRunCount, + rootDeltaStats: domain.rootDeltaStats, } domain.indexedFlushUnits = append(domain.indexedFlushUnits, unit) domain.rootRuns = nil @@ -5586,7 +7042,9 @@ func rotateIndexedMutableToFlushUnitLocked(domain *collectionWriteDomain) bool { domain.rootBaseIDs = nil domain.uniqueValueRuns = nil domain.rootValueArenas = nil + domain.indexedSemanticRecords = nil domain.rootRunCount = 0 + domain.rootDeltaStats = collectionRootDeltaPlanStats{} domain.mutableCount = 0 domain.mutableBytes = 0 return true @@ -5599,7 +7057,7 @@ func removeIndexedPublishingUnitsLocked(domain *collectionWriteDomain, n int) [] if n > len(domain.indexedPublishingUnits) { n = len(domain.indexedPublishingUnits) } - removed := domain.indexedPublishingUnits[:n] + removed := append([]indexedFlushUnit(nil), domain.indexedPublishingUnits[:n]...) remaining := domain.indexedPublishingUnits[n:] if len(remaining) == 0 { domain.indexedPublishingUnits = nil @@ -5613,7 +7071,7 @@ func removeIndexedPublishingWorkUnitsLocked(domain *collectionWriteDomain, units if len(units) == 0 { return nil, true } - if domain == nil || len(domain.indexedPublishingUnits) < len(units) { + if domain == nil || len(domain.indexedPublishingUnits) != len(units) { return nil, false } for i := range units { @@ -5670,6 +7128,43 @@ func retargetPendingIndexedRootBaseIDsLocked(domain *collectionWriteDomain, root retarget(domain.rootBaseIDs) } +func buildCoalescedFlushBatchFromUnits(meta CollectionMeta, catalog *collectionCatalog, units []indexedFlushUnit) (coalescedFlushBatch, error) { + merged := mergedIndexedFlushUnits(units) + rootNames := orderedBufferedRootNames(meta, merged.rootRuns) + batch := coalescedFlushBatch{ + state: coalescedFlushBatchQueued, + units: append([]indexedFlushUnit(nil), units...), + mergedUnit: merged, + semanticRecords: cloneIndexedSemanticRecords(merged.semanticRecords), + rootNames: rootNames, + docCount: merged.docCount, + byteCount: merged.byteCount, + rootRunCount: indexedFlushUnitRootRunCount(merged), + rootCount: len(rootNames), + } + if merged.rootDeltaStats != (collectionRootDeltaPlanStats{}) { + batch.rawRootDeltaStats = merged.rootDeltaStats + batch.rawRootDeltaReady = true + } + if len(rootNames) == 0 { + return batch, nil + } + if catalog == nil { + return coalescedFlushBatch{}, errCollectionNotFound + } + batch.rootBaseIDs = make(map[string]uint64, len(rootNames)) + batch.rootOverlays = make(map[string][]uint64, len(rootNames)) + for _, rootName := range rootNames { + baseRoot, ok := merged.rootBaseIDs[rootName] + if !ok { + return coalescedFlushBatch{}, fmt.Errorf("collections: buffered indexed flush missing base root for %q", rootName) + } + batch.rootBaseIDs[rootName] = baseRoot + batch.rootOverlays[rootName] = append([]uint64(nil), catalog.overlayRootIDs(rootName)...) + } + return batch, nil +} + func mergedIndexedFlushUnits(units []indexedFlushUnit) indexedFlushUnit { if len(units) == 0 { return indexedFlushUnit{} @@ -5698,6 +7193,9 @@ func mergedIndexedFlushUnits(units []indexedFlushUnit) indexedFlushUnit { if len(unit.uniqueValueRuns) == 0 { unit.uniqueValueRuns = nil } + if len(unit.semanticRecords) == 0 { + unit.semanticRecords = nil + } return unit } @@ -5722,8 +7220,10 @@ func mergedIndexedFlushUnitLocked(domain *collectionWriteDomain) indexedFlushUni rootPolicies: domain.rootPolicies, rootBaseIDs: domain.rootBaseIDs, uniqueValueRuns: domain.uniqueValueRuns, + semanticRecords: domain.indexedSemanticRecords, arenaRefs: domain.rootValueArenas, rootRunCount: domain.rootRunCount, + rootDeltaStats: domain.rootDeltaStats, }) if len(unit.rootRuns) == 0 { unit.rootRuns = nil @@ -5737,6 +7237,9 @@ func mergedIndexedFlushUnitLocked(domain *collectionWriteDomain) indexedFlushUni if len(unit.uniqueValueRuns) == 0 { unit.uniqueValueRuns = nil } + if len(unit.semanticRecords) == 0 { + unit.semanticRecords = nil + } return unit } @@ -5746,6 +7249,7 @@ func mergeIndexedFlushUnit(dst *indexedFlushUnit, src indexedFlushUnit) { } appendTableRunMap(dst.rootRuns, src.rootRuns) appendTableRunMap(dst.uniqueValueRuns, src.uniqueValueRuns) + dst.semanticRecords = append(dst.semanticRecords, src.semanticRecords...) dst.arenaRefs = append(dst.arenaRefs, src.arenaRefs...) for rootName, policy := range src.rootPolicies { dst.rootPolicies[rootName] = policy @@ -5758,6 +7262,7 @@ func mergeIndexedFlushUnit(dst *indexedFlushUnit, src indexedFlushUnit) { dst.docCount = saturatingAddNonNegativeInt(dst.docCount, src.docCount) dst.byteCount = saturatingAddNonNegativeInt64(dst.byteCount, src.byteCount) dst.rootRunCount = saturatingAddNonNegativeInt(dst.rootRunCount, indexedFlushUnitRootRunCount(src)) + dst.rootDeltaStats.add(src.rootDeltaStats) } func indexedFlushUnitRootRunCount(unit indexedFlushUnit) int { @@ -5862,11 +7367,16 @@ func (c *Collection) insertOneNoIndex(id, document []byte) ([]byte, error) { defer func() { _ = snap.Close() }() resultID := bytes.Clone(id) - iter := &systemTargetIterator{entries: []systemTargetEntry{{ + entries := []systemTargetEntry{{ key: resultID, value: bytes.Clone(document), - }}} + }} + iter := &systemTargetIterator{entries: entries} defer func() { _ = iter.Close() }() + var deltaStats collectionRootDeltaPlanStats + if c.writeDomain != nil { + deltaStats = collectionRootDeltaPlanStatsFromSystemTargetEntries(c.meta.Name, rootName, entries) + } newSystemRoot, rootIDs, err := c.db.PublishOrderedRootDeltaGroupWithSystemDeltaBuilder([]backenddb.OrderedRootDeltaPublishInput{{ BaseRoot: baseRoot, @@ -5882,6 +7392,10 @@ func (c *Collection) insertOneNoIndex(id, document []byte) ([]byte, error) { return nil, unexpectedOrderedRootCountError(c.meta.Name, 1, len(rootIDs)) } c.rememberCatalogAtSystemRoot(newSystemRoot, cloneCatalogWithRootUpdates(catalog, c.meta, []string{rootName}, rootIDs)) + if c.writeDomain != nil { + c.writeDomain.observeRootDeltaPlanFinal(deltaStats) + c.writeDomain.observeRootDeltaPlan(deltaStats) + } return resultID, nil } @@ -6129,6 +7643,7 @@ func (c *Collection) insertBatchOnce(ids, documents [][]byte, trustedValidBSON b } resetCollectionRunTables(plan.runs) }() + deltaStats := plan.stats.rootDeltaStats for _, run := range plan.runs { iter := run.table.NewIterator(nil, nil) iterators = append(iterators, iter) @@ -6153,6 +7668,10 @@ func (c *Collection) insertBatchOnce(ids, documents [][]byte, trustedValidBSON b nextCatalog := cloneCatalogWithRootUpdates(currentCatalog, meta, rootNames, rootIDs) c.rememberCatalogAtSystemRoot(newSystemRoot, nextCatalog) c.noteWriteDomainCatalog(newSystemRoot, nextCatalog) + if c.writeDomain != nil { + c.writeDomain.observeRootDeltaPlanFinal(deltaStats) + c.writeDomain.observeRootDeltaPlan(deltaStats) + } c.setLastInsertStats(plan.stats.CollectionInsertStats) return plan.resultIDs, nil } @@ -6404,6 +7923,10 @@ func (c *Collection) insertBatchNoIndex( table.Freeze() stats.PrimaryRunBuild = time.Since(phaseStart) iter := table.NewIterator(nil, nil) + var deltaStats collectionRootDeltaPlanStats + if c.writeDomain != nil { + iter = newCollectionRootDeltaStatsIterator(c.meta.Name, rootName, iter, &deltaStats) + } defer func() { _ = iter.Close() resetCollectionRunTable(table) @@ -6429,6 +7952,10 @@ func (c *Collection) insertBatchNoIndex( nextCatalog := cloneCatalogWithRootUpdates(catalog, c.meta, []string{rootName}, rootIDs) c.rememberCatalogAtSystemRoot(newSystemRoot, nextCatalog) c.noteWriteDomainCatalog(newSystemRoot, nextCatalog) + if c.writeDomain != nil { + c.writeDomain.observeRootDeltaPlanFinal(deltaStats) + c.writeDomain.observeRootDeltaPlan(deltaStats) + } c.setLastInsertStats(stats) return resultIDs, nil } @@ -6603,6 +8130,7 @@ func (c *Collection) deleteDocumentOnce(documentID []byte) (bool, error) { c.rememberCatalogAtSystemRoot(newSystemRoot, nextCatalog) c.noteWriteDomainCatalog(newSystemRoot, nextCatalog) if c.writeDomain != nil { + c.writeDomain.observeRootDeltaPlanFinal(deltaStats) c.writeDomain.observeRootDeltaPlan(deltaStats) } return true, nil @@ -6731,17 +8259,23 @@ func (c *Collection) updateBatch(items []UpdateBatchItem, mode updateBatchMode) c.setLastUpdateStats(CollectionUpdateStats{}) return nil, true, nil } + detailedStats := c.updateBatchDetailedStatsEnabled() + var scaffoldStats CollectionUpdateStats + phaseStart := updateBatchStatsNow(detailedStats) if err := validateUpdateBatchItems(items); err != nil { return nil, false, err } + scaffoldStats.BatchValidate += updateBatchStatsSince(detailedStats, phaseStart) + phaseStart = updateBatchStatsNow(detailedStats) items = cloneUpdateBatchItems(items) - return c.updateBatchOwnedItems(items, mode) + scaffoldStats.BatchClone += updateBatchStatsSince(detailedStats, phaseStart) + return c.updateBatchOwnedItems(items, mode, scaffoldStats) } -func (c *Collection) updateBatchOwnedItems(items []UpdateBatchItem, mode updateBatchMode) ([]UpdateBatchResult, bool, error) { +func (c *Collection) updateBatchOwnedItems(items []UpdateBatchItem, mode updateBatchMode, scaffoldStats CollectionUpdateStats) ([]UpdateBatchResult, bool, error) { var lastErr error for attempt := 0; attempt < maxCollectionMutationRetries; attempt++ { - results, err := c.updateBatchOnce(items, mode) + results, err := c.updateBatchOnce(items, mode, scaffoldStats) if errors.Is(err, errUpdateBatchHasSecondaryUniqueIndex) || errors.Is(err, errUpdateBatchChangesSecondaryUniqueIndex) { return make([]UpdateBatchResult, len(items)), false, nil @@ -6770,7 +8304,8 @@ func (c *Collection) ensureWriteDomainOpen() error { } func validateUpdateBatchItems(items []UpdateBatchItem) error { - seen := make(map[string]struct{}, len(items)) + seen := make(map[uint64]int, len(items)) + var collisions map[uint64][]int for i, item := range items { if len(item.DocumentID) == 0 { return fmt.Errorf("collections: document id cannot be empty at index %d", i) @@ -6778,11 +8313,23 @@ func validateUpdateBatchItems(items []UpdateBatchItem) error { if item.Update == nil { return fmt.Errorf("collections: update function is nil at index %d", i) } - key := string(item.DocumentID) - if _, ok := seen[key]; ok { - return fmt.Errorf("%w at index %d", ErrDuplicateDocumentID, i) + hash := xxhash.Sum64(item.DocumentID) + if firstIndexPlusOne := seen[hash]; firstIndexPlusOne != 0 { + if bytes.Equal(items[firstIndexPlusOne-1].DocumentID, item.DocumentID) { + return fmt.Errorf("%w at index %d", ErrDuplicateDocumentID, i) + } + for _, collisionIndex := range collisions[hash] { + if bytes.Equal(items[collisionIndex].DocumentID, item.DocumentID) { + return fmt.Errorf("%w at index %d", ErrDuplicateDocumentID, i) + } + } + if collisions == nil { + collisions = make(map[uint64][]int) + } + collisions[hash] = append(collisions[hash], i) + continue } - seen[key] = struct{}{} + seen[hash] = i + 1 } return nil } @@ -7212,7 +8759,7 @@ func (combiner *collectionUpdateCombiner) runBatch(batch []collectionUpdateCombi Update: req.update, } } - results, batched, err := batch[0].collection.updateBatchOwnedItems(items, updateBatchModeNoSecondaryUniqueIndexChanges) + results, batched, err := batch[0].collection.updateBatchOwnedItems(items, updateBatchModeNoSecondaryUniqueIndexChanges, CollectionUpdateStats{}) clear(items) combiner.itemsScratch = items[:0] if !batched && err == nil { @@ -7387,9 +8934,18 @@ func collectionUpdateCombineHasDuplicateIDs(batch []collectionUpdateCombineReque func cloneUpdateBatchItems(items []UpdateBatchItem) []UpdateBatchItem { out := make([]UpdateBatchItem, len(items)) + totalIDBytes := 0 + for _, item := range items { + totalIDBytes += len(item.DocumentID) + } + idArena := make([]byte, totalIDBytes) + idOffset := 0 for i, item := range items { out[i] = item - out[i].DocumentID = bytes.Clone(item.DocumentID) + idEnd := idOffset + len(item.DocumentID) + copy(idArena[idOffset:idEnd], item.DocumentID) + out[i].DocumentID = idArena[idOffset:idEnd:idEnd] + idOffset = idEnd } return out } @@ -7824,6 +9380,7 @@ func (c *Collection) updateDocumentOnce(documentID []byte, update func(current [ c.rememberCatalogAtSystemRoot(newSystemRoot, nextCatalog) c.noteWriteDomainCatalog(newSystemRoot, nextCatalog) if c.writeDomain != nil { + c.writeDomain.observeRootDeltaPlanFinal(deltaStats) c.writeDomain.observeRootDeltaPlan(deltaStats) if primaryOnlyUpdate { c.writeDomain.observePrimaryOnlyUpdate(true, true, true, deltaStats) @@ -7859,6 +9416,7 @@ type updateBatchPlan struct { policies []backenddb.OrderedRootStoragePolicy deltaTables []memtable.Table directBufferedUpdate *directBufferedUpdatePlan + semanticRecords []indexedSemanticRecord uniqueSecondaryIndexByRoot []int canBufferIndexedUpdateBatch bool bufferedBase bool @@ -7884,7 +9442,7 @@ type directBufferedRootEntry struct { type directBufferedSecondaryRootPlan struct { rootName string - entries []directBufferedSecondaryRootEntry + entries []directBufferedSecondaryRootEntryRef arena []byte deletes int sets int @@ -7892,11 +9450,56 @@ type directBufferedSecondaryRootPlan struct { runtimeIdx int } -type directBufferedSecondaryRootEntry struct { - key []byte +type directBufferedSecondaryRootEntryRef struct { + end uint32 tombstone bool } +func (plan directBufferedSecondaryRootPlan) entryKey(i int) []byte { + if i < 0 || i >= len(plan.entries) { + return nil + } + start := 0 + if i > 0 { + start = int(plan.entries[i-1].end) + } + end := int(plan.entries[i].end) + if start < 0 || end < start || end > len(plan.arena) { + return nil + } + return plan.arena[start:end] +} + +func collectionRootDeltaPlanStatsFromDirectBufferedUpdatePlan(collectionName string, plan *updateBatchPlan) collectionRootDeltaPlanStats { + var stats collectionRootDeltaPlanStats + if plan == nil || plan.directBufferedUpdate == nil { + return stats + } + direct := plan.directBufferedUpdate + if len(direct.templateEntries) > 0 { + kind := stats.addRoot(collectionName, direct.templateRootName) + for _, entry := range direct.templateEntries { + stats.addEntry(kind, uint64(len(entry.key)), uint64(len(entry.value)), entry.flags&node.FlagTombstone != 0) + } + } + if len(direct.primaryEntries) > 0 { + kind := stats.addRoot(collectionName, direct.primaryRootName) + for _, entry := range direct.primaryEntries { + stats.addEntry(kind, uint64(len(entry.key)), uint64(len(entry.value)), entry.flags&node.FlagTombstone != 0) + } + } + for _, secondaryPlan := range direct.secondaryRootPlans { + if len(secondaryPlan.entries) == 0 { + continue + } + kind := stats.addRoot(collectionName, secondaryPlan.rootName) + for i, entry := range secondaryPlan.entries { + stats.addEntry(kind, uint64(len(secondaryPlan.entryKey(i))), 0, entry.tombstone) + } + } + return stats +} + func buildDirectBufferedTemplateRootEntries(records []templateV1Record) []directBufferedRootEntry { if len(records) == 0 { return nil @@ -7957,6 +9560,133 @@ func applyDirectBufferedRootEntries(table memtable.Table, entries []directBuffer }) } +func directBufferedRootTable(rootNames []string, tables []memtable.Table, rootName string) memtable.Table { + if rootName == "" || len(rootNames) != len(tables) { + return nil + } + for i, name := range rootNames { + if name == rootName { + return tables[i] + } + } + return nil +} + +func buildIndexedSemanticUpdateRecords(collectionName string, runtimes []indexRuntime, updates []preparedBatchUpdate, primaryEntries []directBufferedRootEntry) []indexedSemanticRecord { + if len(updates) == 0 { + return nil + } + records := make([]indexedSemanticRecord, 0, len(updates)) + totalIndexDeltas := 0 + totalValueRefs := 0 + totalValueBytes := 0 + if len(runtimes) > 0 { + for _, update := range updates { + if !update.indexStateChanged { + continue + } + for runtimeIdx := range runtimes { + if preparedBatchUpdateIndexChanged(update, runtimeIdx) { + totalIndexDeltas++ + oldValues := update.oldState.valuesAt(runtimeIdx) + newValues := update.newState.valuesAt(runtimeIdx) + totalValueRefs += len(oldValues) + len(newValues) + for _, value := range oldValues { + totalValueBytes += len(value) + } + for _, value := range newValues { + totalValueBytes += len(value) + } + } + } + } + } + if totalIndexDeltas == 0 { + return nil + } + indexDeltas := make([]indexedSemanticIndexDelta, totalIndexDeltas) + valueRefs := make([][]byte, totalValueRefs) + valueArena := make([]byte, totalValueBytes) + indexDeltaPos := 0 + valueRefPos := 0 + valueArenaPos := 0 + for i, update := range updates { + if !update.indexStateChanged || len(runtimes) == 0 { + continue + } + var documentID []byte + if i < len(primaryEntries) && len(primaryEntries[i].key) > 0 { + // Direct primary entries are built from the same changed slice and carry + // an owned, staged document ID clone, so the semantic sidecar can share it. + documentID = primaryEntries[i].key + } else { + documentID = bytes.Clone(update.documentID) + } + record := indexedSemanticRecord{ + kind: indexedSemanticRecordUpdate, + documentID: documentID, + } + indexDeltaStart := indexDeltaPos + for runtimeIdx, runtime := range runtimes { + if !preparedBatchUpdateIndexChanged(update, runtimeIdx) { + continue + } + if runtime.def.unique { + record.fallback = indexedSemanticFallbackRawOnly + } + indexDeltas[indexDeltaPos] = indexedSemanticIndexDelta{ + indexName: runtime.def.name, + rootName: runtimeSecondaryRootName(collectionName, runtime), + runtimeIdx: runtimeIdx, + unique: runtime.def.unique, + oldValues: cloneIndexedSemanticValueSetToArena(update.oldState.valuesAt(runtimeIdx), valueRefs, &valueRefPos, valueArena, &valueArenaPos), + newValues: cloneIndexedSemanticValueSetToArena(update.newState.valuesAt(runtimeIdx), valueRefs, &valueRefPos, valueArena, &valueArenaPos), + } + indexDeltaPos++ + } + if indexDeltaPos == indexDeltaStart { + continue + } + record.indexDeltas = indexDeltas[indexDeltaStart:indexDeltaPos:indexDeltaPos] + records = append(records, record) + } + return records +} + +func cloneIndexedSemanticValueSetToArena(in [][]byte, refs [][]byte, refPos *int, arena []byte, arenaPos *int) [][]byte { + if len(in) == 0 { + return nil + } + start := *refPos + end := start + len(in) + out := refs[start:end:end] + *refPos = end + for i, value := range in { + if value == nil { + continue + } + valueStart := *arenaPos + valueEnd := valueStart + len(value) + copy(arena[valueStart:valueEnd], value) + out[i] = arena[valueStart:valueEnd:valueEnd] + *arenaPos = valueEnd + } + return out +} + +func appendIndexedSemanticRecordsLocked(domain *collectionWriteDomain, records []indexedSemanticRecord) { + if domain == nil || len(records) == 0 { + return + } + // buildIndexedSemanticUpdateRecords already owns cloned document IDs and + // value sets; staging transfers those records into the mutable domain. + if len(domain.indexedSemanticRecords) == 0 { + domain.indexedSemanticRecords = records + return + } + domain.indexedSemanticRecords = append(domain.indexedSemanticRecords, records...) +} + func buildDirectBufferedSecondaryRootPlans(collectionName string, runtimes []indexRuntime, changed []preparedBatchUpdate, stats *CollectionUpdateStats) ([]directBufferedSecondaryRootPlan, int64, error) { if len(runtimes) == 0 || len(changed) == 0 { return nil, 0, nil @@ -7986,7 +9716,7 @@ func buildDirectBufferedSecondaryRootPlans(collectionName string, runtimes []ind } plan := directBufferedSecondaryRootPlan{ rootName: runtimeSecondaryRootName(collectionName, runtime), - entries: make([]directBufferedSecondaryRootEntry, 0, entryCount), + entries: make([]directBufferedSecondaryRootEntryRef, 0, entryCount), arena: make([]byte, 0, runStats.KeyBytes), deletes: runStats.Deletes, sets: runStats.Sets, @@ -8004,16 +9734,21 @@ func buildDirectBufferedSecondaryRootPlans(collectionName string, runtimes []ind if err != nil { return nil, 0, err } - plan.entries = append(plan.entries, directBufferedSecondaryRootEntry{key: key, tombstone: true}) + if len(key) > 0 && len(plan.arena) > int(^uint32(0)) { + return nil, 0, errors.New("collections: direct buffered secondary root plan too large") + } + plan.entries = append(plan.entries, directBufferedSecondaryRootEntryRef{end: uint32(len(plan.arena)), tombstone: true}) } for _, encoded := range item.newState.valuesAt(runtimeIdx) { - var key []byte var err error - plan.arena, key, err = appendIndexEntryKey(plan.arena, encoded, item.documentID) + plan.arena, _, err = appendIndexEntryKey(plan.arena, encoded, item.documentID) if err != nil { return nil, 0, err } - plan.entries = append(plan.entries, directBufferedSecondaryRootEntry{key: key}) + if len(plan.arena) > int(^uint32(0)) { + return nil, 0, errors.New("collections: direct buffered secondary root plan too large") + } + plan.entries = append(plan.entries, directBufferedSecondaryRootEntryRef{end: uint32(len(plan.arena))}) } } stagedBytes = saturatingAddNonNegativeInt64(stagedBytes, int64(runStats.KeyBytes)) @@ -8061,7 +9796,7 @@ var updateBatchPlanScratchPool sync.Pool const ( updateBatchPlanScratchMaxChangedCap = 1 << 15 - updateBatchPlanScratchDocumentBytes = 256 + updateBatchPlanScratchDocumentBytes = 192 updateBatchPlanScratchMaxInitialDocumentArena = 4 << 20 updateBatchPlanScratchMaxDocumentArena = 8 << 20 updateBatchPlanScratchMaxRootNameCap = 64 @@ -8350,11 +10085,21 @@ func (c *Collection) shouldUseDirectBufferedUpdatePlan(meta CollectionMeta, opts return !persistIndexStateForOptions(opts) } -func (c *Collection) updateBatchOnce(items []UpdateBatchItem, mode updateBatchMode) ([]UpdateBatchResult, error) { +func shouldBuildIndexedSemanticUpdateRecords(meta CollectionMeta, canBuffer bool) bool { + if !canBuffer || !meta.Options.BufferedIndexedWrites || len(meta.Indexes) == 0 { + return false + } + // The current semantic publish view only rewrites secondary roots. Template-v1 + // publish still uses the mechanical root-run path because template root + // attribution is not part of this PR3b slice. + return normalizedDocumentFormat(meta.Options.DocumentFormat) != DocumentFormatTemplateV1 +} + +func (c *Collection) updateBatchOnce(items []UpdateBatchItem, mode updateBatchMode, scaffoldStats CollectionUpdateStats) ([]UpdateBatchResult, error) { if c.shouldPlanUpdateBatchWithBufferedWrites(mode) { useBufferedRead := true for { - plan, err := c.buildUpdateBatchPlan(items, mode, useBufferedRead) + plan, err := c.buildUpdateBatchPlan(items, mode, useBufferedRead, scaffoldStats) if err != nil { return nil, err } @@ -8430,7 +10175,9 @@ func (c *Collection) updateBatchOnce(items []UpdateBatchItem, mode updateBatchMo }) primaryOnlyNoPublish := len(plan.meta.Indexes) == 0 && len(plan.deltaTables) == 0 && plan.directBufferedUpdate == nil stats := plan.stats + phaseStart := updateBatchStatsNow(c.updateBatchDetailedStatsEnabled()) plan.close() + stats.PlanClose += updateBatchStatsSince(c.updateBatchDetailedStatsEnabled(), phaseStart) if err != nil { return nil, err } @@ -8451,7 +10198,7 @@ func (c *Collection) updateBatchOnce(items []UpdateBatchItem, mode updateBatchMo return nil, err } - plan, err := c.buildUpdateBatchPlan(items, mode, false) + plan, err := c.buildUpdateBatchPlan(items, mode, false, scaffoldStats) if err != nil { return nil, err } @@ -8754,7 +10501,9 @@ func snapshotUpdateBatchBufferedReadLocked(domain *collectionWriteDomain, meta C return updateBatchBufferedRead{}, nil, false, false, nil } -func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBatchMode, useBufferedRead bool) (*updateBatchPlan, error) { +func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBatchMode, useBufferedRead bool, scaffoldStats CollectionUpdateStats) (*updateBatchPlan, error) { + detailedStats := c.updateBatchDetailedStatsEnabled() + setupStart := updateBatchStatsNow(detailedStats) results := make([]UpdateBatchResult, len(items)) snap := c.db.AcquireSnapshot() if snap == nil { @@ -8774,11 +10523,9 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa return nil, err } meta := catalog.meta - stats := CollectionUpdateStats{ - Items: len(items), - Indexes: len(meta.Indexes), - } - detailedStats := c.updateBatchDetailedStatsEnabled() + stats := scaffoldStats + stats.Items = len(items) + stats.Indexes = len(meta.Indexes) if mode == updateBatchModeNoSecondaryUniqueIndexes && collectionMetaHasSecondaryUniqueIndex(meta) { _ = snap.Close() return nil, errUpdateBatchHasSecondaryUniqueIndex @@ -8801,7 +10548,10 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa defer func() { resetCollectionTables(bufferedTemplateRuns) }() bufferedReadBlocked := false if domain := c.writeDomain; useBufferedRead && domain != nil && mode != updateBatchModeAny { + stats.PlanSetup += updateBatchStatsSince(detailedStats, setupStart) + phaseStart := updateBatchStatsNow(detailedStats) bufferedRead, bufferedTemplateRuns, bufferedReadBlocked, err = snapshotUpdateBatchBufferedRead(domain, meta, baseSystemRoot, items, plannerOptions.documentFormat) + stats.BufferedReadSnapshot += updateBatchStatsSince(detailedStats, phaseStart) if err != nil { _ = snap.Close() return nil, err @@ -8810,6 +10560,7 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa if len(bufferedTemplateRuns) > 0 { plannerOptions = collectionOptionsWithBufferedTemplateV1RunsResolver(plannerOptions, bufferedTemplateRuns) } + setupStart = updateBatchStatsNow(detailedStats) } primaryRootName := catalog.primaryRootName @@ -8826,6 +10577,7 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa } primaryRoot := catalog.rootID(primaryRootName) if primaryRoot == 0 && !bufferedRead.enabled && len(catalog.overlayRootIDs(primaryRootName)) == 0 { + stats.PlanSetup += updateBatchStatsSince(detailedStats, setupStart) plan := newUpdateBatchPlan() *plan = updateBatchPlan{ results: results, @@ -8858,6 +10610,8 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa } primaryReaderOK = true } + validateBSONID := normalizedDocumentFormat(plannerOptions.documentFormat) == DocumentFormatBSON + primaryHasOverlays := len(catalog.overlayRootIDs(primaryRootName)) != 0 scratch := getUpdateBatchPlanScratch(len(items), len(runtimes)) scratchOwnedByPlan := false @@ -8881,10 +10635,16 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa } }() initCollectionUpdateIndexStats(&stats, meta.Name, runtimes, detailedStats) + stats.PlanSetup += updateBatchStatsSince(detailedStats, setupStart) var currentScratch []byte for i, item := range items { phaseStart := updateBatchStatsNow(detailedStats) - current, err := readUpdateBatchCurrentDocumentAtCatalogRoot(snap, catalog, primaryRootName, &primaryReader, primaryReaderOK, i, item.DocumentID, bufferedRead, currentScratch[:0]) + var current updateBatchCurrentDocument + if primaryHasOverlays { + current, err = readUpdateBatchCurrentDocumentAtCatalogRoot(snap, catalog, primaryRootName, &primaryReader, primaryReaderOK, i, item.DocumentID, bufferedRead, currentScratch[:0]) + } else { + current, err = readUpdateBatchCurrentDocument(&primaryReader, primaryReaderOK, i, item.DocumentID, bufferedRead, currentScratch[:0]) + } stats.CurrentRead += updateBatchStatsSince(detailedStats, phaseStart) if err != nil { _ = snap.Close() @@ -8894,10 +10654,15 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa continue } results[i].Matched = true - currentID, err := captureBSONIDSnapshot(current.value, plannerOptions) - if err != nil { - _ = snap.Close() - return nil, updateBatchItemError(i, err) + var currentID bsonIDSnapshot + if validateBSONID { + phaseStart = updateBatchStatsNow(detailedStats) + currentID, err = captureBSONIDSnapshot(current.value, plannerOptions) + stats.BSONIDValidation += updateBatchStatsSince(detailedStats, phaseStart) + if err != nil { + _ = snap.Close() + return nil, updateBatchItemError(i, err) + } } prepared := preparedBatchUpdate{ itemIndex: i, @@ -8929,12 +10694,19 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa _ = snap.Close() return nil, updateBatchItemError(i, errors.New("changed replacement document cannot be empty")) } - if err := validateBSONReplacementPreservesIDSnapshot(currentID, document, plannerOptions); err != nil { - _ = snap.Close() - return nil, updateBatchItemError(i, err) + if validateBSONID { + phaseStart = updateBatchStatsNow(detailedStats) + if err := validateBSONReplacementPreservesIDSnapshot(currentID, document, plannerOptions); err != nil { + stats.BSONIDValidation += updateBatchStatsSince(detailedStats, phaseStart) + _ = snap.Close() + return nil, updateBatchItemError(i, err) + } + stats.BSONIDValidation += updateBatchStatsSince(detailedStats, phaseStart) } + phaseStart = updateBatchStatsNow(detailedStats) changed = append(changed, prepared) changedDocuments = append(changedDocuments, appendUpdateBatchPlanScratchDocument(scratch, document)) + stats.ReplacementStage += updateBatchStatsSince(detailedStats, phaseStart) if !current.buffered { currentScratch = current.value[:0] } @@ -8996,6 +10768,7 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa _ = snap.Close() return nil, updateBatchItemError(changed[i].itemIndex, err) } + phaseStart = updateBatchStatsNow(detailedStats) var changedIndexes uint64 indexStateChanged := false for runtimeIdx, runtime := range runtimes { @@ -9034,6 +10807,7 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa } changed[i].changedIndexes = changedIndexes changed[i].indexStateChanged = indexStateChanged + stats.IndexStateCompare += updateBatchStatsSince(detailedStats, phaseStart) } } if mode == updateBatchModeNoSecondaryUniqueIndexChanges && updateBatchChangesSecondaryUniqueIndex(runtimes, changed) { @@ -9115,6 +10889,12 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa success = true plan := newUpdateBatchPlan() stats = updateCollectionUpdateStatsCounts(stats, results, len(rootNames)) + var semanticRecords []indexedSemanticRecord + if c.writeDomain != nil && shouldBuildIndexedSemanticUpdateRecords(meta, canBufferIndexedUpdateBatch) { + phaseStart = updateBatchStatsNow(detailedStats) + semanticRecords = buildIndexedSemanticUpdateRecords(meta.Name, runtimes, changed, primaryEntries) + stats.SemanticRecordBuild += updateBatchStatsSince(detailedStats, phaseStart) + } *plan = updateBatchPlan{ results: results, stats: stats, @@ -9132,6 +10912,7 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa bufferedReadGeneration: bufferedRead.writeGeneration, bufferedReadBlocked: bufferedReadBlocked, policies: policies, + semanticRecords: semanticRecords, directBufferedUpdate: &directBufferedUpdatePlan{ templateEntries: templateEntries, primaryEntries: primaryEntries, @@ -9314,6 +11095,12 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa success = true plan := newUpdateBatchPlan() stats = updateCollectionUpdateStatsCounts(stats, results, len(deltaTables)) + var semanticRecords []indexedSemanticRecord + if c.writeDomain != nil && shouldBuildIndexedSemanticUpdateRecords(meta, canBufferIndexedUpdateBatch) { + phaseStart = updateBatchStatsNow(detailedStats) + semanticRecords = buildIndexedSemanticUpdateRecords(meta.Name, runtimes, changed, nil) + stats.SemanticRecordBuild += updateBatchStatsSince(detailedStats, phaseStart) + } *plan = updateBatchPlan{ results: results, stats: stats, @@ -9332,6 +11119,7 @@ func (c *Collection) buildUpdateBatchPlan(items []UpdateBatchItem, mode updateBa bufferedReadBlocked: bufferedReadBlocked, policies: policies, deltaTables: deltaTables, + semanticRecords: semanticRecords, scratch: scratch, } scratchOwnedByPlan = true @@ -9381,6 +11169,7 @@ func (c *Collection) publishUpdateBatchPlanLocked(plan *updateBatchPlan) ([]Upda c.rememberCatalogAtSystemRoot(newSystemRoot, nextCatalog) c.noteWriteDomainCatalog(newSystemRoot, nextCatalog) if c.writeDomain != nil { + c.writeDomain.observeRootDeltaPlanFinal(deltaStats) c.writeDomain.observeRootDeltaPlan(deltaStats) if len(plan.meta.Indexes) == 0 { c.writeDomain.observePrimaryOnlyUpdateBatch(plan.stats.Items, plan.stats.Matched, plan.stats.Modified, true, deltaStats) @@ -9428,6 +11217,7 @@ func (c *Collection) bufferDirectUpdateBatchPlanLocked(plan *updateBatchPlan) (b plan.stats.BufferStagePrecheck += updateBatchStatsSince(detailedStats, precheckStart) return false, nil } + rootDeltaStats := collectionRootDeltaPlanStatsFromDirectBufferedUpdatePlan(plan.meta.Name, plan) plan.stats.BufferStagePrecheck += updateBatchStatsSince(detailedStats, precheckStart) domain := c.writeDomain @@ -9516,7 +11306,7 @@ func (c *Collection) bufferDirectUpdateBatchPlanLocked(plan *updateBatchPlan) (b plan.stats.BufferStageDomainPrepare += updateBatchStatsSince(detailedStats, phaseStart) phaseStart = updateBatchStatsNow(detailedStats) - rootTables := make(map[string]memtable.Table, len(plan.rootNames)) + rootTables := make([]memtable.Table, len(plan.rootNames)) actualRootRuns := 0 for i, rootName := range plan.rootNames { baseRoot := plan.baseRootIDs[rootName] @@ -9531,13 +11321,13 @@ func (c *Collection) bufferDirectUpdateBatchPlanLocked(plan *updateBatchPlan) (b if table == nil { return false, fmt.Errorf("collections: UpdateBatch collection %q failed to allocate direct root accumulator for %q", plan.meta.Name, rootName) } - rootTables[rootName] = table + rootTables[i] = table if created { actualRootRuns = saturatingAddNonNegativeInt(actualRootRuns, 1) } } if len(direct.templateEntries) > 0 { - templateTable := rootTables[direct.templateRootName] + templateTable := directBufferedRootTable(plan.rootNames, rootTables, direct.templateRootName) if templateTable == nil { return false, fmt.Errorf("collections: UpdateBatch collection %q missing direct template root accumulator for %q", plan.meta.Name, direct.templateRootName) } @@ -9545,33 +11335,28 @@ func (c *Collection) bufferDirectUpdateBatchPlanLocked(plan *updateBatchPlan) (b return false, err } } - primaryTable := rootTables[direct.primaryRootName] - var primaryIndexKeys [][]byte - if domain.primaryRunIndex != nil { - primaryIndexKeys = make([][]byte, 0, len(direct.primaryEntries)) - } + primaryTable := directBufferedRootTable(plan.rootNames, rootTables, direct.primaryRootName) if err := applyDirectBufferedRootEntries(primaryTable, direct.primaryEntries); err != nil { return false, err } - for _, entry := range direct.primaryEntries { - if primaryIndexKeys != nil { - primaryIndexKeys = append(primaryIndexKeys, entry.key) - } - } - if primaryIndexKeys != nil { - addBufferedPrimaryRunIndexKeys(domain.primaryRunIndex, primaryIndexKeys, primaryTable) + if domain.primaryRunIndex != nil { + addBufferedPrimaryRunIndexDirectEntries(domain.primaryRunIndex, direct.primaryEntries, primaryTable) } for _, secondaryPlan := range direct.secondaryRootPlans { - table := rootTables[secondaryPlan.rootName] + table := directBufferedRootTable(plan.rootNames, rootTables, secondaryPlan.rootName) if table == nil { continue } if err := applyCollectionRunEntriesWithFlags(table, len(secondaryPlan.entries), func(i int) (key, value []byte, ptr page.ValuePtr, flags byte, err error) { entry := secondaryPlan.entries[i] + key = secondaryPlan.entryKey(i) + if len(key) == 0 { + return nil, nil, page.ValuePtr{}, 0, errors.New("collections: empty direct buffered secondary root key") + } if entry.tombstone { - return entry.key, nil, page.ValuePtr{}, node.FlagTombstone, nil + return key, nil, page.ValuePtr{}, node.FlagTombstone, nil } - return entry.key, nil, page.ValuePtr{}, node.FlagInline, nil + return key, nil, page.ValuePtr{}, node.FlagInline, nil }); err != nil { if shouldAutoFlushAfterAdding { rollbackBufferedIndexedDomain(domain, checkpoint) @@ -9597,6 +11382,7 @@ func (c *Collection) bufferDirectUpdateBatchPlanLocked(plan *updateBatchPlan) (b domain.mutableBytes = saturatingAddNonNegativeInt64(domain.mutableBytes, direct.stagedBytes) domain.writeGeneration++ domain.observeIndexedStage(modifiedCount, direct.stagedBytes, actualRootRuns) + domain.rootDeltaStats.add(rootDeltaStats) c.meta = plan.meta compactedObsolete, err := maybeCompactBufferedIndexedMutableRunsLocked(domain, plan.meta.Options) if err != nil { @@ -9606,6 +11392,12 @@ func (c *Collection) bufferDirectUpdateBatchPlanLocked(plan *updateBatchPlan) (b } return false, err } + semanticRecords := plan.semanticRecords + if len(semanticRecords) > 0 { + phaseStart = updateBatchStatsNow(detailedStats) + appendIndexedSemanticRecordsLocked(domain, semanticRecords) + plan.stats.BufferStageSemanticAppend += updateBatchStatsSince(detailedStats, phaseStart) + } if shouldFlushBufferedIndexedWrites(domain, plan.meta.Options) { flushDuration, lockReleased, relockWait, err := c.flushBufferedIndexedAfterThresholdLocked(domain, plan.meta.Options) if lockReleased > 0 { @@ -9622,6 +11414,9 @@ func (c *Collection) bufferDirectUpdateBatchPlanLocked(plan *updateBatchPlan) (b } } resetCollectionTables(compactedObsolete) + if len(semanticRecords) > 0 { + domain.observeIndexedSemanticRawRecords(semanticRecords) + } plan.stats.BufferedBatches = 1 return true, nil } @@ -9670,6 +11465,11 @@ func (c *Collection) bufferUpdateBatchPlanLocked(plan *updateBatchPlan) (bool, e plan.stats.BufferStagePrecheck += updateBatchStatsSince(detailedStats, precheckStart) return false, fmt.Errorf("collections: UpdateBatch collection %q modified rows without delta tables modified=%d roots=%d deltas=%d policies=%d", plan.meta.Name, modifiedCount, len(plan.rootNames), len(plan.deltaTables), len(plan.policies)) } + rootDeltaStats, err := collectionRootDeltaPlanStatsFromRootNameTables(plan.meta.Name, plan.rootNames, plan.deltaTables) + if err != nil { + plan.stats.BufferStagePrecheck += updateBatchStatsSince(detailedStats, precheckStart) + return false, err + } plan.stats.BufferStagePrecheck += updateBatchStatsSince(detailedStats, precheckStart) domain := c.writeDomain lockStart := updateBatchStatsNow(detailedStats) @@ -9839,6 +11639,7 @@ func (c *Collection) bufferUpdateBatchPlanLocked(plan *updateBatchPlan) (bool, e domain.mutableBytes = saturatingAddNonNegativeInt64(domain.mutableBytes, stagedBytes) domain.writeGeneration++ domain.observeIndexedStage(modifiedCount, stagedBytes, stagedRootRuns) + domain.rootDeltaStats.add(rootDeltaStats) c.meta = plan.meta compactedObsolete, err := maybeCompactBufferedIndexedMutableRunsLocked(domain, plan.meta.Options) if err != nil { @@ -9848,6 +11649,12 @@ func (c *Collection) bufferUpdateBatchPlanLocked(plan *updateBatchPlan) (bool, e } return false, err } + semanticRecords := plan.semanticRecords + if len(semanticRecords) > 0 { + phaseStart = updateBatchStatsNow(detailedStats) + appendIndexedSemanticRecordsLocked(domain, semanticRecords) + plan.stats.BufferStageSemanticAppend += updateBatchStatsSince(detailedStats, phaseStart) + } if shouldFlushBufferedIndexedWrites(domain, plan.meta.Options) { flushDuration, lockReleased, relockWait, err := c.flushBufferedIndexedAfterThresholdLocked(domain, plan.meta.Options) if lockReleased > 0 { @@ -9864,6 +11671,9 @@ func (c *Collection) bufferUpdateBatchPlanLocked(plan *updateBatchPlan) (bool, e } } resetCollectionTables(compactedObsolete) + if len(semanticRecords) > 0 { + domain.observeIndexedSemanticRawRecords(semanticRecords) + } plan.stats.BufferedBatches = 1 return true, nil } @@ -10221,6 +12031,7 @@ func (c *Collection) noteWriteDomainCatalog(systemRoot uint64, catalog *collecti domain.rootBaseIDs = nil domain.rootValueArenas = nil domain.rootRunCount = 0 + domain.rootDeltaStats = collectionRootDeltaPlanStats{} domain.primaryIDIndex = nil domain.primaryRunIndex = nil domain.uniqueValueRuns = nil @@ -12511,6 +14322,9 @@ func normalizeCollectionMeta(meta CollectionMeta) (CollectionMeta, error) { if meta.Options.BufferedIndexedAsyncFlushMaxQueuedUnits < 0 { return CollectionMeta{}, errors.New("collections: buffered indexed async flush max queued units cannot be negative") } + if meta.Options.BufferedIndexedReadOnlyPrepareWorkerCount < 0 { + return CollectionMeta{}, errors.New("collections: buffered indexed read-only prepare worker count cannot be negative") + } documentFormat, err := normalizeDocumentFormat(meta.Options.DocumentFormat) if err != nil { return CollectionMeta{}, err @@ -12553,9 +14367,13 @@ func normalizeCollectionMeta(meta CollectionMeta) (CollectionMeta, error) { meta.Options.BufferedIndexedWriteMaxRootRuns = 0 meta.Options.BufferedIndexedAsyncFlush = false meta.Options.BufferedIndexedOverlayRoots = false + meta.Options.BufferedIndexedReadOnlyPrepare = false + meta.Options.BufferedIndexedReadOnlyPrepareWorkerCount = 0 meta.Options.BufferedIndexedAsyncFlushMaxQueuedUnits = 0 } else if len(meta.Indexes) == 0 { meta.Options.BufferedIndexedWrites = false + meta.Options.BufferedIndexedReadOnlyPrepare = false + meta.Options.BufferedIndexedReadOnlyPrepareWorkerCount = 0 } else { meta.Options.BufferedIndexedWrites = true defaultMaxDocuments := DefaultIndexedWriteMemtableMaxDocuments @@ -12574,6 +14392,9 @@ func normalizeCollectionMeta(meta CollectionMeta) (CollectionMeta, error) { if meta.Options.BufferedIndexedAsyncFlush && meta.Options.BufferedIndexedAsyncFlushMaxQueuedUnits == 0 { meta.Options.BufferedIndexedAsyncFlushMaxQueuedUnits = DefaultIndexedWriteMemtableAsyncFlushMaxQueuedUnits } + if !meta.Options.BufferedIndexedReadOnlyPrepare { + meta.Options.BufferedIndexedReadOnlyPrepareWorkerCount = 0 + } } return meta, nil } diff --git a/TreeDB/collections/api_test.go b/TreeDB/collections/api_test.go index 630332ace9..3ec599a789 100644 --- a/TreeDB/collections/api_test.go +++ b/TreeDB/collections/api_test.go @@ -609,13 +609,18 @@ func TestCollectionUpdateBatchStatsExposeIndexRunShape(t *testing.T) { if stats.SecondaryKeyBytes == 0 { t.Fatal("stats secondary key bytes=0 want positive") } - if stats.CurrentRead != 0 || stats.Callback != 0 || stats.BufferStage != 0 || + if stats.BatchValidate != 0 || stats.BatchClone != 0 || + stats.PlanSetup != 0 || stats.BufferedReadSnapshot != 0 || + stats.CurrentRead != 0 || stats.BSONIDValidation != 0 || + stats.Callback != 0 || stats.ReplacementStage != 0 || + stats.IndexStateCompare != 0 || stats.BufferStage != 0 || stats.BufferStagePrecheck != 0 || stats.BufferStageLockWait != 0 || stats.BufferStageLockHold != 0 || stats.BufferStageValidation != 0 || stats.BufferStageRootScan != 0 || stats.BufferStageDomainPrepare != 0 || stats.BufferStagePrimaryIdx != 0 || stats.BufferStageUniqueIdx != 0 || - stats.BufferStageRootAppend != 0 || stats.BufferStageFlush != 0 { + stats.BufferStageRootAppend != 0 || stats.BufferStageFlush != 0 || + stats.PlanClose != 0 { t.Fatalf("default update timings=%+v want zero unless detailed stats enabled", stats) } @@ -677,6 +682,16 @@ func TestCollectionUpdateBatchStatsExposeIndexRunShape(t *testing.T) { } } for _, key := range []string{ + "treedb.collections.write_domain.update_batch.validate_items_ns_total", + "treedb.collections.write_domain.update_batch.clone_items_ns_total", + "treedb.collections.write_domain.update_batch.plan_setup_ns_total", + "treedb.collections.write_domain.update_batch.buffered_read_snapshot_ns_total", + "treedb.collections.write_domain.update_batch.bson_id_validation_ns_total", + "treedb.collections.write_domain.update_batch.replacement_stage_ns_total", + "treedb.collections.write_domain.update_batch.index_state_compare_ns_total", + "treedb.collections.write_domain.indexed_flush.preflight_ns_total", + "treedb.collections.write_domain.indexed_flush.rotate_ns_total", + "treedb.collections.write_domain.indexed_flush.merge_ns_total", "treedb.collections.write_domain.update_batch.buffer_stage_precheck_ns_total", "treedb.collections.write_domain.update_batch.buffer_stage_lock_wait_ns_total", "treedb.collections.write_domain.update_batch.buffer_stage_lock_hold_ns_total", @@ -687,6 +702,7 @@ func TestCollectionUpdateBatchStatsExposeIndexRunShape(t *testing.T) { "treedb.collections.write_domain.update_batch.buffer_stage_unique_index_ns_total", "treedb.collections.write_domain.update_batch.buffer_stage_root_append_ns_total", "treedb.collections.write_domain.update_batch.buffer_stage_flush_ns_total", + "treedb.collections.write_domain.update_batch.plan_close_ns_total", } { if _, ok := exported[key]; !ok { t.Fatalf("manager stats missing %s: keys=%v", key, exported) @@ -842,6 +858,13 @@ func TestCollectionUpdateBufferBreakdownStatsSnapshotAndAdd(t *testing.T) { set func(*CollectionUpdateStats, time.Duration) get func(CollectionManagerStats) time.Duration }{ + {"validate_items", "treedb.collections.write_domain.update_batch.validate_items_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.BatchValidate = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchValidate }}, + {"clone_items", "treedb.collections.write_domain.update_batch.clone_items_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.BatchClone = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchClone }}, + {"plan_setup", "treedb.collections.write_domain.update_batch.plan_setup_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.PlanSetup = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchPlanSetup }}, + {"buffered_read_snapshot", "treedb.collections.write_domain.update_batch.buffered_read_snapshot_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.BufferedReadSnapshot = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchBufferedReadSnapshot }}, + {"bson_id_validation", "treedb.collections.write_domain.update_batch.bson_id_validation_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.BSONIDValidation = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchBSONIDValidation }}, + {"replacement_stage", "treedb.collections.write_domain.update_batch.replacement_stage_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.ReplacementStage = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchReplacementStage }}, + {"index_state_compare", "treedb.collections.write_domain.update_batch.index_state_compare_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.IndexStateCompare = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchIndexStateCompare }}, {"precheck", "treedb.collections.write_domain.update_batch.buffer_stage_precheck_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.BufferStagePrecheck = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchBufferPrecheck }}, {"lock_wait", "treedb.collections.write_domain.update_batch.buffer_stage_lock_wait_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.BufferStageLockWait = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchBufferLockWait }}, {"lock_hold", "treedb.collections.write_domain.update_batch.buffer_stage_lock_hold_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.BufferStageLockHold = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchBufferLockHold }}, @@ -851,7 +874,9 @@ func TestCollectionUpdateBufferBreakdownStatsSnapshotAndAdd(t *testing.T) { {"primary_index", "treedb.collections.write_domain.update_batch.buffer_stage_primary_index_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.BufferStagePrimaryIdx = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchBufferPrimaryIdx }}, {"unique_index", "treedb.collections.write_domain.update_batch.buffer_stage_unique_index_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.BufferStageUniqueIdx = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchBufferUniqueIdx }}, {"root_append", "treedb.collections.write_domain.update_batch.buffer_stage_root_append_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.BufferStageRootAppend = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchBufferRootAppend }}, + {"semantic_append", "treedb.collections.write_domain.update_batch.buffer_stage_semantic_append_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.BufferStageSemanticAppend = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchBufferSemanticAppend }}, {"flush", "treedb.collections.write_domain.update_batch.buffer_stage_flush_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.BufferStageFlush = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchBufferFlush }}, + {"plan_close", "treedb.collections.write_domain.update_batch.plan_close_ns_total", func(s *CollectionUpdateStats, d time.Duration) { s.PlanClose = d }, func(s CollectionManagerStats) time.Duration { return s.UpdateBatchPlanClose }}, } var updateStats CollectionUpdateStats @@ -1311,6 +1336,131 @@ func TestCollectionManagerStatsExposeIndexedWriteDomainMetrics(t *testing.T) { } } +func TestCollectionIndexedFlushReadOnlyPrepareDefaultOff(t *testing.T) { + d, err := backenddb.Open(backenddb.Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = d.Close() }() + + mgr := NewCollectionManager(d) + if _, err := mgr.CreateCollection(&CollectionMeta{ + Name: "users", + Indexes: []IndexDefinition{{Name: "email", Field: "email", ValueType: IndexValueString}}, + }); err != nil { + t.Fatalf("create collection: %v", err) + } + col, err := mgr.OpenCollection("users") + if err != nil { + t.Fatalf("open collection: %v", err) + } + if _, err := col.InsertBatch( + [][]byte{[]byte("u1"), []byte("u2")}, + [][]byte{ + []byte(`{"email":"ada@example.com"}`), + []byte(`{"email":"grace@example.com"}`), + }, + ); err != nil { + t.Fatalf("insert batch: %v", err) + } + + before := d.Stats() + if err := mgr.FlushAll(); err != nil { + t.Fatalf("flush all: %v", err) + } + after := d.Stats() + if got := collectionDBUintStatDelta(t, before, after, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total"); got != 0 { + t.Fatalf("read-only prepare calls delta=%d want 0 by default", got) + } + if got := collectionDBUintStatDelta(t, before, after, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_ranges_total"); got != 0 { + t.Fatalf("read-only prepare worker ranges delta=%d want 0 by default", got) + } +} + +func TestCollectionIndexedFlushReadOnlyPrepareOptInReportsDBStats(t *testing.T) { + d, err := backenddb.Open(backenddb.Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = d.Close() }() + + mgr := NewCollectionManager(d) + meta, err := mgr.CreateCollection(&CollectionMeta{ + Name: "users", + Options: CollectionOptions{ + BufferedIndexedReadOnlyPrepare: true, + BufferedIndexedReadOnlyPrepareWorkerCount: 4, + }, + Indexes: []IndexDefinition{{Name: "email", Field: "email", ValueType: IndexValueString}}, + }) + if err != nil { + t.Fatalf("create collection: %v", err) + } + if !meta.Options.BufferedIndexedReadOnlyPrepare || meta.Options.BufferedIndexedReadOnlyPrepareWorkerCount != 4 { + t.Fatalf("normalized read-only prepare options=%+v want enabled/4", meta.Options) + } + col, err := mgr.OpenCollection("users") + if err != nil { + t.Fatalf("open collection: %v", err) + } + if _, err := col.InsertBatch( + [][]byte{[]byte("u1"), []byte("u2"), []byte("u3")}, + [][]byte{ + []byte(`{"email":"ada@example.com"}`), + []byte(`{"email":"grace@example.com"}`), + []byte(`{"email":"katherine@example.com"}`), + }, + ); err != nil { + t.Fatalf("insert batch: %v", err) + } + + before := d.Stats() + if err := mgr.FlushAll(); err != nil { + t.Fatalf("flush all: %v", err) + } + after := d.Stats() + calls := collectionDBUintStatDelta(t, before, after, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total") + if calls == 0 { + t.Fatal("read-only prepare calls delta=0 want positive") + } + if got := collectionDBUintStatDelta(t, before, after, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ops_total"); got == 0 { + t.Fatal("read-only prepare ops delta=0 want positive") + } + if got := collectionDBUintStatDelta(t, before, after, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_leaf_spans_total"); got == 0 { + t.Fatal("read-only prepare leaf spans delta=0 want positive") + } + targets := collectionDBUintStatDelta(t, before, after, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_targets_total") + if want := calls * 4; targets != want { + t.Fatalf("read-only prepare worker target delta=%d want calls*4=%d", targets, want) + } + if got := collectionDBUintStatDelta(t, before, after, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_ranges_total"); got == 0 { + t.Fatal("read-only prepare worker ranges delta=0 want positive") + } +} + +func collectionDBUintStatDelta(tb testing.TB, before, after map[string]string, key string) uint64 { + tb.Helper() + beforeValue := collectionDBUintStat(tb, before, key) + afterValue := collectionDBUintStat(tb, after, key) + if afterValue < beforeValue { + tb.Fatalf("stat %s decreased from %d to %d", key, beforeValue, afterValue) + } + return afterValue - beforeValue +} + +func collectionDBUintStat(tb testing.TB, stats map[string]string, key string) uint64 { + tb.Helper() + raw, ok := stats[key] + if !ok { + tb.Fatalf("stats missing %s", key) + } + value, err := strconv.ParseUint(raw, 10, 64) + if err != nil { + tb.Fatalf("parse stat %s=%q: %v", key, raw, err) + } + return value +} + func TestCollectionRootDeltaPlanStatsCountsPointerValueBytes(t *testing.T) { delta := batch.New(nil, 0) defer func() { _ = delta.Close() }() @@ -1330,6 +1480,60 @@ func TestCollectionRootDeltaPlanStatsCountsPointerValueBytes(t *testing.T) { } } +func TestCollectionRootDeltaStatsIteratorPreservesFastPathTraits(t *testing.T) { + table := newFreezeSortRunTable() + table.Set([]byte("u1"), []byte(`{"city":"hnl"}`)) + table.Set([]byte("u2"), []byte(`{"city":"sea"}`)) + table.Freeze() + + var stats collectionRootDeltaPlanStats + iter := newCollectionRootDeltaStatsIterator("users", collectionPrimaryRootName("users"), table.NewIterator(nil, nil), &stats) + defer func() { _ = iter.Close() }() + + stable, ok := iter.(interface { + StableUnsafeIteratorSlices() bool + }) + if !ok { + t.Fatalf("stats iterator does not expose StableUnsafeIteratorSlices") + } + if !stable.StableUnsafeIteratorSlices() { + t.Fatalf("stats iterator did not preserve stable unsafe slices") + } + ordered, ok := iter.(interface { + OrderedUniqueUnsafeIterator() bool + }) + if !ok { + t.Fatalf("stats iterator does not expose OrderedUniqueUnsafeIterator") + } + if !ordered.OrderedUniqueUnsafeIterator() { + t.Fatalf("stats iterator did not preserve ordered unique iterator trait") + } + lenHint, ok := iter.(interface { + Len() int + }) + if !ok { + t.Fatalf("stats iterator does not expose Len hint") + } + if got, want := lenHint.Len(), 2; got != want { + t.Fatalf("stats iterator Len=%d want %d", got, want) + } + + delta, err := backenddb.OrderedRootDeltaBatchFromIterator(iter) + if err != nil { + t.Fatalf("materialize delta: %v", err) + } + defer func() { _ = delta.Close() }() + if got, want := len(delta.SortedEntries()), 2; got != want { + t.Fatalf("delta entries=%d want %d", got, want) + } + if got, want := stats.entries, uint64(2); got != want { + t.Fatalf("stats entries=%d want %d", got, want) + } + if got, want := stats.primaryEntries, uint64(2); got != want { + t.Fatalf("primary stats entries=%d want %d", got, want) + } +} + func TestPrimaryOnlyNoIndexUpdateCounters(t *testing.T) { d, err := backenddb.Open(backenddb.Options{Dir: t.TempDir()}) if err != nil { @@ -2093,9 +2297,23 @@ func TestCollectionSingleInsertBufferedNoIndexFlushPersistsAfterReopen(t *testin if _, err := col.Insert([]byte("u1"), []byte(`{"name":"ada"}`)); err != nil { t.Fatalf("insert: %v", err) } + beforeFlush := mgr.StatsSnapshot() if err := col.Flush(); err != nil { t.Fatalf("flush: %v", err) } + afterFlush := mgr.StatsSnapshot() + if got, want := afterFlush.PrimaryOnlyDrainCalls-beforeFlush.PrimaryOnlyDrainCalls, uint64(1); got != want { + t.Fatalf("primary-only drain calls delta=%d want %d", got, want) + } + if got, want := afterFlush.RootDeltaPlanPrimaryRoots-beforeFlush.RootDeltaPlanPrimaryRoots, uint64(1); got != want { + t.Fatalf("root delta primary roots delta=%d want %d", got, want) + } + if got, want := afterFlush.RootDeltaPlanEntries-beforeFlush.RootDeltaPlanEntries, uint64(1); got != want { + t.Fatalf("root delta entries delta=%d want %d", got, want) + } + if got, want := afterFlush.RootDeltaPlanFinalPrimaryEntries-beforeFlush.RootDeltaPlanFinalPrimaryEntries, uint64(1); got != want { + t.Fatalf("final primary entries delta=%d want %d", got, want) + } if err := d.Close(); err != nil { t.Fatalf("close db: %v", err) } @@ -2118,6 +2336,44 @@ func TestCollectionSingleInsertBufferedNoIndexFlushPersistsAfterReopen(t *testin } } +func TestCollectionInsertBatchNoIndexRecordsRootDeltaPlanStats(t *testing.T) { + d, err := backenddb.Open(backenddb.Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = d.Close() }() + + mgr := NewCollectionManager(d) + if _, err := mgr.CreateCollection(&CollectionMeta{Name: "users"}); err != nil { + t.Fatalf("create collection: %v", err) + } + col, err := mgr.OpenCollection("users") + if err != nil { + t.Fatalf("open collection: %v", err) + } + + before := mgr.StatsSnapshot() + if _, err := col.InsertBatch( + [][]byte{[]byte("u1"), []byte("u2")}, + [][]byte{[]byte(`{"name":"ada"}`), []byte(`{"name":"grace"}`)}, + ); err != nil { + t.Fatalf("insert batch: %v", err) + } + after := mgr.StatsSnapshot() + if got, want := after.RootDeltaPlanPrimaryRoots-before.RootDeltaPlanPrimaryRoots, uint64(1); got != want { + t.Fatalf("root delta primary roots delta=%d want %d", got, want) + } + if got, want := after.RootDeltaPlanEntries-before.RootDeltaPlanEntries, uint64(2); got != want { + t.Fatalf("root delta entries delta=%d want %d", got, want) + } + if got, want := after.RootDeltaPlanFinalPrimaryEntries-before.RootDeltaPlanFinalPrimaryEntries, uint64(2); got != want { + t.Fatalf("final primary entries delta=%d want %d", got, want) + } + if got := after.RootDeltaPlanFinalPrimaryBytes - before.RootDeltaPlanFinalPrimaryBytes; got == 0 { + t.Fatal("final primary bytes delta=0 want positive") + } +} + func TestCollectionIndexedWriteMemtablesReadUniqueAndFlush(t *testing.T) { dir := t.TempDir() d, err := backenddb.Open(backenddb.Options{Dir: dir}) @@ -3579,20 +3835,30 @@ func TestCollectionIndexedWriteMemtablesBypassDefaultLargeBatches(t *testing.T) }, Indexes: []IndexDefinition{{Name: "city", Field: "city", ValueType: IndexValueString}}, } - if !col.shouldBufferIndexedInsertBatch(meta, DefaultIndexedWriteMemtableDirectBatchDocuments-1) { - t.Fatal("default indexed memtable path bypassed a below-threshold batch") - } if col.shouldBufferIndexedInsertBatch(meta, DefaultIndexedWriteMemtableDirectBatchDocuments) { t.Fatal("default indexed memtable path buffered a large direct-publish batch") } + if col.shouldBufferIndexedInsertBatch(meta, DefaultIndexedWriteMemtableLowFanoutDirectBatchDocuments) { + t.Fatal("default indexed memtable path buffered a low-fanout direct-publish batch") + } + if !col.shouldBufferIndexedInsertBatch(meta, DefaultIndexedWriteMemtableLowFanoutDirectBatchDocuments-1) { + t.Fatal("default indexed memtable path bypassed a below-threshold low-fanout batch") + } meta.Options.BufferedIndexedAsyncFlush = true meta.Options.BufferedIndexedWriteMaxDocuments = DefaultIndexedWriteMemtableAsyncFlushMaxDocuments - if !col.shouldBufferIndexedInsertBatch(meta, DefaultIndexedWriteMemtableDirectBatchDocuments-1) { - t.Fatal("async default indexed memtable path bypassed a below-threshold batch") - } if col.shouldBufferIndexedInsertBatch(meta, DefaultIndexedWriteMemtableDirectBatchDocuments) { t.Fatal("async default indexed memtable path buffered a large direct-publish batch") } + if col.shouldBufferIndexedInsertBatch(meta, DefaultIndexedWriteMemtableLowFanoutDirectBatchDocuments) { + t.Fatal("async default indexed memtable path buffered a low-fanout direct-publish batch") + } + meta.Indexes = append(meta.Indexes, + IndexDefinition{Name: "state", Field: "state", ValueType: IndexValueString}, + IndexDefinition{Name: "email", Field: "email", ValueType: IndexValueString}, + ) + if !col.shouldBufferIndexedInsertBatch(meta, DefaultIndexedWriteMemtableLowFanoutDirectBatchDocuments) { + t.Fatal("default indexed memtable path bypassed a moderate three-index batch") + } meta.Options.BufferedIndexedAsyncFlush = false meta.Options.BufferedIndexedWriteMaxDocuments = 2 if !col.shouldBufferIndexedInsertBatch(meta, 2) { @@ -4413,7 +4679,7 @@ func TestBuildBufferedRootDeltaBatchPublishInputsParallelPreservesRootOrderAndTo cityName: backenddb.OrderedRootStoragePagerLeaves, } - ordered, cleanup, err := buildBufferedRootDeltaBatchPublishInputs(rootNames, rootRuns, rootBaseIDs, rootPolicies) + ordered, cleanup, err := buildBufferedRootDeltaBatchPublishInputs(rootNames, rootRuns, rootBaseIDs, rootPolicies, false, 0) if err != nil { t.Fatalf("build buffered root deltas: %v", err) } @@ -4435,6 +4701,12 @@ func TestBuildBufferedRootDeltaBatchPublishInputsParallelPreservesRootOrderAndTo if ordered[i].IncludeDeletedOnColdBuild { t.Fatalf("ordered[%d] IncludeDeletedOnColdBuild=true want false", i) } + if ordered[i].PrepareReadOnly || ordered[i].ReadOnlyPrepareWorkerCount != 0 { + t.Fatalf("ordered[%d] read-only prepare=%t/%d want false/0", i, ordered[i].PrepareReadOnly, ordered[i].ReadOnlyPrepareWorkerCount) + } + if ordered[i].ReadOnlyPrepareResult != nil { + t.Fatalf("ordered[%d] ReadOnlyPrepareResult=%p want nil by default", i, ordered[i].ReadOnlyPrepareResult) + } } primaryEntries := ordered[1].Delta.SortedEntries() @@ -4486,7 +4758,7 @@ func TestBuildBufferedRootOverlayDeltaBatchPublishInputsPreservesColdTombstones( cityName: nil, } - ordered, cleanup, err := buildBufferedRootOverlayDeltaBatchPublishInputs(rootNames, rootRuns, rootPolicies, rootOverlays) + ordered, cleanup, err := buildBufferedRootOverlayDeltaBatchPublishInputs(rootNames, rootRuns, rootPolicies, rootOverlays, false, 0) if err != nil { t.Fatalf("build overlay root deltas: %v", err) } @@ -4527,6 +4799,56 @@ func TestBuildBufferedRootOverlayDeltaBatchPublishInputsPreservesColdTombstones( } } +func TestBuildBufferedRootDeltaBatchPublishInputsReusesReadOnlyPrepareResults(t *testing.T) { + const collectionName = "users" + primaryName := collectionPrimaryRootName(collectionName) + cityName := collectionSecondaryRootName(collectionName, "city") + + primaryTable := newCollectionRunTable(1) + setCollectionRunValue(primaryTable, []byte("u1"), []byte(`{"city":"hnl"}`)) + primaryTable.Freeze() + + cityTable := newCollectionRunTable(1) + setCollectionRunValue(cityTable, []byte("city:hnl/u1"), nil) + cityTable.Freeze() + defer resetCollectionTables([]memtable.Table{primaryTable, cityTable}) + + rootNames := []string{primaryName, cityName} + rootRuns := map[string][]memtable.Table{ + primaryName: {primaryTable}, + cityName: {cityTable}, + } + rootBaseIDs := map[string]uint64{ + primaryName: 42, + cityName: 43, + } + rootPolicies := map[string]backenddb.OrderedRootStoragePolicy{ + primaryName: backenddb.OrderedRootStorageValueLogLeaves, + cityName: backenddb.OrderedRootStoragePagerLeaves, + } + + ordered, cleanup, err := buildBufferedRootDeltaBatchPublishInputs(rootNames, rootRuns, rootBaseIDs, rootPolicies, true, 4) + if err != nil { + t.Fatalf("build buffered root deltas: %v", err) + } + defer cleanup() + + if got, want := len(ordered), len(rootNames); got != want { + t.Fatalf("ordered roots=%d want %d", got, want) + } + for i := range ordered { + if !ordered[i].PrepareReadOnly || ordered[i].ReadOnlyPrepareWorkerCount != 4 { + t.Fatalf("ordered[%d] read-only prepare=%t/%d want true/4", i, ordered[i].PrepareReadOnly, ordered[i].ReadOnlyPrepareWorkerCount) + } + if ordered[i].ReadOnlyPrepareResult == nil { + t.Fatalf("ordered[%d] ReadOnlyPrepareResult=nil want reusable result", i) + } + } + if ordered[0].ReadOnlyPrepareResult == ordered[1].ReadOnlyPrepareResult { + t.Fatal("ordered roots share one ReadOnlyPrepareResult") + } +} + func BenchmarkBufferedRootRunsIteratorBuildManyRuns(b *testing.B) { const runCount = 8192 runs := make([]memtable.Table, 0, runCount) @@ -4948,6 +5270,7 @@ func TestCollectionIndexedWriteMemtablesAsyncBackpressureWaitsForPublishingUnit( } backpressureDone := make(chan error, 1) + waitEntered, releaseWait := collectionWaitIndexedAsyncFlushGateForTest(t) go func() { col.writeDomain.mu.Lock() _, _, _, err := col.flushBufferedIndexedAfterThresholdLocked(col.writeDomain, CollectionOptions{ @@ -4959,20 +5282,17 @@ func TestCollectionIndexedWriteMemtablesAsyncBackpressureWaitsForPublishingUnit( col.writeDomain.mu.Unlock() backpressureDone <- err }() - deadline := time.Now().Add(200 * time.Millisecond) - for time.Now().Before(deadline) { - if mgr.StatsSnapshot().IndexedAsyncFlushBackpressure > 0 { - break - } - select { - case err := <-backpressureDone: - t.Fatalf("backpressure flush returned before in-flight async publish drained: %v", err) - case <-time.After(time.Millisecond): - } + select { + case <-waitEntered: + case err := <-backpressureDone: + t.Fatalf("backpressure flush returned before in-flight async publish drained: %v", err) + case <-time.After(collectionTestTimeout(t, 5*time.Second)): + t.Fatal("timed out waiting for backpressure flush to wait on in-flight async publish") } - if got := mgr.StatsSnapshot().IndexedAsyncFlushBackpressure; got == 0 { + if got := col.writeDomain.indexedAsyncFlushBackpressure.Load(); got == 0 { t.Fatal("async backpressure did not wait for in-flight publishing unit") } + releaseWait() publishDone := make(chan error, 1) go func() { @@ -5180,6 +5500,9 @@ func TestCollectionIndexedWriteMemtablesAsyncPublishingUnitsParticipateInReadsAn if err := col.publishPreparedIndexedFlush(work); err != nil { t.Fatalf("publish prepared async flush: %v", err) } + if got := work.batch.state; got != coalescedFlushBatchPublished { + t.Fatalf("published batch state=%d want published", got) + } if got := mgr.StatsSnapshot().PendingDocuments; got != 0 { t.Fatalf("pending docs after publish=%d want 0", got) } @@ -8883,6 +9206,7 @@ func TestCollectionUpdateBatchDirectBufferedTemplateV1AccumulatesRootRuns(t *tes primaryRuns := len(col.writeDomain.rootRuns[collectionPrimaryRootName("users")]) cityRuns := len(col.writeDomain.rootRuns[collectionSecondaryRootName("users", "city")]) rootMutableRuns := len(col.writeDomain.rootMutableRuns) + pendingSemanticRecords := pendingIndexedSemanticRecordCountLocked(col.writeDomain) col.writeDomain.mu.RUnlock() if rootRunCount != 3 { t.Fatalf("rootRunCount=%d want 3 accumulated roots after two template-v1 update batches", rootRunCount) @@ -8893,6 +9217,12 @@ func TestCollectionUpdateBatchDirectBufferedTemplateV1AccumulatesRootRuns(t *tes if rootMutableRuns != 3 { t.Fatalf("rootMutableRuns=%d want 3 active root-local accumulators", rootMutableRuns) } + if pendingSemanticRecords != 0 { + t.Fatalf("pending template-v1 semantic records=%d want 0 mechanical fallback records", pendingSemanticRecords) + } + if got := mgr.StatsSnapshot().IndexedSemanticRawRecords; got != 0 { + t.Fatalf("template-v1 raw semantic records=%d want 0 mechanical fallback records", got) + } seaIDs, err := col.FindByIndex("city", "sea") if err != nil { @@ -9276,7 +9606,7 @@ func TestCollectionUpdateBatchIfNoSecondaryUniqueIndexChangesRejectsStaleBuffere plan, err := col.buildUpdateBatchPlan([]UpdateBatchItem{ {DocumentID: []byte("u1"), Update: setJSONCity("sfo")}, - }, updateBatchModeNoSecondaryUniqueIndexChanges, true) + }, updateBatchModeNoSecondaryUniqueIndexChanges, true, CollectionUpdateStats{}) if err != nil { t.Fatalf("build stale plan: %v", err) } @@ -9328,7 +9658,7 @@ func TestCollectionUpdateBatchIfNoSecondaryUniqueIndexChangesRejectsStaleZeroDel return current, false, nil }, }, - }, updateBatchModeNoSecondaryUniqueIndexChanges, true) + }, updateBatchModeNoSecondaryUniqueIndexChanges, true, CollectionUpdateStats{}) if err != nil { t.Fatalf("build stale zero-delta plan: %v", err) } diff --git a/TreeDB/collections/direct_buffered_update_bench_test.go b/TreeDB/collections/direct_buffered_update_bench_test.go index 72942e8427..78e6134688 100644 --- a/TreeDB/collections/direct_buffered_update_bench_test.go +++ b/TreeDB/collections/direct_buffered_update_bench_test.go @@ -2,6 +2,12 @@ package collections import ( "fmt" + "os" + "path/filepath" + "runtime" + "runtime/pprof" + "strconv" + "strings" "testing" "time" @@ -11,16 +17,140 @@ import ( func BenchmarkCollectionUpdateBatchDirectBufferedTemplateV1NewShape(b *testing.B) { for _, batchSize := range []int{80, 16000} { b.Run(fmt.Sprintf("batch_%d", batchSize), func(b *testing.B) { - benchmarkCollectionUpdateBatchDirectBufferedTemplateV1NewShape(b, batchSize) + benchmarkCollectionUpdateBatchDirectBufferedTemplateV1NewShape(b, batchSize, collectionDirectBufferedBenchmarkOptions{}) }) } } -func benchmarkCollectionUpdateBatchDirectBufferedTemplateV1NewShape(b *testing.B, batchSize int) { +func BenchmarkCollectionUpdateBatchDirectBufferedTemplateV1NewShapeReadOnlyPrepare(b *testing.B) { + for _, tc := range []struct { + name string + opts collectionDirectBufferedBenchmarkOptions + }{ + { + name: "prepare", + opts: collectionDirectBufferedBenchmarkOptions{ + readOnlyPrepare: true, + }, + }, + { + name: "prepare_workers_4", + opts: collectionDirectBufferedBenchmarkOptions{ + readOnlyPrepare: true, + readOnlyPrepareWorkers: 4, + }, + }, + } { + b.Run(tc.name, func(b *testing.B) { + for _, batchSize := range []int{80, 16000} { + b.Run(fmt.Sprintf("batch_%d", batchSize), func(b *testing.B) { + benchmarkCollectionUpdateBatchDirectBufferedTemplateV1NewShape(b, batchSize, tc.opts) + }) + } + }) + } +} + +type collectionDirectBufferedBenchmarkOptions struct { + readOnlyPrepare bool + readOnlyPrepareWorkers int +} + +func configureUpdateBatchTimedAllocsProfileRate(b *testing.B) func() { + b.Helper() + basePath := strings.TrimSpace(os.Getenv("TREEDB_COLLECTION_TIMED_ALLOCS_BASE_PROFILE_PATH")) + afterPath := strings.TrimSpace(os.Getenv("TREEDB_COLLECTION_TIMED_ALLOCS_AFTER_PROFILE_PATH")) + if basePath == "" && afterPath == "" { + return func() {} + } + if basePath == "" || afterPath == "" { + b.Fatalf("set both TREEDB_COLLECTION_TIMED_ALLOCS_BASE_PROFILE_PATH and TREEDB_COLLECTION_TIMED_ALLOCS_AFTER_PROFILE_PATH") + } + rate := 1 + if raw := strings.TrimSpace(os.Getenv("TREEDB_COLLECTION_TIMED_ALLOCS_PROFILE_RATE")); raw != "" { + n, err := strconv.Atoi(raw) + if err != nil || n <= 0 { + b.Fatalf("unsupported TREEDB_COLLECTION_TIMED_ALLOCS_PROFILE_RATE=%q", raw) + } + rate = n + } + prevRate := runtime.MemProfileRate + runtime.MemProfileRate = rate + return func() { + runtime.MemProfileRate = prevRate + } +} + +func startUpdateBatchTimedCPUProfile(b *testing.B) func() { + b.Helper() + profilePath := strings.TrimSpace(os.Getenv("TREEDB_COLLECTION_TIMED_CPU_PROFILE_PATH")) + if profilePath == "" { + return func() {} + } + if dir := filepath.Dir(profilePath); dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0o755); err != nil { + b.Fatalf("create timed cpu profile dir: %v", err) + } + } + file, err := os.Create(profilePath) + if err != nil { + b.Fatalf("create timed cpu profile: %v", err) + } + if err := pprof.StartCPUProfile(file); err != nil { + _ = file.Close() + b.Fatalf("start timed cpu profile: %v; do not also pass go test -cpuprofile", err) + } + stopped := false + return func() { + if stopped { + return + } + pprof.StopCPUProfile() + stopped = true + if err := file.Close(); err != nil { + b.Errorf("close timed cpu profile: %v", err) + } + } +} + +func writeUpdateBatchTimedAllocsSnapshot(b *testing.B, path string) { + b.Helper() + path = strings.TrimSpace(path) + if path == "" { + return + } + if dir := filepath.Dir(path); dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0o755); err != nil { + b.Fatalf("create timed allocs profile dir: %v", err) + } + } + runtime.GC() + runtime.GC() + file, err := os.Create(path) + if err != nil { + b.Fatalf("create timed allocs profile: %v", err) + } + defer func() { + if err := file.Close(); err != nil { + b.Errorf("close timed allocs profile: %v", err) + } + }() + prof := pprof.Lookup("allocs") + if prof == nil { + b.Fatalf("allocs profile unavailable") + } + if err := prof.WriteTo(file, 0); err != nil { + b.Fatalf("write timed allocs profile: %v", err) + } +} + +func benchmarkCollectionUpdateBatchDirectBufferedTemplateV1NewShape(b *testing.B, batchSize int, opts collectionDirectBufferedBenchmarkOptions) { b.Helper() if batchSize <= 0 { b.Fatalf("invalid batch size %d", batchSize) } + restoreMemProfileRate := configureUpdateBatchTimedAllocsProfileRate(b) + defer restoreMemProfileRate() docs := b.N if docs <= 0 { docs = 1 @@ -36,12 +166,14 @@ func benchmarkCollectionUpdateBatchDirectBufferedTemplateV1NewShape(b *testing.B if _, err := mgr.CreateCollection(&CollectionMeta{ Name: "bench", Options: CollectionOptions{ - DocumentFormat: DocumentFormatTemplateV1, - BufferedIndexedWrites: true, - BufferedIndexedWriteMaxDocuments: 1 << 30, - BufferedIndexedWriteMaxBytes: 1 << 40, - BufferedIndexedWriteMaxRootRuns: 1 << 30, - BufferedIndexedAsyncFlushMaxQueuedUnits: 1 << 20, + DocumentFormat: DocumentFormatTemplateV1, + BufferedIndexedWrites: true, + BufferedIndexedWriteMaxDocuments: 1 << 30, + BufferedIndexedWriteMaxBytes: 1 << 40, + BufferedIndexedWriteMaxRootRuns: 1 << 30, + BufferedIndexedAsyncFlushMaxQueuedUnits: 1 << 20, + BufferedIndexedReadOnlyPrepare: opts.readOnlyPrepare, + BufferedIndexedReadOnlyPrepareWorkerCount: opts.readOnlyPrepareWorkers, }, Indexes: []IndexDefinition{ {Name: "email", Field: "email", ValueType: IndexValueString, Unique: true}, @@ -89,8 +221,17 @@ func benchmarkCollectionUpdateBatchDirectBufferedTemplateV1NewShape(b *testing.B batch := make([]UpdateBatchItem, batchSize) statsBefore := mgr.StatsSnapshot() + dbStatsBefore := d.Stats() + writeUpdateBatchTimedAllocsSnapshot(b, os.Getenv("TREEDB_COLLECTION_TIMED_ALLOCS_BASE_PROFILE_PATH")) b.ReportAllocs() b.ResetTimer() + stopProfile := startUpdateBatchTimedCPUProfile(b) + profileActive := true + defer func() { + if profileActive { + stopProfile() + } + }() startTime := time.Now() for start := 0; start < docs; start += batchSize { n := batchSize @@ -113,12 +254,17 @@ func benchmarkCollectionUpdateBatchDirectBufferedTemplateV1NewShape(b *testing.B b.Fatalf("flush updates: %v", err) } elapsed := time.Since(startTime) + stopProfile() + profileActive = false b.StopTimer() + writeUpdateBatchTimedAllocsSnapshot(b, os.Getenv("TREEDB_COLLECTION_TIMED_ALLOCS_AFTER_PROFILE_PATH")) stats := collectionManagerStatsBenchmarkDelta(mgr.StatsSnapshot(), statsBefore) + dbStatsAfter := d.Stats() b.ReportMetric(float64(docs)/elapsed.Seconds(), "docs/sec") b.ReportMetric(float64(elapsed.Nanoseconds())/float64(docs), "ns/doc") reportCollectionUpdateStatsForBenchmark(b, stats, docs) + reportCollectionReadOnlyPrepareDBStatsForBenchmark(b, dbStatsAfter, dbStatsBefore, docs) } func benchmarkTemplateV1UpdateDocID(n int) []byte { @@ -168,45 +314,108 @@ func benchmarkTemplateV1ReplaceWith(raw []byte) func([]byte) ([]byte, bool, erro func collectionManagerStatsBenchmarkDelta(after, before CollectionManagerStats) CollectionManagerStats { return CollectionManagerStats{ - IndexedStageBatches: after.IndexedStageBatches - before.IndexedStageBatches, - IndexedStageDocs: after.IndexedStageDocs - before.IndexedStageDocs, - IndexedStageBytes: after.IndexedStageBytes - before.IndexedStageBytes, - IndexedStageRootRuns: after.IndexedStageRootRuns - before.IndexedStageRootRuns, - IndexedFlushCalls: after.IndexedFlushCalls - before.IndexedFlushCalls, - IndexedFlushErrors: after.IndexedFlushErrors - before.IndexedFlushErrors, - IndexedFlushDocs: after.IndexedFlushDocs - before.IndexedFlushDocs, - IndexedFlushBytes: after.IndexedFlushBytes - before.IndexedFlushBytes, - IndexedFlushRootRuns: after.IndexedFlushRootRuns - before.IndexedFlushRootRuns, - IndexedFlushRoots: after.IndexedFlushRoots - before.IndexedFlushRoots, - IndexedFlushDuration: after.IndexedFlushDuration - before.IndexedFlushDuration, - IndexedFlushMaterialize: after.IndexedFlushMaterialize - before.IndexedFlushMaterialize, - IndexedFlushPublish: after.IndexedFlushPublish - before.IndexedFlushPublish, - UpdateBatchCalls: after.UpdateBatchCalls - before.UpdateBatchCalls, - UpdateBatchItems: after.UpdateBatchItems - before.UpdateBatchItems, - UpdateBatchMatched: after.UpdateBatchMatched - before.UpdateBatchMatched, - UpdateBatchModified: after.UpdateBatchModified - before.UpdateBatchModified, - UpdateBatchRuns: after.UpdateBatchRuns - before.UpdateBatchRuns, - UpdateBatchBufferedBatches: after.UpdateBatchBufferedBatches - before.UpdateBatchBufferedBatches, - UpdateBatchCurrentRead: after.UpdateBatchCurrentRead - before.UpdateBatchCurrentRead, - UpdateBatchCallback: after.UpdateBatchCallback - before.UpdateBatchCallback, - UpdateBatchPrepareDocuments: after.UpdateBatchPrepareDocuments - before.UpdateBatchPrepareDocuments, - UpdateBatchIndexStateExtract: after.UpdateBatchIndexStateExtract - before.UpdateBatchIndexStateExtract, - UpdateBatchUniquePreflight: after.UpdateBatchUniquePreflight - before.UpdateBatchUniquePreflight, - UpdateBatchTemplateRunBuild: after.UpdateBatchTemplateRunBuild - before.UpdateBatchTemplateRunBuild, - UpdateBatchPrimaryRunBuild: after.UpdateBatchPrimaryRunBuild - before.UpdateBatchPrimaryRunBuild, - UpdateBatchSecondaryRunBuild: after.UpdateBatchSecondaryRunBuild - before.UpdateBatchSecondaryRunBuild, - UpdateBatchBufferStage: after.UpdateBatchBufferStage - before.UpdateBatchBufferStage, - UpdateBatchBufferLockWait: after.UpdateBatchBufferLockWait - before.UpdateBatchBufferLockWait, - UpdateBatchBufferLockHold: after.UpdateBatchBufferLockHold - before.UpdateBatchBufferLockHold, - UpdateBatchBufferRootAppend: after.UpdateBatchBufferRootAppend - before.UpdateBatchBufferRootAppend, - UpdateBatchPublish: after.UpdateBatchPublish - before.UpdateBatchPublish, - UpdateBatchSecondaryDeletes: after.UpdateBatchSecondaryDeletes - before.UpdateBatchSecondaryDeletes, - UpdateBatchSecondarySets: after.UpdateBatchSecondarySets - before.UpdateBatchSecondarySets, - UpdateBatchSecondaryKeyBytes: after.UpdateBatchSecondaryKeyBytes - before.UpdateBatchSecondaryKeyBytes, - UpdateBatchIndexValueChanges: after.UpdateBatchIndexValueChanges - before.UpdateBatchIndexValueChanges, - UpdateBatchIndexValueUnchanged: after.UpdateBatchIndexValueUnchanged - before.UpdateBatchIndexValueUnchanged, - UpdateBatchUniqueChecks: after.UpdateBatchUniqueChecks - before.UpdateBatchUniqueChecks, - UpdateBatchUniqueCheckSkips: after.UpdateBatchUniqueCheckSkips - before.UpdateBatchUniqueCheckSkips, + IndexedStageBatches: after.IndexedStageBatches - before.IndexedStageBatches, + IndexedStageDocs: after.IndexedStageDocs - before.IndexedStageDocs, + IndexedStageBytes: after.IndexedStageBytes - before.IndexedStageBytes, + IndexedStageRootRuns: after.IndexedStageRootRuns - before.IndexedStageRootRuns, + IndexedFlushCalls: after.IndexedFlushCalls - before.IndexedFlushCalls, + IndexedFlushErrors: after.IndexedFlushErrors - before.IndexedFlushErrors, + IndexedFlushDocs: after.IndexedFlushDocs - before.IndexedFlushDocs, + IndexedFlushBytes: after.IndexedFlushBytes - before.IndexedFlushBytes, + IndexedFlushRootRuns: after.IndexedFlushRootRuns - before.IndexedFlushRootRuns, + IndexedFlushRoots: after.IndexedFlushRoots - before.IndexedFlushRoots, + IndexedFlushPreflight: after.IndexedFlushPreflight - before.IndexedFlushPreflight, + IndexedFlushRotate: after.IndexedFlushRotate - before.IndexedFlushRotate, + IndexedFlushMerge: after.IndexedFlushMerge - before.IndexedFlushMerge, + IndexedFlushDuration: after.IndexedFlushDuration - before.IndexedFlushDuration, + IndexedFlushMaterialize: after.IndexedFlushMaterialize - before.IndexedFlushMaterialize, + IndexedFlushSemanticPlan: after.IndexedFlushSemanticPlan - before.IndexedFlushSemanticPlan, + IndexedFlushBuildInputs: after.IndexedFlushBuildInputs - before.IndexedFlushBuildInputs, + IndexedFlushPlanStats: after.IndexedFlushPlanStats - before.IndexedFlushPlanStats, + IndexedFlushPublish: after.IndexedFlushPublish - before.IndexedFlushPublish, + CoalescedFlushBatches: after.CoalescedFlushBatches - before.CoalescedFlushBatches, + CoalescedFlushBatchUnits: after.CoalescedFlushBatchUnits - before.CoalescedFlushBatchUnits, + CoalescedFlushBatchDocs: after.CoalescedFlushBatchDocs - before.CoalescedFlushBatchDocs, + CoalescedFlushBatchBytes: after.CoalescedFlushBatchBytes - before.CoalescedFlushBatchBytes, + CoalescedFlushNetZeroBatches: after.CoalescedFlushNetZeroBatches - before.CoalescedFlushNetZeroBatches, + RootDeltaPlanPrimaryRoots: after.RootDeltaPlanPrimaryRoots - before.RootDeltaPlanPrimaryRoots, + RootDeltaPlanTemplateRoots: after.RootDeltaPlanTemplateRoots - before.RootDeltaPlanTemplateRoots, + RootDeltaPlanIndexStateRoots: after.RootDeltaPlanIndexStateRoots - before.RootDeltaPlanIndexStateRoots, + RootDeltaPlanSecondaryRoots: after.RootDeltaPlanSecondaryRoots - before.RootDeltaPlanSecondaryRoots, + RootDeltaPlanEntries: after.RootDeltaPlanEntries - before.RootDeltaPlanEntries, + RootDeltaPlanKeyBytes: after.RootDeltaPlanKeyBytes - before.RootDeltaPlanKeyBytes, + RootDeltaPlanValueBytes: after.RootDeltaPlanValueBytes - before.RootDeltaPlanValueBytes, + RootDeltaPlanTombstones: after.RootDeltaPlanTombstones - before.RootDeltaPlanTombstones, + RootDeltaPlanRawUnitPrimaryEntries: after.RootDeltaPlanRawUnitPrimaryEntries - before.RootDeltaPlanRawUnitPrimaryEntries, + RootDeltaPlanRawUnitPrimaryBytes: after.RootDeltaPlanRawUnitPrimaryBytes - before.RootDeltaPlanRawUnitPrimaryBytes, + RootDeltaPlanRawUnitPrimaryTombstones: after.RootDeltaPlanRawUnitPrimaryTombstones - before.RootDeltaPlanRawUnitPrimaryTombstones, + RootDeltaPlanRawUnitTemplateEntries: after.RootDeltaPlanRawUnitTemplateEntries - before.RootDeltaPlanRawUnitTemplateEntries, + RootDeltaPlanRawUnitTemplateBytes: after.RootDeltaPlanRawUnitTemplateBytes - before.RootDeltaPlanRawUnitTemplateBytes, + RootDeltaPlanRawUnitTemplateTombstones: after.RootDeltaPlanRawUnitTemplateTombstones - before.RootDeltaPlanRawUnitTemplateTombstones, + RootDeltaPlanRawUnitIndexStateEntries: after.RootDeltaPlanRawUnitIndexStateEntries - before.RootDeltaPlanRawUnitIndexStateEntries, + RootDeltaPlanRawUnitIndexStateBytes: after.RootDeltaPlanRawUnitIndexStateBytes - before.RootDeltaPlanRawUnitIndexStateBytes, + RootDeltaPlanRawUnitIndexStateTombstones: after.RootDeltaPlanRawUnitIndexStateTombstones - before.RootDeltaPlanRawUnitIndexStateTombstones, + RootDeltaPlanRawUnitSecondaryEntries: after.RootDeltaPlanRawUnitSecondaryEntries - before.RootDeltaPlanRawUnitSecondaryEntries, + RootDeltaPlanRawUnitSecondaryBytes: after.RootDeltaPlanRawUnitSecondaryBytes - before.RootDeltaPlanRawUnitSecondaryBytes, + RootDeltaPlanRawUnitSecondaryTombstones: after.RootDeltaPlanRawUnitSecondaryTombstones - before.RootDeltaPlanRawUnitSecondaryTombstones, + RootDeltaPlanFinalPrimaryEntries: after.RootDeltaPlanFinalPrimaryEntries - before.RootDeltaPlanFinalPrimaryEntries, + RootDeltaPlanFinalPrimaryBytes: after.RootDeltaPlanFinalPrimaryBytes - before.RootDeltaPlanFinalPrimaryBytes, + RootDeltaPlanFinalPrimaryTombstones: after.RootDeltaPlanFinalPrimaryTombstones - before.RootDeltaPlanFinalPrimaryTombstones, + RootDeltaPlanFinalTemplateEntries: after.RootDeltaPlanFinalTemplateEntries - before.RootDeltaPlanFinalTemplateEntries, + RootDeltaPlanFinalTemplateBytes: after.RootDeltaPlanFinalTemplateBytes - before.RootDeltaPlanFinalTemplateBytes, + RootDeltaPlanFinalTemplateTombstones: after.RootDeltaPlanFinalTemplateTombstones - before.RootDeltaPlanFinalTemplateTombstones, + RootDeltaPlanFinalIndexStateEntries: after.RootDeltaPlanFinalIndexStateEntries - before.RootDeltaPlanFinalIndexStateEntries, + RootDeltaPlanFinalIndexStateBytes: after.RootDeltaPlanFinalIndexStateBytes - before.RootDeltaPlanFinalIndexStateBytes, + RootDeltaPlanFinalIndexStateTombstones: after.RootDeltaPlanFinalIndexStateTombstones - before.RootDeltaPlanFinalIndexStateTombstones, + RootDeltaPlanFinalSecondaryEntries: after.RootDeltaPlanFinalSecondaryEntries - before.RootDeltaPlanFinalSecondaryEntries, + RootDeltaPlanFinalSecondaryBytes: after.RootDeltaPlanFinalSecondaryBytes - before.RootDeltaPlanFinalSecondaryBytes, + RootDeltaPlanFinalSecondaryTombstones: after.RootDeltaPlanFinalSecondaryTombstones - before.RootDeltaPlanFinalSecondaryTombstones, + RootDeltaPlanSquashedEntries: after.RootDeltaPlanSquashedEntries - before.RootDeltaPlanSquashedEntries, + RootDeltaPlanNetZeroPlans: after.RootDeltaPlanNetZeroPlans - before.RootDeltaPlanNetZeroPlans, + UpdateBatchCalls: after.UpdateBatchCalls - before.UpdateBatchCalls, + UpdateBatchItems: after.UpdateBatchItems - before.UpdateBatchItems, + UpdateBatchMatched: after.UpdateBatchMatched - before.UpdateBatchMatched, + UpdateBatchModified: after.UpdateBatchModified - before.UpdateBatchModified, + UpdateBatchRuns: after.UpdateBatchRuns - before.UpdateBatchRuns, + UpdateBatchBufferedBatches: after.UpdateBatchBufferedBatches - before.UpdateBatchBufferedBatches, + UpdateBatchValidate: after.UpdateBatchValidate - before.UpdateBatchValidate, + UpdateBatchClone: after.UpdateBatchClone - before.UpdateBatchClone, + UpdateBatchPlanSetup: after.UpdateBatchPlanSetup - before.UpdateBatchPlanSetup, + UpdateBatchBufferedReadSnapshot: after.UpdateBatchBufferedReadSnapshot - before.UpdateBatchBufferedReadSnapshot, + UpdateBatchCurrentRead: after.UpdateBatchCurrentRead - before.UpdateBatchCurrentRead, + UpdateBatchBSONIDValidation: after.UpdateBatchBSONIDValidation - before.UpdateBatchBSONIDValidation, + UpdateBatchCallback: after.UpdateBatchCallback - before.UpdateBatchCallback, + UpdateBatchReplacementStage: after.UpdateBatchReplacementStage - before.UpdateBatchReplacementStage, + UpdateBatchPrepareDocuments: after.UpdateBatchPrepareDocuments - before.UpdateBatchPrepareDocuments, + UpdateBatchIndexStateExtract: after.UpdateBatchIndexStateExtract - before.UpdateBatchIndexStateExtract, + UpdateBatchIndexStateCompare: after.UpdateBatchIndexStateCompare - before.UpdateBatchIndexStateCompare, + UpdateBatchUniquePreflight: after.UpdateBatchUniquePreflight - before.UpdateBatchUniquePreflight, + UpdateBatchTemplateRunBuild: after.UpdateBatchTemplateRunBuild - before.UpdateBatchTemplateRunBuild, + UpdateBatchPrimaryRunBuild: after.UpdateBatchPrimaryRunBuild - before.UpdateBatchPrimaryRunBuild, + UpdateBatchIndexStateRunBuild: after.UpdateBatchIndexStateRunBuild - before.UpdateBatchIndexStateRunBuild, + UpdateBatchSecondaryRunBuild: after.UpdateBatchSecondaryRunBuild - before.UpdateBatchSecondaryRunBuild, + UpdateBatchSemanticRecordBuild: after.UpdateBatchSemanticRecordBuild - before.UpdateBatchSemanticRecordBuild, + UpdateBatchBufferStage: after.UpdateBatchBufferStage - before.UpdateBatchBufferStage, + UpdateBatchBufferPrecheck: after.UpdateBatchBufferPrecheck - before.UpdateBatchBufferPrecheck, + UpdateBatchBufferLockWait: after.UpdateBatchBufferLockWait - before.UpdateBatchBufferLockWait, + UpdateBatchBufferLockHold: after.UpdateBatchBufferLockHold - before.UpdateBatchBufferLockHold, + UpdateBatchBufferValidation: after.UpdateBatchBufferValidation - before.UpdateBatchBufferValidation, + UpdateBatchBufferRootScan: after.UpdateBatchBufferRootScan - before.UpdateBatchBufferRootScan, + UpdateBatchBufferDomainPrepare: after.UpdateBatchBufferDomainPrepare - before.UpdateBatchBufferDomainPrepare, + UpdateBatchBufferPrimaryIdx: after.UpdateBatchBufferPrimaryIdx - before.UpdateBatchBufferPrimaryIdx, + UpdateBatchBufferUniqueIdx: after.UpdateBatchBufferUniqueIdx - before.UpdateBatchBufferUniqueIdx, + UpdateBatchBufferRootAppend: after.UpdateBatchBufferRootAppend - before.UpdateBatchBufferRootAppend, + UpdateBatchBufferSemanticAppend: after.UpdateBatchBufferSemanticAppend - before.UpdateBatchBufferSemanticAppend, + UpdateBatchBufferFlush: after.UpdateBatchBufferFlush - before.UpdateBatchBufferFlush, + UpdateBatchPlanClose: after.UpdateBatchPlanClose - before.UpdateBatchPlanClose, + UpdateBatchPublish: after.UpdateBatchPublish - before.UpdateBatchPublish, + UpdateBatchSecondaryDeletes: after.UpdateBatchSecondaryDeletes - before.UpdateBatchSecondaryDeletes, + UpdateBatchSecondarySets: after.UpdateBatchSecondarySets - before.UpdateBatchSecondarySets, + UpdateBatchSecondaryKeyBytes: after.UpdateBatchSecondaryKeyBytes - before.UpdateBatchSecondaryKeyBytes, + UpdateBatchIndexValueChanges: after.UpdateBatchIndexValueChanges - before.UpdateBatchIndexValueChanges, + UpdateBatchIndexValueUnchanged: after.UpdateBatchIndexValueUnchanged - before.UpdateBatchIndexValueUnchanged, + UpdateBatchUniqueChecks: after.UpdateBatchUniqueChecks - before.UpdateBatchUniqueChecks, + UpdateBatchUniqueCheckSkips: after.UpdateBatchUniqueCheckSkips - before.UpdateBatchUniqueCheckSkips, } } @@ -242,9 +451,16 @@ func reportCollectionUpdateStatsForBenchmark(b *testing.B, stats CollectionManag reportUintPerDoc(stats.IndexedFlushBytes, "indexed_flush_bytes/doc") reportUintPerDoc(stats.IndexedFlushRootRuns, "indexed_flush_root_runs/doc") reportUintPerDoc(stats.IndexedFlushRoots, "indexed_flush_roots/doc") + reportDurationPerDoc(stats.IndexedFlushPreflight, "indexed_flush_preflight_ns/doc") + reportDurationPerDoc(stats.IndexedFlushRotate, "indexed_flush_rotate_ns/doc") + reportDurationPerDoc(stats.IndexedFlushMerge, "indexed_flush_merge_ns/doc") reportDurationPerDoc(stats.IndexedFlushDuration, "indexed_flush_ns/doc") reportDurationPerDoc(stats.IndexedFlushMaterialize, "indexed_flush_materialize_ns/doc") + reportDurationPerDoc(stats.IndexedFlushSemanticPlan, "indexed_flush_semantic_plan_ns/doc") + reportDurationPerDoc(stats.IndexedFlushBuildInputs, "indexed_flush_build_inputs_ns/doc") + reportDurationPerDoc(stats.IndexedFlushPlanStats, "indexed_flush_plan_stats_ns/doc") reportDurationPerDoc(stats.IndexedFlushPublish, "indexed_flush_publish_ns/doc") + reportCollectionRootDeltaShapeStatsForBenchmark(b, stats, docs) if stats.UpdateBatchCalls > 0 { b.ReportMetric(float64(stats.UpdateBatchCalls), "update_batches") b.ReportMetric(float64(stats.UpdateBatchItems)/float64(stats.UpdateBatchCalls), "update_items/batch") @@ -259,17 +475,137 @@ func reportCollectionUpdateStatsForBenchmark(b *testing.B, stats CollectionManag reportUintPerDoc(stats.UpdateBatchIndexValueUnchanged, "update_index_value_unchanged/doc") reportUintPerDoc(stats.UpdateBatchUniqueChecks, "update_unique_checks/doc") reportUintPerDoc(stats.UpdateBatchUniqueCheckSkips, "update_unique_check_skips/doc") + reportDurationPerDoc(stats.UpdateBatchValidate, "update_validate_items_ns/doc") + reportDurationPerDoc(stats.UpdateBatchClone, "update_clone_items_ns/doc") + reportDurationPerDoc(stats.UpdateBatchPlanSetup, "update_plan_setup_ns/doc") + reportDurationPerDoc(stats.UpdateBatchBufferedReadSnapshot, "update_buffered_read_snapshot_ns/doc") reportDurationPerDoc(stats.UpdateBatchCurrentRead, "update_current_read_ns/doc") + reportDurationPerDoc(stats.UpdateBatchBSONIDValidation, "update_bson_id_validation_ns/doc") reportDurationPerDoc(stats.UpdateBatchCallback, "update_callback_ns/doc") + reportDurationPerDoc(stats.UpdateBatchReplacementStage, "update_replacement_stage_ns/doc") reportDurationPerDoc(stats.UpdateBatchPrepareDocuments, "update_prepare_ns/doc") reportDurationPerDoc(stats.UpdateBatchIndexStateExtract, "update_index_state_extract_ns/doc") + reportDurationPerDoc(stats.UpdateBatchIndexStateCompare, "update_index_state_compare_ns/doc") reportDurationPerDoc(stats.UpdateBatchUniquePreflight, "update_unique_preflight_ns/doc") reportDurationPerDoc(stats.UpdateBatchTemplateRunBuild, "update_template_run_build_ns/doc") reportDurationPerDoc(stats.UpdateBatchPrimaryRunBuild, "update_primary_run_build_ns/doc") + reportDurationPerDoc(stats.UpdateBatchIndexStateRunBuild, "update_index_state_run_build_ns/doc") reportDurationPerDoc(stats.UpdateBatchSecondaryRunBuild, "update_secondary_run_build_ns/doc") + reportDurationPerDoc(stats.UpdateBatchSemanticRecordBuild, "update_semantic_record_build_ns/doc") reportDurationPerDoc(stats.UpdateBatchBufferStage, "update_buffer_stage_ns/doc") + reportDurationPerDoc(stats.UpdateBatchBufferPrecheck, "update_buffer_precheck_ns/doc") reportDurationPerDoc(stats.UpdateBatchBufferLockWait, "update_buffer_lock_wait_ns/doc") reportDurationPerDoc(stats.UpdateBatchBufferLockHold, "update_buffer_lock_hold_ns/doc") + reportDurationPerDoc(stats.UpdateBatchBufferValidation, "update_buffer_validation_ns/doc") + reportDurationPerDoc(stats.UpdateBatchBufferRootScan, "update_buffer_root_scan_ns/doc") + reportDurationPerDoc(stats.UpdateBatchBufferDomainPrepare, "update_buffer_domain_prepare_ns/doc") + reportDurationPerDoc(stats.UpdateBatchBufferPrimaryIdx, "update_buffer_primary_index_ns/doc") + reportDurationPerDoc(stats.UpdateBatchBufferUniqueIdx, "update_buffer_unique_index_ns/doc") reportDurationPerDoc(stats.UpdateBatchBufferRootAppend, "update_buffer_root_append_ns/doc") + reportDurationPerDoc(stats.UpdateBatchBufferSemanticAppend, "update_buffer_semantic_append_ns/doc") + reportDurationPerDoc(stats.UpdateBatchBufferFlush, "update_buffer_flush_ns/doc") + reportDurationPerDoc(stats.UpdateBatchPlanClose, "update_plan_close_ns/doc") reportDurationPerDoc(stats.UpdateBatchPublish, "update_publish_ns/doc") } + +func reportCollectionReadOnlyPrepareDBStatsForBenchmark(b *testing.B, after, before map[string]string, docs int) { + b.Helper() + if docs <= 0 { + return + } + calls := benchmarkDBStatDelta(b, after, before, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total") + if calls == 0 { + return + } + prepareNS := benchmarkDBStatDelta(b, after, before, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ns_total") + ops := benchmarkDBStatDelta(b, after, before, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ops_total") + leafSpans := benchmarkDBStatDelta(b, after, before, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_leaf_spans_total") + workerTargets := benchmarkDBStatDelta(b, after, before, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_targets_total") + workerRanges := benchmarkDBStatDelta(b, after, before, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_ranges_total") + workerMaxOps := benchmarkDBStatDelta(b, after, before, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_range_max_ops_total") + + b.ReportMetric(float64(calls)/float64(docs), "read_only_prepare_calls/doc") + b.ReportMetric(float64(prepareNS)/float64(docs), "read_only_prepare_ns/doc") + b.ReportMetric(float64(prepareNS)/float64(calls), "read_only_prepare_ns/plan") + b.ReportMetric(float64(ops)/float64(docs), "read_only_prepare_ops/doc") + b.ReportMetric(float64(leafSpans)/float64(calls), "read_only_prepare_leaf_spans/plan") + if workerTargets > 0 { + b.ReportMetric(float64(workerTargets)/float64(calls), "read_only_prepare_worker_targets/plan") + } + if workerRanges > 0 { + b.ReportMetric(float64(workerRanges)/float64(calls), "read_only_prepare_worker_ranges/plan") + } + if workerMaxOps > 0 { + b.ReportMetric(float64(workerMaxOps)/float64(calls), "read_only_prepare_worker_max_ops/plan") + } +} + +func reportCollectionRootDeltaShapeStatsForBenchmark(b *testing.B, stats CollectionManagerStats, docs int) { + b.Helper() + if docs <= 0 { + return + } + reportUintPerDoc := func(value uint64, name string) { + if value > 0 { + b.ReportMetric(float64(value)/float64(docs), name) + } + } + if stats.CoalescedFlushBatches > 0 { + b.ReportMetric(float64(stats.CoalescedFlushBatchUnits)/float64(stats.CoalescedFlushBatches), "coalesced_flush_units/batch") + b.ReportMetric(float64(stats.CoalescedFlushBatchDocs)/float64(stats.CoalescedFlushBatches), "coalesced_flush_docs/batch") + } + reportUintPerDoc(stats.CoalescedFlushBatchUnits, "coalesced_flush_units/doc") + reportUintPerDoc(stats.CoalescedFlushBatchDocs, "coalesced_flush_docs/doc") + reportUintPerDoc(stats.CoalescedFlushBatchBytes, "coalesced_flush_bytes/doc") + reportUintPerDoc(stats.CoalescedFlushNetZeroBatches, "coalesced_flush_net_zero_batches/doc") + rawEntries := stats.RootDeltaPlanRawUnitPrimaryEntries + + stats.RootDeltaPlanRawUnitTemplateEntries + + stats.RootDeltaPlanRawUnitIndexStateEntries + + stats.RootDeltaPlanRawUnitSecondaryEntries + finalEntries := stats.RootDeltaPlanFinalPrimaryEntries + + stats.RootDeltaPlanFinalTemplateEntries + + stats.RootDeltaPlanFinalIndexStateEntries + + stats.RootDeltaPlanFinalSecondaryEntries + reportUintPerDoc(rawEntries, "raw_root_delta_entries/doc") + reportUintPerDoc(finalEntries, "final_root_delta_entries/doc") + reportUintPerDoc(stats.RootDeltaPlanEntries, "root_delta_plan_entries/doc") + reportUintPerDoc(stats.RootDeltaPlanRawUnitPrimaryEntries, "raw_primary_entries/doc") + reportUintPerDoc(stats.RootDeltaPlanRawUnitTemplateEntries, "raw_template_entries/doc") + reportUintPerDoc(stats.RootDeltaPlanRawUnitIndexStateEntries, "raw_index_state_entries/doc") + reportUintPerDoc(stats.RootDeltaPlanRawUnitSecondaryEntries, "raw_secondary_entries/doc") + reportUintPerDoc(stats.RootDeltaPlanFinalPrimaryEntries, "final_primary_entries/doc") + reportUintPerDoc(stats.RootDeltaPlanFinalTemplateEntries, "final_template_entries/doc") + reportUintPerDoc(stats.RootDeltaPlanFinalIndexStateEntries, "final_index_state_entries/doc") + reportUintPerDoc(stats.RootDeltaPlanFinalSecondaryEntries, "final_secondary_entries/doc") + reportUintPerDoc(stats.RootDeltaPlanSquashedEntries, "root_delta_squashed_entries/doc") + reportUintPerDoc(stats.RootDeltaPlanNetZeroPlans, "root_delta_net_zero_plans/doc") + if rawEntries > 0 { + b.ReportMetric(float64(finalEntries)/float64(rawEntries), "final/raw_root_delta_entries") + } +} + +func benchmarkDBStatDelta(tb testing.TB, after, before map[string]string, key string) uint64 { + tb.Helper() + afterValue := benchmarkDBStatUint(tb, after, key) + beforeValue := benchmarkDBStatUint(tb, before, key) + if afterValue < beforeValue { + return 0 + } + return afterValue - beforeValue +} + +func benchmarkDBStatUint(tb testing.TB, stats map[string]string, key string) uint64 { + tb.Helper() + if stats == nil { + return 0 + } + value, ok := stats[key] + if !ok || value == "" { + return 0 + } + parsed, err := strconv.ParseUint(value, 10, 64) + if err != nil { + tb.Fatalf("parse db stat %s=%q: %v", key, value, err) + } + return parsed +} diff --git a/TreeDB/collections/flush_after_staging_bench_test.go b/TreeDB/collections/flush_after_staging_bench_test.go new file mode 100644 index 0000000000..19d0d49fc4 --- /dev/null +++ b/TreeDB/collections/flush_after_staging_bench_test.go @@ -0,0 +1,249 @@ +package collections + +import ( + "fmt" + "strconv" + "testing" + "time" + + backenddb "github.com/snissn/gomap/TreeDB/db" +) + +type collectionFlushAfterStagingShape string + +const ( + collectionFlushRepeatedSameIDIndexedChangeBack collectionFlushAfterStagingShape = "repeated_same_id_indexed_change_back" + collectionFlushRepeatedSameIDNonIndexedUpdate collectionFlushAfterStagingShape = "repeated_same_id_non_indexed_update" + collectionFlushManyIDsIndexedChanges collectionFlushAfterStagingShape = "many_ids_indexed_changes" + collectionFlushManyIDsNonIndexedChanges collectionFlushAfterStagingShape = "many_ids_non_indexed_changes" +) + +func BenchmarkCollectionFlushAfterStaging(b *testing.B) { + shapes := []collectionFlushAfterStagingShape{ + collectionFlushRepeatedSameIDIndexedChangeBack, + collectionFlushRepeatedSameIDNonIndexedUpdate, + collectionFlushManyIDsIndexedChanges, + collectionFlushManyIDsNonIndexedChanges, + } + for _, docs := range []int{64, 512, 5000} { + for _, shape := range shapes { + b.Run(fmt.Sprintf("%s/docs_%d", shape, docs), func(b *testing.B) { + benchmarkCollectionFlushAfterStaging(b, docs, shape) + }) + } + } +} + +func benchmarkCollectionFlushAfterStaging(b *testing.B, docs int, shape collectionFlushAfterStagingShape) { + b.Helper() + if docs <= 0 { + b.Fatalf("invalid docs %d", docs) + } + db, err := backenddb.Open(backenddb.Options{Dir: b.TempDir()}) + if err != nil { + b.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + manager := NewCollectionManager(db) + manager.SetUpdateBatchDetailedStatsEnabled(true) + if _, err := manager.CreateCollection(&CollectionMeta{ + Name: "bench", + Options: CollectionOptions{ + DocumentFormat: DocumentFormatJSON, + BufferedIndexedWrites: true, + BufferedIndexedWriteMaxDocuments: 1 << 30, + BufferedIndexedWriteMaxBytes: 1 << 40, + BufferedIndexedWriteMaxRootRuns: 1 << 30, + BufferedIndexedAsyncFlushMaxQueuedUnits: 1 << 20, + }, + Indexes: []IndexDefinition{ + {Name: "city", Field: "city", ValueType: IndexValueString}, + {Name: "score", Field: "score", ValueType: IndexValueInt64}, + }, + }); err != nil { + b.Fatalf("create collection: %v", err) + } + col, err := manager.OpenCollection("bench") + if err != nil { + b.Fatalf("open collection: %v", err) + } + + ids := make([][]byte, docs) + documents := make([][]byte, docs) + for i := 0; i < docs; i++ { + ids[i] = []byte(collectionFlushBenchDocID(i)) + documents[i] = collectionFlushBenchJSON(i, "base-city", 0, 0) + } + if _, err := col.InsertBatch(ids, documents); err != nil { + b.Fatalf("insert preload: %v", err) + } + if err := manager.FlushAll(); err != nil { + b.Fatalf("flush preload: %v", err) + } + if err := db.Checkpoint(); err != nil { + b.Fatalf("checkpoint preload: %v", err) + } + + statsBefore := manager.StatsSnapshot() + dbStatsBefore := db.Stats() + var flushElapsed time.Duration + totalDocs := uint64(0) + + b.ReportAllocs() + b.ResetTimer() + for iter := 0; iter < b.N; iter++ { + b.StopTimer() + stagedDocs := stageCollectionFlushAfterStagingWorkload(b, col, docs, iter, shape) + totalDocs += uint64(stagedDocs) + b.StartTimer() + start := time.Now() + if err := manager.FlushAll(); err != nil { + b.Fatalf("flush staged workload: %v", err) + } + flushElapsed += time.Since(start) + b.StopTimer() + } + + stats := collectionManagerStatsBenchmarkDelta(manager.StatsSnapshot(), statsBefore) + dbStatsDelta := collectionFlushBenchStatsDelta(db.Stats(), dbStatsBefore) + if totalDocs > 0 { + docsFloat := float64(totalDocs) + b.ReportMetric(float64(flushElapsed.Nanoseconds())/docsFloat, "ns/doc") + reportCollectionFlushShapeStatsForBenchmark(b, stats, int(totalDocs)) + collectionFlushBenchReportUintPerDoc(b, dbStatsDelta["treedb.publish.ordered_root_delta_group.root_apply_ns_total"], totalDocs, "root_apply_ns/doc") + collectionFlushBenchReportUintPerDoc(b, dbStatsDelta["treedb.publish.ordered_root_delta_group.root_apply_calls_total"], totalDocs, "root_apply_calls/doc") + } +} + +func stageCollectionFlushAfterStagingWorkload(b *testing.B, col *Collection, docs, iter int, shape collectionFlushAfterStagingShape) int { + b.Helper() + switch shape { + case collectionFlushRepeatedSameIDIndexedChangeBack: + id := []byte(collectionFlushBenchDocID(0)) + for i := 0; i < docs; i++ { + city := "base-city" + if i%2 == 0 { + city = "staged-city" + } + item := UpdateBatchItem{ + DocumentID: id, + Update: collectionFlushBenchReplace(collectionFlushBenchJSON(0, city, 0, 0)), + } + if _, batched, err := col.UpdateBatchIfNoSecondaryUniqueIndexChanges([]UpdateBatchItem{item}); err != nil { + b.Fatalf("stage repeated indexed update %d: %v", i, err) + } else if !batched { + b.Fatalf("stage repeated indexed update %d declined", i) + } + } + return docs + case collectionFlushRepeatedSameIDNonIndexedUpdate: + id := []byte(collectionFlushBenchDocID(0)) + for i := 0; i < docs; i++ { + counter := iter*docs + i + 1 + item := UpdateBatchItem{ + DocumentID: id, + Update: collectionFlushBenchReplace(collectionFlushBenchJSON(0, "base-city", 0, counter)), + } + if _, batched, err := col.UpdateBatchIfNoSecondaryUniqueIndexChanges([]UpdateBatchItem{item}); err != nil { + b.Fatalf("stage repeated non-indexed update %d: %v", i, err) + } else if !batched { + b.Fatalf("stage repeated non-indexed update %d declined", i) + } + } + return docs + case collectionFlushManyIDsIndexedChanges: + items := make([]UpdateBatchItem, docs) + city := "city-even" + if iter%2 != 0 { + city = "city-odd" + } + for i := 0; i < docs; i++ { + items[i] = UpdateBatchItem{ + DocumentID: []byte(collectionFlushBenchDocID(i)), + Update: collectionFlushBenchReplace(collectionFlushBenchJSON(i, city, i%97, iter)), + } + } + if _, batched, err := col.UpdateBatchIfNoSecondaryUniqueIndexChanges(items); err != nil { + b.Fatalf("stage many-id indexed updates: %v", err) + } else if !batched { + b.Fatalf("stage many-id indexed updates declined") + } + return docs + case collectionFlushManyIDsNonIndexedChanges: + items := make([]UpdateBatchItem, docs) + for i := 0; i < docs; i++ { + items[i] = UpdateBatchItem{ + DocumentID: []byte(collectionFlushBenchDocID(i)), + Update: collectionFlushBenchReplace(collectionFlushBenchJSON(i, "base-city", 0, iter+1)), + } + } + if _, batched, err := col.UpdateBatchIfNoSecondaryUniqueIndexChanges(items); err != nil { + b.Fatalf("stage many-id non-indexed updates: %v", err) + } else if !batched { + b.Fatalf("stage many-id non-indexed updates declined") + } + return docs + default: + b.Fatalf("unknown collection flush shape %q", shape) + return 0 + } +} + +func reportCollectionFlushShapeStatsForBenchmark(b *testing.B, stats CollectionManagerStats, docs int) { + b.Helper() + if docs <= 0 { + return + } + reportDurationPerDoc := func(value time.Duration, name string) { + if value > 0 { + b.ReportMetric(float64(value.Nanoseconds())/float64(docs), name) + } + } + if stats.IndexedFlushCalls > 0 { + b.ReportMetric(float64(stats.IndexedFlushCalls), "indexed_flush_calls") + b.ReportMetric(float64(stats.IndexedFlushDocs)/float64(stats.IndexedFlushCalls), "indexed_flush_docs/call") + } + reportDurationPerDoc(stats.IndexedFlushRotate, "indexed_flush_rotate_ns/doc") + reportDurationPerDoc(stats.IndexedFlushMaterialize, "indexed_flush_materialize_ns/doc") + reportDurationPerDoc(stats.IndexedFlushPublish, "indexed_flush_publish_ns/doc") + reportDurationPerDoc(stats.IndexedFlushDuration, "indexed_flush_ns/doc") + reportCollectionRootDeltaShapeStatsForBenchmark(b, stats, docs) +} + +func collectionFlushBenchDocID(n int) string { + return "u-" + strconv.FormatInt(int64(n), 10) +} + +func collectionFlushBenchJSON(n int, city string, score, counter int) []byte { + return []byte(fmt.Sprintf(`{"name":"user-%d","email":"user-%d@example.com","city":%q,"score":%d,"counter":%d}`, n, n, city, score, counter)) +} + +func collectionFlushBenchReplace(raw []byte) func([]byte) ([]byte, bool, error) { + return func([]byte) ([]byte, bool, error) { + return raw, true, nil + } +} + +func collectionFlushBenchStatsDelta(after, before map[string]string) map[string]uint64 { + out := make(map[string]uint64, len(after)) + for key, afterValue := range after { + afterUint, err := strconv.ParseUint(afterValue, 10, 64) + if err != nil { + continue + } + beforeUint, _ := strconv.ParseUint(before[key], 10, 64) + if afterUint >= beforeUint { + out[key] = afterUint - beforeUint + } + } + return out +} + +func collectionFlushBenchReportUintPerDoc(b *testing.B, value uint64, docs uint64, name string) { + b.Helper() + if value == 0 || docs == 0 { + return + } + b.ReportMetric(float64(value)/float64(docs), name) +} diff --git a/TreeDB/collections/freeze_sort_run_table.go b/TreeDB/collections/freeze_sort_run_table.go index 2610b06255..3280f7d4c4 100644 --- a/TreeDB/collections/freeze_sort_run_table.go +++ b/TreeDB/collections/freeze_sort_run_table.go @@ -17,7 +17,7 @@ type freezeSortRunEntry struct { key []byte value []byte ptr page.ValuePtr - seq uint64 + seq uint32 flags byte } @@ -28,9 +28,12 @@ type freezeSortRunTable struct { latestDirty bool frozen bool sizeBytes int64 - nextSeq uint64 + nextSeq uint32 } +const freezeSortRunTablePreallocEntryThreshold = 1024 +const freezeSortRunTableGeometricGrowthEntryThreshold = 1 << 16 + // freezeSortRunTable is a collection-write-domain run table optimized for // root-local accumulation: writes append cheaply while mutable, and rotation to // an immutable flush unit pays the sort/coalesce cost once. @@ -101,6 +104,9 @@ func (t *freezeSortRunTable) ApplyStealEntryFunc(count int, emit func(i int) (ke } t.mu.Lock() defer t.mu.Unlock() + if count >= freezeSortRunTablePreallocEntryThreshold && cap(t.entries)-len(t.entries) < count { + t.entries = growFreezeSortRunEntries(t.entries, count) + } for i := 0; i < count; i++ { key, value, ptr, flags, err := emit(i) if err != nil { @@ -130,6 +136,35 @@ func (t *freezeSortRunTable) ApplyStealEntryFunc(count int, emit func(i int) (ke return nil } +func growFreezeSortRunEntries(entries []freezeSortRunEntry, additional int) []freezeSortRunEntry { + if additional <= 0 || cap(entries)-len(entries) >= additional { + return entries + } + needed := len(entries) + additional + newCap := cap(entries) + if newCap < freezeSortRunTableGeometricGrowthEntryThreshold { + newCap = needed + } else { + for newCap < needed { + growth := newCap / 2 + if growth <= 0 { + growth = additional + } + if growth <= 0 || newCap > maxCollectionInt-growth { + newCap = needed + break + } + newCap += growth + } + } + if newCap < needed { + newCap = needed + } + out := make([]freezeSortRunEntry, len(entries), newCap) + copy(out, entries) + return out +} + func (t *freezeSortRunTable) Delete(key []byte) { t.SetEntry(key, nil, page.ValuePtr{}, node.FlagTombstone) } diff --git a/TreeDB/collections/freeze_sort_run_table_test.go b/TreeDB/collections/freeze_sort_run_table_test.go index 6f3066865b..5175e577ac 100644 --- a/TreeDB/collections/freeze_sort_run_table_test.go +++ b/TreeDB/collections/freeze_sort_run_table_test.go @@ -136,6 +136,20 @@ func TestFreezeSortRunTableApplyStealEntryFunc(t *testing.T) { } } +func TestFreezeSortRunEntriesGrowGeometricallyForLargeAppends(t *testing.T) { + entries := make([]freezeSortRunEntry, freezeSortRunTableGeometricGrowthEntryThreshold) + previousLen := len(entries) + exactNeeded := len(entries) + previousLen + + grown := growFreezeSortRunEntries(entries, previousLen) + if len(grown) != previousLen { + t.Fatalf("grown len=%d want %d", len(grown), previousLen) + } + if cap(grown) <= exactNeeded { + t.Fatalf("grown cap=%d want > exact needed %d", cap(grown), exactNeeded) + } +} + func requireFreezeSortRunIterator(t *testing.T, it interface { Valid() bool Next() diff --git a/TreeDB/collections/indexed_flush_guard_counters_test.go b/TreeDB/collections/indexed_flush_guard_counters_test.go index bd701bd0d8..a47cfdc480 100644 --- a/TreeDB/collections/indexed_flush_guard_counters_test.go +++ b/TreeDB/collections/indexed_flush_guard_counters_test.go @@ -128,7 +128,7 @@ func TestCollectionIndexedFlushGuardCounters(t *testing.T) { col.writeDomain.mu.Lock() col.writeDomain.indexedPublishingUnits = []indexedFlushUnit{{}} col.writeDomain.mu.Unlock() - rootIDs := make([]uint64, len(work.rootNames)) + rootIDs := make([]uint64, len(work.batch.rootNames)) for i := range rootIDs { rootIDs[i] = uint64(1000 + i) } diff --git a/TreeDB/collections/indexed_flush_ownership_test.go b/TreeDB/collections/indexed_flush_ownership_test.go index dffe10d0ed..10703c710a 100644 --- a/TreeDB/collections/indexed_flush_ownership_test.go +++ b/TreeDB/collections/indexed_flush_ownership_test.go @@ -6,6 +6,7 @@ import ( "testing" backenddb "github.com/snissn/gomap/TreeDB/db" + "github.com/snissn/gomap/TreeDB/internal/memtable" ) func TestCollectionIndexedAsyncPublishLostOwnershipDoesNotRemoveCurrentPublishingUnit(t *testing.T) { @@ -44,6 +45,9 @@ func TestCollectionIndexedAsyncPublishLostOwnershipDoesNotRemoveCurrentPublishin if work == nil { t.Fatal("prepare async publish returned nil work") } + if got := work.batch.state; got != coalescedFlushBatchActive { + t.Fatalf("prepared batch state=%d want active", got) + } if work.pin != nil { defer func() { _ = work.pin.Close() }() } @@ -72,7 +76,7 @@ func TestCollectionIndexedAsyncPublishLostOwnershipDoesNotRemoveCurrentPublishin resetIndexedFlushUnits(publishingUnits) }) - rootIDs := make([]uint64, len(work.rootNames)) + rootIDs := make([]uint64, len(work.batch.rootNames)) for i := range rootIDs { rootIDs[i] = uint64(1000 + i) } @@ -80,6 +84,9 @@ func TestCollectionIndexedAsyncPublishLostOwnershipDoesNotRemoveCurrentPublishin if !errors.Is(err, errIndexedFlushLostOwnership) { t.Fatalf("complete lost ownership err=%v want lost ownership", err) } + if got := work.batch.state; got != coalescedFlushBatchLostOwnership { + t.Fatalf("lost-ownership batch state=%d want lost ownership", got) + } col.writeDomain.mu.RLock() publishingUnits := append([]indexedFlushUnit(nil), col.writeDomain.indexedPublishingUnits...) @@ -118,3 +125,38 @@ func TestCollectionIndexedAsyncPublishLostOwnershipDoesNotRemoveCurrentPublishin t.Fatalf("indexed flush lost ownership=%d want 1", got) } } + +func TestIndexedPublishingWorkOwnershipRejectsPrefixOfActiveBatch(t *testing.T) { + unitA := indexedFlushOwnershipUnitForTest("users") + unitB := indexedFlushOwnershipUnitForTest("users") + t.Cleanup(func() { + resetIndexedFlushUnits([]indexedFlushUnit{unitA, unitB}) + }) + + domain := &collectionWriteDomain{ + indexedPublishingUnits: []indexedFlushUnit{unitA, unitB}, + } + removed, owned := removeIndexedPublishingWorkUnitsLocked(domain, []indexedFlushUnit{unitA}) + if owned { + t.Fatal("prefix work unexpectedly owned the active multi-unit batch") + } + if removed != nil { + t.Fatalf("removed prefix units=%+v want nil", removed) + } + if got := len(domain.indexedPublishingUnits); got != 2 { + t.Fatalf("active publishing units after prefix removal attempt=%d want 2", got) + } + if !sameIndexedFlushUnitTables(domain.indexedPublishingUnits[0], unitA) || + !sameIndexedFlushUnitTables(domain.indexedPublishingUnits[1], unitB) { + t.Fatal("active publishing units changed after rejected prefix ownership attempt") + } +} + +func indexedFlushOwnershipUnitForTest(collectionName string) indexedFlushUnit { + rootName := collectionPrimaryRootName(collectionName) + return indexedFlushUnit{ + rootRuns: map[string][]memtable.Table{ + rootName: {newCollectionRunTable(0)}, + }, + } +} diff --git a/TreeDB/collections/indexed_flush_requeue_test.go b/TreeDB/collections/indexed_flush_requeue_test.go index 21118710bc..d47d6948c2 100644 --- a/TreeDB/collections/indexed_flush_requeue_test.go +++ b/TreeDB/collections/indexed_flush_requeue_test.go @@ -48,6 +48,9 @@ func TestCollectionIndexedAsyncPublishFailureRequeuesUnitsAndPreservesUniqueRese if work == nil { t.Fatal("prepare async publish returned nil work") } + if got := work.batch.state; got != coalescedFlushBatchActive { + t.Fatalf("prepared batch state=%d want active", got) + } defer collectionTestCloseIndexedFlushWork(work) if _, err := col.InsertBatch( @@ -77,6 +80,9 @@ func TestCollectionIndexedAsyncPublishFailureRequeuesUnitsAndPreservesUniqueRese if err := col.completePreparedIndexedFlush(work, 0, nil, injectedErr, 0, 0, 0); !errors.Is(err, injectedErr) { t.Fatalf("complete failure err=%v want injected publish failure", err) } + if got := work.batch.state; got != coalescedFlushBatchRequeued { + t.Fatalf("failed batch state=%d want requeued", got) + } col.writeDomain.mu.RLock() publishingUnits := len(col.writeDomain.indexedPublishingUnits) diff --git a/TreeDB/collections/indexed_flush_root_mismatch_test.go b/TreeDB/collections/indexed_flush_root_mismatch_test.go index 25a5f578ce..744d219a32 100644 --- a/TreeDB/collections/indexed_flush_root_mismatch_test.go +++ b/TreeDB/collections/indexed_flush_root_mismatch_test.go @@ -3,6 +3,7 @@ package collections import ( "bytes" "errors" + "strconv" "testing" backenddb "github.com/snissn/gomap/TreeDB/db" @@ -115,6 +116,9 @@ func TestCollectionIndexedAsyncPublishRootBaseMismatchRequeuesFIFOAndCounts(t *t if work == nil { t.Fatal("prepare left async publish returned nil work") } + if got := work.batch.state; got != coalescedFlushBatchActive { + t.Fatalf("prepared batch state=%d want active", got) + } defer collectionTestCloseIndexedFlushWork(work) if _, err := left.InsertBatch( @@ -152,9 +156,17 @@ func TestCollectionIndexedAsyncPublishRootBaseMismatchRequeuesFIFOAndCounts(t *t t.Fatalf("flush right row: %v", err) } + rootApplyCallsBeforeMismatch := orderedRootDeltaGroupRootApplyCallsForTest(t, d) if err := left.publishPreparedIndexedFlush(work); !errors.Is(err, ErrConcurrentMutation) { t.Fatalf("publish prepared left err=%v want ErrConcurrentMutation", err) } + rootApplyCallsAfterMismatch := orderedRootDeltaGroupRootApplyCallsForTest(t, d) + if got := rootApplyCallsAfterMismatch - rootApplyCallsBeforeMismatch; got != 0 { + t.Fatalf("root apply calls during root-mismatched publish=%d want 0", got) + } + if got := work.batch.state; got != coalescedFlushBatchRequeued { + t.Fatalf("root-mismatched batch state=%d want requeued", got) + } prefixA := indexedFlushRequeueEmailPrefix(t, "a@example.com") prefixB := indexedFlushRequeueEmailPrefix(t, "b@example.com") @@ -191,3 +203,13 @@ func TestCollectionIndexedAsyncPublishRootBaseMismatchRequeuesFIFOAndCounts(t *t t.Fatalf("indexed flush requeued units=%d want 1", got) } } + +func orderedRootDeltaGroupRootApplyCallsForTest(tb testing.TB, d *backenddb.DB) uint64 { + tb.Helper() + raw := d.Stats()["treedb.publish.ordered_root_delta_group.root_apply_calls_total"] + got, err := strconv.ParseUint(raw, 10, 64) + if err != nil { + tb.Fatalf("parse root apply calls stat %q: %v", raw, err) + } + return got +} diff --git a/TreeDB/collections/planner.go b/TreeDB/collections/planner.go index 6ba422c021..569fadb0a4 100644 --- a/TreeDB/collections/planner.go +++ b/TreeDB/collections/planner.go @@ -97,7 +97,8 @@ type insertBatchPlan struct { type insertBatchPlanStats struct { CollectionInsertStats - payloadBuilds int + rootDeltaStats collectionRootDeltaPlanStats + payloadBuilds int } type collectionRootRun struct { @@ -671,12 +672,14 @@ func buildUniqueProbeRunsFromSorted(candidates []uniqueProbeCandidate, includeIn func (p insertBatchPlanner) emitPrimaryRun(plan *insertBatchPlan, items []insertBatchItem, order []int) error { table := newCollectionRunTable(len(items)) + kind := plan.stats.rootDeltaStats.addRoot(p.collection, p.primaryRoot) if err := applyCollectionRunEntries(table, len(items), func(i int) (key, value []byte, err error) { idx := orderedItemIndex(order, i) value, err = p.buildPrimaryVal(items[idx].id, items[idx].document) if err != nil { return nil, nil, err } + plan.stats.rootDeltaStats.addEntry(kind, uint64(len(items[idx].id)), uint64(len(value)), false) plan.stats.payloadBuilds++ return items[idx].id, value, nil }); err != nil { @@ -702,11 +705,13 @@ func (p insertBatchPlanner) emitTemplateRun(plan *insertBatchPlan, records []tem return err } table := newCollectionRunTable(len(records)) + kind := plan.stats.rootDeltaStats.addRoot(p.collection, p.templateRoot) if err := applyCollectionRunEntries(table, len(records), func(i int) (key, value []byte, err error) { raw := records[i].raw if p.cloneTemplateRunValues { raw = bytes.Clone(raw) } + plan.stats.rootDeltaStats.addEntry(kind, uint64(len(records[i].id)), uint64(len(raw)), false) return records[i].id[:], raw, nil }); err != nil { return err @@ -778,9 +783,11 @@ func (p insertBatchPlanner) emitIndexStateRun(plan *insertBatchPlan, items []ins } table := newCollectionRunTable(len(items)) valueArena := make([]byte, 0, valueBytes) + kind := plan.stats.rootDeltaStats.addRoot(p.collection, p.indexStateRoot) if err := applyCollectionRunEntries(table, len(items), func(i int) (key, value []byte, err error) { idx := orderedItemIndex(order, i) valueArena, value = appendRuntimeOrderedDocumentIndexState(valueArena, items[idx].state, runtimes, counts[i]) + plan.stats.rootDeltaStats.addEntry(kind, uint64(len(items[idx].id)), uint64(len(value)), false) return items[idx].id, value, nil }); err != nil { return err @@ -822,6 +829,7 @@ func (p insertBatchPlanner) emitSecondaryRuns(plan *insertBatchPlan, items []ins continue } if alreadySorted { + rootName := p.collection + "/index/" + runtime.def.name table := newCollectionRunTable(entryCount) keyArena := make([]byte, 0, keyBytes) itemPos := 0 @@ -845,7 +853,7 @@ func (p insertBatchPlanner) emitSecondaryRuns(plan *insertBatchPlan, items []ins } table.Freeze() plan.runs = append(plan.runs, collectionRootRun{ - name: p.collection + "/index/" + runtime.def.name, + name: rootName, kind: collectionRootSecondary, indexName: runtime.def.name, indexValueType: runtime.def.valueType, @@ -857,14 +865,17 @@ func (p insertBatchPlanner) emitSecondaryRuns(plan *insertBatchPlan, items []ins plan.stats.SecondaryEntries += entryCount plan.stats.SecondaryKeyBytes += keyBytes plan.stats.SecondarySortedRuns++ + kind := plan.stats.rootDeltaStats.addRoot(p.collection, rootName) + plan.stats.rootDeltaStats.addEntries(kind, uint64(entryCount), uint64(keyBytes), 0, 0) continue } if table, ok, err := p.emitGroupedSecondaryRunTable(items, runtimeIdx, runtime.def.name, documentIDOrder, entryCount, keyBytes); err != nil { return err } else if ok { + rootName := p.collection + "/index/" + runtime.def.name plan.runs = append(plan.runs, collectionRootRun{ - name: p.collection + "/index/" + runtime.def.name, + name: rootName, kind: collectionRootSecondary, indexName: runtime.def.name, indexValueType: runtime.def.valueType, @@ -876,6 +887,8 @@ func (p insertBatchPlanner) emitSecondaryRuns(plan *insertBatchPlan, items []ins plan.stats.SecondaryEntries += entryCount plan.stats.SecondaryKeyBytes += keyBytes plan.stats.SecondaryUnsortedRuns++ + kind := plan.stats.rootDeltaStats.addRoot(p.collection, rootName) + plan.stats.rootDeltaStats.addEntries(kind, uint64(entryCount), uint64(keyBytes), 0, 0) continue } @@ -905,8 +918,9 @@ func (p insertBatchPlanner) emitSecondaryRuns(plan *insertBatchPlan, items []ins return err } table.Freeze() + rootName := p.collection + "/index/" + runtime.def.name plan.runs = append(plan.runs, collectionRootRun{ - name: p.collection + "/index/" + runtime.def.name, + name: rootName, kind: collectionRootSecondary, indexName: runtime.def.name, indexValueType: runtime.def.valueType, @@ -918,6 +932,8 @@ func (p insertBatchPlanner) emitSecondaryRuns(plan *insertBatchPlan, items []ins plan.stats.SecondaryEntries += entryCount plan.stats.SecondaryKeyBytes += keyBytes plan.stats.SecondaryUnsortedRuns++ + kind := plan.stats.rootDeltaStats.addRoot(p.collection, rootName) + plan.stats.rootDeltaStats.addEntries(kind, uint64(entryCount), uint64(keyBytes), 0, 0) } return nil } diff --git a/TreeDB/collections/pr3b_semantic_indexed_test.go b/TreeDB/collections/pr3b_semantic_indexed_test.go new file mode 100644 index 0000000000..18f27005af --- /dev/null +++ b/TreeDB/collections/pr3b_semantic_indexed_test.go @@ -0,0 +1,790 @@ +package collections + +import ( + "bytes" + "errors" + "fmt" + "testing" + + backenddb "github.com/snissn/gomap/TreeDB/db" +) + +func TestIndexedSemanticValueSetDiffFastPathsMatchMultiValueSemantics(t *testing.T) { + empty := make([][]byte, 0) + deletes, sets := indexedSemanticValueSetDiff(nil, empty) + if deletes != nil || sets != nil { + t.Fatalf("nil/empty diff got deletes=%v sets=%v want nil/nil", deletes, sets) + } + + base := [][]byte{[]byte("a")} + final := [][]byte{[]byte("b")} + deletes, sets = indexedSemanticValueSetDiff(base, final) + if got, want := string(deletes[0]), "a"; got != want { + t.Fatalf("delete value=%q want %q", got, want) + } + if got, want := string(sets[0]), "b"; got != want { + t.Fatalf("set value=%q want %q", got, want) + } + + base[0] = []byte("base-replaced") + final[0] = []byte("final-replaced") + if got, want := string(deletes[0]), "a"; got != want { + t.Fatalf("delete alias changed after base replacement: got %q want %q", got, want) + } + if got, want := string(sets[0]), "b"; got != want { + t.Fatalf("set alias changed after final replacement: got %q want %q", got, want) + } + + _, sets = indexedSemanticValueSetDiff(nil, final) + final[0] = []byte("second-replacement") + if got, want := string(sets[0]), "final-replaced"; got != want { + t.Fatalf("set-only alias changed after final replacement: got %q want %q", got, want) + } +} + +func TestPR3bSemanticRawRecordsSurviveMutableQueuedActiveRequeued(t *testing.T) { + d, mgr, col := pr3bSemanticTestCollection(t) + defer func() { _ = d.Close() }() + pr3bSeedSemanticUser(t, col) + + if _, batched, err := col.UpdateBatchIfNoSecondaryUniqueIndexChanges([]UpdateBatchItem{{ + DocumentID: []byte("u1"), + Update: setJSONCity("sea"), + }}); err != nil { + t.Fatalf("UpdateBatchIfNoSecondaryUniqueIndexChanges: %v", err) + } else if !batched { + t.Fatal("city update was not buffered") + } + + col.writeDomain.mu.RLock() + mutableRecords := cloneIndexedSemanticRecords(col.writeDomain.indexedSemanticRecords) + queuedBeforeRotate := len(col.writeDomain.indexedFlushUnits) + publishingBeforeRotate := len(col.writeDomain.indexedPublishingUnits) + col.writeDomain.mu.RUnlock() + if queuedBeforeRotate != 0 || publishingBeforeRotate != 0 { + t.Fatalf("pre-rotate queued=%d publishing=%d want 0/0", queuedBeforeRotate, publishingBeforeRotate) + } + pr3bRequireCitySemanticRecord(t, mutableRecords, "hnl", "sea") + pr3bRequireIndexIDs(t, col, "city", "sea", "u1") + pr3bRequireIndexIDs(t, col, "city", "hnl") + + stats := mgr.StatsSnapshot() + if got := stats.IndexedSemanticRawRecords; got != 1 { + t.Fatalf("raw semantic records after stage=%d want 1", got) + } + if got := stats.IndexedSemanticRawIndexDeltas; got != 1 { + t.Fatalf("raw semantic index deltas after stage=%d want 1", got) + } + if got := stats.IndexedSemanticFallbackRecords; got != 0 { + t.Fatalf("fallback semantic records after stage=%d want 0", got) + } + if got := stats.IndexedSemanticEffectiveRecords; got != 0 { + t.Fatalf("effective semantic records after stage=%d want 0", got) + } + if got := stats.PendingIndexedSemanticRecords; got != 1 { + t.Fatalf("pending semantic records after stage=%d want 1", got) + } + pr3bRequireSemanticMetricKeys(t, mgr) + + col.writeDomain.mu.Lock() + if !rotateIndexedMutableToFlushUnitLocked(col.writeDomain) { + col.writeDomain.mu.Unlock() + t.Fatal("rotate indexed mutable state returned false") + } + queuedRecords := cloneIndexedSemanticRecords(col.writeDomain.indexedFlushUnits[0].semanticRecords) + mutableAfterRotate := len(col.writeDomain.indexedSemanticRecords) + col.writeDomain.mu.Unlock() + if mutableAfterRotate != 0 { + t.Fatalf("mutable semantic records after rotate=%d want 0", mutableAfterRotate) + } + pr3bRequireCitySemanticRecord(t, queuedRecords, "hnl", "sea") + pr3bRequireIndexIDs(t, col, "city", "sea", "u1") + + work, err := col.prepareIndexedAsyncPublish() + if err != nil { + t.Fatalf("prepare async publish: %v", err) + } + if work == nil { + t.Fatal("prepare async publish returned nil work") + } + defer collectionTestCloseIndexedFlushWork(work) + if got := work.batch.state; got != coalescedFlushBatchActive { + t.Fatalf("prepared batch state=%d want active", got) + } + pr3bRequireCitySemanticRecord(t, work.batch.semanticRecords, "hnl", "sea") + col.writeDomain.mu.RLock() + activeRecords := cloneIndexedSemanticRecords(col.writeDomain.indexedPublishingUnits[0].semanticRecords) + queuedAfterPrepare := len(col.writeDomain.indexedFlushUnits) + col.writeDomain.mu.RUnlock() + if queuedAfterPrepare != 0 { + t.Fatalf("queued units after prepare=%d want 0", queuedAfterPrepare) + } + pr3bRequireCitySemanticRecord(t, activeRecords, "hnl", "sea") + + injectedErr := errors.New("injected PR3b publish failure") + if err := col.completePreparedIndexedFlush(work, 0, nil, injectedErr, 0, 0, 0); !errors.Is(err, injectedErr) { + t.Fatalf("complete failure err=%v want injected error", err) + } + if got := work.batch.state; got != coalescedFlushBatchRequeued { + t.Fatalf("failed batch state=%d want requeued", got) + } + col.writeDomain.mu.RLock() + requeuedRecords := cloneIndexedSemanticRecords(col.writeDomain.indexedFlushUnits[0].semanticRecords) + publishingAfterFailure := len(col.writeDomain.indexedPublishingUnits) + col.writeDomain.mu.RUnlock() + if publishingAfterFailure != 0 { + t.Fatalf("publishing units after failure=%d want 0", publishingAfterFailure) + } + pr3bRequireCitySemanticRecord(t, requeuedRecords, "hnl", "sea") + pr3bRequireIndexIDs(t, col, "city", "sea", "u1") + + stats = mgr.StatsSnapshot() + if got := stats.IndexedSemanticRawRecords; got != 1 { + t.Fatalf("raw semantic records after requeue=%d want 1", got) + } + if got := stats.PendingIndexedSemanticRecords; got != 1 { + t.Fatalf("pending semantic records after requeue=%d want 1", got) + } + if got := stats.IndexedSemanticEffectiveRecords; got != 0 { + t.Fatalf("effective semantic records after requeue=%d want 0", got) + } + + if err := col.Flush(); err != nil { + t.Fatalf("flush requeued semantic update: %v", err) + } + pr3bRequireIndexIDs(t, col, "city", "sea", "u1") + stats = mgr.StatsSnapshot() + if got := stats.PendingIndexedSemanticRecords; got != 0 { + t.Fatalf("pending semantic records after flush=%d want 0", got) + } + if got := stats.IndexedSemanticRawRecords; got != 1 { + t.Fatalf("raw semantic records after flush=%d want 1", got) + } + if got := stats.IndexedSemanticEffectiveRecords; got != 1 { + t.Fatalf("effective semantic records after flush=%d want 1", got) + } +} + +func TestPR3bRootDeltaCoalescingSkippedSecondaryRootsUseUniqueRoots(t *testing.T) { + raw := collectionRootDeltaPlanStats{ + secondaryRoots: 2, + secondaryUniqueRoots: 1, + entries: 6, + secondaryDetail: collectionRootDeltaKindStats{ + entries: 6, + }, + } + final := collectionRootDeltaPlanStats{ + secondaryRoots: 1, + secondaryUniqueRoots: 1, + entries: 4, + secondaryDetail: collectionRootDeltaKindStats{ + entries: 4, + }, + } + + domain := &collectionWriteDomain{} + domain.observeRootDeltaPlanCoalescing(raw, final) + if got := domain.indexedSemanticSkippedSecondaryRoots.Load(); got != 0 { + t.Fatalf("skipped secondary roots=%d want 0 for repeated raw units that still publish the root", got) + } + domain = &collectionWriteDomain{} + domain.observeRootDeltaPlanCoalescing(raw, collectionRootDeltaPlanStats{}) + if got := domain.indexedSemanticSkippedSecondaryRoots.Load(); got != 1 { + t.Fatalf("skipped secondary roots for net-zero plan=%d want 1 unique root", got) + } +} + +func TestPR3bAsyncNetZeroCoalescingRecordsAcceptanceCounters(t *testing.T) { + d, mgr, col := pr3bSemanticTestCollection(t) + defer func() { _ = d.Close() }() + pr3bSeedSemanticUser(t, col) + + const ( + netZeroDocs = 2 + netZeroBytes = 32 + ) + before := mgr.StatsSnapshot() + col.writeDomain.mu.Lock() + col.writeDomain.indexedFlushUnits = []indexedFlushUnit{{ + docCount: netZeroDocs, + byteCount: netZeroBytes, + rootRunCount: 1, + }} + col.writeDomain.count = netZeroDocs + col.writeDomain.bufferedBytes = netZeroBytes + col.writeDomain.mu.Unlock() + + work, err := col.prepareIndexedAsyncPublish() + if err != nil { + t.Fatalf("prepare async net-zero publish: %v", err) + } + if work != nil { + collectionTestCloseIndexedFlushWork(work) + t.Fatal("prepare async net-zero publish returned work") + } + + stats := mgr.StatsSnapshot() + if got := stats.PendingDocuments; got != 0 { + t.Fatalf("pending docs after async net-zero prepare=%d want 0", got) + } + if got := stats.CoalescedFlushBatches - before.CoalescedFlushBatches; got != 1 { + t.Fatalf("coalesced flush batches after async net-zero prepare=%d want 1", got) + } + if got := stats.CoalescedFlushBatchUnits - before.CoalescedFlushBatchUnits; got != 1 { + t.Fatalf("coalesced flush batch units after async net-zero prepare=%d want 1", got) + } + if got := stats.CoalescedFlushBatchDocs - before.CoalescedFlushBatchDocs; got != netZeroDocs { + t.Fatalf("coalesced flush batch docs after async net-zero prepare=%d want %d", got, netZeroDocs) + } + if got := stats.CoalescedFlushBatchBytes - before.CoalescedFlushBatchBytes; got != netZeroBytes { + t.Fatalf("coalesced flush batch bytes after async net-zero prepare=%d want %d", got, netZeroBytes) + } + if got := stats.CoalescedFlushNetZeroBatches - before.CoalescedFlushNetZeroBatches; got != 1 { + t.Fatalf("coalesced flush net-zero batches after async net-zero prepare=%d want 1", got) + } +} + +func TestPR3bSemanticRepeatedSameDocumentUpdatesSerialEquivalent(t *testing.T) { + d, mgr, col := pr3bSemanticTestCollection(t) + defer func() { _ = d.Close() }() + pr3bSeedSemanticUser(t, col) + before := mgr.StatsSnapshot() + + firstCalls := 0 + if _, batched, err := col.UpdateBatchIfNoSecondaryUniqueIndexChanges([]UpdateBatchItem{{ + DocumentID: []byte("u1"), + Update: func(current []byte) ([]byte, bool, error) { + firstCalls++ + if !bytes.Contains(current, []byte(`"city":"hnl"`)) { + return nil, false, fmt.Errorf("first callback current=%s want city hnl", current) + } + return []byte(`{"email":"a@example.com","city":"sea","score":1}`), true, nil + }, + }}); err != nil { + t.Fatalf("first UpdateBatchIfNoSecondaryUniqueIndexChanges: %v", err) + } else if !batched { + t.Fatal("first update was not buffered") + } + + secondCalls := 0 + if _, batched, err := col.UpdateBatchIfNoSecondaryUniqueIndexChanges([]UpdateBatchItem{{ + DocumentID: []byte("u1"), + Update: func(current []byte) ([]byte, bool, error) { + secondCalls++ + if !bytes.Contains(current, []byte(`"city":"sea"`)) || !bytes.Contains(current, []byte(`"score":1`)) { + return nil, false, fmt.Errorf("second callback current=%s want buffered city sea score 1", current) + } + return []byte(`{"email":"a@example.com","city":"hnl","score":2}`), true, nil + }, + }}); err != nil { + t.Fatalf("second UpdateBatchIfNoSecondaryUniqueIndexChanges: %v", err) + } else if !batched { + t.Fatal("second update was not buffered") + } + if firstCalls != 1 || secondCalls != 1 { + t.Fatalf("callback calls first=%d second=%d want 1/1", firstCalls, secondCalls) + } + + pr3bRequireIndexIDs(t, col, "city", "hnl", "u1") + pr3bRequireIndexIDs(t, col, "city", "sea") + got, err := col.Get([]byte("u1")) + if err != nil { + t.Fatalf("get buffered u1: %v", err) + } + if !bytes.Contains(got, []byte(`"city":"hnl"`)) || !bytes.Contains(got, []byte(`"score":2`)) { + t.Fatalf("buffered u1=%s want city hnl score 2", got) + } + + stats := mgr.StatsSnapshot() + if got := stats.IndexedSemanticRawRecords; got != 2 { + t.Fatalf("raw semantic records before flush=%d want 2", got) + } + if got := stats.IndexedSemanticRawIndexDeltas; got != 2 { + t.Fatalf("raw semantic index deltas before flush=%d want 2", got) + } + if got := stats.IndexedSemanticEffectiveRecords; got != 0 { + t.Fatalf("effective semantic records before flush=%d want 0", got) + } + if got := stats.PendingIndexedSemanticRecords; got != 2 { + t.Fatalf("pending semantic records before flush=%d want 2", got) + } + + if err := col.Flush(); err != nil { + t.Fatalf("flush buffered serial updates: %v", err) + } + pr3bRequireIndexIDs(t, col, "city", "hnl", "u1") + pr3bRequireIndexIDs(t, col, "city", "sea") + stats = mgr.StatsSnapshot() + if got := stats.PendingIndexedSemanticRecords; got != 0 { + t.Fatalf("pending semantic records after flush=%d want 0", got) + } + if got := stats.IndexedSemanticRawRecords; got != 2 { + t.Fatalf("raw semantic records after flush=%d want 2", got) + } + if got := stats.IndexedSemanticEffectiveRecords; got != 0 { + t.Fatalf("effective semantic records after flush=%d want 0", got) + } + if got := stats.RootDeltaPlanRawUnitSecondaryEntries - before.RootDeltaPlanRawUnitSecondaryEntries; got == 0 { + t.Fatalf("raw secondary root entries delta=%d want >0", got) + } + if got := stats.RootDeltaPlanFinalSecondaryEntries - before.RootDeltaPlanFinalSecondaryEntries; got != 0 { + t.Fatalf("final secondary root entries delta=%d want 0", got) + } + if got := stats.IndexedSemanticSkippedSecondaryRoots - before.IndexedSemanticSkippedSecondaryRoots; got == 0 { + t.Fatalf("skipped secondary roots delta=%d want >0", got) + } + if got := stats.RootDeltaPlanSquashedEntries - before.RootDeltaPlanSquashedEntries; got == 0 { + t.Fatalf("squashed entries delta=%d want >0", got) + } +} + +func TestPR3bSemanticNonUniqueChangeChangeBackSkipsSecondaryRoot(t *testing.T) { + d, mgr, col := pr3bSemanticTestCollection(t) + defer func() { _ = d.Close() }() + pr3bSeedSemanticUser(t, col) + before := mgr.StatsSnapshot() + + for _, city := range []string{"sea", "hnl"} { + if _, batched, err := col.UpdateBatchIfNoSecondaryUniqueIndexChanges([]UpdateBatchItem{{ + DocumentID: []byte("u1"), + Update: setJSONCity(city), + }}); err != nil { + t.Fatalf("UpdateBatchIfNoSecondaryUniqueIndexChanges city=%s: %v", city, err) + } else if !batched { + t.Fatalf("city=%s update was not buffered", city) + } + } + + col.writeDomain.mu.RLock() + records := cloneIndexedSemanticRecords(col.writeDomain.indexedSemanticRecords) + col.writeDomain.mu.RUnlock() + if got := len(records); got != 2 { + t.Fatalf("mutable semantic records=%d want 2", got) + } + for i, record := range records { + if record.fallback != indexedSemanticFallbackNone { + t.Fatalf("record %d fallback=%d want none", i, record.fallback) + } + } + pr3bRequireCitySemanticRecord(t, records[:1], "hnl", "sea") + pr3bRequireCitySemanticRecord(t, records[1:], "sea", "hnl") + pr3bRequireIndexIDs(t, col, "city", "hnl", "u1") + pr3bRequireIndexIDs(t, col, "city", "sea") + + if err := col.Flush(); err != nil { + t.Fatalf("flush change-change-back updates: %v", err) + } + pr3bRequireIndexIDs(t, col, "city", "hnl", "u1") + stats := mgr.StatsSnapshot() + if got := stats.IndexedSemanticRawRecords; got != 2 { + t.Fatalf("raw semantic records=%d want 2", got) + } + if got := stats.IndexedSemanticFallbackRecords; got != 0 { + t.Fatalf("fallback semantic records=%d want 0", got) + } + if got := stats.IndexedSemanticEffectiveRecords; got != 0 { + t.Fatalf("effective semantic records=%d want 0", got) + } + if got := stats.RootDeltaPlanRawUnitSecondaryEntries - before.RootDeltaPlanRawUnitSecondaryEntries; got == 0 { + t.Fatalf("raw secondary root entries delta=%d want >0", got) + } + if got := stats.RootDeltaPlanFinalSecondaryEntries - before.RootDeltaPlanFinalSecondaryEntries; got != 0 { + t.Fatalf("final secondary root entries delta=%d want 0", got) + } + if got := stats.IndexedSemanticSkippedSecondaryRoots - before.IndexedSemanticSkippedSecondaryRoots; got == 0 { + t.Fatalf("skipped secondary roots delta=%d want >0", got) + } + if got := stats.RootDeltaPlanSquashedEntries - before.RootDeltaPlanSquashedEntries; got == 0 { + t.Fatalf("squashed entries delta=%d want >0", got) + } +} + +func TestPR3bSemanticAsyncPublishSkipsChangeBackSecondaryRoot(t *testing.T) { + d, mgr, col := pr3bSemanticTestCollection(t) + defer func() { _ = d.Close() }() + pr3bSeedSemanticUser(t, col) + before := mgr.StatsSnapshot() + + for _, city := range []string{"sea", "hnl"} { + if _, batched, err := col.UpdateBatchIfNoSecondaryUniqueIndexChanges([]UpdateBatchItem{{ + DocumentID: []byte("u1"), + Update: setJSONCity(city), + }}); err != nil { + t.Fatalf("UpdateBatchIfNoSecondaryUniqueIndexChanges city=%s: %v", city, err) + } else if !batched { + t.Fatalf("city=%s update was not buffered", city) + } + col.writeDomain.mu.Lock() + if !rotateIndexedMutableToFlushUnitLocked(col.writeDomain) { + col.writeDomain.mu.Unlock() + t.Fatalf("rotate indexed mutable state for city=%s returned false", city) + } + col.writeDomain.mu.Unlock() + } + + work, err := col.prepareIndexedAsyncPublish() + if err != nil { + t.Fatalf("prepare async semantic publish: %v", err) + } + if work == nil { + t.Fatal("prepare async semantic publish returned nil work") + } + if got := len(work.batch.units); got != 2 { + collectionTestCloseIndexedFlushWork(work) + t.Fatalf("prepared semantic units=%d want 2", got) + } + if err := col.publishPreparedIndexedFlush(work); err != nil { + t.Fatalf("publish async semantic work: %v", err) + } + + pr3bRequireIndexIDs(t, col, "city", "hnl", "u1") + pr3bRequireIndexIDs(t, col, "city", "sea") + stats := mgr.StatsSnapshot() + if got := stats.PendingIndexedSemanticRecords; got != 0 { + t.Fatalf("pending semantic records after async publish=%d want 0", got) + } + if got := stats.RootDeltaPlanRawUnitSecondaryEntries - before.RootDeltaPlanRawUnitSecondaryEntries; got == 0 { + t.Fatalf("async raw secondary root entries delta=%d want >0", got) + } + if got := stats.RootDeltaPlanFinalSecondaryEntries - before.RootDeltaPlanFinalSecondaryEntries; got != 0 { + t.Fatalf("async final secondary root entries delta=%d want 0", got) + } + if got := stats.IndexedSemanticSkippedSecondaryRoots - before.IndexedSemanticSkippedSecondaryRoots; got == 0 { + t.Fatalf("async skipped secondary roots delta=%d want >0", got) + } + if got := stats.RootDeltaPlanSquashedEntries - before.RootDeltaPlanSquashedEntries; got == 0 { + t.Fatalf("async squashed entries delta=%d want >0", got) + } +} + +func TestPR3bSemanticMixedInsertUpdateFallsBackToMechanicalPublish(t *testing.T) { + d, err := backenddb.Open(backenddb.Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = d.Close() }() + mgr := NewCollectionManager(d) + if _, err := mgr.CreateCollection(&CollectionMeta{ + Name: "users", + Options: CollectionOptions{ + BufferedIndexedWrites: true, + }, + Indexes: []IndexDefinition{{Name: "city", Field: "city", ValueType: IndexValueString}}, + }); err != nil { + t.Fatalf("create collection: %v", err) + } + col, err := mgr.OpenCollection("users") + if err != nil { + t.Fatalf("open collection: %v", err) + } + if _, err := col.InsertBatch( + [][]byte{[]byte("u1")}, + [][]byte{[]byte(`{"city":"hnl","score":0}`)}, + ); err != nil { + t.Fatalf("insert seed user: %v", err) + } + if err := col.Flush(); err != nil { + t.Fatalf("flush seed user: %v", err) + } + + if _, err := col.InsertBatch( + [][]byte{[]byte("u2")}, + [][]byte{[]byte(`{"city":"lax","score":0}`)}, + ); err != nil { + t.Fatalf("buffer insert u2: %v", err) + } + if _, batched, err := col.UpdateBatchIfNoSecondaryUniqueIndexChanges([]UpdateBatchItem{{ + DocumentID: []byte("u1"), + Update: setJSONCity("sea"), + }}); err != nil { + t.Fatalf("buffer update u1: %v", err) + } else if !batched { + t.Fatal("u1 city update was not buffered") + } + + if err := col.Flush(); err != nil { + t.Fatalf("flush mixed insert/update: %v", err) + } + pr3bRequireIndexIDs(t, col, "city", "sea", "u1") + pr3bRequireIndexIDs(t, col, "city", "lax", "u2") + stats := mgr.StatsSnapshot() + if got := stats.IndexedSemanticRawRecords; got != 1 { + t.Fatalf("raw semantic records after mixed flush=%d want 1", got) + } + if got := stats.IndexedSemanticEffectiveRecords; got != 0 { + t.Fatalf("effective semantic records after mixed flush=%d want 0", got) + } +} + +func TestPR3bSemanticMultiSecondaryChangeCountsOneEffectiveRecord(t *testing.T) { + d, err := backenddb.Open(backenddb.Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = d.Close() }() + mgr := NewCollectionManager(d) + if _, err := mgr.CreateCollection(&CollectionMeta{ + Name: "users", + Options: CollectionOptions{ + BufferedIndexedWrites: true, + }, + Indexes: []IndexDefinition{ + {Name: "city", Field: "city", ValueType: IndexValueString}, + {Name: "score", Field: "score", ValueType: IndexValueInt64}, + }, + }); err != nil { + t.Fatalf("create collection: %v", err) + } + col, err := mgr.OpenCollection("users") + if err != nil { + t.Fatalf("open collection: %v", err) + } + if _, err := col.InsertBatch( + [][]byte{[]byte("u1")}, + [][]byte{[]byte(`{"city":"hnl","score":1}`)}, + ); err != nil { + t.Fatalf("insert seed user: %v", err) + } + if err := col.Flush(); err != nil { + t.Fatalf("flush seed user: %v", err) + } + + if _, batched, err := col.UpdateBatchIfNoSecondaryUniqueIndexChanges([]UpdateBatchItem{{ + DocumentID: []byte("u1"), + Update: func(current []byte) ([]byte, bool, error) { + if !bytes.Contains(current, []byte(`"city":"hnl"`)) { + return nil, false, fmt.Errorf("current=%s want city hnl", current) + } + return []byte(`{"city":"sea","score":2}`), true, nil + }, + }}); err != nil { + t.Fatalf("buffer multi-secondary update: %v", err) + } else if !batched { + t.Fatal("multi-secondary update was not buffered") + } + if err := col.Flush(); err != nil { + t.Fatalf("flush multi-secondary update: %v", err) + } + + pr3bRequireIndexIDs(t, col, "city", "sea", "u1") + pr3bRequireIndexIDs(t, col, "score", int64(2), "u1") + stats := mgr.StatsSnapshot() + if got := stats.IndexedSemanticRawRecords; got != 1 { + t.Fatalf("raw semantic records after multi-secondary flush=%d want 1", got) + } + if got := stats.IndexedSemanticRawIndexDeltas; got != 2 { + t.Fatalf("raw semantic index deltas after multi-secondary flush=%d want 2", got) + } + if got := stats.IndexedSemanticEffectiveRecords; got != 1 { + t.Fatalf("effective semantic records after multi-secondary flush=%d want 1", got) + } +} + +func TestPR3bSemanticUniqueHandoffFallsBackToMechanicalPath(t *testing.T) { + d, err := backenddb.Open(backenddb.Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = d.Close() }() + mgr := NewCollectionManager(d) + if _, err := mgr.CreateCollection(&CollectionMeta{ + Name: "users", + Options: CollectionOptions{ + BufferedIndexedWrites: true, + }, + Indexes: []IndexDefinition{{Name: "email", Field: "email", ValueType: IndexValueString, Unique: true}}, + }); err != nil { + t.Fatalf("create collection: %v", err) + } + col, err := mgr.OpenCollection("users") + if err != nil { + t.Fatalf("open collection: %v", err) + } + if _, err := col.InsertBatch( + [][]byte{[]byte("u1"), []byte("u2")}, + [][]byte{[]byte(`{"email":"a@example.com"}`), []byte(`{"email":"b@example.com"}`)}, + ); err != nil { + t.Fatalf("insert users: %v", err) + } + if err := col.Flush(); err != nil { + t.Fatalf("flush seed users: %v", err) + } + + results, batched, err := col.UpdateBatchIfNoSecondaryUniqueIndexChanges([]UpdateBatchItem{ + {DocumentID: []byte("u1"), Update: setJSONEmail("c@example.com")}, + {DocumentID: []byte("u2"), Update: setJSONEmail("a@example.com")}, + }) + if err != nil { + t.Fatalf("UpdateBatchIfNoSecondaryUniqueIndexChanges unique handoff: %v", err) + } + if batched { + t.Fatalf("unique handoff batched=%v results=%+v want explicit fallback", batched, results) + } + stats := mgr.StatsSnapshot() + if got := stats.IndexedSemanticRawRecords; got != 0 { + t.Fatalf("raw semantic records after unique fallback=%d want 0", got) + } + if got := stats.PendingIndexedSemanticRecords; got != 0 { + t.Fatalf("pending semantic records after unique fallback=%d want 0", got) + } + pr3bRequireIndexIDs(t, col, "email", "a@example.com", "u1") + pr3bRequireIndexIDs(t, col, "email", "b@example.com", "u2") + + published, err := col.UpdateBatch([]UpdateBatchItem{ + {DocumentID: []byte("u1"), Update: setJSONEmail("c@example.com")}, + {DocumentID: []byte("u2"), Update: setJSONEmail("a@example.com")}, + }) + if err != nil { + t.Fatalf("mechanical UpdateBatch unique handoff: %v", err) + } + if len(published) != 2 || !published[0].Modified || !published[1].Modified { + t.Fatalf("mechanical handoff results=%+v want two modified rows", published) + } + pr3bRequireIndexIDs(t, col, "email", "a@example.com", "u2") + pr3bRequireIndexIDs(t, col, "email", "c@example.com", "u1") + stats = mgr.StatsSnapshot() + if got := stats.IndexedSemanticRawRecords; got != 0 { + t.Fatalf("raw semantic records after mechanical handoff=%d want 0", got) + } + if got := stats.PendingIndexedSemanticRecords; got != 0 { + t.Fatalf("pending semantic records after mechanical handoff=%d want 0", got) + } +} + +func pr3bSemanticTestCollection(tb testing.TB) (*backenddb.DB, *CollectionManager, *Collection) { + tb.Helper() + d, err := backenddb.Open(backenddb.Options{Dir: tb.TempDir()}) + if err != nil { + tb.Fatalf("open db: %v", err) + } + mgr := NewCollectionManager(d) + if _, err := mgr.CreateCollection(&CollectionMeta{ + Name: "users", + Options: CollectionOptions{ + BufferedIndexedWrites: true, + BufferedIndexedAsyncFlush: true, + BufferedIndexedAsyncFlushMaxQueuedUnits: 8, + }, + Indexes: []IndexDefinition{ + {Name: "email", Field: "email", ValueType: IndexValueString, Unique: true}, + {Name: "city", Field: "city", ValueType: IndexValueString}, + }, + }); err != nil { + _ = d.Close() + tb.Fatalf("create collection: %v", err) + } + col, err := mgr.OpenCollection("users") + if err != nil { + _ = d.Close() + tb.Fatalf("open collection: %v", err) + } + return d, mgr, col +} + +func pr3bSeedSemanticUser(tb testing.TB, col *Collection) { + tb.Helper() + if _, err := col.InsertBatch( + [][]byte{[]byte("u1")}, + [][]byte{[]byte(`{"email":"a@example.com","city":"hnl","score":0}`)}, + ); err != nil { + tb.Fatalf("insert seed user: %v", err) + } + if err := col.Flush(); err != nil { + tb.Fatalf("flush seed user: %v", err) + } +} + +func pr3bRequireCitySemanticRecord(tb testing.TB, records []indexedSemanticRecord, oldCity, newCity string) { + tb.Helper() + if len(records) != 1 { + tb.Fatalf("semantic records=%d want 1", len(records)) + } + record := records[0] + if record.kind != indexedSemanticRecordUpdate { + tb.Fatalf("semantic record kind=%d want update", record.kind) + } + if !bytes.Equal(record.documentID, []byte("u1")) { + tb.Fatalf("semantic record documentID=%q want u1", record.documentID) + } + if record.fallback != indexedSemanticFallbackNone { + tb.Fatalf("semantic record fallback=%d want none", record.fallback) + } + if len(record.indexDeltas) != 1 { + tb.Fatalf("semantic index deltas=%d want 1", len(record.indexDeltas)) + } + delta := record.indexDeltas[0] + if delta.indexName != "city" || delta.rootName != collectionSecondaryRootName("users", "city") || delta.unique { + tb.Fatalf("city delta identity index=%q root=%q unique=%v", delta.indexName, delta.rootName, delta.unique) + } + wantOld, err := encodeIndexScalar(IndexValueString, oldCity) + if err != nil { + tb.Fatalf("encode old city %q: %v", oldCity, err) + } + wantNew, err := encodeIndexScalar(IndexValueString, newCity) + if err != nil { + tb.Fatalf("encode new city %q: %v", newCity, err) + } + if !pr3bSemanticValueSetsEqual(delta.oldValues, [][]byte{wantOld}) { + tb.Fatalf("old city values=%q want %q", delta.oldValues, [][]byte{wantOld}) + } + if !pr3bSemanticValueSetsEqual(delta.newValues, [][]byte{wantNew}) { + tb.Fatalf("new city values=%q want %q", delta.newValues, [][]byte{wantNew}) + } +} + +func pr3bSemanticValueSetsEqual(left, right [][]byte) bool { + if len(left) != len(right) { + return false + } + for i := range left { + if !bytes.Equal(left[i], right[i]) { + return false + } + } + return true +} + +func pr3bRequireIndexIDs(tb testing.TB, col *Collection, indexName string, value any, want ...string) { + tb.Helper() + ids, err := col.FindByIndexValue(indexName, value) + if err != nil { + tb.Fatalf("find index %s=%v: %v", indexName, value, err) + } + if len(ids) != len(want) { + tb.Fatalf("index %s=%v ids=%q want %q", indexName, value, ids, want) + } + for i := range want { + if !bytes.Equal(ids[i], []byte(want[i])) { + tb.Fatalf("index %s=%v ids=%q want %q", indexName, value, ids, want) + } + } +} + +func pr3bRequireSemanticMetricKeys(tb testing.TB, mgr *CollectionManager) { + tb.Helper() + exported := mgr.Stats() + for _, key := range []string{ + "treedb.collections.write_domain.pending_indexed_semantic_records", + "treedb.collections.write_domain.indexed_semantic.raw_records_total", + "treedb.collections.write_domain.indexed_semantic.raw_index_deltas_total", + "treedb.collections.write_domain.indexed_semantic.fallback_records_total", + "treedb.collections.write_domain.indexed_semantic.effective_records_total", + "treedb.collections.write_domain.indexed_semantic.skipped_secondary_roots_total", + "treedb.collections.write_domain.coalesced_flush_batch.batches_total", + "treedb.collections.write_domain.coalesced_flush_batch.units_total", + "treedb.collections.write_domain.coalesced_flush_batch.docs_total", + "treedb.collections.write_domain.coalesced_flush_batch.bytes_total", + "treedb.collections.write_domain.coalesced_flush_batch.net_zero_batches_total", + "treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total", + "treedb.collections.write_domain.root_delta_plan.final.primary.entries_total", + "treedb.collections.write_domain.root_delta_plan.squashed_entries_total", + "treedb.collections.write_domain.root_delta_plan.net_zero_plans_total", + "treedb.collections.write_domain.primary_only.duplicate_ids_coalesced_total", + "treedb.collections.write_domain.primary_only.drains_total", + } { + if exported[key] == "" { + tb.Fatalf("exported stats missing %s from %#v", key, exported) + } + } +} diff --git a/TreeDB/collections/shape_bench_test.go b/TreeDB/collections/shape_bench_test.go index a67ddbfeeb..7de5d49b10 100644 --- a/TreeDB/collections/shape_bench_test.go +++ b/TreeDB/collections/shape_bench_test.go @@ -2,7 +2,10 @@ package collections_test import ( "fmt" + "os" "runtime" + "strconv" + "strings" "sync" "sync/atomic" "testing" @@ -81,25 +84,22 @@ func benchmarkReportCollectionInsertStats(b *testing.B, docs, batches int, stats } func collectionShapeIndexes(indexCount int) []collections.IndexDefinition { - switch indexCount { - case 0: + if indexCount == 0 { return nil - case 1: - return []collections.IndexDefinition{{Name: "email_idx", Field: "email", ValueType: collections.IndexValueString, Unique: true}} - case 2: - return []collections.IndexDefinition{ - {Name: "email_idx", Field: "email", ValueType: collections.IndexValueString, Unique: true}, - {Name: "city_idx", Field: "city", ValueType: collections.IndexValueString}, - } - case 3: - return []collections.IndexDefinition{ - {Name: "email_idx", Field: "email", ValueType: collections.IndexValueString, Unique: true}, - {Name: "city_idx", Field: "city", ValueType: collections.IndexValueString}, - {Name: "name_idx", Field: "name", ValueType: collections.IndexValueString}, - } - default: - panic(fmt.Sprintf("unsupported collection benchmark index count %d", indexCount)) } + indexes := make([]collections.IndexDefinition, 0, indexCount) + indexes = append(indexes, collections.IndexDefinition{Name: "email_idx", Field: "email", ValueType: collections.IndexValueString, Unique: true}) + if indexCount >= 2 { + indexes = append(indexes, collections.IndexDefinition{Name: "city_idx", Field: "city", ValueType: collections.IndexValueString}) + } + if indexCount >= 3 { + indexes = append(indexes, collections.IndexDefinition{Name: "name_idx", Field: "name", ValueType: collections.IndexValueString}) + } + for i := 4; i <= indexCount; i++ { + field := collectionShapeExtraIndexFieldName(i) + indexes = append(indexes, collections.IndexDefinition{Name: field + "_idx", Field: field, ValueType: collections.IndexValueString}) + } + return indexes } func collectionSingleStringIndexes(indexCount int) []collections.IndexDefinition { @@ -133,6 +133,80 @@ func benchmarkSingleStringDocumentBatch(tb testing.TB, start, count int) ([][]by return ids, docs } +func benchmarkCollectionShapeIndexCounts(b *testing.B) []int { + b.Helper() + raw := strings.TrimSpace(os.Getenv("TREEDB_COLLECTION_SHAPE_INDEX_COUNTS")) + if raw == "" { + return []int{0, 1, 2, 3} + } + parts := strings.Split(raw, ",") + counts := make([]int, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + n, err := strconv.Atoi(part) + if err != nil || n < 0 { + b.Fatalf("unsupported TREEDB_COLLECTION_SHAPE_INDEX_COUNTS=%q", raw) + } + counts = append(counts, n) + } + if len(counts) == 0 { + b.Fatalf("unsupported TREEDB_COLLECTION_SHAPE_INDEX_COUNTS=%q", raw) + } + return counts +} + +func benchmarkCollectionShapeDocumentBatch(tb testing.TB, start, count, indexCount int) ([][]byte, [][]byte) { + tb.Helper() + if indexCount <= 3 { + return benchmarkDocumentBatch(tb, start, count, true) + } + if benchmarkCollectionDocumentFormat(tb) != collections.DocumentFormatJSON { + tb.Skip("shape benchmark index counts above 3 use JSON documents") + } + ids := make([][]byte, count) + docs := make([][]byte, count) + for i := 0; i < count; i++ { + docNum := start + i + ids[i] = benchmarkDocumentID(docNum) + docs[i] = benchmarkCollectionShapeIndexedDocument(docNum, indexCount) + } + return ids, docs +} + +func benchmarkCollectionShapeIndexedDocument(n, indexCount int) []byte { + if indexCount <= 3 { + return benchmarkIndexedDocument(n) + } + out := make([]byte, 0, 112+(indexCount-3)*24) + out = append(out, `{"name":"user-`...) + out = appendZeroPaddedInt(out, n, 9) + out = append(out, `","email":"user-`...) + out = appendZeroPaddedInt(out, n, 9) + out = append(out, `@example.com","city":"city-`...) + out = appendZeroPaddedInt(out, n%collectionBenchCities, 2) + out = append(out, `"`...) + for i := 4; i <= indexCount; i++ { + out = append(out, `,"`...) + out = append(out, collectionShapeExtraIndexFieldName(i)...) + out = append(out, `":"v-`...) + out = appendZeroPaddedInt(out, i, 3) + out = append(out, '-') + out = appendZeroPaddedInt(out, (n+i)%1024, 4) + out = append(out, '"') + } + out = append(out, `,"pad":"`...) + out = append(out, collectionBenchIndexedPad...) + out = append(out, `"}`...) + return out +} + +func collectionShapeExtraIndexFieldName(n int) string { + return fmt.Sprintf("k_%03d", n) +} + func benchmarkCollectionShapeInsertBatch(b *testing.B, indexCount int, checkpoint bool) { backend, collection := openBenchmarkCollection(b, fmt.Sprintf("bench_shape_insert_%d", indexCount), collectionShapeIndexes(indexCount)...) targetBatchSize := benchmarkBatchSize(b) @@ -156,7 +230,7 @@ func benchmarkCollectionShapeInsertBatch(b *testing.B, indexCount int, checkpoin if remaining := b.N - inserted; remaining < batchSize { batchSize = remaining } - ids, docs := benchmarkDocumentBatch(b, inserted, batchSize, true) + ids, docs := benchmarkCollectionShapeDocumentBatch(b, inserted, batchSize, indexCount) b.StartTimer() insertStart := time.Now() @@ -229,7 +303,7 @@ func benchmarkCollectionShapeInsertBatch(b *testing.B, indexCount int, checkpoin } func BenchmarkCollectionShapeInsertBatch(b *testing.B) { - for _, indexCount := range []int{0, 1, 2, 3} { + for _, indexCount := range benchmarkCollectionShapeIndexCounts(b) { b.Run(fmt.Sprintf("indexes_%d", indexCount), func(b *testing.B) { benchmarkCollectionShapeInsertBatch(b, indexCount, false) }) @@ -237,7 +311,7 @@ func BenchmarkCollectionShapeInsertBatch(b *testing.B) { } func BenchmarkCollectionShapeInsertBatchCheckpoint(b *testing.B) { - for _, indexCount := range []int{0, 1, 2, 3} { + for _, indexCount := range benchmarkCollectionShapeIndexCounts(b) { b.Run(fmt.Sprintf("indexes_%d", indexCount), func(b *testing.B) { benchmarkCollectionShapeInsertBatch(b, indexCount, true) }) diff --git a/TreeDB/collections/template_v1.go b/TreeDB/collections/template_v1.go index 4eb89dd274..68e47d763d 100644 --- a/TreeDB/collections/template_v1.go +++ b/TreeDB/collections/template_v1.go @@ -49,6 +49,7 @@ type templateV1Record struct { type templateV1Template struct { id [32]byte + raw []byte fields []string } @@ -57,7 +58,7 @@ type templateV1Resolver interface { } type templateV1MemoryResolver struct { - templates map[string]*templateV1Template + templates map[[32]byte]*templateV1Template } type templateV1CompositeResolver struct { @@ -68,13 +69,13 @@ type templateV1CompositeResolver struct { type templateV1SnapshotResolver struct { snap *backenddb.Snapshot rootID uint64 - cache map[string]*templateV1Template + cache map[[32]byte]*templateV1Template } type templateV1BufferedRunsResolver struct { runs []memtable.Table fallback templateV1Resolver - cache map[string]*templateV1Template + cache map[[32]byte]*templateV1Template } type templateV1ObjectRef struct { @@ -135,7 +136,7 @@ func collectionOptionsWithTemplateV1Resolver(opts collectionOptions, snap *backe opts.templateResolver = &templateV1SnapshotResolver{ snap: snap, rootID: catalog.rootID(collectionTemplateRootName(catalog.meta.Name)), - cache: make(map[string]*templateV1Template), + cache: make(map[[32]byte]*templateV1Template), } return opts } @@ -161,7 +162,7 @@ func collectionOptionsWithBufferedTemplateV1RunsResolver(opts collectionOptions, opts.templateResolver = &templateV1BufferedRunsResolver{ runs: runs, fallback: opts.templateResolver, - cache: make(map[string]*templateV1Template), + cache: make(map[[32]byte]*templateV1Template), } return opts } @@ -171,7 +172,7 @@ func prepareTemplateV1InsertDocuments(documents [][]byte, fallback templateV1Res records := make([]templateV1Record, 0) resolver := &templateV1MemoryResolver{} for i, document := range documents { - stored, docRecords, err := parseTemplateV1InsertDocument(document) + stored, docRecords, err := parseTemplateV1InsertDocumentWithMemoryResolver(document, resolver) if err != nil { return nil, nil, nil, err } @@ -238,7 +239,18 @@ func parseTemplateV1InsertDocument(raw []byte) ([]byte, []templateV1Record, erro return parseTemplateV1InsertEnvelope(raw) } +func parseTemplateV1InsertDocumentWithMemoryResolver(raw []byte, resolver *templateV1MemoryResolver) ([]byte, []templateV1Record, error) { + if bytes.HasPrefix(raw, []byte(templateV1StoredMagic)) { + return raw, nil, nil + } + return parseTemplateV1InsertEnvelopeWithMemoryResolver(raw, resolver) +} + func parseTemplateV1InsertEnvelope(raw []byte) ([]byte, []templateV1Record, error) { + return parseTemplateV1InsertEnvelopeWithMemoryResolver(raw, nil) +} + +func parseTemplateV1InsertEnvelopeWithMemoryResolver(raw []byte, resolver *templateV1MemoryResolver) ([]byte, []templateV1Record, error) { pos := 0 if !consumeMagic(raw, &pos, templateV1InputMagic) { return nil, nil, errors.New("collections: template-v1 insert requires template input envelope") @@ -250,7 +262,7 @@ func parseTemplateV1InsertEnvelope(raw []byte) ([]byte, []templateV1Record, erro if templateCount > uint64(len(raw)) { return nil, nil, errors.New("collections: malformed template-v1 template count") } - records := make([]templateV1Record, 0, int(templateCount)) + var records []templateV1Record for i := uint64(0); i < templateCount; i++ { var id [32]byte if len(raw)-pos < len(id) { @@ -267,6 +279,11 @@ func parseTemplateV1InsertEnvelope(raw []byte) ([]byte, []templateV1Record, erro } recordRaw := raw[pos : pos+int(recordLen)] pos += int(recordLen) + if resolver != nil { + if existing := resolver.templates[id]; existing != nil && bytes.Equal(existing.raw, recordRaw) { + continue + } + } record, err := parseTemplateV1Record(recordRaw) if err != nil { return nil, nil, err @@ -274,6 +291,9 @@ func parseTemplateV1InsertEnvelope(raw []byte) ([]byte, []templateV1Record, erro if record.id != id { return nil, nil, errors.New("collections: template-v1 template id does not match record") } + if records == nil { + records = make([]templateV1Record, 0, int(templateCount-i)) + } records = append(records, record) } stored := raw[pos:] @@ -318,16 +338,15 @@ func validateTemplateV1StoredDocumentTemplates(document []byte, resolver templat func (r *templateV1MemoryResolver) addRecord(record templateV1Record) (bool, error) { if r.templates == nil { - r.templates = make(map[string]*templateV1Template) + r.templates = make(map[[32]byte]*templateV1Template) } - key := string(record.id[:]) - if existing := r.templates[key]; existing != nil { + if existing := r.templates[record.id]; existing != nil { if !equalStringSlices(existing.fields, record.tpl.fields) { return false, errors.New("collections: template-v1 template id collision") } return false, nil } - r.templates[key] = record.tpl + r.templates[record.id] = record.tpl return true, nil } @@ -335,7 +354,7 @@ func (r *templateV1MemoryResolver) lookupTemplateV1(id [32]byte) (*templateV1Tem if r == nil { return nil, errTemplateV1MissingResolver } - tpl := r.templates[string(id[:])] + tpl := r.templates[id] if tpl == nil { return nil, errTemplateV1TemplateNotFound } @@ -347,7 +366,7 @@ func (r *templateV1CompositeResolver) lookupTemplateV1(id [32]byte) (*templateV1 return nil, errTemplateV1MissingResolver } if r.memory != nil && r.memory.templates != nil { - if tpl := r.memory.templates[string(id[:])]; tpl != nil { + if tpl := r.memory.templates[id]; tpl != nil { return tpl, nil } } @@ -361,8 +380,7 @@ func (r *templateV1SnapshotResolver) lookupTemplateV1(id [32]byte) (*templateV1T if r == nil || r.snap == nil || r.rootID == 0 { return nil, errTemplateV1MissingTemplateRoot } - key := string(id[:]) - if tpl := r.cache[key]; tpl != nil { + if tpl := r.cache[id]; tpl != nil { return tpl, nil } entry, err := r.snap.GetEntryAtRoot(r.rootID, id[:]) @@ -380,9 +398,9 @@ func (r *templateV1SnapshotResolver) lookupTemplateV1(id [32]byte) (*templateV1T return nil, errors.New("collections: template-v1 template id mismatch") } if r.cache == nil { - r.cache = make(map[string]*templateV1Template) + r.cache = make(map[[32]byte]*templateV1Template) } - r.cache[key] = record.tpl + r.cache[id] = record.tpl return record.tpl, nil } @@ -390,8 +408,7 @@ func (r *templateV1BufferedRunsResolver) lookupTemplateV1(id [32]byte) (*templat if r == nil { return nil, errTemplateV1MissingResolver } - key := string(id[:]) - if tpl := r.cache[key]; tpl != nil { + if tpl := r.cache[id]; tpl != nil { return tpl, nil } for i := len(r.runs) - 1; i >= 0; i-- { @@ -411,9 +428,9 @@ func (r *templateV1BufferedRunsResolver) lookupTemplateV1(id [32]byte) (*templat return nil, errors.New("collections: template-v1 template id mismatch") } if r.cache == nil { - r.cache = make(map[string]*templateV1Template) + r.cache = make(map[[32]byte]*templateV1Template) } - r.cache[key] = record.tpl + r.cache[id] = record.tpl return record.tpl, nil } if r.fallback != nil { @@ -903,10 +920,11 @@ func parseTemplateV1Record(raw []byte) (templateV1Record, error) { return templateV1Record{}, errors.New("collections: trailing template-v1 template bytes") } id := sha256.Sum256(raw) + rawCopy := bytes.Clone(raw) return templateV1Record{ id: id, - raw: bytes.Clone(raw), - tpl: &templateV1Template{id: id, fields: fields}, + raw: rawCopy, + tpl: &templateV1Template{id: id, raw: rawCopy, fields: fields}, }, nil } diff --git a/TreeDB/collections/template_v1_test.go b/TreeDB/collections/template_v1_test.go index 355474d501..b9c28f0f4a 100644 --- a/TreeDB/collections/template_v1_test.go +++ b/TreeDB/collections/template_v1_test.go @@ -134,7 +134,7 @@ func TestTemplateV1CollectionInsertBatchIndexesAndTemplateRoot(t *testing.T) { _ = snap.Close() t.Fatalf("parse stored document: %v", err) } - resolver := &templateV1SnapshotResolver{snap: snap, rootID: templateRoot, cache: make(map[string]*templateV1Template)} + resolver := &templateV1SnapshotResolver{snap: snap, rootID: templateRoot, cache: make(map[[32]byte]*templateV1Template)} tpl, err := resolver.lookupTemplateV1(root.templateID) _ = snap.Close() if err != nil { @@ -311,7 +311,7 @@ func TestTemplateV1EncoderReusesPersistedTemplateRoot(t *testing.T) { _ = snap.Close() t.Fatalf("parse doc2 root: %v", err) } - resolver := &templateV1SnapshotResolver{snap: snap, rootID: templateRoot, cache: make(map[string]*templateV1Template)} + resolver := &templateV1SnapshotResolver{snap: snap, rootID: templateRoot, cache: make(map[[32]byte]*templateV1Template)} if _, err := resolver.lookupTemplateV1(rootDoc2.templateID); err != nil { _ = snap.Close() t.Fatalf("lookup doc2 template after first insert: %v", err) @@ -647,7 +647,7 @@ func TestTemplateV1UnbufferedSingleInsertUsesTemplateRoot(t *testing.T) { _ = snap.Close() t.Fatalf("parse stored doc: %v", err) } - resolver := &templateV1SnapshotResolver{snap: snap, rootID: templateRoot, cache: make(map[string]*templateV1Template)} + resolver := &templateV1SnapshotResolver{snap: snap, rootID: templateRoot, cache: make(map[[32]byte]*templateV1Template)} if _, err := resolver.lookupTemplateV1(stored.templateID); err != nil { _ = snap.Close() t.Fatalf("lookup template: %v", err) diff --git a/TreeDB/collections/update_combiner_bench_test.go b/TreeDB/collections/update_combiner_bench_test.go new file mode 100644 index 0000000000..ceb423e17a --- /dev/null +++ b/TreeDB/collections/update_combiner_bench_test.go @@ -0,0 +1,145 @@ +package collections + +import ( + "fmt" + "sync/atomic" + "testing" + "time" + + backenddb "github.com/snissn/gomap/TreeDB/db" +) + +func BenchmarkCollectionUpdateCombinerTemplateV1NewShape(b *testing.B) { + for _, parallelism := range []int{1, 8, 32} { + b.Run(fmt.Sprintf("parallel_%d", parallelism), func(b *testing.B) { + benchmarkCollectionUpdateCombinerTemplateV1NewShape(b, parallelism) + }) + } +} + +func benchmarkCollectionUpdateCombinerTemplateV1NewShape(b *testing.B, parallelism int) { + b.Helper() + if parallelism <= 0 { + b.Fatalf("invalid parallelism %d", parallelism) + } + docs := b.N + if docs <= 0 { + docs = 1 + } + d, err := backenddb.Open(backenddb.Options{Dir: b.TempDir()}) + if err != nil { + b.Fatalf("open db: %v", err) + } + defer func() { _ = d.Close() }() + + mgr := NewCollectionManager(d) + if _, err := mgr.CreateCollection(&CollectionMeta{ + Name: "bench", + Options: CollectionOptions{ + DocumentFormat: DocumentFormatTemplateV1, + BufferedIndexedWrites: true, + BufferedIndexedWriteMaxDocuments: 1 << 30, + BufferedIndexedWriteMaxBytes: 1 << 40, + BufferedIndexedWriteMaxRootRuns: 1 << 30, + }, + Indexes: []IndexDefinition{ + {Name: "email", Field: "email", ValueType: IndexValueString, Unique: true}, + {Name: "city", Field: "city", ValueType: IndexValueString}, + }, + }); err != nil { + b.Fatalf("create collection: %v", err) + } + col, err := mgr.OpenCollection("bench") + if err != nil { + b.Fatalf("open collection: %v", err) + } + + ids := make([][]byte, docs) + var preloadEncoder TemplateV1Encoder + const preloadBatchSize = 16000 + for start := 0; start < docs; start += preloadBatchSize { + n := preloadBatchSize + if remaining := docs - start; remaining < n { + n = remaining + } + batchIDs := make([][]byte, n) + batchDocs := make([][]byte, n) + for i := 0; i < n; i++ { + docID := benchmarkTemplateV1UpdateDocID(start + i) + ids[start+i] = docID + batchIDs[i] = docID + batchDocs[i] = benchmarkTemplateV1BaseUpdateDocument(b, &preloadEncoder, start+i) + } + if _, err := col.InsertBatch(batchIDs, batchDocs); err != nil { + b.Fatalf("insert preload batch %d: %v", start/preloadBatchSize, err) + } + } + if err := col.Flush(); err != nil { + b.Fatalf("flush preload: %v", err) + } + + updateDocs := make([][]byte, docs) + for start := 0; start < docs; start += preloadBatchSize { + shapeOrdinal := start / preloadBatchSize + for i := start; i < start+preloadBatchSize && i < docs; i++ { + updateDocs[i] = benchmarkTemplateV1NewShapeUpdateDocument(b, i, shapeOrdinal) + } + } + + statsBefore := mgr.StatsSnapshot() + b.ReportAllocs() + b.SetParallelism(parallelism) + b.ResetTimer() + startTime := time.Now() + var next atomic.Uint64 + var failed atomic.Bool + var benchErr atomic.Value + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if failed.Load() { + return + } + i := int(next.Add(1) - 1) + matched, modified, err := col.Update(ids[i], benchmarkTemplateV1ReplaceWith(updateDocs[i])) + if err != nil { + if failed.CompareAndSwap(false, true) { + benchErr.Store(fmt.Sprintf("update %d: %v", i, err)) + } + return + } + if !matched || !modified { + if failed.CompareAndSwap(false, true) { + benchErr.Store(fmt.Sprintf("update %d matched=%v modified=%v want true/true", i, matched, modified)) + } + return + } + } + }) + if v := benchErr.Load(); v != nil { + b.Fatal(v) + } + if err := col.Flush(); err != nil { + b.Fatalf("flush updates: %v", err) + } + elapsed := time.Since(startTime) + b.StopTimer() + + statsAfter := mgr.StatsSnapshot() + combineRequests := statsAfter.UpdateCombineRequests - statsBefore.UpdateCombineRequests + combineBatches := statsAfter.UpdateCombineBatches - statsBefore.UpdateCombineBatches + combineBatchedRequests := statsAfter.UpdateCombineBatchedRequests - statsBefore.UpdateCombineBatchedRequests + combineFallbackRequests := statsAfter.UpdateCombineFallbackRequests - statsBefore.UpdateCombineFallbackRequests + combineQueueDepthMax := statsAfter.UpdateCombineQueueDepthMax + if statsBefore.UpdateCombineQueueDepthMax > combineQueueDepthMax { + combineQueueDepthMax = 0 + } + b.ReportMetric(float64(docs)/elapsed.Seconds(), "docs/sec") + b.ReportMetric(float64(elapsed.Nanoseconds())/float64(docs), "ns/doc") + b.ReportMetric(float64(combineRequests), "update_combine_requests") + b.ReportMetric(float64(combineBatches), "update_combine_batches") + if combineBatches > 0 { + b.ReportMetric(float64(combineBatchedRequests)/float64(combineBatches), "update_combine_requests/batch") + } + b.ReportMetric(float64(combineFallbackRequests), "update_combine_fallback_requests") + b.ReportMetric(float64(combineQueueDepthMax), "update_combine_queue_depth_max") +} diff --git a/TreeDB/db/alloc_tracker.go b/TreeDB/db/alloc_tracker.go index 0adf077e35..58cd6529bb 100644 --- a/TreeDB/db/alloc_tracker.go +++ b/TreeDB/db/alloc_tracker.go @@ -4,20 +4,33 @@ import ( "sync" "github.com/snissn/gomap/TreeDB/freelist" + "github.com/snissn/gomap/TreeDB/page" ) // allocTracker wraps the freelist allocator and remembers allocated pages so // they can be returned if a write attempt is abandoned. type allocTracker struct { - alloc *freelist.Allocator - mu sync.Mutex - pages []uint64 + alloc *freelist.Allocator + mu sync.Mutex + pages []uint64 + leafLogPtrs []page.LeafLogPtr + + preparedOutputID preparedOutputID + preparedOutputState preparedOutputState } func newAllocTracker(alloc *freelist.Allocator) *allocTracker { return &allocTracker{alloc: alloc} } +func newPreparedOutputAllocTracker(alloc *freelist.Allocator, id preparedOutputID) *allocTracker { + return &allocTracker{ + alloc: alloc, + preparedOutputID: id, + preparedOutputState: preparedOutputStatePrepared, + } +} + func (t *allocTracker) Alloc(hint uint64) (uint64, error) { id, err := t.alloc.Alloc(hint) if err != nil { @@ -38,13 +51,80 @@ func (t *allocTracker) Pages() []uint64 { return append([]uint64(nil), t.pages...) } +func (t *allocTracker) PreparedOutputID() preparedOutputID { + if t == nil { + return 0 + } + t.mu.Lock() + defer t.mu.Unlock() + return t.preparedOutputID +} + +func (t *allocTracker) PreparedOutputSnapshot() preparedOutputSnapshot { + if t == nil { + return preparedOutputSnapshot{} + } + t.mu.Lock() + defer t.mu.Unlock() + return preparedOutputSnapshot{ + ID: t.preparedOutputID, + State: t.preparedOutputState, + Pages: append([]uint64(nil), t.pages...), + LeafLogPtrs: append([]page.LeafLogPtr(nil), t.leafLogPtrs...), + } +} + +func (t *allocTracker) PreparedOutputCounts() (pages, leafLogPtrs uint64) { + if t == nil { + return 0, 0 + } + t.mu.Lock() + defer t.mu.Unlock() + return uint64(len(t.pages)), uint64(len(t.leafLogPtrs)) +} + +func (t *allocTracker) notePreparedLeafLogPtr(ptr page.LeafLogPtr) { + if t == nil { + return + } + t.mu.Lock() + if t.preparedOutputID != 0 && t.preparedOutputState == preparedOutputStatePrepared { + t.leafLogPtrs = append(t.leafLogPtrs, ptr) + } + t.mu.Unlock() +} + +func (t *allocTracker) MarkInstalled() { + if t == nil { + return + } + t.mu.Lock() + if t.preparedOutputID != 0 && t.preparedOutputState == preparedOutputStatePrepared { + t.preparedOutputState = preparedOutputStateInstalled + } + t.mu.Unlock() +} + +// FreeAll releases tracked pages for abandoned write attempts. Prepared output +// trackers that have been marked installed intentionally retain their pages; +// those pages are now reachable through the installed root and must not be +// returned to the allocator by this cleanup path. func (t *allocTracker) FreeAll() error { if t == nil { return nil } t.mu.Lock() + if t.preparedOutputState == preparedOutputStateInstalled { + t.mu.Unlock() + return nil + } pages := append([]uint64(nil), t.pages...) t.pages = nil + hadSideOutput := len(pages) > 0 || len(t.leafLogPtrs) > 0 + t.leafLogPtrs = nil + if t.preparedOutputID != 0 && hadSideOutput { + t.preparedOutputState = preparedOutputStateAbandoned + } t.mu.Unlock() var firstErr error for _, id := range pages { diff --git a/TreeDB/db/api.go b/TreeDB/db/api.go index 88f333d4eb..af3e2bf8b4 100644 --- a/TreeDB/db/api.go +++ b/TreeDB/db/api.go @@ -340,7 +340,7 @@ func (db *DB) Has(key []byte) (bool, error) { // Set sets the value for a key. func (db *DB) Set(key, value []byte) error { unlock := db.lockUpdateKey(key) - defer unlock() + defer unlock.Unlock() return db.setPoint(key, value, false) } @@ -354,14 +354,14 @@ func (db *DB) setPoint(key, value []byte, sync bool) error { // SetSync sets the value and syncs to disk. func (db *DB) SetSync(key, value []byte) error { unlock := db.lockUpdateKey(key) - defer unlock() + defer unlock.Unlock() return db.setPoint(key, value, true) } // Delete removes a key. func (db *DB) Delete(key []byte) error { unlock := db.lockUpdateKey(key) - defer unlock() + defer unlock.Unlock() return db.deletePoint(key, false) } @@ -375,7 +375,7 @@ func (db *DB) deletePoint(key []byte, sync bool) error { // DeleteSync removes a key and syncs. func (db *DB) DeleteSync(key []byte) error { unlock := db.lockUpdateKey(key) - defer unlock() + defer unlock.Unlock() return db.deletePoint(key, true) } @@ -710,6 +710,12 @@ func (db *DB) Stats() map[string]string { // Backend DB path currently doesn't track queue drift; emit a stable default // for suite compatibility and fail-closed checks that require key presence. stats["treedb.publish.watermark.lag_drift_bytes_per_sec"] = "0.000" + stats["treedb.publish.install_guard.ns_total"] = fmt.Sprintf("%d", db.publishInstallGuardNs.Load()) + stats["treedb.publish.install_guard.calls_total"] = fmt.Sprintf("%d", db.publishInstallGuardCalls.Load()) + stats["treedb.publish.install_guard.failures_total"] = fmt.Sprintf("%d", db.publishInstallGuardFailures.Load()) + stats["treedb.publish.install_guard.hook_failures_total"] = fmt.Sprintf("%d", db.publishInstallGuardHookFailures.Load()) + stats["treedb.publish.install_guard.user_root_mismatches_total"] = fmt.Sprintf("%d", db.publishInstallGuardUserRootMismatches.Load()) + stats["treedb.publish.install_guard.system_root_mismatches_total"] = fmt.Sprintf("%d", db.publishInstallGuardSystemRootMismatches.Load()) orderedDeltaStats := db.orderedRootDeltaGroupPublishStats() // Ordered-root delta group stats cover calls that entered the DB write // lock, including failed calls. roots_total counts successfully published @@ -740,6 +746,10 @@ func (db *DB) Stats() map[string]string { stats["treedb.publish.ordered_root_delta_group.root_apply_leaf_log_record_hint_bytes_read_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyLeafLogRecordHintBytesRead) stats["treedb.publish.ordered_root_delta_group.root_apply_leaf_merges_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyLeafMerges) stats["treedb.publish.ordered_root_delta_group.root_apply_internal_merges_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyInternalMerges) + stats["treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_merges_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyInternalParallelMerges) + stats["treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_children_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyInternalParallelChildren) + stats["treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_workers_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyInternalParallelWorkers) + stats["treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_ops_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyInternalParallelOps) stats["treedb.publish.ordered_root_delta_group.root_apply_leaf_pages_written_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyLeafPagesWritten) stats["treedb.publish.ordered_root_delta_group.root_apply_pager_leaf_pages_written_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyPagerLeafPagesWritten) stats["treedb.publish.ordered_root_delta_group.root_apply_leaf_log_pages_written_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyLeafLogPagesWritten) @@ -754,11 +764,51 @@ func (db *DB) Stats() map[string]string { stats["treedb.publish.ordered_root_delta_group.root_apply_internal_leaf_log_refs_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyInternalLeafLogRefs) stats["treedb.publish.ordered_root_delta_group.root_apply_internal_leaf_log_ref_copies_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyInternalLeafLogRefCopies) stats["treedb.publish.ordered_root_delta_group.root_apply_root_split_levels_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyRootSplitLevels) + stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ns_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyReadOnlyPrepareNs) + stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyReadOnlyPrepareCalls) + stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ops_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyReadOnlyPrepareOps) + stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_leaf_spans_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyReadOnlyPrepareLeafSpans) + stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_targets_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyReadOnlyPrepareWorker.targets) + stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_ranges_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyReadOnlyPrepareWorker.ranges) + stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_range_min_ops_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyReadOnlyPrepareWorker.minOps) + stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_range_max_ops_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyReadOnlyPrepareWorker.maxOps) + stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_range_single_span_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyReadOnlyPrepareWorker.singleSpan) + stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_exact_plans_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyReadOnlyPrepareExactPlans) + stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_maintenance_plans_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyReadOnlyPrepareMaintenance) + stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_cold_build_plans_total"] = fmt.Sprintf("%d", orderedDeltaStats.rootApplyReadOnlyPrepareColdBuilds) stats["treedb.publish.ordered_root_delta_group.system_build_ns_total"] = fmt.Sprintf("%d", orderedDeltaStats.systemBuildNs) stats["treedb.publish.ordered_root_delta_group.system_apply_ns_total"] = fmt.Sprintf("%d", orderedDeltaStats.systemApplyNs) stats["treedb.publish.ordered_root_delta_group.system_apply_calls_total"] = fmt.Sprintf("%d", orderedDeltaStats.systemApplyCalls) stats["treedb.publish.ordered_root_delta_group.system_apply_ops_total"] = fmt.Sprintf("%d", orderedDeltaStats.systemApplyOps) stats["treedb.publish.ordered_root_delta_group.system_apply_node_loads_total"] = fmt.Sprintf("%d", orderedDeltaStats.systemApplyNodeLoads) + stats["treedb.publish.ordered_root_delta_group.install_guard_ns_total"] = fmt.Sprintf("%d", orderedDeltaStats.installGuardNs) + stats["treedb.publish.ordered_root_delta_group.install_guard_calls_total"] = fmt.Sprintf("%d", orderedDeltaStats.installGuardCalls) + stats["treedb.publish.ordered_root_delta_group.install_guard_failures_total"] = fmt.Sprintf("%d", orderedDeltaStats.installGuardFailures) + // prepared_root.prepare_ns_total includes metadata planning time, including + // attempts that fail before any root reaches prepared state. Other + // prepared_root.* counters count roots that reached prepared state, including + // optimistic attempts abandoned before retrying through serialized publish; + // they intentionally are not a strict subset of calls_total/roots_total. + // prepared_root.output_* counters count pager and leaf-log side output + // produced by those prepared roots. Leaf-log output remains persistent + // value-log storage; these counters are inventory/ownership observability, not + // reclamation. + stats["treedb.publish.ordered_root_delta_group.prepared_root.prepare_ns_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootPrepareNs) + stats["treedb.publish.ordered_root_delta_group.prepared_root.groups_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootGroups) + stats["treedb.publish.ordered_root_delta_group.prepared_root.roots_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootRoots) + stats["treedb.publish.ordered_root_delta_group.prepared_root.entries_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootEntries) + stats["treedb.publish.ordered_root_delta_group.prepared_root.tombstones_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootTombstones) + stats["treedb.publish.ordered_root_delta_group.prepared_root.key_bytes_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootKeyBytes) + stats["treedb.publish.ordered_root_delta_group.prepared_root.value_bytes_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootValueBytes) + stats["treedb.publish.ordered_root_delta_group.prepared_root.pointer_values_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootPointerValues) + stats["treedb.publish.ordered_root_delta_group.prepared_root.installed_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootInstalled) + stats["treedb.publish.ordered_root_delta_group.prepared_root.abandoned_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootAbandoned) + stats["treedb.publish.ordered_root_delta_group.prepared_root.output_pages_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootOutputPages) + stats["treedb.publish.ordered_root_delta_group.prepared_root.output_leaf_log_ptrs_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootOutputLeafLogPtrs) + stats["treedb.publish.ordered_root_delta_group.prepared_root.installed_output_pages_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootInstalledPages) + stats["treedb.publish.ordered_root_delta_group.prepared_root.installed_output_leaf_log_ptrs_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootInstalledLeafLogPtrs) + stats["treedb.publish.ordered_root_delta_group.prepared_root.abandoned_output_pages_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootAbandonedPages) + stats["treedb.publish.ordered_root_delta_group.prepared_root.abandoned_output_leaf_log_ptrs_total"] = fmt.Sprintf("%d", orderedDeltaStats.preparedRootAbandonedLeafLogPtrs) stats["treedb.publish.ordered_root_delta_group.finalize_ns_total"] = fmt.Sprintf("%d", orderedDeltaStats.finalizeNs) stats["treedb.publish.ordered_root_delta_group.finalize_calls_total"] = fmt.Sprintf("%d", orderedDeltaStats.finalizeCalls) stats["treedb.publish.ordered_root_delta_group.latency_p99_ms"] = fmt.Sprintf("%.3f", float64(orderedDeltaStats.latencyP99)/float64(time.Millisecond)) diff --git a/TreeDB/db/batch.go b/TreeDB/db/batch.go index 41dd12e11b..5e43e1f6d8 100644 --- a/TreeDB/db/batch.go +++ b/TreeDB/db/batch.go @@ -1,10 +1,12 @@ package db import ( + "errors" "fmt" "github.com/snissn/gomap/TreeDB/batch" "github.com/snissn/gomap/TreeDB/page" + "github.com/snissn/gomap/TreeDB/zipper" ) // Batch implements the cosmos-db Batch interface. @@ -192,7 +194,7 @@ func (b *Batch) writeOptimistic(sync bool) (bool, error) { tracker := newAllocTracker(idx.allocator) z := idx.zipper.CloneWithAllocator(tracker) - newRoot, retired, metrics, err := z.Apply(rootID, b.batch) + applyResult, err := z.ApplyWithOptions(rootID, b.batch, zipper.ApplyOptions{}) if err != nil { freeErr := tracker.FreeAll() b.db.writeMu.RUnlock() @@ -201,6 +203,9 @@ func (b *Batch) writeOptimistic(sync bool) (bool, error) { } return false, err } + newRoot := applyResult.RootID + pendingRetiredPages := applyResult.PendingRetiredPages + metrics := applyResult.Metrics entries := b.batch.SortedEntries() vlogRefDelta, err := b.db.buildValueLogRefDelta(idx.pager, rootID, baseSeq, entries) if err != nil { @@ -217,21 +222,24 @@ func (b *Batch) writeOptimistic(sync bool) (bool, error) { } }() b.db.commitMu.Lock() - b.db.mu.RLock() - currentRoot := b.db.meta.UserRootPageID - sysRoot := b.db.meta.SystemRootPageID - b.db.mu.RUnlock() - if currentRoot != rootID { + _, guardErr := b.db.runInstallGuard(rawBatchInstallGuard(rootID)) + if guardErr != nil { b.db.commitMu.Unlock() freeErr := tracker.FreeAll() b.db.writeMu.RUnlock() if freeErr != nil { return false, freeErr } - return false, nil + if errors.Is(guardErr, ErrInstallGuardMismatch) { + return false, nil + } + return false, guardErr } + b.db.mu.RLock() + sysRoot := b.db.meta.SystemRootPageID + b.db.mu.RUnlock() - post, err := b.db.finalizeCommitLocked(newRoot, sysRoot, retired, sync, metrics, touchedValueLogSegments, b.db.indexOuterLeavesInValueLog, vlogRefDelta, nil, nil) + post, err := b.db.finalizeCommitLocked(newRoot, sysRoot, pendingRetiredPages, sync, metrics, touchedValueLogSegments, b.db.indexOuterLeavesInValueLog, vlogRefDelta, nil, nil) b.db.commitMu.Unlock() if err != nil { b.db.writeMu.RUnlock() @@ -266,13 +274,24 @@ func (b *Batch) writeSerialized(sync bool) error { defer idx.registry.Unregister(regID) - newRoot, retired, metrics, err := idx.zipper.Apply(rootID, b.batch) + tracker := newAllocTracker(idx.allocator) + z := idx.zipper.CloneWithAllocator(tracker) + applyResult, err := z.ApplyWithOptions(rootID, b.batch, zipper.ApplyOptions{}) if err != nil { + if freeErr := tracker.FreeAll(); freeErr != nil { + return errors.Join(err, freeErr) + } return err } + newRoot := applyResult.RootID + pendingRetiredPages := applyResult.PendingRetiredPages + metrics := applyResult.Metrics entries := b.batch.SortedEntries() vlogRefDelta, err := b.db.buildValueLogRefDelta(idx.pager, rootID, baseSeq, entries) if err != nil { + if freeErr := tracker.FreeAll(); freeErr != nil { + return errors.Join(err, freeErr) + } return err } defer func() { @@ -281,19 +300,21 @@ func (b *Batch) writeSerialized(sync bool) error { } }() - b.db.mu.Lock() - if b.db.meta.UserRootPageID != rootID { - // This should not happen if writeMu is held and we are the only writer. - b.db.mu.Unlock() - return fmt.Errorf("concurrent modification detected during batch write") + if _, err := b.db.runInstallGuard(rawBatchInstallGuard(rootID)); err != nil { + if freeErr := tracker.FreeAll(); freeErr != nil { + return errors.Join(err, freeErr) + } + return err } + b.db.mu.Lock() sysRoot := b.db.meta.SystemRootPageID b.db.mu.Unlock() - if err := b.db.finalizeCommit(newRoot, sysRoot, retired, sync, metrics, touchedValueLogSegments, b.db.indexOuterLeavesInValueLog, vlogRefDelta, nil, nil); err != nil { + if err := b.db.finalizeCommit(newRoot, sysRoot, pendingRetiredPages, sync, metrics, touchedValueLogSegments, b.db.indexOuterLeavesInValueLog, vlogRefDelta, nil, nil); err != nil { return err } vlogRefDelta = nil + b.db.invalidateLeafGenerationSubtreeStats(tracker.Pages()) b.db.clearLeafGenerationReachabilityCaches() if b.db.vacuum.Active() { b.db.vacuum.RecordEntries(entries) diff --git a/TreeDB/db/bench_test.go b/TreeDB/db/bench_test.go index 82a6635b59..9c5cf136ba 100644 --- a/TreeDB/db/bench_test.go +++ b/TreeDB/db/bench_test.go @@ -436,7 +436,7 @@ func BenchmarkLargeVal(b *testing.B) { if closeErr := wb.Close(); err == nil { err = closeErr } - unlock() + unlock.Unlock() if err != nil { b.Errorf("SetPointer failed: %v", err) } diff --git a/TreeDB/db/db.go b/TreeDB/db/db.go index 5a6fc69b24..1de74046af 100644 --- a/TreeDB/db/db.go +++ b/TreeDB/db/db.go @@ -58,6 +58,14 @@ type snapshotView struct { vlogManager *valuelog.Manager } +type orderedRootDeltaGroupReadOnlyPrepareWorkerCounters struct { + targets atomic.Uint64 + ranges atomic.Uint64 + minOps atomic.Uint64 + maxOps atomic.Uint64 + singleSpan atomic.Uint64 +} + type DB struct { valueLogManager *valuelog.Manager snapshotViewRO atomic.Pointer[snapshotView] @@ -198,6 +206,10 @@ type DB struct { orderedRootDeltaGroupRootApplyLeafLogRecordHintBytesRead atomic.Uint64 orderedRootDeltaGroupRootApplyLeafMerges atomic.Uint64 orderedRootDeltaGroupRootApplyInternalMerges atomic.Uint64 + orderedRootDeltaGroupRootApplyInternalParallelMerges atomic.Uint64 + orderedRootDeltaGroupRootApplyInternalParallelChildren atomic.Uint64 + orderedRootDeltaGroupRootApplyInternalParallelWorkers atomic.Uint64 + orderedRootDeltaGroupRootApplyInternalParallelOps atomic.Uint64 orderedRootDeltaGroupRootApplyLeafPagesWritten atomic.Uint64 orderedRootDeltaGroupRootApplyPagerLeafPagesWritten atomic.Uint64 orderedRootDeltaGroupRootApplyLeafLogPagesWritten atomic.Uint64 @@ -212,14 +224,50 @@ type DB struct { orderedRootDeltaGroupRootApplyInternalLeafLogRefs atomic.Uint64 orderedRootDeltaGroupRootApplyInternalLeafLogRefCopies atomic.Uint64 orderedRootDeltaGroupRootApplyRootSplitLevels atomic.Uint64 + orderedRootDeltaGroupRootApplyReadOnlyPrepareNs atomic.Uint64 + orderedRootDeltaGroupRootApplyReadOnlyPrepareCalls atomic.Uint64 + orderedRootDeltaGroupRootApplyReadOnlyPrepareOps atomic.Uint64 + orderedRootDeltaGroupRootApplyReadOnlyPrepareLeafSpans atomic.Uint64 + orderedRootDeltaGroupRootApplyReadOnlyPrepareWorker orderedRootDeltaGroupReadOnlyPrepareWorkerCounters + orderedRootDeltaGroupRootApplyReadOnlyPrepareExactPlans atomic.Uint64 + orderedRootDeltaGroupRootApplyReadOnlyPrepareMaintenance atomic.Uint64 + orderedRootDeltaGroupRootApplyReadOnlyPrepareColdBuilds atomic.Uint64 orderedRootDeltaGroupSystemBuildNs atomic.Uint64 orderedRootDeltaGroupSystemApplyNs atomic.Uint64 orderedRootDeltaGroupSystemApplyCalls atomic.Uint64 orderedRootDeltaGroupSystemApplyOps atomic.Uint64 orderedRootDeltaGroupSystemApplyNodeLoads atomic.Uint64 + orderedRootDeltaGroupInstallGuardNs atomic.Uint64 + orderedRootDeltaGroupInstallGuardCalls atomic.Uint64 + orderedRootDeltaGroupInstallGuardFailures atomic.Uint64 orderedRootDeltaGroupFinalizeNs atomic.Uint64 orderedRootDeltaGroupFinalizeCalls atomic.Uint64 + orderedRootDeltaGroupPreparedRootPrepareNs atomic.Uint64 + orderedRootDeltaGroupPreparedRootGroups atomic.Uint64 + orderedRootDeltaGroupPreparedRootRoots atomic.Uint64 + orderedRootDeltaGroupPreparedRootEntries atomic.Uint64 + orderedRootDeltaGroupPreparedRootTombstones atomic.Uint64 + orderedRootDeltaGroupPreparedRootKeyBytes atomic.Uint64 + orderedRootDeltaGroupPreparedRootValueBytes atomic.Uint64 + orderedRootDeltaGroupPreparedRootPointerValues atomic.Uint64 + orderedRootDeltaGroupPreparedRootInstalled atomic.Uint64 + orderedRootDeltaGroupPreparedRootAbandoned atomic.Uint64 + orderedRootDeltaGroupPreparedRootOutputPages atomic.Uint64 + orderedRootDeltaGroupPreparedRootOutputLeafLogPtrs atomic.Uint64 + orderedRootDeltaGroupPreparedRootInstalledPages atomic.Uint64 + orderedRootDeltaGroupPreparedRootInstalledLeafLogPtrs atomic.Uint64 + orderedRootDeltaGroupPreparedRootAbandonedPages atomic.Uint64 + orderedRootDeltaGroupPreparedRootAbandonedLeafLogPtrs atomic.Uint64 + preparedOutputNextID atomic.Uint64 + + publishInstallGuardNs atomic.Uint64 + publishInstallGuardCalls atomic.Uint64 + publishInstallGuardFailures atomic.Uint64 + publishInstallGuardHookFailures atomic.Uint64 + publishInstallGuardUserRootMismatches atomic.Uint64 + publishInstallGuardSystemRootMismatches atomic.Uint64 + // R4 warm-publish counters. Warm native apply is used for bounded deltas; // larger or ineligible deltas record an explicit rebuild fallback selection. systemRootWarmPublishAttempts atomic.Uint64 @@ -238,6 +286,8 @@ type DB struct { testFailFinalizeCommit atomic.Bool testBatchCreateHook func() testOrderedRootPublishHook func(baseRoot uint64) + testInstallGuardHook func(dbInstallGuardHookEvent) error + testPreparedRootApplyHook func(preparedRootApplyGroup) testSystemRootWarmMaxDeltaOps int // testFailWriteMeta forces writeMeta to fail before mutating the target meta // page so tests can exercise pre-publish cleanup paths. diff --git a/TreeDB/db/install_guard.go b/TreeDB/db/install_guard.go new file mode 100644 index 0000000000..2a78da8bd2 --- /dev/null +++ b/TreeDB/db/install_guard.go @@ -0,0 +1,138 @@ +package db + +import ( + "errors" + "fmt" + "time" +) + +// ErrInstallGuardMismatch marks a guarded install whose captured root state no +// longer matches the DB's current roots. Callers can retry after replanning. +var ErrInstallGuardMismatch = errors.New("treedb: install guard mismatch") + +type dbInstallGuardKind string +type dbInstallGuardFailureCause uint8 + +const ( + dbInstallGuardRawBatch dbInstallGuardKind = "raw_batch" + dbInstallGuardOrderedRootGroup dbInstallGuardKind = "ordered_root_delta_group" +) + +const ( + dbInstallGuardFailureNone dbInstallGuardFailureCause = iota + dbInstallGuardFailureHook dbInstallGuardFailureCause = 1 << (iota - 1) + dbInstallGuardFailureUserRoot + dbInstallGuardFailureSystemRoot +) + +type dbInstallGuard struct { + kind dbInstallGuardKind + userRoot uint64 + systemRoot uint64 + checkUserRoot bool + checkSystemRoot bool +} + +type dbInstallGuardHookEvent struct { + Kind dbInstallGuardKind + UserRoot uint64 + SystemRoot uint64 + CheckUserRoot bool + CheckSystemRoot bool +} + +func rawBatchInstallGuard(userRoot uint64) dbInstallGuard { + return dbInstallGuard{ + kind: dbInstallGuardRawBatch, + userRoot: userRoot, + checkUserRoot: true, + } +} + +func orderedRootDeltaGroupInstallGuard(userRoot, systemRoot uint64) dbInstallGuard { + return dbInstallGuard{ + kind: dbInstallGuardOrderedRootGroup, + userRoot: userRoot, + systemRoot: systemRoot, + checkUserRoot: true, + checkSystemRoot: true, + } +} + +func orderedRootDeltaGroupSystemInstallGuard(systemRoot uint64) dbInstallGuard { + return dbInstallGuard{ + kind: dbInstallGuardOrderedRootGroup, + systemRoot: systemRoot, + checkSystemRoot: true, + } +} + +func (db *DB) runInstallGuard(guard dbInstallGuard) (uint64, error) { + start := time.Now() + var err error + cause := dbInstallGuardFailureNone + if db == nil { + err = ErrClosed + } else if hook := db.testInstallGuardHook; hook != nil { + err = hook(dbInstallGuardHookEvent{ + Kind: guard.kind, + UserRoot: guard.userRoot, + SystemRoot: guard.systemRoot, + CheckUserRoot: guard.checkUserRoot, + CheckSystemRoot: guard.checkSystemRoot, + }) + if err != nil { + cause = dbInstallGuardFailureHook + } + } + if err == nil { + cause, err = db.checkInstallGuard(guard) + } + elapsed := elapsedDurationNs(start) + if db != nil { + db.publishInstallGuardCalls.Add(1) + db.publishInstallGuardNs.Add(elapsed) + if err != nil { + db.publishInstallGuardFailures.Add(1) + if cause&dbInstallGuardFailureHook != 0 { + db.publishInstallGuardHookFailures.Add(1) + } + if cause&dbInstallGuardFailureUserRoot != 0 { + db.publishInstallGuardUserRootMismatches.Add(1) + } + if cause&dbInstallGuardFailureSystemRoot != 0 { + db.publishInstallGuardSystemRootMismatches.Add(1) + } + } + } + return elapsed, err +} + +func (db *DB) checkInstallGuard(guard dbInstallGuard) (dbInstallGuardFailureCause, error) { + if db == nil { + return dbInstallGuardFailureNone, ErrClosed + } + db.mu.RLock() + currentUserRoot := db.meta.UserRootPageID + currentSystemRoot := db.meta.SystemRootPageID + db.mu.RUnlock() + userMismatch := guard.checkUserRoot && currentUserRoot != guard.userRoot + systemMismatch := guard.checkSystemRoot && currentSystemRoot != guard.systemRoot + var cause dbInstallGuardFailureCause + if userMismatch { + cause |= dbInstallGuardFailureUserRoot + } + if systemMismatch { + cause |= dbInstallGuardFailureSystemRoot + } + if userMismatch && systemMismatch { + return cause, fmt.Errorf("%w: user root changed from %d to %d; system root changed from %d to %d", ErrInstallGuardMismatch, guard.userRoot, currentUserRoot, guard.systemRoot, currentSystemRoot) + } + if userMismatch { + return cause, fmt.Errorf("%w: user root changed from %d to %d", ErrInstallGuardMismatch, guard.userRoot, currentUserRoot) + } + if systemMismatch { + return cause, fmt.Errorf("%w: system root changed from %d to %d", ErrInstallGuardMismatch, guard.systemRoot, currentSystemRoot) + } + return dbInstallGuardFailureNone, nil +} diff --git a/TreeDB/db/install_guard_test.go b/TreeDB/db/install_guard_test.go new file mode 100644 index 0000000000..770065505a --- /dev/null +++ b/TreeDB/db/install_guard_test.go @@ -0,0 +1,270 @@ +package db + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "testing" + + "github.com/snissn/gomap/TreeDB/internal/iterator" +) + +func TestRawBatchInstallGuardMismatchFreesTrackedPagesAndSkipsRetire(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + seed := db.NewBatch() + for i := 0; i < 256; i++ { + if err := seed.Set([]byte(fmt.Sprintf("k%04d", i)), bytes.Repeat([]byte("a"), 128)); err != nil { + t.Fatalf("seed set: %v", err) + } + } + if err := seed.Write(); err != nil { + t.Fatalf("seed write: %v", err) + } + _ = seed.Close() + + before := db.Stats() + beforeFree := installGuardStatUint(t, before, "treedb.freelist.free_pages_total") + beforeGraveyardPages := installGuardStatUint(t, before, "treedb.graveyard.pages") + + batchIface := db.NewBatch() + b, ok := batchIface.(*Batch) + if !ok { + t.Fatalf("new batch type %T, want *Batch", batchIface) + } + defer func() { _ = b.Close() }() + for i := 0; i < 256; i++ { + if err := b.Set([]byte(fmt.Sprintf("k%04d", i)), bytes.Repeat([]byte("b"), 128)); err != nil { + t.Fatalf("update set: %v", err) + } + } + hookCalls := 0 + db.testInstallGuardHook = func(ev dbInstallGuardHookEvent) error { + if ev.Kind != dbInstallGuardRawBatch { + return nil + } + hookCalls++ + return ErrInstallGuardMismatch + } + t.Cleanup(func() { db.testInstallGuardHook = nil }) + committed, err := b.writeOptimistic(false) + db.testInstallGuardHook = nil + if err != nil { + t.Fatalf("writeOptimistic err=%v", err) + } + if committed { + t.Fatal("writeOptimistic committed despite injected install guard mismatch") + } + if hookCalls != 1 { + t.Fatalf("install guard hook calls=%d want 1", hookCalls) + } + + after := db.Stats() + if got := installGuardStatUint(t, after, "treedb.publish.install_guard.failures_total"); got != 1 { + t.Fatalf("install guard failures=%d want 1", got) + } + if got := installGuardStatUint(t, after, "treedb.publish.install_guard.hook_failures_total"); got != 1 { + t.Fatalf("install guard hook failures=%d want 1", got) + } + if got := installGuardStatUint(t, after, "treedb.graveyard.pages"); got != beforeGraveyardPages { + t.Fatalf("graveyard pages=%d want unchanged %d", got, beforeGraveyardPages) + } + if got := installGuardStatUint(t, after, "treedb.freelist.free_pages_total"); got <= beforeFree { + t.Fatalf("freelist free pages=%d want > %d after abandoning tracked pages", got, beforeFree) + } + got, err := db.Get([]byte("k0000")) + if err != nil { + t.Fatalf("get k0000: %v", err) + } + if bytes.Equal(got, bytes.Repeat([]byte("b"), 128)) { + t.Fatalf("guard-failed batch became visible") + } +} + +func TestOrderedRootDeltaBatchGroupInstallGuardFailureAbandonsGroup(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + baseRoot, err := db.PublishOrderedRootIterator(0, mustFrozenSystemMemtable(t, + "root/a", "va", + ).NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish base root: %v", err) + } + beforeState := db.State() + if beforeState == nil { + t.Fatal("state before publish is nil") + } + beforeStats := db.Stats() + beforeFree := installGuardStatUint(t, beforeStats, "treedb.freelist.free_pages_total") + + deltaTable := mustFrozenSystemMemtable(t, "root/b", "vb") + iter := deltaTable.NewIterator(nil, nil) + delta, err := OrderedRootDeltaBatchFromIterator(iter) + _ = iter.Close() + if err != nil { + t.Fatalf("OrderedRootDeltaBatchFromIterator: %v", err) + } + defer func() { _ = delta.Close() }() + + hookCalls := 0 + var captured []preparedRootApplyGroup + db.testInstallGuardHook = func(ev dbInstallGuardHookEvent) error { + if ev.Kind != dbInstallGuardOrderedRootGroup { + return nil + } + hookCalls++ + return ErrInstallGuardMismatch + } + db.testPreparedRootApplyHook = func(group preparedRootApplyGroup) { + captured = append(captured, group) + } + _, _, err = db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: baseRoot, + Delta: delta, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return mustFrozenSystemMemtable(t, "sys/collections/users/primary", strconv.FormatUint(rootIDs[0], 10)).NewIterator(nil, nil), nil + }) + db.testInstallGuardHook = nil + db.testPreparedRootApplyHook = nil + if !errors.Is(err, ErrInstallGuardMismatch) { + t.Fatalf("publish err=%v want install guard mismatch", err) + } + if hookCalls != 1 { + t.Fatalf("install guard hook calls=%d want 1", hookCalls) + } + + afterState := db.State() + if afterState.CommitSeq != beforeState.CommitSeq { + t.Fatalf("commit seq changed after failed install guard: got %d want %d", afterState.CommitSeq, beforeState.CommitSeq) + } + if afterState.RootPageID != beforeState.RootPageID { + t.Fatalf("user root changed after failed install guard: got %d want %d", afterState.RootPageID, beforeState.RootPageID) + } + if afterState.SystemRootPageID != beforeState.SystemRootPageID { + t.Fatalf("system root changed after failed install guard: got %d want %d", afterState.SystemRootPageID, beforeState.SystemRootPageID) + } + afterStats := db.Stats() + if got := installGuardStatUint(t, afterStats, "treedb.publish.ordered_root_delta_group.install_guard_failures_total"); got != 1 { + t.Fatalf("ordered install guard failures=%d want 1", got) + } + if got := installGuardStatUint(t, afterStats, "treedb.publish.install_guard.hook_failures_total"); got != 1 { + t.Fatalf("ordered install guard hook failures=%d want 1", got) + } + if got := installGuardStatUint(t, afterStats, "treedb.publish.ordered_root_delta_group.roots_total"); got != 0 { + t.Fatalf("ordered roots total=%d want 0 after failed install guard", got) + } + if got := installGuardStatUint(t, afterStats, "treedb.publish.ordered_root_delta_group.finalize_calls_total"); got != 0 { + t.Fatalf("ordered finalize calls=%d want 0 after failed install guard", got) + } + if got := installGuardStatUint(t, afterStats, "treedb.freelist.free_pages_total"); got <= beforeFree { + t.Fatalf("freelist free pages=%d want > %d after abandoning ordered output", got, beforeFree) + } + if len(captured) != 1 { + t.Fatalf("prepared apply groups=%d want 1", len(captured)) + } + if captured[0].state != preparedRootApplyStateAbandoned { + t.Fatalf("prepared group state=%v want abandoned", captured[0].state) + } + if got := installGuardStatUint(t, afterStats, "treedb.publish.ordered_root_delta_group.prepared_root.abandoned_total"); got != 2 { + t.Fatalf("prepared abandoned=%d want 2", got) + } + if got := installGuardStatUint(t, afterStats, "treedb.publish.ordered_root_delta_group.prepared_root.installed_total"); got != 0 { + t.Fatalf("prepared installed=%d want 0", got) + } + + snap := db.AcquireSnapshot() + if snap == nil { + t.Fatal("snapshot after failed install guard is nil") + } + defer func() { _ = snap.Close() }() + if _, err := snap.GetEntryAtRoot(afterState.SystemRootPageID, []byte("sys/collections/users/primary")); err == nil { + t.Fatal("unexpected descriptor installed after failed install guard") + } +} + +func TestInstallGuardMismatchCauseCounters(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + if _, err := db.runInstallGuard(rawBatchInstallGuard(999)); !errors.Is(err, ErrInstallGuardMismatch) { + t.Fatalf("user-root guard err=%v want install guard mismatch", err) + } + if _, err := db.runInstallGuard(orderedRootDeltaGroupSystemInstallGuard(999)); !errors.Is(err, ErrInstallGuardMismatch) { + t.Fatalf("system-root guard err=%v want install guard mismatch", err) + } + + stats := db.Stats() + if got := installGuardStatUint(t, stats, "treedb.publish.install_guard.user_root_mismatches_total"); got != 1 { + t.Fatalf("user-root mismatch counter=%d want 1", got) + } + if got := installGuardStatUint(t, stats, "treedb.publish.install_guard.system_root_mismatches_total"); got != 1 { + t.Fatalf("system-root mismatch counter=%d want 1", got) + } + if got := installGuardStatUint(t, stats, "treedb.publish.install_guard.hook_failures_total"); got != 0 { + t.Fatalf("hook failure counter=%d want 0", got) + } +} + +func TestInstallGuardCountsDualRootMismatchCauses(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + state := db.State() + if state == nil { + t.Fatal("state is nil") + } + before := db.Stats() + _, err = db.runInstallGuard(orderedRootDeltaGroupInstallGuard( + state.RootPageID+1, + state.SystemRootPageID+1, + )) + if !errors.Is(err, ErrInstallGuardMismatch) { + t.Fatalf("install guard err=%v want mismatch", err) + } + after := db.Stats() + statDelta := func(name string) uint64 { + gotAfter := installGuardStatUint(t, after, name) + gotBefore := installGuardStatUint(t, before, name) + if gotAfter < gotBefore { + t.Fatalf("%s decreased: before=%d after=%d", name, gotBefore, gotAfter) + } + return gotAfter - gotBefore + } + if got := statDelta("treedb.publish.install_guard.failures_total"); got != 1 { + t.Fatalf("install guard failures delta=%d want 1", got) + } + if got := statDelta("treedb.publish.install_guard.user_root_mismatches_total"); got != 1 { + t.Fatalf("user-root mismatch delta=%d want 1", got) + } + if got := statDelta("treedb.publish.install_guard.system_root_mismatches_total"); got != 1 { + t.Fatalf("system-root mismatch delta=%d want 1", got) + } +} + +func installGuardStatUint(tb testing.TB, stats map[string]string, key string) uint64 { + tb.Helper() + raw, ok := stats[key] + if !ok { + tb.Fatalf("missing stat %s", key) + } + got, err := strconv.ParseUint(raw, 10, 64) + if err != nil { + tb.Fatalf("parse stat %s=%q: %v", key, raw, err) + } + return got +} diff --git a/TreeDB/db/leaf_page_log.go b/TreeDB/db/leaf_page_log.go index eeb5a5b98a..da79d9e098 100644 --- a/TreeDB/db/leaf_page_log.go +++ b/TreeDB/db/leaf_page_log.go @@ -43,6 +43,29 @@ type leafPageLogWithRecordLengthHints struct { inner LeafPageLog } +type preparedOutputLeafPageAppender interface { + AppendLeafPage(leafPage []byte) (page.LeafLogPtr, error) +} + +type preparedOutputLeafPageLog struct { + inner preparedOutputLeafPageAppender + tracker preparedLeafLogOutputRecorder +} + +func (l preparedOutputLeafPageLog) AppendLeafPage(leafPage []byte) (page.LeafLogPtr, error) { + if l.inner == nil { + return page.LeafLogPtr{}, errors.New("leaf page log unavailable") + } + ptr, err := l.inner.AppendLeafPage(leafPage) + if err != nil { + return page.LeafLogPtr{}, err + } + if l.tracker != nil { + l.tracker.notePreparedLeafLogPtr(ptr) + } + return ptr, nil +} + func (l *leafPageLogWithRecordLengthHints) AppendLeafPage(leafPage []byte) (page.LeafLogPtr, error) { if l == nil || l.inner == nil { return page.LeafLogPtr{}, errors.New("leaf page log unavailable") diff --git a/TreeDB/db/ordered_root_delta_shape_bench_test.go b/TreeDB/db/ordered_root_delta_shape_bench_test.go new file mode 100644 index 0000000000..689d0e5afb --- /dev/null +++ b/TreeDB/db/ordered_root_delta_shape_bench_test.go @@ -0,0 +1,174 @@ +package db + +import ( + "fmt" + "strconv" + "testing" + "time" + + "github.com/snissn/gomap/TreeDB/batch" + "github.com/snissn/gomap/TreeDB/internal/iterator" + "github.com/snissn/gomap/TreeDB/internal/memtable" +) + +type orderedRootDeltaShape struct { + name string + rawEntries int + finalEntries int +} + +func BenchmarkOrderedRootDeltaShape(b *testing.B) { + for _, docs := range []int{64, 512, 5000} { + shapes := []orderedRootDeltaShape{ + {name: "repeated_same_id_indexed_change_back_raw", rawEntries: docs, finalEntries: docs}, + {name: "repeated_same_id_indexed_change_back_coalesced", rawEntries: docs, finalEntries: 0}, + {name: "repeated_same_id_non_indexed_update_raw", rawEntries: docs, finalEntries: docs}, + {name: "repeated_same_id_non_indexed_update_coalesced", rawEntries: docs, finalEntries: 1}, + {name: "many_ids_indexed_changes_raw", rawEntries: docs, finalEntries: docs}, + {name: "many_ids_indexed_changes_coalesced", rawEntries: docs, finalEntries: docs}, + {name: "many_ids_non_indexed_changes_raw", rawEntries: docs, finalEntries: docs}, + {name: "many_ids_non_indexed_changes_coalesced", rawEntries: docs, finalEntries: docs}, + } + for _, shape := range shapes { + b.Run(fmt.Sprintf("%s/docs_%d", shape.name, docs), func(b *testing.B) { + benchmarkOrderedRootDeltaShape(b, docs, shape) + }) + } + } +} + +func benchmarkOrderedRootDeltaShape(b *testing.B, docs int, shape orderedRootDeltaShape) { + b.Helper() + if docs <= 0 { + b.Fatalf("invalid docs %d", docs) + } + db, err := Open(Options{Dir: b.TempDir()}) + if err != nil { + b.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + root, err := db.PublishOrderedRootIterator(0, orderedRootDeltaShapeBaseTable(b, docs).NewIterator(nil, nil)) + if err != nil { + b.Fatalf("publish base root: %v", err) + } + statsBefore := db.Stats() + totalDocs := uint64(0) + var elapsed time.Duration + + b.ReportAllocs() + b.ResetTimer() + for iter := 0; iter < b.N; iter++ { + b.StopTimer() + totalDocs += uint64(docs) + if shape.finalEntries == 0 { + b.StartTimer() + start := time.Now() + elapsed += time.Since(start) + b.StopTimer() + continue + } + delta := orderedRootDeltaShapeBatch(b, shape.name, shape.finalEntries, iter) + b.StartTimer() + start := time.Now() + _, rootIDs, err := db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: root, + Delta: delta, + }}, orderedRootDeltaShapeSystemBuilder) + elapsed += time.Since(start) + b.StopTimer() + if err != nil { + _ = delta.Close() + b.Fatalf("publish delta shape %s: %v", shape.name, err) + } + _ = delta.Close() + if len(rootIDs) != 1 || rootIDs[0] == 0 { + b.Fatalf("rootIDs=%v want one non-zero root", rootIDs) + } + root = rootIDs[0] + } + statsDelta := orderedRootDeltaShapeStatsDelta(db.Stats(), statsBefore) + if totalDocs > 0 { + docsFloat := float64(totalDocs) + b.ReportMetric(float64(elapsed.Nanoseconds())/docsFloat, "ns/doc") + b.ReportMetric(float64(uint64(shape.rawEntries)*uint64(b.N))/docsFloat, "raw_root_delta_entries/doc") + b.ReportMetric(float64(uint64(shape.finalEntries)*uint64(b.N))/docsFloat, "final_root_delta_entries/doc") + orderedRootDeltaShapeReportUintPerDoc(b, statsDelta["treedb.publish.ordered_root_delta_group.root_apply_ns_total"], totalDocs, "root_apply_ns/doc") + orderedRootDeltaShapeReportUintPerDoc(b, statsDelta["treedb.publish.ordered_root_delta_group.root_apply_calls_total"], totalDocs, "root_apply_calls/doc") + orderedRootDeltaShapeReportUintPerDoc(b, statsDelta["treedb.publish.ordered_root_delta_group.root_apply_ops_total"], totalDocs, "root_apply_ops/doc") + } +} + +func orderedRootDeltaShapeBaseTable(b *testing.B, docs int) memtable.Table { + b.Helper() + table, err := memtable.NewWithCapacityMode(docs, memtable.ModeHashSorted) + if err != nil { + b.Fatalf("new base table: %v", err) + } + for i := 0; i < docs; i++ { + table.Set(orderedRootDeltaShapeKey("base", i), orderedRootDeltaShapeValue(0, i)) + } + table.Freeze() + return table +} + +func orderedRootDeltaShapeBatch(b *testing.B, shape string, entries, iter int) *batch.Batch { + b.Helper() + table, err := memtable.NewWithCapacityMode(entries, memtable.ModeHashSorted) + if err != nil { + b.Fatalf("new delta table: %v", err) + } + for i := 0; i < entries; i++ { + table.Set(orderedRootDeltaShapeKey(shape, i), orderedRootDeltaShapeValue(iter+1, i)) + } + table.Freeze() + it := table.NewIterator(nil, nil) + delta, err := OrderedRootDeltaBatchFromIterator(it) + _ = it.Close() + if err != nil { + b.Fatalf("materialize delta batch: %v", err) + } + return delta +} + +func orderedRootDeltaShapeSystemBuilder(rootIDs []uint64) (iterator.UnsafeIterator, error) { + table, err := memtable.NewWithCapacityMode(1, memtable.ModeHashSorted) + if err != nil { + return nil, err + } + value := strconv.FormatUint(rootIDs[0], 10) + table.Set([]byte("sys/bench/root"), []byte(value)) + table.Freeze() + return table.NewIterator(nil, nil), nil +} + +func orderedRootDeltaShapeKey(shape string, n int) []byte { + return []byte(shape + "/" + strconv.FormatInt(int64(n), 10)) +} + +func orderedRootDeltaShapeValue(iter, n int) []byte { + return []byte("value-" + strconv.FormatInt(int64(iter), 10) + "-" + strconv.FormatInt(int64(n), 10)) +} + +func orderedRootDeltaShapeStatsDelta(after, before map[string]string) map[string]uint64 { + out := make(map[string]uint64, len(after)) + for key, afterValue := range after { + afterUint, err := strconv.ParseUint(afterValue, 10, 64) + if err != nil { + continue + } + beforeUint, _ := strconv.ParseUint(before[key], 10, 64) + if afterUint >= beforeUint { + out[key] = afterUint - beforeUint + } + } + return out +} + +func orderedRootDeltaShapeReportUintPerDoc(b *testing.B, value uint64, docs uint64, name string) { + b.Helper() + if value == 0 || docs == 0 { + return + } + b.ReportMetric(float64(value)/float64(docs), name) +} diff --git a/TreeDB/db/ordered_root_publish.go b/TreeDB/db/ordered_root_publish.go index ff3f6e793a..bd73c7fa8b 100644 --- a/TreeDB/db/ordered_root_publish.go +++ b/TreeDB/db/ordered_root_publish.go @@ -5,6 +5,7 @@ import ( "errors" "sort" "sync" + "sync/atomic" "time" "github.com/snissn/gomap/TreeDB/batch" @@ -40,6 +41,28 @@ const orderedRootOptimisticSystemDeltaRebaseMaxAttempts = 4 const orderedRootDeltaBatchGroupParallelApplyMinRoots = 2 +var orderedRootReadOnlyPrepareResultPool = sync.Pool{ + New: func() any { + return new(zipper.ReadOnlyPrepareResult) + }, +} + +func acquireOrderedRootReadOnlyPrepareResult() *zipper.ReadOnlyPrepareResult { + result, _ := orderedRootReadOnlyPrepareResultPool.Get().(*zipper.ReadOnlyPrepareResult) + if result == nil { + return new(zipper.ReadOnlyPrepareResult) + } + return result +} + +func releaseOrderedRootReadOnlyPrepareResult(result *zipper.ReadOnlyPrepareResult) { + if result == nil { + return + } + result.ResetForReuse() + orderedRootReadOnlyPrepareResultPool.Put(result) +} + type orderedRootPublishStats struct { warmAttempts uint64 warmNativeApplyAttempts uint64 @@ -51,21 +74,72 @@ type orderedRootPublishStats struct { } type orderedRootPublishOptions struct { - maxWarmDeltaOps int - leafPrefixCompression bool - leafColumnar bool - packedValuePtr bool - internalBaseDelta bool - outerLeavesInValueLog bool - leafPageLog bulk.LeafPageAppender + maxWarmDeltaOps int + leafPrefixCompression bool + leafColumnar bool + packedValuePtr bool + internalBaseDelta bool + outerLeavesInValueLog bool + leafPageLog bulk.LeafPageAppender + applyOptions zipper.ApplyOptions + readOnlyPrepareSummary *zipper.ReadOnlyLeafSpanSummary + readOnlyPrepareWorkerSummary *zipper.ReadOnlyLeafSpanWorkerRangeSummary + readOnlyPrepareCallerResult *zipper.ReadOnlyPrepareResult + readOnlyPrepareNs *uint64 + readOnlyPrepareAttempted *bool + readOnlyPrepareWorkerCount int } type orderedRootDeltaBatchGroupApplyResult struct { - idx int - rootID uint64 - retired []uint64 - metrics adaptive.Metrics - err error + idx int + rootID uint64 + outputID preparedOutputID + output *preparedOutputSnapshot + outputPages uint64 + outputLeafLogPtrs uint64 + pendingRetiredPages []uint64 + metrics adaptive.Metrics + readOnlyPrepareSummary zipper.ReadOnlyLeafSpanSummary + readOnlyPrepareWorkerSummary zipper.ReadOnlyLeafSpanWorkerRangeSummary + readOnlyPrepareNs uint64 + readOnlyPrepareAttempted bool + err error + attempted bool +} + +type preparedLeafLogOutputRecorder interface { + notePreparedLeafLogPtr(page.LeafLogPtr) +} + +type preparedRootApplyOutputCounter struct { + inner zipper.PageAllocator + recorder preparedLeafLogOutputRecorder + + pages atomic.Uint64 + leafLogPtrs atomic.Uint64 +} + +func (c *preparedRootApplyOutputCounter) Alloc(hint uint64) (uint64, error) { + id, err := c.inner.Alloc(hint) + if err != nil { + return 0, err + } + c.pages.Add(1) + return id, nil +} + +func (c *preparedRootApplyOutputCounter) notePreparedLeafLogPtr(ptr page.LeafLogPtr) { + if c.recorder != nil { + c.recorder.notePreparedLeafLogPtr(ptr) + } + c.leafLogPtrs.Add(1) +} + +func (c *preparedRootApplyOutputCounter) counts() (pages, leafLogPtrs uint64) { + if c == nil { + return 0, 0 + } + return c.pages.Load(), c.leafLogPtrs.Load() } // OrderedRootStoragePolicy selects the physical storage policy for a published @@ -116,6 +190,19 @@ type OrderedRootDeltaBatchPublishInput struct { // Callers should opt in only when root deltas are already materialized and // benchmarked as large enough to amortize goroutine and shared backend costs. ParallelApply bool + // PrepareReadOnly runs the read-only leaf-span preparation pass before warm + // root apply and records planning stats. It is observability/planning only; + // it does not change publish output or enable parallel leaf execution. + PrepareReadOnly bool + // ReadOnlyPrepareResult, when non-nil, is both the reuse source and output + // destination for this root's optional preparation metadata. It must be + // owned by this input within the group; sharing one result pointer across + // group inputs is rejected. It is ignored unless PrepareReadOnly is true. + ReadOnlyPrepareResult *zipper.ReadOnlyPrepareResult + // ReadOnlyPrepareWorkerCount, when positive with PrepareReadOnly, records an + // allocation-free summary of deterministic leaf-span worker ranges for this + // target worker count. It is observability/planning only. + ReadOnlyPrepareWorkerCount int } func closeUnconsumedOrderedRootPublishIterators(ordered []OrderedRootPublishInput, consumed []bool) { @@ -635,8 +722,35 @@ func (db *DB) publishOrderedRootDeltaIterator(baseRoot uint64, iter iterator.Uns if err != nil { return 0, nil, metrics, err } - newRoot, retired, metrics, err = rootZipper.Apply(baseRoot, delta) - return + applyOptions := opts.applyOptions + var pooledResult *zipper.ReadOnlyPrepareResult + if applyOptions.PrepareReadOnly && opts.readOnlyPrepareCallerResult == nil { + pooledResult = acquireOrderedRootReadOnlyPrepareResult() + applyOptions.ReadOnlyPrepare = pooledResult.ReuseOptions() + } + if opts.applyOptions.PrepareReadOnly && opts.readOnlyPrepareAttempted != nil { + *opts.readOnlyPrepareAttempted = true + } + newRoot, retired, metrics, readOnlyPrepare, readOnlyPrepareNs, err := applyOrderedRootDeltaWithOptions(rootZipper, baseRoot, delta, applyOptions) + if opts.applyOptions.PrepareReadOnly && opts.readOnlyPrepareSummary != nil { + summary := readOnlyPrepare.LeafSpanSummary() + *opts.readOnlyPrepareSummary = summary + } + if opts.applyOptions.PrepareReadOnly && opts.readOnlyPrepareWorkerSummary != nil { + summary := readOnlyPrepare.LeafSpanWorkerRangeSummary(opts.readOnlyPrepareWorkerCount) + *opts.readOnlyPrepareWorkerSummary = summary + } + if opts.readOnlyPrepareCallerResult != nil { + *opts.readOnlyPrepareCallerResult = readOnlyPrepare + } + if opts.readOnlyPrepareNs != nil { + *opts.readOnlyPrepareNs = readOnlyPrepareNs + } + if pooledResult != nil { + *pooledResult = readOnlyPrepare + releaseOrderedRootReadOnlyPrepareResult(pooledResult) + } + return newRoot, retired, metrics, err } func (db *DB) publishOrderedRootDeltaBatch(baseRoot uint64, delta *batch.Batch, opts orderedRootPublishOptions) (newRoot uint64, retired []uint64, metrics adaptive.Metrics, err error) { @@ -673,10 +787,25 @@ func (db *DB) publishOrderedRootDeltaBatchWithAllocator(idx *indexGen, baseRoot err = errors.New("ordered root value-log leaf storage requires a leaf page log") return } + if opts.outerLeavesInValueLog { + if tracker := preparedOutputTrackerFromAlloc(alloc, coldBuildAlloc); tracker != nil { + opts.leafPageLog = preparedOutputLeafPageLog{inner: opts.leafPageLog, tracker: tracker} + } + } if delta.IsEmpty() { + if opts.applyOptions.PrepareReadOnly { + if err = db.runOrderedRootReadOnlyPrepare(idx, baseRoot, delta, opts, alloc); err != nil { + return 0, nil, metrics, err + } + } return baseRoot, nil, metrics, nil } if baseRoot == 0 { + if opts.applyOptions.PrepareReadOnly { + if err = db.runOrderedRootReadOnlyPrepare(idx, baseRoot, delta, opts, alloc); err != nil { + return 0, nil, metrics, err + } + } iter := newOrderedRootDeltaBatchIterator(delta, includeDeletedOnColdBuild) defer func() { _ = iter.Close() }() if coldBuildAlloc == nil { @@ -700,8 +829,77 @@ func (db *DB) publishOrderedRootDeltaBatchWithAllocator(idx *indexGen, baseRoot if err != nil { return 0, nil, metrics, err } - newRoot, retired, metrics, err = rootZipper.Apply(baseRoot, delta) - return + if opts.applyOptions.PrepareReadOnly { + if err = runOrderedRootReadOnlyPrepare(rootZipper, baseRoot, delta, opts); err != nil { + return 0, nil, metrics, err + } + opts.applyOptions.PrepareReadOnly = false + } + newRoot, retired, metrics, _, _, err = applyOrderedRootDeltaWithOptions(rootZipper, baseRoot, delta, opts.applyOptions) + return newRoot, retired, metrics, err +} + +func (db *DB) runOrderedRootReadOnlyPrepare(idx *indexGen, baseRoot uint64, delta *batch.Batch, opts orderedRootPublishOptions, alloc zipper.PageAllocator) error { + rootZipper, err := db.orderedRootZipperForOptionsWithAllocator(idx, opts, alloc) + if err != nil { + return err + } + return runOrderedRootReadOnlyPrepare(rootZipper, baseRoot, delta, opts) +} + +func runOrderedRootReadOnlyPrepare(rootZipper *zipper.Zipper, baseRoot uint64, delta *batch.Batch, opts orderedRootPublishOptions) error { + if opts.readOnlyPrepareAttempted != nil { + *opts.readOnlyPrepareAttempted = true + } + prepareOptions := opts.applyOptions.ReadOnlyPrepare + var pooledResult *zipper.ReadOnlyPrepareResult + if opts.readOnlyPrepareCallerResult == nil { + pooledResult = acquireOrderedRootReadOnlyPrepareResult() + prepareOptions = pooledResult.ReuseOptions() + } + prepareStart := time.Now() + prepared, err := rootZipper.PrepareReadOnly(baseRoot, delta, prepareOptions) + prepareNs := elapsedDurationNs(prepareStart) + if pooledResult != nil { + *pooledResult = prepared + defer releaseOrderedRootReadOnlyPrepareResult(pooledResult) + } + if opts.readOnlyPrepareSummary != nil { + summary := prepared.LeafSpanSummary() + *opts.readOnlyPrepareSummary = summary + } + if opts.readOnlyPrepareWorkerSummary != nil { + summary := prepared.LeafSpanWorkerRangeSummary(opts.readOnlyPrepareWorkerCount) + *opts.readOnlyPrepareWorkerSummary = summary + } + if opts.readOnlyPrepareCallerResult != nil { + *opts.readOnlyPrepareCallerResult = prepared + } + if opts.readOnlyPrepareNs != nil { + *opts.readOnlyPrepareNs = prepareNs + } + return err +} + +func preparedOutputTrackerFromAlloc(alloc zipper.PageAllocator, coldBuildAlloc bulk.Allocator) preparedLeafLogOutputRecorder { + if tracker, ok := alloc.(preparedLeafLogOutputRecorder); ok && tracker != nil { + return tracker + } + if tracker, ok := coldBuildAlloc.(preparedLeafLogOutputRecorder); ok && tracker != nil { + return tracker + } + return nil +} + +func applyOrderedRootDeltaWithOptions(rootZipper *zipper.Zipper, baseRoot uint64, delta *batch.Batch, opts zipper.ApplyOptions) (uint64, []uint64, adaptive.Metrics, zipper.ReadOnlyPrepareResult, uint64, error) { + applyResult, err := rootZipper.ApplyWithOptions(baseRoot, delta, opts) + // ApplyWithOptions returns its result by value and may include partial + // metrics when err is non-nil; preserve metrics but do not return partial + // root IDs or retired-page ownership on failure. + if err != nil { + return 0, nil, applyResult.Metrics, applyResult.ReadOnlyPrepare, applyResult.ReadOnlyPrepareNs, err + } + return applyResult.RootID, applyResult.PendingRetiredPages, applyResult.Metrics, applyResult.ReadOnlyPrepare, applyResult.ReadOnlyPrepareNs, nil } func buildOrderedRootDeltaBatch(baseIter, targetIter iterator.UnsafeIterator, trackRefs bool) (*batch.Batch, int, *valueLogRefDelta, error) { @@ -904,7 +1102,7 @@ func (db *DB) publishOrderedRootIterator(baseRoot uint64, iter iterator.UnsafeIt err = zipperErr return } - newRoot, retired, metrics, err = rootZipper.Apply(baseRoot, delta) + newRoot, retired, metrics, _, _, err = applyOrderedRootDeltaWithOptions(rootZipper, baseRoot, delta, zipper.ApplyOptions{}) if err != nil { return } @@ -1003,6 +1201,10 @@ func mergeOrderedRootPublishMetrics(dst *adaptive.Metrics, src adaptive.Metrics) dst.ZipperLeafLogRecordHintBytesRead += src.ZipperLeafLogRecordHintBytesRead dst.ZipperLeafMerges += src.ZipperLeafMerges dst.ZipperInternalMerges += src.ZipperInternalMerges + dst.ZipperInternalParallelMerges += src.ZipperInternalParallelMerges + dst.ZipperInternalParallelChildren += src.ZipperInternalParallelChildren + dst.ZipperInternalParallelWorkers += src.ZipperInternalParallelWorkers + dst.ZipperInternalParallelOps += src.ZipperInternalParallelOps dst.ZipperLeafPagesWritten += src.ZipperLeafPagesWritten dst.ZipperPagerLeafPagesWritten += src.ZipperPagerLeafPagesWritten dst.ZipperLeafLogPagesWritten += src.ZipperLeafLogPagesWritten @@ -1418,30 +1620,130 @@ func orderedRootDeltaBatchGroupParallelApplyEligible(ordered []OrderedRootDeltaB return parallelActive >= orderedRootDeltaBatchGroupParallelApplyMinRoots } -func (db *DB) applyOrderedRootDeltaBatchGroupRoots(idx *indexGen, ordered []OrderedRootDeltaBatchPublishInput, alloc zipper.PageAllocator, coldBuildAlloc bulk.Allocator) ([]orderedRootDeltaBatchGroupApplyResult, bool) { +func validateOrderedRootReadOnlyPrepareResultOwnership(ordered []OrderedRootDeltaBatchPublishInput) error { + // Keep this validation allocation-free. Ordered root groups are expected to + // be small, and the read-only prepare reuse path is allocation-sensitive. + for idx := range ordered { + result := ordered[idx].ReadOnlyPrepareResult + if !ordered[idx].PrepareReadOnly || result == nil { + continue + } + for otherIdx := idx + 1; otherIdx < len(ordered); otherIdx++ { + if ordered[otherIdx].PrepareReadOnly && ordered[otherIdx].ReadOnlyPrepareResult == result { + return errors.New("ordered root read-only prepare result reused by multiple inputs") + } + } + } + return nil +} + +func (db *DB) applyOrderedRootDeltaBatchGroupRoots(idx *indexGen, ordered []OrderedRootDeltaBatchPublishInput, alloc zipper.PageAllocator, coldBuildAlloc bulk.Allocator, includeOutputSnapshot bool) ([]orderedRootDeltaBatchGroupApplyResult, bool) { results := make([]orderedRootDeltaBatchGroupApplyResult, len(ordered)) - applyOne := func(orderedIdx int) orderedRootDeltaBatchGroupApplyResult { - result := orderedRootDeltaBatchGroupApplyResult{idx: orderedIdx} + if err := validateOrderedRootReadOnlyPrepareResultOwnership(ordered); err != nil { + if len(results) > 0 { + // recordOrderedRootDeltaBatchGroupApplyResults uses attempted to find + // terminal per-input errors. No root apply metrics are recorded for + // errored results. + results[0] = orderedRootDeltaBatchGroupApplyResult{idx: 0, err: err, attempted: true} + } + return results, false + } + var outputID preparedOutputID + var outputTracker *allocTracker + if tracker, ok := alloc.(*allocTracker); ok { + outputTracker = tracker + outputID = tracker.PreparedOutputID() + } + captureOutputSnapshot := func() { + if !includeOutputSnapshot || outputTracker == nil { + return + } + output := outputTracker.PreparedOutputSnapshot() + for resultIdx := range results { + if !results[resultIdx].attempted || results[resultIdx].err != nil { + continue + } + results[resultIdx].output = &output + results[resultIdx].outputID = output.ID + } + } + applyOne := func(orderedIdx int, isolateOutput bool) orderedRootDeltaBatchGroupApplyResult { + result := orderedRootDeltaBatchGroupApplyResult{ + idx: orderedIdx, + outputID: outputID, + attempted: true, + } opts, err := db.orderedRootPublishOptionsForPolicy(ordered[orderedIdx].StoragePolicy) if err != nil { result.err = err return result } - rootID, retired, metrics, err := db.publishOrderedRootDeltaBatchWithAllocator(idx, ordered[orderedIdx].BaseRoot, ordered[orderedIdx].Delta, opts, alloc, coldBuildAlloc, ordered[orderedIdx].IncludeDeletedOnColdBuild) + if ordered[orderedIdx].PrepareReadOnly { + opts.applyOptions.PrepareReadOnly = true + if resultOut := ordered[orderedIdx].ReadOnlyPrepareResult; resultOut != nil { + opts.applyOptions.ReadOnlyPrepare = resultOut.ReuseOptions() + } + opts.readOnlyPrepareSummary = &result.readOnlyPrepareSummary + if ordered[orderedIdx].ReadOnlyPrepareWorkerCount > 0 { + opts.readOnlyPrepareWorkerSummary = &result.readOnlyPrepareWorkerSummary + opts.readOnlyPrepareWorkerCount = ordered[orderedIdx].ReadOnlyPrepareWorkerCount + } + opts.readOnlyPrepareCallerResult = ordered[orderedIdx].ReadOnlyPrepareResult + opts.readOnlyPrepareNs = &result.readOnlyPrepareNs + opts.readOnlyPrepareAttempted = &result.readOnlyPrepareAttempted + } + beforePages, beforeLeafLogPtrs := uint64(0), uint64(0) + if outputTracker != nil { + beforePages, beforeLeafLogPtrs = outputTracker.PreparedOutputCounts() + } + rootAlloc := alloc + rootColdBuildAlloc := coldBuildAlloc + var counters []*preparedRootApplyOutputCounter + if isolateOutput && outputTracker != nil { + counter := &preparedRootApplyOutputCounter{inner: alloc, recorder: outputTracker} + rootAlloc = counter + counters = append(counters, counter) + if coldBuildAlloc == nil || coldBuildAlloc == alloc { + rootColdBuildAlloc = counter + } else { + coldCounter := &preparedRootApplyOutputCounter{inner: coldBuildAlloc, recorder: outputTracker} + rootColdBuildAlloc = coldCounter + counters = append(counters, coldCounter) + } + } + rootID, pendingRetiredPages, metrics, err := db.publishOrderedRootDeltaBatchWithAllocator(idx, ordered[orderedIdx].BaseRoot, ordered[orderedIdx].Delta, opts, rootAlloc, rootColdBuildAlloc, ordered[orderedIdx].IncludeDeletedOnColdBuild) result.rootID = rootID - result.retired = retired + result.pendingRetiredPages = pendingRetiredPages result.metrics = metrics result.err = err + if err == nil { + if len(counters) == 0 { + afterPages, afterLeafLogPtrs := uint64(0), uint64(0) + if outputTracker != nil { + afterPages, afterLeafLogPtrs = outputTracker.PreparedOutputCounts() + } + result.outputPages = afterPages - beforePages + result.outputLeafLogPtrs = afterLeafLogPtrs - beforeLeafLogPtrs + } else { + for _, counter := range counters { + pages, leafLogPtrs := counter.counts() + result.outputPages += pages + result.outputLeafLogPtrs += leafLogPtrs + } + } + } return result } if !orderedRootDeltaBatchGroupParallelApplyEligible(ordered) { for orderedIdx := range ordered { - results[orderedIdx] = applyOne(orderedIdx) + results[orderedIdx] = applyOne(orderedIdx, false) if results[orderedIdx].err != nil { + captureOutputSnapshot() return results, false } } + captureOutputSnapshot() return results, false } @@ -1460,25 +1762,110 @@ func (db *DB) applyOrderedRootDeltaBatchGroupRoots(idx *indexGen, ordered []Orde wg.Add(1) go func(orderedIdx int) { defer wg.Done() - results[orderedIdx] = applyOne(orderedIdx) + results[orderedIdx] = applyOne(orderedIdx, true) }(orderedIdx) } wg.Wait() for orderedIdx := range ordered { if ordered[orderedIdx].ParallelApply && ordered[orderedIdx].Delta != nil && !ordered[orderedIdx].Delta.IsEmpty() { if results[orderedIdx].err != nil { + captureOutputSnapshot() return results, false } continue } - results[orderedIdx] = applyOne(orderedIdx) + results[orderedIdx] = applyOne(orderedIdx, false) if results[orderedIdx].err != nil { + captureOutputSnapshot() return results, false } } + captureOutputSnapshot() return results, parallelRoots >= orderedRootDeltaBatchGroupParallelApplyMinRoots } +func recordOrderedRootDeltaBatchGroupApplyResults( + preparedGroup *preparedRootApplyGroup, + rootIDs []uint64, + results []orderedRootDeltaBatchGroupApplyResult, + pendingRetiredPages *[]uint64, + mergedMetrics *adaptive.Metrics, + phaseStats *orderedRootDeltaGroupPublishPhaseStats, + rootsObserved *int, +) error { + var firstErr error + for orderedIdx := range results { + result := results[orderedIdx] + if !result.attempted { + continue + } + if result.err != nil { + if firstErr == nil { + firstErr = result.err + } + continue + } + if orderedIdx < len(rootIDs) { + rootIDs[orderedIdx] = result.rootID + } + if preparedGroup != nil { + if result.output != nil { + preparedGroup.markPreparedOutput(orderedIdx, result.rootID, *result.output) + } else { + preparedGroup.markPrepared(orderedIdx, result.rootID, result.outputID) + } + } + if rootsObserved != nil { + (*rootsObserved)++ + } + if pendingRetiredPages != nil { + *pendingRetiredPages = append(*pendingRetiredPages, result.pendingRetiredPages...) + } + if mergedMetrics != nil { + mergeOrderedRootPublishMetrics(mergedMetrics, result.metrics) + } + if phaseStats != nil { + phaseStats.rootApplyMetrics.add(result.metrics) + phaseStats.rootApplyCalls++ + if result.readOnlyPrepareAttempted { + summary := result.readOnlyPrepareSummary + phaseStats.rootApplyReadOnlyPrepareNs += result.readOnlyPrepareNs + phaseStats.rootApplyReadOnlyPrepareCalls++ + phaseStats.rootApplyReadOnlyPrepareOps += uint64(summary.Ops) + phaseStats.rootApplyReadOnlyPrepareLeafSpans += uint64(summary.Spans) + workerSummary := result.readOnlyPrepareWorkerSummary + phaseStats.rootApplyReadOnlyPrepareWorker.targets += uint64(workerSummary.TargetWorkers) + phaseStats.rootApplyReadOnlyPrepareWorker.ranges += uint64(workerSummary.Ranges) + phaseStats.rootApplyReadOnlyPrepareWorker.minOps += uint64(workerSummary.MinRangeOps) + phaseStats.rootApplyReadOnlyPrepareWorker.maxOps += uint64(workerSummary.MaxRangeOps) + phaseStats.rootApplyReadOnlyPrepareWorker.singleSpan += uint64(workerSummary.SingleSpanRanges) + if summary.ExactLeafSpans { + phaseStats.rootApplyReadOnlyPrepareExactPlans++ + } + if summary.Maintenance { + phaseStats.rootApplyReadOnlyPrepareMaintenance++ + } + if summary.ColdBuild { + phaseStats.rootApplyReadOnlyPrepareColdBuilds++ + } + } + } + } + return firstErr +} + +func orderedRootDeltaBatchGroupPreparedOutputCounts(results []orderedRootDeltaBatchGroupApplyResult) (pages, leafLogPtrs uint64) { + for idx := range results { + result := results[idx] + if !result.attempted || result.err != nil { + continue + } + pages += result.outputPages + leafLogPtrs += result.outputLeafLogPtrs + } + return pages, leafLogPtrs +} + func (db *DB) tryPublishOrderedRootDeltaBatchGroupOptimistic(ordered []OrderedRootDeltaBatchPublishInput, buildSystemDeltaIter OrderedRootGroupSystemBuilder) (newSystemRoot uint64, rootIDs []uint64, retrySerialized bool, err error) { if buildSystemDeltaIter == nil { return 0, nil, false, errors.New("nil ordered root group system delta builder") @@ -1524,7 +1911,47 @@ func (db *DB) tryPublishOrderedRootDeltaBatchGroupOptimistic(ordered []OrderedRo return 0, nil, retrySerialized, nil } - rootTracker := newAllocTracker(idx.allocator) + phaseStart := time.Now() + var preparedGroup preparedRootApplyGroup + includePreparedChecksum := db.testPreparedRootApplyHook != nil + initPreparedRootApplyGroup(&preparedGroup, baseUserRoot, baseSystemRoot, ordered, includePreparedChecksum) + phaseStats.preparedRootPrepareNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) + preparedGroupObserved := false + observePreparedGroup := func(state preparedRootApplyState) { + if preparedGroupObserved { + return + } + preparedGroupObserved = true + observePreparedRootApplyGroup(db, &phaseStats, &preparedGroup, state) + } + publishObserved := false + observePublish := func(wait, hold time.Duration, publishErr error) { + if publishObserved { + return + } + publishObserved = true + db.observeOrderedRootDeltaGroupPublish(wait, hold, rootsObserved, phaseStats, publishErr) + } + defer func() { + if err == nil && !retrySerialized { + return + } + // Optimistic attempts can prepare data/system roots before failing or + // falling back to the serialized path. Record those roots as abandoned + // before allocator cleanup discards the prepared output. + if !preparedGroupObserved { + observePreparedGroup(preparedRootApplyStateAbandoned) + } + if err != nil && !publishObserved { + db.observeOrderedRootDeltaGroupPreparedRootApply(phaseStats.preparedRootPrepareNs, phaseStats.preparedRootStats) + return + } + if retrySerialized && !publishObserved { + db.observeOrderedRootDeltaGroupPreparedRootApply(phaseStats.preparedRootPrepareNs, phaseStats.preparedRootStats) + } + }() + + rootTracker := db.newPreparedOutputAllocTracker(idx.allocator) var systemTracker *allocTracker commitStarted := false freeTrackedPages := func() { @@ -1545,10 +1972,10 @@ func (db *DB) tryPublishOrderedRootDeltaBatchGroupOptimistic(ordered []OrderedRo rootIDs = make([]uint64, len(ordered)) systemOpts := systemRootOrderedPublishOptions(db) - var nonSystemRetired []uint64 + var nonSystemPendingRetiredPages []uint64 var nonSystemMetrics adaptive.Metrics - phaseStart := time.Now() - rootApplyResults, parallelRootApply := db.applyOrderedRootDeltaBatchGroupRoots(idx, ordered, rootTracker, rootTracker) + phaseStart = time.Now() + rootApplyResults, parallelRootApply := db.applyOrderedRootDeltaBatchGroupRoots(idx, ordered, rootTracker, rootTracker, includePreparedChecksum) phaseStats.rootApplyNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) if parallelRootApply { phaseStats.rootApplyParallelGroups++ @@ -1558,24 +1985,17 @@ func (db *DB) tryPublishOrderedRootDeltaBatchGroupOptimistic(ordered []OrderedRo } } } - for orderedIdx := range rootApplyResults { - result := rootApplyResults[orderedIdx] - if result.err != nil { - return 0, nil, false, result.err - } - rootIDs[orderedIdx] = result.rootID - rootsObserved++ - nonSystemRetired = append(nonSystemRetired, result.retired...) - mergeOrderedRootPublishMetrics(&nonSystemMetrics, result.metrics) - phaseStats.rootApplyMetrics.add(result.metrics) - phaseStats.rootApplyCalls++ + outputPages, outputLeafLogPtrs := orderedRootDeltaBatchGroupPreparedOutputCounts(rootApplyResults) + preparedGroup.noteSharedOutputCounts(outputPages, outputLeafLogPtrs) + if applyErr := recordOrderedRootDeltaBatchGroupApplyResults(&preparedGroup, rootIDs, rootApplyResults, &nonSystemPendingRetiredPages, &nonSystemMetrics, &phaseStats, &rootsObserved); applyErr != nil { + return 0, nil, false, applyErr } systemBaseRoot := baseSystemRoot var committedRootPages []uint64 var committedSystemPages []uint64 for attempt := 0; ; attempt++ { - systemTracker = newAllocTracker(idx.allocator) + systemTracker = db.newPreparedOutputAllocTracker(idx.allocator) phaseStart := time.Now() iter, err := buildSystemDeltaIter(append([]uint64(nil), rootIDs...)) phaseStats.systemBuildNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) @@ -1591,7 +2011,10 @@ func (db *DB) tryPublishOrderedRootDeltaBatchGroupOptimistic(ordered []OrderedRo return 0, nil, false, err } phaseStart = time.Now() - rootID, systemRetired, systemMetrics, applyErr := db.publishOrderedRootDeltaBatchWithAllocator(idx, systemBaseRoot, systemDelta, systemOpts, systemTracker, systemTracker, false) + systemPreparedIdx := preparedGroup.setSystemRoot(systemBaseRoot, systemDelta, includePreparedChecksum) + phaseStats.preparedRootPrepareNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) + phaseStart = time.Now() + rootID, systemPendingRetiredPages, systemMetrics, applyErr := db.publishOrderedRootDeltaBatchWithAllocator(idx, systemBaseRoot, systemDelta, systemOpts, systemTracker, systemTracker, systemBaseRoot == 0) phaseStats.systemApplyNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) phaseStats.systemApplyCalls++ _ = systemDelta.Close() @@ -1599,6 +2022,12 @@ func (db *DB) tryPublishOrderedRootDeltaBatchGroupOptimistic(ordered []OrderedRo err = applyErr return 0, nil, false, err } + if includePreparedChecksum { + preparedGroup.markPreparedOutput(systemPreparedIdx, rootID, systemTracker.PreparedOutputSnapshot()) + } else { + outputPages, outputLeafLogPtrs := systemTracker.PreparedOutputCounts() + preparedGroup.markPreparedOutputCounts(systemPreparedIdx, rootID, systemTracker.PreparedOutputID(), outputPages, outputLeafLogPtrs) + } phaseStats.systemApplyMetrics.add(systemMetrics) lockStart := time.Now() @@ -1627,8 +2056,8 @@ func (db *DB) tryPublishOrderedRootDeltaBatchGroupOptimistic(ordered []OrderedRo curUserRoot = baseUserRoot } - retired := append([]uint64(nil), nonSystemRetired...) - retired = append(retired, systemRetired...) + pendingRetiredPages := append([]uint64(nil), nonSystemPendingRetiredPages...) + pendingRetiredPages = append(pendingRetiredPages, systemPendingRetiredPages...) merged := nonSystemMetrics mergeOrderedRootPublishMetrics(&merged, systemMetrics) newSystemRoot = rootID @@ -1638,10 +2067,23 @@ func (db *DB) tryPublishOrderedRootDeltaBatchGroupOptimistic(ordered []OrderedRo // delta can change collection descriptors. Keep the incremental ref tracker // conservative by invalidating it after commit. var vlogRefDelta *valueLogRefDelta + preparedGroup.markInstalling() + guardNs, guardErr := db.runInstallGuard(orderedRootDeltaGroupSystemInstallGuard(systemBaseRoot)) + phaseStats.installGuardNs += guardNs + phaseStats.installGuardCalls++ + if guardErr != nil { + phaseStats.installGuardFailures++ + hold := time.Since(holdStart) + db.commitMu.Unlock() + observePreparedGroup(preparedRootApplyStateAbandoned) + observePublish(wait, hold, guardErr) + err = guardErr + return 0, nil, false, err + } phaseStart = time.Now() var post finalizeCommitPost commitStarted = true - post, err = db.finalizeCommitLocked(curUserRoot, newSystemRoot, retired, false, merged, nil, true, vlogRefDelta, nil, nil) + post, err = db.finalizeCommitLocked(curUserRoot, newSystemRoot, pendingRetiredPages, false, merged, nil, true, vlogRefDelta, nil, nil) phaseStats.finalizeNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) phaseStats.finalizeCalls++ hold := time.Since(holdStart) @@ -1649,12 +2091,17 @@ func (db *DB) tryPublishOrderedRootDeltaBatchGroupOptimistic(ordered []OrderedRo committedSystemPages = systemTracker.Pages() db.commitMu.Unlock() if err != nil { + observePreparedGroup(preparedRootApplyStateAbandoned) + observePublish(wait, hold, err) return 0, nil, false, err } + rootTracker.MarkInstalled() + systemTracker.MarkInstalled() db.invalidateLeafGenerationSubtreeStats(append(committedRootPages, committedSystemPages...)) db.finalizeCommitPostWork(post) db.writeMu.RUnlock() - db.observeOrderedRootDeltaGroupPublish(wait, hold, rootsObserved, phaseStats, nil) + observePreparedGroup(preparedRootApplyStateInstalled) + observePublish(wait, hold, nil) return newSystemRoot, rootIDs, false, nil } } @@ -1711,30 +2158,57 @@ func (db *DB) publishOrderedRootDeltaBatchGroupWithSystemDeltaBuilderSerialized( phaseStats.preflightNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) } + phaseStart := time.Now() + var preparedGroup preparedRootApplyGroup + includePreparedChecksum := db.testPreparedRootApplyHook != nil + initPreparedRootApplyGroup(&preparedGroup, userRoot, baseSystemRoot, ordered, includePreparedChecksum) + phaseStats.preparedRootPrepareNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) + preparedGroupObserved := false + observePreparedGroup := func(state preparedRootApplyState) { + if preparedGroupObserved { + return + } + preparedGroupObserved = true + observePreparedRootApplyGroup(db, &phaseStats, &preparedGroup, state) + } + defer func() { + if !preparedGroupObserved && err != nil { + observePreparedGroup(preparedRootApplyStateAbandoned) + } + }() + + rootTracker := db.newPreparedOutputAllocTracker(idxGen.allocator) + systemTracker := db.newPreparedOutputAllocTracker(idxGen.allocator) + commitFinished := false + defer func() { + if err != nil && !commitFinished { + _ = rootTracker.FreeAll() + _ = systemTracker.FreeAll() + } + }() + rootIDs = make([]uint64, len(ordered)) systemOpts := systemRootOrderedPublishOptions(db) - var retired []uint64 + var pendingRetiredPages []uint64 var merged adaptive.Metrics - for idx := range ordered { - opts, err := db.orderedRootPublishOptionsForPolicy(ordered[idx].StoragePolicy) - if err != nil { - return 0, nil, err - } - phaseStart := time.Now() - rootID, rootRetired, metrics, err := db.publishOrderedRootDeltaBatchWithAllocator(idxGen, ordered[idx].BaseRoot, ordered[idx].Delta, opts, idxGen.allocator, &pagerAllocator{p: idxGen.pager}, ordered[idx].IncludeDeletedOnColdBuild) - phaseStats.rootApplyNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) - phaseStats.rootApplyCalls++ - if err != nil { - return 0, nil, err + phaseStart = time.Now() + rootApplyResults, parallelRootApply := db.applyOrderedRootDeltaBatchGroupRoots(idxGen, ordered, rootTracker, rootTracker, includePreparedChecksum) + phaseStats.rootApplyNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) + if parallelRootApply { + phaseStats.rootApplyParallelGroups++ + for orderedIdx := range ordered { + if ordered[orderedIdx].ParallelApply && ordered[orderedIdx].Delta != nil && !ordered[orderedIdx].Delta.IsEmpty() { + phaseStats.rootApplyParallelRoots++ + } } - rootIDs[idx] = rootID - rootsObserved++ - retired = append(retired, rootRetired...) - mergeOrderedRootPublishMetrics(&merged, metrics) - phaseStats.rootApplyMetrics.add(metrics) + } + outputPages, outputLeafLogPtrs := orderedRootDeltaBatchGroupPreparedOutputCounts(rootApplyResults) + preparedGroup.noteSharedOutputCounts(outputPages, outputLeafLogPtrs) + if applyErr := recordOrderedRootDeltaBatchGroupApplyResults(&preparedGroup, rootIDs, rootApplyResults, &pendingRetiredPages, &merged, &phaseStats, &rootsObserved); applyErr != nil { + return 0, nil, applyErr } - phaseStart := time.Now() + phaseStart = time.Now() iter, err := buildSystemDeltaIter(append([]uint64(nil), rootIDs...)) phaseStats.systemBuildNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) if err != nil { @@ -1743,24 +2217,40 @@ func (db *DB) publishOrderedRootDeltaBatchGroupWithSystemDeltaBuilderSerialized( if iter == nil { return 0, nil, errors.New("nil system root delta iterator") } + systemDelta, err := orderedRootDeltaBatchFromIterator(iter) + _ = iter.Close() + if err != nil { + return 0, nil, err + } phaseStart = time.Now() - rootID, rootRetired, metrics, err := db.publishOrderedRootDeltaIterator(baseSystemRoot, iter, systemOpts) + systemPreparedIdx := preparedGroup.setSystemRoot(baseSystemRoot, systemDelta, includePreparedChecksum) + phaseStats.preparedRootPrepareNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) + phaseStart = time.Now() + rootID, systemPendingRetiredPages, metrics, err := db.publishOrderedRootDeltaBatchWithAllocator(idxGen, baseSystemRoot, systemDelta, systemOpts, systemTracker, systemTracker, baseSystemRoot == 0) phaseStats.systemApplyNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) phaseStats.systemApplyCalls++ + _ = systemDelta.Close() if err != nil { return 0, nil, err } + if includePreparedChecksum { + preparedGroup.markPreparedOutput(systemPreparedIdx, rootID, systemTracker.PreparedOutputSnapshot()) + } else { + outputPages, outputLeafLogPtrs := systemTracker.PreparedOutputCounts() + preparedGroup.markPreparedOutputCounts(systemPreparedIdx, rootID, systemTracker.PreparedOutputID(), outputPages, outputLeafLogPtrs) + } newSystemRoot = rootID - retired = append(retired, rootRetired...) + pendingRetiredPages = append(pendingRetiredPages, systemPendingRetiredPages...) mergeOrderedRootPublishMetrics(&merged, metrics) phaseStats.systemApplyMetrics.add(metrics) - db.mu.RLock() - curUserRoot := db.meta.UserRootPageID - curSystemRoot := db.meta.SystemRootPageID - db.mu.RUnlock() - if curUserRoot != userRoot || curSystemRoot != baseSystemRoot { - return 0, nil, errors.New("concurrent modification detected during ordered root group publish") + preparedGroup.markInstalling() + guardNs, guardErr := db.runInstallGuard(orderedRootDeltaGroupInstallGuard(userRoot, baseSystemRoot)) + phaseStats.installGuardNs += guardNs + phaseStats.installGuardCalls++ + if guardErr != nil { + phaseStats.installGuardFailures++ + return 0, nil, guardErr } // Batch-based grouped deltas have the same value-log reachability shape as @@ -1769,12 +2259,19 @@ func (db *DB) publishOrderedRootDeltaBatchGroupWithSystemDeltaBuilderSerialized( // conservative by invalidating it after commit. var vlogRefDelta *valueLogRefDelta phaseStart = time.Now() - err = db.finalizeCommit(userRoot, newSystemRoot, retired, false, merged, nil, true, vlogRefDelta, nil, nil) + committedRootPages := rootTracker.Pages() + committedSystemPages := systemTracker.Pages() + err = db.finalizeCommit(userRoot, newSystemRoot, pendingRetiredPages, false, merged, nil, true, vlogRefDelta, nil, nil) phaseStats.finalizeNs += orderedRootDeltaGroupPhaseDurationNs(phaseStart) phaseStats.finalizeCalls++ if err != nil { return 0, nil, err } + commitFinished = true + rootTracker.MarkInstalled() + systemTracker.MarkInstalled() + db.invalidateLeafGenerationSubtreeStats(append(committedRootPages, committedSystemPages...)) + observePreparedGroup(preparedRootApplyStateInstalled) return newSystemRoot, rootIDs, nil } diff --git a/TreeDB/db/ordered_root_publish_test.go b/TreeDB/db/ordered_root_publish_test.go index d95c76ac17..45581f44f4 100644 --- a/TreeDB/db/ordered_root_publish_test.go +++ b/TreeDB/db/ordered_root_publish_test.go @@ -6,6 +6,7 @@ import ( "errors" "reflect" "strconv" + "strings" "sync" "sync/atomic" "testing" @@ -15,6 +16,7 @@ import ( "github.com/snissn/gomap/TreeDB/internal/iterator" "github.com/snissn/gomap/TreeDB/internal/memtable" "github.com/snissn/gomap/TreeDB/page" + "github.com/snissn/gomap/TreeDB/zipper" ) type closeCountingUnsafeIterator struct { @@ -763,6 +765,9 @@ func TestPublishOrderedRootDeltaGroupWithSystemBuilder_ReportsPublishStats(t *te if got := stats["treedb.publish.ordered_root_delta_group.finalize_calls_total"]; got != "1" { t.Fatalf("finalize calls stat=%q want 1", got) } + if got := stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total"]; got != "0" { + t.Fatalf("readonly prepare calls stat=%q want 0 for default publish", got) + } for _, key := range []string{ "treedb.publish.ordered_root_delta_group.preflight_ns_total", "treedb.publish.ordered_root_delta_group.root_apply_ns_total", @@ -781,16 +786,57 @@ func TestPublishOrderedRootDeltaGroupWithSystemBuilder_ReportsPublishStats(t *te "treedb.publish.ordered_root_delta_group.root_apply_pager_leaf_page_bytes_written_total", "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_page_bytes_written_total", "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_record_hint_bytes_written_total", + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_merges_total", + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_children_total", + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_workers_total", + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_ops_total", "treedb.publish.ordered_root_delta_group.root_apply_internal_page_bytes_written_total", "treedb.publish.ordered_root_delta_group.root_apply_internal_child_refs_total", "treedb.publish.ordered_root_delta_group.root_apply_internal_page_child_refs_total", "treedb.publish.ordered_root_delta_group.root_apply_internal_leaf_log_refs_total", "treedb.publish.ordered_root_delta_group.root_apply_internal_leaf_log_ref_copies_total", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ns_total", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ops_total", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_leaf_spans_total", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_targets_total", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_ranges_total", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_range_min_ops_total", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_range_max_ops_total", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_range_single_span_total", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_exact_plans_total", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_maintenance_plans_total", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_cold_build_plans_total", "treedb.publish.ordered_root_delta_group.system_build_ns_total", "treedb.publish.ordered_root_delta_group.system_apply_ns_total", "treedb.publish.ordered_root_delta_group.system_apply_ops_total", "treedb.publish.ordered_root_delta_group.system_apply_node_loads_total", + "treedb.publish.ordered_root_delta_group.install_guard_ns_total", + "treedb.publish.ordered_root_delta_group.install_guard_calls_total", + "treedb.publish.ordered_root_delta_group.install_guard_failures_total", + "treedb.publish.ordered_root_delta_group.prepared_root.prepare_ns_total", + "treedb.publish.ordered_root_delta_group.prepared_root.groups_total", + "treedb.publish.ordered_root_delta_group.prepared_root.roots_total", + "treedb.publish.ordered_root_delta_group.prepared_root.entries_total", + "treedb.publish.ordered_root_delta_group.prepared_root.tombstones_total", + "treedb.publish.ordered_root_delta_group.prepared_root.key_bytes_total", + "treedb.publish.ordered_root_delta_group.prepared_root.value_bytes_total", + "treedb.publish.ordered_root_delta_group.prepared_root.pointer_values_total", + "treedb.publish.ordered_root_delta_group.prepared_root.installed_total", + "treedb.publish.ordered_root_delta_group.prepared_root.abandoned_total", + "treedb.publish.ordered_root_delta_group.prepared_root.output_pages_total", + "treedb.publish.ordered_root_delta_group.prepared_root.output_leaf_log_ptrs_total", + "treedb.publish.ordered_root_delta_group.prepared_root.installed_output_pages_total", + "treedb.publish.ordered_root_delta_group.prepared_root.installed_output_leaf_log_ptrs_total", + "treedb.publish.ordered_root_delta_group.prepared_root.abandoned_output_pages_total", + "treedb.publish.ordered_root_delta_group.prepared_root.abandoned_output_leaf_log_ptrs_total", "treedb.publish.ordered_root_delta_group.finalize_ns_total", + "treedb.publish.install_guard.ns_total", + "treedb.publish.install_guard.calls_total", + "treedb.publish.install_guard.failures_total", + "treedb.publish.install_guard.hook_failures_total", + "treedb.publish.install_guard.user_root_mismatches_total", + "treedb.publish.install_guard.system_root_mismatches_total", } { if _, ok := stats[key]; !ok { t.Fatalf("missing ordered root delta phase stat %q", key) @@ -798,6 +844,409 @@ func TestPublishOrderedRootDeltaGroupWithSystemBuilder_ReportsPublishStats(t *te } } +func TestPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder_OptionalReadOnlyPrepareStats(t *testing.T) { + dir := t.TempDir() + db, err := Open(Options{Dir: dir}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + baseRoot, err := db.PublishOrderedRootIterator(0, mustFrozenSystemMemtable(t, + "root/a", "va", + "root/m", "vm", + "root/z", "vz", + ).NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish base root: %v", err) + } + + deltaTable := mustFrozenSystemMemtable(t, + "root/b", "vb", + "root/y", "vy", + ) + deltaIter := deltaTable.NewIterator(nil, nil) + delta, err := OrderedRootDeltaBatchFromIterator(deltaIter) + _ = deltaIter.Close() + if err != nil { + t.Fatalf("OrderedRootDeltaBatchFromIterator: %v", err) + } + defer func() { _ = delta.Close() }() + + _, _, err = db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: baseRoot, + Delta: delta, + PrepareReadOnly: true, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return mustFrozenSystemMemtable(t, "sys/collections/users/primary", strconv.FormatUint(rootIDs[0], 10)).NewIterator(nil, nil), nil + }) + if err != nil { + t.Fatalf("publish ordered root delta batch group: %v", err) + } + + stats := db.Stats() + if got := stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total"]; got != "1" { + t.Fatalf("readonly prepare calls=%q want 1", got) + } + if got := stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ops_total"]; got != "2" { + t.Fatalf("readonly prepare ops=%q want 2", got) + } + if spans := requireUintStat(t, stats, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_leaf_spans_total"); spans == 0 { + t.Fatalf("readonly prepare leaf spans=%d want > 0", spans) + } + if got := stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_exact_plans_total"]; got != "1" { + t.Fatalf("readonly prepare exact plans=%q want 1", got) + } + if got := stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_maintenance_plans_total"]; got != "0" { + t.Fatalf("readonly prepare maintenance plans=%q want 0", got) + } + if got := stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_cold_build_plans_total"]; got != "0" { + t.Fatalf("readonly prepare cold build plans=%q want 0", got) + } +} + +func TestPublishOrderedRootDeltaIteratorOptionalReadOnlyPrepareSummary(t *testing.T) { + dir := t.TempDir() + db, err := Open(Options{Dir: dir}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + baseRoot, err := db.PublishOrderedRootIterator(0, mustFrozenSystemMemtable(t, + "root/a", "va", + "root/m", "vm", + "root/z", "vz", + ).NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish base root: %v", err) + } + + opts, err := db.orderedRootPublishOptionsForPolicy(OrderedRootStorageDefault) + if err != nil { + t.Fatalf("ordered root publish options: %v", err) + } + var summary zipper.ReadOnlyLeafSpanSummary + var workerSummary zipper.ReadOnlyLeafSpanWorkerRangeSummary + var attempted bool + opts.applyOptions.PrepareReadOnly = true + opts.readOnlyPrepareSummary = &summary + opts.readOnlyPrepareWorkerSummary = &workerSummary + opts.readOnlyPrepareWorkerCount = 4 + opts.readOnlyPrepareAttempted = &attempted + + newRoot, retired, _, err := db.publishOrderedRootDeltaIterator(baseRoot, mustFrozenSystemMemtable(t, + "root/b", "vb", + "root/y", "vy", + ).NewIterator(nil, nil), opts) + if err != nil { + t.Fatalf("publish ordered root delta iterator: %v", err) + } + if newRoot == 0 || newRoot == baseRoot { + t.Fatalf("new root=%d base=%d want changed non-zero root", newRoot, baseRoot) + } + if len(retired) == 0 { + t.Fatal("retired pages empty; warm iterator apply should retire old root pages") + } + if summary.Ops != 2 || summary.Spans == 0 || !summary.ExactLeafSpans { + t.Fatalf("read-only prepare summary=%+v want ops=2 spans>0 exact", summary) + } + if workerSummary.TargetWorkers != 4 || workerSummary.Ranges == 0 || workerSummary.Ops != summary.Ops { + t.Fatalf("worker summary=%+v want target=4 ranges>0 ops=%d", workerSummary, summary.Ops) + } + if !attempted { + t.Fatal("read-only prepare attempt was not recorded") + } +} + +func TestPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder_ReadOnlyPrepareWorkerRangeStats(t *testing.T) { + dir := t.TempDir() + db, err := Open(Options{Dir: dir}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + baseRoot, err := db.PublishOrderedRootIterator(0, mustFrozenSystemMemtable(t, + "root/a", "va", + "root/m", "vm", + "root/z", "vz", + ).NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish base root: %v", err) + } + delta := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + for _, key := range []string{"root/b", "root/y"} { + if err := delta.Set([]byte(key), []byte("updated")); err != nil { + t.Fatalf("set delta %q: %v", key, err) + } + } + defer func() { _ = delta.Close() }() + + _, _, err = db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: baseRoot, + Delta: delta, + PrepareReadOnly: true, + ReadOnlyPrepareWorkerCount: 4, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return mustFrozenSystemMemtable(t, "sys/collections/users/primary", strconv.FormatUint(rootIDs[0], 10)).NewIterator(nil, nil), nil + }) + if err != nil { + t.Fatalf("publish ordered root delta batch group: %v", err) + } + + stats := db.Stats() + targets := requireUintStat(t, stats, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_targets_total") + ranges := requireUintStat(t, stats, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_ranges_total") + minOps := requireUintStat(t, stats, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_range_min_ops_total") + maxOps := requireUintStat(t, stats, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_range_max_ops_total") + singleSpan := requireUintStat(t, stats, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_range_single_span_total") + if targets != 4 || ranges == 0 { + t.Fatalf("worker targets/ranges=%d/%d want 4/>0", targets, ranges) + } + if minOps == 0 || maxOps < minOps { + t.Fatalf("worker range min/max ops=%d/%d want nonzero ordered values", minOps, maxOps) + } + if singleSpan > ranges { + t.Fatalf("single-span worker ranges=%d exceeds ranges=%d", singleSpan, ranges) + } +} + +func TestPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder_DefaultReadOnlyPrepareStatsZero(t *testing.T) { + dir := t.TempDir() + db, err := Open(Options{Dir: dir}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + baseRoot, err := db.PublishOrderedRootIterator(0, mustFrozenSystemMemtable(t, "root/a", "va").NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish base root: %v", err) + } + delta := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := delta.Set([]byte("root/b"), []byte("vb")); err != nil { + t.Fatalf("set delta: %v", err) + } + defer func() { _ = delta.Close() }() + + _, _, err = db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: baseRoot, + Delta: delta, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return mustFrozenSystemMemtable(t, "sys/collections/users/primary", strconv.FormatUint(rootIDs[0], 10)).NewIterator(nil, nil), nil + }) + if err != nil { + t.Fatalf("publish ordered root delta batch group: %v", err) + } + + stats := db.Stats() + if got := stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total"]; got != "0" { + t.Fatalf("readonly prepare calls=%q want 0", got) + } + if got := stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ops_total"]; got != "0" { + t.Fatalf("readonly prepare ops=%q want 0", got) + } + if got := stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_ranges_total"]; got != "0" { + t.Fatalf("readonly prepare worker ranges=%q want 0", got) + } +} + +func TestPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder_ReadOnlyPreparePlanKindStats(t *testing.T) { + dir := t.TempDir() + db, err := Open(Options{Dir: dir}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + cold := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := cold.Set([]byte("root/a"), []byte("va")); err != nil { + t.Fatalf("set cold delta: %v", err) + } + defer func() { _ = cold.Close() }() + + _, rootIDs, err := db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: 0, + Delta: cold, + PrepareReadOnly: true, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return mustFrozenSystemMemtable(t, "sys/collections/users/primary", strconv.FormatUint(rootIDs[0], 10)).NewIterator(nil, nil), nil + }) + if err != nil { + t.Fatalf("publish cold ordered root delta batch group: %v", err) + } + if len(rootIDs) != 1 || rootIDs[0] == 0 { + t.Fatalf("cold rootIDs=%v want one non-zero root", rootIDs) + } + + maintenance := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := maintenance.Delete([]byte("root/a")); err != nil { + t.Fatalf("delete maintenance delta: %v", err) + } + defer func() { _ = maintenance.Close() }() + + _, _, err = db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: rootIDs[0], + Delta: maintenance, + PrepareReadOnly: true, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return mustFrozenSystemMemtable(t, "sys/collections/users/primary", strconv.FormatUint(rootIDs[0], 10)).NewIterator(nil, nil), nil + }) + if err != nil { + t.Fatalf("publish maintenance ordered root delta batch group: %v", err) + } + stats := db.Stats() + if got := stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total"]; got != "2" { + t.Fatalf("readonly prepare calls=%q want 2", got) + } + if got := stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_cold_build_plans_total"]; got != "1" { + t.Fatalf("readonly prepare cold build plans=%q want 1", got) + } + if got := stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_maintenance_plans_total"]; got != "1" { + t.Fatalf("readonly prepare maintenance plans=%q want 1", got) + } +} + +func TestPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder_ReadOnlyPrepareResultReuse(t *testing.T) { + dir := t.TempDir() + db, err := Open(Options{Dir: dir}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + baseRoot, err := db.PublishOrderedRootIterator(0, mustFrozenSystemMemtable(t, + "root/a", "va", + "root/m", "vm", + "root/z", "vz", + ).NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish base root: %v", err) + } + + first := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := first.Set([]byte("root/b"), []byte("vb")); err != nil { + t.Fatalf("set first delta: %v", err) + } + defer func() { _ = first.Close() }() + + second := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := second.Set([]byte("root/y"), []byte("vy")); err != nil { + t.Fatalf("set second delta: %v", err) + } + defer func() { _ = second.Close() }() + + var prepared zipper.ReadOnlyPrepareResult + publish := func(delta *batch.Batch) uint64 { + t.Helper() + _, rootIDs, err := db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: baseRoot, + Delta: delta, + PrepareReadOnly: true, + ReadOnlyPrepareResult: &prepared, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return mustFrozenSystemMemtable(t, "sys/collections/users/primary", strconv.FormatUint(rootIDs[0], 10)).NewIterator(nil, nil), nil + }) + if err != nil { + t.Fatalf("publish ordered root delta batch group: %v", err) + } + if prepared.RootID != baseRoot { + t.Fatalf("prepared root=%d want base root %d", prepared.RootID, baseRoot) + } + if prepared.Ops != 1 || len(prepared.LeafSpans) == 0 { + t.Fatalf("prepared ops/spans=%d/%d want 1/>0", prepared.Ops, len(prepared.LeafSpans)) + } + if len(rootIDs) != 1 || rootIDs[0] == 0 { + t.Fatalf("rootIDs=%v want one non-zero root", rootIDs) + } + return rootIDs[0] + } + + baseRoot = publish(first) + firstSpanCap := cap(prepared.LeafSpans) + if firstSpanCap == 0 { + t.Fatal("prepared leaf span capacity is zero after first publish") + } + firstSpan := &prepared.LeafSpans[0] + baseRoot = publish(second) + if cap(prepared.LeafSpans) < firstSpanCap { + t.Fatalf("prepared leaf span capacity shrank from %d to %d", firstSpanCap, cap(prepared.LeafSpans)) + } + if &prepared.LeafSpans[0] != firstSpan { + t.Fatal("prepared leaf span backing array was not reused") + } + + stats := db.Stats() + if got := stats["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total"]; got != "2" { + t.Fatalf("readonly prepare calls=%q want 2", got) + } +} + +func TestPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder_ReadOnlyPrepareResultRejectsSharedResult(t *testing.T) { + dir := t.TempDir() + db, err := Open(Options{Dir: dir}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + baseRootA, err := db.PublishOrderedRootIterator(0, mustFrozenSystemMemtable(t, "root/a", "va").NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish base root A: %v", err) + } + baseRootB, err := db.PublishOrderedRootIterator(0, mustFrozenSystemMemtable(t, "root/b", "vb").NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish base root B: %v", err) + } + deltaA := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := deltaA.Set([]byte("root/a"), []byte("next-a")); err != nil { + t.Fatalf("set delta A: %v", err) + } + defer func() { _ = deltaA.Close() }() + deltaB := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := deltaB.Set([]byte("root/b"), []byte("next-b")); err != nil { + t.Fatalf("set delta B: %v", err) + } + defer func() { _ = deltaB.Close() }() + + var shared zipper.ReadOnlyPrepareResult + _, _, err = db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{ + { + BaseRoot: baseRootA, + Delta: deltaA, + PrepareReadOnly: true, + ParallelApply: true, + ReadOnlyPrepareResult: &shared, + }, + { + BaseRoot: baseRootB, + Delta: deltaB, + PrepareReadOnly: true, + ParallelApply: true, + ReadOnlyPrepareResult: &shared, + }, + }, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return mustFrozenSystemMemtable(t, "sys/collections/users/primary", strconv.FormatUint(rootIDs[0], 10)).NewIterator(nil, nil), nil + }) + if err == nil || !strings.Contains(err.Error(), "read-only prepare result reused") { + t.Fatalf("error=%v want shared read-only prepare result rejection", err) + } +} + +func requireUintStat(tb testing.TB, stats map[string]string, key string) uint64 { + tb.Helper() + raw, ok := stats[key] + if !ok { + tb.Fatalf("missing stat %q", key) + } + v, err := strconv.ParseUint(raw, 10, 64) + if err != nil { + tb.Fatalf("stat %q=%q is not uint: %v", key, raw, err) + } + return v +} + func TestPublishOrderedRootDeltaGroupPreflightFailureDoesNotCountRoots(t *testing.T) { dir := t.TempDir() db, err := Open(Options{Dir: dir}) @@ -1520,7 +1969,7 @@ func TestApplyOrderedRootDeltaBatchGroupRoots_MixedOptInStartsParallelBeforeSeri {BaseRoot: 0, Delta: deltaA, ParallelApply: true}, {BaseRoot: baseRootB, Delta: deltaB}, {BaseRoot: 0, Delta: deltaC, ParallelApply: true}, - }, serialAlloc, coldAlloc) + }, serialAlloc, coldAlloc, false) if !parallel { t.Fatal("expected mixed group to use parallel apply") } @@ -1593,6 +2042,71 @@ func TestPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder_SerializedColdB } } +func TestPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder_ColdSystemRootPreservesDeletes(t *testing.T) { + dir := t.TempDir() + db, err := Open(Options{Dir: dir}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + delta := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := delta.Set([]byte("doc/u1"), []byte("document")); err != nil { + t.Fatalf("set doc/u1: %v", err) + } + defer func() { _ = delta.Close() }() + systemDelta := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := systemDelta.Delete([]byte("sys/collections/users/deleted")); err != nil { + t.Fatalf("delete system descriptor: %v", err) + } + defer func() { _ = systemDelta.Close() }() + + idx := db.idx.Load() + oldState := db.state.Load() + if idx == nil || oldState == nil { + t.Fatal("expected initialized DB state") + } + // Open installs the normal format/system root. This regression targets the + // serialized batch-group cold system-root branch directly. + state := *oldState + state.SystemRootPageID = 0 + db.mu.Lock() + db.meta.SystemRootPageID = 0 + db.mu.Unlock() + db.state.Store(&state) + db.publishSnapshotView(idx, &state, db.valueLogManager) + + newSystemRoot, rootIDs, err := db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: 0, + Delta: delta, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + if len(rootIDs) != 1 || rootIDs[0] == 0 { + return nil, errors.New("unexpected cold root ID") + } + return newOrderedRootDeltaBatchIterator(systemDelta, true), nil + }) + if err != nil { + t.Fatalf("publish cold system root tombstone: %v", err) + } + if newSystemRoot == 0 || len(rootIDs) != 1 || rootIDs[0] == 0 { + t.Fatalf("newSystemRoot=%d rootIDs=%v, want non-zero roots", newSystemRoot, rootIDs) + } + + snap := db.AcquireSnapshot() + if snap == nil { + t.Fatal("expected snapshot") + } + defer func() { _ = snap.Close() }() + it, err := snap.IteratorAtRootWithOptions(newSystemRoot, []byte("sys/collections/users/deleted"), nil, IteratorOptions{IncludeTombstones: true}) + if err != nil { + t.Fatalf("IteratorAtRootWithOptions: %v", err) + } + defer func() { _ = it.Close() }() + if !it.Valid() || !bytes.Equal(it.UnsafeKey(), []byte("sys/collections/users/deleted")) || !it.IsDeleted() { + t.Fatalf("iterator valid/key/deleted=%v/%q/%v, want system tombstone", it.Valid(), it.UnsafeKey(), it.Valid() && it.IsDeleted()) + } +} + type orderedRootDeltaBatchGroupTestAllocator struct { delegate interface { Alloc(uint64) (uint64, error) diff --git a/TreeDB/db/prepared_output.go b/TreeDB/db/prepared_output.go new file mode 100644 index 0000000000..731f8d50c5 --- /dev/null +++ b/TreeDB/db/prepared_output.go @@ -0,0 +1,35 @@ +package db + +import ( + "github.com/snissn/gomap/TreeDB/freelist" + "github.com/snissn/gomap/TreeDB/page" +) + +type preparedOutputID uint64 + +type preparedOutputState uint8 + +const ( + preparedOutputStateNone preparedOutputState = iota + preparedOutputStatePrepared + preparedOutputStateInstalled + preparedOutputStateAbandoned +) + +type preparedOutputSnapshot struct { + ID preparedOutputID + State preparedOutputState + Pages []uint64 + LeafLogPtrs []page.LeafLogPtr +} + +func (db *DB) nextPreparedOutputID() preparedOutputID { + if db == nil { + return 0 + } + return preparedOutputID(db.preparedOutputNextID.Add(1)) +} + +func (db *DB) newPreparedOutputAllocTracker(alloc *freelist.Allocator) *allocTracker { + return newPreparedOutputAllocTracker(alloc, db.nextPreparedOutputID()) +} diff --git a/TreeDB/db/prepared_output_test.go b/TreeDB/db/prepared_output_test.go new file mode 100644 index 0000000000..44a5f883f7 --- /dev/null +++ b/TreeDB/db/prepared_output_test.go @@ -0,0 +1,232 @@ +package db + +import ( + "testing" + + "github.com/snissn/gomap/TreeDB/page" +) + +type preparedOutputTestLeafLog struct { + ptrs []page.LeafLogPtr +} + +func (l *preparedOutputTestLeafLog) AppendLeafPage(leafPage []byte) (page.LeafLogPtr, error) { + ptr := page.LeafLogPtr{ + FileID: uint32(len(l.ptrs) + 1), + Offset: uint64(len(l.ptrs)+1) * 100, + RecordLengthHint: uint32(len(leafPage)), + } + l.ptrs = append(l.ptrs, ptr) + return ptr, nil +} + +func (l *preparedOutputTestLeafLog) Flush() error { + return nil +} + +func (l *preparedOutputTestLeafLog) Sync() error { + return nil +} + +func TestPreparedOutputAllocTrackerAbandonsOwnedPagesOnFree(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + idx := db.idx.Load() + if idx == nil { + t.Fatal("missing index") + } + tracker := db.newPreparedOutputAllocTracker(idx.allocator) + if got := tracker.PreparedOutputID(); got == 0 { + t.Fatal("prepared output ID is zero") + } + + pageID, err := tracker.Alloc(0) + if err != nil { + t.Fatalf("alloc prepared page: %v", err) + } + before := tracker.PreparedOutputSnapshot() + if before.State != preparedOutputStatePrepared { + t.Fatalf("state=%v want prepared", before.State) + } + if len(before.Pages) != 1 || before.Pages[0] != pageID { + t.Fatalf("pages=%v want [%d]", before.Pages, pageID) + } + + if err := tracker.FreeAll(); err != nil { + t.Fatalf("free prepared pages: %v", err) + } + after := tracker.PreparedOutputSnapshot() + if after.State != preparedOutputStateAbandoned { + t.Fatalf("state=%v want abandoned", after.State) + } + if len(after.Pages) != 0 { + t.Fatalf("abandoned tracker retained pages: %v", after.Pages) + } +} + +func TestPreparedOutputAllocTrackerInstallPreventsAbandonFree(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + idx := db.idx.Load() + if idx == nil { + t.Fatal("missing index") + } + tracker := db.newPreparedOutputAllocTracker(idx.allocator) + pageID, err := tracker.Alloc(0) + if err != nil { + t.Fatalf("alloc prepared page: %v", err) + } + + tracker.MarkInstalled() + if err := tracker.FreeAll(); err != nil { + t.Fatalf("free installed prepared output: %v", err) + } + after := tracker.PreparedOutputSnapshot() + if after.State != preparedOutputStateInstalled { + t.Fatalf("state=%v want installed", after.State) + } + if len(after.Pages) != 1 || after.Pages[0] != pageID { + t.Fatalf("installed tracker pages=%v want [%d]", after.Pages, pageID) + } +} + +func TestPreparedOutputAllocTrackerAbandonDoesNotBecomeInstalled(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + idx := db.idx.Load() + if idx == nil { + t.Fatal("missing index") + } + tracker := db.newPreparedOutputAllocTracker(idx.allocator) + if _, err := tracker.Alloc(0); err != nil { + t.Fatalf("alloc prepared page: %v", err) + } + if err := tracker.FreeAll(); err != nil { + t.Fatalf("free prepared pages: %v", err) + } + + tracker.MarkInstalled() + after := tracker.PreparedOutputSnapshot() + if after.State != preparedOutputStateAbandoned { + t.Fatalf("state=%v want abandoned", after.State) + } + if len(after.Pages) != 0 { + t.Fatalf("abandoned tracker retained pages: %v", after.Pages) + } +} + +func TestPreparedOutputAllocTrackerEmptyFreeLeavesPreparedState(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + idx := db.idx.Load() + if idx == nil { + t.Fatal("missing index") + } + tracker := db.newPreparedOutputAllocTracker(idx.allocator) + if err := tracker.FreeAll(); err != nil { + t.Fatalf("free empty prepared output: %v", err) + } + + after := tracker.PreparedOutputSnapshot() + if after.State != preparedOutputStatePrepared { + t.Fatalf("state=%v want prepared", after.State) + } + if len(after.Pages) != 0 { + t.Fatalf("empty tracker pages=%v want none", after.Pages) + } +} + +func TestPreparedOutputLeafPageLogTracksPointers(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + idx := db.idx.Load() + if idx == nil { + t.Fatal("missing index") + } + tracker := db.newPreparedOutputAllocTracker(idx.allocator) + log := preparedOutputLeafPageLog{ + inner: &preparedOutputTestLeafLog{}, + tracker: tracker, + } + first, err := log.AppendLeafPage(make([]byte, page.PageSize)) + if err != nil { + t.Fatalf("append first leaf page: %v", err) + } + second, err := log.AppendLeafPage(make([]byte, page.PageSize)) + if err != nil { + t.Fatalf("append second leaf page: %v", err) + } + + before := tracker.PreparedOutputSnapshot() + if before.State != preparedOutputStatePrepared { + t.Fatalf("state=%v want prepared", before.State) + } + if len(before.LeafLogPtrs) != 2 || before.LeafLogPtrs[0] != first || before.LeafLogPtrs[1] != second { + t.Fatalf("leaf log ptrs=%v want [%v %v]", before.LeafLogPtrs, first, second) + } + + if err := tracker.FreeAll(); err != nil { + t.Fatalf("free prepared leaf-log output: %v", err) + } + after := tracker.PreparedOutputSnapshot() + if after.State != preparedOutputStateAbandoned { + t.Fatalf("state=%v want abandoned", after.State) + } + if len(after.LeafLogPtrs) != 0 { + t.Fatalf("abandoned tracker retained leaf-log ptrs: %v", after.LeafLogPtrs) + } +} + +func TestPreparedOutputLeafPageLogInstalledRetainsPointers(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + idx := db.idx.Load() + if idx == nil { + t.Fatal("missing index") + } + tracker := db.newPreparedOutputAllocTracker(idx.allocator) + log := preparedOutputLeafPageLog{ + inner: &preparedOutputTestLeafLog{}, + tracker: tracker, + } + ptr, err := log.AppendLeafPage(make([]byte, page.PageSize)) + if err != nil { + t.Fatalf("append leaf page: %v", err) + } + + tracker.MarkInstalled() + if err := tracker.FreeAll(); err != nil { + t.Fatalf("free installed prepared leaf-log output: %v", err) + } + after := tracker.PreparedOutputSnapshot() + if after.State != preparedOutputStateInstalled { + t.Fatalf("state=%v want installed", after.State) + } + if len(after.LeafLogPtrs) != 1 || after.LeafLogPtrs[0] != ptr { + t.Fatalf("installed tracker leaf-log ptrs=%v want [%v]", after.LeafLogPtrs, ptr) + } +} diff --git a/TreeDB/db/prepared_root_apply.go b/TreeDB/db/prepared_root_apply.go new file mode 100644 index 0000000000..a0fd1d630c --- /dev/null +++ b/TreeDB/db/prepared_root_apply.go @@ -0,0 +1,451 @@ +package db + +import ( + "github.com/snissn/gomap/TreeDB/batch" + "github.com/snissn/gomap/TreeDB/page" +) + +type preparedRootApplyState uint8 + +const ( + preparedRootApplyStatePlanned preparedRootApplyState = iota + preparedRootApplyStatePrepared + preparedRootApplyStateInstalling + preparedRootApplyStateInstalled + preparedRootApplyStateAbandoned +) + +type preparedRootIdentityKind uint8 + +const ( + preparedRootIdentityData preparedRootIdentityKind = iota + preparedRootIdentitySystem +) + +type preparedRootIdentity struct { + kind preparedRootIdentityKind + ordinal int +} + +type preparedRootDeltaPlanSummary struct { + entries uint64 + tombstones uint64 + keyBytes uint64 + valueBytes uint64 + pointerValues uint64 + checksum uint64 + firstKey []byte + lastKey []byte +} + +type preparedRootApply struct { + identity preparedRootIdentity + baseRootID uint64 + preparedRoot uint64 + outputID preparedOutputID + output preparedOutputSnapshot + outputPages uint64 + outputLeafLogPtrs uint64 + prepared bool + storage OrderedRootStoragePolicy + plan preparedRootDeltaPlanSummary + state preparedRootApplyState +} + +type preparedRootApplyGroup struct { + baseUserRootID uint64 + baseSystemRootID uint64 + outputPages uint64 + outputLeafLogPtrs uint64 + state preparedRootApplyState + applyCount int + inlineApplies [4]preparedRootApply + overflowApplies []preparedRootApply +} + +type preparedRootApplyStats struct { + groups uint64 + roots uint64 + entries uint64 + tombstones uint64 + keyBytes uint64 + valueBytes uint64 + pointerValues uint64 + installed uint64 + abandoned uint64 + outputPages uint64 + outputLeafLogPtrs uint64 + installedPages uint64 + installedLeafLogPtrs uint64 + abandonedPages uint64 + abandonedLeafLogPtrs uint64 +} + +const ( + preparedRootPlanChecksumOffset = 14695981039346656037 + preparedRootPlanChecksumPrime = 1099511628211 +) + +func initPreparedRootApplyGroup(group *preparedRootApplyGroup, baseUserRootID, baseSystemRootID uint64, ordered []OrderedRootDeltaBatchPublishInput, includeChecksum bool) { + if group == nil { + return + } + *group = preparedRootApplyGroup{ + baseUserRootID: baseUserRootID, + baseSystemRootID: baseSystemRootID, + state: preparedRootApplyStatePlanned, + } + for i := range ordered { + group.appendApply(preparedRootApply{ + identity: preparedRootIdentity{ + kind: preparedRootIdentityData, + ordinal: i, + }, + baseRootID: ordered[i].BaseRoot, + storage: ordered[i].StoragePolicy, + plan: preparedRootDeltaPlanSummaryFromBatch(ordered[i].Delta, includeChecksum), + state: preparedRootApplyStatePlanned, + }) + } +} + +func (group *preparedRootApplyGroup) appendApply(apply preparedRootApply) int { + if group == nil { + return -1 + } + idx := group.applyCount + if idx < len(group.inlineApplies) { + group.inlineApplies[idx] = apply + } else { + group.overflowApplies = append(group.overflowApplies, apply) + } + group.applyCount++ + return idx +} + +func (group *preparedRootApplyGroup) applyLen() int { + if group == nil { + return 0 + } + return group.applyCount +} + +func (group *preparedRootApplyGroup) applyAt(idx int) *preparedRootApply { + if group == nil || idx < 0 || idx >= group.applyCount { + return nil + } + if idx < len(group.inlineApplies) { + return &group.inlineApplies[idx] + } + return &group.overflowApplies[idx-len(group.inlineApplies)] +} + +func (group *preparedRootApplyGroup) setSystemRoot(baseRootID uint64, delta *batch.Batch, includeChecksum bool) int { + if group == nil { + return -1 + } + group.baseSystemRootID = baseRootID + for i := group.applyCount - 1; i >= 0; i-- { + apply := group.applyAt(i) + if apply != nil && apply.identity.kind == preparedRootIdentitySystem { + if apply.prepared { + if apply.state != preparedRootApplyStateInstalled { + apply.state = preparedRootApplyStateAbandoned + apply.output.State = preparedOutputStateAbandoned + } + break + } + if apply.state == preparedRootApplyStateAbandoned { + break + } + *apply = preparedRootApply{ + identity: preparedRootIdentity{ + kind: preparedRootIdentitySystem, + ordinal: -1, + }, + baseRootID: baseRootID, + storage: OrderedRootStorageDefault, + plan: preparedRootDeltaPlanSummaryFromBatch(delta, includeChecksum), + state: preparedRootApplyStatePlanned, + } + return i + } + } + return group.appendApply(preparedRootApply{ + identity: preparedRootIdentity{ + kind: preparedRootIdentitySystem, + ordinal: -1, + }, + baseRootID: baseRootID, + storage: OrderedRootStorageDefault, + plan: preparedRootDeltaPlanSummaryFromBatch(delta, includeChecksum), + state: preparedRootApplyStatePlanned, + }) +} + +func (group *preparedRootApplyGroup) markPrepared(idx int, rootID uint64, outputID preparedOutputID) { + group.markPreparedOutputCounts(idx, rootID, outputID, 0, 0) +} + +func (group *preparedRootApplyGroup) markPreparedOutput(idx int, rootID uint64, output preparedOutputSnapshot) { + group.markPreparedOutputCounts(idx, rootID, output.ID, uint64(len(output.Pages)), uint64(len(output.LeafLogPtrs))) + apply := group.applyAt(idx) + if apply != nil { + apply.output = clonePreparedOutputSnapshot(output) + } +} + +func (group *preparedRootApplyGroup) markPreparedOutputCounts(idx int, rootID uint64, outputID preparedOutputID, outputPages, outputLeafLogPtrs uint64) { + apply := group.applyAt(idx) + if apply == nil { + return + } + apply.preparedRoot = rootID + apply.outputID = outputID + apply.outputPages = outputPages + apply.outputLeafLogPtrs = outputLeafLogPtrs + apply.prepared = true + apply.state = preparedRootApplyStatePrepared +} + +func (group *preparedRootApplyGroup) noteSharedOutputCounts(outputPages, outputLeafLogPtrs uint64) { + if group == nil { + return + } + group.outputPages = outputPages + group.outputLeafLogPtrs = outputLeafLogPtrs +} + +func (group *preparedRootApplyGroup) markInstalling() { + if group == nil { + return + } + group.state = preparedRootApplyStateInstalling + for i := 0; i < group.applyCount; i++ { + apply := group.applyAt(i) + if apply != nil && apply.prepared && apply.state != preparedRootApplyStateAbandoned { + apply.state = preparedRootApplyStateInstalling + } + } +} + +func (group *preparedRootApplyGroup) markInstalled() { + if group == nil { + return + } + group.state = preparedRootApplyStateInstalled + for i := 0; i < group.applyCount; i++ { + if apply := group.applyAt(i); apply != nil && apply.prepared && apply.state != preparedRootApplyStateAbandoned { + apply.state = preparedRootApplyStateInstalled + apply.output.State = preparedOutputStateInstalled + } + } +} + +func (group *preparedRootApplyGroup) markAbandoned() { + if group == nil { + return + } + group.state = preparedRootApplyStateAbandoned + for i := 0; i < group.applyCount; i++ { + apply := group.applyAt(i) + if apply != nil && apply.prepared && apply.state != preparedRootApplyStateInstalled { + apply.state = preparedRootApplyStateAbandoned + apply.output.State = preparedOutputStateAbandoned + } + } +} + +func (stats *preparedRootApplyStats) observeGroup(group *preparedRootApplyGroup) { + if stats == nil || group == nil || group.applyCount == 0 { + return + } + groupStats := preparedRootApplyStats{} + for i := 0; i < group.applyCount; i++ { + apply := group.applyAt(i) + if apply == nil || !apply.prepared { + continue + } + groupStats.roots++ + outputPages := apply.outputPages + outputLeafLogPtrs := apply.outputLeafLogPtrs + if apply.identity.kind == preparedRootIdentityData { + // Data roots in an ordered-root group share one prepared-output + // tracker. Count that shared output once at the group level below. + outputPages = 0 + outputLeafLogPtrs = 0 + } + switch apply.state { + case preparedRootApplyStateInstalled: + groupStats.installed++ + groupStats.installedPages += outputPages + groupStats.installedLeafLogPtrs += outputLeafLogPtrs + case preparedRootApplyStateAbandoned: + groupStats.abandoned++ + groupStats.abandonedPages += outputPages + groupStats.abandonedLeafLogPtrs += outputLeafLogPtrs + } + groupStats.outputPages += outputPages + groupStats.outputLeafLogPtrs += outputLeafLogPtrs + plan := apply.plan + groupStats.entries += plan.entries + groupStats.tombstones += plan.tombstones + groupStats.keyBytes += plan.keyBytes + groupStats.valueBytes += plan.valueBytes + groupStats.pointerValues += plan.pointerValues + } + if groupStats.roots == 0 { + return + } + groupStats.outputPages += group.outputPages + groupStats.outputLeafLogPtrs += group.outputLeafLogPtrs + switch group.state { + case preparedRootApplyStateInstalled: + groupStats.installedPages += group.outputPages + groupStats.installedLeafLogPtrs += group.outputLeafLogPtrs + case preparedRootApplyStateAbandoned: + groupStats.abandonedPages += group.outputPages + groupStats.abandonedLeafLogPtrs += group.outputLeafLogPtrs + } + groupStats.groups = 1 + stats.groups += groupStats.groups + stats.roots += groupStats.roots + stats.entries += groupStats.entries + stats.tombstones += groupStats.tombstones + stats.keyBytes += groupStats.keyBytes + stats.valueBytes += groupStats.valueBytes + stats.pointerValues += groupStats.pointerValues + stats.installed += groupStats.installed + stats.abandoned += groupStats.abandoned + stats.outputPages += groupStats.outputPages + stats.outputLeafLogPtrs += groupStats.outputLeafLogPtrs + stats.installedPages += groupStats.installedPages + stats.installedLeafLogPtrs += groupStats.installedLeafLogPtrs + stats.abandonedPages += groupStats.abandonedPages + stats.abandonedLeafLogPtrs += groupStats.abandonedLeafLogPtrs +} + +func observePreparedRootApplyGroup(db *DB, phases *orderedRootDeltaGroupPublishPhaseStats, group *preparedRootApplyGroup, state preparedRootApplyState) { + if phases == nil || group == nil || group.applyCount == 0 { + return + } + switch state { + case preparedRootApplyStateInstalled: + group.markInstalled() + case preparedRootApplyStateAbandoned: + group.markAbandoned() + case preparedRootApplyStateInstalling: + group.markInstalling() + } + phases.preparedRootStats.observeGroup(group) + if db != nil { + if hook := db.testPreparedRootApplyHook; hook != nil { + hook(clonePreparedRootApplyGroup(*group)) + } + } +} + +func clonePreparedRootApplyGroup(src preparedRootApplyGroup) preparedRootApplyGroup { + dst := preparedRootApplyGroup{ + baseUserRootID: src.baseUserRootID, + baseSystemRootID: src.baseSystemRootID, + outputPages: src.outputPages, + outputLeafLogPtrs: src.outputLeafLogPtrs, + state: src.state, + } + for i := 0; i < src.applyCount; i++ { + srcApply := src.applyAt(i) + if srcApply == nil { + continue + } + apply := *srcApply + apply.plan.firstKey = append([]byte(nil), apply.plan.firstKey...) + apply.plan.lastKey = append([]byte(nil), apply.plan.lastKey...) + apply.output = clonePreparedOutputSnapshot(apply.output) + dst.appendApply(apply) + } + return dst +} + +func clonePreparedOutputSnapshot(src preparedOutputSnapshot) preparedOutputSnapshot { + return preparedOutputSnapshot{ + ID: src.ID, + State: src.State, + Pages: append([]uint64(nil), src.Pages...), + LeafLogPtrs: append([]page.LeafLogPtr(nil), src.LeafLogPtrs...), + } +} + +func preparedRootDeltaPlanSummaryFromBatch(delta *batch.Batch, includeChecksum bool) preparedRootDeltaPlanSummary { + if delta == nil { + return preparedRootDeltaPlanSummary{} + } + entries := delta.SortedEntries() + if len(entries) == 0 { + if includeChecksum { + return preparedRootDeltaPlanSummary{checksum: preparedRootPlanChecksumOffset} + } + return preparedRootDeltaPlanSummary{} + } + summary := preparedRootDeltaPlanSummary{ + entries: uint64(len(entries)), + } + if includeChecksum { + summary.checksum = preparedRootPlanChecksumOffset + // Key spans are part of hook/debug metadata. They are intentionally not + // captured on the normal hot path so prepared-root accounting stays + // allocation-free unless stable test metadata is requested. + summary.firstKey = append([]byte(nil), entries[0].Key...) + summary.lastKey = append([]byte(nil), entries[len(entries)-1].Key...) + } + for i := range entries { + entry := entries[i] + summary.keyBytes += uint64(len(entry.Key)) + if includeChecksum { + summary.checksum = preparedRootPlanChecksumAddByte(summary.checksum, byte(entry.Type)) + summary.checksum = preparedRootPlanChecksumAddBytes(summary.checksum, entry.Key) + } + if entry.Type == batch.OpDelete { + summary.tombstones++ + continue + } + if entry.IsPtr { + summary.pointerValues++ + summary.valueBytes += uint64(page.ValuePtrSize) + if includeChecksum { + summary.checksum = preparedRootPlanChecksumAddByte(summary.checksum, 1) + summary.checksum = preparedRootPlanChecksumAddUint64(summary.checksum, uint64(entry.ValuePtr.FileID)) + summary.checksum = preparedRootPlanChecksumAddUint64(summary.checksum, entry.ValuePtr.Offset) + summary.checksum = preparedRootPlanChecksumAddUint64(summary.checksum, uint64(entry.ValuePtr.Length)) + } + continue + } + summary.valueBytes += uint64(len(entry.Value)) + if includeChecksum { + summary.checksum = preparedRootPlanChecksumAddByte(summary.checksum, 0) + summary.checksum = preparedRootPlanChecksumAddBytes(summary.checksum, entry.Value) + } + } + return summary +} + +func preparedRootPlanChecksumAddByte(sum uint64, b byte) uint64 { + sum ^= uint64(b) + sum *= preparedRootPlanChecksumPrime + return sum +} + +func preparedRootPlanChecksumAddBytes(sum uint64, b []byte) uint64 { + sum = preparedRootPlanChecksumAddUint64(sum, uint64(len(b))) + for _, c := range b { + sum = preparedRootPlanChecksumAddByte(sum, c) + } + return sum +} + +func preparedRootPlanChecksumAddUint64(sum, v uint64) uint64 { + for i := 0; i < 8; i++ { + sum = preparedRootPlanChecksumAddByte(sum, byte(v>>(uint(i)*8))) + } + return sum +} diff --git a/TreeDB/db/prepared_root_apply_test.go b/TreeDB/db/prepared_root_apply_test.go new file mode 100644 index 0000000000..b9a7625ffc --- /dev/null +++ b/TreeDB/db/prepared_root_apply_test.go @@ -0,0 +1,973 @@ +package db + +import ( + "bytes" + "encoding/binary" + "errors" + "strconv" + "testing" + + "github.com/snissn/gomap/TreeDB/batch" + "github.com/snissn/gomap/TreeDB/internal/iterator" + "github.com/snissn/gomap/TreeDB/page" + "github.com/snissn/gomap/TreeDB/zipper" +) + +func TestPreparedRootDeltaPlanSummaryFromBatch(t *testing.T) { + delta := batch.New(nil, 1<<20) + defer func() { _ = delta.Close() }() + if err := delta.Set([]byte("b"), []byte("value-b")); err != nil { + t.Fatalf("set b: %v", err) + } + if err := delta.Delete([]byte("a")); err != nil { + t.Fatalf("delete a: %v", err) + } + if err := delta.SetPointer([]byte("c"), page.ValuePtr{ + FileID: page.ValueLogFileID(3), + Offset: 17, + Length: 41, + }); err != nil { + t.Fatalf("set pointer c: %v", err) + } + + summary := preparedRootDeltaPlanSummaryFromBatch(delta, true) + if summary.entries != 3 { + t.Fatalf("entries=%d want 3", summary.entries) + } + if summary.tombstones != 1 { + t.Fatalf("tombstones=%d want 1", summary.tombstones) + } + if summary.keyBytes != 3 { + t.Fatalf("key bytes=%d want 3", summary.keyBytes) + } + wantValueBytes := uint64(len("value-b")) + uint64(page.ValuePtrSize) + if summary.valueBytes != wantValueBytes { + t.Fatalf("value bytes=%d want %d", summary.valueBytes, wantValueBytes) + } + if summary.pointerValues != 1 { + t.Fatalf("pointer values=%d want 1", summary.pointerValues) + } + if !bytes.Equal(summary.firstKey, []byte("a")) || !bytes.Equal(summary.lastKey, []byte("c")) { + t.Fatalf("key span=%q..%q want a..c", summary.firstKey, summary.lastKey) + } + if summary.checksum == 0 || summary.checksum == preparedRootPlanChecksumOffset { + t.Fatalf("checksum=%d looks uninitialized", summary.checksum) + } + noSnapshotSummary := preparedRootDeltaPlanSummaryFromBatch(delta, false) + if noSnapshotSummary.firstKey != nil || noSnapshotSummary.lastKey != nil { + t.Fatalf("non-hook summary retained key span=%q..%q", noSnapshotSummary.firstKey, noSnapshotSummary.lastKey) + } + if noSnapshotSummary.checksum != 0 { + t.Fatalf("non-hook checksum=%d want 0", noSnapshotSummary.checksum) + } +} + +func TestPreparedRootDeltaPlanSummaryChecksumDistinguishesPointerAndInline(t *testing.T) { + ptr := page.ValuePtr{ + FileID: page.ValueLogFileID(3), + Offset: 17, + Length: 41, + } + ptrDelta := batch.New(nil, 1<<20) + defer func() { _ = ptrDelta.Close() }() + if err := ptrDelta.SetPointer([]byte("k"), ptr); err != nil { + t.Fatalf("SetPointer: %v", err) + } + + inlineValue := make([]byte, 24) + binary.LittleEndian.PutUint64(inlineValue[0:8], uint64(ptr.FileID)) + binary.LittleEndian.PutUint64(inlineValue[8:16], ptr.Offset) + binary.LittleEndian.PutUint64(inlineValue[16:24], uint64(ptr.Length)) + inlineDelta := batch.New(nil, 1<<20) + defer func() { _ = inlineDelta.Close() }() + if err := inlineDelta.Set([]byte("k"), inlineValue); err != nil { + t.Fatalf("Set inline: %v", err) + } + + ptrSummary := preparedRootDeltaPlanSummaryFromBatch(ptrDelta, true) + inlineSummary := preparedRootDeltaPlanSummaryFromBatch(inlineDelta, true) + if ptrSummary.checksum == inlineSummary.checksum { + t.Fatalf("pointer and inline checksum both %d", ptrSummary.checksum) + } +} + +func TestPreparedRootApplyStatsCountsPreparedZeroRoot(t *testing.T) { + group := preparedRootApplyGroup{} + group.appendApply(preparedRootApply{ + identity: preparedRootIdentity{kind: preparedRootIdentityData}, + plan: preparedRootDeltaPlanSummary{ + entries: 1, + keyBytes: 3, + valueBytes: 5, + }, + state: preparedRootApplyStatePlanned, + }) + group.markPrepared(0, 0, 1) + group.markInstalled() + + var stats preparedRootApplyStats + stats.observeGroup(&group) + if stats.groups != 1 || stats.roots != 1 || stats.installed != 1 || stats.abandoned != 0 { + t.Fatalf("stats groups=%d roots=%d installed=%d abandoned=%d want 1,1,1,0", stats.groups, stats.roots, stats.installed, stats.abandoned) + } + if stats.entries != 1 || stats.keyBytes != 3 || stats.valueBytes != 5 { + t.Fatalf("stats entries/key/value=%d/%d/%d want 1/3/5", stats.entries, stats.keyBytes, stats.valueBytes) + } +} + +func TestPreparedRootApplyRecordsLaterSuccessBeforeEarlierApplyError(t *testing.T) { + sentinel := errors.New("root apply failed") + group := preparedRootApplyGroup{} + group.appendApply(preparedRootApply{ + identity: preparedRootIdentity{kind: preparedRootIdentityData, ordinal: 0}, + plan: preparedRootDeltaPlanSummary{entries: 1}, + state: preparedRootApplyStatePlanned, + }) + group.appendApply(preparedRootApply{ + identity: preparedRootIdentity{kind: preparedRootIdentityData, ordinal: 1}, + plan: preparedRootDeltaPlanSummary{entries: 2}, + state: preparedRootApplyStatePlanned, + }) + rootIDs := make([]uint64, 2) + var pendingRetired []uint64 + var phaseStats orderedRootDeltaGroupPublishPhaseStats + rootsObserved := 0 + + err := recordOrderedRootDeltaBatchGroupApplyResults( + &group, + rootIDs, + []orderedRootDeltaBatchGroupApplyResult{ + {idx: 0, err: sentinel, attempted: true}, + {idx: 1, rootID: 42, pendingRetiredPages: []uint64{7}, attempted: true}, + {idx: 2}, + }, + &pendingRetired, + nil, + &phaseStats, + &rootsObserved, + ) + if !errors.Is(err, sentinel) { + t.Fatalf("error=%v want sentinel", err) + } + if rootIDs[0] != 0 || rootIDs[1] != 42 { + t.Fatalf("root IDs=%v want [0 42]", rootIDs) + } + if rootsObserved != 1 || phaseStats.rootApplyCalls != 1 { + t.Fatalf("roots observed=%d root apply calls=%d want 1/1", rootsObserved, phaseStats.rootApplyCalls) + } + if len(pendingRetired) != 1 || pendingRetired[0] != 7 { + t.Fatalf("pending retired=%v want [7]", pendingRetired) + } + if failed := group.applyAt(0); failed == nil || failed.prepared || failed.state != preparedRootApplyStatePlanned { + t.Fatalf("failed apply=%+v want unprepared planned", failed) + } + later := group.applyAt(1) + if later == nil || !later.prepared || later.preparedRoot != 42 { + t.Fatalf("later apply=%+v want prepared root 42", later) + } + group.markAbandoned() + var stats preparedRootApplyStats + stats.observeGroup(&group) + if stats.roots != 1 || stats.abandoned != 1 || stats.entries != 2 { + t.Fatalf("stats roots=%d abandoned=%d entries=%d want 1/1/2", stats.roots, stats.abandoned, stats.entries) + } +} + +func TestRecordOrderedRootDeltaBatchGroupApplyResultsCountsZeroReadOnlyPrepare(t *testing.T) { + var phaseStats orderedRootDeltaGroupPublishPhaseStats + + err := recordOrderedRootDeltaBatchGroupApplyResults( + nil, + []uint64{0}, + []orderedRootDeltaBatchGroupApplyResult{{ + idx: 0, + attempted: true, + readOnlyPrepareAttempted: true, + readOnlyPrepareSummary: zipper.ReadOnlyLeafSpanSummary{ + ExactLeafSpans: true, + }, + }}, + nil, + nil, + &phaseStats, + nil, + ) + if err != nil { + t.Fatalf("record apply results: %v", err) + } + if phaseStats.rootApplyCalls != 1 { + t.Fatalf("root apply calls=%d want 1", phaseStats.rootApplyCalls) + } + if phaseStats.rootApplyReadOnlyPrepareCalls != 1 { + t.Fatalf("readonly prepare calls=%d want 1", phaseStats.rootApplyReadOnlyPrepareCalls) + } + if phaseStats.rootApplyReadOnlyPrepareOps != 0 || phaseStats.rootApplyReadOnlyPrepareLeafSpans != 0 || phaseStats.rootApplyReadOnlyPrepareNs != 0 { + t.Fatalf("readonly prepare ops/spans/ns=%d/%d/%d want 0/0/0", phaseStats.rootApplyReadOnlyPrepareOps, phaseStats.rootApplyReadOnlyPrepareLeafSpans, phaseStats.rootApplyReadOnlyPrepareNs) + } + if phaseStats.rootApplyReadOnlyPrepareExactPlans != 1 { + t.Fatalf("readonly prepare exact plans=%d want 1", phaseStats.rootApplyReadOnlyPrepareExactPlans) + } +} + +func TestPreparedRootSetSystemRootSupersedesLatestActiveSystemApply(t *testing.T) { + first := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := first.Set([]byte("sys/a"), []byte("1")); err != nil { + t.Fatalf("first set: %v", err) + } + defer func() { _ = first.Close() }() + second := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := second.Set([]byte("sys/b"), []byte("2")); err != nil { + t.Fatalf("second set: %v", err) + } + defer func() { _ = second.Close() }() + third := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := third.Set([]byte("sys/c"), []byte("3")); err != nil { + t.Fatalf("third set: %v", err) + } + defer func() { _ = third.Close() }() + + group := preparedRootApplyGroup{ + baseSystemRootID: 7, + state: preparedRootApplyStatePlanned, + } + firstIdx := group.setSystemRoot(10, first, false) + group.markPreparedOutput(firstIdx, 100, preparedOutputSnapshot{ID: 1, State: preparedOutputStatePrepared}) + secondIdx := group.setSystemRoot(20, second, false) + group.markPreparedOutput(secondIdx, 200, preparedOutputSnapshot{ID: 2, State: preparedOutputStatePrepared}) + thirdIdx := group.setSystemRoot(30, third, false) + + if firstIdx == secondIdx || secondIdx == thirdIdx || firstIdx == thirdIdx { + t.Fatalf("system indexes should be distinct, got %d/%d/%d", firstIdx, secondIdx, thirdIdx) + } + if got := group.baseSystemRootID; got != 30 { + t.Fatalf("group base system root=%d want latest 30", got) + } + if firstApply := group.applyAt(firstIdx); firstApply == nil || firstApply.state != preparedRootApplyStateAbandoned { + t.Fatalf("first system apply=%+v want abandoned", firstApply) + } else if firstApply.output.State != preparedOutputStateAbandoned { + t.Fatalf("first system output state=%v want abandoned", firstApply.output.State) + } + if secondApply := group.applyAt(secondIdx); secondApply == nil || secondApply.state != preparedRootApplyStateAbandoned { + t.Fatalf("second system apply=%+v want abandoned", secondApply) + } else if secondApply.output.State != preparedOutputStateAbandoned { + t.Fatalf("second system output state=%v want abandoned", secondApply.output.State) + } + latest := group.applyAt(thirdIdx) + if latest == nil { + t.Fatal("missing latest system apply") + } + if latest.prepared { + t.Fatalf("latest system apply is already prepared: %+v", latest) + } + if latest.baseRootID != 30 || latest.state != preparedRootApplyStatePlanned { + t.Fatalf("latest system apply=%+v want base 30 planned", latest) + } +} + +func TestPreparedRootPrepareNsRecordedWithoutPreparedRoots(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + before := db.Stats() + db.observeOrderedRootDeltaGroupPreparedRootApply(123, preparedRootApplyStats{}) + after := db.Stats() + beforePrepareNs := installGuardStatUint(t, before, "treedb.publish.ordered_root_delta_group.prepared_root.prepare_ns_total") + afterPrepareNs := installGuardStatUint(t, after, "treedb.publish.ordered_root_delta_group.prepared_root.prepare_ns_total") + if afterPrepareNs-beforePrepareNs != 123 { + t.Fatalf("prepare ns delta=%d want 123", afterPrepareNs-beforePrepareNs) + } + if got := installGuardStatUint(t, after, "treedb.publish.ordered_root_delta_group.prepared_root.groups_total"); got != installGuardStatUint(t, before, "treedb.publish.ordered_root_delta_group.prepared_root.groups_total") { + t.Fatalf("prepared groups changed: before=%s after=%d", before["treedb.publish.ordered_root_delta_group.prepared_root.groups_total"], got) + } +} + +func TestOrderedRootDeltaBatchGroupPreparedRootMetadataRecordsInstall(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + baseRoot, err := db.PublishOrderedRootIterator(0, mustFrozenSystemMemtable(t, "root/a", "va").NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish base root: %v", err) + } + before := db.State() + if before == nil { + t.Fatal("missing state before ordered root group publish") + } + + deltaTable := mustFrozenSystemMemtable(t, "root/b", "vb") + iter := deltaTable.NewIterator(nil, nil) + delta, err := OrderedRootDeltaBatchFromIterator(iter) + _ = iter.Close() + if err != nil { + t.Fatalf("OrderedRootDeltaBatchFromIterator: %v", err) + } + defer func() { _ = delta.Close() }() + + var captured []preparedRootApplyGroup + db.testPreparedRootApplyHook = func(group preparedRootApplyGroup) { + captured = append(captured, group) + } + newSystemRoot, rootIDs, err := db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: baseRoot, + Delta: delta, + StoragePolicy: OrderedRootStoragePagerLeaves, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return mustFrozenSystemMemtable(t, "sys/collections/users/primary", strconv.FormatUint(rootIDs[0], 10)).NewIterator(nil, nil), nil + }) + db.testPreparedRootApplyHook = nil + if err != nil { + t.Fatalf("publish ordered root group: %v", err) + } + if len(rootIDs) != 1 || rootIDs[0] == 0 { + t.Fatalf("root IDs=%v want one nonzero root", rootIDs) + } + if newSystemRoot == 0 { + t.Fatal("new system root is zero") + } + if len(captured) != 1 { + t.Fatalf("captured groups=%d want 1", len(captured)) + } + + group := captured[0] + if group.state != preparedRootApplyStateInstalled { + t.Fatalf("group state=%v want installed", group.state) + } + if group.baseUserRootID != before.RootPageID { + t.Fatalf("base user root=%d want %d", group.baseUserRootID, before.RootPageID) + } + if group.baseSystemRootID != before.SystemRootPageID { + t.Fatalf("base system root=%d want %d", group.baseSystemRootID, before.SystemRootPageID) + } + if group.applyLen() != 2 { + t.Fatalf("applies=%d want data+system", group.applyLen()) + } + + data := *group.applyAt(0) + if data.identity.kind != preparedRootIdentityData || data.identity.ordinal != 0 { + t.Fatalf("data identity=%#v", data.identity) + } + if data.baseRootID != baseRoot { + t.Fatalf("data base root=%d want %d", data.baseRootID, baseRoot) + } + if data.preparedRoot != rootIDs[0] { + t.Fatalf("data prepared root=%d want %d", data.preparedRoot, rootIDs[0]) + } + if data.outputID == 0 { + t.Fatal("data prepared output ID is zero") + } + if data.storage != OrderedRootStoragePagerLeaves { + t.Fatalf("data storage=%d want pager leaves", data.storage) + } + if data.state != preparedRootApplyStateInstalled { + t.Fatalf("data state=%v want installed", data.state) + } + if data.plan.entries != 1 || data.plan.tombstones != 0 { + t.Fatalf("data plan entries/tombstones=%d/%d want 1/0", data.plan.entries, data.plan.tombstones) + } + if !bytes.Equal(data.plan.firstKey, []byte("root/b")) || !bytes.Equal(data.plan.lastKey, []byte("root/b")) { + t.Fatalf("data key span=%q..%q", data.plan.firstKey, data.plan.lastKey) + } + + system := *group.applyAt(1) + if system.identity.kind != preparedRootIdentitySystem { + t.Fatalf("system identity=%#v", system.identity) + } + if system.baseRootID != before.SystemRootPageID { + t.Fatalf("system base root=%d want %d", system.baseRootID, before.SystemRootPageID) + } + if system.preparedRoot != newSystemRoot { + t.Fatalf("system prepared root=%d want %d", system.preparedRoot, newSystemRoot) + } + if system.outputID == 0 { + t.Fatal("system prepared output ID is zero") + } + if system.outputID == data.outputID { + t.Fatalf("system/data prepared output IDs both %d, want distinct owners", system.outputID) + } + if system.state != preparedRootApplyStateInstalled { + t.Fatalf("system state=%v want installed", system.state) + } + if system.plan.entries == 0 { + t.Fatal("system plan entries=0 want descriptor delta") + } + + stats := db.Stats() + if got := installGuardStatUint(t, stats, "treedb.publish.ordered_root_delta_group.prepared_root.groups_total"); got != 1 { + t.Fatalf("prepared groups=%d want 1", got) + } + if got := installGuardStatUint(t, stats, "treedb.publish.ordered_root_delta_group.prepared_root.roots_total"); got != 2 { + t.Fatalf("prepared roots=%d want 2", got) + } + if got := installGuardStatUint(t, stats, "treedb.publish.ordered_root_delta_group.prepared_root.installed_total"); got != 2 { + t.Fatalf("prepared installed=%d want 2", got) + } + if got := installGuardStatUint(t, stats, "treedb.publish.ordered_root_delta_group.prepared_root.abandoned_total"); got != 0 { + t.Fatalf("prepared abandoned=%d want 0", got) + } +} + +func TestOrderedRootDeltaBatchGroupPreparedRootMetadataTracksLeafLogOutput(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + db.SetLeafPageLog(&preparedOutputTestLeafLog{}) + + deltaTable := mustFrozenSystemMemtable(t, "root/a", "va", "root/b", "vb") + iter := deltaTable.NewIterator(nil, nil) + delta, err := OrderedRootDeltaBatchFromIterator(iter) + _ = iter.Close() + if err != nil { + t.Fatalf("OrderedRootDeltaBatchFromIterator: %v", err) + } + defer func() { _ = delta.Close() }() + + var captured []preparedRootApplyGroup + db.testPreparedRootApplyHook = func(group preparedRootApplyGroup) { + captured = append(captured, group) + } + _, rootIDs, err := db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: 0, + Delta: delta, + StoragePolicy: OrderedRootStorageValueLogLeaves, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return mustFrozenSystemMemtable(t, "sys/collections/users/primary", strconv.FormatUint(rootIDs[0], 10)).NewIterator(nil, nil), nil + }) + db.testPreparedRootApplyHook = nil + if err != nil { + t.Fatalf("publish ordered root group: %v", err) + } + if len(rootIDs) != 1 || rootIDs[0] == 0 { + t.Fatalf("root IDs=%v want one nonzero root", rootIDs) + } + if len(captured) != 1 { + t.Fatalf("captured groups=%d want 1", len(captured)) + } + + data := captured[0].applyAt(0) + if data == nil { + t.Fatal("missing data prepared root apply") + } + if data.outputID == 0 || data.output.ID != data.outputID { + t.Fatalf("output ID=%d snapshot ID=%d", data.outputID, data.output.ID) + } + if data.output.State != preparedOutputStateInstalled { + t.Fatalf("output state=%v want installed", data.output.State) + } + if len(data.output.LeafLogPtrs) == 0 { + t.Fatalf("data prepared output did not record leaf-log pointers: %+v", data.output) + } + if len(data.output.Pages) == 0 { + t.Fatalf("data prepared output did not record root/internal pages: %+v", data.output) + } + if data.outputPages != uint64(len(data.output.Pages)) { + t.Fatalf("data output page count=%d want %d", data.outputPages, len(data.output.Pages)) + } + if data.outputLeafLogPtrs != uint64(len(data.output.LeafLogPtrs)) { + t.Fatalf("data output leaf-log count=%d want %d", data.outputLeafLogPtrs, len(data.output.LeafLogPtrs)) + } + + system := captured[0].applyAt(1) + if system == nil { + t.Fatal("missing system prepared root apply") + } + if len(system.output.LeafLogPtrs) != 0 { + t.Fatalf("system prepared output recorded leaf-log pointers: %+v", system.output.LeafLogPtrs) + } + + stats := db.Stats() + wantPages := data.outputPages + system.outputPages + if got := installGuardStatUint(t, stats, "treedb.publish.ordered_root_delta_group.prepared_root.output_pages_total"); got != wantPages { + t.Fatalf("output pages total=%d want %d", got, wantPages) + } + if got := installGuardStatUint(t, stats, "treedb.publish.ordered_root_delta_group.prepared_root.installed_output_pages_total"); got != wantPages { + t.Fatalf("installed output pages total=%d want %d", got, wantPages) + } + if got := installGuardStatUint(t, stats, "treedb.publish.ordered_root_delta_group.prepared_root.abandoned_output_pages_total"); got != 0 { + t.Fatalf("abandoned output pages total=%d want 0", got) + } + if got := installGuardStatUint(t, stats, "treedb.publish.ordered_root_delta_group.prepared_root.output_leaf_log_ptrs_total"); got != data.outputLeafLogPtrs { + t.Fatalf("output leaf-log ptrs total=%d want %d", got, data.outputLeafLogPtrs) + } + if got := installGuardStatUint(t, stats, "treedb.publish.ordered_root_delta_group.prepared_root.installed_output_leaf_log_ptrs_total"); got != data.outputLeafLogPtrs { + t.Fatalf("installed output leaf-log ptrs total=%d want %d", got, data.outputLeafLogPtrs) + } + if got := installGuardStatUint(t, stats, "treedb.publish.ordered_root_delta_group.prepared_root.abandoned_output_leaf_log_ptrs_total"); got != 0 { + t.Fatalf("abandoned output leaf-log ptrs total=%d want 0", got) + } +} + +func TestOrderedRootDeltaBatchGroupPreparedRootOutputStatsCountSharedTrackerOnce(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + leafLog := &preparedOutputTestLeafLog{} + db.SetLeafPageLog(leafLog) + + deltaTableA := mustFrozenSystemMemtable(t, "root/a", "va") + iterA := deltaTableA.NewIterator(nil, nil) + deltaA, err := OrderedRootDeltaBatchFromIterator(iterA) + _ = iterA.Close() + if err != nil { + t.Fatalf("OrderedRootDeltaBatchFromIterator A: %v", err) + } + defer func() { _ = deltaA.Close() }() + + deltaTableB := mustFrozenSystemMemtable(t, "root/b", "vb") + iterB := deltaTableB.NewIterator(nil, nil) + deltaB, err := OrderedRootDeltaBatchFromIterator(iterB) + _ = iterB.Close() + if err != nil { + t.Fatalf("OrderedRootDeltaBatchFromIterator B: %v", err) + } + defer func() { _ = deltaB.Close() }() + + beforeStats := db.Stats() + _, rootIDs, err := db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{ + { + BaseRoot: 0, + Delta: deltaA, + StoragePolicy: OrderedRootStorageValueLogLeaves, + }, + { + BaseRoot: 0, + Delta: deltaB, + StoragePolicy: OrderedRootStorageValueLogLeaves, + }, + }, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + if len(rootIDs) != 2 || rootIDs[0] == 0 || rootIDs[1] == 0 { + return nil, errors.New("unexpected root IDs") + } + return mustFrozenSystemMemtable(t, + "sys/collections/users/primary", + strconv.FormatUint(rootIDs[0], 10), + "sys/collections/users/secondary", + strconv.FormatUint(rootIDs[1], 10), + ).NewIterator(nil, nil), nil + }) + if err != nil { + t.Fatalf("publish ordered root group: %v", err) + } + if len(rootIDs) != 2 || rootIDs[0] == 0 || rootIDs[1] == 0 { + t.Fatalf("root IDs=%v want two nonzero roots", rootIDs) + } + if len(leafLog.ptrs) == 0 { + t.Fatal("publish did not write leaf-log output") + } + + afterStats := db.Stats() + statDelta := func(name string) uint64 { + after := installGuardStatUint(t, afterStats, name) + before := installGuardStatUint(t, beforeStats, name) + if after < before { + t.Fatalf("%s decreased: before=%d after=%d", name, before, after) + } + return after - before + } + pageDelta := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.output_pages_total") + if pageDelta == 0 { + t.Fatal("output pages delta=0 want prepared page output") + } + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.installed_output_pages_total"); got != pageDelta { + t.Fatalf("installed output pages delta=%d want output pages delta %d", got, pageDelta) + } + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.abandoned_output_pages_total"); got != 0 { + t.Fatalf("abandoned output pages delta=%d want 0", got) + } + wantLeafs := uint64(len(leafLog.ptrs)) + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.output_leaf_log_ptrs_total"); got != wantLeafs { + t.Fatalf("output leaf-log ptrs delta=%d want exactly written ptrs %d", got, wantLeafs) + } + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.installed_output_leaf_log_ptrs_total"); got != wantLeafs { + t.Fatalf("installed output leaf-log ptrs delta=%d want exactly written ptrs %d", got, wantLeafs) + } + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.abandoned_output_leaf_log_ptrs_total"); got != 0 { + t.Fatalf("abandoned output leaf-log ptrs delta=%d want 0", got) + } +} + +func TestOrderedRootDeltaBatchGroupPreparedOutputCountsIgnoreFailedRoots(t *testing.T) { + results := []orderedRootDeltaBatchGroupApplyResult{ + {attempted: true, outputPages: 2, outputLeafLogPtrs: 3}, + {attempted: true, err: errors.New("root apply failed"), outputPages: 100, outputLeafLogPtrs: 200}, + {outputPages: 1000, outputLeafLogPtrs: 2000}, + {attempted: true, outputPages: 5, outputLeafLogPtrs: 7}, + } + pages, leafLogPtrs := orderedRootDeltaBatchGroupPreparedOutputCounts(results) + if pages != 7 || leafLogPtrs != 10 { + t.Fatalf("prepared output counts pages/leaf-log=%d/%d want 7/10", pages, leafLogPtrs) + } +} + +func TestOrderedRootDeltaBatchGroupPreparedRootMetadataCapturesFinalSharedOutput(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + leafLog := &preparedOutputTestLeafLog{} + db.SetLeafPageLog(leafLog) + + deltaTableA := mustFrozenSystemMemtable(t, "root/a", "va") + iterA := deltaTableA.NewIterator(nil, nil) + deltaA, err := OrderedRootDeltaBatchFromIterator(iterA) + _ = iterA.Close() + if err != nil { + t.Fatalf("OrderedRootDeltaBatchFromIterator A: %v", err) + } + defer func() { _ = deltaA.Close() }() + + deltaTableB := mustFrozenSystemMemtable(t, "root/b", "vb") + iterB := deltaTableB.NewIterator(nil, nil) + deltaB, err := OrderedRootDeltaBatchFromIterator(iterB) + _ = iterB.Close() + if err != nil { + t.Fatalf("OrderedRootDeltaBatchFromIterator B: %v", err) + } + defer func() { _ = deltaB.Close() }() + + var captured []preparedRootApplyGroup + db.testPreparedRootApplyHook = func(group preparedRootApplyGroup) { + captured = append(captured, group) + } + _, rootIDs, err := db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{ + { + BaseRoot: 0, + Delta: deltaA, + StoragePolicy: OrderedRootStorageValueLogLeaves, + }, + { + BaseRoot: 0, + Delta: deltaB, + StoragePolicy: OrderedRootStorageValueLogLeaves, + }, + }, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + if len(rootIDs) != 2 || rootIDs[0] == 0 || rootIDs[1] == 0 { + return nil, errors.New("unexpected root IDs") + } + return mustFrozenSystemMemtable(t, + "sys/collections/users/primary", + strconv.FormatUint(rootIDs[0], 10), + "sys/collections/users/secondary", + strconv.FormatUint(rootIDs[1], 10), + ).NewIterator(nil, nil), nil + }) + db.testPreparedRootApplyHook = nil + if err != nil { + t.Fatalf("publish ordered root group: %v", err) + } + if len(rootIDs) != 2 || rootIDs[0] == 0 || rootIDs[1] == 0 { + t.Fatalf("root IDs=%v want two nonzero roots", rootIDs) + } + if len(captured) != 1 { + t.Fatalf("captured groups=%d want 1", len(captured)) + } + if len(leafLog.ptrs) == 0 { + t.Fatal("publish did not write leaf-log output") + } + + for idx := 0; idx < 2; idx++ { + data := captured[0].applyAt(idx) + if data == nil { + t.Fatalf("missing data prepared root apply %d", idx) + } + if data.outputID == 0 || data.output.ID != data.outputID { + t.Fatalf("data %d output ID=%d snapshot ID=%d", idx, data.outputID, data.output.ID) + } + if data.output.State != preparedOutputStateInstalled { + t.Fatalf("data %d output state=%v want installed", idx, data.output.State) + } + if got, want := len(data.output.LeafLogPtrs), len(leafLog.ptrs); got != want { + t.Fatalf("data %d leaf-log ptrs=%d want final shared output %d", idx, got, want) + } + } + system := captured[0].applyAt(2) + if system == nil || system.identity.kind != preparedRootIdentitySystem { + t.Fatalf("missing system prepared root apply: %+v", system) + } + if len(system.output.LeafLogPtrs) != 0 { + t.Fatalf("system prepared output recorded leaf-log pointers: %+v", system.output.LeafLogPtrs) + } +} + +func TestOrderedRootDeltaBatchGroupPreparedRootMetadataRecordsOptimisticBuilderError(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + baseRoot, err := db.PublishOrderedRootIterator(0, mustFrozenSystemMemtable(t, "root/a", "va").NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish base root: %v", err) + } + + seedDeltaTable := mustFrozenSystemMemtable(t, "root/b", "vb") + seedIter := seedDeltaTable.NewIterator(nil, nil) + seedDelta, err := OrderedRootDeltaBatchFromIterator(seedIter) + _ = seedIter.Close() + if err != nil { + t.Fatalf("seed OrderedRootDeltaBatchFromIterator: %v", err) + } + defer func() { _ = seedDelta.Close() }() + _, seedRootIDs, err := db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: baseRoot, + Delta: seedDelta, + StoragePolicy: OrderedRootStoragePagerLeaves, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return mustFrozenSystemMemtable(t, "sys/collections/users/primary", strconv.FormatUint(rootIDs[0], 10)).NewIterator(nil, nil), nil + }) + if err != nil { + t.Fatalf("seed publish ordered root group: %v", err) + } + if len(seedRootIDs) != 1 || seedRootIDs[0] == 0 { + t.Fatalf("seed root IDs=%v want one nonzero root", seedRootIDs) + } + beforeStats := db.Stats() + + deltaTable := mustFrozenSystemMemtable(t, "root/c", "vc") + iter := deltaTable.NewIterator(nil, nil) + delta, err := OrderedRootDeltaBatchFromIterator(iter) + _ = iter.Close() + if err != nil { + t.Fatalf("OrderedRootDeltaBatchFromIterator: %v", err) + } + defer func() { _ = delta.Close() }() + + sentinel := errors.New("injected system builder failure") + var captured []preparedRootApplyGroup + db.testPreparedRootApplyHook = func(group preparedRootApplyGroup) { + captured = append(captured, group) + } + _, _, err = db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: seedRootIDs[0], + Delta: delta, + StoragePolicy: OrderedRootStoragePagerLeaves, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return nil, sentinel + }) + db.testPreparedRootApplyHook = nil + if !errors.Is(err, sentinel) { + t.Fatalf("publish err=%v want %v", err, sentinel) + } + if len(captured) != 1 { + t.Fatalf("captured groups=%d want 1", len(captured)) + } + group := captured[0] + if group.state != preparedRootApplyStateAbandoned { + t.Fatalf("group state=%v want abandoned", group.state) + } + if group.applyLen() != 1 { + t.Fatalf("applies=%d want data-only group", group.applyLen()) + } + data := group.applyAt(0) + if data == nil { + t.Fatal("missing data apply") + } + if data.identity.kind != preparedRootIdentityData || data.identity.ordinal != 0 { + t.Fatalf("data identity=%#v", data.identity) + } + if data.state != preparedRootApplyStateAbandoned { + t.Fatalf("data state=%v want abandoned", data.state) + } + if data.preparedRoot == 0 { + t.Fatal("data prepared root is zero") + } + if data.plan.entries != 1 { + t.Fatalf("data plan entries=%d want 1", data.plan.entries) + } + + afterStats := db.Stats() + statDelta := func(name string) uint64 { + after := installGuardStatUint(t, afterStats, name) + before := installGuardStatUint(t, beforeStats, name) + if after < before { + t.Fatalf("%s decreased: before=%d after=%d", name, before, after) + } + return after - before + } + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.groups_total"); got != 1 { + t.Fatalf("prepared groups delta=%d want 1", got) + } + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.roots_total"); got != 1 { + t.Fatalf("prepared roots delta=%d want 1", got) + } + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.entries_total"); got != 1 { + t.Fatalf("prepared entries delta=%d want 1", got) + } + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.installed_total"); got != 0 { + t.Fatalf("prepared installed delta=%d want 0", got) + } + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.abandoned_total"); got != 1 { + t.Fatalf("prepared abandoned delta=%d want 1", got) + } + if got := statDelta("treedb.publish.ordered_root_delta_group.calls_total"); got != 0 { + t.Fatalf("ordered root publish calls delta=%d want 0 before write-lock publish", got) + } + if got := statDelta("treedb.publish.ordered_root_delta_group.errors_total"); got != 0 { + t.Fatalf("ordered root publish errors delta=%d want 0 before write-lock publish", got) + } +} + +func TestOrderedRootDeltaBatchGroupPreparedRootMetadataRecordsOptimisticFallbackToSerialized(t *testing.T) { + db, err := Open(Options{Dir: t.TempDir()}) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + baseRoot, err := db.PublishOrderedRootIterator(0, mustFrozenSystemMemtable(t, "root/a", "va").NewIterator(nil, nil)) + if err != nil { + t.Fatalf("publish base root: %v", err) + } + seedDeltaTable := mustFrozenSystemMemtable(t, "root/b", "vb") + seedIter := seedDeltaTable.NewIterator(nil, nil) + seedDelta, err := OrderedRootDeltaBatchFromIterator(seedIter) + _ = seedIter.Close() + if err != nil { + t.Fatalf("seed OrderedRootDeltaBatchFromIterator: %v", err) + } + defer func() { _ = seedDelta.Close() }() + _, seedRootIDs, err := db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: baseRoot, + Delta: seedDelta, + StoragePolicy: OrderedRootStoragePagerLeaves, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + return mustFrozenSystemMemtable(t, "sys/collections/users/primary", strconv.FormatUint(rootIDs[0], 10)).NewIterator(nil, nil), nil + }) + if err != nil { + t.Fatalf("seed publish ordered root group: %v", err) + } + if len(seedRootIDs) != 1 || seedRootIDs[0] == 0 { + t.Fatalf("seed root IDs=%v want one nonzero root", seedRootIDs) + } + + beforeStats := db.Stats() + targetDeltaTable := mustFrozenSystemMemtable(t, "root/c", "vc") + targetIter := targetDeltaTable.NewIterator(nil, nil) + targetDelta, err := OrderedRootDeltaBatchFromIterator(targetIter) + _ = targetIter.Close() + if err != nil { + t.Fatalf("target OrderedRootDeltaBatchFromIterator: %v", err) + } + defer func() { _ = targetDelta.Close() }() + + builderCalls := 0 + systemRoot, rootIDs, err := db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: seedRootIDs[0], + Delta: targetDelta, + StoragePolicy: OrderedRootStoragePagerLeaves, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + if len(rootIDs) != 1 || rootIDs[0] == 0 { + return nil, errors.New("unexpected target root IDs") + } + builderCalls++ + if builderCalls <= orderedRootOptimisticSystemDeltaRebaseMaxAttempts { + if err := publishPreparedRootMetadataSystemRootChange(t, db, builderCalls); err != nil { + return nil, err + } + } + return mustFrozenSystemMemtable(t, + "sys/collections/users/primary", + strconv.FormatUint(rootIDs[0], 10), + ).NewIterator(nil, nil), nil + }) + if err != nil { + t.Fatalf("publish ordered root group after optimistic fallback: %v", err) + } + if systemRoot == 0 || len(rootIDs) != 1 || rootIDs[0] == 0 { + t.Fatalf("systemRoot=%d rootIDs=%v want nonzero roots", systemRoot, rootIDs) + } + if builderCalls != orderedRootOptimisticSystemDeltaRebaseMaxAttempts+1 { + t.Fatalf("builder calls=%d want %d", builderCalls, orderedRootOptimisticSystemDeltaRebaseMaxAttempts+1) + } + + afterStats := db.Stats() + statDelta := func(name string) uint64 { + after := installGuardStatUint(t, afterStats, name) + before := installGuardStatUint(t, beforeStats, name) + if after < before { + t.Fatalf("%s decreased: before=%d after=%d", name, before, after) + } + return after - before + } + + attempts := uint64(orderedRootOptimisticSystemDeltaRebaseMaxAttempts) + wantCalls := attempts + 1 + if got := statDelta("treedb.publish.ordered_root_delta_group.calls_total"); got != wantCalls { + t.Fatalf("ordered root publish calls delta=%d want %d", got, wantCalls) + } + wantPreparedGroups := wantCalls + 1 + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.groups_total"); got != wantPreparedGroups { + t.Fatalf("prepared groups delta=%d want %d", got, wantPreparedGroups) + } + wantAbandonedRoots := attempts + 1 + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.abandoned_total"); got != wantAbandonedRoots { + t.Fatalf("prepared abandoned delta=%d want %d", got, wantAbandonedRoots) + } + wantInstalledRoots := attempts*2 + 2 + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.installed_total"); got != wantInstalledRoots { + t.Fatalf("prepared installed delta=%d want %d", got, wantInstalledRoots) + } + wantPreparedRoots := wantAbandonedRoots + wantInstalledRoots + if got := statDelta("treedb.publish.ordered_root_delta_group.prepared_root.roots_total"); got != wantPreparedRoots { + t.Fatalf("prepared roots delta=%d want %d", got, wantPreparedRoots) + } + + snap := db.AcquireSnapshot() + if snap == nil { + t.Fatal("expected snapshot") + } + defer func() { _ = snap.Close() }() + entry, err := snap.GetEntryAtRoot(rootIDs[0], []byte("root/c")) + if err != nil { + t.Fatalf("GetEntryAtRoot(root/c): %v", err) + } + if got := string(entry.Value); got != "vc" { + t.Fatalf("root/c=%q want vc", got) + } +} + +func publishPreparedRootMetadataSystemRootChange(t *testing.T, db *DB, ordinal int) error { + t.Helper() + deltaTable := mustFrozenSystemMemtable(t, + "root/concurrent/"+strconv.Itoa(ordinal), + "value-"+strconv.Itoa(ordinal), + ) + iter := deltaTable.NewIterator(nil, nil) + delta, err := OrderedRootDeltaBatchFromIterator(iter) + _ = iter.Close() + if err != nil { + return err + } + defer func() { _ = delta.Close() }() + _, rootIDs, err := db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder([]OrderedRootDeltaBatchPublishInput{{ + BaseRoot: 0, + Delta: delta, + StoragePolicy: OrderedRootStoragePagerLeaves, + }}, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + if len(rootIDs) != 1 || rootIDs[0] == 0 { + return nil, errors.New("unexpected concurrent root IDs") + } + return mustFrozenSystemMemtable(t, + "sys/concurrent/"+strconv.Itoa(ordinal), + strconv.FormatUint(rootIDs[0], 10), + ).NewIterator(nil, nil), nil + }) + if err != nil { + return err + } + if len(rootIDs) != 1 || rootIDs[0] == 0 { + return errors.New("concurrent publish returned invalid root IDs") + } + return nil +} diff --git a/TreeDB/db/publish_watermark_metrics.go b/TreeDB/db/publish_watermark_metrics.go index ecbd383776..9f1887aaac 100644 --- a/TreeDB/db/publish_watermark_metrics.go +++ b/TreeDB/db/publish_watermark_metrics.go @@ -160,6 +160,10 @@ type orderedRootDeltaGroupPublishStats struct { rootApplyLeafLogRecordHintBytesRead uint64 rootApplyLeafMerges uint64 rootApplyInternalMerges uint64 + rootApplyInternalParallelMerges uint64 + rootApplyInternalParallelChildren uint64 + rootApplyInternalParallelWorkers uint64 + rootApplyInternalParallelOps uint64 rootApplyLeafPagesWritten uint64 rootApplyPagerLeafPagesWritten uint64 rootApplyLeafLogPagesWritten uint64 @@ -174,11 +178,38 @@ type orderedRootDeltaGroupPublishStats struct { rootApplyInternalLeafLogRefs uint64 rootApplyInternalLeafLogRefCopies uint64 rootApplyRootSplitLevels uint64 + rootApplyReadOnlyPrepareNs uint64 + rootApplyReadOnlyPrepareCalls uint64 + rootApplyReadOnlyPrepareOps uint64 + rootApplyReadOnlyPrepareLeafSpans uint64 + rootApplyReadOnlyPrepareWorker orderedRootDeltaGroupReadOnlyPrepareWorkerStats + rootApplyReadOnlyPrepareExactPlans uint64 + rootApplyReadOnlyPrepareMaintenance uint64 + rootApplyReadOnlyPrepareColdBuilds uint64 systemBuildNs uint64 systemApplyNs uint64 systemApplyCalls uint64 systemApplyOps uint64 systemApplyNodeLoads uint64 + installGuardNs uint64 + installGuardCalls uint64 + installGuardFailures uint64 + preparedRootPrepareNs uint64 + preparedRootGroups uint64 + preparedRootRoots uint64 + preparedRootEntries uint64 + preparedRootTombstones uint64 + preparedRootKeyBytes uint64 + preparedRootValueBytes uint64 + preparedRootPointerValues uint64 + preparedRootInstalled uint64 + preparedRootAbandoned uint64 + preparedRootOutputPages uint64 + preparedRootOutputLeafLogPtrs uint64 + preparedRootInstalledPages uint64 + preparedRootInstalledLeafLogPtrs uint64 + preparedRootAbandonedPages uint64 + preparedRootAbandonedLeafLogPtrs uint64 finalizeNs uint64 finalizeCalls uint64 latencyP99 time.Duration @@ -188,18 +219,39 @@ type orderedRootDeltaGroupPublishStats struct { } type orderedRootDeltaGroupPublishPhaseStats struct { - preflightNs uint64 - rootApplyNs uint64 - rootApplyCalls uint64 - rootApplyParallelGroups uint64 - rootApplyParallelRoots uint64 - rootApplyMetrics orderedRootDeltaGroupZipperStats - systemBuildNs uint64 - systemApplyNs uint64 - systemApplyCalls uint64 - systemApplyMetrics orderedRootDeltaGroupZipperStats - finalizeNs uint64 - finalizeCalls uint64 + preflightNs uint64 + rootApplyNs uint64 + rootApplyCalls uint64 + rootApplyParallelGroups uint64 + rootApplyParallelRoots uint64 + rootApplyMetrics orderedRootDeltaGroupZipperStats + rootApplyReadOnlyPrepareNs uint64 + rootApplyReadOnlyPrepareCalls uint64 + rootApplyReadOnlyPrepareOps uint64 + rootApplyReadOnlyPrepareLeafSpans uint64 + rootApplyReadOnlyPrepareWorker orderedRootDeltaGroupReadOnlyPrepareWorkerStats + rootApplyReadOnlyPrepareExactPlans uint64 + rootApplyReadOnlyPrepareMaintenance uint64 + rootApplyReadOnlyPrepareColdBuilds uint64 + systemBuildNs uint64 + systemApplyNs uint64 + systemApplyCalls uint64 + systemApplyMetrics orderedRootDeltaGroupZipperStats + installGuardNs uint64 + installGuardCalls uint64 + installGuardFailures uint64 + preparedRootPrepareNs uint64 + preparedRootStats preparedRootApplyStats + finalizeNs uint64 + finalizeCalls uint64 +} + +type orderedRootDeltaGroupReadOnlyPrepareWorkerStats struct { + targets uint64 + ranges uint64 + minOps uint64 + maxOps uint64 + singleSpan uint64 } type orderedRootDeltaGroupZipperStats struct { @@ -216,6 +268,10 @@ type orderedRootDeltaGroupZipperStats struct { ZipperLeafLogRecordHintBytesRead int ZipperLeafMerges int ZipperInternalMerges int + ZipperInternalParallelMerges int + ZipperInternalParallelChildren int + ZipperInternalParallelWorkers int + ZipperInternalParallelOps int ZipperLeafPagesWritten int ZipperPagerLeafPagesWritten int ZipperLeafLogPagesWritten int @@ -249,6 +305,10 @@ func (dst *orderedRootDeltaGroupZipperStats) add(src adaptive.Metrics) { dst.ZipperLeafLogRecordHintBytesRead += src.ZipperLeafLogRecordHintBytesRead dst.ZipperLeafMerges += src.ZipperLeafMerges dst.ZipperInternalMerges += src.ZipperInternalMerges + dst.ZipperInternalParallelMerges += src.ZipperInternalParallelMerges + dst.ZipperInternalParallelChildren += src.ZipperInternalParallelChildren + dst.ZipperInternalParallelWorkers += src.ZipperInternalParallelWorkers + dst.ZipperInternalParallelOps += src.ZipperInternalParallelOps dst.ZipperLeafPagesWritten += src.ZipperLeafPagesWritten dst.ZipperPagerLeafPagesWritten += src.ZipperPagerLeafPagesWritten dst.ZipperLeafLogPagesWritten += src.ZipperLeafLogPagesWritten @@ -272,7 +332,7 @@ func orderedRootDeltaGroupMetricUint(v int) uint64 { return uint64(v) } -func orderedRootDeltaGroupPhaseDurationNs(start time.Time) uint64 { +func elapsedDurationNs(start time.Time) uint64 { elapsed := time.Since(start) if elapsed <= 0 { return 0 @@ -280,6 +340,10 @@ func orderedRootDeltaGroupPhaseDurationNs(start time.Time) uint64 { return uint64(elapsed.Nanoseconds()) } +func orderedRootDeltaGroupPhaseDurationNs(start time.Time) uint64 { + return elapsedDurationNs(start) +} + func (db *DB) observeOrderedRootDeltaGroupPublish(wait, hold time.Duration, roots int, phases orderedRootDeltaGroupPublishPhaseStats, err error) { if db == nil { return @@ -328,6 +392,10 @@ func (db *DB) observeOrderedRootDeltaGroupPublish(wait, hold time.Duration, root db.orderedRootDeltaGroupRootApplyLeafLogRecordHintBytesRead.Add(orderedRootDeltaGroupMetricUint(phases.rootApplyMetrics.ZipperLeafLogRecordHintBytesRead)) db.orderedRootDeltaGroupRootApplyLeafMerges.Add(orderedRootDeltaGroupMetricUint(phases.rootApplyMetrics.ZipperLeafMerges)) db.orderedRootDeltaGroupRootApplyInternalMerges.Add(orderedRootDeltaGroupMetricUint(phases.rootApplyMetrics.ZipperInternalMerges)) + db.orderedRootDeltaGroupRootApplyInternalParallelMerges.Add(orderedRootDeltaGroupMetricUint(phases.rootApplyMetrics.ZipperInternalParallelMerges)) + db.orderedRootDeltaGroupRootApplyInternalParallelChildren.Add(orderedRootDeltaGroupMetricUint(phases.rootApplyMetrics.ZipperInternalParallelChildren)) + db.orderedRootDeltaGroupRootApplyInternalParallelWorkers.Add(orderedRootDeltaGroupMetricUint(phases.rootApplyMetrics.ZipperInternalParallelWorkers)) + db.orderedRootDeltaGroupRootApplyInternalParallelOps.Add(orderedRootDeltaGroupMetricUint(phases.rootApplyMetrics.ZipperInternalParallelOps)) db.orderedRootDeltaGroupRootApplyLeafPagesWritten.Add(orderedRootDeltaGroupMetricUint(phases.rootApplyMetrics.ZipperLeafPagesWritten)) db.orderedRootDeltaGroupRootApplyPagerLeafPagesWritten.Add(orderedRootDeltaGroupMetricUint(phases.rootApplyMetrics.ZipperPagerLeafPagesWritten)) db.orderedRootDeltaGroupRootApplyLeafLogPagesWritten.Add(orderedRootDeltaGroupMetricUint(phases.rootApplyMetrics.ZipperLeafLogPagesWritten)) @@ -342,11 +410,27 @@ func (db *DB) observeOrderedRootDeltaGroupPublish(wait, hold time.Duration, root db.orderedRootDeltaGroupRootApplyInternalLeafLogRefs.Add(orderedRootDeltaGroupMetricUint(phases.rootApplyMetrics.ZipperInternalLeafLogRefs)) db.orderedRootDeltaGroupRootApplyInternalLeafLogRefCopies.Add(orderedRootDeltaGroupMetricUint(phases.rootApplyMetrics.ZipperInternalLeafLogRefCopies)) db.orderedRootDeltaGroupRootApplyRootSplitLevels.Add(orderedRootDeltaGroupMetricUint(phases.rootApplyMetrics.ZipperRootSplitLevels)) + db.orderedRootDeltaGroupRootApplyReadOnlyPrepareNs.Add(phases.rootApplyReadOnlyPrepareNs) + db.orderedRootDeltaGroupRootApplyReadOnlyPrepareCalls.Add(phases.rootApplyReadOnlyPrepareCalls) + db.orderedRootDeltaGroupRootApplyReadOnlyPrepareOps.Add(phases.rootApplyReadOnlyPrepareOps) + db.orderedRootDeltaGroupRootApplyReadOnlyPrepareLeafSpans.Add(phases.rootApplyReadOnlyPrepareLeafSpans) + db.orderedRootDeltaGroupRootApplyReadOnlyPrepareWorker.targets.Add(phases.rootApplyReadOnlyPrepareWorker.targets) + db.orderedRootDeltaGroupRootApplyReadOnlyPrepareWorker.ranges.Add(phases.rootApplyReadOnlyPrepareWorker.ranges) + db.orderedRootDeltaGroupRootApplyReadOnlyPrepareWorker.minOps.Add(phases.rootApplyReadOnlyPrepareWorker.minOps) + db.orderedRootDeltaGroupRootApplyReadOnlyPrepareWorker.maxOps.Add(phases.rootApplyReadOnlyPrepareWorker.maxOps) + db.orderedRootDeltaGroupRootApplyReadOnlyPrepareWorker.singleSpan.Add(phases.rootApplyReadOnlyPrepareWorker.singleSpan) + db.orderedRootDeltaGroupRootApplyReadOnlyPrepareExactPlans.Add(phases.rootApplyReadOnlyPrepareExactPlans) + db.orderedRootDeltaGroupRootApplyReadOnlyPrepareMaintenance.Add(phases.rootApplyReadOnlyPrepareMaintenance) + db.orderedRootDeltaGroupRootApplyReadOnlyPrepareColdBuilds.Add(phases.rootApplyReadOnlyPrepareColdBuilds) db.orderedRootDeltaGroupSystemBuildNs.Add(phases.systemBuildNs) db.orderedRootDeltaGroupSystemApplyNs.Add(phases.systemApplyNs) db.orderedRootDeltaGroupSystemApplyCalls.Add(phases.systemApplyCalls) db.orderedRootDeltaGroupSystemApplyOps.Add(orderedRootDeltaGroupMetricUint(phases.systemApplyMetrics.ZipperApplyOps)) db.orderedRootDeltaGroupSystemApplyNodeLoads.Add(orderedRootDeltaGroupMetricUint(phases.systemApplyMetrics.ZipperNodeLoads)) + db.orderedRootDeltaGroupInstallGuardNs.Add(phases.installGuardNs) + db.orderedRootDeltaGroupInstallGuardCalls.Add(phases.installGuardCalls) + db.orderedRootDeltaGroupInstallGuardFailures.Add(phases.installGuardFailures) + db.observeOrderedRootDeltaGroupPreparedRootApply(phases.preparedRootPrepareNs, phases.preparedRootStats) db.orderedRootDeltaGroupFinalizeNs.Add(phases.finalizeNs) db.orderedRootDeltaGroupFinalizeCalls.Add(phases.finalizeCalls) for { @@ -359,6 +443,33 @@ func (db *DB) observeOrderedRootDeltaGroupPublish(wait, hold time.Duration, root db.orderedRootDeltaGroupLatencyBuckets[bucket].Add(1) } +func (db *DB) observeOrderedRootDeltaGroupPreparedRootApply(prepareNs uint64, stats preparedRootApplyStats) { + if db == nil { + return + } + if prepareNs > 0 { + db.orderedRootDeltaGroupPreparedRootPrepareNs.Add(prepareNs) + } + if stats.groups == 0 || stats.roots == 0 { + return + } + db.orderedRootDeltaGroupPreparedRootGroups.Add(stats.groups) + db.orderedRootDeltaGroupPreparedRootRoots.Add(stats.roots) + db.orderedRootDeltaGroupPreparedRootEntries.Add(stats.entries) + db.orderedRootDeltaGroupPreparedRootTombstones.Add(stats.tombstones) + db.orderedRootDeltaGroupPreparedRootKeyBytes.Add(stats.keyBytes) + db.orderedRootDeltaGroupPreparedRootValueBytes.Add(stats.valueBytes) + db.orderedRootDeltaGroupPreparedRootPointerValues.Add(stats.pointerValues) + db.orderedRootDeltaGroupPreparedRootInstalled.Add(stats.installed) + db.orderedRootDeltaGroupPreparedRootAbandoned.Add(stats.abandoned) + db.orderedRootDeltaGroupPreparedRootOutputPages.Add(stats.outputPages) + db.orderedRootDeltaGroupPreparedRootOutputLeafLogPtrs.Add(stats.outputLeafLogPtrs) + db.orderedRootDeltaGroupPreparedRootInstalledPages.Add(stats.installedPages) + db.orderedRootDeltaGroupPreparedRootInstalledLeafLogPtrs.Add(stats.installedLeafLogPtrs) + db.orderedRootDeltaGroupPreparedRootAbandonedPages.Add(stats.abandonedPages) + db.orderedRootDeltaGroupPreparedRootAbandonedLeafLogPtrs.Add(stats.abandonedLeafLogPtrs) +} + func (db *DB) orderedRootDeltaGroupPublishStats() orderedRootDeltaGroupPublishStats { if db == nil { return orderedRootDeltaGroupPublishStats{} @@ -392,6 +503,10 @@ func (db *DB) orderedRootDeltaGroupPublishStats() orderedRootDeltaGroupPublishSt rootApplyLeafLogRecordHintBytesRead: db.orderedRootDeltaGroupRootApplyLeafLogRecordHintBytesRead.Load(), rootApplyLeafMerges: db.orderedRootDeltaGroupRootApplyLeafMerges.Load(), rootApplyInternalMerges: db.orderedRootDeltaGroupRootApplyInternalMerges.Load(), + rootApplyInternalParallelMerges: db.orderedRootDeltaGroupRootApplyInternalParallelMerges.Load(), + rootApplyInternalParallelChildren: db.orderedRootDeltaGroupRootApplyInternalParallelChildren.Load(), + rootApplyInternalParallelWorkers: db.orderedRootDeltaGroupRootApplyInternalParallelWorkers.Load(), + rootApplyInternalParallelOps: db.orderedRootDeltaGroupRootApplyInternalParallelOps.Load(), rootApplyLeafPagesWritten: db.orderedRootDeltaGroupRootApplyLeafPagesWritten.Load(), rootApplyPagerLeafPagesWritten: db.orderedRootDeltaGroupRootApplyPagerLeafPagesWritten.Load(), rootApplyLeafLogPagesWritten: db.orderedRootDeltaGroupRootApplyLeafLogPagesWritten.Load(), @@ -406,13 +521,46 @@ func (db *DB) orderedRootDeltaGroupPublishStats() orderedRootDeltaGroupPublishSt rootApplyInternalLeafLogRefs: db.orderedRootDeltaGroupRootApplyInternalLeafLogRefs.Load(), rootApplyInternalLeafLogRefCopies: db.orderedRootDeltaGroupRootApplyInternalLeafLogRefCopies.Load(), rootApplyRootSplitLevels: db.orderedRootDeltaGroupRootApplyRootSplitLevels.Load(), - systemBuildNs: db.orderedRootDeltaGroupSystemBuildNs.Load(), - systemApplyNs: db.orderedRootDeltaGroupSystemApplyNs.Load(), - systemApplyCalls: db.orderedRootDeltaGroupSystemApplyCalls.Load(), - systemApplyOps: db.orderedRootDeltaGroupSystemApplyOps.Load(), - systemApplyNodeLoads: db.orderedRootDeltaGroupSystemApplyNodeLoads.Load(), - finalizeNs: db.orderedRootDeltaGroupFinalizeNs.Load(), - finalizeCalls: db.orderedRootDeltaGroupFinalizeCalls.Load(), + rootApplyReadOnlyPrepareNs: db.orderedRootDeltaGroupRootApplyReadOnlyPrepareNs.Load(), + rootApplyReadOnlyPrepareCalls: db.orderedRootDeltaGroupRootApplyReadOnlyPrepareCalls.Load(), + rootApplyReadOnlyPrepareOps: db.orderedRootDeltaGroupRootApplyReadOnlyPrepareOps.Load(), + rootApplyReadOnlyPrepareLeafSpans: db.orderedRootDeltaGroupRootApplyReadOnlyPrepareLeafSpans.Load(), + rootApplyReadOnlyPrepareWorker: orderedRootDeltaGroupReadOnlyPrepareWorkerStats{ + targets: db.orderedRootDeltaGroupRootApplyReadOnlyPrepareWorker.targets.Load(), + ranges: db.orderedRootDeltaGroupRootApplyReadOnlyPrepareWorker.ranges.Load(), + minOps: db.orderedRootDeltaGroupRootApplyReadOnlyPrepareWorker.minOps.Load(), + maxOps: db.orderedRootDeltaGroupRootApplyReadOnlyPrepareWorker.maxOps.Load(), + singleSpan: db.orderedRootDeltaGroupRootApplyReadOnlyPrepareWorker.singleSpan.Load(), + }, + rootApplyReadOnlyPrepareExactPlans: db.orderedRootDeltaGroupRootApplyReadOnlyPrepareExactPlans.Load(), + rootApplyReadOnlyPrepareMaintenance: db.orderedRootDeltaGroupRootApplyReadOnlyPrepareMaintenance.Load(), + rootApplyReadOnlyPrepareColdBuilds: db.orderedRootDeltaGroupRootApplyReadOnlyPrepareColdBuilds.Load(), + systemBuildNs: db.orderedRootDeltaGroupSystemBuildNs.Load(), + systemApplyNs: db.orderedRootDeltaGroupSystemApplyNs.Load(), + systemApplyCalls: db.orderedRootDeltaGroupSystemApplyCalls.Load(), + systemApplyOps: db.orderedRootDeltaGroupSystemApplyOps.Load(), + systemApplyNodeLoads: db.orderedRootDeltaGroupSystemApplyNodeLoads.Load(), + installGuardNs: db.orderedRootDeltaGroupInstallGuardNs.Load(), + installGuardCalls: db.orderedRootDeltaGroupInstallGuardCalls.Load(), + installGuardFailures: db.orderedRootDeltaGroupInstallGuardFailures.Load(), + preparedRootPrepareNs: db.orderedRootDeltaGroupPreparedRootPrepareNs.Load(), + preparedRootGroups: db.orderedRootDeltaGroupPreparedRootGroups.Load(), + preparedRootRoots: db.orderedRootDeltaGroupPreparedRootRoots.Load(), + preparedRootEntries: db.orderedRootDeltaGroupPreparedRootEntries.Load(), + preparedRootTombstones: db.orderedRootDeltaGroupPreparedRootTombstones.Load(), + preparedRootKeyBytes: db.orderedRootDeltaGroupPreparedRootKeyBytes.Load(), + preparedRootValueBytes: db.orderedRootDeltaGroupPreparedRootValueBytes.Load(), + preparedRootPointerValues: db.orderedRootDeltaGroupPreparedRootPointerValues.Load(), + preparedRootInstalled: db.orderedRootDeltaGroupPreparedRootInstalled.Load(), + preparedRootAbandoned: db.orderedRootDeltaGroupPreparedRootAbandoned.Load(), + preparedRootOutputPages: db.orderedRootDeltaGroupPreparedRootOutputPages.Load(), + preparedRootOutputLeafLogPtrs: db.orderedRootDeltaGroupPreparedRootOutputLeafLogPtrs.Load(), + preparedRootInstalledPages: db.orderedRootDeltaGroupPreparedRootInstalledPages.Load(), + preparedRootInstalledLeafLogPtrs: db.orderedRootDeltaGroupPreparedRootInstalledLeafLogPtrs.Load(), + preparedRootAbandonedPages: db.orderedRootDeltaGroupPreparedRootAbandonedPages.Load(), + preparedRootAbandonedLeafLogPtrs: db.orderedRootDeltaGroupPreparedRootAbandonedLeafLogPtrs.Load(), + finalizeNs: db.orderedRootDeltaGroupFinalizeNs.Load(), + finalizeCalls: db.orderedRootDeltaGroupFinalizeCalls.Load(), } if calls > 0 { stats.avgRootsPerCall = float64(roots) / float64(calls) diff --git a/TreeDB/db/publish_watermark_metrics_test.go b/TreeDB/db/publish_watermark_metrics_test.go index 54a856b28f..19f1a0ea22 100644 --- a/TreeDB/db/publish_watermark_metrics_test.go +++ b/TreeDB/db/publish_watermark_metrics_test.go @@ -111,20 +111,24 @@ func TestMergeOrderedRootPublishMetricsIncludesLeafLogAttribution(t *testing.T) ZipperLeafLogRecordHintBytesRead: 11, ZipperLeafMerges: 12, ZipperInternalMerges: 13, - ZipperLeafPagesWritten: 14, - ZipperPagerLeafPagesWritten: 15, - ZipperLeafLogPagesWritten: 16, - ZipperLeafPageBytesWritten: 17, - ZipperPagerLeafPageBytesWritten: 18, - ZipperLeafLogPageBytesWritten: 19, - ZipperLeafLogRecordHintBytesWritten: 20, - ZipperInternalPagesWritten: 21, - ZipperInternalPageBytesWritten: 22, - ZipperInternalChildRefs: 23, - ZipperInternalPageChildRefs: 24, - ZipperInternalLeafLogRefs: 25, - ZipperInternalLeafLogRefCopies: 26, - ZipperRootSplitLevels: 27, + ZipperInternalParallelMerges: 14, + ZipperInternalParallelChildren: 15, + ZipperInternalParallelWorkers: 16, + ZipperInternalParallelOps: 17, + ZipperLeafPagesWritten: 18, + ZipperPagerLeafPagesWritten: 19, + ZipperLeafLogPagesWritten: 20, + ZipperLeafPageBytesWritten: 21, + ZipperPagerLeafPageBytesWritten: 22, + ZipperLeafLogPageBytesWritten: 23, + ZipperLeafLogRecordHintBytesWritten: 24, + ZipperInternalPagesWritten: 25, + ZipperInternalPageBytesWritten: 26, + ZipperInternalChildRefs: 27, + ZipperInternalPageChildRefs: 28, + ZipperInternalLeafLogRefs: 29, + ZipperInternalLeafLogRefCopies: 30, + ZipperRootSplitLevels: 31, } var dst adaptive.Metrics @@ -145,6 +149,10 @@ func TestMergeOrderedRootPublishMetricsIncludesLeafLogAttribution(t *testing.T) want.ZipperLeafLogRecordHintBytesRead *= 2 want.ZipperLeafMerges *= 2 want.ZipperInternalMerges *= 2 + want.ZipperInternalParallelMerges *= 2 + want.ZipperInternalParallelChildren *= 2 + want.ZipperInternalParallelWorkers *= 2 + want.ZipperInternalParallelOps *= 2 want.ZipperLeafPagesWritten *= 2 want.ZipperPagerLeafPagesWritten *= 2 want.ZipperLeafLogPagesWritten *= 2 diff --git a/TreeDB/db/system_root_publish_bench_test.go b/TreeDB/db/system_root_publish_bench_test.go index adf0a0162d..17fd93fe48 100644 --- a/TreeDB/db/system_root_publish_bench_test.go +++ b/TreeDB/db/system_root_publish_bench_test.go @@ -1,6 +1,84 @@ package db -import "testing" +import ( + "bytes" + "strconv" + "testing" + + "github.com/snissn/gomap/TreeDB/batch" + "github.com/snissn/gomap/TreeDB/internal/iterator" + "github.com/snissn/gomap/TreeDB/node" + "github.com/snissn/gomap/TreeDB/page" + "github.com/snissn/gomap/TreeDB/zipper" +) + +type benchSingleKVIterator struct { + key []byte + value []byte + valid bool +} + +func (it *benchSingleKVIterator) Valid() bool { return it != nil && it.valid } + +func (it *benchSingleKVIterator) Next() { it.valid = false } + +func (it *benchSingleKVIterator) Seek(key []byte) { + if it == nil { + return + } + it.valid = bytes.Compare(key, it.key) <= 0 +} + +func (it *benchSingleKVIterator) UnsafeKey() []byte { + if !it.Valid() { + return nil + } + return it.key +} + +func (it *benchSingleKVIterator) UnsafeValue() []byte { + if !it.Valid() { + return nil + } + return it.value +} + +func (it *benchSingleKVIterator) UnsafeEntry() ([]byte, page.ValuePtr, byte) { + if !it.Valid() { + return nil, page.ValuePtr{}, node.FlagInline + } + return it.value, page.ValuePtr{}, node.FlagInline +} + +func (it *benchSingleKVIterator) Key() []byte { + return append([]byte(nil), it.UnsafeKey()...) +} + +func (it *benchSingleKVIterator) Value() []byte { + return append([]byte(nil), it.UnsafeValue()...) +} + +func (it *benchSingleKVIterator) KeyCopy(dst []byte) []byte { + return append(dst[:0], it.UnsafeKey()...) +} + +func (it *benchSingleKVIterator) ValueCopy(dst []byte) []byte { + return append(dst[:0], it.UnsafeValue()...) +} + +func (it *benchSingleKVIterator) IsDeleted() bool { return false } +func (it *benchSingleKVIterator) Error() error { return nil } +func (it *benchSingleKVIterator) Close() error { it.valid = false; return nil } +func (it *benchSingleKVIterator) Domain() ([]byte, []byte) { + return nil, nil +} + +func (it *benchSingleKVIterator) Len() int { + if it.Valid() { + return 1 + } + return 0 +} func BenchmarkPublishSystemRootIterator_WarmSparseDelta(b *testing.B) { dir := b.TempDir() @@ -91,3 +169,89 @@ func BenchmarkPublishSystemRootIterator_WarmDenseDelta(b *testing.B) { } b.ReportMetric(float64(fallbacks), "warm_rebuild_fallback") } + +func BenchmarkPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder_WarmSingleRoot(b *testing.B) { + benchmarkPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilderWarmSingleRoot(b, orderedRootBatchGroupWarmBenchOptions{}) +} + +func BenchmarkPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder_WarmSingleRootReadOnlyPrepare(b *testing.B) { + benchmarkPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilderWarmSingleRoot(b, orderedRootBatchGroupWarmBenchOptions{prepareReadOnly: true}) +} + +func BenchmarkPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder_WarmSingleRootReadOnlyPrepareReuse(b *testing.B) { + benchmarkPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilderWarmSingleRoot(b, orderedRootBatchGroupWarmBenchOptions{prepareReadOnly: true, reusePrepare: true}) +} + +func BenchmarkPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder_WarmSingleRootReadOnlyPrepareWorkerStats(b *testing.B) { + benchmarkPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilderWarmSingleRoot(b, orderedRootBatchGroupWarmBenchOptions{prepareReadOnly: true, prepareWorkerCount: 3}) +} + +type orderedRootBatchGroupWarmBenchOptions struct { + prepareReadOnly bool + reusePrepare bool + prepareWorkerCount int +} + +func benchmarkPublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilderWarmSingleRoot(b *testing.B, benchOpts orderedRootBatchGroupWarmBenchOptions) { + dir := b.TempDir() + db, err := Open(Options{Dir: dir}) + if err != nil { + b.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + baseRoot, err := db.PublishOrderedRootIterator(0, mustFrozenSystemMemtable(b, "root/a", "base").NewIterator(nil, nil)) + if err != nil { + b.Fatalf("publish base root: %v", err) + } + left := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := left.Set([]byte("root/a"), []byte("left")); err != nil { + b.Fatalf("set left delta: %v", err) + } + defer func() { _ = left.Close() }() + right := batch.New(nil, orderedRootDeltaBatchInlineThreshold) + if err := right.Set([]byte("root/a"), []byte("right")); err != nil { + b.Fatalf("set right delta: %v", err) + } + defer func() { _ = right.Close() }() + + ordered := []OrderedRootDeltaBatchPublishInput{{ + StoragePolicy: OrderedRootStorageDefault, + PrepareReadOnly: benchOpts.prepareReadOnly, + ReadOnlyPrepareWorkerCount: benchOpts.prepareWorkerCount, + }} + var prepared zipper.ReadOnlyPrepareResult + systemKey := []byte("sys/collections/users/primary") + var systemValueBuf [20]byte + publish := func(delta *batch.Batch) { + ordered[0].BaseRoot = baseRoot + ordered[0].Delta = delta + if benchOpts.prepareReadOnly && benchOpts.reusePrepare { + ordered[0].ReadOnlyPrepareResult = &prepared + } + _, rootIDs, err := db.PublishOrderedRootDeltaBatchGroupWithSystemDeltaBuilder(ordered, func(rootIDs []uint64) (iterator.UnsafeIterator, error) { + value := strconv.AppendUint(systemValueBuf[:0], rootIDs[0], 10) + return &benchSingleKVIterator{ + key: systemKey, + value: value, + valid: true, + }, nil + }) + if err != nil { + b.Fatalf("publish batch group: %v", err) + } + baseRoot = rootIDs[0] + } + if benchOpts.prepareReadOnly && benchOpts.reusePrepare { + publish(left) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + delta := left + if i&1 == 1 { + delta = right + } + publish(delta) + } +} diff --git a/TreeDB/db/update.go b/TreeDB/db/update.go index 0b25172119..c0ad1f3adb 100644 --- a/TreeDB/db/update.go +++ b/TreeDB/db/update.go @@ -6,6 +6,7 @@ import ( "fmt" batchpkg "github.com/snissn/gomap/TreeDB/batch" + "github.com/snissn/gomap/TreeDB/internal/keyupdate" "github.com/snissn/gomap/TreeDB/tree" ) @@ -82,7 +83,7 @@ func (db *DB) update(key []byte, fn UpdateFunc, syncWrite bool) error { for { unlock := db.lockUpdateKey(key) old, err := db.getForUpdate(key) - unlock() + unlock.Unlock() if err != nil { return err } @@ -102,36 +103,36 @@ func (db *DB) update(key []byte, fn UpdateFunc, syncWrite bool) error { unlock = db.lockUpdateKey(key) latest, err := db.getForUpdate(key) if err != nil { - unlock() + unlock.Unlock() return err } if !sameUpdateValue(observed, latest) { - unlock() + unlock.Unlock() continue } switch result.Op { case UpdateNoop: - unlock() + unlock.Unlock() return nil case UpdateSet: err = db.setPoint(key, result.Value, syncWrite) - unlock() + unlock.Unlock() return err case UpdateDelete: err = db.deletePoint(key, syncWrite) - unlock() + unlock.Unlock() return err default: - unlock() + unlock.Unlock() return fmt.Errorf("treedb: unknown update op %d", result.Op) } } } -func (db *DB) lockUpdateKey(key []byte) func() { +func (db *DB) lockUpdateKey(key []byte) keyupdate.Unlocker { if db == nil { - return func() {} + return keyupdate.Unlocker{} } return db.updateLocks.Lock(key) } diff --git a/TreeDB/docs/spec/collections-write-domain.md b/TreeDB/docs/spec/collections-write-domain.md index d9894a66ae..c9cd60ebde 100644 --- a/TreeDB/docs/spec/collections-write-domain.md +++ b/TreeDB/docs/spec/collections-write-domain.md @@ -26,12 +26,13 @@ escape hatch. It is not the production-mainline path. Pending indexed writes are visible through the collection manager that owns the write domain. -Reads and checks MUST merge these layers with the following newest-to-oldest -precedence: +Reads and checks enumerate pending runs in active, queued, then mutable order. +Lookups and merged iterators use newest-wins shadowing, so effective precedence +is: 1. current mutable indexed runs, -2. queued immutable indexed flush units, -3. in-flight async publishing units, +2. queued immutable indexed flush units, in FIFO order, +3. the active in-flight async publishing batch, in original FIFO unit order, 4. persisted backend roots from the current collection catalog. This applies to: @@ -58,9 +59,13 @@ separate durable log. `BufferedIndexedAsyncFlush` allows threshold-triggered indexed flush units to be published by a background worker. -The async worker may move a queued immutable unit into the publishing state -before root publication completes. Publishing units remain visible to reads, -unique checks, schema-change barriers, and explicit flush barriers. +The async worker may move queued immutable units into one active coalesced flush +batch before root publication completes. The batch preserves the original FIFO +unit boundaries. Ordered-root publish uses either the mechanical merged view or +a narrower semantic effective view for proven-safe non-unique secondary-index +update chains; raw FIFO units remain the visibility, ownership, and requeue +source. Active publishing units remain visible to reads, unique checks, +schema-change barriers, and explicit flush barriers. `BufferedIndexedAsyncFlushMaxQueuedUnits` bounds queued immutable flush units. When the queue is full and a publish is already in flight, writers MUST apply diff --git a/TreeDB/docs/spec/contracts.md b/TreeDB/docs/spec/contracts.md index e738889e15..8441315d62 100644 --- a/TreeDB/docs/spec/contracts.md +++ b/TreeDB/docs/spec/contracts.md @@ -159,8 +159,12 @@ Indexed collection writes use collection-local write memtables by default. Pending indexed writes are visible through the owning collection manager before they are published to persisted roots. Primary reads, secondary index lookups, unique checks, and update/delete planning must merge write-domain state with -explicit newest-to-oldest precedence: current mutable runs, queued immutable -flush units, in-flight async publishing units, then persisted roots. +newest-wins shadowing: current mutable runs, queued immutable flush units, +active in-flight async publishing units, then persisted roots. The active async +publish uses a coalesced flush batch that preserves original FIFO unit +boundaries. Ordered-root publish uses either the mechanical merged view or a +safe semantic effective view for non-unique secondary-index update chains; raw +FIFO units remain the source for visibility, ownership, and requeue behavior. `BufferedIndexedAsyncFlush` is a throughput feature, not a durable-at-ack mutation log. The current contract is flush-boundary durable: callers may treat diff --git a/TreeDB/internal/adaptive/controller.go b/TreeDB/internal/adaptive/controller.go index 7d38f07f6c..0746fa1b71 100644 --- a/TreeDB/internal/adaptive/controller.go +++ b/TreeDB/internal/adaptive/controller.go @@ -37,6 +37,10 @@ type Metrics struct { ZipperLeafLogRecordHintBytesRead int ZipperLeafMerges int ZipperInternalMerges int + ZipperInternalParallelMerges int + ZipperInternalParallelChildren int + ZipperInternalParallelWorkers int + ZipperInternalParallelOps int ZipperLeafPagesWritten int ZipperPagerLeafPagesWritten int ZipperLeafLogPagesWritten int diff --git a/TreeDB/internal/keyupdate/locks.go b/TreeDB/internal/keyupdate/locks.go index c5fa5b265e..4c088b4a54 100644 --- a/TreeDB/internal/keyupdate/locks.go +++ b/TreeDB/internal/keyupdate/locks.go @@ -14,11 +14,24 @@ type Locks struct { stripes [stripes]sync.Mutex } -// Lock locks the stripe for key and returns its unlock function. Hash collisions +// Unlocker releases a lock acquired by Locks.Lock. +type Unlocker struct { + mu *sync.Mutex +} + +// Unlock releases the lock acquired by Locks.Lock. +func (u Unlocker) Unlock() { + if u.mu == nil { + return + } + u.mu.Unlock() +} + +// Lock locks the stripe for key and returns its unlock handle. Hash collisions // only reduce concurrency; they do not affect correctness. -func (l *Locks) Lock(key []byte) func() { +func (l *Locks) Lock(key []byte) Unlocker { idx := xxhash.Sum64(key) & (stripes - 1) mu := &l.stripes[idx] mu.Lock() - return mu.Unlock + return Unlocker{mu: mu} } diff --git a/TreeDB/internal/keyupdate/locks_test.go b/TreeDB/internal/keyupdate/locks_test.go new file mode 100644 index 0000000000..456833f84a --- /dev/null +++ b/TreeDB/internal/keyupdate/locks_test.go @@ -0,0 +1,31 @@ +package keyupdate + +import "testing" + +func TestLocksLockDoesNotAllocate(t *testing.T) { + var locks Locks + key := []byte("alloc-key") + + allocs := testing.AllocsPerRun(1000, func() { + unlock := locks.Lock(key) + unlock.Unlock() + }) + if allocs != 0 { + t.Fatalf("Lock allocations = %v, want 0", allocs) + } +} + +func TestUnlockerZeroValueIsNoop(t *testing.T) { + var unlock Unlocker + unlock.Unlock() +} + +func BenchmarkLocksLockUnlock(b *testing.B) { + var locks Locks + key := []byte("bench-key") + b.ReportAllocs() + for i := 0; i < b.N; i++ { + unlock := locks.Lock(key) + unlock.Unlock() + } +} diff --git a/TreeDB/node/builder.go b/TreeDB/node/builder.go index 9ae7411217..f9c1a39d28 100644 --- a/TreeDB/node/builder.go +++ b/TreeDB/node/builder.go @@ -100,6 +100,7 @@ const ( leafColumnarEntriesPoolMaxCap = 1024 internalBaseEntriesPoolInitCap = 256 internalBaseEntriesPoolMaxCap = 1024 + internalFenceScratchMaxCap = page.PageSize ) var leafColumnarV2EntriesPool = sync.Pool{ @@ -279,6 +280,8 @@ func NewBuilderWithOptions(data []byte, pType page.PageType, opts BuilderOptions // ResetWithOptions reinitializes an existing builder instance for reuse. func (b *Builder) ResetWithOptions(data []byte, pType page.PageType, opts BuilderOptions) { + internalFenceLow := keepInternalFenceScratch(b.internalFenceLow) + internalFenceHigh := keepInternalFenceScratch(b.internalFenceHigh) b.ReleaseScratch() leafPrefix := opts.LeafPrefixCompression @@ -295,6 +298,8 @@ func (b *Builder) ResetWithOptions(data []byte, pType page.PageType, opts Builde leafColumnarV2: leafColumnarV2, leafPackedValuePtr: opts.PackedValuePtr, internalBaseDelta: opts.InternalBaseDelta, + internalFenceLow: internalFenceLow, + internalFenceHigh: internalFenceHigh, } if pType == page.PageTypeLeaf && opts.LeafColumnar { if leafPrefix { @@ -335,8 +340,15 @@ func (b *Builder) ResetWithOptions(data []byte, pType page.PageType, opts Builde } } +func keepInternalFenceScratch(buf []byte) []byte { + if cap(buf) > internalFenceScratchMaxCap { + return nil + } + return buf[:0] +} + // ReleaseScratch returns pooled scratch resources held by the builder and -// drops references so the builder can be reused safely. +// drops large references so the builder can be reused safely. func (b *Builder) ReleaseScratch() { if b == nil { return @@ -346,8 +358,8 @@ func (b *Builder) ReleaseScratch() { b.releaseInternalBaseDeltaScratch() b.data = nil b.leafPrevKey = nil - b.internalFenceLow = nil - b.internalFenceHigh = nil + b.internalFenceLow = keepInternalFenceScratch(b.internalFenceLow) + b.internalFenceHigh = keepInternalFenceScratch(b.internalFenceHigh) } // SetPageID sets the page ID (can be done at finish too). @@ -372,12 +384,12 @@ func (b *Builder) SetInternalFenceBounds(low, high []byte) { } b.internalFenceBounds = true if len(low) == 0 { - b.internalFenceLow = nil + b.internalFenceLow = b.internalFenceLow[:0] } else { b.internalFenceLow = append(b.internalFenceLow[:0], low...) } if len(high) == 0 { - b.internalFenceHigh = nil + b.internalFenceHigh = b.internalFenceHigh[:0] } else { b.internalFenceHigh = append(b.internalFenceHigh[:0], high...) } diff --git a/TreeDB/node/builder_test.go b/TreeDB/node/builder_test.go new file mode 100644 index 0000000000..57b1d486db --- /dev/null +++ b/TreeDB/node/builder_test.go @@ -0,0 +1,51 @@ +package node + +import ( + "bytes" + "testing" + + "github.com/snissn/gomap/TreeDB/page" +) + +func TestBuilderReusesInternalFenceScratch(t *testing.T) { + data := make([]byte, page.PageSize) + b := NewBuilderWithOptions(data, page.PageTypeInternal, BuilderOptions{InternalBaseDelta: true}) + + low := bytes.Repeat([]byte("l"), 32) + high := bytes.Repeat([]byte("h"), 32) + b.SetInternalFenceBounds(low, high) + lowCap := cap(b.internalFenceLow) + highCap := cap(b.internalFenceHigh) + if lowCap < len(low) || highCap < len(high) { + t.Fatalf("initial fence caps low/high=%d/%d want at least %d/%d", lowCap, highCap, len(low), len(high)) + } + + b.ReleaseScratch() + if len(b.internalFenceLow) != 0 || len(b.internalFenceHigh) != 0 { + t.Fatalf("released fence lengths low/high=%d/%d want 0/0", len(b.internalFenceLow), len(b.internalFenceHigh)) + } + if cap(b.internalFenceLow) != lowCap || cap(b.internalFenceHigh) != highCap { + t.Fatalf("released fence caps low/high=%d/%d want %d/%d", cap(b.internalFenceLow), cap(b.internalFenceHigh), lowCap, highCap) + } + + b.ResetWithOptions(data, page.PageTypeInternal, BuilderOptions{InternalBaseDelta: true}) + b.SetInternalFenceBounds(low[:8], high[:8]) + if cap(b.internalFenceLow) != lowCap || cap(b.internalFenceHigh) != highCap { + t.Fatalf("reused fence caps low/high=%d/%d want %d/%d", cap(b.internalFenceLow), cap(b.internalFenceHigh), lowCap, highCap) + } + if !bytes.Equal(b.internalFenceLow, low[:8]) || !bytes.Equal(b.internalFenceHigh, high[:8]) { + t.Fatalf("reused fence values low/high=%q/%q", b.internalFenceLow, b.internalFenceHigh) + } +} + +func TestBuilderDropsOversizedInternalFenceScratch(t *testing.T) { + data := make([]byte, page.PageSize) + b := NewBuilderWithOptions(data, page.PageTypeInternal, BuilderOptions{InternalBaseDelta: true}) + + oversized := bytes.Repeat([]byte("x"), internalFenceScratchMaxCap+1) + b.SetInternalFenceBounds(oversized, oversized) + b.ReleaseScratch() + if cap(b.internalFenceLow) != 0 || cap(b.internalFenceHigh) != 0 { + t.Fatalf("oversized fence caps low/high=%d/%d want 0/0", cap(b.internalFenceLow), cap(b.internalFenceHigh)) + } +} diff --git a/TreeDB/zipper/parallel_policy_test.go b/TreeDB/zipper/parallel_policy_test.go index 0475d59489..f060cbf4fd 100644 --- a/TreeDB/zipper/parallel_policy_test.go +++ b/TreeDB/zipper/parallel_policy_test.go @@ -11,8 +11,8 @@ func TestInternalMergeParallelThresholds_Default(t *testing.T) { func TestInternalMergeParallelThresholds_MaintenanceIgnoresPressure(t *testing.T) { minChildren, minOps := internalMergeParallelThresholds(true, ParallelMergePressureCritical) - if minChildren != mergeInternalMinParallelChildren || minOps != mergeInternalMinParallelOps { - t.Fatalf("maintenance thresholds=(children=%d ops=%d) want (%d,%d)", minChildren, minOps, mergeInternalMinParallelChildren, mergeInternalMinParallelOps) + if minChildren != mergeInternalMinParallelChildren || minOps != mergeInternalMaintenanceMinParallelOps { + t.Fatalf("maintenance thresholds=(children=%d ops=%d) want (%d,%d)", minChildren, minOps, mergeInternalMinParallelChildren, mergeInternalMaintenanceMinParallelOps) } } diff --git a/TreeDB/zipper/zipper.go b/TreeDB/zipper/zipper.go index 3413e1790b..ffe1c6d692 100644 --- a/TreeDB/zipper/zipper.go +++ b/TreeDB/zipper/zipper.go @@ -10,6 +10,7 @@ import ( "sort" "sync" "sync/atomic" + "time" "github.com/snissn/gomap/TreeDB/batch" "github.com/snissn/gomap/TreeDB/internal/adaptive" @@ -56,6 +57,12 @@ var outerLeafBuildPagePool = sync.Pool{ }, } +var clonedZipperApplyScratchPool = sync.Pool{ + New: func() any { + return newMergeScratch() + }, +} + func getLeafPageScratch() []byte { buf, _ := leafPageScratchPool.Get().([]byte) if cap(buf) != page.PageSize { @@ -110,6 +117,8 @@ type Zipper struct { scratchMu sync.Mutex applyScratch *mergeScratch + + pooledApplyScratch bool } type ParallelMergePressureLevel uint8 @@ -139,7 +148,10 @@ const ( mergeNodeKeyScratchMaxCap = 1 << 20 mergeInternalMinParallelChildren = 8 - mergeInternalMinParallelOps = 4096 + mergeInternalMinParallelOps = 1024 + mergeInternalMaintenanceMinParallelOps = 4096 + mergeInternalOuterLeafLogMinParallelOps = 4096 + mergeInternalMaxParallelWorkers = 4 mergeInternalHighPressureMinChildren = 16 mergeInternalHighPressureMinOps = 16 * 1024 mergeInternalCriticalPressureMinChildren = 32 @@ -457,9 +469,20 @@ type childWork struct { childStat adaptive.Metrics } -const maxChildWorkCap = 1 << 14 +type childWorkBuffer struct { + items []childWork +} -var childWorkPool sync.Pool +const ( + maxChildWorkCap = 1 << 14 + maxChildWorkRetiredKeepCap = 8 +) + +var childWorkPool = sync.Pool{ + New: func() any { + return &childWorkBuffer{} + }, +} const maxInternalEntryCap = 1 << 15 @@ -543,30 +566,62 @@ func (b *maintenanceBudget) take(n int64) bool { } } -func getChildWorkSlice(capacity int) []childWork { +func getChildWorkBuffer(capacity int) *childWorkBuffer { if capacity < 0 { capacity = 0 } if capacity > maxChildWorkCap { - return make([]childWork, 0, capacity) + return &childWorkBuffer{items: make([]childWork, 0, capacity)} } - if v := childWorkPool.Get(); v != nil { - s := v.([]childWork) - if cap(s) >= capacity { - return s[:0] - } + buf, _ := childWorkPool.Get().(*childWorkBuffer) + if buf == nil { + buf = &childWorkBuffer{} + } + if cap(buf.items) >= capacity { + buf.items = buf.items[:0] + return buf } - return make([]childWork, 0, capacity) + buf.items = make([]childWork, 0, capacity) + return buf } -func putChildWorkSlice(children []childWork) { +func putChildWorkBuffer(buf *childWorkBuffer) { + if buf == nil { + return + } + children := buf.items if cap(children) > maxChildWorkCap { + buf.items = nil return } for i := range children { + retired := children[i].retired children[i] = childWork{} + if cap(retired) <= maxChildWorkRetiredKeepCap { + children[i].retired = retired[:0] + } } - childWorkPool.Put(children[:0]) + buf.items = children[:0] + childWorkPool.Put(buf) +} + +func appendChildRetiredPages(dst *[]uint64, children []childWork, total int) { + if dst == nil { + return + } + if total == 0 { + return + } + retired := *dst + if cap(retired)-len(retired) < total { + grown := make([]uint64, len(retired), len(retired)+total) + copy(grown, retired) + retired = grown + } + for i := range children { + retired = append(retired, children[i].retired...) + } + *dst = retired } func getInternalEntrySlice(capacity int) []internalEntry { @@ -606,6 +661,14 @@ func (z *Zipper) acquireApplyScratch() *mergeScratch { if z == nil { return newMergeScratch() } + if z.pooledApplyScratch { + s, _ := clonedZipperApplyScratchPool.Get().(*mergeScratch) + if s == nil { + s = newMergeScratch() + } + s.reset() + return s + } z.scratchMu.Lock() s := z.applyScratch z.applyScratch = nil @@ -622,6 +685,10 @@ func (z *Zipper) releaseApplyScratch(s *mergeScratch) { return } s.reset() + if z.pooledApplyScratch { + clonedZipperApplyScratchPool.Put(s) + return + } z.scratchMu.Lock() if z.applyScratch == nil { z.applyScratch = s @@ -650,6 +717,7 @@ func (z *Zipper) CloneWithAllocator(a PageAllocator) *Zipper { adaptiveLeafEncoding: z.adaptiveLeafEncoding, maintenanceOpsPerCoalesce: z.maintenanceOpsPerCoalesce, parallelMergePressure: z.parallelMergePressure, + pooledApplyScratch: true, } } @@ -729,7 +797,7 @@ func internalMergeParallelThresholds(maintenance bool, pressure ParallelMergePre minChildren = mergeInternalMinParallelChildren minOps = mergeInternalMinParallelOps if maintenance { - return minChildren, minOps + return minChildren, mergeInternalMaintenanceMinParallelOps } switch pressure { case ParallelMergePressureCritical: @@ -990,6 +1058,22 @@ func recordZipperInternalLeafLogRefCopy(metrics *adaptive.Metrics) { metrics.ZipperInternalLeafLogRefCopies++ } +func recordZipperInternalParallelMerge(metrics *adaptive.Metrics, activeChildren, workers, ops int) { + if metrics == nil { + return + } + metrics.ZipperInternalParallelMerges++ + if activeChildren > 0 { + metrics.ZipperInternalParallelChildren += activeChildren + } + if workers > 0 { + metrics.ZipperInternalParallelWorkers += workers + } + if ops > 0 { + metrics.ZipperInternalParallelOps += ops + } +} + func validateLoadedLeafLogNode(data []byte) (node.Node, error) { if len(data) != page.PageSize { return node.Node{}, errors.New("zipper: leaf page has invalid size") @@ -1009,8 +1093,534 @@ func validateLoadedLeafLogNodeFrom(source string, data []byte) (node.Node, error return n, nil } +// ApplyOptions configures a root apply attempt. +type ApplyOptions struct { + // PrepareReadOnly asks ApplyWithOptions to run the read-only preparation + // pass before applying the delta. This is opt-in because it traverses the + // existing root in addition to the apply pass. It does not change apply + // output, root installation, or prepared-output ownership. + PrepareReadOnly bool + + // ReadOnlyPrepare reuses buffers for the optional read-only preparation + // pass. It is ignored unless PrepareReadOnly is true. + ReadOnlyPrepare ReadOnlyPrepareOptions +} + +// ApplyResult is the complete in-memory result of a root apply attempt. The +// retired page list is pending until the caller's install guard succeeds and +// the new root is committed. +type ApplyResult struct { + RootID uint64 + PendingRetiredPages []uint64 + Metrics adaptive.Metrics + + // ReadOnlyPrepare is populated only when ApplyOptions.PrepareReadOnly is + // true. It is planning metadata only; it owns no pager or leaf-log output. + ReadOnlyPrepare ReadOnlyPrepareResult + // ReadOnlyPrepareNs is the time spent in the optional read-only preparation + // pass. It is zero when ApplyOptions.PrepareReadOnly is false. + ReadOnlyPrepareNs uint64 +} + +// ReadOnlyPrepareOptions configures a read-only root preparation pass. The zero +// value is the normal caller-constructed form. Non-zero buffer reuse options are +// produced by ReadOnlyPrepareResult.ReuseOptions. +type ReadOnlyPrepareOptions struct { + leafSpans []ReadOnlyLeafSpan + keyArena []byte +} + +// ReadOnlyLeafSpan describes one existing leaf range touched by a sorted delta. +// It contains only in-memory planning metadata; it does not own prepared pager +// pages, leaf-log records, or pending retired pages. +type ReadOnlyLeafSpan struct { + // Ref identifies the existing leaf that owns this span. When ColdBuild is + // true there is no existing leaf; Ref is the zero ChildRef and is not + // actionable. + Ref page.ChildRef + + // LowKey is the inclusive lower bound for the leaf span. HighKey is the + // exclusive upper bound. A nil bound is open-ended. Non-nil bound slices and + // op-key slices are owned by ReadOnlyPrepareResult until ReuseOptions is used. + LowKey []byte + HighKey []byte + + FirstOpKey []byte + LastOpKey []byte + OpCount int +} + +// ReadOnlyLeafSpanSummary is a compact, allocation-free summary of a read-only +// leaf-span plan. It is intended for callers and benchmarks that need to +// report span distribution without requiring each caller to walk or retain the +// span slice. +type ReadOnlyLeafSpanSummary struct { + Ops int + Spans int + ExactLeafSpans bool + ColdBuild bool + Maintenance bool + + // MinSpanOps and MaxSpanOps are zero when Spans is zero. + MinSpanOps int + MaxSpanOps int + SingleOpSpans int + OpenLowSpans int + OpenHighSpans int +} + +// ReadOnlyLeafSpanWorkerRange assigns a contiguous range of read-only leaf +// spans to one future worker. The range is a planning primitive only; it does +// not imply parallel execution or prepared output ownership. +type ReadOnlyLeafSpanWorkerRange struct { + FirstSpan int + SpanCount int + Ops int +} + +// ReadOnlyLeafSpanWorkerRangeSummary is an allocation-free aggregate of the +// deterministic worker ranges derived from a read-only leaf-span plan. +type ReadOnlyLeafSpanWorkerRangeSummary struct { + TargetWorkers int + Ranges int + Ops int + MinRangeOps int + MaxRangeOps int + SingleSpanRanges int +} + +// ReadOnlyPrepareResult is the read-only portion of a root apply attempt. It is +// safe to discard on root mismatch because it has not allocated or persisted +// output pages. +type ReadOnlyPrepareResult struct { + RootID uint64 + Ops int + ColdBuild bool + Maintenance bool + + // ExactLeafSpans is true when LeafSpans fully describe the existing leaves + // touched by the delta. Delete-containing maintenance can merge/rebalance + // adjacent leaves, so its direct key spans are useful planning hints but are + // not complete prepared-output ownership. + ExactLeafSpans bool + + LeafSpans []ReadOnlyLeafSpan + Metrics adaptive.Metrics + + keyArena []byte +} + +const ( + readOnlyPrepareResultReuseLeafSpanKeepCap = 4096 + readOnlyPrepareResultReuseKeyArenaKeepCap = 1 << 20 +) + +// LeafSpanSummary returns an allocation-free aggregate view of r's leaf spans. +func (r ReadOnlyPrepareResult) LeafSpanSummary() ReadOnlyLeafSpanSummary { + summary := ReadOnlyLeafSpanSummary{ + Ops: r.Ops, + Spans: len(r.LeafSpans), + ExactLeafSpans: r.ExactLeafSpans, + ColdBuild: r.ColdBuild, + Maintenance: r.Maintenance, + } + for i := range r.LeafSpans { + span := &r.LeafSpans[i] + if i == 0 || span.OpCount < summary.MinSpanOps { + summary.MinSpanOps = span.OpCount + } + if i == 0 || span.OpCount > summary.MaxSpanOps { + summary.MaxSpanOps = span.OpCount + } + if span.OpCount == 1 { + summary.SingleOpSpans++ + } + if span.LowKey == nil { + summary.OpenLowSpans++ + } + if span.HighKey == nil { + summary.OpenHighSpans++ + } + } + return summary +} + +// AppendLeafSpanWorkerRanges appends deterministic contiguous span partitions +// to dst. It creates at most workers ranges and never creates empty ranges. The +// returned ranges preserve span order and are suitable for future parallel +// preparation steps that still need serial output append and assembly order. +// No-op inputs return dst unchanged. +func (r ReadOnlyPrepareResult) AppendLeafSpanWorkerRanges(dst []ReadOnlyLeafSpanWorkerRange, workers int) []ReadOnlyLeafSpanWorkerRange { + if workers <= 0 || len(r.LeafSpans) == 0 { + return dst + } + if workers > len(r.LeafSpans) { + workers = len(r.LeafSpans) + } + totalOps := int64(r.Ops) + + spanIdx := 0 + cumulativeOps := int64(0) + for rangeIdx := 0; rangeIdx < workers && spanIdx < len(r.LeafSpans); rangeIdx++ { + firstSpan := spanIdx + rangeOps := 0 + remainingRanges := workers - rangeIdx - 1 + lastAllowedSpan := len(r.LeafSpans) - remainingRanges + targetCumulativeOps := readOnlyPrepareCeilDiv64(totalOps*int64(rangeIdx+1), int64(workers)) + + for spanIdx < lastAllowedSpan { + spanOps := r.LeafSpans[spanIdx].OpCount + rangeOps += spanOps + cumulativeOps += int64(spanOps) + spanIdx++ + if remainingRanges > 0 && cumulativeOps >= targetCumulativeOps { + break + } + } + dst = append(dst, ReadOnlyLeafSpanWorkerRange{ + FirstSpan: firstSpan, + SpanCount: spanIdx - firstSpan, + Ops: rangeOps, + }) + } + return dst +} + +// LeafSpanWorkerRangeSummary returns an aggregate of the deterministic worker +// ranges for workers without retaining the ranges themselves. +func (r ReadOnlyPrepareResult) LeafSpanWorkerRangeSummary(workers int) ReadOnlyLeafSpanWorkerRangeSummary { + summary := ReadOnlyLeafSpanWorkerRangeSummary{TargetWorkers: workers} + if workers <= 0 || len(r.LeafSpans) == 0 { + return summary + } + if workers > len(r.LeafSpans) { + workers = len(r.LeafSpans) + } + summary.Ops = r.Ops + totalOps := int64(r.Ops) + + spanIdx := 0 + cumulativeOps := int64(0) + for rangeIdx := 0; rangeIdx < workers && spanIdx < len(r.LeafSpans); rangeIdx++ { + firstSpan := spanIdx + rangeOps := 0 + remainingRanges := workers - rangeIdx - 1 + lastAllowedSpan := len(r.LeafSpans) - remainingRanges + targetCumulativeOps := readOnlyPrepareCeilDiv64(totalOps*int64(rangeIdx+1), int64(workers)) + + for spanIdx < lastAllowedSpan { + spanOps := r.LeafSpans[spanIdx].OpCount + rangeOps += spanOps + cumulativeOps += int64(spanOps) + spanIdx++ + if remainingRanges > 0 && cumulativeOps >= targetCumulativeOps { + break + } + } + summary.Ranges++ + if summary.Ranges == 1 || rangeOps < summary.MinRangeOps { + summary.MinRangeOps = rangeOps + } + if summary.Ranges == 1 || rangeOps > summary.MaxRangeOps { + summary.MaxRangeOps = rangeOps + } + if spanIdx-firstSpan == 1 { + summary.SingleSpanRanges++ + } + } + return summary +} + +func readOnlyPrepareCeilDiv64(n, d int64) int64 { + return (n + d - 1) / d +} + +// ReuseOptions returns buffers from r for a later read-only preparation pass. +// The returned options must not be used while r's LeafSpans are still needed. +func (r ReadOnlyPrepareResult) ReuseOptions() ReadOnlyPrepareOptions { + return ReadOnlyPrepareOptions{ + leafSpans: r.LeafSpans[:0], + keyArena: r.keyArena[:0], + } +} + +// ResetForReuse clears result metadata while retaining bounded reusable +// leaf-span and key-arena buffers for a later ReuseOptions call. Oversized +// buffers are dropped so temporary prepare pools do not retain one-off large +// batch state indefinitely. +func (r *ReadOnlyPrepareResult) ResetForReuse() { + if r == nil { + return + } + leafSpans := r.LeafSpans + keyArena := r.keyArena + *r = ReadOnlyPrepareResult{} + if cap(leafSpans) <= readOnlyPrepareResultReuseLeafSpanKeepCap { + r.LeafSpans = leafSpans[:0] + } + if cap(keyArena) <= readOnlyPrepareResultReuseKeyArenaKeepCap { + r.keyArena = keyArena[:0] + } +} + +// ValidateLeafSpans checks the deterministic planning invariants for r's +// read-only leaf-span view. It is intended for tests and future prepared-output +// callers that want to assert a plan before using it; PrepareReadOnly itself +// does not call this helper on the hot path. +func (r ReadOnlyPrepareResult) ValidateLeafSpans() error { + if r.Ops < 0 { + return fmt.Errorf("zipper: read-only prepare has negative op count %d", r.Ops) + } + if r.Ops == 0 { + if len(r.LeafSpans) != 0 { + return fmt.Errorf("zipper: read-only prepare has %d spans for zero ops", len(r.LeafSpans)) + } + return nil + } + if len(r.LeafSpans) == 0 { + return fmt.Errorf("zipper: read-only prepare has %d ops but no leaf spans", r.Ops) + } + if r.ColdBuild && len(r.LeafSpans) != 1 { + return fmt.Errorf("zipper: cold read-only prepare has %d leaf spans, want 1", len(r.LeafSpans)) + } + totalOps := 0 + var prevLastOp []byte + var prevHigh []byte + for i, span := range r.LeafSpans { + if span.OpCount <= 0 { + return readOnlyPrepareSpanError(i, "has non-positive op count %d", span.OpCount) + } + if len(span.FirstOpKey) == 0 { + return readOnlyPrepareSpanError(i, "has empty first op key") + } + if len(span.LastOpKey) == 0 { + return readOnlyPrepareSpanError(i, "has empty last op key") + } + if bytes.Compare(span.FirstOpKey, span.LastOpKey) > 0 { + return readOnlyPrepareSpanError(i, "first op key %s is after last op key %s", readOnlyPrepareKeyForError(span.FirstOpKey), readOnlyPrepareKeyForError(span.LastOpKey)) + } + if prevLastOp != nil && bytes.Compare(prevLastOp, span.FirstOpKey) >= 0 { + return readOnlyPrepareSpanError(i, "first op key %s is not after previous last op key %s", readOnlyPrepareKeyForError(span.FirstOpKey), readOnlyPrepareKeyForError(prevLastOp)) + } + if span.LowKey != nil && span.HighKey != nil && bytes.Compare(span.LowKey, span.HighKey) >= 0 { + return readOnlyPrepareSpanError(i, "low key %s is not before high key %s", readOnlyPrepareKeyForError(span.LowKey), readOnlyPrepareKeyForError(span.HighKey)) + } + if i > 0 && span.LowKey == nil { + return readOnlyPrepareSpanError(i, "has open low key after earlier span") + } + if i > 0 && prevHigh == nil { + return readOnlyPrepareSpanError(i, "follows previous span with open high key") + } + if i > 0 && bytes.Compare(span.LowKey, prevHigh) < 0 { + return readOnlyPrepareSpanError(i, "low key %s is before previous high key %s", readOnlyPrepareKeyForError(span.LowKey), readOnlyPrepareKeyForError(prevHigh)) + } + if span.LowKey != nil && bytes.Compare(span.FirstOpKey, span.LowKey) < 0 { + return readOnlyPrepareSpanError(i, "first op key %s is before low key %s", readOnlyPrepareKeyForError(span.FirstOpKey), readOnlyPrepareKeyForError(span.LowKey)) + } + if span.HighKey != nil && bytes.Compare(span.LastOpKey, span.HighKey) >= 0 { + return readOnlyPrepareSpanError(i, "last op key %s is not before high key %s", readOnlyPrepareKeyForError(span.LastOpKey), readOnlyPrepareKeyForError(span.HighKey)) + } + totalOps += span.OpCount + prevLastOp = span.LastOpKey + prevHigh = span.HighKey + } + if totalOps != r.Ops { + return fmt.Errorf("zipper: read-only leaf spans cover %d ops, want %d", totalOps, r.Ops) + } + return nil +} + +func readOnlyPrepareSpanError(spanIdx int, format string, args ...any) error { + args = append([]any{spanIdx}, args...) + return fmt.Errorf("zipper: read-only leaf span %d "+format, args...) +} + +func readOnlyPrepareKeyForError(key []byte) string { + const maxPrefix = 8 + if key == nil { + return "nil" + } + if len(key) <= maxPrefix { + return fmt.Sprintf("len=%d hex=%x", len(key), key) + } + return fmt.Sprintf("len=%d hex_prefix=%x", len(key), key[:maxPrefix]) +} + +func (r *ReadOnlyPrepareResult) cloneKey(src []byte) []byte { + if src == nil { + return nil + } + if len(src) == 0 { + return []byte{} + } + start := len(r.keyArena) + r.keyArena = append(r.keyArena, src...) + return r.keyArena[start : start+len(src)] +} + +func (r *ReadOnlyPrepareResult) addLeafSpan(ref page.ChildRef, low, high []byte, ops []batch.Entry) { + if len(ops) == 0 { + return + } + span := ReadOnlyLeafSpan{ + Ref: ref, + LowKey: r.cloneKey(low), + HighKey: r.cloneKey(high), + FirstOpKey: r.cloneKey(ops[0].Key), + LastOpKey: r.cloneKey(ops[len(ops)-1].Key), + OpCount: len(ops), + } + r.LeafSpans = append(r.LeafSpans, span) +} + +func elapsedNsSince(start time.Time) uint64 { + elapsed := time.Since(start) + if elapsed <= 0 { + return 0 + } + return uint64(elapsed.Nanoseconds()) +} + +// ApplyWithOptions applies the batch to the tree rooted at rootID and returns a +// result object suitable for guarded install paths. When opts.PrepareReadOnly is +// true, it first runs PrepareReadOnly and returns that planning metadata on the +// result. If the read-only preparation fails, the returned result may contain +// partial ReadOnlyPrepare metadata and no root output. +func (z *Zipper) ApplyWithOptions(rootID uint64, b *batch.Batch, opts ApplyOptions) (ApplyResult, error) { + var prepared ReadOnlyPrepareResult + var preparedNs uint64 + if opts.PrepareReadOnly { + var err error + prepareStart := time.Now() + prepared, err = z.PrepareReadOnly(rootID, b, opts.ReadOnlyPrepare) + preparedNs = elapsedNsSince(prepareStart) + if err != nil { + return ApplyResult{ReadOnlyPrepare: prepared, ReadOnlyPrepareNs: preparedNs}, err + } + } + newRoot, retired, metrics, err := z.Apply(rootID, b) + return ApplyResult{ + RootID: newRoot, + PendingRetiredPages: retired, + Metrics: metrics, + ReadOnlyPrepare: prepared, + ReadOnlyPrepareNs: preparedNs, + }, err +} + +// PrepareReadOnly discovers the existing leaf spans touched by b without +// allocating or writing pager/leaf-log output. It is the safe preparation phase +// that future prepared-output paths can run before the final install section. +func (z *Zipper) PrepareReadOnly(rootID uint64, b *batch.Batch, opts ReadOnlyPrepareOptions) (ReadOnlyPrepareResult, error) { + result := ReadOnlyPrepareResult{ + LeafSpans: opts.leafSpans[:0], + keyArena: opts.keyArena[:0], + } + if b == nil { + return result, errors.New("zipper: nil batch") + } + ops := b.SortedEntries() + result.RootID = rootID + result.Ops = len(ops) + if len(ops) == 0 { + result.ExactLeafSpans = true + return result, nil + } + maintenance, _ := z.shouldRunMaintenance(ops) + result.Maintenance = maintenance + result.ExactLeafSpans = !maintenance + if rootID == 0 { + result.ColdBuild = true + result.ExactLeafSpans = true + result.addLeafSpan(page.ChildRef{}, nil, nil, ops) + return result, nil + } + + scratch := z.acquireApplyScratch() + defer z.releaseApplyScratch(scratch) + err := z.prepareReadOnlyRecursive(page.PageChildRef(rootID), ops, nil, nil, &result, scratch) + return result, err +} + +func (z *Zipper) prepareReadOnlyRecursive(ref page.ChildRef, ops []batch.Entry, low, high []byte, result *ReadOnlyPrepareResult, scratch *mergeScratch) error { + oldNode, _, leafScratch, leafScratchRef, loadSource, err := z.loadNodeRef(ref, scratch) + if err != nil { + return err + } + recordZipperNodeLoad(&result.Metrics, ref, oldNode, loadSource) + if leafScratchRef { + defer releaseLeafPageScratch(scratch, leafScratch) + } + + switch oldNode.Type() { + case page.PageTypeLeaf, 0: + result.addLeafSpan(ref, low, high, ops) + return nil + case page.PageTypeInternal: + count := oldNode.Count() + opIdx := 0 + for i := uint16(0); i < count; i++ { + key, childRef, err := oldNode.GetInternalEntryRefView(i) + if err != nil { + return err + } + if key == nil { + key = []byte{} + } + useInheritedLow := len(key) == 0 + + var endKey []byte + if i+1 < count { + nextKey, _, err := oldNode.GetInternalEntryRefView(i + 1) + if err != nil { + return err + } + if nextKey == nil { + nextKey = []byte{} + } + endKey = nextKey + } + + startOpIdx := opIdx + for opIdx < len(ops) { + if endKey == nil || bytes.Compare(ops[opIdx].Key, endKey) < 0 { + opIdx++ + continue + } + break + } + if startOpIdx == opIdx { + continue + } + + childLow := low + childHigh := high + if endKey != nil { + childHigh = result.cloneKey(endKey) + } + if !useInheritedLow { + key, _, err := oldNode.GetInternalEntryRefView(i) + if err != nil { + return err + } + if key == nil { + key = []byte{} + } + childLow = result.cloneKey(key) + } + if err := z.prepareReadOnlyRecursive(childRef, ops[startOpIdx:opIdx], childLow, childHigh, result, scratch); err != nil { + return err + } + } + return nil + default: + return page.ErrInvalidPageType + } +} + // Apply applies the batch to the tree rooted at rootID. -// Returns the new root page ID, list of retired pages, and commit metrics. +// Returns the new root page ID, list of pending retired pages, and commit +// metrics. func (z *Zipper) Apply(rootID uint64, b *batch.Batch) (uint64, []uint64, adaptive.Metrics, error) { var metrics adaptive.Metrics ops := b.SortedEntries() @@ -1713,6 +2323,9 @@ func (z *Zipper) mergeInternal(oldNode *node.Node, builder *node.Builder, ops [] useParallel = shouldUseParallelInternalMerge(int(count), len(ops), gomaxprocs, maintenance, pressure) } } + if useParallel && z != nil && z.outerLeavesInValueLog && len(ops) < mergeInternalOuterLeafLogMinParallelOps { + useParallel = false + } copyKeys := oldNode.InternalBaseDeltaEnabled() var keyArena []byte @@ -1865,8 +2478,12 @@ func (z *Zipper) mergeInternal(oldNode *node.Node, builder *node.Builder, ops [] return page.PageChildRef(builder.PageID()), splits, nil } - children := getChildWorkSlice(int(count)) - defer putChildWorkSlice(children) + childBuf := getChildWorkBuffer(int(count)) + children := childBuf.items + defer func() { + childBuf.items = children + putChildWorkBuffer(childBuf) + }() for i := uint16(0); i < count; i++ { key, childRef, err := oldNode.GetInternalEntryRefView(i) @@ -1877,11 +2494,17 @@ func (z *Zipper) mergeInternal(oldNode *node.Node, builder *node.Builder, ops [] key = []byte{} } keyCopy := cloneKey(key) - children = append(children, childWork{ + children = children[:len(children)+1] + child := &children[len(children)-1] + retired := child.retired + *child = childWork{ key: keyCopy, low: keyCopy, child: childRef, - }) + } + if cap(retired) <= maxChildWorkRetiredKeepCap { + child.retired = retired[:0] + } } for i := range children { @@ -1915,7 +2538,7 @@ func (z *Zipper) mergeInternal(oldNode *node.Node, builder *node.Builder, ops [] if useParallel { const ( minParallelActiveChildren = 2 - minParallelOpsPerChild = 256 + minParallelOpsPerChild = 4 ) if activeChildren < minParallelActiveChildren || len(ops)/activeChildren < minParallelOpsPerChild { useParallel = false @@ -1941,9 +2564,13 @@ func (z *Zipper) mergeInternal(oldNode *node.Node, builder *node.Builder, ops [] if activeChildren > 0 && maxParallel > activeChildren { maxParallel = activeChildren } + if maxParallel > mergeInternalMaxParallelWorkers { + maxParallel = mergeInternalMaxParallelWorkers + } if maxParallel < 1 { maxParallel = 1 } + recordZipperInternalParallelMerge(metrics, activeChildren, maxParallel, len(ops)) for i := range children { if len(children[i].ops) == 0 { children[i].newChild = children[i].child @@ -1963,17 +2590,15 @@ func (z *Zipper) mergeInternal(oldNode *node.Node, builder *node.Builder, ops [] if len(children[i].ops) == 0 { continue } - var childMetrics adaptive.Metrics - childRet := children[i].retired[:0] - ncID, cs, err := z.writeRecursive(children[i].child, children[i].ops, maintenance, budget, &childMetrics, children[i].low, children[i].high, &childRet, scratch) + children[i].childStat = adaptive.Metrics{} + children[i].retired = children[i].retired[:0] + ncID, cs, err := z.writeRecursive(children[i].child, children[i].ops, maintenance, budget, &children[i].childStat, children[i].low, children[i].high, &children[i].retired, scratch) if err != nil { errOnce.Do(func() { firstErr = err }) continue } children[i].newChild = ncID children[i].splits = cs - children[i].retired = childRet - children[i].childStat = childMetrics } } for i := 0; i < maxParallel; i++ { @@ -1984,15 +2609,15 @@ func (z *Zipper) mergeInternal(oldNode *node.Node, builder *node.Builder, ops [] if firstErr != nil { return page.ChildRef{}, nil, firstErr } + totalRetired := 0 for i := range children { if len(children[i].ops) == 0 { continue } mergeMetrics(metrics, &children[i].childStat) - if retired != nil && len(children[i].retired) > 0 { - *retired = append(*retired, children[i].retired...) - } + totalRetired += len(children[i].retired) } + appendChildRetiredPages(retired, children, totalRetired) } else { for i := range children { if len(children[i].ops) > 0 { @@ -2111,6 +2736,12 @@ func mergeMetrics(dst, src *adaptive.Metrics) { dst.ZipperLeafLogRecordHintBytesRead += src.ZipperLeafLogRecordHintBytesRead dst.ZipperLeafMerges += src.ZipperLeafMerges dst.ZipperInternalMerges += src.ZipperInternalMerges + if src.ZipperInternalParallelMerges != 0 { + dst.ZipperInternalParallelMerges += src.ZipperInternalParallelMerges + dst.ZipperInternalParallelChildren += src.ZipperInternalParallelChildren + dst.ZipperInternalParallelWorkers += src.ZipperInternalParallelWorkers + dst.ZipperInternalParallelOps += src.ZipperInternalParallelOps + } dst.ZipperLeafPagesWritten += src.ZipperLeafPagesWritten dst.ZipperPagerLeafPagesWritten += src.ZipperPagerLeafPagesWritten dst.ZipperLeafLogPagesWritten += src.ZipperLeafLogPagesWritten diff --git a/TreeDB/zipper/zipper_test.go b/TreeDB/zipper/zipper_test.go index 4bd2074dc8..8b8e38ab26 100644 --- a/TreeDB/zipper/zipper_test.go +++ b/TreeDB/zipper/zipper_test.go @@ -7,7 +7,9 @@ import ( "io" "math/rand" "path/filepath" + "runtime" "strings" + "sync" "sync/atomic" "testing" @@ -27,6 +29,30 @@ func (m *MockAllocator) Alloc(hint uint64) (uint64, error) { return m.p.Alloc(1) } +type recyclingMockAllocator struct { + p *pager.Pager + mu sync.Mutex + retired []uint64 +} + +func (m *recyclingMockAllocator) Alloc(hint uint64) (uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() + n := len(m.retired) + if n > 0 { + id := m.retired[n-1] + m.retired = m.retired[:n-1] + return id, nil + } + return m.p.Alloc(1) +} + +func (m *recyclingMockAllocator) Recycle(ids []uint64) { + m.mu.Lock() + defer m.mu.Unlock() + m.retired = append(m.retired, ids...) +} + type panicValueReader struct{} func (panicValueReader) Read(ptr page.ValuePtr) ([]byte, error) { @@ -142,16 +168,16 @@ func (s *memoryLeafPageStore) resetObservations() { s.sawNoCache = 0 } -func buildOuterLeafInternalRoot(t *testing.T, z *Zipper) uint64 { - t.Helper() +func buildOuterLeafInternalRoot(tb testing.TB, z *Zipper) uint64 { + tb.Helper() rootID, err := z.pager.Alloc(1) if err != nil { - t.Fatalf("alloc root: %v", err) + tb.Fatalf("alloc root: %v", err) } data, err := z.pager.Get(rootID) if err != nil { - t.Fatalf("get root: %v", err) + tb.Fatalf("get root: %v", err) } n := node.NewNode(data) n.SetPageID(rootID) @@ -168,18 +194,1386 @@ func buildOuterLeafInternalRoot(t *testing.T, z *Zipper) uint64 { newRootID, _, _, err := z.Apply(rootID, b) if err != nil { - t.Fatalf("build root apply: %v", err) + tb.Fatalf("build root apply: %v", err) } rootData, err := z.pager.Get(newRootID) if err != nil { - t.Fatalf("get new root: %v", err) + tb.Fatalf("get new root: %v", err) } if got := node.NewNode(rootData).Type(); got != page.PageTypeInternal { - t.Fatalf("new root type=%d want %d", got, page.PageTypeInternal) + tb.Fatalf("new root type=%d want %d", got, page.PageTypeInternal) + } + return newRootID +} + +func buildInternalRootWithKeys(tb testing.TB, z *Zipper, count int) uint64 { + tb.Helper() + + rootID, err := z.pager.Alloc(1) + if err != nil { + tb.Fatalf("alloc root: %v", err) + } + data, err := z.pager.Get(rootID) + if err != nil { + tb.Fatalf("get root: %v", err) + } + n := node.NewNode(data) + n.SetPageID(rootID) + n.SetType(page.PageTypeLeaf) + n.UpdateChecksum() + + b := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = b.Close() }() + value := bytes.Repeat([]byte("v"), 128) + for i := 0; i < count; i++ { + key := []byte(fmt.Sprintf("key-%06d", i)) + b.Set(key, value) + } + + newRootID, _, _, err := z.Apply(rootID, b) + if err != nil { + tb.Fatalf("build %d-key root apply: %v", count, err) } return newRootID } +func rootHasInternalChild(tb testing.TB, z *Zipper, rootID uint64) bool { + tb.Helper() + + scratch := z.acquireApplyScratch() + defer z.releaseApplyScratch(scratch) + + root, _, leafScratch, leafScratchRef, _, err := z.loadNodeRef(page.PageChildRef(rootID), scratch) + if err != nil { + tb.Fatalf("load root: %v", err) + } + if leafScratchRef { + defer releaseLeafPageScratch(scratch, leafScratch) + } + if root.Type() != page.PageTypeInternal { + return false + } + for i := uint16(0); i < root.Count(); i++ { + _, childRef, err := root.GetInternalEntryRefView(i) + if err != nil { + tb.Fatalf("root child %d: %v", i, err) + } + child, _, childLeafScratch, childLeafScratchRef, _, err := z.loadNodeRef(childRef, scratch) + if err != nil { + tb.Fatalf("load child %d: %v", i, err) + } + childType := child.Type() + if childLeafScratchRef { + releaseLeafPageScratch(scratch, childLeafScratch) + } + if childType == page.PageTypeInternal { + return true + } + } + return false +} + +func buildMultiLevelInternalRoot(tb testing.TB, z *Zipper) (uint64, int) { + tb.Helper() + + for count := 1024; count <= 32768; count *= 2 { + rootID := buildInternalRootWithKeys(tb, z, count) + if rootHasInternalChild(tb, z, rootID) { + return rootID, count + } + } + tb.Fatal("failed to build a multi-level internal root") + return 0, 0 +} + +func requireValidReadOnlyPrepare(tb testing.TB, prepared ReadOnlyPrepareResult) { + tb.Helper() + if err := prepared.ValidateLeafSpans(); err != nil { + tb.Fatalf("ValidateLeafSpans: %v", err) + } +} + +func TestZipperPrepareReadOnlyColdBuildDoesNotLoadOrWrite(t *testing.T) { + b := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = b.Close() }() + b.Set([]byte("a"), []byte("1")) + b.Set([]byte("z"), []byte("2")) + + var z Zipper + prepared, err := z.PrepareReadOnly(0, b, ReadOnlyPrepareOptions{}) + if err != nil { + t.Fatalf("PrepareReadOnly: %v", err) + } + requireValidReadOnlyPrepare(t, prepared) + if !prepared.ColdBuild { + t.Fatal("ColdBuild=false want true") + } + if prepared.Maintenance || !prepared.ExactLeafSpans { + t.Fatalf("cold prepare maintenance/exact=%v/%v want false/true", prepared.Maintenance, prepared.ExactLeafSpans) + } + if prepared.RootID != 0 || prepared.Ops != 2 { + t.Fatalf("prepared root/ops=%d/%d want 0/2", prepared.RootID, prepared.Ops) + } + if len(prepared.LeafSpans) != 1 { + t.Fatalf("leaf spans=%d want 1", len(prepared.LeafSpans)) + } + span := prepared.LeafSpans[0] + if span.Ref != (page.ChildRef{}) { + t.Fatalf("cold span ref=%+v want zero ChildRef", span.Ref) + } + if span.OpCount != 2 || string(span.FirstOpKey) != "a" || string(span.LastOpKey) != "z" { + t.Fatalf("cold span=%+v want two ops from a to z", span) + } + if prepared.Metrics.ZipperNodeLoads != 0 || prepared.Metrics.IndexWriteBytes != 0 { + t.Fatalf("cold prepare metrics=%+v want no node load/write", prepared.Metrics) + } +} + +func TestZipperPrepareReadOnlyEmptyBatchDoesNotTraverse(t *testing.T) { + dir := t.TempDir() + p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) + if err != nil { + t.Fatal(err) + } + defer p.Close() + + alloc := &MockAllocator{p: p} + z := New(p, alloc) + rootID := buildOuterLeafInternalRoot(t, z) + beforePages := p.PageCount() + + b := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = b.Close() }() + prepared, err := z.PrepareReadOnly(rootID, b, ReadOnlyPrepareOptions{}) + if err != nil { + t.Fatalf("PrepareReadOnly: %v", err) + } + requireValidReadOnlyPrepare(t, prepared) + if got := p.PageCount(); got != beforePages { + t.Fatalf("page count changed during empty read-only prepare: got %d want %d", got, beforePages) + } + if prepared.RootID != rootID || prepared.Ops != 0 || !prepared.ExactLeafSpans { + t.Fatalf("prepared=%+v want root %d zero ops exact", prepared, rootID) + } + if len(prepared.LeafSpans) != 0 { + t.Fatalf("empty batch spans=%d want 0", len(prepared.LeafSpans)) + } + if prepared.Metrics.ZipperNodeLoads != 0 { + t.Fatalf("empty batch traversed tree metrics=%+v", prepared.Metrics) + } +} + +func TestZipperPrepareReadOnlyExistingLeafRoot(t *testing.T) { + dir := t.TempDir() + p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) + if err != nil { + t.Fatal(err) + } + defer p.Close() + + alloc := &MockAllocator{p: p} + z := New(p, alloc) + rootID := buildInternalRootWithKeys(t, z, 8) + rootData, err := p.Get(rootID) + if err != nil { + t.Fatalf("get root: %v", err) + } + if got := node.NewNode(rootData).Type(); got != page.PageTypeLeaf { + t.Fatalf("root type=%d want leaf", got) + } + + b := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = b.Close() }() + b.Set([]byte("key-000003"), []byte("new")) + + prepared, err := z.PrepareReadOnly(rootID, b, ReadOnlyPrepareOptions{}) + if err != nil { + t.Fatalf("PrepareReadOnly: %v", err) + } + requireValidReadOnlyPrepare(t, prepared) + if prepared.ColdBuild || prepared.Maintenance || !prepared.ExactLeafSpans { + t.Fatalf("prepare cold/maintenance/exact=%v/%v/%v want false/false/true", prepared.ColdBuild, prepared.Maintenance, prepared.ExactLeafSpans) + } + if len(prepared.LeafSpans) != 1 { + t.Fatalf("leaf spans=%d want 1", len(prepared.LeafSpans)) + } + span := prepared.LeafSpans[0] + if span.Ref != page.PageChildRef(rootID) { + t.Fatalf("span ref=%+v want page root %d", span.Ref, rootID) + } + if string(span.FirstOpKey) != "key-000003" || string(span.LastOpKey) != "key-000003" || span.OpCount != 1 { + t.Fatalf("span=%+v want one key-000003 op", span) + } +} + +func TestZipperPrepareReadOnlyDiscoversLeafSpansWithoutWrites(t *testing.T) { + dir := t.TempDir() + p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) + if err != nil { + t.Fatal(err) + } + defer p.Close() + + alloc := &MockAllocator{p: p} + z := New(p, alloc) + rootID := buildOuterLeafInternalRoot(t, z) + beforePages := p.PageCount() + + b := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = b.Close() }() + b.Set([]byte("key-001"), []byte("new-001")) + b.Set([]byte("key-199"), []byte("new-199")) + + prepared, err := z.PrepareReadOnly(rootID, b, ReadOnlyPrepareOptions{}) + if err != nil { + t.Fatalf("PrepareReadOnly: %v", err) + } + requireValidReadOnlyPrepare(t, prepared) + if got := p.PageCount(); got != beforePages { + t.Fatalf("page count changed during read-only prepare: got %d want %d", got, beforePages) + } + if prepared.ColdBuild { + t.Fatal("ColdBuild=true for existing root") + } + if prepared.Maintenance || !prepared.ExactLeafSpans { + t.Fatalf("prepare maintenance/exact=%v/%v want false/true", prepared.Maintenance, prepared.ExactLeafSpans) + } + if prepared.RootID != rootID || prepared.Ops != 2 { + t.Fatalf("prepared root/ops=%d/%d want %d/2", prepared.RootID, prepared.Ops, rootID) + } + if len(prepared.LeafSpans) < 2 { + t.Fatalf("leaf spans=%d want at least 2 for distant keys", len(prepared.LeafSpans)) + } + if prepared.Metrics.ZipperNodeLoads == 0 { + t.Fatalf("node loads=0 want read-only traversal loads") + } + if prepared.Metrics.IndexWriteBytes != 0 || + prepared.Metrics.ZipperLeafPagesWritten != 0 || + prepared.Metrics.ZipperInternalPagesWritten != 0 { + t.Fatalf("read-only prepare wrote output metrics=%+v", prepared.Metrics) + } + for _, span := range prepared.LeafSpans { + if span.OpCount <= 0 { + t.Fatalf("empty span recorded: %+v", span) + } + if len(span.FirstOpKey) == 0 || len(span.LastOpKey) == 0 { + t.Fatalf("span missing op bounds: %+v", span) + } + } +} + +func TestZipperPrepareReadOnlyReuseOptions(t *testing.T) { + dir := t.TempDir() + p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) + if err != nil { + t.Fatal(err) + } + defer p.Close() + + alloc := &MockAllocator{p: p} + z := New(p, alloc) + rootID := buildOuterLeafInternalRoot(t, z) + + firstBatch := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = firstBatch.Close() }() + firstBatch.Set([]byte("key-001"), []byte("one")) + firstBatch.Set([]byte("key-199"), []byte("two")) + + first, err := z.PrepareReadOnly(rootID, firstBatch, ReadOnlyPrepareOptions{}) + if err != nil { + t.Fatalf("first PrepareReadOnly: %v", err) + } + requireValidReadOnlyPrepare(t, first) + if len(first.LeafSpans) == 0 { + t.Fatal("first prepare returned no spans") + } + opts := first.ReuseOptions() + + secondBatch := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = secondBatch.Close() }() + secondBatch.Set([]byte("key-067"), []byte("three")) + + second, err := z.PrepareReadOnly(rootID, secondBatch, opts) + if err != nil { + t.Fatalf("second PrepareReadOnly: %v", err) + } + requireValidReadOnlyPrepare(t, second) + if len(second.LeafSpans) != 1 { + t.Fatalf("second spans=%d want 1", len(second.LeafSpans)) + } + span := second.LeafSpans[0] + if string(span.FirstOpKey) != "key-067" || string(span.LastOpKey) != "key-067" || span.OpCount != 1 { + t.Fatalf("second reused span=%+v want key-067", span) + } +} + +func TestZipperPrepareReadOnlyMarksDeleteMaintenanceSpansNonExact(t *testing.T) { + dir := t.TempDir() + p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) + if err != nil { + t.Fatal(err) + } + defer p.Close() + + alloc := &MockAllocator{p: p} + z := New(p, alloc) + rootID := buildOuterLeafInternalRoot(t, z) + beforePages := p.PageCount() + + b := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = b.Close() }() + b.Delete([]byte("key-001")) + b.Delete([]byte("key-199")) + + prepared, err := z.PrepareReadOnly(rootID, b, ReadOnlyPrepareOptions{}) + if err != nil { + t.Fatalf("PrepareReadOnly: %v", err) + } + requireValidReadOnlyPrepare(t, prepared) + if got := p.PageCount(); got != beforePages { + t.Fatalf("page count changed during read-only prepare: got %d want %d", got, beforePages) + } + if !prepared.Maintenance { + t.Fatal("Maintenance=false want true for delete-containing batch") + } + if prepared.ExactLeafSpans { + t.Fatal("ExactLeafSpans=true want false for delete maintenance") + } + if len(prepared.LeafSpans) == 0 { + t.Fatal("delete maintenance still should expose direct planning spans") + } + if prepared.Metrics.IndexWriteBytes != 0 || + prepared.Metrics.ZipperLeafPagesWritten != 0 || + prepared.Metrics.ZipperInternalPagesWritten != 0 { + t.Fatalf("delete read-only prepare wrote output metrics=%+v", prepared.Metrics) + } +} + +func TestZipperPrepareReadOnlyInternalBaseDeltaKeyBoundsAreStable(t *testing.T) { + dir := t.TempDir() + p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) + if err != nil { + t.Fatal(err) + } + defer p.Close() + + alloc := &MockAllocator{p: p} + z := New(p, alloc) + z.SetIndexInternalBaseDelta(true) + rootID := buildOuterLeafInternalRoot(t, z) + beforePages := p.PageCount() + + b := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = b.Close() }() + b.Set([]byte("key-001"), []byte("new-001")) + b.Set([]byte("key-199"), []byte("new-199")) + + prepared, err := z.PrepareReadOnly(rootID, b, ReadOnlyPrepareOptions{}) + if err != nil { + t.Fatalf("PrepareReadOnly: %v", err) + } + requireValidReadOnlyPrepare(t, prepared) + if got := p.PageCount(); got != beforePages { + t.Fatalf("page count changed during read-only prepare: got %d want %d", got, beforePages) + } + if prepared.Maintenance || !prepared.ExactLeafSpans { + t.Fatalf("prepare maintenance/exact=%v/%v want false/true", prepared.Maintenance, prepared.ExactLeafSpans) + } + if len(prepared.LeafSpans) < 2 { + t.Fatalf("leaf spans=%d want at least 2", len(prepared.LeafSpans)) + } + for _, span := range prepared.LeafSpans { + if len(span.LowKey) > 0 && bytes.Compare(span.LowKey, span.FirstOpKey) > 0 { + t.Fatalf("span low bound %q is after first op %q; span=%+v", span.LowKey, span.FirstOpKey, span) + } + if len(span.HighKey) > 0 && bytes.Compare(span.FirstOpKey, span.HighKey) >= 0 { + t.Fatalf("span high bound %q is not after first op %q; span=%+v", span.HighKey, span.FirstOpKey, span) + } + } +} + +func TestZipperPrepareReadOnlyNestedInternalBoundsInheritParentRange(t *testing.T) { + dir := t.TempDir() + p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) + if err != nil { + t.Fatal(err) + } + defer p.Close() + + alloc := &MockAllocator{p: p} + z := New(p, alloc) + z.SetIndexInternalBaseDelta(true) + rootID, count := buildMultiLevelInternalRoot(t, z) + beforePages := p.PageCount() + + b := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = b.Close() }() + opKeys := [][]byte{ + []byte("key-000001"), + []byte(fmt.Sprintf("key-%06d", count/4)), + []byte(fmt.Sprintf("key-%06d", count/2)), + []byte(fmt.Sprintf("key-%06d", count-2)), + } + for _, key := range opKeys { + b.Set(key, []byte("new")) + } + + prepared, err := z.PrepareReadOnly(rootID, b, ReadOnlyPrepareOptions{}) + if err != nil { + t.Fatalf("PrepareReadOnly: %v", err) + } + requireValidReadOnlyPrepare(t, prepared) + if got := p.PageCount(); got != beforePages { + t.Fatalf("page count changed during read-only prepare: got %d want %d", got, beforePages) + } + if prepared.Maintenance || !prepared.ExactLeafSpans { + t.Fatalf("prepare maintenance/exact=%v/%v want false/true", prepared.Maintenance, prepared.ExactLeafSpans) + } + if len(prepared.LeafSpans) < 3 { + t.Fatalf("leaf spans=%d want at least 3 for sparse multi-level keys", len(prepared.LeafSpans)) + } + for _, span := range prepared.LeafSpans { + if len(span.FirstOpKey) == 0 { + t.Fatalf("span missing first op key: %+v", span) + } + if bytes.Compare(span.FirstOpKey, []byte("key-000001")) > 0 && len(span.LowKey) == 0 { + t.Fatalf("span for non-leftmost op has empty inherited low bound: %+v", span) + } + if span.LowKey != nil && bytes.Compare(span.LowKey, span.FirstOpKey) > 0 { + t.Fatalf("span low bound %q is after first op %q; span=%+v", span.LowKey, span.FirstOpKey, span) + } + if span.HighKey != nil && bytes.Compare(span.FirstOpKey, span.HighKey) >= 0 { + t.Fatalf("span high bound %q is not after first op %q; span=%+v", span.HighKey, span.FirstOpKey, span) + } + } +} + +func newTestZipperWithOuterLeafInternalRoot(tb testing.TB) (*Zipper, uint64) { + tb.Helper() + dir := tb.TempDir() + p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { _ = p.Close() }) + + alloc := &MockAllocator{p: p} + z := New(p, alloc) + rootID := buildOuterLeafInternalRoot(tb, z) + return z, rootID +} + +func TestZipperApplyWithOptionsReturnsReadOnlyPrepare(t *testing.T) { + z, rootID := newTestZipperWithOuterLeafInternalRoot(t) + + delta := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = delta.Close() }() + delta.Set([]byte("key-001"), []byte("new-001")) + delta.Set([]byte("key-067"), []byte("new-067")) + delta.Set([]byte("key-133"), []byte("new-133")) + + result, err := z.ApplyWithOptions(rootID, delta, ApplyOptions{ + PrepareReadOnly: true, + }) + if err != nil { + t.Fatalf("ApplyWithOptions: %v", err) + } + if result.RootID == 0 || result.RootID == rootID { + t.Fatalf("result root=%d want new non-zero root different from %d", result.RootID, rootID) + } + if len(result.PendingRetiredPages) == 0 { + t.Fatal("expected pending retired pages from warm apply") + } + prepared := result.ReadOnlyPrepare + requireValidReadOnlyPrepare(t, prepared) + if prepared.RootID != rootID { + t.Fatalf("prepared root=%d want %d", prepared.RootID, rootID) + } + if prepared.Ops != 3 { + t.Fatalf("prepared ops=%d want 3", prepared.Ops) + } + if len(prepared.LeafSpans) == 0 { + t.Fatal("expected read-only leaf spans") + } + if prepared.ColdBuild || prepared.Maintenance || !prepared.ExactLeafSpans { + t.Fatalf("prepared flags cold/maintenance/exact=%v/%v/%v want false/false/true", prepared.ColdBuild, prepared.Maintenance, prepared.ExactLeafSpans) + } +} + +func TestZipperApplyWithOptionsDefaultSkipsReadOnlyPrepare(t *testing.T) { + z, rootID := newTestZipperWithOuterLeafInternalRoot(t) + + delta := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = delta.Close() }() + delta.Set([]byte("key-001"), []byte("new-001")) + + result, err := z.ApplyWithOptions(rootID, delta, ApplyOptions{}) + if err != nil { + t.Fatalf("ApplyWithOptions: %v", err) + } + if result.RootID == 0 || result.RootID == rootID { + t.Fatalf("result root=%d want new non-zero root different from %d", result.RootID, rootID) + } + if result.ReadOnlyPrepare.RootID != 0 || + result.ReadOnlyPrepare.Ops != 0 || + len(result.ReadOnlyPrepare.LeafSpans) != 0 { + t.Fatalf("read-only prepare populated without opt-in: %+v", result.ReadOnlyPrepare) + } +} + +func TestZipperApplyWarmSparseManyLeafPreservesValues(t *testing.T) { + prevGOMAXPROCS := runtime.GOMAXPROCS(8) + defer runtime.GOMAXPROCS(prevGOMAXPROCS) + + dir := t.TempDir() + p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) + if err != nil { + t.Fatal(err) + } + defer p.Close() + + const ( + keyCount = 8192 + step = 4 + ) + alloc := &MockAllocator{p: p} + z := New(p, alloc) + rootID := buildInternalRootWithKeys(t, z, keyCount) + + delta := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = delta.Close() }() + for i := 0; i < keyCount; i += step { + key := []byte(fmt.Sprintf("key-%06d", i)) + delta.Set(key, []byte("parallel")) + } + + newRoot, retired, metrics, err := z.Apply(rootID, delta) + if err != nil { + t.Fatalf("Apply: %v", err) + } + if newRoot == 0 || newRoot == rootID { + t.Fatalf("new root=%d old root=%d want changed non-zero root", newRoot, rootID) + } + if got := metrics.ZipperApplyOps; got != keyCount/step { + t.Fatalf("ZipperApplyOps=%d want %d", got, keyCount/step) + } + if got := metrics.ZipperLeafMerges; got < 2 { + t.Fatalf("ZipperLeafMerges=%d want multi-leaf apply", got) + } + if got := metrics.ZipperInternalParallelMerges; got == 0 { + t.Fatalf("ZipperInternalParallelMerges=%d want parallel internal merge", got) + } + if got := metrics.ZipperInternalParallelChildren; got < 2 { + t.Fatalf("ZipperInternalParallelChildren=%d want multiple active children", got) + } + if got := metrics.ZipperInternalParallelWorkers; got != mergeInternalMaxParallelWorkers { + t.Fatalf("ZipperInternalParallelWorkers=%d want capped workers %d", got, mergeInternalMaxParallelWorkers) + } + if got := metrics.ZipperInternalParallelOps; got == 0 { + t.Fatalf("ZipperInternalParallelOps=%d want routed parallel ops", got) + } + if len(retired) == 0 { + t.Fatal("expected pending retired pages from warm apply") + } + + tr := tree.New(p, panicValueReader{}, newRoot) + for _, i := range []int{0, 4, 2044, 4096, 8188} { + key := []byte(fmt.Sprintf("key-%06d", i)) + got, err := tr.Get(key) + if err != nil { + t.Fatalf("Get(%q): %v", key, err) + } + if !bytes.Equal(got, []byte("parallel")) { + t.Fatalf("Get(%q)=%q want parallel", key, got) + } + } + originalValue := bytes.Repeat([]byte("v"), 128) + for _, i := range []int{1, 5, 2045, 4097, 8191} { + key := []byte(fmt.Sprintf("key-%06d", i)) + got, err := tr.Get(key) + if err != nil { + t.Fatalf("Get(%q): %v", key, err) + } + if !bytes.Equal(got, originalValue) { + t.Fatalf("Get(%q)=%q want original value", key, got) + } + } +} + +func TestZipperApplyWithOptionsReusesReadOnlyPrepareBuffers(t *testing.T) { + z, rootID := newTestZipperWithOuterLeafInternalRoot(t) + + delta := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = delta.Close() }() + delta.Set([]byte("key-001"), []byte("new-001")) + delta.Set([]byte("key-067"), []byte("new-067")) + + prepared, err := z.PrepareReadOnly(rootID, delta, ReadOnlyPrepareOptions{}) + if err != nil { + t.Fatalf("PrepareReadOnly: %v", err) + } + if len(prepared.LeafSpans) == 0 { + t.Fatal("expected initial read-only leaf spans") + } + opts := ApplyOptions{ + PrepareReadOnly: true, + ReadOnlyPrepare: prepared.ReuseOptions(), + } + if cap(opts.ReadOnlyPrepare.leafSpans) == 0 { + t.Fatal("expected reusable leaf-span capacity") + } + if cap(opts.ReadOnlyPrepare.keyArena) == 0 { + t.Fatal("expected reusable key arena capacity") + } + leafSpanBase := &opts.ReadOnlyPrepare.leafSpans[:cap(opts.ReadOnlyPrepare.leafSpans)][0] + keyArenaBase := &opts.ReadOnlyPrepare.keyArena[:cap(opts.ReadOnlyPrepare.keyArena)][0] + + result, err := z.ApplyWithOptions(rootID, delta, opts) + if err != nil { + t.Fatalf("ApplyWithOptions: %v", err) + } + if len(result.ReadOnlyPrepare.LeafSpans) == 0 { + t.Fatal("expected reused read-only leaf spans") + } + if &result.ReadOnlyPrepare.LeafSpans[0] != leafSpanBase { + t.Fatal("read-only prepare leaf-span buffer was not reused") + } + if len(result.ReadOnlyPrepare.keyArena) == 0 || &result.ReadOnlyPrepare.keyArena[0] != keyArenaBase { + t.Fatal("read-only prepare key arena was not reused") + } +} + +func TestReadOnlyPrepareResultResetForReuseKeepsBoundedBuffers(t *testing.T) { + result := ReadOnlyPrepareResult{ + RootID: 123, + Ops: 2, + ExactLeafSpans: true, + LeafSpans: make([]ReadOnlyLeafSpan, 2, 8), + keyArena: make([]byte, 4, 16), + } + leafBase := &result.LeafSpans[:cap(result.LeafSpans)][0] + keyBase := &result.keyArena[:cap(result.keyArena)][0] + + result.ResetForReuse() + if result.RootID != 0 || result.Ops != 0 || result.ExactLeafSpans { + t.Fatalf("metadata not reset: %+v", result) + } + if len(result.LeafSpans) != 0 || cap(result.LeafSpans) != 8 { + t.Fatalf("leaf spans len/cap=%d/%d want 0/8", len(result.LeafSpans), cap(result.LeafSpans)) + } + if &result.LeafSpans[:cap(result.LeafSpans)][0] != leafBase { + t.Fatal("leaf span buffer was not retained") + } + if len(result.keyArena) != 0 || cap(result.keyArena) != 16 { + t.Fatalf("key arena len/cap=%d/%d want 0/16", len(result.keyArena), cap(result.keyArena)) + } + if &result.keyArena[:cap(result.keyArena)][0] != keyBase { + t.Fatal("key arena buffer was not retained") + } +} + +func TestReadOnlyPrepareResultResetForReuseDropsOversizedBuffers(t *testing.T) { + result := ReadOnlyPrepareResult{ + LeafSpans: make([]ReadOnlyLeafSpan, 1, readOnlyPrepareResultReuseLeafSpanKeepCap+1), + keyArena: make([]byte, 1, readOnlyPrepareResultReuseKeyArenaKeepCap+1), + } + + result.ResetForReuse() + if cap(result.LeafSpans) != 0 { + t.Fatalf("leaf span cap=%d want dropped", cap(result.LeafSpans)) + } + if cap(result.keyArena) != 0 { + t.Fatalf("key arena cap=%d want dropped", cap(result.keyArena)) + } +} + +func TestReadOnlyPrepareResultValidateLeafSpansRejectsInvalidPlans(t *testing.T) { + validSpan := ReadOnlyLeafSpan{ + LowKey: []byte("a"), + HighKey: []byte("z"), + FirstOpKey: []byte("b"), + LastOpKey: []byte("c"), + OpCount: 2, + } + tests := []struct { + name string + in ReadOnlyPrepareResult + }{ + { + name: "zero ops with span", + in: ReadOnlyPrepareResult{ + Ops: 0, + LeafSpans: []ReadOnlyLeafSpan{validSpan}, + }, + }, + { + name: "missing spans", + in: ReadOnlyPrepareResult{Ops: 1}, + }, + { + name: "cold build multiple spans", + in: ReadOnlyPrepareResult{ + Ops: 2, + ColdBuild: true, + LeafSpans: []ReadOnlyLeafSpan{ + {FirstOpKey: []byte("a"), LastOpKey: []byte("a"), OpCount: 1}, + {FirstOpKey: []byte("b"), LastOpKey: []byte("b"), OpCount: 1}, + }, + }, + }, + { + name: "empty op key", + in: ReadOnlyPrepareResult{ + Ops: 1, + LeafSpans: []ReadOnlyLeafSpan{{LastOpKey: []byte("b"), OpCount: 1}}, + }, + }, + { + name: "reversed op keys", + in: ReadOnlyPrepareResult{ + Ops: 1, + LeafSpans: []ReadOnlyLeafSpan{{FirstOpKey: []byte("c"), LastOpKey: []byte("b"), OpCount: 1}}, + }, + }, + { + name: "overlapping op key ranges", + in: ReadOnlyPrepareResult{ + Ops: 2, + LeafSpans: []ReadOnlyLeafSpan{ + {FirstOpKey: []byte("b"), LastOpKey: []byte("d"), OpCount: 1}, + {FirstOpKey: []byte("d"), LastOpKey: []byte("e"), OpCount: 1}, + }, + }, + }, + { + name: "bad bounds", + in: ReadOnlyPrepareResult{ + Ops: 1, + LeafSpans: []ReadOnlyLeafSpan{{LowKey: []byte("z"), HighKey: []byte("a"), FirstOpKey: []byte("m"), LastOpKey: []byte("m"), OpCount: 1}}, + }, + }, + { + name: "second span open low bound", + in: ReadOnlyPrepareResult{ + Ops: 2, + LeafSpans: []ReadOnlyLeafSpan{ + {HighKey: []byte("m"), FirstOpKey: []byte("a"), LastOpKey: []byte("b"), OpCount: 1}, + {FirstOpKey: []byte("n"), LastOpKey: []byte("n"), OpCount: 1}, + }, + }, + }, + { + name: "non-final open high bound", + in: ReadOnlyPrepareResult{ + Ops: 2, + LeafSpans: []ReadOnlyLeafSpan{ + {FirstOpKey: []byte("a"), LastOpKey: []byte("b"), OpCount: 1}, + {LowKey: []byte("m"), FirstOpKey: []byte("n"), LastOpKey: []byte("n"), OpCount: 1}, + }, + }, + }, + { + name: "overlapping bounds", + in: ReadOnlyPrepareResult{ + Ops: 2, + LeafSpans: []ReadOnlyLeafSpan{ + {HighKey: []byte("m"), FirstOpKey: []byte("a"), LastOpKey: []byte("b"), OpCount: 1}, + {LowKey: []byte("c"), FirstOpKey: []byte("d"), LastOpKey: []byte("d"), OpCount: 1}, + }, + }, + }, + { + name: "op before low bound", + in: ReadOnlyPrepareResult{ + Ops: 1, + LeafSpans: []ReadOnlyLeafSpan{{LowKey: []byte("c"), FirstOpKey: []byte("b"), LastOpKey: []byte("b"), OpCount: 1}}, + }, + }, + { + name: "op at high bound", + in: ReadOnlyPrepareResult{ + Ops: 1, + LeafSpans: []ReadOnlyLeafSpan{{HighKey: []byte("b"), FirstOpKey: []byte("b"), LastOpKey: []byte("b"), OpCount: 1}}, + }, + }, + { + name: "op count mismatch", + in: ReadOnlyPrepareResult{ + Ops: 3, + LeafSpans: []ReadOnlyLeafSpan{validSpan}, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if err := tc.in.ValidateLeafSpans(); err == nil { + t.Fatal("ValidateLeafSpans returned nil, want error") + } + }) + } +} + +func TestReadOnlyPrepareResultValidateLeafSpansFormatsKeysSafely(t *testing.T) { + longKey := []byte("0123456789abcdef") + prepared := ReadOnlyPrepareResult{ + Ops: 1, + LeafSpans: []ReadOnlyLeafSpan{ + {FirstOpKey: longKey, LastOpKey: []byte("0"), OpCount: 1}, + }, + } + + err := prepared.ValidateLeafSpans() + if err == nil { + t.Fatal("ValidateLeafSpans returned nil, want key-order error") + } + msg := err.Error() + if strings.Contains(msg, string(longKey)) { + t.Fatalf("error leaked full raw key %q: %s", longKey, msg) + } + if !strings.Contains(msg, "len=16") || !strings.Contains(msg, "hex_prefix=3031323334353637") { + t.Fatalf("error missing safe key summary: %s", msg) + } +} + +func TestReadOnlyPrepareResultValidateLeafSpansAcceptsOpenBounds(t *testing.T) { + prepared := ReadOnlyPrepareResult{ + Ops: 3, + ExactLeafSpans: true, + LeafSpans: []ReadOnlyLeafSpan{ + {HighKey: []byte("m"), FirstOpKey: []byte("a"), LastOpKey: []byte("b"), OpCount: 2}, + {LowKey: []byte("m"), FirstOpKey: []byte("m"), LastOpKey: []byte("m"), OpCount: 1}, + }, + } + requireValidReadOnlyPrepare(t, prepared) +} + +func TestReadOnlyPrepareResultLeafSpanSummary(t *testing.T) { + prepared := ReadOnlyPrepareResult{ + Ops: 6, + ExactLeafSpans: true, + LeafSpans: []ReadOnlyLeafSpan{ + {FirstOpKey: []byte("a"), LastOpKey: []byte("a"), OpCount: 1, HighKey: []byte("d")}, + {LowKey: []byte("d"), HighKey: []byte("m"), FirstOpKey: []byte("e"), LastOpKey: []byte("h"), OpCount: 2}, + {LowKey: []byte("m"), FirstOpKey: []byte("q"), LastOpKey: []byte("z"), OpCount: 3}, + }, + } + + summary := prepared.LeafSpanSummary() + if summary.Ops != 6 || summary.Spans != 3 || !summary.ExactLeafSpans { + t.Fatalf("summary ops/spans/exact=%d/%d/%v want 6/3/true", summary.Ops, summary.Spans, summary.ExactLeafSpans) + } + if summary.MinSpanOps != 1 || summary.MaxSpanOps != 3 || summary.SingleOpSpans != 1 { + t.Fatalf("summary op distribution min/max/single=%d/%d/%d want 1/3/1", summary.MinSpanOps, summary.MaxSpanOps, summary.SingleOpSpans) + } + if summary.OpenLowSpans != 1 || summary.OpenHighSpans != 1 { + t.Fatalf("summary open bounds low/high=%d/%d want 1/1", summary.OpenLowSpans, summary.OpenHighSpans) + } +} + +func TestReadOnlyPrepareResultLeafSpanSummaryEmptyPlan(t *testing.T) { + summary := (ReadOnlyPrepareResult{ExactLeafSpans: true}).LeafSpanSummary() + if summary.Ops != 0 || summary.Spans != 0 || !summary.ExactLeafSpans { + t.Fatalf("empty summary ops/spans/exact=%d/%d/%v want 0/0/true", summary.Ops, summary.Spans, summary.ExactLeafSpans) + } + if summary.MinSpanOps != 0 || summary.MaxSpanOps != 0 || summary.SingleOpSpans != 0 { + t.Fatalf("empty summary op distribution min/max/single=%d/%d/%d want 0/0/0", summary.MinSpanOps, summary.MaxSpanOps, summary.SingleOpSpans) + } + if summary.OpenLowSpans != 0 || summary.OpenHighSpans != 0 { + t.Fatalf("empty summary open bounds low/high=%d/%d want 0/0", summary.OpenLowSpans, summary.OpenHighSpans) + } +} + +func TestZipperPrepareReadOnlyLeafSpanSummaryMatchesPlan(t *testing.T) { + dir := t.TempDir() + p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) + if err != nil { + t.Fatal(err) + } + defer p.Close() + + alloc := &MockAllocator{p: p} + z := New(p, alloc) + rootID := buildOuterLeafInternalRoot(t, z) + + delta := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = delta.Close() }() + delta.Set([]byte("key-001"), []byte("new-001")) + delta.Set([]byte("key-067"), []byte("new-067")) + delta.Set([]byte("key-133"), []byte("new-133")) + delta.Set([]byte("key-199"), []byte("new-199")) + + prepared, err := z.PrepareReadOnly(rootID, delta, ReadOnlyPrepareOptions{}) + if err != nil { + t.Fatalf("PrepareReadOnly: %v", err) + } + requireValidReadOnlyPrepare(t, prepared) + + summary := prepared.LeafSpanSummary() + if summary.Ops != prepared.Ops || summary.Spans != len(prepared.LeafSpans) { + t.Fatalf("summary ops/spans=%d/%d want %d/%d", summary.Ops, summary.Spans, prepared.Ops, len(prepared.LeafSpans)) + } + wantMin, wantMax, wantSingle, wantOpenLow, wantOpenHigh := 0, 0, 0, 0, 0 + for i, span := range prepared.LeafSpans { + if i == 0 || span.OpCount < wantMin { + wantMin = span.OpCount + } + if i == 0 || span.OpCount > wantMax { + wantMax = span.OpCount + } + if span.OpCount == 1 { + wantSingle++ + } + if span.LowKey == nil { + wantOpenLow++ + } + if span.HighKey == nil { + wantOpenHigh++ + } + } + if summary.MinSpanOps != wantMin || summary.MaxSpanOps != wantMax || summary.SingleOpSpans != wantSingle { + t.Fatalf("summary op distribution min/max/single=%d/%d/%d want %d/%d/%d", summary.MinSpanOps, summary.MaxSpanOps, summary.SingleOpSpans, wantMin, wantMax, wantSingle) + } + if summary.OpenLowSpans != wantOpenLow || summary.OpenHighSpans != wantOpenHigh { + t.Fatalf("summary open bounds low/high=%d/%d want %d/%d", summary.OpenLowSpans, summary.OpenHighSpans, wantOpenLow, wantOpenHigh) + } +} + +func TestReadOnlyPrepareResultAppendLeafSpanWorkerRanges(t *testing.T) { + prepared := ReadOnlyPrepareResult{ + Ops: 12, + ExactLeafSpans: true, + LeafSpans: []ReadOnlyLeafSpan{ + {FirstOpKey: []byte("a"), LastOpKey: []byte("a"), OpCount: 1}, + {FirstOpKey: []byte("b"), LastOpKey: []byte("b"), OpCount: 1}, + {FirstOpKey: []byte("c"), LastOpKey: []byte("j"), OpCount: 8}, + {FirstOpKey: []byte("k"), LastOpKey: []byte("k"), OpCount: 1}, + {FirstOpKey: []byte("z"), LastOpKey: []byte("z"), OpCount: 1}, + }, + } + + ranges := prepared.AppendLeafSpanWorkerRanges(nil, 3) + requireLeafSpanWorkerRangesCoverPlan(t, prepared, ranges) + if len(ranges) != 3 { + t.Fatalf("ranges=%d want 3", len(ranges)) + } + if ranges[0].FirstSpan != 0 || ranges[2].FirstSpan+ranges[2].SpanCount != len(prepared.LeafSpans) { + t.Fatalf("ranges do not preserve first/last span: %+v", ranges) + } +} + +func TestReadOnlyPrepareResultAppendLeafSpanWorkerRangesCapsWorkersAtSpanCount(t *testing.T) { + prepared := ReadOnlyPrepareResult{ + Ops: 3, + LeafSpans: []ReadOnlyLeafSpan{ + {FirstOpKey: []byte("a"), LastOpKey: []byte("a"), OpCount: 1}, + {FirstOpKey: []byte("b"), LastOpKey: []byte("b"), OpCount: 2}, + }, + } + + ranges := prepared.AppendLeafSpanWorkerRanges(nil, 8) + requireLeafSpanWorkerRangesCoverPlan(t, prepared, ranges) + if len(ranges) != len(prepared.LeafSpans) { + t.Fatalf("ranges=%d want span count %d", len(ranges), len(prepared.LeafSpans)) + } + for i, r := range ranges { + if r.SpanCount != 1 { + t.Fatalf("range %d span count=%d want 1; ranges=%+v", i, r.SpanCount, ranges) + } + } +} + +func TestReadOnlyPrepareResultAppendLeafSpanWorkerRangesUsesDestination(t *testing.T) { + prepared := ReadOnlyPrepareResult{ + Ops: 2, + LeafSpans: []ReadOnlyLeafSpan{ + {FirstOpKey: []byte("a"), LastOpKey: []byte("a"), OpCount: 1}, + {FirstOpKey: []byte("b"), LastOpKey: []byte("b"), OpCount: 1}, + }, + } + dst := make([]ReadOnlyLeafSpanWorkerRange, 1, 3) + dstBase := &dst[:cap(dst)][0] + dst = dst[:0] + ranges := prepared.AppendLeafSpanWorkerRanges(dst, 2) + requireLeafSpanWorkerRangesCoverPlan(t, prepared, ranges) + if len(ranges) != 2 { + t.Fatalf("ranges=%d want 2", len(ranges)) + } + if &ranges[0] != dstBase { + t.Fatalf("ranges backing array was not reused") + } + allocs := testing.AllocsPerRun(1000, func() { + got := prepared.AppendLeafSpanWorkerRanges(dst, 2) + if len(got) != 2 { + t.Fatalf("ranges=%d want 2", len(got)) + } + }) + if allocs != 0 { + t.Fatalf("AppendLeafSpanWorkerRanges allocations=%v want 0", allocs) + } +} + +func TestReadOnlyPrepareResultLeafSpanWorkerRangeSummaryMatchesRanges(t *testing.T) { + prepared := ReadOnlyPrepareResult{ + Ops: 12, + ExactLeafSpans: true, + LeafSpans: []ReadOnlyLeafSpan{ + {FirstOpKey: []byte("a"), LastOpKey: []byte("a"), OpCount: 1}, + {FirstOpKey: []byte("b"), LastOpKey: []byte("b"), OpCount: 1}, + {FirstOpKey: []byte("c"), LastOpKey: []byte("j"), OpCount: 8}, + {FirstOpKey: []byte("k"), LastOpKey: []byte("k"), OpCount: 1}, + {FirstOpKey: []byte("z"), LastOpKey: []byte("z"), OpCount: 1}, + }, + } + ranges := prepared.AppendLeafSpanWorkerRanges(nil, 3) + requireLeafSpanWorkerRangesCoverPlan(t, prepared, ranges) + + summary := prepared.LeafSpanWorkerRangeSummary(3) + if summary.TargetWorkers != 3 || summary.Ranges != len(ranges) || summary.Ops != prepared.Ops { + t.Fatalf("summary target/ranges/ops=%d/%d/%d want 3/%d/%d", summary.TargetWorkers, summary.Ranges, summary.Ops, len(ranges), prepared.Ops) + } + wantMin, wantMax, wantSingle := 0, 0, 0 + for i, r := range ranges { + if i == 0 || r.Ops < wantMin { + wantMin = r.Ops + } + if i == 0 || r.Ops > wantMax { + wantMax = r.Ops + } + if r.SpanCount == 1 { + wantSingle++ + } + } + if summary.MinRangeOps != wantMin || summary.MaxRangeOps != wantMax || summary.SingleSpanRanges != wantSingle { + t.Fatalf("summary min/max/single=%d/%d/%d want %d/%d/%d", summary.MinRangeOps, summary.MaxRangeOps, summary.SingleSpanRanges, wantMin, wantMax, wantSingle) + } + allocs := testing.AllocsPerRun(1000, func() { + got := prepared.LeafSpanWorkerRangeSummary(3) + if got.Ranges != len(ranges) { + t.Fatalf("ranges=%d want %d", got.Ranges, len(ranges)) + } + }) + if allocs != 0 { + t.Fatalf("LeafSpanWorkerRangeSummary allocations=%v want 0", allocs) + } +} + +func TestReadOnlyPrepareResultLeafSpanWorkerRangeSummaryCapsRangesButKeepsTarget(t *testing.T) { + prepared := ReadOnlyPrepareResult{ + Ops: 3, + LeafSpans: []ReadOnlyLeafSpan{ + {FirstOpKey: []byte("a"), LastOpKey: []byte("a"), OpCount: 1}, + {FirstOpKey: []byte("b"), LastOpKey: []byte("b"), OpCount: 2}, + }, + } + + summary := prepared.LeafSpanWorkerRangeSummary(8) + if summary.TargetWorkers != 8 { + t.Fatalf("target workers=%d want 8", summary.TargetWorkers) + } + if summary.Ranges != len(prepared.LeafSpans) { + t.Fatalf("ranges=%d want span count %d", summary.Ranges, len(prepared.LeafSpans)) + } + if summary.Ops != prepared.Ops { + t.Fatalf("ops=%d want %d", summary.Ops, prepared.Ops) + } + if summary.MinRangeOps != 1 || summary.MaxRangeOps != 2 || summary.SingleSpanRanges != 2 { + t.Fatalf("summary min/max/single=%d/%d/%d want 1/2/2", summary.MinRangeOps, summary.MaxRangeOps, summary.SingleSpanRanges) + } +} + +func TestReadOnlyPrepareResultLeafSpanWorkerRangeSummaryEmptyInputs(t *testing.T) { + for _, workers := range []int{-1, 0, 1} { + summary := (ReadOnlyPrepareResult{}).LeafSpanWorkerRangeSummary(workers) + if summary.TargetWorkers != workers || summary.Ranges != 0 || summary.Ops != 0 { + t.Fatalf("workers=%d summary=%+v want empty", workers, summary) + } + } +} + +func TestReadOnlyPrepareResultAppendLeafSpanWorkerRangesEmptyInputs(t *testing.T) { + prepared := ReadOnlyPrepareResult{} + dst := []ReadOnlyLeafSpanWorkerRange{{FirstSpan: 99, SpanCount: 1, Ops: 1}} + for _, workers := range []int{-1, 0, 1} { + ranges := prepared.AppendLeafSpanWorkerRanges(dst, workers) + if len(ranges) != len(dst) || ranges[0] != dst[0] { + t.Fatalf("workers=%d ranges=%+v want dst unchanged", workers, ranges) + } + } +} + +func requireLeafSpanWorkerRangesCoverPlan(tb testing.TB, prepared ReadOnlyPrepareResult, ranges []ReadOnlyLeafSpanWorkerRange) { + tb.Helper() + spanIdx := 0 + ops := 0 + for i, r := range ranges { + if r.SpanCount <= 0 { + tb.Fatalf("range %d is empty: %+v", i, r) + } + if r.FirstSpan != spanIdx { + tb.Fatalf("range %d first span=%d want %d; ranges=%+v", i, r.FirstSpan, spanIdx, ranges) + } + if r.FirstSpan+r.SpanCount > len(prepared.LeafSpans) { + tb.Fatalf("range %d exceeds span count: %+v spans=%d", i, r, len(prepared.LeafSpans)) + } + rangeOps := 0 + for j := 0; j < r.SpanCount; j++ { + rangeOps += prepared.LeafSpans[r.FirstSpan+j].OpCount + } + if r.Ops != rangeOps { + tb.Fatalf("range %d ops=%d want %d; range=%+v", i, r.Ops, rangeOps, r) + } + ops += r.Ops + spanIdx += r.SpanCount + } + if spanIdx != len(prepared.LeafSpans) { + tb.Fatalf("ranges cover %d spans, want %d; ranges=%+v", spanIdx, len(prepared.LeafSpans), ranges) + } + if ops != prepared.Ops { + tb.Fatalf("ranges cover %d ops, want %d; ranges=%+v", ops, prepared.Ops, ranges) + } +} + +var readOnlyLeafSpanSummaryBenchmarkSink ReadOnlyLeafSpanSummary + +func BenchmarkReadOnlyPrepareResultLeafSpanSummary(b *testing.B) { + prepared := ReadOnlyPrepareResult{ + Ops: 4, + ExactLeafSpans: true, + LeafSpans: []ReadOnlyLeafSpan{ + {FirstOpKey: []byte("a"), LastOpKey: []byte("a"), OpCount: 1, HighKey: []byte("d")}, + {LowKey: []byte("d"), HighKey: []byte("m"), FirstOpKey: []byte("e"), LastOpKey: []byte("h"), OpCount: 2}, + {LowKey: []byte("m"), FirstOpKey: []byte("q"), LastOpKey: []byte("z"), OpCount: 1}, + }, + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + readOnlyLeafSpanSummaryBenchmarkSink = prepared.LeafSpanSummary() + } +} + +var readOnlyLeafSpanWorkerRangesBenchmarkSink []ReadOnlyLeafSpanWorkerRange +var readOnlyLeafSpanWorkerRangeSummaryBenchmarkSink ReadOnlyLeafSpanWorkerRangeSummary + +func BenchmarkReadOnlyPrepareResultLeafSpanWorkerRanges(b *testing.B) { + prepared := ReadOnlyPrepareResult{ + Ops: 12, + ExactLeafSpans: true, + LeafSpans: []ReadOnlyLeafSpan{ + {FirstOpKey: []byte("a"), LastOpKey: []byte("a"), OpCount: 1}, + {FirstOpKey: []byte("b"), LastOpKey: []byte("b"), OpCount: 1}, + {FirstOpKey: []byte("c"), LastOpKey: []byte("j"), OpCount: 8}, + {FirstOpKey: []byte("k"), LastOpKey: []byte("k"), OpCount: 1}, + {FirstOpKey: []byte("z"), LastOpKey: []byte("z"), OpCount: 1}, + }, + } + ranges := make([]ReadOnlyLeafSpanWorkerRange, 0, 3) + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + ranges = prepared.AppendLeafSpanWorkerRanges(ranges[:0], 3) + } + readOnlyLeafSpanWorkerRangesBenchmarkSink = ranges +} + +func BenchmarkReadOnlyPrepareResultLeafSpanWorkerRangeSummary(b *testing.B) { + prepared := ReadOnlyPrepareResult{ + Ops: 12, + ExactLeafSpans: true, + LeafSpans: []ReadOnlyLeafSpan{ + {FirstOpKey: []byte("a"), LastOpKey: []byte("a"), OpCount: 1}, + {FirstOpKey: []byte("b"), LastOpKey: []byte("b"), OpCount: 1}, + {FirstOpKey: []byte("c"), LastOpKey: []byte("j"), OpCount: 8}, + {FirstOpKey: []byte("k"), LastOpKey: []byte("k"), OpCount: 1}, + {FirstOpKey: []byte("z"), LastOpKey: []byte("z"), OpCount: 1}, + }, + } + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + readOnlyLeafSpanWorkerRangeSummaryBenchmarkSink = prepared.LeafSpanWorkerRangeSummary(3) + } +} + +func BenchmarkZipperPrepareReadOnlyWarmSparse(b *testing.B) { + dir := b.TempDir() + p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) + if err != nil { + b.Fatal(err) + } + defer p.Close() + + alloc := &MockAllocator{p: p} + z := New(p, alloc) + rootID := buildOuterLeafInternalRoot(b, z) + + delta := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = delta.Close() }() + delta.Set([]byte("key-001"), []byte("new-001")) + delta.Set([]byte("key-067"), []byte("new-067")) + delta.Set([]byte("key-133"), []byte("new-133")) + delta.Set([]byte("key-199"), []byte("new-199")) + + opts := ReadOnlyPrepareOptions{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + prepared, err := z.PrepareReadOnly(rootID, delta, opts) + if err != nil { + b.Fatalf("PrepareReadOnly: %v", err) + } + if len(prepared.LeafSpans) == 0 { + b.Fatal("no prepared leaf spans") + } + opts = prepared.ReuseOptions() + } +} + +func BenchmarkZipperPrepareReadOnlyWarmSparseMultiLeafReuse(b *testing.B) { + benchmarkZipperPrepareReadOnlyWarmSparseMultiLeaf(b, 257) +} + +func BenchmarkZipperPrepareReadOnlyWarmSparseManyLeafReuse(b *testing.B) { + benchmarkZipperPrepareReadOnlyWarmSparseMultiLeaf(b, 4) +} + +func BenchmarkZipperApplyWarmSparseManyLeaf(b *testing.B) { + dir := b.TempDir() + p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) + if err != nil { + b.Fatal(err) + } + defer p.Close() + + const ( + keyCount = 8192 + step = 4 + workers = 4 + ) + alloc := &recyclingMockAllocator{p: p} + z := New(p, alloc) + rootID := buildInternalRootWithKeys(b, z, keyCount) + + left := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = left.Close() }() + right := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = right.Close() }() + for i := 0; i < keyCount; i += step { + key := []byte(fmt.Sprintf("key-%06d", i)) + left.Set(key, []byte("left")) + right.Set(key, []byte("right")) + } + + prepared, err := z.PrepareReadOnly(rootID, left, ReadOnlyPrepareOptions{}) + if err != nil { + b.Fatalf("PrepareReadOnly: %v", err) + } + requireValidReadOnlyPrepare(b, prepared) + summary := prepared.LeafSpanSummary() + workerSummary := prepared.LeafSpanWorkerRangeSummary(workers) + if summary.Spans < 2 || summary.Ops == 0 || workerSummary.Ranges < 2 { + b.Fatalf("prepare spans/ops/ranges=%d/%d/%d want many-leaf plan", summary.Spans, summary.Ops, workerSummary.Ranges) + } + + var total adaptive.Metrics + var totalRetired int + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + delta := left + if i&1 == 1 { + delta = right + } + newRoot, retired, metrics, err := z.Apply(rootID, delta) + if err != nil { + b.Fatalf("Apply: %v", err) + } + if newRoot == 0 || newRoot == rootID { + b.Fatalf("new root=%d old root=%d want changed non-zero root", newRoot, rootID) + } + rootID = newRoot + totalRetired += len(retired) + alloc.Recycle(retired) + mergeMetrics(&total, &metrics) + } + b.ReportMetric(float64(summary.Spans), "leaf_spans/op") + b.ReportMetric(float64(summary.Ops), "ops/op") + b.ReportMetric(float64(workerSummary.Ranges), "worker_ranges/op") + b.ReportMetric(float64(workerSummary.MaxRangeOps), "max_worker_ops/op") + if b.N > 0 { + b.ReportMetric(float64(total.ZipperLeafMerges)/float64(b.N), "leaf_merges/op") + b.ReportMetric(float64(total.ZipperInternalMerges)/float64(b.N), "internal_merges/op") + b.ReportMetric(float64(total.ZipperInternalParallelMerges)/float64(b.N), "internal_parallel_merges/op") + b.ReportMetric(float64(total.ZipperInternalParallelChildren)/float64(b.N), "internal_parallel_children/op") + b.ReportMetric(float64(total.ZipperInternalParallelWorkers)/float64(b.N), "internal_parallel_workers/op") + b.ReportMetric(float64(total.ZipperInternalParallelOps)/float64(b.N), "internal_parallel_ops/op") + b.ReportMetric(float64(total.IndexWriteBytes)/float64(b.N), "index_write_bytes/op") + b.ReportMetric(float64(totalRetired)/float64(b.N), "pending_retired_pages/op") + } +} + +func benchmarkZipperPrepareReadOnlyWarmSparseMultiLeaf(b *testing.B, step int) { + dir := b.TempDir() + p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) + if err != nil { + b.Fatal(err) + } + defer p.Close() + + const ( + keyCount = 8192 + workers = 4 + ) + alloc := &MockAllocator{p: p} + z := New(p, alloc) + rootID := buildInternalRootWithKeys(b, z, keyCount) + + delta := batch.New(panicValueReader{}, page.DefaultInlineThreshold) + defer func() { _ = delta.Close() }() + for i := 0; i < keyCount; i += step { + key := []byte(fmt.Sprintf("key-%06d", i)) + delta.Set(key, []byte("new")) + } + + first, err := z.PrepareReadOnly(rootID, delta, ReadOnlyPrepareOptions{}) + if err != nil { + b.Fatalf("initial PrepareReadOnly: %v", err) + } + requireValidReadOnlyPrepare(b, first) + if first.Maintenance || !first.ExactLeafSpans { + b.Fatalf("initial prepare maintenance/exact=%v/%v want false/true", first.Maintenance, first.ExactLeafSpans) + } + summary := first.LeafSpanSummary() + workerSummary := first.LeafSpanWorkerRangeSummary(workers) + if summary.Spans < 2 || workerSummary.Ranges < 2 { + b.Fatalf("initial prepare spans/ranges=%d/%d want multi-leaf plan", summary.Spans, workerSummary.Ranges) + } + + opts := first.ReuseOptions() + last := first + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + prepared, err := z.PrepareReadOnly(rootID, delta, opts) + if err != nil { + b.Fatalf("PrepareReadOnly: %v", err) + } + if len(prepared.LeafSpans) != summary.Spans || prepared.Ops != summary.Ops { + b.Fatalf("prepared spans/ops=%d/%d want %d/%d", len(prepared.LeafSpans), prepared.Ops, summary.Spans, summary.Ops) + } + last = prepared + opts = prepared.ReuseOptions() + } + b.StopTimer() + lastSummary := last.LeafSpanSummary() + lastWorkerSummary := last.LeafSpanWorkerRangeSummary(workers) + b.ReportMetric(float64(lastSummary.Spans), "leaf_spans/op") + b.ReportMetric(float64(lastSummary.Ops), "ops/op") + b.ReportMetric(float64(lastWorkerSummary.Ranges), "worker_ranges/op") + b.ReportMetric(float64(lastWorkerSummary.MaxRangeOps), "max_worker_ops/op") +} + func TestZipperLeafRefCacheAvoidsUnflushedReads(t *testing.T) { dir := t.TempDir() p, err := pager.Open(filepath.Join(dir, "index.db"), 65536) diff --git a/cmd/internal/treedbstats/selected.go b/cmd/internal/treedbstats/selected.go index 303486a59e..6ec96ddd57 100644 --- a/cmd/internal/treedbstats/selected.go +++ b/cmd/internal/treedbstats/selected.go @@ -45,6 +45,8 @@ func isSelectedKey(key string) bool { return true case strings.HasPrefix(key, "treedb.publish.ordered_root_delta_group."): return true + case strings.HasPrefix(key, "treedb.publish.install_guard."): + return true case strings.HasPrefix(key, "treedb.collections.write_domain."): return true case strings.HasPrefix(key, "treedb.publish.watermark."): diff --git a/cmd/internal/treedbstats/selected_test.go b/cmd/internal/treedbstats/selected_test.go index 18fe63c386..55b6817043 100644 --- a/cmd/internal/treedbstats/selected_test.go +++ b/cmd/internal/treedbstats/selected_test.go @@ -5,13 +5,22 @@ import "testing" func TestSelectedKeepsSharedTreeDBStats(t *testing.T) { stats := map[string]string{ "treedb.commit_seq": "7", - "treedb.process.read_path.backend_tree.get_append_pointer_hits_total": "5", - "treedb.process.read_path.outer_leaf.cache.hits": "11", - "treedb.vlog.mmap_read.fallback_readat": "13", - "treedb.publish.ordered_root_delta_group.calls_total": "19", - "treedb.publish.watermark.latency_p99_ms": "23", - "treedb.collections.write_domain.indexed_flush.calls_total": "29", - "treedb.unrelated_stat_that_should_not_leave_the_helper": "17", + "treedb.process.read_path.backend_tree.get_append_pointer_hits_total": "5", + "treedb.process.read_path.outer_leaf.cache.hits": "11", + "treedb.vlog.mmap_read.fallback_readat": "13", + "treedb.publish.ordered_root_delta_group.calls_total": "19", + "treedb.publish.install_guard.calls_total": "20", + "treedb.publish.install_guard.failures_total": "21", + "treedb.publish.install_guard.ns_total": "22", + "treedb.publish.watermark.latency_p99_ms": "23", + "treedb.collections.write_domain.indexed_flush.calls_total": "29", + "treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total": "31", + "treedb.collections.write_domain.root_delta_plan.final.secondary.bytes_total": "37", + "treedb.collections.write_domain.root_delta_plan.squashed_entries_total": "41", + "treedb.collections.write_domain.coalesced_flush_batch.net_zero_batches_total": "43", + "treedb.collections.write_domain.primary_only.duplicate_ids_coalesced_total": "53", + "treedb.collections.write_domain.primary_only.drains_total": "59", + "treedb.unrelated_stat_that_should_not_leave_the_helper": "17", } got := Selected(stats) for _, key := range []string{ @@ -20,8 +29,17 @@ func TestSelectedKeepsSharedTreeDBStats(t *testing.T) { "treedb.process.read_path.outer_leaf.cache.hits", "treedb.vlog.mmap_read.fallback_readat", "treedb.publish.ordered_root_delta_group.calls_total", + "treedb.publish.install_guard.calls_total", + "treedb.publish.install_guard.failures_total", + "treedb.publish.install_guard.ns_total", "treedb.publish.watermark.latency_p99_ms", "treedb.collections.write_domain.indexed_flush.calls_total", + "treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total", + "treedb.collections.write_domain.root_delta_plan.final.secondary.bytes_total", + "treedb.collections.write_domain.root_delta_plan.squashed_entries_total", + "treedb.collections.write_domain.coalesced_flush_batch.net_zero_batches_total", + "treedb.collections.write_domain.primary_only.duplicate_ids_coalesced_total", + "treedb.collections.write_domain.primary_only.drains_total", } { if got[key] == "" { t.Fatalf("Selected missing %s from %#v", key, got) diff --git a/cmd/mongo_gateway_bench/README.md b/cmd/mongo_gateway_bench/README.md index fd5babc248..051d0cf540 100644 --- a/cmd/mongo_gateway_bench/README.md +++ b/cmd/mongo_gateway_bench/README.md @@ -97,6 +97,8 @@ collection benchmark profile: - `-treedb-buffered-indexed-write-max-root-runs 0` (explicit `0` disables this trigger; when this flag is omitted while document or byte thresholds are overridden, the tool keeps the matching root-run compatibility default) +- `-treedb-buffered-indexed-read-only-prepare=false` +- `-treedb-buffered-indexed-read-only-prepare-workers=0` - `-treedb-maintenance full` - `-client-mode driver` @@ -126,6 +128,35 @@ tool fills in the matching root-run default; pass `-treedb-buffered-indexed-write-max-root-runs 0` explicitly to keep root-run flushing disabled in that case. +Use `-treedb-buffered-indexed-read-only-prepare` to run TreeDB's read-only +leaf-span preparation pass during indexed collection flush publishes. This is +an observability and planning mode: it records what existing leaf spans the +root delta would touch, but it does not change root publish output, durability, +read visibility, or enable parallel leaf execution. Pair it with +`-treedb-buffered-indexed-read-only-prepare-workers N` to report deterministic +worker-range summaries for an intended worker target. The worker flag is valid +only when read-only prepare is enabled, including explicit `0`. + +JSON output always includes the effective +`treedb_buffered_indexed_read_only_prepare` and +`treedb_buffered_indexed_read_only_prepare_workers` fields so benchmark +artifacts distinguish default-off runs from older runs that predate the mode. +Per-phase `treedb_metrics` include: + +- `read_only_prepare_calls/doc` +- `read_only_prepare_ns/doc` +- `read_only_prepare_ns/plan` +- `read_only_prepare_ops/doc` +- `read_only_prepare_leaf_spans/plan` +- `read_only_prepare_worker_targets/plan` +- `read_only_prepare_worker_ranges/plan` +- `read_only_prepare_worker_max_ops/plan` + +The raw TreeDB stat deltas remain in `treedb_stats_delta` under the +`treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_*` keys. +Use those raw counters when auditing exact totals; use `treedb_metrics` for +normalized writer-sweep comparisons. + The Go profile benchmarks in `profile_bench_test.go` keep their defaults stable, but can opt into the same indexed async flush mode for focused root-publish experiments: @@ -242,6 +273,8 @@ The bundle contains: - `summary.tsv`: machine-readable per-phase comparison rows. - `matrix.tsv`: target/config/document/index/raw-json/physical-byte index. - `raw/*.json`: unmodified `mongo_gateway_bench -format json` output. +- Writer-sweep read-only prepare columns in both `report.md` and `summary.tsv` + when the raw JSON contains the TreeDB read-only prepare metrics. - `profiles/`: per-phase TreeDB pprof artifacts when `--profile-treedb` is used. - `treedb_data/` and, in Docker mode, `mongodb_data/`: final data directories @@ -259,6 +292,23 @@ GOWORK=off go run ./cmd/mongo_gateway_compare_report \ -summary /tmp/gomap_mongo_gateway_compare/summary.tsv ``` +For a TreeDB-only writer metrics TSV from an existing bundle or hand-built +matrix, use: + +```sh +python3 scripts/mongo_gateway_writer_metrics.py \ + /tmp/gomap_mongo_gateway_compare \ + /tmp/gomap_mongo_gateway_compare/matrix.tsv \ + /tmp/gomap_mongo_gateway_compare/writer_metrics.tsv +``` + +The writer metrics TSV exports the same normalized read-only prepare columns as +the compare report, plus exact raw totals: + +- `read_only_prepare_calls_total` +- `read_only_prepare_worker_targets_total` +- `read_only_prepare_worker_ranges_total` + Useful overrides: - `DOCS_LIST="1000 10000 100000"` diff --git a/cmd/mongo_gateway_bench/main.go b/cmd/mongo_gateway_bench/main.go index 760efe78bf..4c0404728f 100644 --- a/cmd/mongo_gateway_bench/main.go +++ b/cmd/mongo_gateway_bench/main.go @@ -77,6 +77,8 @@ type config struct { TreeDBBufferedIndexedWriteMaxRootRuns int TreeDBBufferedIndexedAsyncFlush bool TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits int + TreeDBBufferedIndexedReadOnlyPrepare bool + TreeDBBufferedIndexedReadOnlyPrepareWorkers int TreeDBMaintenance string PrebuildDocuments bool ProfileDir string @@ -123,6 +125,8 @@ type benchmarkResult struct { TreeDBBufferedIndexedWriteMaxRootRuns int `json:"treedb_buffered_indexed_write_max_root_runs"` TreeDBBufferedIndexedAsyncFlush bool `json:"treedb_buffered_indexed_async_flush,omitempty"` TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits int `json:"treedb_buffered_indexed_async_flush_max_queued_units,omitempty"` + TreeDBBufferedIndexedReadOnlyPrepare bool `json:"treedb_buffered_indexed_read_only_prepare"` + TreeDBBufferedIndexedReadOnlyPrepareWorkers int `json:"treedb_buffered_indexed_read_only_prepare_workers"` TreeDBMaintenanceMode string `json:"treedb_maintenance_mode,omitempty"` PrebuildDocuments bool `json:"prebuild_documents,omitempty"` Phases []phaseResult `json:"phases"` @@ -155,6 +159,8 @@ type phaseResult struct { DriverMeanLatencyMicros float64 `json:"driver_mean_latency_us,omitempty"` LatencyMicros latencySummary `json:"latency_micros"` ProducerResults []producerResult `json:"producer_results,omitempty"` + TreeDBDrainMillis float64 `json:"treedb_drain_ms"` + TreeDBDrainStatsDelta map[string]string `json:"treedb_drain_stats_delta,omitempty"` TreeDBStatsDelta map[string]string `json:"treedb_stats_delta,omitempty"` TreeDBMetrics map[string]float64 `json:"treedb_metrics,omitempty"` } @@ -478,6 +484,8 @@ func parseConfig(args []string) (config, error) { fs.IntVar(&cfg.TreeDBBufferedIndexedWriteMaxRootRuns, "treedb-buffered-indexed-write-max-root-runs", cfg.TreeDBBufferedIndexedWriteMaxRootRuns, "TreeDB indexed collection write-domain root-run auto-flush threshold; explicit 0 disables this trigger; omitted with docs/bytes override keeps the compatibility default") fs.BoolVar(&cfg.TreeDBBufferedIndexedAsyncFlush, "treedb-buffered-indexed-async-flush", false, "TreeDB indexed collection threshold flushes publish in the background; explicit Flush/Close still drain before returning") fs.IntVar(&cfg.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits, "treedb-buffered-indexed-async-flush-max-queued-units", 0, "TreeDB indexed collection background flush unit queue limit; 0 uses the collection default when async flush is enabled") + fs.BoolVar(&cfg.TreeDBBufferedIndexedReadOnlyPrepare, "treedb-buffered-indexed-read-only-prepare", false, "TreeDB indexed collection flush publish runs read-only leaf-span preparation for observability; does not change publish output") + fs.IntVar(&cfg.TreeDBBufferedIndexedReadOnlyPrepareWorkers, "treedb-buffered-indexed-read-only-prepare-workers", 0, "TreeDB indexed collection read-only prepare worker target for range summaries; requires -treedb-buffered-indexed-read-only-prepare") fs.StringVar(&cfg.TreeDBMaintenance, "treedb-maintenance", cfg.TreeDBMaintenance, "TreeDB final disk maintenance for -target treedb: full, checkpoint, or none") fs.BoolVar(&cfg.PrebuildDocuments, "prebuild-documents", false, "prebuild benchmark documents before the timed load phase") fs.StringVar(&cfg.ProfileDir, "profile-dir", "", "write per-phase pprof artifacts and a profile_manifest.json into an empty directory") @@ -592,6 +600,12 @@ func parseConfig(args []string) (config, error) { if cfg.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits < 0 { return config{}, errors.New("treedb-buffered-indexed-async-flush-max-queued-units must be >= 0") } + if cfg.TreeDBBufferedIndexedReadOnlyPrepareWorkers < 0 { + return config{}, errors.New("treedb-buffered-indexed-read-only-prepare-workers must be >= 0") + } + if seenFlags["treedb-buffered-indexed-read-only-prepare-workers"] && !cfg.TreeDBBufferedIndexedReadOnlyPrepare { + return config{}, errors.New("treedb-buffered-indexed-read-only-prepare-workers requires -treedb-buffered-indexed-read-only-prepare") + } if cfg.Format != "text" && cfg.Format != "json" { return config{}, fmt.Errorf("unknown format %q", cfg.Format) } @@ -794,14 +808,16 @@ func openTreeDBTarget(ctx context.Context, cfg config) (*benchTarget, error) { server.Collections = manager server.MaxFindScanDocuments = cfg.Documents server.DefaultCollectionOptions = collections.CollectionOptions{ - DocumentFormat: cfg.TreeDBDocumentFormat, - DataRootStoragePolicy: cfg.TreeDBDataRootStorage, - IndexStateStoragePolicy: cfg.TreeDBIndexStateRootStorage, - BufferedIndexedWriteMaxDocuments: cfg.TreeDBBufferedIndexedWriteMaxDocuments, - BufferedIndexedWriteMaxBytes: cfg.TreeDBBufferedIndexedWriteMaxBytes, - BufferedIndexedWriteMaxRootRuns: cfg.TreeDBBufferedIndexedWriteMaxRootRuns, - BufferedIndexedAsyncFlush: cfg.TreeDBBufferedIndexedAsyncFlush, - BufferedIndexedAsyncFlushMaxQueuedUnits: cfg.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits, + DocumentFormat: cfg.TreeDBDocumentFormat, + DataRootStoragePolicy: cfg.TreeDBDataRootStorage, + IndexStateStoragePolicy: cfg.TreeDBIndexStateRootStorage, + BufferedIndexedWriteMaxDocuments: cfg.TreeDBBufferedIndexedWriteMaxDocuments, + BufferedIndexedWriteMaxBytes: cfg.TreeDBBufferedIndexedWriteMaxBytes, + BufferedIndexedWriteMaxRootRuns: cfg.TreeDBBufferedIndexedWriteMaxRootRuns, + BufferedIndexedAsyncFlush: cfg.TreeDBBufferedIndexedAsyncFlush, + BufferedIndexedAsyncFlushMaxQueuedUnits: cfg.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits, + BufferedIndexedReadOnlyPrepare: cfg.TreeDBBufferedIndexedReadOnlyPrepare, + BufferedIndexedReadOnlyPrepareWorkerCount: cfg.TreeDBBufferedIndexedReadOnlyPrepareWorkers, } server.DefaultIndexStoragePolicy = cfg.TreeDBIndexRootStorage @@ -1120,6 +1136,8 @@ func runBenchmark(ctx context.Context, cfg config, target *benchTarget, profiler result.TreeDBBufferedIndexedWriteMaxRootRuns = cfg.TreeDBBufferedIndexedWriteMaxRootRuns result.TreeDBBufferedIndexedAsyncFlush = cfg.TreeDBBufferedIndexedAsyncFlush result.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits = cfg.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits + result.TreeDBBufferedIndexedReadOnlyPrepare = cfg.TreeDBBufferedIndexedReadOnlyPrepare + result.TreeDBBufferedIndexedReadOnlyPrepareWorkers = cfg.TreeDBBufferedIndexedReadOnlyPrepareWorkers result.TreeDBMaintenanceMode = cfg.TreeDBMaintenance } else { result.MongoURI = redactMongoURI(cfg.MongoURI) @@ -1399,10 +1417,13 @@ func runTreeDBProfiledPhase(target *benchTarget, profiler *profileRecorder, name if err != nil { return result, err } + drainBefore := collectLiveTreeDBStats(target) drainElapsed, err := drainTreeDBCollectionsForPhase(target) if err != nil { return result, err } + drainAfter := collectLiveTreeDBStats(target) + attachTreeDBDrainStats(&result, drainBefore, drainAfter, drainElapsed) addPhaseDuration(&result, drainElapsed) return result, nil }) @@ -1540,6 +1561,8 @@ func recordEffectiveTreeDBCollectionOptions(result *benchmarkResult, cfg config, result.TreeDBBufferedIndexedWriteMaxRootRuns = meta.Options.BufferedIndexedWriteMaxRootRuns result.TreeDBBufferedIndexedAsyncFlush = meta.Options.BufferedIndexedAsyncFlush result.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits = meta.Options.BufferedIndexedAsyncFlushMaxQueuedUnits + result.TreeDBBufferedIndexedReadOnlyPrepare = meta.Options.BufferedIndexedReadOnlyPrepare + result.TreeDBBufferedIndexedReadOnlyPrepareWorkers = meta.Options.BufferedIndexedReadOnlyPrepareWorkerCount return nil } @@ -2143,6 +2166,17 @@ func attachTreeDBPhaseStats(result *phaseResult, before, after map[string]string result.TreeDBMetrics = deriveTreeDBPhaseMetrics(numeric, result.Operations, result.DriverCalls) } +func attachTreeDBDrainStats(result *phaseResult, before, after map[string]string, elapsed time.Duration) { + if result == nil { + return + } + result.TreeDBDrainMillis = float64(elapsed) / float64(time.Millisecond) + delta, _ := treeDBStatsDelta(before, after) + if len(delta) > 0 { + result.TreeDBDrainStatsDelta = delta + } +} + func treeDBStatsDelta(before, after map[string]string) (map[string]string, map[string]float64) { if len(after) == 0 { return nil, nil @@ -2248,6 +2282,18 @@ func formatTreeDBStatNumber(value float64) string { return strconv.FormatFloat(value, 'f', -1, 64) } +type treeDBRootDeltaKindMetric struct { + metricToken string + statToken string +} + +var treeDBRootDeltaKindMetrics = [...]treeDBRootDeltaKindMetric{ + {metricToken: "primary", statToken: "primary"}, + {metricToken: "template", statToken: "template"}, + {metricToken: "index_state", statToken: "index_state"}, + {metricToken: "secondary", statToken: "secondary"}, +} + func deriveTreeDBPhaseMetrics(delta map[string]float64, operations, driverCalls int) map[string]float64 { if len(delta) == 0 { return nil @@ -2257,6 +2303,19 @@ func deriveTreeDBPhaseMetrics(delta map[string]float64, operations, driverCalls addPerOperationMetric(metrics, "root_apply_calls/doc", delta, "treedb.publish.ordered_root_delta_group.root_apply_calls_total", operations) addRatioMetric(metrics, "roots/publish", delta, "treedb.publish.ordered_root_delta_group.roots_total", "treedb.publish.ordered_root_delta_group.calls_total") addPerOperationMetric(metrics, "publish_delta_group_root_apply_ns/doc", delta, "treedb.publish.ordered_root_delta_group.root_apply_ns_total", operations) + addPerOperationMetric(metrics, "internal_parallel_merges/doc", delta, "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_merges_total", operations) + addRatioMetric(metrics, "internal_parallel_children/merge", delta, "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_children_total", "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_merges_total") + addRatioMetric(metrics, "internal_parallel_workers/merge", delta, "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_workers_total", "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_merges_total") + addRatioMetric(metrics, "internal_parallel_ops/merge", delta, "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_ops_total", "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_merges_total") + addReadOnlyPrepareRootApplySplitMetrics(metrics, delta, operations) + addPerOperationMetric(metrics, "read_only_prepare_calls/doc", delta, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total", operations) + addPerOperationMetric(metrics, "read_only_prepare_ns/doc", delta, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ns_total", operations) + addRatioMetric(metrics, "read_only_prepare_ns/plan", delta, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ns_total", "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total") + addPerOperationMetric(metrics, "read_only_prepare_ops/doc", delta, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ops_total", operations) + addRatioMetric(metrics, "read_only_prepare_leaf_spans/plan", delta, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_leaf_spans_total", "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total") + addRatioMetric(metrics, "read_only_prepare_worker_targets/plan", delta, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_targets_total", "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total") + addRatioMetric(metrics, "read_only_prepare_worker_ranges/plan", delta, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_ranges_total", "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total") + addRatioMetric(metrics, "read_only_prepare_worker_max_ops/plan", delta, "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_range_max_ops_total", "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total") addPerOperationMetric(metrics, "leaf_log_node_loads/doc", delta, "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_node_loads_total", operations) addPerOperationMetric(metrics, "leaf_log_pages_written/doc", delta, "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_pages_written_total", operations) addPerOperationMetric(metrics, "leaf_log_read_bytes/doc", delta, "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_node_bytes_read_total", operations) @@ -2265,6 +2324,14 @@ func deriveTreeDBPhaseMetrics(delta map[string]float64, operations, driverCalls addRatioMetric(metrics, "indexed_flush_docs/batch", delta, "treedb.collections.write_domain.indexed_flush.docs_total", "treedb.collections.write_domain.indexed_flush.calls_total") addRatioMetric(metrics, "indexed_flush_units/batch", delta, "treedb.collections.write_domain.indexed_flush.units_total", "treedb.collections.write_domain.indexed_flush.calls_total") addPerOperationMetric(metrics, "indexed_flush_root_runs/doc", delta, "treedb.collections.write_domain.indexed_flush.root_runs_total", operations) + addPerOperationMetric(metrics, "indexed_flush_preflight_ns/doc", delta, "treedb.collections.write_domain.indexed_flush.preflight_ns_total", operations) + addPerOperationMetric(metrics, "indexed_flush_rotate_ns/doc", delta, "treedb.collections.write_domain.indexed_flush.rotate_ns_total", operations) + addPerOperationMetric(metrics, "indexed_flush_merge_ns/doc", delta, "treedb.collections.write_domain.indexed_flush.merge_ns_total", operations) + addPerOperationMetric(metrics, "indexed_flush_materialize_ns/doc", delta, "treedb.collections.write_domain.indexed_flush.materialize_ns_total", operations) + addPerOperationMetric(metrics, "indexed_flush_semantic_plan_ns/doc", delta, "treedb.collections.write_domain.indexed_flush.materialize_semantic_plan_ns_total", operations) + addPerOperationMetric(metrics, "indexed_flush_build_inputs_ns/doc", delta, "treedb.collections.write_domain.indexed_flush.materialize_build_inputs_ns_total", operations) + addPerOperationMetric(metrics, "indexed_flush_plan_stats_ns/doc", delta, "treedb.collections.write_domain.indexed_flush.materialize_plan_stats_ns_total", operations) + addPerOperationMetric(metrics, "indexed_flush_publish_ns/doc", delta, "treedb.collections.write_domain.indexed_flush.publish_ns_total", operations) addPerOperationMetric(metrics, "root_delta_plan_entries/doc", delta, "treedb.collections.write_domain.root_delta_plan.entries_total", operations) addPerOperationMetric(metrics, "root_delta_plan_key_bytes/doc", delta, "treedb.collections.write_domain.root_delta_plan.key_bytes_total", operations) addPerOperationMetric(metrics, "root_delta_plan_value_bytes/doc", delta, "treedb.collections.write_domain.root_delta_plan.value_bytes_total", operations) @@ -2273,12 +2340,58 @@ func deriveTreeDBPhaseMetrics(delta map[string]float64, operations, driverCalls addPerOperationMetric(metrics, "affected_template_roots/doc", delta, "treedb.collections.write_domain.root_delta_plan.roots.template_total", operations) addPerOperationMetric(metrics, "affected_index_state_roots/doc", delta, "treedb.collections.write_domain.root_delta_plan.roots.index_state_total", operations) addPerOperationMetric(metrics, "affected_secondary_roots/doc", delta, "treedb.collections.write_domain.root_delta_plan.roots.secondary_total", operations) + addRatioMetric(metrics, "coalesced_batch_units/batch", delta, "treedb.collections.write_domain.coalesced_flush_batch.units_total", "treedb.collections.write_domain.coalesced_flush_batch.batches_total") + addRatioMetric(metrics, "coalesced_batch_docs/batch", delta, "treedb.collections.write_domain.coalesced_flush_batch.docs_total", "treedb.collections.write_domain.coalesced_flush_batch.batches_total") + addRatioMetric(metrics, "coalesced_batch_bytes/batch", delta, "treedb.collections.write_domain.coalesced_flush_batch.bytes_total", "treedb.collections.write_domain.coalesced_flush_batch.batches_total") + addPerOperationMetric(metrics, "net_zero_root_batches/doc", delta, "treedb.collections.write_domain.coalesced_flush_batch.net_zero_batches_total", operations) + addRootDeltaKindMetrics(metrics, "root_delta_plan.raw_unit", "raw", delta, operations) + addRootDeltaKindMetrics(metrics, "root_delta_plan.final", "final", delta, operations) + addPerOperationMetric(metrics, "squashed_root_delta_entries/doc", delta, "treedb.collections.write_domain.root_delta_plan.squashed_entries_total", operations) + addPerOperationMetric(metrics, "net_zero_root_plans/doc", delta, "treedb.collections.write_domain.root_delta_plan.net_zero_plans_total", operations) + addPerOperationMetric(metrics, "skipped_secondary_roots/doc", delta, "treedb.collections.write_domain.indexed_semantic.skipped_secondary_roots_total", operations) + addPerOperationMetric(metrics, "update_validate_items_ns/doc", delta, "treedb.collections.write_domain.update_batch.validate_items_ns_total", operations) + addPerOperationMetric(metrics, "update_clone_items_ns/doc", delta, "treedb.collections.write_domain.update_batch.clone_items_ns_total", operations) + addPerOperationMetric(metrics, "update_plan_setup_ns/doc", delta, "treedb.collections.write_domain.update_batch.plan_setup_ns_total", operations) + addPerOperationMetric(metrics, "update_buffered_read_snapshot_ns/doc", delta, "treedb.collections.write_domain.update_batch.buffered_read_snapshot_ns_total", operations) + addPerOperationMetric(metrics, "update_current_read_ns/doc", delta, "treedb.collections.write_domain.update_batch.current_read_ns_total", operations) + addPerOperationMetric(metrics, "update_bson_id_validation_ns/doc", delta, "treedb.collections.write_domain.update_batch.bson_id_validation_ns_total", operations) + addPerOperationMetric(metrics, "update_callback_ns/doc", delta, "treedb.collections.write_domain.update_batch.callback_ns_total", operations) + addPerOperationMetric(metrics, "update_replacement_stage_ns/doc", delta, "treedb.collections.write_domain.update_batch.replacement_stage_ns_total", operations) + addPerOperationMetric(metrics, "update_prepare_ns/doc", delta, "treedb.collections.write_domain.update_batch.prepare_ns_total", operations) + addPerOperationMetric(metrics, "update_index_state_extract_ns/doc", delta, "treedb.collections.write_domain.update_batch.index_state_extract_ns_total", operations) + addPerOperationMetric(metrics, "update_index_state_compare_ns/doc", delta, "treedb.collections.write_domain.update_batch.index_state_compare_ns_total", operations) + addPerOperationMetric(metrics, "update_unique_preflight_ns/doc", delta, "treedb.collections.write_domain.update_batch.unique_preflight_ns_total", operations) + addPerOperationMetric(metrics, "update_template_run_build_ns/doc", delta, "treedb.collections.write_domain.update_batch.template_run_ns_total", operations) + addPerOperationMetric(metrics, "update_primary_run_build_ns/doc", delta, "treedb.collections.write_domain.update_batch.primary_run_ns_total", operations) + addPerOperationMetric(metrics, "update_index_state_run_build_ns/doc", delta, "treedb.collections.write_domain.update_batch.index_state_run_ns_total", operations) + addPerOperationMetric(metrics, "update_secondary_run_build_ns/doc", delta, "treedb.collections.write_domain.update_batch.secondary_runs_ns_total", operations) + addPerOperationMetric(metrics, "update_semantic_record_build_ns/doc", delta, "treedb.collections.write_domain.update_batch.semantic_record_build_ns_total", operations) + addPerOperationMetric(metrics, "update_buffer_stage_ns/doc", delta, "treedb.collections.write_domain.update_batch.buffer_stage_ns_total", operations) + addPerOperationMetric(metrics, "update_buffer_precheck_ns/doc", delta, "treedb.collections.write_domain.update_batch.buffer_stage_precheck_ns_total", operations) + addPerOperationMetric(metrics, "update_buffer_lock_wait_ns/doc", delta, "treedb.collections.write_domain.update_batch.buffer_stage_lock_wait_ns_total", operations) + addPerOperationMetric(metrics, "update_buffer_lock_hold_ns/doc", delta, "treedb.collections.write_domain.update_batch.buffer_stage_lock_hold_ns_total", operations) + addPerOperationMetric(metrics, "update_buffer_validation_ns/doc", delta, "treedb.collections.write_domain.update_batch.buffer_stage_validation_ns_total", operations) + addPerOperationMetric(metrics, "update_buffer_root_scan_ns/doc", delta, "treedb.collections.write_domain.update_batch.buffer_stage_root_scan_ns_total", operations) + addPerOperationMetric(metrics, "update_buffer_domain_prepare_ns/doc", delta, "treedb.collections.write_domain.update_batch.buffer_stage_domain_prepare_ns_total", operations) + addPerOperationMetric(metrics, "update_buffer_primary_index_ns/doc", delta, "treedb.collections.write_domain.update_batch.buffer_stage_primary_index_ns_total", operations) + addPerOperationMetric(metrics, "update_buffer_unique_index_ns/doc", delta, "treedb.collections.write_domain.update_batch.buffer_stage_unique_index_ns_total", operations) + addPerOperationMetric(metrics, "update_buffer_root_append_ns/doc", delta, "treedb.collections.write_domain.update_batch.buffer_stage_root_append_ns_total", operations) + addPerOperationMetric(metrics, "update_buffer_semantic_append_ns/doc", delta, "treedb.collections.write_domain.update_batch.buffer_stage_semantic_append_ns_total", operations) + addPerOperationMetric(metrics, "update_buffer_flush_ns/doc", delta, "treedb.collections.write_domain.update_batch.buffer_stage_flush_ns_total", operations) + addPerOperationMetric(metrics, "update_plan_close_ns/doc", delta, "treedb.collections.write_domain.update_batch.plan_close_ns_total", operations) addPerOperationMetric(metrics, "primary_root_publishes/doc", delta, "treedb.collections.write_domain.primary_only.root_publishes_total", operations) addPerOperationMetric(metrics, "primary_root_delta_entries/doc", delta, "treedb.collections.write_domain.primary_only.root_delta_entries_total", operations) if bytesTotal, ok := sumTreeDBMetricDeltas(delta, "treedb.collections.write_domain.primary_only.root_delta_key_bytes_total", "treedb.collections.write_domain.primary_only.root_delta_value_bytes_total"); ok { addPerOperationMetricValue(metrics, "primary_root_delta_bytes/doc", bytesTotal, operations) } addRatioMetric(metrics, "primary_only_coalesced_docs/publish", delta, "treedb.collections.write_domain.primary_only.coalesced_docs_total", "treedb.collections.write_domain.primary_only.root_publishes_total") + addPerOperationMetric(metrics, "primary_only_duplicate_ids_coalesced/doc", delta, "treedb.collections.write_domain.primary_only.duplicate_ids_coalesced_total", operations) + addPerOperationMetric(metrics, "primary_only_drains/doc", delta, "treedb.collections.write_domain.primary_only.drains_total", operations) + addRatioMetric(metrics, "primary_only_drain_docs/drain", delta, "treedb.collections.write_domain.primary_only.drain_docs_total", "treedb.collections.write_domain.primary_only.drains_total") + addPerOperationMetric(metrics, "primary_only_drain_bytes/doc", delta, "treedb.collections.write_domain.primary_only.drain_bytes_total", operations) + addPerOperationMetric(metrics, "primary_only_drain_ns/doc", delta, "treedb.collections.write_domain.primary_only.drain_ns_total", operations) + addPerDriverCallMetric(metrics, "primary_only_buffered_calls/driver_call", delta, "treedb.collections.write_domain.primary_only.buffered_calls_total", driverCalls) + addPerDriverCallMetric(metrics, "primary_only_publish_calls/driver_call", delta, "treedb.collections.write_domain.primary_only.root_publishes_total", driverCalls) if uniqueEligible, ok := sumTreeDBMetricDeltas(delta, "treedb.collections.write_domain.update_batch.unique_checks_total", "treedb.collections.write_domain.update_batch.unique_check_skips_total"); ok { addPerOperationMetricValue(metrics, "unique_eligible_checks/doc", uniqueEligible, operations) } @@ -2289,6 +2402,47 @@ func deriveTreeDBPhaseMetrics(delta map[string]float64, operations, driverCalls return metrics } +func addReadOnlyPrepareRootApplySplitMetrics(metrics map[string]float64, delta map[string]float64, operations int) { + rootApplyNS, rootOK := delta["treedb.publish.ordered_root_delta_group.root_apply_ns_total"] + readOnlyPrepareNS, prepareOK := delta["treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ns_total"] + if !rootOK || !prepareOK { + return + } + remainingNS := rootApplyNS - readOnlyPrepareNS + if remainingNS < 0 { + remainingNS = 0 + } + addPerOperationMetricValue(metrics, "publish_delta_group_root_apply_excluding_read_only_prepare_ns/doc", remainingNS, operations) + addRatioMetricValue(metrics, "read_only_prepare_root_apply_share_pct", readOnlyPrepareNS*100, rootApplyNS) +} + +func addRootDeltaKindMetrics(metrics map[string]float64, statPrefix, metricPrefix string, delta map[string]float64, operations int) { + entryKeys := make([]string, 0, len(treeDBRootDeltaKindMetrics)) + byteKeys := make([]string, 0, len(treeDBRootDeltaKindMetrics)) + tombstoneKeys := make([]string, 0, len(treeDBRootDeltaKindMetrics)) + for _, kind := range treeDBRootDeltaKindMetrics { + base := "treedb.collections.write_domain." + statPrefix + "." + kind.statToken + entryKey := base + ".entries_total" + byteKey := base + ".bytes_total" + tombstoneKey := base + ".tombstones_total" + entryKeys = append(entryKeys, entryKey) + byteKeys = append(byteKeys, byteKey) + tombstoneKeys = append(tombstoneKeys, tombstoneKey) + addPerOperationMetric(metrics, metricPrefix+"_"+kind.metricToken+"_root_delta_entries/doc", delta, entryKey, operations) + addPerOperationMetric(metrics, metricPrefix+"_"+kind.metricToken+"_root_delta_bytes/doc", delta, byteKey, operations) + addPerOperationMetric(metrics, metricPrefix+"_"+kind.metricToken+"_root_delta_tombstones/doc", delta, tombstoneKey, operations) + } + if total, ok := sumTreeDBMetricDeltas(delta, entryKeys...); ok { + addPerOperationMetricValue(metrics, metricPrefix+"_root_delta_entries/doc", total, operations) + } + if total, ok := sumTreeDBMetricDeltas(delta, byteKeys...); ok { + addPerOperationMetricValue(metrics, metricPrefix+"_root_delta_bytes/doc", total, operations) + } + if total, ok := sumTreeDBMetricDeltas(delta, tombstoneKeys...); ok { + addPerOperationMetricValue(metrics, metricPrefix+"_root_delta_tombstones/doc", total, operations) + } +} + func addPerOperationMetric(metrics map[string]float64, name string, delta map[string]float64, key string, operations int) { if operations <= 0 { return @@ -3000,12 +3154,14 @@ func writeResult(out io.Writer, format string, result *benchmarkResult) error { fmt.Fprintf(out, "treedb_dir=%s\n", result.TreeDBDir) } if result.TreeDBProfile != "" { - fmt.Fprintf(out, "treedb_profile=%s document_format=%s data_root_storage=%s index_state_root_storage=%s index_root_storage=%s buffered_indexed_max_docs=%d buffered_indexed_max_bytes=%d buffered_indexed_max_root_runs=%d buffered_indexed_async_flush=%t buffered_indexed_async_max_queued_units=%d maintenance=%s\n", + fmt.Fprintf(out, "treedb_profile=%s document_format=%s data_root_storage=%s index_state_root_storage=%s index_root_storage=%s buffered_indexed_max_docs=%d buffered_indexed_max_bytes=%d buffered_indexed_max_root_runs=%d buffered_indexed_async_flush=%t buffered_indexed_async_max_queued_units=%d buffered_indexed_read_only_prepare=%t buffered_indexed_read_only_prepare_workers=%d maintenance=%s\n", result.TreeDBProfile, result.TreeDBDocumentFormat, result.TreeDBDataRootStorage, result.TreeDBIndexStateRootStorage, result.TreeDBIndexRootStorage, result.TreeDBBufferedIndexedWriteMaxDocuments, result.TreeDBBufferedIndexedWriteMaxBytes, result.TreeDBBufferedIndexedWriteMaxRootRuns, result.TreeDBBufferedIndexedAsyncFlush, - result.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits, result.TreeDBMaintenanceMode) + result.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits, + result.TreeDBBufferedIndexedReadOnlyPrepare, result.TreeDBBufferedIndexedReadOnlyPrepareWorkers, + result.TreeDBMaintenanceMode) } if result.MongoURI != "" { fmt.Fprintf(out, "mongo_uri=%s\n", result.MongoURI) @@ -3019,16 +3175,23 @@ func writeResult(out io.Writer, format string, result *benchmarkResult) error { if phase.EffectiveProducers > 0 { fmt.Fprintf(out, " effective_producers=%d", phase.EffectiveProducers) } - fmt.Fprintf(out, " duration_ms=%.1f ops_sec=%.1f sampled_ops_sec=%.1f sampled_ns_op=%.1f driver_aggregate_ms=%.1f driver_mean_us=%.0f p50_us=%.0f p95_us=%.0f p99_us=%.0f\n", + fmt.Fprintf(out, " duration_ms=%.1f ops_sec=%.1f sampled_ops_sec=%.1f sampled_ns_op=%.1f driver_aggregate_ms=%.1f driver_mean_us=%.0f p50_us=%.0f p95_us=%.0f p99_us=%.0f", phase.DurationMillis, phase.OpsPerSecond, phase.SampledOpsPerSecond, phase.SampledNsPerOp, phase.DriverAggregateMillis, phase.DriverMeanLatencyMicros, phase.LatencyMicros.P50, phase.LatencyMicros.P95, phase.LatencyMicros.P99) + if phase.TreeDBDrainMillis > 0 { + fmt.Fprintf(out, " treedb_drain_ms=%.3f", phase.TreeDBDrainMillis) + } + fmt.Fprintln(out) for _, producer := range phase.ProducerResults { fmt.Fprintf(out, " producer=%d ops=%d calls=%d duration_ms=%.1f ops_sec=%.1f driver_aggregate_ms=%.1f driver_mean_us=%.0f p50_us=%.0f p95_us=%.0f p99_us=%.0f\n", producer.Producer, producer.Operations, producer.DriverCalls, producer.DurationMillis, producer.OpsPerSecond, producer.DriverAggregateMillis, producer.DriverMeanLatencyMicros, producer.LatencyMicros.P50, producer.LatencyMicros.P95, producer.LatencyMicros.P99) } + if len(phase.TreeDBDrainStatsDelta) > 0 { + writeTreeDBStats(out, "phase_treedb_drain_stats_delta."+phase.Name, phase.TreeDBDrainStatsDelta) + } if len(phase.TreeDBStatsDelta) > 0 { writeTreeDBStats(out, "phase_treedb_stats_delta."+phase.Name, phase.TreeDBStatsDelta) } diff --git a/cmd/mongo_gateway_bench/main_test.go b/cmd/mongo_gateway_bench/main_test.go index 21b62985f3..e451e74eb1 100644 --- a/cmd/mongo_gateway_bench/main_test.go +++ b/cmd/mongo_gateway_bench/main_test.go @@ -23,6 +23,16 @@ import ( "go.mongodb.org/mongo-driver/v2/event" ) +const ( + testReadOnlyPrepareCallsStat = "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total" + testReadOnlyPrepareNSStat = "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ns_total" + testReadOnlyPrepareOpsStat = "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_ops_total" + testReadOnlyPrepareLeafSpansStat = "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_leaf_spans_total" + testReadOnlyPrepareWorkerTargetStat = "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_targets_total" + testReadOnlyPrepareWorkerRangesStat = "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_ranges_total" + testReadOnlyPrepareWorkerMaxOpsStat = "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_range_max_ops_total" +) + func TestSummarizeLatencyNearestRank(t *testing.T) { summary := summarizeLatency([]time.Duration{ 10 * time.Microsecond, @@ -174,50 +184,158 @@ func TestSelectedTreeDBStats(t *testing.T) { func TestTreeDBStatsDeltaAndPhaseMetrics(t *testing.T) { before := map[string]string{ - "treedb.publish.ordered_root_delta_group.calls_total": "2", - "treedb.publish.ordered_root_delta_group.roots_total": "6", - "treedb.publish.ordered_root_delta_group.root_apply_calls_total": "6", - "treedb.publish.ordered_root_delta_group.root_apply_ns_total": "1000", - "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_node_loads_total": "4", - "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_pages_written_total": "1", - "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_node_bytes_read_total": "128", - "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_page_bytes_written_total": "256", - "treedb.collections.write_domain.indexed_flush.calls_total": "1", - "treedb.collections.write_domain.indexed_flush.docs_total": "8", - "treedb.collections.write_domain.indexed_flush.units_total": "1", - "treedb.collections.write_domain.indexed_flush.root_runs_total": "4", - "treedb.collections.write_domain.root_delta_plan.entries_total": "10", - "treedb.collections.write_domain.root_delta_plan.key_bytes_total": "100", - "treedb.collections.write_domain.root_delta_plan.value_bytes_total": "200", - "treedb.collections.write_domain.root_delta_plan.tombstones_total": "1", - "treedb.collections.write_domain.root_delta_plan.roots.primary_total": "2", - "treedb.collections.write_domain.root_delta_plan.roots.template_total": "0", - "treedb.collections.write_domain.root_delta_plan.roots.index_state_total": "1", - "treedb.collections.write_domain.root_delta_plan.roots.secondary_total": "3", - "treedb.test.large_counter_total": "9007199254740993", + "treedb.publish.ordered_root_delta_group.calls_total": "2", + "treedb.publish.ordered_root_delta_group.roots_total": "6", + "treedb.publish.ordered_root_delta_group.root_apply_calls_total": "6", + "treedb.publish.ordered_root_delta_group.root_apply_ns_total": "1000", + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_merges_total": "1", + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_children_total": "8", + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_workers_total": "4", + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_ops_total": "1024", + testReadOnlyPrepareCallsStat: "1", + testReadOnlyPrepareNSStat: "100", + testReadOnlyPrepareOpsStat: "10", + testReadOnlyPrepareLeafSpansStat: "5", + testReadOnlyPrepareWorkerTargetStat: "4", + testReadOnlyPrepareWorkerRangesStat: "2", + testReadOnlyPrepareWorkerMaxOpsStat: "20", + "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_node_loads_total": "4", + "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_pages_written_total": "1", + "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_node_bytes_read_total": "128", + "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_page_bytes_written_total": "256", + "treedb.collections.write_domain.indexed_flush.calls_total": "1", + "treedb.collections.write_domain.indexed_flush.docs_total": "8", + "treedb.collections.write_domain.indexed_flush.units_total": "1", + "treedb.collections.write_domain.indexed_flush.root_runs_total": "4", + "treedb.collections.write_domain.root_delta_plan.entries_total": "10", + "treedb.collections.write_domain.root_delta_plan.key_bytes_total": "100", + "treedb.collections.write_domain.root_delta_plan.value_bytes_total": "200", + "treedb.collections.write_domain.root_delta_plan.tombstones_total": "1", + "treedb.collections.write_domain.root_delta_plan.roots.primary_total": "2", + "treedb.collections.write_domain.root_delta_plan.roots.template_total": "0", + "treedb.collections.write_domain.root_delta_plan.roots.index_state_total": "1", + "treedb.collections.write_domain.root_delta_plan.roots.secondary_total": "3", + "treedb.collections.write_domain.coalesced_flush_batch.batches_total": "1", + "treedb.collections.write_domain.coalesced_flush_batch.units_total": "2", + "treedb.collections.write_domain.coalesced_flush_batch.docs_total": "20", + "treedb.collections.write_domain.coalesced_flush_batch.bytes_total": "2000", + "treedb.collections.write_domain.coalesced_flush_batch.net_zero_batches_total": "0", + "treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total": "5", + "treedb.collections.write_domain.root_delta_plan.raw_unit.primary.bytes_total": "50", + "treedb.collections.write_domain.root_delta_plan.raw_unit.primary.tombstones_total": "0", + "treedb.collections.write_domain.root_delta_plan.raw_unit.template.entries_total": "0", + "treedb.collections.write_domain.root_delta_plan.raw_unit.template.bytes_total": "0", + "treedb.collections.write_domain.root_delta_plan.raw_unit.template.tombstones_total": "0", + "treedb.collections.write_domain.root_delta_plan.raw_unit.index_state.entries_total": "0", + "treedb.collections.write_domain.root_delta_plan.raw_unit.index_state.bytes_total": "0", + "treedb.collections.write_domain.root_delta_plan.raw_unit.index_state.tombstones_total": "0", + "treedb.collections.write_domain.root_delta_plan.raw_unit.secondary.entries_total": "3", + "treedb.collections.write_domain.root_delta_plan.raw_unit.secondary.bytes_total": "30", + "treedb.collections.write_domain.root_delta_plan.raw_unit.secondary.tombstones_total": "1", + "treedb.collections.write_domain.root_delta_plan.final.primary.entries_total": "5", + "treedb.collections.write_domain.root_delta_plan.final.primary.bytes_total": "50", + "treedb.collections.write_domain.root_delta_plan.final.primary.tombstones_total": "0", + "treedb.collections.write_domain.root_delta_plan.final.template.entries_total": "0", + "treedb.collections.write_domain.root_delta_plan.final.template.bytes_total": "0", + "treedb.collections.write_domain.root_delta_plan.final.template.tombstones_total": "0", + "treedb.collections.write_domain.root_delta_plan.final.index_state.entries_total": "0", + "treedb.collections.write_domain.root_delta_plan.final.index_state.bytes_total": "0", + "treedb.collections.write_domain.root_delta_plan.final.index_state.tombstones_total": "0", + "treedb.collections.write_domain.root_delta_plan.final.secondary.entries_total": "3", + "treedb.collections.write_domain.root_delta_plan.final.secondary.bytes_total": "30", + "treedb.collections.write_domain.root_delta_plan.final.secondary.tombstones_total": "1", + "treedb.collections.write_domain.root_delta_plan.squashed_entries_total": "0", + "treedb.collections.write_domain.root_delta_plan.net_zero_plans_total": "0", + "treedb.collections.write_domain.indexed_semantic.skipped_secondary_roots_total": "0", + "treedb.collections.write_domain.primary_only.root_publishes_total": "4", + "treedb.collections.write_domain.primary_only.root_delta_entries_total": "0", + "treedb.collections.write_domain.primary_only.root_delta_key_bytes_total": "0", + "treedb.collections.write_domain.primary_only.root_delta_value_bytes_total": "0", + "treedb.collections.write_domain.primary_only.coalesced_docs_total": "0", + "treedb.collections.write_domain.primary_only.duplicate_ids_coalesced_total": "0", + "treedb.collections.write_domain.primary_only.drains_total": "1", + "treedb.collections.write_domain.primary_only.drain_docs_total": "10", + "treedb.collections.write_domain.primary_only.drain_bytes_total": "100", + "treedb.collections.write_domain.primary_only.drain_ns_total": "1000", + "treedb.collections.write_domain.primary_only.buffered_calls_total": "2", + "treedb.test.large_counter_total": "9007199254740993", } after := map[string]string{ - "treedb.publish.ordered_root_delta_group.calls_total": "5", - "treedb.publish.ordered_root_delta_group.roots_total": "15", - "treedb.publish.ordered_root_delta_group.root_apply_calls_total": "15", - "treedb.publish.ordered_root_delta_group.root_apply_ns_total": "7000", - "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_node_loads_total": "10", - "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_pages_written_total": "4", - "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_node_bytes_read_total": "640", - "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_page_bytes_written_total": "1280", - "treedb.collections.write_domain.indexed_flush.calls_total": "3", - "treedb.collections.write_domain.indexed_flush.docs_total": "48", - "treedb.collections.write_domain.indexed_flush.units_total": "7", - "treedb.collections.write_domain.indexed_flush.root_runs_total": "16", - "treedb.collections.write_domain.root_delta_plan.entries_total": "50", - "treedb.collections.write_domain.root_delta_plan.key_bytes_total": "500", - "treedb.collections.write_domain.root_delta_plan.value_bytes_total": "1000", - "treedb.collections.write_domain.root_delta_plan.tombstones_total": "5", - "treedb.collections.write_domain.root_delta_plan.roots.primary_total": "6", - "treedb.collections.write_domain.root_delta_plan.roots.template_total": "2", - "treedb.collections.write_domain.root_delta_plan.roots.index_state_total": "3", - "treedb.collections.write_domain.root_delta_plan.roots.secondary_total": "9", - "treedb.test.large_counter_total": "9007199254741000", + "treedb.publish.ordered_root_delta_group.calls_total": "5", + "treedb.publish.ordered_root_delta_group.roots_total": "15", + "treedb.publish.ordered_root_delta_group.root_apply_calls_total": "15", + "treedb.publish.ordered_root_delta_group.root_apply_ns_total": "7000", + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_merges_total": "5", + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_children_total": "40", + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_workers_total": "16", + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_ops_total": "4096", + testReadOnlyPrepareCallsStat: "4", + testReadOnlyPrepareNSStat: "700", + testReadOnlyPrepareOpsStat: "70", + testReadOnlyPrepareLeafSpansStat: "20", + testReadOnlyPrepareWorkerTargetStat: "16", + testReadOnlyPrepareWorkerRangesStat: "11", + testReadOnlyPrepareWorkerMaxOpsStat: "110", + "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_node_loads_total": "10", + "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_pages_written_total": "4", + "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_node_bytes_read_total": "640", + "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_page_bytes_written_total": "1280", + "treedb.collections.write_domain.indexed_flush.calls_total": "3", + "treedb.collections.write_domain.indexed_flush.docs_total": "48", + "treedb.collections.write_domain.indexed_flush.units_total": "7", + "treedb.collections.write_domain.indexed_flush.root_runs_total": "16", + "treedb.collections.write_domain.root_delta_plan.entries_total": "50", + "treedb.collections.write_domain.root_delta_plan.key_bytes_total": "500", + "treedb.collections.write_domain.root_delta_plan.value_bytes_total": "1000", + "treedb.collections.write_domain.root_delta_plan.tombstones_total": "5", + "treedb.collections.write_domain.root_delta_plan.roots.primary_total": "6", + "treedb.collections.write_domain.root_delta_plan.roots.template_total": "2", + "treedb.collections.write_domain.root_delta_plan.roots.index_state_total": "3", + "treedb.collections.write_domain.root_delta_plan.roots.secondary_total": "9", + "treedb.collections.write_domain.coalesced_flush_batch.batches_total": "3", + "treedb.collections.write_domain.coalesced_flush_batch.units_total": "10", + "treedb.collections.write_domain.coalesced_flush_batch.docs_total": "100", + "treedb.collections.write_domain.coalesced_flush_batch.bytes_total": "10000", + "treedb.collections.write_domain.coalesced_flush_batch.net_zero_batches_total": "1", + "treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total": "45", + "treedb.collections.write_domain.root_delta_plan.raw_unit.primary.bytes_total": "450", + "treedb.collections.write_domain.root_delta_plan.raw_unit.primary.tombstones_total": "4", + "treedb.collections.write_domain.root_delta_plan.raw_unit.template.entries_total": "4", + "treedb.collections.write_domain.root_delta_plan.raw_unit.template.bytes_total": "40", + "treedb.collections.write_domain.root_delta_plan.raw_unit.template.tombstones_total": "0", + "treedb.collections.write_domain.root_delta_plan.raw_unit.index_state.entries_total": "6", + "treedb.collections.write_domain.root_delta_plan.raw_unit.index_state.bytes_total": "60", + "treedb.collections.write_domain.root_delta_plan.raw_unit.index_state.tombstones_total": "1", + "treedb.collections.write_domain.root_delta_plan.raw_unit.secondary.entries_total": "33", + "treedb.collections.write_domain.root_delta_plan.raw_unit.secondary.bytes_total": "330", + "treedb.collections.write_domain.root_delta_plan.raw_unit.secondary.tombstones_total": "4", + "treedb.collections.write_domain.root_delta_plan.final.primary.entries_total": "25", + "treedb.collections.write_domain.root_delta_plan.final.primary.bytes_total": "250", + "treedb.collections.write_domain.root_delta_plan.final.primary.tombstones_total": "2", + "treedb.collections.write_domain.root_delta_plan.final.template.entries_total": "2", + "treedb.collections.write_domain.root_delta_plan.final.template.bytes_total": "20", + "treedb.collections.write_domain.root_delta_plan.final.template.tombstones_total": "0", + "treedb.collections.write_domain.root_delta_plan.final.index_state.entries_total": "4", + "treedb.collections.write_domain.root_delta_plan.final.index_state.bytes_total": "40", + "treedb.collections.write_domain.root_delta_plan.final.index_state.tombstones_total": "1", + "treedb.collections.write_domain.root_delta_plan.final.secondary.entries_total": "23", + "treedb.collections.write_domain.root_delta_plan.final.secondary.bytes_total": "230", + "treedb.collections.write_domain.root_delta_plan.final.secondary.tombstones_total": "3", + "treedb.collections.write_domain.root_delta_plan.squashed_entries_total": "34", + "treedb.collections.write_domain.root_delta_plan.net_zero_plans_total": "2", + "treedb.collections.write_domain.indexed_semantic.skipped_secondary_roots_total": "12", + "treedb.collections.write_domain.primary_only.root_publishes_total": "12", + "treedb.collections.write_domain.primary_only.root_delta_entries_total": "20", + "treedb.collections.write_domain.primary_only.root_delta_key_bytes_total": "100", + "treedb.collections.write_domain.primary_only.root_delta_value_bytes_total": "300", + "treedb.collections.write_domain.primary_only.coalesced_docs_total": "32", + "treedb.collections.write_domain.primary_only.duplicate_ids_coalesced_total": "6", + "treedb.collections.write_domain.primary_only.drains_total": "3", + "treedb.collections.write_domain.primary_only.drain_docs_total": "50", + "treedb.collections.write_domain.primary_only.drain_bytes_total": "500", + "treedb.collections.write_domain.primary_only.drain_ns_total": "5000", + "treedb.collections.write_domain.primary_only.buffered_calls_total": "10", + "treedb.test.large_counter_total": "9007199254741000", } phase := summarizePhase("concurrent_id_update_set_w8", 40, 20, time.Second, []time.Duration{time.Millisecond}) attachTreeDBPhaseStats(&phase, before, after) @@ -228,27 +346,73 @@ func TestTreeDBStatsDeltaAndPhaseMetrics(t *testing.T) { t.Fatalf("large counter delta=%q want 7; deltas=%v", got, phase.TreeDBStatsDelta) } for name, want := range map[string]float64{ - "publish_delta_group_calls/doc": 0.075, - "root_apply_calls/doc": 0.225, - "roots/publish": 3, - "publish_delta_group_root_apply_ns/doc": 150, - "leaf_log_node_loads/doc": 0.15, - "leaf_log_pages_written/doc": 0.075, - "leaf_log_read_bytes/doc": 12.8, - "leaf_log_write_bytes/doc": 25.6, - "indexed_flush_calls/doc": 0.05, - "indexed_flush_docs/batch": 20, - "indexed_flush_units/batch": 3, - "indexed_flush_root_runs/doc": 0.3, - "root_delta_plan_entries/doc": 1, - "root_delta_plan_key_bytes/doc": 10, - "root_delta_plan_value_bytes/doc": 20, - "root_delta_plan_tombstones/doc": 0.1, - "affected_primary_roots/doc": 0.1, - "affected_template_roots/doc": 0.05, - "affected_index_state_roots/doc": 0.05, - "affected_secondary_roots/doc": 0.15, - "publish_delta_group_calls/driver_call": 0.15, + "publish_delta_group_calls/doc": 0.075, + "root_apply_calls/doc": 0.225, + "roots/publish": 3, + "publish_delta_group_root_apply_ns/doc": 150, + "internal_parallel_merges/doc": 0.1, + "internal_parallel_children/merge": 8, + "internal_parallel_workers/merge": 3, + "internal_parallel_ops/merge": 768, + "publish_delta_group_root_apply_excluding_read_only_prepare_ns/doc": 135, + "read_only_prepare_calls/doc": 0.075, + "read_only_prepare_ns/doc": 15, + "read_only_prepare_root_apply_share_pct": 10, + "read_only_prepare_ns/plan": 200, + "read_only_prepare_ops/doc": 1.5, + "read_only_prepare_leaf_spans/plan": 5, + "read_only_prepare_worker_targets/plan": 4, + "read_only_prepare_worker_ranges/plan": 3, + "read_only_prepare_worker_max_ops/plan": 30, + "leaf_log_node_loads/doc": 0.15, + "leaf_log_pages_written/doc": 0.075, + "leaf_log_read_bytes/doc": 12.8, + "leaf_log_write_bytes/doc": 25.6, + "indexed_flush_calls/doc": 0.05, + "indexed_flush_docs/batch": 20, + "indexed_flush_units/batch": 3, + "indexed_flush_root_runs/doc": 0.3, + "root_delta_plan_entries/doc": 1, + "root_delta_plan_key_bytes/doc": 10, + "root_delta_plan_value_bytes/doc": 20, + "root_delta_plan_tombstones/doc": 0.1, + "affected_primary_roots/doc": 0.1, + "affected_template_roots/doc": 0.05, + "affected_index_state_roots/doc": 0.05, + "affected_secondary_roots/doc": 0.15, + "coalesced_batch_units/batch": 4, + "coalesced_batch_docs/batch": 40, + "coalesced_batch_bytes/batch": 4000, + "net_zero_root_batches/doc": 0.025, + "raw_root_delta_entries/doc": 2, + "raw_root_delta_bytes/doc": 20, + "raw_root_delta_tombstones/doc": 0.2, + "raw_primary_root_delta_entries/doc": 1, + "raw_primary_root_delta_bytes/doc": 10, + "raw_secondary_root_delta_entries/doc": 0.75, + "raw_secondary_root_delta_bytes/doc": 7.5, + "final_root_delta_entries/doc": 1.15, + "final_root_delta_bytes/doc": 11.5, + "final_root_delta_tombstones/doc": 0.125, + "final_primary_root_delta_entries/doc": 0.5, + "final_primary_root_delta_bytes/doc": 5, + "final_secondary_root_delta_entries/doc": 0.5, + "final_secondary_root_delta_bytes/doc": 5, + "squashed_root_delta_entries/doc": 0.85, + "net_zero_root_plans/doc": 0.05, + "skipped_secondary_roots/doc": 0.3, + "primary_root_publishes/doc": 0.2, + "primary_root_delta_entries/doc": 0.5, + "primary_root_delta_bytes/doc": 10, + "primary_only_coalesced_docs/publish": 4, + "primary_only_duplicate_ids_coalesced/doc": 0.15, + "primary_only_drains/doc": 0.05, + "primary_only_drain_docs/drain": 20, + "primary_only_drain_bytes/doc": 10, + "primary_only_drain_ns/doc": 100, + "primary_only_buffered_calls/driver_call": 0.4, + "primary_only_publish_calls/driver_call": 0.4, + "publish_delta_group_calls/driver_call": 0.15, } { if got := phase.TreeDBMetrics[name]; math.Abs(got-want) > 1e-9 { t.Fatalf("metric %s=%v want %v; metrics=%v", name, got, want, phase.TreeDBMetrics) @@ -257,23 +421,70 @@ func TestTreeDBStatsDeltaAndPhaseMetrics(t *testing.T) { } func TestDeriveTreeDBPhaseMetricsEmitsZeroValues(t *testing.T) { - metrics := deriveTreeDBPhaseMetrics(map[string]float64{ - "treedb.publish.ordered_root_delta_group.calls_total": 2, - "treedb.publish.ordered_root_delta_group.roots_total": 2, - "treedb.publish.ordered_root_delta_group.root_apply_calls_total": 2, - "treedb.publish.ordered_root_delta_group.root_apply_ns_total": 20, - "treedb.collections.write_domain.indexed_flush.calls_total": 2, - "treedb.collections.write_domain.indexed_flush.docs_total": 20, - "treedb.collections.write_domain.indexed_flush.units_total": 2, - "treedb.collections.write_domain.primary_only.root_publishes_total": 2, - "treedb.collections.write_domain.root_delta_plan.tombstones_total": 0, - "treedb.collections.write_domain.primary_only.coalesced_docs_total": 0, - "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_node_loads_total": 0, - }, 10, 2) + delta := map[string]float64{ + "treedb.publish.ordered_root_delta_group.calls_total": 2, + "treedb.publish.ordered_root_delta_group.roots_total": 2, + "treedb.publish.ordered_root_delta_group.root_apply_calls_total": 2, + "treedb.publish.ordered_root_delta_group.root_apply_ns_total": 20, + testReadOnlyPrepareCallsStat: 0, + testReadOnlyPrepareNSStat: 0, + testReadOnlyPrepareOpsStat: 0, + testReadOnlyPrepareLeafSpansStat: 0, + testReadOnlyPrepareWorkerTargetStat: 0, + testReadOnlyPrepareWorkerRangesStat: 0, + testReadOnlyPrepareWorkerMaxOpsStat: 0, + "treedb.collections.write_domain.indexed_flush.calls_total": 2, + "treedb.collections.write_domain.indexed_flush.docs_total": 20, + "treedb.collections.write_domain.indexed_flush.units_total": 2, + "treedb.collections.write_domain.coalesced_flush_batch.batches_total": 2, + "treedb.collections.write_domain.coalesced_flush_batch.units_total": 0, + "treedb.collections.write_domain.coalesced_flush_batch.docs_total": 0, + "treedb.collections.write_domain.coalesced_flush_batch.bytes_total": 0, + "treedb.collections.write_domain.coalesced_flush_batch.net_zero_batches_total": 0, + "treedb.collections.write_domain.primary_only.root_publishes_total": 2, + "treedb.collections.write_domain.primary_only.drains_total": 2, + "treedb.collections.write_domain.primary_only.drain_docs_total": 0, + "treedb.collections.write_domain.primary_only.duplicate_ids_coalesced_total": 0, + "treedb.collections.write_domain.root_delta_plan.tombstones_total": 0, + "treedb.collections.write_domain.root_delta_plan.squashed_entries_total": 0, + "treedb.collections.write_domain.root_delta_plan.net_zero_plans_total": 0, + "treedb.collections.write_domain.indexed_semantic.skipped_secondary_roots_total": 0, + "treedb.collections.write_domain.primary_only.coalesced_docs_total": 0, + "treedb.publish.ordered_root_delta_group.root_apply_leaf_log_node_loads_total": 0, + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_merges_total": 0, + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_children_total": 0, + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_workers_total": 0, + "treedb.publish.ordered_root_delta_group.root_apply_internal_parallel_ops_total": 0, + } + for _, prefix := range []string{"raw_unit", "final"} { + for _, kind := range []string{"primary", "template", "index_state", "secondary"} { + for _, stat := range []string{"entries", "bytes", "tombstones"} { + delta["treedb.collections.write_domain.root_delta_plan."+prefix+"."+kind+"."+stat+"_total"] = 0 + } + } + } + metrics := deriveTreeDBPhaseMetrics(delta, 10, 2) for _, name := range []string{ "root_delta_plan_tombstones/doc", "primary_only_coalesced_docs/publish", "leaf_log_node_loads/doc", + "read_only_prepare_calls/doc", + "read_only_prepare_ns/doc", + "read_only_prepare_root_apply_share_pct", + "read_only_prepare_ops/doc", + "internal_parallel_merges/doc", + "coalesced_batch_units/batch", + "coalesced_batch_docs/batch", + "coalesced_batch_bytes/batch", + "net_zero_root_batches/doc", + "raw_root_delta_entries/doc", + "raw_primary_root_delta_entries/doc", + "final_root_delta_entries/doc", + "squashed_root_delta_entries/doc", + "net_zero_root_plans/doc", + "skipped_secondary_roots/doc", + "primary_only_duplicate_ids_coalesced/doc", + "primary_only_drain_docs/drain", } { got, ok := metrics[name] if !ok { @@ -287,8 +498,12 @@ func TestDeriveTreeDBPhaseMetricsEmitsZeroValues(t *testing.T) { func TestPhaseResultJSONIncludesTreeDBStatsDelta(t *testing.T) { phase := phaseResult{ - Name: "concurrent_id_update_set_w4", - Operations: 10, + Name: "concurrent_id_update_set_w4", + Operations: 10, + TreeDBDrainMillis: 1.25, + TreeDBDrainStatsDelta: map[string]string{ + "treedb.collections.write_domain.indexed_flush.calls_total": "1", + }, TreeDBStatsDelta: map[string]string{ "treedb.publish.ordered_root_delta_group.calls_total": "10", }, @@ -300,6 +515,12 @@ func TestPhaseResultJSONIncludesTreeDBStatsDelta(t *testing.T) { if err != nil { t.Fatalf("marshal phase: %v", err) } + if !bytes.Contains(raw, []byte(`"treedb_drain_ms"`)) { + t.Fatalf("phase JSON missing treedb_drain_ms: %s", raw) + } + if !bytes.Contains(raw, []byte(`"treedb_drain_stats_delta"`)) { + t.Fatalf("phase JSON missing treedb_drain_stats_delta: %s", raw) + } if !bytes.Contains(raw, []byte(`"treedb_stats_delta"`)) { t.Fatalf("phase JSON missing treedb_stats_delta: %s", raw) } @@ -308,16 +529,57 @@ func TestPhaseResultJSONIncludesTreeDBStatsDelta(t *testing.T) { } } +func TestPhaseResultJSONIncludesZeroTreeDBDrainMillis(t *testing.T) { + phase := phaseResult{ + Name: "load", + Operations: 1, + } + raw, err := json.Marshal(phase) + if err != nil { + t.Fatalf("marshal phase: %v", err) + } + if !bytes.Contains(raw, []byte(`"treedb_drain_ms":0`)) { + t.Fatalf("phase JSON missing explicit zero treedb_drain_ms: %s", raw) + } +} + +func TestAttachTreeDBDrainStatsPreservesPhaseLocalDelta(t *testing.T) { + phase := summarizePhase("concurrent_id_update_set_w2", 10, 10, time.Second, nil) + attachTreeDBDrainStats(&phase, + map[string]string{ + "treedb.collections.write_domain.indexed_flush.calls_total": "7", + "treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total": "100", + }, + map[string]string{ + "treedb.collections.write_domain.indexed_flush.calls_total": "8", + "treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total": "125", + }, + 2500*time.Microsecond, + ) + if got := phase.TreeDBDrainMillis; got != 2.5 { + t.Fatalf("drain millis=%v want 2.5", got) + } + if got := phase.TreeDBDrainStatsDelta["treedb.collections.write_domain.indexed_flush.calls_total"]; got != "1" { + t.Fatalf("drain indexed_flush calls delta=%q want 1; deltas=%v", got, phase.TreeDBDrainStatsDelta) + } + if got := phase.TreeDBDrainStatsDelta["treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total"]; got != "25" { + t.Fatalf("drain raw primary delta=%q want 25; deltas=%v", got, phase.TreeDBDrainStatsDelta) + } + if phase.TreeDBStatsDelta != nil || phase.TreeDBMetrics != nil { + t.Fatalf("drain stats should not populate phase stats/metrics: %+v", phase) + } +} + func TestTreeDBStatsDeltaPreservesHugeIntegerStrings(t *testing.T) { delta, numeric := treeDBStatsDelta( - map[string]string{"treedb.test.huge_counter_total": "0"}, - map[string]string{"treedb.test.huge_counter_total": "18446744073709551615"}, + map[string]string{"treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total": "0"}, + map[string]string{"treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total": "18446744073709551615"}, ) - if got := delta["treedb.test.huge_counter_total"]; got != "18446744073709551615" { - t.Fatalf("huge counter delta=%q want exact uint64 max string", got) + if got := delta["treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total"]; got != "18446744073709551615" { + t.Fatalf("huge root-delta counter delta=%q want exact uint64 max string", got) } - if _, ok := numeric["treedb.test.huge_counter_total"]; ok { - t.Fatalf("huge counter unexpectedly present in numeric deltas: %v", numeric) + if _, ok := numeric["treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total"]; ok { + t.Fatalf("huge root-delta counter unexpectedly present in numeric deltas: %v", numeric) } } @@ -408,6 +670,12 @@ func TestRunTreeDBProfiledPhaseDrainsBeforeStatsSnapshot(t *testing.T) { if err != nil { t.Fatalf("run phase: %v", err) } + if got := phase.TreeDBDrainStatsDelta["treedb.collections.write_domain.indexed_flush.calls_total"]; got != "1" { + t.Fatalf("drain indexed flush calls delta=%q want 1; drain deltas=%v phase deltas=%v", got, phase.TreeDBDrainStatsDelta, phase.TreeDBStatsDelta) + } + if phase.TreeDBDrainMillis <= 0 { + t.Fatalf("drain millis=%v want positive", phase.TreeDBDrainMillis) + } if got := phase.TreeDBStatsDelta["treedb.collections.write_domain.indexed_flush.calls_total"]; got != "1" { t.Fatalf("indexed flush calls delta=%q want 1; deltas=%v", got, phase.TreeDBStatsDelta) } @@ -799,6 +1067,12 @@ func TestParseConfigTreeDBCorrectnessDefaults(t *testing.T) { if cfg.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits != 0 { t.Fatalf("TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits=%d want 0", cfg.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits) } + if cfg.TreeDBBufferedIndexedReadOnlyPrepare { + t.Fatal("TreeDBBufferedIndexedReadOnlyPrepare=true want false by default") + } + if cfg.TreeDBBufferedIndexedReadOnlyPrepareWorkers != 0 { + t.Fatalf("TreeDBBufferedIndexedReadOnlyPrepareWorkers=%d want 0", cfg.TreeDBBufferedIndexedReadOnlyPrepareWorkers) + } if cfg.InsertProducers != 1 { t.Fatalf("InsertProducers=%d want 1", cfg.InsertProducers) } @@ -840,6 +1114,8 @@ func TestParseConfigTreeDBBufferedIndexedWriteThresholds(t *testing.T) { "-treedb-buffered-indexed-write-max-root-runs", "90", "-treedb-buffered-indexed-async-flush", "-treedb-buffered-indexed-async-flush-max-queued-units", "3", + "-treedb-buffered-indexed-read-only-prepare", + "-treedb-buffered-indexed-read-only-prepare-workers", "4", }) if err != nil { t.Fatalf("parse buffered indexed thresholds: %v", err) @@ -859,6 +1135,24 @@ func TestParseConfigTreeDBBufferedIndexedWriteThresholds(t *testing.T) { if cfg.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits != 3 { t.Fatalf("TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits=%d want 3", cfg.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits) } + if !cfg.TreeDBBufferedIndexedReadOnlyPrepare { + t.Fatal("TreeDBBufferedIndexedReadOnlyPrepare=false want true") + } + if cfg.TreeDBBufferedIndexedReadOnlyPrepareWorkers != 4 { + t.Fatalf("TreeDBBufferedIndexedReadOnlyPrepareWorkers=%d want 4", cfg.TreeDBBufferedIndexedReadOnlyPrepareWorkers) + } +} + +func TestParseConfigTreeDBBufferedIndexedReadOnlyPrepareWorkersRequirePrepare(t *testing.T) { + _, err := parseConfig([]string{"-treedb-buffered-indexed-read-only-prepare-workers", "4"}) + if err == nil || !strings.Contains(err.Error(), "requires -treedb-buffered-indexed-read-only-prepare") { + t.Fatalf("parse workers without prepare err=%v want requires prepare", err) + } + + _, err = parseConfig([]string{"-treedb-buffered-indexed-read-only-prepare-workers", "0"}) + if err == nil || !strings.Contains(err.Error(), "requires -treedb-buffered-indexed-read-only-prepare") { + t.Fatalf("parse explicit zero workers without prepare err=%v want requires prepare", err) + } } func TestParseConfigAcceptsTreeDBBSONDocumentFormat(t *testing.T) { @@ -1886,6 +2180,8 @@ func TestWriteResultIncludesTreeDBBufferedIndexedThresholds(t *testing.T) { TreeDBBufferedIndexedWriteMaxRootRuns: 90, TreeDBBufferedIndexedAsyncFlush: true, TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits: 3, + TreeDBBufferedIndexedReadOnlyPrepare: true, + TreeDBBufferedIndexedReadOnlyPrepareWorkers: 4, TreeDBMaintenanceMode: "none", } var out bytes.Buffer @@ -1899,6 +2195,8 @@ func TestWriteResultIncludesTreeDBBufferedIndexedThresholds(t *testing.T) { "buffered_indexed_max_root_runs=90", "buffered_indexed_async_flush=true", "buffered_indexed_async_max_queued_units=3", + "buffered_indexed_read_only_prepare=true", + "buffered_indexed_read_only_prepare_workers=4", } { if !strings.Contains(text, want) { t.Fatalf("text output missing %s: %q", want, text) @@ -1917,13 +2215,42 @@ func TestWriteResultIncludesTreeDBBufferedIndexedThresholds(t *testing.T) { decoded.TreeDBBufferedIndexedWriteMaxBytes != 5678 || decoded.TreeDBBufferedIndexedWriteMaxRootRuns != 90 || !decoded.TreeDBBufferedIndexedAsyncFlush || - decoded.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits != 3 { - t.Fatalf("json thresholds docs=%d bytes=%d rootRuns=%d async=%t asyncMax=%d want 1234/5678/90/true/3", + decoded.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits != 3 || + !decoded.TreeDBBufferedIndexedReadOnlyPrepare || + decoded.TreeDBBufferedIndexedReadOnlyPrepareWorkers != 4 { + t.Fatalf("json thresholds docs=%d bytes=%d rootRuns=%d async=%t asyncMax=%d readonly=%t readonlyWorkers=%d want 1234/5678/90/true/3/true/4", decoded.TreeDBBufferedIndexedWriteMaxDocuments, decoded.TreeDBBufferedIndexedWriteMaxBytes, decoded.TreeDBBufferedIndexedWriteMaxRootRuns, decoded.TreeDBBufferedIndexedAsyncFlush, - decoded.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits) + decoded.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits, + decoded.TreeDBBufferedIndexedReadOnlyPrepare, + decoded.TreeDBBufferedIndexedReadOnlyPrepareWorkers) + } +} + +func TestWriteResultIncludesTreeDBBufferedIndexedReadOnlyPrepareDefaultsJSON(t *testing.T) { + result := &benchmarkResult{ + Target: "treedb", + Database: "bench", + Collection: "docs", + Documents: 1, + } + var out bytes.Buffer + if err := writeResult(&out, "json", result); err != nil { + t.Fatalf("writeResult json: %v", err) + } + var decoded map[string]json.RawMessage + if err := json.Unmarshal(out.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal json result: %v", err) + } + for _, key := range []string{ + "treedb_buffered_indexed_read_only_prepare", + "treedb_buffered_indexed_read_only_prepare_workers", + } { + if _, ok := decoded[key]; !ok { + t.Fatalf("json result omitted %s: %s", key, out.String()) + } } } @@ -1937,11 +2264,13 @@ func TestRecordEffectiveTreeDBCollectionOptionsUsesNormalizedMetadata(t *testing if _, err := manager.CreateCollection(&collections.CollectionMeta{ Name: "bench.docs", Options: collections.CollectionOptions{ - BufferedIndexedWriteMaxDocuments: 0, - BufferedIndexedWriteMaxBytes: 777, - BufferedIndexedWriteMaxRootRuns: 0, - BufferedIndexedAsyncFlush: true, - BufferedIndexedAsyncFlushMaxQueuedUnits: 3, + BufferedIndexedWriteMaxDocuments: 0, + BufferedIndexedWriteMaxBytes: 777, + BufferedIndexedWriteMaxRootRuns: 0, + BufferedIndexedAsyncFlush: true, + BufferedIndexedAsyncFlushMaxQueuedUnits: 3, + BufferedIndexedReadOnlyPrepare: true, + BufferedIndexedReadOnlyPrepareWorkerCount: 4, }, Indexes: []collections.IndexDefinition{{Name: "email_1", Field: "email", ValueType: collections.IndexValueString, Unique: true}}, }); err != nil { @@ -1965,13 +2294,17 @@ func TestRecordEffectiveTreeDBCollectionOptionsUsesNormalizedMetadata(t *testing result.TreeDBBufferedIndexedWriteMaxBytes != 777 || result.TreeDBBufferedIndexedWriteMaxRootRuns != 0 || !result.TreeDBBufferedIndexedAsyncFlush || - result.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits != 3 { - t.Fatalf("effective thresholds docs=%d bytes=%d rootRuns=%d async=%t asyncMax=%d want %d/777/0/true/3", + result.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits != 3 || + !result.TreeDBBufferedIndexedReadOnlyPrepare || + result.TreeDBBufferedIndexedReadOnlyPrepareWorkers != 4 { + t.Fatalf("effective thresholds docs=%d bytes=%d rootRuns=%d async=%t asyncMax=%d readonly=%t readonlyWorkers=%d want %d/777/0/true/3/true/4", result.TreeDBBufferedIndexedWriteMaxDocuments, result.TreeDBBufferedIndexedWriteMaxBytes, result.TreeDBBufferedIndexedWriteMaxRootRuns, result.TreeDBBufferedIndexedAsyncFlush, result.TreeDBBufferedIndexedAsyncFlushMaxQueuedUnits, + result.TreeDBBufferedIndexedReadOnlyPrepare, + result.TreeDBBufferedIndexedReadOnlyPrepareWorkers, collections.DefaultIndexedWriteMemtableAsyncFlushMaxDocuments) } } diff --git a/cmd/mongo_gateway_bench/profile_bench_test.go b/cmd/mongo_gateway_bench/profile_bench_test.go index 1ba0553b6d..dd05d1164e 100644 --- a/cmd/mongo_gateway_bench/profile_bench_test.go +++ b/cmd/mongo_gateway_bench/profile_bench_test.go @@ -622,6 +622,28 @@ func BenchmarkDirectCollectionConcurrentUpdateBSONIndexes3CityUpdate(b *testing. }, true) } +func BenchmarkDirectCollectionUpdateBatchBSONIndexes2CityUpdate(b *testing.B) { + for _, batchSize := range []int{1, 8, 64, 512, 5000} { + b.Run("batch_"+strconv.Itoa(batchSize), func(b *testing.B) { + benchmarkDirectCollectionUpdateBatchBSON(b, []collections.IndexDefinition{ + { + Name: "email_1", + Field: "email", + ValueType: collections.IndexValueString, + Unique: true, + StoragePolicy: collections.RootStorageCompressed, + }, + { + Name: "city_1", + Field: "city", + ValueType: collections.IndexValueString, + StoragePolicy: collections.RootStorageCompressed, + }, + }, true, batchSize) + }) + } +} + func benchmarkDirectCollectionConcurrentUpdateBSON(b *testing.B, indexes []collections.IndexDefinition, updateCity bool) { b.Helper() dir := filepath.Join(b.TempDir(), "treedb") @@ -739,6 +761,136 @@ func benchmarkDirectCollectionConcurrentUpdateBSON(b *testing.B, indexes []colle reportDocsPerSecond(b, b.N, timedElapsed) } +func benchmarkDirectCollectionUpdateBatchBSON(b *testing.B, indexes []collections.IndexDefinition, updateCity bool, updateBatchSize int) { + b.Helper() + if updateBatchSize <= 0 { + b.Fatalf("invalid update batch size %d", updateBatchSize) + } + dir := filepath.Join(b.TempDir(), "treedb") + opts := treedb.OptionsFor(treedb.ProfileWALOnFast, dir) + opts.IndexOuterLeavesInValueLog = true + opts.IndexInternalBaseDelta = false + backend, cleanup, err := treedb.OpenBackendWithCachedLeafLog(opts) + if err != nil { + b.Fatalf("open backend: %v", err) + } + defer func() { + if err := cleanup(); err != nil { + b.Fatalf("close backend: %v", err) + } + }() + manager := collections.NewCollectionManager(backend) + if _, err := manager.CreateCollection(&collections.CollectionMeta{ + Name: "bench.docs", + Options: profileBenchCollectionOptions(b, collections.DocumentFormatBSON), + }); err != nil { + b.Fatalf("create collection: %v", err) + } + collection, err := manager.OpenCollection("bench.docs") + if err != nil { + b.Fatalf("open collection: %v", err) + } + for _, idx := range indexes { + if _, err := collection.CreateIndex(idx); err != nil { + b.Fatalf("create index %s: %v", idx.Name, err) + } + } + + documentCount := profileBenchUpdateDocumentCount(b) + preloadBatchSize := profileBenchBatchSize(b) + for inserted := 0; inserted < documentCount; { + count := preloadBatchSize + if remaining := documentCount - inserted; remaining < count { + count = remaining + } + ids := make([][]byte, count) + docs := make([][]byte, count) + for i := 0; i < count; i++ { + docNum := inserted + i + ids[i] = []byte(benchmarkID(docNum)) + raw, err := bson.Marshal(benchmarkDocument(docNum)) + if err != nil { + b.Fatalf("marshal BSON document: %v", err) + } + docs[i] = raw + } + if _, err := collection.InsertBatchValidatedBSON(ids, docs); err != nil { + b.Fatalf("insert preload batch: %v", err) + } + inserted += count + } + if err := manager.FlushAll(); err != nil { + b.Fatalf("flush preload: %v", err) + } + preloadCompactStats := compactProfileBenchOverlayRootsAfterFlush(b, collection, "preload") + if err := backend.Checkpoint(); err != nil { + b.Fatalf("checkpoint preload: %v", err) + } + + warmupUpdateDocs := profileBenchParsedUpdateDocs(b, updateCity, "warmup") + updateDocs := profileBenchParsedUpdateDocs(b, updateCity, "timed") + ids := make([][]byte, documentCount) + for i := range ids { + ids[i] = []byte(benchmarkID(i)) + } + idStride := profileBenchUpdateIDStride(documentCount) + actualBatchSize := updateBatchSize + if actualBatchSize > documentCount { + actualBatchSize = documentCount + } + if actualBatchSize <= 0 { + actualBatchSize = 1 + } + + warmupOps := documentCount + if warmupOps > 100000 { + warmupOps = 100000 + } + if err := runProfileBenchDirectCollectionUpdateBatches(context.Background(), warmupOps, documentCount, idStride, actualBatchSize, ids, warmupUpdateDocs, collection, nil); err != nil { + b.Fatalf("warm up update batches: %v", err) + } + if err := manager.FlushAll(); err != nil { + b.Fatalf("flush warm up: %v", err) + } + warmupCompactStats := compactProfileBenchOverlayRootsAfterFlush(b, collection, "warmup") + if err := backend.Checkpoint(); err != nil { + b.Fatalf("checkpoint warm up: %v", err) + } + + b.ReportAllocs() + manager.SetUpdateBatchDetailedStatsEnabled(true) + manager.ResetUpdateCombineQueueDepthMax() + statsBefore := manager.StatsSnapshot() + backendStatsBefore := backend.Stats() + b.ResetTimer() + started := time.Now() + var directBatchStats profileBenchDirectUpdateBatchRunStats + err = runProfileBenchTimedUpdatePhase(context.Background(), func(ctx context.Context) error { + if err := runProfileBenchDirectCollectionUpdateBatches(ctx, b.N, documentCount, idStride, actualBatchSize, ids, updateDocs, collection, &directBatchStats); err != nil { + return err + } + phaseStart := time.Now() + err := manager.FlushAll() + directBatchStats.FlushAll += time.Since(phaseStart) + return err + }) + timedElapsed := time.Since(started) + b.StopTimer() + if err != nil { + b.Fatalf("run update batches: %v", err) + } + b.ReportMetric(float64(actualBatchSize), "target_update_items/batch") + reportProfileBenchBufferedIndexedWriteOptions(b, collection.Meta().Options) + reportProfileBenchOverlayCompactionStats(b, "preload", preloadCompactStats) + reportProfileBenchOverlayCompactionStats(b, "warmup", warmupCompactStats) + reportProfileBenchDirectUpdateBatchRunStats(b, directBatchStats, b.N) + reportCollectionManagerUpdateStats(b, deltaCollectionManagerUpdateStats(manager.StatsSnapshot(), statsBefore), b.N) + backendStatsAfter := backend.Stats() + reportProfileBenchOrderedRootPublishStats(b, backendStatsAfter, backendStatsBefore, b.N) + reportProfileBenchBackendVlogMmapStats(b, backendStatsAfter, backendStatsBefore, b.N) + reportDocsPerSecond(b, b.N, timedElapsed) +} + func runProfileBenchTimedUpdatePhase(ctx context.Context, run func(context.Context) error) error { if ctx == nil { ctx = context.Background() @@ -989,6 +1141,10 @@ func reportProfileBenchOrderedRootPublishStats(b *testing.B, after, before map[s rootApplyLeafLogRecordHintBytesRead := profileBenchDeltaUintStat(after, before, prefix+"root_apply_leaf_log_record_hint_bytes_read_total") rootApplyLeafMerges := profileBenchDeltaUintStat(after, before, prefix+"root_apply_leaf_merges_total") rootApplyInternalMerges := profileBenchDeltaUintStat(after, before, prefix+"root_apply_internal_merges_total") + rootApplyInternalParallelMerges := profileBenchDeltaUintStat(after, before, prefix+"root_apply_internal_parallel_merges_total") + rootApplyInternalParallelChildren := profileBenchDeltaUintStat(after, before, prefix+"root_apply_internal_parallel_children_total") + rootApplyInternalParallelWorkers := profileBenchDeltaUintStat(after, before, prefix+"root_apply_internal_parallel_workers_total") + rootApplyInternalParallelOps := profileBenchDeltaUintStat(after, before, prefix+"root_apply_internal_parallel_ops_total") rootApplyLeafPagesWritten := profileBenchDeltaUintStat(after, before, prefix+"root_apply_leaf_pages_written_total") rootApplyPagerLeafPagesWritten := profileBenchDeltaUintStat(after, before, prefix+"root_apply_pager_leaf_pages_written_total") rootApplyLeafLogPagesWritten := profileBenchDeltaUintStat(after, before, prefix+"root_apply_leaf_log_pages_written_total") @@ -1034,6 +1190,10 @@ func reportProfileBenchOrderedRootPublishStats(b *testing.B, after, before map[s b.ReportMetric(float64(rootApplyLeafLogRecordHintBytesRead)/float64(docs), "publish_delta_group_root_apply_leaf_log_record_hint_read_bytes/doc") b.ReportMetric(float64(rootApplyLeafMerges)/float64(docs), "publish_delta_group_root_apply_leaf_merges/doc") b.ReportMetric(float64(rootApplyInternalMerges)/float64(docs), "publish_delta_group_root_apply_internal_merges/doc") + b.ReportMetric(float64(rootApplyInternalParallelMerges)/float64(docs), "publish_delta_group_root_apply_internal_parallel_merges/doc") + b.ReportMetric(float64(rootApplyInternalParallelChildren)/float64(docs), "publish_delta_group_root_apply_internal_parallel_children/doc") + b.ReportMetric(float64(rootApplyInternalParallelWorkers)/float64(docs), "publish_delta_group_root_apply_internal_parallel_workers/doc") + b.ReportMetric(float64(rootApplyInternalParallelOps)/float64(docs), "publish_delta_group_root_apply_internal_parallel_ops/doc") b.ReportMetric(float64(rootApplyLeafPagesWritten)/float64(docs), "publish_delta_group_root_apply_leaf_pages_written/doc") b.ReportMetric(float64(rootApplyPagerLeafPagesWritten)/float64(docs), "publish_delta_group_root_apply_pager_leaf_pages_written/doc") b.ReportMetric(float64(rootApplyLeafLogPagesWritten)/float64(docs), "publish_delta_group_root_apply_leaf_log_pages_written/doc") @@ -1075,74 +1235,90 @@ func reportProfileBenchOrderedRootPublishStats(b *testing.B, after, before map[s func deltaCollectionManagerUpdateStats(after, before collections.CollectionManagerStats) collections.CollectionManagerStats { delta := collections.CollectionManagerStats{ - UpdateBatchCalls: after.UpdateBatchCalls - before.UpdateBatchCalls, - UpdateBatchItems: after.UpdateBatchItems - before.UpdateBatchItems, - UpdateBatchMatched: after.UpdateBatchMatched - before.UpdateBatchMatched, - UpdateBatchModified: after.UpdateBatchModified - before.UpdateBatchModified, - UpdateBatchRuns: after.UpdateBatchRuns - before.UpdateBatchRuns, - UpdateBatchBufferedBatches: after.UpdateBatchBufferedBatches - before.UpdateBatchBufferedBatches, - UpdateBatchCurrentRead: after.UpdateBatchCurrentRead - before.UpdateBatchCurrentRead, - UpdateBatchCallback: after.UpdateBatchCallback - before.UpdateBatchCallback, - UpdateBatchPrepareDocuments: after.UpdateBatchPrepareDocuments - before.UpdateBatchPrepareDocuments, - UpdateBatchIndexStateExtract: after.UpdateBatchIndexStateExtract - before.UpdateBatchIndexStateExtract, - UpdateBatchUniquePreflight: after.UpdateBatchUniquePreflight - before.UpdateBatchUniquePreflight, - UpdateBatchTemplateRunBuild: after.UpdateBatchTemplateRunBuild - before.UpdateBatchTemplateRunBuild, - UpdateBatchPrimaryRunBuild: after.UpdateBatchPrimaryRunBuild - before.UpdateBatchPrimaryRunBuild, - UpdateBatchIndexStateRunBuild: after.UpdateBatchIndexStateRunBuild - before.UpdateBatchIndexStateRunBuild, - UpdateBatchSecondaryRunBuild: after.UpdateBatchSecondaryRunBuild - before.UpdateBatchSecondaryRunBuild, - UpdateBatchBufferStage: after.UpdateBatchBufferStage - before.UpdateBatchBufferStage, - UpdateBatchBufferPrecheck: after.UpdateBatchBufferPrecheck - before.UpdateBatchBufferPrecheck, - UpdateBatchBufferLockWait: after.UpdateBatchBufferLockWait - before.UpdateBatchBufferLockWait, - UpdateBatchBufferLockHold: after.UpdateBatchBufferLockHold - before.UpdateBatchBufferLockHold, - UpdateBatchBufferValidation: after.UpdateBatchBufferValidation - before.UpdateBatchBufferValidation, - UpdateBatchBufferRootScan: after.UpdateBatchBufferRootScan - before.UpdateBatchBufferRootScan, - UpdateBatchBufferDomainPrepare: after.UpdateBatchBufferDomainPrepare - before.UpdateBatchBufferDomainPrepare, - UpdateBatchBufferPrimaryIdx: after.UpdateBatchBufferPrimaryIdx - before.UpdateBatchBufferPrimaryIdx, - UpdateBatchBufferUniqueIdx: after.UpdateBatchBufferUniqueIdx - before.UpdateBatchBufferUniqueIdx, - UpdateBatchBufferRootAppend: after.UpdateBatchBufferRootAppend - before.UpdateBatchBufferRootAppend, - UpdateBatchBufferFlush: after.UpdateBatchBufferFlush - before.UpdateBatchBufferFlush, - UpdateBatchPublish: after.UpdateBatchPublish - before.UpdateBatchPublish, - UpdateBatchSecondaryDeletes: after.UpdateBatchSecondaryDeletes - before.UpdateBatchSecondaryDeletes, - UpdateBatchSecondarySets: after.UpdateBatchSecondarySets - before.UpdateBatchSecondarySets, - UpdateBatchSecondaryKeyBytes: after.UpdateBatchSecondaryKeyBytes - before.UpdateBatchSecondaryKeyBytes, - UpdateBatchIndexValueChanges: after.UpdateBatchIndexValueChanges - before.UpdateBatchIndexValueChanges, - UpdateBatchIndexValueUnchanged: after.UpdateBatchIndexValueUnchanged - before.UpdateBatchIndexValueUnchanged, - UpdateBatchMaskFallbacks: after.UpdateBatchMaskFallbacks - before.UpdateBatchMaskFallbacks, - UpdateBatchUniqueChecks: after.UpdateBatchUniqueChecks - before.UpdateBatchUniqueChecks, - UpdateBatchUniqueCheckSkips: after.UpdateBatchUniqueCheckSkips - before.UpdateBatchUniqueCheckSkips, - UpdateCombineRequests: after.UpdateCombineRequests - before.UpdateCombineRequests, - UpdateCombineBatches: after.UpdateCombineBatches - before.UpdateCombineBatches, - UpdateCombineBatchedRequests: after.UpdateCombineBatchedRequests - before.UpdateCombineBatchedRequests, - UpdateCombineFallbackRequests: after.UpdateCombineFallbackRequests - before.UpdateCombineFallbackRequests, - IndexedFlushCalls: after.IndexedFlushCalls - before.IndexedFlushCalls, - IndexedFlushErrors: after.IndexedFlushErrors - before.IndexedFlushErrors, - IndexedFlushForcedDrains: after.IndexedFlushForcedDrains - before.IndexedFlushForcedDrains, - IndexedFlushUnits: after.IndexedFlushUnits - before.IndexedFlushUnits, - IndexedFlushDocs: after.IndexedFlushDocs - before.IndexedFlushDocs, - IndexedFlushBytes: after.IndexedFlushBytes - before.IndexedFlushBytes, - IndexedFlushRootRuns: after.IndexedFlushRootRuns - before.IndexedFlushRootRuns, - IndexedFlushRoots: after.IndexedFlushRoots - before.IndexedFlushRoots, - IndexedFlushDuration: after.IndexedFlushDuration - before.IndexedFlushDuration, - IndexedFlushMaterialize: after.IndexedFlushMaterialize - before.IndexedFlushMaterialize, - IndexedFlushPublish: after.IndexedFlushPublish - before.IndexedFlushPublish, - IndexedAsyncFlushWait: after.IndexedAsyncFlushWait - before.IndexedAsyncFlushWait, - RootDeltaPlanPrimaryRoots: after.RootDeltaPlanPrimaryRoots - before.RootDeltaPlanPrimaryRoots, - RootDeltaPlanTemplateRoots: after.RootDeltaPlanTemplateRoots - before.RootDeltaPlanTemplateRoots, - RootDeltaPlanIndexStateRoots: after.RootDeltaPlanIndexStateRoots - before.RootDeltaPlanIndexStateRoots, - RootDeltaPlanSecondaryRoots: after.RootDeltaPlanSecondaryRoots - before.RootDeltaPlanSecondaryRoots, - RootDeltaPlanEntries: after.RootDeltaPlanEntries - before.RootDeltaPlanEntries, - RootDeltaPlanKeyBytes: after.RootDeltaPlanKeyBytes - before.RootDeltaPlanKeyBytes, - RootDeltaPlanValueBytes: after.RootDeltaPlanValueBytes - before.RootDeltaPlanValueBytes, - RootDeltaPlanTombstones: after.RootDeltaPlanTombstones - before.RootDeltaPlanTombstones, - PrimaryOnlyUpdateCalls: after.PrimaryOnlyUpdateCalls - before.PrimaryOnlyUpdateCalls, - PrimaryOnlyMatched: after.PrimaryOnlyMatched - before.PrimaryOnlyMatched, - PrimaryOnlyModified: after.PrimaryOnlyModified - before.PrimaryOnlyModified, - PrimaryOnlyBufferedCalls: after.PrimaryOnlyBufferedCalls - before.PrimaryOnlyBufferedCalls, - PrimaryOnlyRootPublishes: after.PrimaryOnlyRootPublishes - before.PrimaryOnlyRootPublishes, - PrimaryOnlyRootDeltaEntries: after.PrimaryOnlyRootDeltaEntries - before.PrimaryOnlyRootDeltaEntries, - PrimaryOnlyRootDeltaKeyBytes: after.PrimaryOnlyRootDeltaKeyBytes - before.PrimaryOnlyRootDeltaKeyBytes, - PrimaryOnlyRootDeltaValueBytes: after.PrimaryOnlyRootDeltaValueBytes - before.PrimaryOnlyRootDeltaValueBytes, - PrimaryOnlyCoalescedDocs: after.PrimaryOnlyCoalescedDocs - before.PrimaryOnlyCoalescedDocs, + UpdateBatchCalls: after.UpdateBatchCalls - before.UpdateBatchCalls, + UpdateBatchItems: after.UpdateBatchItems - before.UpdateBatchItems, + UpdateBatchMatched: after.UpdateBatchMatched - before.UpdateBatchMatched, + UpdateBatchModified: after.UpdateBatchModified - before.UpdateBatchModified, + UpdateBatchRuns: after.UpdateBatchRuns - before.UpdateBatchRuns, + UpdateBatchBufferedBatches: after.UpdateBatchBufferedBatches - before.UpdateBatchBufferedBatches, + UpdateBatchValidate: after.UpdateBatchValidate - before.UpdateBatchValidate, + UpdateBatchClone: after.UpdateBatchClone - before.UpdateBatchClone, + UpdateBatchPlanSetup: after.UpdateBatchPlanSetup - before.UpdateBatchPlanSetup, + UpdateBatchBufferedReadSnapshot: after.UpdateBatchBufferedReadSnapshot - before.UpdateBatchBufferedReadSnapshot, + UpdateBatchCurrentRead: after.UpdateBatchCurrentRead - before.UpdateBatchCurrentRead, + UpdateBatchBSONIDValidation: after.UpdateBatchBSONIDValidation - before.UpdateBatchBSONIDValidation, + UpdateBatchCallback: after.UpdateBatchCallback - before.UpdateBatchCallback, + UpdateBatchReplacementStage: after.UpdateBatchReplacementStage - before.UpdateBatchReplacementStage, + UpdateBatchPrepareDocuments: after.UpdateBatchPrepareDocuments - before.UpdateBatchPrepareDocuments, + UpdateBatchIndexStateExtract: after.UpdateBatchIndexStateExtract - before.UpdateBatchIndexStateExtract, + UpdateBatchIndexStateCompare: after.UpdateBatchIndexStateCompare - before.UpdateBatchIndexStateCompare, + UpdateBatchUniquePreflight: after.UpdateBatchUniquePreflight - before.UpdateBatchUniquePreflight, + UpdateBatchTemplateRunBuild: after.UpdateBatchTemplateRunBuild - before.UpdateBatchTemplateRunBuild, + UpdateBatchPrimaryRunBuild: after.UpdateBatchPrimaryRunBuild - before.UpdateBatchPrimaryRunBuild, + UpdateBatchIndexStateRunBuild: after.UpdateBatchIndexStateRunBuild - before.UpdateBatchIndexStateRunBuild, + UpdateBatchSecondaryRunBuild: after.UpdateBatchSecondaryRunBuild - before.UpdateBatchSecondaryRunBuild, + UpdateBatchSemanticRecordBuild: after.UpdateBatchSemanticRecordBuild - before.UpdateBatchSemanticRecordBuild, + UpdateBatchBufferStage: after.UpdateBatchBufferStage - before.UpdateBatchBufferStage, + UpdateBatchBufferPrecheck: after.UpdateBatchBufferPrecheck - before.UpdateBatchBufferPrecheck, + UpdateBatchBufferLockWait: after.UpdateBatchBufferLockWait - before.UpdateBatchBufferLockWait, + UpdateBatchBufferLockHold: after.UpdateBatchBufferLockHold - before.UpdateBatchBufferLockHold, + UpdateBatchBufferValidation: after.UpdateBatchBufferValidation - before.UpdateBatchBufferValidation, + UpdateBatchBufferRootScan: after.UpdateBatchBufferRootScan - before.UpdateBatchBufferRootScan, + UpdateBatchBufferDomainPrepare: after.UpdateBatchBufferDomainPrepare - before.UpdateBatchBufferDomainPrepare, + UpdateBatchBufferPrimaryIdx: after.UpdateBatchBufferPrimaryIdx - before.UpdateBatchBufferPrimaryIdx, + UpdateBatchBufferUniqueIdx: after.UpdateBatchBufferUniqueIdx - before.UpdateBatchBufferUniqueIdx, + UpdateBatchBufferRootAppend: after.UpdateBatchBufferRootAppend - before.UpdateBatchBufferRootAppend, + UpdateBatchBufferSemanticAppend: after.UpdateBatchBufferSemanticAppend - before.UpdateBatchBufferSemanticAppend, + UpdateBatchBufferFlush: after.UpdateBatchBufferFlush - before.UpdateBatchBufferFlush, + UpdateBatchPlanClose: after.UpdateBatchPlanClose - before.UpdateBatchPlanClose, + UpdateBatchPublish: after.UpdateBatchPublish - before.UpdateBatchPublish, + UpdateBatchSecondaryDeletes: after.UpdateBatchSecondaryDeletes - before.UpdateBatchSecondaryDeletes, + UpdateBatchSecondarySets: after.UpdateBatchSecondarySets - before.UpdateBatchSecondarySets, + UpdateBatchSecondaryKeyBytes: after.UpdateBatchSecondaryKeyBytes - before.UpdateBatchSecondaryKeyBytes, + UpdateBatchIndexValueChanges: after.UpdateBatchIndexValueChanges - before.UpdateBatchIndexValueChanges, + UpdateBatchIndexValueUnchanged: after.UpdateBatchIndexValueUnchanged - before.UpdateBatchIndexValueUnchanged, + UpdateBatchMaskFallbacks: after.UpdateBatchMaskFallbacks - before.UpdateBatchMaskFallbacks, + UpdateBatchUniqueChecks: after.UpdateBatchUniqueChecks - before.UpdateBatchUniqueChecks, + UpdateBatchUniqueCheckSkips: after.UpdateBatchUniqueCheckSkips - before.UpdateBatchUniqueCheckSkips, + UpdateCombineRequests: after.UpdateCombineRequests - before.UpdateCombineRequests, + UpdateCombineBatches: after.UpdateCombineBatches - before.UpdateCombineBatches, + UpdateCombineBatchedRequests: after.UpdateCombineBatchedRequests - before.UpdateCombineBatchedRequests, + UpdateCombineFallbackRequests: after.UpdateCombineFallbackRequests - before.UpdateCombineFallbackRequests, + IndexedFlushCalls: after.IndexedFlushCalls - before.IndexedFlushCalls, + IndexedFlushErrors: after.IndexedFlushErrors - before.IndexedFlushErrors, + IndexedFlushForcedDrains: after.IndexedFlushForcedDrains - before.IndexedFlushForcedDrains, + IndexedFlushUnits: after.IndexedFlushUnits - before.IndexedFlushUnits, + IndexedFlushDocs: after.IndexedFlushDocs - before.IndexedFlushDocs, + IndexedFlushBytes: after.IndexedFlushBytes - before.IndexedFlushBytes, + IndexedFlushRootRuns: after.IndexedFlushRootRuns - before.IndexedFlushRootRuns, + IndexedFlushRoots: after.IndexedFlushRoots - before.IndexedFlushRoots, + IndexedFlushPreflight: after.IndexedFlushPreflight - before.IndexedFlushPreflight, + IndexedFlushRotate: after.IndexedFlushRotate - before.IndexedFlushRotate, + IndexedFlushMerge: after.IndexedFlushMerge - before.IndexedFlushMerge, + IndexedFlushDuration: after.IndexedFlushDuration - before.IndexedFlushDuration, + IndexedFlushMaterialize: after.IndexedFlushMaterialize - before.IndexedFlushMaterialize, + IndexedFlushSemanticPlan: after.IndexedFlushSemanticPlan - before.IndexedFlushSemanticPlan, + IndexedFlushBuildInputs: after.IndexedFlushBuildInputs - before.IndexedFlushBuildInputs, + IndexedFlushPlanStats: after.IndexedFlushPlanStats - before.IndexedFlushPlanStats, + IndexedFlushPublish: after.IndexedFlushPublish - before.IndexedFlushPublish, + IndexedAsyncFlushWait: after.IndexedAsyncFlushWait - before.IndexedAsyncFlushWait, + RootDeltaPlanPrimaryRoots: after.RootDeltaPlanPrimaryRoots - before.RootDeltaPlanPrimaryRoots, + RootDeltaPlanTemplateRoots: after.RootDeltaPlanTemplateRoots - before.RootDeltaPlanTemplateRoots, + RootDeltaPlanIndexStateRoots: after.RootDeltaPlanIndexStateRoots - before.RootDeltaPlanIndexStateRoots, + RootDeltaPlanSecondaryRoots: after.RootDeltaPlanSecondaryRoots - before.RootDeltaPlanSecondaryRoots, + RootDeltaPlanEntries: after.RootDeltaPlanEntries - before.RootDeltaPlanEntries, + RootDeltaPlanKeyBytes: after.RootDeltaPlanKeyBytes - before.RootDeltaPlanKeyBytes, + RootDeltaPlanValueBytes: after.RootDeltaPlanValueBytes - before.RootDeltaPlanValueBytes, + RootDeltaPlanTombstones: after.RootDeltaPlanTombstones - before.RootDeltaPlanTombstones, + PrimaryOnlyUpdateCalls: after.PrimaryOnlyUpdateCalls - before.PrimaryOnlyUpdateCalls, + PrimaryOnlyMatched: after.PrimaryOnlyMatched - before.PrimaryOnlyMatched, + PrimaryOnlyModified: after.PrimaryOnlyModified - before.PrimaryOnlyModified, + PrimaryOnlyBufferedCalls: after.PrimaryOnlyBufferedCalls - before.PrimaryOnlyBufferedCalls, + PrimaryOnlyRootPublishes: after.PrimaryOnlyRootPublishes - before.PrimaryOnlyRootPublishes, + PrimaryOnlyRootDeltaEntries: after.PrimaryOnlyRootDeltaEntries - before.PrimaryOnlyRootDeltaEntries, + PrimaryOnlyRootDeltaKeyBytes: after.PrimaryOnlyRootDeltaKeyBytes - before.PrimaryOnlyRootDeltaKeyBytes, + PrimaryOnlyRootDeltaValueBytes: after.PrimaryOnlyRootDeltaValueBytes - before.PrimaryOnlyRootDeltaValueBytes, + PrimaryOnlyCoalescedDocs: after.PrimaryOnlyCoalescedDocs - before.PrimaryOnlyCoalescedDocs, } delta.OverlayMutableDocuments = after.OverlayMutableDocuments delta.OverlayQueuedIndexedFlushUnits = after.OverlayQueuedIndexedFlushUnits @@ -1547,6 +1723,18 @@ func reportCollectionManagerUpdateStats(b *testing.B, stats collections.Collecti b.ReportMetric(float64(stats.IndexedFlushRoots)/float64(stats.IndexedFlushCalls), "indexed_flush_roots/call") b.ReportMetric(float64(stats.IndexedFlushRoots)/float64(docs), "indexed_flush_roots/doc") } + reportFlushDuration := func(value time.Duration, callName, docName string) { + if value <= 0 { + return + } + b.ReportMetric(float64(value.Nanoseconds())/float64(stats.IndexedFlushCalls), callName) + if stats.IndexedFlushDocs > 0 { + b.ReportMetric(float64(value.Nanoseconds())/float64(stats.IndexedFlushDocs), docName) + } + } + reportFlushDuration(stats.IndexedFlushPreflight, "indexed_flush_preflight_ns/call", "indexed_flush_preflight_ns/doc") + reportFlushDuration(stats.IndexedFlushRotate, "indexed_flush_rotate_ns/call", "indexed_flush_rotate_ns/doc") + reportFlushDuration(stats.IndexedFlushMerge, "indexed_flush_merge_ns/call", "indexed_flush_merge_ns/doc") if stats.IndexedFlushErrors > 0 { b.ReportMetric(float64(stats.IndexedFlushErrors), "indexed_flush_errors") } @@ -1566,6 +1754,9 @@ func reportCollectionManagerUpdateStats(b *testing.B, stats collections.Collecti b.ReportMetric(float64(stats.IndexedFlushMaterialize.Nanoseconds())/float64(stats.IndexedFlushDocs), "indexed_flush_materialize_ns/doc") } } + reportFlushDuration(stats.IndexedFlushSemanticPlan, "indexed_flush_semantic_plan_ns/call", "indexed_flush_semantic_plan_ns/doc") + reportFlushDuration(stats.IndexedFlushBuildInputs, "indexed_flush_build_inputs_ns/call", "indexed_flush_build_inputs_ns/doc") + reportFlushDuration(stats.IndexedFlushPlanStats, "indexed_flush_plan_stats_ns/call", "indexed_flush_plan_stats_ns/doc") if stats.IndexedFlushPublish > 0 { b.ReportMetric(float64(stats.IndexedFlushPublish.Nanoseconds())/float64(stats.IndexedFlushCalls), "indexed_flush_publish_ns/call") if stats.IndexedFlushDocs > 0 { @@ -1734,15 +1925,23 @@ func reportCollectionManagerUpdateStats(b *testing.B, stats collections.Collecti b.ReportMetric(float64(d.Nanoseconds())/float64(docs), name) } } + reportDuration("update_validate_items_ns/doc", stats.UpdateBatchValidate) + reportDuration("update_clone_items_ns/doc", stats.UpdateBatchClone) + reportDuration("update_plan_setup_ns/doc", stats.UpdateBatchPlanSetup) + reportDuration("update_buffered_read_snapshot_ns/doc", stats.UpdateBatchBufferedReadSnapshot) reportDuration("update_current_read_ns/doc", stats.UpdateBatchCurrentRead) + reportDuration("update_bson_id_validation_ns/doc", stats.UpdateBatchBSONIDValidation) reportDuration("update_callback_ns/doc", stats.UpdateBatchCallback) + reportDuration("update_replacement_stage_ns/doc", stats.UpdateBatchReplacementStage) reportDuration("update_prepare_ns/doc", stats.UpdateBatchPrepareDocuments) reportDuration("update_index_state_extract_ns/doc", stats.UpdateBatchIndexStateExtract) + reportDuration("update_index_state_compare_ns/doc", stats.UpdateBatchIndexStateCompare) reportDuration("update_unique_preflight_ns/doc", stats.UpdateBatchUniquePreflight) reportDuration("update_template_run_ns/doc", stats.UpdateBatchTemplateRunBuild) reportDuration("update_primary_run_ns/doc", stats.UpdateBatchPrimaryRunBuild) reportDuration("update_index_state_run_ns/doc", stats.UpdateBatchIndexStateRunBuild) reportDuration("update_secondary_runs_ns/doc", stats.UpdateBatchSecondaryRunBuild) + reportDuration("update_semantic_record_build_ns/doc", stats.UpdateBatchSemanticRecordBuild) reportDuration("update_buffer_stage_ns/doc", stats.UpdateBatchBufferStage) reportDuration("update_buffer_precheck_ns/doc", stats.UpdateBatchBufferPrecheck) reportDuration("update_buffer_lock_wait_ns/doc", stats.UpdateBatchBufferLockWait) @@ -1753,7 +1952,9 @@ func reportCollectionManagerUpdateStats(b *testing.B, stats collections.Collecti reportDuration("update_buffer_primary_index_ns/doc", stats.UpdateBatchBufferPrimaryIdx) reportDuration("update_buffer_unique_index_ns/doc", stats.UpdateBatchBufferUniqueIdx) reportDuration("update_buffer_root_append_ns/doc", stats.UpdateBatchBufferRootAppend) + reportDuration("update_buffer_semantic_append_ns/doc", stats.UpdateBatchBufferSemanticAppend) reportDuration("update_buffer_flush_ns/doc", stats.UpdateBatchBufferFlush) + reportDuration("update_plan_close_ns/doc", stats.UpdateBatchPlanClose) reportDuration("update_publish_ns/doc", stats.UpdateBatchPublish) } @@ -1835,6 +2036,112 @@ func runProfileBenchDirectCollectionConcurrentUpdates( return ctx.Err() } +func runProfileBenchDirectCollectionUpdateBatches( + ctx context.Context, + operations, documentCount, idStride, batchSize int, + ids [][]byte, + updateDocs []profileBenchSetUpdate, + collection *collections.Collection, + stats *profileBenchDirectUpdateBatchRunStats, +) error { + if operations <= 0 { + return nil + } + if batchSize <= 0 { + batchSize = operations + } + if batchSize > documentCount { + batchSize = documentCount + } + if batchSize <= 0 { + batchSize = 1 + } + items := make([]collections.UpdateBatchItem, batchSize) + updateScratch := make([][]byte, batchSize) + for start := 0; start < operations; { + if err := ctx.Err(); err != nil { + return err + } + count := batchSize + if remaining := operations - start; remaining < count { + count = remaining + } + phaseStart := time.Now() + for i := 0; i < count; i++ { + slot := i + op := start + i + documentOrdinal := (op * idStride) % documentCount + id := ids[documentOrdinal] + updateDoc := updateDocs[op%len(updateDocs)] + items[slot] = collections.UpdateBatchItem{ + DocumentID: id, + Update: func(stored []byte) ([]byte, bool, error) { + raw := bson.Raw(stored) + originalID := raw.Lookup("_id") + updated, nextScratch, shouldWrite, err := profileBenchApplyParsedSetUpdateToOperation(updateScratch[slot][:0], raw, updateDoc, op, documentOrdinal, documentCount) + updateScratch[slot] = nextScratch + if err != nil { + return nil, false, err + } + if !updated.Lookup("_id").Equal(originalID) { + return nil, false, errUpdatedPrimaryKey + } + return []byte(updated), shouldWrite, nil + }, + } + } + if stats != nil { + stats.ItemBuild += time.Since(phaseStart) + } + phaseStart = time.Now() + results, batched, err := collection.UpdateBatchIfNoSecondaryUniqueIndexChanges(items[:count]) + if stats != nil { + stats.UpdateBatchCall += time.Since(phaseStart) + } + if err != nil { + return err + } + if !batched { + return errors.New("profile benchmark update batch was declined") + } + phaseStart = time.Now() + for i := 0; i < count; i++ { + if !results[i].Matched { + return errProfileBenchUpdateMiss + } + } + if stats != nil { + stats.ResultCheck += time.Since(phaseStart) + } + start += count + } + return ctx.Err() +} + +type profileBenchDirectUpdateBatchRunStats struct { + ItemBuild time.Duration + UpdateBatchCall time.Duration + ResultCheck time.Duration + FlushAll time.Duration +} + +func reportProfileBenchDirectUpdateBatchRunStats(b *testing.B, stats profileBenchDirectUpdateBatchRunStats, docs int) { + b.Helper() + if docs <= 0 { + return + } + report := func(name string, value time.Duration) { + if value <= 0 { + return + } + b.ReportMetric(float64(value.Nanoseconds())/float64(docs), name) + } + report("bench_update_item_build_ns/doc", stats.ItemBuild) + report("bench_update_batch_call_ns/doc", stats.UpdateBatchCall) + report("bench_update_result_check_ns/doc", stats.ResultCheck) + report("bench_update_flush_all_ns/doc", stats.FlushAll) +} + type profileBenchSetField struct { key string keyBytes []byte diff --git a/cmd/mongo_gateway_compare_report/main.go b/cmd/mongo_gateway_compare_report/main.go index 01a5b63e6d..186401b0f8 100644 --- a/cmd/mongo_gateway_compare_report/main.go +++ b/cmd/mongo_gateway_compare_report/main.go @@ -70,10 +70,28 @@ type phaseResult struct { DriverAggregateMillis float64 `json:"driver_aggregate_duration_ms,omitempty"` DriverMeanLatencyMicros float64 `json:"driver_mean_latency_us,omitempty"` LatencyMicros latencySummary `json:"latency_micros"` + TreeDBDrainMillis float64 `json:"treedb_drain_ms,omitempty"` + TreeDBDrainMillisSet bool `json:"-"` + TreeDBDrainStatsDelta map[string]string `json:"treedb_drain_stats_delta,omitempty"` TreeDBStatsDelta map[string]string `json:"treedb_stats_delta,omitempty"` TreeDBMetrics map[string]float64 `json:"treedb_metrics,omitempty"` } +func (p *phaseResult) UnmarshalJSON(data []byte) error { + type phaseResultAlias phaseResult + var alias phaseResultAlias + if err := json.Unmarshal(data, &alias); err != nil { + return err + } + *p = phaseResult(alias) + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + _, p.TreeDBDrainMillisSet = raw["treedb_drain_ms"] + return nil +} + type latencySummary struct { P50 float64 `json:"p50"` P95 float64 `json:"p95"` @@ -1069,8 +1087,42 @@ func renderWriterSweepCounterTable(b *strings.Builder, cells []cellComparison) { } b.WriteString("## 0-Index Writer Sweep Counters\n\n") b.WriteString("These rows preserve TreeDB per-phase counter deltas for `concurrent_id_update_set_wN` phases. Values come from `phase.treedb_metrics` when present, so load counters and writer counters remain separate.\n\n") - b.WriteString("| docs | indexes | TreeDB config | MongoDB baseline config | writers | TreeDB ops/s | MongoDB ops/s | TreeDB p95 us | MongoDB p95 us | TreeDB driver calls | MongoDB driver calls | publish calls/doc | root apply calls/doc | roots/publish | root apply ns/doc | leaf-log loads/doc | leaf-log pages written/doc | leaf-log read bytes/doc | leaf-log write bytes/doc | indexed flush calls/doc | indexed flush units/batch | indexed flush docs/batch | indexed flush root-runs/doc | root-delta entries/doc | root-delta key bytes/doc | root-delta value bytes/doc | root-delta tombstones/doc | affected primary roots/doc | affected secondary roots/doc | primary root publishes/doc | primary root delta entries/doc | primary root delta bytes/doc | primary-only coalesced docs/publish | raw JSON |\n") - b.WriteString("| ---: | ---: | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |\n") + headers := []string{ + "docs", "indexes", "TreeDB config", "MongoDB baseline config", "writers", + "TreeDB ops/s", "MongoDB ops/s", "TreeDB p95 us", "MongoDB p95 us", "TreeDB driver calls", "MongoDB driver calls", "TreeDB drain ms", + "publish calls/doc", "root apply calls/doc", "roots/publish", "root apply ns/doc", "root apply excl. read-only prepare ns/doc", + "internal parallel merges/doc", "internal parallel children/merge", "internal parallel workers/merge", "internal parallel ops/merge", + "read-only prepare share %", + "read-only prepare calls/doc", "read-only prepare ns/doc", "read-only prepare ns/plan", "read-only prepare ops/doc", "read-only prepare leaf spans/plan", + "read-only worker targets/plan", "read-only worker ranges/plan", "read-only worker max ops/plan", + "leaf-log loads/doc", "leaf-log pages written/doc", "leaf-log read bytes/doc", "leaf-log write bytes/doc", + "indexed flush calls/doc", "indexed flush units/batch", "indexed flush docs/batch", "indexed flush root-runs/doc", + "coalesced batch units/batch", "coalesced batch docs/batch", "coalesced batch bytes/batch", + "root-delta entries/doc", "root-delta key bytes/doc", "root-delta value bytes/doc", "root-delta tombstones/doc", + "affected primary roots/doc", "affected secondary roots/doc", + "raw root-delta entries/doc", "raw root-delta bytes/doc", "raw root-delta tombstones/doc", + "raw primary entries/doc", "raw primary bytes/doc", "raw primary tombstones/doc", + "raw template entries/doc", "raw template bytes/doc", "raw template tombstones/doc", + "raw index-state entries/doc", "raw index-state bytes/doc", "raw index-state tombstones/doc", + "raw secondary entries/doc", "raw secondary bytes/doc", "raw secondary tombstones/doc", + "final root-delta entries/doc", "final root-delta bytes/doc", "final root-delta tombstones/doc", + "final primary entries/doc", "final primary bytes/doc", "final primary tombstones/doc", + "final template entries/doc", "final template bytes/doc", "final template tombstones/doc", + "final index-state entries/doc", "final index-state bytes/doc", "final index-state tombstones/doc", + "final secondary entries/doc", "final secondary bytes/doc", "final secondary tombstones/doc", + "squashed entries/doc", "net-zero root batches/doc", "net-zero root plans/doc", "skipped secondary roots/doc", + "primary root publishes/doc", "primary root delta entries/doc", "primary root delta bytes/doc", "primary-only coalesced docs/publish", "primary-only duplicate IDs coalesced/doc", "primary-only drains/doc", "primary-only drain docs/drain", + "raw JSON", + } + b.WriteString("| " + strings.Join(headers, " | ") + " |\n") + align := make([]string, len(headers)) + for i := range align { + align[i] = "---:" + } + for _, idx := range []int{2, 3, len(headers) - 1} { + align[idx] = "---" + } + b.WriteString("| " + strings.Join(align, " | ") + " |\n") for _, cmp := range rows { writers, _ := concurrentUpdateWriters(cmp.Name) cell := findCell(cells, cmp.Cell) @@ -1086,10 +1138,25 @@ func renderWriterSweepCounterTable(b *strings.Builder, cells []cellComparison) { formatPhaseLatency(cmp.HasMongo, cmp.MongoPhase.LatencyMicros.P95), formatPhaseDriverCalls(cmp.HasTreeDB, cmp.TreeDBPhase.DriverCalls), formatPhaseDriverCalls(cmp.HasMongo, cmp.MongoPhase.DriverCalls), + formatPhaseDrainMillis(cmp.HasTreeDB, cmp.TreeDBPhase), formatPhaseMetric(cmp.TreeDBPhase, "publish_delta_group_calls/doc"), formatPhaseMetric(cmp.TreeDBPhase, "root_apply_calls/doc"), formatPhaseMetric(cmp.TreeDBPhase, "roots/publish"), formatPhaseMetric(cmp.TreeDBPhase, "publish_delta_group_root_apply_ns/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "publish_delta_group_root_apply_excluding_read_only_prepare_ns/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "internal_parallel_merges/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "internal_parallel_children/merge"), + formatPhaseMetric(cmp.TreeDBPhase, "internal_parallel_workers/merge"), + formatPhaseMetric(cmp.TreeDBPhase, "internal_parallel_ops/merge"), + formatPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_root_apply_share_pct"), + formatPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_calls/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_ns/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_ns/plan"), + formatPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_ops/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_leaf_spans/plan"), + formatPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_worker_targets/plan"), + formatPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_worker_ranges/plan"), + formatPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_worker_max_ops/plan"), formatPhaseMetric(cmp.TreeDBPhase, "leaf_log_node_loads/doc"), formatPhaseMetric(cmp.TreeDBPhase, "leaf_log_pages_written/doc"), formatPhaseMetric(cmp.TreeDBPhase, "leaf_log_read_bytes/doc"), @@ -1098,16 +1165,56 @@ func renderWriterSweepCounterTable(b *strings.Builder, cells []cellComparison) { formatPhaseMetric(cmp.TreeDBPhase, "indexed_flush_units/batch"), formatPhaseMetric(cmp.TreeDBPhase, "indexed_flush_docs/batch"), formatPhaseMetric(cmp.TreeDBPhase, "indexed_flush_root_runs/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "coalesced_batch_units/batch"), + formatPhaseMetric(cmp.TreeDBPhase, "coalesced_batch_docs/batch"), + formatPhaseMetric(cmp.TreeDBPhase, "coalesced_batch_bytes/batch"), formatPhaseMetric(cmp.TreeDBPhase, "root_delta_plan_entries/doc"), formatPhaseMetric(cmp.TreeDBPhase, "root_delta_plan_key_bytes/doc"), formatPhaseMetric(cmp.TreeDBPhase, "root_delta_plan_value_bytes/doc"), formatPhaseMetric(cmp.TreeDBPhase, "root_delta_plan_tombstones/doc"), formatPhaseMetric(cmp.TreeDBPhase, "affected_primary_roots/doc"), formatPhaseMetric(cmp.TreeDBPhase, "affected_secondary_roots/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_root_delta_entries/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_root_delta_bytes/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_root_delta_tombstones/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_primary_root_delta_entries/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_primary_root_delta_bytes/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_primary_root_delta_tombstones/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_template_root_delta_entries/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_template_root_delta_bytes/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_template_root_delta_tombstones/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_index_state_root_delta_entries/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_index_state_root_delta_bytes/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_index_state_root_delta_tombstones/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_secondary_root_delta_entries/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_secondary_root_delta_bytes/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "raw_secondary_root_delta_tombstones/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_root_delta_entries/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_root_delta_bytes/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_root_delta_tombstones/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_primary_root_delta_entries/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_primary_root_delta_bytes/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_primary_root_delta_tombstones/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_template_root_delta_entries/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_template_root_delta_bytes/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_template_root_delta_tombstones/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_index_state_root_delta_entries/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_index_state_root_delta_bytes/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_index_state_root_delta_tombstones/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_secondary_root_delta_entries/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_secondary_root_delta_bytes/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "final_secondary_root_delta_tombstones/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "squashed_root_delta_entries/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "net_zero_root_batches/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "net_zero_root_plans/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "skipped_secondary_roots/doc"), formatPhaseMetric(cmp.TreeDBPhase, "primary_root_publishes/doc"), formatPhaseMetric(cmp.TreeDBPhase, "primary_root_delta_entries/doc"), formatPhaseMetric(cmp.TreeDBPhase, "primary_root_delta_bytes/doc"), formatPhaseMetric(cmp.TreeDBPhase, "primary_only_coalesced_docs/publish"), + formatPhaseMetric(cmp.TreeDBPhase, "primary_only_duplicate_ids_coalesced/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "primary_only_drains/doc"), + formatPhaseMetric(cmp.TreeDBPhase, "primary_only_drain_docs/drain"), "`" + cell.TreeDB.DisplayRawPath + "`", } b.WriteString("| " + strings.Join(row, " | ") + " |\n") @@ -1357,6 +1464,62 @@ func writeSummaryTSV(path string, cells []cellComparison) error { "mongo_physical_bytes", "treedb_to_mongo_dbstats_total_ratio", "treedb_to_mongo_physical_ratio", + "treedb_drain_ms", + "treedb_publish_delta_group_root_apply_ns_per_doc", + "treedb_publish_delta_group_root_apply_excluding_read_only_prepare_ns_per_doc", + "treedb_internal_parallel_merges_per_doc", + "treedb_internal_parallel_children_per_merge", + "treedb_internal_parallel_workers_per_merge", + "treedb_internal_parallel_ops_per_merge", + "treedb_read_only_prepare_root_apply_share_pct", + "treedb_read_only_prepare_calls_per_doc", + "treedb_read_only_prepare_ns_per_doc", + "treedb_read_only_prepare_ns_per_plan", + "treedb_read_only_prepare_ops_per_doc", + "treedb_read_only_prepare_leaf_spans_per_plan", + "treedb_read_only_prepare_worker_targets_per_plan", + "treedb_read_only_prepare_worker_ranges_per_plan", + "treedb_read_only_prepare_worker_max_ops_per_plan", + "treedb_coalesced_batch_units_per_batch", + "treedb_coalesced_batch_docs_per_batch", + "treedb_coalesced_batch_bytes_per_batch", + "treedb_raw_root_delta_entries_per_doc", + "treedb_raw_root_delta_bytes_per_doc", + "treedb_raw_root_delta_tombstones_per_doc", + "treedb_raw_primary_root_delta_entries_per_doc", + "treedb_raw_primary_root_delta_bytes_per_doc", + "treedb_raw_primary_root_delta_tombstones_per_doc", + "treedb_raw_template_root_delta_entries_per_doc", + "treedb_raw_template_root_delta_bytes_per_doc", + "treedb_raw_template_root_delta_tombstones_per_doc", + "treedb_raw_index_state_root_delta_entries_per_doc", + "treedb_raw_index_state_root_delta_bytes_per_doc", + "treedb_raw_index_state_root_delta_tombstones_per_doc", + "treedb_raw_secondary_root_delta_entries_per_doc", + "treedb_raw_secondary_root_delta_bytes_per_doc", + "treedb_raw_secondary_root_delta_tombstones_per_doc", + "treedb_final_root_delta_entries_per_doc", + "treedb_final_root_delta_bytes_per_doc", + "treedb_final_root_delta_tombstones_per_doc", + "treedb_final_primary_root_delta_entries_per_doc", + "treedb_final_primary_root_delta_bytes_per_doc", + "treedb_final_primary_root_delta_tombstones_per_doc", + "treedb_final_template_root_delta_entries_per_doc", + "treedb_final_template_root_delta_bytes_per_doc", + "treedb_final_template_root_delta_tombstones_per_doc", + "treedb_final_index_state_root_delta_entries_per_doc", + "treedb_final_index_state_root_delta_bytes_per_doc", + "treedb_final_index_state_root_delta_tombstones_per_doc", + "treedb_final_secondary_root_delta_entries_per_doc", + "treedb_final_secondary_root_delta_bytes_per_doc", + "treedb_final_secondary_root_delta_tombstones_per_doc", + "treedb_squashed_root_delta_entries_per_doc", + "treedb_net_zero_root_batches_per_doc", + "treedb_net_zero_root_plans_per_doc", + "treedb_skipped_secondary_roots_per_doc", + "treedb_primary_only_duplicate_ids_coalesced_per_doc", + "treedb_primary_only_drains_per_doc", + "treedb_primary_only_drain_docs_per_drain", } if err := writer.Write(header); err != nil { return err @@ -1404,6 +1567,62 @@ func writeSummaryTSV(path string, cells []cellComparison) error { formatRawInt(hasMongo, mongoPhysical), formatRawMeasuredRatio(treeOK && mongoTotalOK, treeBytes, mongoTotal), formatRawRatio(safeRatio(float64(treePhysical), float64(mongoPhysical))), + formatRawDrainMillis(cmp.HasTreeDB, cmp.TreeDBPhase), + formatRawPhaseMetric(cmp.TreeDBPhase, "publish_delta_group_root_apply_ns/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "publish_delta_group_root_apply_excluding_read_only_prepare_ns/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "internal_parallel_merges/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "internal_parallel_children/merge"), + formatRawPhaseMetric(cmp.TreeDBPhase, "internal_parallel_workers/merge"), + formatRawPhaseMetric(cmp.TreeDBPhase, "internal_parallel_ops/merge"), + formatRawPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_root_apply_share_pct"), + formatRawPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_calls/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_ns/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_ns/plan"), + formatRawPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_ops/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_leaf_spans/plan"), + formatRawPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_worker_targets/plan"), + formatRawPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_worker_ranges/plan"), + formatRawPhaseMetric(cmp.TreeDBPhase, "read_only_prepare_worker_max_ops/plan"), + formatRawPhaseMetric(cmp.TreeDBPhase, "coalesced_batch_units/batch"), + formatRawPhaseMetric(cmp.TreeDBPhase, "coalesced_batch_docs/batch"), + formatRawPhaseMetric(cmp.TreeDBPhase, "coalesced_batch_bytes/batch"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_root_delta_entries/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_root_delta_bytes/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_root_delta_tombstones/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_primary_root_delta_entries/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_primary_root_delta_bytes/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_primary_root_delta_tombstones/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_template_root_delta_entries/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_template_root_delta_bytes/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_template_root_delta_tombstones/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_index_state_root_delta_entries/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_index_state_root_delta_bytes/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_index_state_root_delta_tombstones/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_secondary_root_delta_entries/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_secondary_root_delta_bytes/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "raw_secondary_root_delta_tombstones/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_root_delta_entries/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_root_delta_bytes/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_root_delta_tombstones/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_primary_root_delta_entries/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_primary_root_delta_bytes/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_primary_root_delta_tombstones/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_template_root_delta_entries/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_template_root_delta_bytes/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_template_root_delta_tombstones/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_index_state_root_delta_entries/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_index_state_root_delta_bytes/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_index_state_root_delta_tombstones/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_secondary_root_delta_entries/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_secondary_root_delta_bytes/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "final_secondary_root_delta_tombstones/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "squashed_root_delta_entries/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "net_zero_root_batches/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "net_zero_root_plans/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "skipped_secondary_roots/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "primary_only_duplicate_ids_coalesced/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "primary_only_drains/doc"), + formatRawPhaseMetric(cmp.TreeDBPhase, "primary_only_drain_docs/drain"), } if err := writer.Write(row); err != nil { return err @@ -1603,6 +1822,13 @@ func formatPhaseDriverCalls(ok bool, value int) string { return fmt.Sprintf("%d", value) } +func formatPhaseDrainMillis(ok bool, phase phaseResult) string { + if !ok || !phaseHasDrainMillis(phase) { + return "n/a" + } + return formatNumber(phase.TreeDBDrainMillis) +} + func formatPhaseMetric(phase phaseResult, name string) string { value, ok := phase.TreeDBMetrics[name] if !ok || math.IsNaN(value) || math.IsInf(value, 0) { @@ -1618,6 +1844,25 @@ func formatRawFloat(ok bool, value float64) string { return strconv.FormatFloat(value, 'f', 6, 64) } +func formatRawDrainMillis(ok bool, phase phaseResult) string { + if !ok || !phaseHasDrainMillis(phase) { + return "" + } + return strconv.FormatFloat(phase.TreeDBDrainMillis, 'f', 6, 64) +} + +func phaseHasDrainMillis(phase phaseResult) bool { + return phase.TreeDBDrainMillisSet || phase.TreeDBDrainMillis != 0 +} + +func formatRawPhaseMetric(phase phaseResult, name string) string { + value, ok := phase.TreeDBMetrics[name] + if !ok || math.IsNaN(value) || math.IsInf(value, 0) { + return "" + } + return strconv.FormatFloat(value, 'f', 6, 64) +} + func formatRawInt(ok bool, value int64) string { if !ok { return "" diff --git a/cmd/mongo_gateway_compare_report/main_test.go b/cmd/mongo_gateway_compare_report/main_test.go index e06d01d188..5e98d1964d 100644 --- a/cmd/mongo_gateway_compare_report/main_test.go +++ b/cmd/mongo_gateway_compare_report/main_test.go @@ -1,6 +1,8 @@ package main import ( + "encoding/csv" + "encoding/json" "os" "path/filepath" "strings" @@ -1229,12 +1231,152 @@ func TestMissingTreeDBDiskSnapshotRendersNA(t *testing.T) { } } +func TestWriteSummaryTSVRendersTreeDBCoalescingColumns(t *testing.T) { + phase := phaseResult{ + Name: "concurrent_id_update_set_w4", + Operations: 400, + OpsPerSecond: 1000, + TreeDBDrainMillis: 3.75, + TreeDBMetrics: map[string]float64{ + "coalesced_batch_units/batch": 2, + "coalesced_batch_docs/batch": 64, + "coalesced_batch_bytes/batch": 2048, + "publish_delta_group_root_apply_ns/doc": 200, + "publish_delta_group_root_apply_excluding_read_only_prepare_ns/doc": 187.5, + "internal_parallel_merges/doc": 0.08, + "internal_parallel_children/merge": 7, + "internal_parallel_workers/merge": 4, + "internal_parallel_ops/merge": 625, + "read_only_prepare_root_apply_share_pct": 6.25, + "read_only_prepare_calls/doc": 0.25, + "read_only_prepare_ns/doc": 12.5, + "read_only_prepare_ns/plan": 50, + "read_only_prepare_ops/doc": 1.75, + "read_only_prepare_leaf_spans/plan": 6, + "read_only_prepare_worker_targets/plan": 4, + "read_only_prepare_worker_ranges/plan": 3, + "read_only_prepare_worker_max_ops/plan": 128, + "raw_root_delta_entries/doc": 4, + "raw_root_delta_bytes/doc": 400, + "raw_root_delta_tombstones/doc": 0.1, + "raw_primary_root_delta_entries/doc": 1.5, + "raw_primary_root_delta_bytes/doc": 150, + "raw_secondary_root_delta_entries/doc": 2.5, + "raw_secondary_root_delta_bytes/doc": 250, + "final_root_delta_entries/doc": 2.5, + "final_root_delta_bytes/doc": 250, + "final_root_delta_tombstones/doc": 0, + "final_primary_root_delta_entries/doc": 1, + "final_primary_root_delta_bytes/doc": 100, + "final_secondary_root_delta_entries/doc": 1.5, + "final_secondary_root_delta_bytes/doc": 150, + "squashed_root_delta_entries/doc": 1.5, + "net_zero_root_batches/doc": 0.01, + "net_zero_root_plans/doc": 0.02, + "skipped_secondary_roots/doc": 0.5, + "primary_only_duplicate_ids_coalesced/doc": 0.125, + "primary_only_drains/doc": 0.05, + "primary_only_drain_docs/drain": 20, + }, + } + cells := []cellComparison{{ + Key: cellKey{Documents: 100, SecondaryIndexes: 0, TreeDBConfig: "treedb_writers_4"}, + TreeDB: &runRecord{ + Row: matrixRow{Target: "treedb", Config: "treedb_writers_4", Documents: 100, PhysicalBytes: 1000}, + Result: benchmarkResult{Target: "treedb", Documents: 100, Phases: []phaseResult{phase}, TreeDBDiskAfterCheckpoint: &diskSnapshot{TotalBytes: 500}}, + PhaseMap: map[string]phaseResult{phase.Name: phase}, + }, + Mongo: &runRecord{ + Row: matrixRow{Target: "mongo", Config: "mongo_writers_4", Documents: 100, PhysicalBytes: 2000}, + Result: benchmarkResult{Target: "mongo", Documents: 100, Phases: []phaseResult{{Name: phase.Name, OpsPerSecond: 500}}, MongoDBStatsFinal: map[string]any{"totalSize": float64(1000)}}, + PhaseMap: map[string]phaseResult{phase.Name: {Name: phase.Name, OpsPerSecond: 500}}, + }, + }} + summaryPath := filepath.Join(t.TempDir(), "summary.tsv") + if err := writeSummaryTSV(summaryPath, cells); err != nil { + t.Fatalf("write summary: %v", err) + } + r := csv.NewReader(strings.NewReader(readFile(t, summaryPath))) + r.Comma = '\t' + rows, err := r.ReadAll() + if err != nil { + t.Fatalf("read summary TSV: %v", err) + } + if len(rows) != 2 { + t.Fatalf("summary rows=%d want header+1: %#v", len(rows), rows) + } + values := make(map[string]string) + for i, column := range rows[0] { + values[column] = rows[1][i] + } + for column, want := range map[string]string{ + "treedb_drain_ms": "3.750000", + "treedb_publish_delta_group_root_apply_ns_per_doc": "200.000000", + "treedb_publish_delta_group_root_apply_excluding_read_only_prepare_ns_per_doc": "187.500000", + "treedb_internal_parallel_merges_per_doc": "0.080000", + "treedb_internal_parallel_children_per_merge": "7.000000", + "treedb_internal_parallel_workers_per_merge": "4.000000", + "treedb_internal_parallel_ops_per_merge": "625.000000", + "treedb_read_only_prepare_root_apply_share_pct": "6.250000", + "treedb_read_only_prepare_calls_per_doc": "0.250000", + "treedb_read_only_prepare_ns_per_doc": "12.500000", + "treedb_read_only_prepare_ns_per_plan": "50.000000", + "treedb_read_only_prepare_ops_per_doc": "1.750000", + "treedb_read_only_prepare_leaf_spans_per_plan": "6.000000", + "treedb_read_only_prepare_worker_targets_per_plan": "4.000000", + "treedb_read_only_prepare_worker_ranges_per_plan": "3.000000", + "treedb_read_only_prepare_worker_max_ops_per_plan": "128.000000", + "treedb_coalesced_batch_units_per_batch": "2.000000", + "treedb_raw_root_delta_entries_per_doc": "4.000000", + "treedb_raw_primary_root_delta_entries_per_doc": "1.500000", + "treedb_final_root_delta_entries_per_doc": "2.500000", + "treedb_squashed_root_delta_entries_per_doc": "1.500000", + "treedb_net_zero_root_batches_per_doc": "0.010000", + "treedb_primary_only_duplicate_ids_coalesced_per_doc": "0.125000", + "treedb_primary_only_drains_per_doc": "0.050000", + } { + if got := values[column]; got != want { + t.Fatalf("summary column %s=%q want %q; values=%v", column, got, want, values) + } + } +} + +func TestPhaseResultMissingDrainMillisRendersMissing(t *testing.T) { + var phase phaseResult + if err := json.Unmarshal([]byte(`{"name":"load","operations":1,"ops_per_sec":10}`), &phase); err != nil { + t.Fatalf("unmarshal phase: %v", err) + } + if phase.TreeDBDrainMillisSet { + t.Fatal("missing treedb_drain_ms marked present") + } + if got := formatPhaseDrainMillis(true, phase); got != "n/a" { + t.Fatalf("missing drain millis rendered as %q want n/a", got) + } + if got := formatRawDrainMillis(true, phase); got != "" { + t.Fatalf("missing raw drain millis rendered as %q want empty", got) + } + + if err := json.Unmarshal([]byte(`{"name":"load","operations":1,"ops_per_sec":10,"treedb_drain_ms":0}`), &phase); err != nil { + t.Fatalf("unmarshal present zero phase: %v", err) + } + if !phase.TreeDBDrainMillisSet { + t.Fatal("present zero treedb_drain_ms was not marked present") + } + if got := formatPhaseDrainMillis(true, phase); got != "0" { + t.Fatalf("present zero drain millis rendered as %q want 0", got) + } + if got := formatRawDrainMillis(true, phase); got != "0.000000" { + t.Fatalf("present zero raw drain millis rendered as %q want 0.000000", got) + } +} + func TestRenderWriterSweepCounterTableUsesPhaseMetrics(t *testing.T) { treePhase := phaseResult{ - Name: "concurrent_id_update_set_w8", - Operations: 800, - DriverCalls: 800, - OpsPerSecond: 1200, + Name: "concurrent_id_update_set_w8", + Operations: 800, + DriverCalls: 800, + OpsPerSecond: 1200, + TreeDBDrainMillis: 2.5, LatencyMicros: latencySummary{ P95: 750, }, @@ -1246,24 +1388,78 @@ func TestRenderWriterSweepCounterTableUsesPhaseMetrics(t *testing.T) { "root_apply_calls/doc": 0.5, "roots/publish": 1, "publish_delta_group_root_apply_ns/doc": 2500, - "leaf_log_node_loads/doc": 2, - "leaf_log_pages_written/doc": 0.25, - "leaf_log_read_bytes/doc": 64, - "leaf_log_write_bytes/doc": 128, - "indexed_flush_calls/doc": 0.125, - "indexed_flush_units/batch": 4, - "indexed_flush_docs/batch": 32, - "indexed_flush_root_runs/doc": 0.75, - "root_delta_plan_entries/doc": 1, - "root_delta_plan_key_bytes/doc": 10, - "root_delta_plan_value_bytes/doc": 20, - "root_delta_plan_tombstones/doc": 0.1, - "affected_primary_roots/doc": 0.5, - "affected_secondary_roots/doc": 0, - "primary_root_publishes/doc": 0.5, - "primary_root_delta_entries/doc": 1, - "primary_root_delta_bytes/doc": 42, - "primary_only_coalesced_docs/publish": 0, + "publish_delta_group_root_apply_excluding_read_only_prepare_ns/doc": 2470, + "internal_parallel_merges/doc": 0.08, + "internal_parallel_children/merge": 7, + "internal_parallel_workers/merge": 4, + "internal_parallel_ops/merge": 625, + "read_only_prepare_root_apply_share_pct": 1.2, + "read_only_prepare_calls/doc": 0.03, + "read_only_prepare_ns/doc": 30, + "read_only_prepare_ns/plan": 1000, + "read_only_prepare_ops/doc": 4, + "read_only_prepare_leaf_spans/plan": 8, + "read_only_prepare_worker_targets/plan": 4, + "read_only_prepare_worker_ranges/plan": 3, + "read_only_prepare_worker_max_ops/plan": 512, + "leaf_log_node_loads/doc": 2, + "leaf_log_pages_written/doc": 0.25, + "leaf_log_read_bytes/doc": 64, + "leaf_log_write_bytes/doc": 128, + "indexed_flush_calls/doc": 0.125, + "indexed_flush_units/batch": 4, + "indexed_flush_docs/batch": 32, + "indexed_flush_root_runs/doc": 0.75, + "coalesced_batch_units/batch": 3, + "coalesced_batch_docs/batch": 96, + "coalesced_batch_bytes/batch": 8192, + "root_delta_plan_entries/doc": 1, + "root_delta_plan_key_bytes/doc": 10, + "root_delta_plan_value_bytes/doc": 20, + "root_delta_plan_tombstones/doc": 0.1, + "affected_primary_roots/doc": 0.5, + "affected_secondary_roots/doc": 0, + "raw_root_delta_entries/doc": 2, + "raw_root_delta_bytes/doc": 200, + "raw_root_delta_tombstones/doc": 0.05, + "raw_primary_root_delta_entries/doc": 1.5, + "raw_primary_root_delta_bytes/doc": 150, + "raw_primary_root_delta_tombstones/doc": 0.05, + "raw_template_root_delta_entries/doc": 0.1, + "raw_template_root_delta_bytes/doc": 10, + "raw_template_root_delta_tombstones/doc": 0, + "raw_index_state_root_delta_entries/doc": 0.4, + "raw_index_state_root_delta_bytes/doc": 40, + "raw_index_state_root_delta_tombstones/doc": 0, + "raw_secondary_root_delta_entries/doc": 0.5, + "raw_secondary_root_delta_bytes/doc": 50, + "raw_secondary_root_delta_tombstones/doc": 0, + "final_root_delta_entries/doc": 1.25, + "final_root_delta_bytes/doc": 125, + "final_root_delta_tombstones/doc": 0, + "final_primary_root_delta_entries/doc": 1, + "final_primary_root_delta_bytes/doc": 100, + "final_primary_root_delta_tombstones/doc": 0, + "final_template_root_delta_entries/doc": 0.05, + "final_template_root_delta_bytes/doc": 5, + "final_template_root_delta_tombstones/doc": 0, + "final_index_state_root_delta_entries/doc": 0.2, + "final_index_state_root_delta_bytes/doc": 20, + "final_index_state_root_delta_tombstones/doc": 0, + "final_secondary_root_delta_entries/doc": 0.25, + "final_secondary_root_delta_bytes/doc": 25, + "final_secondary_root_delta_tombstones/doc": 0, + "squashed_root_delta_entries/doc": 0.75, + "net_zero_root_batches/doc": 0.01, + "net_zero_root_plans/doc": 0.02, + "skipped_secondary_roots/doc": 0.44, + "primary_root_publishes/doc": 0.5, + "primary_root_delta_entries/doc": 1, + "primary_root_delta_bytes/doc": 42, + "primary_only_coalesced_docs/publish": 0, + "primary_only_duplicate_ids_coalesced/doc": 0.25, + "primary_only_drains/doc": 0.125, + "primary_only_drain_docs/drain": 8, }, } mongoPhase := phaseResult{ @@ -1294,7 +1490,19 @@ func TestRenderWriterSweepCounterTableUsesPhaseMetrics(t *testing.T) { for _, want := range []string{ "## 0-Index Writer Sweep Counters", "publish calls/doc", - "| 1000 | 0 | `treedb_0idx` | `mongo_baseline` | 8 | 1200 | 2400 | 750 | 500 | 800 | 800 | 0.50 | 0.50 | 1.00 | 2500 | 2.00 | 0.25 | 64.0 | 128 | 0.12 | 4.00 | 32.0 | 0.75 | 1.00 | 10.0 | 20.0 | 0.10 | 0.50 | 0 | 0.50 | 1.00 | 42.0 | 0 | `/tmp/treedb.json` |", + "read-only prepare calls/doc", + "internal parallel merges/doc", + "TreeDB drain ms", + "root apply excl. read-only prepare ns/doc", + "raw root-delta entries/doc", + "final root-delta entries/doc", + "| 1000 | 0 | `treedb_0idx` | `mongo_baseline` | 8 | 1200 | 2400 | 750 | 500 | 800 | 800 | 2.50 |", + "| 1000 | 0 | `treedb_0idx` | `mongo_baseline` | 8 | 1200 | 2400", + "2500 | 2470 | 0.08 | 7.00 | 4.00 | 625 | 1.20 | 0.03 | 30.0 | 1000 | 4.00 | 8.00 | 4.00 | 3.00 | 512", + "3.00 | 96.0 | 8192", + "2.00 | 200 | 0.05 | 1.50 | 150 | 0.05 | 0.10 | 10.0 | 0 | 0.40 | 40.0 | 0 | 0.50 | 50.0 | 0 | 1.25 | 125 | 0 | 1.00 | 100 | 0 | 0.05 | 5.00 | 0 | 0.20 | 20.0 | 0 | 0.25 | 25.0 | 0", + "0.75 | 0.01 | 0.02 | 0.44 | 0.50 | 1.00 | 42.0", + "0.25 | 0.12 | 8.00 | `/tmp/treedb.json` |", } { if !strings.Contains(rendered, want) { t.Fatalf("writer sweep table missing %q:\n%s", want, rendered) diff --git a/scripts/mongo_gateway_writer_metrics.py b/scripts/mongo_gateway_writer_metrics.py index 83d34343e1..7a8f8c689e 100755 --- a/scripts/mongo_gateway_writer_metrics.py +++ b/scripts/mongo_gateway_writer_metrics.py @@ -16,20 +16,90 @@ "ops_per_sec", "sampled_ns_per_op", "driver_calls", + "treedb_drain_ms", + "drain_indexed_flush_calls_total", + "drain_coalesced_flush_batches_total", + "drain_primary_only_drains_total", "publish_delta_group_calls_per_doc", "root_apply_calls_per_doc", "roots_per_publish", + "publish_delta_group_root_apply_ns_per_doc", + "publish_delta_group_root_apply_excluding_read_only_prepare_ns_per_doc", + "internal_parallel_merges_per_doc", + "internal_parallel_children_per_merge", + "internal_parallel_workers_per_merge", + "internal_parallel_ops_per_merge", + "read_only_prepare_root_apply_share_pct", + "read_only_prepare_calls_per_doc", + "read_only_prepare_ns_per_doc", + "read_only_prepare_ns_per_plan", + "read_only_prepare_ops_per_doc", + "read_only_prepare_leaf_spans_per_plan", + "read_only_prepare_worker_targets_per_plan", + "read_only_prepare_worker_ranges_per_plan", + "read_only_prepare_worker_max_ops_per_plan", "primary_root_publishes_per_doc", "primary_root_delta_entries_per_doc", "primary_root_delta_bytes_per_doc", "indexed_flush_units_per_batch", "indexed_flush_docs_per_batch", + "coalesced_batch_units_per_batch", + "coalesced_batch_docs_per_batch", + "coalesced_batch_bytes_per_batch", + "raw_root_delta_entries_per_doc", + "raw_root_delta_bytes_per_doc", + "raw_root_delta_tombstones_per_doc", + "raw_primary_root_delta_entries_per_doc", + "raw_primary_root_delta_bytes_per_doc", + "raw_primary_root_delta_tombstones_per_doc", + "raw_template_root_delta_entries_per_doc", + "raw_template_root_delta_bytes_per_doc", + "raw_template_root_delta_tombstones_per_doc", + "raw_index_state_root_delta_entries_per_doc", + "raw_index_state_root_delta_bytes_per_doc", + "raw_index_state_root_delta_tombstones_per_doc", + "raw_secondary_root_delta_entries_per_doc", + "raw_secondary_root_delta_bytes_per_doc", + "raw_secondary_root_delta_tombstones_per_doc", + "final_root_delta_entries_per_doc", + "final_root_delta_bytes_per_doc", + "final_root_delta_tombstones_per_doc", + "final_primary_root_delta_entries_per_doc", + "final_primary_root_delta_bytes_per_doc", + "final_primary_root_delta_tombstones_per_doc", + "final_template_root_delta_entries_per_doc", + "final_template_root_delta_bytes_per_doc", + "final_template_root_delta_tombstones_per_doc", + "final_index_state_root_delta_entries_per_doc", + "final_index_state_root_delta_bytes_per_doc", + "final_index_state_root_delta_tombstones_per_doc", + "final_secondary_root_delta_entries_per_doc", + "final_secondary_root_delta_bytes_per_doc", + "final_secondary_root_delta_tombstones_per_doc", + "squashed_root_delta_entries_per_doc", + "net_zero_root_batches_per_doc", + "net_zero_root_plans_per_doc", + "skipped_secondary_roots_per_doc", + "primary_only_duplicate_ids_coalesced_per_doc", + "primary_only_drains_per_doc", + "primary_only_drain_docs_per_drain", "leaf_log_node_loads_per_doc", "leaf_log_pages_written_per_doc", "leaf_log_read_bytes_per_doc", "leaf_log_write_bytes_per_doc", "backpressure_sync_total", "root_mismatch_total", + "read_only_prepare_calls_total", + "read_only_prepare_worker_targets_total", + "read_only_prepare_worker_ranges_total", + "root_delta_plan_raw_unit_primary_entries_total", + "root_delta_plan_raw_unit_secondary_entries_total", + "root_delta_plan_final_primary_entries_total", + "root_delta_plan_final_secondary_entries_total", + "root_delta_plan_squashed_entries_total", + "coalesced_flush_net_zero_batches_total", + "primary_only_duplicate_ids_coalesced_total", + "primary_only_drains_total", "raw_json", ] @@ -37,11 +107,66 @@ "publish_delta_group_calls_per_doc": "publish_delta_group_calls/doc", "root_apply_calls_per_doc": "root_apply_calls/doc", "roots_per_publish": "roots/publish", + "publish_delta_group_root_apply_ns_per_doc": "publish_delta_group_root_apply_ns/doc", + "publish_delta_group_root_apply_excluding_read_only_prepare_ns_per_doc": "publish_delta_group_root_apply_excluding_read_only_prepare_ns/doc", + "internal_parallel_merges_per_doc": "internal_parallel_merges/doc", + "internal_parallel_children_per_merge": "internal_parallel_children/merge", + "internal_parallel_workers_per_merge": "internal_parallel_workers/merge", + "internal_parallel_ops_per_merge": "internal_parallel_ops/merge", + "read_only_prepare_root_apply_share_pct": "read_only_prepare_root_apply_share_pct", + "read_only_prepare_calls_per_doc": "read_only_prepare_calls/doc", + "read_only_prepare_ns_per_doc": "read_only_prepare_ns/doc", + "read_only_prepare_ns_per_plan": "read_only_prepare_ns/plan", + "read_only_prepare_ops_per_doc": "read_only_prepare_ops/doc", + "read_only_prepare_leaf_spans_per_plan": "read_only_prepare_leaf_spans/plan", + "read_only_prepare_worker_targets_per_plan": "read_only_prepare_worker_targets/plan", + "read_only_prepare_worker_ranges_per_plan": "read_only_prepare_worker_ranges/plan", + "read_only_prepare_worker_max_ops_per_plan": "read_only_prepare_worker_max_ops/plan", "primary_root_publishes_per_doc": "primary_root_publishes/doc", "primary_root_delta_entries_per_doc": "primary_root_delta_entries/doc", "primary_root_delta_bytes_per_doc": "primary_root_delta_bytes/doc", "indexed_flush_units_per_batch": "indexed_flush_units/batch", "indexed_flush_docs_per_batch": "indexed_flush_docs/batch", + "coalesced_batch_units_per_batch": "coalesced_batch_units/batch", + "coalesced_batch_docs_per_batch": "coalesced_batch_docs/batch", + "coalesced_batch_bytes_per_batch": "coalesced_batch_bytes/batch", + "raw_root_delta_entries_per_doc": "raw_root_delta_entries/doc", + "raw_root_delta_bytes_per_doc": "raw_root_delta_bytes/doc", + "raw_root_delta_tombstones_per_doc": "raw_root_delta_tombstones/doc", + "raw_primary_root_delta_entries_per_doc": "raw_primary_root_delta_entries/doc", + "raw_primary_root_delta_bytes_per_doc": "raw_primary_root_delta_bytes/doc", + "raw_primary_root_delta_tombstones_per_doc": "raw_primary_root_delta_tombstones/doc", + "raw_template_root_delta_entries_per_doc": "raw_template_root_delta_entries/doc", + "raw_template_root_delta_bytes_per_doc": "raw_template_root_delta_bytes/doc", + "raw_template_root_delta_tombstones_per_doc": "raw_template_root_delta_tombstones/doc", + "raw_index_state_root_delta_entries_per_doc": "raw_index_state_root_delta_entries/doc", + "raw_index_state_root_delta_bytes_per_doc": "raw_index_state_root_delta_bytes/doc", + "raw_index_state_root_delta_tombstones_per_doc": "raw_index_state_root_delta_tombstones/doc", + "raw_secondary_root_delta_entries_per_doc": "raw_secondary_root_delta_entries/doc", + "raw_secondary_root_delta_bytes_per_doc": "raw_secondary_root_delta_bytes/doc", + "raw_secondary_root_delta_tombstones_per_doc": "raw_secondary_root_delta_tombstones/doc", + "final_root_delta_entries_per_doc": "final_root_delta_entries/doc", + "final_root_delta_bytes_per_doc": "final_root_delta_bytes/doc", + "final_root_delta_tombstones_per_doc": "final_root_delta_tombstones/doc", + "final_primary_root_delta_entries_per_doc": "final_primary_root_delta_entries/doc", + "final_primary_root_delta_bytes_per_doc": "final_primary_root_delta_bytes/doc", + "final_primary_root_delta_tombstones_per_doc": "final_primary_root_delta_tombstones/doc", + "final_template_root_delta_entries_per_doc": "final_template_root_delta_entries/doc", + "final_template_root_delta_bytes_per_doc": "final_template_root_delta_bytes/doc", + "final_template_root_delta_tombstones_per_doc": "final_template_root_delta_tombstones/doc", + "final_index_state_root_delta_entries_per_doc": "final_index_state_root_delta_entries/doc", + "final_index_state_root_delta_bytes_per_doc": "final_index_state_root_delta_bytes/doc", + "final_index_state_root_delta_tombstones_per_doc": "final_index_state_root_delta_tombstones/doc", + "final_secondary_root_delta_entries_per_doc": "final_secondary_root_delta_entries/doc", + "final_secondary_root_delta_bytes_per_doc": "final_secondary_root_delta_bytes/doc", + "final_secondary_root_delta_tombstones_per_doc": "final_secondary_root_delta_tombstones/doc", + "squashed_root_delta_entries_per_doc": "squashed_root_delta_entries/doc", + "net_zero_root_batches_per_doc": "net_zero_root_batches/doc", + "net_zero_root_plans_per_doc": "net_zero_root_plans/doc", + "skipped_secondary_roots_per_doc": "skipped_secondary_roots/doc", + "primary_only_duplicate_ids_coalesced_per_doc": "primary_only_duplicate_ids_coalesced/doc", + "primary_only_drains_per_doc": "primary_only_drains/doc", + "primary_only_drain_docs_per_drain": "primary_only_drain_docs/drain", "leaf_log_node_loads_per_doc": "leaf_log_node_loads/doc", "leaf_log_pages_written_per_doc": "leaf_log_pages_written/doc", "leaf_log_read_bytes_per_doc": "leaf_log_read_bytes/doc", @@ -145,6 +270,9 @@ def write_writer_metrics(out_dir, matrix_path, writer_metrics_path): delta = phase.get("treedb_stats_delta") or {} if not isinstance(delta, dict): delta = {} + drain_delta = phase.get("treedb_drain_stats_delta") or {} + if not isinstance(drain_delta, dict): + drain_delta = {} out = { "target": target, "config": config, @@ -155,10 +283,20 @@ def write_writer_metrics(out_dir, matrix_path, writer_metrics_path): "ops_per_sec": fmt(phase.get("ops_per_sec")), "sampled_ns_per_op": fmt(phase.get("sampled_ns_per_op")), "driver_calls": fmt(phase.get("driver_calls")), + "treedb_drain_ms": fmt(phase.get("treedb_drain_ms")), "raw_json": raw_json, } for column, metric_name in METRIC_COLUMNS.items(): out[column] = fmt(metrics.get(metric_name)) + out["drain_indexed_flush_calls_total"] = delta_count(drain_delta, [ + "treedb.collections.write_domain.indexed_flush.calls_total", + ], "drain_indexed_flush_calls_total") + out["drain_coalesced_flush_batches_total"] = delta_count(drain_delta, [ + "treedb.collections.write_domain.coalesced_flush_batch.batches_total", + ], "drain_coalesced_flush_batches_total") + out["drain_primary_only_drains_total"] = delta_count(drain_delta, [ + "treedb.collections.write_domain.primary_only.drains_total", + ], "drain_primary_only_drains_total") out["backpressure_sync_total"] = delta_count(delta, [ "treedb.collections.write_domain.indexed_async_flush.backpressure_sync_total", ], "backpressure_sync_total") @@ -167,6 +305,39 @@ def write_writer_metrics(out_dir, matrix_path, writer_metrics_path): "treedb.collections.write_domain.indexed_flush.root_base_mismatch_total", "treedb.collections.write_domain.coordinator_requeue_on_mismatch_total", ], "root_mismatch_total") + out["read_only_prepare_calls_total"] = delta_count(delta, [ + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total", + ], "read_only_prepare_calls_total") + out["read_only_prepare_worker_targets_total"] = delta_count(delta, [ + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_targets_total", + ], "read_only_prepare_worker_targets_total") + out["read_only_prepare_worker_ranges_total"] = delta_count(delta, [ + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_ranges_total", + ], "read_only_prepare_worker_ranges_total") + out["root_delta_plan_raw_unit_primary_entries_total"] = delta_count(delta, [ + "treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total", + ], "root_delta_plan_raw_unit_primary_entries_total") + out["root_delta_plan_raw_unit_secondary_entries_total"] = delta_count(delta, [ + "treedb.collections.write_domain.root_delta_plan.raw_unit.secondary.entries_total", + ], "root_delta_plan_raw_unit_secondary_entries_total") + out["root_delta_plan_final_primary_entries_total"] = delta_count(delta, [ + "treedb.collections.write_domain.root_delta_plan.final.primary.entries_total", + ], "root_delta_plan_final_primary_entries_total") + out["root_delta_plan_final_secondary_entries_total"] = delta_count(delta, [ + "treedb.collections.write_domain.root_delta_plan.final.secondary.entries_total", + ], "root_delta_plan_final_secondary_entries_total") + out["root_delta_plan_squashed_entries_total"] = delta_count(delta, [ + "treedb.collections.write_domain.root_delta_plan.squashed_entries_total", + ], "root_delta_plan_squashed_entries_total") + out["coalesced_flush_net_zero_batches_total"] = delta_count(delta, [ + "treedb.collections.write_domain.coalesced_flush_batch.net_zero_batches_total", + ], "coalesced_flush_net_zero_batches_total") + out["primary_only_duplicate_ids_coalesced_total"] = delta_count(delta, [ + "treedb.collections.write_domain.primary_only.duplicate_ids_coalesced_total", + ], "primary_only_duplicate_ids_coalesced_total") + out["primary_only_drains_total"] = delta_count(delta, [ + "treedb.collections.write_domain.primary_only.drains_total", + ], "primary_only_drains_total") writer.writerow(out) diff --git a/scripts/mongo_gateway_writer_metrics_test.py b/scripts/mongo_gateway_writer_metrics_test.py index 339a4bf523..777c728d9d 100644 --- a/scripts/mongo_gateway_writer_metrics_test.py +++ b/scripts/mongo_gateway_writer_metrics_test.py @@ -87,10 +87,75 @@ def test_exact_integer_composites_and_invalid_present_values(self): "ops_per_sec": 1, "sampled_ns_per_op": 2, "driver_calls": 3, + "treedb_drain_ms": 4.25, + "treedb_drain_stats_delta": { + "treedb.collections.write_domain.indexed_flush.calls_total": "1", + "treedb.collections.write_domain.coalesced_flush_batch.batches_total": "1", + "treedb.collections.write_domain.primary_only.drains_total": "0", + }, + "treedb_metrics": { + "coalesced_batch_units/batch": 2, + "coalesced_batch_docs/batch": 50, + "coalesced_batch_bytes/batch": 4096, + "publish_delta_group_root_apply_ns/doc": 100, + "publish_delta_group_root_apply_excluding_read_only_prepare_ns/doc": 75, + "internal_parallel_merges/doc": 0.08, + "internal_parallel_children/merge": 7, + "internal_parallel_workers/merge": 4, + "internal_parallel_ops/merge": 625, + "read_only_prepare_root_apply_share_pct": 25, + "read_only_prepare_calls/doc": 0.1, + "read_only_prepare_ns/doc": 25, + "read_only_prepare_ns/plan": 250, + "read_only_prepare_ops/doc": 3, + "read_only_prepare_leaf_spans/plan": 6, + "read_only_prepare_worker_targets/plan": 4, + "read_only_prepare_worker_ranges/plan": 3, + "read_only_prepare_worker_max_ops/plan": 512, + "raw_root_delta_entries/doc": 6, + "raw_primary_root_delta_entries/doc": 2, + "raw_primary_root_delta_tombstones/doc": 0.1, + "raw_template_root_delta_entries/doc": 0.25, + "raw_template_root_delta_bytes/doc": 25, + "raw_template_root_delta_tombstones/doc": 0, + "raw_index_state_root_delta_entries/doc": 0.5, + "raw_index_state_root_delta_bytes/doc": 50, + "raw_index_state_root_delta_tombstones/doc": 0, + "raw_secondary_root_delta_entries/doc": 4, + "raw_secondary_root_delta_tombstones/doc": 0, + "final_root_delta_entries/doc": 3, + "final_primary_root_delta_entries/doc": 1, + "final_primary_root_delta_tombstones/doc": 0, + "final_template_root_delta_entries/doc": 0.125, + "final_template_root_delta_bytes/doc": 12.5, + "final_template_root_delta_tombstones/doc": 0, + "final_index_state_root_delta_entries/doc": 0.25, + "final_index_state_root_delta_bytes/doc": 25, + "final_index_state_root_delta_tombstones/doc": 0, + "final_secondary_root_delta_entries/doc": 2, + "final_secondary_root_delta_tombstones/doc": 0, + "squashed_root_delta_entries/doc": 3, + "net_zero_root_batches/doc": 0, + "net_zero_root_plans/doc": 0.25, + "skipped_secondary_roots/doc": 2.5, + "primary_only_duplicate_ids_coalesced/doc": 0.75, + "primary_only_drains/doc": 0.125, + }, "treedb_stats_delta": { "treedb.collections.write_domain.indexed_async_flush.backpressure_sync_total": huge, "treedb.collections.write_domain.collection_root_base_mismatch_total": "bad", "treedb.collections.write_domain.indexed_flush.root_base_mismatch_total": "1", + "treedb.collections.write_domain.root_delta_plan.raw_unit.primary.entries_total": huge, + "treedb.collections.write_domain.root_delta_plan.raw_unit.secondary.entries_total": "11", + "treedb.collections.write_domain.root_delta_plan.final.primary.entries_total": "7", + "treedb.collections.write_domain.root_delta_plan.final.secondary.entries_total": "5", + "treedb.collections.write_domain.root_delta_plan.squashed_entries_total": "9", + "treedb.collections.write_domain.coalesced_flush_batch.net_zero_batches_total": "0", + "treedb.collections.write_domain.primary_only.duplicate_ids_coalesced_total": "6", + "treedb.collections.write_domain.primary_only.drains_total": "2", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_calls_total": "10", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_targets_total": "40", + "treedb.publish.ordered_root_delta_group.root_apply_readonly_prepare_worker_ranges_total": "30", }, }], }), @@ -111,8 +176,52 @@ def test_exact_integer_composites_and_invalid_present_values(self): with output.open(newline="") as out_file: rows = list(csv.DictReader(out_file, delimiter="\t")) self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["treedb_drain_ms"], "4.25") + self.assertEqual(rows[0]["drain_indexed_flush_calls_total"], "1") + self.assertEqual(rows[0]["drain_coalesced_flush_batches_total"], "1") + self.assertEqual(rows[0]["drain_primary_only_drains_total"], "0") + self.assertEqual(rows[0]["coalesced_batch_units_per_batch"], "2") + self.assertEqual(rows[0]["publish_delta_group_root_apply_ns_per_doc"], "100") + self.assertEqual(rows[0]["publish_delta_group_root_apply_excluding_read_only_prepare_ns_per_doc"], "75") + self.assertEqual(rows[0]["internal_parallel_merges_per_doc"], "0.08") + self.assertEqual(rows[0]["internal_parallel_children_per_merge"], "7") + self.assertEqual(rows[0]["internal_parallel_workers_per_merge"], "4") + self.assertEqual(rows[0]["internal_parallel_ops_per_merge"], "625") + self.assertEqual(rows[0]["read_only_prepare_root_apply_share_pct"], "25") + self.assertEqual(rows[0]["read_only_prepare_calls_per_doc"], "0.1") + self.assertEqual(rows[0]["read_only_prepare_ns_per_doc"], "25") + self.assertEqual(rows[0]["read_only_prepare_ns_per_plan"], "250") + self.assertEqual(rows[0]["read_only_prepare_ops_per_doc"], "3") + self.assertEqual(rows[0]["read_only_prepare_leaf_spans_per_plan"], "6") + self.assertEqual(rows[0]["read_only_prepare_worker_targets_per_plan"], "4") + self.assertEqual(rows[0]["read_only_prepare_worker_ranges_per_plan"], "3") + self.assertEqual(rows[0]["read_only_prepare_worker_max_ops_per_plan"], "512") + self.assertEqual(rows[0]["raw_root_delta_entries_per_doc"], "6") + self.assertEqual(rows[0]["raw_primary_root_delta_entries_per_doc"], "2") + self.assertEqual(rows[0]["raw_primary_root_delta_tombstones_per_doc"], "0.1") + self.assertEqual(rows[0]["raw_template_root_delta_entries_per_doc"], "0.25") + self.assertEqual(rows[0]["raw_index_state_root_delta_bytes_per_doc"], "50") + self.assertEqual(rows[0]["raw_secondary_root_delta_entries_per_doc"], "4") + self.assertEqual(rows[0]["raw_secondary_root_delta_tombstones_per_doc"], "0") + self.assertEqual(rows[0]["final_root_delta_entries_per_doc"], "3") + self.assertEqual(rows[0]["final_template_root_delta_bytes_per_doc"], "12.5") + self.assertEqual(rows[0]["final_index_state_root_delta_tombstones_per_doc"], "0") + self.assertEqual(rows[0]["final_secondary_root_delta_tombstones_per_doc"], "0") + self.assertEqual(rows[0]["squashed_root_delta_entries_per_doc"], "3") + self.assertEqual(rows[0]["net_zero_root_batches_per_doc"], "0") self.assertEqual(rows[0]["backpressure_sync_total"], huge) self.assertEqual(rows[0]["root_mismatch_total"], "") + self.assertEqual(rows[0]["read_only_prepare_calls_total"], "10") + self.assertEqual(rows[0]["read_only_prepare_worker_targets_total"], "40") + self.assertEqual(rows[0]["read_only_prepare_worker_ranges_total"], "30") + self.assertEqual(rows[0]["root_delta_plan_raw_unit_primary_entries_total"], huge) + self.assertEqual(rows[0]["root_delta_plan_raw_unit_secondary_entries_total"], "11") + self.assertEqual(rows[0]["root_delta_plan_final_primary_entries_total"], "7") + self.assertEqual(rows[0]["root_delta_plan_squashed_entries_total"], "9") + self.assertEqual(rows[0]["coalesced_flush_net_zero_batches_total"], "0") + self.assertEqual(rows[0]["primary_only_duplicate_ids_coalesced_total"], "6") + self.assertEqual(rows[0]["primary_only_drains_total"], "2") + self.assertIn("treedb_drain_ms", rows[0]) self.assertIn("root_mismatch_total", stderr.getvalue())