Skip to content

[codex] Prepare sorted q2 global ranks lazily #3413

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
snissn wants to merge 3 commits into from
Closed

[codex] Prepare sorted q2 global ranks lazily #3413

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e504ce8
Prepare sorted q2 global ranks lazily
snissn Jul 1, 2026
c25c664
Split sorted q2 local-rank diagnostics
snissn Jul 1, 2026
20e74d4
Merge remote-tracking branch 'origin/main' into codex/3324-q2-sorted-…
snissn Jul 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 49 additions & 36 deletions TreeDB/collections/column_physical_q2_1950_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ func TestTypedColumnQ2SortedGroupedDistinctStreaming1950(t *testing.T) {
}
defer func() { _ = runner.Close() }()
assertTypedColumnQ2SortedGroupedDistinctPostPrepareDiagnostics3324(t, "prepared setup", runner.PrepareDiagnostics(), true)
assertTypedColumnQ2PreparedGlobalRanksNoGlobalCodes1950(t, runner, false)
prepared, err := runner.Run()
if err != nil {
t.Fatalf("prepared q2: %v", err)
Expand Down Expand Up @@ -574,7 +575,7 @@ func TestTypedColumnQ2SortedGroupedDistinctLocalDictionariesAndEmptyValues1950(t
t.Fatalf("PrepareColumnPhysicalQuery(q2 local dictionaries): %v", err)
}
defer func() { _ = runner.Close() }()
assertTypedColumnQ2PreparedGlobalCodes1950(t, runner)
assertTypedColumnQ2PreparedGlobalRanksNoGlobalCodes1950(t, runner, true)
prepared, err := runner.Run()
if err != nil {
t.Fatalf("prepared q2 local dictionaries: %v", err)
Expand Down Expand Up @@ -770,7 +771,7 @@ func flattenColumnPhysicalEvents1950(batches [][]columnPhysicalJSONBenchParityEv
return events
}

func assertTypedColumnQ2PreparedGlobalCodes1950(tb testing.TB, runner *ColumnPhysicalQueryRunner) {
func assertTypedColumnQ2PreparedGlobalRanksNoGlobalCodes1950(tb testing.TB, runner *ColumnPhysicalQueryRunner, requireSharedDidM bool) {
tb.Helper()
if runner == nil || runner.typedColumn == nil {
tb.Fatalf("prepared q2 runner missing typed-column state")
Expand All @@ -785,20 +786,35 @@ func assertTypedColumnQ2PreparedGlobalCodes1950(tb testing.TB, runner *ColumnPhy
if part == nil {
tb.Fatalf("part %d missing sorted grouped-distinct state", partIdx)
}
if len(part.Group.GlobalCodes) != part.Rows || len(part.Distinct.GlobalCodes) != part.Rows {
tb.Fatalf("part %d global code rows group=%d distinct=%d want %d", partIdx, len(part.Group.GlobalCodes), len(part.Distinct.GlobalCodes), part.Rows)
if len(part.Group.GlobalCodes) != 0 || len(part.Distinct.GlobalCodes) != 0 {
tb.Fatalf("part %d global code rows group=%d distinct=%d want 0", partIdx, len(part.Group.GlobalCodes), len(part.Distinct.GlobalCodes))
}
if part.Group.GlobalDictionary == nil || !sort.StringsAreSorted(part.Group.GlobalDictionary) {
tb.Fatalf("part %d group global dictionary not lexicographically sorted group=%v", partIdx, part.Group.GlobalDictionary)
}
if !sort.StringsAreSorted(part.Group.GlobalDictionary) || !sort.StringsAreSorted(part.Distinct.GlobalDictionary) {
tb.Fatalf("part %d global dictionaries not lexicographically sorted group=%v distinct=%v", partIdx, part.Group.GlobalDictionary, part.Distinct.GlobalDictionary)
if !part.Group.GlobalCardinalityOK || part.Group.GlobalCardinality != len(part.Group.GlobalDictionary) {
tb.Fatalf("part %d group global cardinality=%d ok=%t want dictionary=%d", partIdx, part.Group.GlobalCardinality, part.Group.GlobalCardinalityOK, len(part.Group.GlobalDictionary))
}
for row, code := range part.Group.GlobalCodes {
if int(code) >= len(part.Group.GlobalDictionary) {
tb.Fatalf("part %d group global code row=%d code=%d outside cardinality=%d", partIdx, row, code, len(part.Group.GlobalDictionary))
if len(part.Group.GlobalLocalRanks) != len(part.Group.Dictionary) {
tb.Fatalf("part %d group local rank entries=%d want dictionary=%d", partIdx, len(part.Group.GlobalLocalRanks), len(part.Group.Dictionary))
}
for localCode, rank := range part.Group.GlobalLocalRanks {
if int(rank) >= part.Group.GlobalCardinality {
tb.Fatalf("part %d group local code=%d rank=%d outside cardinality=%d", partIdx, localCode, rank, part.Group.GlobalCardinality)
}
}
for row, code := range part.Distinct.GlobalCodes {
if int(code) >= len(part.Distinct.GlobalDictionary) {
tb.Fatalf("part %d distinct global code row=%d code=%d outside cardinality=%d", partIdx, row, code, len(part.Distinct.GlobalDictionary))
if part.Distinct.GlobalDictionary != nil {
tb.Fatalf("part %d distinct global dictionary allocated=%d want nil", partIdx, len(part.Distinct.GlobalDictionary))
}
if !part.Distinct.GlobalCardinalityOK || part.Distinct.GlobalCardinality == 0 {
tb.Fatalf("part %d distinct global cardinality=%d ok=%t want nonzero prepared cardinality", partIdx, part.Distinct.GlobalCardinality, part.Distinct.GlobalCardinalityOK)
}
if len(part.Distinct.GlobalLocalRanks) != len(part.Distinct.Dictionary) {
tb.Fatalf("part %d distinct local rank entries=%d want dictionary=%d", partIdx, len(part.Distinct.GlobalLocalRanks), len(part.Distinct.Dictionary))
}
for localCode, rank := range part.Distinct.GlobalLocalRanks {
if int(rank) >= part.Distinct.GlobalCardinality {
tb.Fatalf("part %d distinct local code=%d rank=%d outside cardinality=%d", partIdx, localCode, rank, part.Distinct.GlobalCardinality)
}
}
localDidM := -1
Expand All @@ -811,36 +827,27 @@ func assertTypedColumnQ2PreparedGlobalCodes1950(tb testing.TB, runner *ColumnPhy
if localDidM < 0 {
continue
}
partDidMRank := -1
for rank, value := range part.Distinct.GlobalDictionary {
if value == "did:m" {
partDidMRank = rank
break
}
}
if partDidMRank < 0 {
tb.Fatalf("part %d distinct global dictionary missing did:m", partIdx)
if localDidM >= len(part.Distinct.GlobalLocalRanks) {
tb.Fatalf("part %d local did:m code=%d outside distinct global local ranks=%d", partIdx, localDidM, len(part.Distinct.GlobalLocalRanks))
}
partDidMRank := int(part.Distinct.GlobalLocalRanks[localDidM])
if didMRank < 0 {
didMRank = partDidMRank
} else if didMRank != partDidMRank {
tb.Fatalf("did:m global rank part %d=%d want %d", partIdx, partDidMRank, didMRank)
}
seenDidMRow := false
for row, localCode := range part.Distinct.Codes {
for _, localCode := range part.Distinct.Codes {
if localCode == int64(localDidM) {
seenDidMRow = true
if part.Distinct.GlobalCodes[row] != uint32(partDidMRank) {
tb.Fatalf("part %d did:m row=%d global code=%d want %d", partIdx, row, part.Distinct.GlobalCodes[row], partDidMRank)
}
}
}
if !seenDidMRow {
tb.Fatalf("part %d local did:m code=%d not referenced by any row", partIdx, localDidM)
}
didMParts++
}
if didMParts < 2 {
if requireSharedDidM && didMParts < 2 {
tb.Fatalf("did:m appeared in %d parts want at least two to prove cross-part global rank reuse", didMParts)
}
}
Expand Down Expand Up @@ -1039,22 +1046,28 @@ func assertTypedColumnQ2DenseGroupCountDistinctDiagnostics1950(tb testing.TB, la

func assertTypedColumnQ2SortedGroupedDistinctPostPrepareDiagnostics3324(tb testing.TB, label string, diag ColumnPhysicalQueryDiagnostics, want bool) {
tb.Helper()
total := diag.TypedColumnPrepareQ2GroupGlobalDictionaryRankNanos +
diag.TypedColumnPrepareQ2DistinctGlobalDictionaryRankNanos +
diag.TypedColumnPrepareQ2GroupGlobalCodeRemapNanos +
remapTotal := diag.TypedColumnPrepareQ2GroupGlobalCodeRemapNanos +
diag.TypedColumnPrepareQ2DistinctGlobalCodeRemapNanos
if remapTotal != 0 {
tb.Fatalf("%s sorted grouped-distinct per-row remap nanos=%d want 0 diagnostics=%+v", label, remapTotal, diag)
}
dictionaryTotal := diag.TypedColumnPrepareQ2GroupGlobalDictionaryRankNanos +
diag.TypedColumnPrepareQ2DistinctGlobalDictionaryRankNanos
localRankTotal := diag.TypedColumnPrepareQ2GroupGlobalLocalRankNanos +
diag.TypedColumnPrepareQ2DistinctGlobalLocalRankNanos
if !want {
if total != 0 {
tb.Fatalf("%s sorted grouped-distinct post-prepare split nanos=%d want 0 diagnostics=%+v", label, total, diag)
if dictionaryTotal+localRankTotal != 0 {
tb.Fatalf("%s sorted grouped-distinct dictionary/rank/local-rank nanos=%d want 0 diagnostics=%+v", label, dictionaryTotal+localRankTotal, diag)
}
return
}
if total == 0 {
if dictionaryTotal+localRankTotal == 0 {
return
}
dictionaryTotal := diag.TypedColumnPrepareQ2GroupGlobalDictionaryRankNanos +
diag.TypedColumnPrepareQ2DistinctGlobalDictionaryRankNanos
if dictionaryTotal <= 0 {
tb.Fatalf("%s sorted grouped-distinct post-prepare split diagnostics=%+v want dictionary/rank split work when timer resolution records split work", label, diag)
if diag.TypedColumnPrepareQ2GroupGlobalDictionaryRankNanos <= 0 || diag.TypedColumnPrepareQ2DistinctGlobalDictionaryRankNanos <= 0 {
tb.Fatalf("%s sorted grouped-distinct post-prepare split diagnostics=%+v want group/distinct dictionary/rank work when timer resolution records split work", label, diag)
}
if diag.TypedColumnPrepareQ2GroupGlobalLocalRankNanos <= 0 || diag.TypedColumnPrepareQ2DistinctGlobalLocalRankNanos <= 0 {
tb.Fatalf("%s sorted grouped-distinct post-prepare split diagnostics=%+v want group/distinct local-rank work when timer resolution records split work", label, diag)
}
}
4 changes: 4 additions & 0 deletions TreeDB/collections/column_physical_query.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,8 @@ type ColumnPhysicalQueryDiagnostics struct {

TypedColumnPrepareQ2GroupGlobalDictionaryRankNanos int64
TypedColumnPrepareQ2DistinctGlobalDictionaryRankNanos int64
TypedColumnPrepareQ2GroupGlobalLocalRankNanos int64
TypedColumnPrepareQ2DistinctGlobalLocalRankNanos int64
TypedColumnPrepareQ2GroupGlobalCodeRemapNanos int64
TypedColumnPrepareQ2DistinctGlobalCodeRemapNanos int64

Expand Down Expand Up @@ -1516,6 +1518,8 @@ func mergeColumnPhysicalQueryDiagnostics(left, right ColumnPhysicalQueryDiagnost
}
left.TypedColumnPrepareQ2GroupGlobalDictionaryRankNanos += right.TypedColumnPrepareQ2GroupGlobalDictionaryRankNanos
left.TypedColumnPrepareQ2DistinctGlobalDictionaryRankNanos += right.TypedColumnPrepareQ2DistinctGlobalDictionaryRankNanos
left.TypedColumnPrepareQ2GroupGlobalLocalRankNanos += right.TypedColumnPrepareQ2GroupGlobalLocalRankNanos
left.TypedColumnPrepareQ2DistinctGlobalLocalRankNanos += right.TypedColumnPrepareQ2DistinctGlobalLocalRankNanos
left.TypedColumnPrepareQ2GroupGlobalCodeRemapNanos += right.TypedColumnPrepareQ2GroupGlobalCodeRemapNanos
left.TypedColumnPrepareQ2DistinctGlobalCodeRemapNanos += right.TypedColumnPrepareQ2DistinctGlobalCodeRemapNanos
return left
Expand Down
2 changes: 2 additions & 0 deletions TreeDB/collections/column_physical_typed_column_one_shot_cache_3088_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -607,6 +607,8 @@ func assertTypedColumnQ2PostPrepareSubphaseDiagnostics3158(tb testing.TB, label
diag.TypedColumnPrepareQ2DensePartLocalRankNanos
globalCodeTotal := diag.TypedColumnPrepareQ2GroupGlobalDictionaryRankNanos +
diag.TypedColumnPrepareQ2DistinctGlobalDictionaryRankNanos +
diag.TypedColumnPrepareQ2GroupGlobalLocalRankNanos +
diag.TypedColumnPrepareQ2DistinctGlobalLocalRankNanos +
diag.TypedColumnPrepareQ2GroupGlobalCodeRemapNanos +
diag.TypedColumnPrepareQ2DistinctGlobalCodeRemapNanos
if !want {
Expand Down
Loading
Loading