Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/app/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -502,7 +502,7 @@ func buildDuckLakeMaterializer(settings *config.Settings, pollInterval time.Dura
dailyMode, ok := materializer.ParseDailyRollupMode(settings.MaterializerDailyRollupMode)
if !ok {
_ = duckSvc.Close()
return nil, nil, nil, fmt.Errorf("invalid MATERIALIZER_DAILY_ROLLUP_MODE %q (off|shadow)", settings.MaterializerDailyRollupMode)
return nil, nil, nil, fmt.Errorf("invalid MATERIALIZER_DAILY_ROLLUP_MODE %q (off|on; shadow retired in dq#55 step 5 — move to on)", settings.MaterializerDailyRollupMode)
}
var dailyDelay time.Duration
if settings.MaterializerDailyRollupDelay != "" {
Expand Down
27 changes: 15 additions & 12 deletions internal/config/settings.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,14 +132,14 @@ type Settings struct {
// reachable from config.
MaterializerMaxSnapshotSpan int `yaml:"MATERIALIZER_MAX_SNAPSHOT_SPAN"`
// MaterializerDailyRollupMode gates the daily signals_latest refresh (dq#55):
// "off" (default) does nothing; "shadow" maintains lake.signals_latest_daily
// by a once-daily watermarked fold while the per-pass fold keeps maintaining
// lake.signals_latest, and diffs the two after each refresh; "on" is the
// step-4 flip — the daily refresh maintains lake.signals_latest itself, the
// per-pass fold is off, and the first boot after shadow PROMOTES the
// validated shadow table (discarding the fold-era table, duplicate
// corruption included). Pair "on" with LAKE_ROLLUP_DAILY_SERVING=true on
// the query fleet. Materializer-only.
// "on" (the default, also for empty) — the once-daily watermarked refresh is
// THE maintainer of lake.signals_latest (the per-pass fold was removed in
// step 5); "off" disables the refresh entirely and leaves the rollup
// UNMAINTAINED — tests/one-off ops only, warned at boot. The retired
// "shadow" value is now invalid: shadow-era configs must move to "on" (a
// leftover lake.signals_latest_daily table is promoted automatically at
// first boot). Pair "on" with LAKE_ROLLUP_DAILY_SERVING=true on the query
// fleet. Materializer-only.
MaterializerDailyRollupMode string `yaml:"MATERIALIZER_DAILY_ROLLUP_MODE"`
// MaterializerDailyRollupDelay is a Go duration: how long after the
// UTC-midnight partition rollover the daily refresh waits before folding the
Expand Down Expand Up @@ -197,10 +197,13 @@ type Settings struct {
// not be off (the store connection rides on it). Query-fleet only.
LatestKVReadModeExtended string `yaml:"LATEST_KV_READ_MODE_EXTENDED"`
// LakeRollupDailyServing marks lake.signals_latest as maintained by the
// DAILY watermarked refresh (the dq#55 step-4 flip): summaries then serve
// the exact (rollup ∪ signals-since-watermark) union instead of the plain
// rollup read. MUST be false while the per-pass fold maintains the rollup
// (the union would double-count the tail). Query-fleet only; default false.
// DAILY watermarked refresh: summaries then serve the exact (rollup ∪
// signals-since-watermark) union instead of the plain rollup read. With the
// per-pass fold removed (dq#55 step 5) the daily refresh is the rollup's
// only maintainer, so this belongs true on the query fleet whenever the
// materializer runs the default MATERIALIZER_DAILY_ROLLUP_MODE=on — a plain
// rollup read under-counts the post-watermark tail. Query-fleet only;
// default false only so a mode=off test/ops setup isn't unioned twice.
LakeRollupDailyServing bool `yaml:"LAKE_ROLLUP_DAILY_SERVING"`
// LatestKVForceBootstrap re-runs the lake.signals_latest → KV bootstrap on
// boot even though the completion marker is present — the repair for a
Expand Down
4 changes: 2 additions & 2 deletions internal/latestkv/latestkv.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
// The bucket is a CACHE of the lake, never the source of truth: the writer
// (the single materializer, via materializer.LatestPublisher) folds each
// decoded batch in last-write-wins by (timestamp DESC, cloud_event_id ASC) —
// the exact recency order foldSignalsRollup uses — so publishes are idempotent
// the exact recency order rollupSelectSQL uses — so publishes are idempotent
// under NATS redelivery, window replay, and backfill. A lost update (KV outage,
// crash) heals per (subject, name) on that signal's next reading, or wholesale
// via BootstrapFromRollup. Readers must treat a miss or an unreachable bucket
Expand Down Expand Up @@ -142,7 +142,7 @@ func (e *Entry) LastSeen() time.Time {
// newerThan reports whether (ts, ceid) beats (oldTS, oldCEID) under the
// rollup's recency order: ORDER BY timestamp DESC, cloud_event_id ASC. On an
// exact timestamp tie the LEXICOGRAPHICALLY SMALLER cloud_event_id wins —
// matching foldSignalsRollup/rollupSelectSQL so the KV and the rollup pick the
// matching rollupSelectSQL so the KV and the rollup pick the
// same winner and the phase-2 fallback path can't flap between two values.
func newerThan(ts time.Time, ceid string, oldTS time.Time, oldCEID string) bool {
if ts.After(oldTS) {
Expand Down
2 changes: 1 addition & 1 deletion internal/latestkv/latestkv_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func TestFold_NewerTimestampWins(t *testing.T) {
}

// The rollup breaks exact-timestamp ties by cloud_event_id ASC
// (foldSignalsRollup's QUALIFY ordering); the KV fold must pick the same
// (rollupSelectSQL's QUALIFY ordering); the KV fold must pick the same
// winner so the phase-2 rollup fallback can't flap between two values.
func TestFold_EqualTimestampSmallerCEIDWins(t *testing.T) {
var e Entry
Expand Down
Loading
Loading