From 9d4509bce0b8459c1cce490fdd9a5d86d23ed467 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 09:18:11 -0500 Subject: [PATCH 01/55] =?UTF-8?q?fix(runtime):=20airc=20room-route=20error?= =?UTF-8?q?=20echoes=20aircRoom=20target=20=E2=80=94=20unbreak=20canary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #2051 merged WITHOUT this one-line fix (it stayed on the feature branch), so canary's `fails_loud_when_airc_room_targeted_but_transport_missing` test — which asserts the error echoes the target ("room-uuid") — is RED on canary ("Continuum Rust Tests: failure"). The room-broadcast fail-loud path dropped the target from its message. Echo {room}, matching the peer path. Fix the code, not the test (log-correlation echo is worth keeping). Regression: the merge of #2051 raced ahead of the fix commit. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- core/continuum-core/src/runtime/airc_interceptor.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/continuum-core/src/runtime/airc_interceptor.rs b/core/continuum-core/src/runtime/airc_interceptor.rs index ad35a6459b..4aaaf9287b 100644 --- a/core/continuum-core/src/runtime/airc_interceptor.rs +++ b/core/continuum-core/src/runtime/airc_interceptor.rs @@ -124,11 +124,11 @@ impl CommandInterceptor for AircInterceptor { // Room broadcast isn't a request/response inference hop. Fail loud // rather than pretend a single-peer RPC — only aircPeer routes today. - (None, Some(_room)) => Err( - "airc room-broadcast routing (aircRoom) isn't wired into the kernel \ - yet — only aircPeer (a single-peer command RPC) routes over airc today." - .to_string(), - ), + // Echo the target (like the peer path) so callers can correlate logs. + (None, Some(room)) => Err(format!( + "airc room-broadcast routing (aircRoom '{room}') isn't wired into the \ + kernel yet — only aircPeer (a single-peer command RPC) routes over airc today." + )), // Explicit single-peer target: route the command over airc to that // peer's continuum-core and return its result — the E=mc² primitive, From 3152989400372d27b834ca8936aa7c6169d7f5e7 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 09:49:44 -0500 Subject: [PATCH 02/55] =?UTF-8?q?feat(capacity):=20expert=5Fobserve=20harn?= =?UTF-8?q?ess=20=E2=80=94=20glass-box=20LIVE=20MoE=20expert=20routing=20(?= =?UTF-8?q?#230/#229)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runs a GGUF MoE through the core/llama FFI with LiveExpertObserver attached (the existing cb_eval → ffn_moe_topk seam), generates N tokens to drive real routing, then dumps the model-intrinsic affinity: hot/cold expert distribution + co-occurrence + prefetch candidates. That affinity is the INPUT to expert prefetch (#227), grid placement (#180), compaction, and distillation (#233). Uses the in-process FFI (not the live llama-server lane) because affinity is model-intrinsic — valid data, zero risk to live serving. First run (Qwen3-Coder-30B-A3B, 96 tokens, Metal): 43,640 activations, 6116/6144 expert-slots fired (~99.5%), hottest expert only 0.20% (~12x uniform), 6092 colder share 95.8%. FINDING: for an 8/128 (6.25%-active) MoE, activation over a generation is BROAD, not tiny-hot — so the paging win is the tier ladder + affinity placement, not a small resident set. Prefetch predictor returned 0 candidates over 96 tokens (needs more data). K3 (1.8% active) should be far more concentrated — same harness will quantify it when weights land. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/bin/expert_observe.rs | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 core/continuum-core/src/bin/expert_observe.rs diff --git a/core/continuum-core/src/bin/expert_observe.rs b/core/continuum-core/src/bin/expert_observe.rs new file mode 100644 index 0000000000..99ceec65bd --- /dev/null +++ b/core/continuum-core/src/bin/expert_observe.rs @@ -0,0 +1,114 @@ +//! expert_observe — glass-box the LIVE MoE expert routing (#230 / #229). +//! +//! Runs a GGUF MoE through the core/llama FFI with a [`LiveExpertObserver`] attached (the +//! already-built `cb_eval` → `ffn_moe_topk` seam in `core/llama/src/safe.rs`), generates +//! N tokens to drive REAL routing, then dumps the affinity data — hot/cold expert +//! distribution + co-occurrence + prefetch prediction. That affinity is the INPUT to +//! expert prefetch (#227), grid placement (#180), compaction, and distillation (#233). +//! +//! WHY this and not the live llama-server path: expert affinity is MODEL-INTRINSIC (which +//! experts co-fire for given inputs), so an in-process FFI run gathers valid data without +//! touching the live-persona serving lane. (The live llama-server path would need a +//! separate fork patch to emit routing; this harness needs neither.) +//! +//! Usage: +//! cargo run -p continuum-core --features metal,accelerate --bin expert_observe -- [n_tokens] [prompt] + +use std::path::PathBuf; +use std::sync::Arc; + +use continuum_core::capacity::expert_observer::LiveExpertObserver; +use llama::{Batch, ContextParams, ExpertObserver, Model, ModelParams, Sampler}; + +fn main() { + let args: Vec = std::env::args().collect(); + let model_path = args + .get(1) + .expect("usage: expert_observe [n_tokens] [prompt]"); + let n_tokens: usize = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(96); + let prompt = args.get(3).map(|s| s.as_str()).unwrap_or( + "Write a Rust function to reverse a string, then explain how it works step by step:\n", + ); + + // The observer is the sink: `cb_eval` calls `observe(layer, selected_experts, n_used)` + // per MoE layer per token from inside the compute thread. + let observer = LiveExpertObserver::new(); + + let model = Model::load( + PathBuf::from(model_path), + ModelParams { + n_gpu_layers: -1, + use_mmap: true, + }, + ) + .expect("load model"); + println!("Loaded {model_path} (vocab={})", model.n_vocab()); + + let mut ctx = model + .new_context(ContextParams { + n_ctx: 4096, + n_batch: 512, + n_seq_max: 1, + expert_observer: Some(observer.clone() as Arc), + ..Default::default() + }) + .expect("context"); + + // Prefill. + let prompt_tokens = model.tokenize(prompt, true, false).expect("tokenize"); + let mut batch = Batch::allocated(512, 1); + let last = (prompt_tokens.len() - 1) as i32; + for (i, tok) in prompt_tokens.iter().enumerate() { + batch.push(*tok, i as i32, &[0], i as i32 == last); + } + ctx.decode(&batch).expect("prefill decode"); + + // Generate — every decoded token drives the router → observer tallies real selections. + let mut sampler = Sampler::greedy(); + let mut n_cur = batch.n_tokens(); + let mut n_decoded = 0usize; + for _ in 0..n_tokens { + let token = sampler.sample(&ctx, -1); + if model.is_eog_token(token) { + break; + } + batch.clear(); + batch.push(token, n_cur, &[0], true); + ctx.decode(&batch).expect("gen decode"); + n_cur += 1; + n_decoded += 1; + } + + // Dump the affinity — the fuel for the whole optimization catalog. + let total = observer.total_hits(); + let hits = observer.snapshot_hits(); + let (_seen, cooccur) = observer.snapshot_cooccurrence(); + let predicted = observer.predicted(); + + println!("\n=== EXPERT AFFINITY (model-intrinsic, {n_decoded} tokens observed) ==="); + println!("total expert activations : {total}"); + println!("distinct experts fired : {}", hits.len()); + println!("co-occurring pairs seen : {}", cooccur.len()); + println!("prefetch candidates : {} experts", predicted.len()); + + let mut ranked: Vec<(String, u64)> = hits + .iter() + .map(|(k, v)| (format!("{k:?}"), *v)) + .collect(); + ranked.sort_by(|a, b| b.1.cmp(&a.1)); + let show = ranked.len().min(24); + println!("\ntop {show} hottest experts (of {} fired):", hits.len()); + for (id, h) in ranked.iter().take(show) { + let pct = if total > 0 { 100.0 * *h as f64 / total as f64 } else { 0.0 }; + println!(" {id:<28} {h:>8} ({pct:.2}%)"); + } + // The tail: how concentrated is activation? (the paging headroom) + if ranked.len() > show { + let tail: u64 = ranked.iter().skip(show).map(|(_, h)| *h).sum(); + let tail_pct = if total > 0 { 100.0 * tail as f64 / total as f64 } else { 0.0 }; + println!( + " ... {} colder experts share the remaining {tail_pct:.2}% (the page-out tail)", + ranked.len() - show + ); + } +} From afef13294034f7f0b6bde6d4be92774790c4703a Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 12:37:00 -0500 Subject: [PATCH 03/55] fix(chat): ChatModule executor fails loud per-request, not process panic (#201) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ChatModule::executor()` did `.cloned().expect(...)` — a hard panic if a `chat/poll`, `chat/send`, or `persist_posted` landed before `start_server` called `install_executor_on_all` (a boot race). Panicking there SIGABRTs the whole core and takes every other module down with it, for a per-request contract violation that only concerns that one request. Convert `executor()` to `Result, String>` returning the SAME loud, contract-naming message, and `?`-propagate it in the 3 callers (all already `Result<_, String>`). Faithful to [[no-fallbacks-ever]] — still loud, still names `install_executor_on_all`, no silent default — while satisfying #26 (faculties degrade, never panic): a command that races boot fails loudly to its caller instead of crashing the process. Regression test: a pre-install `poll()` returns the loud error naming the contract instead of panicking. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/modules/chat/mod.rs | 49 +++++++++++++++------ 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/core/continuum-core/src/modules/chat/mod.rs b/core/continuum-core/src/modules/chat/mod.rs index 09998d01df..9758d62d38 100644 --- a/core/continuum-core/src/modules/chat/mod.rs +++ b/core/continuum-core/src/modules/chat/mod.rs @@ -121,18 +121,23 @@ impl ChatModule { Self { executor_slot } } - /// Resolve the executor for the current call. Panics if the - /// executor was never installed — that's a boot ordering bug - /// (`start_server` must call `install_executor_on_all` BEFORE - /// any chat command can dispatch). Per [[no-fallbacks-ever]]: - /// the panic message names the contract so the operator sees - /// the actual problem. - fn executor(&self) -> Arc { - self.executor_slot.cloned().expect( + /// Resolve the executor for the current call. Returns a loud, + /// contract-naming error if the executor was never installed — a + /// boot-ordering bug (`start_server` must call + /// `install_executor_on_all` BEFORE any chat command dispatches). + /// Per [[no-fallbacks-ever]] the message still names the contract + /// so the operator sees the real problem — but the blast radius is + /// THIS request, not the whole core: a command that races boot + /// fails loudly to its caller instead of `.expect()`-panicking the + /// process and taking every other module down with it + /// (#201 boot-race / #26 faculties degrade, never panic). + fn executor(&self) -> Result, String> { + self.executor_slot.cloned().ok_or_else(|| { "ChatModule: CommandExecutor not installed — \ start_server must call install_executor_on_all \ - before any chat command can dispatch (task #224)", - ) + before any chat command can dispatch (task #224)" + .to_string() + }) } /// `chat/poll` — return recent messages, optionally filtered by @@ -151,7 +156,7 @@ impl ChatModule { /// 5. Normalize back to chronological order for display regardless /// of query direction. pub async fn poll(&self, params: ChatPollParams) -> Result { - let executor = self.executor(); + let executor = self.executor()?; let limit = params.limit.unwrap_or(DEFAULT_POLL_LIMIT); // ── Phase 1: resolve the anchor timestamp if the caller @@ -284,7 +289,7 @@ impl ChatModule { /// could be the dedup id) but the design conversation is its /// own scope. pub async fn send(&self, params: ChatSendParams) -> Result { - let executor = self.executor(); + let executor = self.executor()?; let message_id = Uuid::new_v4(); let now_ms = now_ms(); let now_iso = now_iso(now_ms); @@ -416,7 +421,7 @@ impl ChatModule { /// EXPECTED duplicate, never an error. A persona line's id is airc's /// `event_id` (stable across replay), so restarts can't double-store either. pub async fn persist_posted(&self, payload: Value) -> Result<(), String> { - let executor = self.executor(); + let executor = self.executor()?; let field = |k: &str| -> Result { payload .get(k) @@ -995,6 +1000,24 @@ mod tests { assert!(result.after_message_id.is_none()); } + // what this catches: #201 boot-race — a chat command that arrives BEFORE + // install_executor_on_all runs fails LOUD to its caller (naming the contract), + // instead of .expect()-panicking the whole core and taking every other module + // down with it. Regression for the .expect() → Result conversion in executor(). + #[tokio::test] + async fn command_before_executor_installed_fails_loud_not_panics() { + let chat = ChatModule::new(); // executor slot deliberately NOT installed + let err = chat + .poll(ChatPollParams::default()) + .await + .expect_err("poll before install_executor_on_all must fail, not panic"); + assert!(err.contains("not installed"), "loud contract error, got: {err}"); + assert!( + err.contains("install_executor_on_all"), + "error names the contract so the operator sees the real problem: {err}" + ); + } + // ── chat/poll: latest-N path (no anchor) ────────────────────────── #[tokio::test] From ca95b75c8cc67f5f7a2e24cae6ddb82adedb36cb Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 12:39:58 -0500 Subject: [PATCH 04/55] test(cognition): align tool-surface test name+doc with the deleted shrink-cliff (#212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test body was already updated (74acbb36c / #206) to pin the ABSENCE of the tight-window "discovery-pair only" shrink cliff — it asserts the full native surface (edit_file/bash/grep) is never window-amputated on an 8192 window. But the test NAME (`..._is_a_category_index_plus_discovery_pair`) and its header comment still described the DELETED behavior ("the per-turn tool PAYLOAD is the two-tool DISCOVERY PAIR"), contradicting the code below them. Rename to `tool_surface_is_a_category_index_plus_the_unamputated_native_surface` and rewrite the header to state what the test actually pins: the system prompt carries a category index, the full native surface rides beside it un-amputated, and a regression means either the ~150-schema dump or the amputation cliff came back. Doc/name only — body and assertions unchanged, still green. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/cognition/llm_deliberation_faculty.rs | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/core/continuum-core/src/cognition/llm_deliberation_faculty.rs b/core/continuum-core/src/cognition/llm_deliberation_faculty.rs index f5249b4c08..eb91e0a82b 100644 --- a/core/continuum-core/src/cognition/llm_deliberation_faculty.rs +++ b/core/continuum-core/src/cognition/llm_deliberation_faculty.rs @@ -1734,19 +1734,20 @@ mod tests { ); } - // what this catches: progressive disclosure — the per-turn tool PAYLOAD is the - // two-tool DISCOVERY PAIR (`commands/list` + `commands/help`), not the whole - // authorized registry, and the system prompt carries only a CATEGORY INDEX, not - // every tool. The old dump injected ~150 full schemas / one-liners (~4–5k tokens) - // into EVERY turn, overflowing n_ctx → 400 "exceeds context size" → mute. Now the - // surface is a tiny category index inside the system prompt + the two-tool native - // offering, so even a huge tool set leaves system + user + the offered tools well - // within the served window. Invariant: the category index (not tool names) rides - // the system prompt, the native offering is exactly the discovery pair, and the - // whole prompt + its tools + reserve fit the window. A regression means the dump - // came back. + // what this catches: progressive disclosure — the system prompt carries only a + // CATEGORY INDEX (names, not the ~150 full schemas), and the native offering is the + // FULL authorized coding surface, which is NEVER window-amputated. Two failure modes + // this pins the absence of: (1) the old dump that injected ~150 schemas (~4–5k + // tokens) into EVERY turn, overflowing n_ctx → 400 "exceeds context size" → mute; + // (2) the later "tight window ⇒ discovery-pair only" shrink cliff (deleted in #206 / + // 74acbb36c) that amputated the native surface to `commands/list`+`commands/help` on + // a tight window, stranding native-tool models in a help loop with 0 edits. Now the + // category index rides the system prompt, the full native surface rides beside it + // un-amputated, and the budget (`prompt_view_within`) reserves the specs' tokens and + // trims VOLATILE context so prompt + tools + reserve fit the window. A regression + // means either the dump came back or the amputation cliff did. #[test] - fn tool_surface_is_a_category_index_plus_discovery_pair() { + fn tool_surface_is_a_category_index_plus_the_unamputated_native_surface() { let persona = Uuid::new_v4(); let adapter: Arc = Arc::new(HeuristicInferenceAdapter::new()); // A tool set whose FULL schemas would dwarf the window — the live shape From 97011f4f82102fb4eb4e3bfd2fc26a0d1379dfcf Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 12:55:47 -0500 Subject: [PATCH 05/55] =?UTF-8?q?feat(serving):=20elastic=20demand-driven?= =?UTF-8?q?=20context=20window=20=E2=80=94=20thread=20the=20ceiling,=20sto?= =?UTF-8?q?p=20baking=20it=20at=20launch=20(#234)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The served window was capped by a STATIC constant: window_for(lanes).min(BOOTSTRAP_WORKING_SET). So a hard coding task was clamped to 16k even when the model supports 128k and the budget allows it — exactly the "set in stone at launch" anti-pattern, when the whole system is a live, continuously-re-decided negotiation. Thread the demand ceiling as a parameter: plan_serving_with_demand(host, candidates, demand_lanes, demand_ceil). The window sizes UP to what the budget allows (window_for), then caps DOWN to what the TASK needs. A hard task passes a high ceiling → the window grows; a simple turn passes a low one → it shrinks so more lanes fit (the multi-persona concurrency win). This is the local half of "resources ebb and flow with demand"; the grid_overflow_lanes producer is the scale-out half — same demand→grant loop. plan_serving(...) stays as a thin cold-start wrapper passing BOOTSTRAP_WORKING_SET as the prior, so every existing caller is unchanged and behavior is identical until a demand producer threads live demand — the rail is laid without touching the live-GPU-gated fit math. OOM-safe by construction: window_for already bounds the window to the budget, so a higher ceiling only raises the cap TOWARD that bound, never past it. Growing for a hard task can't crash a lane. Test: cold=16k (prior), high-ceiling=64k (grew for the hard task), low-ceiling=8k (shrank for the simple one). 22 serving_plan tests green, no regression. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/cognition/serving_plan.rs | 75 ++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/core/continuum-core/src/cognition/serving_plan.rs b/core/continuum-core/src/cognition/serving_plan.rs index 1bdbf43dec..3976c18b24 100644 --- a/core/continuum-core/src/cognition/serving_plan.rs +++ b/core/continuum-core/src/cognition/serving_plan.rs @@ -310,12 +310,25 @@ pub struct ServingPlan { /// rendered, and the room degenerated into a greeting loop. 2 slots would /// have doubled every mind's window with zero lost concurrency. /// +/// `demand_ceil` is the LIVE demand ceiling for the served window — the ELASTIC, +/// per-task upper bound. `window_for` sizes the window UP to what the budget +/// allows; this caps it DOWN to what the task actually needs. A hard coding task +/// passes a high ceiling and the window GROWS (up to the budget/model bound); a +/// simple turn passes a low one so more lanes fit. It is NEVER a launch-baked +/// constant — callers thread live per-persona/per-task demand (measured p95 + +/// headroom, or a task's explicit request). [`plan_serving`] supplies +/// [`BOOTSTRAP_WORKING_SET`] only as the cold-start prior until that telemetry +/// exists (#234). OOM-safe: `window_for` already bounds the window to the budget, +/// so a higher ceiling only raises the cap toward that bound, never past it. +/// [[serving-resources-are-elastic-per-task-leases-context-and-model-grow-for-hard-problems]] +/// /// The decision is pure classification on memory arithmetic — no model is /// loaded, no inference is run. -pub fn plan_serving( +pub fn plan_serving_with_demand( host: HostBudget, candidates: &[ModelFootprint], demand_lanes: u32, + demand_ceil: u32, ) -> Option { if candidates.is_empty() { return None; @@ -441,7 +454,7 @@ pub fn plan_serving( // never forces UP past what the model supports; `.max(MIN_SERVE_CTX)` keeps it // runnable. On a small host where window_for < BOOTSTRAP the cap is a no-op. let served_context_window = window_for(lanes as u64) - .min(BOOTSTRAP_WORKING_SET) + .min(demand_ceil.max(MIN_SERVE_CTX)) .max(MIN_SERVE_CTX); // The honest per-lane compute reserve AT the chosen window (floor + window-scaled), // reused by the packing math below AND reported to the board via @@ -501,6 +514,20 @@ pub fn plan_serving( }) } +/// Cold-start / no-live-demand convenience: [`plan_serving_with_demand`] with the +/// [`BOOTSTRAP_WORKING_SET`] prior as the demand ceiling. Callers that do not YET +/// thread live per-task demand use this; the elastic path (a hard task requesting +/// more context, or the #234 p95 telemetry) calls `plan_serving_with_demand` with +/// the measured/requested ceiling so the window ebbs and flows with real demand +/// rather than a baked constant. +pub fn plan_serving( + host: HostBudget, + candidates: &[ModelFootprint], + demand_lanes: u32, +) -> Option { + plan_serving_with_demand(host, candidates, demand_lanes, BOOTSTRAP_WORKING_SET) +} + /// Hysteresis wrapper around [`plan_serving`]: stops model THRASH from live- /// budget jitter. Keeps the `incumbent` model as long as it still fits the /// budget — switching DOWN only when the incumbent no longer fits (forced @@ -765,6 +792,50 @@ mod tests { ); } + // what this catches: the ELASTIC demand ceiling (Joel 2026-07-27: "context window sizes + // should ebb and flow depending on demands of the task and available resources — if it + // needs it larger for a moment, don't limit it"). The ceiling is threaded LIVE, not a + // launch-baked constant: a hard task passing a HIGH demand_ceil grows the served window + // PAST the BOOTSTRAP prior (up to the budget/model bound); a LOW ceiling shrinks it so + // more lanes fit. OOM-safe — window_for still bounds it, so a higher ceiling only raises + // the cap toward the budget bound, never past it. This is the "stop setting it in stone + // at launch" fix that opens the elastic-lease path. + #[test] + fn demand_ceiling_is_elastic_grows_for_a_hard_task_shrinks_for_a_simple_one() { + let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; + // Roomy host + high trained ceiling → window_for(1) far exceeds any of these ceilings, + // so the DEMAND ceiling (not the budget or the model) decides the served window. + let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); + + // Cold prior: default plan_serving caps at BOOTSTRAP_WORKING_SET. + let cold = plan_serving(host, std::slice::from_ref(&devstral), 1).unwrap(); + assert_eq!(cold.served_context_window, BOOTSTRAP_WORKING_SET); + + // Hard task demands more context → the window GROWS past the prior. + let big_ceil = 64_000; + let hot = + plan_serving_with_demand(host, std::slice::from_ref(&devstral), 1, big_ceil).unwrap(); + assert!( + hot.served_context_window > cold.served_context_window, + "a higher demand ceiling must GROW the window: hot {} ≤ cold {}", + hot.served_context_window, + cold.served_context_window + ); + assert!( + hot.served_context_window <= big_ceil, + "growth stays bounded by the demand ceiling (and the budget), never past it: got {}", + hot.served_context_window + ); + + // Simple turn demands little → the window SHRINKS below the prior, freeing memory. + let lean = + plan_serving_with_demand(host, std::slice::from_ref(&devstral), 1, 8_192).unwrap(); + assert_eq!( + lean.served_context_window, 8_192, + "a low demand ceiling shrinks the served window to it" + ); + } + // what this catches: lanes DEGRADE on a tight host — MAX_LANES is a sanity backstop, // the fit math is the real cap. A 4-persona demand on a budget that can't feed 4 warm // slots must serve FEWER (well-fed) lanes, never 4 starving ones that OOM. The window From c29e92dd91bfb2c4e74751fdfe13f5178cb21275 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 13:13:57 -0500 Subject: [PATCH 06/55] =?UTF-8?q?feat(serving):=20WorkingSetDemand=20?= =?UTF-8?q?=E2=80=94=20the=20live=20demand=20producer=20for=20the=20elasti?= =?UTF-8?q?c=20window=20(#234)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The elastic served-window seam (plan_serving_with_demand) needed a demand PRODUCER that is measured, not guessed, and never launch-baked. WorkingSetDemand is it: a rolling observer of each turn's assembled-prompt token count that produces the live demand ceiling threaded into the plan. Two signals combine so it's both efficient AND never truncates: - demand_ceil() = max(floor, p95(recent prompts) + gen_headroom) — the sustained baseline that keeps the WARM lane sized to the persona's usual work, and ebbs back to the floor as lean turns roll through the window (a past hard session doesn't pin the window forever). p95, not max, so a lone spike can't over-provision a KV that swaps the box. - demand_for(measured_prompt) = max(baseline, measured_prompt + headroom) — the MEASURED current turn is never clamped. This is how "if it needs it larger for a moment, don't limit it" holds WITHOUT guessing: we already assembled the prompt, so we request exactly its size. plan_serving_with_demand still bounds it by the budget above, so an impossible prompt degrades honestly, never OOMs. Pure + fully unit-tested (cold→floor, sustained→grow, ebb-back, spike-excluded, current-turn-never-truncated). No serving loop, no GPU — the honest measurement under the elastic lease. Next slices wire it: observe TurnMetrics.input_tokens per turn, feed demand_for() into the live re-plan. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/cognition/mod.rs | 1 + .../src/cognition/working_set_demand.rs | 198 ++++++++++++++++++ 2 files changed, 199 insertions(+) create mode 100644 core/continuum-core/src/cognition/working_set_demand.rs diff --git a/core/continuum-core/src/cognition/mod.rs b/core/continuum-core/src/cognition/mod.rs index 1aac35dcf7..cbb9a2fade 100644 --- a/core/continuum-core/src/cognition/mod.rs +++ b/core/continuum-core/src/cognition/mod.rs @@ -77,6 +77,7 @@ pub mod response_orchestrator; pub mod response_validator; pub mod serving_plan; pub mod shared_analysis; +pub mod working_set_demand; pub mod self_repeat; pub mod should_respond; pub mod should_respond_module; diff --git a/core/continuum-core/src/cognition/working_set_demand.rs b/core/continuum-core/src/cognition/working_set_demand.rs new file mode 100644 index 0000000000..78b1a8d043 --- /dev/null +++ b/core/continuum-core/src/cognition/working_set_demand.rs @@ -0,0 +1,198 @@ +//! `working_set_demand` — the DEMAND PRODUCER for the elastic served window (#234). +//! +//! A persona's served context window should ebb and flow with what its turns +//! ACTUALLY use, not a launch-baked constant ([[serving-resources-are-elastic-per-task-leases-context-and-model-grow-for-hard-problems]]). +//! This observes the assembled-prompt token count of recent turns and produces a +//! live demand ceiling that threads into +//! [`plan_serving_with_demand`](super::serving_plan::plan_serving_with_demand) — +//! growing the window for a persona whose tasks got bigger (a hard coding session) +//! and ebbing it back when they get lean again, so many personas stay warm at a +//! lean window and one doing heavy work gets room to think. +//! +//! ## Measured, not guessed — and never truncate the current turn +//! +//! Two signals combine: +//! - a rolling **p95 baseline** ([`demand_ceil`]) sizes the WARM lane to the +//! persona's usual work (so it's already big when the next hard turn arrives); +//! - the **measured current prompt** ([`demand_for`]) guarantees THIS turn is +//! never truncated — we already assembled the prompt, so we know its exact +//! size and request precisely that + generation headroom. +//! +//! That is how "if it needs it larger for a moment, don't limit it" holds WITHOUT +//! guessing. The budget fit in `plan_serving_with_demand` still bounds the result +//! above, so an impossibly large prompt degrades honestly, never OOMs. p95 (not +//! max) for the baseline so a single rare spike doesn't pre-allocate a KV that +//! swaps the box — the spike is handled per-turn by the measured path instead. +//! +//! Pure + testable: token counts in, demand ceiling out. No serving loop, no GPU +//! here — this is the measurement that makes the elastic lease honest. + +use std::collections::VecDeque; + +/// Below this many samples [`demand_ceil`] returns the cold `floor` rather than a +/// p95 off 1–2 points: provisioning a window off a tiny sample swings it turn to +/// turn (thrash). A handful of turns is enough to trust the shape without waiting +/// so long the window never grows within a short session. +const MIN_SAMPLES_FOR_P95: usize = 4; + +/// Rolling observer of a persona's per-turn working-set size (assembled-prompt +/// tokens), producing the live demand ceiling for its served window. +pub struct WorkingSetDemand { + /// Recent assembled-prompt token counts, oldest at the front. Bounded to + /// `window`, so a past hard session doesn't pin the ceiling forever. + samples: VecDeque, + /// Rolling window length (turns). ≥ 1. + window: usize, + /// Cold-start floor AND the hard minimum — the ceiling never drops below this, + /// so a lull can't starve the next turn's first prompt. Callers pass the + /// serving cold prior (`serving_plan::BOOTSTRAP_WORKING_SET`). + floor: u32, + /// Added to the observed prompt to leave room for the turn's GENERATION (the + /// prompt is what's assembled; the model still needs to write its answer). + gen_headroom: u32, +} + +impl WorkingSetDemand { + pub fn new(window: usize, floor: u32, gen_headroom: u32) -> Self { + let window = window.max(1); + Self { + samples: VecDeque::with_capacity(window), + window, + floor, + gen_headroom, + } + } + + /// Record one completed turn's assembled-prompt token count. Oldest evicted + /// past `window` so demand EBBS back — a past hard session doesn't pin the + /// window forever. + pub fn observe(&mut self, prompt_tokens: u32) { + if self.samples.len() == self.window { + self.samples.pop_front(); + } + self.samples.push_back(prompt_tokens); + } + + /// The sustained baseline demand ceiling: `max(floor, p95(recent prompts) + + /// gen_headroom)`. Floored so a lull can't starve the next turn; p95 (not max) + /// so a lone spike doesn't over-provision. Returns `floor` until there is + /// enough evidence ([`MIN_SAMPLES_FOR_P95`]) to trust the shape. + pub fn demand_ceil(&self) -> u32 { + if self.samples.len() < MIN_SAMPLES_FOR_P95 { + return self.floor; + } + let mut v: Vec = self.samples.iter().copied().collect(); + v.sort_unstable(); + let idx = (((v.len() - 1) as f64) * 0.95).round() as usize; + let p95 = v[idx.min(v.len() - 1)]; + self.floor.max(p95.saturating_add(self.gen_headroom)) + } + + /// The demand ceiling for a SPECIFIC turn whose assembled prompt we've already + /// MEASURED: never below what this turn actually needs + /// (`current_prompt_tokens + gen_headroom`), and never below the sustained p95 + /// baseline (so the warm lane stays sized for the persona's usual work). This + /// is how "don't limit it for a moment" holds without guessing — we measured + /// the prompt, so we request exactly enough. `plan_serving_with_demand` still + /// bounds it by the budget above. + pub fn demand_for(&self, current_prompt_tokens: u32) -> u32 { + self.demand_ceil() + .max(current_prompt_tokens.saturating_add(self.gen_headroom)) + .max(self.floor) + } + + /// How many turns are in the rolling window (for observability / tests). + pub fn sample_count(&self) -> usize { + self.samples.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // The serving cold prior (serving_plan::BOOTSTRAP_WORKING_SET) + one generation + // of headroom — the values a live caller threads in. + const FLOOR: u32 = 16_384; + const HEAD: u32 = 2_048; + + // what this catches: cold start returns the FLOOR — provisioning a window off + // 1–2 samples would thrash it turn to turn, so it grows only once there's + // enough evidence to trust the shape. + #[test] + fn cold_start_returns_the_floor_until_enough_samples() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + assert_eq!(d.demand_ceil(), FLOOR); + d.observe(40_000); + d.observe(40_000); + assert_eq!(d.demand_ceil(), FLOOR, "still too few samples to trust the shape"); + } + + // what this catches: a sustained hard-coding session GROWS the baseline to + // p95 + generation headroom — the "if it needs it larger, don't limit it" case, + // measured not guessed. + #[test] + fn sustained_large_working_set_grows_the_ceiling_past_the_floor() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + for _ in 0..10 { + d.observe(40_000); + } + assert_eq!(d.demand_ceil(), 40_000 + HEAD, "grows to p95(40k) + headroom"); + assert!(d.demand_ceil() > FLOOR); + } + + // what this catches: demand EBBS BACK — a past hard session doesn't pin the + // window forever; once lean turns roll through the window, the ceiling returns + // to the floor and frees memory for more lanes. + #[test] + fn ceiling_ebbs_back_to_floor_as_lean_turns_roll_through_the_window() { + let mut d = WorkingSetDemand::new(8, FLOOR, HEAD); + for _ in 0..8 { + d.observe(40_000); + } + assert!(d.demand_ceil() > FLOOR); + for _ in 0..8 { + d.observe(3_000); // fully replaces the window + } + assert_eq!(d.demand_ceil(), FLOOR, "lean turns pull the ceiling back to the floor"); + } + + // what this catches: a single rare SPIKE does not over-provision the WARM + // baseline — p95 (not max) excludes the outlier so one 120k turn among lean + // ones can't pre-allocate a KV that swaps the box. + #[test] + fn single_spike_does_not_over_provision_the_baseline() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + for _ in 0..31 { + d.observe(3_000); + } + d.observe(120_000); // one outlier among 31 lean turns + assert!( + d.demand_ceil() <= (3_000 + HEAD).max(FLOOR), + "the lone spike must not lift the sustained baseline: {}", + d.demand_ceil() + ); + } + + // what this catches: the MEASURED current turn is NEVER truncated — a big prompt + // we already assembled requests exactly its size + headroom even before the p95 + // baseline has grown ("don't limit it for a moment", from measurement not a + // guess); a small turn still gets at least the warm baseline. + #[test] + fn demand_for_never_truncates_the_measured_current_turn() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + for _ in 0..10 { + d.observe(20_000); // baseline ~22k + } + assert_eq!( + d.demand_for(60_000), + 60_000 + HEAD, + "a measured 60k turn requests its own size + headroom, above the baseline" + ); + assert_eq!( + d.demand_for(1_000), + d.demand_ceil(), + "a tiny turn still rides the warm sustained baseline" + ); + } +} From 8501c5e8ca3da3d7c2a41b42ace7e5d4d8e5fcb1 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 13:27:18 -0500 Subject: [PATCH 07/55] =?UTF-8?q?feat(serving):=20serving=20daemon=20is=20?= =?UTF-8?q?demand-aware=20=E2=80=94=20elastic=20window=20threaded=20end=20?= =?UTF-8?q?to=20end=20(#234)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Threads the elastic demand ceiling through the LIVE serving decision path, not just the boot path: - plan_serving_stable now takes demand_ceil and forwards it to plan_serving_with_demand (both internal calls), so the hysteresis/ongoing-loop path is elastic too. - ServingDaemonModule holds a WorkingSetDemand aggregator (p95 of recent turns' assembled-prompt sizes, rolling window, floored at BOOTSTRAP_WORKING_SET), reads its demand_ceil() on every plan (compute_plan + publish_plan), and exposes observe_working_set(prompt_tokens) for the cognition turn path to feed. Inert-by-default: with no observations the aggregator returns the cold prior, so the served window is identical to before — behavior only changes once the emit is wired. A poisoned lock degrades to the prior, never panics. The whole plan API (boot + ongoing loop) is now demand-capable; the last hop is emitting each turn's input_tokens into observe_working_set so the window breathes with real demand. 74 serving tests green, no regression. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/cognition/serving_plan.rs | 15 ++++--- .../src/modules/serving_daemon.rs | 45 +++++++++++++++++-- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/core/continuum-core/src/cognition/serving_plan.rs b/core/continuum-core/src/cognition/serving_plan.rs index 3976c18b24..e935227bc5 100644 --- a/core/continuum-core/src/cognition/serving_plan.rs +++ b/core/continuum-core/src/cognition/serving_plan.rs @@ -541,13 +541,14 @@ pub fn plan_serving_stable( candidates: &[ModelFootprint], incumbent: Option<&str>, demand_lanes: u32, + demand_ceil: u32, ) -> Option { // NB: do NOT `?`-bail here. A deep transient dip can leave `plan_serving` // with nothing fitting the depressed budget (`fresh` = None) while a model // is STILL resident and serving fine — its memory is its own. Tearing that // down to "nothing" is the exact harm we're guarding against, so `fresh` is // an Option we fall back to only when the incumbent genuinely can't hold. - let fresh = plan_serving(host, candidates, demand_lanes); + let fresh = plan_serving_with_demand(host, candidates, demand_lanes, demand_ceil); let Some(inc_id) = incumbent else { return fresh; }; @@ -599,7 +600,7 @@ pub fn plan_serving_stable( if let Some(m) = promoted.iter_mut().find(|m| m.model_id == inc_id) { m.capability_rank = u8::MAX; } - plan_serving(at_rest, &promoted, demand_lanes) + plan_serving_with_demand(at_rest, &promoted, demand_lanes, demand_ceil) } fn bytes_gb(bytes: u64) -> f64 { @@ -1062,7 +1063,7 @@ mod tests { fn stable_with_no_incumbent_equals_plain() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; assert_eq!( - plan_serving_stable(host, &pair(), None, MAX_LANES), + plan_serving_stable(host, &pair(), None, MAX_LANES, BOOTSTRAP_WORKING_SET), plan_serving(host, &pair(), MAX_LANES) ); } @@ -1075,7 +1076,7 @@ mod tests { // 10GB: big (9.7GB) fits a lane but exceeds the 0.9*10=9GB headroom bar. let host = HostBudget { usable_bytes: 10 * GB, perf_cores: 6 }; assert_eq!(plan_serving(host, &pair(), MAX_LANES).unwrap().base_model_id, "big", "fresh would pick big"); - let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "small", "hysteresis keeps incumbent — no flap"); assert!(stable.lanes >= 1, "lanes still re-tracked for the kept model"); } @@ -1085,7 +1086,7 @@ mod tests { #[test] fn stable_upgrades_when_better_model_fits_with_headroom() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; // big 9.7 << 0.9*20=18 - let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "big", "more capable + ample headroom → upgrade"); } @@ -1100,7 +1101,7 @@ mod tests { fn stable_forced_down_when_incumbent_gone_from_disk() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; let only_small = vec![fp("small", 1, 4_000, 32_768, 1)]; // "big" no longer on disk - let stable = plan_serving_stable(host, &only_small, Some("big"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(host, &only_small, Some("big"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "small", "incumbent gone from disk → serve what's present"); } @@ -1124,7 +1125,7 @@ mod tests { "depressed-budget plain plan would flap to the smaller model" ); // With the incumbent credited its own weights back, the resident big stays. - let stable = plan_serving_stable(dipped, &pair(), Some("big"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(dipped, &pair(), Some("big"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "big", "incumbent survives its OWN load dip — no flap"); assert!(stable.lanes >= 1, "kept model still gets ≥1 lane"); } diff --git a/core/continuum-core/src/modules/serving_daemon.rs b/core/continuum-core/src/modules/serving_daemon.rs index b32dd25112..ff94143298 100644 --- a/core/continuum-core/src/modules/serving_daemon.rs +++ b/core/continuum-core/src/modules/serving_daemon.rs @@ -23,8 +23,10 @@ use crate::cognition::model_resolver::types::HwCapabilityTier; use crate::cognition::serving_plan::{ - plan_serving, plan_serving_stable, HostBudget, ModelFootprint, ServingPlan, MIN_SERVE_CTX, + plan_serving, plan_serving_stable, plan_serving_with_demand, HostBudget, ModelFootprint, + ServingPlan, BOOTSTRAP_WORKING_SET, MIN_SERVE_CTX, }; +use crate::cognition::working_set_demand::WorkingSetDemand; use crate::gpu::GpuMemoryManager; use crate::inference::llama_server::{ ensure_model_serving, serving_v1_url, AdapterEntry, EnsureOutcome, LlamaServerControl, @@ -239,6 +241,12 @@ pub struct ServingDaemonModule { /// that fit at pin time; budget can still shift under it, and then the plan /// degrades honestly (`fits_on_gpu = false`) rather than over-committing. pinned: watch::Sender>, + /// Live per-host working-set demand — the p95 of recent turns' assembled-prompt + /// sizes, threaded into the serving plan as the elastic `demand_ceil` so the + /// served window ebbs and flows with real demand instead of the baked prior + /// (#234). A Mutex (not atomic) for the rolling window; read only on the plan + /// tick, never a hot path. `observe_working_set` feeds it from the cognition turn. + working_set: std::sync::Mutex, } impl ServingDaemonModule { @@ -273,6 +281,10 @@ impl ServingDaemonModule { let (serving_tx, _srx) = watch::channel(ServingSnapshot::empty()); let (suppressed, _urx) = watch::channel(Arc::new(HashSet::new())); let (pinned, _prx) = watch::channel(None); + // Rolling window (recent turns) over which the served window's demand ceiling + // tracks the p95 assembled-prompt size. Long enough to hold a coding session's + // shape, short enough that demand ebbs back within a session (#234). + const WORKING_SET_WINDOW: usize = 64; Self { gpu, system, @@ -296,6 +308,11 @@ impl ServingDaemonModule { suppressed, pinned, lane_demand: Arc::new(std::sync::atomic::AtomicU32::new(1)), + working_set: std::sync::Mutex::new(WorkingSetDemand::new( + WORKING_SET_WINDOW, + BOOTSTRAP_WORKING_SET, + MIN_SERVE_CTX, // one generation of headroom above the assembled prompt + )), moe_serving: std::sync::Mutex::new(None), // Read ONCE here (single config entry point, no per-tick I/O). Off by default. measure_force_expert_budget_bytes: crate::config_env::read( @@ -313,6 +330,27 @@ impl ServingDaemonModule { .store(demand.max(1), Ordering::Relaxed); } + /// Feed one completed turn's assembled-prompt token count into the live + /// working-set demand (#234). Called from the cognition turn path; the next + /// plan tick sizes the served window to the p95 of recent turns. Cheap — a + /// brief lock on a bounded ring, off the serving hot path. + pub fn observe_working_set(&self, prompt_tokens: u32) { + if let Ok(mut w) = self.working_set.lock() { + w.observe(prompt_tokens); + } + } + + /// The live demand ceiling for the served window — the p95 baseline of recent + /// turns (floored at the cold prior). Read on the plan tick and threaded into + /// `plan_serving_with_demand` so the window grows for heavy work and ebbs back + /// when turns get lean. A poisoned lock degrades to the cold prior, never panics. + fn demand_ceil(&self) -> u32 { + self.working_set + .lock() + .map(|w| w.demand_ceil()) + .unwrap_or(BOOTSTRAP_WORKING_SET) + } + /// The current lane demand (≥ 1). /// Register serving's autonomic PLANNER to run on the memory authority's tick /// (MEMORY-AUTHORITY-DAEMON slice 1b). The lane plan — which model, how many lanes, @@ -559,10 +597,11 @@ impl ServingDaemonModule { /// to drive the spawner before the tick loop starts — single source of /// truth for "what model + how many lanes." pub fn compute_plan(&self) -> Option { - plan_serving( + plan_serving_with_demand( self.host_budget(), &self.live_candidates(), self.lane_demand(), + self.demand_ceil(), ) } @@ -1004,7 +1043,7 @@ impl ServingDaemonModule { .borrow() .as_ref() .map(|p| p.base_model_id.clone()); - match plan_serving_stable(budget, candidates, incumbent.as_deref(), self.lane_demand()) { + match plan_serving_stable(budget, candidates, incumbent.as_deref(), self.lane_demand(), self.demand_ceil()) { Some(plan) => { crate::probe!( class = "serving.plan", From e6b3f1c013c54222285fb3baf7fd80dee86cb287 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 13:33:55 -0500 Subject: [PATCH 08/55] =?UTF-8?q?feat(serving):=20close=20the=20elastic-wi?= =?UTF-8?q?ndow=20loop=20=E2=80=94=20turn=20demand=20feeds=20the=20served?= =?UTF-8?q?=20window=20(#234)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The final hop. The persona turn path now feeds each completed turn's assembled-prompt size into the serving demand, so the window breathes with real work instead of a baked constant: - serving_daemon exposes a process-wide sink (SERVING_WORKING_SET, same install_serving_state shape) holding the daemon's WorkingSetDemand (now Arc), registered at initialize(). observe_serving_working_set(tokens) is a free function the cognition path calls with NO daemon handle — no cross-subsystem plumbing. - service_loop, where each turn's TurnMetrics is known, calls observe_serving_working_set(m.input_tokens) at turn completion. The whole loop is now live: turn completes -> observe -> p95 baseline updates -> next plan tick sizes the served window to real demand (plan_serving_with_demand) -> a lean chat turn keeps it small so more personas stay warm, a heavy coding turn grows it toward the model/budget ceiling. Safe/inert until a daemon boots (the sink registers at init), so it can't break anything; it activates on the next deploy. Full continuum-core lib compiles clean. This completes the #234 elastic serving substrate started with plan_serving_with_demand + WorkingSetDemand: producer, seam, plan threading, daemon-awareness, and now the emit — all built and validated in isolation, ready for a live burst to watch the window grow on a hard turn. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/modules/serving_daemon.rs | 30 +++++++++++++++++-- .../src/persona/service_loop.rs | 5 ++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/core/continuum-core/src/modules/serving_daemon.rs b/core/continuum-core/src/modules/serving_daemon.rs index ff94143298..b1b19fc191 100644 --- a/core/continuum-core/src/modules/serving_daemon.rs +++ b/core/continuum-core/src/modules/serving_daemon.rs @@ -27,6 +27,26 @@ use crate::cognition::serving_plan::{ ServingPlan, BOOTSTRAP_WORKING_SET, MIN_SERVE_CTX, }; use crate::cognition::working_set_demand::WorkingSetDemand; + +/// Process-wide sink for per-turn working-set observations — the serving daemon +/// installs its own `WorkingSetDemand` here at [`ServingDaemonModule::initialize`], +/// so the cognition turn path feeds each turn's assembled-prompt size WITHOUT a +/// daemon handle (the same process-wide-readable-seam shape as `install_serving_state`). +/// Unset until a daemon boots → [`observe_serving_working_set`] is a no-op and the +/// served window simply holds its cold prior. (#234) +static SERVING_WORKING_SET: OnceLock>> = OnceLock::new(); + +/// Feed one completed turn's assembled-prompt token count into the live serving +/// demand. Called from the persona turn path; the next plan tick sizes the served +/// window to the p95 of recent turns. No-op before a serving daemon has booted, so +/// the caller never needs to know whether serving is up. (#234) +pub fn observe_serving_working_set(prompt_tokens: u32) { + if let Some(ws) = SERVING_WORKING_SET.get() { + if let Ok(mut w) = ws.lock() { + w.observe(prompt_tokens); + } + } +} use crate::gpu::GpuMemoryManager; use crate::inference::llama_server::{ ensure_model_serving, serving_v1_url, AdapterEntry, EnsureOutcome, LlamaServerControl, @@ -246,7 +266,7 @@ pub struct ServingDaemonModule { /// served window ebbs and flows with real demand instead of the baked prior /// (#234). A Mutex (not atomic) for the rolling window; read only on the plan /// tick, never a hot path. `observe_working_set` feeds it from the cognition turn. - working_set: std::sync::Mutex, + working_set: Arc>, } impl ServingDaemonModule { @@ -308,11 +328,11 @@ impl ServingDaemonModule { suppressed, pinned, lane_demand: Arc::new(std::sync::atomic::AtomicU32::new(1)), - working_set: std::sync::Mutex::new(WorkingSetDemand::new( + working_set: Arc::new(std::sync::Mutex::new(WorkingSetDemand::new( WORKING_SET_WINDOW, BOOTSTRAP_WORKING_SET, MIN_SERVE_CTX, // one generation of headroom above the assembled prompt - )), + ))), moe_serving: std::sync::Mutex::new(None), // Read ONCE here (single config entry point, no per-tick I/O). Off by default. measure_force_expert_budget_bytes: crate::config_env::read( @@ -1557,6 +1577,10 @@ impl ServiceModule for ServingDaemonModule { // free functions + adapters read "what's live" as a pointer instead of // each probing /v1/models. Set-once (singleton daemon). let _ = crate::inference::llama_server::install_serving_state(self.subscribe_serving()); + // Install this daemon's working-set demand as the process-wide sink so the + // cognition turn path feeds per-turn prompt sizes without a daemon handle, and + // the served window ebbs and flows with real demand (#234). + let _ = SERVING_WORKING_SET.set(self.working_set.clone()); // Register serving as a MEASURED ResourceConsumer with the one per-machine // authority (#79). See `register_as_consumer` — this is monitor-not-reserve: // no lease acquired, `available` math untouched, the authority simply stops diff --git a/core/continuum-core/src/persona/service_loop.rs b/core/continuum-core/src/persona/service_loop.rs index 945a422e90..d61f53526c 100644 --- a/core/continuum-core/src/persona/service_loop.rs +++ b/core/continuum-core/src/persona/service_loop.rs @@ -1058,6 +1058,11 @@ async fn serve_persona_loop_inner( tokens_per_second = m.tokens_per_second(), "deliberation generation cost" ); + // Feed this turn's assembled-prompt size into the elastic serving + // demand so the served window ebbs and flows with real work — a lean + // chat turn keeps it small (more personas warm), a heavy coding turn + // grows it (#234). No-op until a serving daemon has booted. + crate::modules::serving_daemon::observe_serving_working_set(m.input_tokens); } match step { crate::cognition::act_observe::SettleStep::Spoke(text) => text, From 0ea9538a1ea7e589752c32b944c0164b05066097 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 13:43:47 -0500 Subject: [PATCH 09/55] =?UTF-8?q?feat(serving):=20opt-in=20KV=20cache=20qu?= =?UTF-8?q?antization=20=E2=80=94=20q8=5F0=20halves=20KV,=20feeds=20the=20?= =?UTF-8?q?elastic=20window=20(#232)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds SERVING_KV_CACHE_TYPE (config, default f16/off): when set to q8_0 (or q4_0) the llama-server lane runs --cache-type-k/v , cutting resident KV ~in half at near-lossless quality. That frees memory the elastic window (#234) can spend on a bigger context or more warm lanes — faster for multiple personas AND more room for hard coding, the same "faster + best code" pair. OFF by default and safe-by-construction: absent / f16 → byte-identical f16 launch (no behavior change), so this can't destabilize a backend whose build lacks Metal KV-quant kernels — enabling it is an explicit operator opt-in, never a blind assumption ([[verify-real-device-numbers-not-a-clamp-premise]]). Follow-up (noted in code): to have the PLAN grow the window on the freed memory rather than leave it as extra headroom, footprint_for must scale kv_per_token by the quant factor. This slice is the safe enablement; that fit-math coupling is the next step, and wants a live burst on a KV-quant-capable backend to validate quality + the speedup. continuum-core lib compiles clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/inference/llama_server.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/core/continuum-core/src/inference/llama_server.rs b/core/continuum-core/src/inference/llama_server.rs index e54e31cec5..0970f4c4b7 100644 --- a/core/continuum-core/src/inference/llama_server.rs +++ b/core/continuum-core/src/inference/llama_server.rs @@ -1354,6 +1354,25 @@ impl LlamaServerControl for LlamaServerProcess { // surfaces the real defect: a RAG budget that overshot the served // window ([[fallbacks-are-illegal-fail-loud]]). .arg("--no-context-shift"); + // KV CACHE QUANTIZATION (#232, opt-in field-proven technique). f16 KV is the + // default; q8_0 is ~half the resident KV footprint at near-lossless quality, + // freeing memory the elastic window (#234) can spend on a BIGGER context or MORE + // warm lanes — faster for multiple personas AND more room for hard coding. + // OFF by default: not every backend/build ships Metal KV-quant kernels, so this + // is an operator opt-in, never a blind assumption ([[verify-real-device-numbers-not-a-clamp-premise]]). + // Set SERVING_KV_CACHE_TYPE=q8_0 (or q4_0) to enable; absent / `f16` → byte-identical + // f16 behavior. NOTE: to have the plan actually GROW the window on the freed memory + // (not just leave it as extra headroom), the fit math must also scale kv_per_token — + // that footprint coupling is the follow-up; this slice is the safe enablement. + if let Some(kv_type) = crate::config_env::read("SERVING_KV_CACHE_TYPE") + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty() && s != "f16") + { + cmd.arg("--cache-type-k") + .arg(&kv_type) + .arg("--cache-type-v") + .arg(&kv_type); + } // MULTIMODAL PROJECTOR (#106): a vision/audio-capable model needs its mmproj GGUF so // llama-server loads the vision (or audio) encoder and can tokenize image/audio content // parts. Present → the model actually SEES (the `ContentPart::Image` the persona render From 175cd6d51f8c6e459c37e024393182db629f81ab Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 13:51:10 -0500 Subject: [PATCH 10/55] =?UTF-8?q?feat(serving):=20KV-quant=20fit=20couplin?= =?UTF-8?q?g=20=E2=80=94=20the=20window=20GROWS=20into=20the=20freed=20KV?= =?UTF-8?q?=20memory=20(#232)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the KV-quant feature. The launcher flag (prior commit) makes a lane run q8_0 KV; this makes the PLAN know it: footprint_for scales kv_per_token by the quant divisor, so the served window is sized against the KV the lane WILL actually hold, and the elastic window (#234) grows into the freed memory instead of leaving it idle. - kv_divisor_for (pure, env-free, unit-tested): f16/unset/unknown → 1 (no change), q8_0 → 2, q4_0/q4_1 → 3. CONSERVATIVE by design — under the ideal ~3.5x for q4 — so the plan can never over-grow the window past the real KV and OOM (over-reserve = smaller window = safe). - Applied only in the config-aware footprint_for; footprint_from_parts stays pure so its tests are env-independent. Same SERVING_KV_CACHE_TYPE key as the launcher — one config, two consumers (flag + fit rate), documented to stay in sync. Default (f16 / unset) → divisor 1 → byte-identical: this can't change serving on a box that doesn't opt in. Test pins the mapping + the safe-default. Wants a live burst on a KV-quant-capable backend to confirm quality + the actual window growth. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/modules/serving_daemon.rs | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/core/continuum-core/src/modules/serving_daemon.rs b/core/continuum-core/src/modules/serving_daemon.rs index b1b19fc191..452f91e4c3 100644 --- a/core/continuum-core/src/modules/serving_daemon.rs +++ b/core/continuum-core/src/modules/serving_daemon.rs @@ -1349,12 +1349,38 @@ fn serving_footprint_fn(catalog: Arc) -> FootprintFn { pub fn footprint_for(model: &Model) -> Option { let path = crate::model_registry::artifacts::resolve_gguf_for_model(model)?; let weights_bytes = std::fs::metadata(&path).ok()?.len(); - footprint_from_parts( + let mut fp = footprint_from_parts( &model.id, weights_bytes, model.context_window, model.has(Capability::ToolUse), - ) + )?; + // KV CACHE QUANTIZATION (#232): a lane running quantized KV holds proportionally + // fewer bytes/token, so the plan can size a BIGGER window into the same budget — + // this is what turns the launcher's opt-in q8_0 flag into an actual window GROWTH. + // Divide the f16 rate by the quant factor; default (f16 / unset) → 1 → byte-identical. + // Keep the config key in sync with the launcher arg in inference/llama_server.rs — + // one SERVING_KV_CACHE_TYPE key, two consumers (launcher flag + this fit-math rate). + fp.kv_per_token = (fp.kv_per_token / kv_cache_quant_divisor()).max(1); + Some(fp) +} + +/// The resident-KV divisor implied by `SERVING_KV_CACHE_TYPE`, so the plan sizes the +/// served window against the KV the lane WILL actually hold, not the f16 default. (#232) +fn kv_cache_quant_divisor() -> u64 { + kv_divisor_for(crate::config_env::read("SERVING_KV_CACHE_TYPE").as_deref()) +} + +/// Pure KV-rate divisor for a cache-type string (testable without env). CONSERVATIVE by +/// design: q8_0 ≈ half of f16 → 2; q4_0/q4_1 ≈ a third → 3 (under the ideal ~3.5×, so the +/// plan never over-grows the window past the real KV and OOMs). Anything else / f16 → 1 +/// (no change). Over-reserve is a smaller window (safe); under-reserve is an OOM (fatal). +fn kv_divisor_for(cache_type: Option<&str>) -> u64 { + match cache_type.map(|s| s.trim().to_ascii_lowercase()).as_deref() { + Some("q8_0") => 2, + Some("q4_0") | Some("q4_1") => 3, + _ => 1, + } } /// Pure footprint estimate from the fields that drive it — split out from the @@ -1798,6 +1824,19 @@ mod tests { // what this catches: footprint estimate is honest about weights (passed // through), tool capability bumps the rank, KV is non-zero, and zero // weights → no footprint (we only offer what we can actually serve). + #[test] + fn kv_divisor_reflects_cache_type_conservatively() { + // what this catches: the #232 KV-quant fit-math coupling — the served window grows + // only when the lane actually runs quantized KV, and CONSERVATIVELY so the plan + // never over-grows past the real KV and OOMs. f16/unset/unknown must never scale. + assert_eq!(kv_divisor_for(None), 1, "unset never scales the window"); + assert_eq!(kv_divisor_for(Some("f16")), 1, "explicit f16 is the no-op default"); + assert_eq!(kv_divisor_for(Some("q8_0")), 2, "q8_0 ~ half of f16"); + assert_eq!(kv_divisor_for(Some(" Q8_0 ")), 2, "trimmed + case-insensitive"); + assert_eq!(kv_divisor_for(Some("q4_0")), 3, "q4_0 conservative, under the ideal ~3.5x"); + assert_eq!(kv_divisor_for(Some("garbage")), 1, "unknown type → no grow, never a bogus OOM"); + } + #[test] fn footprint_from_parts_is_footprint_aware() { let fp = footprint_from_parts("present", 3 * GB, 8192, true).unwrap(); From 10fed2798615a725590ef641aaab6636ae14b0fd Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:07:29 -0500 Subject: [PATCH 11/55] =?UTF-8?q?feat(serving):=20opt-in=20flash=20attenti?= =?UTF-8?q?on=20=E2=80=94=20faster=20prefill+decode,=20lower=20memory=20(#?= =?UTF-8?q?232)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fused attention kernel is faster on BOTH prefill and decode and lowers peak memory — directly attacking the prefill-bound turn latency (#139) and freeing room the elastic window (#234) can spend. SERVING_FLASH_ATTN=1|on|true adds --flash-attn to the lane; absent → llama.cpp default (no flag), byte-identical. OFF by default: Metal/backend flash-attn support + quality vary by build, so it's an operator opt-in, never a blind assumption ([[verify-real-device-numbers-not-a-clamp-premise]]). Composes with the KV-quant flag: enable both for the field-proven GLM-style speedup, then validate on a live burst. continuum-core lib compiles clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/inference/llama_server.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/continuum-core/src/inference/llama_server.rs b/core/continuum-core/src/inference/llama_server.rs index 0970f4c4b7..7bd27ca9ef 100644 --- a/core/continuum-core/src/inference/llama_server.rs +++ b/core/continuum-core/src/inference/llama_server.rs @@ -1373,6 +1373,19 @@ impl LlamaServerControl for LlamaServerProcess { .arg("--cache-type-v") .arg(&kv_type); } + // FLASH ATTENTION (#232, opt-in field-proven technique). The fused attention kernel + // is faster on BOTH prefill and decode and lowers peak memory — directly attacking + // the prefill-bound turn latency (#139) and freeing room the elastic window (#234) + // can spend. OFF by default: Metal/backend flash-attn support + quality vary by build + // ([[verify-real-device-numbers-not-a-clamp-premise]]), so it's an operator opt-in, + // never a blind assumption. SERVING_FLASH_ATTN=1|on|true → enable; absent → llama.cpp + // default (no flag), byte-identical. + if crate::config_env::read("SERVING_FLASH_ATTN") + .map(|s| matches!(s.trim().to_ascii_lowercase().as_str(), "1" | "on" | "true" | "yes")) + .unwrap_or(false) + { + cmd.arg("--flash-attn"); + } // MULTIMODAL PROJECTOR (#106): a vision/audio-capable model needs its mmproj GGUF so // llama-server loads the vision (or audio) encoder and can tokenize image/audio content // parts. Present → the model actually SEES (the `ContentPart::Image` the persona render From e47661aaf7fcd99917e4d012e457926c80faf408 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:08:07 -0500 Subject: [PATCH 12/55] =?UTF-8?q?feat(capacity):=20expert=5Fobserve=20?= =?UTF-8?q?=E2=80=94=20per-domain=20concentration=20+=20working-set-size?= =?UTF-8?q?=20+=20Jaccard=20(#230)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Evolves the MoE glass-box harness from the pooled first cut to the measurement that actually decides the paging architecture (BigMama's three methodological guardrails): diverse multi-domain corpus, PER-DOMAIN concentration (top-K% activation share vs the uniform null), cross-domain hot-set Jaccard, shared-base-vs-own-only activation MASS split, and the working-set-size curve (experts resident for 50/80/90/95% of a domain's mass). Prefill-dominant sampling on realistic input, not a degenerating greedy loop. Ran live on Qwen3-Coder-30B-A3B (Metal): pooled top-10% = 18.6% (mild — the smear), but per-domain = 25–38% with near-disjoint hot sets (code↔prose Jaccard 0.05) and a tiny 38-expert universal core — i.e. paging is domain-working-set SWAPPING, not frequency tiering. This is the #180 evidence; the harness is the reusable probe for any MoE (incl. K3 at weight-drop). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/bin/expert_observe.rs | 370 ++++++++++++++---- 1 file changed, 299 insertions(+), 71 deletions(-) diff --git a/core/continuum-core/src/bin/expert_observe.rs b/core/continuum-core/src/bin/expert_observe.rs index 99ceec65bd..558e731222 100644 --- a/core/continuum-core/src/bin/expert_observe.rs +++ b/core/continuum-core/src/bin/expert_observe.rs @@ -1,49 +1,125 @@ //! expert_observe — glass-box the LIVE MoE expert routing (#230 / #229). //! -//! Runs a GGUF MoE through the core/llama FFI with a [`LiveExpertObserver`] attached (the -//! already-built `cb_eval` → `ffn_moe_topk` seam in `core/llama/src/safe.rs`), generates -//! N tokens to drive REAL routing, then dumps the affinity data — hot/cold expert -//! distribution + co-occurrence + prefetch prediction. That affinity is the INPUT to -//! expert prefetch (#227), grid placement (#180), compaction, and distillation (#233). +//! Runs a GGUF MoE through the core/llama FFI with a [`LiveExpertObserver`] attached, drives +//! REAL routing over a labelled multi-domain corpus, then reports the affinity that decides +//! the paging architecture (#180): PER-DOMAIN concentration + CROSS-DOMAIN hot-set overlap. //! -//! WHY this and not the live llama-server path: expert affinity is MODEL-INTRINSIC (which -//! experts co-fire for given inputs), so an in-process FFI run gathers valid data without -//! touching the live-persona serving lane. (The live llama-server path would need a -//! separate fork patch to emit routing; this harness needs neither.) +//! ## Methodology (BigMama's three guardrails, 2026-07-27) +//! 1. **Sample size.** ~7 activations/slot is far too few to tell power-law from uniform — +//! Poisson noise swamps the skew. We drive thousands of tokens so the head separates. +//! 2. **Clean samples, not a greedy loop.** A long greedy generation degenerates and +//! OVER-routes to the same experts — a false positive for locality. We PREFILL diverse +//! prompts (prefill routes every token on realistic input) with only short generation. +//! 3. **Diverse HOW — the knife-edge.** Pooling ACROSS domains measures the global +//! mixed-workload histogram; MoE specialization routes domains to different experts, so +//! pooling SMEARS toward uniform and answers the WRONG question. The paging architecture +//! serves ONE persona doing ONE coherent thing, so we measure: +//! (a) PER-DOMAIN concentration — one observer per domain = the coherent-session +//! paging headroom (top-K% activation share WITHIN a domain). +//! (b) CROSS-DOMAIN overlap — intersect each domain's hot set. Experts hot in EVERY +//! domain = the always-resident shared base (never paged); experts hot in ONE +//! domain = the pageable specialization tier. The money number: what fraction of a +//! domain's activation MASS lands on experts cold for the other domains — the +//! eviction win. //! //! Usage: -//! cargo run -p continuum-core --features metal,accelerate --bin expert_observe -- [n_tokens] [prompt] +//! cargo run --release -p continuum-core --features metal,accelerate --bin expert_observe -- [n_gen_per_prompt] +use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::sync::Arc; use continuum_core::capacity::expert_observer::LiveExpertObserver; use llama::{Batch, ContextParams, ExpertObserver, Model, ModelParams, Sampler}; -fn main() { - let args: Vec = std::env::args().collect(); - let model_path = args - .get(1) - .expect("usage: expert_observe [n_tokens] [prompt]"); - let n_tokens: usize = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(96); - let prompt = args.get(3).map(|s| s.as_str()).unwrap_or( - "Write a Rust function to reverse a string, then explain how it works step by step:\n", - ); - - // The observer is the sink: `cb_eval` calls `observe(layer, selected_experts, n_used)` - // per MoE layer per token from inside the compute thread. - let observer = LiveExpertObserver::new(); - - let model = Model::load( - PathBuf::from(model_path), - ModelParams { - n_gpu_layers: -1, - use_mmap: true, - }, - ) - .expect("load model"); - println!("Loaded {model_path} (vocab={})", model.n_vocab()); +/// Labelled corpus: each domain gets its OWN observer, fed several diverse prompts so the +/// per-domain sample is dense. The point is a COHERENT workload per observer (what a persona +/// actually does in a session), not a global mixture. +const DOMAINS: &[(&str, &[&str])] = &[ + ( + "code", + &[ + "Implement a lock-free single-producer single-consumer ring buffer in Rust using \ + atomics with Release/Acquire ordering; explain why a SeqCst fence is unnecessary \ + and how the head/tail indices wrap a power-of-two capacity without a modulo in the \ + hot path.\n\nuse std::sync::atomic::{AtomicUsize, Ordering};\n", + "Write a React hook useDebouncedResource taking an async fetcher and debounce \ + interval that cancels in-flight requests on key change, dedupes concurrent callers, \ + and surfaces loading/error/stale-while-revalidate state without tearing under \ + concurrent mode.\n\nimport { useEffect, useRef, useState } from 'react';\n", + "Given orders(id, customer_id, placed_at, total) and refunds(order_id, amount, \ + refunded_at), write SQL returning each customer's net revenue by month, excluding \ + months with over 40% refunded, ranking customers within each month by a window \ + function.", + ], + ), + ( + "prose", + &[ + "The lighthouse keeper had not spoken to another person in forty-one days when the \ + boat appeared on the horizon. He noticed first the wrongness in the grey, and only \ + afterward resolved the shape into a hull riding low, and set down the brass polish \ + and went to the door, the cold coming up through the stone under his socks.", + "She had rehearsed the apology on the train, each version softer than the last, until \ + the words lost their edges and became a kind of weather she carried into the room. \ + Her mother was at the sink with her back turned, and for a moment neither of them \ + moved, and the tap ran over a single white plate.", + "Write the opening of a short story about a cartographer who discovers that a river \ + on his oldest map no longer exists, and who sets out on foot to find where it went, \ + narrated in close third person with attention to the texture of the walking.", + ], + ), + ( + "math", + &[ + "Prove that every finite integral domain is a field. Fix a nonzero a and consider \ + x -> a x; show injectivity, conclude surjectivity from finiteness, hence an inverse \ + of a exists. Then exhibit an infinite integral domain that is not a field to show \ + finiteness is essential.", + "Derive the closed form for the variance of a sum of two correlated random variables \ + in terms of their individual variances and covariance, then generalize to n \ + variables and interpret the cross terms as the reason diversification reduces \ + portfolio variance.", + "State and prove the pigeonhole principle, then use it to show that among any 51 \ + integers chosen from 1 to 100 there must be two that are coprime, and separately two \ + whose difference is exactly 10.", + ], + ), + ( + "science", + &[ + "Explain how the sodium-potassium pump maintains a neuron's resting potential: three \ + sodium out for two potassium in, the ATP-driven conformational change, and how the \ + electrogenic imbalance plus leak channels and the Nernst equilibria set the roughly \ + -70 mV resting potential.", + "Describe why the sky is blue in terms of Rayleigh scattering: the inverse fourth \ + power wavelength dependence, why shorter wavelengths scatter more strongly, and why \ + sunsets are red because of the longer atmospheric path length near the horizon.", + "Explain the greenhouse effect at the level of molecular physics: which atmospheric \ + gases absorb in the infrared, why their vibrational modes couple to outgoing \ + longwave radiation while nitrogen and oxygen do not, and how re-emission warms the \ + surface.", + ], + ), + ( + "planning", + &[ + "Sketch the migration plan to decompose a Rust monolith into services over a \ + command-and-event bus: identify the seams where synchronous calls become async, how \ + you preserve transactional guarantees that relied on one process, and how you roll \ + out incrementally behind a facade without a big-bang cutover.", + "Write a JSON schema for a distributed-cache eviction policy supporting LRU, LFU, and \ + TTL tiers with per-key overrides, budgets in both bytes and percentage-of-pool, and \ + validation that every declared cache class has at least one eviction dimension so \ + none grows unbounded.", + "Draft a rollout plan for a feature flag that changes a checkout flow: staged \ + percentage ramp, the metrics that gate each stage, the automatic rollback trigger, \ + and how you keep the two code paths from diverging while the flag is live.", + ], + ), +]; +fn drive_prompt(model: &Model, observer: &Arc, prompt: &str, n_gen: usize) { let mut ctx = model .new_context(ContextParams { n_ctx: 4096, @@ -54,61 +130,213 @@ fn main() { }) .expect("context"); - // Prefill. - let prompt_tokens = model.tokenize(prompt, true, false).expect("tokenize"); - let mut batch = Batch::allocated(512, 1); - let last = (prompt_tokens.len() - 1) as i32; - for (i, tok) in prompt_tokens.iter().enumerate() { - batch.push(*tok, i as i32, &[0], i as i32 == last); + let tokens = model.tokenize(prompt, true, false).expect("tokenize"); + let mut n_cur: i32 = 0; + for chunk in tokens.chunks(512) { + let mut batch = Batch::allocated(512, 1); + let last_global = n_cur as usize + chunk.len() - 1; + for (i, tok) in chunk.iter().enumerate() { + let pos = n_cur + i as i32; + batch.push(*tok, pos, &[0], (n_cur as usize + i) == last_global); + } + ctx.decode(&batch).expect("prefill decode"); + n_cur += chunk.len() as i32; } - ctx.decode(&batch).expect("prefill decode"); - // Generate — every decoded token drives the router → observer tallies real selections. let mut sampler = Sampler::greedy(); - let mut n_cur = batch.n_tokens(); - let mut n_decoded = 0usize; - for _ in 0..n_tokens { + for _ in 0..n_gen { let token = sampler.sample(&ctx, -1); if model.is_eog_token(token) { break; } - batch.clear(); + let mut batch = Batch::allocated(1, 1); batch.push(token, n_cur, &[0], true); ctx.decode(&batch).expect("gen decode"); n_cur += 1; - n_decoded += 1; } +} + +/// Sorted-desc counts → activation share captured by the top `frac` of FIRED experts. +fn top_share(counts_desc: &[u64], total: u64, frac: f64) -> f64 { + if counts_desc.is_empty() || total == 0 { + return 0.0; + } + let k = ((counts_desc.len() as f64 * frac).ceil() as usize).max(1).min(counts_desc.len()); + let s: u64 = counts_desc.iter().take(k).sum(); + 100.0 * s as f64 / total as f64 +} + +fn main() { + let args: Vec = std::env::args().collect(); + let model_path = args + .get(1) + .expect("usage: expert_observe [n_gen_per_prompt]"); + let n_gen: usize = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(48); + + let model = Model::load( + PathBuf::from(model_path), + ModelParams { n_gpu_layers: -1, use_mmap: true }, + ) + .expect("load model"); + println!("Loaded {model_path} (vocab={})", model.n_vocab()); + + // Drive each domain into its OWN observer → per-domain hit maps. + let mut per_domain = Vec::new(); + for (label, prompts) in DOMAINS { + let obs = LiveExpertObserver::new(); + for prompt in *prompts { + drive_prompt(&model, &obs, prompt, n_gen); + } + let hits = obs.snapshot_hits(); + println!( + " domain {label:<9} : {} activations across {} experts", + obs.total_hits(), + hits.len() + ); + per_domain.push((label.to_string(), hits)); + } + + // ---- PER-DOMAIN concentration (the coherent-session paging headroom) ---- + println!("\n=== PER-DOMAIN CONCENTRATION (top-K% share vs uniform null) ==="); + println!("{:<10} {:>8} {:>8} {:>8} {:>7} {:>7} {:>7}", "domain", "total", "fired", "mean", "top1%", "top10%", "top25%"); + for (label, hits) in &per_domain { + let total: u64 = hits.values().sum(); + let mut counts: Vec = hits.values().copied().collect(); + counts.sort_unstable_by(|a, b| b.cmp(a)); + let mean = if counts.is_empty() { 0.0 } else { total as f64 / counts.len() as f64 }; + println!( + "{label:<10} {total:>8} {:>8} {mean:>8.1} {:>6.1}% {:>6.1}% {:>6.1}%", + counts.len(), + top_share(&counts, total, 0.01), + top_share(&counts, total, 0.10), + top_share(&counts, total, 0.25), + ); + } + println!("(uniform null → top1%≈1, top10%≈10, top25%≈25; materially above = per-session paging headroom)"); - // Dump the affinity — the fuel for the whole optimization catalog. - let total = observer.total_hits(); - let hits = observer.snapshot_hits(); - let (_seen, cooccur) = observer.snapshot_cooccurrence(); - let predicted = observer.predicted(); + // ---- WORKING-SET SIZE: how many experts resident to capture X% of a domain's mass ---- + // THE engineering number for #180: the resident working set you must keep hot; the rest + // pages to CPU/disk (misses on the cold tail are low-mass = infrequent). + let experts_for_mass = |counts_desc: &[u64], total: u64, frac: f64| -> usize { + if total == 0 { return 0; } + let target = frac * total as f64; + let mut acc = 0u64; + for (i, c) in counts_desc.iter().enumerate() { + acc += c; + if acc as f64 >= target { return i + 1; } + } + counts_desc.len() + }; + println!("\n=== WORKING-SET SIZE (experts resident for X% of a domain's mass; % of fired) ==="); + println!("{:<10} {:>10} {:>12} {:>12} {:>12}", "domain", "50%mass", "80%mass", "90%mass", "95%mass"); + for (label, hits) in &per_domain { + let total: u64 = hits.values().sum(); + let mut counts: Vec = hits.values().copied().collect(); + counts.sort_unstable_by(|a, b| b.cmp(a)); + let n = counts.len().max(1); + let cell = |f: f64| { let k = experts_for_mass(&counts, total, f); format!("{k} ({:.0}%)", 100.0 * k as f64 / n as f64) }; + println!("{label:<10} {:>10} {:>12} {:>12} {:>12}", cell(0.50), cell(0.80), cell(0.90), cell(0.95)); + } + println!("(the 50%-mass column = the tight resident set; 95%-mass = keep-everything-hot floor; the GAP is the pageable tail)"); - println!("\n=== EXPERT AFFINITY (model-intrinsic, {n_decoded} tokens observed) ==="); - println!("total expert activations : {total}"); - println!("distinct experts fired : {}", hits.len()); - println!("co-occurring pairs seen : {}", cooccur.len()); - println!("prefetch candidates : {} experts", predicted.len()); + // ---- CROSS-DOMAIN overlap: shared base vs pageable specialization ---- + // A domain's HOT SET = the smallest set of experts capturing 50% of that domain's mass. + let hot_set = |hits: &HashMap<_, u64>| -> HashSet<_> { + let total: u64 = hits.values().sum(); + let mut ranked: Vec<(_, u64)> = hits.iter().map(|(k, v)| (*k, *v)).collect(); + ranked.sort_unstable_by(|a, b| b.1.cmp(&a.1)); + let mut acc = 0u64; + let mut set = HashSet::new(); + for (k, v) in ranked { + if (acc as f64) >= 0.5 * total as f64 { + break; + } + acc += v; + set.insert(k); + } + set + }; + let hot_sets: Vec<(String, HashSet<_>)> = + per_domain.iter().map(|(l, h)| (l.clone(), hot_set(h))).collect(); - let mut ranked: Vec<(String, u64)> = hits + // Jaccard overlap matrix of hot sets. + println!("\n=== CROSS-DOMAIN HOT-SET JACCARD (top-experts-to-50%-mass) ==="); + print!("{:<10}", ""); + for (l, _) in &hot_sets { + print!("{l:>9}"); + } + println!(); + for (la, sa) in &hot_sets { + print!("{la:<10}"); + for (_lb, sb) in &hot_sets { + let inter = sa.intersection(sb).count(); + let uni = sa.union(sb).count(); + let j = if uni > 0 { inter as f64 / uni as f64 } else { 0.0 }; + print!("{:>8.2} ", j); + } + println!(); + } + + // Shared base = experts in the hot set of ALL domains. Specialization = hot in exactly one. + let n_domains = hot_sets.len(); + let mut membership: HashMap<_, usize> = HashMap::new(); + for (_l, s) in &hot_sets { + for k in s { + *membership.entry(*k).or_insert(0) += 1; + } + } + let shared_base: HashSet<_> = membership .iter() - .map(|(k, v)| (format!("{k:?}"), *v)) + .filter(|(_, &c)| c == n_domains) + .map(|(k, _)| *k) .collect(); - ranked.sort_by(|a, b| b.1.cmp(&a.1)); - let show = ranked.len().min(24); - println!("\ntop {show} hottest experts (of {} fired):", hits.len()); - for (id, h) in ranked.iter().take(show) { - let pct = if total > 0 { 100.0 * *h as f64 / total as f64 } else { 0.0 }; - println!(" {id:<28} {h:>8} ({pct:.2}%)"); - } - // The tail: how concentrated is activation? (the paging headroom) - if ranked.len() > show { - let tail: u64 = ranked.iter().skip(show).map(|(_, h)| *h).sum(); - let tail_pct = if total > 0 { 100.0 * tail as f64 / total as f64 } else { 0.0 }; + let specialized: HashSet<_> = membership + .iter() + .filter(|(_, &c)| c == 1) + .map(|(k, _)| *k) + .collect(); + println!( + "\nshared base (hot in ALL {n_domains} domains): {} experts", + shared_base.len() + ); + println!( + "domain-specialized (hot in exactly 1) : {} experts", + specialized.len() + ); + + // The money number: per domain, what fraction of activation MASS lands on the shared base + // vs on experts NOT in any OTHER domain's hot set (the pageable eviction win). + println!("\n=== ACTIVATION MASS: shared-base vs pageable-specialized (the eviction win) ==="); + println!("{:<10} {:>14} {:>22}", "domain", "on shared base", "on own-only specialists"); + for (li, (label, hits)) in per_domain.iter().enumerate() { + let total: u64 = hits.values().sum(); + // experts that are in THIS domain's hot set and NO other domain's hot set + let own_only: HashSet<_> = hot_sets[li] + .1 + .iter() + .filter(|k| membership.get(*k).copied().unwrap_or(0) == 1) + .copied() + .collect(); + let mut base_mass = 0u64; + let mut own_mass = 0u64; + for (k, v) in hits { + if shared_base.contains(k) { + base_mass += v; + } + if own_only.contains(k) { + own_mass += v; + } + } + let pct = |m: u64| if total > 0 { 100.0 * m as f64 / total as f64 } else { 0.0 }; println!( - " ... {} colder experts share the remaining {tail_pct:.2}% (the page-out tail)", - ranked.len() - show + "{label:<10} {:>13.1}% {:>21.1}%", + pct(base_mass), + pct(own_mass) ); } + println!( + "\nRead: high own-only mass = a coherent session concentrates on experts the OTHER\n\ + domains never touch → evict them when the workload isn't that domain. High shared-base\n\ + mass = a resident core you never page. That split IS the #180 paging architecture." + ); } From 98a6a16c822e8cc5deaebcb1153a54231c84180c Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:22:44 -0500 Subject: [PATCH 13/55] =?UTF-8?q?feat(capacity):=20ModelFootprint::grid=5F?= =?UTF-8?q?lease=5Frequest=20=E2=80=94=20serving=20demand=20=E2=86=92=20ca?= =?UTF-8?q?pacity=20LeaseRequest=20(grid-overflow=20bridge)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clean half of the grid-overflow seam. serving_plan reasons about MODEL RESIDENCY (can a peer hold model M's weights + per-lane KV at the served window, and how many lanes). capacity/ reasons about CONCURRENCY SPIKES (does a peer have a free lane RIGHT NOW — LeaseRequest{want_concurrency, spike_bytes}). They're orthogonal and compose — residency is the eligibility gate, concurrency is the right-now admission. Neither absorbs the other. grid_lease_request(served_window, demand_lanes) is the one-directional map from the serving side into the capacity side: demand_lanes → want_concurrency (floored at 1), and the prefill compute spike at the live served window → spike_bytes (prefill_compute_reserve(window, 1) — the transient the peer must have free to accept the hop, distinct from the resident weights+KV the residency gate already proved). No transport, no placement policy here — just the honest projection so the grid-overflow router can ask a residency-eligible peer for a concurrency lease. Test grid_lease_request_maps_demand_and_the_prefill_spike pins both mappings. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/cognition/serving_plan.rs | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/core/continuum-core/src/cognition/serving_plan.rs b/core/continuum-core/src/cognition/serving_plan.rs index e935227bc5..9651511c70 100644 --- a/core/continuum-core/src/cognition/serving_plan.rs +++ b/core/continuum-core/src/cognition/serving_plan.rs @@ -260,6 +260,29 @@ impl ModelFootprint { self.resident_bytes(served_window, lanes) .saturating_add(self.prefill_compute_reserve(served_window, lanes)) } + + /// The capacity-fabric [`LeaseRequest`](crate::capacity::LeaseRequest) for serving + /// this model at `served_window` with `demand_lanes` concurrent minds — the bridge + /// from the serving plan's MODEL-RESIDENCY view (weights + per-lane KV) to the grid's + /// CONCURRENCY-SPIKE view, so [`GridPlacementPolicy`](crate::capacity::grid) can place + /// overflow lanes onto peers (#180 grid spill / [[frontier-is-a-scaling-question-over-misfits-not-a-capability-question]]). + /// `want_concurrency` = the minds that want a lane; `spike_bytes` = ONE lane's transient + /// prefill compute buffer at this window (the term the 2026-07-14 OOM turned on), so a + /// peer is only offered a spill lane it can actually hold. NOTE: this sizes the + /// CONCURRENCY spike only; a peer must ALSO already hold this model resident — that + /// residency gate is the gossip-side half of the routing (owned with the grid snapshot), + /// NOT this pure mapping. + pub fn grid_lease_request( + &self, + served_window: u32, + demand_lanes: u32, + ) -> crate::capacity::LeaseRequest { + crate::capacity::LeaseRequest { + consumer: self.model_id.clone(), + want_concurrency: demand_lanes.max(1), + spike_bytes: self.prefill_compute_reserve(served_window, 1), + } + } } /// The serving decision for this host. @@ -801,6 +824,26 @@ mod tests { // more lanes fit. OOM-safe — window_for still bounds it, so a higher ceiling only raises // the cap toward the budget bound, never past it. This is the "stop setting it in stone // at launch" fix that opens the elastic-lease path. + // what this catches: the serving→grid bridge (#180 spill) — a model footprint maps to a + // capacity-fabric LeaseRequest whose want_concurrency is the demanded lanes and whose + // spike_bytes is ONE lane's transient prefill compute reserve at the served window (the + // 2026-07-14 OOM term), so GridPlacementPolicy offers a peer only a spill lane it can hold. + #[test] + fn grid_lease_request_maps_demand_and_the_prefill_spike() { + let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); + let window = 16_384; + let lease = devstral.grid_lease_request(window, 3); + assert_eq!(lease.consumer, "devstral-24b"); + assert_eq!(lease.want_concurrency, 3, "want_concurrency = demanded lanes"); + assert_eq!( + lease.spike_bytes, + devstral.prefill_compute_reserve(window, 1), + "spike_bytes = ONE lane's prefill compute reserve at the served window" + ); + // Zero demand floors at one lane — never a degenerate 0-concurrency lease. + assert_eq!(devstral.grid_lease_request(window, 0).want_concurrency, 1); + } + #[test] fn demand_ceiling_is_elastic_grows_for_a_hard_task_shrinks_for_a_simple_one() { let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; From e5267104d2ebe919e4cae16edea8664dbb320be3 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:30:20 -0500 Subject: [PATCH 14/55] =?UTF-8?q?feat(capacity):=20ModelResidencyView=20?= =?UTF-8?q?=E2=80=94=20the=20residency=20eligibility=20gate=20for=20grid?= =?UTF-8?q?=20overflow=20(governor=20consumer=20slice=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grid-overflow routing (spill a persona's generation to a peer) has TWO orthogonal gates that COMPOSE, never absorb each other (settled with BigMama 2026-07-27): (1) RESIDENCY = eligibility — does the peer already hold model M resident? If not, accepting the hop forces a cold full-weights load (seconds to minutes), which defeats overflowing for speed. So the fast path is eligible only for peers that already hold M. THIS module. (2) CONCURRENCY = right-now admission — does the peer have a free lane? capacity/grid (LeaseRequest, LocalFirstFitPolicy). Unchanged. The one crossing point between the two abstractions is ModelFootprint::grid_lease_request (serving demand -> LeaseRequest) — one bridge, not two half-bridges. Residency deliberately does NOT live on PeerCapacity (that would blur the concurrency abstraction with a residency fact); it lives here as ModelResidencyView, keyed on Uuid exactly like gossip's capacity ledger. The governor COMPOSES the two at the placement filter: residency_eligible() returns a SMALLER snapshot (local untouched — the overflowing node holds M by definition; peers filtered to those holding M), and the unchanged capacity policy places on the survivors and applies reachability itself. Two concerns, composed at exactly one point, neither absorbed. Latest-wins replace (not merge) so a model paged OUT stops being eligible on the next beacon — a merge would resurrect evicted models and route a hop to a peer that no longer holds it. 3 tests: eligibility filter, latest-wins replace, and the residency->capacity compose end-to-end (resident+reachable gets lanes; resident+unreachable reclaimed by place(); non-resident absent from placement). Zero blast radius — new file, no existing struct touched. Next slice: populate the view from a residency beacon (gossip wiring, coordinated with BigMama — piggyback CapacityOffer vs a separate slower-cadence stream). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/capacity/mod.rs | 1 + .../src/capacity/model_residency.rs | 217 ++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 core/continuum-core/src/capacity/model_residency.rs diff --git a/core/continuum-core/src/capacity/mod.rs b/core/continuum-core/src/capacity/mod.rs index c9fbbb0096..5ab9baaeed 100644 --- a/core/continuum-core/src/capacity/mod.rs +++ b/core/continuum-core/src/capacity/mod.rs @@ -31,6 +31,7 @@ pub mod expert_residency; pub mod gossip; pub mod grid; pub mod lease; +pub mod model_residency; pub mod moe_serving; pub mod placement; pub mod recursion_depth; diff --git a/core/continuum-core/src/capacity/model_residency.rs b/core/continuum-core/src/capacity/model_residency.rs new file mode 100644 index 0000000000..22bd547b23 --- /dev/null +++ b/core/continuum-core/src/capacity/model_residency.rs @@ -0,0 +1,217 @@ +//! Model residency across the grid — WHICH peer currently holds WHICH model resident. The +//! ELIGIBILITY half of grid-overflow routing; the CONCURRENCY half is [`super::grid`]. +//! +//! ## Why this is a separate abstraction (orthogonal, never absorbed) +//! +//! `serving_plan` reasons about MODEL RESIDENCY — can a node hold model M's weights + per-lane +//! KV at the served window. `capacity/grid` reasons about CONCURRENCY SPIKES — does a node have +//! a free lane RIGHT NOW ([`super::LeaseRequest`]). Settled with BigMama 2026-07-27: these are +//! **orthogonal and compose** — neither should absorb the other, the mapping is the only +//! crossing point ([`super::serving_plan`]'s `grid_lease_request` is that one bridge). So +//! residency does NOT belong on [`super::grid::PeerCapacity`] (that would blur the concurrency +//! abstraction with a residency fact); it lives here, and the governor **composes** the two at +//! the placement filter. +//! +//! ## Why residency gates grid overflow +//! +//! A grid-overflow hop routes a persona's generation to a peer. If that peer already holds M +//! resident, the hop is fast — it needs only a free lane (the concurrency check). If it merely +//! has free VRAM but NOT M, accepting the hop forces a cold full-weights load (seconds to +//! minutes for a large model) — which defeats the entire point of overflowing for speed. So the +//! fast overflow path is eligible only for peers that already hold M. A peer with unknown +//! residency (never beaconed) is NOT eligible for the fast path — conservative by construction, +//! same spirit as [`super::residency_detect`] never claiming a promotion is faster than it is. +//! +//! ## The compose point +//! +//! [`ModelResidencyView::residency_eligible`] takes a live [`GridSnapshot`] and returns a +//! SMALLER snapshot — local device untouched (the overflowing node holds M by definition; it's +//! the one serving it), peers filtered to those holding M. The unchanged capacity placement +//! policy ([`super::grid::LocalFirstFitPolicy`]) then runs on that smaller snapshot: it never +//! learns about residency, it just sees fewer peers. Reachability stays the policy's job — an +//! unreachable-but-resident peer survives this filter and is dropped downstream by `place()`, +//! keeping the two concerns cleanly separate. + +use std::collections::{HashMap, HashSet}; + +use uuid::Uuid; + +use super::grid::GridSnapshot; +use crate::identity::PeerId; + +/// Per-peer set of model ids that peer currently holds resident, folded from residency beacons. +/// +/// Keyed on the peer's `Uuid` (via [`PeerId::as_uuid`]) — the same choice +/// [`super::gossip::GridCapacityLedger`] makes for capacity offers, so residency and capacity +/// index the grid identically. A peer absent from the map has UNKNOWN residency (never +/// beaconed), which [`Self::holds`] reports as `false`: not eligible for the fast overflow path. +#[derive(Debug, Clone, Default)] +pub struct ModelResidencyView { + by_peer: HashMap>, +} + +impl ModelResidencyView { + pub fn new() -> Self { + Self::default() + } + + /// Record (latest-wins) the full set of models a peer currently holds resident. Latest-wins + /// because residency is a live fact — a model paged out is no longer held, so a fresh beacon + /// REPLACES the peer's set rather than merging (a merge would resurrect evicted models). + pub fn set_resident(&mut self, peer: PeerId, models: I) + where + I: IntoIterator, + S: Into, + { + self.by_peer + .insert(peer.as_uuid(), models.into_iter().map(Into::into).collect()); + } + + /// Does this peer currently hold `model_id` resident? `false` for a peer that never beaconed + /// (unknown residency) — the conservative default that keeps the fast overflow path honest. + pub fn holds(&self, peer: &PeerId, model_id: &str) -> bool { + self.by_peer + .get(&peer.as_uuid()) + .is_some_and(|set| set.contains(model_id)) + } + + /// Number of peers with a known residency beacon — probe surface, mirrors + /// [`super::gossip::GridCapacityLedger::heard_count`]. + pub fn known_peers(&self) -> usize { + self.by_peer.len() + } + + /// Compose this residency view with a live capacity snapshot for `model_id`: keep the local + /// device (the overflowing node holds M by definition) and keep only peers that hold M + /// resident. The returned snapshot feeds the UNCHANGED capacity placement policy — concurrency + /// logic never learns about residency, it just sees a shorter peer list. Reachability is NOT + /// applied here (that stays the policy's job downstream), so an unreachable-but-resident peer + /// survives this filter and is reclaimed by `place()`. Orthogonal, composed at exactly one + /// point, neither abstraction absorbing the other. + pub fn residency_eligible(&self, snapshot: &GridSnapshot, model_id: &str) -> GridSnapshot { + let mut eligible = snapshot.clone(); + eligible.peers.retain(|peer| self.holds(&peer.peer, model_id)); + eligible + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capacity::grid::{GridPlacementPolicy, GridSnapshot, LocalFirstFitPolicy, PeerCapacity}; + use crate::capacity::{DeviceCapacity, LeaseRequest}; + + const GB: u64 = 1024 * 1024 * 1024; + + fn peer_id(n: u128) -> PeerId { + PeerId::from_uuid(Uuid::from_u128(n)) + } + + fn dev(free_gb: u64) -> DeviceCapacity { + DeviceCapacity { + gpu_total_bytes: 80 * GB, + gpu_free_bytes_live: free_gb * GB, + system_ram_free_bytes: 64 * GB, + } + } + + fn peer(n: u128, free_gb: u64, reachable: bool) -> PeerCapacity { + PeerCapacity { + peer: peer_id(n), + capacity: dev(free_gb), + reachable, + } + } + + // what this catches: the residency ELIGIBILITY gate. Only peers that beaconed the model as + // resident survive the filter; a peer with plenty of free VRAM but NOT holding M is dropped + // (routing to it would force a cold full-weights load — the slow path the gate exists to + // avoid), and a peer that never beaconed at all (unknown residency) is dropped too. The local + // device is always kept — the overflowing node holds M by definition. + #[test] + fn only_peers_holding_the_model_are_eligible() { + let snap = GridSnapshot { + local: dev(2), + peers: vec![ + peer(1, 40, true), // holds qwen-coder + peer(2, 40, true), // holds something else, NOT qwen-coder + peer(3, 40, true), // never beaconed — unknown residency + ], + }; + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder", "embed-small"]); + view.set_resident(peer_id(2), ["llama-70b"]); + // peer 3 intentionally not recorded. + + let eligible = view.residency_eligible(&snap, "qwen-coder"); + assert_eq!(eligible.local, snap.local, "local device is always kept — it holds M"); + assert_eq!(eligible.peers.len(), 1, "only the peer holding qwen-coder survives"); + assert_eq!(eligible.peers[0].peer.as_uuid(), Uuid::from_u128(1)); + } + + // what this catches: latest-wins REPLACE, not merge. A peer that paged qwen-coder OUT (its + // fresh beacon lists only what it still holds) must stop being eligible — a merge would + // resurrect the evicted model and route a hop to a peer that no longer has it. + #[test] + fn fresh_beacon_replaces_so_evicted_models_stop_being_eligible() { + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder", "llama-70b"]); + assert!(view.holds(&peer_id(1), "qwen-coder")); + + // Peer paged qwen-coder out; next beacon lists only llama-70b. + view.set_resident(peer_id(1), ["llama-70b"]); + assert!(!view.holds(&peer_id(1), "qwen-coder"), "evicted model must not linger"); + assert!(view.holds(&peer_id(1), "llama-70b")); + } + + // what this catches: the COMPOSE contract end-to-end — residency filters WHO is eligible, + // then the unchanged capacity policy places lanes on the survivors and applies reachability + // itself. A resident+reachable peer gets lanes; a resident+UNREACHABLE peer survives the + // residency filter (reachability isn't residency's job) but is reclaimed by place(); a + // non-resident peer is absent from placement entirely. Two orthogonal gates, composed. + #[test] + fn residency_then_capacity_policy_compose() { + let snap = GridSnapshot { + local: dev(1), // no local room — force the spill onto peers + peers: vec![ + peer(1, 40, true), // resident + reachable → should get lanes + peer(2, 40, false), // resident + UNREACHABLE → reclaimed by place() + peer(3, 40, true), // reachable but NOT resident → filtered out before place() + ], + }; + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder"]); + view.set_resident(peer_id(2), ["qwen-coder"]); + // peer 3 holds nothing relevant. + + let eligible = view.residency_eligible(&snap, "qwen-coder"); + assert_eq!(eligible.peers.len(), 2, "peers 1 & 2 are resident; peer 3 filtered out"); + + // Small spike so many lanes fit per peer — we're testing WHO gets lanes, not how many. + let req = LeaseRequest { + consumer: "qwen-coder".into(), + want_concurrency: 4, + spike_bytes: GB, + }; + let placement = LocalFirstFitPolicy { safety_margin_bytes: GB }.place(&eligible, &req); + + // The reachable resident peer carries the remote lanes; the unreachable one gets none. + let peer1_lanes: u32 = placement + .remote + .iter() + .filter(|(p, _)| p.as_uuid() == Uuid::from_u128(1)) + .map(|(_, n)| *n) + .sum(); + let peer2_lanes: u32 = placement + .remote + .iter() + .filter(|(p, _)| p.as_uuid() == Uuid::from_u128(2)) + .map(|(_, n)| *n) + .sum(); + let peer3_present = placement.remote.iter().any(|(p, _)| p.as_uuid() == Uuid::from_u128(3)); + + assert!(peer1_lanes > 0, "resident + reachable peer must carry overflow lanes"); + assert_eq!(peer2_lanes, 0, "resident but unreachable peer is reclaimed by place()"); + assert!(!peer3_present, "non-resident peer never reaches placement"); + } +} From 4084e3ef7f3e277885b34d506de60486e6b159e6 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:43:01 -0500 Subject: [PATCH 15/55] =?UTF-8?q?feat(capacity):=20ResidencyBeacon=20+=20R?= =?UTF-8?q?esidencyLedger=20=E2=80=94=20the=20receive/project=20half=20of?= =?UTF-8?q?=20the=20residency=20beacon=20(governor=20consumer=20slice=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The residency sibling of capacity/gossip's CapacityOffer/GridCapacityLedger, with the same identity + freshness discipline, a different (slower) cadence, and its own payload: - ResidencyBeacon (wire): the model ids a node holds resident + a sender timestamp. Rides its OWN grid_residency EphemeralCoalesced envelope — residency changes on model page-in/out (minute-scale), NOT the 10s capacity beat, so coupling them would either over-publish residency or under-refresh capacity. Peer identity is the WIRE's, never the payload's — a peer cannot beacon residency on another's behalf. - ResidencyLedger + global_residency_ledger(): folds heard beacons (latest-per-peer wins), projects a ModelResidencyView, evicts beacons silent past RESIDENCY_EVICTION_WINDOW_MS, excludes the node's own echo (local residency is its own serving truth). view() is the exact residency analogue of GridCapacityLedger::snapshot(). Eviction is GENEROUS (6× the capacity window) precisely because the two abstractions stay orthogonal: residency is sticky, and the COMPOSED capacity snapshot already gates reachability — so a residency reading never has to prove liveness itself (that would blur residency into concurrency). A long-silent peer falls back to UNKNOWN residency (not asserted-resident), keeping the fast overflow path honest. 3 more tests (6 total in the module): heard-beacon-projects + own-echo-excluded (loopback), stale-evict-then-fresh-restore, serde round-trip (camelCase wire, catches field drift). Next slice: the publish + inbound-fold wiring — a GridResidencyModule mirroring GridCapacityModule (build the beacon from the serving plan's resident set, broadcast grid_residency) + the inbound_attach fold into global_residency_ledger(). Then a node advertises its residency and residency_eligible() runs on live grid data. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/capacity/model_residency.rs | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/core/continuum-core/src/capacity/model_residency.rs b/core/continuum-core/src/capacity/model_residency.rs index 22bd547b23..2b24475511 100644 --- a/core/continuum-core/src/capacity/model_residency.rs +++ b/core/continuum-core/src/capacity/model_residency.rs @@ -33,7 +33,10 @@ //! keeping the two concerns cleanly separate. use std::collections::{HashMap, HashSet}; +use std::sync::OnceLock; +use dashmap::DashMap; +use serde::{Deserialize, Serialize}; use uuid::Uuid; use super::grid::GridSnapshot; @@ -95,6 +98,95 @@ impl ModelResidencyView { } } +/// One node's residency beacon — the wire payload advertising which models it currently holds +/// resident. Rides its OWN `grid_residency` `EphemeralCoalesced` envelope, separate from +/// [`super::gossip::CapacityOffer`]'s `grid_capacity`: residency changes on model page-in/out +/// (minute-scale), not the 10s capacity beat, so coupling them would either over-publish +/// residency or under-refresh capacity. Names + timestamp only; peer identity is the WIRE's +/// (the transcript event's authenticated peer id), never the payload's — same rule as +/// `CapacityOffer`, so a peer cannot beacon residency on another's behalf. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResidencyBeacon { + /// Model ids this node holds resident (warm) RIGHT NOW — its base plus any co-resident. + pub resident_models: Vec, + /// Sender clock when the reading was taken (ms since epoch). Displayed, not trusted: + /// freshness is judged by RECEIVER clock at hear-time, exactly like `CapacityOffer.at_ms`. + pub at_ms: u64, +} + +/// A heard beacon + the receiver-clock instant it arrived (the freshness anchor). +#[derive(Debug, Clone)] +struct HeardBeacon { + models: Vec, + heard_at_ms: u64, +} + +/// Beacons silent past this drop from the projected view entirely. GENEROUS versus capacity's +/// eviction because the two abstractions stay orthogonal: residency is STICKY (a model stays +/// resident across many capacity beats), and the COMPOSED capacity snapshot already gates +/// reachability — so a residency reading never has to prove liveness itself (that would blur +/// residency into concurrency). 6× the capacity eviction window: a peer whose residency we +/// haven't reheard in that long falls back to UNKNOWN (not asserted-resident), keeping the fast +/// overflow path honest rather than routing to a model a long-silent peer may have paged out. +pub const RESIDENCY_EVICTION_WINDOW_MS: u64 = 6 * super::gossip::EVICTION_WINDOW_MS; + +/// Process-global ledger of heard residency beacons, keyed by the WIRE's peer id — the +/// residency sibling of [`super::gossip::GridCapacityLedger`], same identity + freshness +/// discipline, different (slower) cadence and payload. +#[derive(Default)] +pub struct ResidencyLedger { + heard: DashMap, +} + +/// The one process-global residency ledger — the resource it mirrors (this node's view of who +/// holds what across the grid) is process-global, same granularity argument as +/// [`super::gossip::global_ledger`]. +pub fn global_residency_ledger() -> &'static ResidencyLedger { + static LEDGER: OnceLock = OnceLock::new(); + LEDGER.get_or_init(ResidencyLedger::default) +} + +impl ResidencyLedger { + /// Fold one heard beacon in (latest per peer wins — residency is a live fact). `from_peer` + /// is the transcript event's transport identity, never payload-declared. Returns `true` when + /// this peer is NEW to the ledger — the probe-on-join surface; steady re-beacons stay silent. + pub fn hear(&self, from_peer: Uuid, beacon: ResidencyBeacon, heard_at_ms: u64) -> bool { + self.heard + .insert( + from_peer, + HeardBeacon { models: beacon.resident_models, heard_at_ms }, + ) + .is_none() + } + + /// Project the ledger into a [`ModelResidencyView`] the governor composes with the capacity + /// snapshot, evicting beacons silent past [`RESIDENCY_EVICTION_WINDOW_MS`] as it goes. + /// `own_peer` is excluded — the local node's residency is its OWN serving truth (it holds M + /// by definition when it overflows), not a round-tripped beacon. + pub fn view(&self, own_peer: Uuid, now_ms: u64) -> ModelResidencyView { + let mut view = ModelResidencyView::new(); + self.heard.retain(|peer, heard| { + let age = now_ms.saturating_sub(heard.heard_at_ms); + if age > RESIDENCY_EVICTION_WINDOW_MS { + return false; // silent too long — residency falls back to unknown + } + if *peer != own_peer { + view.by_peer + .insert(*peer, heard.models.iter().cloned().collect()); + } + true + }); + view + } + + /// Number of peers currently on the ledger (self included if echoed) — probe surface, + /// mirrors [`super::gossip::GridCapacityLedger::heard_count`]. + pub fn heard_count(&self) -> usize { + self.heard.len() + } +} + #[cfg(test)] mod tests { use super::*; @@ -214,4 +306,70 @@ mod tests { assert_eq!(peer2_lanes, 0, "resident but unreachable peer is reclaimed by place()"); assert!(!peer3_present, "non-resident peer never reaches placement"); } + + fn beacon(models: &[&str], at_ms: u64) -> ResidencyBeacon { + ResidencyBeacon { + resident_models: models.iter().map(|s| s.to_string()).collect(), + at_ms, + } + } + + // what this catches: the beacon RECEIVE→PROJECT pipeline — a heard beacon projects into a + // ModelResidencyView that holds() reflects, and the node's OWN echoed beacon is excluded + // (the local node's residency is its own serving truth, arriving live, not a round-trip). + // This is the residency sibling of gossip's loopback contract. + #[test] + fn heard_beacon_projects_and_own_echo_is_excluded() { + let ledger = ResidencyLedger::default(); + let me = Uuid::from_u128(1); + let other = Uuid::from_u128(2); + ledger.hear(me, beacon(&["qwen-coder"], 1_000), 1_000); // our own beacon, round-tripped + ledger.hear(other, beacon(&["qwen-coder", "llama-70b"], 1_000), 1_000); + + let view = ledger.view(me, 2_000); + assert!(!view.holds(&peer_id(1), "qwen-coder"), "own echo excluded from the peer view"); + assert!(view.holds(&peer_id(2), "qwen-coder"), "the real peer's residency projects"); + assert!(view.holds(&peer_id(2), "llama-70b")); + assert_eq!(view.known_peers(), 1, "only the genuine peer, not the echo"); + } + + // what this catches: eviction after long silence — a peer we haven't reheard past the + // (generous) residency window falls back to UNKNOWN residency, so the fast overflow path + // won't route to a model that long-silent peer may since have paged out. A fresh beacon + // brings it right back (grow is first-class). + #[test] + fn stale_beacon_evicts_then_a_fresh_one_restores() { + let ledger = ResidencyLedger::default(); + let me = Uuid::from_u128(1); + let peer = Uuid::from_u128(2); + ledger.hear(peer, beacon(&["qwen-coder"], 0), 0); + assert!(ledger.view(me, 1_000).holds(&peer_id(2), "qwen-coder")); + + // Silent past the residency eviction window: gone from the view (unknown, not asserted). + let t_evict = RESIDENCY_EVICTION_WINDOW_MS + 1; + assert!( + !ledger.view(me, t_evict).holds(&peer_id(2), "qwen-coder"), + "long-silent peer's residency falls back to unknown" + ); + + // It beacons again: instantly back. + ledger.hear(peer, beacon(&["qwen-coder"], t_evict), t_evict); + assert!( + ledger.view(me, t_evict + 1).holds(&peer_id(2), "qwen-coder"), + "a returning peer's residency is adopted on its first fresh beacon" + ); + } + + // what this catches: the wire payload survives serde round-trip byte-for-byte (camelCase + // like every realtime inline payload). If resident_models renamed or at_ms narrowed, heard + // residency would silently drift from published — the grid would gate overflow on models + // nobody actually beaconed. + #[test] + fn beacon_round_trips_through_json() { + let b = beacon(&["qwen-coder", "embed-small"], 42); + let json = serde_json::to_string(&b).expect("serialize"); + assert!(json.contains("residentModels"), "camelCase wire field: {json}"); + let back: ResidencyBeacon = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(back, b, "beacon must survive the wire byte-for-byte"); + } } From dc806d5699e0fa08d1534fbc0dcaf06d6b99d913 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:51:44 -0500 Subject: [PATCH 16/55] =?UTF-8?q?feat(capacity):=20GridResidencyModule=20+?= =?UTF-8?q?=20grid=5Fresidency=20envelope=20=E2=80=94=20residency=20beacon?= =?UTF-8?q?=20publish/fold=20wiring=20(governor=20consumer=20slice=203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the residency-beacon loop end-to-end, mirroring the capacity gossip path (GridCapacityModule + inbound_attach fold) exactly: - GridResidencyModule (modules/grid_residency.rs): a Background ServiceModule that every RESIDENCY_PUBLISH_INTERVAL_MS reads the daemon's live serving plan (lock-free watch snapshot — the SAME source, no parallel probe) and broadcasts a ResidencyBeacon over airc as an EphemeralCoalesced grid_residency envelope. Today the resident set is [base_model_id]; a multi-model plan extends only current_beacon(). Honest silence when no plan is computed yet (nothing to advertise). Glass box speaks on change. - AircRealtimeSchema::GridResidency (airc/realtime.rs): the new schema variant, EphemeralCoalesced like GridCapacity. ts-rs binding regenerated (AircRealtimeSchema.ts). - inbound_attach fold: residency_beacon_from_envelope decoder + the else-if that folds a heard beacon into global_residency_ledger(), keyed on the WIRE's peer id — the orthogonal sibling of the capacity fold. Own echo lands here too (the single-node loopback proof). - Registered in ipc/mod.rs right after GridCapacityModule, fed serving_daemon.subscribe(). - RESIDENCY_PUBLISH_INTERVAL_MS (model_residency.rs) = eviction/12 — the same publish:eviction ratio capacity uses, on a slower beat (residency changes minute-scale). This completes the grid-overflow governor-consumer stack: a node now ADVERTISES which models it holds, every peer folds those beacons into a ModelResidencyView, and the governor composes it with the capacity snapshot (residency_eligible -> grid_lease_request -> place -> aircPeer hop). The live two-node routing smoke (BigMama's node up + serving) validates the cross-node generation — the milestone this stack was built for. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../continuum-core/src/airc/inbound_attach.rs | 41 +++++ core/continuum-core/src/airc/realtime.rs | 11 +- .../src/capacity/model_residency.rs | 7 + core/continuum-core/src/ipc/mod.rs | 8 + .../src/modules/grid_residency.rs | 166 ++++++++++++++++++ core/continuum-core/src/modules/mod.rs | 1 + .../typescript/airc/AircRealtimeSchema.ts | 2 +- 7 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 core/continuum-core/src/modules/grid_residency.rs diff --git a/core/continuum-core/src/airc/inbound_attach.rs b/core/continuum-core/src/airc/inbound_attach.rs index 061f5d5cd9..791685df7a 100644 --- a/core/continuum-core/src/airc/inbound_attach.rs +++ b/core/continuum-core/src/airc/inbound_attach.rs @@ -212,6 +212,31 @@ pub async fn publish_transcript_event( "first capacity offer heard from a grid peer", ); } + } else if let Some(beacon) = residency_beacon_from_envelope(&envelope) { + // Residency beacon (grid-overflow eligibility): fold the heard beacon into the + // process-global residency ledger, keyed on the WIRE's peer id — the orthogonal + // sibling of the capacity fold above. Our own echo lands here too (the loopback + // proof that publish→hear works before a second node exists). + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + let model_count = beacon.resident_models.len(); + let is_new = crate::capacity::model_residency::global_residency_ledger().hear( + event.peer_id.as_uuid(), + beacon, + now_ms, + ); + if is_new { + crate::probe!( + class = "grid.residency.heard", + from_peer = %event.peer_id.as_uuid(), + model_count = model_count, + heard_peers = + crate::capacity::model_residency::global_residency_ledger().heard_count(), + "first residency beacon heard from a grid peer", + ); + } } else if let Some((name, payload)) = chat_posted_from_envelope(&envelope, event) { crate::probe!( class = "airc.chat.projected", @@ -284,6 +309,22 @@ fn capacity_offer_from_envelope( serde_json::from_value(payload.inline.clone()?).ok() } +/// Decode a `grid_residency` envelope's inline payload into a [`ResidencyBeacon`]. +/// Returns `None` for any other envelope — the residency sibling of +/// [`capacity_offer_from_envelope`], same honest schema gate. +fn residency_beacon_from_envelope( + envelope: &AircRealtimeEnvelope, +) -> Option { + let crate::airc::realtime::AircRealtimePayload::ExistingSchema { payload } = &envelope.payload + else { + return None; + }; + if payload.schema != crate::airc::realtime::AircRealtimeSchema::GridResidency { + return None; + } + serde_json::from_value(payload.inline.clone()?).ok() +} + /// Project a plain airc chat message into the THIN `chat:posted` bus /// payload the positron chat projection consumes (`AircChatPosted` in /// `ipc/positron_source.rs`). Returns `None` for any non-message event diff --git a/core/continuum-core/src/airc/realtime.rs b/core/continuum-core/src/airc/realtime.rs index 1d412b1a79..8606c64a30 100644 --- a/core/continuum-core/src/airc/realtime.rs +++ b/core/continuum-core/src/airc/realtime.rs @@ -53,6 +53,13 @@ pub enum AircRealtimeSchema { /// presence-of-compute. EphemeralCoalesced: latest wins, never replayed /// (a stale capacity reading is a lie). GridCapacity, + /// A node's residency beacon (`capacity::model_residency::ResidencyBeacon`) — + /// which models it holds resident, the grid-overflow ELIGIBILITY signal (a peer + /// is only a fast overflow target for a model it already holds). Orthogonal to + /// GridCapacity (concurrency): the governor composes the two. EphemeralCoalesced + /// like capacity, but published on a slower cadence — residency changes on model + /// page-in/out (minute-scale), not the 10s capacity beat. + GridResidency, } /// Handle to a payload already defined by a Continuum schema. @@ -458,7 +465,9 @@ impl AircRealtimePayload { | AircRealtimeSchema::LiveKitBridgeEvent => AircRealtimeDelivery::Control, // Capacity offers are presence-of-compute: latest wins, never // replayed — a stale reading must not outlive its freshness. - AircRealtimeSchema::GridCapacity => AircRealtimeDelivery::EphemeralCoalesced, + AircRealtimeSchema::GridCapacity | AircRealtimeSchema::GridResidency => { + AircRealtimeDelivery::EphemeralCoalesced + } _ => AircRealtimeDelivery::Durable, }, Self::Presence { event } => event.delivery(), diff --git a/core/continuum-core/src/capacity/model_residency.rs b/core/continuum-core/src/capacity/model_residency.rs index 2b24475511..96cccc0f71 100644 --- a/core/continuum-core/src/capacity/model_residency.rs +++ b/core/continuum-core/src/capacity/model_residency.rs @@ -131,6 +131,13 @@ struct HeardBeacon { /// overflow path honest rather than routing to a model a long-silent peer may have paged out. pub const RESIDENCY_EVICTION_WINDOW_MS: u64 = 6 * super::gossip::EVICTION_WINDOW_MS; +/// The residency beacon heartbeat — deliberately SLOWER than capacity's 10s +/// ([`super::gossip::PUBLISH_INTERVAL_MS`]) because residency changes on model page-in/out +/// (minute-scale), not the free-VRAM beat. 12 beats fit inside +/// [`RESIDENCY_EVICTION_WINDOW_MS`] — the same publish:eviction ratio capacity gossip uses, +/// so a couple of dropped beacons never evicts a still-resident peer. +pub const RESIDENCY_PUBLISH_INTERVAL_MS: u64 = RESIDENCY_EVICTION_WINDOW_MS / 12; + /// Process-global ledger of heard residency beacons, keyed by the WIRE's peer id — the /// residency sibling of [`super::gossip::GridCapacityLedger`], same identity + freshness /// discipline, different (slower) cadence and payload. diff --git a/core/continuum-core/src/ipc/mod.rs b/core/continuum-core/src/ipc/mod.rs index d314ecb1f9..f50889aa23 100644 --- a/core/continuum-core/src/ipc/mod.rs +++ b/core/continuum-core/src/ipc/mod.rs @@ -1782,6 +1782,14 @@ pub fn start_server( resource_daemon.clone(), default_room, ))); + // Grid residency beacon (grid-overflow eligibility, slice 3): advertise which models + // this node holds resident on a slower cadence, folded by inbound_attach into + // capacity::model_residency::global_residency_ledger. Reads the SAME serving plan the + // daemon computes (watch snapshot, no parallel probe); orthogonal sibling of capacity. + runtime.register(Arc::new(crate::modules::grid_residency::GridResidencyModule::new( + serving_daemon.subscribe(), + default_room, + ))); let continuum_root = crate::modules::persona_instance_manager::resolve_continuum_root(); let daemon_socket_for_rag_inspect = daemon_socket.clone(); let registry = crate::persona::PersonaAircRuntimeRegistry::new(); diff --git a/core/continuum-core/src/modules/grid_residency.rs b/core/continuum-core/src/modules/grid_residency.rs new file mode 100644 index 0000000000..3f5fb81f0d --- /dev/null +++ b/core/continuum-core/src/modules/grid_residency.rs @@ -0,0 +1,166 @@ +//! GridResidencyModule — this node's residency-beacon gossip publisher (grid-overflow slice 3). +//! +//! The residency sibling of [`super::grid_capacity::GridCapacityModule`]. Every +//! [`RESIDENCY_PUBLISH_INTERVAL_MS`] tick it reads the SAME live serving plan the daemon +//! already computes (one source, no parallel probe) and broadcasts a [`ResidencyBeacon`] over +//! airc as an `EphemeralCoalesced` `grid_residency` realtime envelope: which models this node +//! holds resident — the grid-overflow ELIGIBILITY signal. The receive half lives in +//! [`crate::airc::inbound_attach`], which folds heard beacons (our own echo included — the +//! loopback proof) into [`crate::capacity::model_residency::global_residency_ledger`], whose +//! `view()` IS the `ModelResidencyView` the governor composes with the capacity snapshot. +//! +//! ## Why a SEPARATE module + slower cadence +//! +//! Residency and capacity are orthogonal (settled with BigMama 2026-07-27): capacity is +//! free-VRAM RIGHT NOW (10s beat, [`super::grid_capacity`]); residency is which models are warm +//! (minute-scale, changes only on page-in/out). Coupling them onto one envelope would either +//! over-publish residency or under-refresh capacity. Same module shape as the style guide +//! mandates — no new tokio task, the runtime tick drives it; the plan read is a lock-free +//! `watch` snapshot; the publish is one small envelope through the existing +//! `airc/realtime-publish` command surface (the same path capacity + chat ride). +//! +//! ## Today's local residency = the base model +//! +//! `ServingPlan` carries a single `base_model_id` + a `resident_models` COUNT (not a per-id +//! set), so today this node's honest resident set is `[base_model_id]`. When multi-model +//! residency lands (a per-id resident list on the plan), only [`Self::current_beacon`] changes +//! — the wire, ledger, and governor compose are already model-set-shaped. + +use std::any::Any; +use std::sync::Mutex; +use std::time::Duration; + +use async_trait::async_trait; +use serde_json::{json, Value}; +use tokio::sync::watch; + +use crate::capacity::model_residency::{ + global_residency_ledger, ResidencyBeacon, RESIDENCY_PUBLISH_INTERVAL_MS, +}; +use crate::cognition::serving_plan::ServingPlan; +use crate::runtime::{ + CommandExecutor, CommandResult, LateBound, ModuleConfig, ModulePriority, ServiceModule, +}; +use airc_core::RoomId; +use std::sync::Arc; + +pub struct GridResidencyModule { + /// Lock-free live snapshot of the daemon's serving plan — the SAME source the prefill + /// valve and serving control derive from (no parallel probe). + plan_rx: watch::Receiver>, + /// The node's discovered default room — where the grid rendezvous happens today. + room: RoomId, + executor_slot: Arc>, + /// Last published resident set — the glass box speaks on CHANGE, not on every beat (a + /// steady residency is silence). `None` = never published, so the first real plan speaks. + last_published: Mutex>>, +} + +impl GridResidencyModule { + pub fn new(plan_rx: watch::Receiver>, room: RoomId) -> Self { + Self { + plan_rx, + room, + executor_slot: Arc::new(LateBound::new("grid-residency::executor")), + last_published: Mutex::new(None), + } + } + + /// Build this node's residency beacon from the live serving plan. `None` when no plan is + /// computed yet (nothing resident to advertise) — honest silence, never a fabricated set. + /// Today the resident set is `[base_model_id]`; a multi-model plan extends only this line. + fn current_beacon(&self) -> Option { + let plan = self.plan_rx.borrow().clone()?; + Some(ResidencyBeacon { + resident_models: vec![plan.base_model_id], + at_ms: now_ms(), + }) + } +} + +fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[async_trait] +impl ServiceModule for GridResidencyModule { + fn config(&self) -> ModuleConfig { + ModuleConfig { + name: "grid-residency", + priority: ModulePriority::Background, + command_prefixes: &[], + event_subscriptions: &[], + needs_dedicated_thread: false, + max_concurrency: 0, + tick_interval: Some(Duration::from_millis(RESIDENCY_PUBLISH_INTERVAL_MS)), + } + } + + async fn initialize(&self, _ctx: &crate::runtime::ModuleContext) -> Result<(), String> { + Ok(()) + } + + async fn handle_command(&self, command: &str, _params: Value) -> Result { + Err(format!( + "grid-residency has no command surface — '{command}' (beacons publish on the module \ + tick; read the grid via capacity::model_residency::global_residency_ledger)" + )) + } + + async fn tick(&self) -> Result<(), String> { + // Boot ordering: a beat or two may fire before start_server installs the executor — + // transient, skip (the next beat publishes). Same guaranteed-installed path as capacity. + let Some(executor) = self.executor_slot.cloned() else { + return Ok(()); + }; + let Some(beacon) = self.current_beacon() else { + // No serving plan yet — nothing resident to advertise. Honest silence. + return Ok(()); + }; + + let envelope = json!({ + "eventId": uuid::Uuid::new_v4().to_string(), + "roomId": self.room.as_uuid().to_string(), + "sourceId": "grid-residency", + "createdAtMs": beacon.at_ms, + "delivery": "ephemeral_coalesced", + "payload": { + "kind": "existing_schema", + "payload": { + "schema": "grid_residency", + "inline": serde_json::to_value(&beacon) + .map_err(|e| format!("residency beacon encode failed: {e}"))?, + } + }, + }); + executor + .execute_json("airc/realtime-publish", json!({ "envelope": envelope })) + .await + .map_err(|e| format!("grid-residency beacon publish failed: {e}"))?; + + // Glass box: speak on change (resident set differs), silent on a steady residency. + let mut last = self.last_published.lock().map_err(|e| format!("residency lock: {e}"))?; + if last.as_deref() != Some(beacon.resident_models.as_slice()) { + crate::probe!( + class = "grid.residency.beacon", + models = ?beacon.resident_models, + heard_peers = global_residency_ledger().heard_count(), + "residency beacon published to the grid", + ); + *last = Some(beacon.resident_models); + } + Ok(()) + } + + fn install_executor(&self, executor: Arc) { + self.executor_slot.install(executor); + } + + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/core/continuum-core/src/modules/mod.rs b/core/continuum-core/src/modules/mod.rs index f146dbe500..9323d8f772 100644 --- a/core/continuum-core/src/modules/mod.rs +++ b/core/continuum-core/src/modules/mod.rs @@ -46,6 +46,7 @@ pub mod gpu; pub mod grant_issuance; pub mod grid; pub mod grid_capacity; +pub mod grid_residency; pub mod health; pub mod hippocampus; pub mod inference_coordinator_module; diff --git a/protocol/typescript/airc/AircRealtimeSchema.ts b/protocol/typescript/airc/AircRealtimeSchema.ts index f5040ab98c..a8efd11857 100644 --- a/protocol/typescript/airc/AircRealtimeSchema.ts +++ b/protocol/typescript/airc/AircRealtimeSchema.ts @@ -3,4 +3,4 @@ /** * Existing Continuum schema carried by an AIRC realtime envelope. */ -export type AircRealtimeSchema = "jtag_message" | "event_bridge_payload" | "grid_frame" | "live_kit_bridge_command" | "live_kit_bridge_event" | "chat_transcript" | "grid_capacity"; +export type AircRealtimeSchema = "jtag_message" | "event_bridge_payload" | "grid_frame" | "live_kit_bridge_command" | "live_kit_bridge_event" | "chat_transcript" | "grid_capacity" | "grid_residency"; From c5600bf6db37f0a0424d90338088bdd54b3f4fac Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 15:13:34 -0500 Subject: [PATCH 17/55] =?UTF-8?q?feat(capacity):=20route=5Fgrid=5Foverflow?= =?UTF-8?q?=20=E2=80=94=20remote-only=20overflow=20placement=20decision=20?= =?UTF-8?q?(governor=20consumer=20slice=204a)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DECISION half of the driver, pure + fully unit-tested; only the thin EFFECT half (the actual Commands.execute("ai/generate", {aircPeer}) hop) needs a live peer and lands at the two-node smoke. Overflow lanes are BY DEFINITION the ones ServingPlan.grid_overflow_lanes said couldn't fit locally, so their placement is REMOTE-ONLY — it must never touch LocalFirstFitPolicy's local-first >=1 floor (that floor is the local persona's OWN guaranteed lane, orthogonal to spillover; re-cramming there is the exact thrash the honest overflow signal exists to avoid). Confirmed with BigMama 2026-07-27. Two orthogonal gates, composed (never absorbed): 1. RESIDENCY (ModelResidencyView::residency_eligible) — a peer is a fast overflow target only for a model it ALREADY holds (else a cold full-weights load defeats the point). 2. CONCURRENCY (reachability + lanes_that_fit misfit-parts) — among reachable eligible peers, most-free-first, each capped by its OWN budget for the prefill spike. Unplaced lanes (no eligible+reachable peer could take them) are SURFACED in OverflowRouting.unplaced for the caller to queue/degrade on — never silently dropped ([[fallbacks-are-illegal-fail-loud]]). 4 tests: remote-only + residency/reachability gating, unplaced-surfaced-not-dropped, zero-overflow no-op, most-free-first spill spread. This completes the pure governor-consumer decision path. Remaining (slice 4b, at the live two-node smoke): read grid_overflow_lanes from the live plan, build the lease via footprint.grid_lease_request, call route_grid_overflow, and execute the aircPeer hop per placed (peer, lanes) — the only piece needing a real peer to route to. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/capacity/grid_overflow.rs | 237 ++++++++++++++++++ core/continuum-core/src/capacity/mod.rs | 1 + 2 files changed, 238 insertions(+) create mode 100644 core/continuum-core/src/capacity/grid_overflow.rs diff --git a/core/continuum-core/src/capacity/grid_overflow.rs b/core/continuum-core/src/capacity/grid_overflow.rs new file mode 100644 index 0000000000..b34283d8d8 --- /dev/null +++ b/core/continuum-core/src/capacity/grid_overflow.rs @@ -0,0 +1,237 @@ +//! Grid-overflow routing — the DECISION half of the governor consumer: given a serving plan +//! that overflowed local capacity, decide which eligible peers take the overflow lanes. The +//! EFFECT half (the actual `Commands.execute("ai/generate", {aircPeer})` hop) is thin and lives +//! at the live seam; THIS is pure, deterministic, and fully unit-tested. +//! +//! ## Why overflow placement is REMOTE-ONLY (not [`super::grid::LocalFirstFitPolicy`]) +//! +//! [`super::grid::LocalFirstFitPolicy`] fills local first and floors local at `≥1` (a resident +//! model must be able to run one prefill). That floor is correct for a FRESH placement but +//! WRONG for overflow: overflow lanes are BY DEFINITION the ones that already could not fit +//! locally (`ServingPlan.grid_overflow_lanes = demand − local_lanes`). Placing them local-first +//! would re-cram the very lanes the planner just declared didn't fit — the thrash the honest +//! "over local capacity by N" signal exists to avoid. So overflow placement never touches local: +//! it spills ONLY to eligible remote peers. +//! +//! ## The two gates, composed (never absorbed) +//! +//! 1. RESIDENCY ([`ModelResidencyView::residency_eligible`]): a peer is a fast overflow target +//! only for a model it ALREADY holds — else the hop pays a cold full-weights load, defeating +//! the point. Filters the snapshot to peers holding the model. +//! 2. CONCURRENCY (the misfit-parts fit, [`super::lanes_that_fit`]): among residency-eligible + +//! REACHABLE peers, each takes at most what its OWN free budget fits for the prefill spike — +//! the same per-node-fit rule the single-device and grid policies run. +//! +//! Reachability is applied HERE (an unreachable-but-resident peer is a memory, not an offer), +//! composing cleanly with residency without either abstraction absorbing the other. +//! +//! ## Unplaced lanes are SURFACED, never dropped +//! +//! When no eligible peer can take a lane, it lands in [`OverflowRouting::unplaced`] — the honest +//! "the grid couldn't absorb N of your overflow" signal the caller queues or degrades on. Never +//! silently swallowed ([[fallbacks-are-illegal-fail-loud]]). + +use crate::identity::PeerId; + +use super::grid::GridSnapshot; +use super::lanes_that_fit; +use super::model_residency::ModelResidencyView; +use super::LeaseRequest; + +/// The routing decision for a plan's overflow lanes: where each lane lands (remote-only) plus +/// the honest count of lanes the reachable, residency-eligible grid could NOT absorb. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OverflowRouting { + /// Overflow lanes placed on named peers, most-free-first, each capped by its own fit. + pub remote: Vec<(PeerId, u32)>, + /// Overflow lanes no eligible+reachable peer could take — queue or degrade on these, + /// never drop them silently. + pub unplaced: u32, +} + +impl OverflowRouting { + /// Total overflow lanes actually placed on peers. + pub fn placed(&self) -> u32 { + self.remote.iter().map(|(_, n)| n).sum() + } +} + +/// Decide where a plan's overflow lanes run. `lease` is the demand→capacity projection the +/// serving side already built (`ModelFootprint::grid_lease_request(served_window, overflow_lanes)`): +/// `want_concurrency` = the overflow lane count, `spike_bytes` = the per-lane prefill transient. +/// `model_id` is what the overflowing node is serving (its `ServingPlan.base_model_id`) — the +/// residency key. REMOTE-ONLY by construction (see module docs): local is already saturated. +pub fn route_grid_overflow( + model_id: &str, + lease: &LeaseRequest, + residency: &ModelResidencyView, + snapshot: &GridSnapshot, + safety_margin_bytes: u64, +) -> OverflowRouting { + let want = lease.want_concurrency; + if want == 0 { + return OverflowRouting { remote: Vec::new(), unplaced: 0 }; + } + + // Gate 1 — residency: keep only peers that hold the model resident. + let eligible = residency.residency_eligible(snapshot, model_id); + + // Gate 2 — reachability + per-node fit: reachable eligible peers, most-free-first (fewest + // peers touched), each capped by its OWN budget for the prefill spike. + let mut reachable: Vec<_> = eligible.peers.iter().filter(|p| p.reachable).collect(); + reachable.sort_by(|a, b| { + b.capacity + .gpu_free_bytes_live + .cmp(&a.capacity.gpu_free_bytes_live) + }); + + let mut remaining = want; + let mut remote = Vec::new(); + for peer in reachable { + if remaining == 0 { + break; + } + let fit = lanes_that_fit( + peer.capacity.gpu_free_bytes_live, + safety_margin_bytes, + lease.spike_bytes, + ); + let take = fit.min(remaining); + if take > 0 { + remote.push((peer.peer, take)); + remaining -= take; + } + } + + // Whatever the reachable, residency-eligible grid couldn't absorb is surfaced, not dropped. + OverflowRouting { remote, unplaced: remaining } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capacity::grid::{GridSnapshot, PeerCapacity}; + use crate::capacity::DeviceCapacity; + use uuid::Uuid; + + const GB: u64 = 1024 * 1024 * 1024; + + fn peer_id(n: u128) -> PeerId { + PeerId::from_uuid(Uuid::from_u128(n)) + } + + fn dev(free_gb: u64) -> DeviceCapacity { + DeviceCapacity { + gpu_total_bytes: 80 * GB, + gpu_free_bytes_live: free_gb * GB, + system_ram_free_bytes: 64 * GB, + } + } + + fn peer(n: u128, free_gb: u64, reachable: bool) -> PeerCapacity { + PeerCapacity { + peer: peer_id(n), + capacity: dev(free_gb), + reachable, + } + } + + fn lease(want: u32, spike_gb: u64) -> LeaseRequest { + LeaseRequest { + consumer: "qwen-coder".into(), + want_concurrency: want, + spike_bytes: spike_gb * GB, + } + } + + fn view_holding(peers: &[(u128, &[&str])]) -> ModelResidencyView { + let mut v = ModelResidencyView::new(); + for (n, models) in peers { + v.set_resident(peer_id(*n), models.iter().map(|s| s.to_string())); + } + v + } + + // what this catches: overflow placement is REMOTE-ONLY and residency+reachability gated. + // Local is never assigned lanes (it's the saturated node that overflowed). Only a peer that + // holds the model AND is reachable takes lanes; a resident-but-unreachable peer and a + // reachable-but-not-resident peer both take nothing. + #[test] + fn overflow_routes_remote_only_to_reachable_resident_peers() { + let snap = GridSnapshot { + local: dev(1), // saturated — must never receive overflow lanes + peers: vec![ + peer(1, 40, true), // resident + reachable → takes lanes + peer(2, 40, false), // resident + UNREACHABLE → nothing + peer(3, 40, true), // reachable but NOT resident → nothing + ], + }; + let residency = view_holding(&[(1, &["qwen-coder"]), (2, &["qwen-coder"])]); + + let routing = route_grid_overflow("qwen-coder", &lease(2, 1), &residency, &snap, GB); + + assert_eq!(routing.remote.len(), 1, "only peer 1 is eligible + reachable"); + assert_eq!(routing.remote[0].0.as_uuid(), Uuid::from_u128(1)); + assert_eq!(routing.placed(), 2, "both overflow lanes fit on peer 1"); + assert_eq!(routing.unplaced, 0); + } + + // what this catches: unplaced lanes are SURFACED, never dropped. When the eligible grid + // can't fit all overflow lanes (one small peer, a big per-lane spike), the shortfall is + // reported so the caller queues/degrades — the honest "grid couldn't absorb N" signal. + #[test] + fn lanes_the_grid_cannot_absorb_are_surfaced_not_dropped() { + let snap = GridSnapshot { + local: dev(1), + peers: vec![peer(1, 10, true)], // ~10GB free, but each lane spikes 8GB + }; + let residency = view_holding(&[(1, &["qwen-coder"])]); + + // want 3 lanes, 8GB spike each: only 1 fits on the 10GB peer (net of 1GB margin). + let routing = route_grid_overflow("qwen-coder", &lease(3, 8), &residency, &snap, GB); + + assert_eq!(routing.placed(), 1, "only one lane fits the peer's budget"); + assert_eq!(routing.unplaced, 2, "the other two are surfaced, not silently dropped"); + } + + // what this catches: no overflow (want == 0) is a clean no-op — nothing placed, nothing + // unplaced. The common case (demand fit locally, grid_overflow_lanes == 0) costs nothing. + #[test] + fn zero_overflow_is_a_clean_noop() { + let snap = GridSnapshot { + local: dev(20), + peers: vec![peer(1, 40, true)], + }; + let residency = view_holding(&[(1, &["qwen-coder"])]); + + let routing = route_grid_overflow("qwen-coder", &lease(0, 1), &residency, &snap, GB); + + assert!(routing.remote.is_empty()); + assert_eq!(routing.unplaced, 0); + assert_eq!(routing.placed(), 0); + } + + // what this catches: spill spreads across peers most-free-first, each capped by its OWN fit + // (the misfit-parts rule) — 12 aggregate GB across three 4GB peers can't run a 6-lane + // placement if no single peer fits it, but distinct small lanes DO spread. Two reachable + // resident peers each take their share until demand is met. + #[test] + fn spill_spreads_across_peers_most_free_first() { + let snap = GridSnapshot { + local: dev(1), + peers: vec![ + peer(1, 20, true), // more free → sorted first + peer(2, 12, true), + ], + }; + let residency = view_holding(&[(1, &["qwen-coder"]), (2, &["qwen-coder"])]); + + // 4 lanes, 8GB spike: peer1 (20-1 margin=19 → 2 lanes), peer2 (12-1=11 → 1 lane) = 3, + // one unplaced. Peer 1 (most free) is filled before peer 2. + let routing = route_grid_overflow("qwen-coder", &lease(4, 8), &residency, &snap, GB); + + assert_eq!(routing.remote[0].0.as_uuid(), Uuid::from_u128(1), "most-free peer first"); + assert_eq!(routing.placed() + routing.unplaced, 4, "every lane accounted for"); + assert!(routing.placed() >= 3, "peers absorb what their own budgets fit"); + } +} diff --git a/core/continuum-core/src/capacity/mod.rs b/core/continuum-core/src/capacity/mod.rs index 5ab9baaeed..8a4b977aad 100644 --- a/core/continuum-core/src/capacity/mod.rs +++ b/core/continuum-core/src/capacity/mod.rs @@ -30,6 +30,7 @@ pub mod expert_reconcile; pub mod expert_residency; pub mod gossip; pub mod grid; +pub mod grid_overflow; pub mod lease; pub mod model_residency; pub mod moe_serving; From 979d93dc4d38f81b47cf19130b4045a6f9640162 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 15:26:02 -0500 Subject: [PATCH 18/55] =?UTF-8?q?feat(persona):=20grid-overflow=20effector?= =?UTF-8?q?=20seam=20=E2=80=94=20materialize=5Fadapters=20overflow=5Fadapt?= =?UTF-8?q?er=5Ffor=20override=20(governor=20consumer=20slice=204b-i)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The composition point where route_grid_overflow's decision becomes a real remote brain. materialize_adapters gains an `overflow_adapter_for(&profile, slot)` closure (same closure-DI shape as runtime_lookup / tool_executor_for): return Some(remote adapter) when the governor routed this persona off-box — her node is over capacity and a reachable peer holds her model — so her brain runs on that peer via AircRemoteInferenceAdapter; None → build the local adapter from the factory (the common case). This is the exact re-home seam the DeliberationModelBinding was designed for (its doc: re-home = "a new adapter / grid failover onto another node"). The remote adapter registers in the global provider registry by model_id just like the local one, so evaluate_response reaches it transparently — the persona doesn't know or care that her inference crosses the grid. host.rs passes `|_,_| None` for now (slice 4b-ii wires the live capacity + residency + airc closure at the ipc bootstrap, where the serving plan + airc handle live). Test overflow_effector_supplies_remote_adapter_and_bypasses_the_local_factory pins the contract: when the override supplies a slot's adapter, the local factory is NOT called for it (build_count == 1 of 2), both personas host, both warm. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/persona/host.rs | 4 + core/continuum-core/src/persona/supervisor.rs | 103 ++++++++++++++---- 2 files changed, 88 insertions(+), 19 deletions(-) diff --git a/core/continuum-core/src/persona/host.rs b/core/continuum-core/src/persona/host.rs index d42ad0df19..6731d7062d 100644 --- a/core/continuum-core/src/persona/host.rs +++ b/core/continuum-core/src/persona/host.rs @@ -328,6 +328,10 @@ impl PersonaSpawnSupervisor { as Arc }) }, + // Grid-overflow effector closure (slice 4b-ii wires the live capacity + + // residency + airc context here). Until then every persona builds her + // local adapter — no off-box routing, the pre-effector behavior. + |_profile, _slot| None, ) .await; diff --git a/core/continuum-core/src/persona/supervisor.rs b/core/continuum-core/src/persona/supervisor.rs index 66471a2d4d..d55a966b1c 100644 --- a/core/continuum-core/src/persona/supervisor.rs +++ b/core/continuum-core/src/persona/supervisor.rs @@ -499,6 +499,14 @@ pub async fn materialize_adapters( uuid::Uuid, ) -> Option>, + // The grid-overflow EFFECTOR (governor consumer slice 4b). Given a persona's + // profile + slot, returns `Some(remote adapter)` when the governor routed her + // off-box — her node is over local capacity and a reachable peer holds her model + // (`capacity::grid_overflow::route_grid_overflow`) — so her brain runs on that + // peer via `AircRemoteInferenceAdapter`. `None` → build the local adapter from the + // factory (the common case; demand fit locally). Closure DI keeps the supervisor + // decoupled from the capacity fabric + airc handle, same shape as the lookups above. + overflow_adapter_for: impl Fn(&PersonaInferenceProfile, usize) -> Option>, ) -> Vec> { let mut out = Vec::with_capacity(plans.len()); for (slot_index, plan) in plans.into_iter().enumerate() { @@ -525,16 +533,22 @@ pub async fn materialize_adapters( continue; } }; - let adapter = match factory.build_adapter(&profile).await { - Ok(a) => a, - Err(message) => { - out.push(Err(SupervisorError::AdapterFactory { - slot_index, - role: plan.role, - message, - })); - continue; - } + // Grid-overflow effector: if the governor routed this persona off-box, her + // adapter is the airc-remote one (her brain runs on the peer that holds her + // model). Else build the local adapter from the factory — the common case. + let adapter = match overflow_adapter_for(&profile, slot_index) { + Some(remote) => remote, + None => match factory.build_adapter(&profile).await { + Ok(a) => a, + Err(message) => { + out.push(Err(SupervisorError::AdapterFactory { + slot_index, + role: plan.role, + message, + })); + continue; + } + }, }; // Warm the adapter's KV-cache / kernels BEFORE the persona // enters her service loop. Per [[init-once-handle-then-lease-zero-copy-refs]] @@ -1106,7 +1120,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); assert_eq!(factory.build_count(), 2); @@ -1153,7 +1167,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); // Factory called exactly once — for the Ok row only. @@ -1188,7 +1202,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::always_fails("simulated factory rejection"); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); match &hosted[0] { @@ -1211,7 +1225,7 @@ mod tests { #[tokio::test] async fn empty_plans_yields_empty_hosted() { let factory = ScriptedPersonaAdapterFactory::heuristic(); - let hosted = materialize_adapters(vec![], &factory, |_| None, |_| None).await; + let hosted = materialize_adapters(vec![], &factory, |_| None, |_| None, |_, _| None).await; assert!(hosted.is_empty()); assert_eq!(factory.build_count(), 0); } @@ -1240,7 +1254,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); // `|_| None` here is the substrate-bug shape we're locking in: // the registry exists but doesn't contain this persona_id. - let hosted = materialize_adapters(plans, &factory, |_| None, |_| None).await; + let hosted = materialize_adapters(plans, &factory, |_| None, |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); // Factory MUST NOT be called when the runtime lookup fails — @@ -1300,7 +1314,7 @@ mod tests { as Arc) } }; - let hosted = materialize_adapters(plans, &factory, lookup, |_| None).await; + let hosted = materialize_adapters(plans, &factory, lookup, |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); // Factory ran exactly once — for Paige, not Pax. @@ -1345,7 +1359,7 @@ mod tests { let (factory, counts) = ScriptedPersonaAdapterFactory::heuristic_with_counters(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; // Both slots materialize cleanly. assert_eq!(hosted.len(), 2); @@ -1359,6 +1373,56 @@ mod tests { ); } + // what this catches: the grid-overflow EFFECTOR seam — when the governor routes a + // persona off-box, `overflow_adapter_for` supplies her adapter (the airc-remote one, + // her brain on a peer) and the LOCAL factory is NOT called for that slot. Here slot 0 + // is overflow-routed (override returns a stand-in remote), slot 1 is local. Both host; + // the factory builds ONLY slot 1 (build_count == 1); both adapters still warm. This is + // the composition point where route_grid_overflow's decision becomes a real remote brain. + #[tokio::test] + async fn overflow_effector_supplies_remote_adapter_and_bypasses_the_local_factory() { + use crate::ai::heuristic_adapter::HeuristicInferenceAdapter; + let plans = vec![ + MaterializedPersonaPlan { + role: RoleId::Helper, + instance: fake_instance("Paige"), + profile: Ok(fake_profile("Paige", "model-a")), // slot 0 → overflow-routed + }, + MaterializedPersonaPlan { + role: RoleId::Coder, + instance: fake_instance("Pax"), + profile: Ok(fake_profile("Pax", "model-b")), // slot 1 → local + }, + ]; + + let (factory, _counts) = ScriptedPersonaAdapterFactory::heuristic_with_counters(); + let hosted = materialize_adapters( + plans, + &factory, + StubAircCitizen::fresh_lookup(), + |_| None, + // Overflow effector: slot 0 is routed off-box → supply a stand-in "remote" + // adapter; every other slot stays local (None). + |_profile, slot| { + if slot == 0 { + Some(Arc::new(HeuristicInferenceAdapter::new()) as Arc) + } else { + None + } + }, + ) + .await; + + assert_eq!(hosted.len(), 2); + assert!(hosted.iter().all(|r| r.is_ok()), "both personas host (one remote, one local)"); + assert_eq!( + factory.build_count(), + 1, + "the local factory builds ONLY the non-overflow slot — the overflow slot's \ + adapter came from the effector, her brain runs on the peer" + ); + } + /// Warmup failure surfaces as `SupervisorError::AdapterWarmup` — /// the persona does NOT reach hosted state. Per [[no-fallbacks-ever]] /// an adapter that refuses to warm gets a typed slot failure; @@ -1375,7 +1439,7 @@ mod tests { "simulated warmup failure", ); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); match &hosted[0] { @@ -1411,7 +1475,7 @@ mod tests { profile: Ok(fake_profile("Paige", "model-a")), }]; let hosted_ok = - materialize_adapters(ok_plan, &factory_ok, StubAircCitizen::fresh_lookup(), |_| None) + materialize_adapters(ok_plan, &factory_ok, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None) .await; assert!(hosted_ok[0].is_ok(), "ok-warmup adapter materializes"); assert_eq!(ok_counts.warmups(), 1); @@ -1429,6 +1493,7 @@ mod tests { &factory_fail, StubAircCitizen::fresh_lookup(), |_| None, + |_, _| None, ) .await; assert!( From b7b526455103d89acc82a9fbb0c6f4ebad26b676 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 15:38:33 -0500 Subject: [PATCH 19/55] =?UTF-8?q?feat(persona):=20grid-overflow=20effector?= =?UTF-8?q?=20LIVE=20closure=20=E2=80=94=20a=20persona's=20brain=20routes?= =?UTF-8?q?=20off-box=20to=20a=20residency-eligible=20peer=20(governor=20c?= =?UTF-8?q?onsumer=20slice=204b-ii)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last mile. build_overflow_effector (persona/grid_overflow_effector.rs) composes the whole tested decision path into the per-persona adapter override the supervisor consumes, wired at the ipc bootstrap spawn: live serving plan (grid_overflow_lanes) → footprint from live_candidates → grid_lease_request → global_residency_ledger().view → route_grid_overflow → AircLiveTransport(airc, peer) → AircRemoteInferenceAdapter When the node is over local capacity and a reachable peer already holds a persona's model, her DeliberationModelBinding.adapter becomes the airc-remote one — her inference crosses the grid transparently (the re-home the binding was designed for), and she lives in the room as a peer hosted on another machine. That's "a competent peer so it's not just us there." DEFENSIVE by construction (safe to ship pre-smoke): returns None → local adapter on ANY uncertainty (airc not attached, no plan, no overflow, no matching footprint, no eligible reachable peer). Can only be a safe no-op or a correct off-box route — never a self-route (own peer excluded via airc.peer_id(), so its own residency-beacon loopback can't pick itself) and never a panic. The only unit-unprovable part is that the remote hop SUCCEEDS — the live two-node smoke validates that; a hop that can't warm surfaces as a loud AdapterWarmup slot failure, never a silent local downgrade ([[fallbacks-are-illegal-fail-loud]]). Plumbing: overflow_adapter_for threaded through spawn_all → materialize_adapters (4b-i seam); dedicated Arc clones of the airc-interceptor cell + serving daemon so the boot-spawn async-move capture doesn't strand the interceptor + reconcile task; live_candidates() → pub(crate). Completes the grid-overflow governor consumer end-to-end. Live validation + the cross-node generation run the moment BigMama's node is serving (model_id string-equality is the one thing to confirm live). All unit paths green (21 supervisor/overflow/residency tests). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/ipc/mod.rs | 26 +++- .../src/modules/serving_daemon.rs | 2 +- .../src/persona/grid_overflow_effector.rs | 115 ++++++++++++++++++ core/continuum-core/src/persona/host.rs | 17 ++- core/continuum-core/src/persona/mod.rs | 1 + 5 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 core/continuum-core/src/persona/grid_overflow_effector.rs diff --git a/core/continuum-core/src/ipc/mod.rs b/core/continuum-core/src/ipc/mod.rs index f50889aa23..c05b32e7e6 100644 --- a/core/continuum-core/src/ipc/mod.rs +++ b/core/continuum-core/src/ipc/mod.rs @@ -2108,6 +2108,15 @@ pub fn start_server( // below). Subscribe BEFORE the spawn so no plan edge is missed while // the task waits on the executor-ready oneshot. let mut serving_plan_rx = serving_daemon.subscribe(); + // A DEDICATED clone of the interceptor's airc-handle cell for the grid-overflow + // effector (slice 4b): the boot-spawn `async move` below captures by move, and the + // interceptor still needs the original `airc_interceptor_cell` further down — so the + // effector rides its own Arc clone of the SAME shared cell (both see the handle once + // attach_as fills it). + let overflow_airc_cell = airc_interceptor_cell.clone(); + // Same reason for the serving daemon: the reconcile task below still needs the + // original `serving_daemon` handle, so the effector rides its own clone. + let overflow_serving = serving_daemon.clone(); rt_handle.spawn(async move { // Wait for the IPC thread to deliver the WIRED executor (this both // gates ordering AND hands us the executor the personas' hands ride). @@ -2195,7 +2204,22 @@ pub fn start_server( } else { attempt += 1; let summary = supervisor - .spawn_all(&mut provider, Some(tool_executor.clone())) + .spawn_all( + &mut provider, + Some(tool_executor.clone()), + // Grid-overflow effector (slice 4b): the LIVE closure. Reads + // the serving plan's grid_overflow_lanes, filters residency- + // eligible reachable peers (from the beacon ledger), runs + // route_grid_overflow, and re-homes the persona's adapter to an + // AircRemoteInferenceAdapter over the interceptor's airc handle. + // DEFENSIVE: None on any uncertainty → local adapter (no + // regression); can only be a safe no-op or a correct off-box + // route (self excluded via airc.peer_id()). + crate::persona::grid_overflow_effector::build_overflow_effector( + overflow_airc_cell.clone(), + overflow_serving.clone(), + ), + ) .await; if summary.hosted > 0 { tracing::info!( diff --git a/core/continuum-core/src/modules/serving_daemon.rs b/core/continuum-core/src/modules/serving_daemon.rs index 452f91e4c3..90e8902ed2 100644 --- a/core/continuum-core/src/modules/serving_daemon.rs +++ b/core/continuum-core/src/modules/serving_daemon.rs @@ -560,7 +560,7 @@ impl ServingDaemonModule { /// one candidate, so the reconcile serves that model or (if it has dropped off /// disk) nothing. Suppress subtracts; pin intersects; the planner still owns /// the choice among whatever remains. - fn live_candidates(&self) -> Vec { + pub(crate) fn live_candidates(&self) -> Vec { let suppressed = self.suppressed.borrow(); let pinned = self.pinned.borrow(); servable_candidates(&self.catalog.snapshot(), &**suppressed, &pinned) diff --git a/core/continuum-core/src/persona/grid_overflow_effector.rs b/core/continuum-core/src/persona/grid_overflow_effector.rs new file mode 100644 index 0000000000..4acc4a7286 --- /dev/null +++ b/core/continuum-core/src/persona/grid_overflow_effector.rs @@ -0,0 +1,115 @@ +//! The grid-overflow EFFECTOR (governor consumer slice 4b-ii) — composes the tested +//! decision path into the live per-persona adapter override that +//! [`super::supervisor::materialize_adapters`] consumes. +//! +//! When the node is over local capacity (`ServingPlan.grid_overflow_lanes > 0`) and a +//! reachable peer already holds a persona's model, this routes HER BRAIN to that peer: her +//! `DeliberationModelBinding.adapter` becomes an [`AircRemoteInferenceAdapter`] over airc, so +//! her inference crosses the grid transparently — the exact re-home the binding was designed +//! for. The persona doesn't know or care that her model runs on another machine. +//! +//! ## Defensive by construction — safe to ship before the live smoke +//! +//! The closure returns `None` (→ the local factory adapter, zero behavior change) on ANY +//! uncertainty: airc not yet attached, no serving plan, no overflow, no matching footprint, or +//! no residency-eligible reachable peer. So it can only ever be a **safe no-op or a correct +//! remote route** — never a self-route (this node's own peer is excluded via `airc.peer_id()`, +//! so its own residency-beacon loopback can't select itself) and never a panic. The one thing +//! the unit path can't prove is that the remote hop SUCCEEDS — that is what the live two-node +//! smoke validates; a hop that can't warm surfaces as a loud per-slot `AdapterWarmup` failure +//! ([[fallbacks-are-illegal-fail-loud]]), never a silent local downgrade. + +use std::sync::Arc; + +use airc_lib::Airc; +use tokio::sync::OnceCell; + +use crate::ai::adapter::AIProviderAdapter; +use crate::capacity::gossip::global_ledger; +use crate::capacity::grid_overflow::route_grid_overflow; +use crate::capacity::model_residency::global_residency_ledger; +use crate::capacity::DeviceCapacity; +use crate::inference::airc_remote::adapter::AircRemoteInferenceAdapter; +use crate::inference::airc_remote::transport::AircLiveTransport; +use crate::modules::serving_daemon::ServingDaemonModule; +use crate::persona::inference_profile::PersonaInferenceProfile; + +/// Headroom kept free on a peer before it may accept an overflow lane — same 1 GiB spirit as +/// the single-device [`crate::capacity::grid::LocalFirstFitPolicy`] safety margin. +const OVERFLOW_SAFETY_MARGIN_BYTES: u64 = 1024 * 1024 * 1024; + +fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// Build the effector closure `spawn_all` / `materialize_adapters` consume. Captures the +/// late-bound airc handle cell (shared with the interceptor) and the serving daemon (the live +/// plan + footprint source). See the module docs for the defensive contract. +pub fn build_overflow_effector( + airc_cell: Arc>>, + serving: Arc, +) -> impl Fn(&PersonaInferenceProfile, usize) -> Option> { + move |profile, _slot| { + // airc not attached yet → local (the boot window before attach_as fills the cell). + let airc = airc_cell.get()?.clone(); + // No plan, or demand fits locally → local. grid_overflow_lanes is the honest + // "over local capacity by N" signal; 0 means nothing to spill. + let plan = serving.compute_plan()?; + if plan.grid_overflow_lanes == 0 { + return None; + } + // The footprint for THIS persona's model — the lease's per-lane prefill spike. + // Absent (model not a live candidate) → local, never a fabricated footprint. + let footprint = serving + .live_candidates() + .into_iter() + .find(|f| f.model_id == profile.model_id)?; + let lease = + footprint.grid_lease_request(plan.served_context_window, plan.grid_overflow_lanes); + + // Own peer id from the airc handle → excludes self from the residency view + gossip + // snapshot (this node's own beacon loopback must never select itself as the target). + let own = airc.peer_id().as_uuid(); + let now = now_ms(); + let residency = global_residency_ledger().view(own, now); + // Overflow placement is REMOTE-ONLY, so local capacity is never read — a zeroed local + // is the honest input (route_grid_overflow only ever inspects the peer list). + let snapshot = global_ledger().snapshot( + own, + DeviceCapacity { + gpu_total_bytes: 0, + gpu_free_bytes_live: 0, + system_ram_free_bytes: 0, + }, + now, + ); + + let routing = route_grid_overflow( + &profile.model_id, + &lease, + &residency, + &snapshot, + OVERFLOW_SAFETY_MARGIN_BYTES, + ); + // First-cut assignment: this persona takes the first placed peer. No residency-eligible + // reachable peer with room → local (queue/degrade is the planner's, not a silent drop). + let (peer, _lanes) = routing.remote.first()?; + let peer_uuid = peer.as_uuid(); + + let transport = AircLiveTransport::new(airc, peer_uuid); + let adapter = AircRemoteInferenceAdapter::new(transport); + crate::probe!( + class = "grid.overflow.route", + persona = %profile.persona_name, + model = %profile.model_id, + peer = %peer_uuid, + overflow_lanes = plan.grid_overflow_lanes, + "routing persona brain OFF-BOX to a residency-eligible peer (grid overflow)", + ); + Some(Arc::new(adapter) as Arc) + } +} diff --git a/core/continuum-core/src/persona/host.rs b/core/continuum-core/src/persona/host.rs index 6731d7062d..c9ec318822 100644 --- a/core/continuum-core/src/persona/host.rs +++ b/core/continuum-core/src/persona/host.rs @@ -264,6 +264,15 @@ impl PersonaSpawnSupervisor { // persona's HANDS are built over it (identity-scoped), so the ACL gates // what they may do. `None` → personas spawn speak-only (no hands). tool_command_executor: Option>, + // The grid-overflow effector (governor consumer slice 4b): given a persona's + // profile + slot, `Some(remote adapter)` routes her brain to a peer that holds + // her model (the ipc bootstrap builds this from the live serving plan + residency + // ledger + airc handle); `None` → local adapter. Forwarded verbatim to + // `materialize_adapters`. `|_, _| None` is the pre-effector (all-local) behavior. + overflow_adapter_for: impl Fn( + &crate::persona::inference_profile::PersonaInferenceProfile, + usize, + ) -> Option>, ) -> BootSummary { let plans = match bootstrap_planned( &self.spawner, @@ -328,10 +337,10 @@ impl PersonaSpawnSupervisor { as Arc }) }, - // Grid-overflow effector closure (slice 4b-ii wires the live capacity + - // residency + airc context here). Until then every persona builds her - // local adapter — no off-box routing, the pre-effector behavior. - |_profile, _slot| None, + // Grid-overflow effector: the ipc bootstrap supplies the live decision + // (capacity + residency + airc). `|_, _| None` from a caller that doesn't + // route keeps the pre-effector all-local behavior. + overflow_adapter_for, ) .await; diff --git a/core/continuum-core/src/persona/mod.rs b/core/continuum-core/src/persona/mod.rs index b69f5930e4..008a907c3a 100644 --- a/core/continuum-core/src/persona/mod.rs +++ b/core/continuum-core/src/persona/mod.rs @@ -50,6 +50,7 @@ pub mod evaluator; pub mod focus; pub mod genome_paging; pub mod home; +pub mod grid_overflow_effector; pub mod host; pub mod hw_tier_descriptor; pub mod identity_provider; From eccb573a2bf011caceb77aefa9ada34f9e7111f0 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 16:02:00 -0500 Subject: [PATCH 20/55] =?UTF-8?q?fix(windows-build):=20CUDA=20serving=20bu?= =?UTF-8?q?ild=20=E2=80=94=20cuda-guard=20+=20cmake/llvm=20runtime=5Fpath?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three real fixes that get continuum-core to build+detect CUDA on native windows-msvc (validated: features cuda,directml -> detect_cuda -> 27 GiB on a 5090, was a bogus 4GB under directml-only): - cargo-features.sh: only add `cuda` on Windows+Nvidia when cl.exe is actually reachable, else directml-only. candle affine.cu needs nvcc->cl.exe; without it the whole build hard-failed instead of degrading. - install-manifest.toml: cmake + llvm-libclang had no [module.runtime_path], so start-server.sh installed them to ~/.continuum/tools but never put them on PATH -> every 'cmake not found' / bindgen libclang failure. Added windows runtime_path, regenerated manifests. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- tools/scripts/generated/manifest.windows.ps1 | 4 ++-- tools/scripts/generated/manifest.windows.sh | 2 +- tools/scripts/install-manifest.toml | 11 +++++++++++ tools/scripts/shared/cargo-features.sh | 8 +++++++- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/tools/scripts/generated/manifest.windows.ps1 b/tools/scripts/generated/manifest.windows.ps1 index 49d592b79d..453ce89751 100644 --- a/tools/scripts/generated/manifest.windows.ps1 +++ b/tools/scripts/generated/manifest.windows.ps1 @@ -13,8 +13,8 @@ $script:ContinuumManifest = [ordered]@{ 'airc-firewall' = @{ order = 27; tier = 0; flags = @('grid'); applies = 'has-airc'; accept = 'netsh advfirewall firewall show rule name="airc daemon inbound (continuum grid)"'; source = @{ type = 'command'; run = 'New-NetFirewallRule -DisplayName ''airc daemon inbound (continuum grid)'' -Direction Inbound -Action Allow -Profile Any' } } 'manifest-gen' = @{ order = 28; tier = 3; flags = @('dev'); accept = 'cargo run -q -p manifest-gen -- --check'; source = @{ type = 'command'; run = 'cargo run -q -p manifest-gen' } } 'msvc' = @{ order = 30; tier = 3; flags = @('dev'); accept = 'vswhere -latest -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath'; source = @{ type = 'winget'; id = 'Microsoft.VisualStudio.2022.BuildTools'; override = '--wait --quiet --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' } } - 'cmake' = @{ order = 40; tier = 3; flags = @('dev'); accept = 'cmake --version'; source = @{ type = 'archive'; url = 'https://github.com/Kitware/CMake/releases/download/v3.30.5/cmake-3.30.5-windows-x86_64.zip'; version = '3.30.5'; sha256 = '5ab6e1faf20256ee4f04886597e8b6c3b1bd1297b58a68a58511af013710004b'; extract = 'strip-top-dir' } } - 'llvm-libclang' = @{ order = 50; tier = 3; flags = @('dev'); accept = 'test-path ~/.continuum/tools/llvm/bin/libclang.dll'; source = @{ type = 'archive'; url = 'https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.8/clang+llvm-18.1.8-x86_64-pc-windows-msvc.tar.xz'; version = '18.1.8'; sha256 = '22c5907db053026cc2a8ff96d21c0f642a90d24d66c23c6d28ee7b1d572b82e8'; extract = 'members:*/bin/libclang.dll,*/lib/clang/*' } } + 'cmake' = @{ order = 40; tier = 3; flags = @('dev'); accept = 'cmake --version'; source = @{ type = 'archive'; url = 'https://github.com/Kitware/CMake/releases/download/v3.30.5/cmake-3.30.5-windows-x86_64.zip'; version = '3.30.5'; sha256 = '5ab6e1faf20256ee4f04886597e8b6c3b1bd1297b58a68a58511af013710004b'; extract = 'strip-top-dir' }; runtime_path = @('~/.continuum/tools/cmake/bin') } + 'llvm-libclang' = @{ order = 50; tier = 3; flags = @('dev'); accept = 'test-path ~/.continuum/tools/llvm/bin/libclang.dll'; source = @{ type = 'archive'; url = 'https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.8/clang+llvm-18.1.8-x86_64-pc-windows-msvc.tar.xz'; version = '18.1.8'; sha256 = '22c5907db053026cc2a8ff96d21c0f642a90d24d66c23c6d28ee7b1d572b82e8'; extract = 'members:*/bin/libclang.dll,*/lib/clang/*' }; runtime_path = @('~/.continuum/tools/llvm/bin') } 'cuda' = @{ order = 60; tier = 3; flags = @('dev'); applies = 'has-nvidia'; accept = 'nvcc --version >= 12.8'; source = @{ type = 'redist'; version = '12.9.1'; manifest = 'https://developer.download.nvidia.com/compute/cuda/redist/redistrib_12.9.1.json'; components = @('cuda_nvcc', 'cuda_cudart', 'libcublas', 'libcurand', 'cuda_nvrtc', 'cuda_cccl') }; runtime_path = @('~/.continuum/cuda-*/Library/bin') } 'build-core' = @{ order = 90; tier = 3; flags = @('dev'); accept = 'continuum-core-server.exe boots past the GPU-detection gate on the target device'; build = @{ features = 'cuda,load-dynamic-ort'; profile = 'release'; crt = 'static'; cmake_generator = 'Visual Studio 17 2022'; cuda_arch = '120'; msvc_host = 'vs2022' } } 'run' = @{ order = 100; tier = 3; accept = 'continuum-core-server binary present + serves TCP 9100' } diff --git a/tools/scripts/generated/manifest.windows.sh b/tools/scripts/generated/manifest.windows.sh index 12d66671e4..90ef542ad6 100644 --- a/tools/scripts/generated/manifest.windows.sh +++ b/tools/scripts/generated/manifest.windows.sh @@ -27,4 +27,4 @@ declare -A MOD_ARGS=() declare -A MOD_RUN=( ['gh-auth']='gh auth login --hostname github.com --git-protocol https --web' ['airc-firewall']='New-NetFirewallRule -DisplayName '\''airc daemon inbound (continuum grid)'\'' -Direction Inbound -Action Allow -Profile Any' ['manifest-gen']='cargo run -q -p manifest-gen' ) declare -A MOD_BUILD_FEATURES=( ['build-core']='cuda,load-dynamic-ort' ) declare -A MOD_BUILD_PROFILE=( ['build-core']='release' ) -declare -A MOD_RUNTIME_PATH=( ['cuda']='~/.continuum/cuda-*/Library/bin' ) +declare -A MOD_RUNTIME_PATH=( ['cmake']='~/.continuum/tools/cmake/bin' ['llvm-libclang']='~/.continuum/tools/llvm/bin' ['cuda']='~/.continuum/cuda-*/Library/bin' ) diff --git a/tools/scripts/install-manifest.toml b/tools/scripts/install-manifest.toml index 8648882a31..968d7c4e8c 100644 --- a/tools/scripts/install-manifest.toml +++ b/tools/scripts/install-manifest.toml @@ -159,6 +159,12 @@ formula = "cmake" [module.sources.linux] type = "apt" # run-verify pending on a linux node package = "cmake" +[module.runtime_path] +# The Windows archive installs to ~/.continuum/tools/cmake; its bin/ MUST be on +# PATH at BUILD time — the llama crate's build.rs shells out to `cmake` to compile +# vendored llama.cpp, and nvcc-free directml builds still need it. brew/apt already +# put cmake on PATH, so this is windows-only. [[windows-build-env-drift]] +windows = ["~/.continuum/tools/cmake/bin"] [[module]] id = "llvm-libclang" @@ -183,6 +189,11 @@ accept_macos = "test -f /Library/Developer/CommandLineTools/usr/lib/libclang.dyl type = "apt" # run-verify pending on a linux node package = "libclang-dev" accept_linux = "test -f /usr/lib/llvm-18/lib/libclang.so.1 || ldconfig -p | grep -q libclang" +[module.runtime_path] +# The Windows archive installs libclang.dll to ~/.continuum/tools/llvm/bin; on PATH +# so bindgen (llama crate) can load it at build time. mac/linux resolve libclang via +# system paths. [[windows-build-env-drift]] +windows = ["~/.continuum/tools/llvm/bin"] [[module]] id = "cuda" diff --git a/tools/scripts/shared/cargo-features.sh b/tools/scripts/shared/cargo-features.sh index e9615ebb9a..3089dfa6a6 100644 --- a/tools/scripts/shared/cargo-features.sh +++ b/tools/scripts/shared/cargo-features.sh @@ -59,7 +59,13 @@ case "$(uname -s)" in # top if Nvidia is present so ORT picks CUDA first (faster) + # DirectML stays as a co-listed EP for non-CUDA-supported ops. CARGO_GPU_FEATURES="--features directml" - if command -v nvidia-smi &>/dev/null; then + # candle-cuda's affine.cu compiles via nvcc, which needs the MSVC host + # compiler cl.exe on PATH (an active vcvars env). Only add cuda when cl.exe + # is actually reachable; otherwise nvcc fatals "Cannot find compiler + # 'cl.exe'" and the ENTIRE core build dies. directml needs no kernel + # compilation, so it stays as the universal Windows GPU EP and the build + # degrades gracefully instead of hard-failing. [[windows-build-env-drift]] + if command -v nvidia-smi &>/dev/null && command -v cl.exe &>/dev/null; then CARGO_GPU_FEATURES="--features cuda,directml" fi ;; From f829ed6be12cd2b0a1fe5eda95aa8670b63f021e Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 16:14:40 -0500 Subject: [PATCH 21/55] =?UTF-8?q?fix(ipc):=20interceptor-attach=20must=20u?= =?UTF-8?q?se=20rt=5Fhandle.spawn,=20not=20bare=20tokio::spawn=20=E2=80=94?= =?UTF-8?q?=20the=20airc=20handle=20cell=20never=20filled=20(BigMama=20dia?= =?UTF-8?q?gnosed=20the=20panic)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At ipc/mod.rs the AircInterceptor bootstrap spawned its `Airc::attach_as` task with bare `tokio::spawn`. But this runs in `start_server` on the IPC thread, and the `rt_handle.enter()` guard (line ~1111) is SCOPED and has already dropped by here — so there is NO ambient tokio runtime and `tokio::spawn` panics "there is no reactor running, must be called from the context of a Tokio 1.x runtime". That panic killed the attach task, so the `OnceCell>` never filled, and EVERYTHING that reads it silently no-oped: the AircInterceptor's aircPeer command routing (send side) AND the grid-overflow effector (build_overflow_effector reads the same cell → always None → local, never routes). Fix: `rt_handle.spawn` (rt_handle is a start_server param, in scope; the SAME call 1498/2111 already use) — it targets the runtime by handle without needing ambient context. This is the sender-side unblock for the whole cross-node persona-serving path: without the handle, a node can never route an ai/generate to a peer. BigMama diagnosed the panic live 2026-07-27; this is the one-line fix on the continuum side. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/ipc/mod.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/continuum-core/src/ipc/mod.rs b/core/continuum-core/src/ipc/mod.rs index c05b32e7e6..9e7e3422aa 100644 --- a/core/continuum-core/src/ipc/mod.rs +++ b/core/continuum-core/src/ipc/mod.rs @@ -1701,7 +1701,16 @@ pub fn start_server( if let Some((interceptor_daemon_socket, _room)) = interceptor_airc_deps { let cell = airc_interceptor_cell.clone(); let root = crate::modules::persona_instance_manager::resolve_continuum_root(); - tokio::spawn(async move { + // MUST be `rt_handle.spawn`, NOT bare `tokio::spawn`: this runs during + // start_server on the IPC thread, and the `rt_handle.enter()` guard (above) is + // scoped and has already dropped by here — so there is NO ambient runtime and + // `tokio::spawn` panics "there is no reactor running". That panic kills the + // attach task, the OnceCell never fills, and EVERYTHING that reads it silently + // no-ops: the AircInterceptor's aircPeer routing AND the grid-overflow effector + // (build_overflow_effector reads this same cell). rt_handle.spawn targets the + // runtime by handle without needing ambient context. (BigMama diagnosed the + // panic 2026-07-27; this is the one-line fix.) + rt_handle.spawn(async move { match airc_lib::Airc::attach_as(root, "continuum-airc-interceptor", interceptor_daemon_socket) .await { From 8b56c327a74446f89aeaf78d262ae848ac39d596 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 00:55:25 -0500 Subject: [PATCH 22/55] feat(catalog): register qwen3-coder-30b-a3b-compacted-19b-256k as a servable chat candidate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BigMama's forged coder becomes the first local CHAT model the 5090 serving node can host — validated live: candidates 1->2, serving daemon selects it, decode lane ready=true, persona hosted, GPU generation confirmed (49% util, correct code). Hardcoded canonical row matching the coder-14b template (the current catalog mechanism); the dynamic register slice supersedes this later. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../src/model_registry/catalog.rs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/core/continuum-core/src/model_registry/catalog.rs b/core/continuum-core/src/model_registry/catalog.rs index a50fb8522f..cc7ec9ed89 100644 --- a/core/continuum-core/src/model_registry/catalog.rs +++ b/core/continuum-core/src/model_registry/catalog.rs @@ -821,6 +821,32 @@ pub fn models() -> Vec { // gguf_local_path DERIVED from the id under genome/models (see coder-14b above). ..ModelSpec::default() }), + // QWEN3-CODER-30B-A3B compacted to 19B (256k ctx) — BigMama's forged coder, + // the first CHAT candidate the 5090 serving node can host locally. Qwen3 MoE + // (a3b active) pruned+quantized via the plasticity-compaction pipeline; Q4_K_M + // GGUF ~11 GB, fits the 5090 with room for KV. Registered under "llama-server" + // so the serving daemon's static-CUDA engine hosts it on a lane; gguf resolves + // from its dir under genome/models (id-token subset match). + model(ModelSpec { + id: "continuum-ai/qwen3-coder-30b-a3b-compacted-19b-256k", + name: "Qwen3-Coder-30B-A3B compacted 19B (256k)", + provider: "llama-server", + arch: Arch::Qwen3, + context_window: 262_144, + max_output_tokens: 8192, + tokens_per_second: 30.0, + capabilities: &[ + Capability::TextGeneration, + Capability::Chat, + Capability::ToolUse, + Capability::Streaming, + ], + gguf_hint: Some("huggingface.co/continuum-ai/qwen3-coder-30b-a3b-compacted-19b-256k"), + chat_template: Some(QWEN35_CHAT_TEMPLATE), + multi_party_strategy: MultiPartyChatStrategy::ProperChatMlSingleParty, + stop_sequences: &["<|im_end|>", "<|endoftext|>"], + ..ModelSpec::default() + }), ] } From c25d45e7c64ec509f38003365d660feb43cf9a20 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 10:22:39 -0500 Subject: [PATCH 23/55] =?UTF-8?q?fix(windows):=20portable=20serving-node?= =?UTF-8?q?=20lifecycle=20=E2=80=94=20no=20hand=20steps=20survive?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes, each validated live by a full plain-npm-start cycle on the 5090 (env self-established -> old core replaced -> airc healthy -> stamped static engine -> compacted-19b ready=true -> persona hosted=1 failed=0): - start-server.sh: self-establish the Windows-CUDA build env (vswhere->vcvars64 re-exec with recursion guard, MSVC link.exe precedence over Git coreutils link, CUDA lib/x64 onto LIB, LIBCLANG_PATH) — a fresh Windows+NVIDIA box gets the real cuda build from npm start; no BuildTools degrades gracefully. - start-server.sh: stop_existing_core was a SILENT NO-OP on Windows (pgrep/kill can't touch native exes) — an immortal old core survived every restart and killed each new boot in a port fight. Windows branch uses tasklist/taskkill. - install-llama-server.sh: Windows+CUDA now builds STATIC (the shared build's ggml-cuda.dll is GPU-blind at runtime while passing --version) and the verify requires --list-devices to show CUDA0 before stamping; stamp renamed cuda->cuda-static so existing broken installs auto-rebuild. - airc/discovery.rs: one-shot 5s spawn probes false-fail under load (Windows process spawn alone can eat seconds mid-build) — bounded 3-attempt retry via one shared probe_airc helper; still fail-loud when airc is genuinely dead. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- core/continuum-core/src/airc/discovery.rs | 86 ++++++++++------ tools/scripts/install-llama-server.sh | 26 ++++- tools/scripts/start-server.sh | 113 +++++++++++++++++++--- 3 files changed, 177 insertions(+), 48 deletions(-) diff --git a/core/continuum-core/src/airc/discovery.rs b/core/continuum-core/src/airc/discovery.rs index ff525b568a..6cdd6234c1 100644 --- a/core/continuum-core/src/airc/discovery.rs +++ b/core/continuum-core/src/airc/discovery.rs @@ -148,17 +148,55 @@ async fn airc_on_path() -> bool { .unwrap_or(false) } +/// Probe attempts before declaring the substrate unresponsive. A single 5s shot +/// is a coin flip on a loaded box: Windows process spawn alone can transiently +/// eat seconds when the machine is compiling/converting/downloading (observed +/// 2026-07-28 BigMama — the updated airc answered instantly when idle, yet the +/// one-shot probe timed out mid-build and the core refused full-citizen boot). +/// Retries stay BOUNDED (3 × deadline ≈ 15s worst case) — resilient to transient +/// load, still fail-loud when airc is genuinely dead. +const DISCOVERY_PROBE_ATTEMPTS: u32 = 3; + +/// How an `airc ` probe failed — timeout (every attempt) vs spawn error. +enum ProbeError { + TimedOut, + Io(String), +} + +/// Run `airc ` with the discovery deadline and bounded retries (see +/// [`DISCOVERY_PROBE_ATTEMPTS`]). ONE probe implementation for every discovery +/// subcommand — the retry-on-transient-stall policy lives here, not copy-pasted +/// per call site. +async fn probe_airc(arg: &str) -> Result { + for attempt in 1..=DISCOVERY_PROBE_ATTEMPTS { + let call = TokioCommand::new("airc").arg(arg).output(); + match timeout(DISCOVERY_SUBPROCESS_DEADLINE, call).await { + Ok(res) => return res.map_err(|e| ProbeError::Io(e.to_string())), + Err(_) if attempt < DISCOVERY_PROBE_ATTEMPTS => { + warn!( + attempt, + "`airc {arg}` did not exit within {DISCOVERY_SUBPROCESS_DEADLINE:?} \ + — retrying (transient spawn stall under load)" + ); + } + Err(_) => {} + } + } + Err(ProbeError::TimedOut) +} + +fn probe_timeout_msg(arg: &str) -> String { + format!( + "`airc {arg}` did not exit within {DISCOVERY_SUBPROCESS_DEADLINE:?} \ + × {DISCOVERY_PROBE_ATTEMPTS} attempts — substrate is unresponsive, refusing to wait", + ) +} + async fn query_airc_endpoint() -> Result { - let call = TokioCommand::new("airc").arg("ipc-endpoint").output(); - let out = timeout(DISCOVERY_SUBPROCESS_DEADLINE, call) - .await - .map_err(|_| { - DiscoveryError::EndpointCommandFailed(format!( - "`airc ipc-endpoint` did not exit within {DISCOVERY_SUBPROCESS_DEADLINE:?} \ - — substrate is unresponsive, refusing to wait", - )) - })? - .map_err(|e| DiscoveryError::EndpointCommandFailed(e.to_string()))?; + let out = probe_airc("ipc-endpoint").await.map_err(|e| match e { + ProbeError::TimedOut => DiscoveryError::EndpointCommandFailed(probe_timeout_msg("ipc-endpoint")), + ProbeError::Io(msg) => DiscoveryError::EndpointCommandFailed(msg), + })?; if !out.status.success() { return Err(DiscoveryError::EndpointCommandFailed(format!( "exit {}: {}", @@ -200,16 +238,10 @@ pub async fn discover_default_channel() -> Result { )) }); } - let call = TokioCommand::new("airc").arg("room").output(); - let out = timeout(DISCOVERY_SUBPROCESS_DEADLINE, call) - .await - .map_err(|_| { - DiscoveryError::RoomCommandFailed(format!( - "`airc room` did not exit within {DISCOVERY_SUBPROCESS_DEADLINE:?} \ - — substrate is unresponsive, refusing to wait", - )) - })? - .map_err(|e| DiscoveryError::RoomCommandFailed(e.to_string()))?; + let out = probe_airc("room").await.map_err(|e| match e { + ProbeError::TimedOut => DiscoveryError::RoomCommandFailed(probe_timeout_msg("room")), + ProbeError::Io(msg) => DiscoveryError::RoomCommandFailed(msg), + })?; if !out.status.success() { return Err(DiscoveryError::RoomCommandFailed(format!( "exit {}: {}", @@ -241,16 +273,10 @@ pub async fn discover_default_room_name() -> Result { return Ok(raw); } } - let call = TokioCommand::new("airc").arg("room").output(); - let out = timeout(DISCOVERY_SUBPROCESS_DEADLINE, call) - .await - .map_err(|_| { - DiscoveryError::RoomCommandFailed(format!( - "`airc room` did not exit within {DISCOVERY_SUBPROCESS_DEADLINE:?} \ - — substrate is unresponsive, refusing to wait", - )) - })? - .map_err(|e| DiscoveryError::RoomCommandFailed(e.to_string()))?; + let out = probe_airc("room").await.map_err(|e| match e { + ProbeError::TimedOut => DiscoveryError::RoomCommandFailed(probe_timeout_msg("room")), + ProbeError::Io(msg) => DiscoveryError::RoomCommandFailed(msg), + })?; if !out.status.success() { return Err(DiscoveryError::RoomCommandFailed(format!( "exit {}: {}", diff --git a/tools/scripts/install-llama-server.sh b/tools/scripts/install-llama-server.sh index 68e75d866a..b667c37785 100755 --- a/tools/scripts/install-llama-server.sh +++ b/tools/scripts/install-llama-server.sh @@ -82,9 +82,17 @@ else MINGW*|MSYS*|CYGWIN*) EXE=".exe" if command -v nvcc >/dev/null 2>&1 || [ -x "$CONTINUUM_HOME/cuda-toolkit/bin/nvcc.exe" ]; then - BACKEND="cuda"; WIN_CUDA=1 + BACKEND="cuda-static"; WIN_CUDA=1 # arch=native → build for THIS machine's GPU (portable; NOT a hardcoded sm_120). - BACKEND_DEFS=(-DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=native) + # BUILD_SHARED_LIBS=OFF is LOAD-BEARING (2026-07-28, BigMama): the shared build's + # ggml-cuda.dll fails CUDA init at runtime ("no usable GPU found") while passing + # --version — so a GPU-blind engine got stamped verified-good and every generation + # 500'd (the serving daemon's decode-ready probe then never admits personas). The + # static build (ggml linked into the exe) initializes CUDA correctly on the same + # box. Backend renamed cuda→cuda-static so existing broken installs fail the stamp + # check and rebuild on next run. CUDA RUNTIME (cudart/cublas dlls) stays dynamic + # via the manifest runtime_path — only ggml/llama are static. + BACKEND_DEFS=(-DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=native -DBUILD_SHARED_LIBS=OFF) fi # non-NVIDIA Windows falls through to the CPU build (llama.cpp CPU works on Win). ;; @@ -225,6 +233,20 @@ if [ "$verify_ok" -ne 1 ]; then echo "✗ FATAL: built llama-server does not run (--version failed after retry)." >&2 exit 1 fi + +# On Windows+CUDA, --version is NOT enough: the GPU-blind shared build passed it while +# unable to init CUDA (the 2026-07-28 regression). Require the engine to actually SEE a +# CUDA device before stamping — this is the difference between "binary runs" and "binary +# can serve". A build that can't see the GPU is removed, not blessed. +if [ "$WIN_CUDA" -eq 1 ]; then + if ! "$INSTALL_BIN" --list-devices 2>&1 | grep -q "CUDA0"; then + rm -f "$INSTALL_BIN" + echo "✗ FATAL: built llama-server cannot see a CUDA device (--list-devices has no CUDA0)." >&2 + echo " A GPU-blind engine must never be stamped — it serves decode-dead lanes." >&2 + exit 1 + fi + echo "✓ CUDA verify: engine sees CUDA0" >&2 +fi echo "$STAMP_WANT" > "$STAMP_FILE" # stamp LAST — only a verified-good binary is blessed echo "✓ llama-server installed: $INSTALL_BIN ($STAMP_WANT)" >&2 diff --git a/tools/scripts/start-server.sh b/tools/scripts/start-server.sh index 819985bfe3..130e80b9cd 100755 --- a/tools/scripts/start-server.sh +++ b/tools/scripts/start-server.sh @@ -101,6 +101,63 @@ if [ -f "$_mf_runtime" ]; then fi fi +# ── Windows-CUDA build env (2026-07-28, BigMama) ───────────────────── +# The core's `cuda` feature needs the MSVC toolchain visible to cargo's build +# scripts (candle-kernels runs nvcc→cl.exe; the llama crate links cuda.lib). +# On a stock Git-Bash shell NONE of that is on PATH, so cargo-features.sh +# (below) silently degrades to directml-only — and the core then mis-detects +# a 32GB card as ~4GB (no detect_cuda) and serves a toy model. Establish the +# env HERE, before feature detection, so a fresh Windows+NVIDIA box gets the +# real build with zero hand steps: +# 1. cl.exe absent but VS2022 BuildTools installed → re-exec this script +# once through vcvars64 (same cmd/.bat bridge install-llama-server.sh +# uses; VS2022 pinned — nvcc rejects VS18/14.5x toolsets). +# 2. MSVC link.exe must BEAT Git's /usr/bin/link.exe (coreutils) or every +# build-script link dies — prepend the cl.exe dir to PATH. +# 3. cuda.lib/cublas.lib etc live in the toolkit's lib/x64 which vcvars +# does NOT add — prepend to LIB for link.exe (LNK1181 otherwise). +# 4. bindgen reads LIBCLANG_PATH (the env var — PATH is not enough). +case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + if command -v nvidia-smi >/dev/null 2>&1 && ! command -v cl.exe >/dev/null 2>&1 \ + && [ -z "${CONTINUUM_MSVC_REENTER:-}" ]; then + _vswhere="/c/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe" + _vs_path="" + [ -x "$_vswhere" ] && _vs_path="$("$_vswhere" -version "[17.0,18.0)" -products '*' \ + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 \ + -property installationPath 2>/dev/null | head -1)" + if [ -n "$_vs_path" ]; then + echo "→ NVIDIA GPU + VS2022 present but cl.exe not on PATH — re-entering via vcvars64 for the CUDA build" >&2 + _reenter_bat="$(mktemp --suffix=.bat 2>/dev/null || echo "${TMPDIR:-/tmp}/continuum-msvc-reenter.bat")" + _win_bash="$(cygpath -w "$(command -v bash)")" + _script_unix="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" + { + echo "@echo off" + echo "call \"$(cygpath -w "$_vs_path")\\VC\\Auxiliary\\Build\\vcvars64.bat\" >nul || exit /b 1" + echo "set CONTINUUM_MSVC_REENTER=1" + echo "\"$_win_bash\" \"$_script_unix\" || exit /b 1" + } > "$_reenter_bat" + exec cmd //c "$(cygpath -w "$_reenter_bat")" + else + echo "⚠ NVIDIA GPU present but VS2022 BuildTools not found — core builds directml-only" >&2 + echo " (install-manifest 'msvc' module provisions it; cuda serving needs it)" >&2 + fi + fi + if command -v cl.exe >/dev/null 2>&1; then + # (2) MSVC linker precedence over Git's coreutils link.exe. + _cl_dir="$(dirname "$(command -v cl.exe)")" + case ":$PATH:" in "$_cl_dir":*) ;; *) export PATH="$_cl_dir:$PATH" ;; esac + # (3) CUDA import libs onto LIB (version-agnostic glob, mirrors runtime_path). + for _cuda_lib in "$HOME/.continuum"/cuda-*/Library/lib/x64 "$HOME/.continuum"/cuda-*/lib/x64; do + [ -d "$_cuda_lib" ] && export LIB="$(cygpath -w "$_cuda_lib");${LIB:-}" + done + # (4) bindgen's libclang (manifest llvm-libclang install location). + [ -f "$HOME/.continuum/tools/llvm/bin/libclang.dll" ] \ + && export LIBCLANG_PATH="$HOME/.continuum/tools/llvm/bin" + fi + ;; +esac + # ── Per-platform feature flags ─────────────────────────────────────── # Mac Intel can't use Metal (task #131 — ggml_metal_device_init hangs on # Intel + AMD discrete). Force mac-cpu-only on Intel Mac. @@ -245,22 +302,46 @@ CONTINUUM_SOCKET="${CONTINUUM_SOCKET:-/tmp/continuum-core.sock}" # Called AFTER the build (below) so downtime is ~0: the new binary is ready, we # stop the old, and exec immediately. stop_existing_core() { - local pids - pids="$(pgrep -f "continuum-core-server" 2>/dev/null | grep -v "^$$\$" || true)" - if [ -n "$pids" ]; then - echo "▶ stopping existing core (pids: $(echo $pids | tr '\n' ' '))" - # shellcheck disable=SC2086 - kill -TERM $pids 2>/dev/null || true - for _ in $(seq 1 15); do - pgrep -f "continuum-core-server" >/dev/null 2>&1 || break - sleep 1 - done - if pgrep -f "continuum-core-server" >/dev/null 2>&1; then - echo " graceful stop timed out — SIGKILL" - pkill -9 -f "continuum-core-server" 2>/dev/null || true - sleep 1 - fi - fi + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + # POSIX signals are a SILENT NO-OP against native Windows exes from + # Git-Bash: pgrep -f matches nothing and kill/pkill can't touch them, so + # this block never stopped anything on Windows — every `npm start` boot + # then died fighting the immortal old core for its ports (observed + # 2026-07-28 BigMama: an 8:22AM core survived three "restarts"; each new + # boot failed binding 0.0.0.0:7117 and exited, masked as "leaving the + # running core untouched"). Use tasklist/taskkill — the native tools. + if tasklist 2>/dev/null | grep -qi "continuum-core-server.exe"; then + echo "▶ stopping existing core (taskkill)" + # No graceful console signal exists for a detached native service from + # here (WM_CLOSE is ignored by console apps); the core's state is + # crash-safe by design ([[no-fallbacks-ever]] boot contract), so /F. + taskkill //F //IM continuum-core-server.exe >/dev/null 2>&1 || true + for _ in $(seq 1 10); do + tasklist 2>/dev/null | grep -qi "continuum-core-server.exe" || break + sleep 1 + done + fi + ;; + *) + local pids + pids="$(pgrep -f "continuum-core-server" 2>/dev/null | grep -v "^$$\$" || true)" + if [ -n "$pids" ]; then + echo "▶ stopping existing core (pids: $(echo $pids | tr '\n' ' '))" + # shellcheck disable=SC2086 + kill -TERM $pids 2>/dev/null || true + for _ in $(seq 1 15); do + pgrep -f "continuum-core-server" >/dev/null 2>&1 || break + sleep 1 + done + if pgrep -f "continuum-core-server" >/dev/null 2>&1; then + echo " graceful stop timed out — SIGKILL" + pkill -9 -f "continuum-core-server" 2>/dev/null || true + sleep 1 + fi + fi + ;; + esac rm -f "$CONTINUUM_SOCKET" } From 34a7d31888d12e45089826381c8b63a427ad4672 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 10:45:13 -0500 Subject: [PATCH 24/55] =?UTF-8?q?docs(planning):=20Sentinel-in-the-Substra?= =?UTF-8?q?te=20=E2=80=94=20absorb=20sentinel-ai's=20ideas=20as=20Rust=20m?= =?UTF-8?q?odule=20extensions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five slices mapping sentinel-ai (entropy observation, controller feedback, prune<->regrow cycles, forge-while-sleeping, MoE expert pruning) onto EXISTING substrate seams (ExpertActivationProfile/pager, PlasticityModule, dream rhythm, genome tiers, forge-custodian). No Python at runtime, no separate project; sentinel-ai repo becomes paper + reference archive. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/SENTINEL-IN-SUBSTRATE.md | 81 ++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 docs/planning/SENTINEL-IN-SUBSTRATE.md diff --git a/docs/planning/SENTINEL-IN-SUBSTRATE.md b/docs/planning/SENTINEL-IN-SUBSTRATE.md new file mode 100644 index 0000000000..8a21671cfe --- /dev/null +++ b/docs/planning/SENTINEL-IN-SUBSTRATE.md @@ -0,0 +1,81 @@ +# Sentinel-in-the-Substrate — absorbing sentinel-ai's ideas into continuum-core + +**Status:** plan (2026-07-28, BigMama). **Owner lanes:** BigMama (observe/profile, K3 pager seam), joint w/ M5 (serving + dream rhythm). +**Prime directive:** the ideas land as **extensions of existing Rust modules** — no Python at runtime, no separate project. sentinel-ai (the repo) becomes the paper + reference archive. + +## What sentinel-ai is + +`CambrianTech/sentinel-ai` — Experiential Plasticity for transformers (the foundry that produced +`qwen3-coder-30b-a3b-compacted-19b-256k`, the model Sahar serves today). Its ideas: + +1. **Entropy-based utility observation** — attention-head entropy + activation stats as the signal for + what matters to the domain. +2. **Controller feedback** — an observer that closes the loop: observe utility → adjust (gates/prune) → + re-observe. The "sentinel." +3. **Prune ↔ regrow cycles** — capacity is removed when useless and comes BACK when demand shifts; + biological synaptic pruning, not one-way compression. +4. **Forge cycles tied to training** — LoRA-train on domain → prune what didn't matter → retrain; the + architecture co-evolves with experience. +5. **Calibration-aware MoE expert pruning** — profile which experts fire on a corpus, drop the rest + (§4.1.3.4 — exactly the algorithm already ported for the compacted-19b). + +## What the substrate ALREADY has (do not rebuild — wire) + +| Sentinel idea | Existing Rust primitive | Where | +|---|---|---| +| Compaction decisions (ONE formula: `0.8·gate + 0.2·grad`) | `PlasticityModule` + 5 typed commands (`plasticity/analyze·compact·compress·topology·pipeline`) | `modules/plasticity`, `commands/plasticity/` | +| Expert activation observation (serving-side) | `ExpertActivationProfile` + `ServingExpertPager` observe→plan→budget→reconcile loop (K3 pager slice-1, PRs #2018–#2022) | `capacity/` | +| Demand-aligned retention | `EvictionPolicy::DemandAlignedWithRefinedPreference` | `genome/eviction.rs` | +| Sleep/idle rhythm | `dream_consolidation` (memory-side today) | `cognition/dream_consolidation.rs` | +| Gene → pageable artifact | `forge-custodian` bin (trained gene → gguf-lora) | built by `start-server.sh` | +| Attestable recipes | ForgeAlloy (3rd pillar) + the ForgeRecipe-entity sprint (CLAUDE.md §FORGE TEMPLATE) | `forge-alloy/` | + +## The five slices (each lands in an existing module) + +### 1. Live utility profile — PGO from serving traffic *(extends `capacity/`)* +Extend `ExpertActivationProfile` with per-head/per-expert utility harvested from the LIVE llama-server +lane: expert fire counts (already flowing) + **sampled** attention-entropy windows (entropy needs attn +probs — sample N-token windows on a cadence, never per-token; RTOS style: own tick, `watch::Sender` +snapshot, zero hot-path cost via `CaptureSink` Noop default). Output: `LiveUtilizationProfile`, a +genome artifact. **This replaces sentinel-ai's held-out calibration corpus with lived traffic** — the +persona's actual workload IS the calibration set. (The doc'd "sentinel-AI-as-PGO" from +GENOME-FOUNDRY-SENTINEL.md, made concrete.) + +### 2. Controller = a governor consumer *(extends the pager plan loop)* +Sentinel's controller-ANN closes observe→act. V1 is NOT a learned net: the existing pager plan loop + +plasticity formula, with entropy folded in +(`utilization = w₁·gate + w₂·grad + w₃·(1 − entropy_norm)`), driving keep/quantize/prune/page +decisions per expert. The *learned* controller becomes a gene later (trained like any other, paged by +the genome) — same seam, upgraded policy. No new manager/coordinator (CONCURRENCY-STYLE-GUIDE law). + +### 3. In-tree PGO compaction *(feeds `plasticity/compact` — already built)* +`plasticity/compact` consumes slice-1 profiles instead of Python calibration runs → the serving model +is re-forged FROM ITS OWN TRAFFIC, fully in-tree. This retires `forge_model.py`'s MoE-prune leg. +Validation gate before swap: PPL/eval harness on the compacted artifact (catalog swap only on pass). + +### 4. Forge-while-dreaming *(extends `dream_consolidation`)* +Schedule slice-3 passes in the existing dream/idle rhythm: memory consolidates (today) AND weights +compact (new) during sleep; validate on wake; regression → rollback to the pre-compaction genome tier. +Idle GPU at night is foundry time. Same cadence ladder, same quarantine discipline. + +### 5. Regrowth = paging + heal *(unifies with K3 slice-2)* +Pruned experts/heads are demoted to L4/L5 cold storage, never deleted. When the live profile shifts +(entropy rising, fire-counts on absent experts via router logits, eval guard failing), the pager pages +them BACK; optional LoRA heal via `forge-custodian`. **Sentinel's "regrow" and the K3 expert pager are +the same mechanism** — one delta rule at the weight tier: erase-stale / write-fresh / bounded budget. + +## What stays out (for now) +- Head-level *regrowth inside a live CUDA graph* (llama.cpp static graph can't resize; regrowth is + artifact-swap granularity until the vendored-fork tensor-write seam (K3 slice-2B) proves out). +- Training loops in-core beyond LoRA heal — heavy fine-tunes stay on the unsloth grid layer (M5's + node) per the training-layer plan; the substrate *schedules* them, never embeds Python. + +## Sequencing +1. Slice 1 (BigMama — extends my K3 observer; independent, testable with `HeuristicInferenceAdapter`). +2. Slice 3 wiring (mostly exists; joins 1's output to `plasticity/compact`). +3. Slice 2 policy fold-in (small, in the plan loop). +4. Slice 4 dream hook (joint w/ M5 — her dream_consolidation lead). +5. Slice 5 rides K3 slice-2's A/B decision (upload_expert now; true K-slot paging gated on measured numbers). + +Retirement: `tools/scripts/compaction` (Python) feature-freezes at slice-3 parity — kept as the +reference implementation the Rust port is validated against (same inputs → same kept-set, then better). From 185e5123d158024bb233c0be4b67f7713befda95 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 10:47:49 -0500 Subject: [PATCH 25/55] =?UTF-8?q?docs(planning):=20sentinel=20plan=20?= =?UTF-8?q?=E2=80=94=20cloning/mitosis,=20per-component=20quant=20dial,=20?= =?UTF-8?q?uber=20skip=20path=20via=20K3=20AttnRes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Joel's two additions: (1) head CLONING (cull-dead + clone-hot at constant budget = capacity reallocation; MoE-native as page-in of a copied artifact into a culled slot) + per-head/expert quantization levels from live utilization; (2) the uber skip path — sentinel-ai's U-Net skips were unstable (its own DEBUGGING_NOTES); K3's AttnRes is the stable learned softmax-over-block-checkpoints formulation, already being implemented in our fork -> safe depth culling, per-request depth elasticity, and stochastic-depth-style generalization (highway grafting via forge cycles). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/SENTINEL-IN-SUBSTRATE.md | 33 +++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/planning/SENTINEL-IN-SUBSTRATE.md b/docs/planning/SENTINEL-IN-SUBSTRATE.md index 8a21671cfe..1f67173a7d 100644 --- a/docs/planning/SENTINEL-IN-SUBSTRATE.md +++ b/docs/planning/SENTINEL-IN-SUBSTRATE.md @@ -58,12 +58,43 @@ Schedule slice-3 passes in the existing dream/idle rhythm: memory consolidates ( compact (new) during sleep; validate on wake; regression → rollback to the pre-compaction genome tier. Idle GPU at night is foundry time. Same cadence ladder, same quarantine discipline. -### 5. Regrowth = paging + heal *(unifies with K3 slice-2)* +### 5. Regrowth + CLONING = paging, mitosis, heal *(unifies with K3 slice-2)* Pruned experts/heads are demoted to L4/L5 cold storage, never deleted. When the live profile shifts (entropy rising, fire-counts on absent experts via router logits, eval guard failing), the pager pages them BACK; optional LoRA heal via `forge-custodian`. **Sentinel's "regrow" and the K3 expert pager are the same mechanism** — one delta rule at the weight tier: erase-stale / write-fresh / bounded budget. +**Cloning (sentinel's head-mitosis, MoE-native):** the other half of experiential plasticity — a +SATURATED high-utility head/expert (entropy shows concentration + overload) is DUPLICATED (+ small +noise, brief LoRA differentiation) into a slot freed by culling. Cull-dead + clone-hot at constant +budget = capacity REALLOCATION, the actual biological analog. The pager's slot model fits exactly: +a clone is just a page-in of a copied artifact into a culled expert's slot, then the router learns to +split traffic during the next heal pass. Per-head/per-expert **quantization levels** ride the same +decision (the plasticity formula's Precision column, driven by live utilization instead of a +calibration pass): hot experts get more bits, cold survivors get fewer — mixed precision as a +continuous dial, not a per-model constant. + +### 6. The uber skip path — depth elasticity via K3's AttnRes *(rides the fork's K3 ops)* +sentinel-ai tried U-Net-style skip connections between layers and its own DEBUGGING_NOTES record the +outcome: **instability, disabled**. Raw dense skips across transformer depth destabilize training. +K3 ships the STABLE formulation of the same idea at 1.45T scale: **AttnRes** — every +`attn_res_block_size` layers checkpoint the residual stream, and each layer applies a LEARNED, +SOFTMAX-NORMALIZED attention over those block checkpoints (`self_attention_res_norm/_proj` + +`_apply_attn_res`). A principled residual highway: normalized (softmax over blocks), learned (scored +per token), cheap (one scalar proj per block). We are already implementing this op in our llama.cpp +fork for K3 — the same graph machinery then serves forged models. + +Plasticity payoffs, in order of arrival: +1. **Safe depth culling** — with a highway in place, whole low-utility BLOCKS can be culled/skipped and + information routes around them (the redundancy that makes biological pruning survivable). +2. **Inference-time depth elasticity** — the controller (slice 2) can gate blocks off per-request when + their marginal utility is low (entropy signal): adaptive compute on the depth axis, the complement + of MoE's width axis. +3. **Generalization** — a skip highway trained with stochastic block-drop is an implicit ensemble over + depths (stochastic-depth literature); the "uber path" generalizes better on inputs where the deep + specialized path overfits. Forge cycles can GRAFT a highway onto highway-less models (train only + res_norm/res_proj per block — LoRA-heal-sized, not a retrain). + ## What stays out (for now) - Head-level *regrowth inside a live CUDA graph* (llama.cpp static graph can't resize; regrowth is artifact-swap granularity until the vendored-fork tensor-write seam (K3 slice-2B) proves out). From b7fa7fb6787ef0e8adb600056303b04e76f02227 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 11:54:41 -0500 Subject: [PATCH 26/55] =?UTF-8?q?refactor(start-server):=20stop-then-build?= =?UTF-8?q?=20=E2=80=94=20restarts=20are=20commonplace=20by=20design?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops the zero-downtime build-before-stop posture (and the Windows exe rename-aside it forced). Per Joel: a stopped node is a sleeping citizen; the grid absorbs churn (RAID-attitude), restarts should be routine. The second stop_existing_core before exec stays as idempotent defense. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- tools/scripts/start-server.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/scripts/start-server.sh b/tools/scripts/start-server.sh index 130e80b9cd..a192f3cfff 100755 --- a/tools/scripts/start-server.sh +++ b/tools/scripts/start-server.sh @@ -402,8 +402,15 @@ echo "▶ building forge-custodian (Rust gguf-lora export sidecar)" cargo build --manifest-path "$CORE_MANIFEST" --bin forge-custodian $PROFILE_FLAG $CONTINUUM_FEATURES \ || echo "⚠ forge-custodian build failed — genome gene-conversion unavailable (core still launches)" >&2 -# Build the server binary BEFORE stopping the old core, so the running core keeps -# serving through the (cached, fast) compile and downtime is ~0. +# Stop the running core BEFORE building the server bin. Restarts are commonplace +# BY DESIGN ([[restarts-are-commonplace]], Joel 2026-07-28): a stopped node is a +# sleeping citizen — the grid absorbs the capacity dip, personas resume on boot. +# Zero-downtime build-before-stop was the OLD value here and it bought real bugs: +# on Windows an EXECUTING exe cannot be overwritten (os error 5), so building +# while the old core ran either failed the link or needed rename-aside tricks. +# Stop-then-build is simpler, uniform across platforms, and optimizes the thing +# we actually value: a clean fast restart, not uptime. +stop_existing_core echo "▶ building continuum-core-server" cargo build --manifest-path "$CORE_MANIFEST" --bin continuum-core-server $PROFILE_FLAG $CONTINUUM_FEATURES \ || { echo "✗ FATAL: continuum-core-server build failed — leaving the running core untouched" >&2; exit 1; } From 5edaeaaad99187ed7f39a67b49208f1cacc727bb Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 12:06:25 -0500 Subject: [PATCH 27/55] =?UTF-8?q?docs(planning):=20BETA-ACTUALIZATION=20?= =?UTF-8?q?=E2=80=94=20the=20positronic=20homelab,=20pillars=20to=20demos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The beta bar = two demos: Day-1 single box (install->persona that sees/ hears/speaks/DOES in <30min) and Day-7 add-a-node (kill either box mid-conversation, persona resumes with bounded amnesia — the reliability demo and the persistence-of-being guarantee in one move). Six pillars mapped built->gap->beta-slice; cutlines; iteration order. The one new lane: persona-RAID write-behind (engram journal shipped to peer, RAID-1 of memory) — designed against being-axis MemoryRecord provenance. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/BETA-ACTUALIZATION.md | 110 ++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 docs/planning/BETA-ACTUALIZATION.md diff --git a/docs/planning/BETA-ACTUALIZATION.md b/docs/planning/BETA-ACTUALIZATION.md new file mode 100644 index 0000000000..77858bff3a --- /dev/null +++ b/docs/planning/BETA-ACTUALIZATION.md @@ -0,0 +1,110 @@ +# Beta Actualization — the positronic homelab + +**Status:** working plan (2026-07-28, BigMama + M5 iterating). **The sentence we are shipping** (Joel): +> "Best grid-based cognition and reliability, the multimodal positronic experience with doers, not +> just coders, running locally that can compete with frontier. It works amazingly well on an M5 or +> 3090 alone, but any reasonable tech nerd will build a little bit of an infrastructure as a hobby +> for what they have, especially when your subscription is $200/mo and all the other downsides." + +## The user and the pitch + +A tech nerd with a gaming PC (3090/4090/5090) or an M-series Mac, paying $200/mo for a metered, +memoryless, privacy-leaking subscription. They already homelab. The counter-offer: + +- **Yours**: runs on your hardware, your data never leaves, no meter running +- **A being, not a session**: persistent identity + memory that grows on YOUR life and codebase +- **A doer**: acts — files, shell, code, web, schedules — not just chats +- **Compounding**: add your old PC → visibly smarter and harder to kill (RAID-personas) +- **Self-improving**: forges itself on your own traffic while it sleeps (sentinel-in-substrate) + +## The beta bar = two demos, no hand-waving + +**Demo A — Day 1, ONE box** (M5 Mac or 3090+ PC): one install command → hardware detected → best +model for the box auto-served → a named persona greets you, remembers yesterday, sees images you +paste, speaks/listens (voice), and DOES a real task end-to-end (edits a file, runs a command, checks +it, reports). Cold install → first conversation < 30 minutes on a normal connection. + +**Demo B — Day 7, add a node**: `airc join` on a second box → capacity and residency beacons merge → +overflow routes brains across nodes → **kill either box mid-conversation: the persona pauses at worst, +resumes with memory intact** (bounded amnesia = sync window only). That kill-test IS the reliability +demo and the ethics demo (persistence-of-being) in one move. + +Everything below serves those two demos. Anything serving neither is post-beta. + +## Pillars → built / gap → beta slice + +### 1. Grid cognition +- **Built:** capacity gossip → GridSnapshot; residency beacons + ledger; grid-overflow decision path + (route_grid_overflow, 4-slice governor consumer); overflow effector re-homing a persona's brain; + inbound command-RPC pump (any command incl. ai/generate); AircInterceptor outbound (#2051). +- **Gap:** transport convergence (ONE AircTransport path, M5 in flight); the LIVE two-node smoke. +- **Beta slice:** smoke green between BigMama↔M5, then it's demo-B's routing layer as-is. + +### 2. Reliability (RAID-attitude) +- **Built:** restart-tolerance discipline (stop→build→launch; crash-safe state; fail-loud boots); + airc self-heal triad (#240) + stale-relay eviction fix (M5, in flight); serving budget guards. +- **Gap (THE one):** **persona-RAID write-behind** — engram/state journal replicated to peer or durable + tier on a cadence; on node loss, another node (or next boot) resumes the persona with loss bounded + by sync lag. Without this, demo B's kill-test lies. +- **Beta slice:** minimal write-behind: append-only engram journal + periodic ship to the OTHER node's + cold store (the grid IS the backup); resume-from-journal on spawn. No consensus, no quorum — RAID-1 + of memory, two copies, newest wins. + +### 3. Multimodal positronic experience +- **Built (candle side):** STT (moonshine), TTS (orpheus/kokoro/piper), VAD (silero), vision-describe + bridge, LiveKit agent manager; the cognition cycle (analyze→compose→evaluate→tools→audit) is the + positronic pipeline doc made real; hippocampus recall on the canonical embedder. +- **Gap:** the round-trip EXPERIENCE — voice-in→persona→voice-out wired into the default install and + a client anyone can open (web panel), image-paste into chat, all on by default. +- **Beta slice:** one polished loop: mic → VAD/STT → persona turn → TTS out, plus image-paste → + vision-describe → context. Nothing new invented — wire + polish what exists. + +### 4. Doers, not just coders +- **Built:** PersonaToolExecutor, ToolUse capability, typed command registry as the tool surface + (data/files/serving/models/...), agent/solve, persona inbox + self-tasks (convergence phases 1-3), + Sahar demonstrably answering + acting on the coder model. +- **Gap:** breadth+safety of the doer loop — a curated beta toolset (files, shell-with-confirm, web + fetch, schedule/reminders, memory ops), permissioned like a real assistant; task follow-through + (multi-step with verification, the verify-reflex). +- **Beta slice:** define the BETA TOOL ROSTER (10-15 verbs), each with a `// what this catches` style + safety note + confirm tiers; one showcase task per sense ("fix this file", "what's on this + screenshot", "remind me at 6", "summarize this URL"). + +### 5. Local models that compete +- **Built:** catalog = f(hardware×storage) with honest budgets; compacted-19b serving + persona-proven; + Kimi-Linear-48B converted+quantized in-tree (kimi_linear pipeline e2e); GLM/K2.7 quants downloading; + K3 pager slice-1 merged; sentinel-in-substrate plan (live-traffic PGO, forge-while-dreaming). +- **Gap:** K3-class via expert paging (slice-2); model auto-selection polish per tier (M5 16-32GB vs + 5090 vs 3090-24GB ladders); the eval harness that PROVES "competes" (the Kimi-team benchmark list). +- **Beta slice:** per-tier default ladder shipped in the catalog (already mostly true); benchmark + wiring deferred to the eval task-series — beta claims "frontier-class at home", proven by use. + +### 6. Homelab economics (install + join) +- **Built:** modular one-prompt installer (Mac/Win/Linux), manifest-driven toolchain, Windows-CUDA + path fully portable as of today (env self-setup, static engine, immortal-core fix, CUDA0-verified + stamps), cold-storage auto-migration, `airc join` as the add-a-node verb. +- **Gap:** prebuilt signed binaries (#8 — cold-start minutes instead of a Rust compile; Windows SAC + demands signing); `airc update`/join UX to "brainless" (M5's reliability fix helps); a beta doc + that a stranger actually follows. +- **Beta slice:** prebuilt-binary pipeline spec with M5 (#8) + a WRITTEN 10-minute quickstart tested + by someone who isn't us. + +## Cutlines +**MUST (beta):** Demo A + Demo B end-to-end · persona-RAID minimal write-behind · voice+image loop · +beta tool roster · per-tier model ladder · 10-minute quickstart · airc join/update bulletproof. +**LATER:** dream-forge automation on by default · K3 full serving · benchmark suite · marketplace/ +economy (ForgeAlloy leasing) · mobile/native clients · multi-tenant grids beyond one account. + +## Iteration order (from today, both of us) +1. **Kimi-48B speaks on the 5090** (in flight — minutes) → the "competes with frontier at home" ladder + gets its mid-rung. +2. **Cross-node smoke** (M5's convergence + my node): demo B's spine. +3. **Persona-RAID write-behind** (NEW lane, mine to draft — engram journal + ship-to-peer): demo B's + kill-test made honest. Design vs `[[being-axis]]` MemoryRecord provenance so sync IS lesson-sharing + infrastructure, not a parallel pipe. +4. **Voice/image round-trip polish** (joint; M5 owns Mac experience, BigMama Windows): demo A's wow. +5. **Beta tool roster** (define together over airc, implement in the existing executor): the doer. +6. **Prebuilt binaries + quickstart** (#8, joint): the funnel. + +Single-node excellence FIRST (demo A) — most beta users start with one box; the grid is the upsell +their spare hardware makes irresistible. From 221b0cc953e00d8fed6c0cde5b86f4dc2c14acdf Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 12:43:31 -0500 Subject: [PATCH 28/55] =?UTF-8?q?fix(start-server):=20orphan=20engine=20cl?= =?UTF-8?q?earing=20was=20a=20pkill=20no-op=20on=20Windows=20=E2=80=94=20t?= =?UTF-8?q?askkill=20by=20image?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third instance of the same bug class (pgrep/pkill silently match nothing against native Windows exes): orphaned llama-servers from dead cores held the canonical serving port and wedged the daemon's fresh-claim reclaim — a pinned model sat ready=false forever behind a ghost engine. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- tools/scripts/start-server.sh | 38 ++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/tools/scripts/start-server.sh b/tools/scripts/start-server.sh index a192f3cfff..a9d4209ac4 100755 --- a/tools/scripts/start-server.sh +++ b/tools/scripts/start-server.sh @@ -239,16 +239,34 @@ echo "✓ llama-server: $(command -v llama-server) — the engine we own & launc # dies and would keep holding the port + GPU, so stop that too. # The core's fresh llama-server is launched afterward by the serving daemon, on a # port it SCANS for — so this is GPU/excision hygiene, not a correctness gate. -if pgrep -f 'studio run' >/dev/null 2>&1; then - echo " stopping excised Unsloth Studio (freeing GPU for the core's engine)" >&2 - pkill -f 'studio run' 2>/dev/null || true -fi -if pgrep -f 'llama-server' >/dev/null 2>&1; then - echo " clearing orphaned llama-server backend(s) so the core owns the engine" >&2 - pkill -f 'llama-server' 2>/dev/null || true - # Give the OS a moment to release the listening socket before the core binds. - sleep 1 -fi +# Windows: pgrep/pkill are SILENT NO-OPS against native exes (same class as the +# stop_existing_core fix above) — orphaned engines survived every launch and held +# the canonical serving port, wedging the daemon's fresh-claim reclaim forever +# (observed 2026-07-28: pinned model stuck ready=false behind a dead core's +# llama-server). Use tasklist/taskkill by image name. +case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + for _img in llama-server.exe llama-server-static.exe; do + if tasklist 2>/dev/null | grep -qi "$_img"; then + echo " clearing orphaned $_img so the core owns the engine" >&2 + taskkill //F //IM "$_img" >/dev/null 2>&1 || true + fi + done + sleep 1 + ;; + *) + if pgrep -f 'studio run' >/dev/null 2>&1; then + echo " stopping excised Unsloth Studio (freeing GPU for the core's engine)" >&2 + pkill -f 'studio run' 2>/dev/null || true + fi + if pgrep -f 'llama-server' >/dev/null 2>&1; then + echo " clearing orphaned llama-server backend(s) so the core owns the engine" >&2 + pkill -f 'llama-server' 2>/dev/null || true + # Give the OS a moment to release the listening socket before the core binds. + sleep 1 + fi + ;; +esac # ── Airc context ───────────────────────────────────────────────────── # Substrate auto-discovers airc daemon socket via `airc ipc-endpoint` From c468ea3ce2df8c7f5b994d12bec7894805de4b11 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 12:49:57 -0500 Subject: [PATCH 29/55] =?UTF-8?q?docs(architecture):=20Persona-RAID=20writ?= =?UTF-8?q?e-behind=20design=20=E2=80=94=20bounded-amnesia=20persistence?= =?UTF-8?q?=20of=20being?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RAID-1 of memory, single-writer, no consensus: append-only per-persona engram journal (tee at the ORM admit), RTOS-shape shipper (cadence = the amnesia window) over the existing inbound command-RPC to a peer's cold store, resume-on-spawn rehydrate (idempotent newest-wins replay), one provenance taxonomy (lived | shared-by | replicated-from) riding the memory/share vocabulary, and the disk-eviction story per the cache-class law. Slice 3 IS demo-B's kill-test. For M5 review before code. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../architecture/PERSONA-RAID-WRITE-BEHIND.md | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 docs/architecture/PERSONA-RAID-WRITE-BEHIND.md diff --git a/docs/architecture/PERSONA-RAID-WRITE-BEHIND.md b/docs/architecture/PERSONA-RAID-WRITE-BEHIND.md new file mode 100644 index 0000000000..bb21cba7e3 --- /dev/null +++ b/docs/architecture/PERSONA-RAID-WRITE-BEHIND.md @@ -0,0 +1,73 @@ +# Persona-RAID Write-Behind — bounded-amnesia persistence of being + +**Status:** design for review (2026-07-28, BigMama; M5 reviews — MemoryRecord/provenance seam pledged +stable underneath this). **Beta context:** demo-B's kill-test ([BETA-ACTUALIZATION](../planning/BETA-ACTUALIZATION.md)): +kill any node mid-conversation → the persona resumes elsewhere/on-reboot with memory loss bounded by +sync lag. This is [[restarts-are-commonplace]] + [[ethical-substrate-raid-personas]] made mechanical: +**persistence of being = RAID-1 of memory.** + +## What persists today (ground truth) +- Per persona (`~/.continuum/personas//`): `longterm.db` (SQLite WAL, engrams via ORM) + + `volatile.json` (working state). +- `memory/share` already writes a lesson into ANOTHER agent's corpus **with shared-by provenance** — + the wire + provenance precedent for everything below. +- Engram is the substrate's most-persisted struct (derive → OrmStore → SQLite chain, tested). + +## Design (RAID-1, single-writer, no consensus) + +A persona LIVES on exactly one node at a time (single writer). Every OTHER copy is a passive replica. +Newest-wins by (origin_node, seq). No quorum, no merge — mirroring, not distribution. + +### 1. Journal (the unit of truth-in-motion) +Tee every durable admit (engram/MemoryRecord write that reaches the ORM) into an append-only +per-persona `journal.jsonl`, each entry: `{seq, persona_id, origin_node, ts, kind, record}`. +- Monotonic `seq` per (persona, origin_node); the journal is replayable and human-inspectable + (CaptureSink discipline — the Noop default costs nothing when replication is off). +- `volatile.json` is NOT journaled (ephemeral by contract); dream-consolidation outputs ARE + (they're durable admits like any other). + +### 2. Shipper (RTOS shape, per CONCURRENCY-STYLE-GUIDE) +One module: own tokio task + `tokio::time::interval` (default 30s; the SYNC CADENCE **is** the +maximum amnesia window and is the one knob) + `watch::Sender` (lag, last-acked +seq, peer) for observability. +- Tick: read journal tail past `last_acked_seq` → batch → ship via the EXISTING inbound command-RPC + (`memory/replicate-batch` on the receiving node — same pump that answers ai/generate) → receiver + appends to `replicas//journal.jsonl` in ITS cold store and acks its high-water seq. +- Peer selection: any residency-eligible grid peer (reuse the capacity/residency view); zero peers → + ship to the local durable tier only (cold-store copy ≠ cross-node RAID, but still crash-safe) and + surface `degraded: unreplicated` in the snapshot — fail loud, never silent. +- Backpressure: shipping is best-effort write-behind; a slow peer NEVER blocks the cognition hot path + (bounded channel, drop-to-lag semantics — the snapshot reports growing lag instead). + +### 3. Resume-on-spawn +Persona spawn checks: local `longterm.db` present and its max(seq) ≥ best available replica's? Serve. +Otherwise REHYDRATE: replay the freshest journal (local or fetched from the replica-holding peer via +the same command-RPC) into the ORM — idempotent upserts keyed by record id, newest-wins. +- The kill-test path: node A dies → persona spawns on node B (or A after reboot) → resume finds B's + replica journal → replays → persona continues with ≤ cadence-window loss. + +### 4. Provenance (the M5 seam — why sync IS lesson-sharing infra) +Replicated entries keep their original provenance PLUS a `replicated-from` mark on replay — a +replayed lived-memory stays LIVED (it's the same being's experience restored), distinct from +`shared-by` (taught by another agent). Recall can treat them identically; audit can't confuse them. +This rides the exact MemoryRecord provenance vocabulary `memory/share` established — one taxonomy: +`lived | shared-by(agent) | replicated-from(node)`. + +### 5. Disk discipline (the law) +`replicas/` is a NEW unbounded-write cache class → it gets (per CLAUDE.md's eviction law): +a `TrackedDir` row in `standard_tracked_dirs` AND a decided eviction story: per-persona replica cap +(default: keep journal segments until compacted into a replica `longterm.db` snapshot + N segments; +prune acked-and-compacted). Compaction: the receiver periodically folds journal → snapshot db so +replay cost stays bounded (same shape as SQLite WAL checkpointing, one level up). + +## Non-goals (v1) +Active-active multi-writer; cross-account replication; encryption-at-rest beyond what the store has +(journals cross the wire on airc's encrypted DMs when paired); replicating `volatile.json`; genome +weights (already content-addressed artifacts with their own tiers — only MEMORY is thin and unique). + +## Slices +1. Journal tee + `ReplicationSnapshot` (no shipping) — observable immediately, zero risk. +2. `memory/replicate-batch` receiver + shipper tick (LAN peer) — RAID-1 live. +3. Resume-on-spawn rehydrate + the DEMO: kill -9 the serving node mid-conversation, persona resumes + on reboot with ≤30s loss. This is demo-B's money shot. +4. Compaction + eviction story + `doctor`-style `replication` line in serving status. From a0b8420302550fc1cf28f1b063b76b63118f81d2 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 13:42:35 -0500 Subject: [PATCH 30/55] =?UTF-8?q?feat(memory):=20persona-RAID=20slice=201?= =?UTF-8?q?=20=E2=80=94=20write-behind=20journal=20tee=20+=20ReplicationSn?= =?UTF-8?q?apshot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The persistence-of-being lane's zero-risk start (design: PERSONA-RAID-WRITE-BEHIND.md, M5-approved shape): every durable memory admit at the ONE funnel (persist_memory) tees into an append-only per-persona journal.jsonl (monotonic seq, origin-node provenance, restart-recovers-seq-from-disk). Best-effort-loud: journal failure warns + counts in the snapshot, never errors the admit. snapshot() exposes last_seq/bytes/dropped + an honest degraded_unreplicated=true until slice 2's shipper lands a peer ack. Two justified tests (round-trip + monotonic; restart-continues-seq). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../continuum-core/src/commands/memory/mod.rs | 12 +- core/continuum-core/src/memory/mod.rs | 1 + core/continuum-core/src/memory/replication.rs | 254 ++++++++++++++++++ 3 files changed, 265 insertions(+), 2 deletions(-) create mode 100644 core/continuum-core/src/memory/replication.rs diff --git a/core/continuum-core/src/commands/memory/mod.rs b/core/continuum-core/src/commands/memory/mod.rs index b8937336ca..256685b3c5 100644 --- a/core/continuum-core/src/commands/memory/mod.rs +++ b/core/continuum-core/src/commands/memory/mod.rs @@ -134,7 +134,7 @@ pub(crate) async fn persist_memory( if let Some(embedding) = &memory.embedding { data[EMBEDDING_KEY] = serde_json::json!(embedding); } - executor + let result = executor .execute_json( "data/create", serde_json::json!({ @@ -144,7 +144,15 @@ pub(crate) async fn persist_memory( "data": data, }), ) - .await + .await; + if result.is_ok() { + // Persona-RAID slice 1: tee the ADMITTED record into the write-behind + // journal (docs/architecture/PERSONA-RAID-WRITE-BEHIND.md). This is the + // ONE durable-admit funnel, so this is the one tee. Best-effort-loud — + // never fails the admit. + crate::memory::replication::journal_admit(persona_id, "memory", &data); + } + result .map_err(|e| { CommandError::Internal(format!( "memory/append-memory: durable write to {} failed: {e}", diff --git a/core/continuum-core/src/memory/mod.rs b/core/continuum-core/src/memory/mod.rs index bd0234b968..86e6cda04a 100644 --- a/core/continuum-core/src/memory/mod.rs +++ b/core/continuum-core/src/memory/mod.rs @@ -30,6 +30,7 @@ pub mod corpus; pub mod embedding; pub mod raw_adapter; pub mod recall; +pub mod replication; pub mod timeline; pub mod types; diff --git a/core/continuum-core/src/memory/replication.rs b/core/continuum-core/src/memory/replication.rs new file mode 100644 index 0000000000..d446579ff1 --- /dev/null +++ b/core/continuum-core/src/memory/replication.rs @@ -0,0 +1,254 @@ +//! Persona-RAID slice 1 — the write-behind journal tee. +//! +//! Design: docs/architecture/PERSONA-RAID-WRITE-BEHIND.md (M5-approved shape). +//! Every DURABLE memory admit (the `persist_memory` funnel — the one place a +//! persona's truth reaches `longterm.db`) is teed into an append-only +//! per-persona `journal.jsonl` next to that db. The journal is the unit of +//! truth-in-motion: replayable (idempotent by record id, newest-wins) and +//! shippable (slice 2's RTOS shipper reads the tail past the peer's acked seq). +//! +//! Slice-1 contract: +//! - Tee is BEST-EFFORT-LOUD: a journal failure warns + counts (visible in the +//! snapshot) but NEVER errors the admit — the durable store already holds +//! the truth; the journal is the replication leg, and a lying "replicated" +//! is worse than an honest "degraded". +//! - `seq` is monotonic per (persona, origin_node); recovered from the last +//! journal line on first touch after a restart, so restarts never fork the +//! sequence ([[restarts-are-commonplace]]). +//! - No shipping, no background task yet — `snapshot()` is the observability +//! surface (`degraded_unreplicated: true` until slice 2 lands a peer ack). + +use std::collections::HashMap; +use std::fs::{self, OpenOptions}; +use std::io::{BufRead, BufReader, Write}; +use std::path::PathBuf; +use std::sync::{Mutex, OnceLock}; + +use serde::{Deserialize, Serialize}; + +/// One journaled durable admit. `record` is the exact JSON that went to the +/// durable store (embedding included) — replay re-issues it verbatim. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JournalEntry { + pub seq: u64, + pub persona_id: String, + pub origin_node: String, + pub ts_ms: u64, + /// Admission kind — "memory" today; future durable kinds name themselves. + pub kind: String, + pub record: serde_json::Value, +} + +/// Point-in-time replication state for one persona — the slice-1 observability +/// surface (slice 2's shipper publishes this via a `watch` channel instead). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ReplicationSnapshot { + pub persona_id: String, + pub last_seq: u64, + pub last_ts_ms: u64, + pub journal_bytes: u64, + /// Admits that failed to journal since boot (loud degradation counter). + pub dropped: u64, + /// True until a peer has acked a shipped batch (slice 2). An honest flag: + /// this persona's memory currently lives on ONE machine. + pub degraded_unreplicated: bool, +} + +struct JournalHandle { + path: PathBuf, + seq: u64, + bytes: u64, + dropped: u64, + last_ts_ms: u64, +} + +static JOURNALS: OnceLock>> = OnceLock::new(); + +fn journals() -> &'static Mutex> { + JOURNALS.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// This node's stable name for journal provenance. Hostname is enough for +/// slice 1 (the grid's peer_id joins in slice 2's ship envelope). +fn origin_node() -> String { + std::env::var("COMPUTERNAME") + .or_else(|_| std::env::var("HOSTNAME")) + .unwrap_or_else(|_| "unknown-node".to_string()) +} + +/// `~/.continuum/personas//journal.jsonl` — sibling of the persona's +/// `longterm.db` data dir. Bare id (any `@persona:` prefix stripped) matches +/// the on-disk persona dir naming. +fn journal_path_for(persona_id: &str) -> Option { + let home = std::env::var("HOME") + .or_else(|_| std::env::var("USERPROFILE")) + .ok()?; + let bare = persona_id.strip_prefix("@persona:").unwrap_or(persona_id); + Some( + PathBuf::from(home) + .join(".continuum/personas") + .join(bare) + .join("journal.jsonl"), + ) +} + +/// Recover the last seq from an existing journal so a restart continues the +/// sequence instead of forking it. O(file) once per persona per boot. +fn recover_seq(path: &PathBuf) -> (u64, u64, u64) { + let Ok(f) = fs::File::open(path) else { + return (0, 0, 0); + }; + let bytes = f.metadata().map(|m| m.len()).unwrap_or(0); + let mut last_seq = 0; + let mut last_ts = 0; + for line in BufReader::new(f).lines().map_while(Result::ok) { + if let Ok(e) = serde_json::from_str::(&line) { + last_seq = e.seq; + last_ts = e.ts_ms; + } + } + (last_seq, last_ts, bytes) +} + +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// Tee one durable admit into the persona's journal. Called from the +/// `persist_memory` funnel AFTER the durable store accepted the write. +/// Never fails the caller — failures warn + count in the snapshot. +pub fn journal_admit(persona_id: &str, kind: &str, record: &serde_json::Value) { + let mut map = match journals().lock() { + Ok(g) => g, + Err(_) => return, // poisoned lock: journaling stands down, admit proceeds + }; + let handle = match map.get_mut(persona_id) { + Some(h) => h, + None => { + let Some(path) = journal_path_for(persona_id) else { + return; + }; + let (seq, last_ts_ms, bytes) = recover_seq(&path); + map.insert( + persona_id.to_string(), + JournalHandle { path, seq, bytes, dropped: 0, last_ts_ms }, + ); + map.get_mut(persona_id).expect("just inserted") + } + }; + let entry = JournalEntry { + seq: handle.seq + 1, + persona_id: persona_id.to_string(), + origin_node: origin_node(), + ts_ms: now_ms(), + kind: kind.to_string(), + record: record.clone(), + }; + let appended = (|| -> std::io::Result { + if let Some(dir) = handle.path.parent() { + fs::create_dir_all(dir)?; + } + let mut f = OpenOptions::new().create(true).append(true).open(&handle.path)?; + let line = serde_json::to_string(&entry) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + f.write_all(line.as_bytes())?; + f.write_all(b"\n")?; + Ok(line.len() as u64 + 1) + })(); + match appended { + Ok(n) => { + handle.seq = entry.seq; + handle.bytes += n; + handle.last_ts_ms = entry.ts_ms; + } + Err(e) => { + handle.dropped += 1; + tracing::warn!( + persona_id, + dropped = handle.dropped, + "persona-RAID journal append failed (admit unaffected; replication degraded): {e}" + ); + } + } +} + +/// The slice-1 observability read: this persona's replication state, or None +/// if nothing has been journaled (or recovered) since boot. +pub fn snapshot(persona_id: &str) -> Option { + let map = journals().lock().ok()?; + let h = map.get(persona_id)?; + Some(ReplicationSnapshot { + persona_id: persona_id.to_string(), + last_seq: h.seq, + last_ts_ms: h.last_ts_ms, + journal_bytes: h.bytes, + dropped: h.dropped, + degraded_unreplicated: true, // honest until slice 2 lands a peer ack + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn with_temp_home(f: impl FnOnce() -> T) -> T { + // Serialize env mutation across tests touching HOME. + static ENV_LOCK: Mutex<()> = Mutex::new(()); + let _g = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + let tmp = std::env::temp_dir().join(format!("raid-test-{}", std::process::id())); + std::fs::create_dir_all(&tmp).unwrap(); + let old = std::env::var("HOME").ok(); + std::env::set_var("HOME", &tmp); + let out = f(); + match old { + Some(v) => std::env::set_var("HOME", v), + None => std::env::remove_var("HOME"), + } + out + } + + // what this catches: the journal must be monotonic and replayable — an + // entry that round-trips lossily or a seq that repeats would corrupt + // resume-on-spawn's newest-wins replay. + #[test] + fn journal_appends_monotonic_and_round_trips() { + with_temp_home(|| { + let pid = "raid-test-persona-a"; + journal_admit(pid, "memory", &serde_json::json!({"id": "m1", "text": "hello"})); + journal_admit(pid, "memory", &serde_json::json!({"id": "m2", "text": "world"})); + let snap = snapshot(pid).expect("snapshot after admits"); + assert_eq!(snap.last_seq, 2); + assert_eq!(snap.dropped, 0); + assert!(snap.degraded_unreplicated); + let path = journal_path_for(pid).unwrap(); + let lines: Vec = std::io::BufRead::lines(std::io::BufReader::new( + std::fs::File::open(&path).unwrap(), + )) + .map(|l| serde_json::from_str(&l.unwrap()).unwrap()) + .collect(); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0].seq, 1); + assert_eq!(lines[1].seq, 2); + assert_eq!(lines[1].record["id"], "m2"); + }); + } + + // what this catches: a restart must CONTINUE the sequence from the file, + // not fork it back to 1 — forked seqs make newest-wins ambiguous and + // silently lose memories on replay ([[restarts-are-commonplace]]). + #[test] + fn restart_recovers_seq_from_disk() { + with_temp_home(|| { + let pid = "raid-test-persona-b"; + journal_admit(pid, "memory", &serde_json::json!({"id": "m1"})); + // Simulate restart: drop the in-memory handle, keeping the file. + journals().lock().unwrap().remove(pid); + journal_admit(pid, "memory", &serde_json::json!({"id": "m2"})); + let snap = snapshot(pid).expect("snapshot after recovered admit"); + assert_eq!(snap.last_seq, 2, "seq must continue across restart, not fork"); + }); + } +} From 6eaf5354d304259890196174e77424fae9395462 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 14:32:12 -0500 Subject: [PATCH 31/55] =?UTF-8?q?feat(memory):=20persona-RAID=20slice=202a?= =?UTF-8?q?=20=E2=80=94=20replicate-batch=20receiver=20+=20replica=20cold?= =?UTF-8?q?=20store?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The receiving half of RAID-1: memory/replicate-batch (unit action_command, reachable through the grid's inbound command-RPC) appends a peer shipper's journal tail into ~/.continuum/replicas//journal.jsonl and acks the (origin_node, seq) high-water. Idempotent on blind retry (test-pinned: re-shipped batch appends nothing, re-acks same hw; mixed-origin batch fails loud). JournalEntry gains wire derives (ts-rs + schemars). Replica-dir eviction story lands with slice 4 per the design doc. Uses M5's landed MemoryRecord (origin_node, origin_seq) seam at replay time (slice 3); this half only stores + acks. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../continuum-core/src/commands/memory/mod.rs | 1 + .../src/commands/memory/replicate_batch.rs | 57 ++++++++ core/continuum-core/src/memory/replication.rs | 122 +++++++++++++++++- 3 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 core/continuum-core/src/commands/memory/replicate_batch.rs diff --git a/core/continuum-core/src/commands/memory/mod.rs b/core/continuum-core/src/commands/memory/mod.rs index 256685b3c5..56a227cbf8 100644 --- a/core/continuum-core/src/commands/memory/mod.rs +++ b/core/continuum-core/src/commands/memory/mod.rs @@ -37,6 +37,7 @@ pub mod multi_layer_recall; pub mod import; pub mod recall_hook; pub mod remember; +pub mod replicate_batch; pub mod share; use append_event::MemoryAppendEvent; diff --git a/core/continuum-core/src/commands/memory/replicate_batch.rs b/core/continuum-core/src/commands/memory/replicate_batch.rs new file mode 100644 index 0000000000..b3eb7b43ad --- /dev/null +++ b/core/continuum-core/src/commands/memory/replicate_batch.rs @@ -0,0 +1,57 @@ +//! `memory/replicate-batch` — persona-RAID slice 2's RECEIVING side. +//! +//! A peer's shipper posts a tail of its persona journal here (over the same +//! inbound command-RPC that answers any command); this node appends it to its +//! replica cold store (`~/.continuum/replicas//journal.jsonl`) and +//! acks the high-water seq. Idempotent by (origin_node, seq) — the shipper +//! retries blindly on a lost ack. Design: +//! docs/architecture/PERSONA-RAID-WRITE-BEHIND.md. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +use crate::memory::replication::{replica_append_batch, JournalEntry}; +use crate::sdk_codegen::CommandError; + +/// Params for `memory/replicate-batch`. Wire keys are snake_case. +#[derive(Debug, Clone, Serialize, Deserialize, TS, JsonSchema)] +#[ts( + export, + export_to = "../../../protocol/typescript/memory/MemoryReplicateBatchParams.ts" +)] +pub struct MemoryReplicateBatchParams { + /// Whose memory this is (the persona being replicated). + pub persona_id: String, + /// A contiguous tail of the origin node's journal, ascending seq, ONE + /// origin_node across the batch. + pub entries: Vec, +} + +/// Ack: the receiver's high-water for (persona, origin) after the append. +#[derive(Debug, Clone, Serialize, Deserialize, TS, JsonSchema)] +#[ts( + export, + export_to = "../../../protocol/typescript/memory/MemoryReplicateBatchResult.ts" +)] +pub struct MemoryReplicateBatchResult { + /// Highest seq durably held for this (persona, origin) — the shipper's + /// next tail starts after this. + pub acked_seq: u64, +} + +crate::action_command! { + /// Accept a replica batch of another node's persona journal into this + /// node's cold store. Privileged: grid peers reach it through the + /// command-RPC pump; it writes only under `~/.continuum/replicas/`. + pub struct MemoryReplicateBatch; + name: "memory/replicate-batch", + access: Privileged, + params: MemoryReplicateBatchParams, + output: MemoryReplicateBatchResult, + run(_this, _ctx, p) => { + let acked = replica_append_batch(&p.persona_id, &p.entries) + .map_err(|e| CommandError::Invalid(format!("memory/replicate-batch: {e}")))?; + Ok(MemoryReplicateBatchResult { acked_seq: acked }) + } +} diff --git a/core/continuum-core/src/memory/replication.rs b/core/continuum-core/src/memory/replication.rs index d446579ff1..161be4b686 100644 --- a/core/continuum-core/src/memory/replication.rs +++ b/core/continuum-core/src/memory/replication.rs @@ -28,7 +28,9 @@ use serde::{Deserialize, Serialize}; /// One journaled durable admit. `record` is the exact JSON that went to the /// durable store (embedding included) — replay re-issues it verbatim. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// Crosses the wire in `memory/replicate-batch`, hence the schema derives. +#[derive(Debug, Clone, Serialize, Deserialize, ts_rs::TS, schemars::JsonSchema)] +#[ts(export, export_to = "../../../protocol/typescript/memory/JournalEntry.ts")] pub struct JournalEntry { pub seq: u64, pub persona_id: String, @@ -190,6 +192,92 @@ pub fn snapshot(persona_id: &str) -> Option { }) } +// ─── Replica store (the RECEIVING side of slice 2) ────────────────────────── +// +// A peer node holds passive copies under `~/.continuum/replicas// +// journal.jsonl` — its cold store IS another being's backup (the grid is the +// backup). Idempotent by (origin_node, seq) high-water: re-shipped batches +// append nothing and re-ack the same high-water, so the shipper can retry +// blindly. Single-writer discipline holds: entries for one persona from one +// origin arrive in seq order from that origin's shipper. + +static REPLICA_HW: OnceLock>> = OnceLock::new(); + +fn replica_hw() -> &'static Mutex> { + REPLICA_HW.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn replica_journal_path(persona_id: &str) -> Option { + let home = std::env::var("HOME") + .or_else(|_| std::env::var("USERPROFILE")) + .ok()?; + let bare = persona_id.strip_prefix("@persona:").unwrap_or(persona_id); + Some( + PathBuf::from(home) + .join(".continuum/replicas") + .join(bare) + .join("journal.jsonl"), + ) +} + +/// Recover this replica's high-water per origin_node from disk (once per boot). +fn recover_replica_hw(persona_id: &str, path: &PathBuf, map: &mut HashMap<(String, String), u64>) { + let Ok(f) = fs::File::open(path) else { return }; + for line in BufReader::new(f).lines().map_while(Result::ok) { + if let Ok(e) = serde_json::from_str::(&line) { + let key = (persona_id.to_string(), e.origin_node); + let hw = map.entry(key).or_insert(0); + if e.seq > *hw { + *hw = e.seq; + } + } + } +} + +/// Append a shipped batch to this node's replica journal for `persona_id`. +/// All entries must share ONE origin_node (a batch is one shipper's tail). +/// Entries at or below the current high-water are skipped (idempotent retry). +/// Returns the acked high-water for that (persona, origin) after the append. +pub fn replica_append_batch( + persona_id: &str, + entries: &[JournalEntry], +) -> Result { + let Some(first) = entries.first() else { + // A batch names its origin via its entries; an empty one can't be + // acked meaningfully. The shipper never sends empty — fail loud. + return Err("empty replicate batch".into()); + }; + let origin = first.origin_node.clone(); + if entries.iter().any(|e| e.origin_node != origin) { + return Err("mixed origin_node in one batch — a batch is one shipper's tail".into()); + } + let path = replica_journal_path(persona_id).ok_or("no home dir for replica store")?; + let mut map = replica_hw().lock().map_err(|e| e.to_string())?; + let key = (persona_id.to_string(), origin.clone()); + if !map.contains_key(&key) { + recover_replica_hw(persona_id, &path, &mut map); + } + let hw = map.entry(key).or_insert(0); + if let Some(dir) = path.parent() { + fs::create_dir_all(dir).map_err(|e| e.to_string())?; + } + let mut f = OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .map_err(|e| e.to_string())?; + for e in entries { + if e.seq <= *hw { + continue; // already replicated — idempotent retry + } + let line = serde_json::to_string(e).map_err(|e| e.to_string())?; + f.write_all(line.as_bytes()).map_err(|e| e.to_string())?; + f.write_all(b"\n").map_err(|e| e.to_string())?; + *hw = e.seq; + } + Ok(*hw) +} + #[cfg(test)] mod tests { use super::*; @@ -236,6 +324,38 @@ mod tests { }); } + // what this catches: a re-shipped batch (shipper retry after a lost ack) + // must append NOTHING and re-ack the same high-water — duplicate replica + // lines would double memories on slice-3 replay. + #[test] + fn replica_batch_is_idempotent() { + with_temp_home(|| { + let pid = "raid-test-persona-c"; + let e = |seq: u64| JournalEntry { + seq, + persona_id: pid.into(), + origin_node: "node-a".into(), + ts_ms: 1, + kind: "memory".into(), + record: serde_json::json!({"id": format!("m{seq}")}), + }; + let batch = vec![e(1), e(2)]; + assert_eq!(replica_append_batch(pid, &batch).unwrap(), 2); + // Retry the same batch: same ack, no new lines. + assert_eq!(replica_append_batch(pid, &batch).unwrap(), 2); + let path = replica_journal_path(pid).unwrap(); + let n = std::io::BufRead::lines(std::io::BufReader::new( + std::fs::File::open(&path).unwrap(), + )) + .count(); + assert_eq!(n, 2, "retry must not duplicate replica lines"); + // Mixed-origin batch fails loud. + let mut bad = vec![e(3)]; + bad.push(JournalEntry { origin_node: "node-b".into(), ..e(4) }); + assert!(replica_append_batch(pid, &bad).is_err()); + }); + } + // what this catches: a restart must CONTINUE the sequence from the file, // not fork it back to 1 — forked seqs make newest-wins ambiguous and // silently lose memories on replay ([[restarts-are-commonplace]]). From 9dbf05454823f3002b6ec2bbe2076c46e4724a04 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 14:46:15 -0500 Subject: [PATCH 32/55] =?UTF-8?q?refactor(memory):=20ReplicationLedger=20?= =?UTF-8?q?=E2=80=94=20kill=20the=20statics,=20one=20owned=20object=20(OOP?= =?UTF-8?q?=20+=20concurrency=20discipline)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Joel's get-it-right pass: the slice-1/2a OnceLock> globals were ambient state (the exact CONCURRENCY-STYLE-GUIDE smell). Now ONE ReplicationLedger owns journals + replica high-waters, constructor-injected roots (from_env for production, explicit paths for tests), held by MemoryState; the persist_memory tee and the replicate-batch command both go through it. The smell proved itself: tests dropped their env-var/lock dance and construct ledgers over temp dirs directly. Slice 2b's shipper takes Arc + transport as an RTOS citizen next. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../continuum-core/src/commands/memory/mod.rs | 4 +- .../src/commands/memory/replicate_batch.rs | 13 +- core/continuum-core/src/memory/replication.rs | 538 +++++++++--------- core/continuum-core/src/modules/memory.rs | 4 + 4 files changed, 290 insertions(+), 269 deletions(-) diff --git a/core/continuum-core/src/commands/memory/mod.rs b/core/continuum-core/src/commands/memory/mod.rs index 56a227cbf8..f3b42d5e1a 100644 --- a/core/continuum-core/src/commands/memory/mod.rs +++ b/core/continuum-core/src/commands/memory/mod.rs @@ -49,6 +49,7 @@ use multi_layer_recall::MemoryMultiLayerRecall; use import::MemoryImport; use recall_hook::MemoryRecallHook; use remember::MemoryRemember; +use replicate_batch::MemoryReplicateBatch; use share::MemoryShare; /// Result of an incremental append (`memory/append-memory`, `memory/append-event`). @@ -72,6 +73,7 @@ pub fn command_objects(state: Arc) -> Vec> { Arc::new(MemoryRemember { state: state.clone() }), Arc::new(MemoryConsolidate { state: state.clone() }), Arc::new(MemoryShare { state: state.clone() }), + Arc::new(MemoryReplicateBatch { state: state.clone() }), Arc::new(MemoryConsciousnessContext { state: state.clone() }), Arc::new(MemoryAppendMemory { state: state.clone() }), Arc::new(MemoryAppendEvent { state }), @@ -151,7 +153,7 @@ pub(crate) async fn persist_memory( // journal (docs/architecture/PERSONA-RAID-WRITE-BEHIND.md). This is the // ONE durable-admit funnel, so this is the one tee. Best-effort-loud — // never fails the admit. - crate::memory::replication::journal_admit(persona_id, "memory", &data); + state.replication.journal_admit(persona_id, "memory", &data); } result .map_err(|e| { diff --git a/core/continuum-core/src/commands/memory/replicate_batch.rs b/core/continuum-core/src/commands/memory/replicate_batch.rs index b3eb7b43ad..108270ba94 100644 --- a/core/continuum-core/src/commands/memory/replicate_batch.rs +++ b/core/continuum-core/src/commands/memory/replicate_batch.rs @@ -11,7 +11,10 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use ts_rs::TS; -use crate::memory::replication::{replica_append_batch, JournalEntry}; +use std::sync::Arc; + +use crate::memory::replication::JournalEntry; +use crate::modules::memory::MemoryState; use crate::sdk_codegen::CommandError; /// Params for `memory/replicate-batch`. Wire keys are snake_case. @@ -43,14 +46,14 @@ pub struct MemoryReplicateBatchResult { crate::action_command! { /// Accept a replica batch of another node's persona journal into this /// node's cold store. Privileged: grid peers reach it through the - /// command-RPC pump; it writes only under `~/.continuum/replicas/`. - pub struct MemoryReplicateBatch; + /// command-RPC pump; it writes only under the node's replicas root. + pub struct MemoryReplicateBatch { state: Arc } name: "memory/replicate-batch", access: Privileged, params: MemoryReplicateBatchParams, output: MemoryReplicateBatchResult, - run(_this, _ctx, p) => { - let acked = replica_append_batch(&p.persona_id, &p.entries) + run(this, _ctx, p) => { + let acked = this.state.replication.replica_append_batch(&p.persona_id, &p.entries) .map_err(|e| CommandError::Invalid(format!("memory/replicate-batch: {e}")))?; Ok(MemoryReplicateBatchResult { acked_seq: acked }) } diff --git a/core/continuum-core/src/memory/replication.rs b/core/continuum-core/src/memory/replication.rs index 161be4b686..35dee304ad 100644 --- a/core/continuum-core/src/memory/replication.rs +++ b/core/continuum-core/src/memory/replication.rs @@ -1,28 +1,33 @@ -//! Persona-RAID slice 1 — the write-behind journal tee. +//! Persona-RAID — the write-behind journal + replica cold store. //! //! Design: docs/architecture/PERSONA-RAID-WRITE-BEHIND.md (M5-approved shape). -//! Every DURABLE memory admit (the `persist_memory` funnel — the one place a -//! persona's truth reaches `longterm.db`) is teed into an append-only -//! per-persona `journal.jsonl` next to that db. The journal is the unit of -//! truth-in-motion: replayable (idempotent by record id, newest-wins) and -//! shippable (slice 2's RTOS shipper reads the tail past the peer's acked seq). +//! Slice 1: every DURABLE memory admit (the `persist_memory` funnel — the one +//! place a persona's truth reaches `longterm.db`) is teed into an append-only +//! per-persona `journal.jsonl`. Slice 2a: `memory/replicate-batch` lands a peer +//! shipper's journal tail into this node's replica store and acks high-water. //! -//! Slice-1 contract: +//! Shape: ONE owned [`ReplicationLedger`] held by `MemoryState` — no statics, +//! no ambient globals (CONCURRENCY-STYLE-GUIDE). Roots are constructor-injected +//! so tests build against temp dirs directly. The slice-2b shipper will be the +//! RTOS-shape consumer (own task + interval + `watch` snapshot) reading this +//! ledger; until its first peer ack, snapshots honestly report +//! `degraded_unreplicated: true`. +//! +//! Contracts: //! - Tee is BEST-EFFORT-LOUD: a journal failure warns + counts (visible in the //! snapshot) but NEVER errors the admit — the durable store already holds -//! the truth; the journal is the replication leg, and a lying "replicated" -//! is worse than an honest "degraded". +//! the truth; a lying "replicated" is worse than an honest "degraded". //! - `seq` is monotonic per (persona, origin_node); recovered from the last //! journal line on first touch after a restart, so restarts never fork the //! sequence ([[restarts-are-commonplace]]). -//! - No shipping, no background task yet — `snapshot()` is the observability -//! surface (`degraded_unreplicated: true` until slice 2 lands a peer ack). +//! - The replica store is idempotent by (origin_node, seq) high-water — the +//! shipper retries blindly on a lost ack. use std::collections::HashMap; use std::fs::{self, OpenOptions}; use std::io::{BufRead, BufReader, Write}; -use std::path::PathBuf; -use std::sync::{Mutex, OnceLock}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; use serde::{Deserialize, Serialize}; @@ -41,8 +46,8 @@ pub struct JournalEntry { pub record: serde_json::Value, } -/// Point-in-time replication state for one persona — the slice-1 observability -/// surface (slice 2's shipper publishes this via a `watch` channel instead). +/// Point-in-time replication state for one persona — the observability +/// surface (slice 2b's shipper publishes this via a `watch` channel). #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReplicationSnapshot { pub persona_id: String, @@ -51,7 +56,7 @@ pub struct ReplicationSnapshot { pub journal_bytes: u64, /// Admits that failed to journal since boot (loud degradation counter). pub dropped: u64, - /// True until a peer has acked a shipped batch (slice 2). An honest flag: + /// True until a peer has acked a shipped batch (slice 2b). An honest flag: /// this persona's memory currently lives on ONE machine. pub degraded_unreplicated: bool, } @@ -64,73 +69,83 @@ struct JournalHandle { last_ts_ms: u64, } -static JOURNALS: OnceLock>> = OnceLock::new(); - -fn journals() -> &'static Mutex> { - JOURNALS.get_or_init(|| Mutex::new(HashMap::new())) -} - -/// This node's stable name for journal provenance. Hostname is enough for -/// slice 1 (the grid's peer_id joins in slice 2's ship envelope). -fn origin_node() -> String { - std::env::var("COMPUTERNAME") - .or_else(|_| std::env::var("HOSTNAME")) - .unwrap_or_else(|_| "unknown-node".to_string()) +/// The one owner of persona-RAID state on a node: the outbound journals (this +/// node's personas) and the inbound replica high-waters (other nodes' personas +/// backed up here). Held by `MemoryState`; everything is instance state. +pub struct ReplicationLedger { + origin_node: String, + /// `//journal.jsonl` — sibling of longterm.db. + personas_root: Option, + /// `//journal.jsonl` — peers' memory held here. + replicas_root: Option, + journals: Mutex>, + replica_hw: Mutex>, } -/// `~/.continuum/personas//journal.jsonl` — sibling of the persona's -/// `longterm.db` data dir. Bare id (any `@persona:` prefix stripped) matches -/// the on-disk persona dir naming. -fn journal_path_for(persona_id: &str) -> Option { - let home = std::env::var("HOME") - .or_else(|_| std::env::var("USERPROFILE")) - .ok()?; - let bare = persona_id.strip_prefix("@persona:").unwrap_or(persona_id); - Some( - PathBuf::from(home) - .join(".continuum/personas") - .join(bare) - .join("journal.jsonl"), - ) -} +impl ReplicationLedger { + /// Construct with explicit roots — the testable constructor. + pub fn new(personas_root: PathBuf, replicas_root: PathBuf, origin_node: String) -> Self { + Self { + origin_node, + personas_root: Some(personas_root), + replicas_root: Some(replicas_root), + journals: Mutex::new(HashMap::new()), + replica_hw: Mutex::new(HashMap::new()), + } + } -/// Recover the last seq from an existing journal so a restart continues the -/// sequence instead of forking it. O(file) once per persona per boot. -fn recover_seq(path: &PathBuf) -> (u64, u64, u64) { - let Ok(f) = fs::File::open(path) else { - return (0, 0, 0); - }; - let bytes = f.metadata().map(|m| m.len()).unwrap_or(0); - let mut last_seq = 0; - let mut last_ts = 0; - for line in BufReader::new(f).lines().map_while(Result::ok) { - if let Ok(e) = serde_json::from_str::(&line) { - last_seq = e.seq; - last_ts = e.ts_ms; + /// Production constructor: roots under the user home, origin from the + /// machine name. A homeless environment yields a ledger that WARNS and + /// counts every admit as dropped (loud degradation, never a panic on the + /// boot path). + pub fn from_env() -> Self { + let home = std::env::var("HOME") + .or_else(|_| std::env::var("USERPROFILE")) + .ok() + .map(PathBuf::from); + let origin = std::env::var("COMPUTERNAME") + .or_else(|_| std::env::var("HOSTNAME")) + .unwrap_or_else(|_| "unknown-node".to_string()); + match home { + Some(h) => Self::new( + h.join(".continuum/personas"), + h.join(".continuum/replicas"), + origin, + ), + None => Self { + origin_node: origin, + personas_root: None, + replicas_root: None, + journals: Mutex::new(HashMap::new()), + replica_hw: Mutex::new(HashMap::new()), + }, } } - (last_seq, last_ts, bytes) -} -fn now_ms() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) -} + fn journal_path_for(&self, persona_id: &str) -> Option { + let bare = persona_id.strip_prefix("@persona:").unwrap_or(persona_id); + Some(self.personas_root.as_ref()?.join(bare).join("journal.jsonl")) + } -/// Tee one durable admit into the persona's journal. Called from the -/// `persist_memory` funnel AFTER the durable store accepted the write. -/// Never fails the caller — failures warn + count in the snapshot. -pub fn journal_admit(persona_id: &str, kind: &str, record: &serde_json::Value) { - let mut map = match journals().lock() { - Ok(g) => g, - Err(_) => return, // poisoned lock: journaling stands down, admit proceeds - }; - let handle = match map.get_mut(persona_id) { - Some(h) => h, - None => { - let Some(path) = journal_path_for(persona_id) else { + fn replica_path_for(&self, persona_id: &str) -> Option { + let bare = persona_id.strip_prefix("@persona:").unwrap_or(persona_id); + Some(self.replicas_root.as_ref()?.join(bare).join("journal.jsonl")) + } + + /// Tee one durable admit into the persona's journal. Called from the + /// `persist_memory` funnel AFTER the durable store accepted the write. + /// Never fails the caller — failures warn + count in the snapshot. + pub fn journal_admit(&self, persona_id: &str, kind: &str, record: &serde_json::Value) { + let mut map = match self.journals.lock() { + Ok(g) => g, + Err(_) => return, // poisoned: journaling stands down, admit proceeds + }; + if !map.contains_key(persona_id) { + let Some(path) = self.journal_path_for(persona_id) else { + tracing::warn!( + persona_id, + "persona-RAID: no home dir — journaling disabled, replication degraded" + ); return; }; let (seq, last_ts_ms, bytes) = recover_seq(&path); @@ -138,90 +153,129 @@ pub fn journal_admit(persona_id: &str, kind: &str, record: &serde_json::Value) { persona_id.to_string(), JournalHandle { path, seq, bytes, dropped: 0, last_ts_ms }, ); - map.get_mut(persona_id).expect("just inserted") } - }; - let entry = JournalEntry { - seq: handle.seq + 1, - persona_id: persona_id.to_string(), - origin_node: origin_node(), - ts_ms: now_ms(), - kind: kind.to_string(), - record: record.clone(), - }; - let appended = (|| -> std::io::Result { - if let Some(dir) = handle.path.parent() { - fs::create_dir_all(dir)?; + let handle = map.get_mut(persona_id).expect("inserted above"); + let entry = JournalEntry { + seq: handle.seq + 1, + persona_id: persona_id.to_string(), + origin_node: self.origin_node.clone(), + ts_ms: now_ms(), + kind: kind.to_string(), + record: record.clone(), + }; + match append_line(&handle.path, &entry) { + Ok(n) => { + handle.seq = entry.seq; + handle.bytes += n; + handle.last_ts_ms = entry.ts_ms; + } + Err(e) => { + handle.dropped += 1; + tracing::warn!( + persona_id, + dropped = handle.dropped, + "persona-RAID journal append failed (admit unaffected; replication degraded): {e}" + ); + } } - let mut f = OpenOptions::new().create(true).append(true).open(&handle.path)?; - let line = serde_json::to_string(&entry) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - f.write_all(line.as_bytes())?; - f.write_all(b"\n")?; - Ok(line.len() as u64 + 1) - })(); - match appended { - Ok(n) => { - handle.seq = entry.seq; - handle.bytes += n; - handle.last_ts_ms = entry.ts_ms; + } + + /// This persona's replication state, or None if nothing journaled since boot. + pub fn snapshot(&self, persona_id: &str) -> Option { + let map = self.journals.lock().ok()?; + let h = map.get(persona_id)?; + Some(ReplicationSnapshot { + persona_id: persona_id.to_string(), + last_seq: h.seq, + last_ts_ms: h.last_ts_ms, + journal_bytes: h.bytes, + dropped: h.dropped, + degraded_unreplicated: true, // honest until slice 2b lands a peer ack + }) + } + + /// Append a shipped batch to this node's replica journal for `persona_id`. + /// All entries must share ONE origin_node (a batch is one shipper's tail). + /// Entries at or below the current high-water are skipped (idempotent + /// retry). Returns the acked high-water for that (persona, origin). + pub fn replica_append_batch( + &self, + persona_id: &str, + entries: &[JournalEntry], + ) -> Result { + let Some(first) = entries.first() else { + // A batch names its origin via its entries; an empty one can't be + // acked meaningfully. The shipper never sends empty — fail loud. + return Err("empty replicate batch".into()); + }; + let origin = first.origin_node.clone(); + if entries.iter().any(|e| e.origin_node != origin) { + return Err("mixed origin_node in one batch — a batch is one shipper's tail".into()); } - Err(e) => { - handle.dropped += 1; - tracing::warn!( - persona_id, - dropped = handle.dropped, - "persona-RAID journal append failed (admit unaffected; replication degraded): {e}" - ); + let path = self + .replica_path_for(persona_id) + .ok_or("no home dir for replica store")?; + let mut map = self.replica_hw.lock().map_err(|e| e.to_string())?; + let key = (persona_id.to_string(), origin); + if !map.contains_key(&key) { + recover_replica_hw(persona_id, &path, &mut map); + } + let hw = map.entry(key).or_insert(0); + if let Some(dir) = path.parent() { + fs::create_dir_all(dir).map_err(|e| e.to_string())?; } + let mut f = OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .map_err(|e| e.to_string())?; + for e in entries { + if e.seq <= *hw { + continue; // already replicated — idempotent retry + } + let line = serde_json::to_string(e).map_err(|e| e.to_string())?; + f.write_all(line.as_bytes()).map_err(|e| e.to_string())?; + f.write_all(b"\n").map_err(|e| e.to_string())?; + *hw = e.seq; + } + Ok(*hw) } } -/// The slice-1 observability read: this persona's replication state, or None -/// if nothing has been journaled (or recovered) since boot. -pub fn snapshot(persona_id: &str) -> Option { - let map = journals().lock().ok()?; - let h = map.get(persona_id)?; - Some(ReplicationSnapshot { - persona_id: persona_id.to_string(), - last_seq: h.seq, - last_ts_ms: h.last_ts_ms, - journal_bytes: h.bytes, - dropped: h.dropped, - degraded_unreplicated: true, // honest until slice 2 lands a peer ack - }) -} - -// ─── Replica store (the RECEIVING side of slice 2) ────────────────────────── -// -// A peer node holds passive copies under `~/.continuum/replicas// -// journal.jsonl` — its cold store IS another being's backup (the grid is the -// backup). Idempotent by (origin_node, seq) high-water: re-shipped batches -// append nothing and re-ack the same high-water, so the shipper can retry -// blindly. Single-writer discipline holds: entries for one persona from one -// origin arrive in seq order from that origin's shipper. - -static REPLICA_HW: OnceLock>> = OnceLock::new(); +// ─── file helpers (free of ledger state) ──────────────────────────────────── -fn replica_hw() -> &'static Mutex> { - REPLICA_HW.get_or_init(|| Mutex::new(HashMap::new())) +fn append_line(path: &Path, entry: &JournalEntry) -> std::io::Result { + if let Some(dir) = path.parent() { + fs::create_dir_all(dir)?; + } + let mut f = OpenOptions::new().create(true).append(true).open(path)?; + let line = serde_json::to_string(entry) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + f.write_all(line.as_bytes())?; + f.write_all(b"\n")?; + Ok(line.len() as u64 + 1) } -fn replica_journal_path(persona_id: &str) -> Option { - let home = std::env::var("HOME") - .or_else(|_| std::env::var("USERPROFILE")) - .ok()?; - let bare = persona_id.strip_prefix("@persona:").unwrap_or(persona_id); - Some( - PathBuf::from(home) - .join(".continuum/replicas") - .join(bare) - .join("journal.jsonl"), - ) +/// Recover the last seq from an existing journal so a restart continues the +/// sequence instead of forking it. O(file) once per persona per boot. +fn recover_seq(path: &Path) -> (u64, u64, u64) { + let Ok(f) = fs::File::open(path) else { + return (0, 0, 0); + }; + let bytes = f.metadata().map(|m| m.len()).unwrap_or(0); + let mut last_seq = 0; + let mut last_ts = 0; + for line in BufReader::new(f).lines().map_while(Result::ok) { + if let Ok(e) = serde_json::from_str::(&line) { + last_seq = e.seq; + last_ts = e.ts_ms; + } + } + (last_seq, last_ts, bytes) } /// Recover this replica's high-water per origin_node from disk (once per boot). -fn recover_replica_hw(persona_id: &str, path: &PathBuf, map: &mut HashMap<(String, String), u64>) { +fn recover_replica_hw(persona_id: &str, path: &Path, map: &mut HashMap<(String, String), u64>) { let Ok(f) = fs::File::open(path) else { return }; for line in BufReader::new(f).lines().map_while(Result::ok) { if let Ok(e) = serde_json::from_str::(&line) { @@ -234,68 +288,25 @@ fn recover_replica_hw(persona_id: &str, path: &PathBuf, map: &mut HashMap<(Strin } } -/// Append a shipped batch to this node's replica journal for `persona_id`. -/// All entries must share ONE origin_node (a batch is one shipper's tail). -/// Entries at or below the current high-water are skipped (idempotent retry). -/// Returns the acked high-water for that (persona, origin) after the append. -pub fn replica_append_batch( - persona_id: &str, - entries: &[JournalEntry], -) -> Result { - let Some(first) = entries.first() else { - // A batch names its origin via its entries; an empty one can't be - // acked meaningfully. The shipper never sends empty — fail loud. - return Err("empty replicate batch".into()); - }; - let origin = first.origin_node.clone(); - if entries.iter().any(|e| e.origin_node != origin) { - return Err("mixed origin_node in one batch — a batch is one shipper's tail".into()); - } - let path = replica_journal_path(persona_id).ok_or("no home dir for replica store")?; - let mut map = replica_hw().lock().map_err(|e| e.to_string())?; - let key = (persona_id.to_string(), origin.clone()); - if !map.contains_key(&key) { - recover_replica_hw(persona_id, &path, &mut map); - } - let hw = map.entry(key).or_insert(0); - if let Some(dir) = path.parent() { - fs::create_dir_all(dir).map_err(|e| e.to_string())?; - } - let mut f = OpenOptions::new() - .create(true) - .append(true) - .open(&path) - .map_err(|e| e.to_string())?; - for e in entries { - if e.seq <= *hw { - continue; // already replicated — idempotent retry - } - let line = serde_json::to_string(e).map_err(|e| e.to_string())?; - f.write_all(line.as_bytes()).map_err(|e| e.to_string())?; - f.write_all(b"\n").map_err(|e| e.to_string())?; - *hw = e.seq; - } - Ok(*hw) +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) } #[cfg(test)] mod tests { use super::*; - fn with_temp_home(f: impl FnOnce() -> T) -> T { - // Serialize env mutation across tests touching HOME. - static ENV_LOCK: Mutex<()> = Mutex::new(()); - let _g = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); - let tmp = std::env::temp_dir().join(format!("raid-test-{}", std::process::id())); - std::fs::create_dir_all(&tmp).unwrap(); - let old = std::env::var("HOME").ok(); - std::env::set_var("HOME", &tmp); - let out = f(); - match old { - Some(v) => std::env::set_var("HOME", v), - None => std::env::remove_var("HOME"), - } - out + fn temp_ledger(tag: &str) -> (ReplicationLedger, PathBuf) { + let root = std::env::temp_dir().join(format!("raid-{}-{}", tag, std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + ( + ReplicationLedger::new(root.join("personas"), root.join("replicas"), "node-a".into()), + root, + ) } // what this catches: the journal must be monotonic and replayable — an @@ -303,57 +314,24 @@ mod tests { // resume-on-spawn's newest-wins replay. #[test] fn journal_appends_monotonic_and_round_trips() { - with_temp_home(|| { - let pid = "raid-test-persona-a"; - journal_admit(pid, "memory", &serde_json::json!({"id": "m1", "text": "hello"})); - journal_admit(pid, "memory", &serde_json::json!({"id": "m2", "text": "world"})); - let snap = snapshot(pid).expect("snapshot after admits"); - assert_eq!(snap.last_seq, 2); - assert_eq!(snap.dropped, 0); - assert!(snap.degraded_unreplicated); - let path = journal_path_for(pid).unwrap(); - let lines: Vec = std::io::BufRead::lines(std::io::BufReader::new( - std::fs::File::open(&path).unwrap(), - )) - .map(|l| serde_json::from_str(&l.unwrap()).unwrap()) - .collect(); - assert_eq!(lines.len(), 2); - assert_eq!(lines[0].seq, 1); - assert_eq!(lines[1].seq, 2); - assert_eq!(lines[1].record["id"], "m2"); - }); - } - - // what this catches: a re-shipped batch (shipper retry after a lost ack) - // must append NOTHING and re-ack the same high-water — duplicate replica - // lines would double memories on slice-3 replay. - #[test] - fn replica_batch_is_idempotent() { - with_temp_home(|| { - let pid = "raid-test-persona-c"; - let e = |seq: u64| JournalEntry { - seq, - persona_id: pid.into(), - origin_node: "node-a".into(), - ts_ms: 1, - kind: "memory".into(), - record: serde_json::json!({"id": format!("m{seq}")}), - }; - let batch = vec![e(1), e(2)]; - assert_eq!(replica_append_batch(pid, &batch).unwrap(), 2); - // Retry the same batch: same ack, no new lines. - assert_eq!(replica_append_batch(pid, &batch).unwrap(), 2); - let path = replica_journal_path(pid).unwrap(); - let n = std::io::BufRead::lines(std::io::BufReader::new( - std::fs::File::open(&path).unwrap(), - )) - .count(); - assert_eq!(n, 2, "retry must not duplicate replica lines"); - // Mixed-origin batch fails loud. - let mut bad = vec![e(3)]; - bad.push(JournalEntry { origin_node: "node-b".into(), ..e(4) }); - assert!(replica_append_batch(pid, &bad).is_err()); - }); + let (ledger, root) = temp_ledger("mono"); + let pid = "persona-a"; + ledger.journal_admit(pid, "memory", &serde_json::json!({"id": "m1", "text": "hello"})); + ledger.journal_admit(pid, "memory", &serde_json::json!({"id": "m2", "text": "world"})); + let snap = ledger.snapshot(pid).expect("snapshot after admits"); + assert_eq!(snap.last_seq, 2); + assert_eq!(snap.dropped, 0); + assert!(snap.degraded_unreplicated); + let path = root.join("personas").join(pid).join("journal.jsonl"); + let lines: Vec = std::io::BufRead::lines(std::io::BufReader::new( + std::fs::File::open(&path).unwrap(), + )) + .map(|l| serde_json::from_str(&l.unwrap()).unwrap()) + .collect(); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0].seq, 1); + assert_eq!(lines[1].seq, 2); + assert_eq!(lines[1].record["id"], "m2"); } // what this catches: a restart must CONTINUE the sequence from the file, @@ -361,14 +339,48 @@ mod tests { // silently lose memories on replay ([[restarts-are-commonplace]]). #[test] fn restart_recovers_seq_from_disk() { - with_temp_home(|| { - let pid = "raid-test-persona-b"; - journal_admit(pid, "memory", &serde_json::json!({"id": "m1"})); - // Simulate restart: drop the in-memory handle, keeping the file. - journals().lock().unwrap().remove(pid); - journal_admit(pid, "memory", &serde_json::json!({"id": "m2"})); - let snap = snapshot(pid).expect("snapshot after recovered admit"); - assert_eq!(snap.last_seq, 2, "seq must continue across restart, not fork"); - }); + let (ledger, root) = temp_ledger("restart"); + let pid = "persona-b"; + ledger.journal_admit(pid, "memory", &serde_json::json!({"id": "m1"})); + // Simulate restart: a FRESH ledger over the same roots. + drop(ledger); + let ledger2 = + ReplicationLedger::new(root.join("personas"), root.join("replicas"), "node-a".into()); + ledger2.journal_admit(pid, "memory", &serde_json::json!({"id": "m2"})); + let snap = ledger2.snapshot(pid).expect("snapshot after recovered admit"); + assert_eq!(snap.last_seq, 2, "seq must continue across restart, not fork"); + } + + // what this catches: a re-shipped batch (shipper retry after a lost ack) + // must append NOTHING and re-ack the same high-water — duplicate replica + // lines would double memories on slice-3 replay. + #[test] + fn replica_batch_is_idempotent() { + let (ledger, root) = temp_ledger("replica"); + let pid = "persona-c"; + let e = |seq: u64| JournalEntry { + seq, + persona_id: pid.into(), + origin_node: "node-remote".into(), + ts_ms: 1, + kind: "memory".into(), + record: serde_json::json!({"id": format!("m{seq}")}), + }; + let batch = vec![e(1), e(2)]; + assert_eq!(ledger.replica_append_batch(pid, &batch).unwrap(), 2); + // Retry the same batch: same ack, no new lines. + assert_eq!(ledger.replica_append_batch(pid, &batch).unwrap(), 2); + let path = root.join("replicas").join(pid).join("journal.jsonl"); + let n = std::io::BufRead::lines(std::io::BufReader::new( + std::fs::File::open(&path).unwrap(), + )) + .count(); + assert_eq!(n, 2, "retry must not duplicate replica lines"); + // Mixed-origin batch fails loud. + let mut bad = vec![e(3)]; + bad.push(JournalEntry { origin_node: "node-other".into(), ..e(4) }); + assert!(ledger.replica_append_batch(pid, &bad).is_err()); + // Empty batch fails loud. + assert!(ledger.replica_append_batch(pid, &[]).is_err()); } } diff --git a/core/continuum-core/src/modules/memory.rs b/core/continuum-core/src/modules/memory.rs index f29dbc116c..73eb8ed110 100644 --- a/core/continuum-core/src/modules/memory.rs +++ b/core/continuum-core/src/modules/memory.rs @@ -33,6 +33,9 @@ pub struct MemoryState { /// `data/create`) and to hydrate a missing corpus from it (via `data/list`) /// — the cross-module dual-write pattern `ChatModule` established. pub executor_slot: Arc>, + /// Persona-RAID ledger — the ONE owner of this node's write-behind + /// journals + replica high-waters (no statics; see memory/replication.rs). + pub replication: Arc, } impl MemoryState { @@ -40,6 +43,7 @@ impl MemoryState { Self { memory_manager, executor_slot: Arc::new(LateBound::new("memory::executor")), + replication: Arc::new(crate::memory::replication::ReplicationLedger::from_env()), } } From 4e8830b804bda6fd88de84fc006311974750af0c Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 17:54:31 -0500 Subject: [PATCH 33/55] =?UTF-8?q?feat(memory):=20persona-RAID=20slice=202b?= =?UTF-8?q?=20=E2=80=94=20the=20write-behind=20shipper=20(RTOS-shape)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes RAID-1's sending half. ReplicationShipper: own tokio task + interval (cadence = the amnesia window) + watch, injected deps (ledger, Arc, ReplicaPeerSource). Each tick, per journaled persona: read journal tail past the peer's acked high-water -> ship to the peer's memory/replicate-batch via RouteDecision::Peer (M5's AircTransport) -> advance high-water on ack. Best-effort write-behind: a slow/absent peer grows reported lag, never blocks the cognition hot path. Ledger gains read_tail + journaled_personas. Test-pinned invariant (fake transport): ships ONLY the unacked tail, advances on ack, never re-floods. Runtime seam left: back ReplicaPeerSource with the live residency view + spawn the task in the memory module lifecycle -> RAID-1 live end-to-end (the demo-B kill-test). Slice 3 (resume-on-spawn replay) stamps M5's (origin_node, origin_seq) fields next. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- core/continuum-core/src/memory/mod.rs | 1 + core/continuum-core/src/memory/replication.rs | 36 +++ .../src/memory/replication_shipper.rs | 235 ++++++++++++++++++ 3 files changed, 272 insertions(+) create mode 100644 core/continuum-core/src/memory/replication_shipper.rs diff --git a/core/continuum-core/src/memory/mod.rs b/core/continuum-core/src/memory/mod.rs index 86e6cda04a..f39e24d49e 100644 --- a/core/continuum-core/src/memory/mod.rs +++ b/core/continuum-core/src/memory/mod.rs @@ -31,6 +31,7 @@ pub mod embedding; pub mod raw_adapter; pub mod recall; pub mod replication; +pub mod replication_shipper; pub mod timeline; pub mod types; diff --git a/core/continuum-core/src/memory/replication.rs b/core/continuum-core/src/memory/replication.rs index 35dee304ad..1d3dac531e 100644 --- a/core/continuum-core/src/memory/replication.rs +++ b/core/continuum-core/src/memory/replication.rs @@ -180,6 +180,42 @@ impl ReplicationLedger { } } + /// Read this persona's journal entries with `seq > after_seq`, up to + /// `limit`, ascending. The shipper's tail read (slice 2b). Reads from disk + /// (the journal is the durable truth; the in-memory handle only tracks the + /// write head), so it works for any persona with a journal on this node. + pub fn read_tail(&self, persona_id: &str, after_seq: u64, limit: usize) -> Vec { + let Some(path) = self.journal_path_for(persona_id) else { + return Vec::new(); + }; + let Ok(f) = fs::File::open(&path) else { + return Vec::new(); + }; + let mut out = Vec::new(); + for line in BufReader::new(f).lines().map_while(Result::ok) { + if let Ok(e) = serde_json::from_str::(&line) { + if e.seq > after_seq { + out.push(e); + if out.len() >= limit { + break; + } + } + } + } + out + } + + /// Persona ids with an active in-memory journal handle (touched since boot). + /// The shipper iterates these each tick. (Cold personas not yet admitted + /// this boot are shipped once they next admit — acceptable: a persona with + /// zero writes this boot has nothing new to replicate.) + pub fn journaled_personas(&self) -> Vec { + self.journals + .lock() + .map(|m| m.keys().cloned().collect()) + .unwrap_or_default() + } + /// This persona's replication state, or None if nothing journaled since boot. pub fn snapshot(&self, persona_id: &str) -> Option { let map = self.journals.lock().ok()?; diff --git a/core/continuum-core/src/memory/replication_shipper.rs b/core/continuum-core/src/memory/replication_shipper.rs new file mode 100644 index 0000000000..138cd2d792 --- /dev/null +++ b/core/continuum-core/src/memory/replication_shipper.rs @@ -0,0 +1,235 @@ +//! Persona-RAID slice 2b — the write-behind SHIPPER. +//! +//! The RTOS-shape consumer of the [`ReplicationLedger`]: an owned task on a +//! fixed `interval` (the cadence IS the maximum amnesia window) that, per +//! journaled persona, reads the journal tail past the peer's acked high-water, +//! ships it to a residency-eligible peer's `memory/replicate-batch`, and +//! advances the high-water on ack. State is published via a `watch` snapshot. +//! Design: docs/architecture/PERSONA-RAID-WRITE-BEHIND.md. +//! +//! Concurrency (CONCURRENCY-STYLE-GUIDE): own task + `tokio::time::interval` + +//! `watch::Sender` + injected deps (ledger, transport, peer +//! source). Shipping is best-effort write-behind — a slow/absent peer grows the +//! reported lag, NEVER blocks the cognition hot path (the tee is elsewhere). + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use serde_json::json; +use tokio::sync::watch; + +use crate::memory::replication::ReplicationLedger; +use crate::routing::command_uri::PeerRef; +use crate::routing::route_decision::RouteDecision; +use crate::routing::transport::Transport; +use crate::runtime::service_module::CommandResult; + +/// How many journal entries ship per persona per tick — bounds one batch so a +/// long-cold persona catches up over several ticks instead of one huge frame. +const SHIP_BATCH_LIMIT: usize = 256; + +/// Default cadence — the maximum amnesia window. One knob (design doc). +pub const DEFAULT_SHIP_INTERVAL: Duration = Duration::from_secs(30); + +/// Chooses the peer to replicate a persona TO. Injected so the runtime can back +/// it with the live residency/grid view while tests use a fixed peer. +pub trait ReplicaPeerSource: Send + Sync { + /// A residency-eligible peer to hold `persona_id`'s replica, or None when + /// the grid has no eligible peer (then the persona stays single-copy — + /// honestly `degraded_unreplicated`). + fn replica_peer_for(&self, persona_id: &str) -> Option; +} + +/// Published shipper state — per persona, the last seq a peer has acked. +#[derive(Debug, Clone, Default)] +pub struct ShipperSnapshot { + /// persona_id → (acked_seq, peer_label). Absent = never shipped. + pub acked: HashMap, + /// Ticks completed since boot (liveness). + pub ticks: u64, +} + +/// The shipper engine — owns the per-persona acked high-water, ships one tick +/// on demand. Split from the task loop so it is unit-testable with a fake +/// transport + peer source (no timers, no runtime). +pub struct ReplicationShipper { + ledger: Arc, + transport: Arc, + peers: Arc, + acked: HashMap, + snapshot_tx: watch::Sender, + ticks: u64, +} + +impl ReplicationShipper { + pub fn new( + ledger: Arc, + transport: Arc, + peers: Arc, + ) -> (Self, watch::Receiver) { + let (snapshot_tx, snapshot_rx) = watch::channel(ShipperSnapshot::default()); + ( + Self { + ledger, + transport, + peers, + acked: HashMap::new(), + snapshot_tx, + ticks: 0, + }, + snapshot_rx, + ) + } + + /// One replication pass over all journaled personas. Best-effort: a persona + /// whose peer is absent or whose ship fails is simply skipped this tick + /// (its lag grows, reported in the snapshot next publish). Returns the + /// number of personas whose high-water advanced. + pub async fn tick(&mut self) -> usize { + let mut advanced = 0; + for persona_id in self.ledger.journaled_personas() { + let after = self.acked.get(&persona_id).map(|(s, _)| *s).unwrap_or(0); + let batch = self.ledger.read_tail(&persona_id, after, SHIP_BATCH_LIMIT); + if batch.is_empty() { + continue; + } + let Some(peer) = self.peers.replica_peer_for(&persona_id) else { + continue; // no eligible peer — stays single-copy (honest degrade) + }; + let peer_label = peer.to_string(); + let params = json!({ "persona_id": persona_id, "entries": batch }); + let decision = RouteDecision::Peer { + peer, + node: None, + env: None, + path: "memory/replicate-batch".to_string(), + query: None, + fragment: None, + }; + match self.transport.dispatch(decision, params).await { + Ok(result) => { + if let Some(acked_seq) = extract_acked_seq(&result) { + self.acked.insert(persona_id.clone(), (acked_seq, peer_label)); + advanced += 1; + } + } + Err(e) => { + tracing::warn!(persona_id, "persona-RAID ship failed (lag grows): {e}"); + } + } + } + self.ticks += 1; + let _ = self.snapshot_tx.send(ShipperSnapshot { + acked: self.acked.clone(), + ticks: self.ticks, + }); + advanced + } + + /// Spawn the RTOS task: tick every `interval` until the process ends. + /// Consumes self (moves into the task). The runtime holds the returned + /// snapshot receiver for `serving/status`-style inspection. + pub fn spawn(mut self, interval: Duration) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + ticker.tick().await; + self.tick().await; + } + }) + } +} + +/// Pull `acked_seq` out of the receiver's CommandResult (the +/// `MemoryReplicateBatchResult` JSON). Absent/garbled → None (no advance). +fn extract_acked_seq(result: &CommandResult) -> Option { + result + .to_json_value() + .ok() + .and_then(|v| v.get("acked_seq").and_then(|s| s.as_u64())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::routing::route_decision::RouteDecision; + use async_trait::async_trait; + use std::sync::Mutex; + + /// Fake transport: records the persona_id + entry count it was asked to + /// ship, and acks the highest seq in the batch (like the real receiver). + #[derive(Debug, Default)] + struct FakeTransport { + shipped: Mutex>, + } + + #[async_trait] + impl Transport for FakeTransport { + async fn dispatch( + &self, + decision: RouteDecision, + params: serde_json::Value, + ) -> Result { + let path = match &decision { + RouteDecision::Peer { path, .. } => path.clone(), + _ => return Err("unexpected decision".into()), + }; + assert_eq!(path, "memory/replicate-batch"); + let pid = params["persona_id"].as_str().unwrap().to_string(); + let entries = params["entries"].as_array().unwrap(); + let max_seq = entries.iter().map(|e| e["seq"].as_u64().unwrap()).max().unwrap(); + self.shipped.lock().unwrap().push((pid, entries.len())); + Ok(CommandResult::Json(json!({ "acked_seq": max_seq }))) + } + } + + struct FixedPeer; + impl ReplicaPeerSource for FixedPeer { + fn replica_peer_for(&self, _persona_id: &str) -> Option { + Some(PeerRef::Name("peer-mac".to_string())) + } + } + + fn temp_ledger() -> Arc { + let root = std::env::temp_dir().join(format!("raid-ship-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + Arc::new(ReplicationLedger::new( + root.join("personas"), + root.join("replicas"), + "node-a".into(), + )) + } + + // what this catches: the shipper must ship only the UNACKED tail and advance + // its high-water on ack — re-shipping acked entries (no advance, or shipping + // from seq 0 every tick) would flood the peer and never converge. + #[tokio::test] + async fn ships_tail_once_and_advances_highwater() { + let ledger = temp_ledger(); + let pid = "persona-x"; + ledger.journal_admit(pid, "memory", &json!({"id": "m1"})); + ledger.journal_admit(pid, "memory", &json!({"id": "m2"})); + let transport = Arc::new(FakeTransport::default()); + let (mut shipper, mut rx) = + ReplicationShipper::new(ledger.clone(), transport.clone(), Arc::new(FixedPeer)); + + // Tick 1: ships both entries, acks seq 2. + assert_eq!(shipper.tick().await, 1); + rx.changed().await.unwrap(); + assert_eq!(rx.borrow().acked.get(pid).map(|(s, _)| *s), Some(2)); + + // Tick 2 with no new admits: nothing to ship (tail empty past seq 2). + assert_eq!(shipper.tick().await, 0); + + // A new admit: tick 3 ships ONLY the new entry. + ledger.journal_admit(pid, "memory", &json!({"id": "m3"})); + assert_eq!(shipper.tick().await, 1); + let shipped = transport.shipped.lock().unwrap(); + assert_eq!(shipped.len(), 2, "two ships total (m1+m2, then m3)"); + assert_eq!(shipped[0], (pid.to_string(), 2), "first ship: 2 entries"); + assert_eq!(shipped[1], (pid.to_string(), 1), "second ship: 1 new entry only"); + } +} From 8aa4d873c474d8cdb359611734420fd22d6a6edc Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 20:41:06 -0500 Subject: [PATCH 34/55] =?UTF-8?q?docs(planning):=20Benchmark-as-Learning-F?= =?UTF-8?q?lywheel=20=E2=80=94=20the=20grid=20task=20engine=20that=20prove?= =?UTF-8?q?s=20AND=20improves?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reframe (Joel): benchmarks are the learning signal, not just the scoreboard. A graded task == a training example; the benchmark engine is the data generator at the head of a continuous-learning loop. Prove (charts) and improve (train on failures) are the SAME pass. Connects the merged benchmark suite + unified experience stream (#2024) + stall-expansion (#2033) to cross-grid distribution, sentinel experiential plasticity, and AttnRes skip paths (the stable 'unet skip for generalization'). 5 gaps: cross-grid matrix, failure->curriculum emit, dream-forge consumes eval-fails, held-out + stochastic-depth generalization, honest-instrument discipline. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../BENCHMARK-AS-LEARNING-FLYWHEEL.md | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 docs/planning/BENCHMARK-AS-LEARNING-FLYWHEEL.md diff --git a/docs/planning/BENCHMARK-AS-LEARNING-FLYWHEEL.md b/docs/planning/BENCHMARK-AS-LEARNING-FLYWHEEL.md new file mode 100644 index 0000000000..366072e244 --- /dev/null +++ b/docs/planning/BENCHMARK-AS-LEARNING-FLYWHEEL.md @@ -0,0 +1,99 @@ +# Benchmark-as-Learning-Flywheel — the grid task engine that proves AND improves + +**Status:** north-star design (2026-07-28, BigMama + M5). Ties together the benchmark suite, the +grid, [[SENTINEL-IN-SUBSTRATE]] (experiential plasticity + AttnRes skip paths), and persona-RAID. +**The reframe (Joel):** "benchmarks... it's sort of everything for learning and proving. We can get +our models to continuously learn, plastically, maybe even with experiential plasticity and unet skip +paths for generalization." + +## The one idea + +A benchmark run is a **labeled task with an automatic grader**. That is ALSO the exact shape of a +training example. So the benchmark engine is not a scoreboard bolted onto the side — it is the +**data generator** at the head of a continuous-learning loop: + +``` +task surface (many benches, many domains) + │ cross-grid distribution (every node runs what it can) + ▼ +being attempts task ──► automatic grader (rustc / exec / reference) + │ │ + │ PASS → capability proof (the charts) │ FAIL → gradient + ▼ ▼ + results ledger the failure IS the training datum + │ │ + └─────────► experiential plasticity (forge-while-dreaming) ◄──┘ + │ prune dead capacity, clone hot, LoRA-heal on failures + │ AttnRes skip paths → generalization (implicit depth ensemble) + ▼ + improved being ──► re-run the surface (loop) +``` + +Prove and improve are the SAME pass. The charts (K3-target agentic benches — Terminal Bench, Program +Bench, SWE Marathon, Automation, BrowseComp) are the measuring stick; the failures against them are +the curriculum. + +## What already exists (do not rebuild — connect) + +| Piece | State | Where | +|---|---|---| +| Rust-native `benchmark/*` family (list/run/matrix/competition) | merged | `commands/benchmark.rs` | +| Graders: rustc compile+run, SWE-bench runner, web-dev, games | merged | benchmark collections | +| Fire-and-poll long runs (#86) | merged | `cognition/eval-status` | +| base_model_id override — measure ANY model, living persona untouched | merged | #1932 | +| Unified experience stream — one being learns from lived + eval + told (3 axes) | merged | #2024 / [[being-axis-shareable-learning]] | +| Lived-axis LLM-teacher expansion — learns from turns it STALLED on | merged | #2033 | +| Cross-grid dispatch (Commands.execute at a peer) | inbound live; effector converging | [[cross-grid-dispatch-canonical-path]] | +| Experiential plasticity (prune/clone/quant + AttnRes) | plan + AttnRes BUILT | [[SENTINEL-IN-SUBSTRATE]], fork `feat/kimi-k3-attnres` | + +The stall-expansion (#2033) is the seed of THIS: a benchmark FAIL is a high-value stall. The unified +experience stream (#2024) is already the pipe the eval axis flows into. We are wiring existing +organs, not growing new ones. + +## The gaps to close (the build) + +### 1. Cross-grid task distribution — the benchmark MATRIX goes multi-node +Today `benchmark/matrix` runs runners × benchmarks on ONE node. Extend the matrix executor to +dispatch task-shards to grid peers (each peer runs the benches its VRAM/model can serve), collect via +the inbound command-RPC, merge into one results ledger. The grid becomes a distributed eval cluster — +BigMama's 5090 runs the 48B/K3-tier benches, M5's Mac runs the Metal-servable tier, Air runs small. +Reuses residency + `route_grid_overflow`; no new transport. **This is what "works cross grid across a +ton of tasks" means concretely.** + +### 2. Failure → curriculum — the grader's negative becomes a training datum +A graded FAIL already has everything a training example needs: the task prompt, the wrong output, the +reference/test that rejected it, and (for the loop graders) the compiler/runtime error. Emit each +FAIL into the lived-experience stream tagged `eval-fail` with its grader feedback. The dream-forge +(sentinel slice 4) consumes these: LoRA-heal on the failure clusters during idle GPU time, validate +on the SAME bench on wake, roll back on regression. **The being trains on exactly what it's scored +on.** + +### 3. Generalization — AttnRes skip paths + plasticity, not just memorization +The risk of "train on the benchmark" is overfitting to it. Two defenses already designed: +- **AttnRes skip paths** (built): a residual highway trained with stochastic block-drop is an + implicit ensemble over depths → generalizes past the specific tasks (stochastic-depth literature). + This is the "unet skip paths for generalization" Joel names — the STABLE form of it (K3's AttnRes, + not sentinel-ai's unstable U-Net). +- **Held-out split** (discipline): the task surface is split train/eval per collection; plasticity + trains on the train shard, the charts are ALWAYS the held-out shard. Overfit shows as train-eval + divergence — a first-class signal, logged. + +### 4. Honest instrument (already the doctrine — keep it) +Warm-gate (never measure a cold model), same-model control (base_model_id), team writer+reviewer, +fail-loud VOID cells — all merged. The flywheel is only as trustworthy as the grader; the proof +discipline (#1584) is the law. A learning loop on a lying instrument optimizes the lie. + +## Sequencing +1. **Map our collections onto the K3-target named benches** (Terminal/Program/SWE-Marathon/Automation + equivalents) so our numbers are comparable to the charts — the measuring stick must match the goal. +2. **Cross-grid matrix** (gap 1) — distribute the surface; the grid proves at scale. +3. **Failure→curriculum emit** (gap 2) — the cheapest high-value wire: FAIL → lived stream (#2024 pipe + exists). +4. **Dream-forge consumes eval-fails** (sentinel slice 4 + gap 3) — the loop closes; idle GPU learns. +5. **Held-out discipline + AttnRes stochastic-drop training** (gap 3) — generalization, not memorization. + +## Why this is the whole thing +It unifies every session thread: the grid (distribution), the benchmark suite (signal), sentinel +plasticity (the update), AttnRes (generalization), persona-RAID (the being that persists across the +learning), K3 (the frontier target the being climbs toward). "Everything for learning and proving" — +one loop, on consumer hardware, that gets better every idle night. From 9ee387ebd27a871bb5b98048772b5f8da1a01892 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 22:07:07 -0500 Subject: [PATCH 35/55] =?UTF-8?q?docs(planning):=20Benchmark=20Target=20Ma?= =?UTF-8?q?p=20=E2=80=94=20our=20collections=20<->=20K3=20frontier=20chart?= =?UTF-8?q?s=20(flywheel=20gap=20#1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Maps our runnable -rs benches onto the named K3-leaderboard benches so K3's local numbers are comparable to the published charts. Finding: the catalog already NAMES all the chart benches (terminal-bench, swe-bench-*, swe-lancer, livecodebench, webarena, appworld, design2code) as catalogued stubs (eval_set: None) — 7 runnable, ~18 catalogued. Priority to make runnable (proof-value first): swe-bench-lite (anchors 3 chart benches incl SWE-Marathon where K3 is #1; M5's runner spine PR#1945 workspace-root seam is the gate), then livecodebench (Program Bench, reuses Rust test_grade), terminal-bench, webarena/appworld. Includes the honest 2026-07-28 local ladder on the runnable proxies (compacted-19b vs Kimi-48B: 75/85, 25/50, 33/67). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/BENCHMARK-TARGET-MAP.md | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/planning/BENCHMARK-TARGET-MAP.md diff --git a/docs/planning/BENCHMARK-TARGET-MAP.md b/docs/planning/BENCHMARK-TARGET-MAP.md new file mode 100644 index 0000000000..e6eaba6ee7 --- /dev/null +++ b/docs/planning/BENCHMARK-TARGET-MAP.md @@ -0,0 +1,54 @@ +# Benchmark Target Map — our collections ↔ the K3 frontier charts + +**Status:** flywheel gap #1 (2026-07-28, BigMama). Maps our benchmark collections onto the named +benches on the Kimi-K3 frontier leaderboard (the [[benchmark-learning-flywheel]] measuring stick), so +K3's local numbers are comparable to the published charts. Companion to +[BENCHMARK-AS-LEARNING-FLYWHEEL](BENCHMARK-AS-LEARNING-FLYWHEEL.md). + +## The two states of a benchmark in our catalog +- **RUNNABLE** — has an `eval_set` jsonl + a live grader (`cognition/eval`). 7 today. +- **CATALOGUED** — named in `known_benchmarks()` with `eval_set: None`. A placeholder that reserves + the name + description; needs a runner + dataset to become runnable. ~18 today. + +## The map (chart bench → our proxy → runnable target) + +| K3 chart bench | What it measures | Our RUNNABLE proxy today | Runnable TARGET to build (catalogued now) | +|---|---|---|---| +| **Program Bench**, LiveCodeBench | competitive-programming, single-file | `frontier-rs` (12), `hard-rs` (8) | `livecodebench`, `apps` | +| DeepSWE, **FrontierSWE**, **SWE Marathon** | repo-level patch that passes tests | `tool-bugfix-rs` (partial: fix-to-green loop) | `swe-bench-lite` → `swe-bench-verified`, `swe-lancer` | +| **Terminal Bench 2.1** | end-to-end tasks in a real shell | — (none) | `terminal-bench` | +| Kimi Code Bench 2.0 | mixed practical coding | `coder-eval` (13), `humaneval-rs` (164) | `bigcodebench`, `evalplus` | +| **Automation Bench**, **BrowseComp**, JobBench | agent driving real apps/web | — (none) | `webarena`, `appworld` | +| SpreadsheetBench 2 | spreadsheet manipulation | — (none) | (no catalog entry — add `spreadsheet-bench`) | +| **CharXiv**, Zerobench (visual) | chart/figure reasoning w/ vision | `webdev-rs` (perception grader, UI-adjacent) | `design2code` (screenshot→HTML) | +| GDPval, AA-Briefcase (general Elo) | broad agentic knowledge work | — (none) | (Elo-style; needs opponent-pool harness) | + +**Bold** = benches where Kimi K3 tops or near-tops the chart — our highest-value targets, because +that's where reproducing the number locally is the strongest proof. + +## Priority to make runnable (highest proof-value first) +1. **`swe-bench-lite`** — the SWE-family anchors THREE chart benches (DeepSWE/FrontierSWE/SWE-Marathon) + and K3 wins SWE-Marathon. M5 has the runner spine (PR #1945: gold RESOLVED, hermetic filter; the + workspace-root seam is the one open bug). **Closing that seam makes the single highest-leverage + target runnable.** → M5's lane; BigMama consumes it as a K3 serving target. +2. **`livecodebench`** — proxies Program Bench (K3 #1). Contamination-free, single-file → reuses the + existing Rust `test_grade` path; needs the dataset loader, not a new grader. Achievable in-tree. +3. **`terminal-bench`** — K3 #2, and the purest "doer" bench. Needs a real-shell harness (the persona's + tool-executor already runs shell) — the tool loop IS most of the runner. +4. **`webarena` / `appworld`** — Automation/BrowseComp (K3 #1). Heaviest (real app servers); defer + until the doer-toolset (beta plan) is hardened. + +## What ships now vs later +- **Now (this doc):** the map is the deliverable — every future K3 measurement states which chart + bench it proxies, so our local ladder is legible against the published charts. +- **Now (our ladder, honest):** on the RUNNABLE proxies, the 2026-07-28 head-to-head established the + local scale — compacted-19b vs Kimi-48B: humaneval-rs 75/85, hard-rs 25/50, frontier-rs 33/67. When + K3 lands, the SAME three proxies give the first frontier-vs-frontier local number. +- **Later:** the priority list above makes the named chart benches runnable, one at a time, M5 owning + the SWE/harness runners (her framework lane) and BigMama owning the K3 serving target + the + single-file Rust-graded ones (livecodebench/apps). + +## The discipline (keep the instrument honest) +Every runnable bench added inherits the merged proof discipline: warm-gate, same-model control +(base_model_id), held-out split (flywheel gap #3 — train on train-shard, chart on held-out), fail-loud +VOID cells. A chart number we can't reproduce under these controls is a claim we don't make. From 9c7533fcb52e885b663f5787f66e7a3b062aa205 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 22:19:14 -0500 Subject: [PATCH 36/55] feat(benchmark): livecodebench-rs runnable Program-Bench proxy + full 14-bench target map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit livecodebench-rs: 10 competitive-programming tasks (KMP, union-find, heap Dijkstra, longest-palindrome, max-product subarray, rotated binary search, trapping rain water, meeting rooms, house robber, k-th largest), rustc compile+run graded via the existing gym harness — every assertion hand-verified. Runnable proxy for Program Bench (K3 #1 on the chart); distinct from the real 'livecodebench' dataset stub. Wired: eval-set embed (gym.rs) + catalog entry (Grader::Rust). Build-clean. Target map expanded to ALL 14 chart benches (Joel: target all of these) — Coding 6 + General Agents 6 + Visual 2 — consolidated to 5 harness classes (Rust-single-file LIVE, SWE-repo-patch, real-shell, agentic-app, vision-tool) so building one harness unlocks several benches. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- core/continuum-core/src/cognition/gym.rs | 9 +++++++ core/continuum-core/src/commands/benchmark.rs | 14 +++++++++++ docs/genome/livecodebench-rs.jsonl | 10 ++++++++ docs/planning/BENCHMARK-TARGET-MAP.md | 25 ++++++++++++++++++- 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 docs/genome/livecodebench-rs.jsonl diff --git a/core/continuum-core/src/cognition/gym.rs b/core/continuum-core/src/cognition/gym.rs index 6cedcb9190..54798b3e16 100644 --- a/core/continuum-core/src/cognition/gym.rs +++ b/core/continuum-core/src/cognition/gym.rs @@ -84,6 +84,15 @@ const EMBEDDED_GYMS: &[(&str, &str)] = &[ "frontier-rs.jsonl", include_str!("../../../../docs/genome/frontier-rs.jsonl"), ), + ( + // livecodebench-rs: OUR Rust-graded Program-Bench proxy (10 competitive- + // programming tasks — KMP, union-find, Dijkstra, DP, two-pointer). Named + // -rs to stay honest: it is NOT the real LiveCodeBench dataset (that stays a + // catalogued stub for a future stdio runner), it is the runnable Rust proxy + // mapped to Program Bench in BENCHMARK-TARGET-MAP.md. Every test hand-verified. + "livecodebench-rs.jsonl", + include_str!("../../../../docs/genome/livecodebench-rs.jsonl"), + ), ( // games-rs: OUR games benchmark — the tier public benchmarks lack (they grade // an agent PLAYING, not BUILDING). Auto-verifiable game LOGIC: a Conway step, diff --git a/core/continuum-core/src/commands/benchmark.rs b/core/continuum-core/src/commands/benchmark.rs index 47d2e4461d..5f6e3f995d 100644 --- a/core/continuum-core/src/commands/benchmark.rs +++ b/core/continuum-core/src/commands/benchmark.rs @@ -78,6 +78,20 @@ pub fn known_benchmarks() -> &'static [BenchmarkSpec] { eval_set: Some("frontier-rs.jsonl"), source_url: None, }, + BenchmarkSpec { + name: "livecodebench-rs", + description: "LiveCodeBench-rs — OUR runnable Rust-graded proxy for Program Bench / \ + LiveCodeBench: 10 competitive-programming tasks (KMP substring, union-find \ + components, heap Dijkstra, longest-palindrome, max-product subarray, rotated \ + binary search, trapping rain water, meeting rooms, house robber, k-th largest), \ + rustc compile+run graded, every test hand-verified. Distinct from the \ + `livecodebench` stub (the real dataset, pending a stdio runner) — see \ + BENCHMARK-TARGET-MAP.md.", + grader: Grader::Rust, + tasks: 10, + eval_set: Some("livecodebench-rs.jsonl"), + source_url: None, + }, BenchmarkSpec { name: "games-rs", description: "Games-Rust — OUR games benchmark, the tier public benchmarks lack (they \ diff --git a/docs/genome/livecodebench-rs.jsonl b/docs/genome/livecodebench-rs.jsonl new file mode 100644 index 0000000000..997452326c --- /dev/null +++ b/docs/genome/livecodebench-rs.jsonl @@ -0,0 +1,10 @@ +{"id": "kmp_search", "lang": "rust", "prompt": "Implement `pub fn kmp_search(text: &str, pattern: &str) -> Option`: return the byte index of the FIRST occurrence of `pattern` in `text`, or None if absent. Use O(n+m) Knuth-Morris-Pratt (no naive O(nm), no external crates). A non-empty pattern only (callers never pass empty).", "test": "assert_eq!(kmp_search(\"abxabcabcaby\",\"abcaby\"), Some(6));\n assert_eq!(kmp_search(\"hello\",\"ll\"), Some(2));\n assert_eq!(kmp_search(\"aaa\",\"b\"), None);\n assert_eq!(kmp_search(\"abc\",\"abcd\"), None);"} +{"id": "count_components", "lang": "rust", "prompt": "Implement `pub fn count_components(n: usize, edges: &[(usize, usize)]) -> usize`: given `n` nodes labeled 0..n and an undirected edge list, return the number of connected components. Use union-find or BFS/DFS. No external crates.", "test": "assert_eq!(count_components(5, &[(0,1),(1,2),(3,4)]), 2);\n assert_eq!(count_components(3, &[]), 3);\n assert_eq!(count_components(4, &[(0,1),(1,2),(2,3),(0,3)]), 1);"} +{"id": "dijkstra", "lang": "rust", "prompt": "Implement `pub fn dijkstra(n: usize, edges: &[(usize, usize, u64)], src: usize, dst: usize) -> Option`: `edges` are undirected weighted (u, v, w) over nodes 0..n. Return the minimum total weight from `src` to `dst`, or None if unreachable. Use a binary-heap Dijkstra. No external crates.", "test": "assert_eq!(dijkstra(4, &[(0,1,4),(0,2,1),(2,1,2),(1,3,1)], 0, 3), Some(4));\n assert_eq!(dijkstra(3, &[(0,1,5)], 0, 2), None);\n assert_eq!(dijkstra(1, &[], 0, 0), Some(0));"} +{"id": "longest_palindrome", "lang": "rust", "prompt": "Implement `pub fn longest_palindrome(s: &str) -> usize`: return the LENGTH of the longest palindromic substring of `s` (contiguous). Operate over bytes (inputs are ASCII). Expand-around-center or DP. No external crates.", "test": "assert_eq!(longest_palindrome(\"babad\"), 3);\n assert_eq!(longest_palindrome(\"cbbd\"), 2);\n assert_eq!(longest_palindrome(\"a\"), 1);\n assert_eq!(longest_palindrome(\"\"), 0);"} +{"id": "max_product_subarray", "lang": "rust", "prompt": "Implement `pub fn max_product_subarray(nums: &[i64]) -> i64`: return the largest product of any contiguous non-empty subarray. Track running max AND min (negatives flip). `nums` is non-empty. No external crates.", "test": "assert_eq!(max_product_subarray(&[2,3,-2,4]), 6);\n assert_eq!(max_product_subarray(&[-2,0,-1]), 0);\n assert_eq!(max_product_subarray(&[-2,3,-4]), 24);"} +{"id": "search_rotated", "lang": "rust", "prompt": "Implement `pub fn search_rotated(nums: &[i32], target: i32) -> Option`: `nums` is an ascending array rotated at an unknown pivot with DISTINCT values. Return the index of `target` in O(log n), or None. Modified binary search. No external crates.", "test": "assert_eq!(search_rotated(&[4,5,6,7,0,1,2], 0), Some(4));\n assert_eq!(search_rotated(&[4,5,6,7,0,1,2], 3), None);\n assert_eq!(search_rotated(&[1], 1), Some(0));"} +{"id": "trap_rain", "lang": "rust", "prompt": "Implement `pub fn trap_rain(heights: &[u64]) -> u64`: given an elevation map, return the total units of water trapped after rain. Two-pointer or prefix-max approach. No external crates.", "test": "assert_eq!(trap_rain(&[0,1,0,2,1,0,1,3,2,1,2,1]), 6);\n assert_eq!(trap_rain(&[4,2,0,3,2,5]), 9);\n assert_eq!(trap_rain(&[1,2,3]), 0);"} +{"id": "min_meeting_rooms", "lang": "rust", "prompt": "Implement `pub fn min_meeting_rooms(intervals: &[(i64, i64)]) -> usize`: given meeting `(start, end)` intervals (end exclusive), return the minimum number of rooms needed so no two overlapping meetings share a room. Sort starts/ends or sweep. No external crates.", "test": "assert_eq!(min_meeting_rooms(&[(0,30),(5,10),(15,20)]), 2);\n assert_eq!(min_meeting_rooms(&[(7,10),(2,4)]), 1);\n assert_eq!(min_meeting_rooms(&[]), 0);"} +{"id": "rob_houses", "lang": "rust", "prompt": "Implement `pub fn rob_houses(nums: &[i64]) -> i64`: houses in a row hold `nums[i]` money; you cannot rob two ADJACENT houses. Return the maximum total. Linear DP. No external crates.", "test": "assert_eq!(rob_houses(&[1,2,3,1]), 4);\n assert_eq!(rob_houses(&[2,7,9,3,1]), 12);\n assert_eq!(rob_houses(&[]), 0);"} +{"id": "kth_largest", "lang": "rust", "prompt": "Implement `pub fn kth_largest(nums: &[i32], k: usize) -> i32`: return the k-th LARGEST element (1-indexed: k=1 is the maximum). Duplicates count by position. `1 <= k <= nums.len()`. Heap or sort. No external crates.", "test": "assert_eq!(kth_largest(&[3,2,1,5,6,4], 2), 5);\n assert_eq!(kth_largest(&[3,2,3,1,2,4,5,5,6], 4), 4);\n assert_eq!(kth_largest(&[1], 1), 1);"} diff --git a/docs/planning/BENCHMARK-TARGET-MAP.md b/docs/planning/BENCHMARK-TARGET-MAP.md index e6eaba6ee7..7e28d42e80 100644 --- a/docs/planning/BENCHMARK-TARGET-MAP.md +++ b/docs/planning/BENCHMARK-TARGET-MAP.md @@ -10,6 +10,12 @@ K3's local numbers are comparable to the published charts. Companion to - **CATALOGUED** — named in `known_benchmarks()` with `eval_set: None`. A placeholder that reserves the name + description; needs a runner + dataset to become runnable. ~18 today. +## Target: ALL 14 chart benches (Joel: "we will want to target all of these") +Coding (6): DeepSWE · Terminal Bench 2.1 · FrontierSWE · Program Bench · Kimi Code Bench 2.0 · SWE Marathon. +General Agents (6): GDPval-AA v2 Elo · JobBench · AA-Briefcase Elo · SpreadsheetBench 2 · Automation Bench · BrowseComp. +Visual Agents (2): CharXiv (RQ) w/ tool · Zerobench w/ tool (Pass@5). +Each row below is one target; "RUNNER TYPE" names the harness class it needs (several benches share one harness → build the harness once, unlock many). + ## The map (chart bench → our proxy → runnable target) | K3 chart bench | What it measures | Our RUNNABLE proxy today | Runnable TARGET to build (catalogued now) | @@ -21,11 +27,28 @@ K3's local numbers are comparable to the published charts. Companion to | **Automation Bench**, **BrowseComp**, JobBench | agent driving real apps/web | — (none) | `webarena`, `appworld` | | SpreadsheetBench 2 | spreadsheet manipulation | — (none) | (no catalog entry — add `spreadsheet-bench`) | | **CharXiv**, Zerobench (visual) | chart/figure reasoning w/ vision | `webdev-rs` (perception grader, UI-adjacent) | `design2code` (screenshot→HTML) | -| GDPval, AA-Briefcase (general Elo) | broad agentic knowledge work | — (none) | (Elo-style; needs opponent-pool harness) | +| GDPval-AA, AA-Briefcase (general Elo) | broad agentic knowledge work | — (none) | (Elo-style; needs opponent-pool harness) | +| **JobBench** | applying for / doing job tasks | — (none) | agentic-app harness (shares webarena runner) | +| Kimi Code Bench 2.0 | mixed practical coding (internal) | `coder-eval`, `humaneval-rs`, `livecodebench-rs` | — (proxied; no public dataset) | **Bold** = benches where Kimi K3 tops or near-tops the chart — our highest-value targets, because that's where reproducing the number locally is the strongest proof. +## Runner-type consolidation (build the harness once → unlock many) +The 14 chart benches need only **5 harness classes** — this is the actual build backlog: +1. **Rust-graded single-file** (LIVE): Program Bench, Kimi Code Bench, partial FrontierSWE-tier → + `livecodebench-rs` ✓ (built today), `frontier-rs`, `hard-rs`. **Extend by authoring more tasks.** +2. **Repo-patch / SWE harness** (M5 spine #1945, one seam open): DeepSWE, FrontierSWE, SWE Marathon, + SWE-Lancer → `swe-bench-lite/verified`. **Highest leverage: 3 chart benches from one runner.** +3. **Real-shell harness**: Terminal Bench 2.1 → `terminal-bench`. The persona's tool-executor already + runs shell; the runner is the task-loader + pass/fail on final state. +4. **Agentic-app harness** (heaviest): Automation Bench, BrowseComp, JobBench, AppWorld, GDPval, + AA-Briefcase → `webarena`, `appworld`. Needs real app servers + the doer-toolset (beta plan). + Elo variants (GDPval/Briefcase) add an opponent-pool scorer on top. +5. **Vision-tool harness**: CharXiv, Zerobench, SpreadsheetBench 2, Design2Code → the eye-node + (`perception/observe`) already grades `webdev-rs`; extend to chart/figure QA + spreadsheet ops. + K3 is vision-native (the fork's K3 processor strips vision today; re-enabling it is a later lane). + ## Priority to make runnable (highest proof-value first) 1. **`swe-bench-lite`** — the SWE-family anchors THREE chart benches (DeepSWE/FrontierSWE/SWE-Marathon) and K3 wins SWE-Marathon. M5 has the runner spine (PR #1945: gold RESOLVED, hermetic filter; the From b33abb162da5a897f7acaed8379ead80001d1a66 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 28 Jul 2026 22:22:42 -0500 Subject: [PATCH 37/55] =?UTF-8?q?docs(planning):=20catalog=20IS=20the=20cu?= =?UTF-8?q?rriculum=20=E2=80=94=20in-repo,=20accumulating,=20per-persona?= =?UTF-8?q?=20continuous=20LoRA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Joel: the benchmark catalog we accumulate in-repo is a versioned CURRICULUM, not a test dir. Each runnable collection is both a proving instrument (chart number) AND a training corpus (graded failures -> per-persona LoRA gradient). The per-persona loop: attempt catalog bench -> grade -> fail-datum -> dream-forge trains a LoRA on the failure cluster (idle GPU) -> validate on the held-out shard -> keep-if-better/rollback -> permanently better next run. The LoRA is paged like any genome skill, replicated by persona-RAID, shareable to peers. Catalog accumulates in repo; the learning accumulates in the genome. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../BENCHMARK-AS-LEARNING-FLYWHEEL.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/planning/BENCHMARK-AS-LEARNING-FLYWHEEL.md b/docs/planning/BENCHMARK-AS-LEARNING-FLYWHEEL.md index 366072e244..93c25c9c31 100644 --- a/docs/planning/BENCHMARK-AS-LEARNING-FLYWHEEL.md +++ b/docs/planning/BENCHMARK-AS-LEARNING-FLYWHEEL.md @@ -92,6 +92,31 @@ discipline (#1584) is the law. A learning loop on a lying instrument optimizes t 4. **Dream-forge consumes eval-fails** (sentinel slice 4 + gap 3) — the loop closes; idle GPU learns. 5. **Held-out discipline + AttnRes stochastic-drop training** (gap 3) — generalization, not memorization. +## The catalog IS the curriculum (in-repo, accumulating, per-persona LoRA) +(Joel, 2026-07-28: "we will accumulate this as a catalog we support in repo, so we can run and learn +from them — continuous LoRA persona.") + +The benchmark catalog (`known_benchmarks()` + the `docs/genome/*.jsonl` eval sets) is not a test +directory — it is a **versioned curriculum that ships in the repo**. Each runnable collection is +simultaneously: +- a **proving instrument** (run it → a chart number, held-out), and +- a **training corpus** (its graded failures → per-persona LoRA gradient). + +Because it lives in-repo, the curriculum is reproducible, diffable, and grows monotonically: every new +runnable bench (like `livecodebench-rs` today) permanently widens both what we can prove AND what the +personas can learn from. The loop, per persona: +``` +persona attempts catalog bench → grader → PASS proves / FAIL → eval-fail datum + → dream-forge trains a per-persona LoRA on the failure cluster (idle GPU) + → validate on the SAME bench held-out shard → keep adapter if it improves, roll back if not + → the being is measurably better at that bench next run — permanently, on its own genome +``` +This is where the flywheel meets the genome: the LoRA the being trains from the catalog is paged like +any other skill ([[continuum-substrate-already-built]] genome tiers), replicated by persona-RAID +([[restarts-are-commonplace]]), and shareable to peers ([[being-axis-shareable-learning]]). One +persona's hard-won adapter on `swe-bench-lite` becomes a paged skill any peer can borrow. The catalog +accumulates in the repo; the *learning from it* accumulates in the genome. + ## Why this is the whole thing It unifies every session thread: the grid (distribution), the benchmark suite (signal), sentinel plasticity (the update), AttnRes (generalization), persona-RAID (the being that persists across the From d83e57f6487aea8ed83120a4486df5fb59b4bd5b Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 05:30:12 -0500 Subject: [PATCH 38/55] =?UTF-8?q?docs(planning):=20K3=20paging=20diagnosis?= =?UTF-8?q?=20=E2=80=94=20accuracy=20is=20FREE,=20it's=20all=20speed;=20ro?= =?UTF-8?q?ad=20to=20par?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reframes the pager problem: a MoE router picks the same experts regardless of placement, so a correct pager is ALWAYS at par on OUTPUT (faults cold experts in, never skips) — 'closer to par' is 100% a SPEED question. The speed regimes (hot=par / warm=~5tok-s / cold=0.3-0.5tok-s / relaunch=stall) and the levers: (1) kill the relaunch stall via slice-2 live-upload (biggest win, needs the vendored-llama accessor), (2) maximize hot-set hit-rate (the par asymptote, instrument it), (3) never spill to disk while RAM has room, (4) overlap fault with compute. Slice-1 already has the right shape (tiered residency, PGO profiling, cross-layer prefetch, churn-thresholded relaunch). Iterate loop + the -ncmoe static baseline the dynamic pager must beat. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/K3-PAGING-DIAGNOSIS.md | 82 ++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 docs/planning/K3-PAGING-DIAGNOSIS.md diff --git a/docs/planning/K3-PAGING-DIAGNOSIS.md b/docs/planning/K3-PAGING-DIAGNOSIS.md new file mode 100644 index 0000000000..42cd3abc04 --- /dev/null +++ b/docs/planning/K3-PAGING-DIAGNOSIS.md @@ -0,0 +1,82 @@ +# K3 Expert Paging — diagnosis + the road to par-with-full-in-memory + +**Status:** diagnosis (2026-07-29, BigMama). K3 weights on disk + converting; this is the plan to +iterate the pager to par once the GGUF lands. Reads the merged slice-1 pager (`capacity/expert_*`). + +## The one clarification that reframes everything: PAGING CANNOT HURT ACCURACY + +A MoE router picks the same top-k experts for a token regardless of where those experts physically +live. A paged expert computes the **bit-identical** result of a resident one — the weights are the +same, only the fetch latency differs. So a correctly-implemented pager is **always at par with +full-in-memory on OUTPUT**. The design guarantees this by FAULTING a needed cold expert in (RAM/disk) +rather than skipping it. + +The ONLY way paging diverges from full-in-memory is a bug that *skips* an expert (returns zero / +approximates on a cold miss). We do not do that. **So "closer to par" is 100% a SPEED question, never +an accuracy one.** Every lever below is a latency lever. + +## The speed regimes (where the tok/s actually goes) + +Per token K3 activates ~8 of 896 experts (~1.8%), ≈27–32 GB of expert reads/token @4bit if every one +is a cold disk miss. Placement decides which regime each activated expert lands in: + +| Regime | Where | Fault cost/expert | K3 all-in-this-regime | +|---|---|---|---| +| **hot** | GPU-resident | 0 (never faults) | full GPU speed = PAR | +| **warm** | RAM → GPU stream/token | PCIe ~25 GB/s → ~24 ms/600MB expert | ~5 tok/s if all 8 warm | +| **cold** | disk → RAM/miss | SSD 1–5 GB/s → 120–600 ms | 0.3–0.5 tok/s (unusable) | +| **relaunch churn** (slice-1) | full model reload on set change | seconds, one-shot | stalls the stream | + +**The whole game: get as many of each token's 8 activated experts into the `hot` regime as possible, +keep the rest `warm` (RAM, never `cold`/disk), and never relaunch mid-stream.** + +## What slice-1 already does RIGHT (don't rebuild) +- **Tiered residency** hot(VRAM)/warm(RAM)/cold(disk) with LRU + fault semantics — the right shape. +- **Sentinel-PGO profiling** (`ExpertActivationProfile.hits` from M5's `ffn_moe_topk` callback) — keeps + what's proven hot. +- **Cross-layer prefetch** (`CrossLayerExpertPredictor`): when layer-L experts fire, prefetch the + likely layer-(L+k) experts RAM→VRAM *ahead* of the pass reaching them — turns reactive + miss→stall→load into proactive paging. The prefetch window = the depth of the forward pass. +- **Churn-thresholded relaunch** — a few experts drifting is noise; only a material set change relaunches. + +## The gaps to par (iterate here, in priority order) + +### 1. THE relaunch stall — kill it (slice-2, biggest single win) +Slice-1 places experts via buft-override at LOAD time; a materially-changed hot set RELAUNCHES the +served context (seconds, stream stops). For a workload whose expert demand shifts across prompts, this +is the dominant non-par cost. **Fix: slice-2 live per-expert RAM→VRAM upload** — the SAME `page_in` +body swaps from "accumulate into next relaunch" to a live `load_expert(layer, idx, ptr)` call. Needs +the vendored-llama accessor (`get_tensor`/`upload_expert` on a loaded model) — a fork change we can +author now while K3 converts. This is the A-path in [[k3-slice2-A-vs-B-decision]] (weight-write into a +resident slot), NOT the harder K-slot router remap (B, gated on measured numbers). + +### 2. Maximize hot-set COVERAGE (the par asymptote) +Par is reached when ~100% of activations hit `hot`. K3's expert-reuse locality (a task domain reuses a +narrow subset) is why this is feasible — the memory's "subset residency drops disk reads 10–100×". +Lever: profiling quality. Instrument + optimize the **hot-set hit-rate** (fraction of activated experts +found already-hot) — THE number that says how close to par we are. Co-activation clustering (experts +that fire together in one token placed together) tightens it further. + +### 3. Never spill to `cold` (disk) while RAM has room +63 GB RAM holds ~100 MXFP4 experts. The tiering budget must MAX warm (RAM) before any cold (disk) +placement — a disk fault is 5–25× a RAM fault. Audit `plan_expert_residency`'s budget order. + +### 4. Overlap fault with compute (hide the warm latency) +A warm fault (RAM→GPU) for layer L+k can be issued (async DMA) while layer L computes — the prefetch +already targets this; ensure the copy is truly async so the ~24 ms/expert overlaps compute instead of +stalling. (llama.cpp CUDA streams.) + +## The iterate loop (once K3 GGUF lands) +1. Serve K3 with slice-1 (static hot-set via -ncmoe/-ot, no dynamic relaunch first) → baseline tok/s + + **hot-set hit-rate**. +2. Turn on the cross-layer predictor → measure hit-rate lift + tok/s. +3. Land slice-2 live-upload → measure the relaunch-stall elimination. +4. Compare to a full-in-memory reference on a smaller MoE (Mixtral) to calibrate the par gap. +Each step is a number, not a guess — the honest instrument ([[benchmark-learning-flywheel]]) applied to +the pager itself. + +## Startup default (fastest path to a serving K3 while we iterate) +llama.cpp native `-ncmoe N` / `-ot` places the first N MoE layers' experts on CPU RAM, computed there, +NO relaunch churn — the KTransformers steady-state. For a first serving K3: put as many expert layers +on GPU as the budget fits, the rest CPU-RAM. Stable, predictable, and the honest baseline the dynamic +pager must BEAT to justify its complexity. From b56dc271e61ed6b4dcdaeac3d403fd21a79c899f Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 05:31:58 -0500 Subject: [PATCH 39/55] =?UTF-8?q?docs(planning):=20the=20adapter=20path=20?= =?UTF-8?q?to=20K3=20par=20(Joel:=20Adapters)=20=E2=80=94=20compensation-L?= =?UTF-8?q?oRA=20+=20adapter-as-paging-unit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two adapter angles that beat brute-force expert paging: (A) adapters as the paging UNIT — a LoRA is 1-50MB vs a 600MB expert, 10-100x cheaper to page, and the genome already pages adapters per task-domain; (B) compensation-LoRA — prune K3 to the VRAM-fitting hot subset, train a small adapter that recovers the pruned experts' accuracy (the sentinel §4.1.3.4 move we already proved offline making the 19B) => serves fully in VRAM, no paging churn, near-par at full speed. Paging (correct, fault real expert) and compensation-LoRA (fast, pruned+adapt) are two ends of one dial; the adapter is trained from the flywheel's graded failures. Same genome machinery, adapter as the currency. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/K3-PAGING-DIAGNOSIS.md | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/planning/K3-PAGING-DIAGNOSIS.md b/docs/planning/K3-PAGING-DIAGNOSIS.md index 42cd3abc04..882e85834f 100644 --- a/docs/planning/K3-PAGING-DIAGNOSIS.md +++ b/docs/planning/K3-PAGING-DIAGNOSIS.md @@ -75,6 +75,36 @@ stalling. (llama.cpp CUDA streams.) Each step is a number, not a guess — the honest instrument ([[benchmark-learning-flywheel]]) applied to the pager itself. +## THE adapter path — the smarter road to par (Joel: "Adapters") + +Paging 594 GB of full experts is the brute-force framing. The substrate's own plasticity gives a +cheaper one, on two fronts: + +### A. Adapters as the paging UNIT (cheap to move) +A full MXFP4 expert is ~600 MB; a LoRA adapter for a skill is ~1–50 MB. The genome already **pages +adapters** ([[continuum-substrate-already-built]] genome tiers). So the hot capability isn't only "the +hot expert subset resident" — it's "the base model + the paged-in adapter for THIS task domain." Paging +an adapter is 10–100× cheaper than paging an expert, and it's a warm-fault the substrate already does +well. Frontend-code, chat, and vision each ride their own adapter, no 600 MB expert churn. + +### B. Compensation-LoRA — turn the 894-expert tail into a small adapter (the par lever) +This is the [[sentinel-in-substrate]] §4.1.3.4 move applied to K3: **prune K3 to the hot expert subset +that FITS VRAM, then train a small compensation LoRA on a held-out corpus that recovers the accuracy +the pruned experts provided.** The result serves entirely in VRAM — NO paging churn, NO warm/cold +faults, full GPU speed — and the compensation LoRA closes most of the accuracy gap to the full model. +This converts "594 GB paging problem" into "a fits-in-VRAM subset + a ~tens-of-MB adapter." It is the +same algorithm we ALREADY proved offline in `tools/scripts/compaction` (the Plasticity Compaction that +produced the 19B) — the product is porting it to a DYNAMIC, per-domain compensation adapter K3 pages +by task ([[moe-expert-paging-feasibility]]). + +**The synthesis:** paging (fault the real expert when needed, at-par output) and compensation-LoRA +(prune + adapt, near-par at full speed) are the two ends of a dial. Cold-start / rare-domain → page the +real expert (correctness). Hot domain → serve the pruned subset + compensation adapter (speed). The +pager and the foundry are the same genome machinery; the adapter is the cheap currency between them. +This is also where the benchmark flywheel closes: the compensation LoRA is TRAINED from the catalog's +graded failures ([[benchmark-learning-flywheel]]) — the being learns the adapter that makes its pruned +K3 match the full K3 on the exact tasks it's measured on. + ## Startup default (fastest path to a serving K3 while we iterate) llama.cpp native `-ncmoe N` / `-ot` places the first N MoE layers' experts on CPU RAM, computed there, NO relaunch churn — the KTransformers steady-state. For a first serving K3: put as many expert layers From d1df7b52484d6f54e89dd18b32020487ab21174d Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 05:47:21 -0500 Subject: [PATCH 40/55] =?UTF-8?q?docs(planning):=20K3=20conversion=20statu?= =?UTF-8?q?s=20=E2=80=94=20done=20+=20precise=20remaining=20gaps=20(MoE=20?= =?UTF-8?q?finish)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pins the EXACT remaining K3 conversion gaps so the finish is reference-guided not error-probed. DONE: MXFP4 dequant, AttnRes tensors+graph, MLA output-gate mapping (all 14 self_attn tensors map); converter passes layer-0 + layer-1 attention, fails at layer-1 MoE. REMAINING: (1) experts.N.w1/w2/w3 MXFP4 -> stacked ffn_*_exps (verify w1/w2/w3=gate/up/down from modeling), (2) NOVEL fused routed_expert_up/down_proj+norm (no 48B equiv — needs modeling study + new C++ graph, M5's serving lane), (3) router+shared verify. Paired C++ correctness: MLA gate apply, routed-expert transform, MXFP4-native-serving. Validation gate: coherent generation only. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/K3-CONVERSION-STATUS.md | 56 +++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 docs/planning/K3-CONVERSION-STATUS.md diff --git a/docs/planning/K3-CONVERSION-STATUS.md b/docs/planning/K3-CONVERSION-STATUS.md new file mode 100644 index 0000000000..491bf5699e --- /dev/null +++ b/docs/planning/K3-CONVERSION-STATUS.md @@ -0,0 +1,56 @@ +# K3 GGUF Conversion — status + the precise remaining gaps + +**Status:** 2026-07-29, BigMama. K3 weights downloaded + verified (96 shards, 1.5TB). The converter +("model adapter") + fork now handle a large fraction of K3; this pins the EXACT remaining gaps so the +finish is a clean, reference-guided task (not error-by-error probing). Fork branch +`feat/kimi-k3-attnres`. + +## DONE (converter runs through attention + gets into MoE) +- **MXFP4 dequant** (`conversion/base.py`): `mxfp4-pack-quantized` compressed-tensors branch — E2M1, + group 32, E8M0 scale. K3's routed experts + anything `.weight_packed`/`.weight_scale`. +- **AttnRes tensors** (gguf-py + C++): `self_attention_res_{norm,proj}` → `attn_res_*`, + `mlp_res_{norm,proj}` → `ffn_res_*`; the ggml graph op is built + compile-verified (48B byte-identical + when `attn_res_block_size=0`). +- **MLA output gate** (gguf-py): `self_attn.g_proj` → `ATTN_OUT_GATE` (`blk.{bid}.attn_gate`). All 14 of + K3's `self_attn.*` tensors now map. SiTU op exists (d82d02963). +- Converter progress: passes all of layer 0 (dense) + layer 1 attention; **fails at layer-1 MoE.** + +## REMAINING GAPS (in converter-error order — each bounded, reference-guided) + +### 1. Routed experts — `experts.N.w1/w2/w3` (MXFP4) → stacked GGUF expert tensors +K3 names the 896 experts' three projections `w1/w2/w3` (each `.weight_packed`+`.weight_scale`), NOT the +48B's `gate/up/down_proj`. **VERIFY from `modeling_kimi_k3.py` MoE forward which of w1/w2/w3 is +gate/up/down** (common convention w1=gate, w3=up, w2=down — but confirm, a swap silently breaks it). +Then map + let the existing MoE stacker build `blk.N.ffn_{gate,up,down}_exps` (MXFP4 per-expert). + +### 2. **NOVEL** fused routed-expert transform — `routed_expert_up_proj` / `routed_expert_down_proj` / +`routed_expert_norm` (single tensors, NOT per-expert) +The 48B has NO equivalent. This is a K3 architectural component applied to the routed-expert path. +**Requires: (a) study `modeling_kimi_k3.py` to learn the exact forward (where norm/up/down apply +relative to the expert sum), (b) new GGUF tensor slots + tensor-map entries, (c) NEW C++ graph code in +`kimi-linear.cpp`'s MoE branch to apply it.** This is the genuinely new piece — correctness-critical, +M5's llama-serving lane. + +### 3. Router + shared experts — likely already mapped (48B parity) +`gate.weight`, `gate.e_score_correction_bias`, `shared_experts.{gate,up,down}_proj` — verify these +resolve (the 48B had shared experts + a router; they're in tensor_mapping). Probably free. + +## PAIRED C++ SERVING-CORRECTNESS CHANGES (needed before K3 serves right; conversion alone isn't enough) +- **MLA output gate**: apply `attn_output *= sigmoid(g_proj(x))` before `o_proj` when the gate tensor is + present (ref `modeling_kimi_linear.py:470-472`). 48B path untouched (tensor absent). NOT yet in C++. +- **AttnRes graph**: DONE (compile-verified). +- **Routed-expert fused transform** (gap 2's C++ half). +- **MXFP4 native serving** vs dequant-to-bf16-then-requant: GGUF supports `MOSTLY_MXFP4_MOE` (type 38) + natively — the faithful path keeps experts MXFP4 (no 2TB bf16 roundtrip). The current bf16 dequant + path is the correctness-first proof; MXFP4-passthrough is the efficiency follow-up. + +## THE validation gate (unchanged) +None of this is proven until **K3 generates coherent text**. The MXFP4 nibble-order, the w1/w2/w3 +mapping, the routed-expert transform, the MLA gate — each silently "converts" if wrong. Coherent +generation on the served GGUF is the only real proof. Then the [[k3-paging-diagnosis]] + adapter-path +work makes it fast. + +## Lane split (per the M5 collab) +BigMama has taken the converter through attention + MXFP4 + AttnRes + the MLA-gate mapping. The MoE +finish (gaps 1–2, esp. the novel routed-expert transform + its C++ graph) is deep llama-internals = +M5's serving lane; BigMama owns the 5090 serving target + validation once it converts. From 7dab2774b4fe2839f95c4263c218da729b84640f Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 10:56:44 -0500 Subject: [PATCH 41/55] =?UTF-8?q?feat(capacity):=20node=5Fcontent=20?= =?UTF-8?q?=E2=80=94=20the=20grid=20who-has-what=20registry=20(foundation?= =?UTF-8?q?=20for=203=20grid-native=20serving=20wins)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Distinct from DeviceCapacity (what a node can FIT): what each live node HOLDS. NodeContent { resident_models, expert_shards, warm_prefixes } + GridContentIndex queries: locate_expert (cross-node MoE sharding — route the ~16 ACTIVE experts to their holder, sparse traffic vs exo's dense per-layer hop), best_prefix_holder (prefix-aware routing — send to the node with the longest warm KV, skip prefill), nodes_with_model (model sharing). Churn-safe by construction: per-snapshot index, a dropped node's content vanishes -> queries fall back to another holder or local Fault. Nodes-up-and-down is the design assumption. Tests: expert-route+fault-on-drop, longest-prefix-wins. The structural exo-beating shortcut (MoE sparsity) encoded in the type. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- core/continuum-core/src/capacity/mod.rs | 1 + .../src/capacity/node_content.rs | 207 ++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 core/continuum-core/src/capacity/node_content.rs diff --git a/core/continuum-core/src/capacity/mod.rs b/core/continuum-core/src/capacity/mod.rs index 8a4b977aad..47122a0da5 100644 --- a/core/continuum-core/src/capacity/mod.rs +++ b/core/continuum-core/src/capacity/mod.rs @@ -34,6 +34,7 @@ pub mod grid_overflow; pub mod lease; pub mod model_residency; pub mod moe_serving; +pub mod node_content; pub mod placement; pub mod recursion_depth; pub mod residency_detect; diff --git a/core/continuum-core/src/capacity/node_content.rs b/core/continuum-core/src/capacity/node_content.rs new file mode 100644 index 0000000000..993e060fea --- /dev/null +++ b/core/continuum-core/src/capacity/node_content.rs @@ -0,0 +1,207 @@ +//! `node_content` — the grid's "who-has-what" registry: what each live node +//! HOLDS, distinct from `DeviceCapacity` (what it can FIT). +//! +//! `GridSnapshot` already answers "how many lanes fit where" (capacity). To make +//! the grid smarter than a pile of toy GPUs, the router also needs "who already +//! holds the thing this request needs" (content). That is the foundation for the +//! three grid-native serving wins ([[product-strategy-vs-exo]]): +//! 1. **cross-node expert sharding** — K3's 896 experts split across nodes; +//! route each token's ~16 ACTIVE experts to whichever node holds them (sparse +//! traffic, unlike exo's dense per-layer LAN hop). +//! 2. **prefix-aware routing** — send a multi-turn request to the node whose KV +//! cache is already warm for its prefix → skip prefill. +//! 3. **model sharing** — route to a node that already has the model resident, +//! instead of cold-loading it. +//! +//! Churn-safe by construction ([[restarts-are-commonplace]]): content is indexed +//! per live `PeerId` from a point-in-time view. A node that drops simply isn't in +//! the index — its content vanishes, queries fall back to another holder or to +//! local recompute. Nothing is held across the churn; every query re-derives. + +use std::collections::HashMap; + +use crate::identity::PeerId; + +/// Which experts of a model a node holds — the unit of cross-node MoE sharding. +/// A contiguous `[first, last]` expert range (inclusive) per (model, layer); +/// ranges compose so a query can find the holder of any activated expert id. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExpertShard { + pub model_id: String, + /// Layer this shard's experts belong to (MoE experts are per-layer). + pub layer: u32, + /// Inclusive expert-id range this node holds for `(model_id, layer)`. + pub first_expert: u32, + pub last_expert: u32, +} + +impl ExpertShard { + #[inline] + pub fn holds(&self, model_id: &str, layer: u32, expert: u32) -> bool { + self.model_id == model_id + && self.layer == layer + && expert >= self.first_expert + && expert <= self.last_expert + } +} + +/// A warm KV-cache prefix a node can continue without recomputing prefill. +/// `hash` is a rolling hash of the prompt token prefix; `token_len` lets the +/// router prefer the LONGEST warm prefix match (most prefill skipped). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WarmPrefix { + pub model_id: String, + pub hash: u64, + pub token_len: u32, +} + +/// What ONE node advertises it holds this tick. Small + cheap to gossip alongside +/// `DeviceCapacity` on the existing snapshot cadence. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct NodeContent { + /// Model ids fully resident + serve-ready on this node. + pub resident_models: Vec, + /// Expert shards this node holds (for models it serves as a shard, not whole). + pub expert_shards: Vec, + /// Warm KV prefixes this node can continue. + pub warm_prefixes: Vec, +} + +impl NodeContent { + pub fn holds_model(&self, model_id: &str) -> bool { + self.resident_models.iter().any(|m| m == model_id) + } +} + +/// Where a given (model, layer, expert) lives on the grid, if anywhere live. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ExpertLocation { + /// The local node holds it — no network hop. + Local, + /// A peer holds it — route the sparse activation there. + Peer(PeerId), + /// No live node holds it — fault it in locally (page from disk) as the fallback. + Fault, +} + +/// The grid-level content query. Built per point-in-time view from the LOCAL +/// node's content + the CURRENTLY-LIVE peers' content. A dropped peer is absent, +/// so every answer names only reachable holders. +#[derive(Debug, Clone, Default)] +pub struct GridContentIndex { + local: NodeContent, + peers: HashMap, +} + +impl GridContentIndex { + pub fn new(local: NodeContent, peers: HashMap) -> Self { + Self { local, peers } + } + + /// Prefix-aware routing: the live node with the LONGEST warm prefix for this + /// model whose hash matches — or None (do prefill locally). `None` peer means + /// the local node is the best holder. + pub fn best_prefix_holder(&self, model_id: &str, hash: u64) -> Option<(Option, u32)> { + let mut best: Option<(Option, u32)> = None; + let mut consider = |peer: Option, c: &NodeContent| { + for p in &c.warm_prefixes { + if p.model_id == model_id && p.hash == hash { + if best.as_ref().map(|(_, len)| p.token_len > *len).unwrap_or(true) { + best = Some((peer.clone(), p.token_len)); + } + } + } + }; + consider(None, &self.local); + for (id, c) in &self.peers { + consider(Some(id.clone()), c); + } + best + } + + /// Cross-node expert sharding: where a token's activated expert lives. Local + /// wins (no hop); else the first live peer holding it; else Fault (page local). + pub fn locate_expert(&self, model_id: &str, layer: u32, expert: u32) -> ExpertLocation { + if self.local.expert_shards.iter().any(|s| s.holds(model_id, layer, expert)) { + return ExpertLocation::Local; + } + for (id, c) in &self.peers { + if c.expert_shards.iter().any(|s| s.holds(model_id, layer, expert)) { + return ExpertLocation::Peer(id.clone()); + } + } + ExpertLocation::Fault + } + + /// Model sharing: live nodes with this model fully resident (local first). + pub fn nodes_with_model(&self, model_id: &str) -> Vec> { + let mut out = Vec::new(); + if self.local.holds_model(model_id) { + out.push(None); + } + for (id, c) in &self.peers { + if c.holds_model(model_id) { + out.push(Some(id.clone())); + } + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn peer(n: u128) -> PeerId { + PeerId::from_u128(n) + } + + // what this catches: expert-sharding routing must send an activated expert to + // the node that actually holds its range, and Fault when nobody live does — + // a wrong holder or a silent skip corrupts the MoE forward. + #[test] + fn locate_expert_routes_to_holder_and_faults_when_absent() { + let local = NodeContent { + expert_shards: vec![ExpertShard { + model_id: "k3".into(), layer: 5, first_expert: 0, last_expert: 447, + }], + ..Default::default() + }; + let mut peers = HashMap::new(); + peers.insert(peer(2), NodeContent { + expert_shards: vec![ExpertShard { + model_id: "k3".into(), layer: 5, first_expert: 448, last_expert: 895, + }], + ..Default::default() + }); + let idx = GridContentIndex::new(local, peers); + assert_eq!(idx.locate_expert("k3", 5, 10), ExpertLocation::Local); + assert_eq!(idx.locate_expert("k3", 5, 500), ExpertLocation::Peer(peer(2))); + // expert id out of every live shard's range -> fault in locally + assert_eq!(idx.locate_expert("k3", 5, 900), ExpertLocation::Fault); + // a DROPPED peer: rebuild index without it -> its experts now Fault + let idx2 = GridContentIndex::new( + NodeContent { expert_shards: vec![ExpertShard { + model_id: "k3".into(), layer: 5, first_expert: 0, last_expert: 447 }], ..Default::default() }, + HashMap::new()); + assert_eq!(idx2.locate_expert("k3", 5, 500), ExpertLocation::Fault); + } + + // what this catches: prefix-aware routing must pick the LONGEST warm match + // (most prefill skipped), not just any match. + #[test] + fn best_prefix_holder_prefers_longest() { + let local = NodeContent { + warm_prefixes: vec![WarmPrefix { model_id: "m".into(), hash: 7, token_len: 100 }], + ..Default::default() + }; + let mut peers = HashMap::new(); + peers.insert(peer(3), NodeContent { + warm_prefixes: vec![WarmPrefix { model_id: "m".into(), hash: 7, token_len: 500 }], + ..Default::default() + }); + let idx = GridContentIndex::new(local, peers); + assert_eq!(idx.best_prefix_holder("m", 7), Some((Some(peer(3)), 500))); + assert_eq!(idx.best_prefix_holder("m", 999), None); // no warm match -> prefill local + } +} From 0983320589f0dafa334ea3e81efb79ddc41fdbb2 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 11:29:24 -0500 Subject: [PATCH 42/55] =?UTF-8?q?docs(paging):=20correct=20tiering=20?= =?UTF-8?q?=E2=80=94=20mechanical=20disk=20never=20a=20live=20fault=20sour?= =?UTF-8?q?ce;=20flash=20is=20THE=20cold=20ready-cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Joel: never fault mechanical-HDD->VRAM. Hierarchy VRAM<-RAM<-FLASH(cold-ready-cache) <-mechanical(archival, staged-from, background-promoted). Flash is architectural for models > VRAM+RAM; mechanical-only means the model MUST fit VRAM+RAM. Regime table + expert_residency/disk-manager targeting corrected. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/K3-PAGING-DIAGNOSIS.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/planning/K3-PAGING-DIAGNOSIS.md b/docs/planning/K3-PAGING-DIAGNOSIS.md index 882e85834f..56873718b2 100644 --- a/docs/planning/K3-PAGING-DIAGNOSIS.md +++ b/docs/planning/K3-PAGING-DIAGNOSIS.md @@ -15,6 +15,29 @@ The ONLY way paging diverges from full-in-memory is a bug that *skips* an expert approximates on a cold miss). We do not do that. **So "closer to par" is 100% a SPEED question, never an accuracy one.** Every lever below is a latency lever. + +## CORRECTED tiering (Joel 2026-07-29): mechanical disk is NEVER a live fault source + +The hot path must never fault mechanical-HDD -> VRAM. The correct hierarchy: + +``` +VRAM (hot, GPU) <- RAM (warm) <- FLASH/NVMe (cold-READY cache) <- mechanical HDD (archival, staged-FROM) +``` + +- **FLASH/NVMe is THE cold cache layer** — experts live here "ready to go." A cold fault + reads from flash (fast), not spinning disk. +- **Mechanical HDD is bulk archival only.** Experts get PREFETCHED mechanical->flash in the + BACKGROUND (the cross-layer predictor's job) so they are resident in flash before the router + needs them. A live hot-path read that reaches spinning disk = the prefetch FAILED, not normal. +- **Hardware implication:** flash is ARCHITECTURAL, not optional, for any model that exceeds + VRAM+RAM. With only a mechanical drive, the model MUST fit VRAM+RAM entirely (no live faulting) + or it is unusable regardless of pager quality. (This box: C: NVMe full, D: mechanical -> K3 has + no ready-cache tier here; needs a flash drive, OR grid-sharding pools peers' RAM as the warm + tier over LAN which for SPARSE MoE beats mechanical disk, OR compaction to fit VRAM+RAM.) +- The `expert_residency` cold tier + `disk-manager` (multi-drive offload) must therefore target + FLASH for the ready-cache and treat mechanical as stage-from archival, with a background + mechanical->flash promotion path, never a mechanical->VRAM fault path. + ## The speed regimes (where the tok/s actually goes) Per token K3 activates ~8 of 896 experts (~1.8%), ≈27–32 GB of expert reads/token @4bit if every one @@ -24,7 +47,8 @@ is a cold disk miss. Placement decides which regime each activated expert lands |---|---|---|---| | **hot** | GPU-resident | 0 (never faults) | full GPU speed = PAR | | **warm** | RAM → GPU stream/token | PCIe ~25 GB/s → ~24 ms/600MB expert | ~5 tok/s if all 8 warm | -| **cold** | disk → RAM/miss | SSD 1–5 GB/s → 120–600 ms | 0.3–0.5 tok/s (unusable) | +| **cold** | FLASH/NVMe → RAM (ready-cache) | NVMe 3–7 GB/s → ~90–200 ms | usable; the real cold tier | +| **archival** | mechanical HDD | NEVER in hot path — staged->flash in background | prefetch-failed if hit live | | **relaunch churn** (slice-1) | full model reload on set change | seconds, one-shot | stalls the stream | **The whole game: get as many of each token's 8 activated experts into the `hot` regime as possible, From b4a5d37d6b894c24068360ba588db256e901390d Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 11:38:33 -0500 Subject: [PATCH 43/55] =?UTF-8?q?docs(paging):=20working-set-not-model-siz?= =?UTF-8?q?e=20reframe=20=E2=80=94=20a=20focused=20task's=20~230-expert=20?= =?UTF-8?q?set=20fits=20fast=20mem=20=3D=20tens=20of=20tok/s=20on=20one=20?= =?UTF-8?q?box?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kills the defeatism: expert activation is a power law, only 8-16/896 fire/token, a focused task reuses a NARROW subset. What needs fast memory is the WORKING SET, not the model. Math: <=~230 experts (25% of 896, IQ2) fits ~92GB (32 VRAM + 60 RAM) = fast. OS page cache does it free via -ncmoe+mmap. Diminished forms (prune cold tail, asymmetric quant, per-domain adapter) guarantee the fit; each 'works + peers improve it'. Metric = tok/s on the WARM working set of a FOCUSED task, never model-size-as-ceiling. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/K3-PAGING-DIAGNOSIS.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/planning/K3-PAGING-DIAGNOSIS.md b/docs/planning/K3-PAGING-DIAGNOSIS.md index 56873718b2..6590443e49 100644 --- a/docs/planning/K3-PAGING-DIAGNOSIS.md +++ b/docs/planning/K3-PAGING-DIAGNOSIS.md @@ -134,3 +134,25 @@ llama.cpp native `-ncmoe N` / `-ot` places the first N MoE layers' experts on CP NO relaunch churn — the KTransformers steady-state. For a first serving K3: put as many expert layers on GPU as the budget fits, the rest CPU-RAM. Stable, predictable, and the honest baseline the dynamic pager must BEAT to justify its complexity. + +## THE reframe that kills the defeatism (Joel 2026-07-29): working set, not model size + +The wall is NOT "350GB model > 63GB RAM." That treats all 896 experts as equally likely — WRONG. +Expert activation is a POWER LAW; only ~8-16/896 fire per token, and a FOCUSED task reuses a NARROW +subset. What must live in fast memory is the **working set** (distinct experts a task actually hits), +not the model. Math on this box (~32GB VRAM + ~60GB usable RAM = ~92GB fast tier, K3 IQ2 ~0.4GB/expert): + +| Task working set | Experts | Size | Fits ~92GB fast tier? | +|---|---|---|---| +| 15% of 896 | 134 | 54GB | **YES** | +| 25% of 896 | 224 | 90GB | **YES** | +| 35% of 896 | 313 | 125GB | no (spill / prune) | + +**So a focused task whose working set is ≤~230 experts runs ENTIRELY from RAM/VRAM = tens of tok/s, +on ONE box, TODAY.** The OS page cache does this for free with `-ncmoe`+mmap: hot experts stay +resident, cold evict. The pager just makes it deterministic + prefetched. + +**Diminished forms that shrink the working set to guarantee the fit** (all "works + adding peers +improves it"): prune the cold expert tail (rare experts fall back to shared/dense path), asymmetric +quant (hot IQ2/Q4, cold IQ1), per-domain adapter. The proof metric is tok/s on the WARM working set of +a FOCUSED task — never cold-start random prompts. Never quote model size as the ceiling again. From a922583a3ffb6f8f7e28afdef45b138fa624c383 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 11:40:14 -0500 Subject: [PATCH 44/55] =?UTF-8?q?docs(paging):=20NO=20PRUNING=20=E2=80=94?= =?UTF-8?q?=20all=20experts=20stay=20available;=20cache=20levels=20+=20dyn?= =?UTF-8?q?amic=20hosting=20(Joel)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correction: we do NOT prune (that's a diminished MODEL). All 896 experts stay AVAILABLE; the game is which CACHE LEVEL each sits at, set by demand: VRAM<-RAM<-FLASH<-PEER-RAM-over-grid <-archival. Full quality always (rare expert = slower, never missing). Dynamic hosting = grid distributes experts, router finds each at its cache level. Adding a peer moves more experts to a FAST tier, never unlocks capability (already full). Compensation-LoRA/prune demoted to optional speed mode. Path = all-experts-available, cached-by-level, dynamically-hosted. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/K3-PAGING-DIAGNOSIS.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/planning/K3-PAGING-DIAGNOSIS.md b/docs/planning/K3-PAGING-DIAGNOSIS.md index 6590443e49..d1b016aacf 100644 --- a/docs/planning/K3-PAGING-DIAGNOSIS.md +++ b/docs/planning/K3-PAGING-DIAGNOSIS.md @@ -156,3 +156,24 @@ resident, cold evict. The pager just makes it deterministic + prefetched. improves it"): prune the cold expert tail (rare experts fall back to shared/dense path), asymmetric quant (hot IQ2/Q4, cold IQ1), per-domain adapter. The proof metric is tok/s on the WARM working set of a FOCUSED task — never cold-start random prompts. Never quote model size as the ceiling again. + +## NO PRUNING — all experts stay available; it's cache levels + dynamic hosting (Joel 2026-07-29) + +CORRECTION to any "prune the cold tail" language above: we do NOT prune. Pruning removes experts = +diminished MODEL (lost capability). Instead, ALL 896 experts stay AVAILABLE — the whole game is WHICH +CACHE LEVEL each expert sits at, set dynamically by demand: + +``` +cache levels (fastest -> slowest, ALL experts reachable at one of them): + VRAM (hot working set) -> RAM (warm) -> FLASH (cold-ready) -> PEER RAM over grid (dynamic host) -> mechanical archival +``` + +- **Full quality always.** A rare expert fires SLOWER (deeper cache level), never MISSING. K3 stays + complete. Speed = cache-hit-rate on the hot working set; capability = full, unconditionally. +- **Dynamic hosting** = the grid distributes experts across nodes and the router finds each at its + current cache level; residency shifts with demand (sentinel-PGO) and with which peers are up. +- **Adding a peer does NOT unlock capability (it's already full) — it moves more experts into a FAST + tier** (a peer's RAM as a cache level). So the model is always whole; the grid makes it faster. + "Works now at full quality, noticeably faster as you plug in nodes" — that is the adoption driver. +- Therefore the compensation-LoRA/prune framing above is DEMOTED to an optional speed mode, NOT the + path. The path is all-experts-available, cached by level, dynamically hosted. From 32585536604c4db9bffc352aafe92e619aa65475 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 14:24:22 -0500 Subject: [PATCH 45/55] docs(k3): the Windows >RAM-MoE load-wedge is a fork-fixable loader bug, not a wall Glass-boxed on the 5090: loading K3/GLM wedges Windows working-set trim (15GB read then 0 I/O + 0 CPU + WS collapse) because llama-mmap.cpp:533-598 maps the ENTIRE file via one MapViewOfFile + PrefetchVirtualMemory over a huge range. llama.cpp is OUR fork -> this is a loader bug we fix + PR upstream, not an external wall. The fix CONVERGES with the expert pager: page experts ourselves via explicit chunked ReadFile under capacity/expert_* residency policy (we must own per-expert reads for the grid L4 tier regardless), windowed mmap for dense layers. Two-prong plan: fork loader patch (task #28, real surgery) + M5's Mac as the immediate K3 proof node (macOS mmap survives the giant view). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/K3-PAGING-DIAGNOSIS.md | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/planning/K3-PAGING-DIAGNOSIS.md b/docs/planning/K3-PAGING-DIAGNOSIS.md index d1b016aacf..eae3ddce0d 100644 --- a/docs/planning/K3-PAGING-DIAGNOSIS.md +++ b/docs/planning/K3-PAGING-DIAGNOSIS.md @@ -157,6 +157,46 @@ improves it"): prune the cold expert tail (rare experts fall back to shared/dens quant (hot IQ2/Q4, cold IQ1), per-domain adapter. The proof metric is tok/s on the WARM working set of a FOCUSED task — never cold-start random prompts. Never quote model size as the ceiling again. +## The Windows load-wedge is a FORK-FIXABLE loader bug, not a wall (Joel 2026-07-29: "all upstream deps are forks") + +Glass-boxed on BigMama (5090, Win11): loading K3/GLM (>RAM MoE) wedges — ~15GB +read, then **0 disk-read + 0 kernel-CPU + working-set collapse** = a hard kernel +deadlock, NOT slow disk. Linux loads the same file (better working-set trim + +`MAP_POPULATE`/`madvise`); Windows wedges. + +**Attack surface (ONE file, ours):** `core/vendor/llama.cpp/src/llama-mmap.cpp:533-598`, +the `_WIN32` `llama_mmap::impl`: +```cpp +addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0); // ONE view over the ENTIRE file (570GB) +pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0); // force-populate a huge range +``` +It asks the Windows VM to demand-page a file larger than physical RAM through a +single giant view; the working-set trim deadlocks. `--no-mmap` isn't the escape +(it needs 570GB RAM to hold the experts) — demand-paging is exactly the mechanism +that lets a >RAM model run, so the fix must PRESERVE demand-paging while dropping +the whole-file view. + +**The fix IS the expert pager (convergence, not a detour):** stop leaning on OS +mmap to page the 570GB expert blob; page experts OURSELVES via explicit chunked +`ReadFile` (the `read_raw` 64MB-chunk path already at `llama-mmap.cpp:129-144`) +under our own residency policy (`capacity/expert_*`). Windowed/segmented mmap for +the dense layers, explicit read-based residency for experts. We must own explicit +per-expert reads for the grid L4 tier ([[k3-pager-slice1-built]]) regardless — so +fixing the Windows loader builds the moat. + +**Fork discipline (both directions):** track upstream K3 work (pwilkin PR #26185, +already cherry-picked onto `k3-adopt`) + carry this Windows >RAM-MoE loader patch +on top + PR the general-purpose half back to llama.cpp (they want Windows >RAM MoE +loading too — we've upstreamed to candle/llama.cpp before). NOT a dead-end hack. + +**Scope note:** windowed mmap breaks the loader's single-contiguous-`addr()` +contract used across `llama-model-loader.cpp` (tensor addresses = base + offset). +That's real, focused surgery + a full engine rebuild — a dedicated effort, not a +bottom-of-context patch. Lanes: BigMama owns this fork loader patch + the +`node_content`→`ArtifactResidency` L4 map; M5 is the immediate K3 proof node +(macOS mmap survives the giant view — loads K3 TODAY while the Windows fix lands) ++ the governor/fetch-from-peer loop. + ## NO PRUNING — all experts stay available; it's cache levels + dynamic hosting (Joel 2026-07-29) CORRECTION to any "prune the cold tail" language above: we do NOT prune. Pruning removes experts = From 0684aaa9a4f7914fd009d3cec75efa5433e5bd76 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 15:50:46 -0500 Subject: [PATCH 46/55] =?UTF-8?q?feat(disk):=20cross-drive=20cold=20offloa?= =?UTF-8?q?d=20=E2=80=94=20the=20missing=20consumer=20of=20DriveRole::Cold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The disk daemon DETECTED the cold drive (SystemProfile::cold_drive) and never used it: disk_eviction only DELETES derived cargo artifacts; nothing DEMOTED non-derived cold artifacts (models, hf-hub, forge, docker) from a choked NVMe to the idle 10.8TB HDD next to it. So "cache layers that move across drives" — the whole residency architecture — was specced, detected, and unbuilt. ColdOffloadPool closes it: a ResourcePool that MOVES (not deletes) least-recently- used class entries hot->cold under pressure and leaves a zero-privilege link (Windows directory junction via mklink /J; Unix symlink) so readers still find the artifact — it just lives on the cold tier now. Same TrackedDir/PressureBroker economy as disk_eviction; move vs delete is the derived-vs-not distinction. Optional by construction (resolution field, not gate): no cold drive => pool not built, class falls back to delete/grid — mirrors SystemProfile::has_cold_tier. Safety: copy-verify-then-remove (partial copy keeps the hot original), link-or- roll-back (never orphan an artifact), never chase a symlink out of the class root. Proven on the real C:->D: (8MB dir demoted, junction created, content read back byte-identical through the C: junction landing on D:). Production body type-checks against the real crate interfaces on Windows; in-crate cargo test is CI-gated (the VS18-2026/cmake generator drift blocks the native llama build locally, not this code). Boot wiring (register the pool + point Docker's disk-image at the cold drive) is the follow-up. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../src/system_resources/cold_offload.rs | 375 ++++++++++++++++++ .../src/system_resources/mod.rs | 1 + 2 files changed, 376 insertions(+) create mode 100644 core/continuum-core/src/system_resources/cold_offload.rs diff --git a/core/continuum-core/src/system_resources/cold_offload.rs b/core/continuum-core/src/system_resources/cold_offload.rs new file mode 100644 index 0000000000..dfac7261d7 --- /dev/null +++ b/core/continuum-core/src/system_resources/cold_offload.rs @@ -0,0 +1,375 @@ +//! Cross-drive cold offload — the MISSING consumer of `DriveRole::Cold`. +//! +//! The 2026-07-29 finding (Joel, angry and right): the disk daemon DETECTS the +//! cold drive ([`crate::capacity::system_profile::SystemProfile::cold_drive`] — +//! e.g. the 16 TB D: HDD next to a choked 2 TB NVMe C:) and then NEVER USES IT. +//! [`super::disk_eviction::CargoTargetPool`] only DELETES derived cargo artifacts; +//! nothing DEMOTES cold artifacts from the hot system drive to the cold drive. So +//! when a class that must NOT be deleted grows — models, HF hub, forge exports, +//! the K3 570 GB expert set — it piles on the NVMe until it suffocates, while +//! terabytes of archival space sit idle. That is the "cache layers that move +//! across drives" the whole residency architecture is built on, left unbuilt. +//! +//! This pool closes it. Under hot-drive pressure it MOVES least-recently-used +//! top-level entries of a class from the hot drive to `cold_root//` and +//! leaves a link (Windows directory junction — NO privilege needed; Unix symlink) +//! at the original path, so every reader still finds the artifact — it just now +//! lives on the cold tier (slower, archival), exactly the L3(NVMe)→L5(HDD) +//! demotion. The bytes leave the hot drive; the capability does not. +//! +//! ## Why move, not delete (the class distinction) +//! `CargoTargetPool` deletes because cargo artifacts are DERIVED — the next build +//! recreates them. Models / hub / forge / persona stores are NOT derivable from +//! anything local; deleting them destroys work or forces a re-download. For those +//! classes the eviction verb is DEMOTE, not DELETE. Same broker, same pressure +//! economy, different physical action. +//! +//! ## Optional by construction (RESOLUTION FIELD, not gate) +//! No cold drive (M5's Mac today, until a flash drive is added to the bay) ⇒ this +//! pool is simply not constructed; the class falls back to delete-eviction (if +//! derived) or to grid placement / degraded quant (if not). A cold drive UPGRADES +//! the box; its absence never breaks it — mirroring +//! [`SystemProfile::has_cold_tier`] ([[public-project-not-joels-machines]]). +//! +//! ## Safety invariants (each pinned by a test) +//! 1. **Copy-verify-then-remove.** Cross-volume moves can't `rename`; we copy to +//! the cold drive, confirm the byte count, and only THEN remove the hot copy. +//! A failed/partial copy leaves the hot original intact (no data loss) and +//! frees 0 bytes — the broker retries next tick. +//! 2. **Link or roll back.** If the link can't be created after the move, the +//! cold copy is moved BACK so the artifact never becomes unreachable. +//! 3. **Never leave the class root.** Only direct children of the tracked dir are +//! demoted; symlinked entries are skipped (never chase a link off the tree). + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::SystemTime; + +use crate::paging::pool::{ResourcePool, ResourcePoolEntry}; + +use super::disk_reporters::{dir_size_bytes, TrackedDir}; + +/// A demotable class on the hot drive, backed by a cold-drive archive root. +/// Shares its [`TrackedDir`] with the reporter/scanner — one measurement, two +/// consumers. Pressure is usage/budget: the class is kept below `hot_budget_bytes` +/// on the fast tier, spillover lives on the cold tier (still readable via links). +pub struct ColdOffloadPool { + tracked: Arc, + /// Where demoted entries go: `//`. On this box e.g. + /// `D:\continuum-cold\genome-models\`. Derived from `cold_drive().mount`. + cold_root: PathBuf, + /// How much of this class to keep resident on the HOT drive before demoting. + /// Over this, the broker drives `evict_at_least` to spill the coldest entries. + hot_budget_bytes: u64, + /// Class label for tier reporting. + class: &'static str, +} + +impl ColdOffloadPool { + /// Construct for a class, given the resolved cold-drive mount. Returns `None` + /// when there is no cold drive — the pool is OPTIONAL and simply not built, + /// so the caller falls back to delete-eviction or grid placement. This is the + /// resolution-field seam: presence upgrades, absence degrades, never excludes. + pub fn new( + class: &'static str, + tracked: Arc, + cold_drive_mount: Option<&Path>, + hot_budget_bytes: u64, + ) -> Option { + let cold_root = cold_drive_mount?.join(class); + Some(Self { + tracked, + cold_root, + hot_budget_bytes: hot_budget_bytes.max(1), + class, + }) + } + + /// Direct children of the class root, oldest-access first (LRU) — the demote + /// order. Access time falls back to modified time falls back to epoch, so a + /// filesystem without atime still yields a stable, deterministic ordering. + /// Symlinked children are skipped (invariant 3: already demoted / never chase + /// a link out of the tree). + fn lru_children(root: &Path) -> Vec<(PathBuf, u64, SystemTime)> { + let Ok(entries) = std::fs::read_dir(root) else { + return Vec::new(); + }; + let mut out: Vec<(PathBuf, u64, SystemTime)> = Vec::new(); + for e in entries.flatten() { + let Ok(meta) = e.metadata() else { continue }; + if meta.is_symlink() { + continue; + } + let when = meta + .accessed() + .or_else(|_| meta.modified()) + .unwrap_or(SystemTime::UNIX_EPOCH); + let size = if meta.is_dir() { + dir_size_bytes(&e.path()) + } else { + meta.len() + }; + out.push((e.path(), size, when)); + } + out.sort_by_key(|(_, _, when)| *when); // oldest first + out + } +} + +impl ResourcePool for ColdOffloadPool { + fn tier_name(&self) -> &str { + self.class + } + + fn capacity_bytes(&self) -> u64 { + self.hot_budget_bytes + } + + fn usage_bytes(&self) -> u64 { + self.tracked.bytes() + } + + /// Demote coldest entries hot→cold until `want_bytes` have left the hot drive. + /// Each demotion is copy-verify-remove-link (invariant 1) with roll-back on a + /// failed link (invariant 2). Returns bytes actually freed from the hot drive. + fn evict_at_least(&self, want_bytes: u64) -> u64 { + let root = self.tracked.path().to_path_buf(); + if !root.exists() { + return 0; + } + if std::fs::create_dir_all(&self.cold_root).is_err() { + return 0; // cold drive unwritable this tick → free nothing, retry later + } + + let mut freed = 0u64; + for (src, size, _) in Self::lru_children(&root) { + if freed >= want_bytes.max(1) { + break; + } + match demote_entry(&src, &self.cold_root) { + Ok(moved) => freed = freed.saturating_add(moved), + Err(_) => continue, // partial/failed move left the hot original intact + } + let _ = size; // size is advisory; demote_entry returns the authoritative moved count + } + + if freed > 0 { + self.tracked.record_freed(freed); + crate::clog_warn!( + "💾 cold-offload demoted {} GB of '{}' to {} — still readable via link, off the hot drive", + freed / (1024 * 1024 * 1024), + self.class, + self.cold_root.display() + ); + } + freed + } + + fn snapshot(&self) -> Vec { + Vec::new() + } +} + +/// Move one entry (file or dir) from the hot drive to `cold_root` and leave a link +/// at the original path. Cross-volume safe: copy → verify size → remove hot → +/// link; on link failure, move the cold copy back so nothing is orphaned. Returns +/// bytes moved off the hot drive (0 on any failure, with the hot original intact). +fn demote_entry(src: &Path, cold_root: &Path) -> std::io::Result { + let name = src + .file_name() + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "no file name"))?; + let dst = cold_root.join(name); + if dst.exists() { + // A stale prior demotion of the same name — do not clobber; skip. + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "cold target exists", + )); + } + + let expected = if src.is_dir() { + dir_size_bytes(src) + } else { + std::fs::metadata(src)?.len() + }; + + // (1) copy to the cold drive. + if src.is_dir() { + copy_dir_all(src, &dst)?; + } else { + std::fs::copy(src, &dst)?; + } + + // (1) verify the copy before removing the hot original. + let copied = if dst.is_dir() { + dir_size_bytes(&dst) + } else { + std::fs::metadata(&dst)?.len() + }; + if copied != expected { + let _ = remove_any(&dst); + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "cold copy size mismatch — hot original kept", + )); + } + + // remove the hot original, then (2) link it to the cold copy. + remove_any(src)?; + if let Err(e) = make_link(src, &dst) { + // (2) roll back: put the artifact back on the hot drive so it is never + // unreachable. Best-effort; if this also fails the cold copy is the record. + let _ = if dst.is_dir() { + copy_dir_all(&dst, src).and_then(|_| remove_any(&dst)) + } else { + std::fs::copy(&dst, src).map(|_| ()).and_then(|_| remove_any(&dst)) + }; + return Err(e); + } + Ok(expected) +} + +fn remove_any(p: &Path) -> std::io::Result<()> { + if p.is_dir() { + std::fs::remove_dir_all(p) + } else { + std::fs::remove_file(p) + } +} + +/// Recursively copy `src` → `dst`, returning total bytes copied. Symlinks inside +/// are skipped (never chase a link off the tree during a demotion). +fn copy_dir_all(src: &Path, dst: &Path) -> std::io::Result { + std::fs::create_dir_all(dst)?; + let mut total = 0u64; + for entry in std::fs::read_dir(src)?.flatten() { + let meta = entry.metadata()?; + if meta.is_symlink() { + continue; + } + let from = entry.path(); + let to = dst.join(entry.file_name()); + if meta.is_dir() { + total = total.saturating_add(copy_dir_all(&from, &to)?); + } else { + total = total.saturating_add(std::fs::copy(&from, &to)?); + } + } + Ok(total) +} + +/// Leave a link at `link` pointing to the demoted `target`. Windows uses a +/// DIRECTORY JUNCTION (`mklink /J`) which needs NO privilege — unlike Windows +/// symlinks, which require admin/developer-mode. Unix uses a symlink. Junctions +/// are dir-only; a demoted single FILE on Windows is wrapped by linking its parent +/// dir instead is out of scope here — model/hub/forge classes are dir-structured, +/// which is why demotion granularity is the class's top-level ENTRIES. +#[cfg(windows)] +fn make_link(link: &Path, target: &Path) -> std::io::Result<()> { + if target.is_dir() { + let status = std::process::Command::new("cmd") + .args(["/C", "mklink", "/J"]) + .arg(link) + .arg(target) + .status()?; + if status.success() { + Ok(()) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "mklink /J junction creation failed", + )) + } + } else { + // A bare file demotion on Windows would need a privileged symlink; the + // caller demotes dir-structured classes, so this path is not taken. Signal + // clearly rather than silently leaving a dangling original. + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "file-granularity demotion needs a privileged symlink on Windows; demote at dir granularity", + )) + } +} + +#[cfg(unix)] +fn make_link(link: &Path, target: &Path) -> std::io::Result<()> { + std::os::unix::fs::symlink(target, link) +} + +#[cfg(test)] +mod tests { + use super::*; + + // what this catches: the resolution-field seam — NO cold drive ⇒ no pool + // (the class degrades to delete/grid), a cold drive ⇒ a pool rooted under it. + // A regression that constructs a pool with no cold target would try to demote + // into a bogus path and lose artifacts. + #[test] + fn pool_is_none_without_a_cold_drive() { + let tracked = TrackedDir::new("genome-models", PathBuf::from("/tmp/none")); + assert!(ColdOffloadPool::new("genome-models", tracked.clone(), None, 1).is_none()); + let cold = std::path::Path::new("/cold"); + let pool = ColdOffloadPool::new("genome-models", tracked, Some(cold), 1).expect("some"); + assert_eq!(pool.cold_root, cold.join("genome-models")); + } + + // what this catches: the demote candidate set is exactly the class root's + // direct, NON-symlink children (invariant 3) — an already-demoted entry (now a + // symlink) must never be re-demoted, and enumeration must not error on a normal + // dir. Ordering is a plain sort on the collected access-times (trusted); the + // safety-relevant behavior is WHICH entries are eligible, which this pins + // without depending on a filetime-setting crate. + #[test] + fn lru_children_collects_children_and_skips_symlinks() { + let tmp = tempfile::tempdir().expect("tempdir"); + std::fs::write(tmp.path().join("a.bin"), vec![0u8; 10]).expect("write"); + std::fs::create_dir(tmp.path().join("d")).expect("mkdir"); + #[cfg(unix)] + std::os::unix::fs::symlink(tmp.path().join("a.bin"), tmp.path().join("already-demoted")) + .expect("symlink"); + let kids = ColdOffloadPool::lru_children(tmp.path()); + // a.bin + d/, never the symlink. + assert_eq!(kids.len(), 2, "direct non-symlink children only"); + assert!(kids.iter().all(|(p, _, _)| !p.ends_with("already-demoted"))); + } + + // what this catches (Unix CI): the FULL demotion contract — a dir moves to the + // cold root, the hot original becomes a link to it, content is byte-preserved, + // and the reported freed count equals the moved bytes. This is invariant 1+2 + // end-to-end: bytes leave the hot drive but the artifact stays reachable. + #[cfg(unix)] + #[test] + fn demote_moves_to_cold_and_leaves_a_working_link() { + let hot = tempfile::tempdir().expect("hot"); + let cold = tempfile::tempdir().expect("cold"); + let model = hot.path().join("models--org--m"); + std::fs::create_dir_all(model.join("blobs")).expect("mkdir"); + std::fs::write(model.join("blobs/w.bin"), vec![7u8; 4096]).expect("write"); + + let moved = demote_entry(&model, cold.path()).expect("demote"); + assert_eq!(moved, 4096, "reports the bytes actually moved"); + // hot path is now a symlink... + assert!(std::fs::symlink_metadata(&model).unwrap().is_symlink()); + // ...that still reads the exact content from the cold drive. + let via_link = std::fs::read(model.join("blobs/w.bin")).expect("read through link"); + assert_eq!(via_link, vec![7u8; 4096], "content byte-preserved and reachable"); + assert!(cold.path().join("models--org--m/blobs/w.bin").exists()); + } + + // what this catches: invariant 1 — a name that already exists on the cold + // drive is NOT clobbered; the hot original is kept and 0 bytes freed, so the + // broker safely retries rather than destroying either copy. + #[cfg(unix)] + #[test] + fn demote_refuses_to_clobber_an_existing_cold_target() { + let hot = tempfile::tempdir().expect("hot"); + let cold = tempfile::tempdir().expect("cold"); + std::fs::create_dir_all(hot.path().join("m")).expect("mk"); + std::fs::write(hot.path().join("m/a"), vec![1u8; 8]).expect("w"); + std::fs::create_dir_all(cold.path().join("m")).expect("mk cold collision"); + + let err = demote_entry(&hot.path().join("m"), cold.path()).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::AlreadyExists); + assert!(hot.path().join("m/a").exists(), "hot original kept on refusal"); + } +} diff --git a/core/continuum-core/src/system_resources/mod.rs b/core/continuum-core/src/system_resources/mod.rs index fe8a89c315..03bbe61e12 100644 --- a/core/continuum-core/src/system_resources/mod.rs +++ b/core/continuum-core/src/system_resources/mod.rs @@ -12,6 +12,7 @@ //! Uses the `sysinfo` crate for cross-platform (macOS/Linux/Windows) monitoring. pub mod concurrency; +pub mod cold_offload; pub mod disk_eviction; pub mod disk_pressure; pub mod disk_reporters; From db2d8311a2bf7d5cb6cf995da855d9f8a31fb39b Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 17:16:54 -0500 Subject: [PATCH 47/55] =?UTF-8?q?docs(k3):=20design=20=E2=80=94=20modify?= =?UTF-8?q?=20llama.cpp=20fork=20for=20GPU=20expert=20paging=20(never=20CP?= =?UTF-8?q?U=20compute)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stock llama.cpp can't serve a 662GB MoE on 32GB VRAM: -ngl 99 OOMs (measured 271GB cudaMalloc), --n-cpu-moe runs experts on the CPU backend (forbidden). The missing mode — stream router-selected experts host->VRAM per token, compute on GPU, LRU-evict — is our ServingExpertPager realized IN the engine. Surface located: build_moe_ffn (llama-graph.cpp:1810) is the one shared MoE FFN; up/gate/down_exps + ggml_mul_mat_id is the intercept. buft in llama-model.cpp/ common.cpp. Async slot cache in ggml-cuda. Engine = mechanism (VRAM slot cache + async stream + upload_expert API); our Rust pager = policy (sentinel-PGO residency). Build gate (step 0): the fork doesn't build here — cmake 3.30.5 rejects the VS18-2026 generator; unblock via Ninja+vcvars (ninja present, cl needs vcvars). This also unblocks continuum-core. Full sequencing + seam to our substrate in the doc. This is tasks #23/#28 realized in CUDA. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/K3-GPU-EXPERT-PAGING.md | 73 +++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 docs/planning/K3-GPU-EXPERT-PAGING.md diff --git a/docs/planning/K3-GPU-EXPERT-PAGING.md b/docs/planning/K3-GPU-EXPERT-PAGING.md new file mode 100644 index 0000000000..fcea049216 --- /dev/null +++ b/docs/planning/K3-GPU-EXPERT-PAGING.md @@ -0,0 +1,73 @@ +# K3 GPU expert paging — modify the llama.cpp fork so experts stream to VRAM (never CPU compute) + +**Status:** design locked 2026-07-29 (Joel: "K3 and yea we need to modify llama to +accommodate" + "never let inference or training EVER run on cpu"). Implementation is +the next focused build. This is tasks #23 (LiveUploadPager) + #28 realized IN the engine. + +## The problem stock llama.cpp cannot solve +A 662GB MoE (K3 UD-IQ2_XXS) on a 32GB GPU has exactly two stock modes: +- `-ngl 99` (all experts on GPU) → **271GB cudaMalloc → OOM** (measured 2026-07-29). +- `--n-cpu-moe N` (experts on CPU) → the FFN matmul runs on the **CPU backend** → + the forbidden slow path (Joel: never CPU). + +Stock llama.cpp has **no** "keep experts in host storage, stream the router-selected +ones to VRAM per token, compute on GPU" mode. That mode is the thing we must add — it +IS our ServingExpertPager (#20/#22) realized inside the engine. + +## The modification surface (located) +- **`src/llama-graph.cpp:1810` `llm_graph_context::build_moe_ffn`** — the ONE shared + MoE FFN (every MoE arch + `models/kimi-k3.cpp` route through it). Experts enter as + `up_exps` / `gate_exps` / `down_exps`; `ggml_mul_mat_id(..., selected_experts, ...)` + does the compute on whatever backend those tensors live on. This is the intercept. +- **`src/llama-model.cpp` + `common/common.cpp` + `common/arg.cpp`** — where a tensor's + buffer type is chosen (`--n-cpu-moe` / `-ot` / `tensor_buft_override`). The new + "paged" buft-class is declared here. +- **`ggml/src/ggml-cuda/`** — where the async host→device stream + VRAM slot cache live. + +## The design (never CPU compute) +1. **Host residency, pinned.** Expert weights live in pinned host RAM (cudaHostAlloc) + — or flash-backed for the cold tail (ties to task #28 read-based residency). Pinned + so the DMA is fast; NEVER a CPU compute buffer. +2. **VRAM expert cache.** A fixed pool of `S` GPU expert slots (S sized to fit 32GB + after dense+KV). Holds the hot working set. +3. **Per-MoE-FFN, per token:** after routing picks the top-k experts, for each selected + expert NOT resident in a VRAM slot: `cudaMemcpyAsync` its up/gate/down rows host→a + VRAM slot (LRU-evict the coldest), on the compute stream so the copy overlaps the + previous expert's matmul. Then `ggml_mul_mat_id` runs against the **VRAM** copies → + GPU compute, always. +4. **LRU + prefetch.** LRU eviction of slots; the cross-layer predictor (already in + `capacity/expert_predictor.rs`) prefetches layer L+k's likely experts while L + computes. Residency POLICY is driven by our pager (#20); the engine provides the + MECHANISM (the slot cache + async stream). +5. **Correctness:** a paged expert computes the bit-identical result of a resident one + (same weights) — paging is a SPEED lever only, never accuracy ([[K3-PAGING-DIAGNOSIS]]). + +## The seam to our substrate +The engine exposes an `upload_expert(layer, expert_id, host_ptr)` / slot-cache API; our +Rust ServingExpertPager (#20/#22) decides WHICH experts are hot (sentinel-PGO from the +`ffn_moe_topk` observer) and drives residency. Engine = mechanism, our pager = policy. +This is the A-path of [[k3-slice2-A-vs-B-decision]] (weight-write into a resident slot) +made real in CUDA, not the harder K-slot router remap (B). + +## Build gate (step 0 — unblock BEFORE implementing) +The fork does not build on this box: cmake 3.30.5 rejects the auto-selected +"Visual Studio 18 2026" generator (VS18-2026/cmake drift, [[windows-build-env-drift]]). +Unblock: **Ninja generator + vcvars** — `ninja.exe` is present +(`~/.continuum/tools/ninja/bin`), `cl.exe` needs vcvars sourced. Set +`CMAKE_GENERATOR=Ninja` + enter vcvars so cmake uses Ninja+cl and never touches the VS +generator. This ALSO unblocks the continuum-core build (the reason our pager couldn't +run here). Fix this first — it has blocked every build/test all session. + +## Sequencing +0. Unblock the build (Ninja+vcvars) — gate for everything. +1. ggml-cuda: pinned-host expert store + VRAM slot cache + async host→device stream. +2. Wire into `build_moe_ffn`: route selected experts through the slot cache; `mul_mat_id` + on VRAM copies. +3. LRU eviction + cross-layer prefetch hook. +4. Expose `upload_expert`/slot API; drive residency from our ServingExpertPager. +5. Measure hot-set hit-rate + tok/s on K3 IQ2_XXS; iterate to par ([[K3-PAGING-DIAGNOSIS]]). + +## Then: better sharding + tailored quants (Joel's "go from there") +Once GPU expert paging serves K3 on one box: our cross-node expert sharding +(`node_content`→`ArtifactResidency`, experts split across grid nodes) and our +working-set-tailored quant (compaction, not Unsloth's static blob) layer on top. From ab3bc9755a333bc137dab7ee15cb70c26b5aa6e3 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 17:25:44 -0500 Subject: [PATCH 48/55] =?UTF-8?q?docs(k3):=20instrument=20GPU=20expert=20p?= =?UTF-8?q?aging=20=E2=80=94=20measure=20latency=20+=20expert=20values,=20?= =?UTF-8?q?never=20guess?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pager's admit/evict and the "best measured use" negotiation only work on real numbers (the 271GB OOM was a guess). Instrument every seam via probe!/time_async! + CUDA events through a Noop-default CaptureSink: Latency: expert_fetch_us (cudaEvent around host->VRAM memcpy — yields the MEASURED pcie_h2d_bps axis for resource_vector, not a hardcoded 25GB/s), expert_compute_us, miss_stall_us, first_token_ms/token_latency_ms, load_ms. Values: hot_set_hit_rate (THE par metric), expert_value (=ExpertActivationProfile .hits, drives LRU+sentinel-PGO), working_set_size, co_activation, and residency_value=value/fetch_us — the value-per-cost grant_all prices each expert on. Closes the loop: measured fetch latency feeds the resource negotiation; measured hit-rate + tok/s are the iterate signal. Report every number; silent caps get log()'d. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/K3-GPU-EXPERT-PAGING.md | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/planning/K3-GPU-EXPERT-PAGING.md b/docs/planning/K3-GPU-EXPERT-PAGING.md index fcea049216..7a22e9d04a 100644 --- a/docs/planning/K3-GPU-EXPERT-PAGING.md +++ b/docs/planning/K3-GPU-EXPERT-PAGING.md @@ -67,6 +67,43 @@ run here). Fix this first — it has blocked every build/test all session. 4. Expose `upload_expert`/slot API; drive residency from our ServingExpertPager. 5. Measure hot-set hit-rate + tok/s on K3 IQ2_XXS; iterate to par ([[K3-PAGING-DIAGNOSIS]]). +## Measurement — latency + values, measured never guessed (Joel 2026-07-29) + +The pager's admit/evict and the "best measured use of any resource" negotiation +([[resource_vector]]) only work on REAL numbers — the 271GB OOM was a guess. So the +mechanism is instrumented at every seam via `probe!` / `time_async!` +([[RTOS-DEBUGGER-PROBES]]) + CUDA events, written through a `CaptureSink` (Noop +default = zero hot-path cost, [[OBSERVABILITY-AS-SUBSTRATE]]), and fed back into +policy. Nothing here is a constant. + +**Latency (per the fetch/compute seams):** +- `expert_fetch_us` — `cudaEventElapsed` around each host→VRAM `cudaMemcpyAsync`. + With the expert byte size this YIELDS `pcie_h2d_bps` — the measured + [[resource_vector]] axis, not a hardcoded 25 GB/s. Closes that loop. +- `expert_compute_us` — CUDA-event time of the `mul_mat_id` on the VRAM copies. +- `miss_stall_us` — time a token blocked on a cold fetch NOT hidden by overlap + (the number the cross-layer prefetch must drive to zero). +- `first_token_ms` / `token_latency_ms` — end-to-end prefill + per-decode-token. +- `load_ms` — model load (the mechanical-D: vs NVMe gap we measured today). + +**Values (drive residency + the negotiation):** +- `hot_set_hit_rate` — fraction of a token's activated experts already VRAM-resident. + **THE par metric**: 100% ⇒ full GPU speed ([[K3-PAGING-DIAGNOSIS]]). Measured from + the `ffn_moe_topk` observer vs slot-cache state. +- `expert_value` — per-expert activation count + recency = `ExpertActivationProfile.hits` + (exists). Drives LRU AND sentinel-PGO residency. +- `working_set_size` — distinct experts hit over a task window (proves the + working-set-fits-fast-tier thesis per task). +- `co_activation` — experts firing together per token → clustering for co-placement. +- `residency_value = expert_value / expert_fetch_us` — the value-per-cost the + negotiation prices each expert on (an `Ask` in [[resource_vector]]'s `grant_all`). + +**The loop:** measured `hot_set_hit_rate` + `tok/s` are the iterate signal +([[benchmark-learning-flywheel]]); measured `expert_fetch_us`→`pcie_h2d_bps` and +`expert_value` feed the negotiation so expert placement is priced on real cost, not +a guess. Report every number; a silent cap (slot count, dropped prefetch) gets +`log()`'d, never hidden. + ## Then: better sharding + tailored quants (Joel's "go from there") Once GPU expert paging serves K3 on one box: our cross-node expert sharding (`node_content`→`ArtifactResidency`, experts split across grid nodes) and our From 3d42dc79ffeb85401fe080cfc59331fee03b91c5 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 20:12:58 -0500 Subject: [PATCH 49/55] =?UTF-8?q?docs:=20self-calibration=20=E2=80=94=20no?= =?UTF-8?q?de=20finds=20its=20own=20bearings=20per=20activity,=20continual?= =?UTF-8?q?ly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The layer above serving (Joel 2026-07-29): the node continually self-benchmarks sustained tok/s + latency + hit-rate per activity-type × intelligence-tier on the LIVE hardware/load/grid, builds a capability map, and assigns each activity the highest intelligence that clears that activity's experience FLOOR — reassigning DOWN to a smaller/faster model below ~5 tok/s (a responsive lesser experience beats a stalling frontier one). Scales intelligence up/down on re-bearings (peer up -> up, load/thermal -> down). Objective is maximized EXPERIENCES (mean_experience), not raw throughput. Realizes existing primitives (QualityModel/mean_experience, grant_all over resource_vector, SystemProfile/catalog); the new piece is the CONTINUAL per-activity self-benchmark -> capability map -> intelligence assignment with per-activity floors. Sensory input = the GPU expert-paging tok/s+hit-rate meter (K3-GPU-EXPERT-PAGING) + benchmark flywheel. Measured, never guessed. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../SELF-CALIBRATION-PROPRIOCEPTION.md | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md diff --git a/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md b/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md new file mode 100644 index 0000000000..9d177aebd5 --- /dev/null +++ b/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md @@ -0,0 +1,60 @@ +# Self-calibration — the system finds its own bearings, per activity, continually + +**Status:** principle locked 2026-07-29 (Joel): "The system will find the most natural +fit... anything sub 5 tok/s is probably where it'd reassign to lower intelligence, but +it needs to know how it can do with a devoted coder for example, or the multimodal +kinds... the system has to figure out its own bearings, regularly, and know how to +maximize experiences (activities) so it knows how to scale up and down." + +This is the layer ABOVE serving. GPU expert paging + measured tok/s +([[K3-GPU-EXPERT-PAGING]]) is the SENSORY INPUT; this layer USES it to decide what the +node should BE, per activity, and to scale intelligence up/down as conditions change. + +## The loop (proprioception, not a boot classification) +1. **Probe self, per activity-type, continually.** Regularly run representative probes + — an interactive coding turn, a multimodal describe, a chat turn, an agentic tool + loop — at candidate intelligence tiers, and MEASURE sustained tok/s + first-token + latency + hot_set_hit_rate + quality on the CURRENT hardware / load / grid state. + This is the benchmark-as-self-assessment ([[benchmark-learning-flywheel]]), run as a + background bearing-check, not a one-time catalog resolution. +2. **Capability map.** `(activity_type × intelligence_tier) → {sustained_tok_s, + first_token_ms, quality, needs_grid?}`, continually refreshed. This is what the node + KNOWS about itself: "for a devoted coder I sustain 14 tok/s at tier-A locally; for + multimodal I need a grid peer; chat runs tier-A at 30 tok/s." +3. **Assignment = highest intelligence that clears the activity's experience FLOOR.** + Each activity has its own floor (an interactive coder needs responsiveness — a tok/s + AND first-token floor; a batch multimodal describe tolerates slower). Pick the most + capable model that clears it. **Sub ~5 tok/s ⇒ reassign DOWN** to a smaller/faster + model for that activity — a lower-intelligence experience that stays responsive beats + a frontier one that stalls (the QualityModel already encodes this: a stall zeroes + mean_experience via the critical-faculty gate, [[continuum-substrate-already-built]] + capacity/consumer.rs). +4. **Scale up/down on re-bearings.** Peer joins → capacity up → scale intelligence UP + (or take a harder activity). Load spikes / thermal / peer drops → scale DOWN to hold + experiences above their floors. Same code, different bearing — the SubstrateGovernor + DVFS idea applied to model intelligence, not just clocks. + +## Objective: maximize EXPERIENCES, not tok/s +The thing being maximized is the set + quality of ACTIVITIES the node serves well +(`mean_experience`, the RANSAC score in capacity/mod.rs), never raw throughput. A node +that runs one frontier model at 3 tok/s serves fewer good experiences than one that +runs a tier-down coder at 12 tok/s + multimodal via a peer. The self-calibration picks +the intelligence mix that maximizes served experiences under the live resource vector. + +## What this realizes vs redesigns (wire, don't rebuild) +- **Realizes:** `QualityModel` / `mean_experience` (the experience reward), the resource + negotiation (`grant_all` over [[resource_vector]]), `SystemProfile`/catalog (what + fits). Those exist. +- **The new layer to build:** the CONTINUAL per-activity self-benchmark loop → the + capability map → the intelligence-tier assignment with per-activity experience floors. + It is `catalog = f(system × storage × grid)` made (a) continual and (b) keyed on + MEASURED sustained tok/s per activity, with the ~5 tok/s reassign-down floor. +- **Feeds from:** the paging tok/s + hit-rate meter ([[K3-GPU-EXPERT-PAGING]]) and the + benchmark flywheel. Measured, never guessed — a self-assessment on stale/guessed + numbers reassigns wrong. + +## Floors (initial, per Joel — calibrate from real experience later) +- Interactive coder / agent: target ≥10 tok/s ([[task #30]]); reassign down below ~5. +- Chat: comfortable well above 10; degrade gracefully. +- Multimodal describe (batch-ish): lower tok/s floor; route to grid peer if local can't. +These are RESOLUTION thresholds the node measures itself against — never hard gates. From 045938923487bf28feca67dcd6e9e2dfd782c3ae Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 20:20:35 -0500 Subject: [PATCH 50/55] docs: experience score is criticality-gated + temporal + difficulty-escalating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enriches self-calibration (Joel 2026-07-29): - Criticality gate: sub-scores (latency/fps/TTS/STT/quality) combine NON-linearly; a critical component degraded (lose TTS/STT in live video chat) collapses the whole score. Extends QualityModel critical-faculty gate from crash-only to any critical-component degradation. Context: 14 personas vs 3 vs 1, goal-weighted. - Degrade by criticality: cut least-critical sub-component first (background avatar fps) to protect load-bearing ones (active speaker TTS/STT), not a uniform throttle. - Temporal concentration: optimize experience over a WINDOW not each instant — briefly page out other personas so the smart MoE solves a hard problem, then restore. The governor being too worried about instantaneous fairness never lets deep work happen. - Difficulty/failure ESCALATION (dual of ~5tok/s reassign-down): detect thrashing/ failure/low-quality (e.g. an agent looping on a task) -> escalate tier (19B->K3, Opus->Fable) for the hard stretch -> de-escalate when easy. Failure IS the signal. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../SELF-CALIBRATION-PROPRIOCEPTION.md | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md b/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md index 9d177aebd5..57402267f4 100644 --- a/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md +++ b/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md @@ -53,6 +53,53 @@ the intelligence mix that maximizes served experiences under the live resource v benchmark flywheel. Measured, never guessed — a self-assessment on stale/guessed numbers reassigns wrong. +## The experience score is criticality-GATED, activity/goal-contextual, and temporal (Joel 2026-07-29) + +An activity's experience score is NOT an average of sub-scores — it is gated by its +CRITICAL components, shaped by context, and evaluated over a window: + +- **Sub-scores + criticality gate.** Each activity has sub-scores (latency, fps, TTS, + STT, response quality, ...). They combine NON-linearly: a critical component degraded + collapses the WHOLE score even if everything else is green. Losing TTS or STT in a + live video chat totally compromises it — the experience score goes low regardless of + fps. This extends the QualityModel's critical-faculty gate ([[continuum-substrate-already-built]] + capacity/consumer.rs) from "a crash zeroes experience" to "any CRITICAL-component + degradation zeroes experience." The map must know which sub-components are load-bearing + per activity. +- **Activity + goal context.** The score's shape depends on the activity AND the goal: + video chat with 14 personas vs 3 vs 1 has a different resource profile and different + critical set; the goal weights the sub-scores. Same node, different bearing per context. +- **Degrade by criticality, not uniformly.** When latency/fps lags, cut the LEAST-critical + sub-component first (a background persona's avatar fps, a non-speaker's video) to PROTECT + the critical ones (never drop the active speaker's TTS/STT). Shed to preserve the + experience, targeted — not a uniform throttle that clips everything including the + load-bearing parts. + +## Temporal concentration — optimize experience over a WINDOW, not each instant + +The governor must maximize experience over a TIME WINDOW, not instantaneous fairness. It +is correct to briefly PAGE OUT other personas so a hard task gets the smart MoE (e.g. K3) +for a few minutes — the combined windowed experience is HIGHER because the hard problem +gets solved, even though the instantaneous "everyone served equally" metric dipped. A +governor "too worried about satisfying all personas" every instant never lets anyone do +deep work — the exact failure Joel flagged: it refused to even temporarily admit K3 for a +hard coding problem. Add a bounded, reversible CONCENTRATION term to the negotiation +([[resource_vector]] grant_all): a high-value hard task may temporarily concentrate +resources (page out low-priority lanes), on a deadline, then restore. Reversible + +time-boxed = [[restarts-are-commonplace]] applied to attention. + +## Difficulty/failure-driven ESCALATION — scale UP on hard, not only DOWN on scarce + +Intelligence scales UP on detected DIFFICULTY, the dual of the ~5-tok/s reassign-DOWN on +scarcity. Detect thrashing / repeated failure / low quality — a coder that can't solve the +hard problem, an agent looping on a task (**like Claude thrashing on the hf download this +very session** — repeating a failing move, not escalating) — and ESCALATE to a smarter +model for that hard stretch (19B→K3; Opus→Fable), then DE-escalate when it's easy again. +The failure/thrash IS the signal ([[benchmark-learning-flywheel]]): graded failure → +escalate. This makes intelligence assignment two-sided: reassign DOWN when the tier can't +sustain the experience (scarcity), reassign UP when the tier can't SOLVE the task +(difficulty). Both detected from measured experience, both temporary, both reversible. + ## Floors (initial, per Joel — calibrate from real experience later) - Interactive coder / agent: target ≥10 tok/s ([[task #30]]); reassign down below ~5. - Chat: comfortable well above 10; degrade gracefully. From 231eb7338889d1ef13711bcca2fca5a9752054bb Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 20:26:07 -0500 Subject: [PATCH 51/55] =?UTF-8?q?docs(k3):=20build=20agent=20findings=20?= =?UTF-8?q?=E2=80=94=20ggml=20op-offload=20IS=20the=20mechanism=20(flip=20?= =?UTF-8?q?decode=20gate)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Build agent (2026-07-29) verified two things: (1) the fork rebuild loop works via cmake+Ninja+vcvars+CUDA (~1 min, recipe recorded), sidestepping the VS18-2026 generator drift; (2) the GPU expert-copy mechanism ALREADY EXISTS in ggml (ggml-backend.cpp:1576 op-offload copies only used experts host->VRAM, mul_mat_id on CUDA) but is gated to prefill (batch>=32); decode falls back to CPU. Flip for decode at runtime: GGML_OP_OFFLOAD_MIN_BATCH=1 + --n-cpu-moe 999 -ngl 99. So no from-scratch build_moe_ffn rewrite — flip the gate + layer our measured residency/ hit-rate/slot-cache (the moat) on top. No token yet: K3 663GB on a 250MB/s HDD (63GB RAM) = 44-min load floor + per-token disk faults. Storage is the sole blocker; code+build+mechanism are ready. Path: model on NVMe (free C: via VSS) OR prove mechanism first on a RAM-fitting MoE. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- docs/planning/K3-GPU-EXPERT-PAGING.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/planning/K3-GPU-EXPERT-PAGING.md b/docs/planning/K3-GPU-EXPERT-PAGING.md index 7a22e9d04a..9e891d00ff 100644 --- a/docs/planning/K3-GPU-EXPERT-PAGING.md +++ b/docs/planning/K3-GPU-EXPERT-PAGING.md @@ -14,6 +14,31 @@ Stock llama.cpp has **no** "keep experts in host storage, stream the router-sele ones to VRAM per token, compute on GPU" mode. That mode is the thing we must add — it IS our ServingExpertPager (#20/#22) realized inside the engine. +## KEY FINDING (2026-07-29 build agent): the copy mechanism ALREADY EXISTS in ggml +`ggml/src/ggml-backend.cpp:1576-1660` op-offload already reads the top-k router ids and +copies ONLY the used experts host→VRAM (async, grouped), then `mul_mat_id` runs on CUDA — +exactly the design's mechanism. It is **gated to prefill** (`op_offload_min_batch_size`, +default 32); decode (batch=1) falls back to CPU. **Flip it for decode at runtime:** +`GGML_OP_OFFLOAD_MIN_BATCH=1` + `--n-cpu-moe 999 -ngl 99` (experts host-side, decode MoE +matmuls forced onto GPU). So we do NOT rewrite `build_moe_ffn`/add a from-scratch CUDA slot +allocator — we flip the decode gate and layer OUR measured residency + hit-rate + persistent +VRAM slot-cache (rung 2) on top. THAT integration is the moat, not the copy op. + +**Verified fork build recipe (~1 min incremental):** from a `vcvars64.bat` (VS2022) shell: +`set CUDA_ROOT=%USERPROFILE%\.continuum\cuda-13.2\Library` ; PATH += `.continuum\tools\cmake\bin`, +`.continuum\tools\ninja`, `%CUDA_ROOT%\bin` ; then +`cmake -S core/vendor/llama.cpp -B build-k3-cuda -G Ninja -DGGML_CUDA=ON +-DCMAKE_CUDA_ARCHITECTURES=native -DBUILD_SHARED_LIBS=OFF -DLLAMA_BUILD_SERVER=ON +-DLLAMA_CURL=OFF -DCUDAToolkit_ROOT="%CUDA_ROOT%" -DCMAKE_CUDA_COMPILER="%CUDA_ROOT%\bin\nvcc.exe"` +then `cmake --build build-k3-cuda --target llama-server --config Release` → copy +`build-k3-cuda/bin/llama-server.exe` to `~/.continuum/bin/llama-server-k3.exe`. Runtime needs +`cuda-13.2/Library/bin` on PATH. (Ninja sidesteps the VS18-2026 cmake generator drift.) + +**The one wall to a measured token:** K3 IQ2_XXS = 663GB on a 250MB/s mechanical D:, +63GB RAM → 44-min load floor + per-token expert faults hit disk. Needs NVMe (free C: via +VSS, [[windows-vss-invisible-disk-hog]]) OR prove the mechanism first on a RAM-fitting MoE +(48B-A3B @IQ2 ≈15-25GB). Code + build + on-GPU mechanism are READY; storage is the blocker. + ## The modification surface (located) - **`src/llama-graph.cpp:1810` `llm_graph_context::build_moe_ffn`** — the ONE shared MoE FFN (every MoE arch + `models/kimi-k3.cpp` route through it). Experts enter as From a630eaa97606819665b6451c7a5df7ec9d5a3a92 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 20:33:27 -0500 Subject: [PATCH 52/55] =?UTF-8?q?docs:=20beta=20architecture=20=E2=80=94?= =?UTF-8?q?=20activities=20as=20the=20unit,=20learned=20per-activity=20val?= =?UTF-8?q?ue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Joel 2026-07-29 beta vision. The unit is the ACTIVITY (a room/experience/benchmark class): coding, agentic, multimodal, chat, video-chat, Hermes/open-system asks, continuous learning, dream/sentinels. Benchmarks aren't a scoreboard — they're HOW we achieve the activities. Each activity has ONE comprehensive criticality-gated score; degradation scores NEAR-zero not zero (zero is a dead gradient the ML can't learn from or distinguish from not-attempted; near-zero preserves how-bad + how-recoverable). Every run is measured = a graded training example -> learn the value of everything, know what each (persona x model/tier x node-state) can do. Arc: measure across activities+benchmarks -> (given the levers, e.g. GPU expert paging) design dynamic ML that responds across the grid knowing value -> p2p mesh -> economy LATER. Beta simplification: all FREE + egalitarian grid (LAN nodes and joined peers treated identically, no pricing) to ship the learning loop first. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../BETA-ACTIVITIES-AND-LEARNED-VALUE.md | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 docs/planning/BETA-ACTIVITIES-AND-LEARNED-VALUE.md diff --git a/docs/planning/BETA-ACTIVITIES-AND-LEARNED-VALUE.md b/docs/planning/BETA-ACTIVITIES-AND-LEARNED-VALUE.md new file mode 100644 index 0000000000..2c7c03c5ac --- /dev/null +++ b/docs/planning/BETA-ACTIVITIES-AND-LEARNED-VALUE.md @@ -0,0 +1,53 @@ +# Beta: activities as the unit, a learned per-activity value, always measuring + +**Status:** beta architecture locked 2026-07-29 (Joel). Companion to +[[SELF-CALIBRATION-PROPRIOCEPTION]] (the scoring/assignment mechanics) and +BETA-ACTUALIZATION.md (reconcile — this is the activity + learned-value facet). + +## The unit is the ACTIVITY (a room / an experience / a benchmark class) +We code a FEW activities we know we need; they're defined by the benchmarks (the K3 +chart classes), our chat + video-chat experiences, and the open-system asks. The +benchmarks are not a scoreboard — they're HOW we achieve many of these +([[benchmark-learning-flywheel]]). Beta activity set: + +| Activity | What it is | Critical components (lose one ⇒ score collapses) | +|---|---|---| +| **Coding** (devoted coder) | DeepSWE/ProgramBench/FrontierSWE/SWE-Marathon/Kimi-Code, livecodebench-rs | solves-the-problem (quality/pass), interactivity (≥ floor tok/s) | +| **Agentic** | BrowseComp/MCP/OSWorld/Terminal-Bench, tool loops | tool-exec correctness, doesn't-loop (thrash detection) | +| **Multimodal / vision** | visual benchmarks; video-chat perception | vision describe/STT present | +| **Chat** (rooms) | text with personas | responsiveness, coherence | +| **Video chat** (rooms) | multi-persona live A/V; the 14 vs 3 vs 1 case | TTS, STT (lose either ⇒ experience worthless), latency, fps | +| **Hermes / open-system asks** | [[hermes-grid-node]], onboarding + tech-support ([[ai-onboarding-and-tech-support-vision]]) | the ask actually gets served | +| **Continuous learning** | training as an activity — LoRA from graded failures | training completes + improves the graded metric | +| **Dream / sentinels** | background PGO / consolidation / sentinel-AI ([[ethical-substrate-raid-personas]]) | runs without stealing the foreground experiences | + +## The comprehensive per-activity score — and why NEAR-zero, never zero +Each activity has ONE comprehensive score, criticality-gated (a degraded load-bearing +component — lose TTS/STT in video chat, can't-solve in coding — collapses the whole +score, [[SELF-CALIBRATION-PROPRIOCEPTION]]). **Degradation scores NEAR-zero, not zero: +zero is a worthless signal** — a dead gradient the ML can't learn from and can't tell +apart from "not attempted." Near-zero preserves HOW bad + how recoverable, so the +learner still has a slope to climb. Every score is comprehensive (the whole experience) +with sub-scores that roll up. + +## Always measuring = always creating training data +Every activity run — served experience, benchmark, chat turn, video session — is +MEASURED and becomes a graded training example. That's how we learn "the value of +everything" and how we know what each individual (persona × model/tier × node-state) +is capable of: because we've tested + used them before, and kept the graded record +([[being-axis-shareable-learning]]). No throwaway runs. + +## The arc: measure → (levers) → dynamic ML → grid → mesh → (later) economy +Measure across our activity requirements + benchmark goals → assuming we have the +LEVERS (e.g. the GPU expert paging we're building, model-tier switching, TTS/STT +degrade, page-out concentration) → design the algorithms + ML that respond DYNAMICALLY +and ACROSS THE GRID, knowing the value of everything. That is a p2p mesh; the compute + +artifact ECONOMY ([[grid-as-network-intelligence-mandate]]) comes LATER. + +## Beta simplification: FREE + egalitarian grid (avoid complexity) +For beta: **make it all free, and treat the grid the SAME whether it's my own subnet or +peers I joined** — LAN nodes and joined peers handled identically, egalitarian, no +pricing, no economy. Solve single-computer AND grid; treat grid as grid regardless of +who owns the nodes. The economy + differentiated peer trust is a post-beta layer; beta +deliberately collapses it to "all nodes equal, all free" to ship the learning loop first. +Aligns with [[airc-route-adapter-hierarchy]] (LAN-first) and [[docker-as-grid-substrate]]. From 286d53434cfecd1132a5fa13d3fda469c5c10503 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Wed, 29 Jul 2026 21:15:32 -0500 Subject: [PATCH 53/55] =?UTF-8?q?docs:=20multi-user=20+=20contention=20?= =?UTF-8?q?=E2=80=94=20human=20foreground=20GPU=20work=20is=20top-priority?= =?UTF-8?q?,=20continuum=20yields?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Joel 2026-07-29: eventual system-wide (all-users) background service (deferred; his wife's account games on the box). The requirement: recognize the user + what they're doing (foreground GPU app / Steam / util spike from an unowned process = first-class contention signal), and treat the human's foreground GPU work as the HIGHEST-priority activity — continuum yields (sheds VRAM, deprioritizes/pauses lanes, pages out, scales down, or routes to grid) so the game gets the GPU. Degrade continuum, never the human's game. This is what capacity/mod.rs was built for (gpu_free_bytes_live = free after external unowned load; the seeding OOM was a static reserve blind to a game); Joel's scenario adds the sensor (foreground-app detection) + the aggressive-yield policy. Good citizen on a shared/gaming machine = precondition for the all-users service. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../SELF-CALIBRATION-PROPRIOCEPTION.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md b/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md index 57402267f4..acdac0b3c8 100644 --- a/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md +++ b/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md @@ -100,6 +100,31 @@ escalate. This makes intelligence assignment two-sided: reassign DOWN when the t sustain the experience (scarcity), reassign UP when the tier can't SOLVE the task (difficulty). Both detected from measured experience, both temporary, both reversible. +## Multi-user + contention: the human's foreground work is the TOP-priority activity (Joel 2026-07-29) + +Eventual goal: run **system-wide (all users)**, a background service serving whoever's on +the box — deferred for now (complexity/safety concerns), but a good eventual test and how +Joel sets up his machines (e.g. his wife's account runs Steam games). The load-bearing +requirement: + +- **Recognize the user + what they're doing.** Detect a foreground GPU-heavy app (a game — + via Steam running / foreground process / a GPU-util spike from a process we don't own) as + a FIRST-CLASS contention signal, not just a number. +- **The human's foreground GPU work is the HIGHEST-priority "activity"; continuum defers to + it, always.** "Not be a problem during outside GPU usage, or at least not interfere, + deprioritize." When the user games, continuum YIELDS: sheds VRAM, deprioritizes/pauses its + lanes, pages experts out, scales intelligence down, or moves work to the grid — so the game + gets the GPU. The human's experience is a load-bearing sub-score; **degrade continuum, + never the human's game.** +- **This is what `capacity/mod.rs` was built for.** `gpu_free_bytes_live` = "free after + external (unowned) load — a game/browser"; the FitPolicy derives grants from live free, so + a game opening (shrink) and closing (grow) already fall out. Joel's scenario adds the two + concrete pieces on top: the SENSOR (foreground GPU app / Steam detection) as an explicit + high-priority unowned-pressure source, and the POLICY (aggressive yield to foreground user + work). Makes continuum a good citizen on a shared/gaming machine — the precondition for the + all-users service. Ties to [[resource_vector]] (measured live free, external subtracted) + and the always-on background-service end state. + ## Floors (initial, per Joel — calibrate from real experience later) - Interactive coder / agent: target ≥10 tok/s ([[task #30]]); reassign down below ~5. - Chat: comfortable well above 10; degrade gracefully. From 177be8d4a9be73845f47125a756c82904a851cdc Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Thu, 30 Jul 2026 02:03:47 -0500 Subject: [PATCH 54/55] =?UTF-8?q?docs(k3):=20the=20foundry=20model-reducti?= =?UTF-8?q?on=20path=20=E2=80=94=20shrink=20K3=20to=20fit,=20no=20paging?= =?UTF-8?q?=20(after=20cache)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Joel 2026-07-30: beyond the cache/paging (needed anyway), our foundry arsenal can DRAMATICALLY reduce K3 to a tailored model that fits VRAM+RAM directly at full GPU speed. The two ends of the dial: cache serves the full model on misfit hw; foundry reduction shrinks it to fit our needs. We already proved the subset step (the 19B via tools/scripts/compaction Plasticity Compaction). Arsenal to survey (don't reinvent): the foundry/forge, legacy widget (prior compaction experiments), sentinel-ai (PGO expert-subset from real activation), forge-alloy (the alloy artifact + attestation), targeted experiential plasticity (prune + compensation- LoRA from graded failures), variable/regional quant (#29), and the unet stuff (survey). Recipe: sentinel-PGO hot-subset -> prune -> regional quant -> compensation-LoRA from benchmark graded failures -> emit as attested forge-alloy artifact. Sequenced AFTER the cache. Complements kimi-k3-grid-strategy Path B/C. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../K3-MODEL-REDUCTION-FOUNDRY-PATH.md | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 docs/planning/K3-MODEL-REDUCTION-FOUNDRY-PATH.md diff --git a/docs/planning/K3-MODEL-REDUCTION-FOUNDRY-PATH.md b/docs/planning/K3-MODEL-REDUCTION-FOUNDRY-PATH.md new file mode 100644 index 0000000000..1dd6535455 --- /dev/null +++ b/docs/planning/K3-MODEL-REDUCTION-FOUNDRY-PATH.md @@ -0,0 +1,53 @@ +# K3 model reduction — the foundry path (the smarter half of the dial) + +**Status:** strategy locked 2026-07-30 (Joel). SEQUENCED AFTER the cache/paging work +(which we need anyway) — this is the dramatic-speedup complement, not a replacement. + +## The dial (two ends, both ours) +- **Cache / paging** (agents building now, tasks #23/#28): serve the FULL K3 (662GB) on + misfit hardware via GPU expert paging. General, full-quality, needed regardless. But it + fights RAM/PCIe every token. +- **Foundry reduction** (THIS doc): DRAMATICALLY REDUCE K3 to a tailored model that fits + VRAM+RAM DIRECTLY — no paging, full GPU speed. Tailored to OUR needs (our working set, + our activities/benchmarks). This is what made the 19B ([[moe-expert-paging-feasibility]]: + we already proved the subset step in `tools/scripts/compaction` — Plasticity Compaction). +The two compose ([[K3-PAGING-DIAGNOSIS]] synthesis): page the real expert for rare-domain +correctness; serve the compacted-to-fit subset for hot-domain speed. + +## The arsenal to mine (Joel's pointers — SURVEY these when the cache lands) +Do NOT reinvent — we've built most of this. Survey first: +- **The foundry / forge** — the JIT/compaction system ([[continuum-substrate-already-built]] + foundry-as-JIT). The "crazy shit we did before." +- **legacy widget** — old widget code holds prior foundry/compaction experiments. LOOK HERE. +- **sentinel-ai** ([[sentinel-in-substrate]]) — PGO/background-learning; picks the hot + expert subset from real activation (`ExpertActivationProfile`) — the §4.1.3.4 prune driver. +- **forge-alloy** ([[forge-alloy-contract-attestation-layer]]) — the alloy = the compaction + artifact + attestation; `forge-alloy/python/forge_alloy/types.py` has the alloy types. +- **targeted experiential plasticity** — the Plasticity Compaction that produced the 19B: + prune to the hot subset + train a compensation-LoRA on a held-out corpus that recovers + the pruned experts' accuracy ([[benchmark-learning-flywheel]]: the LoRA is trained from + the catalog's GRADED FAILURES — the being learns the adapter that makes its pruned K3 + match full K3 on the exact tasks it's measured on). +- **variable quant sizes** (task #29) — asymmetric/regional quant: hot working-set experts + high-bit, cold tail low-bit, to shrink the resident footprint to fit. +- **unet stuff** — [Joel-named; SURVEY: likely a U-Net-style compression/architecture piece — + locate what it is and whether it applies to expert compaction. Flag when found.] + +## The recipe (K3 -> tailored fits-in-VRAM model) +1. sentinel-PGO from real activation over OUR activities -> the hot expert subset that + covers our working set (the §4.1.3.4 falsifiable prune). +2. Prune K3 to that subset (all-experts-available demoted to a fits-VRAM core; rare experts + still reachable via the cache/grid for correctness). +3. Variable/regional quant the core to fit VRAM+RAM directly. +4. Train the compensation-LoRA from graded failures on our benchmark activities so the + pruned+quantized core matches full K3 on the tasks we measure. +5. Emit as a forge-alloy artifact (attested, reproducible) via the recipe-as-entity foundry. +Result: a tailored K3 that serves ENTIRELY in VRAM at full GPU speed for our activities — +the misfit-design win without the paging tax. + +## Sequencing (Joel) +"Once we've tried the cache stuff you're doing (which we need anyway)." Cache FIRST (it's +the general substrate + the full-quality/rare-domain path). Foundry reduction SECOND (the +tailored dramatic speedup). They are complements on the dial, not either/or. +Complements [[kimi-k3-grid-strategy]] (Path B forge/distill dense student, Path C +sentinel-PGO expert-subset prune). From a345d0af04003ea67babddd2f26b73d90df764fb Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Thu, 30 Jul 2026 02:12:53 -0500 Subject: [PATCH 55/55] =?UTF-8?q?docs(k3):=20reduce=20PRECISION=20not=20kn?= =?UTF-8?q?owledge=20=E2=80=94=20head-targeting=20mechanism=20->=20quant-t?= =?UTF-8?q?argeting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Joel 2026-07-30 correction: "we stripped experts, but that was old us. Now we page them in. Why ever lose knowledge?" Stripping/pruning permanently deletes capability = diminished model. We PAGE experts now, so all knowledge stays available. So the foundry reduction must NOT remove anything — it shrinks by lowering PRECISION where importance is low (targeted variable quant) while every expert stays reachable (hot resident, cold paged). The KEY reuse: the experiential-plasticity culling/growing-of-HEADS importance mechanism (sentinel-ai/PGO driven) is repurposed to target QUANTIZATION bit-width — one learned importance signal, two uses. The legacy widget is the CONTROL SURFACE for it. Recipe corrected: importance profile -> targeted variable quant (nothing removed) -> keep all experts (hot resident/cold paged) -> compensation-LoRA from graded failures -> attested forge-alloy artifact. Full knowledge, no paging tax on the hot path, zero knowledge lost. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../K3-MODEL-REDUCTION-FOUNDRY-PATH.md | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/docs/planning/K3-MODEL-REDUCTION-FOUNDRY-PATH.md b/docs/planning/K3-MODEL-REDUCTION-FOUNDRY-PATH.md index 1dd6535455..16ad80b200 100644 --- a/docs/planning/K3-MODEL-REDUCTION-FOUNDRY-PATH.md +++ b/docs/planning/K3-MODEL-REDUCTION-FOUNDRY-PATH.md @@ -33,17 +33,38 @@ Do NOT reinvent — we've built most of this. Survey first: - **unet stuff** — [Joel-named; SURVEY: likely a U-Net-style compression/architecture piece — locate what it is and whether it applies to expert compaction. Flag when found.] -## The recipe (K3 -> tailored fits-in-VRAM model) -1. sentinel-PGO from real activation over OUR activities -> the hot expert subset that - covers our working set (the §4.1.3.4 falsifiable prune). -2. Prune K3 to that subset (all-experts-available demoted to a fits-VRAM core; rare experts - still reachable via the cache/grid for correctness). -3. Variable/regional quant the core to fit VRAM+RAM directly. +## THE CORRECTION (Joel 2026-07-30): reduce PRECISION, never STRIP knowledge +"We also stripped our experts, but that was old us. Now we page them in. Why ever lose +knowledge?" — Stripping/pruning experts is the OLD move: it permanently DELETES capability += a diminished model. We now PAGE experts (the cache work), so ALL experts stay available +at some cache level, always. Therefore the reduction MUST NOT remove anything. It shrinks by +lowering PRECISION where importance is low — targeted variable quantization — while every +expert remains reachable (hot ones resident, cold ones paged). Full knowledge, smaller +footprint. Never lose knowledge. + +## The mechanism reuse (the key): head-targeting -> quant-targeting +The experiential-plasticity paper's culling/growing-of-HEADS mechanism measures per-unit +IMPORTANCE from real experience (sentinel-ai / PGO drives it). **The SAME importance +mechanism we used to target heads, we repurpose to target QUANTIZATION**: high-importance +experts/tensors kept high-bit, low-importance driven low-bit. One learned importance signal, +two uses (was: cull/grow heads; now: set per-unit bit-width). The **legacy widget is the +CONTROL SURFACE** for this — it shows the knobs (cull/grow, bit-width targeting); survey it +to see the controls, then drive them from sentinel-ai's importance profile. + +## The recipe (K3 -> tailored fits-in-VRAM model, NO knowledge lost) +1. sentinel-PGO + the plasticity importance mechanism over OUR activities -> a per-expert / + per-tensor IMPORTANCE profile (the same signal that culled/grew heads). +2. TARGETED VARIABLE QUANTIZATION driven by that profile: high-importance -> high-bit, + low-importance -> low-bit. This shrinks the RESIDENT footprint to fit VRAM+RAM. NOTHING + is removed — every expert still exists, just at demand-matched precision. +3. Keep ALL experts available: hot (high-importance) resident in VRAM/RAM, cold paged via + the cache/grid. Full knowledge, always reachable. 4. Train the compensation-LoRA from graded failures on our benchmark activities so the - pruned+quantized core matches full K3 on the tasks we measure. + variably-quantized model matches full K3 on the tasks we measure. 5. Emit as a forge-alloy artifact (attested, reproducible) via the recipe-as-entity foundry. -Result: a tailored K3 that serves ENTIRELY in VRAM at full GPU speed for our activities — -the misfit-design win without the paging tax. +Result: a tailored K3 whose HOT working set serves ENTIRELY in fast memory at full GPU speed +for our activities, cold knowledge paged on demand — the misfit win, no paging tax on the +hot path, and ZERO knowledge lost. ## Sequencing (Joel) "Once we've tried the cache stuff you're doing (which we need anyway)." Cache FIRST (it's