agent_window_test.go

package main

import (
	"encoding/json"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"testing"
)

// `agent_ctx_tokens` is what the harness ADVERTISES: the fleet sizes every delegation
// contract from it. If it understates what the seat actually serves, the node quietly
// works at a fraction of its window and nothing ever errors — the failure mode the
// delegation rules call out by name ("size contracts from the live ceiling, never from
// a written figure", after guidance drifted to a quarter of the real window and work
// that fits trivially was declined for weeks).
//
// blackwell-8 shipped exactly that: its rendered agent seat serves `--ctx-size 32768`
// — a literal in the template, backed by its own fit measurement on the RTX 5060
// (6,344 MiB at 16K, 6,696 MiB at 32K; Qwen3.5's Gated-DeltaNet hybrid KV barely grows
// with context) — while the tier advertised 16,384. Every contract to such a node was
// sized to half the window the seat was already serving.
//
// The rule this asserts: whatever the agent seat serves, the tier advertises.
//   - a seat with a LITERAL --ctx-size  -> agent_ctx_tokens must equal that literal;
//   - a seat carrying __CTX__           -> it serves the tier's window, so
//     agent_ctx_tokens must equal ctx_size.
func TestAgentWindowMatchesWhatTheAgentSeatServes(t *testing.T) {
	raw, err := os.ReadFile(filepath.Join("setup", "templates", "profiles.json"))
	if err != nil {
		t.Fatal(err)
	}
	var doc struct {
		Profiles map[string]struct {
			CtxSize        int                        `json:"ctx_size"`
			AgentCtxTokens int                        `json:"agent_ctx_tokens"`
			Backend        string                     `json:"backend"`
			IncludeQ354B   bool                       `json:"include_qwen35_4b"`
			IncludeQ359B   bool                       `json:"include_qwen35_9b"`
			IncludeQ3827B  bool                       `json:"include_qwen38_27b"`
			ConfigSeed     map[string]json.RawMessage `json:"config_seed"`
			// The RAM overlays can rebind the agent seat too (ampere-6's low-and-up layer binds the
			// spill seat), and the window the tier advertises must match THAT seat's on the boxes
			// the overlay applies to.
			ConfigSeedLowUp   map[string]json.RawMessage `json:"config_seed_ram_low_up"`
			ConfigSeedMidHigh map[string]json.RawMessage `json:"config_seed_ram_mid_high"`
		} `json:"profiles"`
	}
	if err := json.Unmarshal(raw, &doc); err != nil {
		t.Fatalf("profiles.json is not valid JSON: %v", err)
	}

	// Most agent seats live in the Windows CUDA template; both claim the `agent-seat`
	// alias, which is how the harness finds the lane. A "vulkan" tier is the
	// exception (2026-09-24, the amd-gcn onboarding): it renders from
	// llama-swap.linux-vulkan.yaml, whose mimo-9b-agent/qwen3.5-*-agent entries carry
	// the TIER's own __CTX__ macro rather than the CUDA 8GB tiers' literal window —
	// reading the CUDA template's ctx expression for a vulkan-bound tier would compare
	// against a window that tier never actually serves.
	tmplRaw, err := os.ReadFile(filepath.Join("setup", "templates", "llama-swap.win-cuda.yaml"))
	if err != nil {
		t.Fatal(err)
	}
	vulkanTmplRaw, err := os.ReadFile(filepath.Join("setup", "templates", "llama-swap.linux-vulkan.yaml"))
	if err != nil {
		t.Fatal(err)
	}
	seatCtxByBackend := map[string]map[string]string{
		"vulkan": {
			"qwen3.5-4b-agent": ctxExprFor(t, string(vulkanTmplRaw), "qwen3.5-4b-agent"),
			"qwen3.5-9b-agent": ctxExprFor(t, string(vulkanTmplRaw), "qwen3.5-9b-agent"),
			"qwen38-27b-agent": ctxExprFor(t, string(vulkanTmplRaw), "qwen38-27b-agent"),
			"mimo-9b-agent":    ctxExprFor(t, string(vulkanTmplRaw), "mimo-9b-agent"),
		},
	}
	defaultSeatCtx := map[string]string{
		"qwen3.5-4b-agent": ctxExprFor(t, string(tmplRaw), "qwen3.5-4b-agent"),
		"qwen3.5-9b-agent": ctxExprFor(t, string(tmplRaw), "qwen3.5-9b-agent"),
		"qwen38-27b-agent": ctxExprFor(t, string(tmplRaw), "qwen38-27b-agent"),
		"mimo-9b-agent":    ctxExprFor(t, string(tmplRaw), "mimo-9b-agent"),
		// the RAM-spill agent seat: a literal window, bound only through the low-and-up overlay
		"qwen3.6-35b-a3b-agent": ctxExprFor(t, string(tmplRaw), "qwen3.6-35b-a3b-agent"),
	}

	checked := 0
	for tier, p := range doc.Profiles {
		seatCtx := defaultSeatCtx
		if m, ok := seatCtxByBackend[p.Backend]; ok {
			seatCtx = m
		}
		seat := ""
		// A tier may render SEVERAL agent-capable entries (the 27B entry deliberately does
		// not claim the `agent-seat` alias, so it coexists with a 4B/9B fallback). What
		// actually routes the lane is config_seed.agent_model, so the binding — not a gate
		// flag — decides which seat's window the tier must advertise. Resolving this from
		// the gate instead would have hidden exactly the disagreement this test is for.
		if raw, ok := p.ConfigSeed["agent_model"]; ok {
			var bound string
			if err := json.Unmarshal(raw, &bound); err == nil {
				if _, known := seatCtx[bound]; known {
					seat = bound
				}
			}
		}
		switch {
		case seat != "":
			// resolved from the binding above
		case p.IncludeQ359B:
			seat = "qwen3.5-9b-agent"
		case p.IncludeQ354B:
			seat = "qwen3.5-4b-agent"
		default:
			continue
		}
		checked++
		expr := seatCtx[seat]
		if expr == "__CTX__" {
			if p.AgentCtxTokens != p.CtxSize {
				t.Errorf("tier %s: agent seat %q serves the tier window (__CTX__ = %d) but the tier advertises "+
					"agent_ctx_tokens %d — contracts are sized from the advertised figure",
					tier, seat, p.CtxSize, p.AgentCtxTokens)
			}
			continue
		}
		n, err := strconv.Atoi(expr)
		if err != nil {
			t.Errorf("tier %s: cannot read seat %q ctx expression %q", tier, seat, expr)
			continue
		}
		if p.AgentCtxTokens != n {
			t.Errorf("tier %s: agent seat %q serves --ctx-size %d but the tier advertises agent_ctx_tokens %d. "+
				"The fleet sizes every delegation contract from the advertised figure, so the node works at "+
				"%d/%d of the window it is already serving and nothing errors",
				tier, seat, n, p.AgentCtxTokens, p.AgentCtxTokens, n)
		}
	}
	if checked == 0 {
		t.Fatal("no tier declares an agent seat — this gate went blind")
	}
	t.Logf("checked %d tiers with a declared agent seat", checked)

	// The same rule for an agent seat a RAM OVERLAY binds: the base loop above reads only
	// config_seed, so a seat reachable only through an overlay was invisible to it.
	overlayChecked := 0
	for tier, p := range doc.Profiles {
		seatCtx := defaultSeatCtx
		if m, ok := seatCtxByBackend[p.Backend]; ok {
			seatCtx = m
		}
		for layer, seed := range map[string]map[string]json.RawMessage{
			"config_seed_ram_low_up": p.ConfigSeedLowUp, "config_seed_ram_mid_high": p.ConfigSeedMidHigh,
		} {
			raw, ok := seed["agent_model"]
			if !ok {
				continue
			}
			var bound string
			if err := json.Unmarshal(raw, &bound); err != nil {
				continue
			}
			expr, known := seatCtx[bound]
			if !known {
				continue
			}
			overlayChecked++
			want := p.CtxSize
			if expr != "__CTX__" {
				n, err := strconv.Atoi(expr)
				if err != nil {
					t.Errorf("tier %s (%s): cannot read seat %q ctx expression %q", tier, layer, bound, expr)
					continue
				}
				want = n
			}
			if p.AgentCtxTokens != want {
				t.Errorf("tier %s: %s binds agent seat %q, which serves a %d window, but the tier advertises "+
					"agent_ctx_tokens %d — contracts are sized from the advertised figure", tier, layer, bound, want, p.AgentCtxTokens)
			}
		}
	}
	if overlayChecked == 0 {
		t.Fatal("no RAM overlay binds an agent seat this gate knows — it went blind to the spill seat")
	}
}

// ctxExprFor returns the --ctx-size argument of a model block: a literal, or __CTX__.
func ctxExprFor(t *testing.T, tmpl, model string) string {
	t.Helper()
	start := regexp.MustCompile(`(?m)^  ` + regexp.QuoteMeta(model) + `:$`).FindStringIndex(tmpl)
	if start == nil {
		t.Fatalf("template declares no %q model block — this gate went blind", model)
	}
	seg := tmpl[start[1]:]
	if nxt := regexp.MustCompile(`(?m)^  [A-Za-z0-9._-]+:$`).FindStringIndex(seg); nxt != nil {
		seg = seg[:nxt[0]]
	}
	m := regexp.MustCompile(`--ctx-size (\S+)`).FindStringSubmatch(seg)
	if m == nil {
		t.Fatalf("model %q declares no --ctx-size — this gate went blind", model)
	}
	return strings.TrimSpace(m[1])
}