Repository navigation
Expand file tree
/
Copy pathserver.go
More file actions
3460 lines (3342 loc) · 178 KB
/
Copy pathserver.go
File metadata and controls
3460 lines (3342 loc) · 178 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// This file is the HTTP face of the fleet node — the three CONTRACT.md v2
// endpoints (health / dispatch / jobs) over the stores built by the sibling
// files. Handlers hold no GPU lock and spawn nothing: health reads the
// sampler's atomic snapshot + the jobs store's counters, dispatch acks in
// milliseconds and hands the render to Jobs.Accept, and every failure is a
// non-2xx JSON envelope from the spec's error taxonomy. Wrong Content-Type is
// answered 400 (not 415): the taxonomy has exactly one caller-mistake class,
// and one shape beats two.
package fleetnode
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"log"
"math"
"mime"
"net"
"net/http"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/dmmdea/offload-harness/internal/backendchain"
"github.com/dmmdea/offload-harness/internal/config"
"github.com/dmmdea/offload-harness/internal/core"
"github.com/dmmdea/offload-harness/internal/displaystate"
"github.com/dmmdea/offload-harness/internal/fleetqueue"
"github.com/dmmdea/offload-harness/internal/gpuactivity"
"github.com/dmmdea/offload-harness/internal/gpulease"
"github.com/dmmdea/offload-harness/internal/gpuprobe"
"github.com/dmmdea/offload-harness/internal/hostsample"
"github.com/dmmdea/offload-harness/internal/mediacap"
"github.com/dmmdea/offload-harness/internal/netguard"
"github.com/dmmdea/offload-harness/internal/pairworkloads"
"github.com/dmmdea/offload-harness/internal/placement"
"github.com/dmmdea/offload-harness/internal/seatload"
"github.com/dmmdea/offload-harness/internal/seatrate"
"github.com/dmmdea/offload-harness/internal/storesteward"
"github.com/dmmdea/offload-harness/internal/swapclient"
)
// maxDispatchBody is the dispatch request-body cap (1 MiB per the spec — a
// run-graph payload is the biggest legitimate body and fits comfortably).
const maxDispatchBody = 1 << 20
// maxSnapshotAge bounds how stale a VRAM snapshot health may serve. The
// sampler refreshes every 2s and keeps the LAST GOOD snapshot on failure — so
// if nvidia-smi starts failing outright (driver reset), health would otherwise
// serve hours-stale 200s and mislead the dispatcher's routing. Past this bound
// a stale snapshot is the same class of failure as no snapshot: 503.
const maxSnapshotAge = 30 * time.Second
// agentResidencyTTL is how long one roster answer backs agent_seat_resident
// before the next health request triggers a background refresh. Set to the
// SAME bound as maxSnapshotAge on purpose: health already promises its cached
// data (the VRAM snapshot) is no staler than that, and the roster answer
// rides the same freshness contract rather than inventing a second cadence.
const agentResidencyTTL = maxSnapshotAge
// agentResidencyMaxStale is how old a cached residency answer may be before a
// health read WAITS for a fresh one instead of serving it: one more TTL
// window past agentResidencyTTL. Inside that band the read keeps the
// stale-while-revalidate shape (serve the previous answer, refresh behind
// it); beyond it the previous answer was taken before the seat had time to
// load or unload, and serving it charged the delegator's eta a cold load
// for a seat that had been warm for two minutes (the 0.128.1 slot census:
// `chosen eta 326 s (cold 28 + 298 gen)` on a seat whose admission wait was
// 6 ms). The wait is bounded by residencyWaitBound, never open-ended.
const agentResidencyMaxStale = 2 * agentResidencyTTL
// residencyWaitBound caps how long a too-stale health read waits for the
// probe it kicked. It is sized for a RESPONSIVE llama-swap — the three
// refresh legs (/v1/models twice, /running once) answer in milliseconds on a
// healthy box — and it sits UNDER every health client's own budget: the
// accelerator router probes each base with 2 s (internal/accelremote), the
// fleet UI poller with 5 s (internal/fleetview), the delegator with 15 s
// (internal/delegate fetchNodeViewTimeout). A slow or hung llama-swap costs
// a health request this much, once per refresh cycle, and then the previous
// answer is served with a log line — never a client-side timeout. Any new
// /fleet/health client must budget above this. A var so tests can shorten it.
var residencyWaitBound = 1500 * time.Millisecond
// errSeatStateUnresolved marks a /running reading that came back without an
// error and without an ANSWER: the roster could not be read WHILE /running
// listed models, so the seat could only be matched by its bare name, which
// cannot see an alias-bound seat listed under its canonical id (seatload's
// `Ambiguous`). "Not loaded" and "could not tell" are different facts — and a
// failed roster over an EMPTY /running is the first, not the second.
var errSeatStateUnresolved = errors.New("fleet: seat state unresolved (roster unreadable, alias could not be matched)")
// errRefreshIncomplete is the seat-state read's "never ran" value: the deferred
// publish in refreshAgentResidency runs on EVERY exit including a panic, and a
// nil error there would publish "seat not loaded, and we know it" for a read
// that never happened.
var errRefreshIncomplete = errors.New("fleet: seat state not read")
// agentResidencyProbeTimeout bounds ONE background roster GET. Deliberately
// shorter than swapclient.DefaultTimeout: the probe informs a cached health
// field, and against a dead llama-swap a long-hanging refresh would just
// delay the (negative) answer the next TTL window serves anyway.
const agentResidencyProbeTimeout = 5 * time.Second
// Runner runs one translated request to completion. The real pipeline
// satisfies it; tests inject fakes.
type Runner interface {
Run(ctx context.Context, req core.Request) core.Result
}
// Options is the server's static identity + read-only data feeds. Snapshot and
// Footprints are functions (not values) so health always reads live state
// without the handler owning any sampling machinery.
type Options struct {
NodeID string
Snapshot func() (Snapshot, bool)
Footprints func() []FootprintEntry
// Version is advertised as harness_version. Empty = omitted.
Version string
// Reclaim reports how much VRAM this node could free right now. nil (or an
// unknown verdict) omits the reclaim fields entirely rather than guessing.
Reclaim func(freeGiB, totalGiB float64) ReclaimVerdict
GpuVendor string
GpuArch string
// Backends is the serving-backend list from the installer's manifest
// (InstalledInfo.Backends: primary first, then the alternates this install
// rendered — a dual-route node advertises ["vulkan","cpu"]). nil omits the
// field, so every node without a manifest backend is byte-identical.
Backends []string
// Accelerators is the additive-device list from the installer's manifest
// (installed.json `accelerators`, ADR 0024) — advertised in health so a
// delegator can route NPU-owned work to this node, minus the devices that
// stay on the box that carries them (config.FleetVisibleAccelerators,
// register E-08: the standalone Hailo-8L is never advertised, whatever this
// list says). Empty = omitted.
Accelerators []string
// LoopbackListener reports whether the serve listener is bound to a
// loopback address. The verb computes it from the RESOLVED listen address
// (netguard.LoopbackAddr) — NOT from --listen-trusted-network, which is
// permission to bind beyond loopback, not where the bind actually landed.
// The agent lane keys on it: with no Cfg.FleetAuthToken configured, agent
// dispatches on a non-loopback listener are refused 403 at ack time. The
// zero value (false) fails CLOSED for the agent lane — a constructor that
// forgets to set it gets the refusal, never an open agent surface; the
// media lane never consults it.
LoopbackListener bool
Cfg config.Config
// Pair is this node's PAIR emitter (pairworkloads.New(FromConfig(cfg))). The node uses it for ONE
// thing: the card of a job whose asker signalled core.PairCardHeader = "node" because it will not
// card the job itself. nil, or an emitter that is not Enabled, cards nothing.
Pair *pairworkloads.Emitter
// KVSlotDir is the directory the seats' --slot-save-path points at (ADR 0056
// Layer 2); empty = the kvslot lane answers 501. KVSlotCapGiB bounds it (0 = 8).
KVSlotDir string
KVSlotCapGiB int
// Host reports the last host CPU/RAM sample (hostsample.Sampler.Load).
// nil omits the host_* fields; the handler never samples itself.
Host func() (hostsample.Sample, bool)
// Lease reads this node's machine-wide GPU lease (gpulease.InspectDir over
// the config's lease dir). nil = not advertised, never refused.
Lease func() gpulease.Info
// LeaseStanding derives what the held lease is doing (orphaned, overdue, stalled) so
// /fleet/health can publish it beside "held". nil = the lease block carries held,
// class and window only, as before.
LeaseStanding func(gpulease.Info) gpulease.Standing
// Store returns the store steward's last status; nil = no steward.
Store func() storesteward.Status
// ServingConfig reports the rendered serving config's provenance: its spec
// hash and its state (MATCH/STALE/UNSTAMPED/HAND-EDITED). nil, or an empty
// state, omits both health fields.
//
// It is a func and not a path because the re-derive needs the tier table and
// the serving templates, which are embedded in the COMMAND (install_render.go)
// -- this package serves health, it does not own the seeds.
ServingConfig func() (specSHA256, state string)
// PowerProfile reports whether the GPU power profile this node applies at boot took (the state file the boot-apply
// tool writes, found by LOCAL_OFFLOAD_POWER_PROFILE_STATE). Like ServingConfig it is a func: the reader and its
// judgement live in the command, which also owns the doctor row that prints the same reading. nil omits the
// power_profile health field; a func that returns nil does too. It must answer from memory or one small local
// file read, because health never blocks.
PowerProfile func() *PowerProfileHealth
// GpuFaults is this node's driver fault summary (gpuactivity.FaultSummary): each card's incidents over 30 days and
// NVSentinel's repeat rule where it is met. A func for the same reason as PowerProfile: the command owns the
// reader, its background refresh and the retention file. nil, or a func that returns nil (no reading yet, or no
// NVIDIA card), omits gpu_faults. It answers from memory, because health never blocks.
GpuFaults func() *gpuactivity.FaultSummary
}
// Server is the fleet-node HTTP server: three handlers over a Runner + Jobs
// store. Task/family advertisements are derived once at construction (config
// is immutable while serving).
type Server struct {
runner Runner
jobs *Jobs
opts Options
// refusals throttles the journal line written for each request refused for its credential, at every door.
refusals refusalLog
// cardAdmitMu makes a pushed dispatch's id lookup, its node card's queued frame and its Admit one
// step (handleDispatch), so a racing duplicate emits no frame of its own.
cardAdmitMu sync.Mutex
// beforeCardAdmit is a test seam: it runs in that window's lead-in, where a racing duplicate
// has already passed the handler's first lookup.
beforeCardAdmit func()
// queue is the Option B consolidated pull queue (ADR 0030) — non-nil ONLY
// when this node is the config-elected holder (fleet_queue_host). Opened
// by EnableQueueHost; the routes mount only when it is non-nil.
queue *fleetqueue.Queue
// The advertised task and family lists are NOT fields: they are derived per health request and per
// claim (SupportedTasksFor, Families), because the media tasks depend on a disk read that can change
// under a running node (ADR 0077).
//
// accelerators is Options.Accelerators minus the local-only devices
// (config.FleetVisibleAccelerators, register E-08), computed once at construction:
// the list health publishes. The standalone Hailo-8L is never in it, whoever
// built Options and whatever the manifest says. nil on a node with none.
accelerators []string
// imageFamilies is the named-family advertisement (ADR 0058), computed once
// like families; nil on a node without named families (key omitted).
imageFamilies []ImageFamily
// recipeRows is the memoised image_recipes advertisement (image_recipe.go): re-read from the weight files at
// most every mediaRoutesTTL, so polling health does not stat them each time.
recipeMu sync.Mutex
recipeRows []ImageRecipeHealth
recipeAt time.Time
recipeRead bool
// recipeGen is the fingerprinter's generation the rows were built at: a read that settled since turns a pending row
// into its digest without waiting for the memo's minute.
recipeGen uint64
// mediaRecipeMemo is the memoised media_recipes advertisement (media_recipes.go): re-read at most every mediaRoutesTTL.
mediaRecipeMu sync.Mutex
mediaRecipeMemo []MediaRecipeHealth
mediaRecipeAt time.Time
mediaRecipeRead bool
// agentSeat is the resolved agent planner seat (config.AgentPlannerModel:
// agent_model > workhorse Model), computed once here at construction —
// the config cannot change under a running server. Advertised (and probed
// for residency) only when agentLane holds.
agentSeat string
// agentLane is fleetnode.AgentLaneAdmissible over the RESOLVED listener —
// the one predicate dispatch's ack-time guard and this advertisement share,
// computed once here at construction. Health must never advertise a
// lane dispatch would refuse: the delegator reads ONLY the agent_* fields,
// so a lane advertised past a tokenless non-loopback listener sends it
// through placement and straight into a 403.
agentLane bool
// visionLane is VisionLaneAdmissible over the RESOLVED listener (0.116.0)
// — the same one-predicate discipline as agentLane: health publishes
// vision_model (and SupportedTasksFor lists "vision") exactly when
// POST /fleet/vision will admit.
visionLane bool
// textLane is TextLaneAdmissible over the RESOLVED listener (0.154.0), the same
// one-predicate discipline: health publishes text_tasks (and SupportedTasksFor lists
// "text") exactly when POST /fleet/text will admit.
textLane bool
// forecastLane is ForecastLaneAdmissible over the RESOLVED listener, the same one-predicate
// discipline: health publishes forecast_model (and SupportedTasksFor lists "forecast") exactly
// when POST /fleet/forecast will admit.
forecastLane bool
// sttUploadLane is STTUploadAdmissible over the RESOLVED listener (ADR 0072), the same
// one-predicate discipline: health lists "stt-upload" and publishes stt_hq and
// stt_upload_max_mb exactly when POST /fleet/stt will admit.
sttUploadLane bool
// relayLimiter is the PAIR card relay's per-asker and global token bucket (pair_relay.go).
relayLimiter *pairworkloads.RelayLimiter
// sttGate is the node's stt concurrency cap (fleet_stt_max_concurrent), shared by the legacy
// path-taking lane and the upload door, pushed and pulled jobs alike (enterSTT).
sttGate *sttGate
// sttUploadSlots bounds the stt uploads in flight on this node (sttUploadInFlightMax).
sttUploadSlots chan struct{}
// mediaJobSlots bounds the media-job bodies in flight on this node (mediaJobInFlightMax).
mediaJobSlots chan struct{}
// chatLane is ChatLaneAdmissible over the RESOLVED listener (C-41b) — the
// same one-predicate discipline as agentLane and visionLane: health
// publishes `chat_lane` exactly when POST /fleet/chat will admit, because
// the delegator reads that one field to decide a cascade lane base is
// usable at all.
chatLane bool
// seatRateCache / seatRateAt / seatRateMu: the health handler's cached
// read of the seat-rates store (seatRate).
seatRateCache *SeatRateHealth
seatRateAt time.Time
seatRateMu sync.Mutex
// rosterServes answers "does the llama-swap behind endpoint serve seat?"
// (alias-aware). A seam so tests drive the residency cache without a live
// llama-swap; production always gets swapRosterServes.
rosterServes func(ctx context.Context, endpoint, seat string) (bool, error)
// rosterServedModels answers "what names does the llama-swap behind
// endpoint actually serve?" — the roster's own canonical ids PLUS every
// alias (swapclient.Roster.Names), advertised as served_models.
//
// It was rosterIDs, publishing IDs() alone, until the alias-blind gate
// finding: agent seats are normally bound by ALIAS (agent-pool ->
// qwen3.8-27b-vllm, offload-e4b -> gemma-4-e4b — see swapclient.go:9's
// plannerUnserved lesson), so a delegator checking an alias seat name
// against an id-only served_models list would read a healthy node as
// not serving its own advertised agent_seat and silently drop it from
// placement. Names() is IDs() plus every alias, so this field is
// misnamed as "IDs" now and renamed to say what it actually publishes.
//
// A separate seam from rosterServes (tests need to drive them
// independently), but production fetches its own roster per call rather
// than sharing swapRosterServes's fetch — that would mean threading a
// shared Roster through both closures' signatures, which changes tests
// that already inject rosterServes alone. THREE /v1/models GETs per refresh
// since the seat-state read joined them (seatRunning re-fetches the roster
// to resolve the seat's alias before reading /running), once per
// agentResidencyTTL window, is an acceptable cost for keeping today's
// residency seam untouched. What must stay true is the property the cache
// tests pin: one refresh CYCLE per TTL, single-flighted, never a probe per
// health request.
rosterServedModels func(ctx context.Context, endpoint string) ([]string, error)
// seatRunning answers "what does llama-swap's /running say about the agent
// seat RIGHT NOW?" — loaded, starting, or neither. /running ONLY: reading
// `/upstream/<seat>/…` would START an unloaded seat (register C-05; 54
// measured status probes each blocked ~186 s for exactly that reason), so
// the health path is allowed to learn the seat's state and is not allowed
// to touch the seat. Alias-aware through internal/seatload, because the
// harness binds seats by alias and /running lists canonical ids.
seatRunning func(ctx context.Context, endpoint, seat string) (seatload.Reading, error)
// setWriteDeadline extends ONE handler's write deadline past the server's
// blanket WriteTimeout (see extendWrite). A field so a test can observe the
// deadline a handler asks for; production is controllerWriteDeadline.
setWriteDeadline func(w http.ResponseWriter, t time.Time) error
// mediaKey is the media-route cache key of opts.Cfg, computed once in New (mediaKeySet says it was).
mediaKey mediacap.Key
mediaKeyOK bool
mediaKeySet bool
// setReadDeadline is setWriteDeadline's twin for a handler that receives a large body (extendRead).
setReadDeadline func(w http.ResponseWriter, t time.Time) error
// admitting counts the runs this process has claimed that are still in
// their ADMISSION phase (see admittingRuns). A field for the same reason as
// the roster seams: tests drive it without a registry on disk.
admitting func() int
admittingAt time.Time
admittingN int
admittingMu sync.Mutex
activityReg *gpuactivity.Registry
// visionCalls counts the vision calls other processes on this box have on a seat right now
// (see visionCallsRunning); visionN is its reading from the same registry walk as admittingN.
visionCalls func() int
visionN int
// admittingLoggedOpen / admittingLoggedList keep the two registry failures
// to ONE line per process: health is polled every few seconds by every
// delegator, so a per-cycle line would be a log flood, and the failure is
// reported for an operator rather than for a counter.
admittingLoggedOpen bool
admittingLoggedList bool
agentRes agentResidency
// backends / collectBackends: the cached per-lane and per-seat active_backend report
// (activebackend.go). collectBackends is a seam so tests drive the cache without a llama-swap;
// production always gets swapCollectBackends.
backends backendCache
collectBackends func(ctx context.Context, cfg config.Config) backendchain.Report
// agentResultDecodeOnce keeps the "could not decode a finished contract's
// result" line to one per process: the only producer of that JSON is this
// process's own agenttask, so the failure is a wire-shape drift, not a
// per-job event, and once said it is said.
agentResultDecodeOnce sync.Once
}
// agentResidency caches the roster's answer for the agent seat between health
// requests. The cache exists because BOTH of these hold at once:
// - agent_seat_resident must be roster-verified (§S3) — config alone cannot
// know whether the seat is actually served;
// - the health handler must not block on llama-swap as a matter of course
// (the reclaim tracker's rule — fleet_reclaim.go) and must not probe per
// request. Since 0.128.2 there is ONE bounded exception, below.
//
// So a health request serves the cached answer and, when it is older than
// agentResidencyTTL, triggers ONE background refresh (inflight = the
// single-flight latch). Older than agentResidencyMaxStale or never probed,
// the request WAITS for that refresh — at most residencyWaitBound — then
// serves whatever is published, and says so on the log once per refresh
// when the wait ran out. An agent contract that completed a call on the
// advertised seat writes THAT fact (loaded, not starting) straight into the
// cache (noteSeatAnswered) and never forces a probe; a probe that started
// before such a write keeps the job's seat facts when it lands and still
// publishes its roster facts. Residency itself stays roster-verified — a
// job never asserts it — and the cache's age is never touched by a job, so
// the roster keeps refreshing on its own cadence under sustained traffic.
// Until the first probe lands the answer is false — fail CLOSED: an
// unverified seat reads as not resident, and the delegator keeps the work
// local, which is always the safe placement.
type agentResidency struct {
mu sync.Mutex
resident bool
// served is the roster's ids+aliases list from the SAME refresh that set
// resident (rosterServedModels, fetched alongside rosterServes in
// refreshAgentResidency). A failed fetch publishes nil — never a stale
// list — and never flips
// resident on its own; a later placement gate reads absent/empty as
// UNKNOWN, exactly like a cold or failed residency probe.
served []string
// seatLoaded / seatStarting / seatKnown are llama-swap's /running verdict on
// the agent seat, published by the SAME refresh (seatRunning). seatKnown
// false = the read failed or never ran, and health then omits both fields
// rather than publishing "not loaded" for "could not tell".
seatLoaded bool
seatStarting bool
seatKnown bool
at time.Time
inflight bool
// timeoutLogged keeps the "served the previous answer after the wait ran
// out" line to ONE per refresh cycle: set by the read that logs it, reset
// by the publish. Health is polled by every delegator, so a line per
// waiting request would be a flood.
timeoutLogged bool
// seatFactAt is when noteSeatAnswered last wrote the seat state from a
// completed contract call. A refresh that STARTED before it keeps the
// job's seat facts when it lands (its /running reading predates them)
// and still publishes its roster facts.
seatFactAt time.Time
// idle broadcasts when inflight clears, so a caller that needs the answer
// SYNCHRONOUSLY (RefreshAgentResidency) can wait for the probe already
// running instead of starting a second one. Created lazily under mu — a
// sync.Cond needs its Locker, and the zero agentResidency must stay usable.
idle *sync.Cond
}
// idleCond returns the broadcast channel, creating it on first use. Caller
// holds mu.
func (a *agentResidency) idleCond() *sync.Cond {
if a.idle == nil {
a.idle = sync.NewCond(&a.mu)
}
return a.idle
}
// claimProbe takes the single-flight latch, reporting whether THIS caller now
// owns the probe (and must therefore clear the latch when it finishes).
func (a *agentResidency) claimProbe() bool {
a.mu.Lock()
defer a.mu.Unlock()
if a.inflight {
return false
}
a.inflight = true
return true
}
// awaitProbe blocks until whoever owns the latch has published its answer.
// Bounded by the probe's own agentResidencyProbeTimeout, so this cannot hang
// on a dead llama-swap.
func (a *agentResidency) awaitProbe() {
a.mu.Lock()
defer a.mu.Unlock()
for a.inflight {
a.idleCond().Wait()
}
}
// awaitProbeFor is awaitProbe with a ceiling: it returns when the running
// probe publishes OR when bound elapses, whichever is first. The waiter
// goroutine it leaves behind on a timeout ends when the probe does (the
// probe's own per-read timeouts bound that), so nothing here can leak past a
// hung llama-swap.
func (a *agentResidency) awaitProbeFor(bound time.Duration) {
done := make(chan struct{})
go func() {
a.awaitProbe()
close(done)
}()
select {
case <-done:
case <-time.After(bound):
}
}
// noteSeatAnswered records what a completed contract call PROVES: the
// advertised seat is loaded and not starting — right now, whatever the last
// probe said. Written straight into the cache so the next health read serves
// it without a probe and the next placement deal does not charge a cold load
// for a seat that just answered (the 0.128.1 census). It touches ONLY the
// seat state: residency stays roster-verified, the served list stays the last
// probe's, and the cache's age (`at`) is untouched so the roster keeps
// refreshing on its own cadence under sustained traffic.
func (a *agentResidency) noteSeatAnswered() {
a.mu.Lock()
a.seatLoaded, a.seatStarting, a.seatKnown = true, false, true
a.seatFactAt = time.Now()
a.mu.Unlock()
}
// noteAgentResult reads the one fact a finished agent contract proves about
// the advertised seat: whether it completed at least one call on it. Only a
// result that RAN on s.agentSeat counts. A defer that never reached the seat
// (roster-unserved, config, contract, a composite guard) finishes as a
// SUCCESS-shaped job — agenttask's OK is true for every terminal outcome,
// defers included — with zero steps, and proves nothing; a contract that ran
// and hit its wall has steps and proves the seat is loaded; a contract that
// ran on another layer's seat proves nothing about this one.
func (s *Server) noteAgentResult(data json.RawMessage) {
var r struct {
Seat string `json:"seat"`
Steps int `json:"steps"`
}
if err := json.Unmarshal(data, &r); err != nil {
// Not a "proves nothing" outcome: the producer is this process's own
// agenttask, so a decode failure means the wire shape drifted, and
// silently losing the fast path for every job from here on would
// look exactly like a node that never ran agent work. Said once.
s.agentResultDecodeOnce.Do(func() {
log.Printf("fleet: a finished agent contract's result could not be decoded for the seat-state write (%v); until this is fixed the residency cache learns the seat state from /running only", err)
})
return
}
if r.Steps <= 0 || r.Seat == "" || r.Seat != s.agentSeat {
return
}
s.agentRes.noteSeatAnswered()
}
// New builds a Server. The lane verdicts and the image-family list are computed here — the config
// cannot change under a running server. The supported-task and family lists are not: they are derived
// per health request and per claim, because the media tasks follow a cached disk read (ADR 0077).
func New(runner Runner, jobs *Jobs, opts Options) *Server {
s := &Server{
runner: runner,
jobs: jobs,
opts: opts,
accelerators: config.FleetVisibleAccelerators(opts.Accelerators),
imageFamilies: ImageFamilies(opts.Cfg),
agentSeat: opts.Cfg.AgentPlannerModel(""),
agentLane: AgentLaneAdmissible(opts.Cfg, opts.LoopbackListener),
visionLane: VisionLaneAdmissible(opts.Cfg, opts.LoopbackListener),
textLane: TextLaneAdmissible(opts.Cfg, opts.LoopbackListener),
forecastLane: ForecastLaneAdmissible(opts.Cfg, opts.LoopbackListener),
sttUploadLane: STTUploadAdmissible(opts.Cfg, opts.LoopbackListener),
relayLimiter: pairworkloads.DefaultRelayLimiter(),
sttGate: newSTTGate(opts.Cfg.EffectiveSTTMaxConcurrent()),
sttUploadSlots: make(chan struct{}, sttUploadInFlightMax),
mediaJobSlots: make(chan struct{}, mediaJobInFlightMax),
chatLane: ChatLaneAdmissible(opts.Cfg, opts.LoopbackListener),
rosterServes: swapRosterServes,
rosterServedModels: swapRosterServedModels,
seatRunning: swapSeatRunning,
collectBackends: swapCollectBackends,
setWriteDeadline: controllerWriteDeadline,
setReadDeadline: controllerReadDeadline,
}
s.admitting = s.admittingRuns
s.visionCalls = s.visionCallsRunning
// The config cannot change under a running server, so its cache key is computed once: the health
// and admission paths then never encode the config again to look a verdict up.
s.mediaKey, s.mediaKeyOK = mediacap.KeyOf(opts.Cfg)
s.mediaKeySet = true
// The stt transcript retention rides the job store's janitor tick (and runs at fleet-serve start).
if jobs != nil {
jobs.OnSweep(func() { sweepTranscriptsOnTick(opts.Cfg) })
}
return s
}
// mediaView is a fresh single-request view of this node's media route verdicts (see mediaView). It starts
// from the key New computed, so a read is a cache lookup, never a config encoding.
func (s *Server) mediaView() *mediaView {
return &mediaView{cfg: s.opts.Cfg, key: s.mediaKey, keyOK: s.mediaKeyOK, haveKey: s.mediaKeySet}
}
// seatRunningClient reads /running on the node's own llama-swap. netguard's
// transport for the same reason every outbound client here uses it, and a
// timeout equal to the residency probe's: this runs on the background refresh,
// never inside a health request.
var seatRunningClient = &http.Client{Transport: netguard.SafeTransport(nil), Timeout: agentResidencyProbeTimeout}
// swapSeatRunning is the production seat-state read: llama-swap's /running,
// alias-resolved, and nothing else.
func swapSeatRunning(ctx context.Context, endpoint, seat string) (seatload.Reading, error) {
return seatload.Running(ctx, seatRunningClient, endpoint, seat)
}
// admittingTTL bounds how often the health path re-reads the activity registry.
// Health is polled every few seconds by every delegator, and the registry is a
// directory: one read per window, never one per request — the same rule the
// seat-rates store and the residency cache follow.
const admittingTTL = 2 * time.Second
// admittingRuns counts THIS PROCESS's agent runs that are still in their
// ADMISSION phase, from the gpuactivity registry the harness already writes and
// heartbeats (ADR 0041).
//
// WHICH PHASE, and why this source (register S-17): internal/pipeline registers
// every contract run with `Phase: "admission"` and flips it to "running" at the
// first planner step, so "admission" is precisely the window the diagnosis
// measured — cordon → swap pre-flight → warm → coherence probe, up to 300 s
// during which jobs.go has already flipped the job to `running` and the card is
// idle. The job store cannot answer this: it knows a worker took the job, not
// what that worker is waiting for.
//
// PID-scoped on purpose: the registry is machine-wide, and another process's
// `agent_run` sitting in admission is not one of THIS node's jobs. It holds the
// same seat, but it is not part of this node's queue accounting and must not
// subtract from this node's saturation.
func (s *Server) admittingRuns() int {
admitting, _ := s.registryCounts()
return admitting
}
// visionCallsRunning counts the single-shot vision calls (gpuactivity.KindVision) that OTHER
// processes on this box have on a seat right now: a CLI `vqa` / `ocr` loop, an MCP server's
// offload_vqa. They hold no lease and are no job of this node, so jobs_running never counted
// them, and a script that looped one CLI call per image kept a vision seat busy for hours while
// every delegator read the node as idle (2026-10-28). This process's own vision records are its
// /fleet/vision jobs, already in jobs_running, so they are not counted twice. Published as health's
// `vision_calls_running`; a display signal like jobs_admitting, read by no placement rule.
func (s *Server) visionCallsRunning() int {
_, vision := s.registryCounts()
return vision
}
// registryCounts is ONE walk of the activity registry per admittingTTL window, feeding both
// admittingRuns and visionCallsRunning.
func (s *Server) registryCounts() (admitting, vision int) {
s.admittingMu.Lock()
defer s.admittingMu.Unlock()
if !s.admittingAt.IsZero() && time.Since(s.admittingAt) < admittingTTL {
return s.admittingN, s.visionN
}
s.admittingAt = time.Now()
s.admittingN, s.visionN = 0, 0
if s.activityReg == nil {
reg, err := gpuactivity.Open(s.opts.Cfg.GPULockPath, s.opts.Cfg.StateDir)
if err != nil {
// RETRIED on the next cycle, never latched: a root that is briefly
// unresolvable (a state dir not yet created, a transient mount) must
// not silence this field for the life of the process. And it is SAID
// — once — because 0 is a legitimate value, so an unreported failure
// is indistinguishable from a node with nothing in admission.
if !s.admittingLoggedOpen {
s.admittingLoggedOpen = true
log.Printf("fleet: activity registry unreadable under state dir %q; jobs_admitting and vision_calls_running read 0 until it resolves (reported once per process, retried every %s): %v",
s.opts.Cfg.StateDir, admittingTTL, err)
}
return 0, 0
}
s.activityReg = reg
}
runs := s.activityReg.List(time.Now())
if len(runs) == 0 {
// List treats an unreadable directory as an empty one (it is advisory),
// so "nothing in admission" and "cannot read the registry" arrive here
// as the same nil slice. One cheap read tells them apart; a directory
// that does not exist yet is the normal cold state and says nothing.
if _, rerr := os.ReadDir(s.activityReg.Dir()); rerr != nil && !errors.Is(rerr, fs.ErrNotExist) && !s.admittingLoggedList {
s.admittingLoggedList = true
log.Printf("fleet: activity registry directory %q cannot be listed; jobs_admitting and vision_calls_running read 0 (reported once per process): %v",
s.activityReg.Dir(), rerr)
}
}
pid := os.Getpid()
for _, run := range runs {
// The SAME constant internal/pipeline/agenttask.go registers a contract
// run with, and the same one gpuactivity heals at the first planner
// step — a bare literal here would make a rename a silent zero.
if run.PID == pid && run.Phase == gpuactivity.PhaseAdmission {
s.admittingN++
}
if run.PID != pid && run.Kind == gpuactivity.KindVision {
s.visionN++
}
}
return s.admittingN, s.visionN
}
// swapRosterServes is the production residency probe: one GET /v1/models via
// the shared swapclient adapter, answered alias-aware (Roster.Serves matches
// canonical ids AND meta.llamaswap.aliases — the plannerUnserved lesson: every
// harness-bound seat name on the reference deployment is an alias, so an
// id-only match would report a correctly-served seat missing forever).
func swapRosterServes(ctx context.Context, endpoint, seat string) (bool, error) {
roster, err := swapclient.FetchRoster(ctx, endpoint, agentResidencyProbeTimeout)
if err != nil {
return false, err
}
return roster.Serves(seat), nil
}
// swapRosterServedModels is the production served_models probe: one GET
// /v1/models, answered as the roster's own ids PLUS every alias
// (swapclient.Roster.Names — see the rosterServedModels field doc for why
// IDs() alone was the alias-blind gate bug). A separate roster fetch from
// swapRosterServes (see that field's doc) — accepted so the existing
// rosterServes seam and its tests stay untouched.
func swapRosterServedModels(ctx context.Context, endpoint string) ([]string, error) {
roster, err := swapclient.FetchRoster(ctx, endpoint, agentResidencyProbeTimeout)
if err != nil {
return nil, err
}
return roster.Names(), nil
}
// agentResident returns the cached residency answer, kicking off a background
// refresh when the cache has aged past agentResidencyTTL. It never probes more
// than once per TTL window regardless of request rate, and blocks only on a
// too-stale or never-probed cache — for at most residencyWaitBound.
func (s *Server) agentResident() bool {
a := &s.agentRes
a.mu.Lock()
age := time.Since(a.at)
if !a.at.IsZero() && age <= agentResidencyTTL {
v := a.resident
a.mu.Unlock()
return v
}
// Past the TTL. Inside the stale band (one more window) serve the previous
// answer and refresh behind it. Beyond it — or never probed — the previous
// answer describes a seat that has had time to change state: kick the
// refresh if nobody holds it and WAIT for it, bounded, before answering.
tooStale := a.at.IsZero() || age > agentResidencyMaxStale
owner := !a.inflight
if owner {
a.inflight = true
}
if !tooStale {
v := a.resident // serve the previous answer while the refresh runs
a.mu.Unlock()
if owner {
go s.refreshAgentResidency()
}
return v
}
previousAt := a.at
a.mu.Unlock()
if owner {
go s.refreshAgentResidency()
}
a.awaitProbeFor(residencyWaitBound)
a.mu.Lock()
v := a.resident
if !a.at.After(previousAt) && !a.timeoutLogged {
// Nothing was published inside the bound: the previous answer goes
// out, and that is said ONCE per refresh cycle (the publish resets
// the flag), never once per waiting request.
a.timeoutLogged = true
age := "never taken"
if !previousAt.IsZero() {
age = time.Since(previousAt).Round(time.Second).String() + " old"
}
log.Printf("fleet: health waited %s for the residency refresh of %q against %s and it had not published; serving the previous answer (%s)",
residencyWaitBound, s.agentSeat, s.opts.Cfg.Endpoint, age)
}
a.mu.Unlock()
return v
}
// recentAgentWallSamples is how many finished agent jobs recent_agent_wall_sec
// is the median of. Eight: enough that one outlier cannot move the median, few
// enough that the number describes the node NOW rather than an hour ago (the
// store's terminal TTL is an hour).
const recentAgentWallSamples = 8
// medianSeconds is the median of a duration sample, in seconds rounded to
// hundredths. An empty sample is 0 = "this node makes no claim", which the
// payload omits.
func medianSeconds(d []time.Duration) float64 {
if len(d) == 0 {
return 0
}
sorted := append([]time.Duration(nil), d...)
sort.Slice(sorted, func(a, b int) bool { return sorted[a] < sorted[b] })
mid := len(sorted) / 2
sec := sorted[mid].Seconds()
if len(sorted)%2 == 0 {
sec = (sorted[mid-1].Seconds() + sec) / 2
}
return math.Round(sec*100) / 100
}
// queueWaitEstimateSec is how long a caller should expect to wait for a
// worker to free, from the node's OWN measured recent wall — never from
// seat_rate.min_turn_sec, which is a max-final RETRY floor for one seat and
// has no relationship to how deep this node's backlog is (register S-04,
// diagnosis §5.2, the overhaul plan's roast correction).
//
// cappedDepth is the CAPPED backlog only — Jobs.CountsCapped's queued+running,
// whatever phase (an admitting job still holds the worker slot a waiter
// needs; see health's admitting note) — NEVER health's all-jobs queue_depth
// (register review round 1 BLOCKER). max_concurrent_jobs bounds only jobs
// that count against it (the same set runningCappedLocked/IdleSlot measure);
// an uncapped job (a render, an stt, a pipeline route) never waits behind
// that cap, so dividing an ALL-jobs depth by maxConcurrent would inflate the
// estimate for a node whose agent slots are genuinely idle behind unrelated
// media load — exactly the class of bug saturationOf (S-17) and this PR's own
// IdleSlot fix (S-20) both already avoid.
//
// excess = cappedDepth - maxConcurrent is how many admitted CAPPED jobs are
// beyond what can run AT ONCE right now; each of maxConcurrent workers
// retires roughly one job every recentWallSec, so the deepest excess job
// waits about excess x recentWallSec / maxConcurrent. 0 (no claim) when
// maxConcurrent is unlimited (<=0 — nothing ever waits for a worker), there
// is no excess (a worker is free), or the node has no recent wall sample at
// all.
func queueWaitEstimateSec(cappedDepth, maxConcurrent int, recentWallSec float64) float64 {
if maxConcurrent <= 0 || recentWallSec <= 0 {
return 0
}
excess := cappedDepth - maxConcurrent
if excess <= 0 {
return 0
}
return float64(excess) * recentWallSec / float64(maxConcurrent)
}
// newJobWaitSec is how long a job submitted NOW waits for a worker, on the same arithmetic as
// queueWaitEstimateSec but for the arrival and not for the job already at the back of the line: one
// slot deeper, because the new job queues behind everything admitted. queueWaitEstimateSec answers
// "how long does the deepest QUEUED job wait" (excess x wall / workers), which a delegator reads as
// the wait of the job it is about to send and so under-prices by exactly one wall / workers; and at
// depth == max_concurrent_jobs (every worker busy, nothing queued) it is 0 and omitted although a new
// job still waits for a worker to retire, so that estimate was also discontinuous. This one is
// (depth - max + 1) x wall / workers, and it is the arithmetic internal/delegate derives from the
// counters when a node publishes nothing (queueWaitFor), so a published number and a derived one are
// the same number. 0 (no claim) when maxConcurrent is unlimited, a worker is free, or the node has no
// recent wall sample.
func newJobWaitSec(cappedDepth, maxConcurrent int, recentWallSec float64) float64 {
if maxConcurrent <= 0 || recentWallSec <= 0 {
return 0
}
ahead := cappedDepth - maxConcurrent + 1
if ahead <= 0 {
return 0
}
return float64(ahead) * recentWallSec / float64(maxConcurrent)
}
// publishedNewJobWait is newJobWaitSec as health publishes it, rounded to the hundredth: nil when the
// node has no number to give (max_concurrent_jobs unlimited, or no recent wall sample), else a pointer
// to it, 0 included. The pointer is the point: a plain float64 with omitempty drops a genuine 0, and a
// delegator that finds the field absent falls back to counters (jobs_running + jobs_queued) that include
// uncapped jobs, pricing the wait of an idle agent lane from a dozen renders.
func publishedNewJobWait(cappedDepth, maxConcurrent int, recentWallSec float64) *float64 {
if maxConcurrent <= 0 || recentWallSec <= 0 {
return nil
}
sec := math.Round(newJobWaitSec(cappedDepth, maxConcurrent, recentWallSec)*100) / 100
return &sec
}
// retryAfterFor computes the "queue full" 503's Retry-After header value AND
// the human-readable suffix appended to the refusal message, from ONE
// evaluation of the node's own measured recent wall — so the header and the
// text can never disagree (register S-04; review round 1 items 2/3).
//
// The header is always ceil(queueWaitEstimateSec), bounded to [5, 300] so it
// is never "retry at once" (the queue IS full right now) and never an
// unbounded promise. The TEXT distinguishes three shapes the bare number
// cannot:
//
// - No recent wall sample yet (a fresh node, or one that has never
// finished an agent job): the flat 30s is a DEFAULT, not a measurement,
// and must not be worded like one — "no recent completions yet — retry
// in 30 s", never "~30 s until a worker frees" (that phrasing implies a
// precision the node does not have).
// - A genuine measured estimate that lands inside [5, 300]: "~N s until a
// worker frees, from recent completions".
// - An estimate that would exceed 300s: the header is still capped at 300
// (an unbounded promise is worse than an honest floor), but the text says
// so explicitly — ">=300 s until a worker frees" — rather than
// presenting the cap as if it were the precise answer. A raw 600s
// estimate silently rendered as "~300 s" invites every waiter to retry
// in lockstep at the same instant (the overhaul plan's roast correction:
// an admission refusal must be something the delegator can outwait,
// never terminal, and a disguised clamp defeats that).
func retryAfterFor(cappedDepth, maxConcurrent int, recentWallSec float64) (headerSec int, suffix string) {
if recentWallSec <= 0 {
return 30, "no recent completions yet — retry in 30 s"
}
sec := int(math.Ceil(queueWaitEstimateSec(cappedDepth, maxConcurrent, recentWallSec)))
switch {
case sec < 5:
return 5, "~5 s until a worker frees, from recent completions"
case sec > 300:
return 300, ">=300 s until a worker frees, from recent completions"
default:
return sec, fmt.Sprintf("~%d s until a worker frees, from recent completions", sec)
}
}
// seatState reports the cached /running answer for the agent seat: loaded,
// starting, and whether the read is KNOWN at all. Never probes itself — the
// background residency refresh publishes it on the same TTL.
func (s *Server) seatState() (loaded, starting, known bool) {
a := &s.agentRes
a.mu.Lock()
defer a.mu.Unlock()
return a.seatLoaded, a.seatStarting, a.seatKnown
}
// servedModels returns a copy of the cached roster id list, as of the last
// residency refresh. It NEVER triggers a probe itself — agentResident()
// already does, and handleHealth calls both, agentResident() first, so a
// health request that reads servedModels() first would always see the
// PREVIOUS window's answer instead of the refresh it just scheduled.
func (s *Server) servedModels() []string {
a := &s.agentRes
a.mu.Lock()
defer a.mu.Unlock()
if a.served == nil {
return nil
}
return append([]string(nil), a.served...)
}
// refreshAgentResidency runs one bounded roster probe and publishes the
// answer. THE CALLER MUST ALREADY HOLD the single-flight latch
// (agentResidency.claimProbe, or the inline claim in agentResident) — this
// function clears it unconditionally on the way out, so a caller that ran
// without holding it would release someone else's claim and let a duplicate
// probe race the first one's answer.
//
// A probe FAILURE publishes resident=false rather than keeping the
// last good answer (the VRAM sampler's rule): residency feeds a placement
// gate, and advertising a seat as servable off a stale success while
// llama-swap is down would route remote agent work at a node that cannot run
// it — false only costs a conservative local placement. The failure is still
// cached for a full TTL so a dead endpoint is probed once per window, not
// hammered per request.
func (s *Server) refreshAgentResidency() {
probeStart := time.Now()
var resident bool
var err error
var served []string
var seat seatload.Reading
var seatErr error = errRefreshIncomplete
// The publish is DEFERRED, not tail-appended: the probe below runs outside
// any lock, and clearing the latch only on the normal path meant a panic
// anywhere in that seam froze residency for the life of the process — never
// refreshable again — and parked every awaitProbe waiter forever, since
// RefreshAgentResidency has no timeout of its own. On a panic the zero
// values here publish the fail-closed answer (resident=false), which is the
// same verdict a failed probe gets, and the panic still propagates.
defer func() {
a := &s.agentRes
a.mu.Lock()
a.resident = resident && err == nil
a.served = served
if !a.seatFactAt.After(probeStart) {
a.seatLoaded, a.seatStarting, a.seatKnown = seat.Loaded, seat.Starting, seatErr == nil
}
// else: a contract completed a call on the seat while this probe ran
// (noteSeatAnswered) and the /running reading here predates that —
// the job's seat facts stand; the roster facts above are this
// probe's either way.
a.at = time.Now()
a.inflight = false
a.timeoutLogged = false
a.idleCond().Broadcast() // release any synchronous waiter (RefreshAgentResidency)
a.mu.Unlock()
}()
residentCtx, residentCancel := context.WithTimeout(context.Background(), agentResidencyProbeTimeout)
defer residentCancel()
resident, err = s.rosterServes(residentCtx, s.opts.Cfg.Endpoint, s.agentSeat)
if err != nil {
// agent_seat_resident:false is the one field that stops EVERY remote
// placement at this node, and the probe error was the only evidence of
// why. Without this line an operator sees a node advertising itself as
// unusable and has nothing, on either side of the wire, to look at. The
// probe runs at most once per TTL window, so this cannot spam.
log.Printf("fleet: agent seat %q residency probe against %s failed; advertising agent_seat_resident:false for up to %s: %v",
s.agentSeat, s.opts.Cfg.Endpoint, agentResidencyTTL, err)
}
// The served-models fetch gets its OWN full-length timeout rather than
// reusing residentCtx: sharing one context meant a slow-but-healthy
// rosterServes call could burn most of the budget before
// rosterServedModels even starts, starving a perfectly healthy roster
// and publishing served=nil for a seat that is, in fact, resident.
namesCtx, namesCancel := context.WithTimeout(context.Background(), agentResidencyProbeTimeout)
defer namesCancel()
names, nerr := s.rosterServedModels(namesCtx, s.opts.Cfg.Endpoint)
if nerr != nil {
names = nil // unknown on failure — never keep a stale list, never flip resident
}
served = names
// The seat's LOAD state, from /running only (register C-05: nothing here may
// touch a path that could start the seat). Its own full-length timeout, for
// the same reason the served-models fetch has one: a slow roster must not
// starve the read after it. A failure publishes "unknown" — seat_loaded and
// seat_starting are then absent, never false.
if s.seatRunning != nil && strings.TrimSpace(s.opts.Cfg.Endpoint) != "" {
seatCtx, seatCancel := context.WithTimeout(context.Background(), agentResidencyProbeTimeout)
defer seatCancel()
seat, seatErr = s.seatRunning(seatCtx, s.opts.Cfg.Endpoint, s.agentSeat)
switch {
case seatErr != nil:
// Same discipline as the residency probe above: the verdict this
// publishes (two absent fields) is useless to an operator without
// the reason, and the read runs at most once per TTL window, so it
// cannot spam.
log.Printf("fleet: seat state of %q against %s could not be read; omitting seat_loaded/seat_starting for up to %s: %v",
s.agentSeat, s.opts.Cfg.Endpoint, agentResidencyTTL, seatErr)
seat = seatload.Reading{}