Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions frontend/src/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ export function App() {
// exactly when the user has switched away and needs to be told (THI-78).
// Browser throttles background timers (~1Hz, dropping to ~1/min after a few
// minutes hidden), which bounds the bandwidth cost.
const { data: state, consecutiveErrors, refresh } = usePolling(
const { data: state, consecutiveErrors, refresh, degraded: backendDegraded } = usePolling(
fetchState,
pollIntervalMs,
settings.notifyBrowser,
Expand Down Expand Up @@ -587,9 +587,13 @@ export function App() {
settings.pollIntervalMs,
MODAL_OPEN_POLL_MS,
inputActive,
// Adaptive back-off: when /api/state is slow (e.g. the host is swapping),
// usePolling flags `backendDegraded` and the cadence is floored so we
// stop piling requests onto a struggling backend.
backendDegraded,
);
setPollIntervalMs((prev) => (prev === next ? prev : next));
}, [openId, windows, settings.pollIntervalMs, inputActive]);
}, [openId, windows, settings.pollIntervalMs, inputActive, backendDegraded]);

// Apply persisted appearance settings to <html>. The theme/density/
// reduced-motion CSS ships in styles.css; accent is written as CSS vars.
Expand Down
45 changes: 45 additions & 0 deletions frontend/src/api/usePolling.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -110,4 +110,49 @@ describe("usePolling", () => {
get: () => "visible",
});
});

// --- adaptive back-off: `degraded` reflects slow / outpaced backend ---

it("flags the backend degraded when response latency is high", async () => {
// fn resolves after 900 ms of (fake) time; the interval (2 s) is longer so
// the fetch completes and its latency is measured.
const slowFn = () =>
new Promise<string>((res) => {
window.setTimeout(() => res("ok"), 900);
});
const { result } = renderHook(() => usePolling(slowFn, 2000));
await tick(900);
expect(result.current.degraded).toBe(true);
});

it("stays not degraded when responses are fast", async () => {
const fastFn = () =>
new Promise<string>((res) => {
window.setTimeout(() => res("ok"), 20);
});
const { result } = renderHook(() => usePolling(fastFn, 1000));
await tick(20);
await tick(1000);
await tick(20);
expect(result.current.degraded).toBe(false);
});

it("flags degraded when polls are repeatedly superseded (backend outpaced)", async () => {
// fn never resolves on its own; it rejects AbortError when its signal
// fires — i.e. when the next tick aborts it before it completed.
const fn = vi.fn((signal: AbortSignal) => {
return new Promise<string>((_res, rej) => {
signal.addEventListener("abort", () =>
rej(new DOMException("aborted", "AbortError")),
);
});
});
const { result } = renderHook(() => usePolling(fn, 100));
await act(async () => {
await Promise.resolve();
});
await tick(100); // tick 2 aborts tick 1's fetch → supersede #1
await tick(100); // tick 3 aborts tick 2's fetch → supersede #2 → degraded
expect(result.current.degraded).toBe(true);
});
});
53 changes: 49 additions & 4 deletions frontend/src/api/usePolling.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,15 @@
import { startTransition, useEffect, useRef, useState } from "react";

import { nextDegraded, updateLatencyEwma } from "../lib/pollTier";

export interface PollingState<T> {
data: T | null;
error: Error | null;
consecutiveErrors: number;
/** True when responses have been slow or polls keep getting superseded
* (we're outpacing the backend). Consumers can widen the poll cadence to
* let a struggling backend recover — see `pickPollInterval(..., degraded)`. */
degraded: boolean;
refresh: () => void;
}

Expand All @@ -29,23 +35,55 @@ export function usePolling<T>(
const [data, setData] = useState<T | null>(null);
const [error, setError] = useState<Error | null>(null);
const [consecutiveErrors, setConsecutiveErrors] = useState(0);
const [degraded, setDegraded] = useState(false);
const fnRef = useRef(fn);
fnRef.current = fn;
const tickRef = useRef<() => Promise<void>>(async () => {});
// Latency-tracking state lives in refs (not effect-locals) so it survives the
// effect re-run when `ms` changes — backing off (which changes `ms`) must not
// reset the degraded signal that triggered it.
const ewmaRef = useRef<number | null>(null);
const supersedesRef = useRef(0);
const degradedRef = useRef(false);

useEffect(() => {
let alive = true;
let inflight: AbortController | null = null;

// Fold a completed (or superseded) poll into the degraded signal. A
// completed poll records its latency and resets the supersede run; a
// superseded one (aborted by the next tick before finishing) bumps the
// run — the EWMA can't see those since they never complete to be measured.
const recordSample = (latencyMs: number | null, superseded: boolean) => {
if (superseded) {
supersedesRef.current += 1;
} else {
supersedesRef.current = 0;
if (latencyMs !== null) {
ewmaRef.current = updateLatencyEwma(ewmaRef.current, latencyMs);
}
}
const next = nextDegraded(degradedRef.current, ewmaRef.current, supersedesRef.current);
if (next !== degradedRef.current) {
degradedRef.current = next;
startTransition(() => setDegraded(next));
}
};

const tick = async () => {
if (!alive) return;
if (!pollWhenHidden && document.visibilityState === "hidden") return;
inflight?.abort();
const ctrl = new AbortController();
inflight = ctrl;
const startedAt = performance.now();
try {
const v = await fnRef.current(ctrl.signal);
if (!alive || ctrl.signal.aborted) return;
if (!alive) return;
// Completed → a real latency sample, even if a newer tick superseded
// us in the meantime (we still got the response time).
recordSample(performance.now() - startedAt, false);
if (ctrl.signal.aborted) return;
// startTransition marks these as non-urgent so React can interrupt
// the resulting render commit to handle user input (typing in a
// modal, palette search) ahead of the polling update. Polling is
Expand All @@ -56,9 +94,16 @@ export function usePolling<T>(
setConsecutiveErrors(0);
});
} catch (e) {
if (!alive || ctrl.signal.aborted) return;
if (!alive) return;
const err = e as Error;
if (err.name === "AbortError") return;
if (err.name === "AbortError") {
// Aborted while still mounted → superseded by the next tick before
// completing, i.e. we're polling faster than the backend answers.
// (Unmount/re-key sets alive=false first, so it's excluded above.)
recordSample(null, true);
return;
}
if (ctrl.signal.aborted) return;
startTransition(() => {
setError(err);
setConsecutiveErrors((n) => n + 1);
Expand All @@ -82,5 +127,5 @@ export function usePolling<T>(
};
}, [ms, pollWhenHidden]);

return { data, error, consecutiveErrors, refresh: () => void tickRef.current() };
return { data, error, consecutiveErrors, degraded, refresh: () => void tickRef.current() };
}
73 changes: 72 additions & 1 deletion frontend/src/lib/pollTier.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
import { describe, expect, it } from "vitest";

import { ACTIVE_POLL_MS, INPUT_ACTIVE_MIN_MS, pickPollInterval } from "./pollTier";
import {
ACTIVE_POLL_MS,
DEGRADED_POLL_MS,
INPUT_ACTIVE_MIN_MS,
nextDegraded,
pickPollInterval,
updateLatencyEwma,
} from "./pollTier";
import type { Status, Window } from "../types";

const MODAL_OPEN_POLL_MS = 100;
Expand Down Expand Up @@ -162,4 +169,68 @@ describe("pickPollInterval", () => {
pickPollInterval(true, [], 3000, MODAL_OPEN_POLL_MS),
).toBe(MODAL_OPEN_POLL_MS);
});

// --- adaptive back-off (degraded backend) ---
it("floors the interval at DEGRADED_POLL_MS when degraded, overriding modal + input", () => {
expect(pickPollInterval(true, [], 3000, 500, false, true)).toBe(DEGRADED_POLL_MS);
expect(pickPollInterval(true, [], 3000, 500, true, true)).toBe(DEGRADED_POLL_MS);
expect(
pickPollInterval(false, [makeWindow("running")], 3000, 500, false, true),
).toBe(DEGRADED_POLL_MS);
});

it("never lowers an interval that is already slower than the back-off floor", () => {
// Idle tier with a high configured cadence → 2× = 20 000 ms > 5 000 ms floor.
expect(
pickPollInterval(false, [makeWindow("idle")], 10000, 500, false, true),
).toBe(20000);
});

it("is a no-op when not degraded (default arg keeps existing call sites)", () => {
expect(pickPollInterval(true, [], 3000, 500, false, false)).toBe(500);
expect(pickPollInterval(true, [], 3000, 500)).toBe(500);
});
});

describe("updateLatencyEwma", () => {
it("seeds with the first sample", () => {
expect(updateLatencyEwma(null, 120)).toBe(120);
});

it("smooths toward newer samples", () => {
const e1 = updateLatencyEwma(null, 1000);
const e2 = updateLatencyEwma(e1, 0); // 0.4*0 + 0.6*1000
expect(e2).toBeCloseTo(600, 5);
});
});

describe("nextDegraded (hysteresis + supersede escalation)", () => {
it("enters degraded when smoothed latency exceeds the high threshold", () => {
expect(nextDegraded(false, 900, 0)).toBe(true);
});

it("enters degraded after repeated supersedes even when latency reads low", () => {
// Polling faster than the backend can answer: fetches are aborted before
// completing, so latency samples are sparse — the supersede count is the
// signal that we're outpacing the backend.
expect(nextDegraded(false, 50, 2)).toBe(true);
});

it("stays not-degraded for fast responses with no supersedes", () => {
expect(nextDegraded(false, 200, 0)).toBe(false);
});

it("stays degraded in the hysteresis band (between exit and enter)", () => {
expect(nextDegraded(true, 500, 0)).toBe(true);
});

it("exits degraded only when latency is clearly low AND not superseding", () => {
expect(nextDegraded(true, 100, 0)).toBe(false);
expect(nextDegraded(true, 100, 1)).toBe(true); // still superseding → stay
});

it("treats a null (no-sample-yet) latency as no change to the current state", () => {
expect(nextDegraded(false, null, 0)).toBe(false);
expect(nextDegraded(true, null, 0)).toBe(true);
});
});
55 changes: 54 additions & 1 deletion frontend/src/lib/pollTier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,52 @@ export const ACTIVE_POLL_MS = 1000;
// normal tier 800 ms after the last keydown — see `useInputActive`.
export const INPUT_ACTIVE_MIN_MS = 1500;

// --- Adaptive back-off when the backend is slow ---------------------------
// A safety net for when /api/state responses degrade (e.g. the host is under
// memory pressure and tmux subprocess spawns stall). Polling at the normal
// modal-open cadence (500 ms) against a backend that takes seconds to answer
// just stacks requests it can't keep up with — which is exactly what starves
// the keystroke round-trip. When degraded, the cadence is floored here so the
// backend gets room to recover. With batched collect_state the normal latency
// is ~30 ms, so this never trips in the common case.
export const DEGRADED_POLL_MS = 5000;
// EWMA smoothing for response latency — reacts within ~3 samples, damps noise.
const LATENCY_ALPHA = 0.4;
// Hysteresis band: enter "degraded" only when clearly slow, stay until clearly
// recovered. Avoids flapping the cadence around a single threshold.
const DEGRADED_ENTER_MS = 800;
const DEGRADED_EXIT_MS = 400;
// Consecutive superseded polls (aborted before completing because the next
// tick fired first) that flag the backend as outpaced. This is the signal the
// EWMA can't see on its own: when latency exceeds the interval, fetches never
// complete to be measured, so the supersede count is what escalates.
const DEGRADED_SUPERSEDE_COUNT = 2;

/** Exponentially-weighted moving average of poll response latency (ms).
* `null` seeds with the first sample. Pure — the caller threads the prior
* value back in. */
export function updateLatencyEwma(prev: number | null, sampleMs: number): number {
return prev === null ? sampleMs : LATENCY_ALPHA * sampleMs + (1 - LATENCY_ALPHA) * prev;
}

/** Hysteresis decision for the "backend is degraded" flag. Enters when the
* smoothed latency is clearly high OR consecutive polls keep getting
* superseded (we're polling faster than the backend can answer); exits only
* when latency is clearly low AND polls are completing again. `ewmaMs === null`
* (no sample yet) leaves the current state unchanged. Pure. */
export function nextDegraded(
current: boolean,
ewmaMs: number | null,
supersedes: number,
): boolean {
if (current) {
const recovered = supersedes === 0 && ewmaMs !== null && ewmaMs < DEGRADED_EXIT_MS;
return !recovered;
}
if (supersedes >= DEGRADED_SUPERSEDE_COUNT) return true;
return ewmaMs !== null && ewmaMs > DEGRADED_ENTER_MS;
}

/**
* Pure tier selector for the /api/state poll cadence. Pre-hydration / empty
* window list falls through to the user-configured cadence so the first tick
Expand All @@ -24,13 +70,19 @@ export const INPUT_ACTIVE_MIN_MS = 1500;
* When `inputActive` is true (user is mid-burst of typing into a non-xterm
* text input), the chosen interval is clamped to `INPUT_ACTIVE_MIN_MS` so
* polling renders don't compete with keystroke handling (THI-138).
*
* When `degraded` is true (the backend is responding slowly — see
* `nextDegraded`), the cadence is floored at `DEGRADED_POLL_MS`, overriding
* the modal/active/input tiers, so a struggling backend isn't piled with
* requests it can't answer. Never *lowers* an already-slower interval.
*/
export function pickPollInterval(
hasOpenModal: boolean,
windows: Window[],
configured: number,
modalOpenMs: number,
inputActive: boolean = false,
degraded: boolean = false,
): number {
const base = (() => {
if (hasOpenModal) return modalOpenMs;
Expand All @@ -48,5 +100,6 @@ export function pickPollInterval(
return Math.max(8000, configured * 2);
})();

return inputActive ? Math.max(base, INPUT_ACTIVE_MIN_MS) : base;
const withInput = inputActive ? Math.max(base, INPUT_ACTIVE_MIN_MS) : base;
return degraded ? Math.max(withInput, DEGRADED_POLL_MS) : withInput;
}
Loading