fix(evals): keep timeout status, cancel in-flight work, aggregate gate suspensions (#1047)

A run that times out after emitting a failure signal kept the timeout status instead of being recorded failed. The deadline now aborts the in-flight send and the runner quiesces the agent pump and collector before grading, so verify never runs on partial state. Gate suspensions are summed into completion totals and printed in the human summary instead of being collected and dropped.

Author: TheGreatAxios

evals/completion/lib.test.ts

@@ -77,6 +77,14 @@ describe("completion totals", () => {
     expect(totals.meanTurnsToCompletion).toBe(0);
     expect(totals.meanAgentDurationMs).toBe(0);
   });
+
+  test("aggregates gate suspensions across runs", () => {
+    const totals = computeTotals([
+      result({ gateSuspensions: 2 }),
+      result({ taskId: "stall-read", gateSuspensions: 3 }),
+    ]);
+    expect(totals.totalGateSuspensions).toBe(5);
+  });
 });
 
 describe("boundary parsing", () => {
@@ -136,4 +144,26 @@ describe("human summary", () => {
     expect(summary).toContain("version-endpoint r0: complete");
     expect(summary).toContain("stall-read r0: incomplete");
   });
+
+  test("prints the aggregated gate suspensions", () => {
+    const results = [
+      result({ gateSuspensions: 2 }),
+      result({ taskId: "stall-read", gateSuspensions: 3 }),
+    ];
+    const report: CompletionReportType = CompletionReport.assert({
+      harness: "completion-baseline",
+      version: 1,
+      startedAt: "2026-09-14T00:00:00.000Z",
+      finishedAt: "2026-09-14T00:01:00.000Z",
+      commitSha: "deadbeef",
+      provider: "stub-scripted",
+      model: "completion-baseline-v1",
+      repeats: 1,
+      taskSetVersion: 1,
+      taskIds: ["version-endpoint", "stall-read"],
+      results,
+      totals: computeTotals(results),
+    });
+    expect(formatSummary(report)).toContain("gate suspensions 5");
+  });
 });

evals/completion/lib.ts

@@ -85,6 +85,7 @@ export const CompletionTotals = type({
   totalCompactionEvents: "number.integer >= 0",
   totalDoomLoopInterventions: "number.integer >= 0",
   totalThrashInterventions: "number.integer >= 0",
+  totalGateSuspensions: "number.integer >= 0",
 });
 export type CompletionTotals = typeof CompletionTotals.infer;
 
@@ -158,6 +159,7 @@ export function computeTotals(
     totalThrashInterventions: sum(
       results.map((result) => result.thrashInterventions),
     ),
+    totalGateSuspensions: sum(results.map((result) => result.gateSuspensions)),
   };
 }
 
@@ -172,7 +174,7 @@ export function formatSummary(report: CompletionReport): string {
     `task set v${report.taskSetVersion}: ${report.taskIds.join(", ")}`,
     `completion rate ${formatRate(report.totals.completionRate)} (${report.totals.completedRuns}/${report.totals.runsTotal} runs)`,
     `mean turns to completion ${report.totals.meanTurnsToCompletion.toFixed(1)} mean agent time ${Math.round(report.totals.meanAgentDurationMs)}ms`,
-    `retries ${report.totals.totalRetries} compaction events ${report.totals.totalCompactionEvents} doom-loop interventions ${report.totals.totalDoomLoopInterventions} thrash interventions ${report.totals.totalThrashInterventions}`,
+    `retries ${report.totals.totalRetries} compaction events ${report.totals.totalCompactionEvents} doom-loop interventions ${report.totals.totalDoomLoopInterventions} thrash interventions ${report.totals.totalThrashInterventions} gate suspensions ${report.totals.totalGateSuspensions}`,
     "",
     ...report.results.map(
       (result) =>

scripts/eval-completion.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { withTimeout } from "./eval-completion.js";
+import { resolveRunStatus, withTimeout } from "./eval-completion.js";
 
 describe("withTimeout", () => {
   test("rejects a hung run after the timeout", async () => {
@@ -28,4 +28,47 @@ describe("withTimeout", () => {
       withTimeout(Promise.reject(new Error("boom")), 1000, "task"),
     ).rejects.toThrow("boom");
   });
+
+  test("invokes onTimeout when the deadline fires", async () => {
+    const hung = new Promise<never>(() => undefined);
+    let calls = 0;
+    const onTimeout = () => {
+      calls += 1;
+    };
+    const error = await withTimeout(hung, 50, "task", { onTimeout }).catch(
+      (err: unknown) => err,
+    );
+    expect(error).toBeInstanceOf(Error);
+    expect(calls).toBe(1);
+  });
+
+  test("skips onTimeout when the inner promise settles first", async () => {
+    let calls = 0;
+    const onTimeout = () => {
+      calls += 1;
+    };
+    await expect(
+      withTimeout(Promise.resolve("done"), 50, "task", { onTimeout }),
+    ).resolves.toBe("done");
+    const failing = Promise.reject(new Error("boom"));
+    const rejected = withTimeout(failing, 50, "task", { onTimeout });
+    await expect(rejected).rejects.toThrow("boom");
+    expect(calls).toBe(0);
+  });
+});
+
+describe("resolveRunStatus", () => {
+  test("timeout keeps status over a failure signal", () => {
+    const status = resolveRunStatus({ timedOut: true, failed: true });
+    expect(status).toBe("timeout");
+  });
+
+  test("resolves the remaining outcomes", () => {
+    const timeoutOnly = resolveRunStatus({ timedOut: true, failed: false });
+    const failedOnly = resolveRunStatus({ timedOut: false, failed: true });
+    const clean = resolveRunStatus({ timedOut: false, failed: false });
+    expect(timeoutOnly).toBe("timeout");
+    expect(failedOnly).toBe("failed");
+    expect(clean).toBe("completed");
+  });
 });

scripts/eval-completion.ts

@@ -117,6 +117,7 @@ export async function withTimeout<T>(
   promise: Promise<T>,
   ms: number,
   label: string,
+  opts?: { onTimeout?: () => void },
 ): Promise<T> {
   let timer: ReturnType<typeof setTimeout> | undefined;
   try {
@@ -126,17 +127,39 @@ export async function withTimeout<T>(
     return await Promise.race([
       promise,
       new Promise<never>((_, reject) => {
-        timer = setTimeout(
-          () => reject(new Error(`${label} timed out after ${ms}ms`)),
-          ms,
-        );
+        timer = setTimeout(() => {
+          // Cancel first so the caller's in-flight work settles instead of
+          // lingering past the deadline; the race rejects below regardless.
+          opts?.onTimeout?.();
+          reject(new Error(`${label} timed out after ${ms}ms`));
+        }, ms);
       }),
     ]);
   } finally {
     if (timer !== undefined) clearTimeout(timer);
   }
 }
 
+/**
+ * Timeout keeps status over failure signals: a run that timed out did not
+ * fail, it ran out of time — even when the partial stream already carries
+ * a failure event.
+ */
+export function resolveRunStatus(options: {
+  timedOut: boolean;
+  failed: boolean;
+}): RunStatus {
+  if (options.timedOut) return "timeout";
+  if (options.failed) return "failed";
+  return "completed";
+}
+
+// Backstop for the post-timeout quiesce below: the abort plus agent close
+// settle the live paths promptly, so this only bites when the mock pump
+// itself is stuck — and then it keeps a stuck pump from re-hanging the
+// harness at the deadline it just enforced.
+const SETTLE_GRACE_MS = 5_000;
+
 interface PersistedTurn {
   role: string;
 }
@@ -241,31 +264,46 @@ async function runTask(
         if (turnComplete && event.type === "message.run.ended") return;
       }
     })().catch(() => undefined);
+    // Abort the in-flight send when the deadline fires so its promise
+    // settles instead of lingering past the timeout.
+    const controller = new AbortController();
+    const runWork = (async () => {
+      const sendResult = await Promise.all([
+        session.agent
+          .send(task.prompt, { signal: controller.signal })
+          .then((result) => {
+            turnComplete = true;
+            return result;
+          }),
+        session.harness.run({ wallClockBudgetMs: Infinity }),
+        collect,
+      ]).then(([result]) => result);
+      if (sendResult.type !== "reply") {
+        throw new Error(`unexpected send outcome: ${sendResult.type}`);
+      }
+    })();
+    let timedOut = false;
     try {
-      await withTimeout(
-        (async () => {
-          const sendResult = await Promise.all([
-            session.agent.send(task.prompt).then((result) => {
-              turnComplete = true;
-              return result;
-            }),
-            session.harness.run({ wallClockBudgetMs: Infinity }),
-            collect,
-          ]).then(([result]) => result);
-          if (sendResult.type !== "reply") {
-            throw new Error(`unexpected send outcome: ${sendResult.type}`);
-          }
-        })(),
-        timeoutMs,
-        `task ${task.id}`,
-      );
+      await withTimeout(runWork, timeoutMs, `task ${task.id}`, {
+        onTimeout: () => controller.abort(),
+      });
     } catch (err) {
-      runStatus =
-        err instanceof Error && err.message.includes("timed out")
-          ? "timeout"
-          : "failed";
+      timedOut = err instanceof Error && err.message.includes("timed out");
       error = err instanceof Error ? err.message : String(err);
     }
+    if (timedOut) {
+      // Quiesce before grading: the aborted send settles at once, but the
+      // pump and collector lag behind. Closing the agent aborts the reactor
+      // and terminates the stream so the collector settles, then awaiting
+      // the inner work keeps verify below off partial state.
+      await session.agent.close().catch(() => undefined);
+      await Promise.race([
+        runWork.catch(() => undefined),
+        new Promise<void>((resolve) => {
+          setTimeout(resolve, SETTLE_GRACE_MS);
+        }),
+      ]);
+    }
     const agentDurationMs = Date.now() - agentStart;
     const signals = deriveSignals(events);
     // A guard trip rejects send() without a run.ended event; recover the
@@ -277,7 +315,10 @@ async function runTask(
     ) {
       signals.doomLoopInterventions = 1;
     }
-    if (signals.runFailed) runStatus = "failed";
+    runStatus = resolveRunStatus({
+      timedOut,
+      failed: error !== undefined || signals.runFailed,
+    });
 
     const verifyStart = Date.now();
     const verify = spawnSync("bash", [resolve(COMPLETION_ROOT, task.verify)], {