fix(evals): contain completion-harness task paths and validate timeout (#1049)

* fix(evals): contain completion-harness task paths and validate timeout

Confine custom task-set fixture/script/verify references to the harness directory, mark the version-controlled grading trust boundary, reject degenerate timeout values, and re-record the baseline against the current tree.

* fix(evals): align baseline and totals with gate-suspension signal

Carry the gate-suspension total through schema, aggregation, and summary so the checked-in baseline validates. Add a regression test asserting the frozen baseline against the current report schema.

* fix(evals): label estimated turns and failed tool calls honestly (#1062)

Author: TheGreatAxios

evals/completion/baseline-2026-09-14.json

@@ -1,9 +1,9 @@
 {
   "harness": "completion-baseline",
   "version": 1,
-  "startedAt": "2026-09-14T07:30:17.369Z",
-  "finishedAt": "2026-09-14T07:30:19.083Z",
-  "commitSha": "b509ed3edb0db5368eb190528d3beaee8895837b",
+  "startedAt": "2026-09-14T20:11:37.538Z",
+  "finishedAt": "2026-09-14T20:11:41.269Z",
+  "commitSha": "b1bace7e4a6c469050df048d53e7d1dea4b89ab6",
   "provider": "stub-scripted",
   "model": "completion-baseline-v1",
   "repeats": 2,
@@ -25,8 +25,8 @@
       "doomLoopInterventions": 0,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 163,
-      "verifyDurationMs": 149,
+      "agentDurationMs": 567,
+      "verifyDurationMs": 196,
       "verifyExitCode": 0,
       "overBudget": false
     },
@@ -45,8 +45,8 @@
       "doomLoopInterventions": 0,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 62,
-      "verifyDurationMs": 141,
+      "agentDurationMs": 167,
+      "verifyDurationMs": 151,
       "verifyExitCode": 0,
       "overBudget": false
     },
@@ -65,8 +65,8 @@
       "doomLoopInterventions": 0,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 131,
-      "verifyDurationMs": 135,
+      "agentDurationMs": 72,
+      "verifyDurationMs": 188,
       "verifyExitCode": 0,
       "overBudget": false
     },
@@ -85,8 +85,8 @@
       "doomLoopInterventions": 0,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 66,
-      "verifyDurationMs": 127,
+      "agentDurationMs": 68,
+      "verifyDurationMs": 164,
       "verifyExitCode": 0,
       "overBudget": false
     },
@@ -105,8 +105,8 @@
       "doomLoopInterventions": 0,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 30,
-      "verifyDurationMs": 72,
+      "agentDurationMs": 25,
+      "verifyDurationMs": 99,
       "verifyExitCode": 1,
       "overBudget": false
     },
@@ -125,8 +125,8 @@
       "doomLoopInterventions": 0,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 22,
-      "verifyDurationMs": 65,
+      "agentDurationMs": 27,
+      "verifyDurationMs": 86,
       "verifyExitCode": 1,
       "overBudget": false
     },
@@ -145,8 +145,8 @@
       "doomLoopInterventions": 1,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 95,
-      "verifyDurationMs": 16,
+      "agentDurationMs": 318,
+      "verifyDurationMs": 30,
       "verifyExitCode": 2,
       "overBudget": false,
       "error": "reactor error: Doom loop detected: an identical tool batch (read_file) executed 3 times consecutively"
@@ -166,8 +166,8 @@
       "doomLoopInterventions": 1,
       "thrashInterventions": 0,
       "gateSuspensions": 0,
-      "agentDurationMs": 65,
-      "verifyDurationMs": 17,
+      "agentDurationMs": 332,
+      "verifyDurationMs": 52,
       "verifyExitCode": 2,
       "overBudget": false,
       "error": "reactor error: Doom loop detected: an identical tool batch (read_file) executed 3 times consecutively"
@@ -179,10 +179,11 @@
     "completedRuns": 4,
     "completionRate": 0.5,
     "meanTurnsToCompletion": 3,
-    "meanAgentDurationMs": 79.25,
+    "meanAgentDurationMs": 197,
     "totalRetries": 0,
     "totalCompactionEvents": 0,
     "totalDoomLoopInterventions": 2,
-    "totalThrashInterventions": 0
+    "totalThrashInterventions": 0,
+    "totalGateSuspensions": 0
   }
 }

evals/completion/lib.test.ts

@@ -1,11 +1,18 @@
 import { describe, expect, test } from "bun:test";
+import { readFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
 import {
   CompletionReport,
+  REPORT_VERSION,
+  assertTaskSetContained,
   computeTotals,
   formatSummary,
   isCompletedRun,
+  migrateLegacyCompletionReport,
   parseResponderScript,
   parseTaskSetFile,
+  resolveTaskRelativePath,
   TaskResult,
   type CompletionReport as CompletionReportType,
   type TaskResult as TaskResultType,
@@ -20,9 +27,9 @@ const result = (overrides: Partial<TaskResultType> = {}): TaskResultType =>
     completed: true,
     runStatus: "completed",
     turnsUsed: 3,
+    turnsEstimated: false,
     toolCallCount: 2,
     failedToolCalls: 0,
-    retryCount: 0,
     compactionEvents: 0,
     doomLoopInterventions: 0,
     thrashInterventions: 0,
@@ -47,18 +54,19 @@ describe("completion predicate", () => {
 describe("completion totals", () => {
   test("mixed outcomes yield a fractional rate and completion-only turn mean", () => {
     const totals = computeTotals([
-      result({ turnsUsed: 3, agentDurationMs: 1000 }),
+      result({ turnsUsed: 3, agentDurationMs: 1000, verifyDurationMs: 100 }),
       result({
         taskId: "stall-read",
         profile: "stall",
         completed: false,
         runStatus: "failed",
         turnsUsed: 4,
         toolCallCount: 4,
-        retryCount: 1,
+        failedToolCalls: 1,
         doomLoopInterventions: 1,
         verifyExitCode: 1,
         agentDurationMs: 3000,
+        verifyDurationMs: 300,
       }),
     ]);
     expect(totals.tasksTotal).toBe(2);
@@ -67,7 +75,8 @@ describe("completion totals", () => {
     expect(totals.completionRate).toBe(0.5);
     expect(totals.meanTurnsToCompletion).toBe(3);
     expect(totals.meanAgentDurationMs).toBe(2000);
-    expect(totals.totalRetries).toBe(1);
+    expect(totals.meanVerifyDurationMs).toBe(200);
+    expect(totals.totalFailedToolCalls).toBe(1);
     expect(totals.totalDoomLoopInterventions).toBe(1);
   });
 
@@ -76,6 +85,8 @@ describe("completion totals", () => {
     expect(totals.completionRate).toBe(0);
     expect(totals.meanTurnsToCompletion).toBe(0);
     expect(totals.meanAgentDurationMs).toBe(0);
+    expect(totals.meanVerifyDurationMs).toBe(0);
+    expect(totals.totalFailedToolCalls).toBe(0);
   });
 
   test("aggregates gate suspensions across runs", () => {
@@ -117,15 +128,68 @@ describe("boundary parsing", () => {
   });
 });
 
+describe("task path containment", () => {
+  const root = "/repo/evals/completion";
+
+  test("keeps relative fixture paths inside the harness directory", () => {
+    expect(resolveTaskRelativePath(root, "tasks/sum-fix/fixture")).toBe(
+      "/repo/evals/completion/tasks/sum-fix/fixture",
+    );
+  });
+
+  test("rejects absolute escapes", () => {
+    expect(() => resolveTaskRelativePath(root, "/etc/passwd")).toThrow(
+      /escapes the harness directory/,
+    );
+  });
+
+  test("rejects dot-dot escapes outside the harness directory", () => {
+    expect(() => resolveTaskRelativePath(root, "../capability/lib.ts")).toThrow(
+      /escapes the harness directory/,
+    );
+    expect(() =>
+      resolveTaskRelativePath(root, "tasks/../../package.json"),
+    ).toThrow(/escapes the harness directory/);
+  });
+
+  test("rejects an empty reference", () => {
+    expect(() => resolveTaskRelativePath(root, "  ")).toThrow(
+      /escapes the harness directory/,
+    );
+  });
+
+  test("assertTaskSetContained rejects a task set with an escaped grader", () => {
+    const taskSet = parseTaskSetFile({
+      version: 1,
+      note: "x",
+      tasks: [
+        {
+          id: "evil",
+          title: "evil",
+          profile: "solve",
+          prompt: "p",
+          fixture: "tasks/sum-fix/fixture",
+          script: "tasks/sum-fix/script.json",
+          verify: "/tmp/evil.sh",
+          maxTurns: 1,
+        },
+      ],
+    });
+    expect(() => assertTaskSetContained(taskSet, root)).toThrow(
+      /escapes the harness directory/,
+    );
+  });
+});
+
 describe("human summary", () => {
   test("names the harness, provenance, rate, and every run", () => {
     const results = [
       result(),
-      result({ taskId: "stall-read", completed: false }),
+      result({ taskId: "stall-read", completed: false, turnsEstimated: true }),
     ];
     const report: CompletionReportType = CompletionReport.assert({
       harness: "completion-baseline",
-      version: 1,
+      version: REPORT_VERSION,
       startedAt: "2026-09-14T00:00:00.000Z",
       finishedAt: "2026-09-14T00:01:00.000Z",
       commitSha: "deadbeef",
@@ -141,8 +205,106 @@ describe("human summary", () => {
     expect(summary).toContain("completion rate 50.0% (1/2 runs)");
     expect(summary).toContain("deadbeef");
     expect(summary).toContain("stub-scripted");
+    expect(summary).toContain("mean verify time 300ms");
+    expect(summary).toContain("failed tool calls 0");
+    expect(summary).not.toContain("retries");
     expect(summary).toContain("version-endpoint r0: complete");
     expect(summary).toContain("stall-read r0: incomplete");
+    expect(summary).toContain("turns 3 (estimated)");
+    expect(summary).toContain("verify 300ms");
+  });
+});
+
+describe("legacy v1 migration", () => {
+  const legacyResult = (overrides: Record<string, unknown> = {}) => ({
+    taskId: "version-endpoint",
+    title: "Add GET /version",
+    profile: "solve",
+    repeat: 0,
+    completed: true,
+    runStatus: "completed",
+    turnsUsed: 3,
+    toolCallCount: 2,
+    failedToolCalls: 0,
+    retryCount: 0,
+    compactionEvents: 0,
+    doomLoopInterventions: 0,
+    thrashInterventions: 0,
+    gateSuspensions: 0,
+    agentDurationMs: 1200,
+    verifyDurationMs: 300,
+    verifyExitCode: 0,
+    overBudget: false,
+    ...overrides,
+  });
+
+  const legacyReport = (results: Record<string, unknown>[]) => ({
+    harness: "completion-baseline",
+    version: 1,
+    startedAt: "2026-09-14T00:00:00.000Z",
+    finishedAt: "2026-09-14T00:01:00.000Z",
+    commitSha: "deadbeef",
+    provider: "stub-scripted",
+    model: "completion-baseline-v1",
+    repeats: 1,
+    taskSetVersion: 1,
+    taskIds: ["version-endpoint"],
+    results,
+    totals: {
+      tasksTotal: 1,
+      runsTotal: results.length,
+      completedRuns: 1,
+      completionRate: 1,
+      meanTurnsToCompletion: 3,
+      meanAgentDurationMs: 1200,
+      totalRetries: 0,
+      totalCompactionEvents: 0,
+      totalDoomLoopInterventions: 0,
+      totalThrashInterventions: 0,
+      totalGateSuspensions: 0,
+    },
+  });
+
+  test("drops the mislabeled retry count and aggregates verify durations", () => {
+    const migrated = migrateLegacyCompletionReport(
+      legacyReport([legacyResult(), legacyResult({ repeat: 1 })]),
+    );
+    expect(migrated.version).toBe(REPORT_VERSION);
+    expect(migrated.totals.totalFailedToolCalls).toBe(0);
+    expect(migrated.totals.meanVerifyDurationMs).toBe(300);
+    for (const migratedResult of migrated.results) {
+      expect("retryCount" in migratedResult).toBe(false);
+      expect("turnsEstimated" in migratedResult).toBe(false);
+    }
+  });
+
+  test("rejects a legacy result whose retry count is not a failure count", () => {
+    expect(() =>
+      migrateLegacyCompletionReport(
+        legacyReport([legacyResult({ retryCount: 2, failedToolCalls: 1 })]),
+      ),
+    ).toThrow(/mislabels retries/);
+  });
+
+  test("rejects a non-v1 report", () => {
+    const payload = legacyReport([legacyResult()]);
+    payload.version = REPORT_VERSION;
+    expect(() => migrateLegacyCompletionReport(payload)).toThrow(
+      /Only v1 reports can be migrated/,
+    );
+  });
+});
+
+describe("checked-in baseline", () => {
+  test("the frozen v1 baseline migrates to the current report schema", async () => {
+    const dir = dirname(fileURLToPath(import.meta.url));
+    const raw = await readFile(join(dir, "baseline-2026-09-14.json"), "utf8");
+    const report = migrateLegacyCompletionReport(JSON.parse(raw));
+    expect(report.totals.runsTotal).toBe(report.results.length);
+    expect(report.totals.runsTotal).toBe(8);
+    expect(report.totals.completionRate).toBe(0.5);
+    expect(report.totals.totalFailedToolCalls).toBe(0);
+    expect(report.totals.meanVerifyDurationMs).toBe(120.75);
   });
 
   test("prints the aggregated gate suspensions", () => {