test(eval): add 002-dead-config-field functional test case

eval/code/cases/002-dead-config-field/annotations.yaml

@@ -0,0 +1,40 @@
+state: open
+
+expected_files:
+  - config/config.go
+  - config/fields.go
+  - config/config_test.go
+
+labels:
+  forbidden: []
+
+# Consumed by eval.yaml's removed_symbols judge: each symbol must appear
+# only in deletion lines of the captured PR diff. Both the Go identifier
+# and its YAML key must vanish — a fix that drops the struct field but
+# leaves the SetField case, a test assertion, or the raw YAML literal in
+# TestLoad would keep one of these in a non-deletion line and fail.
+removed_symbols:
+  - VerboseLogging
+  - verbose_logging
+
+# Code agent budgets (sandbox work + retries). Unmeasured ceiling: this case
+# has no CI run of its own yet — re-derive from its first run. Values mirror
+# 001-fix-add's ceiling (60 turns / $4.00) and eval.yaml's max_budget_usd.
+# For reference, 001 (a trivial fixture) observed 12 turns / $2.12 (CI run
+# 29424512121) and 35 turns / $0.98 (CI run 30166455238); this cross-file
+# task is harder, so its ceiling is set no lower.
+max_turns: 60
+max_cost_usd: 4.00
+
+# Human reference only; not consumed by judges (unlike triage/review quality).
+code_expectations: |
+  The repo has a dead config field: Config.VerboseLogging is declared, defaulted,
+  parsed, and tested, but never read by any consumer (runner/runner.go doesn't
+  use it). A successful run creates a PR that removes VerboseLogging from: the
+  struct definition (config.go), the Defaults() return value (config.go), the
+  SetField() switch case (fields.go), and all test assertions (config_test.go).
+  Tests must still pass after removal. This case tests cross-file dead-code
+  removal — tracing symbol references across multiple files to determine what
+  to change, not just fixing a single line. The raw YAML literal in TestLoad
+  (verbose_logging: true) must also be removed — SetField rejects unknown keys,
+  so leaving it breaks Load().

eval/code/cases/002-dead-config-field/input.yaml

@@ -0,0 +1,25 @@
+forge: github
+fixture:
+  type: issue
+  title: "config: VerboseLogging field appears unused"
+  body: |
+    ## Bug Report
+
+    **What happened:**
+    We have a `VerboseLogging` field in the config that I can't find being used anywhere
+    in the actual runner logic. I added it a while back thinking we'd need it for debug
+    output, but I'm not sure we ever wired it up to anything.
+
+    **Expected behavior:**
+    If it's truly unused, we should remove it to keep the config clean. But I want to
+    make sure I'm not missing something — maybe there's some code path that reads it
+    that I didn't spot?
+
+    **Additional context:**
+    - The field exists in `config/config.go`
+    - YAML key is `verbose_logging`
+    - I see tests for it, but those might just be testing the config parsing itself
+    - Haven't found where the runner actually checks this value, but I might be wrong
+
+    Can someone verify whether this is actually used? If not, please clean it up.
+    Make sure tests still pass after any changes.

eval/code/cases/002-dead-config-field/repo

@@ -0,0 +1 @@
+../../repos/taskrunner

eval/code/eval.yaml

@@ -2,10 +2,13 @@ name: code-eval
 description: >
   Functional test of the fullsend code agent pipeline (pre → sandbox → post).
   Validates that the post-script opens a PR touching the expected files for
-  a small issue — an end-to-end pipeline guard, not a correctness check.
-  No judge inspects the PR's diff content or runs the fixture's tests
-  against it (see annotations.yaml: "Primary signal is pr_created"), so a PR
-  that opens but contains a cosmetic or outright wrong fix still passes.
+  a small issue — an end-to-end pipeline guard, not a full correctness
+  check. Cases may declare removed_symbols in annotations.yaml; the
+  removed_symbols judge then verifies those symbols appear only in deletion
+  lines of the captured PR diff. Beyond that, no judge runs the fixture's
+  tests against the PR, so a fix that compiles but misbehaves can still
+  pass — and cases with no removed_symbols get file-touch checking only
+  (see annotations.yaml: "Primary signal is pr_created").
   Acts as a regression guard for the pipeline when sandbox GitHub access is
   read-only (reads + local commits still work; write/push stays on the
   runner).
@@ -65,9 +68,15 @@ execution:
     # timeout envelope; matches eval/fix/eval.yaml's value instead (see
     # execution.timeout above).
     #
-    # This fixture is a 2-line arithmetic bug and completes in well under a
-    # minute in practice, so this number essentially never fires. If a
-    # genuine hang did occur: a single-iteration hang gets caught here and
+    # Two cases now share this window: 001-fix-add (a 2-line arithmetic bug
+    # that completes in well under a minute) and 002-dead-config-field (a
+    # cross-file dead-config removal with a larger 60-turn / $4.00 budget).
+    # Neither is expected to approach 1700s in practice: these small fixtures
+    # finish far under it. Note the 1700s outer backstop — not the larger
+    # 2100s per-iteration agent budget — is the binding limit, so a genuine
+    # hang is caught by 1700s first. 002 has no observed CI runtime yet, so
+    # treat its headroom as an open risk to revisit once a real run exists.
+    # If a genuine hang did occur: a single-iteration hang gets caught here and
     # fullsend still writes a partial metrics.json before returning (see
     # writeMetricsJSON in run.go's error path), giving an inconclusive but
     # readable "metrics.json not found"-adjacent judge failure; a
@@ -177,6 +186,62 @@ judges:
           return False, f"Expected files missing from PRs: {missing} (changed: {sorted(changed)})"
       return True, f"All expected files present: {expected}"
 
+  - name: removed_symbols
+    description: >
+      Content-level check for removal cases: every symbol listed in
+      annotations.removed_symbols must appear in at least one deletion line
+      of the captured PR diff (output/pr-<num>.diff, written by
+      capture-fixture.sh) and in no added or context line — i.e. the symbol
+      is gone from every hunk the fix touched. Passes trivially when a case
+      declares no removed_symbols. Diff-scoped only: a symbol surviving in
+      a file the PR never touched is invisible here (expected_files covers
+      the known declaration sites), and no judge runs the fixture's tests.
+    check: |
+      import json
+      symbols = outputs.get("annotations", {}).get("removed_symbols") or []
+      if not symbols:
+          return True, "No removed_symbols declared"
+      raw = outputs["files"].get("output/fixture-state.json")
+      if not raw:
+          return False, "fixture-state.json not found — capture-fixture.sh did not run or failed"
+      state = json.loads(raw)
+      prs = [p for p in (state.get("pull_requests") or [])
+             if str(p.get("state", "")).upper() in ("OPEN", "MERGED")]
+      if not prs:
+          return False, "No open/merged PR to inspect"
+      failed = [p.get("number") for p in prs if p.get("diff_fetch_failed")]
+      if failed:
+          return False, f"Could not fetch diff for PR(s): {failed}"
+      diff_lines = []
+      for pr in prs:
+          chunk = outputs["files"].get(f"output/pr-{pr.get('number')}.diff")
+          if chunk is None:
+              return False, f"output/pr-{pr.get('number')}.diff not captured"
+          diff_lines.extend(chunk.splitlines())
+      # Skip diff metadata: file headers, hunk headers (whose trailing
+      # function context can legitimately mention the symbol), and index
+      # lines. Everything else is a deletion ("-"), addition ("+"), or
+      # unchanged context line — the symbol may only appear in deletions.
+      meta = ("--- ", "+++ ", "diff ", "index ", "@@")
+      problems = []
+      for sym in symbols:
+          deleted = 0
+          survivors = 0
+          for line in diff_lines:
+              if sym not in line or line.startswith(meta):
+                  continue
+              if line.startswith("-"):
+                  deleted += 1
+              else:
+                  survivors += 1
+          if survivors:
+              problems.append(f"{sym}: present in {survivors} non-deletion diff line(s)")
+          elif not deleted:
+              problems.append(f"{sym}: no deletion lines in diff")
+      if problems:
+          return False, "Symbols not fully removed: " + "; ".join(problems)
+      return True, f"All declared symbols removed cleanly: {symbols}"
+
   # forbidden_labels / max_turns / max_cost below are shared verbatim with
   # eval/fix/eval.yaml and eval/review/eval.yaml — update all three if changing.
   - name: forbidden_labels
@@ -237,6 +302,8 @@ thresholds:
     min_pass_rate: 1.0
   expected_files:
     min_pass_rate: 1.0
+  removed_symbols:
+    min_pass_rate: 1.0
   forbidden_labels:
     min_pass_rate: 1.0
   max_turns:

eval/code/repos/taskrunner/README.md

@@ -0,0 +1,3 @@
+# taskrunner
+
+A minimal task runner that reads a YAML config and executes registered tasks.

eval/code/repos/taskrunner/config/config.go

@@ -0,0 +1,47 @@
+package config
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/eval-org/taskrunner/config/internal/yaml"
+)
+
+// Config holds the task runner configuration.
+type Config struct {
+	// MaxRetries controls how many times a failed task is retried.
+	MaxRetries int `yaml:"max_retries"`
+
+	// Timeout is the per-task timeout in seconds.
+	Timeout int `yaml:"timeout"`
+
+	// VerboseLogging enables detailed debug output.
+	VerboseLogging bool `yaml:"verbose_logging"`
+
+	// Workers is the number of concurrent task workers.
+	Workers int `yaml:"workers"`
+}
+
+// Defaults returns a Config with sensible default values.
+func Defaults() Config {
+	return Config{
+		MaxRetries:     3,
+		Timeout:        60,
+		VerboseLogging: false,
+		Workers:        4,
+	}
+}
+
+// Load reads a YAML config file and returns a Config.
+// Missing fields are filled with defaults.
+func Load(path string) (Config, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return Config{}, fmt.Errorf("reading config %s: %w", path, err)
+	}
+	cfg := Defaults()
+	if err := yaml.Unmarshal(data, &cfg); err != nil {
+		return Config{}, fmt.Errorf("parsing config %s: %w", path, err)
+	}
+	return cfg, nil
+}

eval/code/repos/taskrunner/config/config_test.go

@@ -0,0 +1,103 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestDefaults(t *testing.T) {
+	cfg := Defaults()
+	if cfg.MaxRetries != 3 {
+		t.Errorf("MaxRetries = %d, want 3", cfg.MaxRetries)
+	}
+	if cfg.Timeout != 60 {
+		t.Errorf("Timeout = %d, want 60", cfg.Timeout)
+	}
+	if cfg.VerboseLogging != false {
+		t.Errorf("VerboseLogging = %v, want false", cfg.VerboseLogging)
+	}
+	if cfg.Workers != 4 {
+		t.Errorf("Workers = %d, want 4", cfg.Workers)
+	}
+}
+
+func TestLoad(t *testing.T) {
+	content := `max_retries: 5
+timeout: 120
+verbose_logging: true
+workers: 8
+`
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	if err := os.WriteFile(path, []byte(content), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	cfg, err := Load(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if cfg.MaxRetries != 5 {
+		t.Errorf("MaxRetries = %d, want 5", cfg.MaxRetries)
+	}
+	if cfg.Timeout != 120 {
+		t.Errorf("Timeout = %d, want 120", cfg.Timeout)
+	}
+	if cfg.VerboseLogging != true {
+		t.Errorf("VerboseLogging = %v, want true", cfg.VerboseLogging)
+	}
+	if cfg.Workers != 8 {
+		t.Errorf("Workers = %d, want 8", cfg.Workers)
+	}
+}
+
+func TestLoadPartial(t *testing.T) {
+	content := `timeout: 30
+`
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	if err := os.WriteFile(path, []byte(content), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	cfg, err := Load(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if cfg.MaxRetries != 3 {
+		t.Errorf("MaxRetries = %d, want 3 (default)", cfg.MaxRetries)
+	}
+	if cfg.Timeout != 30 {
+		t.Errorf("Timeout = %d, want 30", cfg.Timeout)
+	}
+	if cfg.VerboseLogging != false {
+		t.Errorf("VerboseLogging = %v, want false (default)", cfg.VerboseLogging)
+	}
+}
+
+func TestLoadMalformedLine(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	if err := os.WriteFile(path, []byte("not a valid line\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	_, err := Load(path)
+	if err == nil {
+		t.Error("Load() = nil error for malformed YAML, want error")
+	}
+}
+
+func TestLoadUnknownKey(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	if err := os.WriteFile(path, []byte("bogus_key: 1\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	_, err := Load(path)
+	if err == nil {
+		t.Error("Load() = nil error for unknown key, want error")
+	}
+}

eval/code/repos/taskrunner/config/fields.go

@@ -0,0 +1,40 @@
+package config
+
+import (
+	"fmt"
+
+	"github.com/eval-org/taskrunner/config/internal/yaml"
+)
+
+// SetField implements the configFields interface for the minimal YAML parser.
+func (c *Config) SetField(key, value string) error {
+	switch key {
+	case "max_retries":
+		v, err := yaml.ParseInt(value)
+		if err != nil {
+			return err
+		}
+		c.MaxRetries = v
+	case "timeout":
+		v, err := yaml.ParseInt(value)
+		if err != nil {
+			return err
+		}
+		c.Timeout = v
+	case "verbose_logging":
+		v, err := yaml.ParseBool(value)
+		if err != nil {
+			return err
+		}
+		c.VerboseLogging = v
+	case "workers":
+		v, err := yaml.ParseInt(value)
+		if err != nil {
+			return err
+		}
+		c.Workers = v
+	default:
+		return fmt.Errorf("unknown config key: %s", key)
+	}
+	return nil
+}