diff --git a/eval/code/cases/002-dead-config-field/annotations.yaml b/eval/code/cases/002-dead-config-field/annotations.yaml new file mode 100644 index 00000000..4773c5f7 --- /dev/null +++ b/eval/code/cases/002-dead-config-field/annotations.yaml @@ -0,0 +1,40 @@ +state: open + +expected_files: + - config/config.go + - config/fields.go + - config/config_test.go + +labels: + forbidden: [] + +# Consumed by eval.yaml's removed_symbols judge: each symbol must appear +# only in deletion lines of the captured PR diff. Both the Go identifier +# and its YAML key must vanish — a fix that drops the struct field but +# leaves the SetField case, a test assertion, or the raw YAML literal in +# TestLoad would keep one of these in a non-deletion line and fail. +removed_symbols: + - VerboseLogging + - verbose_logging + +# Code agent budgets (sandbox work + retries). Unmeasured ceiling: this case +# has no CI run of its own yet — re-derive from its first run. Values mirror +# 001-fix-add's ceiling (60 turns / $4.00) and eval.yaml's max_budget_usd. +# For reference, 001 (a trivial fixture) observed 12 turns / $2.12 (CI run +# 29424512121) and 35 turns / $0.98 (CI run 30166455238); this cross-file +# task is harder, so its ceiling is set no lower. +max_turns: 60 +max_cost_usd: 4.00 + +# Human reference only; not consumed by judges (unlike triage/review quality). +code_expectations: | + The repo has a dead config field: Config.VerboseLogging is declared, defaulted, + parsed, and tested, but never read by any consumer (runner/runner.go doesn't + use it). A successful run creates a PR that removes VerboseLogging from: the + struct definition (config.go), the Defaults() return value (config.go), the + SetField() switch case (fields.go), and all test assertions (config_test.go). + Tests must still pass after removal. This case tests cross-file dead-code + removal — tracing symbol references across multiple files to determine what + to change, not just fixing a single line. The raw YAML literal in TestLoad + (verbose_logging: true) must also be removed — SetField rejects unknown keys, + so leaving it breaks Load(). diff --git a/eval/code/cases/002-dead-config-field/input.yaml b/eval/code/cases/002-dead-config-field/input.yaml new file mode 100644 index 00000000..e2c330e1 --- /dev/null +++ b/eval/code/cases/002-dead-config-field/input.yaml @@ -0,0 +1,25 @@ +forge: github +fixture: + type: issue + title: "config: VerboseLogging field appears unused" + body: | + ## Bug Report + + **What happened:** + We have a `VerboseLogging` field in the config that I can't find being used anywhere + in the actual runner logic. I added it a while back thinking we'd need it for debug + output, but I'm not sure we ever wired it up to anything. + + **Expected behavior:** + If it's truly unused, we should remove it to keep the config clean. But I want to + make sure I'm not missing something — maybe there's some code path that reads it + that I didn't spot? + + **Additional context:** + - The field exists in `config/config.go` + - YAML key is `verbose_logging` + - I see tests for it, but those might just be testing the config parsing itself + - Haven't found where the runner actually checks this value, but I might be wrong + + Can someone verify whether this is actually used? If not, please clean it up. + Make sure tests still pass after any changes. diff --git a/eval/code/cases/002-dead-config-field/repo b/eval/code/cases/002-dead-config-field/repo new file mode 120000 index 00000000..2b3908fe --- /dev/null +++ b/eval/code/cases/002-dead-config-field/repo @@ -0,0 +1 @@ +../../repos/taskrunner \ No newline at end of file diff --git a/eval/code/eval.yaml b/eval/code/eval.yaml index 75d262f1..9e98dabd 100644 --- a/eval/code/eval.yaml +++ b/eval/code/eval.yaml @@ -2,10 +2,13 @@ name: code-eval description: > Functional test of the fullsend code agent pipeline (pre → sandbox → post). Validates that the post-script opens a PR touching the expected files for - a small issue — an end-to-end pipeline guard, not a correctness check. - No judge inspects the PR's diff content or runs the fixture's tests - against it (see annotations.yaml: "Primary signal is pr_created"), so a PR - that opens but contains a cosmetic or outright wrong fix still passes. + a small issue — an end-to-end pipeline guard, not a full correctness + check. Cases may declare removed_symbols in annotations.yaml; the + removed_symbols judge then verifies those symbols appear only in deletion + lines of the captured PR diff. Beyond that, no judge runs the fixture's + tests against the PR, so a fix that compiles but misbehaves can still + pass — and cases with no removed_symbols get file-touch checking only + (see annotations.yaml: "Primary signal is pr_created"). Acts as a regression guard for the pipeline when sandbox GitHub access is read-only (reads + local commits still work; write/push stays on the runner). @@ -65,9 +68,15 @@ execution: # timeout envelope; matches eval/fix/eval.yaml's value instead (see # execution.timeout above). # - # This fixture is a 2-line arithmetic bug and completes in well under a - # minute in practice, so this number essentially never fires. If a - # genuine hang did occur: a single-iteration hang gets caught here and + # Two cases now share this window: 001-fix-add (a 2-line arithmetic bug + # that completes in well under a minute) and 002-dead-config-field (a + # cross-file dead-config removal with a larger 60-turn / $4.00 budget). + # Neither is expected to approach 1700s in practice: these small fixtures + # finish far under it. Note the 1700s outer backstop — not the larger + # 2100s per-iteration agent budget — is the binding limit, so a genuine + # hang is caught by 1700s first. 002 has no observed CI runtime yet, so + # treat its headroom as an open risk to revisit once a real run exists. + # If a genuine hang did occur: a single-iteration hang gets caught here and # fullsend still writes a partial metrics.json before returning (see # writeMetricsJSON in run.go's error path), giving an inconclusive but # readable "metrics.json not found"-adjacent judge failure; a @@ -177,6 +186,62 @@ judges: return False, f"Expected files missing from PRs: {missing} (changed: {sorted(changed)})" return True, f"All expected files present: {expected}" + - name: removed_symbols + description: > + Content-level check for removal cases: every symbol listed in + annotations.removed_symbols must appear in at least one deletion line + of the captured PR diff (output/pr-.diff, written by + capture-fixture.sh) and in no added or context line — i.e. the symbol + is gone from every hunk the fix touched. Passes trivially when a case + declares no removed_symbols. Diff-scoped only: a symbol surviving in + a file the PR never touched is invisible here (expected_files covers + the known declaration sites), and no judge runs the fixture's tests. + check: | + import json + symbols = outputs.get("annotations", {}).get("removed_symbols") or [] + if not symbols: + return True, "No removed_symbols declared" + raw = outputs["files"].get("output/fixture-state.json") + if not raw: + return False, "fixture-state.json not found — capture-fixture.sh did not run or failed" + state = json.loads(raw) + prs = [p for p in (state.get("pull_requests") or []) + if str(p.get("state", "")).upper() in ("OPEN", "MERGED")] + if not prs: + return False, "No open/merged PR to inspect" + failed = [p.get("number") for p in prs if p.get("diff_fetch_failed")] + if failed: + return False, f"Could not fetch diff for PR(s): {failed}" + diff_lines = [] + for pr in prs: + chunk = outputs["files"].get(f"output/pr-{pr.get('number')}.diff") + if chunk is None: + return False, f"output/pr-{pr.get('number')}.diff not captured" + diff_lines.extend(chunk.splitlines()) + # Skip diff metadata: file headers, hunk headers (whose trailing + # function context can legitimately mention the symbol), and index + # lines. Everything else is a deletion ("-"), addition ("+"), or + # unchanged context line — the symbol may only appear in deletions. + meta = ("--- ", "+++ ", "diff ", "index ", "@@") + problems = [] + for sym in symbols: + deleted = 0 + survivors = 0 + for line in diff_lines: + if sym not in line or line.startswith(meta): + continue + if line.startswith("-"): + deleted += 1 + else: + survivors += 1 + if survivors: + problems.append(f"{sym}: present in {survivors} non-deletion diff line(s)") + elif not deleted: + problems.append(f"{sym}: no deletion lines in diff") + if problems: + return False, "Symbols not fully removed: " + "; ".join(problems) + return True, f"All declared symbols removed cleanly: {symbols}" + # forbidden_labels / max_turns / max_cost below are shared verbatim with # eval/fix/eval.yaml and eval/review/eval.yaml — update all three if changing. - name: forbidden_labels @@ -237,6 +302,8 @@ thresholds: min_pass_rate: 1.0 expected_files: min_pass_rate: 1.0 + removed_symbols: + min_pass_rate: 1.0 forbidden_labels: min_pass_rate: 1.0 max_turns: diff --git a/eval/code/repos/taskrunner/README.md b/eval/code/repos/taskrunner/README.md new file mode 100644 index 00000000..a9a80f5c --- /dev/null +++ b/eval/code/repos/taskrunner/README.md @@ -0,0 +1,3 @@ +# taskrunner + +A minimal task runner that reads a YAML config and executes registered tasks. diff --git a/eval/code/repos/taskrunner/config/config.go b/eval/code/repos/taskrunner/config/config.go new file mode 100644 index 00000000..4861bd2c --- /dev/null +++ b/eval/code/repos/taskrunner/config/config.go @@ -0,0 +1,47 @@ +package config + +import ( + "fmt" + "os" + + "github.com/eval-org/taskrunner/config/internal/yaml" +) + +// Config holds the task runner configuration. +type Config struct { + // MaxRetries controls how many times a failed task is retried. + MaxRetries int `yaml:"max_retries"` + + // Timeout is the per-task timeout in seconds. + Timeout int `yaml:"timeout"` + + // VerboseLogging enables detailed debug output. + VerboseLogging bool `yaml:"verbose_logging"` + + // Workers is the number of concurrent task workers. + Workers int `yaml:"workers"` +} + +// Defaults returns a Config with sensible default values. +func Defaults() Config { + return Config{ + MaxRetries: 3, + Timeout: 60, + VerboseLogging: false, + Workers: 4, + } +} + +// Load reads a YAML config file and returns a Config. +// Missing fields are filled with defaults. +func Load(path string) (Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return Config{}, fmt.Errorf("reading config %s: %w", path, err) + } + cfg := Defaults() + if err := yaml.Unmarshal(data, &cfg); err != nil { + return Config{}, fmt.Errorf("parsing config %s: %w", path, err) + } + return cfg, nil +} diff --git a/eval/code/repos/taskrunner/config/config_test.go b/eval/code/repos/taskrunner/config/config_test.go new file mode 100644 index 00000000..2043a078 --- /dev/null +++ b/eval/code/repos/taskrunner/config/config_test.go @@ -0,0 +1,103 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDefaults(t *testing.T) { + cfg := Defaults() + if cfg.MaxRetries != 3 { + t.Errorf("MaxRetries = %d, want 3", cfg.MaxRetries) + } + if cfg.Timeout != 60 { + t.Errorf("Timeout = %d, want 60", cfg.Timeout) + } + if cfg.VerboseLogging != false { + t.Errorf("VerboseLogging = %v, want false", cfg.VerboseLogging) + } + if cfg.Workers != 4 { + t.Errorf("Workers = %d, want 4", cfg.Workers) + } +} + +func TestLoad(t *testing.T) { + content := `max_retries: 5 +timeout: 120 +verbose_logging: true +workers: 8 +` + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + + if cfg.MaxRetries != 5 { + t.Errorf("MaxRetries = %d, want 5", cfg.MaxRetries) + } + if cfg.Timeout != 120 { + t.Errorf("Timeout = %d, want 120", cfg.Timeout) + } + if cfg.VerboseLogging != true { + t.Errorf("VerboseLogging = %v, want true", cfg.VerboseLogging) + } + if cfg.Workers != 8 { + t.Errorf("Workers = %d, want 8", cfg.Workers) + } +} + +func TestLoadPartial(t *testing.T) { + content := `timeout: 30 +` + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + + if cfg.MaxRetries != 3 { + t.Errorf("MaxRetries = %d, want 3 (default)", cfg.MaxRetries) + } + if cfg.Timeout != 30 { + t.Errorf("Timeout = %d, want 30", cfg.Timeout) + } + if cfg.VerboseLogging != false { + t.Errorf("VerboseLogging = %v, want false (default)", cfg.VerboseLogging) + } +} + +func TestLoadMalformedLine(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte("not a valid line\n"), 0644); err != nil { + t.Fatal(err) + } + _, err := Load(path) + if err == nil { + t.Error("Load() = nil error for malformed YAML, want error") + } +} + +func TestLoadUnknownKey(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte("bogus_key: 1\n"), 0644); err != nil { + t.Fatal(err) + } + _, err := Load(path) + if err == nil { + t.Error("Load() = nil error for unknown key, want error") + } +} diff --git a/eval/code/repos/taskrunner/config/fields.go b/eval/code/repos/taskrunner/config/fields.go new file mode 100644 index 00000000..00964605 --- /dev/null +++ b/eval/code/repos/taskrunner/config/fields.go @@ -0,0 +1,40 @@ +package config + +import ( + "fmt" + + "github.com/eval-org/taskrunner/config/internal/yaml" +) + +// SetField implements the configFields interface for the minimal YAML parser. +func (c *Config) SetField(key, value string) error { + switch key { + case "max_retries": + v, err := yaml.ParseInt(value) + if err != nil { + return err + } + c.MaxRetries = v + case "timeout": + v, err := yaml.ParseInt(value) + if err != nil { + return err + } + c.Timeout = v + case "verbose_logging": + v, err := yaml.ParseBool(value) + if err != nil { + return err + } + c.VerboseLogging = v + case "workers": + v, err := yaml.ParseInt(value) + if err != nil { + return err + } + c.Workers = v + default: + return fmt.Errorf("unknown config key: %s", key) + } + return nil +} diff --git a/eval/code/repos/taskrunner/config/internal/yaml/yaml.go b/eval/code/repos/taskrunner/config/internal/yaml/yaml.go new file mode 100644 index 00000000..0de49b68 --- /dev/null +++ b/eval/code/repos/taskrunner/config/internal/yaml/yaml.go @@ -0,0 +1,60 @@ +package yaml + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +// Unmarshal is a minimal YAML parser for flat key-value configs. +// It supports string, int, and bool values only. +func Unmarshal(data []byte, v interface{}) error { + lines := strings.Split(string(data), "\n") + kvs := make(map[string]string) + re := regexp.MustCompile(`^(\w+):\s*(.+)$`) + for i, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + m := re.FindStringSubmatch(line) + if m == nil { + return fmt.Errorf("line %d: malformed YAML (expected 'key: value'): %q", i+1, line) + } + kvs[m[1]] = m[2] + } + return applyToStruct(kvs, v) +} + +func applyToStruct(kvs map[string]string, v interface{}) error { + type configFields interface { + SetField(key, value string) error + } + if s, ok := v.(configFields); ok { + for k, val := range kvs { + if err := s.SetField(k, val); err != nil { + return err + } + } + return nil + } + return fmt.Errorf("target does not implement configFields interface") +} + +// ParseBool parses a YAML boolean string. +func ParseBool(s string) (bool, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "true", "yes", "on": + return true, nil + case "false", "no", "off": + return false, nil + default: + return false, fmt.Errorf("invalid bool: %q", s) + } +} + +// ParseInt parses a YAML integer string. +func ParseInt(s string) (int, error) { + return strconv.Atoi(strings.TrimSpace(s)) +} diff --git a/eval/code/repos/taskrunner/go.mod b/eval/code/repos/taskrunner/go.mod new file mode 100644 index 00000000..29ad2e62 --- /dev/null +++ b/eval/code/repos/taskrunner/go.mod @@ -0,0 +1,3 @@ +module github.com/eval-org/taskrunner + +go 1.22 diff --git a/eval/code/repos/taskrunner/runner/runner.go b/eval/code/repos/taskrunner/runner/runner.go new file mode 100644 index 00000000..5e666571 --- /dev/null +++ b/eval/code/repos/taskrunner/runner/runner.go @@ -0,0 +1,79 @@ +package runner + +import ( + "context" + "fmt" + "time" + + "github.com/eval-org/taskrunner/config" +) + +// Task represents a unit of work to execute. +type Task struct { + Name string + Fn func(context.Context) error +} + +// Runner executes tasks according to the provided configuration. +type Runner struct { + cfg config.Config + tasks []Task +} + +// New creates a Runner with the given configuration. +func New(cfg config.Config) (*Runner, error) { + if cfg.Workers < 1 { + return nil, fmt.Errorf("workers must be >= 1, got %d", cfg.Workers) + } + if cfg.Timeout < 1 { + return nil, fmt.Errorf("timeout must be >= 1, got %d", cfg.Timeout) + } + if cfg.MaxRetries < 0 { + return nil, fmt.Errorf("max_retries must be >= 0, got %d", cfg.MaxRetries) + } + return &Runner{cfg: cfg}, nil +} + +// Register adds a task to the runner. +func (r *Runner) Register(t Task) { + r.tasks = append(r.tasks, t) +} + +// Run executes all registered tasks with retry and timeout logic. +// It processes tasks concurrently based on the runner's configuration. +func (r *Runner) Run() error { + sem := make(chan struct{}, r.cfg.Workers) + errs := make(chan error, len(r.tasks)) + + for _, task := range r.tasks { + sem <- struct{}{} + go func(t Task) { + defer func() { <-sem }() + errs <- r.runWithRetry(t) + }(task) + } + + for range r.tasks { + if err := <-errs; err != nil { + return err + } + } + return nil +} + +func (r *Runner) runWithRetry(t Task) error { + timeout := time.Duration(r.cfg.Timeout) * time.Second + var lastErr error + + for attempt := 0; attempt <= r.cfg.MaxRetries; attempt++ { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + err := t.Fn(ctx) + cancel() + + if err == nil { + return nil + } + lastErr = err + } + return fmt.Errorf("task %s failed after %d retries: %w", t.Name, r.cfg.MaxRetries, lastErr) +} diff --git a/eval/code/repos/taskrunner/runner/runner_test.go b/eval/code/repos/taskrunner/runner/runner_test.go new file mode 100644 index 00000000..047efb37 --- /dev/null +++ b/eval/code/repos/taskrunner/runner/runner_test.go @@ -0,0 +1,81 @@ +package runner + +import ( + "context" + "errors" + "sync/atomic" + "testing" + + "github.com/eval-org/taskrunner/config" +) + +func TestNewRejectsInvalidConfig(t *testing.T) { + cases := []struct { + name string + cfg config.Config + }{ + {"zero workers", config.Config{Workers: 0, Timeout: 1, MaxRetries: 0}}, + {"negative timeout", config.Config{Workers: 1, Timeout: -1, MaxRetries: 0}}, + {"negative retries", config.Config{Workers: 1, Timeout: 1, MaxRetries: -1}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := New(tc.cfg) + if err == nil { + t.Errorf("New(%+v) = nil error, want error", tc.cfg) + } + }) + } +} + +func TestRunHappyPath(t *testing.T) { + r, err := New(config.Config{Workers: 1, Timeout: 5, MaxRetries: 0}) + if err != nil { + t.Fatal(err) + } + called := false + r.Register(Task{Name: "test", Fn: func(ctx context.Context) error { + called = true + return nil + }}) + if err := r.Run(); err != nil { + t.Fatal(err) + } + if !called { + t.Error("task was not executed") + } +} + +func TestRunRetriesUntilSuccess(t *testing.T) { + r, err := New(config.Config{Workers: 1, Timeout: 5, MaxRetries: 3}) + if err != nil { + t.Fatal(err) + } + var attempts int32 + r.Register(Task{Name: "flaky", Fn: func(ctx context.Context) error { + if atomic.AddInt32(&attempts, 1) < 3 { + return errors.New("transient") + } + return nil + }}) + if err := r.Run(); err != nil { + t.Fatalf("Run() = %v, want nil after retries", err) + } + if got := atomic.LoadInt32(&attempts); got != 3 { + t.Errorf("attempts = %d, want 3", got) + } +} + +func TestRunTimeoutCancelsAttempt(t *testing.T) { + r, err := New(config.Config{Workers: 1, Timeout: 1, MaxRetries: 0}) + if err != nil { + t.Fatal(err) + } + r.Register(Task{Name: "hang", Fn: func(ctx context.Context) error { + <-ctx.Done() + return ctx.Err() + }}) + if err := r.Run(); err == nil { + t.Fatal("Run() = nil, want error from timed-out task") + } +} diff --git a/eval/scripts/capture-fixture.sh b/eval/scripts/capture-fixture.sh index 160d0e57..adc6c0f9 100755 --- a/eval/scripts/capture-fixture.sh +++ b/eval/scripts/capture-fixture.sh @@ -60,6 +60,21 @@ fetch_pr_files() { return 1 } +# Best-effort gh pr diff, written to output/pr-.diff so content-level +# judges (removed_symbols in eval/code/eval.yaml) can inspect what the PR +# actually changed, not just which files it touched. On persistent failure +# returns non-zero so callers can record diff_fetch_failed instead of a +# missing file being indistinguishable from "capture never ran". +fetch_pr_diff() { + local num="$1" + local diff + if diff=$(retry_cmd gh pr diff "$num" --repo "$EPHEMERAL_REPO"); then + printf '%s\n' "$diff" > "${OUTPUT_DIR}/pr-${num}.diff" + return 0 + fi + return 1 +} + # Resolve branch tip SHA via git refs API, polling if still at baseline. # Poll up to 6 times with linear backoff (~21s total, within 60s after_each timeout). # Prefer refs API over PR headRefOid — the latter can lag briefly after post-fix push. @@ -157,14 +172,19 @@ case "${FIXTURE_TYPE}" in while IFS= read -r pr; do [[ -z "$pr" ]] && continue num=$(printf '%s' "$pr" | jq -r '.number') + diff_failed=false + if ! fetch_pr_diff "$num"; then + echo "WARNING: gh pr diff failed for PR #${num}; marking diff_fetch_failed" >&2 + diff_failed=true + fi if files=$(fetch_pr_files "$num"); then - pr_lines+=("$(printf '%s' "$pr" | jq -c --argjson files "$files" \ - '. + {head: .headRefName, base: .baseRefName, files: $files, files_fetch_failed: false} + pr_lines+=("$(printf '%s' "$pr" | jq -c --argjson files "$files" --argjson diff_failed "$diff_failed" \ + '. + {head: .headRefName, base: .baseRefName, files: $files, files_fetch_failed: false, diff_fetch_failed: $diff_failed} | del(.headRefName, .baseRefName)')") else echo "WARNING: gh pr view failed for PR #${num}; marking files_fetch_failed" >&2 - pr_lines+=("$(printf '%s' "$pr" | jq -c \ - '. + {head: .headRefName, base: .baseRefName, files: null, files_fetch_failed: true} + pr_lines+=("$(printf '%s' "$pr" | jq -c --argjson diff_failed "$diff_failed" \ + '. + {head: .headRefName, base: .baseRefName, files: null, files_fetch_failed: true, diff_fetch_failed: $diff_failed} | del(.headRefName, .baseRefName)')") fi done < <(printf '%s' "$prs_json" | jq -c '.[]') diff --git a/eval/scripts/scrub-eval-results-test.sh b/eval/scripts/scrub-eval-results-test.sh index 078908a8..c95a7fc6 100644 --- a/eval/scripts/scrub-eval-results-test.sh +++ b/eval/scripts/scrub-eval-results-test.sh @@ -20,10 +20,11 @@ run_test() { local test_name="$1" local input_content="$2" local expected_content="$3" + local filename="${4:-output.log}" local test_dir="${TMPDIR}/${test_name}" mkdir -p "${test_dir}" - printf '%s' "${input_content}" > "${test_dir}/output.log" + printf '%s' "${input_content}" > "${test_dir}/${filename}" local exit_code=0 bash "${SCRUB_SCRIPT}" "${test_dir}" > /dev/null 2>&1 || exit_code=$? @@ -35,7 +36,7 @@ run_test() { fi local actual - actual="$(cat "${test_dir}/output.log")" + actual="$(cat "${test_dir}/${filename}")" if [[ "${actual}" != "${expected_content}" ]]; then echo "FAIL: ${test_name}" @@ -114,6 +115,15 @@ Token is realsecret123" \ The quick brown fox jumps over a lazy dog Token is ***" +# --- Captured PR diffs (.diff) are scrubbed like any text artifact --- + +run_test "diff-file-token-redacted" \ + "diff --git a/config b/config ++url = https://x-access-token:ghs_abcdefghijklmnopqrstuv@github.com/o/r" \ + "diff --git a/config b/config ++url = https://x-access-token:***@github.com/o/r" \ + "pr-1.diff" + # --- Summary --- if [[ ${FAILURES} -gt 0 ]]; then diff --git a/eval/scripts/scrub-eval-results.sh b/eval/scripts/scrub-eval-results.sh index 7bbb030d..922067e2 100755 --- a/eval/scripts/scrub-eval-results.sh +++ b/eval/scripts/scrub-eval-results.sh @@ -33,7 +33,10 @@ import sys from pathlib import Path ROOTS = [Path(p) for p in os.environ["EVAL_SCRUB_ROOTS"].splitlines() if p] -TEXT_SUFFIXES = {".log", ".txt", ".json", ".jsonl", ".yaml", ".yml", ".md"} +# .diff: captured PR diffs (capture-fixture.sh writes output/pr-.diff); +# they can carry whatever the agent committed, so they must be scrubbed and +# leak-verified like any other text artifact before upload. +TEXT_SUFFIXES = {".log", ".txt", ".json", ".jsonl", ".yaml", ".yml", ".md", ".diff"} # Actions masks everything to end-of-line (not only \S+). ADD_MASK_RE = re.compile(r"::add-mask::(.+)$", re.MULTILINE) # Reject trivially short mask values that would over-redact (e.g. single