From 6fe7c7a1f49f7608793f13c164a89c02348826fa Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 10 Aug 2026 07:44:17 -0400 Subject: [PATCH 1/8] feat(eval): add measurement manifests for online scoring Introduce per-agent measurement manifest YAML files that select which eval measurement scorers run after managed agent jobs. Every agent except scribe gets trace_fitness (EM-001) at initial ship. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Adam Scerra --- eval/README.md | 7 +++++++ eval/measurements/README.md | 13 +++++++++++++ eval/measurements/code.yaml | 5 +++++ eval/measurements/fix.yaml | 5 +++++ eval/measurements/prioritize.yaml | 5 +++++ eval/measurements/retro.yaml | 5 +++++ eval/measurements/review.yaml | 5 +++++ eval/measurements/triage.yaml | 5 +++++ 8 files changed, 50 insertions(+) create mode 100644 eval/measurements/README.md create mode 100644 eval/measurements/code.yaml create mode 100644 eval/measurements/fix.yaml create mode 100644 eval/measurements/prioritize.yaml create mode 100644 eval/measurements/retro.yaml create mode 100644 eval/measurements/review.yaml create mode 100644 eval/measurements/triage.yaml diff --git a/eval/README.md b/eval/README.md index 6f52d17d..3f0eb245 100644 --- a/eval/README.md +++ b/eval/README.md @@ -136,3 +136,10 @@ Each test case follows this lifecycle: - **`checkStatus` drops string errors.** fullsend's `checkStatus` does not handle string-typed error responses from the GitHub API, causing silent failures. + +## Measurement manifests (online scoring) + +Per-agent manifests under [`eval/measurements/`](./measurements/) select which +**eval measurement** scorers `fullsend eval-measure` runs after managed jobs. +These are **not** functional PR-gate scenarios. See the README in that +directory and fullsend ADR 0087. diff --git a/eval/measurements/README.md b/eval/measurements/README.md new file mode 100644 index 00000000..1eda05a8 --- /dev/null +++ b/eval/measurements/README.md @@ -0,0 +1,13 @@ +# Measurement manifests + +Per-agent YAML that selects which **eval measurement** scorers run after a +managed agent job (`fullsend eval-measure`). This is **not** the functional +eval harness under `eval//` (PR-gate scenarios / fixtures). + +- **Scorers** (Go logic) live in `fullsend-ai/fullsend` (`internal/evalmeasure/`). +- **Manifests** (which scorers for which agent) live here. + +At first ship, every listed agent enables the `trace_fitness` scorer (EM-001). Omit a file to +opt an agent out (e.g. scribe has no forge work-item identity today). + +See fullsend ADR 0087 and the Eval Measurements guide in `fullsend-ai/fullsend`. diff --git a/eval/measurements/code.yaml b/eval/measurements/code.yaml new file mode 100644 index 00000000..5d70fdda --- /dev/null +++ b/eval/measurements/code.yaml @@ -0,0 +1,5 @@ +agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 diff --git a/eval/measurements/fix.yaml b/eval/measurements/fix.yaml new file mode 100644 index 00000000..f1eec35d --- /dev/null +++ b/eval/measurements/fix.yaml @@ -0,0 +1,5 @@ +agent: fix +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 diff --git a/eval/measurements/prioritize.yaml b/eval/measurements/prioritize.yaml new file mode 100644 index 00000000..610cc5e4 --- /dev/null +++ b/eval/measurements/prioritize.yaml @@ -0,0 +1,5 @@ +agent: prioritize +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 diff --git a/eval/measurements/retro.yaml b/eval/measurements/retro.yaml new file mode 100644 index 00000000..62895be8 --- /dev/null +++ b/eval/measurements/retro.yaml @@ -0,0 +1,5 @@ +agent: retro +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 diff --git a/eval/measurements/review.yaml b/eval/measurements/review.yaml new file mode 100644 index 00000000..64e2d91c --- /dev/null +++ b/eval/measurements/review.yaml @@ -0,0 +1,5 @@ +agent: review +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 diff --git a/eval/measurements/triage.yaml b/eval/measurements/triage.yaml new file mode 100644 index 00000000..ef8492df --- /dev/null +++ b/eval/measurements/triage.yaml @@ -0,0 +1,5 @@ +agent: triage +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 From 6f53875c1f6fe1c5aa7d2c69ca958a448311fb54 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 10 Aug 2026 07:50:27 -0400 Subject: [PATCH 2/8] fix(eval): normalize ID casing and add YAML document markers Lowercase EM-001 to em-001 in README to match manifest YAML files. Add --- document start markers to all measurement manifests for consistency with harness/*.yaml convention. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Adam Scerra --- eval/measurements/README.md | 2 +- eval/measurements/code.yaml | 1 + eval/measurements/fix.yaml | 1 + eval/measurements/prioritize.yaml | 1 + eval/measurements/retro.yaml | 1 + eval/measurements/review.yaml | 1 + eval/measurements/triage.yaml | 1 + 7 files changed, 7 insertions(+), 1 deletion(-) diff --git a/eval/measurements/README.md b/eval/measurements/README.md index 1eda05a8..22953c07 100644 --- a/eval/measurements/README.md +++ b/eval/measurements/README.md @@ -7,7 +7,7 @@ eval harness under `eval//` (PR-gate scenarios / fixtures). - **Scorers** (Go logic) live in `fullsend-ai/fullsend` (`internal/evalmeasure/`). - **Manifests** (which scorers for which agent) live here. -At first ship, every listed agent enables the `trace_fitness` scorer (EM-001). Omit a file to +At first ship, every listed agent enables the `trace_fitness` scorer (em-001). Omit a file to opt an agent out (e.g. scribe has no forge work-item identity today). See fullsend ADR 0087 and the Eval Measurements guide in `fullsend-ai/fullsend`. diff --git a/eval/measurements/code.yaml b/eval/measurements/code.yaml index 5d70fdda..da4ff144 100644 --- a/eval/measurements/code.yaml +++ b/eval/measurements/code.yaml @@ -1,3 +1,4 @@ +--- agent: code measurements: - id: em-001 diff --git a/eval/measurements/fix.yaml b/eval/measurements/fix.yaml index f1eec35d..12ce41bd 100644 --- a/eval/measurements/fix.yaml +++ b/eval/measurements/fix.yaml @@ -1,3 +1,4 @@ +--- agent: fix measurements: - id: em-001 diff --git a/eval/measurements/prioritize.yaml b/eval/measurements/prioritize.yaml index 610cc5e4..c8b01981 100644 --- a/eval/measurements/prioritize.yaml +++ b/eval/measurements/prioritize.yaml @@ -1,3 +1,4 @@ +--- agent: prioritize measurements: - id: em-001 diff --git a/eval/measurements/retro.yaml b/eval/measurements/retro.yaml index 62895be8..882d8219 100644 --- a/eval/measurements/retro.yaml +++ b/eval/measurements/retro.yaml @@ -1,3 +1,4 @@ +--- agent: retro measurements: - id: em-001 diff --git a/eval/measurements/review.yaml b/eval/measurements/review.yaml index 64e2d91c..39f0f1b7 100644 --- a/eval/measurements/review.yaml +++ b/eval/measurements/review.yaml @@ -1,3 +1,4 @@ +--- agent: review measurements: - id: em-001 diff --git a/eval/measurements/triage.yaml b/eval/measurements/triage.yaml index ef8492df..f986c783 100644 --- a/eval/measurements/triage.yaml +++ b/eval/measurements/triage.yaml @@ -1,3 +1,4 @@ +--- agent: triage measurements: - id: em-001 From cd3b9bf3b8c63f95aa36098c1155a5d6b5f64826 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 10 Aug 2026 07:53:56 -0400 Subject: [PATCH 3/8] docs(eval): clarify agent coverage in measurements README Replace ambiguous "every listed agent" phrasing with an explicit list of the six agents that enable trace_fitness at initial ship. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Adam Scerra --- eval/measurements/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eval/measurements/README.md b/eval/measurements/README.md index 22953c07..dbc56eeb 100644 --- a/eval/measurements/README.md +++ b/eval/measurements/README.md @@ -7,7 +7,8 @@ eval harness under `eval//` (PR-gate scenarios / fixtures). - **Scorers** (Go logic) live in `fullsend-ai/fullsend` (`internal/evalmeasure/`). - **Manifests** (which scorers for which agent) live here. -At first ship, every listed agent enables the `trace_fitness` scorer (em-001). Omit a file to -opt an agent out (e.g. scribe has no forge work-item identity today). +At first ship, six agents enable the `trace_fitness` scorer (em-001): code, fix, +prioritize, retro, review, and triage. Omit a file to opt an agent out (e.g. +scribe has no forge work-item identity today). See fullsend ADR 0087 and the Eval Measurements guide in `fullsend-ai/fullsend`. From 4d3ed508ce969da81db4b63174dc3a101eceaf50 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 10 Aug 2026 09:32:21 -0400 Subject: [PATCH 4/8] docs(eval): clarify measurement manifest ownership Spell out agents@v0 defaults vs local overrides, engine-vs-policy split with fullsend#6036, and when a change needs fullsend vs agents. Signed-off-by: Adam Scerra Co-authored-by: Cursor Signed-off-by: Adam Scerra --- eval/README.md | 14 +++++++--- eval/measurements/README.md | 53 +++++++++++++++++++++++++++++++++---- 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/eval/README.md b/eval/README.md index 3f0eb245..c363a457 100644 --- a/eval/README.md +++ b/eval/README.md @@ -139,7 +139,13 @@ Each test case follows this lifecycle: ## Measurement manifests (online scoring) -Per-agent manifests under [`eval/measurements/`](./measurements/) select which -**eval measurement** scorers `fullsend eval-measure` runs after managed jobs. -These are **not** functional PR-gate scenarios. See the README in that -directory and fullsend ADR 0087. +Per-agent manifests under [`eval/measurements/`](./measurements/) are the +**default online-scoring policy** for stock agents (which scorers run after +managed jobs via `fullsend eval-measure`). They are **not** functional PR-gate +scenarios under `eval//`. + +Scorer *implementations* live in `fullsend-ai/fullsend`; this repo only +declares defaults. Jobs fetch these files from `agents@v0` unless a consumer +overrides under `FULLSEND_DIR`. See [`eval/measurements/README.md`](./measurements/README.md), +[fullsend#6036](https://github.com/fullsend-ai/fullsend/pull/6036), and +fullsend ADR 0087. diff --git a/eval/measurements/README.md b/eval/measurements/README.md index dbc56eeb..418a6496 100644 --- a/eval/measurements/README.md +++ b/eval/measurements/README.md @@ -4,11 +4,54 @@ Per-agent YAML that selects which **eval measurement** scorers run after a managed agent job (`fullsend eval-measure`). This is **not** the functional eval harness under `eval//` (PR-gate scenarios / fixtures). -- **Scorers** (Go logic) live in `fullsend-ai/fullsend` (`internal/evalmeasure/`). -- **Manifests** (which scorers for which agent) live here. +## Why this lives next to the agents -At first ship, six agents enable the `trace_fitness` scorer (em-001): code, fix, -prioritize, retro, review, and triage. Omit a file to opt an agent out (e.g. +These files are the **default online-scoring policy** for the stock fullsend +agents — the same idea as shipping the agents themselves: “here is `code`, +and here is what we measure on wild `code` runs.” + +Managed fullsend jobs resolve manifests as: + +1. Local `${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml` if present (override / BYOA) +2. Else this repo at the `v0` pin: + `https://raw.githubusercontent.com/fullsend-ai/agents/v0/eval/measurements/${AGENT}.yaml` + +Installs that only use stock agents **do not copy these files**. Local files +are for changing defaults, opting out, or scoring a custom agent. + +## What lives where + +| Concern | Repo | +|---|---| +| Scorer **implementations** (Go), parser, CLI, job wiring | [`fullsend-ai/fullsend`](https://github.com/fullsend-ai/fullsend) (`internal/evalmeasure/`) | +| Default manifests (which `id` / `scorer` / `version` per agent) | **This directory** | +| Org/repo overrides and BYOA manifests | Consumer `FULLSEND_DIR` | + +Executable logic stays in fullsend because `fullsend eval-measure` is the +released binary that reads `run-telemetry.jsonl` (produced by fullsend). This +repo is content/policy, not that binary. Platform checks like EM-001 +(`trace_fitness`) still get **enabled** here for each stock agent. + +| Change | PR | +|---|---| +| New Go scorer or (future) new declarative `assert:` | `fullsend` | +| New measurement id / enable / disable / thresholds for a stock agent using an existing scorer | **agents** (this repo) | +| Custom policy for one org or a BYOA agent | Local override in the consumer repo | + +Companion platform PR: [fullsend-ai/fullsend#6036](https://github.com/fullsend-ai/fullsend/pull/6036) +([ADR 0087](https://github.com/fullsend-ai/fullsend/blob/main/docs/ADRs/0087-eval-measurements-online-trace-scoring.md) +lands with that PR). + +## First ship + +Six agents enable `trace_fitness` (em-001): code, fix, prioritize, retro, +review, and triage. Omit a file to leave an agent without defaults (e.g. scribe has no forge work-item identity today). -See fullsend ADR 0087 and the Eval Measurements guide in `fullsend-ai/fullsend`. +```yaml +agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +``` From a31fc395c1c16a50aa7e01de3e47e438864edcac Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Thu, 13 Aug 2026 07:49:18 -0400 Subject: [PATCH 5/8] docs(eval): hyperlink ADR 0087 in eval README Match eval/measurements/README.md link style for ADR 0087 so the measurement manifest section uses consistent markdown hyperlinks. Signed-off-by: Adam Scerra Co-authored-by: Cursor Signed-off-by: Adam Scerra --- eval/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eval/README.md b/eval/README.md index c363a457..47089c20 100644 --- a/eval/README.md +++ b/eval/README.md @@ -148,4 +148,4 @@ Scorer *implementations* live in `fullsend-ai/fullsend`; this repo only declares defaults. Jobs fetch these files from `agents@v0` unless a consumer overrides under `FULLSEND_DIR`. See [`eval/measurements/README.md`](./measurements/README.md), [fullsend#6036](https://github.com/fullsend-ai/fullsend/pull/6036), and -fullsend ADR 0087. +[ADR 0087](https://github.com/fullsend-ai/fullsend/blob/main/docs/ADRs/0087-eval-measurements-online-trace-scoring.md). From 06d98e491bfcecad2e27ef376d068d2382574ac7 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 17 Aug 2026 15:31:23 -0400 Subject: [PATCH 6/8] docs(eval): lowercase em-001 and drop 404 ADR links Keep the measurement id casing aligned with manifest id fields. Point readers at fullsend#6036 for ADR 0087 until that PR lands on main. Signed-off-by: Adam Scerra Co-authored-by: Cursor Signed-off-by: Adam Scerra --- eval/README.md | 7 ++++--- eval/measurements/README.md | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/eval/README.md b/eval/README.md index 47089c20..9896db2e 100644 --- a/eval/README.md +++ b/eval/README.md @@ -29,6 +29,7 @@ running: ```bash bash eval/lint-cases.sh +bash eval/lint-measurements.sh ``` ## Prerequisites @@ -146,6 +147,6 @@ scenarios under `eval//`. Scorer *implementations* live in `fullsend-ai/fullsend`; this repo only declares defaults. Jobs fetch these files from `agents@v0` unless a consumer -overrides under `FULLSEND_DIR`. See [`eval/measurements/README.md`](./measurements/README.md), -[fullsend#6036](https://github.com/fullsend-ai/fullsend/pull/6036), and -[ADR 0087](https://github.com/fullsend-ai/fullsend/blob/main/docs/ADRs/0087-eval-measurements-online-trace-scoring.md). +overrides under `FULLSEND_DIR`. See [`eval/measurements/README.md`](./measurements/README.md) +and [fullsend#6036](https://github.com/fullsend-ai/fullsend/pull/6036) (ADR 0087 +lands with that PR). diff --git a/eval/measurements/README.md b/eval/measurements/README.md index 418a6496..012aa204 100644 --- a/eval/measurements/README.md +++ b/eval/measurements/README.md @@ -29,7 +29,7 @@ are for changing defaults, opting out, or scoring a custom agent. Executable logic stays in fullsend because `fullsend eval-measure` is the released binary that reads `run-telemetry.jsonl` (produced by fullsend). This -repo is content/policy, not that binary. Platform checks like EM-001 +repo is content/policy, not that binary. Platform checks like em-001 (`trace_fitness`) still get **enabled** here for each stock agent. | Change | PR | @@ -39,8 +39,8 @@ repo is content/policy, not that binary. Platform checks like EM-001 | Custom policy for one org or a BYOA agent | Local override in the consumer repo | Companion platform PR: [fullsend-ai/fullsend#6036](https://github.com/fullsend-ai/fullsend/pull/6036) -([ADR 0087](https://github.com/fullsend-ai/fullsend/blob/main/docs/ADRs/0087-eval-measurements-online-trace-scoring.md) -lands with that PR). +(ADR 0087 lands with that PR; the `docs/ADRs/0087-*.md` path is not on +`fullsend` main until #6036 merges). ## First ship From f8bfbae2a8216dec0fdd2824de12a14b3946735a Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 17 Aug 2026 15:31:23 -0400 Subject: [PATCH 7/8] ci(eval): lint measurement manifests in pre-commit Reject unknown scorers, missing fields, duplicate ids, and agent names that do not match agents/.md so a typo fails CI instead of silently producing zero scores. Signed-off-by: Adam Scerra Co-authored-by: Cursor Signed-off-by: Adam Scerra --- .pre-commit-config.yaml | 6 ++ Makefile | 1 + eval/lint-measurements-test.sh | 123 ++++++++++++++++++++++ eval/lint-measurements.sh | 182 +++++++++++++++++++++++++++++++++ 4 files changed, 312 insertions(+) create mode 100755 eval/lint-measurements-test.sh create mode 100755 eval/lint-measurements.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d690aed7..1f5111bb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,3 +52,9 @@ repos: language: script files: ^(harness/|docs/|hack/lint-agent-docs) pass_filenames: false + - id: lint-measurements + name: lint eval measurement manifests + entry: ./eval/lint-measurements.sh + language: script + files: ^(eval/measurements/|eval/lint-measurements) + pass_filenames: false diff --git a/Makefile b/Makefile index f42539ab..c148f84d 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,7 @@ script-test: $(call run-timed,bash scripts/validate-output-schema-test.sh) $(call run-timed,bash scripts/gitlint-forbidden-type-scope-test.sh) $(call run-timed,bash hack/lint-agent-docs-test.sh) + $(call run-timed,bash eval/lint-measurements-test.sh) $(call run-timed,bash .github/scripts/check-e2e-authorization-test.sh) $(call run-timed,bash .github/scripts/select-eval-agents-test.sh) $(call run-timed,python3 scripts/process-fix-result-test.py) diff --git a/eval/lint-measurements-test.sh b/eval/lint-measurements-test.sh new file mode 100755 index 00000000..890753b9 --- /dev/null +++ b/eval/lint-measurements-test.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# lint-measurements-test.sh — Tests for eval/lint-measurements.sh +# +# Run from the repo root: +# bash eval/lint-measurements-test.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LINTER="${SCRIPT_DIR}/lint-measurements.sh" + +FAILURES=0 +WORKDIR="$(mktemp -d)" +trap 'rm -rf "${WORKDIR}"' EXIT + +VALID_YAML='--- +agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +' + +# run_case NAME YAML EXPECTED_EXIT [EXPECTED_OUTPUT_SUBSTRING] [FILENAME] +run_case() { + local name="$1" yaml="$2" expected_exit="$3" expected_substring="${4:-}" filename="${5:-code.yaml}" + + local case_dir="${WORKDIR}/${name}" + mkdir -p "${case_dir}/eval/measurements" "${case_dir}/agents" + printf '%s\n' "# Code Agent" > "${case_dir}/agents/code.md" + if [[ -n "$yaml" ]]; then + printf '%s' "$yaml" > "${case_dir}/eval/measurements/${filename}" + fi + + local output + local actual_exit=0 + output="$(REPO_ROOT="${case_dir}" MEASUREMENTS_DIR="${case_dir}/eval/measurements" AGENTS_DIR="${case_dir}/agents" "${LINTER}" 2>&1)" || actual_exit=$? + + if [[ "${actual_exit}" != "${expected_exit}" ]]; then + echo "FAIL: ${name} (exit ${actual_exit}, expected ${expected_exit})" + echo "${output}" | sed 's/^/ /' + FAILURES=$((FAILURES + 1)) + return + fi + + if [[ -n "${expected_substring}" ]] && [[ "${output}" != *"${expected_substring}"* ]]; then + echo "FAIL: ${name} (missing expected output: '${expected_substring}')" + echo "${output}" | sed 's/^/ /' + FAILURES=$((FAILURES + 1)) + return + fi + + echo "PASS: ${name}" +} + +run_case "valid-manifest-passes" \ + "${VALID_YAML}" 0 "code.yaml: OK" + +run_case "unknown-scorer" \ + "--- +agent: code +measurements: + - id: em-001 + scorer: trace-fitness + version: 1 +" 1 "unknown scorer 'trace-fitness'" + +run_case "unknown-agent" \ + "--- +agent: not-an-agent +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +" 1 "has no agents/not-an-agent.md" "not-an-agent.yaml" + +run_case "filename-agent-mismatch" \ + "--- +agent: review +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +" 1 "does not match filename stem" "code.yaml" + +run_case "duplicate-id" \ + "--- +agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + - id: em-001 + scorer: trace_fitness + version: 1 +" 1 "duplicate id 'em-001'" + +run_case "uppercase-id" \ + "--- +agent: code +measurements: + - id: EM-001 + scorer: trace_fitness + version: 1 +" 1 "must be lowercase like em-001" + +run_case "missing-version" \ + "--- +agent: code +measurements: + - id: em-001 + scorer: trace_fitness +" 1 "version must be a positive integer" + +run_case "empty-dir" \ + "" 1 "no eval/measurements/*.yaml files found" + +echo "" +if [[ ${FAILURES} -gt 0 ]]; then + echo "${FAILURES} test(s) failed" + exit 1 +fi +echo "All tests passed" diff --git a/eval/lint-measurements.sh b/eval/lint-measurements.sh new file mode 100755 index 00000000..be720099 --- /dev/null +++ b/eval/lint-measurements.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Lint eval/measurements/*.yaml — catch typos that would silently no-op +# at runtime (unknown scorer / agent name that never matches a trace). +# +# Usage: +# ./eval/lint-measurements.sh +# +# Checks: +# - Filename stem equals the top-level `agent:` field +# - `agent:` matches an existing agents/.md +# - Each measurement has id, scorer, and a positive integer version +# - ids are unique per file and match em-001-style lowercase +# - scorer is in the known-scorer allow-list (fullsend evalmeasure registry) +set -euo pipefail +shopt -s nullglob + +REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" +MEASUREMENTS_DIR="${MEASUREMENTS_DIR:-$REPO_ROOT/eval/measurements}" +AGENTS_DIR="${AGENTS_DIR:-$REPO_ROOT/agents}" + +# Keep in sync with fullsend internal/evalmeasure ScorerFitness. +KNOWN_SCORERS="trace_fitness" + +errors=0 +file_count=0 + +echo "Checking measurement manifests..." +echo "================================================" + +if ! command -v python3 >/dev/null 2>&1; then + echo "ERROR: python3 is required" >&2 + exit 1 +fi + +parse_manifest() { + python3 - "$1" <<'PY' +import re, sys + +path = sys.argv[1] +text = open(path, encoding="utf-8").read() +agent = None +items = [] +current = None +in_measurements = False + +for raw in text.splitlines(): + line = raw.split("#", 1)[0].rstrip() + if not line.strip() or line.strip() == "---": + continue + if re.match(r"^agent:\s*\S", line): + agent = line.split(":", 1)[1].strip().strip("'\"") + continue + if re.match(r"^measurements:\s*$", line): + in_measurements = True + continue + if in_measurements and re.match(r"^\s*-\s+", line): + if current is not None: + items.append(current) + current = {} + rest = re.sub(r"^\s*-\s+", "", line) + m = re.match(r"(id|scorer|version):\s*(.*)$", rest) + if m: + current[m.group(1)] = m.group(2).strip().strip("'\"") + continue + if in_measurements and current is not None: + m = re.match(r"^\s+(id|scorer|version):\s*(.*)$", line) + if m: + current[m.group(1)] = m.group(2).strip().strip("'\"") + continue + if re.match(r"^\S", line): + in_measurements = False + items.append(current) + current = None + +if current is not None: + items.append(current) + +print(agent or "") +print(len(items)) +for it in items: + print(it.get("id", "")) + print(it.get("scorer", "")) + print(it.get("version", "")) +PY +} + +for yaml_file in "$MEASUREMENTS_DIR"/*.yaml; do + file_count=$((file_count + 1)) + name="$(basename "$yaml_file")" + stem="${name%.yaml}" + + mapfile -t parsed < <(parse_manifest "$yaml_file") + agent="${parsed[0]:-}" + count="${parsed[1]:-0}" + + file_errors=0 + + if [[ -z "$agent" ]]; then + echo " ERROR: $name: missing 'agent:' field" + errors=$((errors + 1)) + continue + fi + + if [[ "$agent" != "$stem" ]]; then + echo " ERROR: $name: agent '$agent' does not match filename stem '$stem' (jobs fetch \${AGENT}.yaml)" + errors=$((errors + 1)) + file_errors=$((file_errors + 1)) + fi + + if [[ ! -f "$AGENTS_DIR/${agent}.md" ]]; then + echo " ERROR: $name: agent '$agent' has no agents/${agent}.md" + errors=$((errors + 1)) + file_errors=$((file_errors + 1)) + fi + + if [[ "$count" -lt 1 ]]; then + echo " ERROR: $name: measurements list is empty" + errors=$((errors + 1)) + continue + fi + + declare -A seen_ids=() + idx=0 + while [[ $idx -lt $count ]]; do + base=$((2 + idx * 3)) + mid="${parsed[$base]:-}" + scorer="${parsed[$((base + 1))]:-}" + version="${parsed[$((base + 2))]:-}" + idx=$((idx + 1)) + + if [[ -z "$mid" ]]; then + echo " ERROR: $name: measurement #$idx missing id" + errors=$((errors + 1)) + file_errors=$((file_errors + 1)) + continue + fi + if [[ ! "$mid" =~ ^[a-z][a-z0-9]*-[0-9]+$ ]]; then + echo " ERROR: $name: id '$mid' must be lowercase like em-001" + errors=$((errors + 1)) + file_errors=$((file_errors + 1)) + fi + if [[ -n "${seen_ids[$mid]+x}" ]]; then + echo " ERROR: $name: duplicate id '$mid'" + errors=$((errors + 1)) + file_errors=$((file_errors + 1)) + fi + seen_ids["$mid"]=1 + + if [[ -z "$scorer" ]]; then + echo " ERROR: $name: measurement '$mid' missing scorer" + errors=$((errors + 1)) + file_errors=$((file_errors + 1)) + elif [[ " $KNOWN_SCORERS " != *" $scorer "* ]]; then + echo " ERROR: $name: unknown scorer '$scorer' (allowed: $KNOWN_SCORERS)" + errors=$((errors + 1)) + file_errors=$((file_errors + 1)) + fi + + if [[ ! "$version" =~ ^[1-9][0-9]*$ ]]; then + echo " ERROR: $name: measurement '$mid' version must be a positive integer, got '$version'" + errors=$((errors + 1)) + file_errors=$((file_errors + 1)) + fi + done + unset seen_ids + + if [[ $file_errors -eq 0 ]]; then + echo " $name: OK (agent=$agent, $count measurement(s))" + fi +done + +if [[ $file_count -eq 0 ]]; then + echo " ERROR: no eval/measurements/*.yaml files found — expected at least one" + errors=$((errors + 1)) +fi + +echo "" +if [[ $errors -gt 0 ]]; then + echo "ERROR: $errors measurement lint failure(s)" >&2 + exit 1 +fi +echo "OK: all measurement manifests pass lint checks" From 2a545357c9b9244fcd70f5a83fb0223602b402e7 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 17 Aug 2026 21:53:09 -0400 Subject: [PATCH 8/8] fix(eval): address second-round measurement-manifest review Rewrite the pre-commit linter in python3 so it runs on macOS bash 3.2, fails closed on unsupported YAML, and documents em-001 as an agents-repo style rule. Describe the SHA-pinned v0 GetRef fetch instead of a floating raw URL, drop eval/ --- markers, and mention eval/ in README and LOCAL.md. Signed-off-by: Adam Scerra Co-authored-by: Cursor --- LOCAL.md | 6 +- README.md | 1 + eval/lint-measurements-test.sh | 15 ++ eval/lint-measurements.sh | 341 ++++++++++++++++-------------- eval/measurements/README.md | 10 +- eval/measurements/code.yaml | 1 - eval/measurements/fix.yaml | 1 - eval/measurements/prioritize.yaml | 1 - eval/measurements/retro.yaml | 1 - eval/measurements/review.yaml | 1 - eval/measurements/triage.yaml | 1 - 11 files changed, 212 insertions(+), 167 deletions(-) diff --git a/LOCAL.md b/LOCAL.md index e5678d96..fd66998d 100644 --- a/LOCAL.md +++ b/LOCAL.md @@ -142,8 +142,10 @@ fullsend run triage \ ## Functional eval tests The `eval/` directory contains functional test scenarios that run agents -against ephemeral GitHub repos and score the results. See -[eval/README.md](eval/README.md) for setup and usage. +against ephemeral GitHub repos and score the results, plus default +online-scoring manifests under [`eval/measurements/`](eval/measurements/README.md) +consumed by `fullsend eval-measure`. See [eval/README.md](eval/README.md) +for setup and usage. To run triage evals: diff --git a/README.md b/README.md index 076d939b..594d146c 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ schemas/ JSON Schema for validating agent structured output scripts/ Pre-scripts (input validation) and post-scripts (forge mutations) skills/ Reusable skill definitions loaded by agents at runtime plugins/ Sandbox plugins (e.g. gopls LSP for the code agent) +eval/ Functional eval harness and default online-scoring manifests ``` ## Architecture diff --git a/eval/lint-measurements-test.sh b/eval/lint-measurements-test.sh index 890753b9..ca686240 100755 --- a/eval/lint-measurements-test.sh +++ b/eval/lint-measurements-test.sh @@ -115,6 +115,21 @@ measurements: run_case "empty-dir" \ "" 1 "no eval/measurements/*.yaml files found" +run_case "flow-style-unsupported" \ + "agent: code +measurements: [{id: em-001, scorer: trace_fitness, version: 1}] +" 1 "unsupported YAML shape" + +run_case "nested-assert-unsupported" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + assert: + - path: x +" 1 "unsupported YAML shape" + echo "" if [[ ${FAILURES} -gt 0 ]]; then echo "${FAILURES} test(s) failed" diff --git a/eval/lint-measurements.sh b/eval/lint-measurements.sh index be720099..70951921 100755 --- a/eval/lint-measurements.sh +++ b/eval/lint-measurements.sh @@ -2,6 +2,9 @@ # Lint eval/measurements/*.yaml — catch typos that would silently no-op # at runtime (unknown scorer / agent name that never matches a trace). # +# bash 3.2 compatible (macOS /usr/bin/bash). Validation runs in python3; +# this wrapper does not use mapfile or declare -A. +# # Usage: # ./eval/lint-measurements.sh # @@ -9,174 +12,198 @@ # - Filename stem equals the top-level `agent:` field # - `agent:` matches an existing agents/.md # - Each measurement has id, scorer, and a positive integer version -# - ids are unique per file and match em-001-style lowercase +# - ids are unique per file # - scorer is in the known-scorer allow-list (fullsend evalmeasure registry) +# - id matches em-001-style lowercase — agents-repo stock-manifest style, +# stricter than fullsend LoadRegistry (which only requires a non-empty id +# without pipe/newline) +# - YAML must be the shipped block-style three-field shape; other shapes +# fail closed as "unsupported YAML shape" set -euo pipefail -shopt -s nullglob REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" MEASUREMENTS_DIR="${MEASUREMENTS_DIR:-$REPO_ROOT/eval/measurements}" AGENTS_DIR="${AGENTS_DIR:-$REPO_ROOT/agents}" -# Keep in sync with fullsend internal/evalmeasure ScorerFitness. -KNOWN_SCORERS="trace_fitness" - -errors=0 -file_count=0 - -echo "Checking measurement manifests..." -echo "================================================" - if ! command -v python3 >/dev/null 2>&1; then echo "ERROR: python3 is required" >&2 exit 1 fi -parse_manifest() { - python3 - "$1" <<'PY' -import re, sys - -path = sys.argv[1] -text = open(path, encoding="utf-8").read() -agent = None -items = [] -current = None -in_measurements = False - -for raw in text.splitlines(): - line = raw.split("#", 1)[0].rstrip() - if not line.strip() or line.strip() == "---": - continue - if re.match(r"^agent:\s*\S", line): - agent = line.split(":", 1)[1].strip().strip("'\"") - continue - if re.match(r"^measurements:\s*$", line): - in_measurements = True - continue - if in_measurements and re.match(r"^\s*-\s+", line): - if current is not None: - items.append(current) - current = {} - rest = re.sub(r"^\s*-\s+", "", line) - m = re.match(r"(id|scorer|version):\s*(.*)$", rest) - if m: - current[m.group(1)] = m.group(2).strip().strip("'\"") - continue - if in_measurements and current is not None: - m = re.match(r"^\s+(id|scorer|version):\s*(.*)$", line) - if m: - current[m.group(1)] = m.group(2).strip().strip("'\"") +# Command substitution (not process substitution) so python's exit code is visible. +python3 - "$MEASUREMENTS_DIR" "$AGENTS_DIR" <<'PY' +import os +import re +import sys + +MEASUREMENTS_DIR, AGENTS_DIR = sys.argv[1], sys.argv[2] +KNOWN_SCORERS = frozenset({"trace_fitness"}) +ID_STYLE = re.compile(r"^[a-z][a-z0-9]*-[0-9]+$") +FIELD_KEYS = frozenset({"id", "scorer", "version"}) + + +class UnsupportedShape(Exception): + pass + + +def parse_manifest(text): + """Parse the shipped block-style schema. Raise UnsupportedShape otherwise.""" + agent = None + items = [] + current = None + in_measurements = False + measurements_key = False + list_indent = None + + for raw in text.splitlines(): + stripped = raw.split("#", 1)[0].rstrip() + if not stripped or stripped == "---": + continue + if re.search(r"measurements:\s*[\[{]", stripped): + raise UnsupportedShape("flow-style measurements are not supported") + indent = len(raw) - len(raw.lstrip(" ")) + if re.match(r"^agent:\s*\S", stripped): + agent = stripped.split(":", 1)[1].strip().strip("'\"") + continue + if re.match(r"^measurements:\s*$", stripped): + in_measurements = True + measurements_key = True + continue + if in_measurements and re.match(r"^-\s+", stripped.lstrip()): + dash_indent = indent + if list_indent is None: + list_indent = dash_indent + if dash_indent > (list_indent or 0) and current is not None: + raise UnsupportedShape("nested list under a measurement is not supported") + if current is not None: + items.append(current) + current = {} + rest = re.sub(r"^-\s+", "", stripped.lstrip()) + m = re.match(r"(id|scorer|version):\s*(.*)$", rest) + if rest and not m: + raise UnsupportedShape("unsupported field on measurement list item") + if m: + current[m.group(1)] = m.group(2).strip().strip("'\"") + continue + if in_measurements and current is not None: + fm = re.match(r"^\s+(id|scorer|version):\s*(.*)$", stripped) + um = re.match(r"^\s+([A-Za-z0-9_]+):\s*(.*)$", stripped) + if fm: + current[fm.group(1)] = fm.group(2).strip().strip("'\"") + continue + if um and um.group(1) not in FIELD_KEYS: + raise UnsupportedShape("unsupported field %r" % um.group(1)) + if re.match(r"^\S", stripped): + in_measurements = False + items.append(current) + current = None + continue + raise UnsupportedShape("unrecognized line in measurements list") + if re.match(r"^\S", stripped) and stripped.split(":", 1)[0] not in ("agent", "measurements"): + raise UnsupportedShape("unsupported top-level field %r" % stripped.split(":", 1)[0]) + + if current is not None: + items.append(current) + if measurements_key and not items: + raise UnsupportedShape("measurements present but no block-style items parsed") + return agent, items + + +def main(): + print("Checking measurement manifests...") + print("================================================") + errors = 0 + file_count = 0 + + try: + names = sorted(n for n in os.listdir(MEASUREMENTS_DIR) if n.endswith(".yaml")) + except OSError: + names = [] + + for name in names: + file_count += 1 + path = os.path.join(MEASUREMENTS_DIR, name) + stem = name[:-5] + try: + text = open(path, encoding="utf-8").read() + agent, items = parse_manifest(text) + except UnsupportedShape as e: + print(" ERROR: %s: unsupported YAML shape (%s)" % (name, e)) + errors += 1 + continue + except Exception as e: + print(" ERROR: %s: parser failed: %s" % (name, e)) + errors += 1 continue - if re.match(r"^\S", line): - in_measurements = False - items.append(current) - current = None - -if current is not None: - items.append(current) - -print(agent or "") -print(len(items)) -for it in items: - print(it.get("id", "")) - print(it.get("scorer", "")) - print(it.get("version", "")) -PY -} - -for yaml_file in "$MEASUREMENTS_DIR"/*.yaml; do - file_count=$((file_count + 1)) - name="$(basename "$yaml_file")" - stem="${name%.yaml}" - - mapfile -t parsed < <(parse_manifest "$yaml_file") - agent="${parsed[0]:-}" - count="${parsed[1]:-0}" - - file_errors=0 - - if [[ -z "$agent" ]]; then - echo " ERROR: $name: missing 'agent:' field" - errors=$((errors + 1)) - continue - fi - - if [[ "$agent" != "$stem" ]]; then - echo " ERROR: $name: agent '$agent' does not match filename stem '$stem' (jobs fetch \${AGENT}.yaml)" - errors=$((errors + 1)) - file_errors=$((file_errors + 1)) - fi - - if [[ ! -f "$AGENTS_DIR/${agent}.md" ]]; then - echo " ERROR: $name: agent '$agent' has no agents/${agent}.md" - errors=$((errors + 1)) - file_errors=$((file_errors + 1)) - fi - - if [[ "$count" -lt 1 ]]; then - echo " ERROR: $name: measurements list is empty" - errors=$((errors + 1)) - continue - fi - - declare -A seen_ids=() - idx=0 - while [[ $idx -lt $count ]]; do - base=$((2 + idx * 3)) - mid="${parsed[$base]:-}" - scorer="${parsed[$((base + 1))]:-}" - version="${parsed[$((base + 2))]:-}" - idx=$((idx + 1)) - - if [[ -z "$mid" ]]; then - echo " ERROR: $name: measurement #$idx missing id" - errors=$((errors + 1)) - file_errors=$((file_errors + 1)) - continue - fi - if [[ ! "$mid" =~ ^[a-z][a-z0-9]*-[0-9]+$ ]]; then - echo " ERROR: $name: id '$mid' must be lowercase like em-001" - errors=$((errors + 1)) - file_errors=$((file_errors + 1)) - fi - if [[ -n "${seen_ids[$mid]+x}" ]]; then - echo " ERROR: $name: duplicate id '$mid'" - errors=$((errors + 1)) - file_errors=$((file_errors + 1)) - fi - seen_ids["$mid"]=1 - - if [[ -z "$scorer" ]]; then - echo " ERROR: $name: measurement '$mid' missing scorer" - errors=$((errors + 1)) - file_errors=$((file_errors + 1)) - elif [[ " $KNOWN_SCORERS " != *" $scorer "* ]]; then - echo " ERROR: $name: unknown scorer '$scorer' (allowed: $KNOWN_SCORERS)" - errors=$((errors + 1)) - file_errors=$((file_errors + 1)) - fi - - if [[ ! "$version" =~ ^[1-9][0-9]*$ ]]; then - echo " ERROR: $name: measurement '$mid' version must be a positive integer, got '$version'" - errors=$((errors + 1)) - file_errors=$((file_errors + 1)) - fi - done - unset seen_ids - - if [[ $file_errors -eq 0 ]]; then - echo " $name: OK (agent=$agent, $count measurement(s))" - fi -done - -if [[ $file_count -eq 0 ]]; then - echo " ERROR: no eval/measurements/*.yaml files found — expected at least one" - errors=$((errors + 1)) -fi -echo "" -if [[ $errors -gt 0 ]]; then - echo "ERROR: $errors measurement lint failure(s)" >&2 - exit 1 -fi -echo "OK: all measurement manifests pass lint checks" + file_errors = 0 + if not agent: + print(" ERROR: %s: missing 'agent:' field" % name) + errors += 1 + continue + if agent != stem: + print(" ERROR: %s: agent %r does not match filename stem %r (jobs fetch ${AGENT}.yaml)" % (name, agent, stem)) + errors += 1 + file_errors += 1 + if not os.path.isfile(os.path.join(AGENTS_DIR, agent + ".md")): + print(" ERROR: %s: agent %r has no agents/%s.md" % (name, agent, agent)) + errors += 1 + file_errors += 1 + if not items: + print(" ERROR: %s: measurements list is empty" % name) + errors += 1 + continue + + seen = set() + for idx, it in enumerate(items, 1): + mid = it.get("id", "") + scorer = it.get("scorer", "") + version = it.get("version", "") + if not mid: + print(" ERROR: %s: measurement #%d missing id" % (name, idx)) + errors += 1 + file_errors += 1 + continue + if "|" in mid or "\n" in mid: + print(" ERROR: %s: id %r contains characters fullsend LoadRegistry rejects" % (name, mid)) + errors += 1 + file_errors += 1 + if not ID_STYLE.match(mid): + print(" ERROR: %s: id %r must be lowercase like em-001 (agents-repo stock-manifest style)" % (name, mid)) + errors += 1 + file_errors += 1 + if mid in seen: + print(" ERROR: %s: duplicate id %r" % (name, mid)) + errors += 1 + file_errors += 1 + seen.add(mid) + if not scorer: + print(" ERROR: %s: measurement %r missing scorer" % (name, mid)) + errors += 1 + file_errors += 1 + elif scorer not in KNOWN_SCORERS: + print(" ERROR: %s: unknown scorer %r (allowed: %s)" % (name, scorer, ", ".join(sorted(KNOWN_SCORERS)))) + errors += 1 + file_errors += 1 + if not re.match(r"^[1-9][0-9]*$", version or ""): + print(" ERROR: %s: measurement %r version must be a positive integer, got %r" % (name, mid, version)) + errors += 1 + file_errors += 1 + + if file_errors == 0: + print(" %s: OK (agent=%s, %d measurement(s))" % (name, agent, len(items))) + + if file_count == 0: + print(" ERROR: no eval/measurements/*.yaml files found — expected at least one") + errors += 1 + + print("") + if errors: + print("ERROR: %d measurement lint failures" % errors, file=sys.stderr) + return 1 + print("OK: all measurement manifests pass lint checks") + return 0 + + +sys.exit(main()) +PY diff --git a/eval/measurements/README.md b/eval/measurements/README.md index 012aa204..b1346d4c 100644 --- a/eval/measurements/README.md +++ b/eval/measurements/README.md @@ -13,12 +13,18 @@ and here is what we measure on wild `code` runs.” Managed fullsend jobs resolve manifests as: 1. Local `${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml` if present (override / BYOA) -2. Else this repo at the `v0` pin: - `https://raw.githubusercontent.com/fullsend-ai/agents/v0/eval/measurements/${AGENT}.yaml` +2. Else a SHA-pinned fetch from `fullsend-ai/agents` at the `v0` tag: + `fullsend eval-measure` resolves `tags/v0` via GitHub `GetRef` (managed jobs + pass `GH_TOKEN` / `GITHUB_TOKEN`) and then fetches + `eval/measurements/${AGENT}.yaml` at that commit. It does **not** curl the + floating `raw.githubusercontent.com/fullsend-ai/agents/v0/...` URL. Installs that only use stock agents **do not copy these files**. Local files are for changing defaults, opting out, or scoring a custom agent. +Stock manifests in this directory use lowercase ids like `em-001` (an +agents-repo style convention; fullsend's loader only requires a non-empty id). + ## What lives where | Concern | Repo | diff --git a/eval/measurements/code.yaml b/eval/measurements/code.yaml index da4ff144..5d70fdda 100644 --- a/eval/measurements/code.yaml +++ b/eval/measurements/code.yaml @@ -1,4 +1,3 @@ ---- agent: code measurements: - id: em-001 diff --git a/eval/measurements/fix.yaml b/eval/measurements/fix.yaml index 12ce41bd..f1eec35d 100644 --- a/eval/measurements/fix.yaml +++ b/eval/measurements/fix.yaml @@ -1,4 +1,3 @@ ---- agent: fix measurements: - id: em-001 diff --git a/eval/measurements/prioritize.yaml b/eval/measurements/prioritize.yaml index c8b01981..610cc5e4 100644 --- a/eval/measurements/prioritize.yaml +++ b/eval/measurements/prioritize.yaml @@ -1,4 +1,3 @@ ---- agent: prioritize measurements: - id: em-001 diff --git a/eval/measurements/retro.yaml b/eval/measurements/retro.yaml index 882d8219..62895be8 100644 --- a/eval/measurements/retro.yaml +++ b/eval/measurements/retro.yaml @@ -1,4 +1,3 @@ ---- agent: retro measurements: - id: em-001 diff --git a/eval/measurements/review.yaml b/eval/measurements/review.yaml index 39f0f1b7..64e2d91c 100644 --- a/eval/measurements/review.yaml +++ b/eval/measurements/review.yaml @@ -1,4 +1,3 @@ ---- agent: review measurements: - id: em-001 diff --git a/eval/measurements/triage.yaml b/eval/measurements/triage.yaml index f986c783..ef8492df 100644 --- a/eval/measurements/triage.yaml +++ b/eval/measurements/triage.yaml @@ -1,4 +1,3 @@ ---- agent: triage measurements: - id: em-001