diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d690aed7..1f5111bb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,3 +52,9 @@ repos: language: script files: ^(harness/|docs/|hack/lint-agent-docs) pass_filenames: false + - id: lint-measurements + name: lint eval measurement manifests + entry: ./eval/lint-measurements.sh + language: script + files: ^(eval/measurements/|eval/lint-measurements) + pass_filenames: false diff --git a/LOCAL.md b/LOCAL.md index 125105d6..ceb93b04 100644 --- a/LOCAL.md +++ b/LOCAL.md @@ -176,8 +176,10 @@ fullsend run triage \ ## Functional eval tests The `eval/` directory contains functional test scenarios that run agents -against ephemeral GitHub repos and score the results. See -[eval/README.md](eval/README.md) for setup and usage. +against ephemeral GitHub repos and score the results, plus default +online-scoring manifests under [`eval/measurements/`](eval/measurements/README.md) +consumed by `fullsend eval-measure`. See [eval/README.md](eval/README.md) +for setup and usage. To run triage evals: diff --git a/Makefile b/Makefile index 5229f5a9..0a363324 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,7 @@ script-test: $(call run-timed,bash scripts/validate-output-schema-test.sh) $(call run-timed,bash scripts/gitlint-forbidden-type-scope-test.sh) $(call run-timed,bash hack/lint-agent-docs-test.sh) + $(call run-timed,bash eval/lint-measurements-test.sh) $(call run-timed,bash .github/scripts/check-e2e-authorization-test.sh) $(call run-timed,bash .github/scripts/select-eval-agents-test.sh) $(call run-timed,python3 scripts/process-fix-result-test.py) diff --git a/README.md b/README.md index 4ed9d6b8..8ac547bd 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ schemas/ JSON Schema for validating agent structured output scripts/ Pre-scripts (input validation) and post-scripts (forge mutations) skills/ Reusable skill definitions loaded by agents at runtime plugins/ Sandbox plugins (e.g. gopls LSP for the code agent) +eval/ Functional eval harness and default online-scoring manifests ``` ## Architecture diff --git a/eval/README.md b/eval/README.md index ada3df95..df09aeb0 100644 --- a/eval/README.md +++ b/eval/README.md @@ -30,6 +30,7 @@ running: ```bash bash eval/lint-cases.sh +bash eval/lint-measurements.sh ``` ## Prerequisites @@ -137,3 +138,16 @@ Each test case follows this lifecycle: - **`checkStatus` drops string errors.** fullsend's `checkStatus` does not handle string-typed error responses from the GitHub API, causing silent failures. + +## Measurement manifests (online scoring) + +Per-agent manifests under [`eval/measurements/`](./measurements/) are the +**default online-scoring policy** for stock agents (which scorers run after +managed jobs via `fullsend eval-measure`). They are **not** functional PR-gate +scenarios under `eval//`. + +Scorer *implementations* live in `fullsend-ai/fullsend`; this repo only +declares defaults. Jobs fetch these files from `agents@v0` unless a consumer +overrides under `FULLSEND_DIR`. See [`eval/measurements/README.md`](./measurements/README.md) +and [fullsend#6036](https://github.com/fullsend-ai/fullsend/pull/6036) (ADR 0087 +lands with that PR). diff --git a/eval/lint-measurements-test.sh b/eval/lint-measurements-test.sh new file mode 100755 index 00000000..6b4a1865 --- /dev/null +++ b/eval/lint-measurements-test.sh @@ -0,0 +1,301 @@ +#!/usr/bin/env bash +# lint-measurements-test.sh — Tests for eval/lint-measurements.sh +# +# Run from the repo root: +# bash eval/lint-measurements-test.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LINTER="${SCRIPT_DIR}/lint-measurements.sh" + +FAILURES=0 +WORKDIR="$(mktemp -d)" +trap 'rm -rf "${WORKDIR}"' EXIT + +VALID_YAML='--- +agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +' + +# run_case NAME YAML EXPECTED_EXIT [EXPECTED_OUTPUT_SUBSTRING] [FILENAME] +run_case() { + local name="$1" yaml="$2" expected_exit="$3" expected_substring="${4:-}" filename="${5:-code.yaml}" + + local case_dir="${WORKDIR}/${name}" + mkdir -p "${case_dir}/eval/measurements" "${case_dir}/agents" + printf '%s\n' "# Code Agent" > "${case_dir}/agents/code.md" + if [[ -n "$yaml" ]]; then + printf '%s' "$yaml" > "${case_dir}/eval/measurements/${filename}" + fi + + local output + local actual_exit=0 + output="$(REPO_ROOT="${case_dir}" MEASUREMENTS_DIR="${case_dir}/eval/measurements" AGENTS_DIR="${case_dir}/agents" "${LINTER}" 2>&1)" || actual_exit=$? + + if [[ "${actual_exit}" != "${expected_exit}" ]]; then + echo "FAIL: ${name} (exit ${actual_exit}, expected ${expected_exit})" + echo "${output}" | sed 's/^/ /' + FAILURES=$((FAILURES + 1)) + return + fi + + if [[ -n "${expected_substring}" ]] && [[ "${output}" != *"${expected_substring}"* ]]; then + echo "FAIL: ${name} (missing expected output: '${expected_substring}')" + echo "${output}" | sed 's/^/ /' + FAILURES=$((FAILURES + 1)) + return + fi + + echo "PASS: ${name}" +} + +run_case "valid-manifest-passes" \ + "${VALID_YAML}" 0 "code.yaml: OK" + +run_case "unknown-scorer" \ + "--- +agent: code +measurements: + - id: em-001 + scorer: trace-fitness + version: 1 +" 1 "unknown scorer 'trace-fitness'" + +run_case "unknown-agent" \ + "--- +agent: not-an-agent +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +" 1 "has no agents/not-an-agent.md" "not-an-agent.yaml" + +run_case "filename-agent-mismatch" \ + "--- +agent: review +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +" 1 "does not match filename stem" "code.yaml" + +run_case "duplicate-id" \ + "--- +agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + - id: em-001 + scorer: trace_fitness + version: 1 +" 1 "duplicate id 'em-001'" + +run_case "uppercase-id" \ + "--- +agent: code +measurements: + - id: EM-001 + scorer: trace_fitness + version: 1 +" 1 "must be lowercase like em-001" + +run_case "missing-version" \ + "--- +agent: code +measurements: + - id: em-001 + scorer: trace_fitness +" 1 "missing version" + +run_case "empty-dir" \ + "" 1 "no eval/measurements/*.yaml files found" + +run_case "flow-style-unsupported" \ + "agent: code +measurements: [{id: em-001, scorer: trace_fitness, version: 1}] +" 1 "unsupported YAML shape" + +run_case "nested-assert-unsupported" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + assert: + - path: x +" 1 "unsupported YAML shape" + +run_case "trailing-unknown-top-level" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +not_a_real_key: true +" 1 "unsupported top-level field" + +run_case "typo-mesurements-after-list" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +mesurements: + - id: em-002 + scorer: totally_bogus_scorer + version: 1 +" 1 "unsupported top-level field" + +run_case "optional-name-allowed" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + name: Trace Fitness +" 0 "code.yaml: OK" + +run_case "name-pipe-rejected" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + name: bad|name +" 1 "contains characters fullsend LoadRegistry rejects" + +run_case "hash-in-agent-rejected" \ + "agent: code#x +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +" 1 "must not contain '#'" + +run_case "hash-in-scorer-rejected" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness#typo + version: 1 +" 1 "must not contain '#'" + +run_case "quoted-version-rejected" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: \"1\" +" 1 "unquoted integer" + +run_case "duplicate-agent-key-rejected" \ + "agent: code +agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +" 1 "duplicate top-level key" + +run_case "real-comment-still-ok" \ + "agent: code # stock agent +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +" 0 "code.yaml: OK" + +run_case "nested-block-map-under-name-rejected" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + name: + id: em-001 +" 1 "nested block map" + +run_case "empty-then-filled-agent-duplicate" \ + "agent: +agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +" 1 "duplicate top-level key" + +run_case "duplicate-id-in-item-rejected" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + id: em-002 +" 1 "duplicate key" + +run_case "duplicate-name-in-item-rejected" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + name: Trace Fitness + name: Also Fitness +" 1 "duplicate key" + +run_case "flow-map-name-rejected" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + name: {a: 1} +" 1 "flow collection" + +run_case "flow-seq-name-rejected" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + name: [Trace] +" 1 "flow collection" + +run_case "yaml-null-name-rejected" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + name: null +" 1 "YAML literal" + +run_case "quoted-hash-in-name-rejected" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + name: \"foo # bar\" +" 1 "must not contain '#'" + +run_case "quoted-name-with-trailing-comment-ok" \ + "agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + name: \"Trace Fitness\" # display label +" 0 "code.yaml: OK" + +echo "" +if [[ ${FAILURES} -gt 0 ]]; then + echo "${FAILURES} test(s) failed" + exit 1 +fi +echo "All tests passed" diff --git a/eval/lint-measurements.sh b/eval/lint-measurements.sh new file mode 100755 index 00000000..cb51d883 --- /dev/null +++ b/eval/lint-measurements.sh @@ -0,0 +1,395 @@ +#!/usr/bin/env bash +# Lint eval/measurements/*.yaml — catch typos that would silently no-op +# at runtime (unknown scorer / agent name that never matches a trace). +# +# bash 3.2 compatible (macOS /usr/bin/bash). Validation runs in python3; +# this wrapper does not use mapfile or declare -A. +# +# Usage: +# ./eval/lint-measurements.sh +# +# Checks (mirrored LoadRegistry rules + agents-repo style extras): +# - Filename stem equals the top-level `agent:` field +# - `agent:` matches an existing agents/.md +# (necessary but not sufficient for stock fetch: fullsend only pulls +# agents in defaultAgentsRepoKnownAgents — triage/code/fix/review/ +# retro/prioritize today; see measurements README) +# - Each measurement has id, scorer, and a positive unquoted integer version +# (LoadRegistry: Version is int — quoted "1" fails yaml.v3) +# - optional name: allowed; rejects pipe/newline in id/scorer/name +# - ids unique per file; no duplicate top-level agent:/measurements: +# - scorer in known-scorer allow-list (fullsend ScorerFitness) +# - id matches em-001-style lowercase (agents-repo style, not LoadRegistry) +# - YAML comment stripping matches YAML (# only after whitespace / col 0); +# residual # inside agent/id/scorer/name values is rejected +# - Shipped block-style shape only; other shapes fail closed +set -euo pipefail + +REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" +MEASUREMENTS_DIR="${MEASUREMENTS_DIR:-$REPO_ROOT/eval/measurements}" +AGENTS_DIR="${AGENTS_DIR:-$REPO_ROOT/agents}" + +if ! command -v python3 >/dev/null 2>&1; then + echo "ERROR: python3 is required" >&2 + exit 1 +fi + +# Command substitution (not process substitution) so python's exit code is visible. +python3 - "$MEASUREMENTS_DIR" "$AGENTS_DIR" <<'PY' +import os +import re +import sys + +MEASUREMENTS_DIR, AGENTS_DIR = sys.argv[1], sys.argv[2] +KNOWN_SCORERS = frozenset({"trace_fitness"}) +ID_STYLE = re.compile(r"^[a-z][a-z0-9]*-[0-9]+$") +# Matches fullsend MeasurementSpec yaml tags shipped today (id/scorer/version/name). +FIELD_KEYS = frozenset({"id", "scorer", "version", "name"}) +TOP_LEVEL_KEYS = frozenset({"agent", "measurements"}) +FIELD_INLINE = re.compile(r"^(id|scorer|version|name):\s*(.*)$") +FIELD_INDENTED = re.compile(r"^\s+(id|scorer|version|name):\s*(.*)$") +UNKNOWN_INDENTED = re.compile(r"^\s+([A-Za-z0-9_]+):\s*(.*)$") + + +class UnsupportedShape(Exception): + pass + + +def strip_yaml_comment(raw): + """Strip a YAML comment: '#' at column 0 or after whitespace, outside quotes. + + A '#' inside a single- or double-quoted scalar is part of the value, not a + comment start — otherwise quoted values like name: "foo # bar" are truncated + before parse_scalar can reject residual '#'. + """ + in_single = False + in_double = False + i = 0 + while i < len(raw): + c = raw[i] + if in_single: + if c == "'": + # YAML single-quoted escape: '' → literal ' + if i + 1 < len(raw) and raw[i + 1] == "'": + i += 2 + continue + in_single = False + i += 1 + continue + if in_double: + if c == "\\" and i + 1 < len(raw): + i += 2 + continue + if c == '"': + in_double = False + i += 1 + continue + if c == "'": + in_single = True + i += 1 + continue + if c == '"': + in_double = True + i += 1 + continue + if c == "#" and (i == 0 or raw[i - 1].isspace()): + return raw[:i].rstrip() + i += 1 + return raw.rstrip() + + +def parse_scalar(key, raw_token): + """Parse a scalar the way LoadRegistry / yaml.v3 expects for this schema.""" + token = raw_token.strip() + if not token: + if key in ("id", "scorer", "agent", "version"): + raise UnsupportedShape("%s: requires a non-empty value" % key) + return token + # Flow collections cannot unmarshal into string/int MeasurementSpec fields. + if token[0] in "{[": + raise UnsupportedShape( + "%s must be a plain scalar, not a flow collection (got %r)" % (key, token) + ) + if key == "version": + # MeasurementSpec.Version is int; yaml.v3 rejects !!str ("1" / '1'). + if (len(token) >= 2 and token[0] == token[-1] and token[0] in "\"'"): + raise UnsupportedShape( + "version must be an unquoted integer (LoadRegistry int field), got %r" % token + ) + if not re.match(r"^[1-9][0-9]*$", token): + raise UnsupportedShape("version must be a positive integer, got %r" % token) + return token + if len(token) >= 2 and token[0] == token[-1] and token[0] in "\"'": + token = token[1:-1] + if "#" in token: + raise UnsupportedShape( + "%s value must not contain '#' (got %r); YAML treats it as part of the scalar" + % (key, token) + ) + # Unquoted YAML literals that yaml.v3 would not keep as plain strings for our fields. + if token in ("null", "Null", "NULL", "~", "true", "True", "TRUE", "false", "False", "FALSE"): + raise UnsupportedShape( + "%s must not be the unquoted YAML literal %r (LoadRegistry string/int fields)" + % (key, token) + ) + return token + + +def require_top_level(stripped): + key = stripped.split(":", 1)[0] + if key not in TOP_LEVEL_KEYS: + raise UnsupportedShape("unsupported top-level field %r" % key) + return key + + +def parse_manifest(text): + """Parse the shipped block-style schema. Raise UnsupportedShape otherwise.""" + agent = None + items = [] + current = None + in_measurements = False + measurements_key = False + list_indent = None + item_field_indent = None + seen_top = set() + seen_item_keys = set() + + def close_current(): + nonlocal current, item_field_indent, seen_item_keys + if current is not None: + items.append(current) + current = None + item_field_indent = None + seen_item_keys = set() + + def note_top(key): + if key in seen_top: + raise UnsupportedShape("duplicate top-level key %r (yaml.v3 rejects this)" % key) + seen_top.add(key) + + def set_item_field(key, raw_token, field_indent): + nonlocal item_field_indent + if item_field_indent is None: + item_field_indent = field_indent + elif field_indent > item_field_indent: + raise UnsupportedShape( + "nested block map under measurement field is not supported (deeper than item field indent)" + ) + elif field_indent < item_field_indent: + raise UnsupportedShape("inconsistent measurement field indentation") + if key in seen_item_keys: + raise UnsupportedShape("duplicate key %r in measurement item (yaml.v3 rejects this)" % key) + seen_item_keys.add(key) + current[key] = parse_scalar(key, raw_token) + + def set_agent(stripped): + nonlocal agent + note_top("agent") + raw = stripped.split(":", 1)[1] + if not raw.strip(): + # Empty first occurrence still counts so a later agent: value is a duplicate. + return + agent = parse_scalar("agent", raw) + + def start_measurements(): + nonlocal in_measurements, measurements_key, list_indent + note_top("measurements") + in_measurements = True + measurements_key = True + list_indent = None + + def handle_top_level(stripped): + nonlocal in_measurements + key = require_top_level(stripped) + if key == "agent": + set_agent(stripped) + elif key == "measurements": + start_measurements() + else: + note_top(key) + + for raw in text.splitlines(): + stripped = strip_yaml_comment(raw) + if not stripped or stripped == "---": + continue + if re.search(r"measurements:\s*[\[{]", stripped): + raise UnsupportedShape("flow-style measurements are not supported") + indent = len(raw) - len(raw.lstrip(" ")) + + if re.match(r"^agent:", stripped): + if in_measurements: + close_current() + in_measurements = False + set_agent(stripped) + continue + + if re.match(r"^measurements:\s*$", stripped): + if in_measurements: + close_current() + start_measurements() + continue + + if in_measurements and re.match(r"^-\s+", stripped.lstrip()): + dash_indent = indent + if list_indent is None: + list_indent = dash_indent + if dash_indent > (list_indent or 0) and current is not None: + raise UnsupportedShape("nested list under a measurement is not supported") + close_current() + current = {} + rest = re.sub(r"^-\s+", "", stripped.lstrip()) + m = FIELD_INLINE.match(rest) + if rest and not m: + raise UnsupportedShape("unsupported field on measurement list item") + if m: + # Dash-line field records the key but does not set item_field_indent — + # the first indented field below establishes the item's field column. + if m.group(1) in seen_item_keys: + raise UnsupportedShape( + "duplicate key %r in measurement item (yaml.v3 rejects this)" % m.group(1) + ) + seen_item_keys.add(m.group(1)) + current[m.group(1)] = parse_scalar(m.group(1), m.group(2)) + continue + + if in_measurements and current is not None: + fm = FIELD_INDENTED.match(stripped) + um = UNKNOWN_INDENTED.match(stripped) + if fm: + set_item_field(fm.group(1), fm.group(2), indent) + continue + if um and um.group(1) not in FIELD_KEYS: + raise UnsupportedShape("unsupported field %r" % um.group(1)) + if re.match(r"^\S", stripped): + close_current() + in_measurements = False + handle_top_level(stripped) + continue + raise UnsupportedShape("unrecognized line in measurements list") + + if in_measurements and current is None: + if re.match(r"^\S", stripped): + in_measurements = False + handle_top_level(stripped) + continue + raise UnsupportedShape("indented content outside a measurement item") + + if re.match(r"^\S", stripped): + handle_top_level(stripped) + continue + raise UnsupportedShape("indented content outside measurements list") + + close_current() + if measurements_key and not items: + raise UnsupportedShape("measurements present but no block-style items parsed") + return agent, items + + +def main(): + print("Checking measurement manifests...") + print("================================================") + errors = 0 + file_count = 0 + + try: + names = sorted(n for n in os.listdir(MEASUREMENTS_DIR) if n.endswith(".yaml")) + except OSError: + names = [] + + for name in names: + file_count += 1 + path = os.path.join(MEASUREMENTS_DIR, name) + stem = name[:-5] + try: + text = open(path, encoding="utf-8").read() + agent, items = parse_manifest(text) + except UnsupportedShape as e: + print(" ERROR: %s: unsupported YAML shape (%s)" % (name, e)) + errors += 1 + continue + except Exception as e: + print(" ERROR: %s: parser failed: %s" % (name, e)) + errors += 1 + continue + + file_errors = 0 + if not agent: + print(" ERROR: %s: missing 'agent:' field" % name) + errors += 1 + continue + if agent != stem: + print(" ERROR: %s: agent %r does not match filename stem %r (jobs fetch ${AGENT}.yaml)" % (name, agent, stem)) + errors += 1 + file_errors += 1 + if not os.path.isfile(os.path.join(AGENTS_DIR, agent + ".md")): + print(" ERROR: %s: agent %r has no agents/%s.md" % (name, agent, agent)) + errors += 1 + file_errors += 1 + if not items: + print(" ERROR: %s: measurements list is empty" % name) + errors += 1 + continue + + seen = set() + for idx, it in enumerate(items, 1): + mid = it.get("id", "") + scorer = it.get("scorer", "") + display = it.get("name", "") + if not mid: + print(" ERROR: %s: measurement #%d missing id" % (name, idx)) + errors += 1 + file_errors += 1 + continue + if "|" in mid or "\n" in mid: + print(" ERROR: %s: id %r contains characters fullsend LoadRegistry rejects" % (name, mid)) + errors += 1 + file_errors += 1 + if not ID_STYLE.match(mid): + print(" ERROR: %s: id %r must be lowercase like em-001 (agents-repo stock-manifest style)" % (name, mid)) + errors += 1 + file_errors += 1 + if mid in seen: + print(" ERROR: %s: duplicate id %r" % (name, mid)) + errors += 1 + file_errors += 1 + seen.add(mid) + if not scorer: + print(" ERROR: %s: measurement %r missing scorer" % (name, mid)) + errors += 1 + file_errors += 1 + elif "|" in scorer or "\n" in scorer: + print(" ERROR: %s: scorer %r contains characters fullsend LoadRegistry rejects" % (name, scorer)) + errors += 1 + file_errors += 1 + elif scorer not in KNOWN_SCORERS: + print(" ERROR: %s: unknown scorer %r (allowed: %s)" % (name, scorer, ", ".join(sorted(KNOWN_SCORERS)))) + errors += 1 + file_errors += 1 + if "version" not in it: + # parse_scalar already validates present version values; this only + # catches a missing version key (item never saw a version: line). + print(" ERROR: %s: measurement %r missing version" % (name, mid)) + errors += 1 + file_errors += 1 + if display and ("|" in display or "\n" in display): + print(" ERROR: %s: name %r contains characters fullsend LoadRegistry rejects" % (name, display)) + errors += 1 + file_errors += 1 + + if file_errors == 0: + print(" %s: OK (agent=%s, %d measurement(s))" % (name, agent, len(items))) + + if file_count == 0: + print(" ERROR: no eval/measurements/*.yaml files found — expected at least one") + errors += 1 + + print("") + if errors: + print("ERROR: %d measurement lint failures" % errors, file=sys.stderr) + return 1 + print("OK: all measurement manifests pass lint checks") + return 0 + + +sys.exit(main()) +PY diff --git a/eval/measurements/README.md b/eval/measurements/README.md new file mode 100644 index 00000000..1d6b3a1a --- /dev/null +++ b/eval/measurements/README.md @@ -0,0 +1,74 @@ +# Measurement manifests + +Per-agent YAML that selects which **eval measurement** scorers run after a +managed agent job (`fullsend eval-measure`). This is **not** the functional +eval harness under `eval//` (PR-gate scenarios / fixtures). + +## Why this lives next to the agents + +These files are the **default online-scoring policy** for the stock fullsend +agents — the same idea as shipping the agents themselves: “here is `code`, +and here is what we measure on wild `code` runs.” + +Managed fullsend jobs resolve manifests as: + +1. Local `${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml` if present (override / BYOA) +2. Else a SHA-pinned fetch from `fullsend-ai/agents` at the `v0` tag: + `fullsend eval-measure` resolves `tags/v0` via GitHub `GetRef` and then + fetches `eval/measurements/${AGENT}.yaml` at that commit. It does **not** + curl the floating `raw.githubusercontent.com/fullsend-ai/agents/v0/...` URL. + `agents` is public, so the `GetRef` works without a token on both GitHub + Actions and GitLab; unauthenticated calls share GitHub's ~60 req/hr per-IP + limit, so export `GH_TOKEN`/`GITHUB_TOKEN` on busy shared runners. The + managed fullsend Action forwards `inputs.github_token` as `GH_TOKEN` into + the measure step; callers outside that Action must export a token + themselves. A local `FULLSEND_DIR` manifest skips the fetch entirely. + +Installs that only use stock agents **do not copy these files**. Local files +are for changing defaults, opting out, or scoring a custom agent. + +Stock manifests in this directory use lowercase ids like `em-001` (an +agents-repo style convention). fullsend's `LoadRegistry` requires a non-empty +`id` and `scorer`, `version >= 1`, and rejects pipe/newline in `id` / `scorer` +/ optional `name`. + +## What lives where + +| Concern | Repo | +|---|---| +| Scorer **implementations** (Go), parser, CLI, job wiring | [`fullsend-ai/fullsend`](https://github.com/fullsend-ai/fullsend) (`internal/evalmeasure/`) | +| Default manifests (which `id` / `scorer` / `version` per agent) | **This directory** | +| Org/repo overrides and BYOA manifests | Consumer `FULLSEND_DIR` | + +Executable logic stays in fullsend because `fullsend eval-measure` is the +released binary that reads `run-telemetry.jsonl` (produced by fullsend). This +repo is content/policy, not that binary. Platform checks like em-001 +(`trace_fitness`) still get **enabled** here for each stock agent. + +| Change | PR | +|---|---| +| New Go scorer or (future) new declarative `assert:` / thresholds | `fullsend` | +| New measurement id / enable / disable for a stock agent using an existing scorer | **agents** (this repo) | +| Custom policy for one org or a BYOA agent | Local override in the consumer repo | + +Companion platform PR: [fullsend-ai/fullsend#6036](https://github.com/fullsend-ai/fullsend/pull/6036) +(ADR 0087 lands with that PR; the `docs/ADRs/0087-*.md` path is not on +`fullsend` main until #6036 merges). + +## First ship + +Six agents enable `trace_fitness` (em-001): code, fix, prioritize, retro, +review, and triage. Omit a file to leave an agent without defaults (e.g. +scribe has no forge work-item identity today). A file under this directory +only takes effect for agents in fullsend's first-party fetch allow-list +(`defaultAgentsRepoKnownAgents` in `internal/cli/run.go` — currently those +six). Adding a stock manifest for a new agent (or scribe) needs a fullsend +change first; `agents/.md` alone is not enough. + +```yaml +agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +``` diff --git a/eval/measurements/code.yaml b/eval/measurements/code.yaml new file mode 100644 index 00000000..5d70fdda --- /dev/null +++ b/eval/measurements/code.yaml @@ -0,0 +1,5 @@ +agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 diff --git a/eval/measurements/fix.yaml b/eval/measurements/fix.yaml new file mode 100644 index 00000000..f1eec35d --- /dev/null +++ b/eval/measurements/fix.yaml @@ -0,0 +1,5 @@ +agent: fix +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 diff --git a/eval/measurements/prioritize.yaml b/eval/measurements/prioritize.yaml new file mode 100644 index 00000000..610cc5e4 --- /dev/null +++ b/eval/measurements/prioritize.yaml @@ -0,0 +1,5 @@ +agent: prioritize +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 diff --git a/eval/measurements/retro.yaml b/eval/measurements/retro.yaml new file mode 100644 index 00000000..62895be8 --- /dev/null +++ b/eval/measurements/retro.yaml @@ -0,0 +1,5 @@ +agent: retro +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 diff --git a/eval/measurements/review.yaml b/eval/measurements/review.yaml new file mode 100644 index 00000000..64e2d91c --- /dev/null +++ b/eval/measurements/review.yaml @@ -0,0 +1,5 @@ +agent: review +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 diff --git a/eval/measurements/triage.yaml b/eval/measurements/triage.yaml new file mode 100644 index 00000000..ef8492df --- /dev/null +++ b/eval/measurements/triage.yaml @@ -0,0 +1,5 @@ +agent: triage +measurements: + - id: em-001 + scorer: trace_fitness + version: 1