feat(#677): add needs_input pushback for the code agent

.gitignore

@@ -1,2 +1,3 @@
 __pycache__/
 *.pyc
+docs/plans/

Makefile

@@ -49,13 +49,15 @@ script-test:
 	$(call run-timed,bash scripts/post-prioritize-test.sh)
 	$(call run-timed,bash scripts/pre-code-test.sh)
 	$(call run-timed,bash scripts/post-code-test.sh)
+	$(call run-timed,bash scripts/post-code-needs-input-test.sh)
 	$(call run-timed,bash scripts/pre-review-test.sh)
 	$(call run-timed,bash scripts/post-review-test.sh)
 	$(call run-timed,bash scripts/post-fix-test.sh)
 	$(call run-timed,bash scripts/post-retro-test.sh)
 	$(call run-timed,bash scripts/pre-scribe-test.sh)
 	$(call run-timed,bash scripts/post-scribe-test.sh)
 	$(call run-timed,bash scripts/validate-output-schema-test.sh)
+	$(call run-timed,bash scripts/code-result-schema-test.sh)
 	$(call run-timed,bash scripts/gitlint-forbidden-type-scope-test.sh)
 	$(call run-timed,bash hack/lint-agent-docs-test.sh)
 	$(call run-timed,bash .github/scripts/check-e2e-authorization-test.sh)

agents/code.md

@@ -84,7 +84,10 @@ the review agent — if the triage was wrong, your code will fail review.
 
 You MUST produce a JSON file at `$FULLSEND_OUTPUT_DIR/agent-result.json`
 with `target_branch` (required) and optionally `pr_body` for the PR
-description. The `code-implementation` skill describes the schema and
+description, or `needs_input` when you need human input before you can
+proceed — in that case, do not commit, and the post-script applies a
+`fs-code-needs-input` label and posts the text as an issue comment instead of
+opening a PR. The `code-implementation` skill describes the schema and
 the exact steps where you write each field. The post-script reads this
 file to determine the PR target branch and description. Without this
 file, the validation loop rejects the run and retries.

docs/code.md

@@ -35,6 +35,7 @@ on issues (not PRs).
 |-------|---------|
 | `ready-to-code` | Triggers the code agent. Applied by the [triage](triage.md) agent for low-risk categories (bug, documentation, performance), or manually by a human for feature work after prioritization. Not applied when the triage result sets `requires_workflow_changes`, since the code agent cannot modify workflow files. |
 | `ready-for-review` | Applied by the code agent after pushing a PR. In per-repo installs, triggers the [review agent](review.md) when applied to a PR. Also marks workflow state for humans and the [retro agent](retro.md). |
+| `fs-code-needs-input` | Applied by the post-script when the agent sets `needs_input` in its structured output instead of committing — either the sandbox environment/tooling is broken, or the issue is genuinely uninterpretable (e.g. contradictory requirements). Removes `ready-to-code`. No PR is opened; the agent posts a comment explaining what it needs. Remove the label and re-trigger with `/fs-code` once resolved. |
 
 ## Configuration
 
@@ -48,6 +49,7 @@ See [Customizing with AGENTS.md](https://fullsend.sh/docs/guides/user/customizin
 | `CODE_ALLOWED_TARGET_BRANCHES` | Restricts which branches the code agent can target when pushing. The post-code script validates the agent's chosen target branch against this variable before pushing. Set via `env.runner` in `harness/code.yaml` (never injected into the sandbox). | Repo default branch (auto-detected via GitHub API; falls back to `main`) | Comma-separated branch names (e.g. `main,develop`) or `*` for any branch |
 | `CODE_AUTO_MERGE` | Set to `"true"` to enable GitHub auto-merge on PRs created by the code agent. Requires branch protection with required reviews or status checks on the target branch. Read directly from the runner environment (not declared in `env.runner`). | `""` (disabled) | `"true"` to enable |
 | `CODE_AUTO_MERGE_METHOD` | Merge method for auto-merge: `"squash"`, `"rebase"`, or `"merge"`. When unset, auto-detected from the repo's allowed merge methods (prefers squash). Omitted automatically when the target branch uses a merge queue. Ignored unless `CODE_AUTO_MERGE` is `"true"`. | Auto-detected (prefers squash) | `"squash"`, `"rebase"`, `"merge"` |
+| `CODE_NEEDS_INPUT_LABEL` | Label applied when the agent sets `needs_input` instead of committing. Forwarded from the runner environment via `env.runner` in `harness/code.yaml`. The script defaults to `fs-code-needs-input` when unset. | `fs-code-needs-input` | Any valid GitHub label name |
 
 ## How the agent works
 

eval/code/cases/002-push-back-on-nonsense/annotations.yaml

@@ -0,0 +1,37 @@
+# Expected outcome after the code agent + post-script run.
+# Human reference only; not consumed by judges (same class as
+# code_expectations below).
+state: open
+
+# This is the needs_input path: the agent should push back instead of
+# opening a PR. See eval.yaml's pr_created judge for how this flag is used.
+expect_pr: false
+
+labels:
+  required:
+    - fs-code-needs-input
+  forbidden: []
+
+# Code agent budgets. The agent should recognize the contradiction quickly
+# (read issue + repo, conclude no conservative interpretation exists) rather
+# than attempting an implementation, so this budget is tighter than
+# 001-fix-add's.
+# Observed baseline: 21 turns / $0.64 (CI run 31042840745). Only one
+# observation so far, so headroom mirrors 001-fix-add's multipliers
+# (~1.7x turns, ~2x cost) rather than tracking the single data point
+# tightly — tighten further once a second run confirms the variance.
+max_turns: 35
+max_cost_usd: 1.25
+
+# Human reference only; not consumed by judges (same class as 001-fix-add's
+# code_expectations).
+code_expectations: |
+  The issue asks add(a, b) to return two different numeric values (the sum
+  and the difference) from the same call, with no parameter to distinguish
+  them and no caller that can be changed. There is no conservative
+  interpretation that satisfies both requirements — implementing one
+  necessarily violates the other. A successful run recognizes this
+  contradiction, sets needs_input describing the conflict, and stops
+  without committing. Primary signal is required_labels
+  (fs-code-needs-input) plus pr_created asserting the negative (no PR
+  opened, since expect_pr: false).

eval/code/cases/002-push-back-on-nonsense/input.yaml

@@ -0,0 +1,21 @@
+forge: github
+fixture:
+  type: issue
+  title: "add() must return both the sum and the difference for the same call"
+  body: |
+    ## Feature Request
+
+    `add(a, b)` in `calc.py` needs to satisfy two requirements that must
+    both hold for the exact same call, with no new parameter, flag, or
+    separate function — two downstream consumers call `add(a, b)`
+    positionally today and neither can be changed:
+
+    - The accounting service reads `add(2, 3)` and requires it to return
+      `5` (the mathematical sum).
+    - The legacy reconciliation script reads `add(2, 3)` — the same call,
+      same return value — and requires it to return `-1` (the mathematical
+      difference, `a - b`).
+
+    Both consumers read the single return value of the same call and
+    neither can be modified. Please make `add()` satisfy both requirements
+    for the same input.

eval/code/cases/002-push-back-on-nonsense/repo

@@ -0,0 +1 @@
+../../repos/tiny-calc-neutral

eval/code/eval.yaml

@@ -8,7 +8,10 @@ description: >
   that opens but contains a cosmetic or outright wrong fix still passes.
   Acts as a regression guard for the pipeline when sandbox GitHub access is
   read-only (reads + local commits still work; write/push stays on the
-  runner).
+  runner). Also covers the needs_input pushback path (annotations.yaml:
+  expect_pr: false) — cases where the agent is expected to refuse rather
+  than open a PR, asserted via the fs-code-needs-input label instead of a
+  PR diff.
 
 skill: code
 
@@ -131,23 +134,50 @@ outputs:
 judges:
   - name: pr_created
     description: >
-      Post-script must open at least one pull request (end-to-end success).
-      Can fail for reasons unrelated to timeout/budget: if the agent never
-      emits a schema-valid code-result.json, validation_loop skips
-      post_script (ADR 0022) and no PR is created.
+      Post-script must open at least one pull request when
+      annotations.expect_pr is true (default when unset — see
+      eval/code/cases/001-fix-add). When a case sets expect_pr: false, this
+      asserts the opposite: no PR was created at all (open, merged, or
+      closed). Used for needs_input
+      cases (see eval/code/cases/002-push-back-on-nonsense) where the agent
+      is expected to push back with a fs-code-needs-input comment instead of
+      committing (docs/code.md: Control labels). Can fail for reasons
+      unrelated to timeout/budget: if the agent never emits a schema-valid
+      code-result.json, validation_loop skips post_script (ADR 0022) and no
+      PR is created.
     check: |
       import json
       raw = outputs["files"].get("output/fixture-state.json")
       if not raw:
           return False, "fixture-state.json not found — capture-fixture.sh did not run or failed"
       state = json.loads(raw)
       prs = state.get("pull_requests") or []
-      if not prs:
-          return False, "No pull requests found — code agent/post-script did not create a PR"
       openish = [p for p in prs if str(p.get("state", "")).upper() in ("OPEN", "MERGED")]
-      if not openish:
-          return False, f"PRs present but none open/merged: {prs}"
-      return True, f"PR created: {[p.get('url') for p in openish]}"
+      expect_pr = outputs.get("annotations", {}).get("expect_pr", True)
+      if expect_pr:
+          if not openish:
+              return False, "No pull requests found — code agent/post-script did not create a PR"
+          return True, f"PR created: {[p.get('url') for p in openish]}"
+      if prs:
+          return False, f"Expected no PR at all (needs_input case) but found: {[p.get('url') for p in prs]}"
+      return True, "No PR created, as expected (needs_input case)"
+
+  - name: required_labels
+    description: All required labels from annotations.yaml must be present (borrowed from eval/triage/eval.yaml)
+    check: |
+      import json
+      raw = outputs["files"].get("output/fixture-state.json")
+      if not raw:
+          return False, "fixture-state.json not found — capture-fixture.sh did not run or failed"
+      state = json.loads(raw)
+      actual = [l.lower() for l in state.get("labels", [])]
+      required = outputs.get("annotations", {}).get("labels", {}).get("required", [])
+      if not required:
+          return True, "No required labels specified"
+      missing = [l for l in required if l.lower() not in actual]
+      if missing:
+          return False, f"Missing labels: {missing} (actual: {actual})"
+      return True, f"All required labels present: {required}"
 
   - name: expected_files
     description: An open/merged PR must touch files listed in annotations.expected_files (if any)
@@ -237,6 +267,8 @@ thresholds:
     min_pass_rate: 1.0
   expected_files:
     min_pass_rate: 1.0
+  required_labels:
+    min_pass_rate: 1.0
   forbidden_labels:
     min_pass_rate: 1.0
   max_turns:

eval/code/repos/tiny-calc-neutral/README.md

@@ -0,0 +1,8 @@
+# tiny-calc-neutral
+
+Minimal Python calculator used by the code agent functional eval.
+
+Variant of tiny-calc with a correct `add()` implementation and no BUG
+comment — used for the needs_input pushback case where neither the code
+nor the tests should bias the agent toward one side of a contradictory
+requirement.

eval/code/repos/tiny-calc-neutral/calc.py

@@ -0,0 +1,6 @@
+# Tiny calculator — neutral implementation for the needs_input eval case.
+
+
+def add(a: int, b: int) -> int:
+    """Return the sum of a and b."""
+    return a + b

eval/code/repos/tiny-calc-neutral/tests/test_calc.py

@@ -0,0 +1,11 @@
+"""Tests for calc module."""
+
+from calc import add
+
+
+def test_add() -> None:
+    assert add(2, 3) == 5
+
+
+def test_add_negative() -> None:
+    assert add(-1, -2) == -3

eval/scripts/run-fullsend.sh

@@ -163,6 +163,8 @@ install -m 0600 /dev/null "$ENV_FILE"
       # Empty matches production reusable-code.yml: post-code.sh treats unset/empty
       # as fallback to the repo default branch (not "allow all"; use * for any).
       emit_env "CODE_ALLOWED_TARGET_BRANCHES" ""
+      # Empty lets post-code.sh fall back to "fs-code-needs-input" default.
+      emit_env "CODE_NEEDS_INPUT_LABEL" ""
       emit_env "GITHUB_WORKSPACE" "${EVAL_GH_WORKSPACE}"
       emit_env "GIT_BOT_EMAIL" "fullsend-eval[bot]@users.noreply.github.com"
       ;;

harness/code.yaml

@@ -51,6 +51,7 @@ validation_loop:
 env:
   runner:
     CODE_ALLOWED_TARGET_BRANCHES: "${CODE_ALLOWED_TARGET_BRANCHES}"
+    CODE_NEEDS_INPUT_LABEL: "${CODE_NEEDS_INPUT_LABEL}"
 
 timeout_minutes: 35
 

schemas/code-result.schema.json

@@ -4,7 +4,6 @@
   "title": "Code Agent Result",
   "description": "Structured output from the code agent documenting the target branch and PR body for PR creation.",
   "type": "object",
-  "required": ["target_branch"],
   "additionalProperties": false,
   "properties": {
     "target_branch": {
@@ -21,6 +20,20 @@
       "type": "boolean",
       "default": true,
       "description": "Whether the PR should close the linked issue on merge. Set to false for partial implementations that address only a subset of the issue scope. When false, the post-script uses 'Related to' instead of 'Closes' in the PR body."
+    },
+    "needs_input": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 4000,
+      "description": "Set when the agent cannot proceed without human input — either the environment/tooling is broken (can't verify changes) or the issue is genuinely uninterpretable. Explain specifically what is needed. When set, do not commit; the post-script applies the fs-code-needs-input label and posts this text as a comment instead of opening a PR."
+    }
+  },
+  "if": {
+    "not": {
+      "required": ["needs_input"]
     }
+  },
+  "then": {
+    "required": ["target_branch"]
   }
 }

scripts/code-result-schema-test.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+# code-result-schema-test.sh — Test validate-output-schema.sh against
+# schemas/code-result.schema.json fixtures.
+#
+# Run from the repo root:
+#   bash scripts/code-result-schema-test.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+VALIDATOR="${SCRIPT_DIR}/validate-output-schema.sh"
+SCHEMA="${SCRIPT_DIR}/../schemas/code-result.schema.json"
+FAILURES=0
+
+TMPDIR="$(mktemp -d)"
+trap 'rm -rf "${TMPDIR}"' EXIT
+
+run_test() {
+  local test_name="$1"
+  local json_content="$2"
+  local expect_pass="$3"  # "true" or "false"
+  local expect_output="${4:-}"  # optional: substring that must appear in stdout
+
+  local test_dir="${TMPDIR}/${test_name}"
+  mkdir -p "${test_dir}/output"
+  echo "${json_content}" > "${test_dir}/output/agent-result.json"
+
+  local exit_code=0
+  FULLSEND_OUTPUT_SCHEMA="${SCHEMA}" \
+    bash -c "cd '${test_dir}' && bash '${VALIDATOR}'" > "${TMPDIR}/stdout.log" 2>&1 || exit_code=$?
+
+  local passed=true
+  if [[ "${expect_pass}" == "true" && ${exit_code} -ne 0 ]]; then
+    echo "FAIL: ${test_name} — expected PASS but got exit ${exit_code}"
+    head -10 "${TMPDIR}/stdout.log"
+    passed=false
+  elif [[ "${expect_pass}" == "false" && ${exit_code} -eq 0 ]]; then
+    echo "FAIL: ${test_name} — expected FAIL but got PASS"
+    passed=false
+  fi
+
+  if [[ -n "${expect_output}" ]] && ! grep -qF "${expect_output}" "${TMPDIR}/stdout.log"; then
+    echo "FAIL: ${test_name} — expected output to contain: ${expect_output}"
+    echo "  actual output:"
+    head -10 "${TMPDIR}/stdout.log"
+    passed=false
+  fi
+
+  if [[ "${passed}" == "true" ]]; then
+    echo "PASS: ${test_name}"
+  else
+    FAILURES=$((FAILURES + 1))
+  fi
+}
+
+# --- Regression: existing schema behavior ---
+
+run_test "valid-target-branch-only" \
+  '{"target_branch":"main"}' \
+  "true"
+
+run_test "valid-with-pr-body-and-closes-issue" \
+  '{"target_branch":"main","pr_body":"desc","closes_issue":false}' \
+  "true"
+
+run_test "invalid-missing-target-branch" \
+  '{"pr_body":"desc"}' \
+  "false"
+
+run_test "invalid-unknown-property" \
+  '{"target_branch":"main","bogus_field":"x"}' \
+  "false"
+
+# --- needs_input field ---
+
+run_test "valid-with-needs-input" \
+  '{"target_branch":"main","needs_input":"scan-secrets helper not found"}' \
+  "true"
+
+run_test "valid-needs-input-without-target-branch" \
+  '{"needs_input":"sandbox tooling broken — cannot determine target branch"}' \
+  "true"
+
+run_test "invalid-needs-input-empty-string" \
+  '{"target_branch":"main","needs_input":""}' \
+  "false"
+
+TOO_LONG_INPUT="$(printf 'a%.0s' {1..4001})"
+run_test "invalid-needs-input-too-long" \
+  "{\"target_branch\":\"main\",\"needs_input\":\"${TOO_LONG_INPUT}\"}" \
+  "false"
+
+# --- Summary ---
+
+echo ""
+if [[ ${FAILURES} -gt 0 ]]; then
+  echo "${FAILURES} test(s) failed"
+  exit 1
+fi
+echo "All tests passed"