Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -533,7 +533,9 @@ for different things:

When a run does fail, the advice it prints comes from the condition it ended on. If the
rejected proposal included notes, the failure prints those too instead of hiding useful
diagnostic context. Run `backpass propose` again to start a fresh synthesis.
diagnostic context. A failed synthesis prompt call reports the assembled prompt's measured
UTF-8 byte size without printing its contents. That size is diagnostic context, not a
claim that the prompt was too large. Run `backpass propose` again to start a fresh synthesis.

Token deltas shown to you are measured by backpass from the actual text - never taken from
the model's own arithmetic.
Expand Down
2 changes: 1 addition & 1 deletion src/agents.js
Original file line number Diff line number Diff line change
Expand Up @@ -567,7 +567,7 @@ function pinnedFailureError(role, pick, verdict, err) {
} else if (verdict) {
hint = pin;
}
const detail = err?.message ? ` (${String(err.message).split("\n")[0]})` : "";
const detail = err?.message ? ` (${String(err.message)})` : "";
return new UserError(`pinned ${role} agent ${pick.agent}${model} ${label}${detail}`, hint);
}

Expand Down
16 changes: 12 additions & 4 deletions src/synthesize.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import fs from "node:fs";
import path from "node:path";

import { extractJson, isBlankOutput, openSession, usageRecord } from "./acpx.js";
import { AcpxError, extractJson, isBlankOutput, openSession, usageRecord } from "./acpx.js";
import { expandHomePath, userClaudeSkillsDir } from "./config.js";
import { renderEvidenceForPrompt } from "./fold.js";
import { renderInstructionIndex, resolveMemoryPath } from "./memory.js";
Expand Down Expand Up @@ -704,9 +704,17 @@ export async function synthesizeProposal({
const holder = {
session: null,
ranWith,
prompt(args) {
async prompt(args) {
if (!this.session) throw new Error("synthesis session is not open");
return this.session.prompt(args);
// Measure the assembled UTF-8 artifact before handing it to the harness. Size is
// diagnostic context for any failed turn, not a verdict about why it failed.
const promptBytes = fs.statSync(args.promptFile).size;
try {
return await this.session.prompt(args);
} catch (err) {
if (err instanceof AcpxError) err.message += `\nSynthesis prompt size: ${promptBytes} UTF-8 bytes.`;
throw err;
}
},
};
let attempted = false;
Expand All @@ -731,7 +739,7 @@ export async function synthesizeProposal({
writeAccess: true,
});
try {
return await holder.session.prompt({
return await holder.prompt({
promptFile: editPromptFile,
approveAll: true,
timeoutSeconds,
Expand Down
66 changes: 64 additions & 2 deletions test/synthesize.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,15 @@ if (argv.includes("sessions")) {
if (argv.includes("new")) process.stdout.write("fake-session-id\\n");
process.exit(0);
}
if (argv.includes("set")) process.exit(0);
if (argv.includes("set") || argv.includes("set-mode")) process.exit(0);
const fileAt = argv.indexOf("--file");
if (fileAt < 0) process.exit(2);
const prompt = fs.readFileSync(argv[fileAt + 1], "utf8");
const phase = prompt.includes("## Measured changes") ? "annotate" : "edit";
if (script.fail === phase) {
process.stderr.write("synthetic adapter failure");
process.exit(1);
}
const statePath = process.env.FAKE_ACPX_STATE;
const state = fs.existsSync(statePath) ? JSON.parse(fs.readFileSync(statePath, "utf8")) : { annotate: 0 };
function applyEdits(edits) {
Expand Down Expand Up @@ -79,6 +84,8 @@ const { parseMemoryUnits, readMemoryFile } = await import("../src/memory.js");
const { foldEvidence } = await import("../src/fold.js");
const { ProposalViolation } = await import("../src/proposal.js");
const { State } = await import("../src/state.js");
const { AcpxError } = await import("../src/acpx.js");
const { AgentResolver } = await import("../src/agents.js");
const { UserError, setLoggerSink } = await import("../src/logger.js");
const { workspacePathFor } = await import("../src/workspace.js");
const { makeRepo } = await import("./helpers/staging.js");
Expand Down Expand Up @@ -205,6 +212,57 @@ function setup(
return { repo, config, memoryFile, externalSkillsDir, run, calls };
}

test("failed synthesis reports the failed prompt's UTF-8 bytes without its contents", async () => {
for (const { phase, repeats } of [
{ phase: "edit", repeats: 1 },
{ phase: "edit", repeats: 180_000 },
{ phase: "annotate", repeats: 1 },
]) {
const privateText = `SYNTH_PRIVATE_${"密🙂".repeat(repeats)}`;
const { config, run, calls } = setup(
{ fail: phase, edit: {}, annotations: [] },
{
text: `# Memory\n\n- ${privateText}\n`,
overrides: { synthesis: { agent: "codex", model: null } },
},
);
config.agents = new AgentResolver(config);
await assert.rejects(run, (err) => {
if (phase === "edit") {
assert.ok(err instanceof UserError);
assert.match(err.message, /pinned synthesis agent codex/);
assert.match(err.hint, /check the codex\/acpx failure above and retry/);
} else {
assert.ok(err instanceof AcpxError);
assert.equal(err.code, 1);
assert.equal(err.stderr, "synthetic adapter failure");
}
assert.equal(err.cause, undefined);
assert.match(err.message, /synthetic adapter failure/);
const invocation = calls()
.filter((call) => call.argv.includes("--file"))
.at(-1);
const promptFile = invocation.argv[invocation.argv.indexOf("--file") + 1];
const prompt = fs.readFileSync(promptFile, "utf8");
const bytes = Buffer.byteLength(prompt, "utf8");
assert.match(err.message, new RegExp(`synthesis prompt size: ${bytes} UTF-8 bytes`, "i"));
assert.match(err.stack, new RegExp(`synthesis prompt size: ${bytes} UTF-8 bytes`, "i"));
if (phase === "edit") {
assert.ok(bytes > prompt.length, "non-ASCII bytes must not be counted as characters");
assert.equal(bytes > 1_000_000, repeats > 1);
}
assert.ok(!err.message.includes("SYNTH_PRIVATE_"));
assert.ok(!err.stack.includes("SYNTH_PRIVATE_"));
assert.ok(!err.message.includes("too large"), "size is context, not an inferred cause");
return true;
});
assert.ok(
calls().some((call) => call.argv.includes("close")),
"failed sessions are closed",
);
}
});

test("root and nested synthesis both refuse direct writes to the other memory file", async () => {
for (const pass of ["root", "nested"]) {
const { repo, config, run } = setup(
Expand Down Expand Up @@ -531,7 +589,11 @@ test("a skill target in a relative external directory is staged and uses its act
});

test("an agent that changes nothing yields an empty proposal, never an invented edit", async () => {
const { run } = setup({ edit: {}, annotations: [{ reply: { edits: [], notes: ["the evidence is too thin"] } }] });
const { config, run } = setup(
{ edit: {}, annotations: [{ reply: { edits: [], notes: ["the evidence is too thin"] } }] },
{ overrides: { synthesis: { agent: "codex", model: null } } },
);
config.agents = new AgentResolver(config);
const { proposal, violations } = await run();
assert.deepEqual(violations, []);
assert.equal(proposal.edits.length, 0);
Expand Down
Loading