|
| 1 | +import { readdir, readFile, stat } from "node:fs/promises"; |
| 2 | +import { relative, resolve } from "node:path"; |
| 3 | +import { compact } from "../src/compact.js"; |
| 4 | +import { JevJudge } from "../src/jev.js"; |
| 5 | +import type { Judge, Message } from "../src/types.js"; |
| 6 | + |
| 7 | +const astroRoot = resolve(process.cwd(), "../../node_modules/astro"); |
| 8 | +const task = "Trace how Astro resolves routes and builds the route manifest without changing its public API."; |
| 9 | + |
| 10 | +async function filesUnder(root: string): Promise<string[]> { |
| 11 | + const output: string[] = []; |
| 12 | + async function walk(directory: string): Promise<void> { |
| 13 | + for (const name of await readdir(directory)) { |
| 14 | + if (name === "node_modules" || name.startsWith(".")) continue; |
| 15 | + const path = resolve(directory, name); |
| 16 | + const info = await stat(path); |
| 17 | + if (info.isDirectory()) await walk(path); |
| 18 | + else if (/\.(?:js|mjs|d\.ts)$/.test(name) && info.size > 200 && info.size < 120_000) output.push(path); |
| 19 | + } |
| 20 | + } |
| 21 | + await walk(root); |
| 22 | + return output.sort(); |
| 23 | +} |
| 24 | + |
| 25 | +const available = await filesUnder(astroRoot); |
| 26 | +const relevantFiles = available.filter((file) => /(route|routing|manifest)/i.test(relative(astroRoot, file))); |
| 27 | +const irrelevantFiles = available.filter((file) => !/(route|routing|manifest)/i.test(relative(astroRoot, file))); |
| 28 | +const files = [...relevantFiles.slice(0, 60), ...irrelevantFiles.slice(0, 180)].sort(); |
| 29 | +if (files.length < 50) throw new Error(`Expected a framework-sized fixture, found ${files.length} files`); |
| 30 | + |
| 31 | +const messages: Message[] = [ |
| 32 | + { id: "goal", role: "user", parts: [{ type: "text", text: task }] }, |
| 33 | +]; |
| 34 | +const relevant = new Set<string>(); |
| 35 | + |
| 36 | +for (const [index, file] of files.entries()) { |
| 37 | + const path = relative(astroRoot, file); |
| 38 | + const callId = `read-${index}`; |
| 39 | + if (/(route|routing|manifest)/i.test(path)) relevant.add(callId); |
| 40 | + messages.push({ |
| 41 | + id: `assistant-${index}`, |
| 42 | + role: "assistant", |
| 43 | + parts: [{ type: "tool_call", id: callId, name: "Read", input: { path } }], |
| 44 | + }); |
| 45 | + messages.push({ |
| 46 | + id: `tool-${index}`, |
| 47 | + role: "tool", |
| 48 | + parts: [{ type: "tool_result", callId, output: await readFile(file, "utf8") }], |
| 49 | + }); |
| 50 | +} |
| 51 | +messages.push({ |
| 52 | + id: "tail", |
| 53 | + role: "assistant", |
| 54 | + parts: [{ type: "text", text: "Continue with the route and manifest implementation files." }], |
| 55 | +}); |
| 56 | + |
| 57 | +const pathOracle: Judge = { |
| 58 | + async judge({ candidates }) { |
| 59 | + return candidates.map((candidate) => ({ |
| 60 | + callId: candidate.id, |
| 61 | + keepCall: relevant.has(candidate.id) ? 0.98 : 0.05, |
| 62 | + keepResult: relevant.has(candidate.id) ? 0.95 : 0.02, |
| 63 | + })); |
| 64 | + }, |
| 65 | +}; |
| 66 | + |
| 67 | +const live = process.argv.includes("--jev"); |
| 68 | +const judge: Judge = live ? new JevJudge({ batchSize: 40 }) : pathOracle; |
| 69 | + |
| 70 | +const started = performance.now(); |
| 71 | +const result = await compact(messages, judge, { |
| 72 | + goal: task, |
| 73 | + preserveRecentMessages: 6, |
| 74 | +}); |
| 75 | +const elapsed = performance.now() - started; |
| 76 | +const relevantDecisions = result.decisions.filter((decision) => relevant.has(decision.callId)); |
| 77 | +const captured = new Map(result.decisions.map((decision) => [decision.callId, decision])); |
| 78 | +const threshold03 = await compact(messages, { |
| 79 | + async judge({ candidates }) { |
| 80 | + return candidates.map((candidate) => { |
| 81 | + const decision = captured.get(candidate.id); |
| 82 | + if (!decision) throw new Error(`Missing captured judgment for ${candidate.id}`); |
| 83 | + return { callId: candidate.id, keepCall: decision.keepCall, keepResult: decision.keepResult }; |
| 84 | + }); |
| 85 | + }, |
| 86 | +}, { |
| 87 | + goal: task, |
| 88 | + preserveRecentMessages: 6, |
| 89 | + keepThreshold: 0.3, |
| 90 | +}); |
| 91 | +const thresholdSweep = [0.2, 0.3, 0.4, 0.5].map((threshold) => { |
| 92 | + const retained = (keepCall: number, keepResult: number, reason: string) => |
| 93 | + reason === "policy" || keepCall >= threshold || keepResult >= threshold; |
| 94 | + return { |
| 95 | + threshold, |
| 96 | + relevantRetained: result.decisions.filter( |
| 97 | + (decision) => relevant.has(decision.callId) && |
| 98 | + retained(decision.keepCall, decision.keepResult, decision.reason), |
| 99 | + ).length, |
| 100 | + irrelevantRetained: result.decisions.filter( |
| 101 | + (decision) => !relevant.has(decision.callId) && |
| 102 | + retained(decision.keepCall, decision.keepResult, decision.reason), |
| 103 | + ).length, |
| 104 | + }; |
| 105 | +}); |
| 106 | + |
| 107 | +console.log(JSON.stringify({ |
| 108 | + fixture: "installed astro package", |
| 109 | + frameworkFilesRead: files.length, |
| 110 | + goldRelevantCalls: relevant.size, |
| 111 | + goldRelevantRetained: relevantDecisions.filter((decision) => decision.action !== "drop").length, |
| 112 | + irrelevantRetained: result.decisions.filter( |
| 113 | + (decision) => !relevant.has(decision.callId) && decision.action !== "drop", |
| 114 | + ).length, |
| 115 | + thresholdSweep, |
| 116 | + threshold03Stats: threshold03.stats, |
| 117 | + ...result.stats, |
| 118 | + localDecisionMs: Number(elapsed.toFixed(2)), |
| 119 | + judge: live ? "jev-live" : "path-oracle", |
| 120 | + caveat: live |
| 121 | + ? "Jev sees call metadata and conversation state, not tool-result bodies." |
| 122 | + : "Path oracle measures the reducible upper bound; it is not a Jev accuracy result.", |
| 123 | +}, null, 2)); |
0 commit comments