diff --git a/src/benchmarks/agent-cli/harness.ts b/src/benchmarks/agent-cli/harness.ts index dba8549..e034649 100644 --- a/src/benchmarks/agent-cli/harness.ts +++ b/src/benchmarks/agent-cli/harness.ts @@ -1,4 +1,8 @@ -import type { ChatMessage, ModelUsage, ResponseItem } from "../../harness/core"; +import type { + ModelMessage, + ModelUsage, + ResponseItem, +} from "../../harness/core"; import { MessageRole } from "../../harness/core"; import { Either } from "../../internal/either"; import { isRecord } from "../../internal/guards"; @@ -40,7 +44,7 @@ export interface OriAgentRun { readonly generationIds: readonly string[]; readonly generationTimeMs: number | undefined; readonly finalText: string | undefined; - readonly assistantMessages: readonly ChatMessage[]; + readonly assistantMessages: readonly ModelMessage[]; readonly responseItems: readonly ResponseItem[]; readonly isError: boolean; readonly apiErrorStatus: string | undefined; @@ -175,7 +179,7 @@ function usageFromResult(result: Record): ModelUsage { function parseClaudeStream(stdout: string): OriAgentRun { const generationIds: string[] = []; - const assistantMessages: ChatMessage[] = []; + const assistantMessages: ModelMessage[] = []; const responseItems: ResponseItem[] = []; let usage: ModelUsage | undefined; let generationTimeMs: number | undefined; @@ -363,7 +367,7 @@ function parsePiStream(stdout: string): OriAgentRun { let apiErrorStatus: string | undefined; let finalText: string | undefined; const generationIds: string[] = []; - const assistantMessages: ChatMessage[] = []; + const assistantMessages: ModelMessage[] = []; const responseItems: ResponseItem[] = []; for (const line of stdout.split("\n")) { const trimmed = line.trim(); diff --git a/src/benchmarks/define-chat-benchmark.ts b/src/benchmarks/define-single-turn-benchmark.ts similarity index 95% rename from src/benchmarks/define-chat-benchmark.ts rename to src/benchmarks/define-single-turn-benchmark.ts index f4ae005..5fd5d53 100644 --- a/src/benchmarks/define-chat-benchmark.ts +++ b/src/benchmarks/define-single-turn-benchmark.ts @@ -21,7 +21,7 @@ import type { RetryConfig } from "../runtime/retry"; import type { BenchmarkRunConfig } from "./benchmark-config"; import type { Benchmark, BenchmarkRunInput } from "./types"; -export interface ChatBenchmarkDefinition< +export interface SingleTurnBenchmarkDefinition< C extends BenchmarkRunConfig & { readonly model: string; }, @@ -39,11 +39,11 @@ export interface ChatBenchmarkDefinition< readonly makeSolver: (model: ModelService, config: C) => SolverService; } -export function defineChatBenchmark< +export function defineSingleTurnBenchmark< C extends BenchmarkRunConfig & { readonly model: string; }, ->(definition: ChatBenchmarkDefinition): Benchmark { +>(definition: SingleTurnBenchmarkDefinition): Benchmark { function makeLayer( input: BenchmarkRunInput ): Layer { diff --git a/src/benchmarks/draco/solver.ts b/src/benchmarks/draco/solver.ts index 2435ea0..3430c8b 100644 --- a/src/benchmarks/draco/solver.ts +++ b/src/benchmarks/draco/solver.ts @@ -16,7 +16,7 @@ import { } from "effect/Effect"; import type { - ChatMessage, + ModelMessage, ModelError, ModelOutput, ModelUsage, @@ -476,7 +476,7 @@ function completedState( generation: GenerationResult, verdicts: JudgeRun[] ): TaskState { - const messages: ChatMessage[] = [ + const messages: ModelMessage[] = [ { role: MessageRole.User, content: state.sample.input }, ...(generation.content ? [ diff --git a/src/benchmarks/gpqa-solver.test.ts b/src/benchmarks/gpqa-solver.test.ts index 531afe1..4e25610 100644 --- a/src/benchmarks/gpqa-solver.test.ts +++ b/src/benchmarks/gpqa-solver.test.ts @@ -10,7 +10,7 @@ import { noopProgressLayer, noopCheckpointLayer, } from "../../test/helpers/noop-progress-layer"; -import type { ChatMessage, ModelError, ModelOutput } from "../harness/core"; +import type { ModelMessage, ModelError, ModelOutput } from "../harness/core"; import { initialTaskState, MessageRole } from "../harness/core"; import type { GenerateConfig, ModelService } from "../harness/model"; import { Model } from "../harness/model"; @@ -23,7 +23,7 @@ function recordingModel(record: { config: GenerateConfig | undefined }): { } { const service: ModelService = { generate: ( - _messages: readonly ChatMessage[], + _messages: readonly ModelMessage[], config: GenerateConfig ): Effect => { record.config = config; diff --git a/src/benchmarks/gpqa.ts b/src/benchmarks/gpqa.ts index 545f44d..1c71bba 100644 --- a/src/benchmarks/gpqa.ts +++ b/src/benchmarks/gpqa.ts @@ -14,7 +14,7 @@ import type { GpqaBenchmarkConfig, } from "./benchmark-config"; import { GPQA_META } from "./benchmark-meta"; -import { defineChatBenchmark } from "./define-chat-benchmark"; +import { defineSingleTurnBenchmark } from "./define-single-turn-benchmark"; import { mcqScorer } from "./scorers/mcq/scorer"; import { seededPermutation } from "./scorers/mcq/shuffle"; import type { Benchmark } from "./types"; @@ -119,7 +119,7 @@ export function makeGpqaDatasetLayer( }); } -export const GPQA_BENCHMARK: Benchmark = defineChatBenchmark({ +export const GPQA_BENCHMARK: Benchmark = defineSingleTurnBenchmark({ id: "gpqa_diamond", temperature: GPQA_TEMPERATURE, defaultEpochs: GPQA_META.defaultEpochs, diff --git a/src/benchmarks/harbor/agent-loop.test.ts b/src/benchmarks/harbor/agent-loop.test.ts index f110cf1..79127da 100644 --- a/src/benchmarks/harbor/agent-loop.test.ts +++ b/src/benchmarks/harbor/agent-loop.test.ts @@ -8,7 +8,7 @@ import { SolverError } from "../../harness/core"; import { assertRight } from "../../internal/testing"; import { parseSchema, z } from "../../internal/zod"; import type { ResponsesModelService } from "../../providers/responses-model"; -import { itemsToChatMessages, runAgentLoop } from "./agent-loop"; +import { itemsToModelMessages, runAgentLoop } from "./agent-loop"; import { makeHarborStreamTracker } from "./agent-progress"; import { SUBMIT_SENTINEL } from "./prompts"; import type { ExecResult, SandboxSessionInstance } from "./sandbox"; @@ -314,7 +314,7 @@ describe("Harbor stream progress", () => { describe("Responses item round-tripping", () => { it("converts advisor advice from the real terminal fixture into an assistant message", async () => { const terminal = await readTerminalFixture(); - expect(itemsToChatMessages(terminal.output)).toContainEqual({ + expect(itemsToModelMessages(terminal.output)).toContainEqual({ role: "assistant", content: "Confirmed: 2 + 2 = 4 in standard base-10 arithmetic; edge cases include alternate numeric bases or string concatenation in programming contexts.", diff --git a/src/benchmarks/harbor/agent-loop.ts b/src/benchmarks/harbor/agent-loop.ts index 2ce05d1..386bab3 100644 --- a/src/benchmarks/harbor/agent-loop.ts +++ b/src/benchmarks/harbor/agent-loop.ts @@ -9,7 +9,7 @@ import { } from "effect/Effect"; import type { - ChatMessage, + ModelMessage, ModelError, ModelUsage, SolverError, @@ -68,7 +68,7 @@ export interface AgentLoopInput { export interface AgentLoopResult { readonly input: ResponsesInputItem[]; - readonly messages: ChatMessage[]; + readonly messages: ModelMessage[]; readonly usage: ModelUsage; readonly generationTimeMs: number; readonly finalText: string; @@ -173,7 +173,7 @@ export function runAgentLoop( } return { input: conversation, - messages: itemsToChatMessages(conversation), + messages: itemsToModelMessages(conversation), usage: toModelUsage(acc), generationTimeMs, finalText, @@ -329,10 +329,10 @@ function truncateCommand(command: string): string { : command; } -export function itemsToChatMessages( +export function itemsToModelMessages( items: readonly ResponsesInputItem[] -): ChatMessage[] { - const messages: ChatMessage[] = []; +): ModelMessage[] { + const messages: ModelMessage[] = []; for (const item of items) { const type = item["type"]; if (type === "message" || type === undefined) { diff --git a/src/benchmarks/ifstruct/benchmark.ts b/src/benchmarks/ifstruct/benchmark.ts index 8685fcb..4eed115 100644 --- a/src/benchmarks/ifstruct/benchmark.ts +++ b/src/benchmarks/ifstruct/benchmark.ts @@ -16,7 +16,7 @@ import type { InferenceOverride, } from "../benchmark-config"; import { IFSTRUCT_META } from "../benchmark-meta"; -import { defineChatBenchmark } from "../define-chat-benchmark"; +import { defineSingleTurnBenchmark } from "../define-single-turn-benchmark"; import type { Benchmark } from "../types"; import type { IfStructRequirements, @@ -143,7 +143,7 @@ export function makeIfStructDatasetLayer( }); } -export const IFSTRUCT_BENCHMARK: Benchmark = defineChatBenchmark({ +export const IFSTRUCT_BENCHMARK: Benchmark = defineSingleTurnBenchmark({ id: "ifstruct", temperature: IFSTRUCT_TEMPERATURE, defaultEpochs: IFSTRUCT_META.defaultEpochs, diff --git a/src/benchmarks/mmlu-pro-solver.test.ts b/src/benchmarks/mmlu-pro-solver.test.ts index 82e4e58..2c3f980 100644 --- a/src/benchmarks/mmlu-pro-solver.test.ts +++ b/src/benchmarks/mmlu-pro-solver.test.ts @@ -8,7 +8,7 @@ import { noopCheckpointLayer, noopProgressLayer, } from "../../test/helpers/noop-progress-layer"; -import type { ChatMessage, ModelError, ModelOutput } from "../harness/core"; +import type { ModelMessage, ModelError, ModelOutput } from "../harness/core"; import { initialTaskState, MessageRole } from "../harness/core"; import type { GenerateConfig, ModelService } from "../harness/model"; import { MMLU_PRO_TEMPERATURE, mmluProSolver } from "./mmlu-pro"; @@ -16,11 +16,11 @@ describe("mmluProSolver", () => { it("uses canonical sampling defaults and sends one user message", async () => { const recorded: { config?: GenerateConfig; - messages?: readonly ChatMessage[]; + messages?: readonly ModelMessage[]; } = {}; const model: ModelService = { generate: ( - messages: readonly ChatMessage[], + messages: readonly ModelMessage[], config: GenerateConfig ): Effect => { recorded.messages = messages; diff --git a/src/benchmarks/mmlu-pro.ts b/src/benchmarks/mmlu-pro.ts index f19cd9a..928ef98 100644 --- a/src/benchmarks/mmlu-pro.ts +++ b/src/benchmarks/mmlu-pro.ts @@ -15,7 +15,7 @@ import type { MmluProBenchmarkConfig, } from "./benchmark-config"; import { MMLU_PRO_META } from "./benchmark-meta"; -import { defineChatBenchmark } from "./define-chat-benchmark"; +import { defineSingleTurnBenchmark } from "./define-single-turn-benchmark"; import { makeMmluProFewShotDatasetLayer } from "./mmlu-pro-dataset"; import type { MmluProCotExamplesByCategory } from "./mmlu-pro-prompt"; import { buildMmluProPrompt } from "./mmlu-pro-prompt"; @@ -148,7 +148,7 @@ function mmluProRunLevelScores(result: RunResult): readonly { ]; } -const MMLU_PRO_CHAT_BENCHMARK = defineChatBenchmark({ +const MMLU_PRO_SINGLE_TURN_BENCHMARK = defineSingleTurnBenchmark({ id: "mmlu_pro", temperature: MMLU_PRO_TEMPERATURE, defaultEpochs: MMLU_PRO_META.defaultEpochs, @@ -176,6 +176,6 @@ const MMLU_PRO_CHAT_BENCHMARK = defineChatBenchmark({ }); export const MMLU_PRO_BENCHMARK: Benchmark = { - ...MMLU_PRO_CHAT_BENCHMARK, + ...MMLU_PRO_SINGLE_TURN_BENCHMARK, runLevelScores: mmluProRunLevelScores, }; diff --git a/src/benchmarks/mmmu-pro-vision.ts b/src/benchmarks/mmmu-pro-vision.ts index 4adef85..1304ffd 100644 --- a/src/benchmarks/mmmu-pro-vision.ts +++ b/src/benchmarks/mmmu-pro-vision.ts @@ -17,7 +17,7 @@ import type { MmmuProVisionBenchmarkConfig, } from "./benchmark-config"; import { MMMU_PRO_VISION_META } from "./benchmark-meta"; -import { defineChatBenchmark } from "./define-chat-benchmark"; +import { defineSingleTurnBenchmark } from "./define-single-turn-benchmark"; import { MMMU_SYSTEM_MESSAGE, parseOptions } from "./mmmu-shared"; import { buildDynamicMcqPrompt } from "./scorers/mcq/dynamic-prompt"; import { mcqScorer } from "./scorers/mcq/scorer"; @@ -139,7 +139,7 @@ export function mmmuProVisionSolver( return chain(systemMessage(MMMU_SYSTEM_MESSAGE), generate(model, config)); } -export const MMMU_PRO_VISION_BENCHMARK: Benchmark = defineChatBenchmark({ +export const MMMU_PRO_VISION_BENCHMARK: Benchmark = defineSingleTurnBenchmark({ id: "mmmu_pro_vision", temperature: 0, defaultEpochs: MMMU_PRO_VISION_META.defaultEpochs, diff --git a/src/benchmarks/tau-bench-airline/benchmark.ts b/src/benchmarks/tau-bench-airline/benchmark.ts index 70f4902..3b7646f 100644 --- a/src/benchmarks/tau-bench-airline/benchmark.ts +++ b/src/benchmarks/tau-bench-airline/benchmark.ts @@ -19,6 +19,10 @@ import { Solver } from "../../harness/solver"; import { Either } from "../../internal/either"; import { parseSchema } from "../../internal/zod"; import { makeOpenRouterModelLayer } from "../../providers/openrouter-model"; +import { + makeResponsesModelLayer, + ResponsesModel, +} from "../../providers/responses-model"; import type { RetryConfig } from "../../runtime/retry"; import { TAU_BENCH_AIRLINE_META } from "../benchmark-meta"; import type { Benchmark, BenchmarkRunInput } from "../types"; @@ -123,20 +127,33 @@ function makeAirlineLayer( sessionId: input.sessionId, ...(input.modelRetry !== undefined && { retry: input.modelRetry }), }); + const userModelLayer = makeResponsesModelLayer({ + model: benchmarkConfig.userModel, + apiKey: input.apiKey, + ...(input.baseUrl !== undefined && { baseUrl: input.baseUrl }), + sessionId: input.sessionId, + }); const solverLayer = layerEffect(Solver)( gen(function* () { const model = yield* Model; + const userModel = yield* ResponsesModel; const client = yield* HttpClient.HttpClient; const dataFetchLock = yield* makeSemaphore(1); return Solver.of( - airlineSolver({ model, client, dataFetchLock, opts: solverOpts }) + airlineSolver({ + model, + userModel, + client, + dataFetchLock, + opts: solverOpts, + }) ); }) ); const scorerLayer = layerSucceed(Scorer, Scorer.of(airlineScorer)); return layerMergeAll( datasetLayer, - solverLayer.pipe(layerProvide(modelLayer)), + solverLayer.pipe(layerProvide(layerMergeAll(modelLayer, userModelLayer))), scorerLayer ); } diff --git a/src/benchmarks/tau-bench-airline/scorer.ts b/src/benchmarks/tau-bench-airline/scorer.ts index df9818e..654e7e4 100644 --- a/src/benchmarks/tau-bench-airline/scorer.ts +++ b/src/benchmarks/tau-bench-airline/scorer.ts @@ -1,7 +1,12 @@ import type { Effect } from "effect/Effect"; import { succeed } from "effect/Effect"; -import type { ChatMessage, Score, Target, TaskState } from "../../harness/core"; +import type { + ModelMessage, + Score, + Target, + TaskState, +} from "../../harness/core"; import { MessageRole, ScoreValue } from "../../harness/core"; import type { ScorerService } from "../../harness/scorer"; import { Either } from "../../internal/either"; @@ -25,7 +30,7 @@ function isTau2Task(val: unknown): val is Tau2Task { } function collectToolCalls( - messages: readonly ChatMessage[] + messages: readonly ModelMessage[] ): PredictedToolCall[] { const calls: PredictedToolCall[] = []; for (const m of messages) { diff --git a/src/benchmarks/tau-bench-airline/solver.ts b/src/benchmarks/tau-bench-airline/solver.ts index 146b703..af61c73 100644 --- a/src/benchmarks/tau-bench-airline/solver.ts +++ b/src/benchmarks/tau-bench-airline/solver.ts @@ -2,12 +2,13 @@ import { HttpClient } from "@effect/platform"; import type { Semaphore } from "effect/Effect"; import { gen, mapError, provideService } from "effect/Effect"; -import type { ChatMessage, ModelUsage, ToolCall } from "../../harness/core"; +import type { ModelMessage, ModelUsage, ToolCall } from "../../harness/core"; import { MessageRole, SolverError } from "../../harness/core"; import type { GenerateConfig, ModelService } from "../../harness/model"; import type { SolverService } from "../../harness/solver"; import { Either } from "../../internal/either"; import { definedValues, isRecord } from "../../internal/guards"; +import type { ResponsesModelService } from "../../providers/responses-model"; import { buildAgentSystemPrompt, DEFAULT_FIRST_AGENT_MESSAGE, @@ -40,11 +41,13 @@ type Role = (typeof Role)[keyof typeof Role]; export function airlineSolver({ model, + userModel, client, dataFetchLock, opts, }: { readonly model: ModelService; + readonly userModel: ResponsesModelService; readonly client: HttpClient.HttpClient; readonly dataFetchLock: Semaphore; readonly opts?: SolverOpts; @@ -69,9 +72,9 @@ export function airlineSolver({ ); const task = state.sample.metadata?.["task"]; const data: AirlineData = loadAirlineData(); - const userSim = new UserSimulator(userModelConfig); + const userSim = new UserSimulator(userModel, userModelConfig); userSim.reset(state.sample.input, DEFAULT_FIRST_AGENT_MESSAGE); - const messages: ChatMessage[] = [ + const messages: ModelMessage[] = [ { role: MessageRole.System, content: buildAgentSystemPrompt(AIRLINE_POLICY), @@ -205,7 +208,7 @@ export function airlineSolver({ }); } -function lastAssistantText(messages: readonly ChatMessage[]): string { +function lastAssistantText(messages: readonly ModelMessage[]): string { for (let i = messages.length - 1; i >= 0; i--) { const m = messages[i]; if (m?.role === MessageRole.Assistant) { diff --git a/src/benchmarks/tau-bench-airline/user-simulator.test.ts b/src/benchmarks/tau-bench-airline/user-simulator.test.ts index 2e6a312..e21bc67 100644 --- a/src/benchmarks/tau-bench-airline/user-simulator.test.ts +++ b/src/benchmarks/tau-bench-airline/user-simulator.test.ts @@ -1,350 +1,66 @@ -import { describe, expect, it, spyOn } from "bun:test"; -import assert from "node:assert/strict"; +import { describe, expect, it } from "bun:test"; -import { FetchHttpClient } from "@effect/platform"; -import { flatMap, provide, runPromise } from "effect/Effect"; +import { runPromise, succeed } from "effect/Effect"; -import type { CapturedRequest } from "../../../test/helpers/fetch-sequence"; -import { installFetchSequence } from "../../../test/helpers/fetch-sequence"; -import { runHarnessPromise } from "../../internal/effect-logger"; -import { isRecord } from "../../internal/guards"; -import { - getCollectedGenerationIds, - resetGenerationIds, -} from "../../runtime/generation-ids"; -import { setCurrentEpoch } from "../../runtime/response-cache"; +import type { + ResponsesInputItem, + ResponsesModelService, + ResponsesTurn, +} from "../../providers/responses-model"; import { UserSimulator } from "./user-simulator"; -describe("UserSimulator", () => { - it.serial( - "records the generation id from a successful user-model response", - async () => { - const originalFetch = globalThis.fetch; - globalThis.fetch = async () => - Response.json({ - id: "tau-user-gen-1", - choices: [{ message: { content: "Hello" } }], - }); - try { - const simulator = new UserSimulator({ - apiKey: "sk-test", - model: "openai/gpt-4o-mini", - baseUrl: "https://example.test", - }); - simulator.reset("scenario", "Hi"); - const ids = await runPromise( - resetGenerationIds.pipe( - flatMap(() => simulator.generateInitial()), - flatMap(() => getCollectedGenerationIds), - provide(FetchHttpClient.layer) - ) - ); - expect(ids).toEqual(["tau-user-gen-1"]); - } finally { - globalThis.fetch = originalFetch; - } - } - ); - it.serial("sends the cache salt as a header, not a body field", async () => { - const requests: CapturedRequest[] = []; - const restore = installFetchSequence( - [{ id: "tau-user-gen-1", choices: [{ message: { content: "Hello" } }] }], - requests + +const config = { + apiKey: "sk-test", + model: "openai/gpt-5", + sessionId: "session-1", +} as const; + +function modelFor( + turns: readonly ResponsesTurn[], + inputs: ResponsesInputItem[][] +): ResponsesModelService { + let index = 0; + return { + generate: (input) => { + inputs.push([...input]); + return succeed(turns[Math.min(index++, turns.length - 1)]!); + }, + }; +} + +describe("tau-bench airline user simulator", () => { + it("uses Responses turns and replays output items", async () => { + const inputs: ResponsesInputItem[][] = []; + const responseItems = [ + { type: "reasoning", encrypted_content: "opaque" }, + { type: "message", content: [{ type: "output_text", text: "Hello" }] }, + ]; + const model = modelFor( + [ + { + outputItems: responseItems, + functionCalls: [], + text: "Hello", + generationTimeMs: 1, + }, + { + outputItems: [], + functionCalls: [], + text: "Goodbye", + generationTimeMs: 1, + }, + ], + inputs ); - try { - const simulator = new UserSimulator({ - apiKey: "sk-test", - model: "openai/gpt-4o-mini", - baseUrl: "https://example.test", - sessionId: "wf-123", - }); - simulator.reset("scenario", "Hi"); - await runPromise( - setCurrentEpoch(2).pipe( - flatMap(() => simulator.generateInitial()), - provide(FetchHttpClient.layer) - ) - ); - expect(requests[0]?.headers["x-openrouter-cache-salt"]).toBe( - "wf-123:epoch-2" - ); - expect(requests[0]?.body["cache_salt"]).toBeUndefined(); - } finally { - restore(); - } + const simulator = new UserSimulator(model, config); + simulator.reset("scenario", "Hi"); + expect(await runPromise(simulator.generateInitial())).toBe("Hello"); + expect(await runPromise(simulator.step("How are you?"))).toBe("Goodbye"); + expect(inputs[1]).toEqual([ + { type: "message", role: "system", content: expect.any(String) }, + { type: "message", role: "user", content: "Hi" }, + ...responseItems, + { type: "message", role: "user", content: "How are you?" }, + ]); }); - it.serial( - "replays opaque reasoning_details and omits absent details", - async () => { - const requests: CapturedRequest[] = []; - const reasoningDetails = [ - { type: "opaque", payload: { step: 1 } }, - { value: "keep" }, - ]; - const restore = installFetchSequence( - [ - { - model: "openai/gpt-4o-mini", - choices: [ - { - message: { - content: "User turn 1", - reasoning_details: reasoningDetails, - }, - }, - ], - }, - { - model: "openai/gpt-4o-mini", - choices: [{ message: { content: "User turn 2" } }], - }, - { - choices: [{ message: { content: "User turn 3" } }], - }, - ], - requests - ); - try { - const simulator = new UserSimulator({ - apiKey: "sk-test", - model: "openai/gpt-4o-mini", - baseUrl: "https://example.test", - }); - simulator.reset("scenario", "Hi"); - await runPromise( - simulator.generateInitial().pipe(provide(FetchHttpClient.layer)) - ); - await runPromise( - simulator.step("Agent reply").pipe(provide(FetchHttpClient.layer)) - ); - await runPromise( - simulator - .step("Agent reply again") - .pipe(provide(FetchHttpClient.layer)) - ); - const secondRequest = requests[1]; - const thirdRequest = requests[2]; - assert(secondRequest); - assert(thirdRequest); - const secondMessages = secondRequest.body["messages"]; - const thirdMessages = thirdRequest.body["messages"]; - assert(Array.isArray(secondMessages)); - assert(Array.isArray(thirdMessages)); - const secondAssistant = secondMessages.find( - (message) => isRecord(message) && message["content"] === "User turn 1" - ); - const thirdAssistant = thirdMessages.find( - (message) => isRecord(message) && message["content"] === "User turn 2" - ); - assert(isRecord(secondAssistant)); - assert(isRecord(thirdAssistant)); - expect(secondAssistant["reasoning_details"]).toEqual(reasoningDetails); - expect(thirdAssistant["reasoning_details"]).toBeUndefined(); - } finally { - restore(); - } - } - ); - it.serial( - "replays primary-model details on the fallback-model request", - async () => { - const requests: CapturedRequest[] = []; - const reasoningDetails = [ - { type: "future_variant", payload: { step: 1 } }, - ]; - const restore = installFetchSequence( - [ - { - model: "openai/gpt-4o-mini", - choices: [ - { - message: { - content: "Primary turn", - reasoning_details: reasoningDetails, - }, - }, - ], - }, - { choices: [{ message: { content: null } }] }, - { choices: [{ message: { content: null } }] }, - { choices: [{ message: { content: null } }] }, - { - model: "openai/gpt-5.4-mini", - choices: [{ message: { content: "Fallback turn" } }], - }, - ], - requests - ); - try { - const simulator = new UserSimulator({ - apiKey: "sk-test", - model: "openai/gpt-4o-mini", - baseUrl: "https://example.test", - }); - simulator.reset("scenario", "Hi"); - await runPromise( - simulator.generateInitial().pipe(provide(FetchHttpClient.layer)) - ); - await runPromise( - simulator.step("Agent reply").pipe(provide(FetchHttpClient.layer)) - ); - const fallbackRequest = requests[4]; - assert(fallbackRequest); - expect(fallbackRequest.body["model"]).toBe("openai/gpt-5.4-mini"); - const messages = fallbackRequest.body["messages"]; - assert(Array.isArray(messages)); - const assistant = messages.find( - (message) => - isRecord(message) && message["content"] === "Primary turn" - ); - assert(isRecord(assistant)); - expect(assistant["reasoning_details"]).toEqual(reasoningDetails); - } finally { - restore(); - } - } - ); - it.serial( - "keeps non-reasoning history in the pre-replay wire shape", - async () => { - const requests: CapturedRequest[] = []; - const restore = installFetchSequence( - [ - { - model: "openai/gpt-4o-mini", - choices: [{ message: { content: "First user turn" } }], - }, - { choices: [{ message: { content: "Second user turn" } }] }, - ], - requests - ); - try { - const simulator = new UserSimulator({ - apiKey: "sk-test", - model: "openai/gpt-4o-mini", - baseUrl: "https://example.test", - }); - simulator.reset("scenario", "Hi"); - await runPromise( - simulator.generateInitial().pipe(provide(FetchHttpClient.layer)) - ); - await runPromise( - simulator.step("Agent reply").pipe(provide(FetchHttpClient.layer)) - ); - const secondRequest = requests[1]; - assert(secondRequest); - const messages = secondRequest.body["messages"]; - assert(Array.isArray(messages)); - const assistant = messages.find( - (message) => - isRecord(message) && message["content"] === "First user turn" - ); - assert(isRecord(assistant)); - expect(assistant).toEqual({ - role: "assistant", - content: "First user turn", - }); - } finally { - restore(); - } - } - ); - it.serial( - "retries null message content before returning a recovered response", - async () => { - const originalFetch = globalThis.fetch; - let callCount = 0; - const responses = [ - { choices: [{ message: { content: null } }] }, - { choices: [{ message: { content: null } }] }, - { choices: [{ message: { content: "Recovered" } }] }, - ]; - globalThis.fetch = async () => { - const response = responses[callCount] ?? responses.at(-1); - callCount++; - return Response.json(response); - }; - try { - const simulator = new UserSimulator({ - apiKey: "sk-test", - model: "openai/gpt-4o-mini", - baseUrl: "https://example.test", - }); - simulator.reset("scenario", "Hi"); - const result = await runPromise( - simulator.generateInitial().pipe(provide(FetchHttpClient.layer)) - ); - expect(result).toBe("Recovered"); - expect(callCount).toBe(3); - } finally { - globalThis.fetch = originalFetch; - } - } - ); - it.serial( - "logs a structured retry warning for each user-sim retry", - async () => { - const warn = spyOn(console, "warn").mockImplementation(() => {}); - const originalFetch = globalThis.fetch; - let callCount = 0; - const responses = [ - { choices: [{ message: { content: null } }] }, - { choices: [{ message: { content: null } }] }, - { choices: [{ message: { content: "Recovered" } }] }, - ]; - globalThis.fetch = async () => { - const response = responses[callCount] ?? responses.at(-1); - callCount++; - return Response.json(response); - }; - try { - const simulator = new UserSimulator({ - apiKey: "sk-test", - model: "openai/gpt-4o-mini", - baseUrl: "https://example.test", - }); - simulator.reset("scenario", "Hi"); - const result = await runHarnessPromise( - simulator.generateInitial().pipe(provide(FetchHttpClient.layer)) - ); - expect(result).toBe("Recovered"); - expect(warn).toHaveBeenCalledTimes(2); - expect(warn.mock.calls[0]?.[0]).toBe("Retrying after transient error"); - expect(warn.mock.calls[0]?.[1]).toMatchObject({ - attempt: 1, - error_tag: "UserSimError", - }); - expect(warn.mock.calls[1]?.[1]).toMatchObject({ attempt: 2 }); - } finally { - globalThis.fetch = originalFetch; - warn.mockRestore(); - } - } - ); - it.serial( - "preserves UserSimError after bounded retries are exhausted", - async () => { - const originalFetch = globalThis.fetch; - let callCount = 0; - globalThis.fetch = async () => { - callCount++; - return Response.json({ choices: [{ message: { content: null } }] }); - }; - try { - const simulator = new UserSimulator({ - apiKey: "sk-test", - model: "openai/gpt-4o-mini", - baseUrl: "https://example.test", - }); - simulator.reset("scenario", "Hi"); - await expect( - runPromise( - simulator.generateInitial().pipe(provide(FetchHttpClient.layer)) - ) - ).rejects.toThrow("User simulator response parse error"); - expect(callCount).toBe(6); - } finally { - globalThis.fetch = originalFetch; - } - } - ); }); diff --git a/src/benchmarks/tau-bench-airline/user-simulator.ts b/src/benchmarks/tau-bench-airline/user-simulator.ts index b681d3b..729efd5 100644 --- a/src/benchmarks/tau-bench-airline/user-simulator.ts +++ b/src/benchmarks/tau-bench-airline/user-simulator.ts @@ -1,57 +1,17 @@ -import { - HttpClient, - HttpClientError, - HttpClientRequest, -} from "@effect/platform"; import { TaggedError } from "effect/Data"; import type { Effect } from "effect/Effect"; -import { catchAll, fail, gen } from "effect/Effect"; +import { catchAll, gen, map, mapError } from "effect/Effect"; import { fixed, passthrough, whileInput } from "effect/Schedule"; -import type { ReasoningDetails } from "../../harness/reasoning-details"; -import { - ReasoningDetailsSchema, - hasReasoningDetails, -} from "../../harness/reasoning-details"; -import { Either } from "../../internal/either"; -import { parseSchema, z } from "../../internal/zod"; -import { - BENCH_HARNESS_APP_REFERRER, - BENCH_HARNESS_APP_TITLE, -} from "../../providers/openrouter-model"; -import { - recordGenerationId, - withAuxiliaryUsage, -} from "../../runtime/generation-ids"; -import { - buildResponseCacheSalt, - getCurrentCallSalt, - getCurrentEpoch, - getCurrentRetryAttempt, - RESPONSE_CACHE_HEADER, - RESPONSE_CACHE_SALT_HEADER, - RESPONSE_CACHE_SOURCE_ID_HEADER, - RESPONSE_CACHE_STATUS_HEADER, - RESPONSE_CACHE_STATUS_HIT, - RESPONSE_CACHE_TTL_HEADER, - RESPONSE_CACHE_TTL_SECONDS, -} from "../../runtime/response-cache"; +import type { ModelMessage } from "../../harness/core"; +import { MessageRole } from "../../harness/core"; +import { messagesToResponses } from "../../providers/messages-to-responses"; +import type { ResponsesModelService } from "../../providers/responses-model"; +import { withAuxiliaryUsage } from "../../runtime/generation-ids"; import { retrySalted, withRetryAttemptLogging } from "../../runtime/retry"; import type { UserModelConfig } from "./types"; import { USER_SIM_GUIDELINES } from "./user-sim-guidelines"; -const ChatCompletionResponseSchema = z.object({ - id: z.string().nullish(), - choices: z.array( - z.object({ - message: z.object({ - content: z.string(), - reasoning_details: ReasoningDetailsSchema.optional(), - }), - }) - ), -}); - const USER_FALLBACK_MODEL = "openai/gpt-5.4-mini"; class UserSimError extends TaggedError("UserSimError")<{ @@ -59,7 +19,7 @@ class UserSimError extends TaggedError("UserSimError")<{ readonly retryable?: boolean; }> {} -type SimError = UserSimError | HttpClientError.HttpClientError; +type SimError = UserSimError; const USER_SIM_MAX_RETRIES = 2; @@ -78,150 +38,72 @@ function buildUserSystemPrompt(scenarioInstructions: string): string { return `${USER_SIM_GUIDELINES}\n\n\n${scenarioInstructions}\n`; } -interface UserMessage { - readonly role: "system" | "user" | "assistant"; - readonly content: string; - readonly reasoning_details?: ReasoningDetails; -} - interface UserModelResponse { readonly content: string; - readonly reasoningDetails?: ReasoningDetails; + readonly responseItems: readonly Record[]; } export class UserSimulator { - private readonly messages: UserMessage[] = []; + private readonly messages: ModelMessage[] = []; private readonly config: UserModelConfig; - private readonly baseUrl: string; - constructor(config: UserModelConfig) { + private readonly model: ResponsesModelService; + + constructor(model: ResponsesModelService, config: UserModelConfig) { + this.model = model; this.config = config; - const raw = config.baseUrl ?? "https://openrouter.ai"; - const trimmed = raw.replace(/\/+$/, ""); - this.baseUrl = trimmed.endsWith("/api/v1") ? trimmed : `${trimmed}/api/v1`; } + reset(scenarioInstructions: string, firstAgentMessage: string): void { this.messages.length = 0; this.messages.push( - { role: "system", content: buildUserSystemPrompt(scenarioInstructions) }, - { role: "user", content: firstAgentMessage } + { + role: MessageRole.System, + content: buildUserSystemPrompt(scenarioInstructions), + }, + { role: MessageRole.User, content: firstAgentMessage } ); } - generateInitial(): Effect { + + generateInitial(): Effect { return this.callModel(); } - step(agentMessage: string): Effect { - this.messages.push({ role: "user", content: agentMessage }); + + step(agentMessage: string): Effect { + this.messages.push({ role: MessageRole.User, content: agentMessage }); return this.callModel(); } - private callModel(): Effect { - const callModelOnce = this.callModelOnce; - const messages = this.messages; - const config = this.config; - return gen(function* () { - const response = yield* retrySalted( - withAuxiliaryUsage(callModelOnce(config.model)), - USER_SIM_RESPONSE_RETRY_SCHEDULE - ).pipe( - catchAll(() => - retrySalted( - withAuxiliaryUsage(callModelOnce(USER_FALLBACK_MODEL)), - USER_SIM_RESPONSE_RETRY_SCHEDULE - ) - ) + + private callModel(): Effect { + const response = (model: string) => + retrySalted(this.callModelOnce(model), USER_SIM_RESPONSE_RETRY_SCHEDULE); + return gen(this, function* (this: UserSimulator) { + const result = yield* response(this.config.model).pipe( + withAuxiliaryUsage, + catchAll(() => withAuxiliaryUsage(response(USER_FALLBACK_MODEL))) ); - messages.push({ - role: "assistant", - content: response.content, - ...(hasReasoningDetails(response.reasoningDetails) && { - reasoning_details: response.reasoningDetails, + this.messages.push({ + role: MessageRole.Assistant, + content: result.content, + ...(result.responseItems.length > 0 && { + responseItems: result.responseItems, }), }); - return response.content; + return result.content; }); } - private readonly callModelOnce = ( - model: string - ): Effect => { - const { baseUrl, config, messages } = this; - return gen(function* () { - const epoch = yield* getCurrentEpoch; - const retryAttempt = yield* getCurrentRetryAttempt; - const callSalt = yield* getCurrentCallSalt; - const cacheSalt = buildResponseCacheSalt( - config.sessionId, - epoch, - retryAttempt, - callSalt - ); - const request = HttpClientRequest.post( - `${baseUrl}/chat/completions` - ).pipe( - HttpClientRequest.setHeaders({ - Authorization: `Bearer ${config.apiKey}`, - "Content-Type": "application/json", - "HTTP-Referer": BENCH_HARNESS_APP_REFERRER, - "X-OpenRouter-Title": BENCH_HARNESS_APP_TITLE, - [RESPONSE_CACHE_HEADER]: "true", - ...(cacheSalt !== undefined && { - [RESPONSE_CACHE_SALT_HEADER]: cacheSalt, - }), - [RESPONSE_CACHE_TTL_HEADER]: `${RESPONSE_CACHE_TTL_SECONDS}`, - ...(config.sessionId !== undefined && { - "x-session-id": config.sessionId, - }), - }), - HttpClientRequest.bodyUnsafeJson({ - model, - messages, - temperature: 0, - }) - ); - const client = yield* HttpClient.HttpClient; - const response = yield* client.execute(request); - if (response.status < 200 || response.status >= 300) { - const text = yield* response.text; - return yield* fail( - new UserSimError({ - message: `User simulator HTTP ${response.status}: ${text}`, - }) - ); - } - const json: unknown = yield* response.json; - const parsed = parseSchema(ChatCompletionResponseSchema, json); - if (Either.isLeft(parsed)) { - const hasInvalidMessageContent = parsed.left.issues.some( - (issue) => - issue.path.length === 4 && - issue.path[0] === "choices" && - issue.path[1] === 0 && - issue.path[2] === "message" && - issue.path[3] === "content" - ); - return yield* fail( - new UserSimError({ - message: `User simulator response parse error: ${parsed.left.message}`, - retryable: hasInvalidMessageContent, - }) - ); - } - const isCacheHit = - response.headers[RESPONSE_CACHE_STATUS_HEADER] === - RESPONSE_CACHE_STATUS_HIT; - const cacheSourceId = response.headers[RESPONSE_CACHE_SOURCE_ID_HEADER]; - const hasSourceId = isCacheHit && cacheSourceId !== undefined; - yield* recordGenerationId( - hasSourceId ? cacheSourceId : parsed.right.id, - isCacheHit, - hasSourceId + + private callModelOnce(model: string): Effect { + return this.model + .generate(messagesToResponses(this.messages), { + model, + temperature: 0, + }) + .pipe( + mapError((error) => new UserSimError({ message: error.message })), + map((turn) => ({ + content: turn.text, + responseItems: turn.outputItems, + })) ); - const message = parsed.right.choices[0]?.message; - const content = message?.content ?? ""; - return { - content, - ...(message?.reasoning_details !== undefined && { - reasoningDetails: message?.reasoning_details, - }), - }; - }); - }; + } } diff --git a/src/benchmarks/tau3-bench-banking/benchmark.ts b/src/benchmarks/tau3-bench-banking/benchmark.ts index 23c5161..b98ec0e 100644 --- a/src/benchmarks/tau3-bench-banking/benchmark.ts +++ b/src/benchmarks/tau3-bench-banking/benchmark.ts @@ -16,6 +16,10 @@ import { Solver } from "../../harness/solver"; import { Either } from "../../internal/either"; import { parseSchema } from "../../internal/zod"; import { makeOpenRouterModelLayer } from "../../providers/openrouter-model"; +import { + makeResponsesModelLayer, + ResponsesModel, +} from "../../providers/responses-model"; import { Tau3BenchBankingConfigSchema } from "../benchmark-config"; import { TAU3_BENCH_BANKING_META } from "../benchmark-meta"; import type { Benchmark, BenchmarkRunInput } from "../types"; @@ -78,20 +82,33 @@ function makeBankingLayer( sessionId: input.sessionId, ...(input.modelRetry !== undefined && { retry: input.modelRetry }), }); + const userModelLayer = makeResponsesModelLayer({ + model: config.userModel, + apiKey: input.apiKey, + ...(input.baseUrl !== undefined && { baseUrl: input.baseUrl }), + sessionId: input.sessionId, + }); const solverLayer = layerEffect(Solver)( gen(function* () { const model = yield* Model; + const userModel = yield* ResponsesModel; const client = yield* HttpClient.HttpClient; const dataFetchLock = yield* makeSemaphore(1); return Solver.of( - bankingSolver({ model, client, dataFetchLock, opts: solverOpts }) + bankingSolver({ + model, + userModel, + client, + dataFetchLock, + opts: solverOpts, + }) ); }) ); const scorerLayer = layerSucceed(Scorer, Scorer.of(bankingScorer)); return layerMergeAll( datasetLayer, - solverLayer.pipe(layerProvide(modelLayer)), + solverLayer.pipe(layerProvide(layerMergeAll(modelLayer, userModelLayer))), scorerLayer ); } diff --git a/src/benchmarks/tau3-bench-banking/scorer.ts b/src/benchmarks/tau3-bench-banking/scorer.ts index 2f04ffa..00feeec 100644 --- a/src/benchmarks/tau3-bench-banking/scorer.ts +++ b/src/benchmarks/tau3-bench-banking/scorer.ts @@ -1,7 +1,12 @@ import type { Effect } from "effect/Effect"; import { succeed } from "effect/Effect"; -import type { ChatMessage, Score, Target, TaskState } from "../../harness/core"; +import type { + ModelMessage, + Score, + Target, + TaskState, +} from "../../harness/core"; import { MessageRole, ScoreValue } from "../../harness/core"; import type { ScorerService } from "../../harness/scorer"; import { isDefinedAndNotNull, isRecord } from "../../internal/guards"; @@ -25,7 +30,7 @@ function isTau3Task(val: unknown): val is Tau3Task { } function collectToolCalls( - messages: readonly ChatMessage[] + messages: readonly ModelMessage[] ): PredictedToolCall[] { const calls: PredictedToolCall[] = []; for (const m of messages) { diff --git a/src/benchmarks/tau3-bench-banking/solver.ts b/src/benchmarks/tau3-bench-banking/solver.ts index 428f19d..dc1b5cc 100644 --- a/src/benchmarks/tau3-bench-banking/solver.ts +++ b/src/benchmarks/tau3-bench-banking/solver.ts @@ -3,7 +3,7 @@ import type { Semaphore } from "effect/Effect"; import { gen, mapError, provideService } from "effect/Effect"; import type { - ChatMessage, + ModelMessage, ModelUsage, TaskState, ToolDefinition, @@ -14,6 +14,7 @@ import type { SolverService } from "../../harness/solver"; import { Either } from "../../internal/either"; import { definedValues } from "../../internal/guards"; import { parseSchema } from "../../internal/zod"; +import type { ResponsesModelService } from "../../providers/responses-model"; import { buildBankingAgentSystemPrompt, DEFAULT_FIRST_AGENT_MESSAGE, @@ -70,7 +71,7 @@ const Role = { type Role = (typeof Role)[keyof typeof Role]; -function lastAssistantText(messages: readonly ChatMessage[]): string { +function lastAssistantText(messages: readonly ModelMessage[]): string { for (let i = messages.length - 1; i >= 0; i--) { if (messages[i]!.role === MessageRole.Assistant) { return messages[i]!.content; @@ -93,7 +94,7 @@ type UserTurnState = "initial" | "continuing_tools" | "awaiting_reply"; function selectUserTurnEffect(opts: { readonly userSim: UserSimulator; readonly turnState: UserTurnState; - readonly messages: readonly ChatMessage[]; + readonly messages: readonly ModelMessage[]; }): ReturnType { switch (opts.turnState) { case "initial": { @@ -117,11 +118,13 @@ function isToolErrorResult(result: string): boolean { export function bankingSolver({ model, + userModel, client, dataFetchLock, opts, }: { readonly model: ModelService; + readonly userModel: ResponsesModelService; readonly client: HttpClient.HttpClient; readonly dataFetchLock: Semaphore; readonly opts?: SolverOpts; @@ -197,14 +200,14 @@ export function bankingSolver({ ); registerInitialDiscoverableTools(); const userToolDefs = selectUserToolDefinitions(task.user_tools ?? []); - const userSim = new UserSimulator(userModelConfig); + const userSim = new UserSimulator(userModel, userModelConfig); userSim.setAvailableTools(userToolDefs); userSim.reset(state.sample.input, DEFAULT_FIRST_AGENT_MESSAGE); const systemPrompt = buildBankingAgentSystemPrompt({ requiredDocIds, retrievalConfig, }); - const messages: ChatMessage[] = [ + const messages: ModelMessage[] = [ { role: MessageRole.System, content: systemPrompt }, { role: MessageRole.Assistant, content: DEFAULT_FIRST_AGENT_MESSAGE }, ]; diff --git a/src/benchmarks/tau3-bench-banking/user-simulator.test.ts b/src/benchmarks/tau3-bench-banking/user-simulator.test.ts index 0c3a0ee..3f27ad9 100644 --- a/src/benchmarks/tau3-bench-banking/user-simulator.test.ts +++ b/src/benchmarks/tau3-bench-banking/user-simulator.test.ts @@ -1,584 +1,102 @@ -import { describe, it, expect, beforeEach, afterEach } from "bun:test"; -import assert from "node:assert/strict"; +import { describe, expect, it } from "bun:test"; -import { FetchHttpClient, HttpClient } from "@effect/platform"; -import type { Effect } from "effect/Effect"; -import { flatMap, provide, runPromise } from "effect/Effect"; +import { runPromise, succeed } from "effect/Effect"; -import type { CapturedRequest } from "../../../test/helpers/fetch-sequence"; -import { installFetchSequence } from "../../../test/helpers/fetch-sequence"; -import { isRecord } from "../../internal/guards"; -import { setCurrentEpoch } from "../../runtime/response-cache"; +import type { + ResponsesInputItem, + ResponsesModelService, + ResponsesTurn, +} from "../../providers/responses-model"; import { UserSimulator } from "./user-simulator"; -const TEST_API_KEY = "test-key-123"; +const config = { + apiKey: "sk-test", + model: "openai/gpt-5", + sessionId: "session-1", + userReasoningEffort: "low", +} as const; -const TEST_MODEL = "openai/gpt-4o-mini"; - -const TEST_SESSION = "test-session-1"; - -function createTestConfig() { - return { - apiKey: TEST_API_KEY, - model: TEST_MODEL, - sessionId: TEST_SESSION, - }; -} - -let originalFetch: typeof global.fetch; -beforeEach(() => { - originalFetch = global.fetch; - global.fetch = async () => - new Response( - JSON.stringify({ - choices: [{ message: { content: "Default response" } }], - }), - { status: 200 } - ); -}); -afterEach(() => { - global.fetch = originalFetch; -}); - -function runSim( - effect: Effect -): Promise { - return runPromise(provide(effect, FetchHttpClient.layer)); -} -describe("UserSimulator", () => { - describe("initialization", () => { - it("normalizes trailing-slash and already-suffixed base URLs to the same endpoint", async () => { - const requestedUrls: string[] = []; - global.fetch = async (input, init) => { - requestedUrls.push(new Request(input, init).url); - return new Response( - JSON.stringify({ choices: [{ message: { content: "ok" } }] }), +describe("tau3 banking user simulator", () => { + it("uses Responses function calls and preserves call_id in tool results", async () => { + const inputs: ResponsesInputItem[][] = []; + const turns: ResponsesTurn[] = [ + { + outputItems: [ { - status: 200, - } - ); - }; - for (const baseUrl of [ - "https://api.example.com/api/v1/", - "https://api.example.com", - ]) { - const sim = new UserSimulator({ ...createTestConfig(), baseUrl }); - sim.reset("Scenario", "Hi."); - await runSim(sim.generateInitial()); - } - expect(requestedUrls).toEqual([ - "https://api.example.com/api/v1/chat/completions", - "https://api.example.com/api/v1/chat/completions", - ]); - }); - }); - describe("reset", () => { - it("seeds the scenario system prompt and first agent message", async () => { - const scenario = "Help me reset my password."; - const firstAgentMessage = "Hi, I need help with my account."; - let requestBody: { - messages?: { - role: string; - content: string; - }[]; - } = {}; - global.fetch = async (input, init) => { - requestBody = await new Request(input, init).json(); - return new Response( - JSON.stringify({ choices: [{ message: { content: "ok" } }] }), - { - status: 200, - } - ); - }; - const sim = new UserSimulator(createTestConfig()); - sim.reset(scenario, firstAgentMessage); - await runSim(sim.generateInitial()); - const [systemMessage, userMessage] = requestBody.messages ?? []; - assert(systemMessage); - assert(userMessage); - expect(systemMessage.role).toBe("system"); - expect(systemMessage.content).toContain(scenario); - expect(userMessage.role).toBe("user"); - expect(userMessage.content).toBe(firstAgentMessage); - }); - }); - describe("generateInitial", () => { - it("returns text turn with content", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help me with my account", "Hi there."); - let callCount = 0; - global.fetch = async () => { - callCount++; - return new Response( - JSON.stringify({ - choices: [ - { - message: { - content: "Hello! How can I help you today?", - }, - }, - ], - }), - { status: 200 } - ); - }; - const result = await runSim(sim.generateInitial()); - expect(result.kind).toBe("text"); - if (result.kind === "text") { - expect(result.content).toBe("Hello! How can I help you today?"); - } - expect(callCount).toBe(1); - }); - it("sends auth headers", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help", "Hi"); - let authorization: string | null = null; - global.fetch = async (input, init) => { - const request = new Request(input, init); - authorization = request.headers.get("Authorization"); - return new Response( - JSON.stringify({ - choices: [{ message: { content: "Hi" } }], - }), - { status: 200 } - ); - }; - await runSim(sim.generateInitial()); - expect(authorization).toBe(`Bearer ${TEST_API_KEY}`); - }); - it("sends the response-cache headers and an epoch-scoped salt", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help", "Hi"); - let cacheHeader: string | null = null; - let saltHeader: string | null = null; - let ttlHeader: string | null = null; - let capturedBody: unknown; - global.fetch = async (input, init) => { - const request = new Request(input, init); - cacheHeader = request.headers.get("x-openrouter-cache"); - saltHeader = request.headers.get("x-openrouter-cache-salt"); - ttlHeader = request.headers.get("x-openrouter-cache-ttl"); - capturedBody = await request.clone().json(); - return new Response( - JSON.stringify({ - choices: [{ message: { content: "Hi" } }], - }), - { status: 200 } - ); - }; - await runSim( - setCurrentEpoch(2).pipe(flatMap(() => sim.generateInitial())) - ); - expect(cacheHeader).toBe("true"); - expect(saltHeader).toBe(`${TEST_SESSION}:epoch-2`); - expect(ttlHeader).toBe("7200"); - assert(isRecord(capturedBody)); - expect(capturedBody["cache_salt"]).toBeUndefined(); - }); - }); - describe("step", () => { - it("appends agent message and generates response", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help me", "Hi"); - global.fetch = async () => - new Response( - JSON.stringify({ - choices: [{ message: { content: "Will do" } }], - }), - { status: 200 } - ); - const result = await runSim(sim.step("How can I help?")); - expect(result.kind).toBe("text"); - if (result.kind === "text") { - expect(result.content).toBe("Will do"); - } - }); - it("does not mutate history when the effect is constructed but never run", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help me", "Hi"); - const discarded = sim.step("SHOULD NOT APPEAR"); - expect(discarded).toBeDefined(); - let capturedMessages: unknown; - global.fetch = async (input, init) => { - const request = new Request(input, init); - const body: unknown = await request.json(); - assert(isRecord(body)); - capturedMessages = body["messages"]; - return new Response( - JSON.stringify({ - choices: [{ message: { content: "Hello" } }], - }), - { status: 200 } - ); - }; - await runSim(sim.generateInitial()); - assert(Array.isArray(capturedMessages)); - expect(capturedMessages).toHaveLength(2); - expect(JSON.stringify(capturedMessages)).not.toContain( - "SHOULD NOT APPEAR" - ); - }); - it("keeps prior user-simulator text in the next request history", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help me", "Hi"); - let callCount = 0; - global.fetch = async (input, init) => { - callCount++; - const request = new Request(input, init); - const requestBody: unknown = await request.json(); - assert(isRecord(requestBody)); - const messages = requestBody["messages"]; - assert(Array.isArray(messages)); - if (callCount === 2) { - expect(messages).toContainEqual({ - role: "assistant", - content: "First user turn", - }); - } - return Response.json({ - choices: [ - { - message: { - content: - callCount === 1 ? "First user turn" : "Second user turn", - }, - }, - ], - }); - }; - await runSim(sim.generateInitial()); - await runSim(sim.step("Agent reply")); - expect(callCount).toBe(2); - }); - it("keeps non-reasoning history in the pre-replay wire shape", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help me", "Hi"); - const requests: CapturedRequest[] = []; - const restore = installFetchSequence( - [ - { - model: TEST_MODEL, - choices: [{ message: { content: "First user turn" } }], - }, - { choices: [{ message: { content: "Second user turn" } }] }, - ], - requests - ); - try { - await runSim(sim.generateInitial()); - await runSim(sim.step("Agent reply")); - const secondRequest = requests[1]; - assert(secondRequest); - const messages = secondRequest.body["messages"]; - assert(Array.isArray(messages)); - const assistant = messages.find( - (message) => - isRecord(message) && message["content"] === "First user turn" - ); - assert(isRecord(assistant)); - expect(assistant).toEqual({ - role: "assistant", - content: "First user turn", - }); - } finally { - restore(); - } - }); - }); - describe("tool-call handling", () => { - it("replays opaque reasoning_details across tool and text turns", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help me", "Hi"); - const reasoningDetails = [ - { type: "summary", summary: "opaque" }, - { type: "future_variant", future_payload: { step: 1 } }, - ]; - const requests: CapturedRequest[] = []; - const restore = installFetchSequence( - [ - { - model: TEST_MODEL, - choices: [ - { - message: { - content: "", - reasoning_details: reasoningDetails, - tool_calls: [ - { - id: "call_1", - type: "function", - function: { name: "lookup", arguments: "{}" }, - }, - ], - }, - }, - ], + type: "reasoning", + encrypted_content: "opaque", }, { - model: "openai/gpt-4o", - choices: [ - { - message: { - content: "Done", - reasoning_details: reasoningDetails, - }, - }, - ], + type: "function_call", + call_id: "call-1", + name: "get_balance", + arguments: '{"account_id":"a1"}', }, + ], + functionCalls: [ { - choices: [{ message: { content: "Done" } }], + callId: "call-1", + name: "get_balance", + arguments: '{"account_id":"a1"}', }, ], - requests - ); - try { - const first = await runSim(sim.generateInitial()); - expect(first.kind).toBe("toolCalls"); - sim.addToolResult("call_1", "tool result"); - await runSim(sim.continueAfterTools()); - const secondRequest = requests[1]; - assert(secondRequest); - const secondMessages = secondRequest.body["messages"]; - assert(Array.isArray(secondMessages)); - const assistant = secondMessages.find( - (message) => isRecord(message) && message["role"] === "assistant" - ); - assert(isRecord(assistant)); - expect(assistant).toMatchObject({ - reasoning_details: reasoningDetails, - tool_calls: [ - { - id: "call_1", - type: "function", - function: { name: "lookup", arguments: "{}" }, - }, - ], - }); - const third = await runSim(sim.step("Agent follow-up")); - expect(third.kind).toBe("text"); - const thirdRequest = requests[2]; - assert(thirdRequest); - const thirdMessages = thirdRequest.body["messages"]; - assert(Array.isArray(thirdMessages)); - const textAssistant = thirdMessages.find( - (message) => isRecord(message) && message["content"] === "Done" - ); - assert(isRecord(textAssistant)); - expect(textAssistant["reasoning_details"]).toEqual(reasoningDetails); - } finally { - restore(); - } - }); - it("returns tool calls when response includes them", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help me", "Hi"); - global.fetch = async () => - new Response( - JSON.stringify({ - choices: [ - { - message: { - content: null, - tool_calls: [ - { - id: "call_123", - type: "function", - function: { - name: "check_balance", - arguments: '{"account_id": "acc_1"}', - }, - }, - ], - }, - }, - ], - }), - { status: 200 } - ); - const result = await runSim(sim.generateInitial()); - expect(result.kind).toBe("toolCalls"); - if (result.kind === "toolCalls") { - expect(result.calls).toHaveLength(1); - expect(result.calls[0]?.name).toBe("check_balance"); - expect(result.calls[0]?.id).toBe("call_123"); - } - }); - it("addToolResult appends result message", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help me", "Hi"); - sim.addToolResult("call_123", "Balance: $1000"); - global.fetch = async (input, init) => { - const request = new Request(input, init); - const bodyJson: unknown = await request.json(); - assert(isRecord(bodyJson)); - const messages = bodyJson["messages"]; - assert(Array.isArray(messages)); - expect(messages.length).toBeGreaterThanOrEqual(3); - return new Response( - JSON.stringify({ - choices: [{ message: { content: "Got it" } }], - }), - { status: 200 } - ); - }; - const result = await runSim(sim.step("Here is the balance")); - expect(result.kind).toBe("text"); - }); - it("continues after tool results without duplicating the agent message", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help me", "Hi"); - sim.addToolResult("call_123", "Balance: $1000"); - let capturedMessages: unknown[] | undefined; - global.fetch = async (input, init) => { - const request = new Request(input, init); - const bodyJson: unknown = await request.json(); - assert(isRecord(bodyJson)); - const messages = bodyJson["messages"]; - assert(Array.isArray(messages)); - capturedMessages = messages; - return new Response( - JSON.stringify({ choices: [{ message: { content: "Got it" } }] }), - { - status: 200, - } - ); - }; - await runSim(sim.continueAfterTools()); - expect(capturedMessages).toHaveLength(3); - expect(capturedMessages?.at(-1)).toEqual({ - role: "tool", - content: "Balance: $1000", - tool_call_id: "call_123", - }); - }); - }); - describe("setAvailableTools", () => { - it("includes tools in request when set", async () => { - const sim = new UserSimulator(createTestConfig()); - const toolDef = { - type: "function" as const, + text: "", + generationTimeMs: 1, + }, + { + outputItems: [ + { type: "message", content: [{ type: "output_text", text: "Done" }] }, + ], + functionCalls: [], + text: "Done", + generationTimeMs: 1, + }, + ]; + let index = 0; + const model: ResponsesModelService = { + generate: (input, generateConfig) => { + inputs.push([...input]); + expect(generateConfig.reasoningEffort).toBe("low"); + return succeed(turns[index++]!); + }, + }; + const simulator = new UserSimulator(model, config); + simulator.setAvailableTools([ + { + type: "function", function: { - name: "test_tool", - description: "A test tool", - parameters: { type: "object" as const, properties: {} }, + name: "get_balance", + description: undefined, + parameters: { type: "object" }, }, - }; - sim.setAvailableTools([toolDef]); - sim.reset("Help", "Hi"); - let hasTools = false; - let systemPrompt = ""; - global.fetch = async (input, init) => { - const request = new Request(input, init); - const bodyJson: unknown = await request.json(); - assert(isRecord(bodyJson)); - const tools = bodyJson["tools"]; - hasTools = Array.isArray(tools) && tools.length > 0; - const messages = bodyJson["messages"]; - if (Array.isArray(messages) && isRecord(messages[0])) { - systemPrompt = - typeof messages[0]["content"] === "string" - ? messages[0]["content"] - : ""; - } - return new Response( - JSON.stringify({ - choices: [{ message: { content: "OK" } }], - }), - { status: 200 } - ); - }; - await runSim(sim.generateInitial()); - expect(hasTools).toBe(true); - expect(systemPrompt).toContain( - "Make a tool call to perform an action requested by the agent." - ); - }); - }); - describe("error handling", () => { - it("throws on HTTP error", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help", "Hi"); - global.fetch = async () => - new Response(JSON.stringify({ error: "Server error" }), { - status: 500, - }); - await expect(runSim(sim.generateInitial())).rejects.toThrow("HTTP 500"); - }); - it("propagates parse errors without retrying another model", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help", "Hi"); - let callCount = 0; - global.fetch = async () => { - callCount++; - return new Response(JSON.stringify({}), { status: 200 }); - }; - await expect(runSim(sim.generateInitial())).rejects.toThrow( - "response parse error" - ); - expect(callCount).toBe(1); - }); - }); - describe("temperature and model", () => { - it("sends temperature: 0", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help", "Hi"); - let capturedTemp: unknown; - global.fetch = async (input, init) => { - const request = new Request(input, init); - const bodyJson: unknown = await request.json(); - assert(isRecord(bodyJson)); - capturedTemp = bodyJson["temperature"]; - return new Response( - JSON.stringify({ - choices: [{ message: { content: "OK" } }], - }), - { status: 200 } - ); - }; - await runSim(sim.generateInitial()); - expect(capturedTemp).toBe(0); - }); - it("sends configured model", async () => { - const sim = new UserSimulator(createTestConfig()); - sim.reset("Help", "Hi"); - let capturedModel: unknown; - global.fetch = async (input, init) => { - const request = new Request(input, init); - const bodyJson: unknown = await request.json(); - assert(isRecord(bodyJson)); - capturedModel = bodyJson["model"]; - return new Response( - JSON.stringify({ - choices: [{ message: { content: "OK" } }], - }), - { status: 200 } - ); - }; - await runSim(sim.generateInitial()); - expect(capturedModel).toBe(TEST_MODEL); - }); - it("sends configured user reasoning effort", async () => { - const sim = new UserSimulator({ - ...createTestConfig(), - userReasoningEffort: "medium", - }); - sim.reset("Help", "Hi"); - let capturedReasoningEffort: unknown; - global.fetch = async (input, init) => { - const request = new Request(input, init); - const bodyJson: unknown = await request.json(); - assert(isRecord(bodyJson)); - capturedReasoningEffort = bodyJson["reasoning_effort"]; - return new Response( - JSON.stringify({ - choices: [{ message: { content: "OK" } }], - }), - { status: 200 } - ); - }; - await runSim(sim.generateInitial()); - expect(capturedReasoningEffort).toBe("medium"); - }); + }, + ]); + simulator.reset("scenario", "Hi"); + const toolTurn = await runPromise(simulator.generateInitial()); + expect(toolTurn).toEqual({ + kind: "toolCalls", + calls: [ + { + id: "call-1", + name: "get_balance", + arguments: '{"account_id":"a1"}', + }, + ], + }); + simulator.addToolResult("call-1", "Balance: $10"); + expect(await runPromise(simulator.continueAfterTools())).toEqual({ + kind: "text", + content: "Done", + }); + expect(inputs[1]).toEqual([ + { type: "message", role: "system", content: expect.any(String) }, + { type: "message", role: "user", content: "Hi" }, + ...turns[0]!.outputItems, + { + type: "function_call_output", + call_id: "call-1", + output: "Balance: $10", + }, + ]); }); }); diff --git a/src/benchmarks/tau3-bench-banking/user-simulator.ts b/src/benchmarks/tau3-bench-banking/user-simulator.ts index 66f19b6..38891f2 100644 --- a/src/benchmarks/tau3-bench-banking/user-simulator.ts +++ b/src/benchmarks/tau3-bench-banking/user-simulator.ts @@ -1,34 +1,14 @@ -import { - HttpClient, - HttpClientError, - HttpClientRequest, -} from "@effect/platform"; import { TaggedError } from "effect/Data"; import type { Effect } from "effect/Effect"; -import { fail, gen, suspend } from "effect/Effect"; +import { map, mapError } from "effect/Effect"; -import type { ToolDefinition } from "../../harness/core"; -import type { ReasoningDetails } from "../../harness/reasoning-details"; -import { - ReasoningDetailsSchema, - hasReasoningDetails, -} from "../../harness/reasoning-details"; -import { Either } from "../../internal/either"; -import { parseSchema, z } from "../../internal/zod"; -import { - BENCH_HARNESS_APP_REFERRER, - BENCH_HARNESS_APP_TITLE, -} from "../../providers/openrouter-model"; +import type { ModelMessage, ToolDefinition } from "../../harness/core"; +import { MessageRole } from "../../harness/core"; import { - buildResponseCacheSalt, - getCurrentCallSalt, - getCurrentEpoch, - getCurrentRetryAttempt, - RESPONSE_CACHE_HEADER, - RESPONSE_CACHE_SALT_HEADER, - RESPONSE_CACHE_TTL_HEADER, - RESPONSE_CACHE_TTL_SECONDS, -} from "../../runtime/response-cache"; + messagesToResponses, + toolDefinitionToResponses, +} from "../../providers/messages-to-responses"; +import type { ResponsesModelService } from "../../providers/responses-model"; import type { UserModelConfig } from "./types"; import { USER_SIM_GUIDELINES, @@ -39,32 +19,9 @@ class UserSimError extends TaggedError("UserSimError")<{ readonly message: string; }> {} -type SimError = UserSimError | HttpClientError.HttpClientError; - -const ToolCallFunctionSchema = z.object({ - name: z.string(), - arguments: z.string(), -}); - -const ChatCompletionToolCallSchema = z.object({ - id: z.string(), - type: z.literal("function"), - function: ToolCallFunctionSchema, -}); +type SimError = UserSimError; -const ChatCompletionChoiceSchema = z.object({ - message: z.object({ - content: z.string().nullish(), - tool_calls: z.array(ChatCompletionToolCallSchema).optional(), - reasoning_details: ReasoningDetailsSchema.optional(), - }), -}); - -const ChatCompletionResponseSchema = z.object({ - choices: z.array(ChatCompletionChoiceSchema), -}); - -type SimulatorTurn = TextTurn | ToolCallsTurn; +export type SimulatorTurn = TextTurn | ToolCallsTurn; interface TextTurn { readonly kind: "text"; @@ -80,23 +37,6 @@ interface ToolCallsTurn { }[]; } -interface ToolCall { - readonly id: string; - readonly type: "function"; - readonly function: { - readonly name: string; - readonly arguments: string; - }; -} - -interface UserMessage { - readonly role: "system" | "user" | "assistant" | "tool"; - readonly content?: string; - readonly tool_call_id?: string; - readonly tool_calls?: ToolCall[]; - readonly reasoning_details?: ReasoningDetails; -} - function buildUserSystemPrompt( scenarioInstructions: string, useTools: boolean @@ -106,151 +46,91 @@ function buildUserSystemPrompt( } export class UserSimulator { - private readonly messages: UserMessage[] = []; + private readonly messages: ModelMessage[] = []; private readonly config: UserModelConfig; - private readonly baseUrl: string; - constructor(config: UserModelConfig) { + private readonly model: ResponsesModelService; + private availableTools: readonly ToolDefinition[] = []; + + constructor(model: ResponsesModelService, config: UserModelConfig) { + this.model = model; this.config = config; - const raw = config.baseUrl ?? "https://openrouter.ai"; - const trimmed = raw.replace(/\/+$/, ""); - this.baseUrl = trimmed.endsWith("/api/v1") ? trimmed : `${trimmed}/api/v1`; } + reset(scenarioInstructions: string, firstAgentMessage: string): void { const useTools = this.availableTools.length > 0; this.messages.length = 0; this.messages.push( { - role: "system", + role: MessageRole.System, content: buildUserSystemPrompt(scenarioInstructions, useTools), }, - { role: "user", content: firstAgentMessage } + { role: MessageRole.User, content: firstAgentMessage } ); } - generateInitial(): Effect { - return this.callModel(); + + generateInitial(): Effect { + return this.callModel(this.config.model); } - step( - agentMessage: string - ): Effect { - return suspend(() => { - this.messages.push({ role: "user", content: agentMessage }); - return this.callModel(); - }); + + step(agentMessage: string): Effect { + this.messages.push({ role: MessageRole.User, content: agentMessage }); + return this.callModel(this.config.model); } - continueAfterTools(): Effect { - return this.callModel(); + + continueAfterTools(): Effect { + return this.callModel(this.config.model); } + setAvailableTools(toolDefs: readonly ToolDefinition[]): void { this.availableTools = toolDefs; } - private availableTools: readonly ToolDefinition[] = []; + addToolResult(toolCallId: string, content: string): void { this.messages.push({ - role: "tool", + role: MessageRole.Tool, content, - tool_call_id: toolCallId, + toolCallId, }); } - private callModel(): Effect { - return this.callModelOnce(this.config.model); + + private callModel(model: string): Effect { + return this.callModelOnce(model).pipe( + map((turn) => { + const assistantMessage: ModelMessage = { + role: MessageRole.Assistant, + content: turn.text, + ...(turn.outputItems.length > 0 && { + responseItems: turn.outputItems, + }), + }; + this.messages.push(assistantMessage); + if (turn.functionCalls.length > 0) { + return { + kind: "toolCalls", + calls: turn.functionCalls.map((call) => ({ + id: call.callId, + name: call.name, + arguments: call.arguments, + })), + }; + } + return { kind: "text", content: turn.text }; + }) + ); } - private callModelOnce( - model: string - ): Effect { - return gen(this, function* (this: UserSimulator) { - const epoch = yield* getCurrentEpoch; - const retryAttempt = yield* getCurrentRetryAttempt; - const callSalt = yield* getCurrentCallSalt; - const cacheSalt = buildResponseCacheSalt( - this.config.sessionId, - epoch, - retryAttempt, - callSalt - ); - const requestBody: Record = { + + private callModelOnce(model: string) { + return this.model + .generate(messagesToResponses(this.messages), { model, - messages: this.messages, temperature: 0, ...(this.config.userReasoningEffort !== undefined && { - reasoning_effort: this.config.userReasoningEffort, - }), - }; - if (this.availableTools.length > 0) { - requestBody.tools = [...this.availableTools]; - } - const request = HttpClientRequest.post( - `${this.baseUrl}/chat/completions` - ).pipe( - HttpClientRequest.setHeaders({ - Authorization: `Bearer ${this.config.apiKey}`, - "Content-Type": "application/json", - "HTTP-Referer": BENCH_HARNESS_APP_REFERRER, - "X-OpenRouter-Title": BENCH_HARNESS_APP_TITLE, - [RESPONSE_CACHE_HEADER]: "true", - ...(cacheSalt !== undefined && { - [RESPONSE_CACHE_SALT_HEADER]: cacheSalt, - }), - [RESPONSE_CACHE_TTL_HEADER]: `${RESPONSE_CACHE_TTL_SECONDS}`, - ...(this.config.sessionId !== undefined && { - "x-session-id": this.config.sessionId, - }), + reasoningEffort: this.config.userReasoningEffort, }), - HttpClientRequest.bodyUnsafeJson(requestBody) - ); - const client = yield* HttpClient.HttpClient; - const response = yield* client.execute(request); - if (response.status < 200 || response.status >= 300) { - const text = yield* response.text; - return yield* fail( - new UserSimError({ - message: `User simulator HTTP ${response.status}: ${text}`, - }) - ); - } - const json: unknown = yield* response.json; - const parsed = parseSchema(ChatCompletionResponseSchema, json); - if (Either.isLeft(parsed)) { - return yield* fail( - new UserSimError({ - message: `User simulator response parse error: ${parsed.left.message}`, - }) - ); - } - const choice = parsed.right.choices[0]; - if (!choice) { - return yield* fail( - new UserSimError({ - message: "User simulator: no choices in response", - }) - ); - } - const { message } = choice; - const content = message.content ?? ""; - if (message.tool_calls && message.tool_calls.length > 0) { - const toolCalls = message.tool_calls.map((tc) => ({ - id: tc.id, - name: tc.function.name, - arguments: tc.function.arguments, - })); - const assistantMsg: UserMessage = { - role: "assistant", - content: content.length > 0 ? content : undefined, - tool_calls: message.tool_calls, - ...(hasReasoningDetails(message.reasoning_details) && { - reasoning_details: message.reasoning_details, - }), - }; - this.messages.push(assistantMsg); - return { kind: "toolCalls", calls: toolCalls }; - } - this.messages.push({ - role: "assistant", - content, - ...(hasReasoningDetails(message.reasoning_details) && { - reasoning_details: message.reasoning_details, + ...(this.availableTools.length > 0 && { + tools: this.availableTools.map(toolDefinitionToResponses), }), - }); - return { kind: "text", content }; - }); + }) + .pipe(mapError((error) => new UserSimError({ message: error.message }))); } } diff --git a/src/benchmarks/terminal-bench/ori-solver.ts b/src/benchmarks/terminal-bench/ori-solver.ts index 24959c3..a6fcd52 100644 --- a/src/benchmarks/terminal-bench/ori-solver.ts +++ b/src/benchmarks/terminal-bench/ori-solver.ts @@ -1,6 +1,6 @@ import { gen, tryPromise } from "effect/Effect"; -import type { ChatMessage, ModelUsage } from "../../harness/core"; +import type { ModelMessage, ModelUsage } from "../../harness/core"; import { MessageRole, SolverError } from "../../harness/core"; import type { SolverService } from "../../harness/solver"; import type { OriHarnessDef } from "../agent-cli/harness"; @@ -76,7 +76,7 @@ export function oriSolver( ? `${run.failureDetail}\n\n${testResult.output}` : testResult.output; const completion = run.finalText ?? run.rawStream; - const messages: ChatMessage[] = [ + const messages: ModelMessage[] = [ { role: MessageRole.User, content: state.sample.input }, ...run.assistantMessages, ]; diff --git a/src/benchmarks/vgi-bench/benchmark.ts b/src/benchmarks/vgi-bench/benchmark.ts index bd1f920..48e14e6 100644 --- a/src/benchmarks/vgi-bench/benchmark.ts +++ b/src/benchmarks/vgi-bench/benchmark.ts @@ -17,7 +17,7 @@ import type { VgiBenchmarkConfig, } from "../benchmark-config"; import { VGI_BENCH_META } from "../benchmark-meta"; -import { defineChatBenchmark } from "../define-chat-benchmark"; +import { defineSingleTurnBenchmark } from "../define-single-turn-benchmark"; import { mcqScorer } from "../scorers/mcq/scorer"; import type { Benchmark } from "../types"; import type { VgiBenchMediaManifest } from "./media-manifest"; @@ -272,7 +272,7 @@ function vgiBenchRunLevelScores(result: RunResult): readonly { ]; } -const VGI_BENCH_CHAT_BENCHMARK = defineChatBenchmark({ +const VGI_BENCH_SINGLE_TURN_BENCHMARK = defineSingleTurnBenchmark({ id: VGI_BENCH_META.id, temperature: VGI_BENCH_TEMPERATURE, defaultEpochs: VGI_BENCH_META.defaultEpochs, @@ -315,6 +315,6 @@ const VGI_BENCH_CHAT_BENCHMARK = defineChatBenchmark({ }); export const VGI_BENCH_BENCHMARK: Benchmark = { - ...VGI_BENCH_CHAT_BENCHMARK, + ...VGI_BENCH_SINGLE_TURN_BENCHMARK, runLevelScores: vgiBenchRunLevelScores, }; diff --git a/src/harness/core.ts b/src/harness/core.ts index 8c1f668..885e2c5 100644 --- a/src/harness/core.ts +++ b/src/harness/core.ts @@ -68,7 +68,7 @@ export const ContentPartSchema = z.discriminatedUnion("type", [ VideoContentPartSchema, ]); -export const ChatMessageSchema = z +export const ModelMessageSchema = z .object({ role: z.enum(MESSAGE_ROLE_VALUES), content: z.string(), @@ -77,6 +77,10 @@ export const ChatMessageSchema = z toolCallId: z.string().optional(), reasoning: z.string().optional(), reasoningDetails: ReasoningDetailsSchema.optional(), + responseItems: z + .array(z.record(z.string(), z.unknown())) + .readonly() + .optional(), citations: z.array(CitationSchema).readonly().optional(), model: z.string().optional(), }) @@ -111,7 +115,7 @@ export interface VideoContentPart { export type ContentPart = TextContentPart | ImageContentPart | VideoContentPart; -export type ChatMessage = z.infer; +export type ModelMessage = z.infer; export interface Sample { readonly id: string; @@ -148,7 +152,7 @@ export type ResponseItem = Readonly>; export interface ModelOutput { readonly completion: string; - readonly message: ChatMessage; + readonly message: ModelMessage; readonly usage?: ModelUsage; readonly generationTimeMs?: number; readonly rawResponse?: Readonly>; @@ -156,7 +160,7 @@ export interface ModelOutput { export interface TaskState { readonly sample: Sample; - readonly messages: readonly ChatMessage[]; + readonly messages: readonly ModelMessage[]; readonly responseItems?: readonly ResponseItem[]; readonly requestBody?: Readonly>; readonly output?: ModelOutput; diff --git a/src/harness/metric.ts b/src/harness/metric.ts index b75b190..73c4ce1 100644 --- a/src/harness/metric.ts +++ b/src/harness/metric.ts @@ -1,11 +1,11 @@ -import type { ChatMessage, ResponseItem, Score, ScoreValue } from "./core"; +import type { ModelMessage, ResponseItem, Score, ScoreValue } from "./core"; import { ScoreValue as SV, scoreToNumber } from "./core"; export interface SampleScore { readonly sampleId: string; readonly epoch: number; readonly score: Score; - readonly messages?: readonly ChatMessage[]; + readonly messages?: readonly ModelMessage[]; readonly responseItems?: readonly ResponseItem[]; readonly requestBody?: Readonly>; readonly generationIds?: readonly string[]; diff --git a/src/harness/model.ts b/src/harness/model.ts index c9d5bd0..0f85c28 100644 --- a/src/harness/model.ts +++ b/src/harness/model.ts @@ -4,7 +4,7 @@ import type { Effect } from "effect/Effect"; import type { ProviderSort } from "../internal/enums"; import type { CostTier, ReasoningEffort } from "./constants"; import type { - ChatMessage, + ModelMessage, ModelError, ModelOutput, ToolDefinition, @@ -40,7 +40,7 @@ export class Model extends Tag("@openrouter/bench-harness/model")< export interface ModelService { readonly generate: ( - messages: readonly ChatMessage[], + messages: readonly ModelMessage[], config: GenerateConfig ) => Effect; } diff --git a/src/providers/app-identity.ts b/src/providers/app-identity.ts new file mode 100644 index 0000000..c02eeaa --- /dev/null +++ b/src/providers/app-identity.ts @@ -0,0 +1,4 @@ +export const BENCH_HARNESS_APP_REFERRER = + "https://bench-harness.openrouter.ai/"; + +export const BENCH_HARNESS_APP_TITLE = "OpenRouter: Bench Harness"; diff --git a/src/providers/messages-to-responses.test.ts b/src/providers/messages-to-responses.test.ts new file mode 100644 index 0000000..1b0adfc --- /dev/null +++ b/src/providers/messages-to-responses.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it } from "bun:test"; + +import type { ModelMessage } from "../harness/core"; +import { MessageRole } from "../harness/core"; +import { + messagesToResponses, + responsesTurnToModelOutput, + toolDefinitionToResponses, +} from "./messages-to-responses"; + +describe("messages-to-responses", () => { + it("maps system and user messages", () => { + expect( + messagesToResponses([ + { role: MessageRole.System, content: "rules" }, + { role: MessageRole.User, content: "question" }, + ]) + ).toEqual([ + { type: "message", role: "system", content: "rules" }, + { type: "message", role: "user", content: "question" }, + ]); + }); + + it("maps multimodal content with default and explicit image detail", () => { + expect( + messagesToResponses([ + { + role: MessageRole.User, + content: "", + contentParts: [ + { type: "text", text: "look" }, + { type: "image_url", imageUrl: { url: "a.png" } }, + { type: "image_url", imageUrl: { url: "b.png", detail: "high" } }, + { type: "video_url", videoUrl: { url: "v.mp4" } }, + ], + }, + ]) + ).toEqual([ + { + type: "message", + role: "user", + content: [ + { type: "input_text", text: "look" }, + { type: "input_image", image_url: "a.png", detail: "auto" }, + { type: "input_image", image_url: "b.png", detail: "high" }, + { type: "input_video", video_url: "v.mp4" }, + ], + }, + ]); + }); + + it("maps assistant tool calls and tool results", () => { + expect( + messagesToResponses([ + { + role: MessageRole.Assistant, + content: "", + toolCalls: [ + { + id: "call-1", + type: "function", + function: { name: "lookup", arguments: '{"x":1}' }, + }, + ], + }, + { + role: MessageRole.Tool, + content: "found", + toolCallId: "call-1", + }, + ]) + ).toEqual([ + { + type: "function_call", + call_id: "call-1", + name: "lookup", + arguments: '{"x":1}', + }, + { type: "function_call_output", call_id: "call-1", output: "found" }, + ]); + }); + + it("replays response items verbatim before synthesized fields", () => { + const responseItems = [ + { type: "reasoning", encrypted_content: "opaque" }, + { type: "function_call", call_id: "call-2" }, + ]; + const message: ModelMessage = { + role: MessageRole.Assistant, + content: "ignored", + responseItems, + toolCalls: [ + { + id: "call-1", + type: "function", + function: { name: "ignored", arguments: "{}" }, + }, + ], + }; + expect(messagesToResponses([message])).toEqual(responseItems); + }); + + it("omits empty assistant content", () => { + expect( + messagesToResponses([{ role: MessageRole.Assistant, content: "" }]) + ).toEqual([]); + }); + + it("maps tool definitions and omits undefined descriptions", () => { + expect( + toolDefinitionToResponses({ + type: "function", + function: { + name: "lookup", + parameters: { type: "object" }, + strict: true, + }, + }) + ).toEqual({ + type: "function", + name: "lookup", + parameters: { type: "object" }, + strict: true, + }); + }); + + it("uses call_id when converting Responses function calls", () => { + const output = responsesTurnToModelOutput({ + text: "", + outputItems: [{ type: "function_call", id: "item-1" }], + functionCalls: [ + { callId: "call-1", name: "lookup", arguments: '{"x":1}' }, + ], + generationTimeMs: 7, + }); + expect(output.message.toolCalls).toEqual([ + { + id: "call-1", + type: "function", + function: { name: "lookup", arguments: '{"x":1}' }, + }, + ]); + }); +}); diff --git a/src/providers/messages-to-responses.ts b/src/providers/messages-to-responses.ts new file mode 100644 index 0000000..1faab4c --- /dev/null +++ b/src/providers/messages-to-responses.ts @@ -0,0 +1,131 @@ +import type { + ModelMessage, + ContentPart, + ModelOutput, + ToolDefinition, +} from "../harness/core"; +import { MessageRole } from "../harness/core"; +import type { + ResponsesFunctionTool, + ResponsesInputItem, + ResponsesTurn, +} from "./responses-model"; + +export function messagesToResponses( + messages: readonly ModelMessage[] +): readonly ResponsesInputItem[] { + return messages.flatMap(messageToResponses); +} + +function messageToResponses(message: ModelMessage): ResponsesInputItem[] { + switch (message.role) { + case MessageRole.System: + case MessageRole.User: { + return [ + { + type: "message", + role: message.role, + content: + message.role === MessageRole.User && + message.contentParts !== undefined + ? message.contentParts.map(contentPartToResponses) + : message.content, + }, + ]; + } + case MessageRole.Assistant: { + if (message.responseItems !== undefined) { + return [...message.responseItems]; + } + const items: ResponsesInputItem[] = []; + if (message.content.length > 0) { + items.push({ + type: "message", + role: "assistant", + content: message.content, + }); + } + for (const toolCall of message.toolCalls ?? []) { + items.push({ + type: "function_call", + call_id: toolCall.id, + name: toolCall.function.name, + arguments: toolCall.function.arguments, + }); + } + return items; + } + case MessageRole.Tool: { + return [ + { + type: "function_call_output", + call_id: message.toolCallId ?? "", + output: message.content, + }, + ]; + } + default: { + return message.role satisfies never; + } + } +} + +function contentPartToResponses(part: ContentPart): ResponsesInputItem { + switch (part.type) { + case "text": { + return { type: "input_text", text: part.text }; + } + case "image_url": { + return { + type: "input_image", + image_url: part.imageUrl.url, + detail: part.imageUrl.detail ?? "auto", + }; + } + case "video_url": { + return { type: "input_video", video_url: part.videoUrl.url }; + } + default: { + return part satisfies never; + } + } +} + +export function toolDefinitionToResponses( + tool: ToolDefinition +): ResponsesFunctionTool { + return { + type: "function", + name: tool.function.name, + ...(tool.function.description !== undefined && { + description: tool.function.description, + }), + parameters: tool.function.parameters ?? {}, + ...(tool.function.strict !== undefined && { + strict: tool.function.strict, + }), + }; +} + +export function responsesTurnToModelOutput(turn: ResponsesTurn): ModelOutput { + return { + completion: turn.text, + message: { + role: MessageRole.Assistant, + content: turn.text, + ...(turn.functionCalls.length > 0 && { + toolCalls: turn.functionCalls.map((call) => ({ + id: call.callId, + type: "function" as const, + function: { + name: call.name, + arguments: call.arguments, + }, + })), + }), + responseItems: turn.outputItems, + }, + ...(turn.usage !== undefined && { usage: turn.usage }), + generationTimeMs: turn.generationTimeMs, + }; +} diff --git a/src/providers/openrouter-model.test.ts b/src/providers/openrouter-model.test.ts index 80da0cc..ffb85e5 100644 --- a/src/providers/openrouter-model.test.ts +++ b/src/providers/openrouter-model.test.ts @@ -1,1322 +1,146 @@ -import type { Mock } from "bun:test"; -import { afterEach, describe, expect, it, spyOn } from "bun:test"; -import assert from "node:assert"; +import { afterEach, describe, expect, it } from "bun:test"; +import { readFile } from "node:fs/promises"; import { FetchHttpClient } from "@effect/platform"; -import { failureOption } from "effect/Cause"; -import { - flatMap, - gen, - provide, - runPromise, - runPromiseExit, -} from "effect/Effect"; -import type { Exit, Failure } from "effect/Exit"; -import { isFailure, isSuccess } from "effect/Exit"; +import { gen, provide, runPromiseExit } from "effect/Effect"; import { provide as layerProvide } from "effect/Layer"; -import { getOrUndefined } from "effect/Option"; -import { assertFailure, assertSuccess } from "../../test/helpers/exit-asserts"; -import type { CapturedRequest } from "../../test/helpers/fetch-sequence"; -import { installFetchSequence } from "../../test/helpers/fetch-sequence"; -import type { ModelError, ModelOutput } from "../harness/core"; -import { MessageRole } from "../harness/core"; +import { assertSuccess } from "../../test/helpers/exit-asserts"; import { Model } from "../harness/model"; -import { ProviderSort } from "../internal/enums"; -import { isRecord } from "../internal/guards"; -import { - getCollectedGenerationIdEntries, - getCollectedGenerationIds, - resetGenerationIds, -} from "../runtime/generation-ids"; -import { setCurrentEpoch, withRunAttempt } from "../runtime/response-cache"; import { makeOpenRouterModelLayer } from "./openrouter-model"; -const CHAT_RESULT = { - id: "1", - object: "chat.completion", - created: 0, - model: "m", - system_fingerprint: "", - choices: [ - { - index: 0, - message: { role: "assistant", content: "Answer: A" }, - finish_reason: "stop", - }, - ], - usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, -}; - -const CHAT_RESULT_JSON = JSON.stringify(CHAT_RESULT); - -const CHAT_RESULT_WITH_RAW_USAGE = { - ...CHAT_RESULT, - choices: [ - { - ...CHAT_RESULT.choices[0], - logprobs: null, - }, - ], - usage: { - ...CHAT_RESULT.usage, - cost: 0.25, - completion_tokens_details: { reasoning_tokens: 3 }, - }, -}; - -const REASONING_CHAT_RESULT = { - ...CHAT_RESULT, - model: "openai/gpt-4o", -}; - -function installFetchCapture(captured: { - value: CapturedRequest | undefined; -}): () => void { - const original = globalThis.fetch; - const stub: typeof fetch = async (input, init) => { - const req = input instanceof Request ? input : new Request(input, init); - const { url } = req; - const rawBody = await req.clone().text(); - const body: Record = parseJsonObject(rawBody); - const headers: Record = {}; - for (const [k, v] of req.headers.entries()) { - headers[k.toLowerCase()] = v; - } - captured.value = { url, body, headers, signal: req.signal }; - return new Response(JSON.stringify(CHAT_RESULT), { - status: 200, - headers: { "content-type": "application/json" }, - }); - }; - globalThis.fetch = stub; - return () => { - globalThis.fetch = original; - }; -} - -interface CapturedHolder { - value: CapturedRequest | undefined; -} - -function newHolder(): CapturedHolder { - return { value: undefined }; -} - -function parseJsonObject(raw: string): Record { - if (raw.length === 0) { - return {}; - } - const parsed: unknown = JSON.parse(raw); - if (!isRecord(parsed)) { - return {}; - } - return parsed; +function withFunctionCall(stream: string): string { + return stream + .split("\n") + .map((line) => { + if (!line.startsWith("data: ") || line === "data: [DONE]") { + return line; + } + const event: unknown = JSON.parse(line.slice(6)); + if ( + typeof event !== "object" || + event === null || + !("type" in event) || + event.type !== "response.completed" + ) { + return line; + } + const response = event.response; + if ( + typeof response !== "object" || + response === null || + !("output" in response) || + !Array.isArray(response.output) + ) { + return line; + } + return `data: ${JSON.stringify({ + ...event, + response: { + ...response, + output: [ + ...response.output, + { + type: "function_call", + id: "item-1", + call_id: "call-1", + name: "lookup", + arguments: '{"x":1}', + }, + ], + }, + })}`; + }) + .join("\n"); } -const MESSAGES = [{ role: MessageRole.User, content: "q" }] as const; - -function modelErrorFrom( - exit: Failure -): ModelError | undefined { - return getOrUndefined(failureOption(exit.cause)); -} -describe("openrouter-model request parity", () => { +describe("openrouter-model", () => { let restore: (() => void) | undefined; + afterEach(() => { restore?.(); restore = undefined; }); - it("sends provider.sort on unpinned runs", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, { sort: ProviderSort.Price }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["provider"]).toEqual({ sort: "price" }); - expect(captured.value?.body["provider"]).not.toHaveProperty("ignore"); - }); - it("sends provider.only with fallbacks disabled on pinned runs", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, { - providerOnly: ["google-vertex"], - providerIgnore: ["azure"], - allowFallbacks: false, - }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["provider"]).toEqual({ - only: ["google-vertex"], - ignore: ["azure"], - allow_fallbacks: false, - }); - }); - it("merges sort with provider.only on pinned runs", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, { - sort: ProviderSort.Price, - providerOnly: ["google-vertex"], - providerIgnore: ["azure"], - allowFallbacks: false, - }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["provider"]).toEqual({ - sort: "price", - only: ["google-vertex"], - ignore: ["azure"], - allow_fallbacks: false, - }); - }); - it("sends provider.ignore without other provider preferences", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, { - providerIgnore: ["azure"], - }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["provider"]).toEqual({ - ignore: ["azure"], - }); - }); - it("records the chat completion generation id", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - const ids = await runPromise( - resetGenerationIds.pipe( - flatMap(() => - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, {}); - }) + + it("uses the Responses endpoint with streaming and cache control", async () => { + const stream = withFunctionCall( + await readFile( + new URL( + "../../test/fixtures/advisor-responses-stream.sse", + import.meta.url ), - flatMap(() => getCollectedGenerationIds), - provide(layer.pipe(layerProvide(FetchHttpClient.layer))) + "utf8" ) ); - expect(ids).toEqual(["1"]); - }); - it("records the cache source id from the response header on cache hits", async () => { - const original = globalThis.fetch; - globalThis.fetch = (async () => - new Response(CHAT_RESULT_JSON, { - status: 200, - headers: { - "content-type": "application/json", - "x-openrouter-cache-status": "HIT", - "x-openrouter-cache-source-id": "gen-source", - }, - })) as typeof fetch; - try { - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - const entries = await runPromise( - resetGenerationIds.pipe( - flatMap(() => - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, {}); - }) - ), - flatMap(() => getCollectedGenerationIdEntries), - provide(layer.pipe(layerProvide(FetchHttpClient.layer))) - ) - ); - expect(entries).toEqual([ - { - id: "gen-source", - isCacheHit: true, - countsTowardUsage: true, - isResolvedSource: true, - }, - ]); - } finally { - globalThis.fetch = original; - } - }); - it("records the dummy id when a cache hit has no source id header", async () => { - const original = globalThis.fetch; - globalThis.fetch = (async () => - new Response(CHAT_RESULT_JSON, { - status: 200, - headers: { - "content-type": "application/json", - "x-openrouter-cache-status": "HIT", - }, - })) as typeof fetch; - try { - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - const entries = await runPromise( - resetGenerationIds.pipe( - flatMap(() => - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, {}); - }) - ), - flatMap(() => getCollectedGenerationIdEntries), - provide(layer.pipe(layerProvide(FetchHttpClient.layer))) - ) - ); - expect(entries).toEqual([ - { - id: "1", - isCacheHit: true, - countsTowardUsage: true, - isResolvedSource: false, - }, - ]); - } finally { - globalThis.fetch = original; - } - }); - it("suppresses sort when endpointId is set (pinning overrides sorting)", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, { - sort: ProviderSort.Price, - endpointId: "ep-1", - }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["provider"]).toBeUndefined(); - expect(captured.value?.headers["x-or-endpoint-id"]).toBe("ep-1"); - }); - it("serializes a video_url content part by url", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "google/gemini-2.5-flash", - apiKey: "sk-test", - }); - const messages = [ - { - role: MessageRole.User, - content: "", - contentParts: [ - { - type: "video_url" as const, - videoUrl: { url: "https://cdn.seldon.global/v/clip.mp4" }, - }, - { type: "text" as const, text: "describe the video" }, - ], - }, - ]; - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(messages, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - const sentMessages = captured.value?.body["messages"] as unknown[]; - expect(sentMessages).toBeDefined(); - expect(sentMessages[0]).toEqual({ - role: "user", - content: [ - { - type: "video_url", - video_url: { url: "https://cdn.seldon.global/v/clip.mp4" }, - }, - { type: "text", text: "describe the video" }, - ], - }); - }); - it("sends reasoning_effort, maxTokens, temperature", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, { - temperature: 0, - maxTokens: 128, - reasoningEffort: "high", - }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["temperature"]).toBe(0); - expect(captured.value?.body["max_tokens"]).toBe(128); - expect(captured.value?.body["reasoning_effort"]).toBe("high"); - }); - it("sends the Cloudflare version override header", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, { cloudflareVersion: "ver-9" }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect( - captured.value?.headers["cloudflare-workers-version-overrides"] - ).toBe("ver-9"); - }); - it("sends x-session-id header when sessionId is configured", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - sessionId: "bench-run-abc123", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.headers["x-session-id"]).toBe("bench-run-abc123"); - }); - it("omits x-session-id header when sessionId is not configured", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.headers["x-session-id"]).toBeUndefined(); - }); - it("sends app attribution headers on every call", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.headers["http-referer"]).toBe( - "https://bench-harness.openrouter.ai/" - ); - expect(captured.value?.headers["x-openrouter-title"]).toBe( - "OpenRouter: Bench Harness" - ); - }); - it("always sends top-level cache_control", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["cache_control"]).toEqual({ - type: "ephemeral", - }); - }); - it("omits provider, reasoning_effort, and headers when unset", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, { temperature: 0.5 }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["provider"]).toBeUndefined(); - expect(captured.value?.body["reasoning_effort"]).toBeUndefined(); - expect(captured.value?.headers["x-or-endpoint-id"]).toBeUndefined(); - expect( - captured.value?.headers["cloudflare-workers-version-overrides"] - ).toBeUndefined(); - }); - it("preserves the raw non-streaming response alongside the mapped output", async () => { - const captured: CapturedRequest[] = []; - restore = installFetchSequence([CHAT_RESULT_WITH_RAW_USAGE], captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - const exit = await runPromiseExit( - gen(function* run() { - const model = yield* Model; - return yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - assertSuccess(exit); - expect(exit.value.rawResponse).toEqual(CHAT_RESULT_WITH_RAW_USAGE); - expect(exit.value.completion).toBe("Answer: A"); - expect(exit.value.message).toEqual({ - role: MessageRole.Assistant, - content: "Answer: A", - model: "m", - }); - expect(exit.value.usage).toEqual({ - inputTokens: 1, - outputTokens: 1, - totalTokens: 2, - reasoningTokens: 3, - totalCost: 0.25, - }); - }); -}); -describe("openrouter-model response caching", () => { - let restore: (() => void) | undefined; - afterEach(() => { - restore?.(); - restore = undefined; - }); - it("always sends the response-cache header", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.headers["x-openrouter-cache"]).toBe("true"); - }); - it("sends a session- and epoch-scoped cache salt header", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - sessionId: "wf-123", - }); - await runPromiseExit( - gen(function* run() { - yield* setCurrentEpoch(1); - const model = yield* Model; - yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.headers["x-openrouter-cache-salt"]).toBe( - "wf-123:epoch-1" - ); - expect(captured.value?.body["cache_salt"]).toBeUndefined(); - }); - it("varies the cache salt header across epochs", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - sessionId: "wf-123", - }); - const salts: unknown[] = []; - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* setCurrentEpoch(0); - yield* model.generate(MESSAGES, {}); - salts.push(captured.value?.headers["x-openrouter-cache-salt"]); - yield* setCurrentEpoch(1); - yield* model.generate(MESSAGES, {}); - salts.push(captured.value?.headers["x-openrouter-cache-salt"]); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(salts).toEqual(["wf-123:epoch-0", "wf-123:epoch-1"]); - }); - it("appends the retry attempt to the cache salt header on in-process retries", async () => { - const salts: unknown[] = []; - const original = globalThis.fetch; - let callCount = 0; - const stub: typeof fetch = async (input, init) => { - const req = input instanceof Request ? input : new Request(input, init); - salts.push(req.headers.get("x-openrouter-cache-salt")); - callCount += 1; - if (callCount === 1) { - return new Response(JSON.stringify({ error: { message: "slow" } }), { - status: 429, - headers: { "content-type": "application/json", "retry-after": "0" }, - }); - } - return new Response(CHAT_RESULT_JSON, { + const originalFetch = globalThis.fetch; + let request: Request | undefined; + globalThis.fetch = async (input, init) => { + request = input instanceof Request ? input : new Request(input, init); + return new Response(stream, { status: 200, - headers: { "content-type": "application/json" }, + headers: { "content-type": "text/event-stream" }, }); }; - globalThis.fetch = stub; restore = () => { - globalThis.fetch = original; + globalThis.fetch = originalFetch; }; - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - sessionId: "wf-123", - retry: { maxRetries: 2, baseDelayMs: 1 }, - }); - await runPromiseExit( - gen(function* run() { - yield* setCurrentEpoch(1); - const model = yield* Model; - yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(salts).toEqual(["wf-123:epoch-1", "wf-123:epoch-1:attempt-1"]); - }); - it("omits the cache salt header when session id and epoch are unset", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.headers["x-openrouter-cache-salt"]).toBeUndefined(); - expect(captured.value?.body["cache_salt"]).toBeUndefined(); - }); -}); -describe("openrouter-model auto-router plugin", () => { - let restore: (() => void) | undefined; - afterEach(() => { - restore?.(); - restore = undefined; - }); - it("sends pin_model on the auto-beta-router plugin", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openrouter/auto-beta", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, { pinModel: true }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["plugins"]).toEqual([ - { id: "auto-beta-router", pin_model: true }, - ]); - }); - it("sends pin_model on the auto-router plugin for openrouter/auto", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openrouter/auto", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, { pinModel: true }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["plugins"]).toEqual([ - { id: "auto-router", pin_model: true }, - ]); - }); - for (const [model, pluginId] of [ - ["openrouter/auto", "auto-router"], - ["openrouter/auto-beta", "auto-beta-router"], - ] as const) { - it(`sends cost_tier on the ${pluginId} plugin`, async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ model, apiKey: "sk-test" }); - await runPromiseExit( - gen(function* run() { - const modelService = yield* Model; - yield* modelService.generate(MESSAGES, { costTier: "xhigh" }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["plugins"]).toEqual([ - { id: pluginId, cost_tier: "xhigh" }, - ]); - }); - } - it("merges pin_model and cost_quality_tradeoff into one plugin entry", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openrouter/auto-beta", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, { - pinModel: true, - costQualityTradeoff: 8, - }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["plugins"]).toEqual([ - { id: "auto-beta-router", cost_quality_tradeoff: 8, pin_model: true }, - ]); - }); - it("omits plugins when no auto-router option is set", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openrouter/auto-beta", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["plugins"]).toBeUndefined(); - }); - it("omits plugins for non-auto models even when pinModel is set", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(MESSAGES, { pinModel: true }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["plugins"]).toBeUndefined(); - }); - it("echoes the served model on the returned assistant message", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openrouter/auto-beta", - apiKey: "sk-test", - }); + const exit = await runPromiseExit( - gen(function* run() { + gen(function* () { const model = yield* Model; - return yield* model.generate(MESSAGES, { pinModel: true }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - assertSuccess(exit); - expect(exit.value.message.model).toBe("m"); - }); - it("re-emits model on assistant messages sent back in history", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openrouter/auto-beta", - apiKey: "sk-test", - }); - const history = [ - { role: MessageRole.User, content: "q" }, - { role: MessageRole.Assistant, content: "a", model: "openai/gpt-4o" }, - ] as const; - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(history, { pinModel: true }); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - const sentMessages = captured.value?.body["messages"]; - const assistant = Array.isArray(sentMessages) - ? sentMessages.find((m) => isRecord(m) && m["role"] === "assistant") - : undefined; - expect( - assistant && isRecord(assistant) ? assistant["model"] : undefined - ).toBe("openai/gpt-4o"); - }); - it("keeps history without reasoning_details in the pre-replay wire shape", async () => { - const captured = newHolder(); - restore = installFetchCapture(captured); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - const history = [ - { role: MessageRole.User, content: "q" }, - { role: MessageRole.Assistant, content: "a" }, - ] as const; - await runPromiseExit( - gen(function* run() { - const model = yield* Model; - yield* model.generate(history, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(captured.value?.body["messages"]).toEqual([ - { role: "user", content: "q" }, - { role: "assistant", content: "a" }, - ]); - }); - it("replays opaque reasoning_details with and without tool calls", async () => { - const captured: CapturedRequest[] = []; - const reasoningDetails = [ - { type: "summary", summary: "opaque" }, - { type: "future_variant", future_payload: { step: 1 } }, - ]; - const textReasoningDetails = [{ type: "summary", summary: "text" }]; - restore = installFetchSequence( - [ - { - ...REASONING_CHAT_RESULT, - choices: [ - { - ...CHAT_RESULT.choices[0], - message: { - role: "assistant", - content: "tool response", - reasoning_details: reasoningDetails, - tool_calls: [ - { - id: "call_1", - type: "function", - function: { name: "lookup", arguments: "{}" }, - }, - ], - }, - }, + return yield* model.generate( + [ + { role: "system", content: "rules" }, + { role: "user", content: "question" }, ], - }, - { - ...REASONING_CHAT_RESULT, - choices: [ - { - ...CHAT_RESULT.choices[0], - message: { - role: "assistant", - content: "text response", - reasoning_details: textReasoningDetails, + { + temperature: 0, + tools: [ + { + type: "function", + function: { + name: "lookup", + description: "Find a record", + parameters: { type: "object" }, + }, }, - }, - ], - }, - REASONING_CHAT_RESULT, - ], - captured - ); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - const first = await runPromise( - gen(function* run() { - const model = yield* Model; - return yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - const text = await runPromise( - gen(function* run() { - const model = yield* Model; - return yield* model.generate([MESSAGES[0], first.message], {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - await runPromise( - gen(function* run() { - const model = yield* Model; - yield* model.generate([MESSAGES[0], first.message, text.message], {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - const secondMessages = captured[1]?.body["messages"]; - assert(Array.isArray(secondMessages)); - const assistant = secondMessages.find( - (message) => isRecord(message) && message["role"] === "assistant" - ); - assert(isRecord(assistant)); - expect(assistant).toMatchObject({ - content: "tool response", - reasoning_details: reasoningDetails, - tool_calls: [ - { - id: "call_1", - type: "function", - function: { name: "lookup", arguments: "{}" }, - }, - ], - }); - const thirdMessages = captured[2]?.body["messages"]; - assert(Array.isArray(thirdMessages)); - const textAssistant = thirdMessages.find( - (message) => isRecord(message) && message["content"] === "text response" - ); - assert(isRecord(textAssistant)); - expect(textAssistant).toMatchObject({ - content: "text response", - reasoning_details: textReasoningDetails, - }); - expect(text.message.reasoningDetails).toEqual(textReasoningDetails); - }); - it("omits reasoning_details when the provider returns none", async () => { - const captured: CapturedRequest[] = []; - restore = installFetchSequence( - [REASONING_CHAT_RESULT, REASONING_CHAT_RESULT], - captured - ); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - }); - const first = await runPromise( - gen(function* run() { - const model = yield* Model; - return yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - await runPromise( - gen(function* run() { - const model = yield* Model; - yield* model.generate([MESSAGES[0], first.message], {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - const secondMessages = captured[1]?.body["messages"]; - assert(Array.isArray(secondMessages)); - const assistant = secondMessages.find( - (message) => isRecord(message) && message["role"] === "assistant" - ); - assert(isRecord(assistant)); - expect(assistant["reasoning_details"]).toBeUndefined(); - }); -}); -describe("openrouter-model transient fetch retry", () => { - let restore: (() => void) | undefined; - afterEach(() => { - restore?.(); - restore = undefined; - }); - function installFlakyFetch( - failTimes: number, - failure: "network" | "5xx" | "429", - fetchCalls: { - count: number; - } - ): () => void { - const original = globalThis.fetch; - const stub: typeof fetch = async () => { - fetchCalls.count += 1; - if (fetchCalls.count <= failTimes) { - if (failure === "network") { - throw new TypeError("terminated"); - } - const status = failure === "5xx" ? 503 : 429; - const headers: Record = - failure === "429" ? { "retry-after": "0" } : {}; - return new Response(`http ${status}`, { status, headers }); - } - return new Response(CHAT_RESULT_JSON, { - status: 200, - headers: { "content-type": "application/json" }, - }); - }; - globalThis.fetch = stub; - return () => { - globalThis.fetch = original; - }; - } - it("retries a transient network error (terminated) then succeeds", async () => { - const fetchCalls = { count: 0 }; - restore = installFlakyFetch(2, "network", fetchCalls); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - retry: { baseDelayMs: 0, maxRetries: 5 }, - }); - const exit = await runPromiseExit( - gen(function* run() { - const model = yield* Model; - return yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(isSuccess(exit)).toBe(true); - expect(fetchCalls.count).toBe(3); - }); - it("retries a 5xx then succeeds", async () => { - const fetchCalls = { count: 0 }; - restore = installFlakyFetch(1, "5xx", fetchCalls); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - retry: { baseDelayMs: 0, maxRetries: 5 }, - }); - const exit = await runPromiseExit( - gen(function* run() { - const model = yield* Model; - return yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(isSuccess(exit)).toBe(true); - expect(fetchCalls.count).toBe(2); - }); - it("does not retry a non-retryable 4xx (400)", async () => { - const fetchCalls = { count: 0 }; - const original = globalThis.fetch; - globalThis.fetch = async () => { - fetchCalls.count += 1; - return new Response("bad request", { status: 400 }); - }; - restore = () => { - globalThis.fetch = original; - }; - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - retry: { baseDelayMs: 0, maxRetries: 5 }, - }); - const exit = await runPromiseExit( - gen(function* run() { - const model = yield* Model; - return yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - expect(isFailure(exit)).toBe(true); - expect(fetchCalls.count).toBe(1); - }); - it("generationTimeMs excludes retry backoff delays", async () => { - const fetchCalls = { count: 0 }; - restore = installFlakyFetch(2, "5xx", fetchCalls); - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - retry: { baseDelayMs: 200, maxRetries: 5 }, - }); - const exit = await runPromiseExit( - gen(function* run() { - const model = yield* Model; - return yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) + ], + } + ); + }).pipe( + provide( + makeOpenRouterModelLayer({ + model: "openai/gpt-5", + apiKey: "sk-test", + baseUrl: "https://example.test", + }).pipe(layerProvide(FetchHttpClient.layer)) + ) + ) ); assertSuccess(exit); - expect(exit.value.generationTimeMs).toBeLessThan(600); - }); -}); -describe("openrouter-model 2xx error envelope", () => { - let restore: (() => void) | undefined; - const warnSpies: { - mockRestore: () => void; - }[] = []; - afterEach(() => { - restore?.(); - restore = undefined; - for (const warn of warnSpies.splice(0)) { - warn.mockRestore(); - } - }); - function silenceWarnings(): Mock<(...args: unknown[]) => void> { - const warn = spyOn(console, "warn").mockImplementation(() => {}); - warnSpies.push(warn); - return warn; - } - function installBodyFetch( - body: string, - bodyTimes: number, - fetchCalls: { - count: number; - } - ): () => void { - const original = globalThis.fetch; - globalThis.fetch = async () => { - fetchCalls.count += 1; - const payload = fetchCalls.count <= bodyTimes ? body : CHAT_RESULT_JSON; - return new Response(payload, { - status: 200, - headers: { "content-type": "application/json" }, - }); - }; - return () => { - globalThis.fetch = original; - }; - } - function generateWithRetries( - maxRetries: number - ): Promise> { - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - retry: { baseDelayMs: 0, maxRetries }, + expect(request?.url).toBe("https://example.test/api/v1/responses"); + const body = JSON.parse(await request!.clone().text()) as Record< + string, + unknown + >; + expect(body).toMatchObject({ + input: [ + { type: "message", role: "system", content: "rules" }, + { type: "message", role: "user", content: "question" }, + ], + stream: true, + cache_control: { type: "ephemeral" }, + temperature: 0, }); - return runPromiseExit( - gen(function* run() { - const model = yield* Model; - return yield* model.generate(MESSAGES, {}); - }).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - } - function warnContextFor( - warn: Mock<(...args: unknown[]) => void> - ): Record { - const call = warn.mock.calls.find( - (args) => args[0] === "OpenRouter 2xx response did not yield a completion" - ); - const context = call?.[1]; - assert(isRecord(context), "expected an unusable-body log record"); - return context; - } - it("retries a 200 error envelope with code 429 and then succeeds", async () => { - silenceWarnings(); - const fetchCalls = { count: 0 }; - restore = installBodyFetch( - JSON.stringify({ - error: { message: "Provider returned error", code: 429 }, - }), - 1, - fetchCalls - ); - const exit = await generateWithRetries(5); - assertSuccess(exit); - expect(fetchCalls.count).toBe(2); - }); - it("surfaces status 429 from a persistent 200 error envelope", async () => { - silenceWarnings(); - const fetchCalls = { count: 0 }; - restore = installBodyFetch( - JSON.stringify({ - error: { message: "Provider returned error", code: 429 }, - }), - Number.POSITIVE_INFINITY, - fetchCalls - ); - const exit = await generateWithRetries(1); - assertFailure(exit); - expect(modelErrorFrom(exit)?.status).toBe(429); - expect(fetchCalls.count).toBe(2); - }); - it("surfaces status 429 from an error envelope whose code is a numeric string", async () => { - silenceWarnings(); - const fetchCalls = { count: 0 }; - restore = installBodyFetch( - JSON.stringify({ - error: { message: "Provider returned error", code: "429" }, - }), - Number.POSITIVE_INFINITY, - fetchCalls - ); - const exit = await generateWithRetries(1); - assertFailure(exit); - expect(modelErrorFrom(exit)?.status).toBe(429); - expect(fetchCalls.count).toBe(2); - }); - it("does not retry a 200 error envelope carrying a non-retryable code", async () => { - silenceWarnings(); - const fetchCalls = { count: 0 }; - restore = installBodyFetch( - JSON.stringify({ error: { message: "bad request", code: 400 } }), - Number.POSITIVE_INFINITY, - fetchCalls - ); - const exit = await generateWithRetries(5); - assertFailure(exit); - expect(modelErrorFrom(exit)?.status).toBe(400); - expect(fetchCalls.count).toBe(1); - }); - it("retries an envelope whose code is not an HTTP status", async () => { - silenceWarnings(); - const fetchCalls = { count: 0 }; - restore = installBodyFetch( - JSON.stringify({ error: { message: "provider overloaded", code: 1000 } }), - 1, - fetchCalls - ); - const exit = await generateWithRetries(5); - assertSuccess(exit); - expect(fetchCalls.count).toBe(2); - }); - it("decodes a 200 body that carries both choices and an error", async () => { - silenceWarnings(); - const fetchCalls = { count: 0 }; - restore = installBodyFetch( - JSON.stringify({ ...CHAT_RESULT, error: { code: 429 } }), - 1, - fetchCalls - ); - const exit = await generateWithRetries(5); - assertSuccess(exit); - expect(fetchCalls.count).toBe(1); - }); - it("decodes a 200 body that omits required unused fields", async () => { - silenceWarnings(); - const fetchCalls = { count: 0 }; - const { - system_fingerprint: _systemFingerprint, - created: _created, - object: _object, - ...body - } = CHAT_RESULT; - restore = installBodyFetch(JSON.stringify(body), 1, fetchCalls); - const exit = await generateWithRetries(5); - assertSuccess(exit); - expect(exit.value.completion).toBe("Answer: A"); - expect(fetchCalls.count).toBe(1); - }); - it("treats an envelope with null choices as an error envelope", async () => { - silenceWarnings(); - const fetchCalls = { count: 0 }; - restore = installBodyFetch( - JSON.stringify({ - choices: null, - error: { message: "bad request", code: 400 }, - }), - Number.POSITIVE_INFINITY, - fetchCalls - ); - const exit = await generateWithRetries(5); - assertFailure(exit); - expect(modelErrorFrom(exit)?.status).toBe(400); - expect(fetchCalls.count).toBe(1); - }); - it("retries an unparseable 200 body and logs it truncated to 2000 characters", async () => { - const warn = silenceWarnings(); - const fetchCalls = { count: 0 }; - const body = `not json ${"x".repeat(3000)}`; - restore = installBodyFetch(body, 1, fetchCalls); - const exit = await generateWithRetries(5); - assertSuccess(exit); - expect(fetchCalls.count).toBe(2); - expect(warnContextFor(warn)["raw_body"]).toBe(`${body.slice(0, 1997)}...`); - }); - it("retries a 200 body that fails schema validation and logs the raw body", async () => { - const warn = silenceWarnings(); - const fetchCalls = { count: 0 }; - const body = JSON.stringify({ unexpected: true }); - restore = installBodyFetch(body, 1, fetchCalls); - const exit = await generateWithRetries(5); - assertSuccess(exit); - expect(fetchCalls.count).toBe(2); - expect(warnContextFor(warn)["raw_body"]).toBe(body); - }); -}); - -describe("openrouter-model response cache miss logging", () => { - let restore: (() => void) | undefined; - let warn: Mock<(...args: unknown[]) => void> | undefined; - afterEach(() => { - restore?.(); - restore = undefined; - warn?.mockRestore(); - warn = undefined; - }); - function installCacheStatusFetch(cacheStatus: string): () => void { - const original = globalThis.fetch; - globalThis.fetch = async () => - new Response(CHAT_RESULT_JSON, { - status: 200, - headers: { - "content-type": "application/json", - "x-openrouter-cache-status": cacheStatus, - }, - }); - return () => { - globalThis.fetch = original; - }; - } - function generateOnRunAttempt( - runAttempt: number - ): Promise> { - const layer = makeOpenRouterModelLayer({ - model: "openai/gpt-4o", - apiKey: "sk-test", - sessionId: "wf-123", + expect(exit.value.completion).toBe("4"); + expect(exit.value.message.toolCalls?.[0]).toEqual({ + id: "call-1", + type: "function", + function: { name: "lookup", arguments: '{"x":1}' }, }); - return runPromiseExit( - withRunAttempt( - runAttempt, - gen(function* run() { - const model = yield* Model; - return yield* model.generate(MESSAGES, {}); - }) - ).pipe(provide(layer.pipe(layerProvide(FetchHttpClient.layer)))) - ); - } - it("warns when a retried run misses the response cache", async () => { - warn = spyOn(console, "warn").mockImplementation(() => {}); - restore = installCacheStatusFetch("MISS"); - const exit = await generateOnRunAttempt(2); - assertSuccess(exit); - expect(warn).toHaveBeenCalledTimes(1); - const [message, context] = warn.mock.calls[0] ?? []; - expect(message).toBe("Expected response cache hit on run retry but missed"); - expect(context).toMatchObject({ - run_attempt: 2, - cache_salt: "wf-123", - cache_status: "MISS", - model: "openai/gpt-4o", + expect(exit.value.message.responseItems?.at(-1)).toMatchObject({ + call_id: "call-1", }); - }); - it("stays silent when a retried run hits the response cache", async () => { - warn = spyOn(console, "warn").mockImplementation(() => {}); - restore = installCacheStatusFetch("HIT"); - const exit = await generateOnRunAttempt(2); - assertSuccess(exit); - expect(warn).not.toHaveBeenCalled(); - }); - it("stays silent on the first run attempt", async () => { - warn = spyOn(console, "warn").mockImplementation(() => {}); - restore = installCacheStatusFetch("MISS"); - const exit = await generateOnRunAttempt(1); - assertSuccess(exit); - expect(warn).not.toHaveBeenCalled(); + expect(exit.value.rawResponse).toBeUndefined(); }); }); diff --git a/src/providers/openrouter-model.ts b/src/providers/openrouter-model.ts index de3a42d..12624ce 100644 --- a/src/providers/openrouter-model.ts +++ b/src/providers/openrouter-model.ts @@ -1,70 +1,20 @@ -import { HttpClient, HttpClientRequest } from "@effect/platform"; -import type { ChatUsage } from "@openrouter/sdk/models"; -import { ChatResult$inboundSchema } from "@openrouter/sdk/models/chatresult"; -import { millis } from "effect/Duration"; -import type { Effect } from "effect/Effect"; -import { - fail, - flatMap, - gen, - mapError, - succeed, - sync, - tapError, - timeout, - catchTag, -} from "effect/Effect"; +import { gen, map } from "effect/Effect"; import type { Layer } from "effect/Layer"; -import { effect } from "effect/Layer"; +import { effect, provide } from "effect/Layer"; -import type { - ChatMessage, - ContentPart, - ModelOutput, - ModelUsage, -} from "../harness/core"; -import { MessageRole, ModelError } from "../harness/core"; -import type { GenerateConfig } from "../harness/model"; -import { Model, stripVariantSuffix } from "../harness/model"; -import type { ReasoningDetails } from "../harness/reasoning-details"; -import { hasReasoningDetails } from "../harness/reasoning-details"; -import { Either } from "../internal/either"; -import { unknownErrorToString } from "../internal/errors"; -import { isDefinedAndNotNull, isRecord } from "../internal/guards"; -import { wLog } from "../internal/log"; -import { parseSchema, z } from "../internal/zod"; -import { recordGenerationId } from "../runtime/generation-ids"; -import { - buildResponseCacheSalt, - getCurrentCallSalt, - getCurrentEpoch, - getCurrentRetryAttempt, - getCurrentRunAttempt, - logUnexpectedResponseCacheMiss, - RESPONSE_CACHE_HEADER, - RESPONSE_CACHE_SALT_HEADER, - RESPONSE_CACHE_SOURCE_ID_HEADER, - RESPONSE_CACHE_STATUS_HEADER, - RESPONSE_CACHE_STATUS_HIT, - RESPONSE_CACHE_TTL_HEADER, - RESPONSE_CACHE_TTL_SECONDS, -} from "../runtime/response-cache"; +import { Model } from "../harness/model"; import type { RetryConfig } from "../runtime/retry"; -import { rateLimitRetrySchedule, retrySalted } from "../runtime/retry"; import { - buildAutoRouterPlugin, - toWireAutoRouterPlugin, -} from "./auto-router-plugin"; -import type { ModelErrorIdentifiers } from "./request-identifiers"; -import { - appendModelErrorIdentifiers, - modelErrorIdentifiersFromHeaders, -} from "./request-identifiers"; - -export const BENCH_HARNESS_APP_REFERRER = - "https://bench-harness.openrouter.ai/"; + messagesToResponses, + responsesTurnToModelOutput, + toolDefinitionToResponses, +} from "./messages-to-responses"; +import { makeResponsesModelLayer, ResponsesModel } from "./responses-model"; -export const BENCH_HARNESS_APP_TITLE = "OpenRouter: Bench Harness"; +export { + BENCH_HARNESS_APP_REFERRER, + BENCH_HARNESS_APP_TITLE, +} from "./app-identity"; export interface OpenRouterModelConfig { readonly model: string; @@ -81,503 +31,32 @@ export function normalizeBaseUrl(baseUrl: string): string { export function makeOpenRouterModelLayer( config: OpenRouterModelConfig -): Layer { - const baseUrl = normalizeBaseUrl( - config.baseUrl ?? "https://openrouter.ai/api/v1" - ); +): Layer { + const responsesLayer = makeResponsesModelLayer({ + model: config.model, + apiKey: config.apiKey, + ...(config.baseUrl !== undefined && { + baseUrl: normalizeBaseUrl(config.baseUrl), + }), + ...(config.sessionId !== undefined && { sessionId: config.sessionId }), + ...(config.retry !== undefined && { retry: config.retry }), + }); return effect(Model)( gen(function* () { - const client = yield* HttpClient.HttpClient; + const responsesModel = yield* ResponsesModel; return Model.of({ - generate: (messages, genConfig) => - generate( - { - model: config.model, - messages, - genConfig, - sessionId: config.sessionId, - apiKey: config.apiKey, - baseUrl, - retry: config.retry, - }, - client - ), + generate: (messages, generateConfig) => { + const { tools, ...rest } = generateConfig; + return responsesModel + .generate(messagesToResponses(messages), { + ...rest, + ...(tools !== undefined && { + tools: tools.map(toolDefinitionToResponses), + }), + }) + .pipe(map(responsesTurnToModelOutput)); + }, }); }) - ); -} - -interface GenerateOpts { - readonly model: string; - readonly messages: readonly ChatMessage[]; - readonly genConfig: GenerateConfig; - readonly sessionId?: string; - readonly apiKey: string; - readonly baseUrl: string; - readonly retry?: RetryConfig; -} - -export function generate( - opts: GenerateOpts, - client: HttpClient.HttpClient -): Effect { - const { model, messages, genConfig } = opts; - const headers: Record = { - Authorization: `Bearer ${opts.apiKey}`, - "Content-Type": "application/json", - "HTTP-Referer": BENCH_HARNESS_APP_REFERRER, - "X-OpenRouter-Title": BENCH_HARNESS_APP_TITLE, - [RESPONSE_CACHE_HEADER]: "true", - [RESPONSE_CACHE_TTL_HEADER]: `${RESPONSE_CACHE_TTL_SECONDS}`, - }; - if (genConfig.endpointId !== undefined) { - headers["X-OR-Endpoint-Id"] = genConfig.endpointId; - } - if (genConfig.cloudflareVersion !== undefined) { - headers["Cloudflare-Workers-Version-Overrides"] = - genConfig.cloudflareVersion; - } - if (opts.sessionId !== undefined) { - headers["x-session-id"] = opts.sessionId; - } - const sendSort = - genConfig.sort !== undefined && genConfig.endpointId === undefined; - const providerPreferences = { - ...(sendSort && { sort: genConfig.sort }), - ...(genConfig.providerOnly !== undefined && { - only: [...genConfig.providerOnly], - }), - ...(genConfig.providerIgnore !== undefined && { - ignore: [...genConfig.providerIgnore], - }), - ...(genConfig.allowFallbacks !== undefined && { - allow_fallbacks: genConfig.allowFallbacks, - }), - }; - const sendProvider = Object.keys(providerPreferences).length > 0; - const hasTimeout = - genConfig.timeoutMs !== undefined && genConfig.timeoutMs > 0; - const baseModel = stripVariantSuffix(model); - const autoRouterPlugin = buildAutoRouterPlugin(baseModel, genConfig); - const wireAutoRouterPlugin = - autoRouterPlugin === undefined - ? undefined - : toWireAutoRouterPlugin(autoRouterPlugin); - const attempt = gen(function* () { - const startedAt = performance.now(); - const epoch = yield* getCurrentEpoch; - const retryAttempt = yield* getCurrentRetryAttempt; - const runAttempt = yield* getCurrentRunAttempt; - const callSalt = yield* getCurrentCallSalt; - const cacheSalt = buildResponseCacheSalt( - opts.sessionId, - epoch, - retryAttempt, - callSalt - ); - const body = { - model, - messages: messages.map(toApiMessage), - stream: false, - cache_control: { type: "ephemeral" }, - ...(genConfig.temperature !== undefined && { - temperature: genConfig.temperature, - }), - ...(genConfig.maxTokens !== undefined && { - max_tokens: genConfig.maxTokens, - }), - ...(genConfig.tools !== undefined && - genConfig.tools.length > 0 && { tools: [...genConfig.tools] }), - ...(genConfig.reasoningEffort !== undefined && { - reasoning_effort: genConfig.reasoningEffort, - }), - ...(sendProvider && { provider: providerPreferences }), - ...(wireAutoRouterPlugin !== undefined && { - plugins: [wireAutoRouterPlugin], - }), - ...genConfig.extraBody, - }; - const request = HttpClientRequest.post( - `${opts.baseUrl}/chat/completions` - ).pipe( - HttpClientRequest.setHeaders({ - ...headers, - ...(cacheSalt !== undefined && { - [RESPONSE_CACHE_SALT_HEADER]: cacheSalt, - }), - }), - HttpClientRequest.bodyUnsafeJson(body) - ); - const response = yield* hasTimeout - ? client.execute(request).pipe( - timeout(millis(genConfig.timeoutMs!)), - catchTag("TimeoutException", () => - fail( - new ModelError({ - status: 408, - message: `Request timed out after ${genConfig.timeoutMs}ms`, - }) - ) - ) - ) - : client.execute(request); - const identifiers = modelErrorIdentifiersFromHeaders(response.headers); - const retryAfterHeader = response.headers["retry-after"] ?? null; - if (response.status < 200 || response.status >= 300) { - const text = yield* response.text; - return yield* fail( - new ModelError({ - status: response.status, - message: appendModelErrorIdentifiers( - `OpenRouter HTTP ${response.status}: ${text}`, - identifiers - ), - ...identifiers, - ...(response.status === 429 && { - retryAfterMs: parseRetryAfter(retryAfterHeader), - }), - }) - ); - } - const rawBody = yield* response.text; - const json = yield* parseJsonBody(rawBody, identifiers); - const envelopeError = errorEnvelopeError( - json, - identifiers, - retryAfterHeader - ); - if (envelopeError) { - logUnusableBody(rawBody, envelopeError, identifiers); - return yield* fail(envelopeError); - } - const isCacheHit = - response.headers[RESPONSE_CACHE_STATUS_HEADER] === - RESPONSE_CACHE_STATUS_HIT; - const cacheSourceId = response.headers[RESPONSE_CACHE_SOURCE_ID_HEADER]; - logUnexpectedResponseCacheMiss({ - isCacheHit, - runAttempt, - retryAttempt, - cacheSalt, - model, - ...(response.headers[RESPONSE_CACHE_STATUS_HEADER] !== undefined && { - cacheStatus: response.headers[RESPONSE_CACHE_STATUS_HEADER], - }), - ...identifiers, - }); - return yield* decodeResult( - json, - startedAt, - identifiers, - isCacheHit, - cacheSourceId - ).pipe( - tapError((error) => - sync(() => { - logUnusableBody(rawBody, error, identifiers); - }) - ) - ); - }); - return retrySalted( - attempt.pipe(mapError(toModelError)), - rateLimitRetrySchedule(opts.retry ?? {}) - ); -} - -type ResponseIdentifiers = Pick; - -const RAW_BODY_LOG_LIMIT = 2000; - -const errorEnvelopeSchema = z.object({ - choices: z.array(z.unknown()).nullish(), - error: z.object({ - message: z.string().optional(), - code: z.union([z.number(), z.string()]).optional(), - }), -}); - -function parseJsonBody( - rawBody: string, - identifiers: ResponseIdentifiers -): Effect { - try { - const parsed: unknown = JSON.parse(rawBody); - return succeed(parsed); - } catch (cause) { - const error = new ModelError({ - message: appendModelErrorIdentifiers( - `OpenRouter 2xx body was not JSON: ${unknownErrorToString(cause)}`, - identifiers - ), - ...identifiers, - }); - logUnusableBody(rawBody, error, identifiers); - return fail(error); - } -} - -function errorEnvelopeError( - json: unknown, - identifiers: ResponseIdentifiers, - retryAfterHeader: string | null -): ModelError | undefined { - const parsed = parseSchema(errorEnvelopeSchema, json); - if (Either.isLeft(parsed)) { - return undefined; - } - const { choices } = parsed.right; - if (isDefinedAndNotNull(choices) && choices.length > 0) { - return undefined; - } - const { code, message } = parsed.right.error; - const status = toStatus(code); - const details = [ - code === undefined ? undefined : `code ${code}`, - message, - ].filter((detail): detail is string => detail !== undefined); - return new ModelError({ - ...(status !== undefined && { status }), - message: appendModelErrorIdentifiers( - details.length > 0 - ? `OpenRouter HTTP 200 error envelope (${details.join(": ")})` - : "OpenRouter HTTP 200 error envelope", - identifiers - ), - ...identifiers, - ...(status === 429 && { retryAfterMs: parseRetryAfter(retryAfterHeader) }), - }); -} - -function toStatus(code: number | string | undefined): number | undefined { - if (code === undefined) { - return undefined; - } - const parsed = typeof code === "number" ? code : Number(code); - const isHttpStatus = - Number.isInteger(parsed) && parsed >= 100 && parsed <= 599; - return isHttpStatus ? parsed : undefined; -} - -function logUnusableBody( - rawBody: string, - error: ModelError, - identifiers: ResponseIdentifiers -): void { - wLog("OpenRouter 2xx response did not yield a completion", { - error_message: error.message, - ...(error.status !== undefined && { error_status: error.status }), - raw_body: - rawBody.length > RAW_BODY_LOG_LIMIT - ? `${rawBody.slice(0, RAW_BODY_LOG_LIMIT - 3)}...` - : rawBody, - ...(identifiers.cfRay !== undefined && { cf_ray: identifiers.cfRay }), - ...(identifiers.xRequestId !== undefined && { - x_request_id: identifiers.xRequestId, - }), - }); -} - -function toApiContentItem(part: ContentPart) { - switch (part.type) { - case "text": { - return { type: "text", text: part.text }; - } - case "image_url": { - return { - type: "image_url", - image_url: { - url: part.imageUrl.url, - ...(part.imageUrl.detail !== undefined && { - detail: part.imageUrl.detail, - }), - }, - }; - } - case "video_url": { - return { - type: "video_url", - video_url: { - url: part.videoUrl.url, - }, - }; - } - default: { - return part satisfies never; - } - } -} - -function toApiMessage(message: ChatMessage) { - switch (message.role) { - case MessageRole.System: { - return { role: "system", content: message.content }; - } - case MessageRole.User: { - if (message.contentParts && message.contentParts.length > 0) { - return { - role: "user", - content: message.contentParts.map(toApiContentItem), - }; - } - return { role: "user", content: message.content }; - } - case MessageRole.Assistant: { - const base: { - role: "assistant"; - content: string; - tool_calls?: unknown[]; - reasoning_details?: ReasoningDetails; - model?: string; - } = { - role: "assistant", - content: message.content, - }; - if (message.model !== undefined) { - base.model = message.model; - } - if (hasReasoningDetails(message.reasoningDetails)) { - base.reasoning_details = message.reasoningDetails; - } - if (message.toolCalls && message.toolCalls.length > 0) { - return { ...base, tool_calls: [...message.toolCalls] }; - } - return base; - } - case MessageRole.Tool: { - return { - role: "tool", - content: message.content, - tool_call_id: message.toolCallId ?? "", - }; - } - default: { - return message.role satisfies never; - } - } -} - -function decodeResult( - raw: unknown, - startedAt: number, - identifiers: ResponseIdentifiers, - isCacheHit: boolean, - cacheSourceId?: string -): Effect { - const parseResult = parseSchema( - ChatResult$inboundSchema, - normalizeResultForSchema(raw) - ); - if (Either.isLeft(parseResult)) { - return fail( - new ModelError({ - message: appendModelErrorIdentifiers( - `OpenRouter response failed validation: ${parseResult.left.message}`, - identifiers - ), - ...identifiers, - }) - ); - } - const result = parseResult.right; - const responseIdentifiers: ModelErrorIdentifiers = { - ...identifiers, - ...(result.id !== undefined && { generationId: result.id }), - }; - const choice = result.choices[0]; - if (!choice) { - return fail( - new ModelError({ - message: appendModelErrorIdentifiers( - "OpenRouter response had no choices", - responseIdentifiers - ), - ...responseIdentifiers, - }) - ); - } - const rawContent = choice.message.content; - const completion = typeof rawContent === "string" ? rawContent : ""; - const reasoning = choice.message.reasoning ?? undefined; - const reasoningDetails = extractReasoningDetails(raw); - const usage = toModelUsage(result.usage); - const toolCalls = choice.message.toolCalls ?? []; - const hasSourceId = isCacheHit && cacheSourceId !== undefined; - return recordGenerationId( - hasSourceId ? cacheSourceId : result.id, - isCacheHit, - hasSourceId - ).pipe( - flatMap(() => - succeed({ - completion, - message: { - role: MessageRole.Assistant, - content: completion, - ...(toolCalls.length > 0 && { toolCalls }), - ...(reasoning !== undefined && { reasoning }), - ...(reasoningDetails !== undefined && { reasoningDetails }), - ...(result.model !== undefined && { model: result.model }), - }, - generationTimeMs: Math.round(performance.now() - startedAt), - ...(usage && { usage }), - ...(isRecord(raw) && { rawResponse: raw }), - }) - ) - ); -} - -function normalizeResultForSchema(raw: unknown): unknown { - if (!isRecord(raw)) { - return raw; - } - return { - ...raw, - ...(!("system_fingerprint" in raw) && { system_fingerprint: null }), - ...(!("created" in raw) && { created: 0 }), - ...(!("object" in raw) && { object: "chat.completion" }), - }; -} - -function extractReasoningDetails(raw: unknown): ReasoningDetails | undefined { - if (!isRecord(raw) || !Array.isArray(raw["choices"])) { - return undefined; - } - const choice = raw["choices"][0]; - if (!isRecord(choice) || !isRecord(choice["message"])) { - return undefined; - } - const details = choice["message"]["reasoning_details"]; - return hasReasoningDetails(details) ? details : undefined; -} - -function toModelUsage(usage: ChatUsage | undefined): ModelUsage | undefined { - if (!usage) { - return undefined; - } - const reasoningTokens = usage.completionTokensDetails?.reasoningTokens; - return { - inputTokens: usage.promptTokens, - outputTokens: usage.completionTokens, - totalTokens: usage.totalTokens, - ...(isDefinedAndNotNull(reasoningTokens) && { reasoningTokens }), - ...(isDefinedAndNotNull(usage.cost) && { totalCost: usage.cost }), - }; -} - -function toModelError(cause: unknown): ModelError { - if (cause instanceof ModelError) { - return cause; - } - return new ModelError({ - message: `OpenRouter request failed: ${String(cause)}`, - }); -} - -function parseRetryAfter(value: string | null): number | undefined { - if (value === null) { - return undefined; - } - const seconds = Number(value); - return Number.isFinite(seconds) && seconds >= 0 ? seconds * 1e3 : undefined; + ).pipe(provide(responsesLayer)); } diff --git a/src/providers/responses-client.ts b/src/providers/responses-client.ts index e843367..4d71787 100644 --- a/src/providers/responses-client.ts +++ b/src/providers/responses-client.ts @@ -47,7 +47,7 @@ import { import { BENCH_HARNESS_APP_REFERRER, BENCH_HARNESS_APP_TITLE, -} from "./openrouter-model"; +} from "./app-identity"; import type { ModelErrorIdentifiers } from "./request-identifiers"; import { appendModelErrorIdentifiers, diff --git a/src/providers/responses-model.ts b/src/providers/responses-model.ts index 6e0a2de..9dbb174 100644 --- a/src/providers/responses-model.ts +++ b/src/providers/responses-model.ts @@ -60,9 +60,11 @@ export interface ResponsesFunctionTool { readonly name: string; readonly description?: string; readonly parameters: Record; + readonly strict?: boolean | null; } export interface ResponsesGenerateConfig extends Omit { + readonly model?: string; readonly instructions?: string; readonly tools?: readonly ResponsesFunctionTool[]; } @@ -158,10 +160,11 @@ export function generate( }), }; const sendProvider = Object.keys(providerPreferences).length > 0; - const baseModel = stripVariantSuffix(opts.model); + const requestModel = genConfig.model ?? opts.model; + const baseModel = stripVariantSuffix(requestModel); const autoRouterPlugin = buildAutoRouterPlugin(baseModel, genConfig); const body = { - model: opts.model, + model: requestModel, input: toSdkInput(opts.input), store: false, include: ["reasoning.encrypted_content"], @@ -244,7 +247,7 @@ export function generate( const RAW_PAYLOAD_KEYS = new Set(["arguments", "output"]); -function toSdkInput(input: readonly ResponsesInputItem[]): InputsUnion { +export function toSdkInput(input: readonly ResponsesInputItem[]): InputsUnion { return input.map(toSdkValue) as InputsUnion; } diff --git a/src/providers/responses-wire.test.ts b/src/providers/responses-wire.test.ts new file mode 100644 index 0000000..f99eea3 --- /dev/null +++ b/src/providers/responses-wire.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "bun:test"; + +import { ResponsesRequest$outboundSchema } from "@openrouter/sdk/models/responsesrequest"; + +import { MessageRole } from "../harness/core"; +import { + messagesToResponses, + toolDefinitionToResponses, +} from "./messages-to-responses"; +import { toSdkInput } from "./responses-model"; + +function expectValidResponsesRequest( + messages: Parameters[0], + tools?: Parameters[0][] +) { + const result = ResponsesRequest$outboundSchema.safeParse({ + model: "openai/gpt-4o-mini", + input: toSdkInput(messagesToResponses(messages)), + store: false, + stream: true, + serviceTier: null, + ...(tools !== undefined && { + tools: tools.map(toolDefinitionToResponses), + }), + }); + expect(result.success).toBe(true); +} + +describe("Responses wire contract", () => { + it("validates text-only system and user messages", () => { + expectValidResponsesRequest([ + { role: MessageRole.System, content: "Follow the rules." }, + { role: MessageRole.User, content: "Answer the question." }, + ]); + }); + + it("validates vision messages with default and explicit detail", () => { + expectValidResponsesRequest([ + { + role: MessageRole.User, + content: "", + contentParts: [ + { type: "image_url", imageUrl: { url: "without-detail.png" } }, + { + type: "image_url", + imageUrl: { url: "high-detail.png", detail: "high" }, + }, + ], + }, + ]); + }); + + it("validates video content", () => { + expectValidResponsesRequest([ + { + role: MessageRole.User, + content: "", + contentParts: [{ type: "video_url", videoUrl: { url: "video.mp4" } }], + }, + ]); + }); + + it("validates an assistant function call and its output", () => { + expectValidResponsesRequest( + [ + { role: MessageRole.System, content: "You are helpful." }, + { role: MessageRole.User, content: "Look up my balance." }, + { + role: MessageRole.Assistant, + content: "", + toolCalls: [ + { + id: "call-1", + type: "function", + function: { + name: "get_balance", + arguments: '{"account_id":"acct-1"}', + }, + }, + ], + }, + { + role: MessageRole.Tool, + content: '{"balance":100}', + toolCallId: "call-1", + }, + ], + [ + { + type: "function", + function: { + name: "get_balance", + description: "Get the current balance.", + parameters: { + type: "object", + properties: { account_id: { type: "string" } }, + required: ["account_id"], + additionalProperties: false, + }, + strict: true, + }, + }, + ] + ); + }); +}); diff --git a/src/results/parquet.test.ts b/src/results/parquet.test.ts index 8c484a5..d7fab87 100644 --- a/src/results/parquet.test.ts +++ b/src/results/parquet.test.ts @@ -4,7 +4,7 @@ import assert from "node:assert"; import type { AsyncBuffer } from "hyparquet"; import { parquetMetadata } from "hyparquet"; -import type { ChatMessage, ResponseItem } from "../harness/core"; +import type { ModelMessage, ResponseItem } from "../harness/core"; import { MessageRole, ScoreValue } from "../harness/core"; import type { SampleScore } from "../harness/metric"; import { assertRight, assertLeft } from "../internal/testing"; @@ -344,7 +344,7 @@ describe("runResultToParquet", () => { expect(rowScoreToNumber("I")).toBe(0); }); it("serializes message trajectories as a JSON column", async () => { - const messages: readonly ChatMessage[] = [ + const messages: readonly ModelMessage[] = [ { role: MessageRole.System, content: "You are a helpful assistant." }, { role: MessageRole.User, content: "What is 2+2?" }, { role: MessageRole.Assistant, content: "Answer: B" }, @@ -378,7 +378,7 @@ describe("runResultToParquet", () => { ]); }); it("serializes tool calls and tool_call_id in the messages JSON", async () => { - const messages: readonly ChatMessage[] = [ + const messages: readonly ModelMessage[] = [ { role: MessageRole.Assistant, content: "", @@ -424,7 +424,7 @@ describe("runResultToParquet", () => { expect(parsed[1]).not.toHaveProperty("tool_calls"); }); it("serializes assistant reasoning traces in the messages JSON", async () => { - const messages: readonly ChatMessage[] = [ + const messages: readonly ModelMessage[] = [ { role: MessageRole.Assistant, content: "Answer: B", @@ -455,7 +455,7 @@ describe("runResultToParquet", () => { expect(parsed[0]?.["reasoning"]).toBe("Step 1: ..."); }); it("serializes multimodal content parts (image_url) in the messages JSON", async () => { - const messages: readonly ChatMessage[] = [ + const messages: readonly ModelMessage[] = [ { role: MessageRole.User, content: "What is this image?", @@ -498,7 +498,7 @@ describe("runResultToParquet", () => { ]); }); it("serializes assistant citations (camelCase→snake_case) in the messages JSON", async () => { - const messages: readonly ChatMessage[] = [ + const messages: readonly ModelMessage[] = [ { role: MessageRole.Assistant, content: "Answer based on sources.", diff --git a/src/results/parquet.ts b/src/results/parquet.ts index aa8a06f..9556146 100644 --- a/src/results/parquet.ts +++ b/src/results/parquet.ts @@ -6,7 +6,7 @@ import { parquetWriteBuffer } from "hyparquet-writer"; import type { BenchmarkRunConfig } from "../benchmarks/benchmark-config"; import type { BenchmarkPrimaryScore } from "../benchmarks/types"; import type { - ChatMessage, + ModelMessage, ContentPart, ToolCall, UsageTotals, @@ -255,7 +255,7 @@ function cellValue(name: ColumnName, ctx: RowContext, s: SampleScore): unknown { } case "messages": { return s.messages !== undefined && s.messages.length > 0 - ? JSON.stringify(s.messages.map(chatMessageToPojo)) + ? JSON.stringify(s.messages.map(messageToPojo)) : null; } case "metadata": { @@ -280,7 +280,7 @@ interface RowContext { readonly benchmarkConfigJson: string | null; } -function chatMessageToPojo(msg: ChatMessage): Record { +function messageToPojo(msg: ModelMessage): Record { const pojo: Record = { role: msg.role, content: msg.content,