refactor(providers): drop the chat-completions client

src/benchmarks/agent-cli/harness.ts

@@ -1,4 +1,8 @@
-import type { ChatMessage, ModelUsage, ResponseItem } from "../../harness/core";
+import type {
+  ModelMessage,
+  ModelUsage,
+  ResponseItem,
+} from "../../harness/core";
 import { MessageRole } from "../../harness/core";
 import { Either } from "../../internal/either";
 import { isRecord } from "../../internal/guards";
@@ -40,7 +44,7 @@ export interface OriAgentRun {
   readonly generationIds: readonly string[];
   readonly generationTimeMs: number | undefined;
   readonly finalText: string | undefined;
-  readonly assistantMessages: readonly ChatMessage[];
+  readonly assistantMessages: readonly ModelMessage[];
   readonly responseItems: readonly ResponseItem[];
   readonly isError: boolean;
   readonly apiErrorStatus: string | undefined;
@@ -175,7 +179,7 @@ function usageFromResult(result: Record<string, unknown>): ModelUsage {
 
 function parseClaudeStream(stdout: string): OriAgentRun {
   const generationIds: string[] = [];
-  const assistantMessages: ChatMessage[] = [];
+  const assistantMessages: ModelMessage[] = [];
   const responseItems: ResponseItem[] = [];
   let usage: ModelUsage | undefined;
   let generationTimeMs: number | undefined;
@@ -363,7 +367,7 @@ function parsePiStream(stdout: string): OriAgentRun {
   let apiErrorStatus: string | undefined;
   let finalText: string | undefined;
   const generationIds: string[] = [];
-  const assistantMessages: ChatMessage[] = [];
+  const assistantMessages: ModelMessage[] = [];
   const responseItems: ResponseItem[] = [];
   for (const line of stdout.split("\n")) {
     const trimmed = line.trim();

src/benchmarks/define-chat-benchmark.ts → src/benchmarks/define-single-turn-benchmark.ts

@@ -21,7 +21,7 @@ import type { RetryConfig } from "../runtime/retry";
 import type { BenchmarkRunConfig } from "./benchmark-config";
 import type { Benchmark, BenchmarkRunInput } from "./types";
 
-export interface ChatBenchmarkDefinition<
+export interface SingleTurnBenchmarkDefinition<
   C extends BenchmarkRunConfig & {
     readonly model: string;
   },
@@ -39,11 +39,11 @@ export interface ChatBenchmarkDefinition<
   readonly makeSolver: (model: ModelService, config: C) => SolverService;
 }
 
-export function defineChatBenchmark<
+export function defineSingleTurnBenchmark<
   C extends BenchmarkRunConfig & {
     readonly model: string;
   },
->(definition: ChatBenchmarkDefinition<C>): Benchmark {
+>(definition: SingleTurnBenchmarkDefinition<C>): Benchmark {
   function makeLayer(
     input: BenchmarkRunInput
   ): Layer<Dataset | Solver | Scorer, Error, HttpClient.HttpClient> {

src/benchmarks/draco/solver.ts

@@ -16,7 +16,7 @@ import {
 } from "effect/Effect";
 
 import type {
-  ChatMessage,
+  ModelMessage,
   ModelError,
   ModelOutput,
   ModelUsage,
@@ -476,7 +476,7 @@ function completedState(
   generation: GenerationResult,
   verdicts: JudgeRun[]
 ): TaskState {
-  const messages: ChatMessage[] = [
+  const messages: ModelMessage[] = [
     { role: MessageRole.User, content: state.sample.input },
     ...(generation.content
       ? [

src/benchmarks/gpqa-solver.test.ts

@@ -10,7 +10,7 @@ import {
   noopProgressLayer,
   noopCheckpointLayer,
 } from "../../test/helpers/noop-progress-layer";
-import type { ChatMessage, ModelError, ModelOutput } from "../harness/core";
+import type { ModelMessage, ModelError, ModelOutput } from "../harness/core";
 import { initialTaskState, MessageRole } from "../harness/core";
 import type { GenerateConfig, ModelService } from "../harness/model";
 import { Model } from "../harness/model";
@@ -23,7 +23,7 @@ function recordingModel(record: { config: GenerateConfig | undefined }): {
 } {
   const service: ModelService = {
     generate: (
-      _messages: readonly ChatMessage[],
+      _messages: readonly ModelMessage[],
       config: GenerateConfig
     ): Effect<ModelOutput, ModelError> => {
       record.config = config;

src/benchmarks/gpqa.ts

@@ -14,7 +14,7 @@ import type {
   GpqaBenchmarkConfig,
 } from "./benchmark-config";
 import { GPQA_META } from "./benchmark-meta";
-import { defineChatBenchmark } from "./define-chat-benchmark";
+import { defineSingleTurnBenchmark } from "./define-single-turn-benchmark";
 import { mcqScorer } from "./scorers/mcq/scorer";
 import { seededPermutation } from "./scorers/mcq/shuffle";
 import type { Benchmark } from "./types";
@@ -119,7 +119,7 @@ export function makeGpqaDatasetLayer(
   });
 }
 
-export const GPQA_BENCHMARK: Benchmark = defineChatBenchmark({
+export const GPQA_BENCHMARK: Benchmark = defineSingleTurnBenchmark({
   id: "gpqa_diamond",
   temperature: GPQA_TEMPERATURE,
   defaultEpochs: GPQA_META.defaultEpochs,

src/benchmarks/harbor/agent-loop.test.ts

@@ -8,7 +8,7 @@ import { SolverError } from "../../harness/core";
 import { assertRight } from "../../internal/testing";
 import { parseSchema, z } from "../../internal/zod";
 import type { ResponsesModelService } from "../../providers/responses-model";
-import { itemsToChatMessages, runAgentLoop } from "./agent-loop";
+import { itemsToModelMessages, runAgentLoop } from "./agent-loop";
 import { makeHarborStreamTracker } from "./agent-progress";
 import { SUBMIT_SENTINEL } from "./prompts";
 import type { ExecResult, SandboxSessionInstance } from "./sandbox";
@@ -314,7 +314,7 @@ describe("Harbor stream progress", () => {
 describe("Responses item round-tripping", () => {
   it("converts advisor advice from the real terminal fixture into an assistant message", async () => {
     const terminal = await readTerminalFixture();
-    expect(itemsToChatMessages(terminal.output)).toContainEqual({
+    expect(itemsToModelMessages(terminal.output)).toContainEqual({
       role: "assistant",
       content:
         "Confirmed: 2 + 2 = 4 in standard base-10 arithmetic; edge cases include alternate numeric bases or string concatenation in programming contexts.",

src/benchmarks/harbor/agent-loop.ts

@@ -9,7 +9,7 @@ import {
 } from "effect/Effect";
 
 import type {
-  ChatMessage,
+  ModelMessage,
   ModelError,
   ModelUsage,
   SolverError,
@@ -68,7 +68,7 @@ export interface AgentLoopInput {
 
 export interface AgentLoopResult {
   readonly input: ResponsesInputItem[];
-  readonly messages: ChatMessage[];
+  readonly messages: ModelMessage[];
   readonly usage: ModelUsage;
   readonly generationTimeMs: number;
   readonly finalText: string;
@@ -173,7 +173,7 @@ export function runAgentLoop(
     }
     return {
       input: conversation,
-      messages: itemsToChatMessages(conversation),
+      messages: itemsToModelMessages(conversation),
       usage: toModelUsage(acc),
       generationTimeMs,
       finalText,
@@ -329,10 +329,10 @@ function truncateCommand(command: string): string {
     : command;
 }
 
-export function itemsToChatMessages(
+export function itemsToModelMessages(
   items: readonly ResponsesInputItem[]
-): ChatMessage[] {
-  const messages: ChatMessage[] = [];
+): ModelMessage[] {
+  const messages: ModelMessage[] = [];
   for (const item of items) {
     const type = item["type"];
     if (type === "message" || type === undefined) {

src/benchmarks/ifstruct/benchmark.ts

@@ -16,7 +16,7 @@ import type {
   InferenceOverride,
 } from "../benchmark-config";
 import { IFSTRUCT_META } from "../benchmark-meta";
-import { defineChatBenchmark } from "../define-chat-benchmark";
+import { defineSingleTurnBenchmark } from "../define-single-turn-benchmark";
 import type { Benchmark } from "../types";
 import type {
   IfStructRequirements,
@@ -143,7 +143,7 @@ export function makeIfStructDatasetLayer(
   });
 }
 
-export const IFSTRUCT_BENCHMARK: Benchmark = defineChatBenchmark({
+export const IFSTRUCT_BENCHMARK: Benchmark = defineSingleTurnBenchmark({
   id: "ifstruct",
   temperature: IFSTRUCT_TEMPERATURE,
   defaultEpochs: IFSTRUCT_META.defaultEpochs,

src/benchmarks/mmlu-pro-solver.test.ts

@@ -8,19 +8,19 @@ import {
   noopCheckpointLayer,
   noopProgressLayer,
 } from "../../test/helpers/noop-progress-layer";
-import type { ChatMessage, ModelError, ModelOutput } from "../harness/core";
+import type { ModelMessage, ModelError, ModelOutput } from "../harness/core";
 import { initialTaskState, MessageRole } from "../harness/core";
 import type { GenerateConfig, ModelService } from "../harness/model";
 import { MMLU_PRO_TEMPERATURE, mmluProSolver } from "./mmlu-pro";
 describe("mmluProSolver", () => {
   it("uses canonical sampling defaults and sends one user message", async () => {
     const recorded: {
       config?: GenerateConfig;
-      messages?: readonly ChatMessage[];
+      messages?: readonly ModelMessage[];
     } = {};
     const model: ModelService = {
       generate: (
-        messages: readonly ChatMessage[],
+        messages: readonly ModelMessage[],
         config: GenerateConfig
       ): Effect<ModelOutput, ModelError> => {
         recorded.messages = messages;

src/benchmarks/mmlu-pro.ts

@@ -15,7 +15,7 @@ import type {
   MmluProBenchmarkConfig,
 } from "./benchmark-config";
 import { MMLU_PRO_META } from "./benchmark-meta";
-import { defineChatBenchmark } from "./define-chat-benchmark";
+import { defineSingleTurnBenchmark } from "./define-single-turn-benchmark";
 import { makeMmluProFewShotDatasetLayer } from "./mmlu-pro-dataset";
 import type { MmluProCotExamplesByCategory } from "./mmlu-pro-prompt";
 import { buildMmluProPrompt } from "./mmlu-pro-prompt";
@@ -148,7 +148,7 @@ function mmluProRunLevelScores(result: RunResult): readonly {
   ];
 }
 
-const MMLU_PRO_CHAT_BENCHMARK = defineChatBenchmark({
+const MMLU_PRO_SINGLE_TURN_BENCHMARK = defineSingleTurnBenchmark({
   id: "mmlu_pro",
   temperature: MMLU_PRO_TEMPERATURE,
   defaultEpochs: MMLU_PRO_META.defaultEpochs,
@@ -176,6 +176,6 @@ const MMLU_PRO_CHAT_BENCHMARK = defineChatBenchmark({
 });
 
 export const MMLU_PRO_BENCHMARK: Benchmark = {
-  ...MMLU_PRO_CHAT_BENCHMARK,
+  ...MMLU_PRO_SINGLE_TURN_BENCHMARK,
   runLevelScores: mmluProRunLevelScores,
 };

src/benchmarks/mmmu-pro-vision.ts

@@ -17,7 +17,7 @@ import type {
   MmmuProVisionBenchmarkConfig,
 } from "./benchmark-config";
 import { MMMU_PRO_VISION_META } from "./benchmark-meta";
-import { defineChatBenchmark } from "./define-chat-benchmark";
+import { defineSingleTurnBenchmark } from "./define-single-turn-benchmark";
 import { MMMU_SYSTEM_MESSAGE, parseOptions } from "./mmmu-shared";
 import { buildDynamicMcqPrompt } from "./scorers/mcq/dynamic-prompt";
 import { mcqScorer } from "./scorers/mcq/scorer";
@@ -139,7 +139,7 @@ export function mmmuProVisionSolver(
   return chain(systemMessage(MMMU_SYSTEM_MESSAGE), generate(model, config));
 }
 
-export const MMMU_PRO_VISION_BENCHMARK: Benchmark = defineChatBenchmark({
+export const MMMU_PRO_VISION_BENCHMARK: Benchmark = defineSingleTurnBenchmark({
   id: "mmmu_pro_vision",
   temperature: 0,
   defaultEpochs: MMMU_PRO_VISION_META.defaultEpochs,

src/benchmarks/tau-bench-airline/benchmark.ts

@@ -19,6 +19,10 @@ import { Solver } from "../../harness/solver";
 import { Either } from "../../internal/either";
 import { parseSchema } from "../../internal/zod";
 import { makeOpenRouterModelLayer } from "../../providers/openrouter-model";
+import {
+  makeResponsesModelLayer,
+  ResponsesModel,
+} from "../../providers/responses-model";
 import type { RetryConfig } from "../../runtime/retry";
 import { TAU_BENCH_AIRLINE_META } from "../benchmark-meta";
 import type { Benchmark, BenchmarkRunInput } from "../types";
@@ -123,20 +127,33 @@ function makeAirlineLayer(
       sessionId: input.sessionId,
       ...(input.modelRetry !== undefined && { retry: input.modelRetry }),
     });
+  const userModelLayer = makeResponsesModelLayer({
+    model: benchmarkConfig.userModel,
+    apiKey: input.apiKey,
+    ...(input.baseUrl !== undefined && { baseUrl: input.baseUrl }),
+    sessionId: input.sessionId,
+  });
   const solverLayer = layerEffect(Solver)(
     gen(function* () {
       const model = yield* Model;
+      const userModel = yield* ResponsesModel;
       const client = yield* HttpClient.HttpClient;
       const dataFetchLock = yield* makeSemaphore(1);
       return Solver.of(
-        airlineSolver({ model, client, dataFetchLock, opts: solverOpts })
+        airlineSolver({
+          model,
+          userModel,
+          client,
+          dataFetchLock,
+          opts: solverOpts,
+        })
       );
     })
   );
   const scorerLayer = layerSucceed(Scorer, Scorer.of(airlineScorer));
   return layerMergeAll(
     datasetLayer,
-    solverLayer.pipe(layerProvide(modelLayer)),
+    solverLayer.pipe(layerProvide(layerMergeAll(modelLayer, userModelLayer))),
     scorerLayer
   );
 }

src/benchmarks/tau-bench-airline/scorer.ts

@@ -1,7 +1,12 @@
 import type { Effect } from "effect/Effect";
 import { succeed } from "effect/Effect";
 
-import type { ChatMessage, Score, Target, TaskState } from "../../harness/core";
+import type {
+  ModelMessage,
+  Score,
+  Target,
+  TaskState,
+} from "../../harness/core";
 import { MessageRole, ScoreValue } from "../../harness/core";
 import type { ScorerService } from "../../harness/scorer";
 import { Either } from "../../internal/either";
@@ -25,7 +30,7 @@ function isTau2Task(val: unknown): val is Tau2Task {
 }
 
 function collectToolCalls(
-  messages: readonly ChatMessage[]
+  messages: readonly ModelMessage[]
 ): PredictedToolCall[] {
   const calls: PredictedToolCall[] = [];
   for (const m of messages) {

src/benchmarks/tau-bench-airline/solver.ts

@@ -2,12 +2,13 @@ import { HttpClient } from "@effect/platform";
 import type { Semaphore } from "effect/Effect";
 import { gen, mapError, provideService } from "effect/Effect";
 
-import type { ChatMessage, ModelUsage, ToolCall } from "../../harness/core";
+import type { ModelMessage, ModelUsage, ToolCall } from "../../harness/core";
 import { MessageRole, SolverError } from "../../harness/core";
 import type { GenerateConfig, ModelService } from "../../harness/model";
 import type { SolverService } from "../../harness/solver";
 import { Either } from "../../internal/either";
 import { definedValues, isRecord } from "../../internal/guards";
+import type { ResponsesModelService } from "../../providers/responses-model";
 import {
   buildAgentSystemPrompt,
   DEFAULT_FIRST_AGENT_MESSAGE,
@@ -40,11 +41,13 @@ type Role = (typeof Role)[keyof typeof Role];
 
 export function airlineSolver({
   model,
+  userModel,
   client,
   dataFetchLock,
   opts,
 }: {
   readonly model: ModelService;
+  readonly userModel: ResponsesModelService;
   readonly client: HttpClient.HttpClient;
   readonly dataFetchLock: Semaphore;
   readonly opts?: SolverOpts;
@@ -69,9 +72,9 @@ export function airlineSolver({
       );
       const task = state.sample.metadata?.["task"];
       const data: AirlineData = loadAirlineData();
-      const userSim = new UserSimulator(userModelConfig);
+      const userSim = new UserSimulator(userModel, userModelConfig);
       userSim.reset(state.sample.input, DEFAULT_FIRST_AGENT_MESSAGE);
-      const messages: ChatMessage[] = [
+      const messages: ModelMessage[] = [
         {
           role: MessageRole.System,
           content: buildAgentSystemPrompt(AIRLINE_POLICY),
@@ -205,7 +208,7 @@ export function airlineSolver({
     });
 }
 
-function lastAssistantText(messages: readonly ChatMessage[]): string {
+function lastAssistantText(messages: readonly ModelMessage[]): string {
   for (let i = messages.length - 1; i >= 0; i--) {
     const m = messages[i];
     if (m?.role === MessageRole.Assistant) {