diff --git a/evals/README.md b/evals/README.md index b14b5b2..ff118b8 100644 --- a/evals/README.md +++ b/evals/README.md @@ -34,6 +34,10 @@ Passing requires every functional browser/API, TypeScript, data-preservation/res See [the full protocol](PROTOCOL.md), [task definitions](yoloeval/catalog.py), [grading code](browser/checks.mjs), and [acceptance/early-stop rules](yoloeval/unseen.py). Jev routes the candidate; deterministic checks grade the results. There is no LLM judge in this protocol. +## Recorded evidence + +The [recorded result](results/unseen-002/README.md) includes all 120 outcomes, 393 model-call records, source diffs, provenance, and an offline audit. Run `python3 evals/audit_results.py` to recalculate its acceptance and timings without API calls. + ## Interpretation The recorded unseen result was **45/60 → 53/60**, mean **70.58s → 47.37s**, P50 **54.30s → 30.92s**. This is a complete-bundle result, not an isolated Jev ablation. Campaign 001 was invalidated for a native-search-input grader bug; campaign 002 restarted all 120 attempts after correcting it. A provider outage was held between attempts; all scored service errors remained in the final data. diff --git a/evals/audit_results.py b/evals/audit_results.py new file mode 100644 index 0000000..db6d4d3 --- /dev/null +++ b/evals/audit_results.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Audit the published 120-attempt historical result without API calls.""" +import hashlib +import json +import math +import statistics +from pathlib import Path + +from yoloeval.catalog import tasks +from yoloeval.scoring import quantile +from yoloeval.unseen import decision + +ROOT = Path(__file__).resolve().parent/'results/unseen-002' + + +def read_rows(name): + rows = [json.loads(line) for line in (ROOT/name).read_text().splitlines()] + keys = [(row['role'], row['task_id'], row['trial']) for row in rows] + expected = {(role, task.id, trial) for role in ('baseline', 'candidate') + for task in tasks() for trial in (1, 2, 3)} + if len(keys) != len(expected) or set(keys) != expected: + raise ValueError('Missing, duplicate or unexpected attempts in '+name) + return {key: row for key, row in zip(keys, rows)} + + +def main(): + for line in (ROOT/'SHA256SUMS').read_text().splitlines(): + expected, name = line.split(' ', 1) + if hashlib.sha256((ROOT/name).read_bytes()).hexdigest() != expected: + raise ValueError('Published evidence changed: '+name) + results, calls, changes = (read_rows(name) for name in ('results.jsonl', 'calls.jsonl', 'changes.jsonl')) + rows = {role: [] for role in ('baseline', 'candidate')} + for key, envelope in results.items(): + role, task, trial = key + row = envelope['result'] + if (row['task_id'], row['trial']) != (task, trial) or row['status'] != 'completed': + raise ValueError('Incomplete or misidentified result: '+str(key)) + if row['passed'] != (not row['agent_error'] and all(c['passed'] for c in row['checks'])): + raise ValueError('Pass flag differs from checks/error: '+str(key)) + trace = calls[key]['calls'] + if row['usage']['model_calls'] != len(trace) or not all(c['complete'] for c in trace): + raise ValueError('Missing or undrained calls: '+str(key)) + allowed = {'muse-spark-1-3', 'jev-1.13.0'} if role == 'candidate' else {'muse-spark-1-3'} + for call in trace: + if call['requested_model'] not in allowed: + raise ValueError('Unexpected model: '+str(key)) + if call['status'] == 200 and call.get('returned_model') != call['requested_model']: + raise ValueError('Successful call returned a different model: '+str(key)) + rows[role].append(row) + computed = decision(rows) + recorded = json.loads((ROOT/'decision.json').read_text()) + for field in ('status', 'accepted', 'complete', 'counts', 'critical_regressions'): + if computed[field] != recorded[field]: + raise ValueError('Acceptance result differs: '+field) + for field in ('candidate_mean_s', 'candidate_p50_s'): + if not math.isclose(computed[field], recorded[field], abs_tol=1e-9): + raise ValueError('Timing differs: '+field) + summary = {} + for role, samples in rows.items(): + durations = [r['timing']['agent_wall_s'] for r in samples] + summary[role] = {'attempts': len(samples), 'passed': sum(r['passed'] for r in samples), + 'mean_s': statistics.mean(durations), 'p50_s': quantile(durations, .5), + 'p95_s': quantile(durations, .95)} + print(json.dumps({'audit_passed': True, 'acceptance': computed['accepted'], 'summary': summary, + 'scored_calls': sum(len(c['calls']) for c in calls.values()), + 'preserved_patches': len(changes)}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/evals/results/unseen-002/README.md b/evals/results/unseen-002/README.md new file mode 100644 index 0000000..2b78aec --- /dev/null +++ b/evals/results/unseen-002/README.md @@ -0,0 +1,32 @@ +# Recorded unseen validation: 26 September 2026 + +| Metric | Original vanilla | Frozen winner | +|---|---:|---:| +| Passed | 45/60 | 53/60 | +| Mean agent time | 70.58s | 47.37s | +| Nearest-rank P50 | 54.30s | 30.92s | +| P95 | 180.06s | 130.77s | + +The winner met the preregistered gate: at least vanilla's passes, no paired storage-check regressions, and lower mean and P50. Mean time fell **32.9%** and P50 **43.1%**, including failures. This comparison used the original vanilla revision `6a00e38` and the frozen Round 032 winner (`b1d5a3f`, published with identical product source at `review/frozen-r032`). Both used Muse `muse-spark-1-3`; the winner also used Jev `jev-1.13.0`. + +## Audit without running models + +From the repository root: + +```sh +python3 evals/audit_results.py +``` + +The audit checks evidence hashes, all 120 task/repetition identities, every pass flag against its checks/error, all 393 drained model calls and model identities, and recalculates the acceptance decision and timings. `results.jsonl`, `calls.jsonl`, and `changes.jsonl` each retain one entry per attempt, including failures. `preregistration.json` contains the original schedule and hashes. `provenance.json` documents source hashes and replacement of local filesystem prefixes. + +The full 109 MB working archive includes screenshots, traces, app trees and offline controls. It is retained locally and **not** uploaded here. The compact published data supports numeric and check-outcome auditing; a fresh [replication](../../README.md) regenerates browser and application evidence. It is not a byte-for-byte substitute for that archive. + +## Failures and limitations + +Vanilla had five attempts ending in upstream service errors, four timeouts and six requirement failures. The winner had four upstream service errors, one timeout and two requirement failures. Its three non-service failures were quiz cases: creation used the wrong option labels, a feature repair exhausted the edit budget, and one bug fix retained the old grading loop alongside its replacement. + +A service outage caused seven consecutive upstream failures. The campaign paused for approximately 561 seconds between completed attempts, then resumed after recovery. Ten tiny operational probes are logged separately and excluded; every scored service error remained in the 120 results. Removing both sides of any pair with a non-200 call leaves 54 pairs: vanilla 44/54, winner 51/54, with approximately 35.2% lower winner mean time. That is a post-hoc sensitivity check, not the acceptance rule. + +Campaign 001 was invalidated after a native search-input grader false negative. Campaign 002 reran all 120 attempts with the corrected grader and unchanged actors. None of campaign 001's rows is substituted here. The previously tuned, different suite reached 60/60; that exposed result is not this unseen validation. + +Five app families and correlated repetitions do not establish universal reliability. These tasks are now published and exposed. The later integration with current main has offline test coverage but has **not** been assigned these frozen binary's performance numbers. diff --git a/evals/results/unseen-002/SHA256SUMS b/evals/results/unseen-002/SHA256SUMS new file mode 100644 index 0000000..1b65688 --- /dev/null +++ b/evals/results/unseen-002/SHA256SUMS @@ -0,0 +1,9 @@ +339370d90d44f810b93555c2a88f297933f38e17a307a2eabbf88f5a4707f99d README.md +5e13560e4c23037dc9294720953a1d16bd956fa8ac5917ea6be132ef5ac21492 calls.jsonl +81ab76dc51abc26f258c71b30d53fd64db9dfdedc704b278cc822b66b0e76b7e changes.jsonl +9231973b7574c7f2da3b18950e8d176069580b0717a4f9f8a582fa5c60f4058c decision.json +b2efac2d65b7eb95c9b501ab96c2a3440a48a3be10c0ff8e73c7ff6fb1caeaf8 preregistration.json +d891753fe6d9616eb1e1c8f9bde81066f0d4110dc50cd629ff71c6a66b166461 provenance.json +3949da99f46048850d95ccc54ea7e957c6548ce7c60ff7b1598ff27a846f306a provider-health-probes.jsonl +2d006eb27e50e6126740a0337559d7d966bc98e804ddfd2e8e427ada9ffe6334 provider-outage-hold.json +73d8f9fa5cac0b06f62e2cf7a698a4b11985fb77ff7f8617097fa4e616a8d3a9 results.jsonl diff --git a/evals/results/unseen-002/calls.jsonl b/evals/results/unseen-002/calls.jsonl new file mode 100644 index 0000000..acc7fa0 --- /dev/null +++ b/evals/results/unseen-002/calls.jsonl @@ -0,0 +1,120 @@ +{"role":"baseline","task_id":"application.bug","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.2452176669612527,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8932,"response_bytes":6670,"usage":{"input":13,"output":118,"cached":2289,"total":131,"consistent":false},"usage_raw":{"input_tokens":13,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":118,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":131}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.923824750003405,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17975,"response_bytes":6799,"usage":{"input":2168,"output":276,"cached":2289,"total":2444,"consistent":false},"usage_raw":{"input_tokens":2168,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":276,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2444}}]} +{"role":"baseline","task_id":"application.bug","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.998847375041805,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8932,"response_bytes":6694,"usage":{"input":13,"output":122,"cached":2289,"total":135,"consistent":false},"usage_raw":{"input_tokens":13,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":122,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":135}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":10.61312958295457,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17987,"response_bytes":6882,"usage":{"input":2169,"output":980,"cached":2289,"total":3149,"consistent":false},"usage_raw":{"input_tokens":2169,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":980,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3149}}]} +{"role":"baseline","task_id":"application.bug","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":1.8143655839376152,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8932,"response_bytes":6664,"usage":{"input":2302,"output":111,"cached":0,"total":2413,"consistent":true},"usage_raw":{"input_tokens":2302,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":111,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2413}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":8.952426709001884,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17774,"response_bytes":6732,"usage":{"input":4414,"output":836,"cached":0,"total":5250,"consistent":true},"usage_raw":{"input_tokens":4414,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":836,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5250}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":10.796613333048299,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21596,"response_bytes":6898,"usage":{"input":1140,"output":1089,"cached":4337,"total":2229,"consistent":false},"usage_raw":{"input_tokens":1140,"input_tokens_details":{"cached_tokens":4337,"cache_write_tokens":0},"output_tokens":1089,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2229}}]} +{"role":"baseline","task_id":"application.create","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.6400115829892457,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8785,"response_bytes":6807,"usage":{"input":114,"output":151,"cached":2161,"total":265,"consistent":false},"usage_raw":{"input_tokens":114,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":151,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":265}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":53.76175737497397,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15860,"response_bytes":8106,"usage":{"input":1963,"output":5795,"cached":2161,"total":7758,"consistent":false},"usage_raw":{"input_tokens":1963,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":5795,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7758}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":50.68825841706712,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17914,"response_bytes":10824,"usage":{"input":2457,"output":6727,"cached":2161,"total":9184,"consistent":true},"usage_raw":{"input_tokens":2457,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":6727,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":9184}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":43.51088258298114,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":22751,"response_bytes":14713,"usage":{"input":1808,"output":6193,"cached":4081,"total":8001,"consistent":false},"usage_raw":{"input_tokens":1808,"input_tokens_details":{"cached_tokens":4081,"cache_write_tokens":0},"output_tokens":6193,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":8001}}]} +{"role":"baseline","task_id":"application.create","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.0048170420341194,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8785,"response_bytes":6847,"usage":{"input":114,"output":147,"cached":2161,"total":261,"consistent":false},"usage_raw":{"input_tokens":114,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":147,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":261}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":105.13343808299396,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15880,"response_bytes":19656,"usage":{"input":1963,"output":13775,"cached":2161,"total":15738,"consistent":false},"usage_raw":{"input_tokens":1963,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":13775,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":15738}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":16.617096042027697,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":30533,"response_bytes":7128,"usage":{"input":5434,"output":1504,"cached":2161,"total":6938,"consistent":true},"usage_raw":{"input_tokens":5434,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":1504,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6938}}]} +{"role":"baseline","task_id":"application.create","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.233978584059514,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8785,"response_bytes":6811,"usage":{"input":114,"output":145,"cached":2161,"total":259,"consistent":false},"usage_raw":{"input_tokens":114,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":145,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":259}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":84.08382950001396,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15862,"response_bytes":12009,"usage":{"input":1961,"output":11081,"cached":2161,"total":13042,"consistent":false},"usage_raw":{"input_tokens":1961,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":11081,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":13042}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":26.521535417065024,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21876,"response_bytes":15500,"usage":{"input":1555,"output":4210,"cached":4081,"total":5765,"consistent":false},"usage_raw":{"input_tokens":1555,"input_tokens_details":{"cached_tokens":4081,"cache_write_tokens":0},"output_tokens":4210,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5765}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":34.20582516700961,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":32358,"response_bytes":7913,"usage":{"input":7941,"output":3420,"cached":0,"total":11361,"consistent":true},"usage_raw":{"input_tokens":7941,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":3420,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":11361}}]} +{"role":"baseline","task_id":"application.edit","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.057860792032443,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8943,"response_bytes":6575,"usage":{"input":25,"output":130,"cached":2289,"total":155,"consistent":false},"usage_raw":{"input_tokens":25,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":130,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":155}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.191573249991052,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":16813,"response_bytes":6854,"usage":{"input":2038,"output":437,"cached":2289,"total":2475,"consistent":false},"usage_raw":{"input_tokens":2038,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":437,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2475}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.382085583056323,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17992,"response_bytes":6617,"usage":{"input":2317,"output":434,"cached":2289,"total":2751,"consistent":true},"usage_raw":{"input_tokens":2317,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":434,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2751}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":15.543914165929891,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21585,"response_bytes":8964,"usage":{"input":1132,"output":2196,"cached":4209,"total":3328,"consistent":false},"usage_raw":{"input_tokens":1132,"input_tokens_details":{"cached_tokens":4209,"cache_write_tokens":0},"output_tokens":2196,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3328}}]} +{"role":"baseline","task_id":"application.edit","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.519915833021514,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8943,"response_bytes":6606,"usage":{"input":25,"output":109,"cached":2289,"total":134,"consistent":false},"usage_raw":{"input_tokens":25,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":109,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":134}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.718238083063625,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17321,"response_bytes":6918,"usage":{"input":2231,"output":432,"cached":2289,"total":2663,"consistent":false},"usage_raw":{"input_tokens":2231,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":432,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2663}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":9.098899250035174,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18590,"response_bytes":8991,"usage":{"input":348,"output":1819,"cached":4465,"total":2167,"consistent":false},"usage_raw":{"input_tokens":348,"input_tokens_details":{"cached_tokens":4465,"cache_write_tokens":0},"output_tokens":1819,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2167}}]} +{"role":"baseline","task_id":"application.edit","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.3311902499990538,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8943,"response_bytes":6715,"usage":{"input":2314,"output":119,"cached":0,"total":2433,"consistent":true},"usage_raw":{"input_tokens":2314,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":119,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2433}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.291170583921485,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19902,"response_bytes":6900,"usage":{"input":5048,"output":514,"cached":0,"total":5562,"consistent":true},"usage_raw":{"input_tokens":5048,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":514,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5562}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":7.445799042005092,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21165,"response_bytes":8961,"usage":{"input":363,"output":1717,"cached":4977,"total":2080,"consistent":false},"usage_raw":{"input_tokens":363,"input_tokens_details":{"cached_tokens":4977,"cache_write_tokens":0},"output_tokens":1717,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2080}}]} +{"role":"baseline","task_id":"application.feature","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.489238415961154,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":9279,"response_bytes":6720,"usage":{"input":2372,"output":129,"cached":0,"total":2501,"consistent":true},"usage_raw":{"input_tokens":2372,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":129,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2501}},{"index":1,"complete":true,"status":502,"endpoint":"/responses","duration_s":33.97763741703238,"upstream_status":502,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":null,"request_bytes":20683,"response_bytes":16,"usage":null,"usage_raw":null}]} +{"role":"baseline","task_id":"application.feature","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.8780365410493687,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":9279,"response_bytes":6682,"usage":{"input":83,"output":111,"cached":2289,"total":194,"consistent":false},"usage_raw":{"input_tokens":83,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":111,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":194}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.291980165988207,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18092,"response_bytes":6667,"usage":{"input":2823,"output":350,"cached":1649,"total":3173,"consistent":true},"usage_raw":{"input_tokens":2823,"input_tokens_details":{"cached_tokens":1649,"cache_write_tokens":0},"output_tokens":350,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3173}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":62.84187645895872,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21880,"response_bytes":10918,"usage":{"input":1060,"output":8255,"cached":4465,"total":9315,"consistent":false},"usage_raw":{"input_tokens":1060,"input_tokens_details":{"cached_tokens":4465,"cache_write_tokens":0},"output_tokens":8255,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":9315}}]} +{"role":"baseline","task_id":"application.feature","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.2635080830659717,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":9279,"response_bytes":6638,"usage":{"input":83,"output":108,"cached":2289,"total":191,"consistent":false},"usage_raw":{"input_tokens":83,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":108,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":191}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.1912323330761865,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18070,"response_bytes":6702,"usage":{"input":2180,"output":348,"cached":2289,"total":2528,"consistent":false},"usage_raw":{"input_tokens":2180,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":348,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2528}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":46.87802349997219,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21533,"response_bytes":7914,"usage":{"input":3141,"output":5106,"cached":2289,"total":8247,"consistent":true},"usage_raw":{"input_tokens":3141,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":5106,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":8247}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.621413750108331,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23456,"response_bytes":7658,"usage":{"input":583,"output":477,"cached":5361,"total":1060,"consistent":false},"usage_raw":{"input_tokens":583,"input_tokens_details":{"cached_tokens":5361,"cache_write_tokens":0},"output_tokens":477,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1060}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":17.177266499958932,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":25074,"response_bytes":7266,"usage":{"input":1892,"output":2082,"cached":4465,"total":3974,"consistent":false},"usage_raw":{"input_tokens":1892,"input_tokens_details":{"cached_tokens":4465,"cache_write_tokens":0},"output_tokens":2082,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3974}},{"index":5,"complete":true,"status":200,"endpoint":"/responses","duration_s":17.72425533295609,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":26271,"response_bytes":6537,"usage":{"input":423,"output":1874,"cached":6257,"total":2297,"consistent":false},"usage_raw":{"input_tokens":423,"input_tokens_details":{"cached_tokens":6257,"cache_write_tokens":0},"output_tokens":1874,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2297}},{"index":6,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.557824749965221,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":29827,"response_bytes":6537,"usage":{"input":1530,"output":449,"cached":5873,"total":1979,"consistent":false},"usage_raw":{"input_tokens":1530,"input_tokens_details":{"cached_tokens":5873,"cache_write_tokens":0},"output_tokens":449,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1979}},{"index":7,"complete":true,"status":200,"endpoint":"/responses","duration_s":13.446417417027988,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":33817,"response_bytes":7340,"usage":{"input":1704,"output":1389,"cached":6641,"total":3093,"consistent":false},"usage_raw":{"input_tokens":1704,"input_tokens_details":{"cached_tokens":6641,"cache_write_tokens":0},"output_tokens":1389,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3093}},{"index":8,"complete":true,"status":200,"endpoint":"/responses","duration_s":16.199758416973054,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":35145,"response_bytes":6655,"usage":{"input":364,"output":1131,"cached":8305,"total":1495,"consistent":false},"usage_raw":{"input_tokens":364,"input_tokens_details":{"cached_tokens":8305,"cache_write_tokens":0},"output_tokens":1131,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1495}},{"index":9,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.719247667002492,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":35761,"response_bytes":6992,"usage":{"input":1559,"output":400,"cached":7281,"total":1959,"consistent":false},"usage_raw":{"input_tokens":1559,"input_tokens_details":{"cached_tokens":7281,"cache_write_tokens":0},"output_tokens":400,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1959}}]} +{"role":"baseline","task_id":"csv.bug","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.728177750017494,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8420,"response_bytes":6646,"usage":{"input":39,"output":117,"cached":2161,"total":156,"consistent":false},"usage_raw":{"input_tokens":39,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":117,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":156}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.089516083011404,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15354,"response_bytes":7030,"usage":{"input":1739,"output":394,"cached":2161,"total":2133,"consistent":false},"usage_raw":{"input_tokens":1739,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":394,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2133}}]} +{"role":"baseline","task_id":"csv.bug","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.018846374936402,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8420,"response_bytes":6704,"usage":{"input":2200,"output":118,"cached":0,"total":2318,"consistent":true},"usage_raw":{"input_tokens":2200,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":118,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2318}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":7.283793249982409,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15383,"response_bytes":7035,"usage":{"input":1744,"output":578,"cached":2161,"total":2322,"consistent":false},"usage_raw":{"input_tokens":1744,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":578,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2322}}]} +{"role":"baseline","task_id":"csv.bug","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.0627218750305474,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8420,"response_bytes":6690,"usage":{"input":2200,"output":121,"cached":0,"total":2321,"consistent":true},"usage_raw":{"input_tokens":2200,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":121,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2321}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.709934083046392,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15376,"response_bytes":6626,"usage":{"input":1744,"output":535,"cached":2161,"total":2279,"consistent":false},"usage_raw":{"input_tokens":1744,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":535,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2279}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.293196124956012,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18603,"response_bytes":7120,"usage":{"input":994,"output":627,"cached":3825,"total":1621,"consistent":false},"usage_raw":{"input_tokens":994,"input_tokens_details":{"cached_tokens":3825,"cache_write_tokens":0},"output_tokens":627,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1621}}]} +{"role":"baseline","task_id":"csv.create","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.1667996670585126,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8268,"response_bytes":6826,"usage":{"input":11,"output":156,"cached":2161,"total":167,"consistent":false},"usage_raw":{"input_tokens":11,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":156,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":167}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.8594435000559315,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8940,"response_bytes":6761,"usage":{"input":181,"output":147,"cached":2161,"total":328,"consistent":false},"usage_raw":{"input_tokens":181,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":147,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":328}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":30.495307542034425,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15416,"response_bytes":6511,"usage":{"input":1915,"output":3049,"cached":2161,"total":4964,"consistent":false},"usage_raw":{"input_tokens":1915,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":3049,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4964}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":24.246342541999184,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15758,"response_bytes":6604,"usage":{"input":4163,"output":2369,"cached":0,"total":6532,"consistent":true},"usage_raw":{"input_tokens":4163,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":2369,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6532}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":87.54210204188712,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17344,"response_bytes":7496,"usage":{"input":643,"output":10249,"cached":3953,"total":10892,"consistent":false},"usage_raw":{"input_tokens":643,"input_tokens_details":{"cached_tokens":3953,"cache_write_tokens":0},"output_tokens":10249,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":10892}},{"index":5,"complete":true,"status":200,"endpoint":"/responses","duration_s":40.886275541968644,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18788,"response_bytes":11325,"usage":{"input":2670,"output":4959,"cached":2289,"total":7629,"consistent":true},"usage_raw":{"input_tokens":2670,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":4959,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7629}}]} +{"role":"baseline","task_id":"csv.create","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.477088209008798,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8268,"response_bytes":6816,"usage":{"input":11,"output":152,"cached":2161,"total":163,"consistent":false},"usage_raw":{"input_tokens":11,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":152,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":163}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":98.79369216703344,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14831,"response_bytes":10202,"usage":{"input":1755,"output":9724,"cached":2161,"total":11479,"consistent":false},"usage_raw":{"input_tokens":1755,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":9724,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":11479}},{"index":2,"complete":true,"status":503,"endpoint":"/responses","duration_s":9.031703208922409,"upstream_status":503,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":null,"request_bytes":19083,"response_bytes":153,"usage":null,"usage_raw":null}]} +{"role":"baseline","task_id":"csv.create","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.455412667011842,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8268,"response_bytes":6828,"usage":{"input":2172,"output":423,"cached":0,"total":2595,"consistent":true},"usage_raw":{"input_tokens":2172,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":423,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2595}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.1232944160001352,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8941,"response_bytes":6745,"usage":{"input":182,"output":153,"cached":2161,"total":335,"consistent":false},"usage_raw":{"input_tokens":182,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":153,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":335}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.7436079169856384,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15468,"response_bytes":6567,"usage":{"input":4083,"output":380,"cached":0,"total":4463,"consistent":true},"usage_raw":{"input_tokens":4083,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":380,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4463}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.766992499935441,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":16814,"response_bytes":6527,"usage":{"input":2094,"output":454,"cached":2289,"total":2548,"consistent":false},"usage_raw":{"input_tokens":2094,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":454,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2548}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":111.83617129200138,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17229,"response_bytes":17608,"usage":{"input":411,"output":11305,"cached":4081,"total":11716,"consistent":false},"usage_raw":{"input_tokens":411,"input_tokens_details":{"cached_tokens":4081,"cache_write_tokens":0},"output_tokens":11305,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":11716}},{"index":5,"complete":true,"status":200,"endpoint":"/responses","duration_s":53.195898915990256,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":29508,"response_bytes":7312,"usage":{"input":3242,"output":3711,"cached":4337,"total":6953,"consistent":false},"usage_raw":{"input_tokens":3242,"input_tokens_details":{"cached_tokens":4337,"cache_write_tokens":0},"output_tokens":3711,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6953}}]} +{"role":"baseline","task_id":"csv.edit","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.115364750032313,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8421,"response_bytes":6744,"usage":{"input":2210,"output":135,"cached":0,"total":2345,"consistent":true},"usage_raw":{"input_tokens":2210,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":135,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2345}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":10.845205125049688,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17806,"response_bytes":7228,"usage":{"input":4610,"output":1256,"cached":0,"total":5866,"consistent":true},"usage_raw":{"input_tokens":4610,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":1256,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5866}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":22.72906112507917,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19639,"response_bytes":8135,"usage":{"input":2843,"output":3026,"cached":2161,"total":5869,"consistent":true},"usage_raw":{"input_tokens":2843,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":3026,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5869}}]} +{"role":"baseline","task_id":"csv.edit","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.457751125097275,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8421,"response_bytes":6623,"usage":{"input":49,"output":153,"cached":2161,"total":202,"consistent":false},"usage_raw":{"input_tokens":49,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":153,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":202}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.396125750034116,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14801,"response_bytes":6472,"usage":{"input":1737,"output":481,"cached":2161,"total":2218,"consistent":false},"usage_raw":{"input_tokens":1737,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":481,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2218}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":8.314476375002414,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":16188,"response_bytes":8162,"usage":{"input":2159,"output":1126,"cached":2161,"total":3285,"consistent":false},"usage_raw":{"input_tokens":2159,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":1126,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3285}}]} +{"role":"baseline","task_id":"csv.edit","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":1.9413104580016807,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8421,"response_bytes":6623,"usage":{"input":49,"output":105,"cached":2161,"total":154,"consistent":false},"usage_raw":{"input_tokens":49,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":105,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":154}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":7.266571875079535,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15761,"response_bytes":7210,"usage":{"input":4168,"output":804,"cached":0,"total":4972,"consistent":true},"usage_raw":{"input_tokens":4168,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":804,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4972}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.332408167072572,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17603,"response_bytes":6830,"usage":{"input":2403,"output":719,"cached":2161,"total":3122,"consistent":true},"usage_raw":{"input_tokens":2403,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":719,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3122}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":10.632157832966186,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18707,"response_bytes":8129,"usage":{"input":366,"output":2026,"cached":4465,"total":2392,"consistent":false},"usage_raw":{"input_tokens":366,"input_tokens_details":{"cached_tokens":4465,"cache_write_tokens":0},"output_tokens":2026,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2392}}]} +{"role":"baseline","task_id":"csv.feature","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.598010416026227,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8718,"response_bytes":6707,"usage":{"input":2260,"output":121,"cached":0,"total":2381,"consistent":true},"usage_raw":{"input_tokens":2260,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":121,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2381}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.332724582985975,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15955,"response_bytes":6613,"usage":{"input":1864,"output":216,"cached":2161,"total":2080,"consistent":false},"usage_raw":{"input_tokens":1864,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":216,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2080}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":37.98254429199733,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18931,"response_bytes":9102,"usage":{"input":931,"output":4088,"cached":3953,"total":5019,"consistent":false},"usage_raw":{"input_tokens":931,"input_tokens_details":{"cached_tokens":3953,"cache_write_tokens":0},"output_tokens":4088,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5019}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":17.519733541994356,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":22887,"response_bytes":9725,"usage":{"input":943,"output":2696,"cached":4849,"total":3639,"consistent":false},"usage_raw":{"input_tokens":943,"input_tokens_details":{"cached_tokens":4849,"cache_write_tokens":0},"output_tokens":2696,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3639}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":14.934053792036138,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":27794,"response_bytes":7821,"usage":{"input":4715,"output":1100,"cached":2161,"total":5815,"consistent":true},"usage_raw":{"input_tokens":4715,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":1100,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5815}}]} +{"role":"baseline","task_id":"csv.feature","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.872353875078261,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8718,"response_bytes":6764,"usage":{"input":2260,"output":133,"cached":0,"total":2393,"consistent":true},"usage_raw":{"input_tokens":2260,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":133,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2393}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":38.06995420798194,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18113,"response_bytes":9743,"usage":{"input":2501,"output":5344,"cached":2161,"total":7845,"consistent":true},"usage_raw":{"input_tokens":2501,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":5344,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7845}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":19.061962000094354,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23055,"response_bytes":7727,"usage":{"input":1162,"output":1491,"cached":4593,"total":2653,"consistent":false},"usage_raw":{"input_tokens":1162,"input_tokens_details":{"cached_tokens":4593,"cache_write_tokens":0},"output_tokens":1491,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2653}}]} +{"role":"baseline","task_id":"csv.feature","trial":3,"calls":[{"index":0,"complete":true,"status":500,"endpoint":"/responses","duration_s":60.178625625092536,"upstream_status":500,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":null,"request_bytes":8718,"response_bytes":170,"usage":null,"usage_raw":null}]} +{"role":"baseline","task_id":"inventory.bug","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.4028602089965716,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8413,"response_bytes":6679,"usage":{"input":40,"output":123,"cached":2161,"total":163,"consistent":false},"usage_raw":{"input_tokens":40,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":123,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":163}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":12.018915707943961,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15820,"response_bytes":6628,"usage":{"input":3999,"output":929,"cached":0,"total":4928,"consistent":true},"usage_raw":{"input_tokens":3999,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":929,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4928}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":28.25549608306028,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19120,"response_bytes":7696,"usage":{"input":979,"output":3016,"cached":3953,"total":3995,"consistent":false},"usage_raw":{"input_tokens":979,"input_tokens_details":{"cached_tokens":3953,"cache_write_tokens":0},"output_tokens":3016,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3995}}]} +{"role":"baseline","task_id":"inventory.bug","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.2488382500596344,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8413,"response_bytes":6738,"usage":{"input":2201,"output":132,"cached":0,"total":2333,"consistent":true},"usage_raw":{"input_tokens":2201,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":132,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2333}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":14.93197858298663,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17881,"response_bytes":7783,"usage":{"input":2432,"output":1452,"cached":2161,"total":3884,"consistent":true},"usage_raw":{"input_tokens":2432,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":1452,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3884}}]} +{"role":"baseline","task_id":"inventory.bug","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.1501184169901535,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8413,"response_bytes":6662,"usage":{"input":40,"output":114,"cached":2161,"total":154,"consistent":false},"usage_raw":{"input_tokens":40,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":114,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":154}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":8.281764833023772,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15469,"response_bytes":6633,"usage":{"input":1739,"output":686,"cached":2161,"total":2425,"consistent":false},"usage_raw":{"input_tokens":1739,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":686,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2425}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":30.302430624957196,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19042,"response_bytes":7836,"usage":{"input":1085,"output":3045,"cached":3825,"total":4130,"consistent":false},"usage_raw":{"input_tokens":1085,"input_tokens_details":{"cached_tokens":3825,"cache_write_tokens":0},"output_tokens":3045,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4130}}]} +{"role":"baseline","task_id":"inventory.create","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.7612153330119327,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8232,"response_bytes":6841,"usage":{"input":5,"output":152,"cached":2161,"total":157,"consistent":false},"usage_raw":{"input_tokens":5,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":152,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":157}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":59.031943833921105,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14772,"response_bytes":7305,"usage":{"input":3906,"output":5802,"cached":0,"total":9708,"consistent":true},"usage_raw":{"input_tokens":3906,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":5802,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":9708}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":43.96288879099302,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":16023,"response_bytes":16252,"usage":{"input":2067,"output":5566,"cached":2161,"total":7633,"consistent":false},"usage_raw":{"input_tokens":2067,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":5566,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7633}}]} +{"role":"baseline","task_id":"inventory.create","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.01713158399798,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8232,"response_bytes":6837,"usage":{"input":5,"output":149,"cached":2161,"total":154,"consistent":false},"usage_raw":{"input_tokens":5,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":149,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":154}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.929617458023131,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14770,"response_bytes":6650,"usage":{"input":1744,"output":609,"cached":2161,"total":2353,"consistent":false},"usage_raw":{"input_tokens":1744,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":609,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2353}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":63.79093449993525,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":16399,"response_bytes":7537,"usage":{"input":4351,"output":7009,"cached":0,"total":11360,"consistent":true},"usage_raw":{"input_tokens":4351,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":7009,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":11360}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":50.68205924995709,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17864,"response_bytes":17045,"usage":{"input":384,"output":6799,"cached":4337,"total":7183,"consistent":false},"usage_raw":{"input_tokens":384,"input_tokens_details":{"cached_tokens":4337,"cache_write_tokens":0},"output_tokens":6799,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7183}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":26.723977167042904,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":29654,"response_bytes":8118,"usage":{"input":3799,"output":2649,"cached":3825,"total":6448,"consistent":false},"usage_raw":{"input_tokens":3799,"input_tokens_details":{"cached_tokens":3825,"cache_write_tokens":0},"output_tokens":2649,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6448}}]} +{"role":"baseline","task_id":"inventory.create","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.7561136659933254,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8232,"response_bytes":6839,"usage":{"input":5,"output":151,"cached":2161,"total":156,"consistent":false},"usage_raw":{"input_tokens":5,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":151,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":156}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":87.13549899996724,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14771,"response_bytes":15794,"usage":{"input":1745,"output":10690,"cached":2161,"total":12435,"consistent":false},"usage_raw":{"input_tokens":1745,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":10690,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":12435}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":26.70143316697795,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":25248,"response_bytes":7076,"usage":{"input":2631,"output":1738,"cached":3825,"total":4369,"consistent":false},"usage_raw":{"input_tokens":2631,"input_tokens_details":{"cached_tokens":3825,"cache_write_tokens":0},"output_tokens":1738,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4369}}]} +{"role":"baseline","task_id":"inventory.edit","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.1383967919973657,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8380,"response_bytes":6627,"usage":{"input":43,"output":126,"cached":2161,"total":169,"consistent":false},"usage_raw":{"input_tokens":43,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":126,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":169}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.6096494579687715,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15095,"response_bytes":6550,"usage":{"input":1750,"output":376,"cached":2161,"total":2126,"consistent":false},"usage_raw":{"input_tokens":1750,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":376,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2126}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":17.169453124981374,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":16521,"response_bytes":8429,"usage":{"input":515,"output":2373,"cached":3825,"total":2888,"consistent":false},"usage_raw":{"input_tokens":515,"input_tokens_details":{"cached_tokens":3825,"cache_write_tokens":0},"output_tokens":2373,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2888}}]} +{"role":"baseline","task_id":"inventory.edit","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.4215339579386637,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8380,"response_bytes":6691,"usage":{"input":2204,"output":129,"cached":0,"total":2333,"consistent":true},"usage_raw":{"input_tokens":2204,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":129,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2333}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":8.563890916993842,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17345,"response_bytes":7033,"usage":{"input":2347,"output":972,"cached":2161,"total":3319,"consistent":true},"usage_raw":{"input_tokens":2347,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":972,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3319}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":9.37115266697947,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18520,"response_bytes":8582,"usage":{"input":4800,"output":1805,"cached":0,"total":6605,"consistent":true},"usage_raw":{"input_tokens":4800,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":1805,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6605}}]} +{"role":"baseline","task_id":"inventory.edit","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.9420469170436263,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8380,"response_bytes":6719,"usage":{"input":43,"output":130,"cached":2161,"total":173,"consistent":false},"usage_raw":{"input_tokens":43,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":130,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":173}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":8.151513167074881,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":16674,"response_bytes":7466,"usage":{"input":2178,"output":835,"cached":2161,"total":3013,"consistent":true},"usage_raw":{"input_tokens":2178,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":835,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3013}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":14.017446708050556,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18972,"response_bytes":7026,"usage":{"input":2671,"output":1292,"cached":2161,"total":3963,"consistent":true},"usage_raw":{"input_tokens":2671,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":1292,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3963}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":13.46985999995377,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":20233,"response_bytes":6524,"usage":{"input":792,"output":1800,"cached":4337,"total":2592,"consistent":false},"usage_raw":{"input_tokens":792,"input_tokens_details":{"cached_tokens":4337,"cache_write_tokens":0},"output_tokens":1800,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2592}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":17.739838292007335,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":22885,"response_bytes":8458,"usage":{"input":576,"output":2495,"cached":5105,"total":3071,"consistent":false},"usage_raw":{"input_tokens":576,"input_tokens_details":{"cached_tokens":5105,"cache_write_tokens":0},"output_tokens":2495,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3071}}]} +{"role":"baseline","task_id":"inventory.feature","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.9371685839723796,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8642,"response_bytes":6734,"usage":{"input":88,"output":132,"cached":2161,"total":220,"consistent":false},"usage_raw":{"input_tokens":88,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":132,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":220}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":43.22117370797787,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18096,"response_bytes":10905,"usage":{"input":2475,"output":4791,"cached":2161,"total":7266,"consistent":true},"usage_raw":{"input_tokens":2475,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":4791,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7266}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":18.42779079102911,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24264,"response_bytes":7711,"usage":{"input":3832,"output":1752,"cached":2161,"total":5584,"consistent":true},"usage_raw":{"input_tokens":3832,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":1752,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5584}}]} +{"role":"baseline","task_id":"inventory.feature","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.6936429580673575,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8642,"response_bytes":6774,"usage":{"input":2249,"output":146,"cached":0,"total":2395,"consistent":true},"usage_raw":{"input_tokens":2249,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":146,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2395}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":45.42312212497927,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18927,"response_bytes":10740,"usage":{"input":4869,"output":5029,"cached":0,"total":9898,"consistent":true},"usage_raw":{"input_tokens":4869,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":5029,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":9898}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":39.021651459042914,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24946,"response_bytes":8969,"usage":{"input":4014,"output":4589,"cached":2161,"total":8603,"consistent":true},"usage_raw":{"input_tokens":4014,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":4589,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":8603}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.803598582977429,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":33172,"response_bytes":8957,"usage":{"input":1743,"output":698,"cached":6129,"total":2441,"consistent":false},"usage_raw":{"input_tokens":1743,"input_tokens_details":{"cached_tokens":6129,"cache_write_tokens":0},"output_tokens":698,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2441}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":14.535840083961375,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":40665,"response_bytes":6819,"usage":{"input":4505,"output":1667,"cached":4849,"total":6172,"consistent":false},"usage_raw":{"input_tokens":4505,"input_tokens_details":{"cached_tokens":4849,"cache_write_tokens":0},"output_tokens":1667,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6172}},{"index":5,"complete":true,"status":200,"endpoint":"/responses","duration_s":35.11787345900666,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":41459,"response_bytes":7473,"usage":{"input":1751,"output":2794,"cached":7793,"total":4545,"consistent":false},"usage_raw":{"input_tokens":1751,"input_tokens_details":{"cached_tokens":7793,"cache_write_tokens":0},"output_tokens":2794,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4545}}]} +{"role":"baseline","task_id":"inventory.feature","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.0509444169001654,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8642,"response_bytes":6730,"usage":{"input":88,"output":130,"cached":2161,"total":218,"consistent":false},"usage_raw":{"input_tokens":88,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":130,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":218}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":61.06724270794075,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18094,"response_bytes":10235,"usage":{"input":2474,"output":6545,"cached":2161,"total":9019,"consistent":true},"usage_raw":{"input_tokens":2474,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":6545,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":9019}}]} +{"role":"baseline","task_id":"notes.bug","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.600552332936786,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8408,"response_bytes":6668,"usage":{"input":2201,"output":119,"cached":0,"total":2320,"consistent":true},"usage_raw":{"input_tokens":2201,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":119,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2320}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":11.459770833957009,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14821,"response_bytes":6658,"usage":{"input":3789,"output":783,"cached":0,"total":4572,"consistent":true},"usage_raw":{"input_tokens":3789,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":783,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4572}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":17.405557792051695,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18408,"response_bytes":7043,"usage":{"input":1106,"output":1737,"cached":3697,"total":2843,"consistent":false},"usage_raw":{"input_tokens":1106,"input_tokens_details":{"cached_tokens":3697,"cache_write_tokens":0},"output_tokens":1737,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2843}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.380954458960332,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":20157,"response_bytes":6526,"usage":{"input":3094,"output":86,"cached":2161,"total":3180,"consistent":true},"usage_raw":{"input_tokens":3094,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":86,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3180}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.860596042010002,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":22262,"response_bytes":6902,"usage":{"input":1072,"output":180,"cached":4721,"total":1252,"consistent":false},"usage_raw":{"input_tokens":1072,"input_tokens_details":{"cached_tokens":4721,"cache_write_tokens":0},"output_tokens":180,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1252}}]} +{"role":"baseline","task_id":"notes.bug","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.574600583058782,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8408,"response_bytes":6644,"usage":{"input":40,"output":110,"cached":2161,"total":150,"consistent":false},"usage_raw":{"input_tokens":40,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":110,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":150}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":10.587343625025824,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14611,"response_bytes":6649,"usage":{"input":1584,"output":1202,"cached":2161,"total":2786,"consistent":false},"usage_raw":{"input_tokens":1584,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":1202,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2786}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":11.773886458948255,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18047,"response_bytes":6989,"usage":{"input":2541,"output":1318,"cached":2161,"total":3859,"consistent":true},"usage_raw":{"input_tokens":2541,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":1318,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3859}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.2787492919014767,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19748,"response_bytes":6520,"usage":{"input":1447,"output":83,"cached":3697,"total":1530,"consistent":false},"usage_raw":{"input_tokens":1447,"input_tokens_details":{"cached_tokens":3697,"cache_write_tokens":0},"output_tokens":83,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1530}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.557481833035126,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21850,"response_bytes":7077,"usage":{"input":576,"output":420,"cached":5105,"total":996,"consistent":false},"usage_raw":{"input_tokens":576,"input_tokens_details":{"cached_tokens":5105,"cache_write_tokens":0},"output_tokens":420,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":996}}]} +{"role":"baseline","task_id":"notes.bug","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.8329767499817535,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8408,"response_bytes":6654,"usage":{"input":40,"output":114,"cached":2161,"total":154,"consistent":false},"usage_raw":{"input_tokens":40,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":114,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":154}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.424339334014803,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14616,"response_bytes":6701,"usage":{"input":1585,"output":554,"cached":2161,"total":2139,"consistent":false},"usage_raw":{"input_tokens":1585,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":554,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2139}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":7.88118495803792,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18421,"response_bytes":7210,"usage":{"input":2647,"output":752,"cached":2161,"total":3399,"consistent":true},"usage_raw":{"input_tokens":2647,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":752,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3399}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":1.9158964169910178,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":20318,"response_bytes":6508,"usage":{"input":570,"output":82,"cached":4721,"total":652,"consistent":false},"usage_raw":{"input_tokens":570,"input_tokens_details":{"cached_tokens":4721,"cache_write_tokens":0},"output_tokens":82,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":652}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.1415232089348137,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":22414,"response_bytes":6875,"usage":{"input":2129,"output":194,"cached":3697,"total":2323,"consistent":false},"usage_raw":{"input_tokens":2129,"input_tokens_details":{"cached_tokens":3697,"cache_write_tokens":0},"output_tokens":194,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2323}}]} +{"role":"baseline","task_id":"notes.create","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.494786625029519,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8234,"response_bytes":6803,"usage":{"input":6,"output":144,"cached":2161,"total":150,"consistent":false},"usage_raw":{"input_tokens":6,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":144,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":150}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":61.17870087502524,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14757,"response_bytes":14964,"usage":{"input":1744,"output":9454,"cached":2161,"total":11198,"consistent":false},"usage_raw":{"input_tokens":1744,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":9454,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":11198}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":16.970749041065574,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24232,"response_bytes":7686,"usage":{"input":4077,"output":1547,"cached":2161,"total":5624,"consistent":true},"usage_raw":{"input_tokens":4077,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":1547,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5624}}]} +{"role":"baseline","task_id":"notes.create","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.17383487499319,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8234,"response_bytes":6820,"usage":{"input":2167,"output":151,"cached":0,"total":2318,"consistent":true},"usage_raw":{"input_tokens":2167,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":151,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2318}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":29.359173250035383,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14765,"response_bytes":6554,"usage":{"input":1746,"output":2799,"cached":2161,"total":4545,"consistent":false},"usage_raw":{"input_tokens":1746,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":2799,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4545}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":60.81327029201202,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15193,"response_bytes":15991,"usage":{"input":4018,"output":8555,"cached":0,"total":12573,"consistent":true},"usage_raw":{"input_tokens":4018,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":8555,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":12573}}]} +{"role":"baseline","task_id":"notes.create","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.659389708074741,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8234,"response_bytes":6791,"usage":{"input":6,"output":149,"cached":2161,"total":155,"consistent":false},"usage_raw":{"input_tokens":6,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":149,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":155}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":16.375365625019185,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14751,"response_bytes":6603,"usage":{"input":1745,"output":1536,"cached":2161,"total":3281,"consistent":false},"usage_raw":{"input_tokens":1745,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":1536,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3281}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":9.92556154099293,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":16304,"response_bytes":6544,"usage":{"input":2172,"output":928,"cached":2161,"total":3100,"consistent":true},"usage_raw":{"input_tokens":2172,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":928,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3100}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":26.79606562491972,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17713,"response_bytes":7363,"usage":{"input":447,"output":2516,"cached":4209,"total":2963,"consistent":false},"usage_raw":{"input_tokens":447,"input_tokens_details":{"cached_tokens":4209,"cache_write_tokens":0},"output_tokens":2516,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2963}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":50.02193979197182,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19032,"response_bytes":13616,"usage":{"input":393,"output":6960,"cached":4593,"total":7353,"consistent":false},"usage_raw":{"input_tokens":393,"input_tokens_details":{"cached_tokens":4593,"cache_write_tokens":0},"output_tokens":6960,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7353}},{"index":5,"complete":true,"status":200,"endpoint":"/responses","duration_s":15.738563166931272,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":27080,"response_bytes":9046,"usage":{"input":2006,"output":1627,"cached":4977,"total":3633,"consistent":false},"usage_raw":{"input_tokens":2006,"input_tokens_details":{"cached_tokens":4977,"cache_write_tokens":0},"output_tokens":1627,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3633}},{"index":6,"complete":true,"status":200,"endpoint":"/responses","duration_s":19.10761929198634,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":30150,"response_bytes":10803,"usage":{"input":893,"output":2829,"cached":6897,"total":3722,"consistent":false},"usage_raw":{"input_tokens":893,"input_tokens_details":{"cached_tokens":6897,"cache_write_tokens":0},"output_tokens":2829,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3722}},{"index":7,"complete":true,"status":200,"endpoint":"/responses","duration_s":16.446771417045966,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":35378,"response_bytes":7528,"usage":{"input":5197,"output":1268,"cached":3825,"total":6465,"consistent":true},"usage_raw":{"input_tokens":5197,"input_tokens_details":{"cached_tokens":3825,"cache_write_tokens":0},"output_tokens":1268,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6465}},{"index":8,"complete":true,"status":200,"endpoint":"/responses","duration_s":13.265026958077215,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":36904,"response_bytes":8689,"usage":{"input":1713,"output":1100,"cached":7665,"total":2813,"consistent":false},"usage_raw":{"input_tokens":1713,"input_tokens_details":{"cached_tokens":7665,"cache_write_tokens":0},"output_tokens":1100,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2813}}]} +{"role":"baseline","task_id":"notes.edit","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.0036291669821367,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8384,"response_bytes":6621,"usage":{"input":2205,"output":102,"cached":0,"total":2307,"consistent":true},"usage_raw":{"input_tokens":2205,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":102,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2307}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.394400957971811,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14466,"response_bytes":7254,"usage":{"input":1641,"output":521,"cached":2161,"total":2162,"consistent":false},"usage_raw":{"input_tokens":1641,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":521,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2162}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":16.438896416919306,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":16310,"response_bytes":8086,"usage":{"input":501,"output":2271,"cached":3697,"total":2772,"consistent":false},"usage_raw":{"input_tokens":501,"input_tokens_details":{"cached_tokens":3697,"cache_write_tokens":0},"output_tokens":2271,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2772}}]} +{"role":"baseline","task_id":"notes.edit","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.3462537500308827,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8384,"response_bytes":6639,"usage":{"input":44,"output":106,"cached":2161,"total":150,"consistent":false},"usage_raw":{"input_tokens":44,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":106,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":150}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.6392255841055885,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15212,"response_bytes":7312,"usage":{"input":1886,"output":693,"cached":2161,"total":2579,"consistent":false},"usage_raw":{"input_tokens":1886,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":693,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2579}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.844504208071157,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17129,"response_bytes":6612,"usage":{"input":503,"output":560,"cached":3953,"total":1063,"consistent":false},"usage_raw":{"input_tokens":503,"input_tokens_details":{"cached_tokens":3953,"cache_write_tokens":0},"output_tokens":560,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1063}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.709055999992415,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17766,"response_bytes":6548,"usage":{"input":279,"output":303,"cached":4337,"total":582,"consistent":false},"usage_raw":{"input_tokens":279,"input_tokens_details":{"cached_tokens":4337,"cache_write_tokens":0},"output_tokens":303,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":582}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":17.064813541946933,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":20887,"response_bytes":6549,"usage":{"input":947,"output":1631,"cached":4593,"total":2578,"consistent":false},"usage_raw":{"input_tokens":947,"input_tokens_details":{"cached_tokens":4593,"cache_write_tokens":0},"output_tokens":1631,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2578}},{"index":5,"complete":true,"status":200,"endpoint":"/responses","duration_s":15.624965166905895,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21314,"response_bytes":8123,"usage":{"input":160,"output":2713,"cached":5489,"total":2873,"consistent":false},"usage_raw":{"input_tokens":160,"input_tokens_details":{"cached_tokens":5489,"cache_write_tokens":0},"output_tokens":2713,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2873}}]} +{"role":"baseline","task_id":"notes.edit","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.755863250000402,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8384,"response_bytes":6627,"usage":{"input":44,"output":105,"cached":2161,"total":149,"consistent":false},"usage_raw":{"input_tokens":44,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":105,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":149}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.312183666974306,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14469,"response_bytes":6545,"usage":{"input":3805,"output":329,"cached":0,"total":4134,"consistent":true},"usage_raw":{"input_tokens":3805,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":329,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4134}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":16.925014375010505,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15894,"response_bytes":8257,"usage":{"input":2073,"output":2522,"cached":2161,"total":4595,"consistent":false},"usage_raw":{"input_tokens":2073,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":2522,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4595}}]} +{"role":"baseline","task_id":"notes.feature","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.920690874918364,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8795,"response_bytes":6659,"usage":{"input":113,"output":119,"cached":2161,"total":232,"consistent":false},"usage_raw":{"input_tokens":113,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":119,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":232}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.999468625057489,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15822,"response_bytes":6614,"usage":{"input":4050,"output":233,"cached":0,"total":4283,"consistent":true},"usage_raw":{"input_tokens":4050,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":233,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4283}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":43.872449874994345,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18774,"response_bytes":9025,"usage":{"input":920,"output":4242,"cached":3953,"total":5162,"consistent":false},"usage_raw":{"input_tokens":920,"input_tokens_details":{"cached_tokens":3953,"cache_write_tokens":0},"output_tokens":4242,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5162}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":31.48408974998165,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21868,"response_bytes":9507,"usage":{"input":823,"output":3754,"cached":4849,"total":4577,"consistent":false},"usage_raw":{"input_tokens":823,"input_tokens_details":{"cached_tokens":4849,"cache_write_tokens":0},"output_tokens":3754,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4577}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":7.017157500027679,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":26893,"response_bytes":8869,"usage":{"input":4696,"output":1317,"cached":2161,"total":6013,"consistent":true},"usage_raw":{"input_tokens":4696,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":1317,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6013}},{"index":5,"complete":true,"status":200,"endpoint":"/responses","duration_s":13.05637754208874,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":30663,"response_bytes":6695,"usage":{"input":978,"output":1444,"cached":6769,"total":2422,"consistent":false},"usage_raw":{"input_tokens":978,"input_tokens_details":{"cached_tokens":6769,"cache_write_tokens":0},"output_tokens":1444,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2422}},{"index":6,"complete":true,"status":200,"endpoint":"/responses","duration_s":20.814110250095837,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":31320,"response_bytes":7699,"usage":{"input":251,"output":1331,"cached":7665,"total":1582,"consistent":false},"usage_raw":{"input_tokens":251,"input_tokens_details":{"cached_tokens":7665,"cache_write_tokens":0},"output_tokens":1331,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1582}},{"index":7,"complete":true,"status":200,"endpoint":"/responses","duration_s":15.104644792038016,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":33086,"response_bytes":6582,"usage":{"input":2764,"output":981,"cached":5617,"total":3745,"consistent":false},"usage_raw":{"input_tokens":2764,"input_tokens_details":{"cached_tokens":5617,"cache_write_tokens":0},"output_tokens":981,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3745}}]} +{"role":"baseline","task_id":"notes.feature","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.1480295000365004,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8795,"response_bytes":6764,"usage":{"input":2274,"output":141,"cached":0,"total":2415,"consistent":true},"usage_raw":{"input_tokens":2274,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":141,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2415}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":37.34998958394863,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18447,"response_bytes":13052,"usage":{"input":2623,"output":5549,"cached":2161,"total":8172,"consistent":true},"usage_raw":{"input_tokens":2623,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":5549,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":8172}}]} +{"role":"baseline","task_id":"notes.feature","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.2357745830668136,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8795,"response_bytes":6601,"usage":{"input":113,"output":105,"cached":2161,"total":218,"consistent":false},"usage_raw":{"input_tokens":113,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":105,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":218}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.635766208055429,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14982,"response_bytes":6649,"usage":{"input":1653,"output":282,"cached":2161,"total":1935,"consistent":false},"usage_raw":{"input_tokens":1653,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":282,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1935}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":31.764039832982235,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17950,"response_bytes":9194,"usage":{"input":941,"output":4044,"cached":3697,"total":4985,"consistent":false},"usage_raw":{"input_tokens":941,"input_tokens_details":{"cached_tokens":3697,"cache_write_tokens":0},"output_tokens":4044,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4985}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":20.189157583983615,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21188,"response_bytes":10838,"usage":{"input":871,"output":2595,"cached":4593,"total":3466,"consistent":false},"usage_raw":{"input_tokens":871,"input_tokens_details":{"cached_tokens":4593,"cache_write_tokens":0},"output_tokens":2595,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3466}}]} +{"role":"baseline","task_id":"quiz.bug","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.0158086660085246,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8455,"response_bytes":6628,"usage":{"input":2225,"output":107,"cached":0,"total":2332,"consistent":true},"usage_raw":{"input_tokens":2225,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":107,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2332}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.821267000050284,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15973,"response_bytes":7630,"usage":{"input":1892,"output":787,"cached":2161,"total":2679,"consistent":false},"usage_raw":{"input_tokens":1892,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":787,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2679}}]} +{"role":"baseline","task_id":"quiz.bug","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.3361459999578074,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8455,"response_bytes":6664,"usage":{"input":64,"output":114,"cached":2161,"total":178,"consistent":false},"usage_raw":{"input_tokens":64,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":114,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":178}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.8539059170288965,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15991,"response_bytes":7004,"usage":{"input":4056,"output":743,"cached":0,"total":4799,"consistent":true},"usage_raw":{"input_tokens":4056,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":743,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4799}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":12.346493041026406,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17036,"response_bytes":7540,"usage":{"input":364,"output":1607,"cached":3953,"total":1971,"consistent":false},"usage_raw":{"input_tokens":364,"input_tokens_details":{"cached_tokens":3953,"cache_write_tokens":0},"output_tokens":1607,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1971}}]} +{"role":"baseline","task_id":"quiz.bug","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.10052183305379,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8455,"response_bytes":6630,"usage":{"input":64,"output":115,"cached":2161,"total":179,"consistent":false},"usage_raw":{"input_tokens":64,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":115,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":179}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.680324084009044,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":16172,"response_bytes":6757,"usage":{"input":1935,"output":337,"cached":2161,"total":2272,"consistent":false},"usage_raw":{"input_tokens":1935,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":337,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2272}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":9.61975537496619,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":16962,"response_bytes":7496,"usage":{"input":217,"output":1140,"cached":4081,"total":1357,"consistent":false},"usage_raw":{"input_tokens":217,"input_tokens_details":{"cached_tokens":4081,"cache_write_tokens":0},"output_tokens":1140,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1357}}]} +{"role":"baseline","task_id":"quiz.create","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.506881167064421,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8316,"response_bytes":6796,"usage":{"input":2197,"output":148,"cached":0,"total":2345,"consistent":true},"usage_raw":{"input_tokens":2197,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":148,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2345}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":61.58719300001394,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14917,"response_bytes":11477,"usage":{"input":3964,"output":7110,"cached":0,"total":11074,"consistent":true},"usage_raw":{"input_tokens":3964,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":7110,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":11074}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":40.783467290923,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":20467,"response_bytes":10538,"usage":{"input":1441,"output":4822,"cached":3953,"total":6263,"consistent":false},"usage_raw":{"input_tokens":1441,"input_tokens_details":{"cached_tokens":3953,"cache_write_tokens":0},"output_tokens":4822,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6263}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":38.13290766708087,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24998,"response_bytes":13037,"usage":{"input":6659,"output":4388,"cached":0,"total":11047,"consistent":true},"usage_raw":{"input_tokens":6659,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":4388,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":11047}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":19.346264417050406,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":32790,"response_bytes":6694,"usage":{"input":3073,"output":1711,"cached":5361,"total":4784,"consistent":false},"usage_raw":{"input_tokens":3073,"input_tokens_details":{"cached_tokens":5361,"cache_write_tokens":0},"output_tokens":1711,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4784}},{"index":5,"complete":true,"status":200,"endpoint":"/responses","duration_s":30.201864000060596,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":33489,"response_bytes":6951,"usage":{"input":1966,"output":2892,"cached":6641,"total":4858,"consistent":false},"usage_raw":{"input_tokens":1966,"input_tokens_details":{"cached_tokens":6641,"cache_write_tokens":0},"output_tokens":2892,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4858}}]} +{"role":"baseline","task_id":"quiz.create","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.3806182079715654,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8316,"response_bytes":6822,"usage":{"input":36,"output":151,"cached":2161,"total":187,"consistent":false},"usage_raw":{"input_tokens":36,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":151,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":187}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":49.95889391703531,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14930,"response_bytes":7027,"usage":{"input":1806,"output":5736,"cached":2161,"total":7542,"consistent":false},"usage_raw":{"input_tokens":1806,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":5736,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7542}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":39.633181084063835,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15902,"response_bytes":10206,"usage":{"input":259,"output":4549,"cached":3953,"total":4808,"consistent":false},"usage_raw":{"input_tokens":259,"input_tokens_details":{"cached_tokens":3953,"cache_write_tokens":0},"output_tokens":4549,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4808}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":25.352803249959834,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":20109,"response_bytes":12796,"usage":{"input":1165,"output":3232,"cached":4209,"total":4397,"consistent":false},"usage_raw":{"input_tokens":1165,"input_tokens_details":{"cached_tokens":4209,"cache_write_tokens":0},"output_tokens":3232,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4397}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":28.59040637500584,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":27624,"response_bytes":6809,"usage":{"input":1681,"output":2921,"cached":5361,"total":4602,"consistent":false},"usage_raw":{"input_tokens":1681,"input_tokens_details":{"cached_tokens":5361,"cache_write_tokens":0},"output_tokens":2921,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4602}},{"index":5,"complete":true,"status":200,"endpoint":"/responses","duration_s":20.579641124932095,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":28403,"response_bytes":7016,"usage":{"input":5082,"output":1978,"cached":2161,"total":7060,"consistent":true},"usage_raw":{"input_tokens":5082,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":1978,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7060}},{"index":6,"complete":true,"status":200,"endpoint":"/responses","duration_s":11.461088542011566,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":29399,"response_bytes":6799,"usage":{"input":339,"output":976,"cached":7153,"total":1315,"consistent":false},"usage_raw":{"input_tokens":339,"input_tokens_details":{"cached_tokens":7153,"cache_write_tokens":0},"output_tokens":976,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1315}},{"index":7,"complete":true,"status":200,"endpoint":"/responses","duration_s":12.593177416943945,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":30179,"response_bytes":8867,"usage":{"input":672,"output":991,"cached":7025,"total":1663,"consistent":false},"usage_raw":{"input_tokens":672,"input_tokens_details":{"cached_tokens":7025,"cache_write_tokens":0},"output_tokens":991,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1663}}]} +{"role":"baseline","task_id":"quiz.create","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.1901169589255005,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8316,"response_bytes":6812,"usage":{"input":36,"output":149,"cached":2161,"total":185,"consistent":false},"usage_raw":{"input_tokens":36,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":149,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":185}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":18.21906408306677,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14925,"response_bytes":6618,"usage":{"input":1805,"output":727,"cached":2161,"total":2532,"consistent":false},"usage_raw":{"input_tokens":1805,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":727,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2532}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":31.936896749888547,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":16482,"response_bytes":6520,"usage":{"input":443,"output":3321,"cached":3953,"total":3764,"consistent":false},"usage_raw":{"input_tokens":443,"input_tokens_details":{"cached_tokens":3953,"cache_write_tokens":0},"output_tokens":3321,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3764}},{"index":3,"complete":true,"status":524,"endpoint":"/responses","duration_s":125.35409224999603,"upstream_status":524,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":null,"request_bytes":17852,"response_bytes":16,"usage":null,"usage_raw":null}]} +{"role":"baseline","task_id":"quiz.edit","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.2423027500044554,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8474,"response_bytes":6609,"usage":{"input":74,"output":129,"cached":2161,"total":203,"consistent":false},"usage_raw":{"input_tokens":74,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":129,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":203}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.106178208021447,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15763,"response_bytes":6568,"usage":{"input":1924,"output":361,"cached":2161,"total":2285,"consistent":false},"usage_raw":{"input_tokens":1924,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":361,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2285}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":9.357727125054225,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17198,"response_bytes":6839,"usage":{"input":2354,"output":881,"cached":2161,"total":3235,"consistent":true},"usage_raw":{"input_tokens":2354,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":881,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3235}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":19.960298707941547,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18312,"response_bytes":8924,"usage":{"input":699,"output":2306,"cached":4081,"total":3005,"consistent":false},"usage_raw":{"input_tokens":699,"input_tokens_details":{"cached_tokens":4081,"cache_write_tokens":0},"output_tokens":2306,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3005}}]} +{"role":"baseline","task_id":"quiz.edit","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.744099666015245,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8474,"response_bytes":6610,"usage":{"input":586,"output":102,"cached":1649,"total":688,"consistent":false},"usage_raw":{"input_tokens":586,"input_tokens_details":{"cached_tokens":1649,"cache_write_tokens":0},"output_tokens":102,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":688}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.241149083944038,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15763,"response_bytes":6554,"usage":{"input":1923,"output":496,"cached":2161,"total":2419,"consistent":false},"usage_raw":{"input_tokens":1923,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":496,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2419}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":7.073009499930777,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17191,"response_bytes":6867,"usage":{"input":4513,"output":760,"cached":0,"total":5273,"consistent":true},"usage_raw":{"input_tokens":4513,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":760,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5273}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":13.260727875051089,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18370,"response_bytes":8849,"usage":{"input":325,"output":2191,"cached":4465,"total":2516,"consistent":false},"usage_raw":{"input_tokens":325,"input_tokens_details":{"cached_tokens":4465,"cache_write_tokens":0},"output_tokens":2191,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2516}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":20.092903167009354,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":27319,"response_bytes":6508,"usage":{"input":1833,"output":1620,"cached":4721,"total":3453,"consistent":false},"usage_raw":{"input_tokens":1833,"input_tokens_details":{"cached_tokens":4721,"cache_write_tokens":0},"output_tokens":1620,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3453}},{"index":5,"complete":true,"status":200,"endpoint":"/responses","duration_s":17.186724166967906,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":30334,"response_bytes":8873,"usage":{"input":684,"output":2333,"cached":6513,"total":3017,"consistent":false},"usage_raw":{"input_tokens":684,"input_tokens_details":{"cached_tokens":6513,"cache_write_tokens":0},"output_tokens":2333,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3017}}]} +{"role":"baseline","task_id":"quiz.edit","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.6046008750563487,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8474,"response_bytes":6612,"usage":{"input":74,"output":102,"cached":2161,"total":176,"consistent":false},"usage_raw":{"input_tokens":74,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":102,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":176}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.485765708959661,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":16034,"response_bytes":6907,"usage":{"input":2087,"output":678,"cached":2161,"total":2765,"consistent":false},"usage_raw":{"input_tokens":2087,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":678,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2765}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":11.784762291936204,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":17354,"response_bytes":8911,"usage":{"input":2389,"output":1736,"cached":2161,"total":4125,"consistent":true},"usage_raw":{"input_tokens":2389,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":1736,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4125}}]} +{"role":"baseline","task_id":"quiz.feature","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.316778791020624,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8837,"response_bytes":6770,"usage":{"input":2286,"output":133,"cached":0,"total":2419,"consistent":true},"usage_raw":{"input_tokens":2286,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":133,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2419}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":41.870638708001934,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18976,"response_bytes":8507,"usage":{"input":2687,"output":4615,"cached":2161,"total":7302,"consistent":true},"usage_raw":{"input_tokens":2687,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":4615,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7302}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":23.05169058300089,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21483,"response_bytes":10567,"usage":{"input":737,"output":3311,"cached":4721,"total":4048,"consistent":false},"usage_raw":{"input_tokens":737,"input_tokens_details":{"cached_tokens":4721,"cache_write_tokens":0},"output_tokens":3311,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4048}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":18.71830679103732,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":27288,"response_bytes":7424,"usage":{"input":6740,"output":1674,"cached":0,"total":8414,"consistent":true},"usage_raw":{"input_tokens":6740,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":1674,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":8414}}]} +{"role":"baseline","task_id":"quiz.feature","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.451374207972549,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":8837,"response_bytes":6752,"usage":{"input":125,"output":129,"cached":2161,"total":254,"consistent":false},"usage_raw":{"input_tokens":125,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":129,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":254}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":73.2982571250759,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":18499,"response_bytes":7403,"usage":{"input":2477,"output":8225,"cached":2289,"total":10702,"consistent":true},"usage_raw":{"input_tokens":2477,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":8225,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":10702}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":45.502383750048466,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19862,"response_bytes":11330,"usage":{"input":2796,"output":7058,"cached":2289,"total":9854,"consistent":true},"usage_raw":{"input_tokens":2796,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":7058,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":9854}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":31.603159709018655,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":26450,"response_bytes":7395,"usage":{"input":1572,"output":3159,"cached":4977,"total":4731,"consistent":false},"usage_raw":{"input_tokens":1572,"input_tokens_details":{"cached_tokens":4977,"cache_write_tokens":0},"output_tokens":3159,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4731}}]} +{"role":"baseline","task_id":"quiz.feature","trial":3,"calls":[{"index":0,"complete":true,"status":503,"endpoint":"/responses","duration_s":57.109958291985095,"upstream_status":503,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":null,"request_bytes":8837,"response_bytes":153,"usage":null,"usage_raw":null}]} +{"role":"candidate","task_id":"application.bug","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8775758750271052,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11482,"response_bytes":1115,"usage":{"input":2638,"output":302,"cached":0,"total":2940,"consistent":true},"usage_raw":{"input_tokens":2638,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.03039833297953,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":12827,"response_bytes":10377,"usage":{"input":52,"output":125,"cached":2929,"total":177,"consistent":false},"usage_raw":{"input_tokens":52,"input_tokens_details":{"cached_tokens":2929,"cache_write_tokens":0},"output_tokens":125,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":177}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.803337416960858,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24069,"response_bytes":10436,"usage":{"input":2853,"output":547,"cached":2929,"total":3400,"consistent":false},"usage_raw":{"input_tokens":2853,"input_tokens_details":{"cached_tokens":2929,"cache_write_tokens":0},"output_tokens":547,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3400}}]} +{"role":"candidate","task_id":"application.bug","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.9386614169925451,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11482,"response_bytes":1109,"usage":{"input":2638,"output":302,"cached":0,"total":2940,"consistent":true},"usage_raw":{"input_tokens":2638,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.04433866601903,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":12827,"response_bytes":10284,"usage":{"input":692,"output":111,"cached":2289,"total":803,"consistent":false},"usage_raw":{"input_tokens":692,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":111,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":803}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":7.26659087499138,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21648,"response_bytes":10315,"usage":{"input":2802,"output":624,"cached":2289,"total":3426,"consistent":true},"usage_raw":{"input_tokens":2802,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":624,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3426}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":12.690880291978829,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":25087,"response_bytes":10421,"usage":{"input":3120,"output":1144,"cached":2929,"total":4264,"consistent":true},"usage_raw":{"input_tokens":3120,"input_tokens_details":{"cached_tokens":2929,"cache_write_tokens":0},"output_tokens":1144,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4264}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":5.783798917080276,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":26419,"response_bytes":10147,"usage":{"input":400,"output":78,"cached":6001,"total":478,"consistent":false},"usage_raw":{"input_tokens":400,"input_tokens_details":{"cached_tokens":6001,"cache_write_tokens":0},"output_tokens":78,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":478}},{"index":5,"complete":true,"status":200,"endpoint":"/responses","duration_s":8.3906414170051,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":30161,"response_bytes":10700,"usage":{"input":1013,"output":739,"cached":6257,"total":1752,"consistent":false},"usage_raw":{"input_tokens":1013,"input_tokens_details":{"cached_tokens":6257,"cache_write_tokens":0},"output_tokens":739,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1752}}]} +{"role":"candidate","task_id":"application.bug","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.4274132079444826,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11482,"response_bytes":1115,"usage":{"input":2638,"output":302,"cached":0,"total":2940,"consistent":true},"usage_raw":{"input_tokens":2638,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":1.8578229160048068,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":12827,"response_bytes":10354,"usage":{"input":2981,"output":120,"cached":0,"total":3101,"consistent":true},"usage_raw":{"input_tokens":2981,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":120,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3101}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":8.273425417020917,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21928,"response_bytes":10473,"usage":{"input":2220,"output":754,"cached":2929,"total":2974,"consistent":false},"usage_raw":{"input_tokens":2220,"input_tokens_details":{"cached_tokens":2929,"cache_write_tokens":0},"output_tokens":754,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2974}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":1.7825294160284102,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23295,"response_bytes":10150,"usage":{"input":5510,"output":79,"cached":0,"total":5589,"consistent":true},"usage_raw":{"input_tokens":5510,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":79,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5589}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.873644166975282,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":27038,"response_bytes":10979,"usage":{"input":1403,"output":348,"cached":4977,"total":1751,"consistent":false},"usage_raw":{"input_tokens":1403,"input_tokens_details":{"cached_tokens":4977,"cache_write_tokens":0},"output_tokens":348,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1751}}]} +{"role":"candidate","task_id":"application.create","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.5190132909920067,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11335,"response_bytes":1109,"usage":{"input":2613,"output":302,"cached":0,"total":2915,"consistent":true},"usage_raw":{"input_tokens":2613,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":46.99805120797828,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21032,"response_bytes":11595,"usage":{"input":5209,"output":5523,"cached":0,"total":10732,"consistent":true},"usage_raw":{"input_tokens":5209,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":5523,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":10732}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":34.09436616592575,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23415,"response_bytes":23560,"usage":{"input":692,"output":5645,"cached":5105,"total":6337,"consistent":false},"usage_raw":{"input_tokens":692,"input_tokens_details":{"cached_tokens":5105,"cache_write_tokens":0},"output_tokens":5645,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6337}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":43.52131420804653,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":39014,"response_bytes":21834,"usage":{"input":4402,"output":5854,"cached":5105,"total":10256,"consistent":false},"usage_raw":{"input_tokens":4402,"input_tokens_details":{"cached_tokens":5105,"cache_write_tokens":0},"output_tokens":5854,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":10256}}]} +{"role":"candidate","task_id":"application.create","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.971888417028822,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11335,"response_bytes":1106,"usage":{"input":2613,"output":302,"cached":0,"total":2915,"consistent":true},"usage_raw":{"input_tokens":2613,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":107.6471907501109,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21032,"response_bytes":25185,"usage":{"input":104,"output":14990,"cached":5105,"total":15094,"consistent":false},"usage_raw":{"input_tokens":104,"input_tokens_details":{"cached_tokens":5105,"cache_write_tokens":0},"output_tokens":14990,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":15094}}]} +{"role":"candidate","task_id":"application.create","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.5911080840742216,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11335,"response_bytes":1109,"usage":{"input":2613,"output":302,"cached":0,"total":2915,"consistent":true},"usage_raw":{"input_tokens":2613,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":72.38674841693137,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21032,"response_bytes":15737,"usage":{"input":5209,"output":8566,"cached":0,"total":13775,"consistent":true},"usage_raw":{"input_tokens":5209,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":8566,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":13775}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":41.26221416704357,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":27640,"response_bytes":18325,"usage":{"input":1806,"output":5218,"cached":5105,"total":7024,"consistent":false},"usage_raw":{"input_tokens":1806,"input_tokens_details":{"cached_tokens":5105,"cache_write_tokens":0},"output_tokens":5218,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7024}}]} +{"role":"candidate","task_id":"application.edit","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.531210832996294,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11493,"response_bytes":1110,"usage":{"input":2649,"output":303,"cached":0,"total":2952,"consistent":true},"usage_raw":{"input_tokens":2649,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":9.002368167042732,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14985,"response_bytes":6681,"usage":{"input":1781,"output":943,"cached":2161,"total":2724,"consistent":false},"usage_raw":{"input_tokens":1781,"input_tokens_details":{"cached_tokens":2161,"cache_write_tokens":0},"output_tokens":943,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2724}}]} +{"role":"candidate","task_id":"application.edit","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8601069579599425,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11493,"response_bytes":1113,"usage":{"input":2649,"output":303,"cached":0,"total":2952,"consistent":true},"usage_raw":{"input_tokens":2649,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":14.467825458035804,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14985,"response_bytes":6645,"usage":{"input":117,"output":1641,"cached":3825,"total":1758,"consistent":false},"usage_raw":{"input_tokens":117,"input_tokens_details":{"cached_tokens":3825,"cache_write_tokens":0},"output_tokens":1641,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1758}}]} +{"role":"candidate","task_id":"application.edit","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8373663329984993,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11493,"response_bytes":1111,"usage":{"input":2649,"output":303,"cached":0,"total":2952,"consistent":true},"usage_raw":{"input_tokens":2649,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.6969339590286836,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":13310,"response_bytes":6276,"usage":{"input":3388,"output":315,"cached":0,"total":3703,"consistent":true},"usage_raw":{"input_tokens":3388,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":315,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3703}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":7.546911832992919,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":15325,"response_bytes":6708,"usage":{"input":4028,"output":690,"cached":0,"total":4718,"consistent":true},"usage_raw":{"input_tokens":4028,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":690,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4718}}]} +{"role":"candidate","task_id":"application.feature","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.425622042035684,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11829,"response_bytes":1112,"usage":{"input":2721,"output":302,"cached":0,"total":3023,"consistent":true},"usage_raw":{"input_tokens":2721,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":40.197538457927294,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":26709,"response_bytes":13623,"usage":{"input":4238,"output":6161,"cached":2289,"total":10399,"consistent":true},"usage_raw":{"input_tokens":4238,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":6161,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":10399}}]} +{"role":"candidate","task_id":"application.feature","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.5057388750137761,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11829,"response_bytes":1112,"usage":{"input":2721,"output":302,"cached":0,"total":3023,"consistent":true},"usage_raw":{"input_tokens":2721,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":28.368688000016846,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":26709,"response_bytes":17567,"usage":{"input":14,"output":3951,"cached":6513,"total":3965,"consistent":false},"usage_raw":{"input_tokens":14,"input_tokens_details":{"cached_tokens":6513,"cache_write_tokens":0},"output_tokens":3951,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3965}}]} +{"role":"candidate","task_id":"application.feature","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.5189679170725867,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11829,"response_bytes":1110,"usage":{"input":2721,"output":302,"cached":0,"total":3023,"consistent":true},"usage_raw":{"input_tokens":2721,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":36.0352498749271,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":26709,"response_bytes":14084,"usage":{"input":14,"output":6087,"cached":6513,"total":6101,"consistent":false},"usage_raw":{"input_tokens":14,"input_tokens_details":{"cached_tokens":6513,"cache_write_tokens":0},"output_tokens":6087,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6101}}]} +{"role":"candidate","task_id":"csv.bug","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.0547940420219675,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10970,"response_bytes":1111,"usage":{"input":2519,"output":302,"cached":0,"total":2821,"consistent":true},"usage_raw":{"input_tokens":2519,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.033323541982099,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23791,"response_bytes":10552,"usage":{"input":43,"output":400,"cached":5873,"total":443,"consistent":false},"usage_raw":{"input_tokens":43,"input_tokens_details":{"cached_tokens":5873,"cache_write_tokens":0},"output_tokens":400,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":443}}]} +{"role":"candidate","task_id":"csv.bug","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8747527080122381,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10970,"response_bytes":1111,"usage":{"input":2519,"output":302,"cached":0,"total":2821,"consistent":true},"usage_raw":{"input_tokens":2519,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.441596749937162,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23791,"response_bytes":10487,"usage":{"input":3627,"output":496,"cached":2289,"total":4123,"consistent":true},"usage_raw":{"input_tokens":3627,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":496,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4123}}]} +{"role":"candidate","task_id":"csv.bug","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.0284747499972582,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10970,"response_bytes":1113,"usage":{"input":2519,"output":302,"cached":0,"total":2821,"consistent":true},"usage_raw":{"input_tokens":2519,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":7.874042917042971,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23791,"response_bytes":10508,"usage":{"input":3627,"output":673,"cached":2289,"total":4300,"consistent":true},"usage_raw":{"input_tokens":3627,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":673,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4300}}]} +{"role":"candidate","task_id":"csv.create","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.3550079159904271,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10818,"response_bytes":1109,"usage":{"input":2492,"output":302,"cached":0,"total":2794,"consistent":true},"usage_raw":{"input_tokens":2492,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":53.56754662503954,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19998,"response_bytes":10707,"usage":{"input":154,"output":4764,"cached":4849,"total":4918,"consistent":false},"usage_raw":{"input_tokens":154,"input_tokens_details":{"cached_tokens":4849,"cache_write_tokens":0},"output_tokens":4764,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4918}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":53.9230706669623,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21449,"response_bytes":14567,"usage":{"input":5368,"output":6058,"cached":0,"total":11426,"consistent":true},"usage_raw":{"input_tokens":5368,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":6058,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":11426}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":20.088729250011966,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":26461,"response_bytes":14052,"usage":{"input":1884,"output":2635,"cached":4849,"total":4519,"consistent":false},"usage_raw":{"input_tokens":1884,"input_tokens_details":{"cached_tokens":4849,"cache_write_tokens":0},"output_tokens":2635,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4519}}]} +{"role":"candidate","task_id":"csv.create","trial":2,"calls":[{"index":0,"complete":true,"status":503,"endpoint":"/decisions","duration_s":10.446022582938895,"upstream_status":503,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":null,"request_bytes":10818,"response_bytes":153,"usage":null,"usage_raw":null},{"index":1,"complete":true,"status":503,"endpoint":"/responses","duration_s":11.629067708039656,"upstream_status":503,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":null,"request_bytes":12163,"response_bytes":153,"usage":null,"usage_raw":null},{"index":2,"complete":true,"status":500,"endpoint":"/responses","duration_s":60.208322833059356,"upstream_status":500,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":null,"request_bytes":12163,"response_bytes":170,"usage":null,"usage_raw":null}]} +{"role":"candidate","task_id":"csv.create","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.3419595829909667,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10818,"response_bytes":1107,"usage":{"input":2492,"output":302,"cached":0,"total":2794,"consistent":true},"usage_raw":{"input_tokens":2492,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":53.653479166910984,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19998,"response_bytes":10698,"usage":{"input":5003,"output":4371,"cached":0,"total":9374,"consistent":true},"usage_raw":{"input_tokens":5003,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":4371,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":9374}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":70.31544616597239,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21469,"response_bytes":19773,"usage":{"input":5383,"output":8835,"cached":0,"total":14218,"consistent":true},"usage_raw":{"input_tokens":5383,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":8835,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":14218}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":43.58899774996098,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":32381,"response_bytes":10533,"usage":{"input":3226,"output":3914,"cached":4849,"total":7140,"consistent":false},"usage_raw":{"input_tokens":3226,"input_tokens_details":{"cached_tokens":4849,"cache_write_tokens":0},"output_tokens":3914,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7140}}]} +{"role":"candidate","task_id":"csv.edit","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.2455356249120086,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10971,"response_bytes":1110,"usage":{"input":2527,"output":303,"cached":0,"total":2830,"consistent":true},"usage_raw":{"input_tokens":2527,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":17.99224383395631,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":13184,"response_bytes":6625,"usage":{"input":3523,"output":1885,"cached":0,"total":5408,"consistent":true},"usage_raw":{"input_tokens":3523,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":1885,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5408}}]} +{"role":"candidate","task_id":"csv.edit","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8002357500372455,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10971,"response_bytes":1114,"usage":{"input":2527,"output":303,"cached":0,"total":2830,"consistent":true},"usage_raw":{"input_tokens":2527,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":9.632705666008405,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":13184,"response_bytes":6642,"usage":{"input":82,"output":947,"cached":3441,"total":1029,"consistent":false},"usage_raw":{"input_tokens":82,"input_tokens_details":{"cached_tokens":3441,"cache_write_tokens":0},"output_tokens":947,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1029}}]} +{"role":"candidate","task_id":"csv.edit","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.3704882910242304,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10971,"response_bytes":1112,"usage":{"input":2527,"output":303,"cached":0,"total":2830,"consistent":true},"usage_raw":{"input_tokens":2527,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":8.715296749956906,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":13184,"response_bytes":6617,"usage":{"input":3523,"output":992,"cached":0,"total":4515,"consistent":true},"usage_raw":{"input_tokens":3523,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":992,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4515}}]} +{"role":"candidate","task_id":"csv.feature","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8598794160643592,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11268,"response_bytes":1113,"usage":{"input":2587,"output":302,"cached":0,"total":2889,"consistent":true},"usage_raw":{"input_tokens":2587,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":46.96248895803001,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24117,"response_bytes":13457,"usage":{"input":3694,"output":6020,"cached":2289,"total":9714,"consistent":true},"usage_raw":{"input_tokens":3694,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":6020,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":9714}}]} +{"role":"candidate","task_id":"csv.feature","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":2.6979112500557676,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11268,"response_bytes":1113,"usage":{"input":2587,"output":302,"cached":0,"total":2889,"consistent":true},"usage_raw":{"input_tokens":2587,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":36.84399629198015,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24117,"response_bytes":14101,"usage":{"input":5983,"output":4700,"cached":0,"total":10683,"consistent":true},"usage_raw":{"input_tokens":5983,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":4700,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":10683}}]} +{"role":"candidate","task_id":"csv.feature","trial":3,"calls":[{"index":0,"complete":true,"status":500,"endpoint":"/decisions","duration_s":60.1993414160097,"upstream_status":500,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":null,"request_bytes":11268,"response_bytes":170,"usage":null,"usage_raw":null},{"index":1,"complete":true,"status":500,"endpoint":"/responses","duration_s":60.14810908399522,"upstream_status":500,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":null,"request_bytes":12613,"response_bytes":170,"usage":null,"usage_raw":null}]} +{"role":"candidate","task_id":"inventory.bug","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8290331249590963,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10963,"response_bytes":1109,"usage":{"input":2535,"output":302,"cached":0,"total":2837,"consistent":true},"usage_raw":{"input_tokens":2535,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":23.6502559579676,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23898,"response_bytes":11171,"usage":{"input":42,"output":2319,"cached":5873,"total":2361,"consistent":false},"usage_raw":{"input_tokens":42,"input_tokens_details":{"cached_tokens":5873,"cache_write_tokens":0},"output_tokens":2319,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2361}}]} +{"role":"candidate","task_id":"inventory.bug","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.238432832993567,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10963,"response_bytes":1113,"usage":{"input":2535,"output":302,"cached":0,"total":2837,"consistent":true},"usage_raw":{"input_tokens":2535,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":14.221318625030108,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23898,"response_bytes":11226,"usage":{"input":5915,"output":1472,"cached":0,"total":7387,"consistent":true},"usage_raw":{"input_tokens":5915,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":1472,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7387}}]} +{"role":"candidate","task_id":"inventory.bug","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.39168304100167,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10963,"response_bytes":1111,"usage":{"input":2535,"output":302,"cached":0,"total":2837,"consistent":true},"usage_raw":{"input_tokens":2535,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":17.19266099995002,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23898,"response_bytes":11180,"usage":{"input":42,"output":1746,"cached":5873,"total":1788,"consistent":false},"usage_raw":{"input_tokens":42,"input_tokens_details":{"cached_tokens":5873,"cache_write_tokens":0},"output_tokens":1746,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1788}}]} +{"role":"candidate","task_id":"inventory.create","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.9171701659215614,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10782,"response_bytes":1108,"usage":{"input":2499,"output":302,"cached":0,"total":2801,"consistent":true},"usage_raw":{"input_tokens":2499,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":43.113897041999735,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19926,"response_bytes":19895,"usage":{"input":4991,"output":5742,"cached":0,"total":10733,"consistent":true},"usage_raw":{"input_tokens":4991,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":5742,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":10733}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":9.318022790946998,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":31390,"response_bytes":19792,"usage":{"input":2910,"output":2718,"cached":4849,"total":5628,"consistent":false},"usage_raw":{"input_tokens":2910,"input_tokens_details":{"cached_tokens":4849,"cache_write_tokens":0},"output_tokens":2718,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5628}}]} +{"role":"candidate","task_id":"inventory.create","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.0128026659367606,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10782,"response_bytes":1111,"usage":{"input":2499,"output":302,"cached":0,"total":2801,"consistent":true},"usage_raw":{"input_tokens":2499,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":59.57377212494612,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19926,"response_bytes":10752,"usage":{"input":14,"output":5393,"cached":4977,"total":5407,"consistent":false},"usage_raw":{"input_tokens":14,"input_tokens_details":{"cached_tokens":4977,"cache_write_tokens":0},"output_tokens":5393,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5407}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":55.61870629200712,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":21462,"response_bytes":10233,"usage":{"input":5389,"output":4079,"cached":0,"total":9468,"consistent":true},"usage_raw":{"input_tokens":5389,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":4079,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":9468}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":40.64372066606302,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":22473,"response_bytes":19528,"usage":{"input":402,"output":5411,"cached":5233,"total":5813,"consistent":false},"usage_raw":{"input_tokens":402,"input_tokens_details":{"cached_tokens":5233,"cache_write_tokens":0},"output_tokens":5411,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5813}}]} +{"role":"candidate","task_id":"inventory.create","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.6492161670466885,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10782,"response_bytes":1106,"usage":{"input":2499,"output":302,"cached":0,"total":2801,"consistent":true},"usage_raw":{"input_tokens":2499,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":42.48584600002505,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19926,"response_bytes":14671,"usage":{"input":14,"output":5039,"cached":4977,"total":5053,"consistent":false},"usage_raw":{"input_tokens":14,"input_tokens_details":{"cached_tokens":4977,"cache_write_tokens":0},"output_tokens":5039,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5053}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":23.242594249895774,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":25448,"response_bytes":16068,"usage":{"input":1635,"output":3021,"cached":4849,"total":4656,"consistent":false},"usage_raw":{"input_tokens":1635,"input_tokens_details":{"cached_tokens":4849,"cache_write_tokens":0},"output_tokens":3021,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4656}}]} +{"role":"candidate","task_id":"inventory.edit","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.427395167062059,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10930,"response_bytes":1113,"usage":{"input":2534,"output":303,"cached":0,"total":2837,"consistent":true},"usage_raw":{"input_tokens":2534,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":22.167674458003603,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":13455,"response_bytes":6678,"usage":{"input":3580,"output":2187,"cached":0,"total":5767,"consistent":true},"usage_raw":{"input_tokens":3580,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":2187,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5767}}]} +{"role":"candidate","task_id":"inventory.edit","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.827856709016487,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10930,"response_bytes":1116,"usage":{"input":2534,"output":303,"cached":0,"total":2837,"consistent":true},"usage_raw":{"input_tokens":2534,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":10.047854749951512,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":13455,"response_bytes":6610,"usage":{"input":3580,"output":1083,"cached":0,"total":4663,"consistent":true},"usage_raw":{"input_tokens":3580,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":1083,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4663}}]} +{"role":"candidate","task_id":"inventory.edit","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.01075725001283,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10930,"response_bytes":1114,"usage":{"input":2534,"output":303,"cached":0,"total":2837,"consistent":true},"usage_raw":{"input_tokens":2534,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":7.301086417050101,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":13455,"response_bytes":6662,"usage":{"input":11,"output":644,"cached":3569,"total":655,"consistent":false},"usage_raw":{"input_tokens":11,"input_tokens_details":{"cached_tokens":3569,"cache_write_tokens":0},"output_tokens":644,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":655}}]} +{"role":"candidate","task_id":"inventory.feature","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.913082042010501,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11192,"response_bytes":1110,"usage":{"input":2584,"output":302,"cached":0,"total":2886,"consistent":true},"usage_raw":{"input_tokens":2584,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":45.40003991697449,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24115,"response_bytes":14745,"usage":{"input":86,"output":4692,"cached":5873,"total":4778,"consistent":false},"usage_raw":{"input_tokens":86,"input_tokens_details":{"cached_tokens":5873,"cache_write_tokens":0},"output_tokens":4692,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4778}}]} +{"role":"candidate","task_id":"inventory.feature","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":2.2662433329969645,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11192,"response_bytes":1113,"usage":{"input":2584,"output":302,"cached":0,"total":2886,"consistent":true},"usage_raw":{"input_tokens":2584,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":57.03180387499742,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24115,"response_bytes":13248,"usage":{"input":5959,"output":7772,"cached":0,"total":13731,"consistent":true},"usage_raw":{"input_tokens":5959,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":7772,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":13731}}]} +{"role":"candidate","task_id":"inventory.feature","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.3550680830376223,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11192,"response_bytes":1111,"usage":{"input":2584,"output":302,"cached":0,"total":2886,"consistent":true},"usage_raw":{"input_tokens":2584,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":48.86727787507698,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24115,"response_bytes":14961,"usage":{"input":86,"output":4699,"cached":5873,"total":4785,"consistent":false},"usage_raw":{"input_tokens":86,"input_tokens_details":{"cached_tokens":5873,"cache_write_tokens":0},"output_tokens":4699,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4785}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.171667999937199,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":31207,"response_bytes":10145,"usage":{"input":5272,"output":92,"cached":2289,"total":5364,"consistent":true},"usage_raw":{"input_tokens":5272,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":92,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5364}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":12.919106959016062,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":34714,"response_bytes":10161,"usage":{"input":2458,"output":1187,"cached":5873,"total":3645,"consistent":false},"usage_raw":{"input_tokens":2458,"input_tokens_details":{"cached_tokens":5873,"cache_write_tokens":0},"output_tokens":1187,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3645}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":17.549795874976553,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":36751,"response_bytes":11382,"usage":{"input":805,"output":1775,"cached":8177,"total":2580,"consistent":false},"usage_raw":{"input_tokens":805,"input_tokens_details":{"cached_tokens":8177,"cache_write_tokens":0},"output_tokens":1775,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2580}}]} +{"role":"candidate","task_id":"notes.bug","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8766952500445768,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10958,"response_bytes":1113,"usage":{"input":2520,"output":302,"cached":0,"total":2822,"consistent":true},"usage_raw":{"input_tokens":2520,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":21.016714582918212,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23247,"response_bytes":10507,"usage":{"input":5800,"output":1634,"cached":0,"total":7434,"consistent":true},"usage_raw":{"input_tokens":5800,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":1634,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7434}}]} +{"role":"candidate","task_id":"notes.bug","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8661050419323146,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10958,"response_bytes":1113,"usage":{"input":2520,"output":302,"cached":0,"total":2822,"consistent":true},"usage_raw":{"input_tokens":2520,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":14.736077042063698,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23247,"response_bytes":10835,"usage":{"input":55,"output":1673,"cached":5745,"total":1728,"consistent":false},"usage_raw":{"input_tokens":55,"input_tokens_details":{"cached_tokens":5745,"cache_write_tokens":0},"output_tokens":1673,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1728}}]} +{"role":"candidate","task_id":"notes.bug","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.5340035000117496,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10958,"response_bytes":1108,"usage":{"input":2520,"output":302,"cached":0,"total":2822,"consistent":true},"usage_raw":{"input_tokens":2520,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":8.782469250028953,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23247,"response_bytes":10542,"usage":{"input":5800,"output":900,"cached":0,"total":6700,"consistent":true},"usage_raw":{"input_tokens":5800,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":900,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6700}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.317055125022307,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24910,"response_bytes":10161,"usage":{"input":496,"output":83,"cached":5745,"total":579,"consistent":false},"usage_raw":{"input_tokens":496,"input_tokens_details":{"cached_tokens":5745,"cache_write_tokens":0},"output_tokens":83,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":579}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.174531124997884,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":27004,"response_bytes":10511,"usage":{"input":1032,"output":293,"cached":5745,"total":1325,"consistent":false},"usage_raw":{"input_tokens":1032,"input_tokens_details":{"cached_tokens":5745,"cache_write_tokens":0},"output_tokens":293,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1325}}]} +{"role":"candidate","task_id":"notes.create","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.4193647910142317,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10784,"response_bytes":1106,"usage":{"input":2488,"output":302,"cached":0,"total":2790,"consistent":true},"usage_raw":{"input_tokens":2488,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":43.17564712499734,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19930,"response_bytes":20072,"usage":{"input":16,"output":5959,"cached":4977,"total":5975,"consistent":false},"usage_raw":{"input_tokens":16,"input_tokens_details":{"cached_tokens":4977,"cache_write_tokens":0},"output_tokens":5959,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5975}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":10.663197749992833,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":31262,"response_bytes":17440,"usage":{"input":3127,"output":2536,"cached":4849,"total":5663,"consistent":false},"usage_raw":{"input_tokens":3127,"input_tokens_details":{"cached_tokens":4849,"cache_write_tokens":0},"output_tokens":2536,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5663}}]} +{"role":"candidate","task_id":"notes.create","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.3961579579627141,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10784,"response_bytes":1110,"usage":{"input":2488,"output":302,"cached":0,"total":2790,"consistent":true},"usage_raw":{"input_tokens":2488,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":69.5190698329825,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19930,"response_bytes":19316,"usage":{"input":4993,"output":10406,"cached":0,"total":15399,"consistent":true},"usage_raw":{"input_tokens":4993,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":10406,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":15399}}]} +{"role":"candidate","task_id":"notes.create","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":2.355416417005472,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10784,"response_bytes":1107,"usage":{"input":2488,"output":302,"cached":0,"total":2790,"consistent":true},"usage_raw":{"input_tokens":2488,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":49.16647979198024,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":19930,"response_bytes":17570,"usage":{"input":16,"output":6103,"cached":4977,"total":6119,"consistent":false},"usage_raw":{"input_tokens":16,"input_tokens_details":{"cached_tokens":4977,"cache_write_tokens":0},"output_tokens":6103,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6119}}]} +{"role":"candidate","task_id":"notes.edit","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8716071670642123,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10934,"response_bytes":1108,"usage":{"input":2523,"output":303,"cached":0,"total":2826,"consistent":true},"usage_raw":{"input_tokens":2523,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":9.841228249948472,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":12965,"response_bytes":6633,"usage":{"input":3511,"output":1022,"cached":0,"total":4533,"consistent":true},"usage_raw":{"input_tokens":3511,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":1022,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4533}}]} +{"role":"candidate","task_id":"notes.edit","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.462756042019464,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10934,"response_bytes":1112,"usage":{"input":2523,"output":303,"cached":0,"total":2826,"consistent":true},"usage_raw":{"input_tokens":2523,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":7.9333320839796215,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":12965,"response_bytes":6712,"usage":{"input":70,"output":881,"cached":3441,"total":951,"consistent":false},"usage_raw":{"input_tokens":70,"input_tokens_details":{"cached_tokens":3441,"cache_write_tokens":0},"output_tokens":881,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":951}}]} +{"role":"candidate","task_id":"notes.edit","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8209806251106784,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10934,"response_bytes":1113,"usage":{"input":2523,"output":303,"cached":0,"total":2826,"consistent":true},"usage_raw":{"input_tokens":2523,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":12.462591457995586,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":12965,"response_bytes":6621,"usage":{"input":3511,"output":1034,"cached":0,"total":4545,"consistent":true},"usage_raw":{"input_tokens":3511,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":1034,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4545}}]} +{"role":"candidate","task_id":"notes.feature","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.3577352080028504,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11345,"response_bytes":1110,"usage":{"input":2593,"output":302,"cached":0,"total":2895,"consistent":true},"usage_raw":{"input_tokens":2593,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":41.21185349998996,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23640,"response_bytes":15780,"usage":{"input":5875,"output":6195,"cached":0,"total":12070,"consistent":true},"usage_raw":{"input_tokens":5875,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":6195,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":12070}}]} +{"role":"candidate","task_id":"notes.feature","trial":2,"calls":[{"index":0,"complete":true,"status":500,"endpoint":"/decisions","duration_s":60.14615629205946,"upstream_status":500,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":null,"request_bytes":11345,"response_bytes":170,"usage":null,"usage_raw":null},{"index":1,"complete":true,"status":500,"endpoint":"/responses","duration_s":60.224620084045455,"upstream_status":500,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":null,"request_bytes":12690,"response_bytes":170,"usage":null,"usage_raw":null}]} +{"role":"candidate","task_id":"notes.feature","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.255110041005537,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11345,"response_bytes":1108,"usage":{"input":2593,"output":302,"cached":0,"total":2895,"consistent":true},"usage_raw":{"input_tokens":2593,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":17.542540916940197,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":23640,"response_bytes":10178,"usage":{"input":5875,"output":1699,"cached":0,"total":7574,"consistent":true},"usage_raw":{"input_tokens":5875,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":1699,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7574}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":40.86900233302731,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":25132,"response_bytes":12971,"usage":{"input":520,"output":4891,"cached":5745,"total":5411,"consistent":false},"usage_raw":{"input_tokens":520,"input_tokens_details":{"cached_tokens":5745,"cache_write_tokens":0},"output_tokens":4891,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5411}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":46.36288454104215,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":28972,"response_bytes":13670,"usage":{"input":1135,"output":6880,"cached":6129,"total":8015,"consistent":false},"usage_raw":{"input_tokens":1135,"input_tokens_details":{"cached_tokens":6129,"cache_write_tokens":0},"output_tokens":6880,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":8015}}]} +{"role":"candidate","task_id":"quiz.bug","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8870314999949187,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11005,"response_bytes":1115,"usage":{"input":2549,"output":302,"cached":0,"total":2851,"consistent":true},"usage_raw":{"input_tokens":2549,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":10.207126833032817,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24617,"response_bytes":11040,"usage":{"input":3566,"output":1106,"cached":2545,"total":4672,"consistent":true},"usage_raw":{"input_tokens":3566,"input_tokens_details":{"cached_tokens":2545,"cache_write_tokens":0},"output_tokens":1106,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4672}}]} +{"role":"candidate","task_id":"quiz.bug","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.4462660000426695,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11005,"response_bytes":1115,"usage":{"input":2549,"output":302,"cached":0,"total":2851,"consistent":true},"usage_raw":{"input_tokens":2549,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":12.490085791912861,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24617,"response_bytes":10414,"usage":{"input":3822,"output":1089,"cached":2289,"total":4911,"consistent":true},"usage_raw":{"input_tokens":3822,"input_tokens_details":{"cached_tokens":2289,"cache_write_tokens":0},"output_tokens":1089,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4911}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.264513292000629,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":25740,"response_bytes":10176,"usage":{"input":385,"output":123,"cached":6001,"total":508,"consistent":false},"usage_raw":{"input_tokens":385,"input_tokens_details":{"cached_tokens":6001,"cache_write_tokens":0},"output_tokens":123,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":508}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":7.988844917039387,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":26145,"response_bytes":10974,"usage":{"input":489,"output":1075,"cached":6001,"total":1564,"consistent":false},"usage_raw":{"input_tokens":489,"input_tokens_details":{"cached_tokens":6001,"cache_write_tokens":0},"output_tokens":1075,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1564}}]} +{"role":"candidate","task_id":"quiz.bug","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.9353029580088332,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11005,"response_bytes":1113,"usage":{"input":2549,"output":302,"cached":0,"total":2851,"consistent":true},"usage_raw":{"input_tokens":2549,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":6.415509874932468,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24617,"response_bytes":10333,"usage":{"input":110,"output":456,"cached":6001,"total":566,"consistent":false},"usage_raw":{"input_tokens":110,"input_tokens_details":{"cached_tokens":6001,"cache_write_tokens":0},"output_tokens":456,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":566}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":1.9761110830586404,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":25683,"response_bytes":10210,"usage":{"input":6377,"output":91,"cached":0,"total":6468,"consistent":true},"usage_raw":{"input_tokens":6377,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":91,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6468}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":18.18070374999661,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":26106,"response_bytes":10958,"usage":{"input":484,"output":1215,"cached":6001,"total":1699,"consistent":false},"usage_raw":{"input_tokens":484,"input_tokens_details":{"cached_tokens":6001,"cache_write_tokens":0},"output_tokens":1215,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1699}}]} +{"role":"candidate","task_id":"quiz.create","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.2587280420120806,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10866,"response_bytes":1105,"usage":{"input":2523,"output":302,"cached":0,"total":2825,"consistent":true},"usage_raw":{"input_tokens":2523,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":51.55863500002306,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":20094,"response_bytes":19735,"usage":{"input":5053,"output":7106,"cached":0,"total":12159,"consistent":true},"usage_raw":{"input_tokens":5053,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":7106,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":12159}}]} +{"role":"candidate","task_id":"quiz.create","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.9072963329963386,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10866,"response_bytes":1107,"usage":{"input":2523,"output":302,"cached":0,"total":2825,"consistent":true},"usage_raw":{"input_tokens":2523,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":39.649984750081785,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":20094,"response_bytes":21019,"usage":{"input":76,"output":5405,"cached":4977,"total":5481,"consistent":false},"usage_raw":{"input_tokens":76,"input_tokens_details":{"cached_tokens":4977,"cache_write_tokens":0},"output_tokens":5405,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5481}}]} +{"role":"candidate","task_id":"quiz.create","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.3577790419803932,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":10866,"response_bytes":1109,"usage":{"input":2523,"output":302,"cached":0,"total":2825,"consistent":true},"usage_raw":{"input_tokens":2523,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":74.43968883296475,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":20094,"response_bytes":18120,"usage":{"input":76,"output":9568,"cached":4977,"total":9644,"consistent":false},"usage_raw":{"input_tokens":76,"input_tokens_details":{"cached_tokens":4977,"cache_write_tokens":0},"output_tokens":9568,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":9644}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":27.622696666046977,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":29174,"response_bytes":16405,"usage":{"input":2496,"output":3622,"cached":4977,"total":6118,"consistent":false},"usage_raw":{"input_tokens":2496,"input_tokens_details":{"cached_tokens":4977,"cache_write_tokens":0},"output_tokens":3622,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6118}}]} +{"role":"candidate","task_id":"quiz.edit","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8767297910526395,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11024,"response_bytes":1110,"usage":{"input":2558,"output":303,"cached":0,"total":2861,"consistent":true},"usage_raw":{"input_tokens":2558,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":8.28847345802933,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14080,"response_bytes":6613,"usage":{"input":53,"output":955,"cached":3697,"total":1008,"consistent":false},"usage_raw":{"input_tokens":53,"input_tokens_details":{"cached_tokens":3697,"cache_write_tokens":0},"output_tokens":955,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1008}}]} +{"role":"candidate","task_id":"quiz.edit","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.3339689170243219,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11024,"response_bytes":1113,"usage":{"input":2558,"output":303,"cached":0,"total":2861,"consistent":true},"usage_raw":{"input_tokens":2558,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":10.950811250018887,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14080,"response_bytes":6711,"usage":{"input":3750,"output":1219,"cached":0,"total":4969,"consistent":true},"usage_raw":{"input_tokens":3750,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":1219,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4969}}]} +{"role":"candidate","task_id":"quiz.edit","trial":3,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":0.8822282499168068,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11024,"response_bytes":1114,"usage":{"input":2558,"output":303,"cached":0,"total":2861,"consistent":true},"usage_raw":{"input_tokens":2558,"output_tokens":303}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":11.69803137506824,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":14080,"response_bytes":6648,"usage":{"input":3750,"output":1164,"cached":0,"total":4914,"consistent":true},"usage_raw":{"input_tokens":3750,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":1164,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4914}}]} +{"role":"candidate","task_id":"quiz.feature","trial":1,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.2834129999391735,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11387,"response_bytes":1113,"usage":{"input":2615,"output":302,"cached":0,"total":2917,"consistent":true},"usage_raw":{"input_tokens":2615,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":43.74556458299048,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24977,"response_bytes":15960,"usage":{"input":6169,"output":6299,"cached":0,"total":12468,"consistent":true},"usage_raw":{"input_tokens":6169,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":6299,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":12468}}]} +{"role":"candidate","task_id":"quiz.feature","trial":2,"calls":[{"index":0,"complete":true,"status":200,"endpoint":"/decisions","duration_s":1.328058790997602,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":"jev-1.13.0","request_bytes":11387,"response_bytes":1113,"usage":{"input":2615,"output":302,"cached":0,"total":2917,"consistent":true},"usage_raw":{"input_tokens":2615,"output_tokens":302}},{"index":1,"complete":true,"status":200,"endpoint":"/responses","duration_s":43.504078791011125,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":24977,"response_bytes":12965,"usage":{"input":40,"output":5807,"cached":6129,"total":5847,"consistent":false},"usage_raw":{"input_tokens":40,"input_tokens_details":{"cached_tokens":6129,"cache_write_tokens":0},"output_tokens":5807,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":5847}},{"index":2,"complete":true,"status":200,"endpoint":"/responses","duration_s":17.62198795902077,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":29956,"response_bytes":10171,"usage":{"input":1319,"output":2224,"cached":6001,"total":3543,"consistent":false},"usage_raw":{"input_tokens":1319,"input_tokens_details":{"cached_tokens":6001,"cache_write_tokens":0},"output_tokens":2224,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3543}},{"index":3,"complete":true,"status":200,"endpoint":"/responses","duration_s":24.488433417049237,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":30438,"response_bytes":10936,"usage":{"input":289,"output":3370,"cached":7153,"total":3659,"consistent":false},"usage_raw":{"input_tokens":289,"input_tokens_details":{"cached_tokens":7153,"cache_write_tokens":0},"output_tokens":3370,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":3659}},{"index":4,"complete":true,"status":200,"endpoint":"/responses","duration_s":9.511400333954953,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":32197,"response_bytes":10245,"usage":{"input":7886,"output":101,"cached":0,"total":7987,"consistent":true},"usage_raw":{"input_tokens":7886,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":101,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":7987}},{"index":5,"complete":true,"status":200,"endpoint":"/responses","duration_s":22.525051041971892,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":34741,"response_bytes":10553,"usage":{"input":730,"output":3820,"cached":7793,"total":4550,"consistent":false},"usage_raw":{"input_tokens":730,"input_tokens_details":{"cached_tokens":7793,"cache_write_tokens":0},"output_tokens":3820,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":4550}},{"index":6,"complete":true,"status":200,"endpoint":"/responses","duration_s":31.01686862495262,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":35687,"response_bytes":14922,"usage":{"input":1489,"output":5013,"cached":7281,"total":6502,"consistent":false},"usage_raw":{"input_tokens":1489,"input_tokens_details":{"cached_tokens":7281,"cache_write_tokens":0},"output_tokens":5013,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":6502}},{"index":7,"complete":true,"status":200,"endpoint":"/responses","duration_s":2.2132954159751534,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":42515,"response_bytes":10155,"usage":{"input":1617,"output":78,"cached":8689,"total":1695,"consistent":false},"usage_raw":{"input_tokens":1617,"input_tokens_details":{"cached_tokens":8689,"cache_write_tokens":0},"output_tokens":78,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1695}},{"index":8,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.6555265419883654,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":45002,"response_bytes":10507,"usage":{"input":2496,"output":201,"cached":8433,"total":2697,"consistent":false},"usage_raw":{"input_tokens":2496,"input_tokens_details":{"cached_tokens":8433,"cache_write_tokens":0},"output_tokens":201,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2697}},{"index":9,"complete":true,"status":200,"endpoint":"/responses","duration_s":3.2480682500172406,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":45826,"response_bytes":10457,"usage":{"input":931,"output":257,"cached":10225,"total":1188,"consistent":false},"usage_raw":{"input_tokens":931,"input_tokens_details":{"cached_tokens":10225,"cache_write_tokens":0},"output_tokens":257,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1188}},{"index":10,"complete":true,"status":200,"endpoint":"/responses","duration_s":14.33719300001394,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":46625,"response_bytes":10201,"usage":{"input":513,"output":1344,"cached":10865,"total":1857,"consistent":false},"usage_raw":{"input_tokens":513,"input_tokens_details":{"cached_tokens":10865,"cache_write_tokens":0},"output_tokens":1344,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1857}},{"index":11,"complete":true,"status":200,"endpoint":"/responses","duration_s":4.939458334003575,"upstream_status":200,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":"muse-spark-1-3","request_bytes":50218,"response_bytes":10467,"usage":{"input":943,"output":216,"cached":11249,"total":1159,"consistent":false},"usage_raw":{"input_tokens":943,"input_tokens_details":{"cached_tokens":11249,"cache_write_tokens":0},"output_tokens":216,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1159}}]} +{"role":"candidate","task_id":"quiz.feature","trial":3,"calls":[{"index":0,"complete":true,"status":500,"endpoint":"/decisions","duration_s":59.85060445801355,"upstream_status":500,"response_origin":"upstream","transport_error_type":null,"requested_model":"jev-1.13.0","returned_model":null,"request_bytes":11387,"response_bytes":170,"usage":null,"usage_raw":null},{"index":1,"complete":true,"status":500,"endpoint":"/responses","duration_s":60.19044558296446,"upstream_status":500,"response_origin":"upstream","transport_error_type":null,"requested_model":"muse-spark-1-3","returned_model":null,"request_bytes":12732,"response_bytes":170,"usage":null,"usage_raw":null}]} diff --git a/evals/results/unseen-002/changes.jsonl b/evals/results/unseen-002/changes.jsonl new file mode 100644 index 0000000..5b064d7 --- /dev/null +++ b/evals/results/unseen-002/changes.jsonl @@ -0,0 +1,120 @@ +{"role":"baseline","task_id":"application.bug","trial":1,"diff":"--- a/frontend/src/App.tsx\n+++ b/frontend/src/App.tsx\n@@ -5,7 +5,7 @@\n const load=async()=>{const r:Row[]=await fetch('/api/applications').then(r=>r.json());R(r);return r;};useEffect(()=>{void load().then(r=>{const draft=r.filter(a=>a.status==='draft').at(-1);if(draft){D(draft);I(draft.id);}});},[]);\n function field(k:keyof Data,v:string){D({...data,[k]:v});}\n function next(){if(step===1&&(!data.name.trim()||!/^\\S+@\\S+\\.\\S+$/.test(data.email.trim()))||step===2&&(!['Design','Engineering'].includes(data.track)||data.statement.trim().length<10)){E('Complete required fields');return;}E('');S(step+1);}\n- function back(){if(step===3)D({...data,statement:''});S(step-1);}\n+ function back(){S(step-1);}\n async function save(){const r=await fetch(id===null?'/api/applications':`/api/applications/${id}`,{method:id===null?'POST':'PUT',headers:{'Content-Type':'application/json'},body:JSON.stringify(data)});if(!r.ok){E('Cannot save draft');return null;}const row:Row=await r.json();I(row.id);E('');await load();return row;}\n async function submit(){const row=await save();if(!row)return;const r=await fetch(`/api/applications/${row.id}/submit`,{method:'POST'});if(!r.ok){E('Cannot submit');return;}T('Application submitted');I(null);D(empty);S(1);await load();}\n return

Application form

{step===1&&<>}{step===2&&<>