feat(gooddata-eval): populate best_run_latency_s for agentic evaluators

ItemReport.best_run_latency_s has only ever been set on the single-shot path
(_run_one_item times each of its K attempts directly and keeps the best
one's own latency). The agentic path drives its K runs and best-of-K
selection inside each evaluator, so the runner never had a per-run latency
to read -- every agentic kind reported best_run_latency_s: null, which is
why tools consuming it (e.g. the Latency Explorer's tool-calls-vs-run-time
panel) have always been empty for agentic_*/vis_agentic.

Adds AgenticEvalOutcome.best_run_latency_s (and the matching bare annotation
on AgenticAssertionError/JudgeResponseError, same pattern already used for
timings) and wires it through all 11 agentic evaluators:
- general_question, metric_skill, dashboard_skill already track per-run
  PhaseTimings; best_run_latency_s is derived from the rank-selected run's
  own timings via the new run_latency_s() helper (excludes langfuse_s,
  which is deliberately off the critical path).
- The other 8 (guardrail, search_tool, alert_skill, anomaly_detection,
  kda_skill, visualization, what_if, conversation) had no per-run wall-clock
  at all; each gets a new run_latency_s field on its per-run result,
  measured the same way _run_one_item already does (time.monotonic()
  bracketing the whole run).
agentic_runner.py copies it onto the ItemReport via a new
_apply_best_run_latency(), mirroring the existing _apply_timings().

Verified: full gooddata-eval suite (1382 tests), ruff lint/format, and ty
type-check all pass. Added focused tests for the new run_latency_s() helper,
_apply_best_run_latency()'s three paths (success/failure/absent), and
end-to-end value assertions for general_question and metric_skill (the two
kinds with fully deterministic mocked clocks).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Author: Tomkess

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -340,6 +340,18 @@ def _apply_timings(item_report: ItemReport, timings: Any) -> None:
     item_report.simulated_user_latency_s = timings.simulated_user_s
 
 
+def _apply_best_run_latency(item_report: ItemReport, source: Any) -> None:
+    """Copy the rank-selected run's own wall time, mirroring the single-shot path's
+    ``ItemReport.best_run_latency_s`` (see ``core/runner.py``'s ``_run_one_item``).
+
+    Kinds not yet wired to measure it pass None and keep the field at its own None
+    default rather than reporting an invented number.
+    """
+    best_run_latency_s = getattr(source, "best_run_latency_s", None)
+    if best_run_latency_s is not None:
+        item_report.best_run_latency_s = best_run_latency_s
+
+
 def run_agentic_items(
     items: list[DatasetItem],
     host: str,
@@ -425,6 +437,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport:
             item_report.response_id = response_id
             item_report.best_detail = detail or {}
             _apply_timings(item_report, getattr(outcome, "timings", None))
+            _apply_best_run_latency(item_report, outcome)
             _apply_run_counts(item_report, outcome)
         except AssertionError as exc:
             item_report.gate_passed = False if gated else None
@@ -434,6 +447,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport:
             item_report.response_id = getattr(exc, "response_id", None)
             item_report.best_detail = getattr(exc, "detail", None) or {}
             _apply_timings(item_report, getattr(exc, "timings", None))
+            _apply_best_run_latency(item_report, exc)
             _apply_run_counts(item_report, exc)
             # Read off the counts, not off the gate: pass^K fails items where runs did pass,
             # and reporting those as pass_at_k False would contradict the Langfuse score of
@@ -448,6 +462,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport:
             # judge broke should not also report the agent as costing 0s. Kinds that
             # attach no timings to the exception keep their 0.0 defaults.
             _apply_timings(item_report, getattr(exc, "timings", None))
+            _apply_best_run_latency(item_report, exc)
         finally:
             item_report.latency_s = time.perf_counter() - t0
 

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -6,6 +6,7 @@
 import json
 import os
 import re
+import time
 from dataclasses import dataclass, field
 from typing import Any
 
@@ -490,6 +491,9 @@ class AlertRunResult:
     # also report operator/threshold/metric/recipients as False.
     exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED
     turns_used: int = 0
+    # This run's own wall time, across all its turns -- mirrors the single-shot
+    # path's best_run_latency_s (see core/runner.py's _run_one_item).
+    run_latency_s: float = 0.0
 
 
 @dataclass
@@ -670,6 +674,7 @@ def run_agentic_alert_skill(
 
     def _run_once(conv_id: str) -> AlertRunResult:
         alert_id_to_delete: str | None = None
+        run_started = time.monotonic()
         try:
             alert_id: str | None = None
             actual_args: dict = {}
@@ -788,6 +793,7 @@ def _run_once(conv_id: str) -> AlertRunResult:
                 reasoning_step_events=all_reasoning_step_events,
                 exit_reason=exit_reason,
                 turns_used=turns_used,
+                run_latency_s=time.monotonic() - run_started,
             )
         finally:
             if alert_id_to_delete:
@@ -982,6 +988,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         exc.detail = detail
         exc.runs_passed = runs_passed
         exc.runs_effective = runs_effective
+        exc.best_run_latency_s = best.run_latency_s
         raise exc
     return AgenticEvalOutcome(
         runs_passed=runs_passed,
@@ -990,4 +997,5 @@ def _write_scores(ctx: RunTraceContext) -> None:
         conversation_id=best.conversation_id,
         response_id=best.response_id,
         detail=detail,
+        best_run_latency_s=best.run_latency_s,
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py

@@ -28,6 +28,7 @@
 import logging
 import os
 import re
+import time
 from dataclasses import dataclass, field
 from typing import Any
 
@@ -282,6 +283,10 @@ class AnomalyRunResult:
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
     reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
+    # This run's own wall time, across all its turns -- mirrors the single-shot
+    # path's best_run_latency_s (see core/runner.py's _run_one_item). Distinct from
+    # turn_wall_clock_sec above, which is only the final triggering turn.
+    run_latency_s: float = 0.0
 
 
 @dataclass
@@ -360,6 +365,7 @@ def run_agentic_anomaly_detection(
     )
 
     def _run_once(conv_id: str) -> AnomalyRunResult:
+        run_started = time.monotonic()
         viz_args: dict | None = None
         execute_result: dict | None = None
         turn_wall_clock_sec: float | None = None
@@ -432,6 +438,7 @@ def _accumulate(result: ChatResult) -> None:
             response_id=response_id,
             tool_call_events=all_tool_call_events,
             reasoning_step_events=all_reasoning_step_events,
+            run_latency_s=time.monotonic() - run_started,
         )
 
     try:
@@ -630,6 +637,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         exc.detail = detail
         exc.runs_passed = runs_passed
         exc.runs_effective = len(summary.run_results)
+        exc.best_run_latency_s = best.run_latency_s
         raise exc
 
     return AgenticEvalOutcome(
@@ -639,4 +647,5 @@ def _write_scores(ctx: RunTraceContext) -> None:
         conversation_id=best.conversation_id,
         response_id=best.response_id,
         detail=detail,
+        best_run_latency_s=best.run_latency_s,
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -19,6 +19,7 @@
 import json
 import os
 import re
+import time
 from dataclasses import dataclass, field
 from datetime import date
 from typing import ClassVar, Literal
@@ -593,6 +594,11 @@ class ConversationResult:
     stalled_turns: int = 0
     mode: ConversationMode = "legacy"
     judge_model: str | None = None
+    # This conversation's own wall time, across every turn -- mirrors the single-shot
+    # path's best_run_latency_s (see core/runner.py's _run_one_item). There is no
+    # K-of-N selection for this kind (see run_agentic_conversation's docstring), so
+    # this is simply the one conversation's own time, not a "best of" pick.
+    run_latency_s: float = 0.0
 
 
 def _metric_creations(tool_call_events: list[ToolCallEvent]) -> list[tuple[str, bool]]:
@@ -707,6 +713,7 @@ def run_agentic_conversation(
     agent sees no history. It is the no-memory baseline a context score is calibrated
     against: context-dependent turns are expected to fail there.
     """
+    run_started = time.monotonic()
     resolved_mode = resolve_conversation_mode(mode)
     if fresh_conversation_per_turn and initial_conversation_id is not None:
         raise ValueError("fresh_conversation_per_turn needs conversations this run creates itself")
@@ -1011,6 +1018,7 @@ def run_agentic_conversation(
         stalled_turns=stalled,
         mode=resolved_mode,
         judge_model=judge.model_name if judged else None,
+        run_latency_s=time.monotonic() - run_started,
     )
 
 
@@ -1216,6 +1224,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         # for. Saying so explicitly stops the report claiming K runs that never happened.
         exc.runs_passed = 0
         exc.runs_effective = 1
+        exc.best_run_latency_s = result.run_latency_s
         raise exc
     return AgenticEvalOutcome(
         reasoning_steps=result.reasoning_steps,
@@ -1224,4 +1233,5 @@ def _write_scores(ctx: RunTraceContext) -> None:
         detail=detail,
         runs_passed=1,
         runs_effective=1,
+        best_run_latency_s=result.run_latency_s,
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py

@@ -38,7 +38,7 @@
     build_latency_breakdown,
     shift_and_index_events,
 )
-from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings
+from gooddata_eval.core.timing import PhaseTimings, log_timer, run_latency_s, sum_timings
 
 _DEFAULT_K = 1
 # Matches kda_skill and visualization. The reply this loop sends is built from the expectation
@@ -1137,6 +1137,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         exc.conversation_id = best.conversation_id
         exc.response_id = best.response_id
         exc.timings = item_timings
+        exc.best_run_latency_s = run_latency_s(best.timings)
         exc.detail = detail
         exc.runs_passed = runs_passed
         exc.runs_effective = runs_effective
@@ -1149,4 +1150,5 @@ def _write_scores(ctx: RunTraceContext) -> None:
         response_id=best.response_id,
         detail=detail,
         timings=item_timings,
+        best_run_latency_s=run_latency_s(best.timings),
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py

@@ -34,7 +34,7 @@
     ToolCallEvent,
     build_latency_breakdown,
 )
-from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings
+from gooddata_eval.core.timing import PhaseTimings, log_timer, run_latency_s, sum_timings
 
 _DEFAULT_K = 1
 
@@ -318,6 +318,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
             + " | ".join(unscored)
         )
         exc.timings = item_timings
+        exc.best_run_latency_s = run_latency_s(summary.best.timings)
         raise exc
 
     runs_passed = sum(1 for r in summary.scored_run_results if r.passed)
@@ -344,6 +345,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         exc.conversation_id = best.conversation_id
         exc.response_id = best.response_id
         exc.timings = item_timings
+        exc.best_run_latency_s = run_latency_s(best.timings)
         exc.detail = detail
         exc.runs_passed = runs_passed
         exc.runs_effective = runs_effective
@@ -356,4 +358,5 @@ def _write_scores(ctx: RunTraceContext) -> None:
         response_id=best.response_id,
         detail=detail,
         timings=item_timings,
+        best_run_latency_s=run_latency_s(best.timings),
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import time
 from dataclasses import dataclass, field
 
 from gooddata_eval.core.agentic._gate import (
@@ -83,6 +84,9 @@ class GuardrailResult:
     # Set when the judge returned something unreadable for THIS run. Excluded from pass@K
     # and from Langfuse scoring rather than counted as a failure -- see score_run.
     judge_error: str | None = None
+    # This run's own wall time (agent call + its grading) -- mirrors the single-shot
+    # path's best_run_latency_s (see core/runner.py's _run_one_item).
+    run_latency_s: float = 0.0
 
 
 @dataclass
@@ -117,6 +121,7 @@ def _run_single_guardrail(
     Extracted so the two call sites below (the first conversation, which may be supplied,
     and the remaining K-1) cannot drift -- they had already duplicated the whole body once.
     """
+    run_started = time.monotonic()
     chat_result = client.send_message(conversation_id, question)
     actual_output = render_answer_text(chat_result)
     verdict = score_run(judge, input=question, expected_output=expected_output, actual_output=actual_output)
@@ -131,6 +136,7 @@ def _run_single_guardrail(
         tool_call_events=list(chat_result.tool_call_events or []),
         reasoning_step_events=list(chat_result.reasoning_step_events or []),
         judge_error=verdict.error,
+        run_latency_s=time.monotonic() - run_started,
     )
 
 
@@ -314,6 +320,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         exc.detail = detail
         exc.runs_passed = runs_passed
         exc.runs_effective = runs_effective
+        exc.best_run_latency_s = best.run_latency_s
         raise exc
     return AgenticEvalOutcome(
         runs_passed=runs_passed,
@@ -322,4 +329,5 @@ def _write_scores(ctx: RunTraceContext) -> None:
         conversation_id=best.conversation_id,
         response_id=best.response_id,
         detail=detail,
+        best_run_latency_s=best.run_latency_s,
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py

@@ -5,6 +5,7 @@
 
 import logging
 import os
+import time
 from dataclasses import dataclass, field
 
 import httpx
@@ -198,6 +199,10 @@ class KdaRunResult:
     # separate a refusal from a run that hit max_iterations while still on track.
     exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED
     turns_used: int = 0
+    # This run's own wall time, across all its turns -- mirrors the single-shot
+    # path's best_run_latency_s (see core/runner.py's _run_one_item). Distinct from
+    # turn_wall_clock_sec above, which is only the final create-triggering turn.
+    run_latency_s: float = 0.0
 
 
 @dataclass
@@ -260,6 +265,7 @@ def run_agentic_kda_skill(
     )
 
     def _run_once(conv_id: str) -> KdaRunResult:
+        run_started = time.monotonic()
         create_args: dict | None = None
         execute_result: dict | None = None
         turn_wall_clock_sec: float | None = None
@@ -367,6 +373,7 @@ def _accumulate(result: ChatResult) -> None:
             reasoning_step_events=all_reasoning_step_events,
             exit_reason=exit_reason,
             turns_used=turns_used,
+            run_latency_s=time.monotonic() - run_started,
         )
 
     try:
@@ -552,6 +559,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         exc.detail = detail
         exc.runs_passed = runs_passed
         exc.runs_effective = runs_effective
+        exc.best_run_latency_s = best.run_latency_s
         raise exc
     return AgenticEvalOutcome(
         runs_passed=runs_passed,
@@ -560,4 +568,5 @@ def _write_scores(ctx: RunTraceContext) -> None:
         conversation_id=best.conversation_id,
         response_id=best.response_id,
         detail=detail,
+        best_run_latency_s=best.run_latency_s,
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py

@@ -41,7 +41,7 @@
     shift_and_index_events,
     timeline_detail,
 )
-from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings
+from gooddata_eval.core.timing import PhaseTimings, log_timer, run_latency_s, sum_timings
 
 try:
     from openai import OpenAI as _OpenAI
@@ -575,6 +575,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         exc.conversation_id = best.conversation_id
         exc.response_id = best.response_id
         exc.timings = item_timings
+        exc.best_run_latency_s = run_latency_s(best.timings)
         exc.detail = detail
         exc.runs_passed = runs_passed
         exc.runs_effective = runs_effective
@@ -587,4 +588,5 @@ def _write_scores(ctx: RunTraceContext) -> None:
         response_id=best.response_id,
         detail=detail,
         timings=item_timings,
+        best_run_latency_s=run_latency_s(best.timings),
     )