Skip to content

Commit af862ad

Browse files
Tomkessclaude
andcommitted
fix(eval): agentic search_tool scored tool_correctness ~1% against valid calls
The agentic and single-shot search_tool scorers disagree, and the agentic one -- which is what `--kind search_tool` dispatches to -- is the stale one. `core/evaluators/search_tool.py` already documents and applies the exclusion: # limit is optional with a server-side default; emit_widget was renamed to # user_requested_search in the tool schema -- neither affects search quality. `core/agentic/search_tool.py` never got it. It compares every key the fixture names, so a fixture written against the older tool signature fails on `limit` alone (fixtures say 10, the agent sends the server default 20). The second cause is list matching. Expected elements were compared with exact string equality, so a fixture that tokenizes -- ["abandonment", "rate", "3ds"] -- never matches an agent that searches the object's title as one phrase, ["Abandonment Rate - 3DS"]. Both name the same object. Measured over 143 recorded search_objects calls from a live corpus (12 distinct questions, October 2026): tool_correctness was true in 3 of 244 evaluations, ~1%. Every one of those 12 questions failed on `keywords` + `limit`, and 11 of 12 failed on nothing else. This changes two things in `_tool_correctness`: - ignore `limit`, `emit_widget` and `user_requested_search`, matching the single-shot evaluator's existing rationale - match list elements by case-insensitive substring in either direction rather than exact equality, so differing tokenization of the same object name passes Replayed against those same 143 calls, 11 of 12 questions now score correct on every run. The twelfth scores 1 of 7 and should: it phrases its search differently on every run, which is real instability rather than a scorer artifact. Nothing in the suite surfaced this, because an item passes on `tool_selection` alone -- pass@k stayed at 91-100% while `tool_correctness` sat at 1% and the derived quality_score was pinned at exactly 0.5 for every search_tool item. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> An empty search term matches nothing: "" is a substring of every string, so _loosely_contains would otherwise accept an empty agent keyword against any expected keyword, and the reverse. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
1 parent 4b505e4 commit af862ad

2 files changed

Lines changed: 82 additions & 4 deletions

File tree

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,22 +43,53 @@ def _tool_selection(tool_call_events: list[ToolCallEvent]) -> bool:
4343
return any(tc.function_name == "search_objects" for tc in tool_call_events)
4444

4545

46+
# Arguments that do not determine search quality, so a fixture naming them must not
47+
# fail the call. `limit` has a server-side default the agent is free to choose;
48+
# `emit_widget` was renamed to `user_requested_search` in the tool schema. The
49+
# single-shot evaluator (core/evaluators/search_tool.py) has ignored these since the
50+
# rename -- this path did not, so every fixture written against the older signature
51+
# scored tool_correctness=False while still passing on tool_selection alone.
52+
_IGNORED_SEARCH_ARGS = frozenset({"limit", "emit_widget", "user_requested_search"})
53+
54+
55+
def _loosely_contains(expected: str, actual: str) -> bool:
56+
"""Case-insensitive containment in either direction; an empty string matches nothing."""
57+
if not expected.strip() or not actual.strip():
58+
return False
59+
return expected.lower() in actual.lower() or actual.lower() in expected.lower()
60+
61+
62+
def _list_matches(expected: list, actual: list) -> bool:
63+
"""Every expected element must appear in some actual element, and vice versa is not required.
64+
65+
Substring rather than equality, because the two sides legitimately tokenize
66+
differently: a fixture says ["abandonment", "rate", "3ds"] while the agent searches
67+
the object's title as one phrase, ["Abandonment Rate - 3DS"]. Both name the same
68+
object, and exact element equality calls that a failure.
69+
"""
70+
return all(
71+
any(isinstance(a, str) and isinstance(e, str) and _loosely_contains(e, a) for a in actual) for e in expected
72+
)
73+
74+
4675
def _tool_correctness(tool_call_events: list[ToolCallEvent], expected_tool_call: dict) -> bool:
4776
"""Return True if the search_objects call arguments match expected.
4877
49-
List fields (e.g. keywords, object_types) use subset matching: all expected
50-
values must appear in the actual call, but the agent may include extras.
78+
List fields (e.g. keywords, object_types) use subset matching: every expected value
79+
must appear in the actual call, by substring, and the agent may include extras.
5180
"""
5281
for tc in tool_call_events:
5382
if tc.function_name == "search_objects":
5483
args = tc.parsed_arguments() or {}
5584
for key, exp_val in expected_tool_call.items():
85+
if key in _IGNORED_SEARCH_ARGS:
86+
continue
5687
act_val = args.get(key)
5788
if isinstance(exp_val, list) and isinstance(act_val, list):
58-
if not set(exp_val).issubset(set(act_val)):
89+
if not _list_matches(exp_val, act_val):
5990
return False
6091
elif isinstance(exp_val, str) and isinstance(act_val, str):
61-
if exp_val.lower() not in act_val.lower() and act_val.lower() not in exp_val.lower():
92+
if not _loosely_contains(exp_val, act_val):
6293
return False
6394
elif exp_val != act_val:
6495
return False

‎packages/gooddata-eval/tests/test_agentic_search_tool.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,3 +250,50 @@ def test_evaluate_agentic_search_tool_attaches_reasoning_steps_to_exception_on_f
250250
"latency_breakdown": [],
251251
}
252252
assert len(exc_info.value.run_latencies_s) == exc_info.value.runs_effective == 1
253+
254+
255+
def test_tool_correctness_tokenized_keywords_match_a_title_phrase():
256+
"""A fixture tokenizes; the agent searches the object's title as one phrase."""
257+
tcs = [_mock_tc("search_objects", {"keywords": ["Abandonment Rate - 3DS"], "object_types": ["metric"]})]
258+
assert _tool_correctness(tcs, {"keywords": ["abandonment", "rate", "3ds"]}) is True
259+
260+
261+
def test_tool_correctness_still_rejects_an_unrelated_keyword():
262+
tcs = [_mock_tc("search_objects", {"keywords": ["Abandonment Rate - 3DS"]})]
263+
assert _tool_correctness(tcs, {"keywords": ["interchange"]}) is False
264+
265+
266+
def test_tool_correctness_ignores_limit():
267+
"""`limit` has a server-side default; the single-shot evaluator already ignores it."""
268+
tcs = [_mock_tc("search_objects", {"keywords": ["revenue"], "limit": 20})]
269+
assert _tool_correctness(tcs, {"keywords": ["revenue"], "limit": 10}) is True
270+
271+
272+
def test_tool_correctness_ignores_the_renamed_widget_argument():
273+
tcs = [_mock_tc("search_objects", {"keywords": ["revenue"], "user_requested_search": True})]
274+
assert _tool_correctness(tcs, {"keywords": ["revenue"], "emit_widget": False}) is True
275+
276+
277+
def test_tool_correctness_allows_extra_object_types():
278+
"""Searching more broadly than the fixture asked is not a failure."""
279+
tcs = [_mock_tc("search_objects", {"keywords": ["revenue"], "object_types": ["metric", "fact", "attribute"]})]
280+
assert _tool_correctness(tcs, {"keywords": ["revenue"], "object_types": ["metric"]}) is True
281+
282+
283+
def test_tool_correctness_rejects_a_missing_object_type():
284+
tcs = [_mock_tc("search_objects", {"keywords": ["revenue"], "object_types": ["metric"]})]
285+
assert _tool_correctness(tcs, {"keywords": ["revenue"], "object_types": ["dashboard"]}) is False
286+
287+
288+
def test_tool_correctness_ignores_arguments_the_agent_adds():
289+
"""compatible_with_afm and kda_eligible appeared after these fixtures were written."""
290+
tcs = [_mock_tc("search_objects", {"keywords": ["revenue"], "compatible_with_afm": "x", "kda_eligible": True})]
291+
assert _tool_correctness(tcs, {"keywords": ["revenue"]}) is True
292+
293+
294+
def test_tool_correctness_rejects_an_empty_search_term():
295+
""" "" is a substring of everything, so it must not satisfy an expected keyword."""
296+
tcs = [_mock_tc("search_objects", {"keywords": [""]})]
297+
assert _tool_correctness(tcs, {"keywords": ["revenue"]}) is False
298+
tcs = [_mock_tc("search_objects", {"keywords": ["revenue"]})]
299+
assert _tool_correctness(tcs, {"keywords": [""]}) is False

0 commit comments

Comments
 (0)