fix(eval): agentic search_tool scored tool_correctness ~1% against valid calls

The agentic and single-shot search_tool scorers disagree, and the agentic one --
which is what `--kind search_tool` dispatches to -- is the stale one.

`core/evaluators/search_tool.py` already documents and applies the exclusion:

    # limit is optional with a server-side default; emit_widget was renamed to
    # user_requested_search in the tool schema -- neither affects search quality.

`core/agentic/search_tool.py` never got it. It compares every key the fixture
names, so a fixture written against the older tool signature fails on `limit`
alone (fixtures say 10, the agent sends the server default 20).

The second cause is list matching. Expected elements were compared with exact
string equality, so a fixture that tokenizes -- ["abandonment", "rate", "3ds"] --
never matches an agent that searches the object's title as one phrase,
["Abandonment Rate - 3DS"]. Both name the same object.

Measured over 143 recorded search_objects calls from a live corpus (12 distinct
questions, October 2026): tool_correctness was true in 3 of 244 evaluations,
~1%. Every one of those 12 questions failed on `keywords` + `limit`, and 11 of
12 failed on nothing else.

This changes two things in `_tool_correctness`:

- ignore `limit`, `emit_widget` and `user_requested_search`, matching the
  single-shot evaluator's existing rationale
- match list elements by case-insensitive substring in either direction rather
  than exact equality, so differing tokenization of the same object name passes

Replayed against those same 143 calls, 11 of 12 questions now score correct on
every run. The twelfth scores 1 of 7 and should: it phrases its search
differently on every run, which is real instability rather than a scorer
artifact.

Nothing in the suite surfaced this, because an item passes on `tool_selection`
alone -- pass@k stayed at 91-100% while `tool_correctness` sat at 1% and the
derived quality_score was pinned at exactly 0.5 for every search_tool item.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: Tomkess

packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py

@@ -42,22 +42,51 @@ def _tool_selection(tool_call_events: list[ToolCallEvent]) -> bool:
     return any(tc.function_name == "search_objects" for tc in tool_call_events)
 
 
+# Arguments that do not determine search quality, so a fixture naming them must not
+# fail the call. `limit` has a server-side default the agent is free to choose;
+# `emit_widget` was renamed to `user_requested_search` in the tool schema. The
+# single-shot evaluator (core/evaluators/search_tool.py) has ignored these since the
+# rename -- this path did not, so every fixture written against the older signature
+# scored tool_correctness=False while still passing on tool_selection alone.
+_IGNORED_SEARCH_ARGS = frozenset({"limit", "emit_widget", "user_requested_search"})
+
+
+def _loosely_contains(expected: str, actual: str) -> bool:
+    """Case-insensitive containment in either direction."""
+    return expected.lower() in actual.lower() or actual.lower() in expected.lower()
+
+
+def _list_matches(expected: list, actual: list) -> bool:
+    """Every expected element must appear in some actual element, and vice versa is not required.
+
+    Substring rather than equality, because the two sides legitimately tokenize
+    differently: a fixture says ["abandonment", "rate", "3ds"] while the agent searches
+    the object's title as one phrase, ["Abandonment Rate - 3DS"]. Both name the same
+    object, and exact element equality calls that a failure.
+    """
+    return all(
+        any(isinstance(a, str) and isinstance(e, str) and _loosely_contains(e, a) for a in actual) for e in expected
+    )
+
+
 def _tool_correctness(tool_call_events: list[ToolCallEvent], expected_tool_call: dict) -> bool:
     """Return True if the search_objects call arguments match expected.
 
-    List fields (e.g. keywords, object_types) use subset matching: all expected
-    values must appear in the actual call, but the agent may include extras.
+    List fields (e.g. keywords, object_types) use subset matching: every expected value
+    must appear in the actual call, by substring, and the agent may include extras.
     """
     for tc in tool_call_events:
         if tc.function_name == "search_objects":
             args = tc.parsed_arguments() or {}
             for key, exp_val in expected_tool_call.items():
+                if key in _IGNORED_SEARCH_ARGS:
+                    continue
                 act_val = args.get(key)
                 if isinstance(exp_val, list) and isinstance(act_val, list):
-                    if not set(exp_val).issubset(set(act_val)):
+                    if not _list_matches(exp_val, act_val):
                         return False
                 elif isinstance(exp_val, str) and isinstance(act_val, str):
-                    if exp_val.lower() not in act_val.lower() and act_val.lower() not in exp_val.lower():
+                    if not _loosely_contains(exp_val, act_val):
                         return False
                 elif exp_val != act_val:
                     return False

packages/gooddata-eval/tests/test_agentic_search_tool.py

@@ -245,3 +245,42 @@ def test_evaluate_agentic_search_tool_attaches_reasoning_steps_to_exception_on_f
         "tool_call_names": [],
         "latency_breakdown": [],
     }
+
+
+def test_tool_correctness_tokenized_keywords_match_a_title_phrase():
+    """A fixture tokenizes; the agent searches the object's title as one phrase."""
+    tcs = [_mock_tc("search_objects", {"keywords": ["Abandonment Rate - 3DS"], "object_types": ["metric"]})]
+    assert _tool_correctness(tcs, {"keywords": ["abandonment", "rate", "3ds"]}) is True
+
+
+def test_tool_correctness_still_rejects_an_unrelated_keyword():
+    tcs = [_mock_tc("search_objects", {"keywords": ["Abandonment Rate - 3DS"]})]
+    assert _tool_correctness(tcs, {"keywords": ["interchange"]}) is False
+
+
+def test_tool_correctness_ignores_limit():
+    """`limit` has a server-side default; the single-shot evaluator already ignores it."""
+    tcs = [_mock_tc("search_objects", {"keywords": ["revenue"], "limit": 20})]
+    assert _tool_correctness(tcs, {"keywords": ["revenue"], "limit": 10}) is True
+
+
+def test_tool_correctness_ignores_the_renamed_widget_argument():
+    tcs = [_mock_tc("search_objects", {"keywords": ["revenue"], "user_requested_search": True})]
+    assert _tool_correctness(tcs, {"keywords": ["revenue"], "emit_widget": False}) is True
+
+
+def test_tool_correctness_allows_extra_object_types():
+    """Searching more broadly than the fixture asked is not a failure."""
+    tcs = [_mock_tc("search_objects", {"keywords": ["revenue"], "object_types": ["metric", "fact", "attribute"]})]
+    assert _tool_correctness(tcs, {"keywords": ["revenue"], "object_types": ["metric"]}) is True
+
+
+def test_tool_correctness_rejects_a_missing_object_type():
+    tcs = [_mock_tc("search_objects", {"keywords": ["revenue"], "object_types": ["metric"]})]
+    assert _tool_correctness(tcs, {"keywords": ["revenue"], "object_types": ["dashboard"]}) is False
+
+
+def test_tool_correctness_ignores_arguments_the_agent_adds():
+    """compatible_with_afm and kda_eligible appeared after these fixtures were written."""
+    tcs = [_mock_tc("search_objects", {"keywords": ["revenue"], "compatible_with_afm": "x", "kda_eligible": True})]
+    assert _tool_correctness(tcs, {"keywords": ["revenue"]}) is True