Merge pull request #1743 from gooddata/feat/eval-visualization-filter-detail

feat(gooddata-eval): report compared filters in visualization evaluation detail

Author: Tomkess

packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py

@@ -254,6 +254,18 @@ class VisualizationAssertionError(AssertionError):
     __tracebackhide__ = True
 
 
+def _filter_diff(category: str, ev: EvaluationResult) -> str:
+    """Expected-vs-actual lines for one filter category, or "" when they matched.
+
+    Filters compare by exact equality on their canonical form, so "False" alone leaves
+    the reader guessing which part differed — the period, the granularity or the dataset.
+    """
+    expected, actual = ev.expected_filters.get(category, []), ev.actual_filters.get(category, [])
+    if expected == actual:
+        return ""
+    return f"      expected : {expected or 'none'}\n      actual   : {actual or 'none'}\n"
+
+
 def evaluate_agentic_visualization(
     host: str,
     token: str,
@@ -393,8 +405,11 @@ def evaluate_agentic_visualization(
             f"    actual   : {sorted(ev.actual_dim_uris)}\n"
             f"  Filters Correct       : {ev.filters_correct}\n"
             f"    date      : {ev.filter_date_score}\n"
+            f"{_filter_diff('date', ev)}"
             f"    ranking   : {ev.filter_ranking_score}\n"
+            f"{_filter_diff('ranking', ev)}"
             f"    attribute : {ev.filter_attribute_score}\n"
+            f"{_filter_diff('attribute', ev)}"
             f"  Viz Type Hard         : {ev.viz_type_hard}\n"
             "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
         )

packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py

@@ -10,9 +10,12 @@
     check_viz_type,
     get_dimension_uri_set,
     get_metric_uri_set,
+    normalized_filters,
     validate_cross_references,
 )
 
+_NO_FILTERS: dict[str, list[str]] = {"date": [], "ranking": [], "attribute": []}
+
 
 @dataclass
 class EvaluationResult:
@@ -31,6 +34,11 @@ class EvaluationResult:
     actual_metric_uris: set[str]
     expected_dim_uris: set[str]
     actual_dim_uris: set[str]
+    # Filters in the canonical form equality is tested on, keyed by the category each
+    # `filter_*_score` covers. Reported alongside the booleans because a filter mismatch
+    # is otherwise undiagnosable from a finished run.
+    expected_filters: dict[str, list[str]]
+    actual_filters: dict[str, list[str]]
 
     @property
     def strict_pass(self) -> bool:
@@ -91,6 +99,8 @@ def _evaluate_visualization(
             actual_metric_uris=set(),
             expected_dim_uris=exp_dim_uris,
             actual_dim_uris=set(),
+            expected_filters=normalized_filters(expected),
+            actual_filters={category: values.copy() for category, values in _NO_FILTERS.items()},
         )
     cross_ref_valid, cross_ref_errors = validate_cross_references(actual)
     act_metric_uris = get_metric_uri_set(actual)
@@ -112,6 +122,8 @@ def _evaluate_visualization(
         actual_metric_uris=act_metric_uris,
         expected_dim_uris=exp_dim_uris,
         actual_dim_uris=act_dim_uris,
+        expected_filters=normalized_filters(expected),
+        actual_filters=normalized_filters(actual),
     )
 
 
@@ -174,5 +186,7 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation
                 "actual_metric_uris": sorted(ev.actual_metric_uris),
                 "expected_dim_uris": sorted(ev.expected_dim_uris),
                 "actual_dim_uris": sorted(ev.actual_dim_uris),
+                "expected_filters": ev.expected_filters,
+                "actual_filters": ev.actual_filters,
             },
         )

packages/gooddata-eval/src/gooddata_eval/core/scoring.py

@@ -188,6 +188,19 @@ def _split_and_normalize_filters(viz: CreatedVisualization) -> tuple[set[str], s
     return date_set, ranking_set, attr_set
 
 
+def normalized_filters(viz: CreatedVisualization) -> dict[str, list[str]]:
+    """A visualization's filters exactly as `check_filters` compares them.
+
+    Grouped by the three categories it scores separately and sorted for stable output.
+    Each entry is the canonical JSON string equality is tested on, so reporting the
+    expected and actual side by side shows precisely why a category did not match —
+    a `filter_date_score` of False otherwise gives no clue whether the period differed,
+    the granularity did, or the dataset the filter hangs off did.
+    """
+    date_set, ranking_set, attr_set = _split_and_normalize_filters(viz)
+    return {"date": sorted(date_set), "ranking": sorted(ranking_set), "attribute": sorted(attr_set)}
+
+
 def check_filters(expected: CreatedVisualization, actual: CreatedVisualization) -> FilterScores:
     exp_date, exp_rank, exp_attr = _split_and_normalize_filters(expected)
     act_date, act_rank, act_attr = _split_and_normalize_filters(actual)

packages/gooddata-eval/tests/test_scoring.py

@@ -5,6 +5,7 @@
     check_viz_type,
     get_dimension_uri_set,
     get_metric_uri_set,
+    normalized_filters,
     uri_to_display_name,
     validate_cross_references,
 )
@@ -161,3 +162,46 @@ def test_validate_cross_references_accepts_omitted_attribute_but_flags_missing_u
     ok, errors = validate_cross_references(no_using)
     assert ok is False
     assert "is required" in errors[0]
+
+
+def test_normalized_filters_groups_by_scored_category():
+    """The three categories `check_filters` scores separately, in the form it compares."""
+    viz = CreatedVisualization.model_validate(
+        {
+            "id": "x",
+            "type": "bar_chart",
+            "query": {
+                "fields": {"m": {"using": "metric/spend"}, "d": {"using": "label/merchant.name"}},
+                "filter_by": {
+                    "f0": {
+                        "type": "date_filter",
+                        "using": "dataset/date",
+                        "granularity": "MONTH",
+                        "from": -1,
+                        "to": -1,
+                    },
+                    "f1": {"type": "ranking_filter", "using": "m", "top": 5},
+                    "f2": {"type": "attribute_filter", "using": "label/region", "state": {"include": ["EMEA"]}},
+                },
+            },
+            "metrics": ["m"],
+            "view_by": ["d"],
+        }
+    )
+    grouped = normalized_filters(viz)
+    assert set(grouped) == {"date", "ranking", "attribute"}
+    assert all(len(v) == 1 for v in grouped.values())
+    # The ranking entry carries the substituted sole dimension, matching what equality sees.
+    assert '"dim_uri": "label/merchant.name"' in grouped["ranking"][0]
+
+
+def test_normalized_filters_is_empty_per_category_when_unfiltered():
+    viz = CreatedVisualization.model_validate(
+        {
+            "id": "x",
+            "type": "headline",
+            "query": {"fields": {"m": {"using": "metric/spend"}}, "filter_by": {}},
+            "metrics": ["m"],
+        }
+    )
+    assert normalized_filters(viz) == {"date": [], "ranking": [], "attribute": []}

packages/gooddata-eval/tests/test_visualization_evaluator.py

@@ -127,3 +127,46 @@ def test_evaluator_passes_when_agent_omits_ranking_attribute_on_single_dim_viz()
     assert result.detail["filter_ranking_score"] is True
     assert result.detail["filters_correct"] is True
     assert result.passed is True
+
+
+def _dated(granularity: str, frm: int, to: int):
+    return {
+        "id": "x",
+        "type": "column_chart",
+        "query": {
+            "fields": {"m_rev": {"using": "metric/revenue"}, "d_q": {"using": "label/date.quarter"}},
+            "filter_by": {
+                "f_date": {
+                    "type": "date_filter",
+                    "using": "dataset/date",
+                    "granularity": granularity,
+                    "from": frm,
+                    "to": to,
+                }
+            },
+        },
+        "metrics": ["m_rev"],
+        "view_by": ["d_q"],
+    }
+
+
+def test_detail_reports_the_filters_that_were_compared():
+    """A `filter_date_score` of False is undiagnosable from a finished run without them:
+    two encodings of the same period compare unequal and the booleans don't say which."""
+    ev = get_evaluator("visualization")
+    result = ev.evaluate(_item(_dated("MONTH", -11, 0)), _chat_result_with(_dated("MONTH", -12, -1)))
+
+    assert result.detail["filter_date_score"] is False
+    expected, actual = result.detail["expected_filters"], result.detail["actual_filters"]
+    assert '"from": -11' in expected["date"][0]
+    assert '"from": -12' in actual["date"][0]
+    assert expected["ranking"] == actual["ranking"] == []
+    assert expected["attribute"] == actual["attribute"] == []
+
+
+def test_detail_filters_are_empty_when_no_visualization_was_created():
+    ev = get_evaluator("visualization")
+    empty = ChatResult.model_validate({"textResponse": "what metric?", "toolCallEvents": []})
+    result = ev.evaluate(_item(_dated("MONTH", -11, 0)), empty)
+    assert result.detail["actual_filters"] == {"date": [], "ranking": [], "attribute": []}
+    assert len(result.detail["expected_filters"]["date"]) == 1