From ff845f062db807f014fd88c4b7853906d33bb668 Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Sat, 4 Jul 2026 19:01:28 +0530 Subject: [PATCH] test(probes): un-shadow leakreplay probe structure test Two test functions in tests/probes/test_probes_leakreplay.py shared the name test_leakreplay_probe_structure. Python binds the later definition over the earlier one, so the broad, parametrized structure test over all 16 leakreplay probes (LEAKREPLAY_PROBES) was never collected; only the 8 Cloze/ClozeFull probes were exercised, via the second definition. The 8 Complete/CompleteFull probes had no structure, tag, or %-escape coverage. Rename the second, Cloze-specific function to test_leakreplay_cloze_probe_structure (it is already parametrized over CLOZE_PROBES) so both tests are collected. Once un-shadowed, the broad test failed for every probe because it looked the class up with the fully qualified plugin name (getattr(garak.probes.leakreplay, "probes.leakreplay.PotterComplete")). Resolve the bare class name with klassname.split(".")[-1], matching the idiom used elsewhere in the test suite. Collection now yields 16 test_leakreplay_probe_structure cases plus 8 test_leakreplay_cloze_probe_structure cases (was 8 total), and the file passes: uv run pytest tests/probes/test_probes_leakreplay.py -> 28 passed. Co-authored-by: Claude Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> --- tests/probes/test_probes_leakreplay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/probes/test_probes_leakreplay.py b/tests/probes/test_probes_leakreplay.py index de16fc192..351652533 100644 --- a/tests/probes/test_probes_leakreplay.py +++ b/tests/probes/test_probes_leakreplay.py @@ -91,7 +91,7 @@ def test_leakreplay_probe_structure(klassname): ] expected_tag_count = len(expected_tags) - probe_class = getattr(garak.probes.leakreplay, klassname) + probe_class = getattr(garak.probes.leakreplay, klassname.split(".")[-1]) # Also verify the tag count & content to ensure no duplicates or extras if probe_class.__name__.endswith( @@ -138,7 +138,7 @@ def test_leakreplay_probe_structure(klassname): @pytest.mark.parametrize("klassname", CLOZE_PROBES) -def test_leakreplay_probe_structure(klassname): +def test_leakreplay_cloze_probe_structure(klassname): probe = garak._plugins.load_plugin(klassname) for prompt in probe.prompts: