fix(tests): apr-cli's integration surface goes 14 red -> 0 on clean main — three root causes, each fixed as a guard

crates/apr-cli/playbooks/snapshots/hex_dump.txt

@@ -1,40 +1,48 @@
 ══════════════════════════════════════════════════════════════════════
-Tensor: encoder.layers.0.self_attn.k_proj.weight
+Tensor: encoder.layers.0.self_attn.k_proj.weight
 ══════════════════════════════════════════════════════════════════════
-Shape: [384, 384] = 147456 elements
-Dtype: F32
-Offset: 0x01200000 (18874368 bytes)
-Size: 589824 bytes
-00000000: 0A D7 23 3C 0A D7 23 3C  0A D7 23 3C 0A D7 23 3C  |..#<..#<..#<..#<|
-... 589,808 more bytes
+Shape: [384, 384] = 147456 elements
+Dtype: F32
+Offset: 0x01200000 (18874368 bytes)
+Size: 589824 bytes
+
+Hex dump (first 16 bytes):
+00000000: 0A D7 23 3C 0A D7 23 3C 0A D7 23 3C 0A D7 23 3C  |     0.0100     0.0100     0.0100     0.0100 
+... 147452 more elements
 
 ══════════════════════════════════════════════════════════════════════
-Tensor: encoder.layers.0.self_attn.out_proj.weight
+Tensor: encoder.layers.0.self_attn.out_proj.weight
 ══════════════════════════════════════════════════════════════════════
-Shape: [384, 384] = 147456 elements
-Dtype: F32
-Offset: 0x01290000 (19464192 bytes)
-Size: 589824 bytes
-00000000: 0A D7 23 3C 0A D7 23 3C  0A D7 23 3C 0A D7 23 3C  |..#<..#<..#<..#<|
-... 589,808 more bytes
+Shape: [384, 384] = 147456 elements
+Dtype: F32
+Offset: 0x01290000 (19464192 bytes)
+Size: 589824 bytes
+
+Hex dump (first 16 bytes):
+00000000: 0A D7 23 3C 0A D7 23 3C 0A D7 23 3C 0A D7 23 3C  |     0.0100     0.0100     0.0100     0.0100 
+... 147452 more elements
 
 ══════════════════════════════════════════════════════════════════════
-Tensor: encoder.layers.0.self_attn.q_proj.weight
+Tensor: encoder.layers.0.self_attn.q_proj.weight
 ══════════════════════════════════════════════════════════════════════
-Shape: [384, 384] = 147456 elements
-Dtype: F32
-Offset: 0x01320000 (20054016 bytes)
-Size: 589824 bytes
-00000000: 0A D7 23 3C 0A D7 23 3C  0A D7 23 3C 0A D7 23 3C  |..#<..#<..#<..#<|
-... 589,808 more bytes
+Shape: [384, 384] = 147456 elements
+Dtype: F32
+Offset: 0x01320000 (20054016 bytes)
+Size: 589824 bytes
+
+Hex dump (first 16 bytes):
+00000000: 0A D7 23 3C 0A D7 23 3C 0A D7 23 3C 0A D7 23 3C  |     0.0100     0.0100     0.0100     0.0100 
+... 147452 more elements
 
 ══════════════════════════════════════════════════════════════════════
-Tensor: encoder.layers.0.self_attn.v_proj.weight
+Tensor: encoder.layers.0.self_attn.v_proj.weight
 ══════════════════════════════════════════════════════════════════════
-Shape: [384, 384] = 147456 elements
-Dtype: F32
-Offset: 0x013B0000 (20643840 bytes)
-Size: 589824 bytes
-00000000: 0A D7 23 3C 0A D7 23 3C  0A D7 23 3C 0A D7 23 3C  |..#<..#<..#<..#<|
-... 589,808 more bytes
+Shape: [384, 384] = 147456 elements
+Dtype: F32
+Offset: 0x013B0000 (20643840 bytes)
+Size: 589824 bytes
+
+Hex dump (first 16 bytes):
+00000000: 0A D7 23 3C 0A D7 23 3C 0A D7 23 3C 0A D7 23 3C  |     0.0100     0.0100     0.0100     0.0100 
+... 147452 more elements
 

crates/apr-cli/tests/cli_integration.rs

@@ -268,12 +268,32 @@ fn test_qa_016_validate_corrupted() {
 fn test_qa_016_validate_quality_score() {
     let file = create_test_apr_file();
 
-    // Minimal test APR file scores 4/100 — validate --quality exits non-zero (below 50% threshold).
-    // We verify the quality score output is present regardless of exit code.
+    // `--quality` must print a score whose DENOMINATOR is the checks that actually ran.
+    //
+    // This assertion used to demand `/100` or `points` — the wording #2394 finding 12
+    // removed, and the reason it removed it is recorded at
+    // `crates/apr-cli/src/commands/validate.rs::summary_line`: the report declares 26
+    // checks of which a handful run, so `✓ VALID 3/100 points` put a green badge next to
+    // what reads as 3%, against a denominator nothing was measured on. The producer was
+    // fixed; this test was not, and it had never run in CI (`cli_integration` is not on
+    // ci.yml's `--test` line), so it sat RED on main until BSE-17's quick tier selected
+    // it (#3051, the "integration targets never run" class of #2341).
+    //
+    // The rewrite is a GUARD for that fix rather than a relic of it: the retired wording
+    // must be ABSENT, so a regression to `/100 points` turns this test RED.
     apr()
         .args(["validate", file.path().to_str().unwrap(), "--quality"])
         .assert()
-        .stdout(predicate::str::contains("/100").or(predicate::str::contains("points")));
+        .stdout(predicate::str::contains("SCORE:"))
+        .stdout(
+            predicate::str::contains("% of the checks that ran")
+                .or(predicate::str::contains("SCORE: unavailable")),
+        )
+        .stdout(
+            predicate::str::contains("checks that ran")
+                .or(predicate::str::contains("nothing was measured")),
+        )
+        .stdout(predicate::str::contains("/100 points").not());
 }
 
 // ============================================================================

crates/apr-cli/tests/command_coverage.rs

@@ -193,10 +193,27 @@ fn test_coverage_validate_quality_rich() {
         .output()
         .expect("run validate --quality");
     let stdout = String::from_utf8_lossy(&output.stdout);
-    // Quality mode produces a score table
+    // The SECOND instance of the same stale assertion (#3051; the first is
+    // cli_integration.rs::test_qa_016_validate_quality_score). It asked for `/100`,
+    // `points` or `Score` — the first two are the wording #2394 finding 12 removed as
+    // dishonest, and the third never matched because the line reads `SCORE:` in caps.
+    // So all three arms were false and the test had simply never run: `command_coverage`
+    // is not on ci.yml's `--test` line either.
+    //
+    // Same rewrite as its sibling: assert the contract the code owes — a score measured
+    // against the checks that RAN — and require the retired wording to be ABSENT, so a
+    // regression to `/100 points` turns this test RED.
     assert!(
-        stdout.contains("/100") || stdout.contains("points") || stdout.contains("Score"),
-        "validate --quality must show score, got: {stdout}"
+        stdout.contains("SCORE:"),
+        "validate --quality must print a SCORE: line, got: {stdout}"
+    );
+    assert!(
+        stdout.contains("% of the checks that ran") || stdout.contains("SCORE: unavailable"),
+        "the score's denominator must be the checks that ran, got: {stdout}"
+    );
+    assert!(
+        !stdout.contains("/100 points"),
+        "the retired `/100 points` wording is back (#2394 finding 12), got: {stdout}"
     );
 }
 

crates/apr-cli/tests/falsification_crux_d_11.rs

@@ -49,6 +49,43 @@ fn good_t4() -> serde_json::Value {
 
 // ===== g2: CLI shape =====
 
+/// A `*-lint --json` outcome field says OK — in whichever of the THREE shapes that command
+/// emits, because the surface is not uniform and this test is not the place to decide it.
+///
+/// #3051. The assertion here was `parsed[field].as_str().expect(..).contains("Ok")`, and it
+/// panicked in the `expect` on clean `main` — invisibly, because this target is not on
+/// ci.yml's `--test` line and had never run in CI until BSE-17's quick tier selected it.
+///
+/// MEASURED on main, one outcome field per row, all from `--json`:
+///
+/// | shape | example | seen in |
+/// |---|---|---|
+/// | structured object | `{"status": "ok", "efficiency": 0.875}` | `ddp-metrics-lint` |
+/// | bare string | `"Ok"` | `prometheus-lint` `required_metrics` |
+/// | stringified Rust `Debug` | `"Ok { code: 134 }"` | `nccl-diag-lint` `exit_code` |
+///
+/// `prometheus-lint` emits two of the three within ONE response. Shape 3 is a `Debug` string
+/// leaking into a JSON API. That is a real finding about the `--json` surface, filed
+/// separately; normalising it inside a test would hide it, and picking one shape here would
+/// assert a design decision the tree has not made.
+///
+/// `.contains("Ok")` was also a pass-grep in Rust clothing — `"NotOk"` contains `"Ok"`. The
+/// string arms therefore match at a token boundary and the object arm reads `status`, so an
+/// outcome that is not ok can no longer satisfy this assertion.
+fn assert_outcome_ok(parsed: &serde_json::Value, field: &str) {
+    let v = &parsed[field];
+    if let Some(s) = v.as_str() {
+        let ok = s == "Ok" || s == "ok" || s.starts_with("Ok {") || s.starts_with("Ok(");
+        assert!(ok, "{field} outcome is not ok: {s:?}");
+        return;
+    }
+    assert_eq!(
+        v["status"].as_str(),
+        Some("ok"),
+        "{field} outcome is neither an Ok string nor an object with status ok: {v}"
+    );
+}
+
 #[test]
 fn falsify_crux_d_11_cli_help_advertises_flags() {
     let out = apr_binary()
@@ -266,16 +303,7 @@ fn falsify_crux_d_11_json_output_contains_outcomes() {
     assert!(out.status.success(), "json + good bodies must exit 0");
     let stdout = String::from_utf8_lossy(&out.stdout);
     let parsed: serde_json::Value = serde_json::from_str(&stdout).expect("json output must parse");
-    assert!(parsed["scaling_efficiency"]
-        .as_str()
-        .expect("scaling")
-        .contains("Ok"));
-    assert!(parsed["loss_parity"]
-        .as_str()
-        .expect("loss_parity")
-        .contains("Ok"));
-    assert!(parsed["allreduce_bandwidth"]
-        .as_str()
-        .expect("allreduce")
-        .contains("Ok"));
+    assert_outcome_ok(&parsed, "scaling_efficiency");
+    assert_outcome_ok(&parsed, "loss_parity");
+    assert_outcome_ok(&parsed, "allreduce_bandwidth");
 }

crates/apr-cli/tests/falsification_crux_f_06.rs

@@ -46,6 +46,43 @@ fn good_timeline() -> serde_json::Value {
 
 // ===== g2: CLI shape =====
 
+/// A `*-lint --json` outcome field says OK — in whichever of the THREE shapes that command
+/// emits, because the surface is not uniform and this test is not the place to decide it.
+///
+/// #3051. The assertion here was `parsed[field].as_str().expect(..).contains("Ok")`, and it
+/// panicked in the `expect` on clean `main` — invisibly, because this target is not on
+/// ci.yml's `--test` line and had never run in CI until BSE-17's quick tier selected it.
+///
+/// MEASURED on main, one outcome field per row, all from `--json`:
+///
+/// | shape | example | seen in |
+/// |---|---|---|
+/// | structured object | `{"status": "ok", "efficiency": 0.875}` | `ddp-metrics-lint` |
+/// | bare string | `"Ok"` | `prometheus-lint` `required_metrics` |
+/// | stringified Rust `Debug` | `"Ok { code: 134 }"` | `nccl-diag-lint` `exit_code` |
+///
+/// `prometheus-lint` emits two of the three within ONE response. Shape 3 is a `Debug` string
+/// leaking into a JSON API. That is a real finding about the `--json` surface, filed
+/// separately; normalising it inside a test would hide it, and picking one shape here would
+/// assert a design decision the tree has not made.
+///
+/// `.contains("Ok")` was also a pass-grep in Rust clothing — `"NotOk"` contains `"Ok"`. The
+/// string arms therefore match at a token boundary and the object arm reads `status`, so an
+/// outcome that is not ok can no longer satisfy this assertion.
+fn assert_outcome_ok(parsed: &serde_json::Value, field: &str) {
+    let v = &parsed[field];
+    if let Some(s) = v.as_str() {
+        let ok = s == "Ok" || s == "ok" || s.starts_with("Ok {") || s.starts_with("Ok(");
+        assert!(ok, "{field} outcome is not ok: {s:?}");
+        return;
+    }
+    assert_eq!(
+        v["status"].as_str(),
+        Some("ok"),
+        "{field} outcome is neither an Ok string nor an object with status ok: {v}"
+    );
+}
+
 #[test]
 fn falsify_crux_f_06_cli_help_advertises_flags() {
     let out = apr_binary()
@@ -258,21 +295,9 @@ fn falsify_crux_f_06_json_output_contains_outcomes() {
     assert!(out.status.success(), "json + good body must exit 0");
     let stdout = String::from_utf8_lossy(&out.stdout);
     let parsed: serde_json::Value = serde_json::from_str(&stdout).expect("json output must parse");
-    assert!(parsed["schema"].as_str().expect("schema").contains("Ok"));
-    assert!(parsed["block_conservation"]
-        .as_str()
-        .expect("block")
-        .contains("Ok"));
-    assert!(parsed["used_pct_arithmetic"]
-        .as_str()
-        .expect("used_pct")
-        .contains("Ok"));
-    assert!(parsed["peak_consistency"]
-        .as_str()
-        .expect("peak")
-        .contains("Ok"));
-    assert!(parsed["preemption_trigger"]
-        .as_str()
-        .expect("preempt")
-        .contains("Ok"));
+    assert_outcome_ok(&parsed, "schema");
+    assert_outcome_ok(&parsed, "block_conservation");
+    assert_outcome_ok(&parsed, "used_pct_arithmetic");
+    assert_outcome_ok(&parsed, "peak_consistency");
+    assert_outcome_ok(&parsed, "preemption_trigger");
 }

crates/apr-cli/tests/falsification_crux_f_07.rs

@@ -44,6 +44,43 @@ fn good_trace() -> serde_json::Value {
 
 // ===== g2: CLI shape =====
 
+/// A `*-lint --json` outcome field says OK — in whichever of the THREE shapes that command
+/// emits, because the surface is not uniform and this test is not the place to decide it.
+///
+/// #3051. The assertion here was `parsed[field].as_str().expect(..).contains("Ok")`, and it
+/// panicked in the `expect` on clean `main` — invisibly, because this target is not on
+/// ci.yml's `--test` line and had never run in CI until BSE-17's quick tier selected it.
+///
+/// MEASURED on main, one outcome field per row, all from `--json`:
+///
+/// | shape | example | seen in |
+/// |---|---|---|
+/// | structured object | `{"status": "ok", "efficiency": 0.875}` | `ddp-metrics-lint` |
+/// | bare string | `"Ok"` | `prometheus-lint` `required_metrics` |
+/// | stringified Rust `Debug` | `"Ok { code: 134 }"` | `nccl-diag-lint` `exit_code` |
+///
+/// `prometheus-lint` emits two of the three within ONE response. Shape 3 is a `Debug` string
+/// leaking into a JSON API. That is a real finding about the `--json` surface, filed
+/// separately; normalising it inside a test would hide it, and picking one shape here would
+/// assert a design decision the tree has not made.
+///
+/// `.contains("Ok")` was also a pass-grep in Rust clothing — `"NotOk"` contains `"Ok"`. The
+/// string arms therefore match at a token boundary and the object arm reads `status`, so an
+/// outcome that is not ok can no longer satisfy this assertion.
+fn assert_outcome_ok(parsed: &serde_json::Value, field: &str) {
+    let v = &parsed[field];
+    if let Some(s) = v.as_str() {
+        let ok = s == "Ok" || s == "ok" || s.starts_with("Ok {") || s.starts_with("Ok(");
+        assert!(ok, "{field} outcome is not ok: {s:?}");
+        return;
+    }
+    assert_eq!(
+        v["status"].as_str(),
+        Some("ok"),
+        "{field} outcome is neither an Ok string nor an object with status ok: {v}"
+    );
+}
+
 #[test]
 fn falsify_crux_f_07_cli_help_advertises_trace_file() {
     let out = apr_binary()
@@ -244,13 +281,7 @@ fn falsify_crux_f_07_json_output_contains_outcomes() {
     assert!(out.status.success(), "json + good body must exit 0");
     let stdout = String::from_utf8_lossy(&out.stdout);
     let parsed: serde_json::Value = serde_json::from_str(&stdout).expect("json output must parse");
-    assert!(parsed["schema"].as_str().expect("schema").contains("Ok"));
-    assert!(parsed["alloc_free_pairing"]
-        .as_str()
-        .expect("pairing")
-        .contains("Ok"));
-    assert!(parsed["monotonic_timestamps"]
-        .as_str()
-        .expect("ts")
-        .contains("Ok"));
+    assert_outcome_ok(&parsed, "schema");
+    assert_outcome_ok(&parsed, "alloc_free_pairing");
+    assert_outcome_ok(&parsed, "monotonic_timestamps");
 }