From f0248ca169b9906387a6c220f9b6575f483624fd Mon Sep 17 00:00:00 2001 From: copyleftdev Date: Wed, 29 Jul 2026 22:10:54 -0700 Subject: [PATCH] feat(invariants): consume relationship hints, reporting expected-but-absent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `VajraPlugin::relationship_hints` was declared by every domain plugin and consumed by nothing — vajra-cli, vajra-essence and vajra-core never read it. Roughly 68 curated hints across six plugins were inert. That is dead surface under CLAUDE.md rule 6, and worse, it misleads plugin authors into thinking declaring a hint does something. Wired up rather than deleted, because the hint data is real domain knowledge worth keeping. `invariants` now matches each applicable hint against what it discovered and reports the outcome. The valuable half is absence, not confirmation. `observed: false` means the domain expects those fields to relate, the fields are present, and the data does not show it. On GitHub-shaped input the `pr_review_assignment` hint expects pr_author to determine reviewer; measured strength is 0.0, so reviewers are being assigned independently of who wrote the PR. Unobserved hints sort first for that reason. Deliberate constraints: - Hints never weight the information theory. Conditional entropy and mutual information are measured from the data alone; a hint is context laid alongside them. Weighting would make the numbers unfalsifiable. - A hint is evaluated only when two or more of its fields are present. One present field is untestable, and reporting it as a failure would be noise. - Confirmation requires spanning two *distinct* patterns of the hint. Two paths matching the same pattern prove nothing about it. - `max_strength` is always reported, so a near-miss below the confirmation threshold stays visible rather than collapsing to a bare false. - When no hint applies — the common case — output stays the documented flat array rather than gaining a wrapper. Caught while testing: the first implementation silently never fired. Two path vocabularies are in play — `invariants` names fields relative to each record (`$.reviewer`) while the document trie names them relative to the whole document (`$[*].reviewer`) — and intersecting the two sets never matches. Matching is now on the trailing key throughout, with a regression test pinning the exact mismatch. Closes #74. --- Cargo.lock | 11 ++ docs/src/cmd-invariants.md | 26 ++++ vajra-cli/src/hints.rs | 274 +++++++++++++++++++++++++++++++++++++ vajra-cli/src/main.rs | 82 ++++++++++- 4 files changed, 391 insertions(+), 2 deletions(-) create mode 100644 vajra-cli/src/hints.rs diff --git a/Cargo.lock b/Cargo.lock index 77865ce..ca2f7a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1863,6 +1863,7 @@ dependencies = [ "vajra-domain-encoding", "vajra-domain-github", "vajra-domain-med", + "vajra-domain-package", "vajra-domain-sec", "vajra-domain-source", "vajra-drift", @@ -1944,6 +1945,16 @@ dependencies = [ "vajra-types", ] +[[package]] +name = "vajra-domain-package" +version = "0.5.0" +dependencies = [ + "proptest", + "regex", + "serde_json", + "vajra-types", +] + [[package]] name = "vajra-domain-sec" version = "0.5.0" diff --git a/docs/src/cmd-invariants.md b/docs/src/cmd-invariants.md index f41241e..35290aa 100644 --- a/docs/src/cmd-invariants.md +++ b/docs/src/cmd-invariants.md @@ -73,6 +73,32 @@ Every result records `field_x_binned` / `field_y_binned` so you can tell which v **Known limitation.** Binning fixes *numeric* degeneracy only. A high-cardinality **string** field — a primary key, a UUID, a filename — is still near-unique and will still report a perfect relationship with everything. Treat any near-1.0 strength involving an identifier-like field as an artefact, not a finding. +### Domain Expectations + +Domain plugins declare *relationship hints* — "in GitHub data, `reviewer`, `pr_author` and `review_state` co-occur". When any plugin's hints apply to the document, `invariants` reports how each fared: + +```json +{ + "relationships": [ ... ], + "domain_hints": [ + { "name": "pr_review_assignment", "relationship": "FunctionalDependency", + "weight": 0.85, "observed": false, "max_strength": 0.0, + "matched_fields": ["$[*].pr_author", "$[*].reviewer"] }, + { "name": "review_network", "relationship": "CoOccurrence", + "weight": 0.9, "observed": true, "max_strength": 1.0, + "matched_fields": ["$[*].review_state", "$[*].reviewer"] } + ] +} +``` + +**The absent ones are usually the interesting result.** `observed: false` means the domain expects those fields to relate, the fields *are* present, and this data does not show it. Above, `pr_review_assignment` expects `pr_author` to determine `reviewer`; the measured strength is 0.0, so reviewers are being assigned independently of who wrote the PR. Unobserved hints are listed first for that reason. + +A hint is only evaluated when at least two of its fields are present — one field is untestable, and reporting it as a failure would be noise. Confirmation requires a relationship spanning two *distinct* patterns of the hint: two paths matching the same pattern prove nothing about it. + +**Hints never weight the information theory.** Conditional entropy and mutual information are measured from the data alone; a hint is context laid alongside them, never an adjustment to them. `max_strength` reports the strongest relationship actually found among the hint's fields, so a near-miss below the confirmation threshold is still visible. + +When no hint applies — the common case for non-domain data — the output stays the flat array documented below, rather than gaining a wrapper. + ### Direction Matters: `relationship_strength` vs `mutual_information` `relationship_strength` normalises by the target's own entropy: diff --git a/vajra-cli/src/hints.rs b/vajra-cli/src/hints.rs new file mode 100644 index 0000000..f04ffa4 --- /dev/null +++ b/vajra-cli/src/hints.rs @@ -0,0 +1,274 @@ +//! Matching discovered relationships against domain relationship hints. +//! +//! Domain plugins declare hints — "in GitHub data, `reviewer`, `pr_author` and +//! `review_state` co-occur" — and `invariants` discovers relationships +//! empirically. Connecting the two gives two things neither has alone: +//! +//! - **confirmation**: a discovered relationship that a domain expected, which +//! is stronger evidence than an unexplained correlation. +//! - **absence**: a relationship the domain expected, whose fields are present, +//! that was *not* found. That is often the more interesting result — a merge +//! gate that should tie `state` to `merged_by` and doesn't is a finding. +//! +//! Hints deliberately do **not** weight the information theory. Conditional +//! entropy is measured from the data; a hint is context laid alongside it, never +//! an adjustment to it. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::Serialize; +use vajra_types::traits::RelationshipHint; + +/// A hint matched against what was discovered. +#[derive(Debug, Clone, Serialize)] +pub struct HintOutcome { + /// The hint's name. + pub name: String, + /// The relationship the domain expects. + pub relationship: String, + /// The hint's declared weight, as authored by the plugin. + pub weight: f64, + /// Document paths matching the hint's field patterns. + pub matched_fields: Vec, + /// Whether a discovered relationship connected two of those fields. + pub observed: bool, + /// Strongest relationship strength found among the hint's fields. + pub max_strength: f64, +} + +/// Does a document path match a hint field pattern? +/// +/// Patterns are authored as `**/name`, so matching is on the trailing key. That +/// keeps a hint portable across `$.pr.reviewer`, `$[*].reviewer` and +/// `$.items[*].pr.reviewer` without the plugin having to know the shape. +fn pattern_matches(pattern: &str, path: &str) -> bool { + let key = pattern.rsplit('/').next().unwrap_or(pattern); + if key.is_empty() { + return false; + } + path.rsplit('.').next().is_some_and(|last| last == key) +} + +/// Evaluate every hint against the document's paths and discovered relationships. +/// +/// `relationships` is `(field_x, field_y, strength)`. A hint counts as observed +/// when some discovered relationship connects two *different* patterns of that +/// hint at or above `min_strength` — a field related to itself, or two paths +/// matching the same pattern, proves nothing about the hint. +pub fn evaluate_hints( + hints: &[RelationshipHint], + document_paths: &BTreeSet, + relationships: &[(String, String, f64)], + min_strength: f64, +) -> Vec { + let mut out = Vec::new(); + + for hint in hints { + // Testability: which of the hint's patterns reach a real document path? + // A hint needs at least two present fields to say anything. + // + // Note the two path vocabularies in play. `invariants` reports fields + // relative to each *record* (`$.reviewer`) while the document trie + // reports them relative to the whole document (`$[*].reviewer`). + // Everything here therefore matches on the trailing key rather than + // intersecting the two sets, which would never match. + let mut present: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for pattern in &hint.fields { + let hit: BTreeSet<&str> = document_paths + .iter() + .filter(|p| pattern_matches(pattern, p)) + .map(String::as_str) + .collect(); + if !hit.is_empty() { + present.insert(pattern.as_str(), hit); + } + } + if present.len() < 2 { + continue; + } + + let matched_fields: BTreeSet<&str> = + present.values().flat_map(|s| s.iter().copied()).collect(); + + let mut max_strength = 0.0_f64; + let mut observed = false; + for (x, y, strength) in relationships { + // Which of this hint's present patterns does each side match? + let x_pat: Vec<&str> = present + .keys() + .copied() + .filter(|pat| pattern_matches(pat, x)) + .collect(); + let y_pat: Vec<&str> = present + .keys() + .copied() + .filter(|pat| pattern_matches(pat, y)) + .collect(); + // Must span two *distinct* patterns: a field related to another + // field matching the same pattern proves nothing about the hint. + let spans = x_pat.iter().any(|xp| y_pat.iter().any(|yp| xp != yp)); + if !spans { + continue; + } + if *strength > max_strength { + max_strength = *strength; + } + if *strength >= min_strength { + observed = true; + } + } + + out.push(HintOutcome { + name: hint.name.clone(), + relationship: format!("{:?}", hint.relationship), + weight: hint.weight, + matched_fields: matched_fields.into_iter().map(str::to_owned).collect(), + observed, + max_strength, + }); + } + + // Unobserved first — an expected relationship that is missing is the more + // actionable result. Then by name, so ordering is fully specified. + out.sort_by(|a, b| { + a.observed + .cmp(&b.observed) + .then_with(|| a.name.cmp(&b.name)) + }); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use vajra_types::traits::RelationshipType; + + fn hint(name: &str, fields: &[&str]) -> RelationshipHint { + RelationshipHint { + name: name.to_owned(), + fields: fields.iter().map(|f| (*f).to_owned()).collect(), + relationship: RelationshipType::CoOccurrence, + weight: 0.9, + } + } + + fn paths(items: &[&str]) -> BTreeSet { + items.iter().map(|s| (*s).to_owned()).collect() + } + + #[test] + fn pattern_matches_trailing_key_at_any_depth() { + assert!(pattern_matches("**/reviewer", "$.pr.reviewer")); + assert!(pattern_matches("**/reviewer", "$[*].reviewer")); + assert!(pattern_matches("**/reviewer", "$.items[*].pr.reviewer")); + assert!(!pattern_matches("**/reviewer", "$.reviewer_id")); + assert!(!pattern_matches("**/reviewer", "$.pr.author")); + } + + #[test] + fn hint_is_observed_when_a_relationship_spans_two_of_its_fields() { + let hints = [hint("gate", &["**/state", "**/merged_by"])]; + let p = paths(&["$[*].state", "$[*].merged_by"]); + let rels = vec![("$[*].state".to_owned(), "$[*].merged_by".to_owned(), 0.8)]; + let out = evaluate_hints(&hints, &p, &rels, 0.5); + assert_eq!(out.len(), 1); + assert!(out[0].observed); + assert!((out[0].max_strength - 0.8).abs() < 1e-9); + assert_eq!(out[0].matched_fields.len(), 2); + } + + /// The point of the feature: fields present, relationship absent. + #[test] + fn hint_is_unobserved_when_the_relationship_is_missing() { + let hints = [hint("gate", &["**/state", "**/merged_by"])]; + let p = paths(&["$[*].state", "$[*].merged_by"]); + let out = evaluate_hints(&hints, &p, &[], 0.5); + assert_eq!(out.len(), 1); + assert!(!out[0].observed, "no relationship was discovered"); + assert!(out[0].max_strength.abs() < 1e-9); + } + + #[test] + fn weak_relationship_does_not_count_as_observed() { + let hints = [hint("gate", &["**/state", "**/merged_by"])]; + let p = paths(&["$[*].state", "$[*].merged_by"]); + let rels = vec![("$[*].state".to_owned(), "$[*].merged_by".to_owned(), 0.1)]; + let out = evaluate_hints(&hints, &p, &rels, 0.5); + assert!(!out[0].observed, "0.1 is below the 0.5 threshold"); + // But the strength found is still reported, so the near-miss is visible. + assert!((out[0].max_strength - 0.1).abs() < 1e-9); + } + + /// A hint whose fields are mostly absent is untestable and must be skipped + /// rather than reported as a failure. + #[test] + fn hint_needs_two_present_fields_to_be_evaluated() { + let hints = [hint("gate", &["**/state", "**/merged_by"])]; + let out = evaluate_hints(&hints, &paths(&["$[*].state"]), &[], 0.5); + assert!(out.is_empty(), "one field present is not testable"); + let none = evaluate_hints(&hints, &paths(&["$[*].unrelated"]), &[], 0.5); + assert!(none.is_empty()); + } + + /// Two paths matching the *same* pattern prove nothing about the hint. + #[test] + fn relationship_within_one_pattern_does_not_confirm() { + let hints = [hint("gate", &["**/state", "**/merged_by"])]; + let p = paths(&["$.a.state", "$.b.state", "$[*].merged_by"]); + let rels = vec![("$.a.state".to_owned(), "$.b.state".to_owned(), 0.99)]; + let out = evaluate_hints(&hints, &p, &rels, 0.5); + assert!(!out[0].observed, "state<->state spans one pattern, not two"); + } + + #[test] + fn unobserved_hints_are_listed_first() { + let hints = [ + hint("zzz_seen", &["**/a", "**/b"]), + hint("aaa_missing", &["**/c", "**/d"]), + ]; + let p = paths(&["$.a", "$.b", "$.c", "$.d"]); + let rels = vec![("$.a".to_owned(), "$.b".to_owned(), 0.9)]; + let out = evaluate_hints(&hints, &p, &rels, 0.5); + assert_eq!(out.len(), 2); + assert_eq!(out[0].name, "aaa_missing", "missing first"); + assert!(!out[0].observed); + assert!(out[1].observed); + } + + /// Regression: `invariants` names fields relative to each record + /// (`$.reviewer`) while the document trie names them relative to the whole + /// document (`$[*].reviewer`). Matching must bridge that, or hints silently + /// never fire. + #[test] + fn bridges_record_and_document_path_vocabularies() { + let hints = [hint("review_network", &["**/reviewer", "**/review_state"])]; + // Document paths as the trie reports them. + let p = paths(&["$[*].reviewer", "$[*].review_state"]); + // Relationships as `invariants` reports them. + let rels = vec![( + "$.reviewer".to_owned(), + "$.review_state".to_owned(), + 1.0_f64, + )]; + let out = evaluate_hints(&hints, &p, &rels, 0.25); + assert_eq!(out.len(), 1); + assert!( + out[0].observed, + "record-relative relationship must match document-relative paths" + ); + assert!((out[0].max_strength - 1.0).abs() < 1e-9); + } + + #[test] + fn is_deterministic() { + let hints = [hint("b", &["**/x", "**/y"]), hint("a", &["**/x", "**/y"])]; + let p = paths(&["$.x", "$.y"]); + let rels = vec![("$.x".to_owned(), "$.y".to_owned(), 0.7)]; + let one = evaluate_hints(&hints, &p, &rels, 0.5); + let two = evaluate_hints(&hints, &p, &rels, 0.5); + let names = + |v: &[HintOutcome]| -> Vec { v.iter().map(|h| h.name.clone()).collect() }; + assert_eq!(names(&one), names(&two)); + assert_eq!(names(&one), vec!["a", "b"]); + } +} diff --git a/vajra-cli/src/main.rs b/vajra-cli/src/main.rs index 8b1af13..c0cbf6f 100644 --- a/vajra-cli/src/main.rs +++ b/vajra-cli/src/main.rs @@ -1,5 +1,6 @@ mod batch; mod corpus; +mod hints; mod treediff; use std::collections::BTreeMap; @@ -1075,6 +1076,31 @@ fn collect_recognizers() -> Vec> { recognizers } +/// Collect relationship hints from enabled domain plugins. +fn collect_hints() -> Vec { + let mut hints: Vec = Vec::new(); + macro_rules! add { + ($plugin:expr) => { + hints.extend(vajra_types::traits::VajraPlugin::relationship_hints( + &$plugin, + )); + }; + } + #[cfg(feature = "medical")] + add!(vajra_domain_med::MedicalPlugin); + #[cfg(feature = "security")] + add!(vajra_domain_sec::SecurityPlugin); + #[cfg(feature = "devops")] + add!(vajra_domain_devops::DevOpsPlugin); + #[cfg(feature = "source")] + add!(vajra_domain_source::SourcePlugin); + #[cfg(feature = "github")] + add!(vajra_domain_github::GitHubPlugin); + #[cfg(feature = "encoding")] + add!(vajra_domain_encoding::EncodingPlugin); + hints +} + /// Collect structural detectors from enabled domain plugins. fn collect_detectors() -> Vec> { let mut detectors: Vec> = Vec::new(); @@ -2728,6 +2754,31 @@ fn cmd_invariants(input: &str, top_k: usize, bin: &str, cli: &Cli) -> Result<()> let relationships = vajra_stats::relationships::discover_relationships_binned(&doc, top_k, bins); + // Domain hints declare relationships a domain expects. Matching them against + // what was discovered surfaces both confirmation and, more usefully, + // expected relationships that are absent. Hints never weight the + // information theory — they are context laid alongside it. + let hint_outcomes = { + let all = collect_hints(); + if all.is_empty() { + Vec::new() + } else { + let document_paths: std::collections::BTreeSet = + doc.trie().all_paths().iter().map(|p| p.as_str()).collect(); + let pairs: Vec<(String, String, f64)> = relationships + .iter() + .map(|r| { + ( + r.field_x.as_str(), + r.field_y.as_str(), + r.relationship_strength, + ) + }) + .collect(); + hints::evaluate_hints(&all, &document_paths, &pairs, 0.25) + } + }; + match cli.format { Format::Json => { let rels_json: Vec = relationships @@ -2745,8 +2796,17 @@ fn cmd_invariants(input: &str, top_k: usize, bin: &str, cli: &Cli) -> Result<()> }) }) .collect(); - let json = - serde_json::to_string_pretty(&rels_json).context("JSON serialization failed")?; + let json = if hint_outcomes.is_empty() { + // Preserve the documented flat-array contract when no domain + // hint applies, which is the common case. + serde_json::to_string_pretty(&rels_json) + } else { + serde_json::to_string_pretty(&serde_json::json!({ + "relationships": rels_json, + "domain_hints": hint_outcomes, + })) + } + .context("JSON serialization failed")?; println!("{json}"); } Format::Text | Format::Markdown | Format::CompactAi => { @@ -2787,6 +2847,24 @@ fn cmd_invariants(input: &str, top_k: usize, bin: &str, cli: &Cli) -> Result<()> println!(" [b] = numeric field discretised before analysis (see --bin)."); } } + + if !hint_outcomes.is_empty() { + println!(); + println!("=== Domain Expectations ==="); + for h in &hint_outcomes { + println!( + " {:<10} {:<28} {:<20} max strength {:.4}", + if h.observed { "observed" } else { "MISSING" }, + h.name, + h.relationship, + h.max_strength + ); + } + println!(); + println!(" MISSING means the domain expects these fields to relate and this"); + println!(" document's data does not show it. Hints are context only — they"); + println!(" never weight the entropy measures above."); + } } }