Bound sequence spars_merge with partial Ruzicka on indexed multisets

DESIGN.md

@@ -937,7 +937,7 @@ the row's value at that key, so the accumulated conjunction is the row's exact
 similarity and no further comparison is needed. Signature keys carry no usable
 value, and a sequence's terms bound its similarity without determining it, so in
 both cases the merge generator scores each surviving candidate through the
-comparator, as `spars` does. `inverted_hybrid` applies the generator
+comparator. `inverted_hybrid` applies the generator
 independently to each of the two, so its term index scores from the conjunction
 while its signature index verifies.
 
@@ -948,11 +948,12 @@ the exact similarity. Jaccard, Ruzicka, and L2 implement the hooks, so
 `spars_merge` applies partial-conjunction bounds there. Inverted-list values are
 materialized only where those bounds read them.
 
-Sequence comparators implement prefix bounds for `spars` but not conjunction
-hooks, so `spars_merge` does not accumulate partial conjunctions over their
-indexed multisets. On `inverted_term`, `gld` and `ngld` may still use
-`spars_merge`: the generator aligns rows on shared keys and scores each surviving
-candidate through the comparator, as `spars` does. Signature-keyed structures
+Sequence comparators implement prefix bounds for `spars` but not `ConjunctionScored`
+hooks on ordered sequences. On `inverted_term`, `spars_merge`
+still bounds rows with partial Ruzicka conjunction over the indexed term
+multiset: the configured comparator supplies the minimum shared-key fraction,
+and verification scores each surviving candidate through the edit distance.
+Signature-keyed structures
 and the signature half of `inverted_hybrid` still reject `spars_merge` with
 sequence comparators, because that half requires a conjunction-scoring comparator.
 

README.md

@@ -427,8 +427,9 @@ row as soon as no completion of it can reach the current minimum similarity. It
 pays off when queries have many keys and the minimum similarity rejects most
 rows early. It is available for `l2`, `jaccard`, and `ruzicka`, which implement
 partial-conjunction bounds, and for `gld` and `ngld` on `inverted_term`, where
-`spars_merge` aligns rows on shared keys and scores each candidate through the
-configured comparator, as `spars` does.
+`spars_merge` bounds rows with partial Ruzicka conjunction over the indexed
+multiset and verifies each surviving candidate with the configured sequence
+comparator on the ordered terms.
 
 ## What USSI Does Not Do
 

src/main/java/com/uber/ussi/comparator/BaseRuzickaComparator.java

@@ -163,6 +163,17 @@ public double conjunctionContribution(float value1, float value2) {
     return Math.min(getUniTransformedValue(value1), getUniTransformedValue(value2));
   }
 
+  @Override
+  public double conjunctionUnionContribution(
+      Comparator comparator, float value1, float value2) {
+    double transformedValue1 = getUniTransformedValue(value1);
+    double transformedValue2 = comparator.getUniTransformedValue(value2);
+    if (Math.signum(value1) == Math.signum(value2)) {
+      return Math.max(transformedValue1, transformedValue2);
+    }
+    return transformedValue1 + transformedValue2;
+  }
+
   @Override
   public double similarityFromConjunction(
       double conjunction,
@@ -179,18 +190,22 @@ public double similarityFromConjunction(
   }
 
   @Override
-  public double maxSimilarityFromPartialConjunction(
+  public final double maxSimilarityFromPartialConjunction(
       double conjunction,
+      double scannedUnion,
       double unscannedKeysUniValue,
       double partialUniValue1,
       double uniValue1,
       double partialUniValue2,
       double uniValue2) {
-    // The intersection can never exceed either row's Uni value, whatever the unscanned keys hold.
-    double maxConjunction =
-        Math.min(conjunction + unscannedKeysUniValue, Math.min(uniValue1, uniValue2));
-    return similarityFromConjunction(
-        maxConjunction, partialUniValue1, uniValue1, partialUniValue2, uniValue2);
+    return comparatorNormalizer.comparatorValueToNormalizedSimilarityValue(
+        computeMaxPossibleComparatorValue(
+            partialUniValue1,
+            uniValue1,
+            partialUniValue2,
+            uniValue2,
+            conjunction,
+            scannedUnion));
   }
 
   @Override

src/main/java/com/uber/ussi/comparator/BaseSequenceComparator.java

@@ -25,9 +25,9 @@
  * generation verifies each surviving candidate against the ordered sequences.
  *
  * <p>{@code spars} generates candidates with prefix bounds from this comparator over those indexed
- * keys and scores ordered sequences. {@code spars_merge} on {@code inverted_term} uses the same
- * scoring path but row-major merge over the lists instead of key-major {@code spars}
- * candidate generation.
+ * keys and scores ordered sequences. {@code spars_merge} on {@code inverted_term} bounds rows with
+ * partial Ruzicka conjunction over the indexed multiset, then scores each surviving candidate
+ * through this comparator on the ordered sequences.
  */
 abstract class BaseSequenceComparator extends Comparator implements KeyShareBounded {
 

src/main/java/com/uber/ussi/comparator/Comparator.java

@@ -22,6 +22,11 @@ public abstract class Comparator implements Serializable {
   protected final ComparatorNormalizer comparatorNormalizer;
   private final double zeroSimilarityComparatorValue;
 
+  /** Returns the normalizer that maps between comparator values and normalized similarities. */
+  public final ComparatorNormalizer getComparatorNormalizer() {
+    return comparatorNormalizer;
+  }
+
   protected Comparator(ComparatorNormalizer comparatorNormalizer) {
     if (comparatorNormalizer == null) {
       throw new NullPointerException("The comparatorNormalizer is null.");

src/main/java/com/uber/ussi/comparator/ComparatorFactory.java

@@ -9,6 +9,7 @@
 import com.uber.ussi.comparator.signaturegenerator.SignatureGeneratorFactory.SignatureGeneratorType;
 import com.uber.ussi.comparatornormalizer.ComparatorNormalizer;
 import com.uber.ussi.comparatornormalizer.ComparatorNormalizerFactory;
+import com.uber.ussi.comparatornormalizer.IdentityComparatorNormalizer;
 import com.uber.ussi.config.ConfigVocabulary;
 import com.uber.ussi.config.NamespaceConfig;
 import com.uber.ussi.config.NamespaceConfigParams;
@@ -21,6 +22,15 @@ public class ComparatorFactory {
 
   private ComparatorFactory() {}
 
+  /**
+   * Returns a {@link ConjunctionScored} measure over indexed term counts, for partial-conjunction
+   * bounds during sequence {@code spars_merge}. Verification still uses the configured sequence
+   * comparator on ordered sequences.
+   */
+  public static ConjunctionScored createIndexedMultisetMergeConjunctionScored() {
+    return new RuzickaComparator(new IdentityComparatorNormalizer());
+  }
+
   public static Comparator createComparator(
       String comparatorType,
       Map<String, String> comparatorParams,
@@ -67,6 +77,57 @@ public static Comparator tryCreateComparator(NamespaceConfig namespaceConfig) {
     }
   }
 
+  /** Returns the configured comparator type, or null when the param names none. */
+  @Nullable
+  public static ComparatorType getComparatorType(NamespaceConfig namespaceConfig) {
+    return ConfigVocabulary.fromParamValue(
+        ComparatorType.class, namespaceConfig.getComparatorType());
+  }
+
+  /**
+   * Returns the configured comparator as a {@link ConjunctionScored} measure. The {@code
+   * comparatorType} must state that the configured measure supports merge conjunction on itself.
+   */
+  public static ConjunctionScored conjunctionScored(
+      Comparator comparator, ComparatorType comparatorType) {
+    if (!comparatorType.similarityFromConfiguredConjunction()) {
+      throw new IllegalArgumentException(
+          String.format(
+              "Comparator type %s does not determine similarity from a merge conjunction.",
+              comparatorType.getParamValue()));
+    }
+    return (ConjunctionScored) comparator;
+  }
+
+  /**
+   * Returns the configured comparator as a {@link KeyShareBounded} measure. The {@code
+   * comparatorType} must state that the measure bounds key share.
+   */
+  public static KeyShareBounded keyShareBounded(
+      Comparator comparator, ComparatorType comparatorType) {
+    if (!comparatorType.boundsKeyShare()) {
+      throw new IllegalArgumentException(
+          String.format(
+              "Comparator type %s does not bound key share.", comparatorType.getParamValue()));
+    }
+    return (KeyShareBounded) comparator;
+  }
+
+  /**
+   * Returns the configured comparator as a {@link DotProductScored} measure. The {@code
+   * comparatorType} must state that similarity comes from a dot product.
+   */
+  public static DotProductScored dotProductScored(
+      Comparator comparator, ComparatorType comparatorType) {
+    if (!comparatorType.similarityFromDotProduct()) {
+      throw new IllegalArgumentException(
+          String.format(
+              "Comparator type %s does not derive similarity from a dot product.",
+              comparatorType.getParamValue()));
+    }
+    return (DotProductScored) comparator;
+  }
+
   /**
    * Returns the configured edit distance, defaulting to {@link SequenceDistanceType#LEVENSHTEIN}
    * when the param is absent.

src/main/java/com/uber/ussi/comparator/ComparatorType.java

@@ -12,24 +12,59 @@
  */
 public enum ComparatorType implements ConfigVocabulary {
   /** Generalized Levenshtein distance between two sequences. */
-  GLD(Generators.CONSISTENT_WEIGHTED_SAMPLING, /* comparesSequences */ true),
+  GLD(
+      Generators.CONSISTENT_WEIGHTED_SAMPLING,
+      /* comparesSequences */ true,
+      /* boundsKeyShare */ true,
+      /* similarityFromConfiguredConjunction */ false,
+      /* similarityFromDotProduct */ false),
 
-  JACCARD(Set.of(SignatureGeneratorType.MINHASH), /* comparesSequences */ false),
+  JACCARD(
+      Set.of(SignatureGeneratorType.MINHASH),
+      /* comparesSequences */ false,
+      /* boundsKeyShare */ true,
+      /* similarityFromConfiguredConjunction */ true,
+      /* similarityFromDotProduct */ false),
 
-  L2(Set.of(), /* comparesSequences */ false),
+  L2(
+      Set.of(),
+      /* comparesSequences */ false,
+      /* boundsKeyShare */ false,
+      /* similarityFromConfiguredConjunction */ true,
+      /* similarityFromDotProduct */ true),
 
   /** Normalized generalized Levenshtein distance between two sequences. */
-  NGLD(Generators.CONSISTENT_WEIGHTED_SAMPLING, /* comparesSequences */ true),
+  NGLD(
+      Generators.CONSISTENT_WEIGHTED_SAMPLING,
+      /* comparesSequences */ true,
+      /* boundsKeyShare */ true,
+      /* similarityFromConfiguredConjunction */ false,
+      /* similarityFromDotProduct */ false),
 
-  RUZICKA(Generators.CONSISTENT_WEIGHTED_SAMPLING, /* comparesSequences */ false);
+  RUZICKA(
+      Generators.CONSISTENT_WEIGHTED_SAMPLING,
+      /* comparesSequences */ false,
+      /* boundsKeyShare */ true,
+      /* similarityFromConfiguredConjunction */ true,
+      /* similarityFromDotProduct */ false);
 
   private final Set<SignatureGeneratorType> supportedSignatureGeneratorTypes;
   private final boolean comparesSequences;
+  private final boolean boundsKeyShare;
+  private final boolean similarityFromConfiguredConjunction;
+  private final boolean similarityFromDotProduct;
 
   ComparatorType(
-      Set<SignatureGeneratorType> supportedSignatureGeneratorTypes, boolean comparesSequences) {
+      Set<SignatureGeneratorType> supportedSignatureGeneratorTypes,
+      boolean comparesSequences,
+      boolean boundsKeyShare,
+      boolean similarityFromConfiguredConjunction,
+      boolean similarityFromDotProduct) {
     this.supportedSignatureGeneratorTypes = supportedSignatureGeneratorTypes;
     this.comparesSequences = comparesSequences;
+    this.boundsKeyShare = boundsKeyShare;
+    this.similarityFromConfiguredConjunction = similarityFromConfiguredConjunction;
+    this.similarityFromDotProduct = similarityFromDotProduct;
   }
 
   public Set<SignatureGeneratorType> getSupportedSignatureGeneratorTypes() {
@@ -45,6 +80,24 @@ public boolean comparesSequences() {
     return comparesSequences;
   }
 
+  /** Returns whether this measure can bound how much of a record's keys a candidate must share. */
+  public boolean boundsKeyShare() {
+    return boundsKeyShare;
+  }
+
+  /**
+   * Returns whether merge may treat the configured comparator's conjunction as the row's
+   * similarity on structures where conjunction determines similarity.
+   */
+  public boolean similarityFromConfiguredConjunction() {
+    return similarityFromConfiguredConjunction;
+  }
+
+  /** Returns whether this measure's similarity is determined by a dot product and two Uni values. */
+  public boolean similarityFromDotProduct() {
+    return similarityFromDotProduct;
+  }
+
   /**
    * The generator sets more than one comparator draws on. They live in a nested class because an
    * enum constructor cannot read a static field of its own enum.

src/main/java/com/uber/ussi/comparator/ConjunctionScored.java

@@ -20,6 +20,15 @@ public interface ConjunctionScored {
   /** Returns what one key the query and an indexed row share adds to the conjunction. */
   double conjunctionContribution(float value1, float value2);
 
+  /**
+   * Returns what one shared key adds to the scanned union mass during partial conjunction merge.
+   * Measures that do not use a multiset union return 0.0.
+   */
+  default double conjunctionUnionContribution(
+      Comparator comparator, float value1, float value2) {
+    return 0.0;
+  }
+
   /** Returns the normalized similarity implied by a complete conjunction. */
   double similarityFromConjunction(
       double conjunction,
@@ -29,11 +38,13 @@ public interface ConjunctionScored {
       double uniValue2);
 
   /**
-   * Returns the highest normalized similarity still reachable from a partial conjunction, where
-   * {@code unscannedKeysUniValue} bounds what the query's not-yet-merged keys can add.
+   * Returns the highest normalized similarity still reachable from a partial conjunction. For
+   * multiset measures, {@code scannedUnion} is the union mass accumulated over shared keys so
+   * far. For other measures it is unused.
    */
   double maxSimilarityFromPartialConjunction(
       double conjunction,
+      double scannedUnion,
       double unscannedKeysUniValue,
       double partialUniValue1,
       double uniValue1,

src/main/java/com/uber/ussi/comparator/KeyShareBounded.java

@@ -1,6 +1,8 @@
 /* AUTHOR: Shijie Lu (shijie@uber.com), Shalini Kedlaya (skedlaya@uber.com), Ahmed Metwally (ametwally@uber.com) */
 package com.uber.ussi.comparator;
 
+import com.uber.ussi.comparatornormalizer.ComparatorNormalizer;
+
 /**
  * A similarity that can say how much of a record's keys a qualifying candidate has to share with
  * it, rather than only how many of them it may miss.
@@ -37,4 +39,26 @@ public interface KeyShareBounded {
    * Comparator#maxPrefixSumFromSharedFraction} turn it into a prefix over either key space.
    */
   double getMinSharedKeyFraction(double recordUniValue, double comparatorValue);
+
+  /**
+   * Returns the minimum multiset similarity a partial conjunction must still be able to reach for
+   * a row to remain a candidate, given the query's indexed Uni value and the search's minimum
+   * normalized similarity.
+   */
+  default double minMultisetSimilarityForMergePartialConjunction(
+      ComparatorNormalizer comparatorNormalizer,
+      double queryUniValue,
+      double minNormalizedSimilarity) {
+    if (comparatorNormalizer == null) {
+      throw new NullPointerException("The comparatorNormalizer is null.");
+    }
+    if (minNormalizedSimilarity < 0.0 || minNormalizedSimilarity > 1.0) {
+      throw new IllegalArgumentException(
+          String.format(
+              "minNormalizedSimilarity must be in [0.0, 1.0], got %s.", minNormalizedSimilarity));
+    }
+    double comparatorValue =
+        comparatorNormalizer.normalizedSimilarityValueToComparatorValue(minNormalizedSimilarity);
+    return getMinSharedKeyFraction(queryUniValue, comparatorValue);
+  }
 }

src/main/java/com/uber/ussi/comparator/L2Comparator.java

@@ -171,6 +171,7 @@ public double similarityFromConjunction(
   @Override
   public double maxSimilarityFromPartialConjunction(
       double conjunction,
+      double scannedUnion,
       double unscannedKeysUniValue,
       double partialUniValue1,
       double uniValue1,