diff --git a/src/main/java/com/uber/ussi/searchablestructure/cache/InvertedTermCache.java b/src/main/java/com/uber/ussi/searchablestructure/cache/InvertedTermCache.java index f3e619b..4d1fd68 100644 --- a/src/main/java/com/uber/ussi/searchablestructure/cache/InvertedTermCache.java +++ b/src/main/java/com/uber/ussi/searchablestructure/cache/InvertedTermCache.java @@ -12,7 +12,8 @@ import com.uber.ussi.entity.termsandvalues.LongTermsAndValues; import com.uber.ussi.searchablestructure.result.RowNumAndSimilarity; import com.uber.ussi.searchablestructure.result.ResultHeaps; -import com.uber.ussi.searchablestructure.index.inverted.KeyAndPrefixFilteringData; +import com.uber.ussi.searchablestructure.utils.inverted.KeyAndPrefixFilteringData; +import com.uber.ussi.searchablestructure.utils.inverted.PopularTermDiscardPolicy; import com.uber.ussi.searchablestructure.utils.metadata.PreFilteringResult; import com.uber.ussi.searchablestructure.utils.parallel.ParallelRowScan; import com.uber.ussi.searchablestructure.utils.parallel.SharedMinSimilarity; @@ -52,13 +53,15 @@ public final class InvertedTermCache extends Cache { public InvertedTermCache(NamespaceConfig namespaceConfig) { super(namespaceConfig); - this.maxFractionIdsPerTerm = parseMaxFractionIdsPerTerm(namespaceConfig); + this.maxFractionIdsPerTerm = + PopularTermDiscardPolicy.maxFractionIdsPerTermFromCacheConfig(namespaceConfig); this.popularTermDiscardScope = namespaceConfig.getCachePopularTermDiscardScope(); this.fullReevaluationCacheSizeDecreaseFraction = parseFullReevaluationCacheSizeDecreaseFraction(namespaceConfig); this.popularityConfidenceTester = new MathUtils.ProportionConfidenceInterval1Sided( - parseMaxFractionIdsPerTermConfidence(namespaceConfig)); + PopularTermDiscardPolicy.maxFractionIdsPerTermConfidenceFromCacheConfig( + namespaceConfig)); this.termAndRowNumsIndex = new LongObjectHashMap<>(CACHE_INITIAL_CAPACITY); this.discardedTerms = new LongHashSet(); } @@ -357,12 +360,6 @@ private void rebuildDiscardedTerms() { } - private static double parseMaxFractionIdsPerTerm(NamespaceConfig namespaceConfig) { - return namespaceConfig.readDoubleCacheParam( - ConfigKeys.MAX_FRACTION_IDS_PER_TERM, - ConfigKeys.DEFAULT_MAX_FRACTION_IDS_PER_TERM); - } - private static double parseFullReevaluationCacheSizeDecreaseFraction( NamespaceConfig namespaceConfig) { return namespaceConfig.readDoubleCacheParam( @@ -370,13 +367,4 @@ private static double parseFullReevaluationCacheSizeDecreaseFraction( ConfigKeys.DEFAULT_FULL_REEVALUATION_CACHE_SIZE_DECREASE_FRACTION); } - /** - * Parses the one-sided confidence used to declare a term high-popularity. A confidence of 0.5 - * degenerates to comparing the observed popularity against maxFractionIdsPerTerm directly. - */ - private static double parseMaxFractionIdsPerTermConfidence(NamespaceConfig namespaceConfig) { - return namespaceConfig.readDoubleCacheParam( - ConfigKeys.MAX_FRACTION_IDS_PER_TERM_CONFIDENCE, - ConfigKeys.DEFAULT_MAX_FRACTION_IDS_PER_TERM_CONFIDENCE); - } } diff --git a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/BaseInvertedIndex.java b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/BaseInvertedIndex.java index 549cbc4..254bd0a 100644 --- a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/BaseInvertedIndex.java +++ b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/BaseInvertedIndex.java @@ -30,7 +30,9 @@ import com.uber.ussi.searchablestructure.index.inverted.generator.InvertedList; import com.uber.ussi.searchablestructure.index.inverted.generator.MergeSearch; import com.uber.ussi.searchablestructure.result.ResultHeaps; -import com.uber.ussi.searchablestructure.index.inverted.KeyAndPrefixFilteringData; +import com.uber.ussi.searchablestructure.utils.inverted.KeyAndPrefixFilteringData; +import com.uber.ussi.searchablestructure.utils.inverted.KeyAndUniTransformedValue; +import com.uber.ussi.searchablestructure.utils.inverted.PopularTermDiscardPolicy; import com.uber.ussi.searchablestructure.utils.metadata.MetadataFilteringStrategy; import com.uber.ussi.searchablestructure.utils.parallel.ParallelShardSearch; import com.uber.ussi.searchablestructure.utils.parallel.SharedMinSimilarity; @@ -169,7 +171,8 @@ protected BaseInvertedIndex( this.mergeScoresFromAccumulatedConjunction = sparsMerge.mergeScoresFromAccumulatedConjunction(); this.partialConjunctionPolicy = sparsMerge.partialConjunctionPolicy(); this.doesStoreMergePostingValues = sparsMerge.doesStoreMergePostingValues(); - this.maxFractionIdsPerTerm = PopularTermDiscardPolicy.maxFractionIdsPerTerm(namespaceConfig); + this.maxFractionIdsPerTerm = + PopularTermDiscardPolicy.maxFractionIdsPerTermFromIndexConfig(namespaceConfig); validateRows(); this.discardedTerms = structureDiscardedTerms == null diff --git a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/HybridIndex.java b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/HybridIndex.java index 3d4899d..5a0df33 100644 --- a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/HybridIndex.java +++ b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/HybridIndex.java @@ -11,6 +11,7 @@ import com.uber.ussi.entity.termsandvalues.LongTermsAndValues; import com.uber.ussi.searchablestructure.result.RowNumAndSimilarity; import com.uber.ussi.searchablestructure.index.Index; +import com.uber.ussi.searchablestructure.utils.inverted.PopularTermDiscardPolicy; import com.uber.ussi.utils.BoundedSizeMaxHeap; import java.util.List; @@ -26,7 +27,8 @@ public HybridIndex( LongObjectHashMap rowNumToTermsAndValuesMap, LongObjectHashMap rowNumToMetaMap) { super(namespaceConfig); - double maxFractionIdsPerTerm = PopularTermDiscardPolicy.maxFractionIdsPerTerm(namespaceConfig); + double maxFractionIdsPerTerm = + PopularTermDiscardPolicy.maxFractionIdsPerTermFromIndexConfig(namespaceConfig); LongObjectHashMap exactRows = new LongObjectHashMap<>(); LongObjectHashMap signatureRows = new LongObjectHashMap<>(); for (LongObjectCursor entry : rowNumToTermsAndValuesMap) { diff --git a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/SignatureIndex.java b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/SignatureIndex.java index 7f78af1..0b7c59a 100644 --- a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/SignatureIndex.java +++ b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/SignatureIndex.java @@ -7,6 +7,7 @@ import com.uber.ussi.entity.meta.LongMeta; import com.uber.ussi.entity.termsandvalues.LongTermsAndValues; import com.uber.ussi.searchablestructure.index.IndexType; +import com.uber.ussi.searchablestructure.utils.inverted.KeyAndUniTransformedValue; import java.util.Arrays; import javax.annotation.Nullable; diff --git a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/TermIndex.java b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/TermIndex.java index a48513b..9726c1e 100644 --- a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/TermIndex.java +++ b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/TermIndex.java @@ -7,6 +7,7 @@ import com.uber.ussi.entity.meta.LongMeta; import com.uber.ussi.entity.termsandvalues.LongTermsAndValues; import com.uber.ussi.searchablestructure.index.IndexType; +import com.uber.ussi.searchablestructure.utils.inverted.KeyAndUniTransformedValue; import javax.annotation.Nullable; /** diff --git a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/generator/FilteredSearch.java b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/generator/FilteredSearch.java index 7e3aa0e..e4a47a7 100644 --- a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/generator/FilteredSearch.java +++ b/src/main/java/com/uber/ussi/searchablestructure/index/inverted/generator/FilteredSearch.java @@ -7,7 +7,7 @@ import com.uber.ussi.entity.termsandvalues.LongTermsAndValues; import com.uber.ussi.searchablestructure.result.RowNumAndSimilarity; import com.uber.ussi.searchablestructure.result.ResultHeaps; -import com.uber.ussi.searchablestructure.index.inverted.KeyAndPrefixFilteringData; +import com.uber.ussi.searchablestructure.utils.inverted.KeyAndPrefixFilteringData; import com.uber.ussi.searchablestructure.utils.parallel.SharedMinSimilarity; import com.uber.ussi.utils.BoundedSizeMaxHeap; import com.uber.ussi.utils.MathUtils; diff --git a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/KeyAndPrefixFilteringData.java b/src/main/java/com/uber/ussi/searchablestructure/utils/inverted/KeyAndPrefixFilteringData.java similarity index 96% rename from src/main/java/com/uber/ussi/searchablestructure/index/inverted/KeyAndPrefixFilteringData.java rename to src/main/java/com/uber/ussi/searchablestructure/utils/inverted/KeyAndPrefixFilteringData.java index 95a67ff..a9f915c 100644 --- a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/KeyAndPrefixFilteringData.java +++ b/src/main/java/com/uber/ussi/searchablestructure/utils/inverted/KeyAndPrefixFilteringData.java @@ -1,5 +1,5 @@ /* AUTHOR: Shijie Lu (shijie@uber.com), Shalini Kedlaya (skedlaya@uber.com), Ahmed Metwally (ametwally@uber.com) */ -package com.uber.ussi.searchablestructure.index.inverted; +package com.uber.ussi.searchablestructure.utils.inverted; /** * Query key data ordered for low-cost prefix candidate generation. Shared by the inverted indexes diff --git a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/KeyAndUniTransformedValue.java b/src/main/java/com/uber/ussi/searchablestructure/utils/inverted/KeyAndUniTransformedValue.java similarity index 80% rename from src/main/java/com/uber/ussi/searchablestructure/index/inverted/KeyAndUniTransformedValue.java rename to src/main/java/com/uber/ussi/searchablestructure/utils/inverted/KeyAndUniTransformedValue.java index 8f0324b..cba6245 100644 --- a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/KeyAndUniTransformedValue.java +++ b/src/main/java/com/uber/ussi/searchablestructure/utils/inverted/KeyAndUniTransformedValue.java @@ -1,5 +1,5 @@ /* AUTHOR: Shijie Lu (shijie@uber.com), Shalini Kedlaya (skedlaya@uber.com), Ahmed Metwally (ametwally@uber.com) */ -package com.uber.ussi.searchablestructure.index.inverted; +package com.uber.ussi.searchablestructure.utils.inverted; /** A key and its contribution to the comparator-specific unilateral value. */ public final class KeyAndUniTransformedValue { @@ -11,11 +11,11 @@ public KeyAndUniTransformedValue(long key, double uniTransformedValue) { this.uniTransformedValue = uniTransformedValue; } - long getKey() { + public long getKey() { return key; } - double getUniTransformedValue() { + public double getUniTransformedValue() { return uniTransformedValue; } } diff --git a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicy.java b/src/main/java/com/uber/ussi/searchablestructure/utils/inverted/PopularTermDiscardPolicy.java similarity index 59% rename from src/main/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicy.java rename to src/main/java/com/uber/ussi/searchablestructure/utils/inverted/PopularTermDiscardPolicy.java index e8e99eb..ceb52aa 100644 --- a/src/main/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicy.java +++ b/src/main/java/com/uber/ussi/searchablestructure/utils/inverted/PopularTermDiscardPolicy.java @@ -1,5 +1,5 @@ /* AUTHOR: Ahmed Metwally (ametwally@uber.com) */ -package com.uber.ussi.searchablestructure.index.inverted; +package com.uber.ussi.searchablestructure.utils.inverted; import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.LongIntHashMap; @@ -12,25 +12,52 @@ import java.util.Objects; /** - * Popularity-based term discard policy for inverted term-keyed indexes. + * Popularity-based term discard policy for inverted term-keyed structures. * - *

Hybrid and term indexes share this policy so a composite structure can compute discards over - * its term-keyed partition without depending on {@link BaseInvertedIndex}. + *

Graduated indexes read thresholds from {@code index_params}; the writable inverted term cache + * reads the same keys from {@code cache_params}. Batch discard over a complete row map applies only + * to index construction. */ public final class PopularTermDiscardPolicy { private PopularTermDiscardPolicy() {} - /** Returns the configured maximum fraction of rows a term may appear in before it is discarded. */ - public static double maxFractionIdsPerTerm(NamespaceConfig namespaceConfig) { + /** + * Returns the configured maximum fraction of rows a term may appear in before it is discarded, from + * {@code index_params}. + */ + public static double maxFractionIdsPerTermFromIndexConfig(NamespaceConfig namespaceConfig) { Objects.requireNonNull(namespaceConfig, "namespaceConfig is null."); return namespaceConfig.readDoubleIndexParam( ConfigKeys.MAX_FRACTION_IDS_PER_TERM, ConfigKeys.DEFAULT_MAX_FRACTION_IDS_PER_TERM); } - /** Returns whether any term may be discarded under the configured popularity threshold. */ - public static boolean doesDiscardPopularTerms(NamespaceConfig namespaceConfig) { - return maxFractionIdsPerTerm(namespaceConfig) < 1.0; + /** + * Returns the configured maximum fraction of rows a term may appear in before it is discarded, + * from {@code cache_params}. + */ + public static double maxFractionIdsPerTermFromCacheConfig(NamespaceConfig namespaceConfig) { + Objects.requireNonNull(namespaceConfig, "namespaceConfig is null."); + return namespaceConfig.readDoubleCacheParam( + ConfigKeys.MAX_FRACTION_IDS_PER_TERM, ConfigKeys.DEFAULT_MAX_FRACTION_IDS_PER_TERM); + } + + /** + * Returns the one-sided confidence used to declare a term high-popularity in the inverted term + * cache, from {@code cache_params}. A confidence of 0.5 degenerates to comparing observed + * popularity against {@link #maxFractionIdsPerTermFromCacheConfig} directly. + */ + public static double maxFractionIdsPerTermConfidenceFromCacheConfig( + NamespaceConfig namespaceConfig) { + Objects.requireNonNull(namespaceConfig, "namespaceConfig is null."); + return namespaceConfig.readDoubleCacheParam( + ConfigKeys.MAX_FRACTION_IDS_PER_TERM_CONFIDENCE, + ConfigKeys.DEFAULT_MAX_FRACTION_IDS_PER_TERM_CONFIDENCE); + } + + /** Returns whether any term may be discarded under the index popularity threshold. */ + public static boolean doesDiscardPopularTermsFromIndexConfig(NamespaceConfig namespaceConfig) { + return maxFractionIdsPerTermFromIndexConfig(namespaceConfig) < 1.0; } /** Returns whether any term may be discarded under {@code maxFractionIdsPerTerm}. */ @@ -39,14 +66,16 @@ public static boolean doesDiscardPopularTerms(double maxFractionIdsPerTerm) { } /** - * Returns the terms a structure holding {@code rowNumToTermsAndValuesMap} discards as popular. + * Returns the terms a graduated index holding {@code rowNumToTermsAndValuesMap} discards as + * popular, using {@code index_params}. */ public static LongHashSet discardedTermsOf( NamespaceConfig namespaceConfig, LongObjectHashMap rowNumToTermsAndValuesMap) { Objects.requireNonNull(namespaceConfig, "namespaceConfig is null."); Objects.requireNonNull(rowNumToTermsAndValuesMap, "rowNumToTermsAndValuesMap is null."); - return discardedTermsOf(rowNumToTermsAndValuesMap, maxFractionIdsPerTerm(namespaceConfig)); + return discardedTermsOf( + rowNumToTermsAndValuesMap, maxFractionIdsPerTermFromIndexConfig(namespaceConfig)); } /** diff --git a/src/test/java/com/uber/ussi/searchablestructure/index/inverted/TermIndexTest.java b/src/test/java/com/uber/ussi/searchablestructure/index/inverted/TermIndexTest.java index 6e20a6a..6c67894 100644 --- a/src/test/java/com/uber/ussi/searchablestructure/index/inverted/TermIndexTest.java +++ b/src/test/java/com/uber/ussi/searchablestructure/index/inverted/TermIndexTest.java @@ -23,6 +23,7 @@ import com.uber.ussi.searchablestructure.index.IndexType; import com.uber.ussi.searchablestructure.index.scan.ScanIndex; import com.uber.ussi.searchablestructure.result.RowNumAndSimilarity; +import com.uber.ussi.searchablestructure.utils.inverted.KeyAndUniTransformedValue; import com.uber.ussi.searchablestructure.utils.metadata.MetadataFilteringStrategy; import com.uber.ussi.utils.ConfigKeys; import java.lang.reflect.Field; diff --git a/src/test/java/com/uber/ussi/searchablestructure/index/inverted/generator/FilteredSearchTest.java b/src/test/java/com/uber/ussi/searchablestructure/index/inverted/generator/FilteredSearchTest.java index 4dd99ec..97f6998 100644 --- a/src/test/java/com/uber/ussi/searchablestructure/index/inverted/generator/FilteredSearchTest.java +++ b/src/test/java/com/uber/ussi/searchablestructure/index/inverted/generator/FilteredSearchTest.java @@ -11,7 +11,7 @@ import com.uber.ussi.entity.termsandvalues.LongTermsAndValues; import com.uber.ussi.entity.termsandvalues.LongTermsAndValuesTestFactory; import com.uber.ussi.searchablestructure.result.RowNumAndSimilarity; -import com.uber.ussi.searchablestructure.index.inverted.KeyAndPrefixFilteringData; +import com.uber.ussi.searchablestructure.utils.inverted.KeyAndPrefixFilteringData; import com.uber.ussi.searchablestructure.utils.parallel.SharedMinSimilarity; import java.util.ArrayList; import java.util.List; diff --git a/src/test/java/com/uber/ussi/searchablestructure/index/inverted/KeyAndPrefixFilteringDataTest.java b/src/test/java/com/uber/ussi/searchablestructure/utils/inverted/KeyAndPrefixFilteringDataTest.java similarity index 94% rename from src/test/java/com/uber/ussi/searchablestructure/index/inverted/KeyAndPrefixFilteringDataTest.java rename to src/test/java/com/uber/ussi/searchablestructure/utils/inverted/KeyAndPrefixFilteringDataTest.java index 5dcbb70..b272974 100644 --- a/src/test/java/com/uber/ussi/searchablestructure/index/inverted/KeyAndPrefixFilteringDataTest.java +++ b/src/test/java/com/uber/ussi/searchablestructure/utils/inverted/KeyAndPrefixFilteringDataTest.java @@ -1,4 +1,4 @@ -package com.uber.ussi.searchablestructure.index.inverted; +package com.uber.ussi.searchablestructure.utils.inverted; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; diff --git a/src/test/java/com/uber/ussi/searchablestructure/index/inverted/KeyAndUniTransformedValueTest.java b/src/test/java/com/uber/ussi/searchablestructure/utils/inverted/KeyAndUniTransformedValueTest.java similarity index 87% rename from src/test/java/com/uber/ussi/searchablestructure/index/inverted/KeyAndUniTransformedValueTest.java rename to src/test/java/com/uber/ussi/searchablestructure/utils/inverted/KeyAndUniTransformedValueTest.java index 8e20905..9d042dd 100644 --- a/src/test/java/com/uber/ussi/searchablestructure/index/inverted/KeyAndUniTransformedValueTest.java +++ b/src/test/java/com/uber/ussi/searchablestructure/utils/inverted/KeyAndUniTransformedValueTest.java @@ -1,4 +1,4 @@ -package com.uber.ussi.searchablestructure.index.inverted; +package com.uber.ussi.searchablestructure.utils.inverted; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/src/test/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicyTest.java b/src/test/java/com/uber/ussi/searchablestructure/utils/inverted/PopularTermDiscardPolicyTest.java similarity index 67% rename from src/test/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicyTest.java rename to src/test/java/com/uber/ussi/searchablestructure/utils/inverted/PopularTermDiscardPolicyTest.java index 42058db..89ab760 100644 --- a/src/test/java/com/uber/ussi/searchablestructure/index/inverted/PopularTermDiscardPolicyTest.java +++ b/src/test/java/com/uber/ussi/searchablestructure/utils/inverted/PopularTermDiscardPolicyTest.java @@ -1,5 +1,5 @@ /* AUTHOR: Ahmed Metwally (ametwally@uber.com) */ -package com.uber.ussi.searchablestructure.index.inverted; +package com.uber.ussi.searchablestructure.utils.inverted; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -31,10 +31,30 @@ void doesDiscardPopularTermsWhenMaxFractionIsBelowOne() { .maxNumSearchableStructures(3) .maxNumSimilarities(100) .build(); - assertTrue(PopularTermDiscardPolicy.doesDiscardPopularTerms(config)); + assertTrue(PopularTermDiscardPolicy.doesDiscardPopularTermsFromIndexConfig(config)); assertFalse(PopularTermDiscardPolicy.doesDiscardPopularTerms(1.0)); } + @Test + void maxFractionIdsPerTermReadsIndexAndCacheParamBagsSeparately() { + NamespaceConfig config = + NamespaceConfig.builder() + .minTermsAndValuesLength(0) + .maxTermsAndValuesLength(100) + .maxCacheSize(100) + .cacheType("inverted_term") + .indexType("inverted_term") + .indexParams(Map.of(ConfigKeys.MAX_FRACTION_IDS_PER_TERM, "0.3")) + .cacheParams(Map.of(ConfigKeys.MAX_FRACTION_IDS_PER_TERM, "0.7")) + .comparatorType("jaccard") + .comparatorNormalizerType("complement") + .maxNumSearchableStructures(3) + .maxNumSimilarities(100) + .build(); + assertEquals(0.3, PopularTermDiscardPolicy.maxFractionIdsPerTermFromIndexConfig(config)); + assertEquals(0.7, PopularTermDiscardPolicy.maxFractionIdsPerTermFromCacheConfig(config)); + } + @Test void discardedTermsOfMarksTermsAboveTheRowFractionThreshold() { LongObjectHashMap rows = new LongObjectHashMap<>();