diff --git a/AzureSearchEmulator.IntegrationTests/SynonymMapIntegrationTests.cs b/AzureSearchEmulator.IntegrationTests/SynonymMapIntegrationTests.cs new file mode 100644 index 0000000..7a0ebb6 --- /dev/null +++ b/AzureSearchEmulator.IntegrationTests/SynonymMapIntegrationTests.cs @@ -0,0 +1,350 @@ +using Azure; +using Azure.Search.Documents; +using Azure.Search.Documents.Indexes; +using Azure.Search.Documents.Indexes.Models; +using Azure.Search.Documents.Models; +using Xunit; +using SearchIndex = Azure.Search.Documents.Indexes.Models.SearchIndex; + +namespace AzureSearchEmulator.IntegrationTests; + +/// +/// Integration tests for synonym map support (issue #69), run against a containerized emulator +/// through the real Azure Search SDK. +/// +/// +/// The SDK is the point of these tests. It decides the routes a synonym map is created and read +/// on, the value wrapper a listing comes back in, and the format/synonyms +/// property names that go on the wire — none of which the unit tests exercise, because they +/// call the emulator's own types directly. A map the emulator stored under a different shape, +/// or a synonymMaps field property it accepted and ignored, would fail here rather than +/// pass unnoticed. +/// +/// The scenarios are the ones Azure's documentation gives: a query for a word the indexed +/// documents do not contain, matched through a rule. +/// +public class SynonymMapIntegrationTests(EmulatorFactory factory) + : IClassFixture +{ + private class Document + { + public string Id { get; set; } = ""; + + public string? Name { get; set; } + } + + private static readonly Document[] Products = + [ + new() { Id = "1", Name = "canine chew toy" }, + new() { Id = "2", Name = "feline scratcher" }, + new() { Id = "3", Name = "united states map" }, + ]; + + private static SearchIndex CreateIndex(string indexName, string? synonymMapName) + { + var name = new SearchField(nameof(Document.Name), SearchFieldDataType.String) + { + IsSearchable = true, + IsStored = true, + }; + + if (synonymMapName != null) + { + name.SynonymMapNames.Add(synonymMapName); + } + + return new SearchIndex(indexName) + { + Fields = + { + new SearchField(nameof(Document.Id), SearchFieldDataType.String) + { IsKey = true, IsSearchable = true, IsFilterable = true, IsStored = true }, + name, + } + }; + } + + private static async Task PopulateAsync(SearchIndexClient indexClient, SearchIndex index) + { + await indexClient.CreateIndexAsync(index, TestContext.Current.CancellationToken); + + var searchClient = indexClient.GetSearchClient(index.Name); + + var response = await searchClient.IndexDocumentsAsync( + IndexDocumentsBatch.Upload(Products), + cancellationToken: TestContext.Current.CancellationToken); + + Assert.All(response.Value.Results, i => Assert.True(i.Succeeded)); + + await WaitForCountAsync(searchClient, Products.Length); + + return searchClient; + } + + private static async Task> IdsAsync(SearchClient searchClient, string search) + { + var response = await searchClient.SearchAsync( + search, new SearchOptions(), TestContext.Current.CancellationToken); + + var ids = new List(); + + await foreach (var result in response.Value.GetResultsAsync()) + { + ids.Add(result.Document.Id); + } + + return ids; + } + + /// + /// A synonym map survives being written and read back through the SDK, with its rules + /// intact. + /// + /// + /// Read back before any search, so a persistence fault shows as a definition mismatch here + /// rather than as a mysteriously empty result later. + /// + [Fact] + public async Task SynonymMap_RoundTrips() + { + const string mapName = "test-synonyms-roundtrip"; + + var indexClient = factory.CreateSearchIndexClient(); + + await indexClient.CreateSynonymMapAsync( + new SynonymMap(mapName, "usa, united states\ndog => canine"), + TestContext.Current.CancellationToken); + + var read = await indexClient.GetSynonymMapAsync(mapName, TestContext.Current.CancellationToken); + + Assert.Equal(mapName, read.Value.Name); + Assert.Contains("usa, united states", read.Value.Synonyms); + Assert.Contains("dog => canine", read.Value.Synonyms); + + await indexClient.DeleteSynonymMapAsync(mapName, TestContext.Current.CancellationToken); + } + + /// + /// A created map appears in the listing, which the SDK reads out of the value wrapper. + /// + [Fact] + public async Task SynonymMaps_AreListed() + { + const string mapName = "test-synonyms-listed"; + + var indexClient = factory.CreateSearchIndexClient(); + + await indexClient.CreateSynonymMapAsync( + new SynonymMap(mapName, "usa, united states"), + TestContext.Current.CancellationToken); + + var names = await indexClient.GetSynonymMapNamesAsync(TestContext.Current.CancellationToken); + + Assert.Contains(mapName, names.Value); + + await indexClient.DeleteSynonymMapAsync(mapName, TestContext.Current.CancellationToken); + } + + /// + /// A deleted map is gone. + /// + [Fact] + public async Task DeletedSynonymMap_IsNotFound() + { + const string mapName = "test-synonyms-deleted"; + + var indexClient = factory.CreateSearchIndexClient(); + + await indexClient.CreateSynonymMapAsync( + new SynonymMap(mapName, "usa, united states"), + TestContext.Current.CancellationToken); + + await indexClient.DeleteSynonymMapAsync(mapName, TestContext.Current.CancellationToken); + + var ex = await Assert.ThrowsAsync(() => + indexClient.GetSynonymMapAsync(mapName, TestContext.Current.CancellationToken)); + + Assert.Equal(404, ex.Status); + } + + /// + /// An edit replaces the rules, and the map reads back with the new ones. + /// + [Fact] + public async Task SynonymMap_CanBeEdited() + { + const string mapName = "test-synonyms-edited"; + + var indexClient = factory.CreateSearchIndexClient(); + + await indexClient.CreateSynonymMapAsync( + new SynonymMap(mapName, "usa, united states"), + TestContext.Current.CancellationToken); + + await indexClient.CreateOrUpdateSynonymMapAsync( + new SynonymMap(mapName, "dog, canine"), + onlyIfUnchanged: false, + cancellationToken: TestContext.Current.CancellationToken); + + var read = await indexClient.GetSynonymMapAsync(mapName, TestContext.Current.CancellationToken); + + Assert.Equal("dog, canine", read.Value.Synonyms); + + await indexClient.DeleteSynonymMapAsync(mapName, TestContext.Current.CancellationToken); + } + + /// + /// The scenario the feature exists for: a query for a word no document contains matches + /// through an equivalency rule. + /// + [Fact] + public async Task EquivalencyRule_WidensTheQuery() + { + const string indexName = "test-synonym-equivalency"; + const string mapName = "test-synonyms-equivalency"; + + var indexClient = factory.CreateSearchIndexClient(); + + await indexClient.CreateSynonymMapAsync( + new SynonymMap(mapName, "dog, canine"), + TestContext.Current.CancellationToken); + + var searchClient = await PopulateAsync(indexClient, CreateIndex(indexName, mapName)); + + Assert.Equal(["1"], await IdsAsync(searchClient, "dog")); + + await indexClient.DeleteIndexAsync(indexName, TestContext.Current.CancellationToken); + await indexClient.DeleteSynonymMapAsync(mapName, TestContext.Current.CancellationToken); + } + + /// + /// Sits beside the test above: the same query against a field naming no map matches + /// nothing, so the match there comes from the expansion rather than from looser matching. + /// + [Fact] + public async Task WithoutASynonymMap_TheSameQueryMatchesNothing() + { + const string indexName = "test-synonym-none"; + + var indexClient = factory.CreateSearchIndexClient(); + + var searchClient = await PopulateAsync(indexClient, CreateIndex(indexName, synonymMapName: null)); + + Assert.Empty(await IdsAsync(searchClient, "dog")); + + await indexClient.DeleteIndexAsync(indexName, TestContext.Current.CancellationToken); + } + + /// + /// A mapping rule replaces the query's term rather than adding to it. + /// + [Fact] + public async Task MappingRule_SearchesForTheReplacement() + { + const string indexName = "test-synonym-mapping"; + const string mapName = "test-synonyms-mapping"; + + var indexClient = factory.CreateSearchIndexClient(); + + await indexClient.CreateSynonymMapAsync( + new SynonymMap(mapName, "cat => feline"), + TestContext.Current.CancellationToken); + + var searchClient = await PopulateAsync(indexClient, CreateIndex(indexName, mapName)); + + Assert.Equal(["2"], await IdsAsync(searchClient, "cat")); + + await indexClient.DeleteIndexAsync(indexName, TestContext.Current.CancellationToken); + await indexClient.DeleteSynonymMapAsync(mapName, TestContext.Current.CancellationToken); + } + + /// + /// A multi-word rule matches as a phrase. + /// + [Fact] + public async Task MultiWordRule_MatchesThePhrase() + { + const string indexName = "test-synonym-phrase"; + const string mapName = "test-synonyms-phrase"; + + var indexClient = factory.CreateSearchIndexClient(); + + await indexClient.CreateSynonymMapAsync( + new SynonymMap(mapName, "usa, united states"), + TestContext.Current.CancellationToken); + + var searchClient = await PopulateAsync(indexClient, CreateIndex(indexName, mapName)); + + Assert.Equal(["3"], await IdsAsync(searchClient, "usa")); + + await indexClient.DeleteIndexAsync(indexName, TestContext.Current.CancellationToken); + await indexClient.DeleteSynonymMapAsync(mapName, TestContext.Current.CancellationToken); + } + + /// + /// Editing a map changes what a query matches without the documents being touched, which is + /// the property that makes a map safe to edit. + /// + [Fact] + public async Task EditedSynonymMap_TakesEffectWithoutReindexing() + { + const string indexName = "test-synonym-reedit"; + const string mapName = "test-synonyms-reedit"; + + var indexClient = factory.CreateSearchIndexClient(); + + await indexClient.CreateSynonymMapAsync( + new SynonymMap(mapName, "bird, avian"), + TestContext.Current.CancellationToken); + + var searchClient = await PopulateAsync(indexClient, CreateIndex(indexName, mapName)); + + Assert.Empty(await IdsAsync(searchClient, "dog")); + + await indexClient.CreateOrUpdateSynonymMapAsync( + new SynonymMap(mapName, "dog, canine"), + onlyIfUnchanged: false, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.Equal(["1"], await IdsAsync(searchClient, "dog")); + + await indexClient.DeleteIndexAsync(indexName, TestContext.Current.CancellationToken); + await indexClient.DeleteSynonymMapAsync(mapName, TestContext.Current.CancellationToken); + } + + /// + /// An index naming a synonym map that does not exist is refused when it is created, rather + /// than accepted and left to search unexpanded. + /// + [Fact] + public async Task IndexNamingAMissingSynonymMap_IsRejected() + { + const string indexName = "test-synonym-missing"; + + var indexClient = factory.CreateSearchIndexClient(); + + var ex = await Assert.ThrowsAsync(() => + indexClient.CreateIndexAsync( + CreateIndex(indexName, "no-such-map"), + TestContext.Current.CancellationToken)); + + Assert.Equal(400, ex.Status); + } + + private static async Task WaitForCountAsync(SearchClient searchClient, int expected) + { + for (var attempt = 0; attempt < 20; attempt++) + { + var count = await searchClient.GetDocumentCountAsync(TestContext.Current.CancellationToken); + + if (count.Value >= expected) + { + return; + } + + await Task.Delay(100, TestContext.Current.CancellationToken); + } + + Assert.Fail($"{expected} documents were expected to become searchable, but the count never reached it."); + } +} diff --git a/AzureSearchEmulator.UnitTests/SynonymMapEndToEndTests.cs b/AzureSearchEmulator.UnitTests/SynonymMapEndToEndTests.cs new file mode 100644 index 0000000..38e06ad --- /dev/null +++ b/AzureSearchEmulator.UnitTests/SynonymMapEndToEndTests.cs @@ -0,0 +1,229 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using AzureSearchEmulator.Indexing; +using AzureSearchEmulator.Models; +using AzureSearchEmulator.Repositories; +using AzureSearchEmulator.SearchData; +using AzureSearchEmulator.Searching; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Xunit; +using SynonymMap = AzureSearchEmulator.Models.SynonymMap; + +namespace AzureSearchEmulator.UnitTests; + +/// +/// Exercises synonym maps through the real indexing and search path (issue #69). +/// +/// +/// This is the part of the feature that only a search can prove. The documents are indexed with +/// no synonym applied — Azure expands at query time only — so a match here can come from nothing +/// but the query having been widened, and a test that passed while the expansion silently did +/// nothing would have to match on a term the document does not contain. +/// +/// Every test names the map through a field's synonymMaps and resolves it through the +/// repository, so the wiring the controller depends on is covered too rather than being stubbed +/// past. See for the rules in isolation. +/// +public class SynonymMapEndToEndTests +{ + private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web); + + private const string IndexJson = + """ + { + "name": "products", + "fields": [ + { "name": "id", "type": "Edm.String", "key": true }, + { "name": "name", "type": "Edm.String", "searchable": true, "synonymMaps": ["products"] }, + { "name": "description", "type": "Edm.String", "searchable": true } + ] + } + """; + + private static readonly string[] Products = + [ + """{ "id": "1", "name": "canine chew toy", "description": "for a canine" }""", + """{ "id": "2", "name": "united states map", "description": "a map" }""", + """{ "id": "3", "name": "feline scratcher", "description": "for a feline" }""", + ]; + + private static SearchIndex CreateIndex() => JsonSerializer.Deserialize(IndexJson, Options)!; + + private static SynonymMap CreateMap(string synonyms) => + new() { Name = "products", Synonyms = synonyms }; + + /// + /// Builds documents through the real indexing path, so the tests read the same Lucene + /// fields an uploaded document would produce. + /// + private static List CreateDocuments(SearchIndex index) + { + var documents = new List(); + + foreach (var json in Products) + { + var item = JsonNode.Parse(json)!.AsObject(); + var document = new Document(); + + foreach (var field in index.Fields) + { + var value = item.FirstOrDefault(p => + string.Equals(p.Key, field.Name, StringComparison.OrdinalIgnoreCase)).Value; + + if (value is null) + { + continue; + } + + foreach (var luceneField in field.CreateFields(value, index)) + { + document.Add(luceneField); + } + } + + documents.Add(document); + } + + return documents; + } + + private static async Task> Search(SynonymMap? synonymMap, string search) + { + var index = CreateIndex(); + + using var helper = new LuceneTestHelper(index, CreateDocuments(index)); + + var searcher = new LuceneNetIndexSearcher( + new StubIndexReaderFactory(helper.Directory), + new StubSynonymMapRepository(synonymMap)); + + var response = await searcher.Search(index, new SearchRequest { Search = search, Top = 10 }); + + return response.Results.Select(i => i["id"]!.GetValue()).Order().ToList(); + } + + /// + /// The scenario Azure's documentation opens with: a query for a word the document does not + /// contain matches through an equivalency rule. + /// + [Fact] + public async Task EquivalencyRule_MatchesADocumentWithTheOtherTerm() + { + Assert.Equal(["1"], await Search(CreateMap("dog, canine"), "dog")); + } + + /// + /// Sits beside the test above deliberately: without the map the same query matches nothing, + /// so the match there comes from the expansion rather than from any looser matching. + /// + [Fact] + public async Task WithoutTheMap_TheSameQueryMatchesNothing() + { + Assert.Empty(await Search(null, "dog")); + } + + /// + /// Expansion runs in both directions, so the term the document holds still matches. + /// + [Fact] + public async Task EquivalencyRule_StillMatchesTheOriginalTerm() + { + Assert.Equal(["1"], await Search(CreateMap("dog, canine"), "canine")); + } + + /// + /// A mapping rule replaces the query's term, so a document holding only the replacement + /// matches and one holding only the original no longer does. + /// + [Fact] + public async Task MappingRule_SearchesForTheReplacement() + { + Assert.Equal(["3"], await Search(CreateMap("cat => feline"), "cat")); + } + + /// + /// A multi-word rule matches as a phrase. + /// + [Fact] + public async Task MultiWordRule_MatchesThePhrase() + { + Assert.Equal(["2"], await Search(CreateMap("usa, united states"), "usa")); + } + + /// + /// A field that names no map is unaffected, even while another field on the same index is + /// expanding — which is what makes the map a property of the field rather than the index. + /// + [Fact] + public async Task FieldWithoutAMap_DoesNotExpand() + { + var index = CreateIndex(); + + using var helper = new LuceneTestHelper(index, CreateDocuments(index)); + + var searcher = new LuceneNetIndexSearcher( + new StubIndexReaderFactory(helper.Directory), + new StubSynonymMapRepository(CreateMap("dog, canine"))); + + var response = await searcher.Search(index, new SearchRequest + { + Search = "dog", + SearchFields = "description", + Top = 10, + }); + + Assert.Empty(response.Results); + } + + /// + /// A query is widened by the rules the map holds now, not the ones it held when the + /// documents were indexed — the property that makes a map safe to edit. + /// + [Fact] + public async Task EditedMap_TakesEffectWithoutReindexing() + { + Assert.Empty(await Search(CreateMap("bird, avian"), "dog")); + Assert.Equal(["1"], await Search(CreateMap("dog, canine"), "dog")); + } + + /// + /// A field naming a map the service no longer holds searches unexpanded rather than + /// failing, for the reason given on . + /// + [Fact] + public async Task MissingMap_LeavesTheQueryUnexpanded() + { + Assert.Equal(["1"], await Search(null, "canine")); + } + + private class StubIndexReaderFactory(Lucene.Net.Store.Directory directory) : ILuceneIndexReaderFactory + { + public IndexReader GetIndexReader(string indexName) => DirectoryReader.Open(directory); + + public IndexReader RefreshReader(string indexName) => GetIndexReader(indexName); + + public void ClearCachedReader(string indexName) { } + } + + private class StubSynonymMapRepository(SynonymMap? synonymMap) : ISynonymMapRepository + { + public async IAsyncEnumerable GetAll() + { + if (synonymMap != null) + { + yield return synonymMap; + } + + await Task.CompletedTask; + } + + public Task Get(string key) => Task.FromResult(synonymMap); + + public Task Create(SynonymMap map) => Task.CompletedTask; + + public Task Update(SynonymMap map) => Task.CompletedTask; + + public Task Delete(SynonymMap map) => Task.FromResult(true); + } +} diff --git a/AzureSearchEmulator.UnitTests/SynonymMapTests.cs b/AzureSearchEmulator.UnitTests/SynonymMapTests.cs new file mode 100644 index 0000000..7749dec --- /dev/null +++ b/AzureSearchEmulator.UnitTests/SynonymMapTests.cs @@ -0,0 +1,241 @@ +using AzureSearchEmulator.SearchData; +using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Core; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Analysis.TokenAttributes; +using Lucene.Net.Util; +using Xunit; +using SynonymMap = AzureSearchEmulator.Models.SynonymMap; + +namespace AzureSearchEmulator.UnitTests; + +/// +/// Exercises synonym expansion in isolation, before any index or query is involved (issue #69). +/// +/// +/// The two Solr rule forms mean different things, and the difference is the part of this +/// feature most easily got wrong: an equivalency rule widens a query while keeping the term the +/// caller typed, and a mapping rule replaces it. Asserting on the token stream shows which of +/// the two a rule produced, which a search result alone would not — a query that matched could +/// have matched on the original term. +/// +/// See for the same rules driving a real search. +/// +public class SynonymMapTests +{ + private const LuceneVersion Version = LuceneVersion.LUCENE_48; + + private static SynonymMap CreateMap(string synonyms) => + new() { Name = "test", Synonyms = synonyms }; + + /// + /// Expands text through a lower-casing chain with the map applied, the way a searchable + /// field's analyzer does. + /// + private static List Expand(string synonyms, string text) + { + var analyzer = SynonymMapHelper.Wrap(CreateBaseAnalyzer(), [CreateMap(synonyms)]); + + using var stream = analyzer.GetTokenStream("field", text); + var term = stream.AddAttribute(); + var tokens = new List(); + + stream.Reset(); + + while (stream.IncrementToken()) + { + tokens.Add(term.ToString()); + } + + stream.End(); + + return tokens; + } + + private static Analyzer CreateBaseAnalyzer() => + Analyzer.NewAnonymous((_, reader) => + { + Tokenizer source = new StandardTokenizer(Version, reader); + + return new TokenStreamComponents(source, new LowerCaseFilter(Version, source)); + }); + + /// + /// An equivalency rule adds every alternative and keeps the term that was typed, so a + /// document holding any one of them matches a query using any other. + /// + [Theory] + [InlineData("usa")] + [InlineData("united states")] + public void EquivalencyRule_ExpandsInEveryDirection(string query) + { + var tokens = Expand("usa, united states", query); + + Assert.Contains("usa", tokens); + Assert.Contains("united", tokens); + Assert.Contains("states", tokens); + } + + /// + /// A mapping rule replaces its left side, so the original term is gone from the query. + /// + /// + /// This is the assertion that separates the two rule forms. Were the arrow treated as an + /// equivalency, "dog" would survive alongside "canine" and a search for it would still + /// match documents about dogs — which is the opposite of what the rule asks for. + /// + [Fact] + public void MappingRule_ReplacesTheTermItMatches() + { + var tokens = Expand("dog => canine", "dog"); + + Assert.Equal(["canine"], tokens); + } + + /// + /// A mapping rule may name several replacements, and all of them are produced. + /// + [Fact] + public void MappingRule_ProducesEveryReplacement() + { + var tokens = Expand("dog => canine, hound", "dog"); + + Assert.Contains("canine", tokens); + Assert.Contains("hound", tokens); + Assert.DoesNotContain("dog", tokens); + } + + /// + /// A term no rule mentions passes through untouched. + /// + [Fact] + public void UnmatchedTerm_IsLeftAlone() + { + Assert.Equal(["cat"], Expand("dog => canine", "cat")); + } + + /// + /// Matching ignores case, because the rules are lower-cased when parsed and the field's own + /// analyzer lower-cases the query before the filter sees it. + /// + /// + /// Covers the case that would otherwise fail silently: a map written in lower case and a + /// query typed in capitals is the ordinary way both are used, and an expansion that only + /// worked when the two happened to agree would look correct in every hand-written test. + /// + [Theory] + [InlineData("USA")] + [InlineData("Usa")] + public void Matching_IgnoresCase(string query) + { + Assert.Contains("united", Expand("usa, united states", query)); + } + + /// + /// A multi-word rule matches the words in sequence rather than individually. + /// + [Fact] + public void MultiWordRule_MatchesThePhrase() + { + var synonyms = "united states of america, usa"; + + Assert.Contains("usa", Expand(synonyms, "united states of america")); + + // The leading word on its own is not the phrase, so it must not trigger the rule. + Assert.DoesNotContain("usa", Expand(synonyms, "united")); + } + + /// + /// Rules are separated by newlines, and every one of them applies. + /// + [Fact] + public void SeveralRules_AllApply() + { + var synonyms = "usa, united states\ndog => canine"; + + Assert.Contains("united", Expand(synonyms, "usa")); + Assert.Equal(["canine"], Expand(synonyms, "dog")); + } + + /// + /// A field naming several maps gets all of them, each as its own filter. + /// + [Fact] + public void SeveralMaps_AreAllApplied() + { + var analyzer = SynonymMapHelper.Wrap( + CreateBaseAnalyzer(), + [CreateMap("usa, united states"), CreateMap("dog => canine")]); + + Assert.Contains("united", Tokens(analyzer, "usa")); + Assert.Equal(["canine"], Tokens(analyzer, "dog")); + } + + /// + /// Wrapping with no maps returns the analyzer untouched, so a field that names none pays + /// nothing for the feature. + /// + [Fact] + public void NoMaps_LeavesTheAnalyzerAlone() + { + var analyzer = CreateBaseAnalyzer(); + + Assert.Same(analyzer, SynonymMapHelper.Wrap(analyzer, [])); + } + + /// + /// A compiled map is reused while its definition is, so the rules are not re-parsed on + /// every search. + /// + [Fact] + public void CompiledMap_IsCachedPerDefinition() + { + var map = CreateMap("usa, united states"); + + Assert.Same(SynonymMapHelper.Compile(map), SynonymMapHelper.Compile(map)); + } + + /// + /// An edited map is a new definition, so it compiles afresh rather than answering from the + /// rules it replaced. + /// + [Fact] + public void EditedMap_IsNotServedFromTheCache() + { + Assert.NotSame( + SynonymMapHelper.Compile(CreateMap("usa, united states")), + SynonymMapHelper.Compile(CreateMap("usa, us"))); + } + + /// + /// Only Azure's one supported format is accepted; anything else would be stored and then + /// expand nothing. + /// + [Fact] + public void UnsupportedFormat_IsRejected() + { + var map = new SynonymMap { Name = "test", Format = "wordnet", Synonyms = "usa, us" }; + + var ex = Assert.Throws(() => SynonymMapBuilder.Build(map)); + + Assert.Contains("wordnet", ex.Message); + } + + private static List Tokens(Analyzer analyzer, string text) + { + using var stream = analyzer.GetTokenStream("field", text); + var term = stream.AddAttribute(); + var tokens = new List(); + + stream.Reset(); + + while (stream.IncrementToken()) + { + tokens.Add(term.ToString()); + } + + stream.End(); + + return tokens; + } +} diff --git a/AzureSearchEmulator.UnitTests/SynonymMapValidationTests.cs b/AzureSearchEmulator.UnitTests/SynonymMapValidationTests.cs new file mode 100644 index 0000000..a548a55 --- /dev/null +++ b/AzureSearchEmulator.UnitTests/SynonymMapValidationTests.cs @@ -0,0 +1,204 @@ +using System.Text.Json; +using AzureSearchEmulator.Indexing; +using AzureSearchEmulator.Models; +using Xunit; +using SynonymMap = AzureSearchEmulator.Models.SynonymMap; + +namespace AzureSearchEmulator.UnitTests; + +/// +/// Covers what the emulator refuses to store or to search with (issue #69). +/// +/// +/// Each rejection here is one a caller would otherwise discover as a search that quietly +/// returned the wrong results: a map naming an unsupported format, or a field naming a map that +/// does not exist or that no query could ever reach. Reporting them when the definition is +/// written points at the mistake instead. +/// +public class SynonymMapValidationTests +{ + private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web); + + private static SearchIndex CreateIndex(string fieldsJson) => + JsonSerializer.Deserialize( + $$"""{ "name": "test", "fields": {{fieldsJson}} }""", Options)!; + + private static IReadOnlySet Existing(params string[] names) + => names.ToHashSet(StringComparer.OrdinalIgnoreCase); + + [Fact] + public void ValidMap_IsAccepted() + { + var map = new SynonymMap { Name = "places", Synonyms = "usa, united states" }; + + Assert.Null(SynonymMapValidator.FindInvalidSynonymMap(map)); + } + + [Fact] + public void MapWithoutName_IsRejected() + { + var map = new SynonymMap { Name = "", Synonyms = "usa, united states" }; + + Assert.Contains("must have a name", SynonymMapValidator.FindInvalidSynonymMap(map)); + } + + /// + /// Azure supports only the Solr format, and a map claiming another would be stored and then + /// expand nothing. + /// + [Fact] + public void MapWithUnsupportedFormat_IsRejected() + { + var map = new SynonymMap { Name = "places", Format = "wordnet", Synonyms = "usa, us" }; + + var error = SynonymMapValidator.FindInvalidSynonymMap(map); + + Assert.Contains("wordnet", error); + Assert.Contains("solr", error); + } + + /// + /// An empty map would look to a caller as though it had been applied while doing nothing. + /// + [Theory] + [InlineData("")] + [InlineData(" ")] + public void MapWithoutRules_IsRejected(string synonyms) + { + var map = new SynonymMap { Name = "places", Synonyms = synonyms }; + + Assert.Contains("at least one synonym rule", SynonymMapValidator.FindInvalidSynonymMap(map)); + } + + /// + /// The format check is case-insensitive, so a map written as "Solr" is still usable. + /// + [Fact] + public void FormatComparison_IgnoresCase() + { + var map = new SynonymMap { Name = "places", Format = "SOLR", Synonyms = "usa, us" }; + + Assert.Null(SynonymMapValidator.FindInvalidSynonymMap(map)); + } + + [Fact] + public void FieldNamingAnExistingMap_IsAccepted() + { + var index = CreateIndex( + """ + [ { "name": "id", "type": "Edm.String", "key": true }, + { "name": "city", "type": "Edm.String", "searchable": true, "synonymMaps": ["places"] } ] + """); + + Assert.Null(SynonymMapValidator.FindInvalidFieldReference(index, Existing("places"))); + } + + [Fact] + public void FieldNamingAMissingMap_IsRejected() + { + var index = CreateIndex( + """ + [ { "name": "id", "type": "Edm.String", "key": true }, + { "name": "city", "type": "Edm.String", "searchable": true, "synonymMaps": ["nope"] } ] + """); + + var error = SynonymMapValidator.FindInvalidFieldReference(index, Existing("places")); + + Assert.Contains("'nope'", error); + Assert.Contains("does not exist", error); + } + + /// + /// Synonyms only ever widen a full-text query, so a field no query can reach would name a + /// map that never applied. + /// + [Fact] + public void NonSearchableField_IsRejected() + { + var index = CreateIndex( + """ + [ { "name": "id", "type": "Edm.String", "key": true }, + { "name": "city", "type": "Edm.String", "searchable": false, "synonymMaps": ["places"] } ] + """); + + Assert.Contains("not searchable", SynonymMapValidator.FindInvalidFieldReference(index, Existing("places"))); + } + + /// + /// A synonym map transforms text, so it is meaningless on a field that holds none. + /// + [Fact] + public void NonStringField_IsRejected() + { + var index = CreateIndex( + """ + [ { "name": "id", "type": "Edm.String", "key": true }, + { "name": "rating", "type": "Edm.Int32", "searchable": true, "synonymMaps": ["places"] } ] + """); + + Assert.Contains("cannot have a synonym map", + SynonymMapValidator.FindInvalidFieldReference(index, Existing("places"))); + } + + /// + /// A string collection is still text, so it may carry a map. + /// + [Fact] + public void StringCollectionField_IsAccepted() + { + var index = CreateIndex( + """ + [ { "name": "id", "type": "Edm.String", "key": true }, + { "name": "tags", "type": "Collection(Edm.String)", "searchable": true, "synonymMaps": ["places"] } ] + """); + + Assert.Null(SynonymMapValidator.FindInvalidFieldReference(index, Existing("places"))); + } + + /// + /// A mistake on a sub-field is reported against the path that identifies it, rather than + /// against the complex field that contains it. + /// + [Fact] + public void SubFieldOfComplexType_IsReportedByPath() + { + var index = CreateIndex( + """ + [ { "name": "id", "type": "Edm.String", "key": true }, + { "name": "address", "type": "Edm.ComplexType", "fields": [ + { "name": "city", "type": "Edm.String", "searchable": true, "synonymMaps": ["nope"] } ] } ] + """); + + Assert.Contains("address/city", SynonymMapValidator.FindInvalidFieldReference(index, Existing("places"))); + } + + /// + /// Map names are matched case-insensitively, as Azure matches them. + /// + [Fact] + public void MapNameComparison_IgnoresCase() + { + var index = CreateIndex( + """ + [ { "name": "id", "type": "Edm.String", "key": true }, + { "name": "city", "type": "Edm.String", "searchable": true, "synonymMaps": ["PLACES"] } ] + """); + + Assert.Null(SynonymMapValidator.FindInvalidFieldReference(index, Existing("places"))); + } + + /// + /// A field naming no map is unaffected, whatever maps the service holds. + /// + [Fact] + public void FieldNamingNoMap_IsAccepted() + { + var index = CreateIndex( + """ + [ { "name": "id", "type": "Edm.String", "key": true }, + { "name": "city", "type": "Edm.String", "searchable": true } ] + """); + + Assert.Null(SynonymMapValidator.FindInvalidFieldReference(index, Existing())); + } +} diff --git a/AzureSearchEmulator/Controllers/IndexesController.cs b/AzureSearchEmulator/Controllers/IndexesController.cs index 1748559..50bf957 100644 --- a/AzureSearchEmulator/Controllers/IndexesController.cs +++ b/AzureSearchEmulator/Controllers/IndexesController.cs @@ -12,6 +12,7 @@ namespace AzureSearchEmulator.Controllers; public class IndexesController( JsonSerializerOptions jsonSerializerOptions, ISearchIndexRepository searchIndexRepository, + ISynonymMapRepository synonymMapRepository, ILuceneDirectoryFactory luceneDirectoryFactory, ILuceneIndexReaderFactory luceneIndexReaderFactory, ILuceneIndexWriterFactory luceneIndexWriterFactory) @@ -83,6 +84,11 @@ public async Task Post([FromBody] SearchIndex? index) return BadRequest(normalizerError); } + if (await FindInvalidSynonymMapReference(index) is { } synonymMapError) + { + return BadRequest(synonymMapError); + } + try { await searchIndexRepository.Create(index); @@ -141,6 +147,11 @@ public async Task Put(string key, [FromBody] SearchIndex? index) return BadRequest(normalizerError); } + if (await FindInvalidSynonymMapReference(index) is { } synonymMapError) + { + return BadRequest(synonymMapError); + } + var existing = await searchIndexRepository.Get(key); if (existing == null) @@ -230,6 +241,27 @@ private ContentResult IndexJson(SearchIndex index, int statusCode = StatusCodes. }; } + /// + /// Checks the synonym maps the index's fields name against the ones the service holds + /// (issue #69). + /// + /// + /// Async, and so kept out of the static validator chain above: unlike every other check + /// here, this one cannot be answered from the index definition alone, because a synonym map + /// lives outside it. + /// + private async Task FindInvalidSynonymMapReference(SearchIndex index) + { + var names = new HashSet(StringComparer.OrdinalIgnoreCase); + + await foreach (var synonymMap in synonymMapRepository.GetAll()) + { + names.Add(synonymMap.Name); + } + + return SynonymMapValidator.FindInvalidFieldReference(index, names); + } + /// /// Checks the index's complex fields, returning an error message when one is malformed, /// or null when the schema is acceptable. diff --git a/AzureSearchEmulator/Controllers/ServiceStatsController.cs b/AzureSearchEmulator/Controllers/ServiceStatsController.cs index 355876f..1b40625 100644 --- a/AzureSearchEmulator/Controllers/ServiceStatsController.cs +++ b/AzureSearchEmulator/Controllers/ServiceStatsController.cs @@ -6,7 +6,9 @@ namespace AzureSearchEmulator.Controllers; [ApiController] [Route("servicestats")] -public class ServiceStatsController(ISearchIndexRepository searchIndexRepository) : ControllerBase +public class ServiceStatsController( + ISearchIndexRepository searchIndexRepository, + ISynonymMapRepository synonymMapRepository) : ControllerBase { [HttpGet] public async Task Get() @@ -16,6 +18,12 @@ public async Task Get() { indexes.Add(index); } + + var synonymMapCount = 0; + await foreach (var _ in synonymMapRepository.GetAll()) + { + synonymMapCount++; + } var stats = new Dictionary { @@ -53,7 +61,7 @@ public async Task Get() }, synonymMaps = new { - usage = 0, + usage = synonymMapCount, quota = 3 }, skillsetCount = new diff --git a/AzureSearchEmulator/Controllers/SynonymMapsController.cs b/AzureSearchEmulator/Controllers/SynonymMapsController.cs new file mode 100644 index 0000000..d3630ef --- /dev/null +++ b/AzureSearchEmulator/Controllers/SynonymMapsController.cs @@ -0,0 +1,165 @@ +using System.Text.Json; +using AzureSearchEmulator.Indexing; +using AzureSearchEmulator.Models; +using AzureSearchEmulator.Repositories; +using Microsoft.AspNetCore.Mvc; +using SynonymMap = AzureSearchEmulator.Models.SynonymMap; + +namespace AzureSearchEmulator.Controllers; + +/// +/// The service-level /synonymmaps routes (issue #69). +/// +/// +/// A plain rather than an ODataController, and every +/// response is written with System.Text.Json. explains +/// why that matters for a type with a [JsonExtensionData] bag: OData serializes strictly +/// from the EDM model, so a map's encryptionKey — which the emulator stores but does not +/// model — would come back as an empty object or vanish. Unlike the index routes there is no +/// paired collection endpoint left on OData, so the value wrapper the Azure Search SDK +/// expects from a listing is written by hand in . +/// +[ApiController] +public class SynonymMapsController( + JsonSerializerOptions jsonSerializerOptions, + ISynonymMapRepository synonymMapRepository) + : ControllerBase +{ + [HttpGet] + [Route("synonymmaps")] + public async Task Get() + { + var synonymMaps = new List(); + + await foreach (var synonymMap in synonymMapRepository.GetAll()) + { + synonymMaps.Add(synonymMap); + } + + // The SDK reads a listing out of "value"; returning the bare array fails to deserialize. + return Json(new { value = synonymMaps }); + } + + [HttpGet] + [Route("synonymmaps({key})")] + [Route("synonymmaps/{key}")] + public async Task Get(string key) + { + var synonymMap = await synonymMapRepository.Get(key.Trim('\'')); + + if (synonymMap == null) + { + return NotFound(); + } + + return Json(synonymMap); + } + + [HttpPost] + [Route("synonymmaps")] + public async Task Post([FromBody] SynonymMap? synonymMap) + { + if (synonymMap == null || !ModelState.IsValid) + { + return BadRequest(ModelState); + } + + if (SynonymMapValidator.FindInvalidSynonymMap(synonymMap) is { } error) + { + return BadRequest(error); + } + + try + { + await synonymMapRepository.Create(synonymMap); + } + catch (SynonymMapExistsException ex) + { + return Conflict(ex.Message); + } + + return Json(synonymMap, StatusCodes.Status201Created); + } + + /// + /// Create-or-update, which is what the SDK's CreateOrUpdateSynonymMap calls. Nothing + /// about a map is immutable — the rules are read afresh on every search — so unlike an index + /// there is no schema-change check to make here, and replacing the definition wholesale is + /// the whole of the update. + /// + [HttpPut] + [Route("synonymmaps({key})")] + [Route("synonymmaps/{key}")] + public async Task Put(string key, [FromBody] SynonymMap? synonymMap) + { + key = key.Trim('\''); + + if (synonymMap == null || !ModelState.IsValid) + { + return BadRequest(ModelState); + } + + if (!string.Equals(synonymMap.Name, key, StringComparison.OrdinalIgnoreCase)) + { + ModelState.AddModelError( + nameof(synonymMap.Name), + "The synonym map name in the request body must match the name in the URL."); + + return BadRequest(ModelState); + } + + if (SynonymMapValidator.FindInvalidSynonymMap(synonymMap) is { } error) + { + return BadRequest(error); + } + + var existing = await synonymMapRepository.Get(key); + + if (existing == null) + { + await synonymMapRepository.Create(synonymMap); + + return Json(synonymMap, StatusCodes.Status201Created); + } + + await synonymMapRepository.Update(synonymMap); + + return Json(synonymMap); + } + + [HttpDelete] + [Route("synonymmaps({key})")] + [Route("synonymmaps/{key}")] + public async Task Delete(string key) + { + var synonymMap = await synonymMapRepository.Get(key.Trim('\'')); + + if (synonymMap == null) + { + return NotFound(); + } + + await synonymMapRepository.Delete(synonymMap); + + return NoContent(); + } + + /// + /// Writes a response body with System.Text.Json, for the reason given on the class. + /// + private ContentResult Json(object value, int statusCode = StatusCodes.Status200OK) + { + if (statusCode == StatusCodes.Status201Created) + { + Response.Headers.Location = + $"{Request.Scheme}://{Request.Host}{Request.PathBase}/synonymmaps('{((SynonymMap)value).Name}')"; + } + + return new ContentResult + { + Content = JsonSerializer.Serialize(value, jsonSerializerOptions), + ContentType = "application/json", + StatusCode = statusCode + }; + } +} diff --git a/AzureSearchEmulator/Indexing/SynonymMapValidator.cs b/AzureSearchEmulator/Indexing/SynonymMapValidator.cs new file mode 100644 index 0000000..2c4f9e3 --- /dev/null +++ b/AzureSearchEmulator/Indexing/SynonymMapValidator.cs @@ -0,0 +1,124 @@ +using AzureSearchEmulator.Models; +using AzureSearchEmulator.SearchData; +using SynonymMap = AzureSearchEmulator.Models.SynonymMap; + +namespace AzureSearchEmulator.Indexing; + +/// +/// Checks synonym map definitions and the names an index's fields refer to (issue #69). +/// +/// +/// Split across two entry points because the two halves are checked at different times: a map +/// is validated on its own routes when it is written, while the field references are validated +/// with the index that makes them. Following , both report the +/// mistake when the definition is written rather than leaving it to surface against whichever +/// query first reached the field. +/// +public static class SynonymMapValidator +{ + /// + /// Returns a message describing the first problem with the map, or null when it is usable. + /// + public static string? FindInvalidSynonymMap(SynonymMap synonymMap) + { + if (string.IsNullOrWhiteSpace(synonymMap.Name)) + { + return "A synonym map must have a name."; + } + + if (!string.Equals(synonymMap.Format, SynonymMap.SolrFormat, StringComparison.OrdinalIgnoreCase)) + { + return $"Synonym map '{synonymMap.Name}' declares format '{synonymMap.Format}'; only " + + $"'{SynonymMap.SolrFormat}' is supported."; + } + + // Azure requires the rules, and an empty map would expand nothing while looking to a + // caller as though it had been applied. + if (string.IsNullOrWhiteSpace(synonymMap.Synonyms)) + { + return $"Synonym map '{synonymMap.Name}' must define at least one synonym rule."; + } + + try + { + SynonymMapBuilder.Build(synonymMap); + } + catch (SynonymMapDefinitionException ex) + { + return ex.Message; + } + + return null; + } + + /// + /// Returns a message describing the first field that misuses a synonym map, or null when + /// the index's references are acceptable. + /// + /// + /// is the set of map names the service currently holds. It is + /// passed in rather than looked up here so this stays a pure function, in keeping with the + /// other validators the controller chains together. + /// + public static string? FindInvalidFieldReference(SearchIndex index, IReadOnlySet existing) + { + foreach (var (field, path) in EnumerateFields(index.Fields)) + { + if (field.SynonymMaps.Count == 0) + { + continue; + } + + if (!IsStringField(field)) + { + return $"Field '{path}' is type {field.Type} and cannot have a synonym map; " + + "synonym maps apply to Edm.String and Collection(Edm.String) fields."; + } + + // Synonyms only ever widen a full-text query, so a field that no query can reach + // would name a map that never applied. + if (!field.Searchable.GetValueOrDefault()) + { + return $"Field '{path}' names a synonym map but is not searchable; synonym maps " + + "apply only to full-text search."; + } + + foreach (var name in field.SynonymMaps) + { + if (!existing.Contains(name)) + { + return $"Field '{path}' names synonym map '{name}', which does not exist."; + } + } + } + + return null; + } + + private static bool IsStringField(SearchField field) + => field.Type == "Edm.String" + || (field.IsCollection() + && SearchFieldExtensions.GetCollectionElementType(field.Type) == "Edm.String"); + + /// + /// Walks the index's fields, including the sub-fields of complex types, pairing each with + /// its full path, so that a mistake one level down is reported against the path that + /// identifies it. + /// + private static IEnumerable<(SearchField Field, string Path)> EnumerateFields( + IEnumerable fields, + string prefix = "") + { + foreach (var field in fields) + { + var path = prefix + field.Name; + + yield return (field, path); + + foreach (var subField in EnumerateFields(field.Fields, path + ComplexTypeSupport.PathSeparator)) + { + yield return subField; + } + } + } +} diff --git a/AzureSearchEmulator/Models/SynonymMap.cs b/AzureSearchEmulator/Models/SynonymMap.cs new file mode 100644 index 0000000..1d7b18d --- /dev/null +++ b/AzureSearchEmulator/Models/SynonymMap.cs @@ -0,0 +1,76 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AzureSearchEmulator.Models; + +/// +/// A synonym map: a named set of rules that widen a query's terms at search time (issue #69). +/// +/// +/// Unlike an analyzer or a normalizer, this is not part of an index definition. It is a +/// service-level resource with its own /synonymmaps routes and its own lifetime, and a +/// field opts into it by naming it in . That indirection is +/// the point of the feature: one map can be edited once and take effect across every field of +/// every index that names it, without any of them being rewritten. +/// +/// Azure applies the rules at query time only — never while indexing. Expanding at index time +/// would bake today's rules into the stored terms, so editing a map afterwards would leave the +/// documents indexed before the edit disagreeing with the ones after it, and the only repair +/// would be a full reindex. Expanding the query instead keeps the rules a property of the +/// search, which is why is reached from the search +/// analyzer and not from the indexing path. +/// +public class SynonymMap +{ + /// + /// The only format Azure supports, and the one the parser understands. + /// + public const string SolrFormat = "solr"; + + /// + /// The name a field's synonymMaps entry refers to. + /// + public string Name { get; set; } = ""; + + /// + /// The format the rules in are written in. + /// + /// + /// Defaulted rather than required, for the reason + /// is: Azure defines exactly one accepted value, so a definition that omits it is + /// unambiguous. rejects any other value, because + /// a map claiming a format the emulator cannot parse would otherwise be stored and then + /// silently expand nothing. + /// + public string Format { get; set; } = SolrFormat; + + /// + /// The rules, in Solr syntax, separated by newlines. + /// + /// + /// Azure carries these as one newline-delimited string rather than an array, which is why + /// this is a string here too — splitting it into a list would change the wire shape the + /// Azure Search SDK writes and reads. + /// + /// Two forms are meaningful, and they do different things: + /// usa, united states makes every term equivalent to every other, while + /// dog => canine, hound replaces the left side with the right and drops the + /// original term from the query. + /// + public string Synonyms { get; set; } = ""; + + /// + /// The map's entity tag, echoed back so a client's optimistic concurrency check has + /// something to compare against. + /// + [JsonPropertyName("@odata.etag")] + public string? ETag { get; set; } + + /// + /// Present for the reason given on : + /// without it, properties the emulator does not model — encryptionKey among them — + /// would be dropped from a map that is read back and written again. + /// + [JsonExtensionData] + public Dictionary? AdditionalProperties { get; set; } +} diff --git a/AzureSearchEmulator/Models/SynonymMapExistsException.cs b/AzureSearchEmulator/Models/SynonymMapExistsException.cs new file mode 100644 index 0000000..aa63d99 --- /dev/null +++ b/AzureSearchEmulator/Models/SynonymMapExistsException.cs @@ -0,0 +1,3 @@ +namespace AzureSearchEmulator.Models; + +public class SynonymMapExistsException(string synonymMapKey) : Exception($"Synonym map with key {synonymMapKey} already exists"); diff --git a/AzureSearchEmulator/Program.cs b/AzureSearchEmulator/Program.cs index 7315da0..271e70f 100644 --- a/AzureSearchEmulator/Program.cs +++ b/AzureSearchEmulator/Program.cs @@ -89,6 +89,7 @@ }); builder.Services.AddTransient(); +builder.Services.AddTransient(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); // Singleton so writers stay open across requests; the container disposes it on shutdown, diff --git a/AzureSearchEmulator/Repositories/FileSynonymMapRepository.cs b/AzureSearchEmulator/Repositories/FileSynonymMapRepository.cs new file mode 100644 index 0000000..f44cdc8 --- /dev/null +++ b/AzureSearchEmulator/Repositories/FileSynonymMapRepository.cs @@ -0,0 +1,117 @@ +using System.Text.Json; +using AzureSearchEmulator.Models; +using Microsoft.Extensions.Options; +using static System.IO.File; + +namespace AzureSearchEmulator.Repositories; + +/// +/// Keeps each synonym map in its own JSON file, next to the index definitions. +/// +/// +/// Deliberately the same shape as , down to the +/// lower-cased file name and the guard on path characters — a synonym map name arrives from the +/// same untrusted route an index name does, and .. in it would otherwise escape the +/// directory. +/// +/// Unlike an index, a map owns no Lucene directory, so has only the one +/// file to remove. +/// +public class FileSynonymMapRepository(JsonSerializerOptions jsonSerializerOptions, IOptions options) + : ISynonymMapRepository +{ + private const string FileSuffix = ".synonymmap.json"; + + private readonly EmulatorOptions _options = options.Value; + + public async IAsyncEnumerable GetAll() + { + if (!Directory.Exists(_options.IndexesDirectory)) + { + yield break; + } + + var files = Directory.GetFiles(_options.IndexesDirectory, $"*{FileSuffix}"); + + foreach (var file in files) + { + yield return JsonSerializer.Deserialize(await ReadAllTextAsync(file), jsonSerializerOptions) + ?? throw new InvalidOperationException($"Invalid synonym map definition file: {file}"); + } + } + + public async Task Get(string key) + { + if (!Directory.Exists(_options.IndexesDirectory)) + { + return null; + } + + string file = GetSynonymMapFileName(key); + + if (!Exists(file)) + { + return null; + } + + return JsonSerializer.Deserialize(await ReadAllTextAsync(file), jsonSerializerOptions); + } + + public Task Create(SynonymMap synonymMap) + { + if (!Directory.Exists(_options.IndexesDirectory)) + { + Directory.CreateDirectory(_options.IndexesDirectory); + } + + string file = GetSynonymMapFileName(synonymMap.Name); + + if (Exists(file)) + { + throw new SynonymMapExistsException(synonymMap.Name); + } + + return WriteAllTextAsync(file, JsonSerializer.Serialize(synonymMap, jsonSerializerOptions)); + } + + public Task Update(SynonymMap synonymMap) + { + if (!Directory.Exists(_options.IndexesDirectory)) + { + Directory.CreateDirectory(_options.IndexesDirectory); + } + + string file = GetSynonymMapFileName(synonymMap.Name); + + return WriteAllTextAsync(file, JsonSerializer.Serialize(synonymMap, jsonSerializerOptions)); + } + + public Task Delete(SynonymMap synonymMap) + { + if (!Directory.Exists(_options.IndexesDirectory)) + { + return Task.FromResult(false); + } + + string file = GetSynonymMapFileName(synonymMap.Name); + + if (!Exists(file)) + { + return Task.FromResult(false); + } + + File.Delete(file); + + return Task.FromResult(true); + } + + private string GetSynonymMapFileName(string key) + { + if (key.Contains('.') || key.Contains('/') || key.Contains('\\')) + { + throw new ArgumentException("Synonym map file name cannot contain any of the following characters: . \\ /"); + } + + return Path.Combine(_options.IndexesDirectory, $"{key.ToLowerInvariant()}{FileSuffix}"); + } +} diff --git a/AzureSearchEmulator/Repositories/ISynonymMapRepository.cs b/AzureSearchEmulator/Repositories/ISynonymMapRepository.cs new file mode 100644 index 0000000..21be26c --- /dev/null +++ b/AzureSearchEmulator/Repositories/ISynonymMapRepository.cs @@ -0,0 +1,24 @@ +using AzureSearchEmulator.Models; + +namespace AzureSearchEmulator.Repositories; + +/// +/// Stores the service's synonym maps (issue #69). +/// +/// +/// Separate from because a synonym map is a service-level +/// resource rather than part of an index: it is created, edited and deleted on its own routes, +/// and the indexes that name it are unaware of when that happens. +/// +public interface ISynonymMapRepository +{ + IAsyncEnumerable GetAll(); + + Task Get(string key); + + Task Create(SynonymMap synonymMap); + + Task Update(SynonymMap synonymMap); + + Task Delete(SynonymMap synonymMap); +} diff --git a/AzureSearchEmulator/SearchData/AnalyzerHelper.cs b/AzureSearchEmulator/SearchData/AnalyzerHelper.cs index eac545c..ebd0f02 100644 --- a/AzureSearchEmulator/SearchData/AnalyzerHelper.cs +++ b/AzureSearchEmulator/SearchData/AnalyzerHelper.cs @@ -224,6 +224,32 @@ public static Analyzer GetAnalyzer(SearchIndex? index, string? name) public static Analyzer GetPerFieldSearchAnalyzer(SearchIndex index) => GetPerFieldAnalyzer(index, i => i.SearchAnalyzer ?? i.Analyzer); + /// + /// The search analyzer, with each field's synonym maps applied on top of it (issue #69). + /// + /// + /// Only the search side has this overload. Azure expands synonyms when a query is parsed + /// and never while indexing, so there is deliberately no equivalent for + /// — see for why. + /// + /// is keyed by name, case-insensitively, and holds every map + /// the service currently has; the fields pick out the ones they name. + /// + public static Analyzer GetPerFieldSearchAnalyzer( + SearchIndex index, + IReadOnlyDictionary synonymMaps) + { + if (synonymMaps.Count == 0) + { + return GetPerFieldSearchAnalyzer(index); + } + + return GetPerFieldAnalyzer( + index, + i => i.SearchAnalyzer ?? i.Analyzer, + (field, analyzer) => SynonymMapHelper.Wrap(analyzer, SynonymMapHelper.Resolve(field, synonymMaps))); + } + public static Analyzer GetPerFieldIndexAnalyzer(SearchIndex index) => GetPerFieldAnalyzer(index, i => i.IndexAnalyzer ?? i.Analyzer); @@ -235,14 +261,34 @@ public static Analyzer GetPerFieldIndexAnalyzer(SearchIndex index) /// they are registered under that path here too — keying them by their bare name would /// silently leave them on the default analyzer. /// - private static Analyzer GetPerFieldAnalyzer(SearchIndex index, Func selectAnalyzer) + /// + /// Applied to each field's resolved analyzer, for a caller that needs to add to the chain — + /// synonym expansion is the one that does. Fields that name no analyzer at all are included + /// when a wrapper is given, because a synonym map on a field that never named an analyzer + /// still has to reach the default one. + /// + private static Analyzer GetPerFieldAnalyzer( + SearchIndex index, + Func selectAnalyzer, + Func? wrap = null) { - var analyzers = index.Fields + var fields = index.Fields .SelectMany(i => ComplexTypeSupport.EnumerateLeafFields(i)) - .Select(i => (i.Path, Analyzer: selectAnalyzer(i.Field))) - .Where(i => i.Analyzer != null) - .ToDictionary(i => i.Path, i => GetAnalyzer(index, i.Analyzer)); + .Select(i => (i.Path, i.Field, Analyzer: selectAnalyzer(i.Field))); + + if (wrap == null) + { + var named = fields + .Where(i => i.Analyzer != null) + .ToDictionary(i => i.Path, i => GetAnalyzer(index, i.Analyzer)); + + return new PerFieldAnalyzerWrapper(CreateDefault(), named); + } + + var wrapped = fields + .Select(i => (i.Path, Analyzer: wrap(i.Field, GetAnalyzer(index, i.Analyzer)))) + .ToDictionary(i => i.Path, i => i.Analyzer); - return new PerFieldAnalyzerWrapper(CreateDefault(), analyzers); + return new PerFieldAnalyzerWrapper(CreateDefault(), wrapped); } } diff --git a/AzureSearchEmulator/SearchData/SynonymMapBuilder.cs b/AzureSearchEmulator/SearchData/SynonymMapBuilder.cs new file mode 100644 index 0000000..76d6313 --- /dev/null +++ b/AzureSearchEmulator/SearchData/SynonymMapBuilder.cs @@ -0,0 +1,81 @@ +using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Core; +using Lucene.Net.Analysis.Synonym; +using Lucene.Net.Util; +using LuceneSynonymMap = Lucene.Net.Analysis.Synonym.SynonymMap; +using SynonymMap = AzureSearchEmulator.Models.SynonymMap; + +namespace AzureSearchEmulator.SearchData; + +/// +/// Turns a synonym map's Solr rules into the compiled form Lucene matches against (issue #69). +/// +public static class SynonymMapBuilder +{ + private const LuceneVersion Version = LuceneVersion.LUCENE_48; + + /// + /// Parses the map's rules. + /// + /// + /// The parser is given expand: true so that a comma-separated rule makes every term + /// equivalent to every other, which is what Azure documents an equivalency rule to mean. It + /// does not affect the explicit a => b form: an arrow rule always replaces its left + /// side, so dog => canine drops "dog" from the query whichever way expand is set. + /// + /// Rules are tokenized with the same analyzer the terms will be matched against a + /// lower-casing whitespace chain rather than the field's own. Azure matches synonyms on + /// whole words, and using the field's analyzer here would let a stemming or n-gram chain + /// rewrite the rule text into tokens the rule's author never wrote. + /// + /// + /// The rules are malformed. refuses those when + /// the map is created, so reaching this means a map was written before that validation + /// existed. + /// + public static LuceneSynonymMap Build(SynonymMap synonymMap) + { + if (!string.Equals(synonymMap.Format, SynonymMap.SolrFormat, StringComparison.OrdinalIgnoreCase)) + { + throw new SynonymMapDefinitionException( + $"Synonym map '{synonymMap.Name}' declares format '{synonymMap.Format}', but only " + + $"'{SynonymMap.SolrFormat}' is supported."); + } + + var parser = new SolrSynonymParser(dedup: true, expand: true, analyzer: CreateRuleAnalyzer()); + + try + { + parser.Parse(new StringReader(synonymMap.Synonyms)); + + return parser.Build(); + } + catch (Exception ex) when (ex is not SynonymMapDefinitionException) + { + throw new SynonymMapDefinitionException( + $"Synonym map '{synonymMap.Name}' could not be parsed: {ex.Message}", ex); + } + } + + /// + /// The analyzer the rule text itself is read with. + /// + /// + /// Whitespace-split and lower-cased, so a multi-word rule such as united states + /// becomes the two tokens the filter needs to match in sequence, and the map is + /// case-insensitive the way Azure's is. + /// + private static Analyzer CreateRuleAnalyzer() => + Analyzer.NewAnonymous((_, reader) => + { + Tokenizer source = new WhitespaceTokenizer(Version, reader); + + return new TokenStreamComponents(source, new LowerCaseFilter(Version, source)); + }); +} + +/// +/// Raised when a synonym map's declared format or rules cannot be turned into a usable map. +/// +public class SynonymMapDefinitionException(string message, Exception? innerException = null) + : Exception(message, innerException); diff --git a/AzureSearchEmulator/SearchData/SynonymMapHelper.cs b/AzureSearchEmulator/SearchData/SynonymMapHelper.cs new file mode 100644 index 0000000..04cc03e --- /dev/null +++ b/AzureSearchEmulator/SearchData/SynonymMapHelper.cs @@ -0,0 +1,114 @@ +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; +using AzureSearchEmulator.Models; +using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Synonym; +using LuceneSynonymMap = Lucene.Net.Analysis.Synonym.SynonymMap; +using SynonymMap = AzureSearchEmulator.Models.SynonymMap; + +namespace AzureSearchEmulator.SearchData; + +/// +/// Applies synonym maps to the analyzer a query's terms are read with (issue #69). +/// +/// +/// Query time only, which is what makes the feature safe to edit: see +/// for why expanding at index time would strand the documents +/// indexed before a rule change. +/// +public static class SynonymMapHelper +{ + /// + /// Compiled maps, keyed by the definition instance they were built from. + /// + /// + /// Parsing rules into an FST is the expensive part of this feature and it would otherwise + /// happen on every search. Keying on the instance rather than the name means an edited map + /// — a fresh object from the file repository — misses the cache and is rebuilt, so a + /// search never answers from rules the caller has already replaced. + /// + private static readonly ConditionalWeakTable Cache = new(); + + /// + /// Wraps so the tokens it produces are expanded by + /// , in the order the field names them. + /// + /// + /// Chained rather than merged: Azure lets a field name several maps, and running each as + /// its own filter keeps a rule in one map from being combined with a rule in another into + /// an expansion neither map describes. + /// + public static Analyzer Wrap(Analyzer analyzer, IReadOnlyList synonymMaps) + { + if (synonymMaps.Count == 0) + { + return analyzer; + } + + var compiled = synonymMaps.Select(Compile).ToList(); + + return new SynonymAnalyzerWrapper(analyzer, compiled); + } + + /// + /// Compiles a map, reusing the previous result when the definition has not been replaced. + /// + public static LuceneSynonymMap Compile(SynonymMap synonymMap) => + Cache.GetValue(synonymMap, SynonymMapBuilder.Build); + + /// + /// Resolves the maps a field names, skipping any the service no longer holds. + /// + /// + /// A missing map is skipped rather than thrown on. Azure refuses to delete a synonym map + /// while an index still names it, but the emulator's indexes and maps are separate files + /// that a user may edit or restore independently, and failing every search against the + /// index would be a harsh answer to a dangling name that only widens results. + /// + public static IReadOnlyList Resolve( + SearchField field, + IReadOnlyDictionary available) + { + if (field.SynonymMaps.Count == 0) + { + return []; + } + + return field.SynonymMaps + .Select(i => available.GetValueOrDefault(i)) + .OfType() + .ToList(); + } + + /// + /// An analyzer that appends a per map to another analyzer's chain. + /// + /// + /// exists for exactly this: it delegates the tokenizer and the + /// rest of the chain to the wrapped analyzer and lets the wrapper add to the token stream, + /// so a field's own analyzer — custom, predefined or the default — keeps deciding how the + /// text is split before any synonym is considered. + /// + private sealed class SynonymAnalyzerWrapper(Analyzer inner, IReadOnlyList maps) + : AnalyzerWrapper(inner.Strategy) + { + protected override Analyzer GetWrappedAnalyzer(string fieldName) => inner; + + protected override TokenStreamComponents WrapComponents( + string fieldName, + TokenStreamComponents components) + { + var stream = components.TokenStream; + + foreach (var map in maps) + { + // ignoreCase is false because the rules were already lower-cased when parsed, + // and a field whose analyzer does not lower-case its tokens should not have + // synonym matching quietly do it for them. + stream = new SynonymFilter(stream, map, ignoreCase: false); + } + + return new TokenStreamComponents(components.Tokenizer, stream); + } + } +} diff --git a/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs b/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs index 1315b63..2d049f5 100644 --- a/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs +++ b/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs @@ -2,6 +2,7 @@ using System.Text.Json.Nodes; using AzureSearchEmulator.Indexing; using AzureSearchEmulator.Models; +using AzureSearchEmulator.Repositories; using AzureSearchEmulator.SearchData; using Lucene.Net.Analysis; using Lucene.Net.Index; @@ -14,7 +15,14 @@ namespace AzureSearchEmulator.Searching; -public class LuceneNetIndexSearcher(ILuceneIndexReaderFactory indexReaderFactory) : IIndexSearcher +/// +/// The service's synonym maps, or null where there are none to apply (issue #69). Optional so +/// that a caller with no interest in the feature — the unit tests build a searcher over a +/// bare Lucene directory — need not supply a store for it. +/// +public class LuceneNetIndexSearcher( + ILuceneIndexReaderFactory indexReaderFactory, + ISynonymMapRepository? synonymMapRepository = null) : IIndexSearcher { private const string SearchScoreFunction = "search.score("; @@ -53,7 +61,7 @@ public Task GetDocCount(SearchIndex index) } } - public Task Search(SearchIndex index, SearchRequest request) + public async Task Search(SearchIndex index, SearchRequest request) { // Validated against the index definition before the reader is opened, so a malformed // request is reported as the fault it is rather than as whatever the storage layer @@ -73,15 +81,20 @@ public Task Search(SearchIndex index, SearchRequest request) throw new InvalidOperationException(missing); } + // Resolved before the filter, because a search.ismatch inside one is a full-text query + // and expands its terms the same way the main query does (issue #69). + var synonymMaps = await GetSynonymMaps(index); + // Built before the query because a vector query needs it: under preFilter — the default // and the only mode Azure applies to the scan itself — the filter decides which // documents the nearest-neighbour search considers at all (issue #46). - var filter = GetFilterFromRequest(request, index); + var filter = GetFilterFromRequest(request, index, synonymMaps); // Captured once so every document in the response is scored against the same instant; // a freshness function reading the clock per document would rank inconsistently within // a single query. - var query = GetQueryFromRequest(index, request, profile, scoringParameters, DateTimeOffset.UtcNow, filter); + var query = GetQueryFromRequest( + index, request, profile, scoringParameters, DateTimeOffset.UtcNow, filter, synonymMaps); // Facet expressions are parsed even when nothing can match, so that an invalid one is // still reported as the error it is rather than silently returning no facets. @@ -89,7 +102,7 @@ public Task Search(SearchIndex index, SearchRequest request) if (query == null) { - return Task.FromResult(new SearchResponse + return new SearchResponse { Count = 0, // An empty match set still produces the requested facets, with every bucket @@ -98,7 +111,7 @@ public Task Search(SearchIndex index, SearchRequest request) // Coverage describes how much of the index was searched, not how much of it // matched, so a query that cannot match anything still reports it in full. Coverage = SearchCoverage.GetCoverage(request), - }); + }; } var sort = GetSortFromRequest(index, request, query); @@ -135,7 +148,7 @@ public Task Search(SearchIndex index, SearchRequest request) response.Facets = FacetCounter.Count(searcher, facets, query, filter); } - return Task.FromResult(response); + return response; } var docs = searcher.Search(query, filter, hitsWanted, sort, true, true); @@ -171,7 +184,7 @@ public Task Search(SearchIndex index, SearchRequest request) response.Facets = FacetCounter.Count(searcher, facets, query, filter); } - return Task.FromResult(response); + return response; } public Task Suggest(SearchIndex index, SuggestRequest request) @@ -669,7 +682,43 @@ private static SortFieldType GetSortFieldType(SearchField field) }; } - private static Filter? GetFilterFromRequest(SearchRequest request, SearchIndex? index = null) + /// + /// The service's synonym maps, keyed by name, or an empty set when no field names one + /// (issue #69). + /// + /// + /// Short-circuited on the index definition so an index that uses no synonym maps — which is + /// most of them — never reads the synonym map directory at all, and the feature costs a + /// search that does not use it nothing. + /// + private async Task> GetSynonymMaps(SearchIndex index) + { + var referenced = ComplexTypeSupport.EnumerateLeafFields(index) + .SelectMany(i => i.Field.SynonymMaps) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + if (referenced.Count == 0 || synonymMapRepository == null) + { + return new Dictionary(); + } + + var maps = new Dictionary(StringComparer.OrdinalIgnoreCase); + + await foreach (var synonymMap in synonymMapRepository.GetAll()) + { + if (referenced.Contains(synonymMap.Name)) + { + maps[synonymMap.Name] = synonymMap; + } + } + + return maps; + } + + private static Filter? GetFilterFromRequest( + SearchRequest request, + SearchIndex? index = null, + IReadOnlyDictionary? synonymMaps = null) { if (string.IsNullOrEmpty(request.Filter)) { @@ -684,7 +733,7 @@ private static SortFieldType GetSortFieldType(SearchField field) return null; } - var query = filterQuery.Accept(new ODataQueryVisitor(index)); + var query = filterQuery.Accept(new ODataQueryVisitor(index, synonymMaps)); return new QueryWrapperFilter(query); } @@ -707,7 +756,8 @@ private static SortFieldType GetSortFieldType(SearchField field) ScoringProfile? profile, ScoringParameterCollection parameters, DateTimeOffset now, - Filter? filter) + Filter? filter, + IReadOnlyDictionary synonymMaps) { var preFilter = GetVectorPreFilter(request, filter); @@ -715,7 +765,7 @@ private static SortFieldType GetSortFieldType(SearchField field) ? VectorQuerySupport.BuildQueries(index, request, preFilter) : []; - var query = GetBaseQueryFromRequest(index, request, profile, vectorQueries.Count > 0); + var query = GetBaseQueryFromRequest(index, request, profile, vectorQueries.Count > 0, synonymMaps); if (query != null && profile != null && IsScored(request)) { @@ -756,7 +806,8 @@ private static bool IsScored(SearchRequest request) SearchIndex index, SearchRequest request, ScoringProfile? profile, - bool hasVectorQueries = false) + bool hasVectorQueries, + IReadOnlyDictionary synonymMaps) { // A wildcard counts as no text query here, not as one matching everything: alongside // vector queries it is a pure vector search, which is how Azure reads it and how REST @@ -794,7 +845,7 @@ private static bool IsScored(SearchRequest request) throw new InvalidOperationException("Unable to search with no searchable fields"); } - var analyzer = AnalyzerHelper.GetPerFieldSearchAnalyzer(index); + var analyzer = AnalyzerHelper.GetPerFieldSearchAnalyzer(index, synonymMaps); return request.QueryType switch { diff --git a/AzureSearchEmulator/Searching/ODataQueryVisitor.cs b/AzureSearchEmulator/Searching/ODataQueryVisitor.cs index 608351b..9115442 100644 --- a/AzureSearchEmulator/Searching/ODataQueryVisitor.cs +++ b/AzureSearchEmulator/Searching/ODataQueryVisitor.cs @@ -14,10 +14,23 @@ namespace AzureSearchEmulator.Searching; -public class ODataQueryVisitor(SearchIndex? index = null) : ISyntacticTreeVisitor +public class ODataQueryVisitor( + SearchIndex? index = null, + IReadOnlyDictionary? synonymMaps = null) : ISyntacticTreeVisitor { private readonly SearchIndex? _index = index; + /// + /// The synonym maps a search.ismatch inside a filter expands its terms with + /// (issue #69). + /// + /// + /// Defaulted to none so the many callers that parse a filter with no full-text function in + /// it — and the tests that construct this directly — need not supply any. + /// + private readonly IReadOnlyDictionary _synonymMaps = + synonymMaps ?? new Dictionary(); + // When walking into a lambda (any/all), pushes the (path, parameter) context so that // child RangeVariableTokens can be resolved back to the collection's field path. private readonly Stack _lambdaContexts = new(); @@ -1176,7 +1189,7 @@ private Query ParseFullTextSearchQuery(string searchText, string? searchFields, } // Get the analyzer for this index - var analyzer = AnalyzerHelper.GetPerFieldSearchAnalyzer(_index); + var analyzer = AnalyzerHelper.GetPerFieldSearchAnalyzer(_index, _synonymMaps); // Parse the search text var query = queryType switch diff --git a/README.md b/README.md index 85fa7c6..68d6699 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,8 @@ Currently, there is support (to varying degrees) for the following Azure Search (see Analyzers below) * Normalizers for filter, facet and sort values: every predefined `LexicalNormalizerName`, plus custom `normalizers` defined on the index (see Normalizers below) +* Synonym maps: service-level `synonymmaps` in Solr format, named by a field's `synonymMaps` + to widen queries at search time (see Synonym maps below) * Vector search: `Collection(Edm.Single)` fields with `vectorSearch` profiles and algorithms, `vectorQueries` with all three metrics and both filter modes, and hybrid search fusing text and vector rankings with Reciprocal Rank Fusion (see Vector fields below) @@ -350,6 +352,54 @@ fixed once the index exists; changing it requires rebuilding the index. * **Options that name a file**, such as a synonym map on disk. Word lists and mappings given inline in the index definition work normally. +### Synonym maps + +A synonym map widens a full-text query so that a search for one term also matches documents +holding an equivalent one, without either the query or the documents being rewritten. Maps are +service-level resources with their own routes, and a field opts in by naming one: + +```http +POST /synonymmaps?api-version=2024-07-01 +Content-Type: application/json + +{ + "name": "products", + "format": "solr", + "synonyms": "usa, united states, united states of america\ndog => canine, hound" +} +``` + +```json +{ + "name": "name", "type": "Edm.String", "searchable": true, + "synonymMaps": ["products"] +} +``` + +`GET`, `PUT` and `DELETE` on `/synonymmaps('products')` behave as the service's do, and the map +appears in the `synonymMaps` counter of `/servicestats`. + +The Solr format has two rule forms, and they do different things: + +* **Equivalency** — `usa, united states` — makes every term match every other, keeping the term + that was typed. A search for either matches documents holding either. +* **Explicit mapping** — `dog => canine, hound` — replaces the left side with the right, so a + search for `dog` matches documents holding `canine` or `hound`, but no longer matches + documents holding only `dog`. + +Rules are separated by newlines, matched case-insensitively, and may span several words +(`united states of america`), which match as a phrase. + +Expansion happens at query time only, never while indexing — the same as the service. That is +what makes a map safe to edit: the rules a search uses are the ones the map holds now, so +changing them takes effect immediately and never requires reindexing. A field may name several +maps, and each is applied in turn. + +Synonym maps apply to `searchable` `Edm.String` and `Collection(Edm.String)` fields. A field +that names a map it cannot use — one that is not searchable, or holds no text — is rejected when +the index is created, as is a field naming a map the service does not have. Only the `solr` +format is supported, which is the only one Azure Search itself supports. + ### Filter semantics Null comparison, lexicographic string ranges, and the two full-text filter functions behave