diff --git a/Backend.Tests/Controllers/WordControllerTests.cs b/Backend.Tests/Controllers/WordControllerTests.cs index 839c84251a..35036fd7b4 100644 --- a/Backend.Tests/Controllers/WordControllerTests.cs +++ b/Backend.Tests/Controllers/WordControllerTests.cs @@ -15,6 +15,7 @@ namespace Backend.Tests.Controllers internal sealed class WordControllerTests : IDisposable { private WordRepositoryMock _wordRepo = null!; + private SemanticDomainCountRepositoryMock _semDomCountRepo = null!; private IPermissionService _permissionService = null!; private IWordService _wordService = null!; private WordController _wordController = null!; @@ -32,9 +33,10 @@ public void Dispose() public void Setup() { _wordRepo = new WordRepositoryMock(); + _semDomCountRepo = new SemanticDomainCountRepositoryMock(); _wordService = new WordService(_wordRepo); _permissionService = new PermissionServiceMock(); - _wordController = new WordController(_wordRepo, _wordService, _permissionService); + _wordController = new WordController(_wordRepo, _wordService, _semDomCountRepo, _permissionService); } [Test] @@ -464,8 +466,11 @@ public async Task TestGetDomainWordCountNoPermission() [Test] public async Task TestGetDomainWordCount() { + _semDomCountRepo.SetCount(ProjId, "1", 3); + var result = await _wordController.GetDomainWordCount(ProjId, "1"); Assert.That(result, Is.InstanceOf()); + Assert.That(((OkObjectResult)result).Value, Is.EqualTo(3)); } } } diff --git a/Backend.Tests/Mocks/SemanticDomainCountRepositoryMock.cs b/Backend.Tests/Mocks/SemanticDomainCountRepositoryMock.cs new file mode 100644 index 0000000000..c09c7a92a3 --- /dev/null +++ b/Backend.Tests/Mocks/SemanticDomainCountRepositoryMock.cs @@ -0,0 +1,71 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using BackendFramework.Interfaces; +using BackendFramework.Models; +using MongoDB.Driver; + +namespace Backend.Tests.Mocks +{ + internal sealed class SemanticDomainCountRepositoryMock : ISemanticDomainCountRepository + { + private readonly List _counts = []; + + public Task GetCount(string projectId, string domainId) + { + var count = _counts + .FirstOrDefault(c => c.ProjectId == projectId && c.DomainId == domainId)?.Count ?? 0; + return Task.FromResult(count); + } + + public Task> GetAllCounts(string projectId) + { + return Task.FromResult(_counts.Where(c => c.ProjectId == projectId).Select(c => c.Clone()).ToList()); + } + + // The session is ignored: this mock does not simulate transactions. + public Task ApplyDeltas( + IClientSessionHandle session, string projectId, IReadOnlyDictionary domainDeltas) + { + foreach (var (domainId, delta) in domainDeltas) + { + if (delta == 0) + { + continue; + } + + var existing = _counts.FirstOrDefault(c => c.ProjectId == projectId && c.DomainId == domainId); + if (existing is null) + { + _counts.Add(new ProjectSemanticDomainCount(projectId, domainId, delta)); + } + else + { + existing.Count += delta; + } + } + + return Task.CompletedTask; + } + + public Task DeleteAllCounts(IClientSessionHandle session, string projectId) + { + _counts.RemoveAll(c => c.ProjectId == projectId); + return Task.CompletedTask; + } + + /// Test helper to seed a count directly, without a transaction. + public void SetCount(string projectId, string domainId, int count) + { + var existing = _counts.FirstOrDefault(c => c.ProjectId == projectId && c.DomainId == domainId); + if (existing is null) + { + _counts.Add(new ProjectSemanticDomainCount(projectId, domainId, count)); + } + else + { + existing.Count = count; + } + } + } +} diff --git a/Backend.Tests/Mocks/WordRepositoryMock.cs b/Backend.Tests/Mocks/WordRepositoryMock.cs index 4d5f976a69..b29772f893 100644 --- a/Backend.Tests/Mocks/WordRepositoryMock.cs +++ b/Backend.Tests/Mocks/WordRepositoryMock.cs @@ -306,12 +306,5 @@ public async Task RevertReplaceFrontier( return true; } - - public Task CountFrontierWordsWithDomain(string projectId, string domainId) - { - var count = _frontier.Count( - w => w.ProjectId == projectId && w.Senses.Any(s => s.SemanticDomains.Any(sd => sd.Id == domainId))); - return Task.FromResult(count); - } } } diff --git a/Backend.Tests/Repositories/MongoDbSetUpFixture.cs b/Backend.Tests/Repositories/MongoDbSetUpFixture.cs new file mode 100644 index 0000000000..6ef083527d --- /dev/null +++ b/Backend.Tests/Repositories/MongoDbSetUpFixture.cs @@ -0,0 +1,26 @@ +using NUnit.Framework; + +namespace Backend.Tests.Repositories +{ + /// + /// Starts a single shared MongoDB instance for all repository integration tests. + /// Fixtures isolate themselves by using distinct database names. + /// + [SetUpFixture] + public sealed class MongoDbSetUpFixture + { + internal static MongoDbTestRunner Runner { get; private set; } = null!; + + [OneTimeSetUp] + public void StartMongo() + { + Runner = MongoDbTestRunner.Start(); + } + + [OneTimeTearDown] + public void StopMongo() + { + Runner?.Dispose(); + } + } +} diff --git a/Backend.Tests/Repositories/MongoDbTestRunner.cs b/Backend.Tests/Repositories/MongoDbTestRunner.cs index ae61cfa6db..cc13c147d4 100644 --- a/Backend.Tests/Repositories/MongoDbTestRunner.cs +++ b/Backend.Tests/Repositories/MongoDbTestRunner.cs @@ -173,9 +173,16 @@ private static void WaitForReplicaSetReady(int port, int timeoutSeconds = 30) { var admin = client.GetDatabase("admin"); var status = admin.RunCommand(new BsonDocument("replSetGetStatus", 1)); + // myState == 1 means PRIMARY, but a freshly elected primary briefly rejects writes with + // "not primary" until it finishes transitioning. Also confirm it is writable via hello so + // callers that write immediately (e.g. index creation) don't race that window. if (status["ok"].ToInt32() == 1 && status["myState"].ToInt32() == 1) { - return; + var hello = admin.RunCommand(new BsonDocument("hello", 1)); + if (hello.GetValue("isWritablePrimary", BsonBoolean.False).ToBoolean()) + { + return; + } } } catch (Exception ex) diff --git a/Backend.Tests/Repositories/SemanticDomainCountRepositoryTests.cs b/Backend.Tests/Repositories/SemanticDomainCountRepositoryTests.cs new file mode 100644 index 0000000000..54280bfa61 --- /dev/null +++ b/Backend.Tests/Repositories/SemanticDomainCountRepositoryTests.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using BackendFramework.Contexts; +using BackendFramework.Models; +using BackendFramework.Repositories; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using NUnit.Framework; + +namespace Backend.Tests.Repositories +{ + /// + /// Integration tests for that spin up an actual MongoDB instance. + /// A single-node replica set is required because the write methods run inside transactions. + /// + [TestFixture] + [Category("IntegrationTest")] + public sealed class SemanticDomainCountRepositoryTests + { + private MongoDbContext _dbContext = null!; + private SemanticDomainCountRepository _repo = null!; + private string _projectId = null!; + + [SetUp] + public void SetUp() + { + _projectId = Guid.NewGuid().ToString(); + var options = Options.Create(new BackendFramework.Startup.Settings + { + ConnectionString = MongoDbSetUpFixture.Runner.ConnectionString, + CombineDatabase = "SemanticDomainCountRepositoryTests", + }); + _dbContext = new MongoDbContext(options); + _repo = new SemanticDomainCountRepository(_dbContext); + } + + private Task ApplyDeltas(IReadOnlyDictionary deltas) + { + return _dbContext.ExecuteInTransaction(async session => + { + await _repo.ApplyDeltas(session, _projectId, deltas); + return true; + }); + } + + [Test] + public async Task ApplyDeltasUpsertsThenIncrements() + { + await ApplyDeltas(new Dictionary { ["1"] = 2, ["1.1"] = 1 }); + Assert.That(await _repo.GetCount(_projectId, "1"), Is.EqualTo(2)); + Assert.That(await _repo.GetCount(_projectId, "1.1"), Is.EqualTo(1)); + + await ApplyDeltas(new Dictionary { ["1"] = 3 }); + Assert.That(await _repo.GetCount(_projectId, "1"), Is.EqualTo(5)); + } + + [Test] + public async Task ApplyDeltasDecrements() + { + await ApplyDeltas(new Dictionary { ["1"] = 5 }); + await ApplyDeltas(new Dictionary { ["1"] = -2 }); + Assert.That(await _repo.GetCount(_projectId, "1"), Is.EqualTo(3)); + } + + [Test] + public async Task ApplyDeltasSkipsZeroDeltas() + { + await ApplyDeltas(new Dictionary { ["1"] = 0 }); + Assert.That(await _repo.GetAllCounts(_projectId), Is.Empty); + } + + [Test] + public async Task GetCountReturnsZeroWhenAbsent() + { + Assert.That(await _repo.GetCount(_projectId, "9.9"), Is.EqualTo(0)); + } + + [Test] + public async Task GetAllCountsReturnsProjectCounts() + { + await ApplyDeltas(new Dictionary { ["1"] = 1, ["2"] = 4 }); + var all = await _repo.GetAllCounts(_projectId); + Assert.That(all, Has.Count.EqualTo(2)); + Assert.That(all.Find(c => c.DomainId == "2")!.Count, Is.EqualTo(4)); + } + + [Test] + public async Task GetAllCountsIsolatesByProject() + { + await ApplyDeltas(new Dictionary { ["1"] = 1 }); + Assert.That(await _repo.GetAllCounts(Guid.NewGuid().ToString()), Is.Empty); + } + + [Test] + public async Task DeleteAllCountsRemovesProjectCounts() + { + await ApplyDeltas(new Dictionary { ["1"] = 1, ["2"] = 2 }); + await _dbContext.ExecuteInTransaction(async session => + { + await _repo.DeleteAllCounts(session, _projectId); + return true; + }); + Assert.That(await _repo.GetAllCounts(_projectId), Is.Empty); + } + + [Test] + public void UniqueIndexPreventsDuplicatePairs() + { + var collection = + _dbContext.Db.GetCollection("SemanticDomainCountCollection"); + collection.InsertOne(new ProjectSemanticDomainCount(_projectId, "1", 1)); + Assert.That( + async () => await collection.InsertOneAsync(new ProjectSemanticDomainCount(_projectId, "1", 1)), + Throws.InstanceOf()); + } + + [Test] + public void CountDomainsTalliesEachSenseOccurrence() + { + var word = new Word + { + Senses = + [ + new Sense { SemanticDomains = [new SemanticDomain { Id = "1" }, new SemanticDomain { Id = "2" }] }, + new Sense { SemanticDomains = [new SemanticDomain { Id = "1" }] }, + ], + }; + + var counts = SemanticDomainCountRepository.CountDomains(word); + Assert.That(counts["1"], Is.EqualTo(2)); + Assert.That(counts["2"], Is.EqualTo(1)); + } + } +} diff --git a/Backend.Tests/Repositories/WordRepositoryTests.cs b/Backend.Tests/Repositories/WordRepositoryTests.cs index e87e541360..383e255b12 100644 --- a/Backend.Tests/Repositories/WordRepositoryTests.cs +++ b/Backend.Tests/Repositories/WordRepositoryTests.cs @@ -17,33 +17,22 @@ namespace Backend.Tests.Repositories [Category("IntegrationTest")] public sealed class WordRepositoryTests { - private static MongoDbTestRunner _runner = null!; private WordRepository _repo = null!; + private SemanticDomainCountRepository _semDomCountRepo = null!; private string _projectId = null!; - [OneTimeSetUp] - public static void StartMongo() - { - _runner?.Dispose(); - _runner = MongoDbTestRunner.Start(); - } - - [OneTimeTearDown] - public static void StopMongo() - { - _runner?.Dispose(); - } - [SetUp] public void SetUp() { _projectId = Guid.NewGuid().ToString(); var options = Options.Create(new BackendFramework.Startup.Settings { - ConnectionString = _runner.ConnectionString, + ConnectionString = MongoDbSetUpFixture.Runner.ConnectionString, CombineDatabase = "WordRepositoryTests", }); - _repo = new WordRepository(new MongoDbContext(options)); + var dbContext = new MongoDbContext(options); + _semDomCountRepo = new SemanticDomainCountRepository(dbContext); + _repo = new WordRepository(dbContext, _semDomCountRepo); } private Task CreateWord(string? vernacular = null, string? domainId = null) @@ -704,23 +693,88 @@ public async Task TestRevertReplaceFrontierOverlappingIdsThrows() } [Test] - public async Task TestCountFrontierWordsWithDomainReturnsCorrectCount() + public async Task TestCreateIncrementsDomainCount() { const string domainId = "1.1"; await CreateWord(domainId: domainId); await CreateWord(domainId: domainId); - await CreateWord(); + Assert.That(await _semDomCountRepo.GetCount(_projectId, domainId), Is.EqualTo(2)); + } - var count = await _repo.CountFrontierWordsWithDomain(_projectId, domainId); - Assert.That(count, Is.EqualTo(2)); + [Test] + public async Task TestDeleteFrontierDecrementsDomainCount() + { + const string domainId = "1.1"; + var word = await CreateWord(domainId: domainId); + await _repo.DeleteFrontier(_projectId, word.Id, _ => { }); + Assert.That(await _semDomCountRepo.GetCount(_projectId, domainId), Is.Zero); } [Test] - public async Task TestCountFrontierWordsWithDomainNoneMatchReturnsZero() + public async Task TestUpdateFrontierAdjustsDomainCounts() { - await CreateWord(); - var count = await _repo.CountFrontierWordsWithDomain(_projectId, "99.99"); - Assert.That(count, Is.Zero); + const string oldDomain = "1.1"; + const string newDomain = "2.2"; + var word = await CreateWord(domainId: oldDomain); + + await _repo.UpdateFrontier(_projectId, word.Id, + w => w.Senses[0].SemanticDomains = [new SemanticDomain { Id = newDomain, Name = "Test" }]); + + Assert.That(await _semDomCountRepo.GetCount(_projectId, oldDomain), Is.Zero); + Assert.That(await _semDomCountRepo.GetCount(_projectId, newDomain), Is.EqualTo(1)); + } + + [Test] + public async Task TestReplaceFrontierAdjustsDomainCounts() + { + const string oldDomain = "1.1"; + const string newDomain = "2.2"; + var word = await CreateWord(domainId: oldDomain); + var updated = word.Clone(); + updated.Senses[0].SemanticDomains = [new SemanticDomain { Id = newDomain, Name = "Test" }]; + + await _repo.ReplaceFrontier(_projectId, [updated], [word.Id], (_, _) => { }, _ => { }); + + Assert.That(await _semDomCountRepo.GetCount(_projectId, oldDomain), Is.Zero); + Assert.That(await _semDomCountRepo.GetCount(_projectId, newDomain), Is.EqualTo(1)); + } + + [Test] + public async Task TestRestoreFrontierRestoresDomainCount() + { + const string domainId = "1.1"; + var word = await CreateWord(domainId: domainId); + await _repo.DeleteFrontier(_projectId, word.Id, _ => { }); + Assert.That(await _semDomCountRepo.GetCount(_projectId, domainId), Is.Zero); + + await _repo.RestoreFrontier(_projectId, word.Id); + Assert.That(await _semDomCountRepo.GetCount(_projectId, domainId), Is.EqualTo(1)); + } + + [Test] + public async Task TestDeleteAllFrontierWordsClearsDomainCounts() + { + await CreateWord(domainId: "1.1"); + await CreateWord(domainId: "2.2"); + + await _repo.DeleteAllFrontierWords(_projectId); + + Assert.That(await _semDomCountRepo.GetAllCounts(_projectId), Is.Empty); + } + + [Test] + public async Task TestDeleteFrontierModifyThrowsRollsBackDomainCount() + { + const string domainId = "1.1"; + var word = await CreateWord(domainId: domainId); + Assert.That(await _semDomCountRepo.GetCount(_projectId, domainId), Is.EqualTo(1)); + + // The count delta is applied inside the same transaction as the word write; when the modify + // action throws, the whole transaction aborts and the delta must roll back with it. + Assert.ThrowsAsync(() => + _repo.DeleteFrontier(_projectId, word.Id, _ => throw new InvalidOperationException())); + + Assert.That(await _semDomCountRepo.GetCount(_projectId, domainId), Is.EqualTo(1)); } } } diff --git a/Backend.Tests/Services/StatisticsServiceTests.cs b/Backend.Tests/Services/StatisticsServiceTests.cs index a49ee6df6e..147ae90104 100644 --- a/Backend.Tests/Services/StatisticsServiceTests.cs +++ b/Backend.Tests/Services/StatisticsServiceTests.cs @@ -12,6 +12,7 @@ namespace Backend.Tests.Services internal sealed class StatisticsServiceTests { private ISemanticDomainRepository _domainRepo = null!; + private SemanticDomainCountRepositoryMock _semDomCountRepo = null!; private IUserRepository _userRepo = null!; private WordRepositoryMock _wordRepo = null!; private IStatisticsService _statsService = null!; @@ -46,16 +47,17 @@ private static Word GetWordWithDomain(string semDomId = SemDomId) public void Setup() { _domainRepo = new SemanticDomainRepositoryMock(); + _semDomCountRepo = new SemanticDomainCountRepositoryMock(); _userRepo = new UserRepositoryMock(); _wordRepo = new WordRepositoryMock(); - _statsService = new StatisticsService(_wordRepo, _domainRepo, _userRepo); + _statsService = new StatisticsService(_wordRepo, _domainRepo, _semDomCountRepo, _userRepo); } [Test] public void GetSemanticDomainCountsTestNullDomainList() { - // Add a word to the database and leave the semantic domain list null - _wordRepo.AddFrontier(GetWordWithDomain()); + // Leave the semantic domain tree-node list null; the cached count is irrelevant without domains. + _semDomCountRepo.SetCount(ProjId, SemDomId, 1); var result = _statsService.GetSemanticDomainCounts(ProjId, "").Result; Assert.That(result, Is.Empty); @@ -64,42 +66,41 @@ public void GetSemanticDomainCountsTestNullDomainList() [Test] public void GetSemanticDomainCountsTestEmptyDomainList() { - // Add to the database a word and an empty list of semantic domains ((SemanticDomainRepositoryMock)_domainRepo).SetNextResponse(new List()); - _wordRepo.AddFrontier(GetWordWithDomain()); + _semDomCountRepo.SetCount(ProjId, SemDomId, 1); var result = _statsService.GetSemanticDomainCounts(ProjId, "").Result; Assert.That(result, Is.Empty); } [Test] - public void GetSemanticDomainCountsTestEmptyFrontier() + public void GetSemanticDomainCountsTestEmptyCounts() { - // Add to the database a semantic domain but no word + // With domains present but no cached counts, every node is reported with a count of 0. ((SemanticDomainRepositoryMock)_domainRepo).SetNextResponse(TreeNodes); var result = _statsService.GetSemanticDomainCounts(ProjId, "").Result; - Assert.That(result, Is.Empty); + Assert.That(result, Has.Count.EqualTo(1)); + Assert.That(result.First().Count, Is.Zero); } [Test] public void GetSemanticDomainCountsTestIdMismatch() { - // Add to the database a semantic domain and a word with a different semantic domain + // Cache a count for a different domain than the one in the tree; the node's count stays 0. ((SemanticDomainRepositoryMock)_domainRepo).SetNextResponse(TreeNodes); - _wordRepo.AddFrontier(GetWordWithDomain("different-id")); + _semDomCountRepo.SetCount(ProjId, "different-id", 1); var result = _statsService.GetSemanticDomainCounts(ProjId, "").Result; Assert.That(result, Has.Count.EqualTo(1)); - Assert.That(result.First(), Is.Empty); + Assert.That(result.First().Count, Is.Zero); } [Test] public void GetSemanticDomainCountsTestIdMatch() { - // Add to the database a semantic domain and a word with the same semantic domain ((SemanticDomainRepositoryMock)_domainRepo).SetNextResponse(TreeNodes); - _wordRepo.AddFrontier(GetWordWithDomain()); + _semDomCountRepo.SetCount(ProjId, SemDomId, 1); var result = _statsService.GetSemanticDomainCounts(ProjId, "").Result; Assert.That(result, Has.Count.EqualTo(1)); diff --git a/Backend/Controllers/WordController.cs b/Backend/Controllers/WordController.cs index 9623d69e62..04923d5975 100644 --- a/Backend/Controllers/WordController.cs +++ b/Backend/Controllers/WordController.cs @@ -13,10 +13,11 @@ namespace BackendFramework.Controllers [Authorize] [Produces("application/json")] [Route("v1/projects/{projectId}/words")] - public class WordController( - IWordRepository wordRepo, IWordService wordService, IPermissionService permissionService) : Controller + public class WordController(IWordRepository wordRepo, IWordService wordService, + ISemanticDomainCountRepository semDomCountRepo, IPermissionService permissionService) : Controller { private readonly IWordRepository _wordRepo = wordRepo; + private readonly ISemanticDomainCountRepository _semDomCountRepo = semDomCountRepo; private readonly IPermissionService _permissionService = permissionService; private readonly IWordService _wordService = wordService; @@ -299,7 +300,7 @@ public async Task RevertWords( return Ok(updates); } - /// Get the count of frontier words with senses in a specific semantic domain + /// Get the count of frontier word senses in a specific semantic domain /// An integer count [HttpGet("domainwordcount/{domainId}", Name = "GetDomainWordCount")] [ProducesResponseType(StatusCodes.Status200OK, Type = typeof(int))] @@ -313,7 +314,7 @@ public async Task GetDomainWordCount(string projectId, string dom return Forbid(); } - return Ok(await _wordRepo.CountFrontierWordsWithDomain(projectId, domainId)); + return Ok(await _semDomCountRepo.GetCount(projectId, domainId)); } } } diff --git a/Backend/Interfaces/ISemanticDomainCountRepository.cs b/Backend/Interfaces/ISemanticDomainCountRepository.cs new file mode 100644 index 0000000000..6dd979dfa6 --- /dev/null +++ b/Backend/Interfaces/ISemanticDomainCountRepository.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using BackendFramework.Models; +using MongoDB.Driver; + +namespace BackendFramework.Interfaces +{ + /// + /// Database functions for cached per-project semantic domain sense counts + /// (see ). + /// + /// + /// Writes take an so they can run inside the same transaction as the + /// word write that changed the Frontier, keeping the counts atomically in sync. Reads run outside any + /// transaction (used by the statistics and word-count endpoints). + /// + public interface ISemanticDomainCountRepository + { + /// Gets the cached count for a single semantic domain in a project (0 when absent). + Task GetCount(string projectId, string domainId); + + /// Gets all cached semantic domain counts for a project. + Task> GetAllCounts(string projectId); + + /// + /// Applies signed per-domain deltas to a project's cached counts within a transaction, upserting as needed. + /// + /// Mongo transaction session. + /// Id of the project whose counts are updated. + /// Map of semantic domain id to the signed amount to add (may be negative). + Task ApplyDeltas(IClientSessionHandle session, string projectId, IReadOnlyDictionary domainDeltas); + + /// Removes all cached counts for a project within a transaction. + /// Mongo transaction session. + /// Id of the project whose counts are removed. + Task DeleteAllCounts(IClientSessionHandle session, string projectId); + } +} diff --git a/Backend/Interfaces/IWordRepository.cs b/Backend/Interfaces/IWordRepository.cs index a38531dc08..27e45a093f 100644 --- a/Backend/Interfaces/IWordRepository.cs +++ b/Backend/Interfaces/IWordRepository.cs @@ -28,6 +28,5 @@ Task> ReplaceFrontier(string projectId, List newWords, List modifyUpdatedWord, Action modifyDeletedWord); Task RevertReplaceFrontier(string projectId, List idsToRestore, List idsToDelete, Action modifyDeletedWord); - Task CountFrontierWordsWithDomain(string projectId, string domainId); } } diff --git a/Backend/Models/ProjectSemanticDomainCount.cs b/Backend/Models/ProjectSemanticDomainCount.cs new file mode 100644 index 0000000000..3e21798a0a --- /dev/null +++ b/Backend/Models/ProjectSemanticDomainCount.cs @@ -0,0 +1,52 @@ +using System.ComponentModel.DataAnnotations; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace BackendFramework.Models +{ + /// + /// A cached tally of how many frontier sense-occurrences of a semantic domain exist within a project. + /// There is one document per (, ) pair. + /// + public class ProjectSemanticDomainCount + { + [Required] + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string Id { get; set; } + + [Required] + [BsonElement("projectId")] + public string ProjectId { get; set; } + + [Required] + [BsonElement("domainId")] + public string DomainId { get; set; } + + [Required] + [BsonElement("count")] + public int Count { get; set; } + + public ProjectSemanticDomainCount() + { + Id = ""; + ProjectId = ""; + DomainId = ""; + Count = 0; + } + + public ProjectSemanticDomainCount(string projectId, string domainId, int count = 0) : this() + { + ProjectId = projectId; + DomainId = domainId; + Count = count; + } + + /// Create a deep copy. + public ProjectSemanticDomainCount Clone() + { + // Shallow copy is sufficient. + return (ProjectSemanticDomainCount)MemberwiseClone(); + } + } +} diff --git a/Backend/Repositories/SemanticDomainCountRepository.cs b/Backend/Repositories/SemanticDomainCountRepository.cs new file mode 100644 index 0000000000..197441090d --- /dev/null +++ b/Backend/Repositories/SemanticDomainCountRepository.cs @@ -0,0 +1,124 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using BackendFramework.Interfaces; +using BackendFramework.Models; +using BackendFramework.Otel; +using MongoDB.Driver; + +namespace BackendFramework.Repositories +{ + /// Atomic database functions for cached s. + public class SemanticDomainCountRepository : ISemanticDomainCountRepository + { + private readonly IMongoCollection _counts; + + private const string otelTagName = "otel.SemanticDomainCountRepository"; + + public SemanticDomainCountRepository(IMongoDbContext dbContext) + { + _counts = dbContext.Db.GetCollection("SemanticDomainCountCollection"); + + // The unique compound index enforces one document per (project, domain). Creating it here also + // guarantees the collection exists before any transactional upsert touches it. + var keys = Builders.IndexKeys + .Ascending(c => c.ProjectId) + .Ascending(c => c.DomainId); + _counts.Indexes.CreateOne( + new CreateIndexModel(keys, new CreateIndexOptions { Unique = true })); + } + + /// Gets the cached count for a single semantic domain in a project (0 when absent). + public async Task GetCount(string projectId, string domainId) + { + using var activity = OtelService.StartActivityWithTag(otelTagName, "getting a semantic domain count"); + + var count = await _counts.Find(ProjectDomainFilter(projectId, domainId)).FirstOrDefaultAsync(); + return count?.Count ?? 0; + } + + /// Gets all cached semantic domain counts for a project. + public async Task> GetAllCounts(string projectId) + { + using var activity = OtelService.StartActivityWithTag(otelTagName, "getting all semantic domain counts"); + + return await _counts.Find(c => c.ProjectId == projectId).ToListAsync(); + } + + /// + /// Applies signed per-domain deltas to a project's cached counts within a transaction, upserting as needed. + /// + /// Mongo transaction session. + /// Id of the project whose counts are updated. + /// Map of semantic domain id to the signed amount to add (may be negative). + public async Task ApplyDeltas( + IClientSessionHandle session, string projectId, IReadOnlyDictionary domainDeltas) + { + using var activity = OtelService.StartActivityWithTag(otelTagName, "applying semantic domain count deltas"); + + var models = new List>(); + foreach (var (domainId, delta) in domainDeltas) + { + if (delta == 0) + { + continue; + } + + var update = Builders.Update.Inc(c => c.Count, delta); + models.Add(new UpdateOneModel(ProjectDomainFilter(projectId, domainId), + update) + { IsUpsert = true }); + } + + if (models.Count == 0) + { + return; + } + + await _counts.BulkWriteAsync(session, models); + } + + /// Removes all cached counts for a project within a transaction. + /// Mongo transaction session. + /// Id of the project whose counts are removed. + public async Task DeleteAllCounts(IClientSessionHandle session, string projectId) + { + using var activity = OtelService.StartActivityWithTag(otelTagName, "deleting all semantic domain counts"); + + await _counts.DeleteManyAsync(session, c => c.ProjectId == projectId); + } + + /// + /// Tallies, per semantic domain id, how many sense-occurrences of that domain appear across the given words. + /// A word with two senses in the same domain contributes 2, matching the historical per-sense statistics. + /// + public static Dictionary CountDomains(IEnumerable words) + { + var counts = new Dictionary(); + foreach (var word in words) + { + foreach (var sense in word.Senses) + { + foreach (var domain in sense.SemanticDomains) + { + counts[domain.Id] = counts.GetValueOrDefault(domain.Id) + 1; + } + } + } + + return counts; + } + + /// Tallies semantic domain sense-occurrences for a single word. + public static Dictionary CountDomains(Word word) + { + return CountDomains([word]); + } + + private static FilterDefinition ProjectDomainFilter( + string projectId, string domainId) + { + var filterDef = new FilterDefinitionBuilder(); + return filterDef.And(filterDef.Eq(c => c.ProjectId, projectId), filterDef.Eq(c => c.DomainId, domainId)); + } + } +} diff --git a/Backend/Repositories/WordRepository.cs b/Backend/Repositories/WordRepository.cs index 0365fdb851..74e35c1b75 100644 --- a/Backend/Repositories/WordRepository.cs +++ b/Backend/Repositories/WordRepository.cs @@ -10,9 +10,11 @@ namespace BackendFramework.Repositories { /// Atomic database functions for s. - public class WordRepository(IMongoDbContext dbContext) : IWordRepository + public class WordRepository(IMongoDbContext dbContext, ISemanticDomainCountRepository semDomCountRepo) + : IWordRepository { private readonly IMongoDbContext _dbContext = dbContext; + private readonly ISemanticDomainCountRepository _semDomCountRepo = semDomCountRepo; private readonly IMongoCollection _frontier = dbContext.Db.GetCollection("FrontierCollection"); private readonly IMongoCollection _words = dbContext.Db.GetCollection("WordsCollection"); @@ -135,11 +137,8 @@ public async Task DeleteAllFrontierWords(string projectId) { using var activity = OtelService.StartActivityWithTag(otelTagName, "deleting all words from Frontier"); - var filterDef = new FilterDefinitionBuilder(); - var filter = filterDef.Eq(x => x.ProjectId, projectId); - - var deleted = await _frontier.DeleteManyAsync(filter); - return deleted.DeletedCount != 0; + return await _dbContext.ExecuteInTransaction( + async s => await DeleteAllFrontierWordsWithSession(s, projectId)); } /// Checks if Words collection for specified has any words. @@ -365,24 +364,6 @@ public async Task RevertReplaceFrontier( s, projectId, idsToRestore, idsToDelete, modifyDeletedWord)) ?? false; } - /// - /// Counts the number of Frontier words that have the specified semantic domain. - /// - /// The project id - /// The semantic domain id - /// The count of words containing at least one sense with the specified domain. - public async Task CountFrontierWordsWithDomain(string projectId, string domainId) - { - using var activity = OtelService.StartActivityWithTag(otelTagName, "counting frontier words with domain"); - - var filterDef = new FilterDefinitionBuilder(); - var filter = filterDef.And( - filterDef.Eq(w => w.ProjectId, projectId), - filterDef.ElemMatch(w => w.Senses, s => s.SemanticDomains.Any(sd => sd.Id == domainId))); - - return (int)await _frontier.CountDocumentsAsync(filter); - } - #endregion #region Private with-session helper methods @@ -406,6 +387,7 @@ private async Task> CreateWithSession(IClientSessionHandle session, L // The first collection insert will generate the id, which should match in the second collection. await _words.InsertManyAsync(session, words); await _frontier.InsertManyAsync(session, words); + await ApplyDomainDeltas(session, words, 1); return words; } @@ -428,6 +410,8 @@ private async Task> CreateWithSession(IClientSessionHandle session, L return null; } + await ApplyDomainDeltas(session, [deletedWord], -1); + var modifiedWord = deletedWord.Clone(); modifyDeletedWord(modifiedWord); modifiedWord.Id = ""; @@ -462,6 +446,7 @@ private async Task RestoreFrontierWithSession(IClientSessionHandle session } await _frontier.InsertOneAsync(session, word); + await ApplyDomainDeltas(session, [word], 1); return true; } @@ -482,6 +467,8 @@ private async Task RestoreFrontierWithSession(IClientSessionHandle session return null; } + await ApplyDomainDeltas(session, [deletedWord], -1); + var word = deletedWord.Clone(); modifyUpdatedWord(word); await CreateWithSession(session, [word]); @@ -512,6 +499,11 @@ private async Task RestoreFrontierWithSession(IClientSessionHandle session return null; } + if (deletedWord is not null) + { + await ApplyDomainDeltas(session, [deletedWord], -1); + } + modifyUpdatedWord(word, deletedWord?.Clone()); await CreateWithSession(session, [word]); return word; @@ -599,6 +591,43 @@ private async Task> ReplaceFrontierWithSession(IClientSessionHandle s return true; } + /// + /// Deletes all Frontier words for a project and clears its cached semantic domain counts in one transaction. + /// + /// Mongo transaction session. + /// Id of the project whose Frontier words are removed. + /// True if at least one Frontier word was deleted; otherwise false. + private async Task DeleteAllFrontierWordsWithSession(IClientSessionHandle session, string projectId) + { + var filterDef = new FilterDefinitionBuilder(); + var filter = filterDef.Eq(x => x.ProjectId, projectId); + + var deleted = await _frontier.DeleteManyAsync(session, filter); + await _semDomCountRepo.DeleteAllCounts(session, projectId); + return deleted.DeletedCount != 0; + } + + /// + /// Applies semantic domain sense-count deltas for the given words within the transaction session, one + /// call per distinct project. is +1 when the words enter the Frontier and -1 + /// when they leave it, so the cached counts commit or roll back with the same transaction as the word write. + /// + /// Mongo transaction session. + /// Words whose semantic domain occurrences changed in the Frontier. + /// +1 to add the words' domain occurrences, -1 to remove them. + private async Task ApplyDomainDeltas(IClientSessionHandle session, IEnumerable words, int sign) + { + foreach (var wordsByProject in words.GroupBy(w => w.ProjectId)) + { + var deltas = SemanticDomainCountRepository.CountDomains(wordsByProject); + if (sign < 0) + { + deltas = deltas.ToDictionary(kv => kv.Key, kv => -kv.Value); + } + await _semDomCountRepo.ApplyDeltas(session, wordsByProject.Key, deltas); + } + } + #endregion } } diff --git a/Backend/Services/StatisticsService.cs b/Backend/Services/StatisticsService.cs index f51a897ecc..7671559fa7 100644 --- a/Backend/Services/StatisticsService.cs +++ b/Backend/Services/StatisticsService.cs @@ -13,15 +13,17 @@ public class StatisticsService : IStatisticsService { private readonly IWordRepository _wordRepo; private readonly ISemanticDomainRepository _domainRepo; + private readonly ISemanticDomainCountRepository _semDomCountRepo; private readonly IUserRepository _userRepo; private const string otelTagName = "otel.StatisticsService"; - public StatisticsService( - IWordRepository wordRepo, ISemanticDomainRepository domainRepo, IUserRepository userRepo) + public StatisticsService(IWordRepository wordRepo, ISemanticDomainRepository domainRepo, + ISemanticDomainCountRepository semDomCountRepo, IUserRepository userRepo) { _wordRepo = wordRepo; _domainRepo = domainRepo; + _semDomCountRepo = semDomCountRepo; _userRepo = userRepo; } @@ -39,32 +41,17 @@ public async Task> GetSemanticDomainCounts(string proj { using var activity = OtelService.StartActivityWithTag(otelTagName, "getting semantic domain counts"); - var hashMap = new Dictionary(); var domainTreeNodeList = await _domainRepo.GetAllSemanticDomainTreeNodes(lang); - var wordList = await _wordRepo.GetAllFrontier(projectId); - - if (domainTreeNodeList is null || domainTreeNodeList.Count == 0 || wordList.Count == 0) + if (domainTreeNodeList is null || domainTreeNodeList.Count == 0) { return []; } - foreach (var word in wordList) - { - foreach (var sense in word.Senses) - { - foreach (var sd in sense.SemanticDomains) - { - hashMap[sd.Id] = hashMap.GetValueOrDefault(sd.Id, 0) + 1; - } - } - } + var domainCounts = + (await _semDomCountRepo.GetAllCounts(projectId)).ToDictionary(dc => dc.DomainId, dc => dc.Count); - var resList = new List(); - foreach (var domainTreeNode in domainTreeNodeList) - { - resList.Add(new(domainTreeNode, hashMap.GetValueOrDefault(domainTreeNode.Id, 0))); - } - return resList; + return domainTreeNodeList + .Select(node => new SemanticDomainCount(node, domainCounts.GetValueOrDefault(node.Id, 0))).ToList(); } /// diff --git a/Backend/Startup.cs b/Backend/Startup.cs index 642e1fd5eb..23f9a977e0 100644 --- a/Backend/Startup.cs +++ b/Backend/Startup.cs @@ -272,6 +272,9 @@ public void ConfigureServices(IServiceCollection services) // Semantic Domain types services.AddSingleton(); + // Singleton so the repository's unique-index creation runs once per process rather than on + // every word edit (it is a dependency of the per-word-operation WordRepository). + services.AddSingleton(); // Speaker types services.AddTransient(); diff --git a/database/backfill-semantic-domain-counts.js b/database/backfill-semantic-domain-counts.js new file mode 100644 index 0000000000..2b6df84f89 --- /dev/null +++ b/database/backfill-semantic-domain-counts.js @@ -0,0 +1,97 @@ +// Backfill script: populate SemanticDomainCountCollection from the current Frontier. +// +// The backend keeps a cached count, per (project, semantic domain), of how many Frontier +// sense-occurrences reference that domain. The count is maintained transactionally as words are +// created/updated/deleted, but existing projects need their counts computed once from the current +// Frontier. This script does that backfill. +// +// Usage (local): +// mongosh CombineDatabase database/backfill-semantic-domain-counts.js +// +// Usage (Kubernetes, e.g. production): +// kubectl -n thecombine cp database/backfill-semantic-domain-counts.js \ +// :/tmp/backfill-semantic-domain-counts.js +// kubectl -n thecombine exec -- \ +// mongosh CombineDatabase /tmp/backfill-semantic-domain-counts.js +// +// IMPORTANT: +// - This is NOT a breaking schema change: an old backend simply ignores the new collection, and the +// new backend maintains it going forward. It is safe to deploy the count-maintaining backend first. +// - Run the backfill with word editing paused (backend scaled down, or no active users), because a +// Frontier write that lands between the aggregation and this run could be double-counted (the new +// backend already incremented it) or a delete missed. Editing while stopped avoids drift. +// - The script is idempotent: it fully rebuilds the collection, which is a pure cache derived from +// the Frontier. Re-run it any time the counts are suspected to have drifted, or after restoring a +// backup taken before the count collection existed. +// +// Count document (must match ProjectSemanticDomainCount in Backend/Models/ProjectSemanticDomainCount.cs): +// { _id: ObjectId, projectId: string, domainId: string, count: int } + +var frontier = db.getCollection("FrontierCollection"); +var counts = db.getCollection("SemanticDomainCountCollection"); + +// Rebuild from scratch: the collection is fully derived from the Frontier. +var removed = counts.deleteMany({}).deletedCount; +print("Cleared " + removed + " existing count document(s)."); + +// Tally every (sense, semantic domain) occurrence per project. A word with two senses in the same +// domain contributes 2, matching how the backend maintains the counts. +var aggregated = frontier + .aggregate( + [ + { $unwind: "$senses" }, + { $unwind: "$senses.SemanticDomains" }, + { + $group: { + _id: { projectId: "$projectId", domainId: "$senses.SemanticDomains.id" }, + count: { $sum: 1 }, + }, + }, + ], + { allowDiskUse: true } + ) + .toArray(); + +var docs = aggregated.map(function (g) { + return { + _id: new ObjectId(), + projectId: g._id.projectId, + domainId: g._id.domainId, + count: g.count, + }; +}); + +if (docs.length > 0) { + counts.insertMany(docs, { ordered: false }); +} +print("Inserted " + docs.length + " count document(s)."); + +// Unique compound index matches the one the backend creates on start-up. +counts.createIndex({ projectId: 1, domainId: 1 }, { unique: true }); +print("Ensured unique index { projectId: 1, domainId: 1 } on SemanticDomainCountCollection."); + +// Verify: total counted occurrences equal the number of (sense, domain) pairs in the Frontier. +var expected = frontier + .aggregate( + [ + { $unwind: "$senses" }, + { $unwind: "$senses.SemanticDomains" }, + { $count: "n" }, + ], + { allowDiskUse: true } + ) + .toArray(); +var expectedTotal = expected.length > 0 ? expected[0].n : 0; + +var actualTotal = 0; +counts.find({}, { count: 1 }).forEach(function (d) { + actualTotal += d.count; +}); + +if (actualTotal === expectedTotal) { + print( + "Verification passed: " + actualTotal + " occurrence(s) across " + docs.length + " (project, domain) pair(s)." + ); +} else { + print("WARNING: counted " + actualTotal + " occurrence(s) but Frontier has " + expectedTotal + "."); +}