Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 68 additions & 2 deletions grit-lib-server/src/cached.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ use crate::cache::{Cache, CacheKey, CacheValue};
use crate::error::{Error, Result};
use crate::ids::{RepositoryId, TenantId};
use crate::storage::{
BrowseIndex, ConfigStore, IndexedTreeEntry, ObjectStore, RefStore, ReflogEntry, ReflogStore,
StoredObject, StoredRef,
BrowseIndex, CommitGraphStore, ConfigStore, IndexedCommit, IndexedTreeEntry, ObjectStore,
RefStore, ReflogEntry, ReflogStore, StoredObject, StoredRef,
};

/// Storage wrapper that caches hot object and ref reads.
Expand Down Expand Up @@ -309,6 +309,72 @@ where
}
}

#[async_trait]
impl<S, C> CommitGraphStore for CachedStorage<S, C>
where
S: CommitGraphStore,
C: Cache,
{
async fn upsert_commits(
&self,
tenant: &TenantId,
repository: &RepositoryId,
commits: &[IndexedCommit],
) -> Result<()> {
self.storage
.upsert_commits(tenant, repository, commits)
.await
}

async fn replace_commit_graph(
&self,
tenant: &TenantId,
repository: &RepositoryId,
commits: &[IndexedCommit],
) -> Result<()> {
self.storage
.replace_commit_graph(tenant, repository, commits)
.await
}

async fn read_indexed_commit(
&self,
tenant: &TenantId,
repository: &RepositoryId,
oid: &ObjectId,
) -> Result<Option<IndexedCommit>> {
self.storage
.read_indexed_commit(tenant, repository, oid)
.await
}

async fn commit_parents(
&self,
tenant: &TenantId,
repository: &RepositoryId,
oid: &ObjectId,
) -> Result<Vec<ObjectId>> {
self.storage.commit_parents(tenant, repository, oid).await
}

async fn commit_children(
&self,
tenant: &TenantId,
repository: &RepositoryId,
oid: &ObjectId,
) -> Result<Vec<ObjectId>> {
self.storage.commit_children(tenant, repository, oid).await
}

async fn list_indexed_commits(
&self,
tenant: &TenantId,
repository: &RepositoryId,
) -> Result<Vec<IndexedCommit>> {
self.storage.list_indexed_commits(tenant, repository).await
}
}

fn encode_object(object: &StoredObject) -> Vec<u8> {
let mut out = Vec::with_capacity(object.data.len() + 1);
out.push(kind_code(object.kind));
Expand Down
3 changes: 3 additions & 0 deletions grit-lib-server/src/import.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ pub struct ImportReport {
pub objects: usize,
/// Number of tree entries indexed.
pub tree_entries: usize,
/// Number of commit graph rows rebuilt from imported commits.
pub commit_graph_entries: usize,
}

/// Import reachable repository data from a filesystem-backed [`Repository`].
Expand Down Expand Up @@ -143,6 +145,7 @@ where
}
}

report.commit_graph_entries = destination.repair_commit_graph().await?;
Ok(report)
}

Expand Down
9 changes: 5 additions & 4 deletions grit-lib-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,12 @@ pub mod prelude {
pub use crate::import::{import_repository, ImportReport};
pub use crate::repository::ServerRepository;
pub use crate::storage::{
BrowseIndex, ConfigStore, ObjectStore, RefStore, ReflogEntry, ReflogStore, StoredObject,
StoredRef,
BrowseIndex, CommitGraphStore, ConfigStore, IndexedCommit, ObjectStore, RefStore,
ReflogEntry, ReflogStore, StoredObject, StoredRef,
};
pub use crate::views::{
BlobView, BranchView, CommitSummary, CompareInputs, DiscoveredFile, RepositorySummary,
TagView, TreeEntryView, TreeView,
BlobView, BranchView, CommitComparison, CommitHistoryOptions, CommitHistoryPage,
CommitSummary, CompareInputs, DiscoveredFile, RepositorySummary, TagView, TreeEntryView,
TreeView,
};
}
164 changes: 159 additions & 5 deletions grit-lib-server/src/memory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,22 @@ use std::collections::{BTreeMap, HashMap};
use std::sync::RwLock;

use async_trait::async_trait;
use grit_lib::objects::{ObjectId, ObjectKind};
use grit_lib::objects::{parse_commit, ObjectId, ObjectKind};

use crate::cache::{Cache, CacheKey, CacheValue, EventPublisher, InvalidationEvent};
use crate::error::{Error, Result};
use crate::ids::{RepositoryId, TenantId};
use crate::storage::{
BrowseIndex, ConfigStore, IndexedTreeEntry, ObjectStore, RefStore, ReflogEntry, ReflogStore,
StoredObject, StoredRef,
commit_time_from_identity, BrowseIndex, CommitGraphStore, ConfigStore, IndexedCommit,
IndexedTreeEntry, ObjectStore, RefStore, ReflogEntry, ReflogStore, StoredObject, StoredRef,
};

type RepoKey = (TenantId, RepositoryId);
type ObjectKey = (RepoKey, ObjectId);
type RefKey = (RepoKey, String);
type ConfigKey = (RepoKey, String);
type TreeKey = (RepoKey, ObjectId, String);
type CommitKey = (RepoKey, ObjectId);
type CacheEntryKey = (RepoKey, CacheKey);

/// In-memory repository backend.
Expand All @@ -29,6 +30,7 @@ pub struct MemoryBackend {
reflogs: RwLock<BTreeMap<RefKey, Vec<ReflogEntry>>>,
config: RwLock<BTreeMap<ConfigKey, String>>,
trees: RwLock<BTreeMap<TreeKey, IndexedTreeEntry>>,
commits: RwLock<BTreeMap<CommitKey, IndexedCommit>>,
cache: RwLock<HashMap<CacheEntryKey, CacheValue>>,
events: RwLock<Vec<InvalidationEvent>>,
}
Expand All @@ -55,6 +57,36 @@ impl MemoryBackend {
.map(|guard| guard.clone())
.map_err(|_| Error::Backend("memory event log lock poisoned".to_owned()))
}

fn indexed_commit(
&self,
repo: RepoKey,
oid: ObjectId,
object: &StoredObject,
) -> Result<Option<IndexedCommit>> {
if object.kind != ObjectKind::Commit {
return Ok(None);
}
let commit = parse_commit(&object.data)?;
let generation = self
.commits
.read()
.map_err(|_| Error::Backend("memory commit graph lock poisoned".to_owned()))?
.iter()
.filter(|((candidate_repo, candidate_oid), _)| {
candidate_repo == &repo && commit.parents.contains(candidate_oid)
})
.map(|(_, parent)| parent.generation.saturating_add(1))
.max()
.unwrap_or(1);
Ok(Some(IndexedCommit {
oid,
tree: commit.tree,
parents: commit.parents,
commit_time: commit_time_from_identity(&commit.committer),
generation,
}))
}
}

#[async_trait]
Expand All @@ -78,14 +110,25 @@ impl ObjectStore for MemoryBackend {
oid: &ObjectId,
object: &StoredObject,
) -> Result<()> {
let repo = repo_key(tenant, repository);
self.objects
.write()
.map(|mut objects| {
objects
.entry((repo_key(tenant, repository), *oid))
.entry((repo.clone(), *oid))
.or_insert_with(|| object.clone());
})
.map_err(|_| Error::Backend("memory object lock poisoned".to_owned()))
.map_err(|_| Error::Backend("memory object lock poisoned".to_owned()))?;

if let Some(commit) = self.indexed_commit(repo.clone(), *oid, object)? {
self.commits
.write()
.map(|mut commits| {
commits.insert((repo, *oid), commit);
})
.map_err(|_| Error::Backend("memory commit graph lock poisoned".to_owned()))?;
}
Ok(())
}

async fn object_exists(
Expand Down Expand Up @@ -382,6 +425,117 @@ impl BrowseIndex for MemoryBackend {
}
}

#[async_trait]
impl CommitGraphStore for MemoryBackend {
async fn upsert_commits(
&self,
tenant: &TenantId,
repository: &RepositoryId,
commits: &[IndexedCommit],
) -> Result<()> {
let repo = repo_key(tenant, repository);
self.commits
.write()
.map(|mut stored| {
for commit in commits {
stored.insert((repo.clone(), commit.oid), commit.clone());
}
})
.map_err(|_| Error::Backend("memory commit graph lock poisoned".to_owned()))
}

async fn replace_commit_graph(
&self,
tenant: &TenantId,
repository: &RepositoryId,
commits: &[IndexedCommit],
) -> Result<()> {
let repo = repo_key(tenant, repository);
self.commits
.write()
.map(|mut stored| {
stored.retain(|(candidate_repo, _), _| candidate_repo != &repo);
for commit in commits {
stored.insert((repo.clone(), commit.oid), commit.clone());
}
})
.map_err(|_| Error::Backend("memory commit graph lock poisoned".to_owned()))
}

async fn read_indexed_commit(
&self,
tenant: &TenantId,
repository: &RepositoryId,
oid: &ObjectId,
) -> Result<Option<IndexedCommit>> {
self.commits
.read()
.map(|commits| commits.get(&(repo_key(tenant, repository), *oid)).cloned())
.map_err(|_| Error::Backend("memory commit graph lock poisoned".to_owned()))
}

async fn commit_parents(
&self,
tenant: &TenantId,
repository: &RepositoryId,
oid: &ObjectId,
) -> Result<Vec<ObjectId>> {
Ok(self
.read_indexed_commit(tenant, repository, oid)
.await?
.map(|commit| commit.parents)
.unwrap_or_default())
}

async fn commit_children(
&self,
tenant: &TenantId,
repository: &RepositoryId,
oid: &ObjectId,
) -> Result<Vec<ObjectId>> {
let repo = repo_key(tenant, repository);
self.commits
.read()
.map(|commits| {
let mut children = commits
.iter()
.filter(|((candidate_repo, _), commit)| {
candidate_repo == &repo && commit.parents.contains(oid)
})
.map(|((_, child), _)| *child)
.collect::<Vec<_>>();
children.sort();
children
})
.map_err(|_| Error::Backend("memory commit graph lock poisoned".to_owned()))
}

async fn list_indexed_commits(
&self,
tenant: &TenantId,
repository: &RepositoryId,
) -> Result<Vec<IndexedCommit>> {
let repo = repo_key(tenant, repository);
self.commits
.read()
.map(|commits| {
let mut commits = commits
.iter()
.filter(|((candidate_repo, _), _)| candidate_repo == &repo)
.map(|(_, commit)| commit.clone())
.collect::<Vec<_>>();
commits.sort_by(|left, right| {
right
.commit_time
.cmp(&left.commit_time)
.then_with(|| left.oid.cmp(&right.oid))
});
commits
})
.map_err(|_| Error::Backend("memory commit graph lock poisoned".to_owned()))
}
}

#[async_trait]
impl Cache for MemoryBackend {
async fn get(
Expand Down
Loading