diff --git a/src/database/indexer.rs b/src/database/indexer.rs
@@ -1,4 +1,5 @@
use git2::Sort;
+use std::collections::HashSet;
use std::path::{Path, PathBuf};
use time::OffsetDateTime;
use tracing::{info, info_span};
@@ -6,12 +7,25 @@ use tracing::{info, info_span};
use crate::database::schema::{
commit::Commit,
repository::{Repository, RepositoryId},
+ tag::Tag,
};
pub fn run(db: &sled::Db) {
+ let span = info_span!("index_update");
+ let _entered = span.enter();
+
+ info!("Starting index update");
+
let scan_path = Path::new("/Users/jordan/Code/test-git");
update_repository_metadata(scan_path, db);
update_repository_reflog(scan_path, db);
+ update_repository_tags(scan_path, db);
+
+ info!("Flushing to disk");
+
+ db.flush().unwrap();
+
+ info!("Finished index update");
}
fn update_repository_metadata(scan_path: &Path, db: &sled::Db) {
@@ -52,7 +66,7 @@ fn update_repository_reflog(scan_path: &Path, db: &sled::Db) {
}
let span = info_span!(
- "index_update",
+ "branch_index_update",
reference = reference_name.as_ref(),
repository = relative_path
);
@@ -98,6 +112,56 @@ fn update_repository_reflog(scan_path: &Path, db: &sled::Db) {
}
}
+fn update_repository_tags(scan_path: &Path, db: &sled::Db) {
+ for (relative_path, db_repository) in Repository::fetch_all(db).unwrap() {
+ let git_repository = git2::Repository::open(scan_path.join(&relative_path)).unwrap();
+
+ let tag_tree = db_repository.get().tag_tree(db).unwrap();
+
+ let git_tags: HashSet<_> = git_repository
+ .references()
+ .unwrap()
+ .filter_map(Result::ok)
+ .filter(|v| v.name_bytes().starts_with(b"refs/tags/"))
+ .map(|v| String::from_utf8_lossy(v.name_bytes()).into_owned())
+ .collect();
+ let indexed_tags: HashSet<String> = tag_tree.list().into_iter().collect();
+
+ // insert any git tags that are missing from the index
+ for tag_name in git_tags.difference(&indexed_tags) {
+ let span = info_span!(
+ "tag_index_update",
+ reference = tag_name,
+ repository = relative_path
+ );
+ let _entered = span.enter();
+
+ let reference = git_repository.find_reference(tag_name).unwrap();
+
+ if let Ok(tag) = reference.peel_to_tag() {
+ info!("Inserting newly discovered tag to index");
+
+ Tag::new(tag.tagger().as_ref()).insert(&tag_tree, tag_name);
+ }
+ }
+
+ // remove any extra tags that the index has
+ // TODO: this also needs to check peel_to_tag
+ for tag_name in indexed_tags.difference(&git_tags) {
+ let span = info_span!(
+ "tag_index_update",
+ reference = tag_name,
+ repository = relative_path
+ );
+ let _entered = span.enter();
+
+ info!("Removing stale tag from index");
+
+ tag_tree.remove(tag_name);
+ }
+ }
+}
+
fn get_relative_path<'a>(relative_to: &Path, full_path: &'a Path) -> &'a Path {
full_path.strip_prefix(relative_to).unwrap()
}
diff --git a/src/database/schema/commit.rs b/src/database/schema/commit.rs
@@ -85,7 +85,9 @@ impl<'a, 'de: 'a> Deserialize<'de> for CommitHash<'a> {
#[derive(Serialize, Deserialize, Debug)]
pub struct Author<'a> {
+ #[serde(borrow)]
pub name: Cow<'a, str>,
+ #[serde(borrow)]
pub email: Cow<'a, str>,
pub time: OffsetDateTime,
}
diff --git a/src/database/schema/mod.rs b/src/database/schema/mod.rs
@@ -6,5 +6,6 @@ use yoke::Yoke;
pub mod commit;
pub mod prefixes;
pub mod repository;
+pub mod tag;
pub type Yoked<T> = Yoke<T, Box<IVec>>;
diff --git a/src/database/schema/prefixes.rs b/src/database/schema/prefixes.rs
@@ -5,7 +5,7 @@ use std::path::Path;
pub enum TreePrefix {
Repository = 0,
Commit = 100,
- _Tag = 101,
+ Tag = 101,
}
impl TreePrefix {
@@ -34,4 +34,14 @@ impl TreePrefix {
prefixed
}
+
+ pub fn tag_id(repository: RepositoryId) -> Vec<u8> {
+ let mut prefixed = Vec::with_capacity(
+ std::mem::size_of::<TreePrefix>() + std::mem::size_of::<RepositoryId>(),
+ );
+ prefixed.push(TreePrefix::Tag as u8);
+ prefixed.extend_from_slice(&repository.to_ne_bytes());
+
+ prefixed
+ }
}
diff --git a/src/database/schema/repository.rs b/src/database/schema/repository.rs
@@ -1,7 +1,9 @@
use crate::database::schema::commit::CommitTree;
use crate::database::schema::prefixes::TreePrefix;
+use crate::database::schema::tag::TagTree;
use crate::database::schema::Yoked;
use anyhow::{Context, Result};
+use nom::AsBytes;
use serde::{Deserialize, Serialize};
use sled::IVec;
use std::borrow::Cow;
@@ -85,6 +87,27 @@ impl Repository<'_> {
Ok(CommitTree::new(tree))
}
+
+ pub fn tag_tree(&self, database: &sled::Db) -> Result<TagTree> {
+ let tree = database
+ .open_tree(TreePrefix::tag_id(self.id))
+ .context("Failed to open tag tree")?;
+
+ Ok(TagTree::new(tree))
+ }
+
+ pub fn heads(&self, database: &sled::Db) -> Vec<String> {
+ let prefix = TreePrefix::commit_id(self.id, "");
+
+ database
+ .tree_names()
+ .into_iter()
+ .filter_map(|v| {
+ v.strip_prefix(prefix.as_bytes())
+ .map(|v| String::from_utf8_lossy(v).into_owned())
+ })
+ .collect()
+ }
}
#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash)]
diff --git a/src/database/schema/tag.rs b/src/database/schema/tag.rs
@@ -0,0 +1,92 @@
+use crate::database::schema::commit::Author;
+use crate::database::schema::Yoked;
+use git2::Signature;
+use serde::{Deserialize, Serialize};
+use sled::IVec;
+use std::collections::HashSet;
+use std::ops::Deref;
+use yoke::{Yoke, Yokeable};
+
+#[derive(Serialize, Deserialize, Debug, Yokeable)]
+pub struct Tag<'a> {
+ #[serde(borrow)]
+ pub tagger: Option<Author<'a>>,
+}
+
+impl<'a> Tag<'a> {
+ pub fn new(tagger: Option<&'a Signature<'_>>) -> Self {
+ Self {
+ tagger: tagger.map(Into::into),
+ }
+ }
+
+ pub fn insert(&self, batch: &TagTree, name: &str) {
+ batch
+ .insert(&name.as_bytes(), bincode::serialize(self).unwrap())
+ .unwrap();
+ }
+}
+
+pub struct TagTree(sled::Tree);
+
+impl Deref for TagTree {
+ type Target = sled::Tree;
+
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
+}
+
+pub type YokedTag = Yoked<Tag<'static>>;
+
+impl TagTree {
+ pub(super) fn new(tree: sled::Tree) -> Self {
+ Self(tree)
+ }
+
+ pub fn remove(&self, name: &str) -> bool {
+ self.0.remove(name).unwrap().is_some()
+ }
+
+ pub fn list(&self) -> HashSet<String> {
+ self.iter()
+ .keys()
+ .filter_map(Result::ok)
+ .map(|v| String::from_utf8_lossy(&v).into_owned())
+ .collect()
+ }
+
+ pub fn fetch_all(&self) -> Vec<(String, YokedTag)> {
+ let mut res = self
+ .iter()
+ .map(|res| {
+ let (name, value) = res?;
+
+ let name = String::from_utf8_lossy(&name)
+ .strip_prefix("refs/tags/")
+ .unwrap()
+ .to_string();
+
+ // internally value is an Arc so it should already be stablederef but because
+ // of reasons unbeknownst to me, sled has its own Arc implementation so we need
+ // to box the value as well to get a stablederef...
+ let value = Box::new(value);
+
+ Ok((
+ name,
+ Yoke::try_attach_to_cart(value, |data: &IVec| bincode::deserialize(data))
+ .unwrap(),
+ ))
+ })
+ .collect::<Result<Vec<(String, YokedTag)>, sled::Error>>()
+ .unwrap();
+
+ res.sort_unstable_by(|a, b| {
+ let a_tagger = a.1.get().tagger.as_ref().map(|v| v.time);
+ let b_tagger = b.1.get().tagger.as_ref().map(|v| v.time);
+ b_tagger.cmp(&a_tagger)
+ });
+
+ res
+ }
+}
diff --git a/src/git.rs b/src/git.rs
@@ -20,7 +20,6 @@ use tracing::instrument;
pub struct Git {
commits: Cache<Oid, Arc<Commit>>,
readme_cache: Cache<PathBuf, Option<(ReadmeFormat, Arc<str>)>>,
- refs: Cache<PathBuf, Arc<Refs>>,
syntax_set: SyntaxSet,
}
@@ -36,10 +35,6 @@ impl Git {
.time_to_live(Duration::from_secs(10))
.max_capacity(100)
.build(),
- refs: Cache::builder()
- .time_to_live(Duration::from_secs(10))
- .max_capacity(100)
- .build(),
syntax_set,
}
}
@@ -202,56 +197,6 @@ impl OpenRepository {
.context("Failed to join Tokio task")?
}
- #[instrument(skip(self))]
- pub async fn refs(self: Arc<Self>) -> Result<Arc<Refs>, Arc<anyhow::Error>> {
- let git = self.git.clone();
-
- git.refs
- .try_get_with(self.cache_key.clone(), async move {
- tokio::task::spawn_blocking(move || {
- let repo = self.repo.lock();
-
- let ref_iter = repo.references().context("Couldn't get list of references for repository")?;
-
- let mut built_refs = Refs::default();
-
- for ref_ in ref_iter {
- let ref_ = ref_?;
-
- if ref_.is_branch() {
- let commit = ref_.peel_to_commit().context("Reference is apparently a branch but I couldn't get to the HEAD of it")?;
-
- built_refs.branch.push(Branch {
- name: String::from_utf8_lossy(ref_.shorthand_bytes()).into_owned(),
- commit: commit.try_into()?,
- });
- } else if ref_.is_tag() {
- if let Ok(tag) = ref_.peel_to_tag() {
- built_refs.tag.push(Tag {
- name: String::from_utf8_lossy(ref_.shorthand_bytes()).into_owned(),
- tagger: tag.tagger().map(TryInto::try_into).transpose()?,
- });
- }
- }
- }
-
- built_refs.branch.sort_unstable_by(|one, two| {
- two.commit.committer.time.cmp(&one.commit.committer.time)
- });
- built_refs.tag.sort_unstable_by(|one, two| {
- let one_tagger = one.tagger.as_ref().map(|v| v.time);
- let two_tagger = two.tagger.as_ref().map(|v| v.time);
- two_tagger.cmp(&one_tagger)
- });
-
- Ok(Arc::new(built_refs))
- })
- .await
- .context("Failed to join Tokio task")?
- })
- .await
- }
-
#[instrument(skip(self))]
pub async fn readme(
self: Arc<Self>,
@@ -411,12 +356,6 @@ pub struct FileWithContent {
pub content: String,
}
-#[derive(Debug, Default)]
-pub struct Refs {
- pub branch: Vec<Branch>,
- pub tag: Vec<Tag>,
-}
-
#[derive(Debug)]
pub struct Branch {
pub name: String,
@@ -442,17 +381,10 @@ pub struct DetailedTag {
pub tagged_object: Option<TaggedObject>,
}
-#[derive(Debug)]
-pub struct Tag {
- pub name: String,
- pub tagger: Option<CommitUser>,
-}
-
#[derive(Debug)]
pub struct CommitUser {
name: String,
email: String,
- email_md5: String,
time: OffsetDateTime,
}
@@ -463,7 +395,6 @@ impl TryFrom<Signature<'_>> for CommitUser {
Ok(CommitUser {
name: String::from_utf8_lossy(v.name_bytes()).into_owned(),
email: String::from_utf8_lossy(v.email_bytes()).into_owned(),
- email_md5: format!("{:x}", md5::compute(v.email_bytes())),
time: OffsetDateTime::from_unix_timestamp(v.when().seconds())?,
})
}
@@ -478,10 +409,6 @@ impl CommitUser {
&self.email
}
- pub fn email_md5(&self) -> &str {
- &self.email_md5
- }
-
pub fn time(&self) -> OffsetDateTime {
self.time
}
diff --git a/src/methods/filters.rs b/src/methods/filters.rs
@@ -1,8 +1,11 @@
// sorry clippy, we don't have a choice. askama forces this on us
#![allow(clippy::unnecessary_wraps, clippy::trivially_copy_pass_by_ref)]
-pub fn timeago(s: time::OffsetDateTime) -> Result<String, askama::Error> {
- Ok(timeago::Formatter::new().convert((time::OffsetDateTime::now_utc() - s).unsigned_abs()))
+use std::borrow::Borrow;
+
+pub fn timeago(s: impl Borrow<time::OffsetDateTime>) -> Result<String, askama::Error> {
+ Ok(timeago::Formatter::new()
+ .convert((time::OffsetDateTime::now_utc() - *s.borrow()).unsigned_abs()))
}
pub fn file_perms(s: &i32) -> Result<String, askama::Error> {
diff --git a/src/methods/repo.rs b/src/methods/repo.rs
@@ -1,4 +1,5 @@
use anyhow::Context;
+use std::collections::BTreeMap;
use std::{
fmt::{Debug, Display, Formatter},
io::Write,
@@ -27,7 +28,9 @@ use tower::{util::BoxCloneService, Service};
use yoke::Yoke;
use super::filters;
-use crate::git::{DetailedTag, FileWithContent, PathDestination, ReadmeFormat, Refs, TreeItem};
+use crate::database::schema::commit::YokedCommit;
+use crate::database::schema::tag::YokedTag;
+use crate::git::{DetailedTag, FileWithContent, PathDestination, ReadmeFormat, TreeItem};
use crate::{git::Commit, into_response, layers::UnwrapInfallible, Git};
#[derive(Clone)]
@@ -159,32 +162,48 @@ where
.into_response()
}
+pub struct Refs {
+ heads: BTreeMap<String, YokedCommit>,
+ tags: Vec<(String, YokedTag)>,
+}
+
#[derive(Template)]
#[template(path = "repo/summary.html")]
pub struct SummaryView<'a> {
repo: Repository,
- refs: Arc<Refs>,
+ refs: Refs,
commit_list: Vec<&'a crate::database::schema::commit::Commit<'a>>,
}
pub async fn handle_summary(
Extension(repo): Extension<Repository>,
- Extension(RepositoryPath(repository_path)): Extension<RepositoryPath>,
- Extension(git): Extension<Arc<Git>>,
Extension(db): Extension<sled::Db>,
) -> Result<Response> {
- let open_repo = git.repo(repository_path).await?;
- let refs = open_repo.refs().await?;
-
let repository = crate::database::schema::repository::Repository::open(&db, &*repo)?
.context("Repository does not exist")?;
let commit_tree = repository.get().commit_tree(&db, "refs/heads/master")?;
let commits = commit_tree.fetch_latest(11, 0).await;
let commit_list = commits.iter().map(Yoke::get).collect();
+ let mut heads = BTreeMap::new();
+ for head in repository.get().heads(&db) {
+ let commit_tree = repository.get().commit_tree(&db, &head)?;
+ let name = head.strip_prefix("refs/heads/");
+
+ if let (Some(name), Some(commit)) = (name, commit_tree.fetch_latest_one()) {
+ heads.insert(name.to_string(), commit);
+ }
+ }
+
+ let tags = repository
+ .get()
+ .tag_tree(&db)
+ .context("Failed to fetch indexed tags")?
+ .fetch_all();
+
Ok(into_response(&SummaryView {
repo,
- refs,
+ refs: Refs { heads, tags },
commit_list,
}))
}
@@ -309,18 +328,36 @@ pub async fn handle_git_upload_pack(
#[template(path = "repo/refs.html")]
pub struct RefsView {
repo: Repository,
- refs: Arc<Refs>,
+ refs: Refs,
}
pub async fn handle_refs(
Extension(repo): Extension<Repository>,
- Extension(RepositoryPath(repository_path)): Extension<RepositoryPath>,
- Extension(git): Extension<Arc<Git>>,
+ Extension(db): Extension<sled::Db>,
) -> Result<Response> {
- let open_repo = git.repo(repository_path).await?;
- let refs = open_repo.refs().await?;
+ let repository = crate::database::schema::repository::Repository::open(&db, &*repo)?
+ .context("Repository does not exist")?;
+
+ let mut heads = BTreeMap::new();
+ for head in repository.get().heads(&db) {
+ let commit_tree = repository.get().commit_tree(&db, &head)?;
+ let name = head.strip_prefix("refs/heads/");
- Ok(into_response(&RefsView { repo, refs }))
+ if let (Some(name), Some(commit)) = (name, commit_tree.fetch_latest_one()) {
+ heads.insert(name.to_string(), commit);
+ }
+ }
+
+ let tags = repository
+ .get()
+ .tag_tree(&db)
+ .context("Failed to fetch indexed tags")?
+ .fetch_all();
+
+ Ok(into_response(&RefsView {
+ repo,
+ refs: Refs { heads, tags },
+ }))
}
#[derive(Template)]
diff --git a/templates/repo/macros/refs.html b/templates/repo/macros/refs.html
@@ -9,17 +9,17 @@
</thead>
<tbody>
-{% for branch in branches %}
+{% for (name, commit) in branches %}
<tr>
- <td><a href="/{{ repo.display() }}/log/?h={{ branch.name }}">{{ branch.name }}</a></td>
- <td><a href="/{{ repo.display() }}/commit/?id={{ branch.commit.oid() }}">{{ branch.commit.summary() }}</a></td>
+ <td><a href="/{{ repo.display() }}/log/?h={{ name }}">{{ name }}</a></td>
+ <td><a href="/{{ repo.display() }}/commit/?id={{ commit.get().hash|hex }}">{{ commit.get().summary }}</a></td>
<td>
- <img src="https://www.gravatar.com/avatar/{{ branch.commit.author().email_md5() }}?s=13&d=retro" width="13" height="13">
- {{ branch.commit.author().name() }}
+ <img src="https://www.gravatar.com/avatar/{{ commit.get().author.email|md5 }}?s=13&d=retro" width="13" height="13">
+ {{ commit.get().author.name }}
</td>
<td>
- <time datetime="{{ branch.commit.author().time() }}" title="{{ branch.commit.author().time() }}">
- {{ branch.commit.author().time()|timeago }}
+ <time datetime="{{ commit.get().author.time }}" title="{{ commit.get().author.time }}">
+ {{ commit.get().author.time|timeago }}
</time>
</td>
</tr>
@@ -38,20 +38,20 @@
</thead>
<tbody>
-{% for tag in tags %}
+{% for (name, tag) in tags %}
<tr>
- <td><a href="/{{ repo.display() }}/tag/?h={{ tag.name }}">{{ tag.name }}</a></td>
+ <td><a href="/{{ repo.display() }}/tag/?h={{ name }}">{{ name }}</a></td>
<td></td>
<td>
- {% if let Some(tagger) = tag.tagger %}
- <img src="https://www.gravatar.com/avatar/{{ tagger.email_md5() }}?s=13&d=retro" width="13" height="13">
- {{ tagger.name() }}
+ {% if let Some(tagger) = tag.get().tagger %}
+ <img src="https://www.gravatar.com/avatar/{{ tagger.email|md5 }}?s=13&d=retro" width="13" height="13">
+ {{ tagger.name }}
{% endif %}
</td>
<td>
- {% if let Some(tagger) = tag.tagger %}
- <time datetime="{{ tagger.time() }}" title="{{ tagger.time() }}">
- {{ tagger.time()|timeago }}
+ {% if let Some(tagger) = tag.get().tagger %}
+ <time datetime="{{ tagger.time }}" title="{{ tagger.time }}">
+ {{ tagger.time|timeago }}
</time>
{% endif %}
</td>
diff --git a/templates/repo/refs.html b/templates/repo/refs.html
@@ -5,7 +5,7 @@
{% block content %}
<table class="repositories">
- {% call refs::branch_table(refs.branch) %}
+ {% call refs::branch_table(refs.heads) %}
<tbody>
<tr class="separator">
@@ -16,6 +16,6 @@
</tr>
</tbody>
- {% call refs::tag_table(refs.tag) %}
+ {% call refs::tag_table(refs.tags) %}
</table>
{% endblock %}
\ No newline at end of file
diff --git a/templates/repo/summary.html b/templates/repo/summary.html
@@ -5,8 +5,8 @@
{% block content %}
<table class="repositories">
- {% call refs::branch_table(refs.branch.iter().take(10)) %}
- {% if refs.branch.len() > 10 %}
+ {% call refs::branch_table(refs.heads.iter().take(10)) %}
+ {% if refs.heads.len() > 10 %}
<tbody>
<tr class="no-background">
<td><a href="/{{ repo.display() }}/refs" class="no-style">[...]</a></td>
@@ -26,8 +26,8 @@
</tr>
</tbody>
- {% call refs::tag_table(refs.tag.iter().take(10)) %}
- {% if refs.tag.len() > 10 %}
+ {% call refs::tag_table(refs.tags.iter().take(10)) %}
+ {% if refs.tags.len() > 10 %}
<tbody>
<tr class="no-background">
<td><a href="/{{ repo.display() }}/refs" class="no-style">[...]</a></td>