1use std::{
2 borrow::Cow,
3 collections::HashSet,
4 ffi::OsStr,
5 fmt::Debug,
6 path::{Path, PathBuf},
7 sync::Arc,
8};
9
10use anyhow::Context;
11use git2::{ErrorCode, Reference, Sort};
12use ini::Ini;
13use itertools::Itertools;
14use rocksdb::WriteBatch;
15use time::OffsetDateTime;
16use tracing::{error, info, info_span, instrument, warn};
17
18use crate::database::schema::{
19 commit::Commit,
20 repository::{Repository, RepositoryId},
21 tag::{Tag, TagTree},
22};
23
24pub fn run(scan_path: &Path, db: &Arc<rocksdb::DB>) {
25 let span = info_span!("index_update");
26 let _entered = span.enter();
27
28 info!("Starting index update");
29
30 update_repository_metadata(scan_path, db);
31 update_repository_reflog(scan_path, db.clone());
32 update_repository_tags(scan_path, db.clone());
33
34 info!("Flushing to disk");
35
36 if let Err(error) = db.flush() {
37 error!(%error, "Failed to flush database to disk");
38 }
39
40 info!("Finished index update");
41}
42
43#[instrument(skip(db))]
44fn update_repository_metadata(scan_path: &Path, db: &rocksdb::DB) {
45 let mut discovered = Vec::new();
46 discover_repositories(scan_path, &mut discovered);
47
48 for repository in discovered {
49 let Some(relative) = get_relative_path(scan_path, &repository) else {
50 continue;
51 };
52
53 let id = match Repository::open(db, relative) {
54 Ok(v) => v.map_or_else(RepositoryId::new, |v| v.get().id),
55 Err(error) => {
56 // maybe we could nuke it ourselves, but we need to instantly trigger
57 // a reindex and we could enter into an infinite loop if there's a bug
58 // or something
59 error!(%error, "Failed to open repository index {}, please consider nuking database", relative.display());
60 continue;
61 }
62 };
63
64 let Some(name) = relative.file_name().map(OsStr::to_string_lossy) else {
65 continue;
66 };
67 let description = std::fs::read(repository.join("description")).unwrap_or_default();
68 let description = Some(String::from_utf8_lossy(&description)).filter(|v| !v.is_empty());
69
70 let repository_path = scan_path.join(relative);
71
72 let git_repository = match git2::Repository::open(repository_path.clone()) {
73 Ok(v) => v,
74 Err(error) => {
75 warn!(%error, "Failed to open repository {} to update metadata, skipping", relative.display());
76 continue;
77 }
78 };
79
80 let res = Repository {
81 id,
82 name,
83 description,
84 owner: find_gitweb_owner(repository_path.as_path()),
85 last_modified: find_last_committed_time(&git_repository)
86 .unwrap_or(OffsetDateTime::UNIX_EPOCH),
87 default_branch: find_default_branch(&git_repository)
88 .ok()
89 .flatten()
90 .map(Cow::Owned),
91 }
92 .insert(db, relative);
93
94 if let Err(error) = res {
95 warn!(%error, "Failed to insert repository");
96 }
97 }
98}
99
100fn find_default_branch(repo: &git2::Repository) -> Result<Option<String>, git2::Error> {
101 Ok(repo.head()?.name().map(ToString::to_string))
102}
103
104fn find_last_committed_time(repo: &git2::Repository) -> Result<OffsetDateTime, git2::Error> {
105 let mut timestamp = OffsetDateTime::UNIX_EPOCH;
106
107 for reference in repo.references()? {
108 let Ok(commit) = reference?.peel_to_commit() else {
109 continue;
110 };
111
112 let committed_time = commit.committer().when().seconds();
113 let committed_time = OffsetDateTime::from_unix_timestamp(committed_time)
114 .unwrap_or(OffsetDateTime::UNIX_EPOCH);
115
116 if committed_time > timestamp {
117 timestamp = committed_time;
118 }
119 }
120
121 Ok(timestamp)
122}
123
124#[instrument(skip(db))]
125fn update_repository_reflog(scan_path: &Path, db: Arc<rocksdb::DB>) {
126 let repos = match Repository::fetch_all(&db) {
127 Ok(v) => v,
128 Err(error) => {
129 error!(%error, "Failed to read repository index to update reflog, consider deleting database directory");
130 return;
131 }
132 };
133
134 for (relative_path, db_repository) in repos {
135 let Some(git_repository) = open_repo(scan_path, &relative_path, db_repository.get(), &db)
136 else {
137 continue;
138 };
139
140 let references = match git_repository.references() {
141 Ok(v) => v,
142 Err(error) => {
143 error!(%error, "Failed to read references for {relative_path}");
144 continue;
145 }
146 };
147
148 let mut valid_references = Vec::new();
149
150 for reference in references.filter_map(Result::ok) {
151 let reference_name = String::from_utf8_lossy(reference.name_bytes());
152 if !reference_name.starts_with("refs/heads/")
153 && !reference_name.starts_with("refs/tags/")
154 {
155 continue;
156 }
157
158 valid_references.push(reference_name.to_string());
159
160 if let Err(error) = branch_index_update(
161 &reference,
162 &reference_name,
163 &relative_path,
164 db_repository.get(),
165 db.clone(),
166 &git_repository,
167 false,
168 ) {
169 error!(%error, "Failed to update reflog for {relative_path}@{reference_name}");
170 }
171 }
172
173 if let Err(error) = db_repository.get().replace_heads(&db, &valid_references) {
174 error!(%error, "Failed to update heads");
175 }
176 }
177}
178
179#[instrument(skip(reference, db_repository, db, git_repository))]
180fn branch_index_update(
181 reference: &Reference<'_>,
182 reference_name: &str,
183 relative_path: &str,
184 db_repository: &Repository<'_>,
185 db: Arc<rocksdb::DB>,
186 git_repository: &git2::Repository,
187 force_reindex: bool,
188) -> Result<(), anyhow::Error> {
189 info!("Refreshing indexes");
190
191 let commit_tree = db_repository.commit_tree(db.clone(), reference_name);
192
193 if force_reindex {
194 commit_tree.drop_commits()?;
195 }
196
197 let commit = reference.peel_to_commit()?;
198
199 let latest_indexed = if let Some(latest_indexed) = commit_tree.fetch_latest_one()? {
200 if commit.id().as_bytes() == &*latest_indexed.get().hash {
201 info!("No commits since last index");
202 return Ok(());
203 }
204
205 Some(latest_indexed)
206 } else {
207 None
208 };
209
210 let mut revwalk = git_repository.revwalk()?;
211 revwalk.set_sorting(Sort::REVERSE)?;
212 revwalk.push_ref(reference_name)?;
213
214 let tree_len = commit_tree.len()?;
215 let mut seen = false;
216 let mut i = 0;
217 for revs in &revwalk.chunks(250) {
218 let mut batch = WriteBatch::default();
219
220 for rev in revs {
221 let rev = rev?;
222
223 if let (false, Some(latest_indexed)) = (seen, &latest_indexed) {
224 if rev.as_bytes() == &*latest_indexed.get().hash {
225 seen = true;
226 }
227
228 continue;
229 }
230
231 seen = true;
232
233 if ((i + 1) % 25_000) == 0 {
234 info!("{} commits ingested", i + 1);
235 }
236
237 let commit = git_repository.find_commit(rev)?;
238 let author = commit.author();
239 let committer = commit.committer();
240
241 Commit::new(&commit, &author, &committer).insert(
242 &commit_tree,
243 tree_len + i,
244 &mut batch,
245 )?;
246 i += 1;
247 }
248
249 commit_tree.update_counter(tree_len + i, &mut batch)?;
250 db.write_without_wal(batch)?;
251 }
252
253 if !seen && !force_reindex {
254 warn!("Detected converged history, forcing reindex");
255
256 return branch_index_update(
257 reference,
258 reference_name,
259 relative_path,
260 db_repository,
261 db,
262 git_repository,
263 true,
264 );
265 }
266
267 Ok(())
268}
269
270#[instrument(skip(db))]
271fn update_repository_tags(scan_path: &Path, db: Arc<rocksdb::DB>) {
272 let repos = match Repository::fetch_all(&db) {
273 Ok(v) => v,
274 Err(error) => {
275 error!(%error, "Failed to read repository index to update tags, consider deleting database directory");
276 return;
277 }
278 };
279
280 for (relative_path, db_repository) in repos {
281 let Some(git_repository) = open_repo(scan_path, &relative_path, db_repository.get(), &db)
282 else {
283 continue;
284 };
285
286 if let Err(error) = tag_index_scan(
287 &relative_path,
288 db_repository.get(),
289 db.clone(),
290 &git_repository,
291 ) {
292 error!(%error, "Failed to update tags for {relative_path}");
293 }
294 }
295}
296
297#[instrument(skip(db_repository, db, git_repository))]
298fn tag_index_scan(
299 relative_path: &str,
300 db_repository: &Repository<'_>,
301 db: Arc<rocksdb::DB>,
302 git_repository: &git2::Repository,
303) -> Result<(), anyhow::Error> {
304 let tag_tree = db_repository.tag_tree(db);
305
306 let git_tags: HashSet<_> = git_repository
307 .references()
308 .context("Failed to scan indexes on git repository")?
309 .filter_map(Result::ok)
310 .filter(|v| v.name_bytes().starts_with(b"refs/tags/"))
311 .map(|v| String::from_utf8_lossy(v.name_bytes()).into_owned())
312 .collect();
313 let indexed_tags: HashSet<String> = tag_tree.list()?.into_iter().collect();
314
315 // insert any git tags that are missing from the index
316 for tag_name in git_tags.difference(&indexed_tags) {
317 tag_index_update(tag_name, git_repository, &tag_tree)?;
318 }
319
320 // remove any extra tags that the index has
321 // TODO: this also needs to check peel_to_tag
322 for tag_name in indexed_tags.difference(&git_tags) {
323 tag_index_delete(tag_name, &tag_tree)?;
324 }
325
326 Ok(())
327}
328
329#[instrument(skip(git_repository, tag_tree))]
330fn tag_index_update(
331 tag_name: &str,
332 git_repository: &git2::Repository,
333 tag_tree: &TagTree,
334) -> Result<(), anyhow::Error> {
335 let reference = git_repository
336 .find_reference(tag_name)
337 .context("Failed to read newly discovered tag")?;
338
339 if let Ok(tag) = reference.peel_to_tag() {
340 info!("Inserting newly discovered tag to index");
341
342 Tag::new(tag.tagger().as_ref()).insert(tag_tree, tag_name)?;
343 }
344
345 Ok(())
346}
347
348#[instrument(skip(tag_tree))]
349fn tag_index_delete(tag_name: &str, tag_tree: &TagTree) -> Result<(), anyhow::Error> {
350 info!("Removing stale tag from index");
351 tag_tree.remove(tag_name)?;
352
353 Ok(())
354}
355
356#[instrument(skip(scan_path, db_repository, db))]
357fn open_repo<P: AsRef<Path> + Debug>(
358 scan_path: &Path,
359 relative_path: P,
360 db_repository: &Repository<'_>,
361 db: &rocksdb::DB,
362) -> Option<git2::Repository> {
363 match git2::Repository::open(scan_path.join(relative_path.as_ref())) {
364 Ok(v) => Some(v),
365 Err(e) if e.code() == ErrorCode::NotFound => {
366 warn!("Repository gone from disk, removing from db");
367
368 if let Err(error) = db_repository.delete(db, relative_path) {
369 warn!(%error, "Failed to delete dangling index");
370 }
371
372 None
373 }
374 Err(error) => {
375 warn!(%error, "Failed to reindex, skipping");
376 None
377 }
378 }
379}
380
381fn get_relative_path<'a>(relative_to: &Path, full_path: &'a Path) -> Option<&'a Path> {
382 full_path.strip_prefix(relative_to).ok()
383}
384
385fn discover_repositories(current: &Path, discovered_repos: &mut Vec<PathBuf>) {
386 let current = match std::fs::read_dir(current) {
387 Ok(v) => v,
388 Err(error) => {
389 error!(%error, "Failed to enter repository directory {}", current.display());
390 return;
391 }
392 };
393
394 let dirs = current
395 .filter_map(Result::ok)
396 .map(|v| v.path())
397 .filter(|path| path.is_dir());
398
399 for dir in dirs {
400 if dir.join("packed-refs").is_file() {
401 // we've hit what looks like a bare git repo, lets take it
402 discovered_repos.push(dir);
403 } else {
404 // probably not a bare git repo, lets recurse deeper
405 discover_repositories(&dir, discovered_repos);
406 }
407 }
408}
409
410fn find_gitweb_owner(repository_path: &Path) -> Option<Cow<'_, str>> {
411 // Load the Git config file and attempt to extract the owner from the "gitweb" section.
412 // If the owner is not found, an empty string is returned.
413 Ini::load_from_file(repository_path.join("config"))
414 .ok()?
415 .section(Some("gitweb"))
416 .and_then(|section| section.get("owner"))
417 .map(String::from)
418 .map(Cow::Owned)
419}