From f9518d6b7de2bb78e1e69d638ea505c248c0101d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 30 May 2024 10:49:44 +0200 Subject: [PATCH] refactor: ensure repo dirs exist; rename some things --- server/src/cli.rs | 10 ++-- .../m20230730_000001_create_repo_tables.rs | 2 +- server/src/distro.rs | 51 +++++++++-------- server/src/main.rs | 5 +- server/src/repo/manager.rs | 57 +++++++++---------- server/src/repo/mod.rs | 2 +- 6 files changed, 61 insertions(+), 66 deletions(-) diff --git a/server/src/cli.rs b/server/src/cli.rs index 740e2ad..4fc94f1 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -1,10 +1,9 @@ -use crate::{distro::MetaDistroMgr, repo::MetaRepoMgr, Config, Global}; +use crate::{distro::MetaDistroMgr, Config, Global}; -use axum::{extract::FromRef, Router}; +use axum::Router; use clap::Parser; use sea_orm_migration::MigratorTrait; -use std::{io, path::PathBuf, sync::Arc}; -use tokio::sync::RwLock; +use std::{io, path::PathBuf}; use tower_http::trace::TraceLayer; use tracing::debug; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; @@ -76,8 +75,7 @@ impl Cli { data_dir: self.data_dir.clone(), }; - let mgr = MetaDistroMgr::new(&self.data_dir.join("distros"), db.clone()); - mgr.bootstrap().await?; + let mgr = MetaDistroMgr::new(&self.data_dir.join("distros"), db.clone()).await?; let global = Global { config, mgr, db }; diff --git a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs index b1e1fbe..2deb05f 100644 --- a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs +++ b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs @@ -81,7 +81,7 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Package::PgpSig).string_len(255)) .col(ColumnDef::new(Package::PgpSigSize).big_integer()) .col(ColumnDef::new(Package::Sha256Sum).char_len(64).not_null()) - .col(ColumnDef::new(Package::Compression).char_len(16).not_null()) + .col(ColumnDef::new(Package::Compression).string_len(16).not_null()) .foreign_key( ForeignKey::create() .name("fk-package-repo_id") diff --git a/server/src/distro.rs b/server/src/distro.rs index 209d227..22563ff 100644 --- a/server/src/distro.rs +++ b/server/src/distro.rs @@ -1,4 +1,4 @@ -use crate::{db, MetaRepoMgr}; +use crate::{db, DistroMgr}; use std::{ collections::HashMap, @@ -13,42 +13,40 @@ use tokio::sync::Mutex; pub struct MetaDistroMgr { distro_dir: PathBuf, conn: DbConn, - distros: Arc>>>, + distros: Arc>>>, } impl MetaDistroMgr { - pub fn new>(distro_dir: P, conn: DbConn) -> Self { - Self { - distro_dir: distro_dir.as_ref().to_path_buf(), - conn, - distros: Arc::new(Mutex::new(HashMap::new())), + pub async fn new>(distro_dir: P, conn: DbConn) -> crate::Result { + if !tokio::fs::try_exists(&distro_dir).await? { + tokio::fs::create_dir(&distro_dir).await?; } - } - /// Populate the manager with the currently known distros from the database. - pub async fn bootstrap(&self) -> crate::Result<()> { - let mut map = self.distros.lock().await; - let distros = db::Distro::find().all(&self.conn).await?; + let distro_dir = distro_dir.as_ref().to_path_buf(); + let mut map: HashMap> = HashMap::new(); + + let distros = db::Distro::find().all(&conn).await?; for distro in distros { - let mgr = MetaRepoMgr::new( - self.distro_dir.join(&distro.name), - distro.id, - self.conn.clone(), - ); + let mgr = + DistroMgr::new(distro_dir.join(&distro.name), distro.id, conn.clone()).await?; map.insert(distro.name, Arc::new(mgr)); } - Ok(()) + Ok(Self { + distro_dir, + conn, + distros: Arc::new(Mutex::new(map)), + }) } - pub async fn get_mgr(&self, distro: &str) -> Option> { + pub async fn get_mgr(&self, distro: &str) -> Option> { let map = self.distros.lock().await; map.get(distro).map(|mgr| Arc::clone(mgr)) } - pub async fn get_or_create_mgr(&self, distro: &str) -> crate::Result> { + pub async fn get_or_create_mgr(&self, distro: &str) -> crate::Result> { let mut map = self.distros.lock().await; if let Some(mgr) = map.get(distro) { @@ -56,11 +54,14 @@ impl MetaDistroMgr { } else { let distro = db::query::distro::insert(&self.conn, distro, None).await?; - let mgr = Arc::new(MetaRepoMgr::new( - self.distro_dir.join(&distro.name), - distro.id, - self.conn.clone(), - )); + let mgr = Arc::new( + DistroMgr::new( + self.distro_dir.join(&distro.name), + distro.id, + self.conn.clone(), + ) + .await?, + ); map.insert(distro.name, Arc::clone(&mgr)); Ok(mgr) diff --git a/server/src/main.rs b/server/src/main.rs index a428e39..d3cbdf0 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -6,11 +6,10 @@ mod error; mod repo; pub use error::{Result, ServerError}; -use repo::MetaRepoMgr; +use repo::DistroMgr; use clap::Parser; -use std::{path::PathBuf, sync::Arc}; -use tokio::sync::RwLock; +use std::path::PathBuf; #[derive(Clone)] pub struct Config { diff --git a/server/src/repo/manager.rs b/server/src/repo/manager.rs index a877d0d..23d693d 100644 --- a/server/src/repo/manager.rs +++ b/server/src/repo/manager.rs @@ -10,19 +10,23 @@ use uuid::Uuid; pub const ANY_ARCH: &'static str = "any"; -pub struct MetaRepoMgr { - repo_dir: PathBuf, +pub struct DistroMgr { + distro_dir: PathBuf, distro_id: i32, conn: DbConn, } -impl MetaRepoMgr { - pub fn new>(repo_dir: P, distro_id: i32, conn: DbConn) -> Self { - MetaRepoMgr { - repo_dir: repo_dir.as_ref().to_path_buf(), +impl DistroMgr { + pub async fn new>(distro_dir: P, distro_id: i32, conn: DbConn) -> Result { + if !tokio::fs::try_exists(&distro_dir).await? { + tokio::fs::create_dir(&distro_dir).await?; + } + + Ok(Self { + distro_dir: distro_dir.as_ref().to_path_buf(), distro_id, conn, - } + }) } /// Generate archive databases for all known architectures in the repository, including the @@ -62,9 +66,6 @@ impl MetaRepoMgr { let repo = repo.unwrap(); - let parent_dir = self.repo_dir.join(&repo.name); - tokio::fs::create_dir_all(&parent_dir).await?; - let [tmp_ar_db_path, tmp_ar_files_path, files_tmp_file_path, desc_tmp_file_path] = self.random_file_paths(); let ar_db = archive::RepoArchiveWriter::open(&tmp_ar_db_path).await?; @@ -102,15 +103,13 @@ impl MetaRepoMgr { ar_db.close().await?; ar_files.close().await?; + let repo_dir = self.distro_dir.join(&repo.name); + // Move the db archives to their respective places - tokio::fs::rename( - tmp_ar_db_path, - parent_dir.join(format!("{}.db.tar.gz", arch)), - ) - .await?; + tokio::fs::rename(tmp_ar_db_path, repo_dir.join(format!("{}.db.tar.gz", arch))).await?; tokio::fs::rename( tmp_ar_files_path, - parent_dir.join(format!("{}.files.tar.gz", arch)), + repo_dir.join(format!("{}.files.tar.gz", arch)), ) .await?; @@ -131,7 +130,7 @@ impl MetaRepoMgr { repo_entry.delete(&self.conn).await?; // Remove files from file system - tokio::fs::remove_dir_all(self.repo_dir.join(repo)).await?; + tokio::fs::remove_dir_all(self.distro_dir.join(repo)).await?; Ok(true) } else { @@ -152,7 +151,7 @@ impl MetaRepoMgr { while let Some(pkg) = pkgs.next().await.transpose()? { let path = self - .repo_dir + .distro_dir .join(&repo.name) .join(super::package::filename(&pkg)); tokio::fs::remove_file(path).await?; @@ -161,13 +160,13 @@ impl MetaRepoMgr { } tokio::fs::remove_file( - self.repo_dir + self.distro_dir .join(&repo.name) .join(format!("{}.db.tar.gz", arch)), ) .await?; tokio::fs::remove_file( - self.repo_dir + self.distro_dir .join(&repo.name) .join(format!("{}.files.tar.gz", arch)), ) @@ -194,7 +193,7 @@ impl MetaRepoMgr { if let Some(pkg) = pkg { // Remove package from database & file system tokio::fs::remove_file( - self.repo_dir + self.distro_dir .join(&repo.name) .join(super::package::filename(&pkg)), ) @@ -221,24 +220,23 @@ impl MetaRepoMgr { reader: &mut R, repo: &str, ) -> crate::Result<(String, String, String)> { - // Copy file contents to temporary path so libarchive can work with it let [path] = self.random_file_paths(); let mut temp_file = tokio::fs::File::create(&path).await?; tokio::io::copy(reader, &mut temp_file).await?; - // Parse the package let path_clone = path.clone(); let pkg = tokio::task::spawn_blocking(move || package::Package::open(path_clone)) .await .unwrap()?; - // Query the repo for its ID, or create it if it does not already exist - let res = db::query::repo::by_name(&self.conn, &repo).await?; + let repo_dir = self.distro_dir.join(repo); - let repo_id = if let Some(repo_entity) = res { - repo_entity.id + let repo_id = if let Some(repo) = db::query::repo::by_name(&self.conn, &repo).await? { + repo.id } else { + tokio::fs::create_dir(&repo_dir).await?; + db::query::repo::insert(&self.conn, self.distro_id, repo, None) .await? .id @@ -259,7 +257,7 @@ impl MetaRepoMgr { entry.delete(&self.conn).await?; } - let dest_pkg_path = self.repo_dir.join(repo).join(pkg.file_name()); + let dest_pkg_path = repo_dir.join(pkg.file_name()); // Insert new package into database let name = pkg.info.name.clone(); @@ -268,7 +266,6 @@ impl MetaRepoMgr { db::query::package::insert(&self.conn, repo_id, pkg).await?; // Move the package to its final resting place - tokio::fs::create_dir_all(dest_pkg_path.parent().unwrap()).await?; tokio::fs::rename(path, dest_pkg_path).await?; // Synchronize archive databases @@ -285,7 +282,7 @@ impl MetaRepoMgr { pub fn random_file_paths(&self) -> [PathBuf; C] { std::array::from_fn(|_| { let uuid: uuid::fmt::Simple = Uuid::new_v4().into(); - self.repo_dir.join(uuid.to_string()) + self.distro_dir.join(uuid.to_string()) }) } } diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index 7544faa..7d7e321 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -2,7 +2,7 @@ mod archive; mod manager; pub mod package; -pub use manager::MetaRepoMgr; +pub use manager::DistroMgr; use axum::{ body::Body,