Chewing_Bever
/
rieter
1
0
Fork 0
Code Issues 11 Pull Requests Packages Releases Activity

refactor: ensure repo dirs exist; rename some things

concurrent-repos
Jef Roosens 2024-05-30 10:49:44 +02:00
parent 58def483aa
commit f9518d6b7d
Signed by: Jef Roosens
GPG Key ID: 02D4C0997E74717B
6 changed files with 61 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
server/src/cli.rs
View File

@ -1,10 +1,9 @@
use crate::{distro::MetaDistroMgr, repo::MetaRepoMgr, Config, Global};
use crate::{distro::MetaDistroMgr, Config, Global};
use axum::{extract::FromRef, Router};
use axum::Router;
use clap::Parser;
use sea_orm_migration::MigratorTrait;
use std::{io, path::PathBuf, sync::Arc};
use tokio::sync::RwLock;
use std::{io, path::PathBuf};
use tower_http::trace::TraceLayer;
use tracing::debug;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
@ -76,8 +75,7 @@ impl Cli {
data_dir: self.data_dir.clone(),
};
let mgr = MetaDistroMgr::new(&self.data_dir.join("distros"), db.clone());
mgr.bootstrap().await?;
let mgr = MetaDistroMgr::new(&self.data_dir.join("distros"), db.clone()).await?;
let global = Global { config, mgr, db };

2
server/src/db/migrator/m20230730_000001_create_repo_tables.rs
View File

@ -81,7 +81,7 @@ impl MigrationTrait for Migration {
.col(ColumnDef::new(Package::PgpSig).string_len(255))
.col(ColumnDef::new(Package::PgpSigSize).big_integer())
.col(ColumnDef::new(Package::Sha256Sum).char_len(64).not_null())
.col(ColumnDef::new(Package::Compression).char_len(16).not_null())
.col(ColumnDef::new(Package::Compression).string_len(16).not_null())
.foreign_key(
ForeignKey::create()
.name("fk-package-repo_id")

45
server/src/distro.rs
View File

@ -1,4 +1,4 @@
use crate::{db, MetaRepoMgr};
use crate::{db, DistroMgr};
use std::{
collections::HashMap,
@ -13,42 +13,40 @@ use tokio::sync::Mutex;
pub struct MetaDistroMgr {
distro_dir: PathBuf,
conn: DbConn,
distros: Arc<Mutex<HashMap<String, Arc<MetaRepoMgr>>>>,
distros: Arc<Mutex<HashMap<String, Arc<DistroMgr>>>>,
}
impl MetaDistroMgr {
pub fn new<P: AsRef<Path>>(distro_dir: P, conn: DbConn) -> Self {
Self {
distro_dir: distro_dir.as_ref().to_path_buf(),
conn,
distros: Arc::new(Mutex::new(HashMap::new())),
}
pub async fn new<P: AsRef<Path>>(distro_dir: P, conn: DbConn) -> crate::Result<Self> {
if !tokio::fs::try_exists(&distro_dir).await? {
tokio::fs::create_dir(&distro_dir).await?;
}
/// Populate the manager with the currently known distros from the database.
pub async fn bootstrap(&self) -> crate::Result<()> {
let mut map = self.distros.lock().await;
let distros = db::Distro::find().all(&self.conn).await?;
let distro_dir = distro_dir.as_ref().to_path_buf();
let mut map: HashMap<String, Arc<DistroMgr>> = HashMap::new();
let distros = db::Distro::find().all(&conn).await?;
for distro in distros {
let mgr = MetaRepoMgr::new(
self.distro_dir.join(&distro.name),
distro.id,
self.conn.clone(),
);
let mgr =
DistroMgr::new(distro_dir.join(&distro.name), distro.id, conn.clone()).await?;
map.insert(distro.name, Arc::new(mgr));
}
Ok(())
Ok(Self {
distro_dir,
conn,
distros: Arc::new(Mutex::new(map)),
})
}
pub async fn get_mgr(&self, distro: &str) -> Option<Arc<MetaRepoMgr>> {
pub async fn get_mgr(&self, distro: &str) -> Option<Arc<DistroMgr>> {
let map = self.distros.lock().await;
map.get(distro).map(|mgr| Arc::clone(mgr))
}
pub async fn get_or_create_mgr(&self, distro: &str) -> crate::Result<Arc<MetaRepoMgr>> {
pub async fn get_or_create_mgr(&self, distro: &str) -> crate::Result<Arc<DistroMgr>> {
let mut map = self.distros.lock().await;
if let Some(mgr) = map.get(distro) {
@ -56,11 +54,14 @@ impl MetaDistroMgr {
} else {
let distro = db::query::distro::insert(&self.conn, distro, None).await?;
let mgr = Arc::new(MetaRepoMgr::new(
let mgr = Arc::new(
DistroMgr::new(
self.distro_dir.join(&distro.name),
distro.id,
self.conn.clone(),
));
)
.await?,
);
map.insert(distro.name, Arc::clone(&mgr));
Ok(mgr)

5
server/src/main.rs
View File

@ -6,11 +6,10 @@ mod error;
mod repo;
pub use error::{Result, ServerError};
use repo::MetaRepoMgr;
use repo::DistroMgr;
use clap::Parser;
use std::{path::PathBuf, sync::Arc};
use tokio::sync::RwLock;
use std::path::PathBuf;
#[derive(Clone)]
pub struct Config {

57
server/src/repo/manager.rs
View File

@ -10,19 +10,23 @@ use uuid::Uuid;
pub const ANY_ARCH: &'static str = "any";
pub struct MetaRepoMgr {
repo_dir: PathBuf,
pub struct DistroMgr {
distro_dir: PathBuf,
distro_id: i32,
conn: DbConn,
}
impl MetaRepoMgr {
pub fn new<P: AsRef<Path>>(repo_dir: P, distro_id: i32, conn: DbConn) -> Self {
MetaRepoMgr {
repo_dir: repo_dir.as_ref().to_path_buf(),
impl DistroMgr {
pub async fn new<P: AsRef<Path>>(distro_dir: P, distro_id: i32, conn: DbConn) -> Result<Self> {
if !tokio::fs::try_exists(&distro_dir).await? {
tokio::fs::create_dir(&distro_dir).await?;
}
Ok(Self {
distro_dir: distro_dir.as_ref().to_path_buf(),
distro_id,
conn,
}
})
}
/// Generate archive databases for all known architectures in the repository, including the
@ -62,9 +66,6 @@ impl MetaRepoMgr {
let repo = repo.unwrap();
let parent_dir = self.repo_dir.join(&repo.name);
tokio::fs::create_dir_all(&parent_dir).await?;
let [tmp_ar_db_path, tmp_ar_files_path, files_tmp_file_path, desc_tmp_file_path] =
self.random_file_paths();
let ar_db = archive::RepoArchiveWriter::open(&tmp_ar_db_path).await?;
@ -102,15 +103,13 @@ impl MetaRepoMgr {
ar_db.close().await?;
ar_files.close().await?;
let repo_dir = self.distro_dir.join(&repo.name);
// Move the db archives to their respective places
tokio::fs::rename(
tmp_ar_db_path,
parent_dir.join(format!("{}.db.tar.gz", arch)),
)
.await?;
tokio::fs::rename(tmp_ar_db_path, repo_dir.join(format!("{}.db.tar.gz", arch))).await?;
tokio::fs::rename(
tmp_ar_files_path,
parent_dir.join(format!("{}.files.tar.gz", arch)),
repo_dir.join(format!("{}.files.tar.gz", arch)),
)
.await?;
@ -131,7 +130,7 @@ impl MetaRepoMgr {
repo_entry.delete(&self.conn).await?;
// Remove files from file system
tokio::fs::remove_dir_all(self.repo_dir.join(repo)).await?;
tokio::fs::remove_dir_all(self.distro_dir.join(repo)).await?;
Ok(true)
} else {
@ -152,7 +151,7 @@ impl MetaRepoMgr {
while let Some(pkg) = pkgs.next().await.transpose()? {
let path = self
.repo_dir
.distro_dir
.join(&repo.name)
.join(super::package::filename(&pkg));
tokio::fs::remove_file(path).await?;
@ -161,13 +160,13 @@ impl MetaRepoMgr {
}
tokio::fs::remove_file(
self.repo_dir
self.distro_dir
.join(&repo.name)
.join(format!("{}.db.tar.gz", arch)),
)
.await?;
tokio::fs::remove_file(
self.repo_dir
self.distro_dir
.join(&repo.name)
.join(format!("{}.files.tar.gz", arch)),
)
@ -194,7 +193,7 @@ impl MetaRepoMgr {
if let Some(pkg) = pkg {
// Remove package from database & file system
tokio::fs::remove_file(
self.repo_dir
self.distro_dir
.join(&repo.name)
.join(super::package::filename(&pkg)),
)
@ -221,24 +220,23 @@ impl MetaRepoMgr {
reader: &mut R,
repo: &str,
) -> crate::Result<(String, String, String)> {
// Copy file contents to temporary path so libarchive can work with it
let [path] = self.random_file_paths();
let mut temp_file = tokio::fs::File::create(&path).await?;
tokio::io::copy(reader, &mut temp_file).await?;
// Parse the package
let path_clone = path.clone();
let pkg = tokio::task::spawn_blocking(move || package::Package::open(path_clone))
.await
.unwrap()?;
// Query the repo for its ID, or create it if it does not already exist
let res = db::query::repo::by_name(&self.conn, &repo).await?;
let repo_dir = self.distro_dir.join(repo);
let repo_id = if let Some(repo_entity) = res {
repo_entity.id
let repo_id = if let Some(repo) = db::query::repo::by_name(&self.conn, &repo).await? {
repo.id
} else {
tokio::fs::create_dir(&repo_dir).await?;
db::query::repo::insert(&self.conn, self.distro_id, repo, None)
.await?
.id
@ -259,7 +257,7 @@ impl MetaRepoMgr {
entry.delete(&self.conn).await?;
}
let dest_pkg_path = self.repo_dir.join(repo).join(pkg.file_name());
let dest_pkg_path = repo_dir.join(pkg.file_name());
// Insert new package into database
let name = pkg.info.name.clone();
@ -268,7 +266,6 @@ impl MetaRepoMgr {
db::query::package::insert(&self.conn, repo_id, pkg).await?;
// Move the package to its final resting place
tokio::fs::create_dir_all(dest_pkg_path.parent().unwrap()).await?;
tokio::fs::rename(path, dest_pkg_path).await?;
// Synchronize archive databases
@ -285,7 +282,7 @@ impl MetaRepoMgr {
pub fn random_file_paths<const C: usize>(&self) -> [PathBuf; C] {
std::array::from_fn(|_| {
let uuid: uuid::fmt::Simple = Uuid::new_v4().into();
self.repo_dir.join(uuid.to_string())
self.distro_dir.join(uuid.to_string())
})
}
}

2
server/src/repo/mod.rs
View File

@ -2,7 +2,7 @@ mod archive;
mod manager;
pub mod package;
pub use manager::MetaRepoMgr;
pub use manager::DistroMgr;
use axum::{
body::Body,