diff --git a/CHANGELOG.md b/CHANGELOG.md index f715cfe..8e00689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,5 +14,3 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Serve packages from any number of repositories & architectures * Publish packages to and delete packages from repositories using HTTP requests - * Packages of architecture "any" are part of every architecture's - database diff --git a/server/src/api/pagination.rs b/server/src/api/pagination.rs index 3904ea4..ae3812d 100644 --- a/server/src/api/pagination.rs +++ b/server/src/api/pagination.rs @@ -1,3 +1,5 @@ +use axum::response::{IntoResponse, Response}; +use axum::Json; use serde::{Deserialize, Serialize}; pub const DEFAULT_PAGE: u64 = 0; diff --git a/server/src/cli.rs b/server/src/cli.rs index 0725e64..be4e9f5 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -18,6 +18,8 @@ pub struct Cli { pub pkg_dir: PathBuf, /// Directory where repository metadata & SQLite database is stored pub data_dir: PathBuf, + /// Default architecture to add packages with arch "any" to + pub default_arch: String, /// Database connection URL; either sqlite:// or postgres://. Defaults to rieter.sqlite in the /// data directory @@ -73,7 +75,8 @@ impl Cli { repo_dir: self.data_dir.join("repos"), pkg_dir: self.pkg_dir.clone(), }; - let repo_manager = RepoGroupManager::new(&config.repo_dir, &self.pkg_dir); + let repo_manager = + RepoGroupManager::new(&config.repo_dir, &self.pkg_dir, &self.default_arch); let global = Global { config, @@ -84,7 +87,7 @@ impl Cli { // build our application with a single route let app = Router::new() .nest("/api", crate::api::router()) - .merge(crate::repo::router()) + .merge(crate::repo::router(&global)) .with_state(global) .layer(TraceLayer::new_for_http()); diff --git a/server/src/error.rs b/server/src/error.rs index 4fbb7c4..c11ad8d 100644 --- a/server/src/error.rs +++ b/server/src/error.rs @@ -19,8 +19,8 @@ impl fmt::Display for ServerError { match self { ServerError::IO(err) => write!(fmt, "{}", err), ServerError::Axum(err) => write!(fmt, "{}", err), - ServerError::Status(status) => write!(fmt, "{}", status), ServerError::Db(err) => write!(fmt, "{}", err), + ServerError::Status(status) => write!(fmt, "{}", status), } } } @@ -34,11 +34,11 @@ impl IntoResponse for ServerError { match self { ServerError::IO(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), ServerError::Axum(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), - ServerError::Status(status) => status.into_response(), ServerError::Db(sea_orm::DbErr::RecordNotFound(_)) => { StatusCode::NOT_FOUND.into_response() } ServerError::Db(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + ServerError::Status(status) => status.into_response(), } } } @@ -61,14 +61,14 @@ impl From for ServerError { } } -impl From for ServerError { - fn from(status: StatusCode) -> Self { - Self::Status(status) - } -} - impl From for ServerError { fn from(err: sea_orm::DbErr) -> Self { ServerError::Db(err) } } + +impl From for ServerError { + fn from(status: StatusCode) -> Self { + ServerError::Status(status) + } +} diff --git a/server/src/repo/manager.rs b/server/src/repo/manager.rs index f5aad8c..d036aab 100644 --- a/server/src/repo/manager.rs +++ b/server/src/repo/manager.rs @@ -1,33 +1,28 @@ use super::package::Package; use libarchive::write::{Builder, WriteEntry}; use libarchive::{Entry, WriteFilter, WriteFormat}; +use std::collections::HashSet; use std::fs; use std::io; use std::path::{Path, PathBuf}; -pub const ANY_ARCH: &str = "any"; - /// Overarching abstraction that orchestrates updating the repositories stored on the server pub struct RepoGroupManager { repo_dir: PathBuf, pkg_dir: PathBuf, -} - -fn parse_pkg_filename(file_name: &str) -> (String, &str, &str, &str) { - let name_parts = file_name.split('-').collect::>(); - let name = name_parts[..name_parts.len() - 3].join("-"); - let version = name_parts[name_parts.len() - 3]; - let release = name_parts[name_parts.len() - 2]; - let (arch, _) = name_parts[name_parts.len() - 1].split_once('.').unwrap(); - - (name, version, release, arch) + default_arch: String, } impl RepoGroupManager { - pub fn new, P2: AsRef>(repo_dir: P1, pkg_dir: P2) -> Self { + pub fn new, P2: AsRef>( + repo_dir: P1, + pkg_dir: P2, + default_arch: &str, + ) -> Self { RepoGroupManager { repo_dir: repo_dir.as_ref().to_path_buf(), pkg_dir: pkg_dir.as_ref().to_path_buf(), + default_arch: String::from(default_arch), } } @@ -42,23 +37,11 @@ impl RepoGroupManager { ar_files.add_filter(WriteFilter::Gzip)?; ar_files.set_format(WriteFormat::PaxRestricted)?; - let mut ar_db = ar_db.open_file(subrepo_path.join(format!("{}.db.tar.gz", repo)))?; + let mut ar_db = ar_db.open_file(subrepo_path.join(format!("{}.tar.gz", repo)))?; let mut ar_files = ar_files.open_file(subrepo_path.join(format!("{}.files.tar.gz", repo)))?; - // All architectures should also include the "any" architecture, except for the "any" - // architecture itself. - let repo_any_dir = self.repo_dir.join(repo).join(ANY_ARCH); - - let any_entries_iter = if arch != ANY_ARCH && repo_any_dir.try_exists()? { - Some(repo_any_dir.read_dir()?) - } else { - None - } - .into_iter() - .flatten(); - - for entry in subrepo_path.read_dir()?.chain(any_entries_iter) { + for entry in subrepo_path.read_dir()? { let entry = entry?; if entry.file_type()?.is_dir() { @@ -85,23 +68,7 @@ impl RepoGroupManager { } } - ar_db.close()?; - ar_files.close()?; - - Ok(()) - } - - /// Synchronize all present architectures' db archives in the given repository. - pub fn sync_all(&mut self, repo: &str) -> io::Result<()> { - for entry in self.repo_dir.join(repo).read_dir()? { - let entry = entry?; - - if entry.file_type()?.is_dir() { - self.sync(repo, &entry.file_name().to_string_lossy())?; - } - } - - Ok(()) + ar_db.close().and(ar_files.close()).map_err(Into::into) } pub fn add_pkg_from_path>( @@ -111,97 +78,118 @@ impl RepoGroupManager { ) -> io::Result { let pkg = Package::open(&path)?; - self.add_pkg(repo, &pkg)?; + let archs = self.add_pkg_in_repo(repo, &pkg)?; - // After successfully adding the package, we move it to the packages directory - let dest_pkg_path = self - .pkg_dir - .join(repo) - .join(&pkg.info.arch) - .join(pkg.file_name()); + // We add the package to each architecture it was added to by hard-linking the provided + // package file. This prevents storing a package of type "any" multiple times on disk. + for arch in archs { + let arch_repo_pkg_path = self.pkg_dir.join(repo).join(arch); + let dest_pkg_path = arch_repo_pkg_path.join(pkg.file_name()); - fs::create_dir_all(dest_pkg_path.parent().unwrap())?; - fs::rename(&path, dest_pkg_path)?; + fs::create_dir_all(&arch_repo_pkg_path)?; + fs::hard_link(&path, dest_pkg_path)?; + } + + fs::remove_file(path)?; Ok(pkg) } /// Add a package to the given repo, returning to what architectures the package was added. - pub fn add_pkg(&mut self, repo: &str, pkg: &Package) -> io::Result<()> { - // We first remove any existing version of the package - self.remove_pkg(repo, &pkg.info.arch, &pkg.info.name, false)?; + pub fn add_pkg_in_repo(&mut self, repo: &str, pkg: &Package) -> io::Result> { + let mut arch_repos: HashSet = HashSet::new(); - // Write the `desc` and `files` metadata files to disk - let metadata_dir = self - .repo_dir - .join(repo) - .join(&pkg.info.arch) - .join(format!("{}-{}", pkg.info.name, pkg.info.version)); + if pkg.info.arch != "any" { + self.add_pkg_in_arch_repo(repo, &pkg.info.arch, pkg)?; + arch_repos.insert(pkg.info.arch.clone()); + } + // Packages of arch "any" are added to every existing arch + else { + arch_repos.insert(self.default_arch.clone()); - fs::create_dir_all(&metadata_dir)?; + let repo_dir = self.repo_dir.join(repo); - let mut desc_file = fs::File::create(metadata_dir.join("desc"))?; - pkg.write_desc(&mut desc_file)?; + if repo_dir.exists() { + for entry in repo_dir.read_dir()? { + arch_repos.insert(entry?.file_name().to_string_lossy().to_string()); + } + } - let mut files_file = fs::File::create(metadata_dir.join("files"))?; - pkg.write_files(&mut files_file)?; - - // If a package of type "any" is added, we need to update every existing database - if pkg.info.arch == ANY_ARCH { - self.sync_all(repo)?; - } else { - self.sync(repo, &pkg.info.arch)?; + for arch in arch_repos.iter() { + self.add_pkg_in_arch_repo(repo, arch, pkg)?; + } } - Ok(()) + Ok(arch_repos) + } + + pub fn add_pkg_in_arch_repo( + &mut self, + repo: &str, + arch: &str, + pkg: &Package, + ) -> io::Result<()> { + let pkg_dir = self + .repo_dir + .join(repo) + .join(arch) + .join(format!("{}-{}", pkg.info.name, pkg.info.version)); + + // We first remove the previous version of the package, if present + self.remove_pkg_from_arch_repo(repo, arch, &pkg.info.name, false)?; + + fs::create_dir_all(&pkg_dir)?; + + let mut desc_file = fs::File::create(pkg_dir.join("desc"))?; + pkg.write_desc(&mut desc_file)?; + + let mut files_file = fs::File::create(pkg_dir.join("files"))?; + pkg.write_files(&mut files_file)?; + + self.sync(repo, arch) } pub fn remove_repo(&mut self, repo: &str) -> io::Result { - let repo_dir = self.repo_dir.join(repo); + let repo_dir = self.repo_dir.join(&repo); if !repo_dir.exists() { Ok(false) } else { - fs::remove_dir_all(&repo_dir)?; - fs::remove_dir_all(self.pkg_dir.join(repo))?; + fs::remove_dir_all(&repo_dir) + .and_then(|_| fs::remove_dir_all(self.pkg_dir.join(repo)))?; Ok(true) } } - pub fn remove_repo_arch(&mut self, repo: &str, arch: &str) -> io::Result { + pub fn remove_arch_repo(&mut self, repo: &str, arch: &str) -> io::Result { let sub_path = PathBuf::from(repo).join(arch); let repo_dir = self.repo_dir.join(&sub_path); if !repo_dir.exists() { - return Ok(false); + Ok(false) + } else { + fs::remove_dir_all(&repo_dir) + .and_then(|_| fs::remove_dir_all(self.pkg_dir.join(sub_path)))?; + + Ok(true) } - - fs::remove_dir_all(&repo_dir)?; - fs::remove_dir_all(self.pkg_dir.join(sub_path))?; - - // Removing the "any" architecture updates all other repositories - if arch == ANY_ARCH { - self.sync_all(repo)?; - } - - Ok(true) } - pub fn remove_pkg( + pub fn remove_pkg_from_arch_repo( &mut self, repo: &str, arch: &str, pkg_name: &str, sync: bool, ) -> io::Result { - let repo_arch_dir = self.repo_dir.join(repo).join(arch); + let arch_repo_dir = self.repo_dir.join(repo).join(arch); - if !repo_arch_dir.exists() { + if !arch_repo_dir.exists() { return Ok(false); } - for entry in repo_arch_dir.read_dir()? { + for entry in arch_repo_dir.read_dir()? { let entry = entry?; // Make sure we skip the archive files @@ -221,13 +209,16 @@ impl RepoGroupManager { fs::remove_dir_all(entry.path())?; // Also remove the old package archive - let repo_arch_pkg_dir = self.pkg_dir.join(repo).join(arch); + let arch_repo_pkg_dir = self.pkg_dir.join(repo).join(arch); - repo_arch_pkg_dir.read_dir()?.try_for_each(|res| { + arch_repo_pkg_dir.read_dir()?.try_for_each(|res| { res.and_then(|entry: fs::DirEntry| { let file_name = entry.file_name(); let file_name = file_name.to_string_lossy(); - let (name, _, _, _) = parse_pkg_filename(&file_name); + + // Same trick, but for package files, we also need to trim the arch + let name_parts = file_name.split('-').collect::>(); + let name = name_parts[..name_parts.len() - 3].join("-"); if name == pkg_name { fs::remove_file(entry.path()) @@ -238,11 +229,7 @@ impl RepoGroupManager { })?; if sync { - if arch == ANY_ARCH { - self.sync_all(repo)?; - } else { - self.sync(repo, arch)?; - } + self.sync(repo, arch)?; } return Ok(true); diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index 0a28969..e4be526 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -4,29 +4,30 @@ mod package; pub use manager::RepoGroupManager; use crate::db::entities::{package as db_package, repo as db_repo}; -use axum::body::Body; use axum::extract::{BodyStream, Path, State}; -use axum::http::Request; use axum::http::StatusCode; -use axum::response::IntoResponse; -use axum::routing::{delete, post}; +use axum::routing::{delete, get_service, post}; use axum::Router; use futures::StreamExt; use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter}; use std::sync::Arc; use tokio::{fs, io::AsyncWriteExt}; -use tower::util::ServiceExt; -use tower_http::services::{ServeDir, ServeFile}; +use tower_http::services::ServeDir; use uuid::Uuid; -pub fn router() -> Router { +pub fn router(global: &crate::Global) -> Router { + // Try to serve packages by default, and try the database files instead if not found + let serve_repos = get_service( + ServeDir::new(&global.config.pkg_dir).fallback(ServeDir::new(&global.config.repo_dir)), + ); Router::new() .route("/:repo", post(post_package_archive).delete(delete_repo)) .route("/:repo/:arch", delete(delete_arch_repo)) .route( "/:repo/:arch/:filename", - delete(delete_package).get(get_file), + delete(delete_package).get(serve_repos.clone()), ) + .fallback(serve_repos) } async fn post_package_archive( @@ -44,104 +45,40 @@ async fn post_package_archive( } let clone = Arc::clone(&global.repo_manager); - let path_clone = path.clone(); + let repo_clone = repo.clone(); - let res = tokio::task::spawn_blocking(move || { - clone - .write() - .unwrap() - .add_pkg_from_path(&repo_clone, &path_clone) + let pkg = tokio::task::spawn_blocking(move || { + clone.write().unwrap().add_pkg_from_path(&repo_clone, &path) }) - .await?; + .await??; - match res { - // Insert the newly added package into the database - Ok(pkg) => { - // Query the repo for its ID, or create it if it does not already exist - let repo_entity = db_repo::Entity::find() - .filter(db_repo::Column::Name.eq(&repo)) - .one(&global.db) - .await?; + // Query the repo for its ID, or create it if it does not already exist + let repo_entity = db_repo::Entity::find() + .filter(db_repo::Column::Name.eq(&repo)) + .one(&global.db) + .await?; - let repo_id = if let Some(repo_entity) = repo_entity { - repo_entity.id - } else { - let model = db_repo::ActiveModel { - name: sea_orm::Set(repo.clone()), - ..Default::default() - }; - - db_repo::Entity::insert(model) - .exec(&global.db) - .await? - .last_insert_id - }; - - // Insert the package's data into the database - let mut model: db_package::ActiveModel = pkg.into(); - model.repo_id = sea_orm::Set(repo_id); - - model.insert(&global.db).await?; - - Ok(()) - } - // Remove the uploaded file and return the error - Err(err) => { - tokio::fs::remove_file(path).await?; - - Err(err.into()) - } - } -} - -/// Serve the package archive files and database archives. If files are requested for an -/// architecture that does not have any explicit packages, a repository containing only "any" files -/// is returned. -async fn get_file( - State(global): State, - Path((repo, arch, mut file_name)): Path<(String, String, String)>, - req: Request, -) -> crate::Result { - let repo_dir = global.config.repo_dir.join(&repo).join(&arch); - let repo_exists = tokio::fs::try_exists(&repo_dir).await?; - - let res = if file_name.ends_with(".db") || file_name.ends_with(".db.tar.gz") { - // Append tar extension to ensure we find the file - if file_name.ends_with(".db") { - file_name.push_str(".tar.gz"); + let repo_id = if let Some(repo_entity) = repo_entity { + repo_entity.id + } else { + let model = db_repo::ActiveModel { + name: sea_orm::Set(repo.clone()), + ..Default::default() }; - if repo_exists { - ServeFile::new(repo_dir.join(file_name)).oneshot(req).await - } else { - let path = global - .config - .repo_dir - .join(repo) - .join(manager::ANY_ARCH) - .join(file_name); - - ServeFile::new(path).oneshot(req).await - } - } else { - let any_file = global - .config - .pkg_dir - .join(repo) - .join(manager::ANY_ARCH) - .join(file_name); - - if repo_exists { - ServeDir::new(global.config.pkg_dir) - .fallback(ServeFile::new(any_file)) - .oneshot(req) - .await - } else { - ServeFile::new(any_file).oneshot(req).await - } + db_repo::Entity::insert(model) + .exec(&global.db) + .await? + .last_insert_id }; - Ok(res) + // Insert the package's data into the database + let mut model: db_package::ActiveModel = pkg.into(); + model.repo_id = sea_orm::Set(repo_id); + + model.insert(&global.db).await?; + + Ok(()) } async fn delete_repo( @@ -167,7 +104,7 @@ async fn delete_arch_repo( let clone = Arc::clone(&global.repo_manager); let repo_removed = - tokio::task::spawn_blocking(move || clone.write().unwrap().remove_repo_arch(&repo, &arch)) + tokio::task::spawn_blocking(move || clone.write().unwrap().remove_arch_repo(&repo, &arch)) .await??; if repo_removed { @@ -194,7 +131,10 @@ async fn delete_package( let clone = Arc::clone(&global.repo_manager); let pkg_removed = tokio::task::spawn_blocking(move || { - clone.write().unwrap().remove_pkg(&repo, &arch, &name, true) + clone + .write() + .unwrap() + .remove_pkg_from_arch_repo(&repo, &arch, &name, true) }) .await??;