diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e00689..f715cfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,3 +14,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Serve packages from any number of repositories & architectures * Publish packages to and delete packages from repositories using HTTP requests + * Packages of architecture "any" are part of every architecture's + database diff --git a/server/src/api/pagination.rs b/server/src/api/pagination.rs index ae3812d..3904ea4 100644 --- a/server/src/api/pagination.rs +++ b/server/src/api/pagination.rs @@ -1,5 +1,3 @@ -use axum::response::{IntoResponse, Response}; -use axum::Json; use serde::{Deserialize, Serialize}; pub const DEFAULT_PAGE: u64 = 0; diff --git a/server/src/cli.rs b/server/src/cli.rs index be4e9f5..0725e64 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -18,8 +18,6 @@ pub struct Cli { pub pkg_dir: PathBuf, /// Directory where repository metadata & SQLite database is stored pub data_dir: PathBuf, - /// Default architecture to add packages with arch "any" to - pub default_arch: String, /// Database connection URL; either sqlite:// or postgres://. Defaults to rieter.sqlite in the /// data directory @@ -75,8 +73,7 @@ impl Cli { repo_dir: self.data_dir.join("repos"), pkg_dir: self.pkg_dir.clone(), }; - let repo_manager = - RepoGroupManager::new(&config.repo_dir, &self.pkg_dir, &self.default_arch); + let repo_manager = RepoGroupManager::new(&config.repo_dir, &self.pkg_dir); let global = Global { config, @@ -87,7 +84,7 @@ impl Cli { // build our application with a single route let app = Router::new() .nest("/api", crate::api::router()) - .merge(crate::repo::router(&global)) + .merge(crate::repo::router()) .with_state(global) .layer(TraceLayer::new_for_http()); diff --git a/server/src/error.rs b/server/src/error.rs index c11ad8d..4fbb7c4 100644 --- a/server/src/error.rs +++ b/server/src/error.rs @@ -19,8 +19,8 @@ impl fmt::Display for ServerError { match self { ServerError::IO(err) => write!(fmt, "{}", err), ServerError::Axum(err) => write!(fmt, "{}", err), - ServerError::Db(err) => write!(fmt, "{}", err), ServerError::Status(status) => write!(fmt, "{}", status), + ServerError::Db(err) => write!(fmt, "{}", err), } } } @@ -34,11 +34,11 @@ impl IntoResponse for ServerError { match self { ServerError::IO(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), ServerError::Axum(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + ServerError::Status(status) => status.into_response(), ServerError::Db(sea_orm::DbErr::RecordNotFound(_)) => { StatusCode::NOT_FOUND.into_response() } ServerError::Db(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), - ServerError::Status(status) => status.into_response(), } } } @@ -61,14 +61,14 @@ impl From for ServerError { } } +impl From for ServerError { + fn from(status: StatusCode) -> Self { + Self::Status(status) + } +} + impl From for ServerError { fn from(err: sea_orm::DbErr) -> Self { ServerError::Db(err) } } - -impl From for ServerError { - fn from(status: StatusCode) -> Self { - ServerError::Status(status) - } -} diff --git a/server/src/repo/manager.rs b/server/src/repo/manager.rs index d036aab..f5aad8c 100644 --- a/server/src/repo/manager.rs +++ b/server/src/repo/manager.rs @@ -1,28 +1,33 @@ use super::package::Package; use libarchive::write::{Builder, WriteEntry}; use libarchive::{Entry, WriteFilter, WriteFormat}; -use std::collections::HashSet; use std::fs; use std::io; use std::path::{Path, PathBuf}; +pub const ANY_ARCH: &str = "any"; + /// Overarching abstraction that orchestrates updating the repositories stored on the server pub struct RepoGroupManager { repo_dir: PathBuf, pkg_dir: PathBuf, - default_arch: String, +} + +fn parse_pkg_filename(file_name: &str) -> (String, &str, &str, &str) { + let name_parts = file_name.split('-').collect::>(); + let name = name_parts[..name_parts.len() - 3].join("-"); + let version = name_parts[name_parts.len() - 3]; + let release = name_parts[name_parts.len() - 2]; + let (arch, _) = name_parts[name_parts.len() - 1].split_once('.').unwrap(); + + (name, version, release, arch) } impl RepoGroupManager { - pub fn new, P2: AsRef>( - repo_dir: P1, - pkg_dir: P2, - default_arch: &str, - ) -> Self { + pub fn new, P2: AsRef>(repo_dir: P1, pkg_dir: P2) -> Self { RepoGroupManager { repo_dir: repo_dir.as_ref().to_path_buf(), pkg_dir: pkg_dir.as_ref().to_path_buf(), - default_arch: String::from(default_arch), } } @@ -37,11 +42,23 @@ impl RepoGroupManager { ar_files.add_filter(WriteFilter::Gzip)?; ar_files.set_format(WriteFormat::PaxRestricted)?; - let mut ar_db = ar_db.open_file(subrepo_path.join(format!("{}.tar.gz", repo)))?; + let mut ar_db = ar_db.open_file(subrepo_path.join(format!("{}.db.tar.gz", repo)))?; let mut ar_files = ar_files.open_file(subrepo_path.join(format!("{}.files.tar.gz", repo)))?; - for entry in subrepo_path.read_dir()? { + // All architectures should also include the "any" architecture, except for the "any" + // architecture itself. + let repo_any_dir = self.repo_dir.join(repo).join(ANY_ARCH); + + let any_entries_iter = if arch != ANY_ARCH && repo_any_dir.try_exists()? { + Some(repo_any_dir.read_dir()?) + } else { + None + } + .into_iter() + .flatten(); + + for entry in subrepo_path.read_dir()?.chain(any_entries_iter) { let entry = entry?; if entry.file_type()?.is_dir() { @@ -68,7 +85,23 @@ impl RepoGroupManager { } } - ar_db.close().and(ar_files.close()).map_err(Into::into) + ar_db.close()?; + ar_files.close()?; + + Ok(()) + } + + /// Synchronize all present architectures' db archives in the given repository. + pub fn sync_all(&mut self, repo: &str) -> io::Result<()> { + for entry in self.repo_dir.join(repo).read_dir()? { + let entry = entry?; + + if entry.file_type()?.is_dir() { + self.sync(repo, &entry.file_name().to_string_lossy())?; + } + } + + Ok(()) } pub fn add_pkg_from_path>( @@ -78,118 +111,97 @@ impl RepoGroupManager { ) -> io::Result { let pkg = Package::open(&path)?; - let archs = self.add_pkg_in_repo(repo, &pkg)?; + self.add_pkg(repo, &pkg)?; - // We add the package to each architecture it was added to by hard-linking the provided - // package file. This prevents storing a package of type "any" multiple times on disk. - for arch in archs { - let arch_repo_pkg_path = self.pkg_dir.join(repo).join(arch); - let dest_pkg_path = arch_repo_pkg_path.join(pkg.file_name()); + // After successfully adding the package, we move it to the packages directory + let dest_pkg_path = self + .pkg_dir + .join(repo) + .join(&pkg.info.arch) + .join(pkg.file_name()); - fs::create_dir_all(&arch_repo_pkg_path)?; - fs::hard_link(&path, dest_pkg_path)?; - } - - fs::remove_file(path)?; + fs::create_dir_all(dest_pkg_path.parent().unwrap())?; + fs::rename(&path, dest_pkg_path)?; Ok(pkg) } /// Add a package to the given repo, returning to what architectures the package was added. - pub fn add_pkg_in_repo(&mut self, repo: &str, pkg: &Package) -> io::Result> { - let mut arch_repos: HashSet = HashSet::new(); + pub fn add_pkg(&mut self, repo: &str, pkg: &Package) -> io::Result<()> { + // We first remove any existing version of the package + self.remove_pkg(repo, &pkg.info.arch, &pkg.info.name, false)?; - if pkg.info.arch != "any" { - self.add_pkg_in_arch_repo(repo, &pkg.info.arch, pkg)?; - arch_repos.insert(pkg.info.arch.clone()); - } - // Packages of arch "any" are added to every existing arch - else { - arch_repos.insert(self.default_arch.clone()); - - let repo_dir = self.repo_dir.join(repo); - - if repo_dir.exists() { - for entry in repo_dir.read_dir()? { - arch_repos.insert(entry?.file_name().to_string_lossy().to_string()); - } - } - - for arch in arch_repos.iter() { - self.add_pkg_in_arch_repo(repo, arch, pkg)?; - } - } - - Ok(arch_repos) - } - - pub fn add_pkg_in_arch_repo( - &mut self, - repo: &str, - arch: &str, - pkg: &Package, - ) -> io::Result<()> { - let pkg_dir = self + // Write the `desc` and `files` metadata files to disk + let metadata_dir = self .repo_dir .join(repo) - .join(arch) + .join(&pkg.info.arch) .join(format!("{}-{}", pkg.info.name, pkg.info.version)); - // We first remove the previous version of the package, if present - self.remove_pkg_from_arch_repo(repo, arch, &pkg.info.name, false)?; + fs::create_dir_all(&metadata_dir)?; - fs::create_dir_all(&pkg_dir)?; - - let mut desc_file = fs::File::create(pkg_dir.join("desc"))?; + let mut desc_file = fs::File::create(metadata_dir.join("desc"))?; pkg.write_desc(&mut desc_file)?; - let mut files_file = fs::File::create(pkg_dir.join("files"))?; + let mut files_file = fs::File::create(metadata_dir.join("files"))?; pkg.write_files(&mut files_file)?; - self.sync(repo, arch) + // If a package of type "any" is added, we need to update every existing database + if pkg.info.arch == ANY_ARCH { + self.sync_all(repo)?; + } else { + self.sync(repo, &pkg.info.arch)?; + } + + Ok(()) } pub fn remove_repo(&mut self, repo: &str) -> io::Result { - let repo_dir = self.repo_dir.join(&repo); + let repo_dir = self.repo_dir.join(repo); if !repo_dir.exists() { Ok(false) } else { - fs::remove_dir_all(&repo_dir) - .and_then(|_| fs::remove_dir_all(self.pkg_dir.join(repo)))?; + fs::remove_dir_all(&repo_dir)?; + fs::remove_dir_all(self.pkg_dir.join(repo))?; Ok(true) } } - pub fn remove_arch_repo(&mut self, repo: &str, arch: &str) -> io::Result { + pub fn remove_repo_arch(&mut self, repo: &str, arch: &str) -> io::Result { let sub_path = PathBuf::from(repo).join(arch); let repo_dir = self.repo_dir.join(&sub_path); if !repo_dir.exists() { - Ok(false) - } else { - fs::remove_dir_all(&repo_dir) - .and_then(|_| fs::remove_dir_all(self.pkg_dir.join(sub_path)))?; - - Ok(true) + return Ok(false); } + + fs::remove_dir_all(&repo_dir)?; + fs::remove_dir_all(self.pkg_dir.join(sub_path))?; + + // Removing the "any" architecture updates all other repositories + if arch == ANY_ARCH { + self.sync_all(repo)?; + } + + Ok(true) } - pub fn remove_pkg_from_arch_repo( + pub fn remove_pkg( &mut self, repo: &str, arch: &str, pkg_name: &str, sync: bool, ) -> io::Result { - let arch_repo_dir = self.repo_dir.join(repo).join(arch); + let repo_arch_dir = self.repo_dir.join(repo).join(arch); - if !arch_repo_dir.exists() { + if !repo_arch_dir.exists() { return Ok(false); } - for entry in arch_repo_dir.read_dir()? { + for entry in repo_arch_dir.read_dir()? { let entry = entry?; // Make sure we skip the archive files @@ -209,16 +221,13 @@ impl RepoGroupManager { fs::remove_dir_all(entry.path())?; // Also remove the old package archive - let arch_repo_pkg_dir = self.pkg_dir.join(repo).join(arch); + let repo_arch_pkg_dir = self.pkg_dir.join(repo).join(arch); - arch_repo_pkg_dir.read_dir()?.try_for_each(|res| { + repo_arch_pkg_dir.read_dir()?.try_for_each(|res| { res.and_then(|entry: fs::DirEntry| { let file_name = entry.file_name(); let file_name = file_name.to_string_lossy(); - - // Same trick, but for package files, we also need to trim the arch - let name_parts = file_name.split('-').collect::>(); - let name = name_parts[..name_parts.len() - 3].join("-"); + let (name, _, _, _) = parse_pkg_filename(&file_name); if name == pkg_name { fs::remove_file(entry.path()) @@ -229,7 +238,11 @@ impl RepoGroupManager { })?; if sync { - self.sync(repo, arch)?; + if arch == ANY_ARCH { + self.sync_all(repo)?; + } else { + self.sync(repo, arch)?; + } } return Ok(true); diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index e4be526..0a28969 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -4,30 +4,29 @@ mod package; pub use manager::RepoGroupManager; use crate::db::entities::{package as db_package, repo as db_repo}; +use axum::body::Body; use axum::extract::{BodyStream, Path, State}; +use axum::http::Request; use axum::http::StatusCode; -use axum::routing::{delete, get_service, post}; +use axum::response::IntoResponse; +use axum::routing::{delete, post}; use axum::Router; use futures::StreamExt; use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter}; use std::sync::Arc; use tokio::{fs, io::AsyncWriteExt}; -use tower_http::services::ServeDir; +use tower::util::ServiceExt; +use tower_http::services::{ServeDir, ServeFile}; use uuid::Uuid; -pub fn router(global: &crate::Global) -> Router { - // Try to serve packages by default, and try the database files instead if not found - let serve_repos = get_service( - ServeDir::new(&global.config.pkg_dir).fallback(ServeDir::new(&global.config.repo_dir)), - ); +pub fn router() -> Router { Router::new() .route("/:repo", post(post_package_archive).delete(delete_repo)) .route("/:repo/:arch", delete(delete_arch_repo)) .route( "/:repo/:arch/:filename", - delete(delete_package).get(serve_repos.clone()), + delete(delete_package).get(get_file), ) - .fallback(serve_repos) } async fn post_package_archive( @@ -45,40 +44,104 @@ async fn post_package_archive( } let clone = Arc::clone(&global.repo_manager); - + let path_clone = path.clone(); let repo_clone = repo.clone(); - let pkg = tokio::task::spawn_blocking(move || { - clone.write().unwrap().add_pkg_from_path(&repo_clone, &path) + let res = tokio::task::spawn_blocking(move || { + clone + .write() + .unwrap() + .add_pkg_from_path(&repo_clone, &path_clone) }) - .await??; + .await?; - // Query the repo for its ID, or create it if it does not already exist - let repo_entity = db_repo::Entity::find() - .filter(db_repo::Column::Name.eq(&repo)) - .one(&global.db) - .await?; + match res { + // Insert the newly added package into the database + Ok(pkg) => { + // Query the repo for its ID, or create it if it does not already exist + let repo_entity = db_repo::Entity::find() + .filter(db_repo::Column::Name.eq(&repo)) + .one(&global.db) + .await?; - let repo_id = if let Some(repo_entity) = repo_entity { - repo_entity.id - } else { - let model = db_repo::ActiveModel { - name: sea_orm::Set(repo.clone()), - ..Default::default() + let repo_id = if let Some(repo_entity) = repo_entity { + repo_entity.id + } else { + let model = db_repo::ActiveModel { + name: sea_orm::Set(repo.clone()), + ..Default::default() + }; + + db_repo::Entity::insert(model) + .exec(&global.db) + .await? + .last_insert_id + }; + + // Insert the package's data into the database + let mut model: db_package::ActiveModel = pkg.into(); + model.repo_id = sea_orm::Set(repo_id); + + model.insert(&global.db).await?; + + Ok(()) + } + // Remove the uploaded file and return the error + Err(err) => { + tokio::fs::remove_file(path).await?; + + Err(err.into()) + } + } +} + +/// Serve the package archive files and database archives. If files are requested for an +/// architecture that does not have any explicit packages, a repository containing only "any" files +/// is returned. +async fn get_file( + State(global): State, + Path((repo, arch, mut file_name)): Path<(String, String, String)>, + req: Request, +) -> crate::Result { + let repo_dir = global.config.repo_dir.join(&repo).join(&arch); + let repo_exists = tokio::fs::try_exists(&repo_dir).await?; + + let res = if file_name.ends_with(".db") || file_name.ends_with(".db.tar.gz") { + // Append tar extension to ensure we find the file + if file_name.ends_with(".db") { + file_name.push_str(".tar.gz"); }; - db_repo::Entity::insert(model) - .exec(&global.db) - .await? - .last_insert_id + if repo_exists { + ServeFile::new(repo_dir.join(file_name)).oneshot(req).await + } else { + let path = global + .config + .repo_dir + .join(repo) + .join(manager::ANY_ARCH) + .join(file_name); + + ServeFile::new(path).oneshot(req).await + } + } else { + let any_file = global + .config + .pkg_dir + .join(repo) + .join(manager::ANY_ARCH) + .join(file_name); + + if repo_exists { + ServeDir::new(global.config.pkg_dir) + .fallback(ServeFile::new(any_file)) + .oneshot(req) + .await + } else { + ServeFile::new(any_file).oneshot(req).await + } }; - // Insert the package's data into the database - let mut model: db_package::ActiveModel = pkg.into(); - model.repo_id = sea_orm::Set(repo_id); - - model.insert(&global.db).await?; - - Ok(()) + Ok(res) } async fn delete_repo( @@ -104,7 +167,7 @@ async fn delete_arch_repo( let clone = Arc::clone(&global.repo_manager); let repo_removed = - tokio::task::spawn_blocking(move || clone.write().unwrap().remove_arch_repo(&repo, &arch)) + tokio::task::spawn_blocking(move || clone.write().unwrap().remove_repo_arch(&repo, &arch)) .await??; if repo_removed { @@ -131,10 +194,7 @@ async fn delete_package( let clone = Arc::clone(&global.repo_manager); let pkg_removed = tokio::task::spawn_blocking(move || { - clone - .write() - .unwrap() - .remove_pkg_from_arch_repo(&repo, &arch, &name, true) + clone.write().unwrap().remove_pkg(&repo, &arch, &name, true) }) .await??;