diff --git a/Cargo.lock b/Cargo.lock index 333bc72..d0c00b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1660,6 +1660,7 @@ dependencies = [ "futures", "http-body-util", "libarchive", + "regex", "sea-orm", "sea-orm-migration", "serde", diff --git a/server/Cargo.toml b/server/Cargo.toml index c2c5d74..1d5dc1d 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -13,6 +13,7 @@ clap = { version = "4.3.12", features = ["env", "derive"] } futures = "0.3.28" http-body-util = "0.1.1" libarchive = { path = "../libarchive" } +regex = "1.10.4" sea-orm-migration = "0.12.1" serde = { version = "1.0.178", features = ["derive"] } sha256 = "1.1.4" diff --git a/server/src/cli.rs b/server/src/cli.rs index 0c802f2..2b85492 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -16,9 +16,6 @@ use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; #[derive(Parser)] #[command(author, version, about, long_about = None)] pub struct Cli { - /// Directory where package archives will be stored - #[arg(env = "RIETER_PKG_DIR")] - pub pkg_dir: PathBuf, /// Directory where repository metadata & SQLite database is stored #[arg(env = "RIETER_DATA_DIR")] pub data_dir: PathBuf, @@ -85,16 +82,17 @@ impl Cli { let config = Config { data_dir: self.data_dir.clone(), - repo_dir: self.data_dir.join("repos"), - pkg_dir: self.pkg_dir.clone(), api_key: self.api_key.clone(), }; - let repo_manager = MetaRepoMgr::new(&config.repo_dir, &self.pkg_dir); + let repo_manager = MetaRepoMgr::new(&self.data_dir.join("repos")); + + let pkg_filename_re = regex::Regex::new(r"^([a-z0-9@_+][a-z0-9@_+-.]*)-((?:[0-9]+:)?[a-z0-9._]+-[0-9.]+)-([a-z0-9_]+)\.pkg\.tar\.([a-z0-9]+)$").unwrap(); let global = Global { config, repo_manager: Arc::new(RwLock::new(repo_manager)), db, + pkg_filename_re, }; // build our application with a single route diff --git a/server/src/db/entities/package.rs b/server/src/db/entities/package.rs index c09a310..112cde4 100644 --- a/server/src/db/entities/package.rs +++ b/server/src/db/entities/package.rs @@ -1,5 +1,6 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 +use chrono::NaiveDateTime; use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; @@ -17,7 +18,7 @@ pub struct Model { pub c_size: i64, pub description: Option, pub url: Option, - pub build_date: String, + pub build_date: NaiveDateTime, pub packager: Option, pub pgp_sig: Option, pub pgp_sig_size: Option, diff --git a/server/src/db/query/package.rs b/server/src/db/query/package.rs index ecd306e..31f3464 100644 --- a/server/src/db/query/package.rs +++ b/server/src/db/query/package.rs @@ -47,13 +47,23 @@ pub async fn by_fields( repo_id: i32, arch: &str, name: &str, + version: Option<&str>, + compression: Option<&str>, ) -> Result> { - Package::find() + let mut query = Package::find() .filter(package::Column::RepoId.eq(repo_id)) .filter(package::Column::Name.eq(name)) - .filter(package::Column::Arch.eq(arch)) - .one(conn) - .await + .filter(package::Column::Arch.eq(arch)); + + if let Some(version) = version { + query = query.filter(package::Column::Version.eq(version)); + } + + if let Some(compression) = compression { + query = query.filter(package::Column::Compression.eq(compression)); + } + + query.one(conn).await } pub async fn delete_with_arch(conn: &DbConn, repo_id: i32, arch: &str) -> Result { @@ -78,7 +88,7 @@ pub async fn insert(conn: &DbConn, repo_id: i32, pkg: crate::repo::package::Pack c_size: Set(info.csize), description: Set(info.description), url: Set(info.url), - build_date: Set(info.build_date.to_string()), + build_date: Set(info.build_date), packager: Set(info.packager), pgp_sig: Set(info.pgpsig), pgp_sig_size: Set(info.pgpsigsize), diff --git a/server/src/main.rs b/server/src/main.rs index 7038203..eea6dc3 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -15,8 +15,6 @@ use tokio::sync::RwLock; #[derive(Clone)] pub struct Config { data_dir: PathBuf, - repo_dir: PathBuf, - pkg_dir: PathBuf, api_key: String, } @@ -25,6 +23,7 @@ pub struct Global { config: Config, repo_manager: Arc>, db: sea_orm::DbConn, + pkg_filename_re: regex::Regex, } #[tokio::main] diff --git a/server/src/repo/manager_new.rs b/server/src/repo/manager_new.rs index 9728764..b61dd57 100644 --- a/server/src/repo/manager_new.rs +++ b/server/src/repo/manager_new.rs @@ -5,7 +5,6 @@ use uuid::Uuid; use futures::StreamExt; use tokio::io::AsyncRead; -use tokio::io::AsyncWriteExt; use super::archive; use super::package; @@ -16,14 +15,12 @@ pub const ANY_ARCH: &'static str = "any"; pub struct MetaRepoMgr { repo_dir: PathBuf, - pkg_dir: PathBuf, } impl MetaRepoMgr { - pub fn new, P2: AsRef>(repo_dir: P1, pkg_dir: P2) -> Self { + pub fn new>(repo_dir: P) -> Self { MetaRepoMgr { repo_dir: repo_dir.as_ref().to_path_buf(), - pkg_dir: pkg_dir.as_ref().to_path_buf(), } } @@ -64,11 +61,14 @@ impl MetaRepoMgr { let repo = repo.unwrap(); - let parent_dir = self.repo_dir.join(&repo.name).join(arch); + let parent_dir = self.repo_dir.join(&repo.name); tokio::fs::create_dir_all(&parent_dir).await?; + let ar_db = + archive::RepoArchiveWriter::open(parent_dir.join(format!("{}.db.tar.gz", arch))) + .await?; let ar_files = - archive::RepoArchiveWriter::open(parent_dir.join(format!("{}.db.tar.gz", repo.name))) + archive::RepoArchiveWriter::open(parent_dir.join(format!("{}.files.tar.gz", arch))) .await?; // Query all packages in the repo that have the given architecture or the "any" @@ -79,11 +79,12 @@ impl MetaRepoMgr { .stream(conn) .await?; + // Create two temp file paths to write our entries to let uuid: uuid::fmt::Simple = Uuid::new_v4().into(); - let files_tmp_file_path = self.pkg_dir.join(uuid.to_string()); + let files_tmp_file_path = self.repo_dir.join(uuid.to_string()); let uuid: uuid::fmt::Simple = Uuid::new_v4().into(); - let desc_tmp_file_path = self.pkg_dir.join(uuid.to_string()); + let desc_tmp_file_path = self.repo_dir.join(uuid.to_string()); while let Some(pkg) = pkgs.next().await.transpose()? { let mut files_tmp_file = tokio::fs::File::create(&files_tmp_file_path).await?; @@ -93,6 +94,10 @@ impl MetaRepoMgr { package::write_desc(conn, &mut desc_tmp_file, &pkg).await?; let full_name = format!("{}-{}", pkg.name, pkg.version); + + ar_db + .add_entry(&full_name, &desc_tmp_file_path, true) + .await?; ar_files .add_entry(&full_name, &desc_tmp_file_path, true) .await?; @@ -101,7 +106,11 @@ impl MetaRepoMgr { .await?; } + // Cleanup + ar_db.close().await?; ar_files.close().await?; + + tokio::fs::remove_file(desc_tmp_file_path).await?; tokio::fs::remove_file(files_tmp_file_path).await?; Ok(()) @@ -117,7 +126,6 @@ impl MetaRepoMgr { // Remove files from file system tokio::fs::remove_dir_all(self.repo_dir.join(repo)).await?; - tokio::fs::remove_dir_all(self.pkg_dir.join(repo)).await?; Ok(true) } else { @@ -135,7 +143,7 @@ impl MetaRepoMgr { let repo = db::query::repo::by_name(conn, repo).await?; if let Some(repo) = repo { - let pkg = db::query::package::by_fields(conn, repo.id, arch, name).await?; + let pkg = db::query::package::by_fields(conn, repo.id, arch, name, None, None).await?; if let Some(pkg) = pkg { // Remove package from database @@ -157,7 +165,7 @@ impl MetaRepoMgr { ) -> crate::Result<()> { // Copy file contents to temporary path so libarchive can work with it let uuid: uuid::fmt::Simple = Uuid::new_v4().into(); - let path = self.pkg_dir.join(uuid.to_string()); + let path = self.repo_dir.join(uuid.to_string()); let mut temp_file = tokio::fs::File::create(&path).await?; tokio::io::copy(reader, &mut temp_file).await?; @@ -180,18 +188,21 @@ impl MetaRepoMgr { }; // If the package already exists in the database, we remove it first - let res = - db::query::package::by_fields(conn, repo_id, &pkg.info.arch, &pkg.info.name).await?; + let res = db::query::package::by_fields( + conn, + repo_id, + &pkg.info.arch, + &pkg.info.name, + None, + None, + ) + .await?; if let Some(entry) = res { entry.delete(conn).await?; } - let dest_pkg_path = self - .pkg_dir - .join(repo) - .join(&pkg.info.arch) - .join(pkg.file_name()); + let dest_pkg_path = self.repo_dir.join(repo).join(pkg.file_name()); // Insert new package into database let arch = pkg.info.arch.clone(); diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index 1992e22..a50f4fd 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -19,6 +19,7 @@ use axum::routing::{delete, post}; use axum::Router; use futures::TryStreamExt; use futures::{Stream, StreamExt}; +use regex::Regex; use sea_orm::ModelTrait; use std::sync::Arc; use tokio::{fs, io::AsyncWriteExt}; @@ -56,52 +57,25 @@ pub fn router(api_key: &str) -> Router { /// is returned. async fn get_file( State(global): State, - Path((repo, arch, mut file_name)): Path<(String, String, String)>, + Path((repo, arch, file_name)): Path<(String, String, String)>, req: Request, ) -> crate::Result { - let repo_dir = global.config.repo_dir.join(&repo).join(&arch); - let repo_exists = tokio::fs::try_exists(&repo_dir).await?; + let repo_dir = global.config.data_dir.join("repos").join(&repo); - let res = if DB_FILE_EXTS.iter().any(|ext| file_name.ends_with(ext)) { - // Append tar extension to ensure we find the file - if !file_name.ends_with(".tar.gz") { - file_name.push_str(".tar.gz"); + let file_name = + if file_name == format!("{}.db", repo) || file_name == format!("{}.db.tar.gz", repo) { + format!("{}.db.tar.gz", arch) + } else if file_name == format!("{}.files", repo) + || file_name == format!("{}.files.tar.gz", repo) + { + format!("{}.files.tar.gz", arch) + } else { + file_name }; - if repo_exists { - ServeFile::new(repo_dir.join(file_name)).oneshot(req).await - } else { - let path = global - .config - .repo_dir - .join(repo) - .join(manager::ANY_ARCH) - .join(file_name); - - ServeFile::new(path).oneshot(req).await - } - } else { - let any_file = global - .config - .pkg_dir - .join(repo) - .join(manager::ANY_ARCH) - .join(file_name); - - if repo_exists { - ServeDir::new(global.config.pkg_dir) - .fallback(ServeFile::new(any_file)) - .oneshot(req) - .await - } else { - ServeFile::new(any_file).oneshot(req).await - } - }; - - Ok(res) + Ok(ServeFile::new(repo_dir.join(file_name)).oneshot(req).await) } -#[axum::debug_handler] async fn post_package_archive( State(global): State, Path(repo): Path, @@ -122,7 +96,6 @@ async fn post_package_archive( //tracing::info!("Added '{}' to repository '{}'", pkg.file_name(), repo); } -#[axum::debug_handler] async fn delete_repo( State(global): State, Path(repo): Path, @@ -207,11 +180,3 @@ async fn delete_package( // Ok(StatusCode::NOT_FOUND) //} } - -fn help_me_figure_this_out(_: impl Send) {} - -#[allow(dead_code)] -fn assert_my_handler_is_ok() { - // This is the fun part: we need to call our handler, and *not* await it's Future. - help_me_figure_this_out(delete_repo) -} diff --git a/server/src/repo/package.rs b/server/src/repo/package.rs index 7c9a778..980cbb6 100644 --- a/server/src/repo/package.rs +++ b/server/src/repo/package.rs @@ -283,7 +283,7 @@ impl From for package::ActiveModel { c_size: Set(info.csize), description: Set(info.description), url: Set(info.url), - build_date: Set(info.build_date.to_string()), + build_date: Set(info.build_date), packager: Set(info.packager), pgp_sig: Set(info.pgpsig), pgp_sig_size: Set(info.pgpsigsize), @@ -359,7 +359,12 @@ pub async fn write_desc( write_attribute(writer, "ARCH", &pkg.arch).await?; // TODO build date - //write_attribute(writer, "BUILDDATE", &pkg.build_) + write_attribute( + writer, + "BUILDDATE", + &pkg.build_date.and_utc().timestamp().to_string(), + ) + .await?; if let Some(ref packager) = pkg.packager { write_attribute(writer, "PACKAGER", packager).await?;