feat: simplified repo structure; fully implemented repo db archives

Cargo.lock

@@ -1660,6 +1660,7 @@ dependencies = [
  "futures",
  "http-body-util",
  "libarchive",
+ "regex",
  "sea-orm",
  "sea-orm-migration",
  "serde",

server/Cargo.toml

@@ -13,6 +13,7 @@ clap = { version = "4.3.12", features = ["env", "derive"] }
 futures = "0.3.28"
 http-body-util = "0.1.1"
 libarchive = { path = "../libarchive" }
+regex = "1.10.4"
 sea-orm-migration = "0.12.1"
 serde = { version = "1.0.178", features = ["derive"] }
 sha256 = "1.1.4"

server/src/cli.rs

@@ -16,9 +16,6 @@ use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
 #[derive(Parser)]
 #[command(author, version, about, long_about = None)]
 pub struct Cli {
-    /// Directory where package archives will be stored
-    #[arg(env = "RIETER_PKG_DIR")]
-    pub pkg_dir: PathBuf,
     /// Directory where repository metadata & SQLite database is stored
     #[arg(env = "RIETER_DATA_DIR")]
     pub data_dir: PathBuf,
@@ -85,16 +82,17 @@ impl Cli {
 
         let config = Config {
             data_dir: self.data_dir.clone(),
-            repo_dir: self.data_dir.join("repos"),
-            pkg_dir: self.pkg_dir.clone(),
             api_key: self.api_key.clone(),
         };
-        let repo_manager = MetaRepoMgr::new(&config.repo_dir, &self.pkg_dir);
+        let repo_manager = MetaRepoMgr::new(&self.data_dir.join("repos"));
+
+        let pkg_filename_re = regex::Regex::new(r"^([a-z0-9@_+][a-z0-9@_+-.]*)-((?:[0-9]+:)?[a-z0-9._]+-[0-9.]+)-([a-z0-9_]+)\.pkg\.tar\.([a-z0-9]+)$").unwrap();
 
         let global = Global {
             config,
             repo_manager: Arc::new(RwLock::new(repo_manager)),
             db,
+            pkg_filename_re,
         };
 
         // build our application with a single route

server/src/db/entities/package.rs

@@ -1,5 +1,6 @@
 //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1
 
+use chrono::NaiveDateTime;
 use sea_orm::entity::prelude::*;
 use serde::{Deserialize, Serialize};
 
@@ -17,7 +18,7 @@ pub struct Model {
     pub c_size: i64,
     pub description: Option<String>,
     pub url: Option<String>,
-    pub build_date: String,
+    pub build_date: NaiveDateTime,
     pub packager: Option<String>,
     pub pgp_sig: Option<String>,
     pub pgp_sig_size: Option<i64>,

server/src/db/query/package.rs

@@ -47,13 +47,23 @@ pub async fn by_fields(
     repo_id: i32,
     arch: &str,
     name: &str,
+    version: Option<&str>,
+    compression: Option<&str>,
 ) -> Result<Option<package::Model>> {
-    Package::find()
+    let mut query = Package::find()
         .filter(package::Column::RepoId.eq(repo_id))
         .filter(package::Column::Name.eq(name))
-        .filter(package::Column::Arch.eq(arch))
-        .one(conn)
-        .await
+        .filter(package::Column::Arch.eq(arch));
+
+    if let Some(version) = version {
+        query = query.filter(package::Column::Version.eq(version));
+    }
+
+    if let Some(compression) = compression {
+        query = query.filter(package::Column::Compression.eq(compression));
+    }
+
+    query.one(conn).await
 }
 
 pub async fn delete_with_arch(conn: &DbConn, repo_id: i32, arch: &str) -> Result<DeleteResult> {
@@ -78,7 +88,7 @@ pub async fn insert(conn: &DbConn, repo_id: i32, pkg: crate::repo::package::Pack
         c_size: Set(info.csize),
         description: Set(info.description),
         url: Set(info.url),
-        build_date: Set(info.build_date.to_string()),
+        build_date: Set(info.build_date),
         packager: Set(info.packager),
         pgp_sig: Set(info.pgpsig),
         pgp_sig_size: Set(info.pgpsigsize),

server/src/main.rs

@@ -15,8 +15,6 @@ use tokio::sync::RwLock;
 #[derive(Clone)]
 pub struct Config {
     data_dir: PathBuf,
-    repo_dir: PathBuf,
-    pkg_dir: PathBuf,
     api_key: String,
 }
 
@@ -25,6 +23,7 @@ pub struct Global {
     config: Config,
     repo_manager: Arc<RwLock<MetaRepoMgr>>,
     db: sea_orm::DbConn,
+    pkg_filename_re: regex::Regex,
 }
 
 #[tokio::main]

server/src/repo/manager_new.rs

@@ -5,7 +5,6 @@ use uuid::Uuid;
 
 use futures::StreamExt;
 use tokio::io::AsyncRead;
-use tokio::io::AsyncWriteExt;
 
 use super::archive;
 use super::package;
@@ -16,14 +15,12 @@ pub const ANY_ARCH: &'static str = "any";
 
 pub struct MetaRepoMgr {
     repo_dir: PathBuf,
-    pkg_dir: PathBuf,
 }
 
 impl MetaRepoMgr {
-    pub fn new<P1: AsRef<Path>, P2: AsRef<Path>>(repo_dir: P1, pkg_dir: P2) -> Self {
+    pub fn new<P: AsRef<Path>>(repo_dir: P) -> Self {
         MetaRepoMgr {
             repo_dir: repo_dir.as_ref().to_path_buf(),
-            pkg_dir: pkg_dir.as_ref().to_path_buf(),
         }
     }
 
@@ -64,11 +61,14 @@ impl MetaRepoMgr {
 
         let repo = repo.unwrap();
 
-        let parent_dir = self.repo_dir.join(&repo.name).join(arch);
+        let parent_dir = self.repo_dir.join(&repo.name);
         tokio::fs::create_dir_all(&parent_dir).await?;
 
+        let ar_db =
+            archive::RepoArchiveWriter::open(parent_dir.join(format!("{}.db.tar.gz", arch)))
+                .await?;
         let ar_files =
-            archive::RepoArchiveWriter::open(parent_dir.join(format!("{}.db.tar.gz", repo.name)))
+            archive::RepoArchiveWriter::open(parent_dir.join(format!("{}.files.tar.gz", arch)))
                 .await?;
 
         // Query all packages in the repo that have the given architecture or the "any"
@@ -79,11 +79,12 @@ impl MetaRepoMgr {
             .stream(conn)
             .await?;
 
+        // Create two temp file paths to write our entries to
         let uuid: uuid::fmt::Simple = Uuid::new_v4().into();
-        let files_tmp_file_path = self.pkg_dir.join(uuid.to_string());
+        let files_tmp_file_path = self.repo_dir.join(uuid.to_string());
 
         let uuid: uuid::fmt::Simple = Uuid::new_v4().into();
-        let desc_tmp_file_path = self.pkg_dir.join(uuid.to_string());
+        let desc_tmp_file_path = self.repo_dir.join(uuid.to_string());
 
         while let Some(pkg) = pkgs.next().await.transpose()? {
             let mut files_tmp_file = tokio::fs::File::create(&files_tmp_file_path).await?;
@@ -93,6 +94,10 @@ impl MetaRepoMgr {
             package::write_desc(conn, &mut desc_tmp_file, &pkg).await?;
 
             let full_name = format!("{}-{}", pkg.name, pkg.version);
+
+            ar_db
+                .add_entry(&full_name, &desc_tmp_file_path, true)
+                .await?;
             ar_files
                 .add_entry(&full_name, &desc_tmp_file_path, true)
                 .await?;
@@ -101,7 +106,11 @@ impl MetaRepoMgr {
                 .await?;
         }
 
+        // Cleanup
+        ar_db.close().await?;
         ar_files.close().await?;
+
+        tokio::fs::remove_file(desc_tmp_file_path).await?;
         tokio::fs::remove_file(files_tmp_file_path).await?;
 
         Ok(())
@@ -117,7 +126,6 @@ impl MetaRepoMgr {
 
             // Remove files from file system
             tokio::fs::remove_dir_all(self.repo_dir.join(repo)).await?;
-            tokio::fs::remove_dir_all(self.pkg_dir.join(repo)).await?;
 
             Ok(true)
         } else {
@@ -135,7 +143,7 @@ impl MetaRepoMgr {
         let repo = db::query::repo::by_name(conn, repo).await?;
 
         if let Some(repo) = repo {
-            let pkg = db::query::package::by_fields(conn, repo.id, arch, name).await?;
+            let pkg = db::query::package::by_fields(conn, repo.id, arch, name, None, None).await?;
 
             if let Some(pkg) = pkg {
                 // Remove package from database
@@ -157,7 +165,7 @@ impl MetaRepoMgr {
     ) -> crate::Result<()> {
         // Copy file contents to temporary path so libarchive can work with it
         let uuid: uuid::fmt::Simple = Uuid::new_v4().into();
-        let path = self.pkg_dir.join(uuid.to_string());
+        let path = self.repo_dir.join(uuid.to_string());
         let mut temp_file = tokio::fs::File::create(&path).await?;
 
         tokio::io::copy(reader, &mut temp_file).await?;
@@ -180,18 +188,21 @@ impl MetaRepoMgr {
         };
 
         // If the package already exists in the database, we remove it first
-        let res =
-            db::query::package::by_fields(conn, repo_id, &pkg.info.arch, &pkg.info.name).await?;
+        let res = db::query::package::by_fields(
+            conn,
+            repo_id,
+            &pkg.info.arch,
+            &pkg.info.name,
+            None,
+            None,
+        )
+        .await?;
 
         if let Some(entry) = res {
             entry.delete(conn).await?;
         }
 
-        let dest_pkg_path = self
-            .pkg_dir
-            .join(repo)
-            .join(&pkg.info.arch)
-            .join(pkg.file_name());
+        let dest_pkg_path = self.repo_dir.join(repo).join(pkg.file_name());
 
         // Insert new package into database
         let arch = pkg.info.arch.clone();

server/src/repo/mod.rs

@@ -19,6 +19,7 @@ use axum::routing::{delete, post};
 use axum::Router;
 use futures::TryStreamExt;
 use futures::{Stream, StreamExt};
+use regex::Regex;
 use sea_orm::ModelTrait;
 use std::sync::Arc;
 use tokio::{fs, io::AsyncWriteExt};
@@ -56,52 +57,25 @@ pub fn router(api_key: &str) -> Router<crate::Global> {
 /// is returned.
 async fn get_file(
     State(global): State<crate::Global>,
-    Path((repo, arch, mut file_name)): Path<(String, String, String)>,
+    Path((repo, arch, file_name)): Path<(String, String, String)>,
     req: Request<Body>,
 ) -> crate::Result<impl IntoResponse> {
-    let repo_dir = global.config.repo_dir.join(&repo).join(&arch);
-    let repo_exists = tokio::fs::try_exists(&repo_dir).await?;
+    let repo_dir = global.config.data_dir.join("repos").join(&repo);
 
-    let res = if DB_FILE_EXTS.iter().any(|ext| file_name.ends_with(ext)) {
-        // Append tar extension to ensure we find the file
-        if !file_name.ends_with(".tar.gz") {
-            file_name.push_str(".tar.gz");
+    let file_name =
+        if file_name == format!("{}.db", repo) || file_name == format!("{}.db.tar.gz", repo) {
+            format!("{}.db.tar.gz", arch)
+        } else if file_name == format!("{}.files", repo)
+            || file_name == format!("{}.files.tar.gz", repo)
+        {
+            format!("{}.files.tar.gz", arch)
+        } else {
+            file_name
         };
 
-        if repo_exists {
-            ServeFile::new(repo_dir.join(file_name)).oneshot(req).await
-        } else {
-            let path = global
-                .config
-                .repo_dir
-                .join(repo)
-                .join(manager::ANY_ARCH)
-                .join(file_name);
-
-            ServeFile::new(path).oneshot(req).await
-        }
-    } else {
-        let any_file = global
-            .config
-            .pkg_dir
-            .join(repo)
-            .join(manager::ANY_ARCH)
-            .join(file_name);
-
-        if repo_exists {
-            ServeDir::new(global.config.pkg_dir)
-                .fallback(ServeFile::new(any_file))
-                .oneshot(req)
-                .await
-        } else {
-            ServeFile::new(any_file).oneshot(req).await
-        }
-    };
-
-    Ok(res)
+    Ok(ServeFile::new(repo_dir.join(file_name)).oneshot(req).await)
 }
 
-#[axum::debug_handler]
 async fn post_package_archive(
     State(global): State<crate::Global>,
     Path(repo): Path<String>,
@@ -122,7 +96,6 @@ async fn post_package_archive(
     //tracing::info!("Added '{}' to repository '{}'", pkg.file_name(), repo);
 }
 
-#[axum::debug_handler]
 async fn delete_repo(
     State(global): State<crate::Global>,
     Path(repo): Path<String>,
@@ -207,11 +180,3 @@ async fn delete_package(
     //    Ok(StatusCode::NOT_FOUND)
     //}
 }
-
-fn help_me_figure_this_out(_: impl Send) {}
-
-#[allow(dead_code)]
-fn assert_my_handler_is_ok() {
-    // This is the fun part: we need to call our handler, and *not* await it's Future.
-    help_me_figure_this_out(delete_repo)
-}

server/src/repo/package.rs

@@ -283,7 +283,7 @@ impl From<Package> for package::ActiveModel {
             c_size: Set(info.csize),
             description: Set(info.description),
             url: Set(info.url),
-            build_date: Set(info.build_date.to_string()),
+            build_date: Set(info.build_date),
             packager: Set(info.packager),
             pgp_sig: Set(info.pgpsig),
             pgp_sig_size: Set(info.pgpsigsize),
@@ -359,7 +359,12 @@ pub async fn write_desc<W: AsyncWrite + std::marker::Unpin>(
     write_attribute(writer, "ARCH", &pkg.arch).await?;
 
     // TODO build date
-    //write_attribute(writer, "BUILDDATE", &pkg.build_)
+    write_attribute(
+        writer,
+        "BUILDDATE",
+        &pkg.build_date.and_utc().timestamp().to_string(),
+    )
+    .await?;
 
     if let Some(ref packager) = pkg.packager {
         write_attribute(writer, "PACKAGER", packager).await?;