feat: simplified repo structure; fully implemented repo db archives

server/src/cli.rs

@@ -16,9 +16,6 @@ use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
 #[derive(Parser)]
 #[command(author, version, about, long_about = None)]
 pub struct Cli {
-    /// Directory where package archives will be stored
-    #[arg(env = "RIETER_PKG_DIR")]
-    pub pkg_dir: PathBuf,
     /// Directory where repository metadata & SQLite database is stored
     #[arg(env = "RIETER_DATA_DIR")]
     pub data_dir: PathBuf,
@@ -85,11 +82,9 @@ impl Cli {
 
         let config = Config {
             data_dir: self.data_dir.clone(),
-            repo_dir: self.data_dir.join("repos"),
-            pkg_dir: self.pkg_dir.clone(),
             api_key: self.api_key.clone(),
         };
-        let repo_manager = MetaRepoMgr::new(&config.repo_dir, &self.pkg_dir);
+        let repo_manager = MetaRepoMgr::new(&self.data_dir.join("repos"));
 
         let pkg_filename_re = regex::Regex::new(r"^([a-z0-9@_+][a-z0-9@_+-.]*)-((?:[0-9]+:)?[a-z0-9._]+-[0-9.]+)-([a-z0-9_]+)\.pkg\.tar\.([a-z0-9]+)$").unwrap();
 

server/src/db/entities/package.rs

@@ -1,5 +1,6 @@
 //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1
 
+use chrono::NaiveDateTime;
 use sea_orm::entity::prelude::*;
 use serde::{Deserialize, Serialize};
 
@@ -17,7 +18,7 @@ pub struct Model {
     pub c_size: i64,
     pub description: Option<String>,
     pub url: Option<String>,
-    pub build_date: String,
+    pub build_date: NaiveDateTime,
     pub packager: Option<String>,
     pub pgp_sig: Option<String>,
     pub pgp_sig_size: Option<i64>,

server/src/db/query/package.rs

@@ -88,7 +88,7 @@ pub async fn insert(conn: &DbConn, repo_id: i32, pkg: crate::repo::package::Pack
         c_size: Set(info.csize),
         description: Set(info.description),
         url: Set(info.url),
-        build_date: Set(info.build_date.to_string()),
+        build_date: Set(info.build_date),
         packager: Set(info.packager),
         pgp_sig: Set(info.pgpsig),
         pgp_sig_size: Set(info.pgpsigsize),

server/src/main.rs

@@ -15,8 +15,6 @@ use tokio::sync::RwLock;
 #[derive(Clone)]
 pub struct Config {
     data_dir: PathBuf,
-    repo_dir: PathBuf,
-    pkg_dir: PathBuf,
     api_key: String,
 }
 

server/src/repo/manager_new.rs

@@ -5,7 +5,6 @@ use uuid::Uuid;
 
 use futures::StreamExt;
 use tokio::io::AsyncRead;
-use tokio::io::AsyncWriteExt;
 
 use super::archive;
 use super::package;
@@ -16,14 +15,12 @@ pub const ANY_ARCH: &'static str = "any";
 
 pub struct MetaRepoMgr {
     repo_dir: PathBuf,
-    pkg_dir: PathBuf,
 }
 
 impl MetaRepoMgr {
-    pub fn new<P1: AsRef<Path>, P2: AsRef<Path>>(repo_dir: P1, pkg_dir: P2) -> Self {
+    pub fn new<P: AsRef<Path>>(repo_dir: P) -> Self {
         MetaRepoMgr {
             repo_dir: repo_dir.as_ref().to_path_buf(),
-            pkg_dir: pkg_dir.as_ref().to_path_buf(),
         }
     }
 
@@ -64,11 +61,14 @@ impl MetaRepoMgr {
 
         let repo = repo.unwrap();
 
-        let parent_dir = self.repo_dir.join(&repo.name).join(arch);
+        let parent_dir = self.repo_dir.join(&repo.name);
         tokio::fs::create_dir_all(&parent_dir).await?;
 
+        let ar_db =
+            archive::RepoArchiveWriter::open(parent_dir.join(format!("{}.db.tar.gz", arch)))
+                .await?;
         let ar_files =
-            archive::RepoArchiveWriter::open(parent_dir.join(format!("{}.db.tar.gz", repo.name)))
+            archive::RepoArchiveWriter::open(parent_dir.join(format!("{}.files.tar.gz", arch)))
                 .await?;
 
         // Query all packages in the repo that have the given architecture or the "any"
@@ -79,11 +79,12 @@ impl MetaRepoMgr {
             .stream(conn)
             .await?;
 
+        // Create two temp file paths to write our entries to
         let uuid: uuid::fmt::Simple = Uuid::new_v4().into();
-        let files_tmp_file_path = self.pkg_dir.join(uuid.to_string());
+        let files_tmp_file_path = self.repo_dir.join(uuid.to_string());
 
         let uuid: uuid::fmt::Simple = Uuid::new_v4().into();
-        let desc_tmp_file_path = self.pkg_dir.join(uuid.to_string());
+        let desc_tmp_file_path = self.repo_dir.join(uuid.to_string());
 
         while let Some(pkg) = pkgs.next().await.transpose()? {
             let mut files_tmp_file = tokio::fs::File::create(&files_tmp_file_path).await?;
@@ -93,6 +94,10 @@ impl MetaRepoMgr {
             package::write_desc(conn, &mut desc_tmp_file, &pkg).await?;
 
             let full_name = format!("{}-{}", pkg.name, pkg.version);
+
+            ar_db
+                .add_entry(&full_name, &desc_tmp_file_path, true)
+                .await?;
             ar_files
                 .add_entry(&full_name, &desc_tmp_file_path, true)
                 .await?;
@@ -101,7 +106,11 @@ impl MetaRepoMgr {
                 .await?;
         }
 
+        // Cleanup
+        ar_db.close().await?;
         ar_files.close().await?;
+
+        tokio::fs::remove_file(desc_tmp_file_path).await?;
         tokio::fs::remove_file(files_tmp_file_path).await?;
 
         Ok(())
@@ -117,7 +126,6 @@ impl MetaRepoMgr {
 
             // Remove files from file system
             tokio::fs::remove_dir_all(self.repo_dir.join(repo)).await?;
-            tokio::fs::remove_dir_all(self.pkg_dir.join(repo)).await?;
 
             Ok(true)
         } else {
@@ -157,7 +165,7 @@ impl MetaRepoMgr {
     ) -> crate::Result<()> {
         // Copy file contents to temporary path so libarchive can work with it
         let uuid: uuid::fmt::Simple = Uuid::new_v4().into();
-        let path = self.pkg_dir.join(uuid.to_string());
+        let path = self.repo_dir.join(uuid.to_string());
         let mut temp_file = tokio::fs::File::create(&path).await?;
 
         tokio::io::copy(reader, &mut temp_file).await?;
@@ -194,11 +202,7 @@ impl MetaRepoMgr {
             entry.delete(conn).await?;
         }
 
-        let dest_pkg_path = self
+        let dest_pkg_path = self.repo_dir.join(repo).join(pkg.file_name());
-            .pkg_dir
-            .join(repo)
-            .join(&pkg.info.arch)
-            .join(pkg.file_name());
 
         // Insert new package into database
         let arch = pkg.info.arch.clone();

server/src/repo/mod.rs

@@ -57,41 +57,23 @@ pub fn router(api_key: &str) -> Router<crate::Global> {
 /// is returned.
 async fn get_file(
     State(global): State<crate::Global>,
-    Path((repo, _arch, file_name)): Path<(String, String, String)>,
+    Path((repo, arch, file_name)): Path<(String, String, String)>,
     req: Request<Body>,
 ) -> crate::Result<impl IntoResponse> {
-    // Query the repo to see if it exists
+    let repo_dir = global.config.data_dir.join("repos").join(&repo);
-    let repo = db::query::repo::by_name(&global.db, &repo)
-        .await?
-        .ok_or(StatusCode::NOT_FOUND)?;
 
-    // Match the filename
+    let file_name =
-    let caps = global
+        if file_name == format!("{}.db", repo) || file_name == format!("{}.db.tar.gz", repo) {
-        .pkg_filename_re
+            format!("{}.db.tar.gz", arch)
-        .captures(&file_name)
+        } else if file_name == format!("{}.files", repo)
-        .ok_or(StatusCode::NOT_FOUND)?;
+            || file_name == format!("{}.files.tar.gz", repo)
-    let arch = caps.get(3).unwrap().as_str();
+        {
+            format!("{}.files.tar.gz", arch)
+        } else {
+            file_name
+        };
 
-    // Query the package to see if it exists
+    Ok(ServeFile::new(repo_dir.join(file_name)).oneshot(req).await)
-    let pkg = db::query::package::by_fields(
-        &global.db,
-        repo.id,
-        arch,
-        caps.get(1).unwrap().as_str(),
-        Some(caps.get(2).unwrap().as_str()),
-        Some(caps.get(4).unwrap().as_str()),
-    )
-    .await?
-    .ok_or(StatusCode::NOT_FOUND)?;
-
-    // Serve the file if it idoes
-    let pkg_path = global
-        .config
-        .pkg_dir
-        .join(repo.name)
-        .join(arch)
-        .join(package::filename(&pkg));
-    Ok(ServeFile::new(pkg_path).oneshot(req).await.unwrap())
 }
 
 async fn post_package_archive(

server/src/repo/package.rs

@@ -283,7 +283,7 @@ impl From<Package> for package::ActiveModel {
             c_size: Set(info.csize),
             description: Set(info.description),
             url: Set(info.url),
-            build_date: Set(info.build_date.to_string()),
+            build_date: Set(info.build_date),
             packager: Set(info.packager),
             pgp_sig: Set(info.pgpsig),
             pgp_sig_size: Set(info.pgpsigsize),
@@ -359,7 +359,12 @@ pub async fn write_desc<W: AsyncWrite + std::marker::Unpin>(
     write_attribute(writer, "ARCH", &pkg.arch).await?;
 
     // TODO build date
-    //write_attribute(writer, "BUILDDATE", &pkg.build_)
+    write_attribute(
+        writer,
+        "BUILDDATE",
+        &pkg.build_date.and_utc().timestamp().to_string(),
+    )
+    .await?;
 
     if let Some(ref packager) = pkg.packager {
         write_attribute(writer, "PACKAGER", packager).await?;