diff --git a/Cargo.lock b/Cargo.lock index e59252c..333bc72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -151,7 +151,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -162,7 +162,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -194,6 +194,7 @@ checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", "axum-core", + "axum-macros", "bytes", "futures-util", "http", @@ -241,6 +242,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00c055ee2d014ae5981ce1016374e8213682aa14d9bf40e48ab48b5f3ef20eaa" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "backtrace" version = "0.3.71" @@ -335,7 +348,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", "syn_derive", ] @@ -381,9 +394,9 @@ checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" [[package]] name = "cfg-if" @@ -443,7 +456,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -505,9 +518,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crypto-common" @@ -617,12 +630,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" -[[package]] -name = "finl_unicode" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" - [[package]] name = "flume" version = "0.11.0" @@ -722,7 +729,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -963,9 +970,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +checksum = "3d8d52be92d09acc2e01dddb7fde3ad983fc6489c7db4837e605bc3fca4cb63e" dependencies = [ "bytes", "futures-util", @@ -973,7 +980,6 @@ dependencies = [ "http-body", "hyper", "pin-project-lite", - "socket2", "tokio", ] @@ -1028,7 +1034,7 @@ checksum = "0122b7114117e64a63ac49f752a5ca4624d534c7b1c7de796ac196381cd2d947" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -1342,7 +1348,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -1353,9 +1359,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "parking_lot" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -1412,7 +1418,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -1501,9 +1507,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.82" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" +checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" dependencies = [ "unicode-ident", ] @@ -1652,6 +1658,7 @@ dependencies = [ "chrono", "clap", "futures", + "http-body-util", "libarchive", "sea-orm", "sea-orm-migration", @@ -1833,7 +1840,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -1891,7 +1898,7 @@ dependencies = [ "proc-macro2", "quote", "sea-bae", - "syn 2.0.64", + "syn 2.0.66", "unicode-ident", ] @@ -1955,7 +1962,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", "thiserror", ] @@ -1990,22 +1997,22 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "serde" -version = "1.0.202" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.202" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -2395,13 +2402,13 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "stringprep" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ - "finl_unicode", "unicode-bidi", "unicode-normalization", + "unicode-properties", ] [[package]] @@ -2435,9 +2442,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.64" +version = "2.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ad3dee41f36859875573074334c200d1add8e4a87bb37113ebd31d926b7b11f" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" dependencies = [ "proc-macro2", "quote", @@ -2453,7 +2460,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -2503,7 +2510,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -2589,7 +2596,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -2707,7 +2714,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -2785,6 +2792,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" + [[package]] name = "unicode-segmentation" version = "1.11.0" @@ -2887,7 +2900,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", "wasm-bindgen-shared", ] @@ -2909,7 +2922,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3141,11 +3154,11 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] name = "zeroize" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/server/Cargo.toml b/server/Cargo.toml index a234377..c2c5d74 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -7,10 +7,11 @@ authors = ["Jef Roosens"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -axum = { version = "0.7.5", features = ["http2"] } +axum = { version = "0.7.5", features = ["http2", "macros"] } chrono = { version = "0.4.26", features = ["serde"] } clap = { version = "4.3.12", features = ["env", "derive"] } futures = "0.3.28" +http-body-util = "0.1.1" libarchive = { path = "../libarchive" } sea-orm-migration = "0.12.1" serde = { version = "1.0.178", features = ["derive"] } diff --git a/server/src/cli.rs b/server/src/cli.rs index 2976d1c..91569de 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -1,4 +1,4 @@ -use crate::repo::RepoGroupManager; +use crate::repo::{MetaRepoMgr, RepoGroupManager}; use crate::{Config, Global}; use axum::extract::FromRef; @@ -7,7 +7,8 @@ use clap::Parser; use sea_orm_migration::MigratorTrait; use std::io; use std::path::PathBuf; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; +use tokio::sync::RwLock; use tower_http::trace::TraceLayer; use tracing::debug; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; @@ -48,7 +49,7 @@ pub struct Cli { pub log: String, } -impl FromRef for Arc> { +impl FromRef for Arc> { fn from_ref(global: &Global) -> Self { Arc::clone(&global.repo_manager) } @@ -85,7 +86,7 @@ impl Cli { pkg_dir: self.pkg_dir.clone(), api_key: self.api_key.clone(), }; - let repo_manager = RepoGroupManager::new(&config.repo_dir, &self.pkg_dir); + let repo_manager = MetaRepoMgr::new(&config.repo_dir, &self.pkg_dir); let global = Global { config, diff --git a/server/src/main.rs b/server/src/main.rs index 9068bd7..7038203 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -6,9 +6,11 @@ mod repo; use clap::Parser; pub use error::{Result, ServerError}; +use repo::MetaRepoMgr; use repo::RepoGroupManager; use std::path::PathBuf; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; +use tokio::sync::RwLock; #[derive(Clone)] pub struct Config { @@ -21,7 +23,7 @@ pub struct Config { #[derive(Clone)] pub struct Global { config: Config, - repo_manager: Arc>, + repo_manager: Arc>, db: sea_orm::DbConn, } diff --git a/server/src/repo/manager_new.rs b/server/src/repo/manager_new.rs index a7f2f1b..f7c0fb1 100644 --- a/server/src/repo/manager_new.rs +++ b/server/src/repo/manager_new.rs @@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex}; use libarchive::write::{Builder, WriteEntry}; use libarchive::{Entry, WriteFilter, WriteFormat}; -use sea_orm::{ColumnTrait, DbConn, EntityTrait, ModelTrait, QueryFilter}; +use sea_orm::{ColumnTrait, DbConn, EntityTrait, ModelTrait, QueryFilter, QuerySelect}; use uuid::Uuid; use futures::StreamExt; @@ -40,6 +40,7 @@ impl MetaRepoMgr { let repo = repo.unwrap(); let parent_dir = self.repo_dir.join(&repo.name).join(arch); + tokio::fs::create_dir_all(&parent_dir).await?; let ar_files = archive::RepoArchiveWriter::open(parent_dir.join(format!("{}.db.tar.gz", repo.name))) @@ -112,7 +113,12 @@ impl MetaRepoMgr { } } - pub async fn add_pkg_from_reader(&self, conn: &DbConn, reader: &mut R, repo: &str) -> crate::Result<()> { + pub async fn add_pkg_from_reader( + &self, + conn: &DbConn, + reader: &mut R, + repo: &str, + ) -> crate::Result<()> { // Copy file contents to temporary path so libarchive can work with it let uuid: uuid::fmt::Simple = Uuid::new_v4().into(); let path = self.pkg_dir.join(uuid.to_string()); @@ -122,7 +128,9 @@ impl MetaRepoMgr { // Parse the package let path_clone = path.clone(); - let pkg = tokio::task::spawn_blocking(move || package::Package::open(path_clone)).await.unwrap()?; + let pkg = tokio::task::spawn_blocking(move || package::Package::open(path_clone)) + .await + .unwrap()?; // Query the repo for its ID, or create it if it does not already exist let res = db::query::repo::by_name(conn, &repo).await?; @@ -137,8 +145,7 @@ impl MetaRepoMgr { // If the package already exists in the database, we remove it first let res = - db::query::package::by_fields(conn, repo_id, &pkg.info.arch, &pkg.info.name) - .await?; + db::query::package::by_fields(conn, repo_id, &pkg.info.arch, &pkg.info.name).await?; if let Some(entry) = res { entry.delete(conn).await?; diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index 4b3ff29..1992e22 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -4,9 +4,12 @@ mod manager_new; pub mod package; pub use manager::RepoGroupManager; +pub use manager_new::MetaRepoMgr; +use tokio_util::io::StreamReader; use std::path::PathBuf; +use crate::db; use axum::body::Body; use axum::extract::{Path, State}; use axum::http::Request; @@ -14,7 +17,8 @@ use axum::http::StatusCode; use axum::response::IntoResponse; use axum::routing::{delete, post}; use axum::Router; -use futures::StreamExt; +use futures::TryStreamExt; +use futures::{Stream, StreamExt}; use sea_orm::ModelTrait; use std::sync::Arc; use tokio::{fs, io::AsyncWriteExt}; @@ -23,8 +27,6 @@ use tower_http::services::{ServeDir, ServeFile}; use tower_http::validate_request::ValidateRequestHeaderLayer; use uuid::Uuid; -use crate::db; - const DB_FILE_EXTS: [&str; 4] = [".db", ".files", ".db.tar.gz", ".files.tar.gz"]; pub fn router(api_key: &str) -> Router { @@ -99,90 +101,40 @@ async fn get_file( Ok(res) } +#[axum::debug_handler] async fn post_package_archive( State(global): State, Path(repo): Path, body: Body, ) -> crate::Result<()> { - // We first stream the uploaded file to disk - let uuid: uuid::fmt::Simple = Uuid::new_v4().into(); - let path = global.config.pkg_dir.join(uuid.to_string()); - let mut f = fs::File::create(&path).await?; + let body = body.into_data_stream(); + let body = body.map_err(std::io::Error::other); + let mut body = StreamReader::new(body); + global + .repo_manager + .write() + .await + .add_pkg_from_reader(&global.db, &mut body, &repo) + .await?; - let mut body = body.into_data_stream(); + Ok(()) - while let Some(chunk) = body.next().await { - f.write_all(&chunk?).await?; - } - - let clone = Arc::clone(&global.repo_manager); - let path_clone = path.clone(); - let repo_clone = repo.clone(); - let res = tokio::task::spawn_blocking(move || { - global - .repo_manager - .write() - .unwrap() - .add_pkg_from_path(&repo_clone, &path_clone) - }) - .await?; - - match res { - // Insert the newly added package into the database - Ok(pkg) => { - tracing::info!("Added '{}' to repository '{}'", pkg.file_name(), repo); - - // Query the repo for its ID, or create it if it does not already exist - let res = db::query::repo::by_name(&global.db, &repo).await?; - - let repo_id = if let Some(repo_entity) = res { - repo_entity.id - } else { - db::query::repo::insert(&global.db, &repo, None) - .await? - .last_insert_id - }; - - // If the package already exists in the database, we remove it first - let res = - db::query::package::by_fields(&global.db, repo_id, &pkg.info.arch, &pkg.info.name) - .await?; - - if let Some(entry) = res { - entry.delete(&global.db).await?; - } - - db::query::package::insert(&global.db, repo_id, pkg).await?; - - Ok(()) - } - // Remove the uploaded file and return the error - Err(err) => { - tokio::fs::remove_file(path).await?; - - Err(err.into()) - } - } + //tracing::info!("Added '{}' to repository '{}'", pkg.file_name(), repo); } +#[axum::debug_handler] async fn delete_repo( State(global): State, Path(repo): Path, ) -> crate::Result { - let clone = Arc::clone(&global.repo_manager); - - let repo_clone = repo.clone(); - let repo_removed = - tokio::task::spawn_blocking(move || clone.write().unwrap().remove_repo(&repo_clone)) - .await??; + let repo_removed = global + .repo_manager + .write() + .await + .remove_repo(&global.db, &repo) + .await?; if repo_removed { - let res = db::query::repo::by_name(&global.db, &repo).await?; - - if let Some(repo_entry) = res { - repo_entry.delete(&global.db).await?; - } - tracing::info!("Removed repository '{}'", repo); Ok(StatusCode::OK) @@ -195,60 +147,71 @@ async fn delete_arch_repo( State(global): State, Path((repo, arch)): Path<(String, String)>, ) -> crate::Result { - let clone = Arc::clone(&global.repo_manager); - - let arch_clone = arch.clone(); - let repo_clone = repo.clone(); - let repo_removed = tokio::task::spawn_blocking(move || { - clone - .write() - .unwrap() - .remove_repo_arch(&repo_clone, &arch_clone) - }) - .await??; - - if repo_removed { - let res = db::query::repo::by_name(&global.db, &repo).await?; - - if let Some(repo_entry) = res { - db::query::package::delete_with_arch(&global.db, repo_entry.id, &arch).await?; - } - tracing::info!("Removed architecture '{}' from repository '{}'", arch, repo); - - Ok(StatusCode::OK) - } else { - Ok(StatusCode::NOT_FOUND) - } + Ok(StatusCode::NOT_FOUND) + //let clone = Arc::clone(&global.repo_manager); + // + //let arch_clone = arch.clone(); + //let repo_clone = repo.clone(); + //let repo_removed = tokio::task::spawn_blocking(move || { + // clone + // .write() + // .unwrap() + // .remove_repo_arch(&repo_clone, &arch_clone) + //}) + //.await??; + // + //if repo_removed { + // let res = db::query::repo::by_name(&global.db, &repo).await?; + // + // if let Some(repo_entry) = res { + // db::query::package::delete_with_arch(&global.db, repo_entry.id, &arch).await?; + // } + // tracing::info!("Removed architecture '{}' from repository '{}'", arch, repo); + // + // Ok(StatusCode::OK) + //} else { + // Ok(StatusCode::NOT_FOUND) + //} } async fn delete_package( State(global): State, Path((repo, arch, file_name)): Path<(String, String, String)>, ) -> crate::Result { - let clone = Arc::clone(&global.repo_manager); - let path = PathBuf::from(&repo).join(arch).join(&file_name); - - let res = tokio::task::spawn_blocking(move || { - clone.write().unwrap().remove_pkg_from_path(path, true) - }) - .await??; - - if let Some((name, version, release, arch)) = res { - let res = db::query::repo::by_name(&global.db, &repo).await?; - - if let Some(repo_entry) = res { - let res = - db::query::package::by_fields(&global.db, repo_entry.id, &arch, &name).await?; - - if let Some(entry) = res { - entry.delete(&global.db).await?; - } - } - - tracing::info!("Removed '{}' from repository '{}'", file_name, repo); - - Ok(StatusCode::OK) - } else { - Ok(StatusCode::NOT_FOUND) - } + Ok(StatusCode::NOT_FOUND) + //global.repo_manager.write().unwrap().remove_pkg(&global.db, &repo, &arch, name) + //let clone = Arc::clone(&global.repo_manager); + //let path = PathBuf::from(&repo).join(arch).join(&file_name); + // + //let res = tokio::task::spawn_blocking(move || { + // clone.write().unwrap().remove_pkg_from_path(path, true) + //}) + //.await??; + // + //if let Some((name, version, release, arch)) = res { + // let res = db::query::repo::by_name(&global.db, &repo).await?; + // + // if let Some(repo_entry) = res { + // let res = + // db::query::package::by_fields(&global.db, repo_entry.id, &arch, &name).await?; + // + // if let Some(entry) = res { + // entry.delete(&global.db).await?; + // } + // } + // + // tracing::info!("Removed '{}' from repository '{}'", file_name, repo); + // + // Ok(StatusCode::OK) + //} else { + // Ok(StatusCode::NOT_FOUND) + //} +} + +fn help_me_figure_this_out(_: impl Send) {} + +#[allow(dead_code)] +fn assert_my_handler_is_ok() { + // This is the fun part: we need to call our handler, and *not* await it's Future. + help_me_figure_this_out(delete_repo) }