From 0ff225dddb9fe802b6211f7434337e6bd0ddd155 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 4 Aug 2023 17:33:03 +0200 Subject: [PATCH 1/3] refactor(server): further abstract db --- server/src/db/mod.rs | 62 ++++++++++++++++++++++++++++++++++-------- server/src/repo/mod.rs | 43 +++++++++++------------------ 2 files changed, 67 insertions(+), 38 deletions(-) diff --git a/server/src/db/mod.rs b/server/src/db/mod.rs index 02c4284..1dbaba5 100644 --- a/server/src/db/mod.rs +++ b/server/src/db/mod.rs @@ -2,19 +2,14 @@ mod conn; pub mod entities; mod migrator; -use migrator::Migrator; -use sea_orm::ColumnTrait; -use sea_orm::ConnectOptions; -use sea_orm::Database; -use sea_orm::DatabaseConnection; -use sea_orm::EntityTrait; -use sea_orm::PaginatorTrait; -use sea_orm::QueryFilter; -use sea_orm::QueryOrder; +use sea_orm::{ + ColumnTrait, ConnectOptions, Database, DatabaseConnection, DeleteResult, EntityTrait, + InsertResult, NotSet, PaginatorTrait, QueryFilter, QueryOrder, Set, +}; use sea_orm_migration::MigratorTrait; -pub use entities::prelude::*; -pub use entities::*; +pub use entities::{prelude::*, *}; +use migrator::Migrator; type Result = std::result::Result; @@ -53,6 +48,20 @@ impl RieterDb { .await } + pub async fn insert_repo( + &self, + name: &str, + description: Option<&str>, + ) -> Result> { + let model = repo::ActiveModel { + id: NotSet, + name: Set(String::from(name)), + description: Set(description.map(String::from)), + }; + + Repo::insert(model).exec(&self.conn).await + } + pub async fn packages(&self, per_page: u64, page: u64) -> Result<(u64, Vec)> { let paginator = Package::find() .order_by_asc(package::Column::Id) @@ -66,4 +75,35 @@ impl RieterDb { pub async fn package(&self, id: i32) -> Result> { package::Entity::find_by_id(id).one(&self.conn).await } + + pub async fn package_by_fields( + &self, + repo_id: i32, + name: &str, + version: Option<&str>, + arch: &str, + ) -> Result> { + let mut query = Package::find() + .filter(package::Column::RepoId.eq(repo_id)) + .filter(package::Column::Name.eq(name)) + .filter(package::Column::Arch.eq(arch)); + + if let Some(version) = version { + query = query.filter(package::Column::Version.eq(version)); + } + + query.one(&self.conn).await + } + + pub async fn delete_packages_with_arch( + &self, + repo_id: i32, + arch: &str, + ) -> Result { + Package::delete_many() + .filter(package::Column::RepoId.eq(repo_id)) + .filter(package::Column::Arch.eq(arch)) + .exec(&self.conn) + .await + } } diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index 8ab2b89..f87d572 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -14,7 +14,7 @@ use axum::response::IntoResponse; use axum::routing::{delete, post}; use axum::Router; use futures::StreamExt; -use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, ModelTrait, QueryFilter}; +use sea_orm::{ActiveModelTrait, EntityTrait, ModelTrait}; use std::sync::Arc; use tokio::{fs, io::AsyncWriteExt}; use tower::util::ServiceExt; @@ -132,23 +132,13 @@ async fn post_package_archive( let repo_id = if let Some(repo_entity) = res { repo_entity.id } else { - let model = db::repo::ActiveModel { - name: sea_orm::Set(repo.clone()), - ..Default::default() - }; - - db::Repo::insert(model) - .exec(&global.db) - .await? - .last_insert_id + global.db.insert_repo(&repo, None).await?.last_insert_id }; // If the package already exists in the database, we remove it first - let res = db::Package::find() - .filter(db::package::Column::RepoId.eq(repo_id)) - .filter(db::package::Column::Name.eq(&pkg.info.name)) - .filter(db::package::Column::Arch.eq(&pkg.info.arch)) - .one(&global.db) + let res = global + .db + .package_by_fields(repo_id, &pkg.info.name, None, &pkg.info.arch) .await?; if let Some(entry) = res { @@ -226,11 +216,9 @@ async fn delete_arch_repo( let res = global.db.repo_by_name(&repo).await?; if let Some(repo_entry) = res { - // Also remove all packages for that architecture from database - db::Package::delete_many() - .filter(db::package::Column::RepoId.eq(repo_entry.id)) - .filter(db::package::Column::Arch.eq(&arch)) - .exec(&global.db) + global + .db + .delete_packages_with_arch(repo_entry.id, &arch) .await?; } tracing::info!("Removed architecture '{}' from repository '{}'", arch, repo); @@ -257,13 +245,14 @@ async fn delete_package( let res = global.db.repo_by_name(&repo).await?; if let Some(repo_entry) = res { - // Also remove entry from database - let res = db::Package::find() - .filter(db::package::Column::RepoId.eq(repo_entry.id)) - .filter(db::package::Column::Name.eq(name)) - .filter(db::package::Column::Version.eq(format!("{}-{}", version, release))) - .filter(db::package::Column::Arch.eq(arch)) - .one(&global.db) + let res = global + .db + .package_by_fields( + repo_entry.id, + &name, + Some(&format!("{}-{}", version, release)), + &arch, + ) .await?; if let Some(entry) = res { From 2df52320d1b3934b2165aa627bf300b9cc3e2cb5 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 4 Aug 2023 18:40:17 +0200 Subject: [PATCH 2/3] feat(server): store all package info in database --- server/src/db/entities/mod.rs | 1 + server/src/db/entities/package.rs | 8 ++ server/src/db/entities/package_depends.rs | 2 +- server/src/db/entities/package_file.rs | 33 +++++ server/src/db/entities/prelude.rs | 1 + .../m20230730_000001_create_repo_tables.rs | 41 +++++- server/src/db/mod.rs | 118 +++++++++++++++++- server/src/repo/mod.rs | 19 +-- 8 files changed, 203 insertions(+), 20 deletions(-) create mode 100644 server/src/db/entities/package_file.rs diff --git a/server/src/db/entities/mod.rs b/server/src/db/entities/mod.rs index 828fec2..1111e7a 100644 --- a/server/src/db/entities/mod.rs +++ b/server/src/db/entities/mod.rs @@ -5,6 +5,7 @@ pub mod prelude; pub mod package; pub mod package_conflicts; pub mod package_depends; +pub mod package_file; pub mod package_group; pub mod package_license; pub mod package_provides; diff --git a/server/src/db/entities/package.rs b/server/src/db/entities/package.rs index c34abb1..b2e2b0b 100644 --- a/server/src/db/entities/package.rs +++ b/server/src/db/entities/package.rs @@ -30,6 +30,8 @@ pub enum Relation { PackageConflicts, #[sea_orm(has_many = "super::package_depends::Entity")] PackageDepends, + #[sea_orm(has_many = "super::package_file::Entity")] + PackageFile, #[sea_orm(has_many = "super::package_group::Entity")] PackageGroup, #[sea_orm(has_many = "super::package_license::Entity")] @@ -60,6 +62,12 @@ impl Related for Entity { } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::PackageFile.def() + } +} + impl Related for Entity { fn to() -> RelationDef { Relation::PackageGroup.def() diff --git a/server/src/db/entities/package_depends.rs b/server/src/db/entities/package_depends.rs index b22f717..7e94374 100644 --- a/server/src/db/entities/package_depends.rs +++ b/server/src/db/entities/package_depends.rs @@ -9,7 +9,7 @@ pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub package_id: i32, #[sea_orm(primary_key, auto_increment = false)] - pub r#type: i32, + pub r#type: crate::db::PackageDepend, #[sea_orm(primary_key, auto_increment = false)] pub value: String, } diff --git a/server/src/db/entities/package_file.rs b/server/src/db/entities/package_file.rs new file mode 100644 index 0000000..6e994e0 --- /dev/null +++ b/server/src/db/entities/package_file.rs @@ -0,0 +1,33 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "package_file")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub package_id: i32, + #[sea_orm(primary_key, auto_increment = false)] + pub value: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::package::Entity", + from = "Column::PackageId", + to = "super::package::Column::Id", + on_update = "NoAction", + on_delete = "Cascade" + )] + Package, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Package.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/db/entities/prelude.rs b/server/src/db/entities/prelude.rs index 9eae171..bee503c 100644 --- a/server/src/db/entities/prelude.rs +++ b/server/src/db/entities/prelude.rs @@ -3,6 +3,7 @@ pub use super::package::Entity as Package; pub use super::package_conflicts::Entity as PackageConflicts; pub use super::package_depends::Entity as PackageDepends; +pub use super::package_file::Entity as PackageFile; pub use super::package_group::Entity as PackageGroup; pub use super::package_license::Entity as PackageLicense; pub use super::package_provides::Entity as PackageProvides; diff --git a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs index aceae78..adefe56 100644 --- a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs +++ b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs @@ -212,7 +212,11 @@ impl MigrationTrait for Migration { .integer() .not_null(), ) - .col(ColumnDef::new(PackageDepends::Type).integer().not_null()) + .col( + ColumnDef::new(PackageDepends::Type) + .string_len(6) + .not_null(), + ) .col( ColumnDef::new(PackageDepends::Value) .string_len(255) @@ -234,6 +238,31 @@ impl MigrationTrait for Migration { .to_owned(), ) .await?; + manager + .create_table( + Table::create() + .table(PackageFile::Table) + .col(ColumnDef::new(PackageFile::PackageId).integer().not_null()) + .col( + ColumnDef::new(PackageFile::Value) + .string_len(255) + .not_null(), + ) + .primary_key( + Index::create() + .col(PackageFile::PackageId) + .col(PackageFile::Value), + ) + .foreign_key( + ForeignKey::create() + .name("fk-package_file-package_id") + .from(PackageFile::Table, PackageFile::PackageId) + .to(Package::Table, Package::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; Ok(()) } @@ -258,6 +287,9 @@ impl MigrationTrait for Migration { manager .drop_table(Table::drop().table(PackageDepends::Table).to_owned()) .await?; + manager + .drop_table(Table::drop().table(PackageFile::Table).to_owned()) + .await?; manager .drop_table(Table::drop().table(Package::Table).to_owned()) .await?; @@ -337,3 +369,10 @@ pub enum PackageDepends { Type, Value, } + +#[derive(Iden)] +pub enum PackageFile { + Table, + PackageId, + Value, +} diff --git a/server/src/db/mod.rs b/server/src/db/mod.rs index 1dbaba5..bc5204c 100644 --- a/server/src/db/mod.rs +++ b/server/src/db/mod.rs @@ -3,16 +3,31 @@ pub mod entities; mod migrator; use sea_orm::{ - ColumnTrait, ConnectOptions, Database, DatabaseConnection, DeleteResult, EntityTrait, - InsertResult, NotSet, PaginatorTrait, QueryFilter, QueryOrder, Set, + ActiveModelTrait, ColumnTrait, ConnectOptions, Database, DatabaseConnection, DeleteResult, + DeriveActiveEnum, EntityTrait, EnumIter, InsertResult, NotSet, PaginatorTrait, QueryFilter, + QueryOrder, Set, }; use sea_orm_migration::MigratorTrait; +use serde::{Deserialize, Serialize}; pub use entities::{prelude::*, *}; use migrator::Migrator; type Result = std::result::Result; +#[derive(EnumIter, DeriveActiveEnum, Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] +#[sea_orm(rs_type = "String", db_type = "String(Some(6))")] +pub enum PackageDepend { + #[sea_orm(string_value = "depend")] + Depend, + #[sea_orm(string_value = "make")] + Make, + #[sea_orm(string_value = "check")] + Check, + #[sea_orm(string_value = "opt")] + Opt, +} + #[derive(Clone, Debug)] pub struct RieterDb { pub conn: DatabaseConnection, @@ -106,4 +121,103 @@ impl RieterDb { .exec(&self.conn) .await } + + pub async fn insert_package( + &self, + repo_id: i32, + pkg: crate::repo::package::Package, + ) -> Result<()> { + let info = pkg.info; + + let model = package::ActiveModel { + id: NotSet, + repo_id: Set(repo_id), + base: Set(info.base), + name: Set(info.name), + version: Set(info.version), + arch: Set(info.arch), + size: Set(info.size), + c_size: Set(info.csize), + description: Set(info.description), + url: Set(info.url), + build_date: Set(info.build_date), + packager: Set(info.packager), + pgp_sig: Set(info.pgpsig), + pgp_sig_size: Set(info.pgpsigsize), + sha256_sum: Set(info.sha256sum), + }; + + let pkg_entry = model.insert(&self.conn).await?; + + // Insert all the related tables + PackageLicense::insert_many(info.licenses.iter().map(|s| package_license::ActiveModel { + package_id: Set(pkg_entry.id), + value: Set(s.to_string()), + })) + .on_empty_do_nothing() + .exec(self) + .await?; + + PackageGroup::insert_many(info.groups.iter().map(|s| package_group::ActiveModel { + package_id: Set(pkg_entry.id), + value: Set(s.to_string()), + })) + .on_empty_do_nothing() + .exec(self) + .await?; + + PackageReplaces::insert_many(info.replaces.iter().map(|s| package_replaces::ActiveModel { + package_id: Set(pkg_entry.id), + value: Set(s.to_string()), + })) + .on_empty_do_nothing() + .exec(self) + .await?; + + PackageConflicts::insert_many(info.conflicts.iter().map(|s| { + package_conflicts::ActiveModel { + package_id: Set(pkg_entry.id), + value: Set(s.to_string()), + } + })) + .on_empty_do_nothing() + .exec(self) + .await?; + + PackageProvides::insert_many(info.provides.iter().map(|s| package_provides::ActiveModel { + package_id: Set(pkg_entry.id), + value: Set(s.to_string()), + })) + .on_empty_do_nothing() + .exec(self) + .await?; + + PackageFile::insert_many(pkg.files.iter().map(|s| package_file::ActiveModel { + package_id: Set(pkg_entry.id), + value: Set(s.display().to_string()), + })) + .on_empty_do_nothing() + .exec(self) + .await?; + + let deps = info + .depends + .iter() + .map(|d| (PackageDepend::Depend, d)) + .chain(info.makedepends.iter().map(|d| (PackageDepend::Make, d))) + .chain(info.checkdepends.iter().map(|d| (PackageDepend::Check, d))) + .chain(info.optdepends.iter().map(|d| (PackageDepend::Opt, d))) + .map(|(t, s)| package_depends::ActiveModel { + package_id: Set(pkg_entry.id), + r#type: Set(t), + value: Set(s.to_string()), + }); + + PackageDepends::insert_many(deps) + .on_empty_do_nothing() + .exec(self) + .await?; + + Ok(()) + } } diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index f87d572..ba8fbf7 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -1,11 +1,10 @@ mod manager; -mod package; +pub mod package; pub use manager::RepoGroupManager; use std::path::PathBuf; -use crate::db; use axum::body::Body; use axum::extract::{BodyStream, Path, State}; use axum::http::Request; @@ -14,7 +13,7 @@ use axum::response::IntoResponse; use axum::routing::{delete, post}; use axum::Router; use futures::StreamExt; -use sea_orm::{ActiveModelTrait, EntityTrait, ModelTrait}; +use sea_orm::ModelTrait; use std::sync::Arc; use tokio::{fs, io::AsyncWriteExt}; use tower::util::ServiceExt; @@ -145,19 +144,7 @@ async fn post_package_archive( entry.delete(&global.db).await?; } - // Insert the package's data into the database - let mut model: db::package::ActiveModel = pkg.clone().into(); - model.repo_id = sea_orm::Set(repo_id); - - let pkg_entry = model.insert(&global.db).await?; - db::PackageLicense::insert_many(pkg.info.licenses.iter().map(|s| { - db::package_license::ActiveModel { - package_id: sea_orm::Set(pkg_entry.id), - value: sea_orm::Set(s.to_string()), - } - })) - .exec(&global.db) - .await?; + global.db.insert_package(repo_id, pkg).await?; Ok(()) } From 5c8b7ac3e0276d8f28bab5a1542cb42f9b38d0b2 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 4 Aug 2023 19:08:55 +0200 Subject: [PATCH 3/3] feat(server): serve full package info from api --- server/src/api/mod.rs | 23 ++----------- server/src/db/mod.rs | 75 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 22 deletions(-) diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index 184c1a4..cf6f94c 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -4,7 +4,6 @@ use axum::extract::{Path, Query, State}; use axum::routing::get; use axum::Json; use axum::Router; -use sea_orm::ModelTrait; use pagination::PaginatedResponse; @@ -60,31 +59,15 @@ async fn get_packages( Ok(Json(pagination.res(total_pages, pkgs))) } -#[derive(serde::Serialize)] -pub struct PackageRes { - #[serde(flatten)] - entry: db::package::Model, - licenses: Vec, -} - async fn get_single_package( State(global): State, Path(id): Path, -) -> crate::Result> { +) -> crate::Result> { let entry = global .db - .package(id) + .full_package(id) .await? .ok_or(axum::http::StatusCode::NOT_FOUND)?; - let licenses = entry - .find_related(db::PackageLicense) - .all(&global.db) - .await? - .iter() - .map(|e| e.value.clone()) - .collect(); - let res = PackageRes { entry, licenses }; - - Ok(Json(res)) + Ok(Json(entry)) } diff --git a/server/src/db/mod.rs b/server/src/db/mod.rs index bc5204c..1da82dd 100644 --- a/server/src/db/mod.rs +++ b/server/src/db/mod.rs @@ -4,8 +4,8 @@ mod migrator; use sea_orm::{ ActiveModelTrait, ColumnTrait, ConnectOptions, Database, DatabaseConnection, DeleteResult, - DeriveActiveEnum, EntityTrait, EnumIter, InsertResult, NotSet, PaginatorTrait, QueryFilter, - QueryOrder, Set, + DeriveActiveEnum, EntityTrait, EnumIter, InsertResult, ModelTrait, NotSet, PaginatorTrait, + QueryFilter, QueryOrder, Set, }; use sea_orm_migration::MigratorTrait; use serde::{Deserialize, Serialize}; @@ -28,6 +28,18 @@ pub enum PackageDepend { Opt, } +#[derive(Serialize)] +pub struct FullPackage { + #[serde(flatten)] + entry: package::Model, + licenses: Vec, + groups: Vec, + replaces: Vec, + provides: Vec, + depends: Vec<(PackageDepend, String)>, + files: Vec, +} + #[derive(Clone, Debug)] pub struct RieterDb { pub conn: DatabaseConnection, @@ -220,4 +232,63 @@ impl RieterDb { Ok(()) } + + pub async fn full_package(&self, id: i32) -> Result> { + if let Some(entry) = self.package(id).await? { + let licenses = entry + .find_related(PackageLicense) + .all(self) + .await? + .into_iter() + .map(|e| e.value) + .collect(); + let groups = entry + .find_related(PackageGroup) + .all(self) + .await? + .into_iter() + .map(|e| e.value) + .collect(); + let replaces = entry + .find_related(PackageReplaces) + .all(self) + .await? + .into_iter() + .map(|e| e.value) + .collect(); + let provides = entry + .find_related(PackageProvides) + .all(self) + .await? + .into_iter() + .map(|e| e.value) + .collect(); + let depends = entry + .find_related(PackageDepends) + .all(self) + .await? + .into_iter() + .map(|e| (e.r#type, e.value)) + .collect(); + let files = entry + .find_related(PackageFile) + .all(self) + .await? + .into_iter() + .map(|e| e.value) + .collect(); + + Ok(Some(FullPackage { + entry, + licenses, + groups, + replaces, + provides, + depends, + files, + })) + } else { + Ok(None) + } + } }