Compare commits

...

3 Commits

Author SHA1 Message Date
Jef Roosens 5c8b7ac3e0
feat(server): serve full package info from api
ci/woodpecker/push/lint Pipeline was successful Details
ci/woodpecker/push/clippy Pipeline failed Details
ci/woodpecker/push/build Pipeline was successful Details
2023-08-04 19:08:55 +02:00
Jef Roosens 2df52320d1
feat(server): store all package info in database 2023-08-04 18:40:17 +02:00
Jef Roosens 0ff225dddb
refactor(server): further abstract db 2023-08-04 17:59:20 +02:00
9 changed files with 341 additions and 75 deletions

View File

@ -4,7 +4,6 @@ use axum::extract::{Path, Query, State};
use axum::routing::get;
use axum::Json;
use axum::Router;
use sea_orm::ModelTrait;
use pagination::PaginatedResponse;
@ -60,31 +59,15 @@ async fn get_packages(
Ok(Json(pagination.res(total_pages, pkgs)))
}
#[derive(serde::Serialize)]
pub struct PackageRes {
#[serde(flatten)]
entry: db::package::Model,
licenses: Vec<String>,
}
async fn get_single_package(
State(global): State<crate::Global>,
Path(id): Path<i32>,
) -> crate::Result<Json<PackageRes>> {
) -> crate::Result<Json<crate::db::FullPackage>> {
let entry = global
.db
.package(id)
.full_package(id)
.await?
.ok_or(axum::http::StatusCode::NOT_FOUND)?;
let licenses = entry
.find_related(db::PackageLicense)
.all(&global.db)
.await?
.iter()
.map(|e| e.value.clone())
.collect();
let res = PackageRes { entry, licenses };
Ok(Json(res))
Ok(Json(entry))
}

View File

@ -5,6 +5,7 @@ pub mod prelude;
pub mod package;
pub mod package_conflicts;
pub mod package_depends;
pub mod package_file;
pub mod package_group;
pub mod package_license;
pub mod package_provides;

View File

@ -30,6 +30,8 @@ pub enum Relation {
PackageConflicts,
#[sea_orm(has_many = "super::package_depends::Entity")]
PackageDepends,
#[sea_orm(has_many = "super::package_file::Entity")]
PackageFile,
#[sea_orm(has_many = "super::package_group::Entity")]
PackageGroup,
#[sea_orm(has_many = "super::package_license::Entity")]
@ -60,6 +62,12 @@ impl Related<super::package_depends::Entity> for Entity {
}
}
impl Related<super::package_file::Entity> for Entity {
fn to() -> RelationDef {
Relation::PackageFile.def()
}
}
impl Related<super::package_group::Entity> for Entity {
fn to() -> RelationDef {
Relation::PackageGroup.def()

View File

@ -9,7 +9,7 @@ pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub package_id: i32,
#[sea_orm(primary_key, auto_increment = false)]
pub r#type: i32,
pub r#type: crate::db::PackageDepend,
#[sea_orm(primary_key, auto_increment = false)]
pub value: String,
}

View File

@ -0,0 +1,33 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1
use sea_orm::entity::prelude::*;
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
#[sea_orm(table_name = "package_file")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub package_id: i32,
#[sea_orm(primary_key, auto_increment = false)]
pub value: String,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {
#[sea_orm(
belongs_to = "super::package::Entity",
from = "Column::PackageId",
to = "super::package::Column::Id",
on_update = "NoAction",
on_delete = "Cascade"
)]
Package,
}
impl Related<super::package::Entity> for Entity {
fn to() -> RelationDef {
Relation::Package.def()
}
}
impl ActiveModelBehavior for ActiveModel {}

View File

@ -3,6 +3,7 @@
pub use super::package::Entity as Package;
pub use super::package_conflicts::Entity as PackageConflicts;
pub use super::package_depends::Entity as PackageDepends;
pub use super::package_file::Entity as PackageFile;
pub use super::package_group::Entity as PackageGroup;
pub use super::package_license::Entity as PackageLicense;
pub use super::package_provides::Entity as PackageProvides;

View File

@ -212,7 +212,11 @@ impl MigrationTrait for Migration {
.integer()
.not_null(),
)
.col(ColumnDef::new(PackageDepends::Type).integer().not_null())
.col(
ColumnDef::new(PackageDepends::Type)
.string_len(6)
.not_null(),
)
.col(
ColumnDef::new(PackageDepends::Value)
.string_len(255)
@ -234,6 +238,31 @@ impl MigrationTrait for Migration {
.to_owned(),
)
.await?;
manager
.create_table(
Table::create()
.table(PackageFile::Table)
.col(ColumnDef::new(PackageFile::PackageId).integer().not_null())
.col(
ColumnDef::new(PackageFile::Value)
.string_len(255)
.not_null(),
)
.primary_key(
Index::create()
.col(PackageFile::PackageId)
.col(PackageFile::Value),
)
.foreign_key(
ForeignKey::create()
.name("fk-package_file-package_id")
.from(PackageFile::Table, PackageFile::PackageId)
.to(Package::Table, Package::Id)
.on_delete(ForeignKeyAction::Cascade),
)
.to_owned(),
)
.await?;
Ok(())
}
@ -258,6 +287,9 @@ impl MigrationTrait for Migration {
manager
.drop_table(Table::drop().table(PackageDepends::Table).to_owned())
.await?;
manager
.drop_table(Table::drop().table(PackageFile::Table).to_owned())
.await?;
manager
.drop_table(Table::drop().table(Package::Table).to_owned())
.await?;
@ -337,3 +369,10 @@ pub enum PackageDepends {
Type,
Value,
}
#[derive(Iden)]
pub enum PackageFile {
Table,
PackageId,
Value,
}

View File

@ -2,22 +2,44 @@ mod conn;
pub mod entities;
mod migrator;
use migrator::Migrator;
use sea_orm::ColumnTrait;
use sea_orm::ConnectOptions;
use sea_orm::Database;
use sea_orm::DatabaseConnection;
use sea_orm::EntityTrait;
use sea_orm::PaginatorTrait;
use sea_orm::QueryFilter;
use sea_orm::QueryOrder;
use sea_orm::{
ActiveModelTrait, ColumnTrait, ConnectOptions, Database, DatabaseConnection, DeleteResult,
DeriveActiveEnum, EntityTrait, EnumIter, InsertResult, ModelTrait, NotSet, PaginatorTrait,
QueryFilter, QueryOrder, Set,
};
use sea_orm_migration::MigratorTrait;
use serde::{Deserialize, Serialize};
pub use entities::prelude::*;
pub use entities::*;
pub use entities::{prelude::*, *};
use migrator::Migrator;
type Result<T> = std::result::Result<T, sea_orm::DbErr>;
#[derive(EnumIter, DeriveActiveEnum, Serialize, Deserialize, PartialEq, Eq, Clone, Debug)]
#[sea_orm(rs_type = "String", db_type = "String(Some(6))")]
pub enum PackageDepend {
#[sea_orm(string_value = "depend")]
Depend,
#[sea_orm(string_value = "make")]
Make,
#[sea_orm(string_value = "check")]
Check,
#[sea_orm(string_value = "opt")]
Opt,
}
#[derive(Serialize)]
pub struct FullPackage {
#[serde(flatten)]
entry: package::Model,
licenses: Vec<String>,
groups: Vec<String>,
replaces: Vec<String>,
provides: Vec<String>,
depends: Vec<(PackageDepend, String)>,
files: Vec<String>,
}
#[derive(Clone, Debug)]
pub struct RieterDb {
pub conn: DatabaseConnection,
@ -53,6 +75,20 @@ impl RieterDb {
.await
}
pub async fn insert_repo(
&self,
name: &str,
description: Option<&str>,
) -> Result<InsertResult<repo::ActiveModel>> {
let model = repo::ActiveModel {
id: NotSet,
name: Set(String::from(name)),
description: Set(description.map(String::from)),
};
Repo::insert(model).exec(&self.conn).await
}
pub async fn packages(&self, per_page: u64, page: u64) -> Result<(u64, Vec<package::Model>)> {
let paginator = Package::find()
.order_by_asc(package::Column::Id)
@ -66,4 +102,193 @@ impl RieterDb {
pub async fn package(&self, id: i32) -> Result<Option<package::Model>> {
package::Entity::find_by_id(id).one(&self.conn).await
}
pub async fn package_by_fields(
&self,
repo_id: i32,
name: &str,
version: Option<&str>,
arch: &str,
) -> Result<Option<package::Model>> {
let mut query = Package::find()
.filter(package::Column::RepoId.eq(repo_id))
.filter(package::Column::Name.eq(name))
.filter(package::Column::Arch.eq(arch));
if let Some(version) = version {
query = query.filter(package::Column::Version.eq(version));
}
query.one(&self.conn).await
}
pub async fn delete_packages_with_arch(
&self,
repo_id: i32,
arch: &str,
) -> Result<DeleteResult> {
Package::delete_many()
.filter(package::Column::RepoId.eq(repo_id))
.filter(package::Column::Arch.eq(arch))
.exec(&self.conn)
.await
}
pub async fn insert_package(
&self,
repo_id: i32,
pkg: crate::repo::package::Package,
) -> Result<()> {
let info = pkg.info;
let model = package::ActiveModel {
id: NotSet,
repo_id: Set(repo_id),
base: Set(info.base),
name: Set(info.name),
version: Set(info.version),
arch: Set(info.arch),
size: Set(info.size),
c_size: Set(info.csize),
description: Set(info.description),
url: Set(info.url),
build_date: Set(info.build_date),
packager: Set(info.packager),
pgp_sig: Set(info.pgpsig),
pgp_sig_size: Set(info.pgpsigsize),
sha256_sum: Set(info.sha256sum),
};
let pkg_entry = model.insert(&self.conn).await?;
// Insert all the related tables
PackageLicense::insert_many(info.licenses.iter().map(|s| package_license::ActiveModel {
package_id: Set(pkg_entry.id),
value: Set(s.to_string()),
}))
.on_empty_do_nothing()
.exec(self)
.await?;
PackageGroup::insert_many(info.groups.iter().map(|s| package_group::ActiveModel {
package_id: Set(pkg_entry.id),
value: Set(s.to_string()),
}))
.on_empty_do_nothing()
.exec(self)
.await?;
PackageReplaces::insert_many(info.replaces.iter().map(|s| package_replaces::ActiveModel {
package_id: Set(pkg_entry.id),
value: Set(s.to_string()),
}))
.on_empty_do_nothing()
.exec(self)
.await?;
PackageConflicts::insert_many(info.conflicts.iter().map(|s| {
package_conflicts::ActiveModel {
package_id: Set(pkg_entry.id),
value: Set(s.to_string()),
}
}))
.on_empty_do_nothing()
.exec(self)
.await?;
PackageProvides::insert_many(info.provides.iter().map(|s| package_provides::ActiveModel {
package_id: Set(pkg_entry.id),
value: Set(s.to_string()),
}))
.on_empty_do_nothing()
.exec(self)
.await?;
PackageFile::insert_many(pkg.files.iter().map(|s| package_file::ActiveModel {
package_id: Set(pkg_entry.id),
value: Set(s.display().to_string()),
}))
.on_empty_do_nothing()
.exec(self)
.await?;
let deps = info
.depends
.iter()
.map(|d| (PackageDepend::Depend, d))
.chain(info.makedepends.iter().map(|d| (PackageDepend::Make, d)))
.chain(info.checkdepends.iter().map(|d| (PackageDepend::Check, d)))
.chain(info.optdepends.iter().map(|d| (PackageDepend::Opt, d)))
.map(|(t, s)| package_depends::ActiveModel {
package_id: Set(pkg_entry.id),
r#type: Set(t),
value: Set(s.to_string()),
});
PackageDepends::insert_many(deps)
.on_empty_do_nothing()
.exec(self)
.await?;
Ok(())
}
pub async fn full_package(&self, id: i32) -> Result<Option<FullPackage>> {
if let Some(entry) = self.package(id).await? {
let licenses = entry
.find_related(PackageLicense)
.all(self)
.await?
.into_iter()
.map(|e| e.value)
.collect();
let groups = entry
.find_related(PackageGroup)
.all(self)
.await?
.into_iter()
.map(|e| e.value)
.collect();
let replaces = entry
.find_related(PackageReplaces)
.all(self)
.await?
.into_iter()
.map(|e| e.value)
.collect();
let provides = entry
.find_related(PackageProvides)
.all(self)
.await?
.into_iter()
.map(|e| e.value)
.collect();
let depends = entry
.find_related(PackageDepends)
.all(self)
.await?
.into_iter()
.map(|e| (e.r#type, e.value))
.collect();
let files = entry
.find_related(PackageFile)
.all(self)
.await?
.into_iter()
.map(|e| e.value)
.collect();
Ok(Some(FullPackage {
entry,
licenses,
groups,
replaces,
provides,
depends,
files,
}))
} else {
Ok(None)
}
}
}

View File

@ -1,11 +1,10 @@
mod manager;
mod package;
pub mod package;
pub use manager::RepoGroupManager;
use std::path::PathBuf;
use crate::db;
use axum::body::Body;
use axum::extract::{BodyStream, Path, State};
use axum::http::Request;
@ -14,7 +13,7 @@ use axum::response::IntoResponse;
use axum::routing::{delete, post};
use axum::Router;
use futures::StreamExt;
use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, ModelTrait, QueryFilter};
use sea_orm::ModelTrait;
use std::sync::Arc;
use tokio::{fs, io::AsyncWriteExt};
use tower::util::ServiceExt;
@ -132,42 +131,20 @@ async fn post_package_archive(
let repo_id = if let Some(repo_entity) = res {
repo_entity.id
} else {
let model = db::repo::ActiveModel {
name: sea_orm::Set(repo.clone()),
..Default::default()
};
db::Repo::insert(model)
.exec(&global.db)
.await?
.last_insert_id
global.db.insert_repo(&repo, None).await?.last_insert_id
};
// If the package already exists in the database, we remove it first
let res = db::Package::find()
.filter(db::package::Column::RepoId.eq(repo_id))
.filter(db::package::Column::Name.eq(&pkg.info.name))
.filter(db::package::Column::Arch.eq(&pkg.info.arch))
.one(&global.db)
let res = global
.db
.package_by_fields(repo_id, &pkg.info.name, None, &pkg.info.arch)
.await?;
if let Some(entry) = res {
entry.delete(&global.db).await?;
}
// Insert the package's data into the database
let mut model: db::package::ActiveModel = pkg.clone().into();
model.repo_id = sea_orm::Set(repo_id);
let pkg_entry = model.insert(&global.db).await?;
db::PackageLicense::insert_many(pkg.info.licenses.iter().map(|s| {
db::package_license::ActiveModel {
package_id: sea_orm::Set(pkg_entry.id),
value: sea_orm::Set(s.to_string()),
}
}))
.exec(&global.db)
.await?;
global.db.insert_package(repo_id, pkg).await?;
Ok(())
}
@ -226,11 +203,9 @@ async fn delete_arch_repo(
let res = global.db.repo_by_name(&repo).await?;
if let Some(repo_entry) = res {
// Also remove all packages for that architecture from database
db::Package::delete_many()
.filter(db::package::Column::RepoId.eq(repo_entry.id))
.filter(db::package::Column::Arch.eq(&arch))
.exec(&global.db)
global
.db
.delete_packages_with_arch(repo_entry.id, &arch)
.await?;
}
tracing::info!("Removed architecture '{}' from repository '{}'", arch, repo);
@ -257,13 +232,14 @@ async fn delete_package(
let res = global.db.repo_by_name(&repo).await?;
if let Some(repo_entry) = res {
// Also remove entry from database
let res = db::Package::find()
.filter(db::package::Column::RepoId.eq(repo_entry.id))
.filter(db::package::Column::Name.eq(name))
.filter(db::package::Column::Version.eq(format!("{}-{}", version, release)))
.filter(db::package::Column::Arch.eq(arch))
.one(&global.db)
let res = global
.db
.package_by_fields(
repo_entry.id,
&name,
Some(&format!("{}-{}", version, release)),
&arch,
)
.await?;
if let Some(entry) = res {