Chewing_Bever
/
rieter
1
0
Fork 0
Code Issues 11 Pull Requests Packages Releases Activity

feat: simplified repo structure; fully implemented repo db archives

concurrent-repos
Jef Roosens 2024-05-27 10:59:32 +02:00
parent 633e670840
commit 513a760040
Signed by: Jef Roosens
GPG Key ID: B75D4F293C7052DB
7 changed files with 43 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
server/src/cli.rs
View File

@ -16,9 +16,6 @@ use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
#[derive(Parser)]
#[command(author, version, about, long_about = None)]
pub struct Cli {
/// Directory where package archives will be stored
#[arg(env = "RIETER_PKG_DIR")]
pub pkg_dir: PathBuf,
/// Directory where repository metadata & SQLite database is stored
#[arg(env = "RIETER_DATA_DIR")]
pub data_dir: PathBuf,
@ -85,11 +82,9 @@ impl Cli {
let config = Config {
data_dir: self.data_dir.clone(),
repo_dir: self.data_dir.join("repos"),
pkg_dir: self.pkg_dir.clone(),
api_key: self.api_key.clone(),
};
let repo_manager = MetaRepoMgr::new(&config.repo_dir, &self.pkg_dir);
let repo_manager = MetaRepoMgr::new(&self.data_dir.join("repos"));
let pkg_filename_re = regex::Regex::new(r"^([a-z0-9@_+][a-z0-9@_+-.]*)-((?:[0-9]+:)?[a-z0-9._]+-[0-9.]+)-([a-z0-9_]+)\.pkg\.tar\.([a-z0-9]+)$").unwrap();

3
server/src/db/entities/package.rs
View File

@ -1,5 +1,6 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1
use chrono::NaiveDateTime;
use sea_orm::entity::prelude::*;
use serde::{Deserialize, Serialize};
@ -17,7 +18,7 @@ pub struct Model {
pub c_size: i64,
pub description: Option<String>,
pub url: Option<String>,
pub build_date: String,
pub build_date: NaiveDateTime,
pub packager: Option<String>,
pub pgp_sig: Option<String>,
pub pgp_sig_size: Option<i64>,

2
server/src/db/query/package.rs
View File

@ -88,7 +88,7 @@ pub async fn insert(conn: &DbConn, repo_id: i32, pkg: crate::repo::package::Pack
c_size: Set(info.csize),
description: Set(info.description),
url: Set(info.url),
build_date: Set(info.build_date.to_string()),
build_date: Set(info.build_date),
packager: Set(info.packager),
pgp_sig: Set(info.pgpsig),
pgp_sig_size: Set(info.pgpsigsize),

2
server/src/main.rs
View File

@ -15,8 +15,6 @@ use tokio::sync::RwLock;
#[derive(Clone)]
pub struct Config {
data_dir: PathBuf,
repo_dir: PathBuf,
pkg_dir: PathBuf,
api_key: String,
}

34
server/src/repo/manager_new.rs
View File

@ -5,7 +5,6 @@ use uuid::Uuid;
use futures::StreamExt;
use tokio::io::AsyncRead;
use tokio::io::AsyncWriteExt;
use super::archive;
use super::package;
@ -16,14 +15,12 @@ pub const ANY_ARCH: &'static str = "any";
pub struct MetaRepoMgr {
repo_dir: PathBuf,
pkg_dir: PathBuf,
}
impl MetaRepoMgr {
pub fn new<P1: AsRef<Path>, P2: AsRef<Path>>(repo_dir: P1, pkg_dir: P2) -> Self {
pub fn new<P: AsRef<Path>>(repo_dir: P) -> Self {
MetaRepoMgr {
repo_dir: repo_dir.as_ref().to_path_buf(),
pkg_dir: pkg_dir.as_ref().to_path_buf(),
}
}
@ -64,11 +61,14 @@ impl MetaRepoMgr {
let repo = repo.unwrap();
let parent_dir = self.repo_dir.join(&repo.name).join(arch);
let parent_dir = self.repo_dir.join(&repo.name);
tokio::fs::create_dir_all(&parent_dir).await?;
let ar_db =
archive::RepoArchiveWriter::open(parent_dir.join(format!("{}.db.tar.gz", arch)))
.await?;
let ar_files =
archive::RepoArchiveWriter::open(parent_dir.join(format!("{}.db.tar.gz", repo.name)))
archive::RepoArchiveWriter::open(parent_dir.join(format!("{}.files.tar.gz", arch)))
.await?;
// Query all packages in the repo that have the given architecture or the "any"
@ -79,11 +79,12 @@ impl MetaRepoMgr {
.stream(conn)
.await?;
// Create two temp file paths to write our entries to
let uuid: uuid::fmt::Simple = Uuid::new_v4().into();
let files_tmp_file_path = self.pkg_dir.join(uuid.to_string());
let files_tmp_file_path = self.repo_dir.join(uuid.to_string());
let uuid: uuid::fmt::Simple = Uuid::new_v4().into();
let desc_tmp_file_path = self.pkg_dir.join(uuid.to_string());
let desc_tmp_file_path = self.repo_dir.join(uuid.to_string());
while let Some(pkg) = pkgs.next().await.transpose()? {
let mut files_tmp_file = tokio::fs::File::create(&files_tmp_file_path).await?;
@ -93,6 +94,10 @@ impl MetaRepoMgr {
package::write_desc(conn, &mut desc_tmp_file, &pkg).await?;
let full_name = format!("{}-{}", pkg.name, pkg.version);
ar_db
.add_entry(&full_name, &desc_tmp_file_path, true)
.await?;
ar_files
.add_entry(&full_name, &desc_tmp_file_path, true)
.await?;
@ -101,7 +106,11 @@ impl MetaRepoMgr {
.await?;
}
// Cleanup
ar_db.close().await?;
ar_files.close().await?;
tokio::fs::remove_file(desc_tmp_file_path).await?;
tokio::fs::remove_file(files_tmp_file_path).await?;
Ok(())
@ -117,7 +126,6 @@ impl MetaRepoMgr {
// Remove files from file system
tokio::fs::remove_dir_all(self.repo_dir.join(repo)).await?;
tokio::fs::remove_dir_all(self.pkg_dir.join(repo)).await?;
Ok(true)
} else {
@ -157,7 +165,7 @@ impl MetaRepoMgr {
) -> crate::Result<()> {
// Copy file contents to temporary path so libarchive can work with it
let uuid: uuid::fmt::Simple = Uuid::new_v4().into();
let path = self.pkg_dir.join(uuid.to_string());
let path = self.repo_dir.join(uuid.to_string());
let mut temp_file = tokio::fs::File::create(&path).await?;
tokio::io::copy(reader, &mut temp_file).await?;
@ -194,11 +202,7 @@ impl MetaRepoMgr {
entry.delete(conn).await?;
}
let dest_pkg_path = self
.pkg_dir
.join(repo)
.join(&pkg.info.arch)
.join(pkg.file_name());
let dest_pkg_path = self.repo_dir.join(repo).join(pkg.file_name());
// Insert new package into database
let arch = pkg.info.arch.clone();

44
server/src/repo/mod.rs
View File

@ -57,41 +57,23 @@ pub fn router(api_key: &str) -> Router<crate::Global> {
/// is returned.
async fn get_file(
State(global): State<crate::Global>,
Path((repo, _arch, file_name)): Path<(String, String, String)>,
Path((repo, arch, file_name)): Path<(String, String, String)>,
req: Request<Body>,
) -> crate::Result<impl IntoResponse> {
// Query the repo to see if it exists
let repo = db::query::repo::by_name(&global.db, &repo)
.await?
.ok_or(StatusCode::NOT_FOUND)?;
let repo_dir = global.config.data_dir.join("repos").join(&repo);
// Match the filename
let caps = global
.pkg_filename_re
.captures(&file_name)
.ok_or(StatusCode::NOT_FOUND)?;
let arch = caps.get(3).unwrap().as_str();
let file_name =
if file_name == format!("{}.db", repo) || file_name == format!("{}.db.tar.gz", repo) {
format!("{}.db.tar.gz", arch)
} else if file_name == format!("{}.files", repo)
|| file_name == format!("{}.files.tar.gz", repo)
{
format!("{}.files.tar.gz", arch)
} else {
file_name
};
// Query the package to see if it exists
let pkg = db::query::package::by_fields(
&global.db,
repo.id,
arch,
caps.get(1).unwrap().as_str(),
Some(caps.get(2).unwrap().as_str()),
Some(caps.get(4).unwrap().as_str()),
)
.await?
.ok_or(StatusCode::NOT_FOUND)?;
// Serve the file if it idoes
let pkg_path = global
.config
.pkg_dir
.join(repo.name)
.join(arch)
.join(package::filename(&pkg));
Ok(ServeFile::new(pkg_path).oneshot(req).await.unwrap())
Ok(ServeFile::new(repo_dir.join(file_name)).oneshot(req).await)
}
async fn post_package_archive(

9
server/src/repo/package.rs
View File

@ -283,7 +283,7 @@ impl From<Package> for package::ActiveModel {
c_size: Set(info.csize),
description: Set(info.description),
url: Set(info.url),
build_date: Set(info.build_date.to_string()),
build_date: Set(info.build_date),
packager: Set(info.packager),
pgp_sig: Set(info.pgpsig),
pgp_sig_size: Set(info.pgpsigsize),
@ -359,7 +359,12 @@ pub async fn write_desc<W: AsyncWrite + std::marker::Unpin>(
write_attribute(writer, "ARCH", &pkg.arch).await?;
// TODO build date
//write_attribute(writer, "BUILDDATE", &pkg.build_)
write_attribute(
writer,
"BUILDDATE",
&pkg.build_date.and_utc().timestamp().to_string(),
)
.await?;
if let Some(ref packager) = pkg.packager {
write_attribute(writer, "PACKAGER", packager).await?;