Compare commits
No commits in common. "86ab143271ecb866a8cce1db2234e5d0d587dbd3" and "d375df0ff4fde9fc8095e41e712c5cef27a86867" have entirely different histories.
86ab143271
...
d375df0ff4
|
@ -4,9 +4,6 @@ use sea_orm::{sea_query::IntoCondition, *};
|
||||||
use sea_query::{Alias, Expr, Query, SelectStatement};
|
use sea_query::{Alias, Expr, Query, SelectStatement};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
|
|
||||||
/// How many fields may be inserted at once into the database.
|
|
||||||
const PACKAGE_INSERT_LIMIT: usize = 1000;
|
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
pub struct Filter {
|
pub struct Filter {
|
||||||
repo: Option<i32>,
|
repo: Option<i32>,
|
||||||
|
@ -163,34 +160,23 @@ pub async fn insert(
|
||||||
.iter()
|
.iter()
|
||||||
.map(|s| (PackageRelatedEnum::Optdepend, s)),
|
.map(|s| (PackageRelatedEnum::Optdepend, s)),
|
||||||
);
|
);
|
||||||
let related = crate::util::Chunked::new(related, PACKAGE_INSERT_LIMIT);
|
|
||||||
|
|
||||||
for chunk in related {
|
PackageRelated::insert_many(related.map(|(t, s)| package_related::ActiveModel {
|
||||||
PackageRelated::insert_many(
|
package_id: Set(pkg_entry.id),
|
||||||
chunk
|
r#type: Set(t),
|
||||||
.into_iter()
|
name: Set(s.to_string()),
|
||||||
.map(|(t, s)| package_related::ActiveModel {
|
}))
|
||||||
package_id: Set(pkg_entry.id),
|
.on_empty_do_nothing()
|
||||||
r#type: Set(t),
|
.exec(&txn)
|
||||||
name: Set(s.to_string()),
|
.await?;
|
||||||
}),
|
|
||||||
)
|
|
||||||
.on_empty_do_nothing()
|
|
||||||
.exec(&txn)
|
|
||||||
.await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
let files = crate::util::Chunked::new(pkg.files, PACKAGE_INSERT_LIMIT);
|
PackageFile::insert_many(pkg.files.iter().map(|s| package_file::ActiveModel {
|
||||||
|
package_id: Set(pkg_entry.id),
|
||||||
for chunk in files {
|
path: Set(s.display().to_string()),
|
||||||
PackageFile::insert_many(chunk.into_iter().map(|s| package_file::ActiveModel {
|
}))
|
||||||
package_id: Set(pkg_entry.id),
|
.on_empty_do_nothing()
|
||||||
path: Set(s.display().to_string()),
|
.exec(&txn)
|
||||||
}))
|
.await?;
|
||||||
.on_empty_do_nothing()
|
|
||||||
.exec(&txn)
|
|
||||||
.await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
txn.commit().await?;
|
txn.commit().await?;
|
||||||
|
|
||||||
|
|
|
@ -3,7 +3,6 @@ mod config;
|
||||||
pub mod db;
|
pub mod db;
|
||||||
mod error;
|
mod error;
|
||||||
mod repo;
|
mod repo;
|
||||||
mod util;
|
|
||||||
mod web;
|
mod web;
|
||||||
|
|
||||||
pub use config::{Config, DbConfig, FsConfig};
|
pub use config::{Config, DbConfig, FsConfig};
|
||||||
|
|
|
@ -13,6 +13,8 @@ use libarchive::{
|
||||||
};
|
};
|
||||||
use sea_orm::ActiveValue::Set;
|
use sea_orm::ActiveValue::Set;
|
||||||
|
|
||||||
|
const IGNORED_FILES: [&str; 5] = [".BUILDINFO", ".INSTALL", ".MTREE", ".PKGINFO", ".CHANGELOG"];
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct Package {
|
pub struct Package {
|
||||||
pub path: PathBuf,
|
pub path: PathBuf,
|
||||||
|
@ -156,9 +158,11 @@ impl Package {
|
||||||
let entry = entry?;
|
let entry = entry?;
|
||||||
let path_name = entry.pathname();
|
let path_name = entry.pathname();
|
||||||
|
|
||||||
if !path_name.starts_with('.') {
|
if !IGNORED_FILES.iter().any(|p| p == &path_name) {
|
||||||
files.push(PathBuf::from(path_name));
|
files.push(PathBuf::from(path_name));
|
||||||
} else if path_name == ".PKGINFO" {
|
}
|
||||||
|
|
||||||
|
if path_name == ".PKGINFO" {
|
||||||
info = Some(PkgInfo::parse(entry)?);
|
info = Some(PkgInfo::parse(entry)?);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,23 +0,0 @@
|
||||||
pub struct Chunked<I> {
|
|
||||||
iter: I,
|
|
||||||
chunk_size: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<I: Iterator> Chunked<I> {
|
|
||||||
pub fn new<T: IntoIterator<IntoIter = I>>(into: T, chunk_size: usize) -> Self {
|
|
||||||
Self {
|
|
||||||
iter: into.into_iter(),
|
|
||||||
chunk_size,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// https://users.rust-lang.org/t/how-to-breakup-an-iterator-into-chunks/87915/5
|
|
||||||
impl<I: Iterator> Iterator for Chunked<I> {
|
|
||||||
type Item = Vec<I::Item>;
|
|
||||||
|
|
||||||
fn next(&mut self) -> Option<Self::Item> {
|
|
||||||
Some(self.iter.by_ref().take(self.chunk_size).collect())
|
|
||||||
.filter(|chunk: &Vec<_>| !chunk.is_empty())
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
Reference in New Issue