diff --git a/server/src/db/query/package.rs b/server/src/db/query/package.rs index ad9d74a..9a8be5f 100644 --- a/server/src/db/query/package.rs +++ b/server/src/db/query/package.rs @@ -4,6 +4,9 @@ use sea_orm::{sea_query::IntoCondition, *}; use sea_query::{Alias, Expr, Query, SelectStatement}; use serde::Deserialize; +/// How many fields may be inserted at once into the database. +const PACKAGE_INSERT_LIMIT: usize = 1000; + #[derive(Deserialize)] pub struct Filter { repo: Option, @@ -160,23 +163,34 @@ pub async fn insert( .iter() .map(|s| (PackageRelatedEnum::Optdepend, s)), ); + let related = crate::util::Chunked::new(related, PACKAGE_INSERT_LIMIT); - PackageRelated::insert_many(related.map(|(t, s)| package_related::ActiveModel { - package_id: Set(pkg_entry.id), - r#type: Set(t), - name: Set(s.to_string()), - })) - .on_empty_do_nothing() - .exec(&txn) - .await?; + for chunk in related { + PackageRelated::insert_many( + chunk + .into_iter() + .map(|(t, s)| package_related::ActiveModel { + package_id: Set(pkg_entry.id), + r#type: Set(t), + name: Set(s.to_string()), + }), + ) + .on_empty_do_nothing() + .exec(&txn) + .await?; + } - PackageFile::insert_many(pkg.files.iter().map(|s| package_file::ActiveModel { - package_id: Set(pkg_entry.id), - path: Set(s.display().to_string()), - })) - .on_empty_do_nothing() - .exec(&txn) - .await?; + let files = crate::util::Chunked::new(pkg.files, PACKAGE_INSERT_LIMIT); + + for chunk in files { + PackageFile::insert_many(chunk.into_iter().map(|s| package_file::ActiveModel { + package_id: Set(pkg_entry.id), + path: Set(s.display().to_string()), + })) + .on_empty_do_nothing() + .exec(&txn) + .await?; + } txn.commit().await?; diff --git a/server/src/main.rs b/server/src/main.rs index 5a91fdb..cb66668 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -3,6 +3,7 @@ mod config; pub mod db; mod error; mod repo; +mod util; mod web; pub use config::{Config, DbConfig, FsConfig}; diff --git a/server/src/util.rs b/server/src/util.rs new file mode 100644 index 0000000..9aad122 --- /dev/null +++ b/server/src/util.rs @@ -0,0 +1,23 @@ +pub struct Chunked { + iter: I, + chunk_size: usize, +} + +impl Chunked { + pub fn new>(into: T, chunk_size: usize) -> Self { + Self { + iter: into.into_iter(), + chunk_size, + } + } +} + +// https://users.rust-lang.org/t/how-to-breakup-an-iterator-into-chunks/87915/5 +impl Iterator for Chunked { + type Item = Vec; + + fn next(&mut self) -> Option { + Some(self.iter.by_ref().take(self.chunk_size).collect()) + .filter(|chunk: &Vec<_>| !chunk.is_empty()) + } +}