server/src/db/query/package.rs

@@ -4,9 +4,6 @@ use sea_orm::{sea_query::IntoCondition, *};
 use sea_query::{Alias, Expr, Query, SelectStatement};
 use serde::Deserialize;
 
-/// How many fields may be inserted at once into the database.
-const PACKAGE_INSERT_LIMIT: usize = 1000;
-
 #[derive(Deserialize)]
 pub struct Filter {
     repo: Option<i32>,
@@ -163,34 +160,23 @@ pub async fn insert(
                 .iter()
                 .map(|s| (PackageRelatedEnum::Optdepend, s)),
         );
-    let related = crate::util::Chunked::new(related, PACKAGE_INSERT_LIMIT);
 
-    for chunk in related {
-        PackageRelated::insert_many(
-            chunk
-                .into_iter()
-                .map(|(t, s)| package_related::ActiveModel {
-                    package_id: Set(pkg_entry.id),
-                    r#type: Set(t),
-                    name: Set(s.to_string()),
-                }),
-        )
-        .on_empty_do_nothing()
-        .exec(&txn)
-        .await?;
-    }
+    PackageRelated::insert_many(related.map(|(t, s)| package_related::ActiveModel {
+        package_id: Set(pkg_entry.id),
+        r#type: Set(t),
+        name: Set(s.to_string()),
+    }))
+    .on_empty_do_nothing()
+    .exec(&txn)
+    .await?;
 
-    let files = crate::util::Chunked::new(pkg.files, PACKAGE_INSERT_LIMIT);
-
-    for chunk in files {
-        PackageFile::insert_many(chunk.into_iter().map(|s| package_file::ActiveModel {
-            package_id: Set(pkg_entry.id),
-            path: Set(s.display().to_string()),
-        }))
-        .on_empty_do_nothing()
-        .exec(&txn)
-        .await?;
-    }
+    PackageFile::insert_many(pkg.files.iter().map(|s| package_file::ActiveModel {
+        package_id: Set(pkg_entry.id),
+        path: Set(s.display().to_string()),
+    }))
+    .on_empty_do_nothing()
+    .exec(&txn)
+    .await?;
 
     txn.commit().await?;
 

server/src/main.rs

@@ -3,7 +3,6 @@ mod config;
 pub mod db;
 mod error;
 mod repo;
-mod util;
 mod web;
 
 pub use config::{Config, DbConfig, FsConfig};

server/src/repo/package.rs

@@ -13,6 +13,8 @@ use libarchive::{
 };
 use sea_orm::ActiveValue::Set;
 
+const IGNORED_FILES: [&str; 5] = [".BUILDINFO", ".INSTALL", ".MTREE", ".PKGINFO", ".CHANGELOG"];
+
 #[derive(Debug, Clone)]
 pub struct Package {
     pub path: PathBuf,
@@ -156,9 +158,11 @@ impl Package {
             let entry = entry?;
             let path_name = entry.pathname();
 
-            if !path_name.starts_with('.') {
+            if !IGNORED_FILES.iter().any(|p| p == &path_name) {
                 files.push(PathBuf::from(path_name));
-            } else if path_name == ".PKGINFO" {
+            }
+
+            if path_name == ".PKGINFO" {
                 info = Some(PkgInfo::parse(entry)?);
             }
         }

server/src/util.rs

@@ -1,23 +0,0 @@
-pub struct Chunked<I> {
-    iter: I,
-    chunk_size: usize,
-}
-
-impl<I: Iterator> Chunked<I> {
-    pub fn new<T: IntoIterator<IntoIter = I>>(into: T, chunk_size: usize) -> Self {
-        Self {
-            iter: into.into_iter(),
-            chunk_size,
-        }
-    }
-}
-
-// https://users.rust-lang.org/t/how-to-breakup-an-iterator-into-chunks/87915/5
-impl<I: Iterator> Iterator for Chunked<I> {
-    type Item = Vec<I::Item>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        Some(self.iter.by_ref().take(self.chunk_size).collect())
-            .filter(|chunk: &Vec<_>| !chunk.is_empty())
-    }
-}