Attempted split at multi-repo/arch support, segfaults atm [CI SKIP]

Makefile

@@ -5,7 +5,9 @@ SOURCES != find '$(SRC_DIR)' -iname '*.v'
 V_RELEASE := weekly.2022.05
 V_PATH ?= v-$(V_RELEASE)/v
 V := $(V_PATH) -showcc -gc boehm
+ENV := API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repos PKG_DIR=data/pkgs LOG_LEVEL=DEBUG
 
+.PHONY: all
 all: vieter
 
 # =====COMPILATION=====
@@ -32,19 +34,23 @@ c:
 
 
 # =====EXECUTION=====
+# Run gdb on the debug binary
+.PHONY: gdb
+gdb: vieter
+	$(ENV) gdb ./vieter
 # Run the server in the default 'data' directory
 .PHONY: run
 run: vieter
-	 API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter
+	 $(ENV) ./vieter
 
 .PHONY: run-prod
 run-prod: prod
-	API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./pvieter
+	$(ENV) ./pvieter
 
 # Same as run, but restart when the source code changes
 .PHONY: watch
 watch:
-	API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG $(V) watch run vieter
+	$(ENV) $(V) watch run vieter
 
 
 # =====OTHER=====

src/main.v

@@ -18,7 +18,9 @@ pub:
 	api_key string [required; web_global]
 	dl_dir  string [required; web_global]
 pub mut:
-	repo repo.Repo [required; web_global]
+	repo         repo.MultiArchRepo [required; web_global]
+	default_repo string             [required; web_global]
+	default_arch string             [required; web_global]
 }
 
 [noreturn]
@@ -85,11 +87,11 @@ fn main() {
 	dl_dir := os.getenv_opt('DOWNLOAD_DIR') or {
 		exit_with_message(1, 'No download directory was configured.')
 	}
+	default_repo := os.getenv_opt('DEFAULT_REPO') or { 'vieter' }
+	default_arch := os.getenv_opt('DEFAULT_ARCH') or { 'x86_64' }
 
-	// This also creates the directories if needed
+	repo := repo.new_multi(repo_dir, pkg_dir) or {
-	repo := repo.new(repo_dir, pkg_dir) or {
+		exit_with_message(1, 'Failed to create repo: $err.msg')
-		logger.error(err.msg)
-		exit(1)
 	}
 
 	os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') }
@@ -99,5 +101,7 @@ fn main() {
 		api_key: key
 		dl_dir: dl_dir
 		repo: repo
+		default_repo: default_repo
+		default_arch: default_arch
 	}, port)
 }

src/repo/arch.v

@@ -0,0 +1,62 @@
+module repo
+
+import os
+import package
+
+// A architecture repo acts as a single repo, but manages multiple repos
+// internally that each manage a repo for a specific architecture
+pub struct ArchRepo {
+mut:
+	// Each key is the name of an architecture, e.g. x86_64
+	repos shared map[string]Repo [required]
+pub:
+	repo_dir string [required]
+	pkg_dir  string [required]
+}
+
+pub fn new_arch(repo_dir string, pkg_dir string) ?ArchRepo {
+	if !os.is_dir(pkg_dir) {
+		os.mkdir(pkg_dir) or { return error('Failed to create package directory: $err.msg') }
+	}
+
+	mut repos := map[string]Repo{}
+
+	if !os.is_dir(repo_dir) {
+		os.mkdir(repo_dir) or { return error('Failed to create repo directory: $err.msg') }
+	}
+	// If the directory already exists, we can check which repos already exist
+	else {
+		for name in os.ls(repo_dir) ? {
+			d := os.join_path_single(repo_dir, name)
+
+			if os.is_dir(d) {
+				// The name is expected to be a correct architecture name
+				repos[name] = new(d, pkg_dir) ?
+			}
+		}
+	}
+
+	return ArchRepo{
+		repo_dir: repo_dir
+		pkg_dir: pkg_dir
+		repos: repos
+	}
+}
+
+fn (r &ArchRepo) add(pkg &package.Pkg) ?bool {
+	// TODO add 'any' architecture to every repo
+	arch := pkg.info.arch
+
+	lock r.repos {
+		// If a repo for this architecture doesn't exist yet, create it
+		if arch !in r.repos {
+			r.repos[arch] = new(os.join_path_single(r.repo_dir, arch), r.pkg_dir) ?
+		}
+	}
+
+	repo := rlock r.repos {
+		r.repos[arch]
+	}
+
+	return repo.add(pkg)
+}

src/repo/multi.v

@@ -0,0 +1,68 @@
+module repo
+
+import os
+import package
+
+// Represents a collection of Repos
+pub struct MultiArchRepo {
+mut:
+	repos shared map[string]ArchRepo [required]
+pub:
+	pkg_dir  string [required]
+	repo_dir string [required]
+}
+
+// Initializes a new multi-repository.
+pub fn new_multi(repo_dir string, pkg_dir string) ?MultiArchRepo {
+	mut repos := map[string]ArchRepo{}
+
+	// We initialize a repo for each directory inside the repos dir
+	if !os.is_dir(repo_dir) {
+		os.mkdir_all(repo_dir) or {
+			return error("Failed to create directory '$repo_dir': $err.msg")
+		}
+	} else {
+		for name in os.ls(repo_dir) ? {
+			d := os.join_path_single(repo_dir, name)
+
+			if os.is_dir(d) {
+				repos[name] = new_arch(d, pkg_dir) ?
+			}
+		}
+	}
+
+	return MultiArchRepo{
+		pkg_dir: pkg_dir
+		repo_dir: repo_dir
+		repos: repos
+	}
+}
+
+pub fn (r &MultiArchRepo) add_from_path(repo_name string, pkg_path string) ?(bool, &package.Pkg) {
+	// First, we create the repo if it isn't present yet
+	repo := lock r.repos {
+		if repo_name !in r.repos {
+			r.repos[repo_name] = new_arch(os.join_path_single(r.repo_dir, repo_name),
+				r.pkg_dir) ?
+		}
+
+		r.repos[repo_name] ?
+	}
+
+	// We read in the package file
+	pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') }
+
+	added := repo.add(pkg) ?
+
+	// Move over package to pkg dir if it was successfully added
+	if added {
+		dest_path := os.real_path(os.join_path_single(r.pkg_dir, pkg.filename()))
+
+		// Only move the file if it's not already in the package directory
+		if dest_path != os.real_path(pkg_path) {
+			os.mv(pkg_path, dest_path) ?
+		}
+	}
+
+	return added, &pkg
+}

src/repo/repo.v

@@ -20,20 +20,14 @@ pub:
 	pkg_dir string [required]
 }
 
-pub struct RepoAddResult {
-pub:
-	added bool         [required]
-	pkg   &package.Pkg [required]
-}
-
 // new creates a new Repo & creates the directories as needed
 pub fn new(repo_dir string, pkg_dir string) ?Repo {
 	if !os.is_dir(repo_dir) {
-		os.mkdir_all(repo_dir) or { return error('Failed to create repo directory: $err.msg') }
+		os.mkdir(repo_dir) or { return error('Failed to create repo directory: $err.msg') }
 	}
 
 	if !os.is_dir(pkg_dir) {
-		os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg') }
+		os.mkdir(pkg_dir) or { return error('Failed to create package directory: $err.msg') }
 	}
 
 	return Repo{
@@ -44,7 +38,7 @@ pub fn new(repo_dir string, pkg_dir string) ?Repo {
 
 // add_from_path adds a package from an arbitrary path & moves it into the pkgs
 // directory if necessary.
-pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult {
+pub fn (r &Repo) add_from_path(pkg_path string) ?(bool, &package.Pkg) {
 	pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') }
 
 	added := r.add(pkg) ?
@@ -59,10 +53,7 @@ pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult {
 		}
 	}
 
-	return RepoAddResult{
+	return added, &pkg
-		added: added
-		pkg: &pkg
-	}
 }
 
 // add adds a given Pkg to the repository

src/repo/sync.v

@@ -2,6 +2,10 @@ module repo
 
 import os
 
+const db_filename = 'repo.db.tar.gz'
+
+const files_filename = 'repo.files.tar.gz'
+
 fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &string, inner_path &string) {
 	st := C.stat{}
 
@@ -31,7 +35,6 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri
 
 // Re-generate the repo archive files
 fn (r &Repo) sync() ? {
-	// TODO also write files archive
 	lock r.mutex {
 		a_db := C.archive_write_new()
 		a_files := C.archive_write_new()
@@ -44,8 +47,8 @@ fn (r &Repo) sync() ? {
 		C.archive_write_add_filter_gzip(a_files)
 		C.archive_write_set_format_pax_restricted(a_files)
 
-		db_path := os.join_path_single(r.repo_dir, 'vieter.db.tar.gz')
+		db_path := os.join_path_single(r.repo_dir, repo.db_filename)
-		files_path := os.join_path_single(r.repo_dir, 'vieter.files.tar.gz')
+		files_path := os.join_path_single(r.repo_dir, repo.files_filename)
 
 		C.archive_write_open_filename(a_db, &char(db_path.str))
 		C.archive_write_open_filename(a_files, &char(files_path.str))

src/routes.v

@@ -21,10 +21,6 @@ fn pretty_bytes(bytes int) string {
 	return '${n:.2}${prefixes[i]}'
 }
 
-fn is_pkg_name(s string) bool {
-	return s.contains('.pkg')
-}
-
 // healthcheck just returns a string, but can be used to quickly check if the
 // server is still responsive.
 ['/health'; get]
@@ -82,22 +78,22 @@ fn (mut app App) put_package() web.Result {
 		return app.text("Content-Type header isn't set.")
 	}
 
-	res := app.repo.add_from_path(pkg_path) or {
+	added, pkg := app.repo.add_from_path(app.default_repo, pkg_path) or {
 		app.lerror('Error while adding package: $err.msg')
 
 		os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }
 
 		return app.text('Failed to add package.')
 	}
-	if !res.added {
+	if !added {
 		os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }
 
-		app.lwarn("Duplicate package '$res.pkg.full_name()'.")
+		app.lwarn("Duplicate package '$pkg.full_name()'.")
 
 		return app.text('File already exists.')
 	}
 
-	app.linfo("Added '$res.pkg.full_name()' to repository.")
+	app.linfo("Added '$pkg.full_name()' to repository.")
 
 	return app.text('Package added successfully.')
 }