From f7171b822d2cef2f00168f928e8920f6b72b8222 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 4 Feb 2022 16:04:05 +0100 Subject: [PATCH] Attempted split at multi-repo/arch support, segfaults atm [CI SKIP] --- Makefile | 12 ++++++--- src/main.v | 14 ++++++---- src/repo/arch.v | 62 +++++++++++++++++++++++++++++++++++++++++++ src/repo/multi.v | 68 ++++++++++++++++++++++++++++++++++++++++++++++++ src/repo/repo.v | 17 +++--------- src/repo/sync.v | 9 ++++--- src/routes.v | 12 +++------ 7 files changed, 162 insertions(+), 32 deletions(-) create mode 100644 src/repo/arch.v create mode 100644 src/repo/multi.v diff --git a/Makefile b/Makefile index 1cf4d3f..8fa5dce 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,9 @@ SOURCES != find '$(SRC_DIR)' -iname '*.v' V_RELEASE := weekly.2022.05 V_PATH ?= v-$(V_RELEASE)/v V := $(V_PATH) -showcc -gc boehm +ENV := API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repos PKG_DIR=data/pkgs LOG_LEVEL=DEBUG +.PHONY: all all: vieter # =====COMPILATION===== @@ -32,19 +34,23 @@ c: # =====EXECUTION===== +# Run gdb on the debug binary +.PHONY: gdb +gdb: vieter + $(ENV) gdb ./vieter # Run the server in the default 'data' directory .PHONY: run run: vieter - API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter + $(ENV) ./vieter .PHONY: run-prod run-prod: prod - API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./pvieter + $(ENV) ./pvieter # Same as run, but restart when the source code changes .PHONY: watch watch: - API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG $(V) watch run vieter + $(ENV) $(V) watch run vieter # =====OTHER===== diff --git a/src/main.v b/src/main.v index cddd9ae..f8628dc 100644 --- a/src/main.v +++ b/src/main.v @@ -18,7 +18,9 @@ pub: api_key string [required; web_global] dl_dir string [required; web_global] pub mut: - repo repo.Repo [required; web_global] + repo repo.MultiArchRepo [required; web_global] + default_repo string [required; web_global] + default_arch string [required; web_global] } [noreturn] @@ -85,11 +87,11 @@ fn main() { dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { exit_with_message(1, 'No download directory was configured.') } + default_repo := os.getenv_opt('DEFAULT_REPO') or { 'vieter' } + default_arch := os.getenv_opt('DEFAULT_ARCH') or { 'x86_64' } - // This also creates the directories if needed - repo := repo.new(repo_dir, pkg_dir) or { - logger.error(err.msg) - exit(1) + repo := repo.new_multi(repo_dir, pkg_dir) or { + exit_with_message(1, 'Failed to create repo: $err.msg') } os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } @@ -99,5 +101,7 @@ fn main() { api_key: key dl_dir: dl_dir repo: repo + default_repo: default_repo + default_arch: default_arch }, port) } diff --git a/src/repo/arch.v b/src/repo/arch.v new file mode 100644 index 0000000..a5632f9 --- /dev/null +++ b/src/repo/arch.v @@ -0,0 +1,62 @@ +module repo + +import os +import package + +// A architecture repo acts as a single repo, but manages multiple repos +// internally that each manage a repo for a specific architecture +pub struct ArchRepo { +mut: + // Each key is the name of an architecture, e.g. x86_64 + repos shared map[string]Repo [required] +pub: + repo_dir string [required] + pkg_dir string [required] +} + +pub fn new_arch(repo_dir string, pkg_dir string) ?ArchRepo { + if !os.is_dir(pkg_dir) { + os.mkdir(pkg_dir) or { return error('Failed to create package directory: $err.msg') } + } + + mut repos := map[string]Repo{} + + if !os.is_dir(repo_dir) { + os.mkdir(repo_dir) or { return error('Failed to create repo directory: $err.msg') } + } + // If the directory already exists, we can check which repos already exist + else { + for name in os.ls(repo_dir) ? { + d := os.join_path_single(repo_dir, name) + + if os.is_dir(d) { + // The name is expected to be a correct architecture name + repos[name] = new(d, pkg_dir) ? + } + } + } + + return ArchRepo{ + repo_dir: repo_dir + pkg_dir: pkg_dir + repos: repos + } +} + +fn (r &ArchRepo) add(pkg &package.Pkg) ?bool { + // TODO add 'any' architecture to every repo + arch := pkg.info.arch + + lock r.repos { + // If a repo for this architecture doesn't exist yet, create it + if arch !in r.repos { + r.repos[arch] = new(os.join_path_single(r.repo_dir, arch), r.pkg_dir) ? + } + } + + repo := rlock r.repos { + r.repos[arch] + } + + return repo.add(pkg) +} diff --git a/src/repo/multi.v b/src/repo/multi.v new file mode 100644 index 0000000..bf83dfe --- /dev/null +++ b/src/repo/multi.v @@ -0,0 +1,68 @@ +module repo + +import os +import package + +// Represents a collection of Repos +pub struct MultiArchRepo { +mut: + repos shared map[string]ArchRepo [required] +pub: + pkg_dir string [required] + repo_dir string [required] +} + +// Initializes a new multi-repository. +pub fn new_multi(repo_dir string, pkg_dir string) ?MultiArchRepo { + mut repos := map[string]ArchRepo{} + + // We initialize a repo for each directory inside the repos dir + if !os.is_dir(repo_dir) { + os.mkdir_all(repo_dir) or { + return error("Failed to create directory '$repo_dir': $err.msg") + } + } else { + for name in os.ls(repo_dir) ? { + d := os.join_path_single(repo_dir, name) + + if os.is_dir(d) { + repos[name] = new_arch(d, pkg_dir) ? + } + } + } + + return MultiArchRepo{ + pkg_dir: pkg_dir + repo_dir: repo_dir + repos: repos + } +} + +pub fn (r &MultiArchRepo) add_from_path(repo_name string, pkg_path string) ?(bool, &package.Pkg) { + // First, we create the repo if it isn't present yet + repo := lock r.repos { + if repo_name !in r.repos { + r.repos[repo_name] = new_arch(os.join_path_single(r.repo_dir, repo_name), + r.pkg_dir) ? + } + + r.repos[repo_name] ? + } + + // We read in the package file + pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') } + + added := repo.add(pkg) ? + + // Move over package to pkg dir if it was successfully added + if added { + dest_path := os.real_path(os.join_path_single(r.pkg_dir, pkg.filename())) + + // Only move the file if it's not already in the package directory + if dest_path != os.real_path(pkg_path) { + os.mv(pkg_path, dest_path) ? + } + } + + return added, &pkg +} diff --git a/src/repo/repo.v b/src/repo/repo.v index 554b744..d9f2d0d 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -20,20 +20,14 @@ pub: pkg_dir string [required] } -pub struct RepoAddResult { -pub: - added bool [required] - pkg &package.Pkg [required] -} - // new creates a new Repo & creates the directories as needed pub fn new(repo_dir string, pkg_dir string) ?Repo { if !os.is_dir(repo_dir) { - os.mkdir_all(repo_dir) or { return error('Failed to create repo directory: $err.msg') } + os.mkdir(repo_dir) or { return error('Failed to create repo directory: $err.msg') } } if !os.is_dir(pkg_dir) { - os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg') } + os.mkdir(pkg_dir) or { return error('Failed to create package directory: $err.msg') } } return Repo{ @@ -44,7 +38,7 @@ pub fn new(repo_dir string, pkg_dir string) ?Repo { // add_from_path adds a package from an arbitrary path & moves it into the pkgs // directory if necessary. -pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult { +pub fn (r &Repo) add_from_path(pkg_path string) ?(bool, &package.Pkg) { pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') } added := r.add(pkg) ? @@ -59,10 +53,7 @@ pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult { } } - return RepoAddResult{ - added: added - pkg: &pkg - } + return added, &pkg } // add adds a given Pkg to the repository diff --git a/src/repo/sync.v b/src/repo/sync.v index d6080e0..1bea816 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -2,6 +2,10 @@ module repo import os +const db_filename = 'repo.db.tar.gz' + +const files_filename = 'repo.files.tar.gz' + fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &string, inner_path &string) { st := C.stat{} @@ -31,7 +35,6 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri // Re-generate the repo archive files fn (r &Repo) sync() ? { - // TODO also write files archive lock r.mutex { a_db := C.archive_write_new() a_files := C.archive_write_new() @@ -44,8 +47,8 @@ fn (r &Repo) sync() ? { C.archive_write_add_filter_gzip(a_files) C.archive_write_set_format_pax_restricted(a_files) - db_path := os.join_path_single(r.repo_dir, 'vieter.db.tar.gz') - files_path := os.join_path_single(r.repo_dir, 'vieter.files.tar.gz') + db_path := os.join_path_single(r.repo_dir, repo.db_filename) + files_path := os.join_path_single(r.repo_dir, repo.files_filename) C.archive_write_open_filename(a_db, &char(db_path.str)) C.archive_write_open_filename(a_files, &char(files_path.str)) diff --git a/src/routes.v b/src/routes.v index 8b7ddeb..5bcac50 100644 --- a/src/routes.v +++ b/src/routes.v @@ -21,10 +21,6 @@ fn pretty_bytes(bytes int) string { return '${n:.2}${prefixes[i]}' } -fn is_pkg_name(s string) bool { - return s.contains('.pkg') -} - // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. ['/health'; get] @@ -82,22 +78,22 @@ fn (mut app App) put_package() web.Result { return app.text("Content-Type header isn't set.") } - res := app.repo.add_from_path(pkg_path) or { + added, pkg := app.repo.add_from_path(app.default_repo, pkg_path) or { app.lerror('Error while adding package: $err.msg') os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } return app.text('Failed to add package.') } - if !res.added { + if !added { os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } - app.lwarn("Duplicate package '$res.pkg.full_name()'.") + app.lwarn("Duplicate package '$pkg.full_name()'.") return app.text('File already exists.') } - app.linfo("Added '$res.pkg.full_name()' to repository.") + app.linfo("Added '$pkg.full_name()' to repository.") return app.text('Package added successfully.') }