From a47cace2964740833916e181084273cec8381a48 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 16:33:06 +0200 Subject: [PATCH] Very alpha support for multiple & multi-arch repos --- Makefile | 4 +- src/env.v | 2 +- src/package.v | 2 +- src/repo/repo.v | 93 ++++++++++++++++++++++++++++++--------------- src/repo/sync.v | 15 ++++---- src/server/routes.v | 8 ++-- src/server/server.v | 4 +- test.py | 2 +- 8 files changed, 83 insertions(+), 47 deletions(-) diff --git a/Makefile b/Makefile index 7b0fb1b1..9996cc3c 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ dvieter: $(SOURCES) gdb: dvieter VIETER_API_KEY=test \ VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ + VIETER_DATA_DIR=data/repo \ VIETER_PKG_DIR=data/pkgs \ VIETER_LOG_LEVEL=DEBUG \ VIETER_REPOS_FILE=data/repos.json \ @@ -60,7 +60,7 @@ cli-prod: run: vieter VIETER_API_KEY=test \ VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ + VIETER_DATA_DIR=data/repo \ VIETER_PKG_DIR=data/pkgs \ VIETER_LOG_LEVEL=DEBUG \ VIETER_REPOS_FILE=data/repos.json \ diff --git a/src/env.v b/src/env.v index bd544cfc..3e71a091 100644 --- a/src/env.v +++ b/src/env.v @@ -16,7 +16,7 @@ pub: pkg_dir string download_dir string api_key string - repo_dir string + data_dir string repos_file string } diff --git a/src/package.v b/src/package.v index d643ff1e..e0eeac28 100644 --- a/src/package.v +++ b/src/package.v @@ -101,7 +101,7 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { // read_pkg extracts the file list & .PKGINFO contents from an archive // NOTE: this command currently only supports zstd-compressed tarballs -pub fn read_pkg(pkg_path string) ?Pkg { +pub fn read_pkg_archive(pkg_path string) ?Pkg { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") } diff --git a/src/repo/repo.v b/src/repo/repo.v index f1419aca..d9ae06e3 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -4,15 +4,20 @@ import os import package import util -// This struct manages a single repository. -pub struct Repo { +// Manages a group of repositories. Each repository contains one or more +// arch-repositories, each of which represent a specific architecture. +pub struct RepoGroupManager { mut: mutex shared util.Dummy pub: - // Where to store repository files - repo_dir string [required] - // Where to find packages; packages are expected to all be in the same directory + // Where to store repositories' files + data_dir string [required] + // Where packages are stored; each architecture gets its own subdirectory pkg_dir string [required] + // The default architecture to use for a repository. In reality, this value + // is only required when a package with architecture "any" is added as the + // first package to a repository. + default_arch string [required] } pub struct RepoAddResult { @@ -21,28 +26,29 @@ pub: pkg &package.Pkg [required] } -// new creates a new Repo & creates the directories as needed -pub fn new(repo_dir string, pkg_dir string) ?Repo { - if !os.is_dir(repo_dir) { - os.mkdir_all(repo_dir) or { return error('Failed to create repo directory: $err.msg') } +// new creates a new RepoGroupManager & creates the directories as needed +pub fn new(data_dir string, pkg_dir string, default_arch string) ?RepoGroupManager { + if !os.is_dir(data_dir) { + os.mkdir_all(data_dir) or { return error('Failed to create repo directory: $err.msg') } } if !os.is_dir(pkg_dir) { os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg') } } - return Repo{ - repo_dir: repo_dir + return RepoGroupManager{ + data_dir: data_dir pkg_dir: pkg_dir + default_arch: default_arch } } // add_from_path adds a package from an arbitrary path & moves it into the pkgs // directory if necessary. -pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult { - pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') } +pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult { + pkg := package.read_pkg_archive(pkg_path) or { return error('Failed to read package file: $err.msg') } - added := r.add(pkg) ? + added := r.add_pkg_in_repo(repo, pkg) ? // If the add was successful, we move the file to the packages directory if added { @@ -60,9 +66,33 @@ pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult { } } -// add adds a given Pkg to the repository -fn (r &Repo) add(pkg &package.Pkg) ?bool { - pkg_dir := r.pkg_path(pkg) +fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { + if pkg.info.arch == "any" { + repo_dir := os.join_path_single(r.data_dir, repo) + + // We get a listing of all currently present arch-repos in the given repo + mut arch_repos := os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, it))) + + if arch_repos.len == 0 { + arch_repos << r.default_arch + } + + for arch in arch_repos { + r.add_pkg_in_arch_repo(repo, arch, pkg) ? + } + }else{ + r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) ? + } + + // TODO properly handle this + return true +} + +// add_pkg_in_repo adds the given package to the specified repo. A repo is an +// arbitrary subdirectory of r.repo_dir, but in practice, it will always be an +// architecture-specific version of some sub-repository. +fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool { + pkg_dir := os.join_path(r.data_dir, repo, arch, '$pkg.info.name-$pkg.info.version') // We can't add the same package twice if os.exists(pkg_dir) { @@ -70,9 +100,9 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool { } // We remove the older package version first, if present - r.remove(pkg.info.name, false) ? + r.remove_pkg_from_arch_repo(repo, arch, pkg, false) ? - os.mkdir(pkg_dir) or { return error('Failed to create package directory.') } + os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') } os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or { os.rmdir_all(pkg_dir) ? @@ -85,27 +115,35 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool { return error('Failed to write files file.') } - r.sync() ? + r.sync(repo, arch) ? return true } // remove removes a package from the database. It returns false if the package // wasn't present in the database. -fn (r &Repo) remove(pkg_name string, sync bool) ?bool { +fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg &package.Pkg, sync bool) ?bool { + repo_dir := os.join_path(r.data_dir, repo, arch) + + // If the repository doesn't exist yet, the result is automatically false + if !os.exists(repo_dir) { + return false + } + // We iterate over every directory in the repo dir - for d in os.ls(r.repo_dir) ? { + // TODO filter so we only check directories + for d in os.ls(repo_dir) ? { name := d.split('-')#[..-2].join('-') - if name == pkg_name { + if name == pkg.info.name { // We lock the mutex here to prevent other routines from creating a // new archive while we removed an entry lock r.mutex { - os.rmdir_all(os.join_path_single(r.repo_dir, d)) ? + os.rmdir_all(os.join_path_single(repo_dir, d)) ? } if sync { - r.sync() ? + r.sync(repo, arch) ? } return true @@ -114,8 +152,3 @@ fn (r &Repo) remove(pkg_name string, sync bool) ?bool { return false } - -// Returns the path where the given package's desc & files files are stored -fn (r &Repo) pkg_path(pkg &package.Pkg) string { - return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version') -} diff --git a/src/repo/sync.v b/src/repo/sync.v index d6080e07..c0c0de95 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -30,8 +30,9 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri } // Re-generate the repo archive files -fn (r &Repo) sync() ? { - // TODO also write files archive +fn (r &RepoGroupManager) sync(repo string, arch string) ? { + subrepo_path := os.join_path(r.data_dir, repo, arch) + lock r.mutex { a_db := C.archive_write_new() a_files := C.archive_write_new() @@ -44,18 +45,18 @@ fn (r &Repo) sync() ? { C.archive_write_add_filter_gzip(a_files) C.archive_write_set_format_pax_restricted(a_files) - db_path := os.join_path_single(r.repo_dir, 'vieter.db.tar.gz') - files_path := os.join_path_single(r.repo_dir, 'vieter.files.tar.gz') + db_path := os.join_path_single(subrepo_path, 'vieter.db.tar.gz') + files_path := os.join_path_single(subrepo_path, 'vieter.files.tar.gz') C.archive_write_open_filename(a_db, &char(db_path.str)) C.archive_write_open_filename(a_files, &char(files_path.str)) // Iterate over each directory - for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, + for d in os.ls(subrepo_path) ?.filter(os.is_dir(os.join_path_single(subrepo_path, it))) { // desc mut inner_path := os.join_path_single(d, 'desc') - mut actual_path := os.join_path_single(r.repo_dir, inner_path) + mut actual_path := os.join_path_single(subrepo_path, inner_path) archive_add_entry(a_db, entry, actual_path, inner_path) archive_add_entry(a_files, entry, actual_path, inner_path) @@ -64,7 +65,7 @@ fn (r &Repo) sync() ? { // files inner_path = os.join_path_single(d, 'files') - actual_path = os.join_path_single(r.repo_dir, inner_path) + actual_path = os.join_path_single(subrepo_path, inner_path) archive_add_entry(a_files, entry, actual_path, inner_path) diff --git a/src/server/routes.v b/src/server/routes.v index 0090666e..9d92192a 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -8,6 +8,8 @@ import rand import util import net.http +const default_repo = "vieter" + // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. ['/health'; get] @@ -21,9 +23,9 @@ fn (mut app App) get_root(filename string) web.Result { mut full_path := '' if filename.ends_with('.db') || filename.ends_with('.files') { - full_path = os.join_path_single(app.repo.repo_dir, '${filename}.tar.gz') + full_path = os.join_path_single(app.repo.data_dir, '${filename}.tar.gz') } else if filename.ends_with('.db.tar.gz') || filename.ends_with('.files.tar.gz') { - full_path = os.join_path_single(app.repo.repo_dir, '$filename') + full_path = os.join_path_single(app.repo.data_dir, '$filename') } else { full_path = os.join_path_single(app.repo.pkg_dir, filename) } @@ -74,7 +76,7 @@ fn (mut app App) put_package() web.Result { return app.text("Content-Type header isn't set.") } - res := app.repo.add_from_path(pkg_path) or { + res := app.repo.add_pkg_from_path(default_repo, pkg_path) or { app.lerror('Error while adding package: $err.msg') os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } diff --git a/src/server/server.v b/src/server/server.v index 4b31b99e..27321ed1 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -14,7 +14,7 @@ struct App { pub: conf env.ServerConfig [required; web_global] pub mut: - repo repo.Repo [required; web_global] + repo repo.RepoGroupManager [required; web_global] // This is used to claim the file lock on the repos file git_mutex shared util.Dummy } @@ -42,7 +42,7 @@ pub fn server() ? { } // This also creates the directories if needed - repo := repo.new(conf.repo_dir, conf.pkg_dir) or { + repo := repo.new(conf.data_dir, conf.pkg_dir, "x86_64") or { logger.error(err.msg) exit(1) } diff --git a/test.py b/test.py index 5721310b..cb817b7e 100644 --- a/test.py +++ b/test.py @@ -46,7 +46,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len): "pkgname": name, "pkgbase": name, "pkgver": ver, - "arch": "x86_64" + "arch": "any" } return "\n".join(f"{key} = {value}" for key, value in data.items())