From 013ce511d774dc60d78423fd2dec954ab5cbd1b2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 15:10:45 +0200 Subject: [PATCH 01/17] Loosed up SRCINFO field requirements (fixes #111) --- src/package.v | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/package.v b/src/package.v index 4e3b97f..d643ff1 100644 --- a/src/package.v +++ b/src/package.v @@ -72,14 +72,11 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { 'pkgbase' { pkg_info.base = value } 'pkgver' { pkg_info.version = value } 'pkgdesc' { pkg_info.description = value } - 'csize' { continue } 'size' { pkg_info.size = value.int() } 'url' { pkg_info.url = value } 'arch' { pkg_info.arch = value } 'builddate' { pkg_info.build_date = value.int() } 'packager' { pkg_info.packager = value } - 'md5sum' { continue } - 'sha256sum' { continue } 'pgpsig' { pkg_info.pgpsig = value } 'pgpsigsize' { pkg_info.pgpsigsize = value.int() } // Array values @@ -92,7 +89,10 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { 'optdepend' { pkg_info.optdepends << value } 'makedepend' { pkg_info.makedepends << value } 'checkdepend' { pkg_info.checkdepends << value } - else { return error("Invalid key '$key'.") } + // There's no real point in trying to exactly manage which fields + // are allowed, so we just ignore any we don't explicitely need for + // in the db file + else { continue } } } -- 2.40.1 From a47cace2964740833916e181084273cec8381a48 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 16:33:06 +0200 Subject: [PATCH 02/17] Very alpha support for multiple & multi-arch repos --- Makefile | 4 +- src/env.v | 2 +- src/package.v | 2 +- src/repo/repo.v | 93 ++++++++++++++++++++++++++++++--------------- src/repo/sync.v | 15 ++++---- src/server/routes.v | 8 ++-- src/server/server.v | 4 +- test.py | 2 +- 8 files changed, 83 insertions(+), 47 deletions(-) diff --git a/Makefile b/Makefile index 7b0fb1b..9996cc3 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ dvieter: $(SOURCES) gdb: dvieter VIETER_API_KEY=test \ VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ + VIETER_DATA_DIR=data/repo \ VIETER_PKG_DIR=data/pkgs \ VIETER_LOG_LEVEL=DEBUG \ VIETER_REPOS_FILE=data/repos.json \ @@ -60,7 +60,7 @@ cli-prod: run: vieter VIETER_API_KEY=test \ VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ + VIETER_DATA_DIR=data/repo \ VIETER_PKG_DIR=data/pkgs \ VIETER_LOG_LEVEL=DEBUG \ VIETER_REPOS_FILE=data/repos.json \ diff --git a/src/env.v b/src/env.v index bd544cf..3e71a09 100644 --- a/src/env.v +++ b/src/env.v @@ -16,7 +16,7 @@ pub: pkg_dir string download_dir string api_key string - repo_dir string + data_dir string repos_file string } diff --git a/src/package.v b/src/package.v index d643ff1..e0eeac2 100644 --- a/src/package.v +++ b/src/package.v @@ -101,7 +101,7 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { // read_pkg extracts the file list & .PKGINFO contents from an archive // NOTE: this command currently only supports zstd-compressed tarballs -pub fn read_pkg(pkg_path string) ?Pkg { +pub fn read_pkg_archive(pkg_path string) ?Pkg { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") } diff --git a/src/repo/repo.v b/src/repo/repo.v index f1419ac..d9ae06e 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -4,15 +4,20 @@ import os import package import util -// This struct manages a single repository. -pub struct Repo { +// Manages a group of repositories. Each repository contains one or more +// arch-repositories, each of which represent a specific architecture. +pub struct RepoGroupManager { mut: mutex shared util.Dummy pub: - // Where to store repository files - repo_dir string [required] - // Where to find packages; packages are expected to all be in the same directory + // Where to store repositories' files + data_dir string [required] + // Where packages are stored; each architecture gets its own subdirectory pkg_dir string [required] + // The default architecture to use for a repository. In reality, this value + // is only required when a package with architecture "any" is added as the + // first package to a repository. + default_arch string [required] } pub struct RepoAddResult { @@ -21,28 +26,29 @@ pub: pkg &package.Pkg [required] } -// new creates a new Repo & creates the directories as needed -pub fn new(repo_dir string, pkg_dir string) ?Repo { - if !os.is_dir(repo_dir) { - os.mkdir_all(repo_dir) or { return error('Failed to create repo directory: $err.msg') } +// new creates a new RepoGroupManager & creates the directories as needed +pub fn new(data_dir string, pkg_dir string, default_arch string) ?RepoGroupManager { + if !os.is_dir(data_dir) { + os.mkdir_all(data_dir) or { return error('Failed to create repo directory: $err.msg') } } if !os.is_dir(pkg_dir) { os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg') } } - return Repo{ - repo_dir: repo_dir + return RepoGroupManager{ + data_dir: data_dir pkg_dir: pkg_dir + default_arch: default_arch } } // add_from_path adds a package from an arbitrary path & moves it into the pkgs // directory if necessary. -pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult { - pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') } +pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult { + pkg := package.read_pkg_archive(pkg_path) or { return error('Failed to read package file: $err.msg') } - added := r.add(pkg) ? + added := r.add_pkg_in_repo(repo, pkg) ? // If the add was successful, we move the file to the packages directory if added { @@ -60,9 +66,33 @@ pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult { } } -// add adds a given Pkg to the repository -fn (r &Repo) add(pkg &package.Pkg) ?bool { - pkg_dir := r.pkg_path(pkg) +fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { + if pkg.info.arch == "any" { + repo_dir := os.join_path_single(r.data_dir, repo) + + // We get a listing of all currently present arch-repos in the given repo + mut arch_repos := os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, it))) + + if arch_repos.len == 0 { + arch_repos << r.default_arch + } + + for arch in arch_repos { + r.add_pkg_in_arch_repo(repo, arch, pkg) ? + } + }else{ + r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) ? + } + + // TODO properly handle this + return true +} + +// add_pkg_in_repo adds the given package to the specified repo. A repo is an +// arbitrary subdirectory of r.repo_dir, but in practice, it will always be an +// architecture-specific version of some sub-repository. +fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool { + pkg_dir := os.join_path(r.data_dir, repo, arch, '$pkg.info.name-$pkg.info.version') // We can't add the same package twice if os.exists(pkg_dir) { @@ -70,9 +100,9 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool { } // We remove the older package version first, if present - r.remove(pkg.info.name, false) ? + r.remove_pkg_from_arch_repo(repo, arch, pkg, false) ? - os.mkdir(pkg_dir) or { return error('Failed to create package directory.') } + os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') } os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or { os.rmdir_all(pkg_dir) ? @@ -85,27 +115,35 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool { return error('Failed to write files file.') } - r.sync() ? + r.sync(repo, arch) ? return true } // remove removes a package from the database. It returns false if the package // wasn't present in the database. -fn (r &Repo) remove(pkg_name string, sync bool) ?bool { +fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg &package.Pkg, sync bool) ?bool { + repo_dir := os.join_path(r.data_dir, repo, arch) + + // If the repository doesn't exist yet, the result is automatically false + if !os.exists(repo_dir) { + return false + } + // We iterate over every directory in the repo dir - for d in os.ls(r.repo_dir) ? { + // TODO filter so we only check directories + for d in os.ls(repo_dir) ? { name := d.split('-')#[..-2].join('-') - if name == pkg_name { + if name == pkg.info.name { // We lock the mutex here to prevent other routines from creating a // new archive while we removed an entry lock r.mutex { - os.rmdir_all(os.join_path_single(r.repo_dir, d)) ? + os.rmdir_all(os.join_path_single(repo_dir, d)) ? } if sync { - r.sync() ? + r.sync(repo, arch) ? } return true @@ -114,8 +152,3 @@ fn (r &Repo) remove(pkg_name string, sync bool) ?bool { return false } - -// Returns the path where the given package's desc & files files are stored -fn (r &Repo) pkg_path(pkg &package.Pkg) string { - return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version') -} diff --git a/src/repo/sync.v b/src/repo/sync.v index d6080e0..c0c0de9 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -30,8 +30,9 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri } // Re-generate the repo archive files -fn (r &Repo) sync() ? { - // TODO also write files archive +fn (r &RepoGroupManager) sync(repo string, arch string) ? { + subrepo_path := os.join_path(r.data_dir, repo, arch) + lock r.mutex { a_db := C.archive_write_new() a_files := C.archive_write_new() @@ -44,18 +45,18 @@ fn (r &Repo) sync() ? { C.archive_write_add_filter_gzip(a_files) C.archive_write_set_format_pax_restricted(a_files) - db_path := os.join_path_single(r.repo_dir, 'vieter.db.tar.gz') - files_path := os.join_path_single(r.repo_dir, 'vieter.files.tar.gz') + db_path := os.join_path_single(subrepo_path, 'vieter.db.tar.gz') + files_path := os.join_path_single(subrepo_path, 'vieter.files.tar.gz') C.archive_write_open_filename(a_db, &char(db_path.str)) C.archive_write_open_filename(a_files, &char(files_path.str)) // Iterate over each directory - for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, + for d in os.ls(subrepo_path) ?.filter(os.is_dir(os.join_path_single(subrepo_path, it))) { // desc mut inner_path := os.join_path_single(d, 'desc') - mut actual_path := os.join_path_single(r.repo_dir, inner_path) + mut actual_path := os.join_path_single(subrepo_path, inner_path) archive_add_entry(a_db, entry, actual_path, inner_path) archive_add_entry(a_files, entry, actual_path, inner_path) @@ -64,7 +65,7 @@ fn (r &Repo) sync() ? { // files inner_path = os.join_path_single(d, 'files') - actual_path = os.join_path_single(r.repo_dir, inner_path) + actual_path = os.join_path_single(subrepo_path, inner_path) archive_add_entry(a_files, entry, actual_path, inner_path) diff --git a/src/server/routes.v b/src/server/routes.v index 0090666..9d92192 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -8,6 +8,8 @@ import rand import util import net.http +const default_repo = "vieter" + // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. ['/health'; get] @@ -21,9 +23,9 @@ fn (mut app App) get_root(filename string) web.Result { mut full_path := '' if filename.ends_with('.db') || filename.ends_with('.files') { - full_path = os.join_path_single(app.repo.repo_dir, '${filename}.tar.gz') + full_path = os.join_path_single(app.repo.data_dir, '${filename}.tar.gz') } else if filename.ends_with('.db.tar.gz') || filename.ends_with('.files.tar.gz') { - full_path = os.join_path_single(app.repo.repo_dir, '$filename') + full_path = os.join_path_single(app.repo.data_dir, '$filename') } else { full_path = os.join_path_single(app.repo.pkg_dir, filename) } @@ -74,7 +76,7 @@ fn (mut app App) put_package() web.Result { return app.text("Content-Type header isn't set.") } - res := app.repo.add_from_path(pkg_path) or { + res := app.repo.add_pkg_from_path(default_repo, pkg_path) or { app.lerror('Error while adding package: $err.msg') os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } diff --git a/src/server/server.v b/src/server/server.v index 4b31b99..27321ed 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -14,7 +14,7 @@ struct App { pub: conf env.ServerConfig [required; web_global] pub mut: - repo repo.Repo [required; web_global] + repo repo.RepoGroupManager [required; web_global] // This is used to claim the file lock on the repos file git_mutex shared util.Dummy } @@ -42,7 +42,7 @@ pub fn server() ? { } // This also creates the directories if needed - repo := repo.new(conf.repo_dir, conf.pkg_dir) or { + repo := repo.new(conf.data_dir, conf.pkg_dir, "x86_64") or { logger.error(err.msg) exit(1) } diff --git a/test.py b/test.py index 5721310..cb817b7 100644 --- a/test.py +++ b/test.py @@ -46,7 +46,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len): "pkgname": name, "pkgbase": name, "pkgver": ver, - "arch": "x86_64" + "arch": "any" } return "\n".join(f"{key} = {value}" for key, value in data.items()) -- 2.40.1 From 014ade50929d22224c78cd493f7f4151895387cc Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 17:00:11 +0200 Subject: [PATCH 03/17] Updated routes for multi-repo setup (untested) --- src/repo/repo.v | 8 ++++++-- src/server/routes.v | 27 +++++++++++++++------------ test.py | 2 +- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/repo/repo.v b/src/repo/repo.v index d9ae06e..bf5ab02 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -70,8 +70,12 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { if pkg.info.arch == "any" { repo_dir := os.join_path_single(r.data_dir, repo) - // We get a listing of all currently present arch-repos in the given repo - mut arch_repos := os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, it))) + mut arch_repos := []string{} + + if os.exists(repo_dir) { + // We get a listing of all currently present arch-repos in the given repo + arch_repos = os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, it))) + } if arch_repos.len == 0 { arch_repos << r.default_arch diff --git a/src/server/routes.v b/src/server/routes.v index 9d92192..55964db 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -8,8 +8,6 @@ import rand import util import net.http -const default_repo = "vieter" - // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. ['/health'; get] @@ -17,15 +15,20 @@ pub fn (mut app App) healthcheck() web.Result { return app.text('Healthy') } -// get_root handles a GET request for a file on the root -['/:filename'; get; head] -fn (mut app App) get_root(filename string) web.Result { +['/:repo/:arch/:filename'; get; head] +fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result { mut full_path := '' - if filename.ends_with('.db') || filename.ends_with('.files') { - full_path = os.join_path_single(app.repo.data_dir, '${filename}.tar.gz') - } else if filename.ends_with('.db.tar.gz') || filename.ends_with('.files.tar.gz') { - full_path = os.join_path_single(app.repo.data_dir, '$filename') + db_exts := ['.db', '.files', '.db.tar.gz', '.files.tar.gz'] + + if db_exts.any(filename.ends_with(it)) { + full_path = os.join_path(app.repo.data_dir, repo, arch, filename) + + // repo-add does this using symlinks, but we just change the requested + // path + if !full_path.ends_with('.tar.gz') { + full_path += '.tar.gz' + } } else { full_path = os.join_path_single(app.repo.pkg_dir, filename) } @@ -42,8 +45,8 @@ fn (mut app App) get_root(filename string) web.Result { return app.file(full_path) } -['/publish'; post] -fn (mut app App) put_package() web.Result { +['/:repo/publish'; post] +fn (mut app App) put_package(repo string) web.Result { if !app.is_authorized() { return app.text('Unauthorized.') } @@ -76,7 +79,7 @@ fn (mut app App) put_package() web.Result { return app.text("Content-Type header isn't set.") } - res := app.repo.add_pkg_from_path(default_repo, pkg_path) or { + res := app.repo.add_pkg_from_path(repo, pkg_path) or { app.lerror('Error while adding package: $err.msg') os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } diff --git a/test.py b/test.py index cb817b7..9b0116e 100644 --- a/test.py +++ b/test.py @@ -97,7 +97,7 @@ async def upload_random_package(tar_path, sem): async with sem: with open(tar_path, 'rb') as f: async with aiohttp.ClientSession() as s: - async with s.post("http://localhost:8000/publish", data=f.read(), headers={"x-api-key": "test"}) as r: + async with s.post("http://localhost:8000/vieter/publish", data=f.read(), headers={"x-api-key": "test"}) as r: return await check_output(r) -- 2.40.1 From cb2ba862000b3b492c1cd4e80bc8aebbae12aa47 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 18:22:30 +0200 Subject: [PATCH 04/17] Updated logging for multi-repo setup --- CHANGELOG.md | 6 +++++- src/{archive.v => archive.c.v} | 0 src/repo/repo.v | 17 +++++++++-------- src/server/routes.v | 5 +++-- test.py | 4 ++-- 5 files changed, 19 insertions(+), 13 deletions(-) rename src/{archive.v => archive.c.v} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eaf477..754f04e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,17 +18,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Very basic build system * Build is triggered by separate cron container * Packages build on cron container's system - * Packages are always rebuilt, even if they haven't changed + * A HEAD request is used to determine whether a package should be rebuilt + or not * Hardcoded planning of builds * Builds are sequential * API for managing Git repositories to build * CLI to list, add & remove Git repos to build * Published packages on my Vieter instance +* Support for multiple repositories +* Support for multiple architectures per repository ## Fixed * Each package can now only have one version in the repository at once (required by Pacman) +* Packages with unknown fields in .PKGINFO are now allowed ## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0) diff --git a/src/archive.v b/src/archive.c.v similarity index 100% rename from src/archive.v rename to src/archive.c.v diff --git a/src/repo/repo.v b/src/repo/repo.v index bf5ab02..e1c9fdb 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -81,15 +81,16 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { arch_repos << r.default_arch } - for arch in arch_repos { - r.add_pkg_in_arch_repo(repo, arch, pkg) ? - } - }else{ - r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) ? - } + mut added := false - // TODO properly handle this - return true + for arch in arch_repos { + added = added || r.add_pkg_in_arch_repo(repo, arch, pkg) ? + } + + return added + }else{ + return r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) + } } // add_pkg_in_repo adds the given package to the specified repo. A repo is an diff --git a/src/server/routes.v b/src/server/routes.v index 55964db..0f697f9 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -86,15 +86,16 @@ fn (mut app App) put_package(repo string) web.Result { return app.text('Failed to add package.') } + if !res.added { os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } - app.lwarn("Duplicate package '$res.pkg.full_name()'.") + app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo ($res.pkg.info.arch)'.") return app.text('File already exists.') } - app.linfo("Added '$res.pkg.full_name()' to repository.") + app.linfo("Added '$res.pkg.full_name()' to repo '$repo ($res.pkg.info.arch)'.") return app.text('Package added successfully.') } diff --git a/test.py b/test.py index 9b0116e..f1668d7 100644 --- a/test.py +++ b/test.py @@ -46,7 +46,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len): "pkgname": name, "pkgbase": name, "pkgver": ver, - "arch": "any" + "arch": "x86_64" } return "\n".join(f"{key} = {value}" for key, value in data.items()) @@ -97,7 +97,7 @@ async def upload_random_package(tar_path, sem): async with sem: with open(tar_path, 'rb') as f: async with aiohttp.ClientSession() as s: - async with s.post("http://localhost:8000/vieter/publish", data=f.read(), headers={"x-api-key": "test"}) as r: + async with s.post("http://localhost:8000/vieter2/publish", data=f.read(), headers={"x-api-key": "test"}) as r: return await check_output(r) -- 2.40.1 From 0dd4534e208cf84c3f1ecda18e2213295c634a63 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 19:54:49 +0200 Subject: [PATCH 05/17] Added extra comments; made linter happy --- src/package.v | 4 +-- src/repo/repo.v | 84 ++++++++++++++++++++++++++++----------------- src/server/server.v | 2 +- 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/src/package.v b/src/package.v index e0eeac2..8bd30d0 100644 --- a/src/package.v +++ b/src/package.v @@ -99,8 +99,8 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { return pkg_info } -// read_pkg extracts the file list & .PKGINFO contents from an archive -// NOTE: this command currently only supports zstd-compressed tarballs +// read_pkg_archive extracts the file list & .PKGINFO contents from an archive +// NOTE: this command only supports zstd- & gzip-compressed tarballs pub fn read_pkg_archive(pkg_path string) ?Pkg { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") diff --git a/src/repo/repo.v b/src/repo/repo.v index e1c9fdb..4e4fa34 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -16,7 +16,7 @@ pub: pkg_dir string [required] // The default architecture to use for a repository. In reality, this value // is only required when a package with architecture "any" is added as the - // first package to a repository. + // first package of a repository. default_arch string [required] } @@ -43,10 +43,14 @@ pub fn new(data_dir string, pkg_dir string, default_arch string) ?RepoGroupManag } } -// add_from_path adds a package from an arbitrary path & moves it into the pkgs -// directory if necessary. +// add_pkg_from_path adds a package to a given repo, given the file path to the +// pkg archive. It's a wrapper around add_pkg_in_repo that parses the archive +// file, passes the result to add_pkg_in_repo, and moves the archive to +// r.pkg_dir if it was successfully added. pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult { - pkg := package.read_pkg_archive(pkg_path) or { return error('Failed to read package file: $err.msg') } + pkg := package.read_pkg_archive(pkg_path) or { + return error('Failed to read package file: $err.msg') + } added := r.add_pkg_in_repo(repo, pkg) ? @@ -66,36 +70,48 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re } } +// add_pkg_in_repo adds a package to a given repo. This function is responsible +// for inspecting the package architecture. If said architecture is 'any', the +// package is added to each arch-repository within the given repo. If none +// exist, one is created for provided r.default_arch value. If the architecture +// isn't 'any', the package is only added to the specific architecture. fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { - if pkg.info.arch == "any" { - repo_dir := os.join_path_single(r.data_dir, repo) - - mut arch_repos := []string{} - - if os.exists(repo_dir) { - // We get a listing of all currently present arch-repos in the given repo - arch_repos = os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, it))) - } - - if arch_repos.len == 0 { - arch_repos << r.default_arch - } - - mut added := false - - for arch in arch_repos { - added = added || r.add_pkg_in_arch_repo(repo, arch, pkg) ? - } - - return added - }else{ + if pkg.info.arch != 'any' { return r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) } + + repo_dir := os.join_path_single(r.data_dir, repo) + + mut arch_repos := []string{} + + // If this is the first package to be added to the repository, it won't + // contain any arch-repos yet. + if os.exists(repo_dir) { + // We get a listing of all currently present arch-repos in the given repo + arch_repos = os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, + it))) + } + + if arch_repos.len == 0 { + arch_repos << r.default_arch + } + + mut added := false + + // We add the package to each repository. If any of the repositories + // return true, the result of the function is also true. + for arch in arch_repos { + added = added || r.add_pkg_in_arch_repo(repo, arch, pkg) ? + } + + return added } -// add_pkg_in_repo adds the given package to the specified repo. A repo is an -// arbitrary subdirectory of r.repo_dir, but in practice, it will always be an -// architecture-specific version of some sub-repository. +// add_pkg_in_arch_repo is the function that actually adds a package to a given +// arch-repo. It records the package's data in the arch-repo's desc & files +// files, and afterwards updates the db & files archives to reflect these +// changes. The function returns false if the package was already present in +// the repo, and true otherwise. fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool { pkg_dir := os.join_path(r.data_dir, repo, arch, '$pkg.info.name-$pkg.info.version') @@ -125,8 +141,9 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac return true } -// remove removes a package from the database. It returns false if the package -// wasn't present in the database. +// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It +// returns false if the package wasn't present in the database. It also +// optionally re-syncs the repo archives. fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg &package.Pkg, sync bool) ?bool { repo_dir := os.join_path(r.data_dir, repo, arch) @@ -138,11 +155,14 @@ fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg // We iterate over every directory in the repo dir // TODO filter so we only check directories for d in os.ls(repo_dir) ? { + // Because a repository only allows a single version of each package, + // we need only compare whether the name of the package is the same, + // not the version. name := d.split('-')#[..-2].join('-') if name == pkg.info.name { // We lock the mutex here to prevent other routines from creating a - // new archive while we removed an entry + // new archive while we remove an entry lock r.mutex { os.rmdir_all(os.join_path_single(repo_dir, d)) ? } diff --git a/src/server/server.v b/src/server/server.v index 27321ed..db2c9ae 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -42,7 +42,7 @@ pub fn server() ? { } // This also creates the directories if needed - repo := repo.new(conf.data_dir, conf.pkg_dir, "x86_64") or { + repo := repo.new(conf.data_dir, conf.pkg_dir, 'x86_64') or { logger.error(err.msg) exit(1) } -- 2.40.1 From aa632c7cd92a1b930929a6a6ce0bc6d3cb8a3676 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 20:26:58 +0200 Subject: [PATCH 06/17] Switched to correct filenames --- src/repo/sync.v | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/repo/sync.v b/src/repo/sync.v index c0c0de9..afb8a4b 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -45,8 +45,8 @@ fn (r &RepoGroupManager) sync(repo string, arch string) ? { C.archive_write_add_filter_gzip(a_files) C.archive_write_set_format_pax_restricted(a_files) - db_path := os.join_path_single(subrepo_path, 'vieter.db.tar.gz') - files_path := os.join_path_single(subrepo_path, 'vieter.files.tar.gz') + db_path := os.join_path_single(subrepo_path, '${repo}.db.tar.gz') + files_path := os.join_path_single(subrepo_path, '${repo}.files.tar.gz') C.archive_write_open_filename(a_db, &char(db_path.str)) C.archive_write_open_filename(a_files, &char(files_path.str)) -- 2.40.1 From 4139c50780cf14408b6b34a5eacb8920066d0f6c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 23:30:38 +0200 Subject: [PATCH 07/17] Small changes to Makefile --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9996cc3..7e3c7ea 100644 --- a/Makefile +++ b/Makefile @@ -70,9 +70,10 @@ run: vieter run-prod: prod VIETER_API_KEY=test \ VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ + VIETER_DATA_DIR=data/repo \ VIETER_PKG_DIR=data/pkgs \ VIETER_LOG_LEVEL=DEBUG \ + VIETER_REPOS_FILE=data/repos.json \ ./pvieter server # =====OTHER===== -- 2.40.1 From 56cb23cc7e16a1444927b245d5c92f114f97cc98 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 23:33:33 +0200 Subject: [PATCH 08/17] Just some changes to poke CI --- .editorconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 630e4fa..e23a3c7 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,4 +1,3 @@ -# top-most EditorConfig file root = true # Unix-style newlines with a newline ending every file -- 2.40.1 From 9d8491a77a3fee0d6a2a325fec98a98b065a20da Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 13:47:06 +0200 Subject: [PATCH 09/17] Changed behavior for default_arch variable --- src/repo/repo.v | 17 +++++++++++------ src/server/cli.v | 1 + src/server/routes.v | 4 ---- src/server/server.v | 7 ++++++- test.py | 4 ++-- vieter.toml | 1 + 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/repo/repo.v b/src/repo/repo.v index 4e4fa34..334c894 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -72,10 +72,13 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re // add_pkg_in_repo adds a package to a given repo. This function is responsible // for inspecting the package architecture. If said architecture is 'any', the -// package is added to each arch-repository within the given repo. If none -// exist, one is created for provided r.default_arch value. If the architecture -// isn't 'any', the package is only added to the specific architecture. +// package is added to each arch-repository within the given repo. A package of +// architecture 'any' will always be added to the arch-repo defined by +// r.default_arch. If this arch-repo doesn't exist yet, it will be created. If +// the architecture isn't 'any', the package is only added to the specific +// architecture. fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { + // A package without arch 'any' can be handled without any further checks if pkg.info.arch != 'any' { return r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) } @@ -84,15 +87,17 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { mut arch_repos := []string{} - // If this is the first package to be added to the repository, it won't - // contain any arch-repos yet. + // If this is the first package that's added to the repo, the directory + // won't exist yet if os.exists(repo_dir) { // We get a listing of all currently present arch-repos in the given repo arch_repos = os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, it))) } - if arch_repos.len == 0 { + // The default_arch should always be updated when a package with arch 'any' + // is added. + if !arch_repos.contains(r.default_arch) { arch_repos << r.default_arch } diff --git a/src/server/cli.v b/src/server/cli.v index 9820cf8..b3f37e3 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -12,6 +12,7 @@ pub: api_key string data_dir string repos_file string + default_arch string } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/routes.v b/src/server/routes.v index 55f6f12..8c8cdff 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -58,10 +58,6 @@ fn (mut app App) put_package(repo string) web.Result { // Generate a random filename for the temp file pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4()) - for os.exists(pkg_path) { - pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4()) - } - app.ldebug("Uploading $length bytes (${util.pretty_bytes(length.int())}) to '$pkg_path'.") // This is used to time how long it takes to upload a file diff --git a/src/server/server.v b/src/server/server.v index 7b59896..f93a5ae 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -20,6 +20,11 @@ pub mut: // server starts the web server & starts listening for requests pub fn server(conf Config) ? { + // Prevent using 'any' as the default arch + if conf.default_arch == 'any' { + util.exit_with_message(1, "'any' is not allowed as the value for default_arch.") + } + // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -39,7 +44,7 @@ pub fn server(conf Config) ? { } // This also creates the directories if needed - repo := repo.new(conf.data_dir, conf.pkg_dir, 'x86_64') or { + repo := repo.new(conf.data_dir, conf.pkg_dir, conf.default_arch) or { logger.error(err.msg) exit(1) } diff --git a/test.py b/test.py index f1668d7..9b0116e 100644 --- a/test.py +++ b/test.py @@ -46,7 +46,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len): "pkgname": name, "pkgbase": name, "pkgver": ver, - "arch": "x86_64" + "arch": "any" } return "\n".join(f"{key} = {value}" for key, value in data.items()) @@ -97,7 +97,7 @@ async def upload_random_package(tar_path, sem): async with sem: with open(tar_path, 'rb') as f: async with aiohttp.ClientSession() as s: - async with s.post("http://localhost:8000/vieter2/publish", data=f.read(), headers={"x-api-key": "test"}) as r: + async with s.post("http://localhost:8000/vieter/publish", data=f.read(), headers={"x-api-key": "test"}) as r: return await check_output(r) diff --git a/vieter.toml b/vieter.toml index f3904b6..10b3855 100644 --- a/vieter.toml +++ b/vieter.toml @@ -5,5 +5,6 @@ data_dir = "data" pkg_dir = "data/pkgs" log_level = "DEBUG" repos_file = "data/repos.json" +default_arch = "x86_64" address = "http://localhost:8000" -- 2.40.1 From 7eab7afa982ea24232460374210e5d2576b9288d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 14:28:21 +0200 Subject: [PATCH 10/17] Expanded git repos api to store repository to publish to --- src/git/cli.v | 12 ++++++------ src/git/client.v | 43 +++++++++++++++++++++++++++---------------- src/git/git.v | 2 ++ 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/src/git/cli.v b/src/git/cli.v index 4a066d5..376592d 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -26,14 +26,14 @@ pub fn cmd() cli.Command { }, cli.Command{ name: 'add' - required_args: 2 - usage: 'url branch arch...' + required_args: 4 + usage: 'url branch repo arch...' description: 'Add a new repository.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file') ? conf := env.load(config_file) ? - add(conf, cmd.args[0], cmd.args[1], cmd.args[2..]) ? + add(conf, cmd.args[0], cmd.args[1], cmd.args[2], cmd.args[3..]) ? } }, cli.Command{ @@ -56,12 +56,12 @@ fn list(conf Config) ? { repos := get_repos(conf.address, conf.api_key) ? for id, details in repos { - println('${id[..8]}\t$details.url\t$details.branch\t$details.arch') + println('${id[..8]}\t$details.url\t$details.branch\t$details.repo\t$details.arch') } } -fn add(conf Config, url string, branch string, arch []string) ? { - res := add_repo(conf.address, conf.api_key, url, branch, arch) ? +fn add(conf Config, url string, branch string, repo string, arch []string) ? { + res := add_repo(conf.address, conf.api_key, url, branch, repo, arch) ? println(res.message) } diff --git a/src/git/client.v b/src/git/client.v index 97fe9fb..7e1e55b 100644 --- a/src/git/client.v +++ b/src/git/client.v @@ -4,36 +4,47 @@ import json import response { Response } import net.http -// get_repos returns the current list of repos. -pub fn get_repos(address string, api_key string) ?map[string]GitRepo { - mut req := http.new_request(http.Method.get, '$address/api/repos', '') ? +fn send_request(method http.Method, address string, url string, api_key string, params map[string]string) ?Response { + mut full_url := '$address$url' + + if params.len > 0 { + params_str := params.keys().map('$it=${params[it]}').join('&') + + full_url = "$full_url?$params_str" + } + + mut req := http.new_request(method, full_url, '') ? req.add_custom_header('X-API-Key', api_key) ? res := req.do() ? - data := json.decode(Response, res.text) ? + data := json.decode(Response, res.text) ? + + return data +} + +// get_repos returns the current list of repos. +pub fn get_repos(address string, api_key string) ?map[string]GitRepo { + data := send_request(http.Method.get, address, '/api/repos', api_key, {}) ? return data.data } // add_repo adds a new repo to the server. -pub fn add_repo(address string, api_key string, url string, branch string, arch []string) ?Response { - mut req := http.new_request(http.Method.post, '$address/api/repos?url=$url&branch=$branch&arch=${arch.join(',')}', - '') ? - req.add_custom_header('X-API-Key', api_key) ? - - res := req.do() ? - data := json.decode(Response, res.text) ? +pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response { + params := { + 'url': url + 'branch': branch + 'repo': repo + 'arch': arch.join(',') + } + data := send_request(http.Method.post, address, '/api/repos', api_key, params) ? return data } // remove_repo removes the repo with the given ID from the server. pub fn remove_repo(address string, api_key string, id string) ?Response { - mut req := http.new_request(http.Method.delete, '$address/api/repos/$id', '') ? - req.add_custom_header('X-API-Key', api_key) ? - - res := req.do() ? - data := json.decode(Response, res.text) ? + data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, {}) ? return data } diff --git a/src/git/git.v b/src/git/git.v index c5390b6..eaec895 100644 --- a/src/git/git.v +++ b/src/git/git.v @@ -12,6 +12,8 @@ pub mut: // On which architectures the package is allowed to be built. In reality, // this controls which builders will periodically build the image. arch []string + // Which repo the builder should publish packages to + repo string } // patch_from_params patches a GitRepo from a map[string]string, usually -- 2.40.1 From 8ecac2ff28895151a2cc1adc0d4c306d3ef71581 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 14:40:49 +0200 Subject: [PATCH 11/17] Updated build system to respect repo arch settings --- src/build/build.v | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index c42c98d..31e7c3f 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -4,11 +4,14 @@ import docker import encoding.base64 import time import git +import os const container_build_dir = '/build' const build_image_repo = 'vieter-build' +const base_image = 'archlinux:latest' + fn create_build_image() ?string { commands := [ // Update repos & install required packages @@ -26,7 +29,7 @@ fn create_build_image() ?string { cmds_str := base64.encode_str(commands.join('\n')) c := docker.NewContainer{ - image: 'archlinux:latest' + image: base_image env: ['BUILD_SCRIPT=$cmds_str'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] @@ -60,27 +63,34 @@ fn create_build_image() ?string { } fn build(conf Config) ? { - // We get the repos list from the Vieter instance - repos := git.get_repos(conf.address, conf.api_key) ? + build_arch := os.uname().machine + + // We get the repos map from the Vieter instance + repos_map := git.get_repos(conf.address, conf.api_key) ? + + // We filter out any repos that aren't allowed to be built on this + // architecture + filtered_repos := repos_map.keys().map(repos_map[it]).filter(it.arch.contains(build_arch)) // No point in doing work if there's no repos present - if repos.len == 0 { + if filtered_repos.len == 0 { return } // First, we create a base image which has updated repos n stuff image_id := create_build_image() ? - for _, repo in repos { + for repo in filtered_repos { // TODO what to do with PKGBUILDs that build multiple packages? commands := [ 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', 'cd repo', 'makepkg --nobuild --nodeps', 'source PKGBUILD', - // The build container checks whether the package is already present on the server - 'curl --head --fail $conf.address/\$pkgname-\$pkgver-\$pkgrel-\$(uname -m).pkg.tar.zst && exit 0', - 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done', + // The build container checks whether the package is already + // present on the server + 'curl --head --fail $conf.address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel-${build_arch}.pkg.tar.zst && exit 0', + 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/$repo.repo/publish; done', ] // We convert the list of commands into a base64 string, which then gets -- 2.40.1 From d11ad78bff759d050029cc9641c8d3662d52832a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 14:43:41 +0200 Subject: [PATCH 12/17] Updated CI to use new system --- .woodpecker/.arch.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index d646353..23c3408 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -3,10 +3,10 @@ branches: [dev] pipeline: build: - image: 'archlinux:latest' + image: 'archlinux:base-devel' commands: # Update packages - - pacman -Syu --needed --noconfirm base-devel + - pacman -Syu # Create non-root user to perform build & switch to their home - groupadd -g 1000 builder - useradd -mg builder builder @@ -17,9 +17,9 @@ pipeline: - makepkg -s --noconfirm --needed publish: - image: 'archlinux:latest' + image: 'curlimages/curl:latest' commands: # Publish the package - - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/publish; done' + - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done' secrets: - vieter_api_key -- 2.40.1 From ab72c800c360c996a7f4beb1321c718105056b1f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 14:50:07 +0200 Subject: [PATCH 13/17] Ran vfmt --- src/build/build.v | 2 +- src/git/client.v | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 31e7c3f..4270e9d 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -29,7 +29,7 @@ fn create_build_image() ?string { cmds_str := base64.encode_str(commands.join('\n')) c := docker.NewContainer{ - image: base_image + image: build.base_image env: ['BUILD_SCRIPT=$cmds_str'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] diff --git a/src/git/client.v b/src/git/client.v index 7e1e55b..92d27ee 100644 --- a/src/git/client.v +++ b/src/git/client.v @@ -10,7 +10,7 @@ fn send_request(method http.Method, address string, url string, api_key strin if params.len > 0 { params_str := params.keys().map('$it=${params[it]}').join('&') - full_url = "$full_url?$params_str" + full_url = '$full_url?$params_str' } mut req := http.new_request(method, full_url, '') ? @@ -24,7 +24,8 @@ fn send_request(method http.Method, address string, url string, api_key strin // get_repos returns the current list of repos. pub fn get_repos(address string, api_key string) ?map[string]GitRepo { - data := send_request(http.Method.get, address, '/api/repos', api_key, {}) ? + data := send_request(http.Method.get, address, '/api/repos', api_key, + {}) ? return data.data } @@ -32,10 +33,10 @@ pub fn get_repos(address string, api_key string) ?map[string]GitRepo { // add_repo adds a new repo to the server. pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response { params := { - 'url': url + 'url': url 'branch': branch - 'repo': repo - 'arch': arch.join(',') + 'repo': repo + 'arch': arch.join(',') } data := send_request(http.Method.post, address, '/api/repos', api_key, params) ? @@ -44,7 +45,8 @@ pub fn add_repo(address string, api_key string, url string, branch string, repo // remove_repo removes the repo with the given ID from the server. pub fn remove_repo(address string, api_key string, id string) ?Response { - data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, {}) ? + data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, + {}) ? return data } -- 2.40.1 From c6d176f426738c7282757927b2a516f5fe136e26 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 15:08:27 +0200 Subject: [PATCH 14/17] Added patch support to repos cli --- src/git/cli.v | 96 +++++++++++++++++++++++++++++++++++++----------- src/git/client.v | 9 +++++ 2 files changed, 84 insertions(+), 21 deletions(-) diff --git a/src/git/cli.v b/src/git/cli.v index 376592d..a75f8a0 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -48,10 +48,76 @@ pub fn cmd() cli.Command { remove(conf, cmd.args[0]) ? } }, + cli.Command{ + name: 'edit' + required_args: 1 + usage: 'id' + description: 'Edit the repository that matches the given ID prefix.' + flags: [ + cli.Flag{ + name: 'url' + description: 'URL of the Git repository.' + flag: cli.FlagType.string + }, + cli.Flag{ + name: 'branch' + description: 'Branch of the Git repository.' + flag: cli.FlagType.string + }, + cli.Flag{ + name: 'repo' + description: 'Repo to publish builds to.' + flag: cli.FlagType.string + }, + cli.Flag{ + name: 'arch' + description: 'Comma-separated list of architectures to build on.' + flag: cli.FlagType.string + }, + ] + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? + + found := cmd.flags.get_all_found() + + mut params := map[string]string{} + + for f in found { + if f.name != 'config-file' { + params[f.name] = f.get_string() ? + } + } + + patch(conf, cmd.args[0], params) ? + } + }, ] } } +fn get_repo_id_by_prefix(conf Config, id_prefix string) ?string { + repos := get_repos(conf.address, conf.api_key) ? + + mut res := []string{} + + for id, _ in repos { + if id.starts_with(id_prefix) { + res << id + } + } + + if res.len == 0 { + eprintln('No repo found for given prefix.') + } + + if res.len > 1 { + eprintln('Multiple repos found for given prefix.') + } + + return res[0] +} + fn list(conf Config) ? { repos := get_repos(conf.address, conf.api_key) ? @@ -67,27 +133,15 @@ fn add(conf Config, url string, branch string, repo string, arch []string) ? { } fn remove(conf Config, id_prefix string) ? { - repos := get_repos(conf.address, conf.api_key) ? - - mut to_remove := []string{} - - for id, _ in repos { - if id.starts_with(id_prefix) { - to_remove << id - } - } - - if to_remove.len == 0 { - eprintln('No repo found for given prefix.') - exit(1) - } - - if to_remove.len > 1 { - eprintln('Multiple repos found for given prefix.') - exit(1) - } - - res := remove_repo(conf.address, conf.api_key, to_remove[0]) ? + id := get_repo_id_by_prefix(conf, id_prefix) ? + res := remove_repo(conf.address, conf.api_key, id) ? + + println(res.message) +} + +fn patch(conf Config, id_prefix string, params map[string]string) ? { + id := get_repo_id_by_prefix(conf, id_prefix) ? + res := patch_repo(conf.address, conf.api_key, id, params) ? println(res.message) } diff --git a/src/git/client.v b/src/git/client.v index 92d27ee..e4a39ac 100644 --- a/src/git/client.v +++ b/src/git/client.v @@ -50,3 +50,12 @@ pub fn remove_repo(address string, api_key string, id string) ?Response return data } + +// patch_repo sends a PATCH request to the given repo with the params as +// payload. +pub fn patch_repo(address string, api_key string, id string, params map[string]string) ?Response { + data := send_request(http.Method.patch, address, '/api/repos/$id', api_key, + params) ? + + return data +} -- 2.40.1 From a5ac0b69562da2d9558fcc4d19860a5e816ef16e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 15:12:48 +0200 Subject: [PATCH 15/17] Fixed dumb mistake in repos cli --- src/git/cli.v | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/git/cli.v b/src/git/cli.v index a75f8a0..463f1ba 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -108,11 +108,11 @@ fn get_repo_id_by_prefix(conf Config, id_prefix string) ?string { } if res.len == 0 { - eprintln('No repo found for given prefix.') + return error('No repo found for given prefix.') } if res.len > 1 { - eprintln('Multiple repos found for given prefix.') + return error('Multiple repos found for given prefix.') } return res[0] -- 2.40.1 From 3b555efa916179e0cbd885d5a229151b67f761cc Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 15:21:27 +0200 Subject: [PATCH 16/17] Renamed data_dir to repos_dir --- Dockerfile | 2 +- src/repo/repo.v | 16 ++++++++-------- src/repo/sync.v | 2 +- src/server/cli.v | 2 +- src/server/routes.v | 2 +- src/server/server.v | 2 +- vieter.toml | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8b62521..58087ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN if [ -n "${CI_COMMIT_SHA}" ]; then \ FROM busybox:1.35.0 ENV PATH=/bin \ - VIETER_REPO_DIR=/data/repo \ + VIETER_REPOS_DIR=/data/repos \ VIETER_PKG_DIR=/data/pkgs \ VIETER_DOWNLOAD_DIR=/data/downloads \ VIETER_REPOS_FILE=/data/repos.json diff --git a/src/repo/repo.v b/src/repo/repo.v index 334c894..228b17e 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -11,7 +11,7 @@ mut: mutex shared util.Dummy pub: // Where to store repositories' files - data_dir string [required] + repos_dir string [required] // Where packages are stored; each architecture gets its own subdirectory pkg_dir string [required] // The default architecture to use for a repository. In reality, this value @@ -27,9 +27,9 @@ pub: } // new creates a new RepoGroupManager & creates the directories as needed -pub fn new(data_dir string, pkg_dir string, default_arch string) ?RepoGroupManager { - if !os.is_dir(data_dir) { - os.mkdir_all(data_dir) or { return error('Failed to create repo directory: $err.msg') } +pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupManager { + if !os.is_dir(repos_dir) { + os.mkdir_all(repos_dir) or { return error('Failed to create repos directory: $err.msg') } } if !os.is_dir(pkg_dir) { @@ -37,7 +37,7 @@ pub fn new(data_dir string, pkg_dir string, default_arch string) ?RepoGroupManag } return RepoGroupManager{ - data_dir: data_dir + repos_dir: repos_dir pkg_dir: pkg_dir default_arch: default_arch } @@ -83,7 +83,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { return r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) } - repo_dir := os.join_path_single(r.data_dir, repo) + repo_dir := os.join_path_single(r.repos_dir, repo) mut arch_repos := []string{} @@ -118,7 +118,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { // changes. The function returns false if the package was already present in // the repo, and true otherwise. fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool { - pkg_dir := os.join_path(r.data_dir, repo, arch, '$pkg.info.name-$pkg.info.version') + pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version') // We can't add the same package twice if os.exists(pkg_dir) { @@ -150,7 +150,7 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac // returns false if the package wasn't present in the database. It also // optionally re-syncs the repo archives. fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg &package.Pkg, sync bool) ?bool { - repo_dir := os.join_path(r.data_dir, repo, arch) + repo_dir := os.join_path(r.repos_dir, repo, arch) // If the repository doesn't exist yet, the result is automatically false if !os.exists(repo_dir) { diff --git a/src/repo/sync.v b/src/repo/sync.v index afb8a4b..e2b7aac 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -31,7 +31,7 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri // Re-generate the repo archive files fn (r &RepoGroupManager) sync(repo string, arch string) ? { - subrepo_path := os.join_path(r.data_dir, repo, arch) + subrepo_path := os.join_path(r.repos_dir, repo, arch) lock r.mutex { a_db := C.archive_write_new() diff --git a/src/server/cli.v b/src/server/cli.v index b3f37e3..ae9e89c 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -10,7 +10,7 @@ pub: pkg_dir string download_dir string api_key string - data_dir string + repos_dir string repos_file string default_arch string } diff --git a/src/server/routes.v b/src/server/routes.v index 8c8cdff..a926425 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -23,7 +23,7 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re db_exts := ['.db', '.files', '.db.tar.gz', '.files.tar.gz'] if db_exts.any(filename.ends_with(it)) { - full_path = os.join_path(app.repo.data_dir, repo, arch, filename) + full_path = os.join_path(app.repo.repos_dir, repo, arch, filename) // repo-add does this using symlinks, but we just change the requested // path diff --git a/src/server/server.v b/src/server/server.v index f93a5ae..5bf9a87 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -44,7 +44,7 @@ pub fn server(conf Config) ? { } // This also creates the directories if needed - repo := repo.new(conf.data_dir, conf.pkg_dir, conf.default_arch) or { + repo := repo.new(conf.repos_dir, conf.pkg_dir, conf.default_arch) or { logger.error(err.msg) exit(1) } diff --git a/vieter.toml b/vieter.toml index 10b3855..8e0447b 100644 --- a/vieter.toml +++ b/vieter.toml @@ -1,7 +1,7 @@ # This file contains settings used during development api_key = "test" download_dir = "data/downloads" -data_dir = "data" +repos_dir = "data/repos" pkg_dir = "data/pkgs" log_level = "DEBUG" repos_file = "data/repos.json" -- 2.40.1 From 97f35322042c655b39483ec6fe36f2d2b2640561 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 16:11:05 +0200 Subject: [PATCH 17/17] Ran vfmt --- src/server/cli.v | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/cli.v b/src/server/cli.v index ae9e89c..bea223d 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -10,7 +10,7 @@ pub: pkg_dir string download_dir string api_key string - repos_dir string + repos_dir string repos_file string default_arch string } -- 2.40.1