From cec5ecce7f936e118fb2bca41006fa8daa390f59 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 9 Apr 2022 17:41:41 +0200 Subject: [PATCH 1/3] Each repo now has its own subdir in pkg_dir --- src/build/cli.v | 4 ++-- src/repo/repo.v | 6 ++++-- src/server/routes.v | 8 ++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/build/cli.v b/src/build/cli.v index 7cdcf83..0131396 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -5,8 +5,8 @@ import env pub struct Config { pub: - api_key string - address string + api_key string + address string base_image string = 'archlinux:base-devel' } diff --git a/src/repo/repo.v b/src/repo/repo.v index 228b17e..9c2b0ec 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -12,7 +12,7 @@ mut: pub: // Where to store repositories' files repos_dir string [required] - // Where packages are stored; each architecture gets its own subdirectory + // Where packages are stored; each repository gets its own subdirectory pkg_dir string [required] // The default architecture to use for a repository. In reality, this value // is only required when a package with architecture "any" is added as the @@ -56,10 +56,12 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re // If the add was successful, we move the file to the packages directory if added { - dest_path := os.real_path(os.join_path_single(r.pkg_dir, pkg.filename())) + repo_pkg_path := os.real_path(os.join_path_single(r.pkg_dir, repo)) + dest_path := os.join_path_single(repo_pkg_path, pkg.filename()) // Only move the file if it's not already in the package directory if dest_path != os.real_path(pkg_path) { + os.mkdir_all(repo_pkg_path) ? os.mv(pkg_path, dest_path) ? } } diff --git a/src/server/routes.v b/src/server/routes.v index f259af4..c449d7d 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -33,11 +33,11 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re full_path += '.tar.gz' } } else if filename.contains('.pkg') { - full_path = os.join_path_single(app.repo.pkg_dir, filename) + full_path = os.join_path(app.repo.pkg_dir, repo, filename) - // Default behavior is to return the desc file for the package, if present. - // This can then also be used by the build system to properly check whether - // a package is present in an arch-repo. + // Default behavior is to return the desc file for the package, if present. + // This can then also be used by the build system to properly check whether + // a package is present in an arch-repo. } else { full_path = os.join_path(app.repo.repos_dir, repo, arch, filename, 'desc') } -- 2.40.1 From a65207f961c022898b631681a466088726518d65 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 9 Apr 2022 21:08:54 +0200 Subject: [PATCH 2/3] Solved the "removing old packages" problem (I think) --- src/repo/repo.v | 87 +++++++++++++++++++++++++++------------------ src/server/routes.v | 12 +++---- test.py | 4 +-- 3 files changed, 60 insertions(+), 43 deletions(-) diff --git a/src/repo/repo.v b/src/repo/repo.v index 9c2b0ec..42ef3c9 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -14,9 +14,9 @@ pub: repos_dir string [required] // Where packages are stored; each repository gets its own subdirectory pkg_dir string [required] - // The default architecture to use for a repository. In reality, this value - // is only required when a package with architecture "any" is added as the - // first package of a repository. + // The default architecture to use for a repository. Whenever a package of + // arch "any" is added to a repo, it will also be added to this + // architecture. default_arch string [required] } @@ -45,8 +45,8 @@ pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupMana // add_pkg_from_path adds a package to a given repo, given the file path to the // pkg archive. It's a wrapper around add_pkg_in_repo that parses the archive -// file, passes the result to add_pkg_in_repo, and moves the archive to -// r.pkg_dir if it was successfully added. +// file, passes the result to add_pkg_in_repo, and hard links the archive to +// the right subdirectories in r.pkg_dir if it was successfully added. pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult { pkg := package.read_pkg_archive(pkg_path) or { return error('Failed to read package file: $err.msg') @@ -55,19 +55,22 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re added := r.add_pkg_in_repo(repo, pkg) ? // If the add was successful, we move the file to the packages directory - if added { - repo_pkg_path := os.real_path(os.join_path_single(r.pkg_dir, repo)) + for arch in added { + repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch)) dest_path := os.join_path_single(repo_pkg_path, pkg.filename()) - // Only move the file if it's not already in the package directory - if dest_path != os.real_path(pkg_path) { - os.mkdir_all(repo_pkg_path) ? - os.mv(pkg_path, dest_path) ? - } + os.mkdir_all(repo_pkg_path) ? + + // We create hard links so that "any" arch packages aren't stored + // multiple times + os.link(pkg_path, dest_path) ? } + // After linking, we can remove the original file + os.rm(pkg_path) ? + return RepoAddResult{ - added: added + added: added.len > 0 pkg: &pkg } } @@ -75,26 +78,32 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re // add_pkg_in_repo adds a package to a given repo. This function is responsible // for inspecting the package architecture. If said architecture is 'any', the // package is added to each arch-repository within the given repo. A package of -// architecture 'any' will always be added to the arch-repo defined by -// r.default_arch. If this arch-repo doesn't exist yet, it will be created. If -// the architecture isn't 'any', the package is only added to the specific +// architecture 'any' is always added to the arch-repo defined by +// r.default_arch. If this arch-repo doesn't exist yet, it is created. If the +// architecture isn't 'any', the package is only added to the specific // architecture. -fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { - // A package without arch 'any' can be handled without any further checks +fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]string { + // A package not of arch 'any' can be handled easily by adding it to the + // respective repo if pkg.info.arch != 'any' { - return r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) + if r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) ? { + return [pkg.info.arch] + } else { + return [] + } } - repo_dir := os.join_path_single(r.repos_dir, repo) - mut arch_repos := []string{} + // If it is an "any" package, the package gets added to every currently + // present arch-repo. It will always get added to the r.default_arch repo, + // even if no or multiple others are present. + repo_dir := os.join_path_single(r.repos_dir, repo) + // If this is the first package that's added to the repo, the directory // won't exist yet if os.exists(repo_dir) { - // We get a listing of all currently present arch-repos in the given repo - arch_repos = os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, - it))) + arch_repos = os.ls(repo_dir) ? } // The default_arch should always be updated when a package with arch 'any' @@ -103,12 +112,14 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { arch_repos << r.default_arch } - mut added := false + mut added := []string{} // We add the package to each repository. If any of the repositories // return true, the result of the function is also true. for arch in arch_repos { - added = added || r.add_pkg_in_arch_repo(repo, arch, pkg) ? + if r.add_pkg_in_arch_repo(repo, arch, pkg) ? { + added << arch + } } return added @@ -122,13 +133,8 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool { pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version') - // We can't add the same package twice - if os.exists(pkg_dir) { - return false - } - - // We remove the older package version first, if present - r.remove_pkg_from_arch_repo(repo, arch, pkg, false) ? + // Remove the previous version of the package, if present + r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false) ? os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') } @@ -151,7 +157,7 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac // remove_pkg_from_arch_repo removes a package from an arch-repo's database. It // returns false if the package wasn't present in the database. It also // optionally re-syncs the repo archives. -fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg &package.Pkg, sync bool) ?bool { +fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool { repo_dir := os.join_path(r.repos_dir, repo, arch) // If the repository doesn't exist yet, the result is automatically false @@ -167,13 +173,24 @@ fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg // not the version. name := d.split('-')#[..-2].join('-') - if name == pkg.info.name { + if name == pkg_name { // We lock the mutex here to prevent other routines from creating a // new archive while we remove an entry lock r.mutex { os.rmdir_all(os.join_path_single(repo_dir, d)) ? } + // Also remove the package archive + repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch) + + archives := os.ls(repo_pkg_dir) ?.filter(it.split('-')#[..-3].join('-') == name) + + for archive_name in archives { + full_path := os.join_path_single(repo_pkg_dir, archive_name) + os.rm(full_path) ? + } + + // Sync the db archives if requested if sync { r.sync(repo, arch) ? } diff --git a/src/server/routes.v b/src/server/routes.v index c449d7d..ccc0e0e 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -33,12 +33,12 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re full_path += '.tar.gz' } } else if filename.contains('.pkg') { - full_path = os.join_path(app.repo.pkg_dir, repo, filename) - - // Default behavior is to return the desc file for the package, if present. - // This can then also be used by the build system to properly check whether - // a package is present in an arch-repo. - } else { + full_path = os.join_path(app.repo.pkg_dir, repo, arch, filename) + } + // Default behavior is to return the desc file for the package, if present. + // This can then also be used by the build system to properly check whether + // a package is present in an arch-repo. + else { full_path = os.join_path(app.repo.repos_dir, repo, arch, filename, 'desc') } diff --git a/test.py b/test.py index 9b0116e..46dac52 100644 --- a/test.py +++ b/test.py @@ -38,7 +38,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len): Generates a random .PKGINFO """ name = "-".join(random_words(words, name_min_len, name_max_len)) - ver = "0.1.0-1" # doesn't matter what it is anyways + ver = "0.1.0-3" # doesn't matter what it is anyways # TODO add random dependencies (all types) @@ -46,7 +46,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len): "pkgname": name, "pkgbase": name, "pkgver": ver, - "arch": "any" + "arch": "x86_64" } return "\n".join(f"{key} = {value}" for key, value in data.items()) -- 2.40.1 From 62bee78955ea205f587c4b2cfe8c7a31572a16b6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 9 Apr 2022 21:17:22 +0200 Subject: [PATCH 3/3] Updated changelog --- CHANGELOG.md | 1 + src/repo/repo.v | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1daaec..66ff9d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Each package can now only have one version in the repository at once (required by Pacman) * Packages with unknown fields in .PKGINFO are now allowed +* Old packages are now properly removed ## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0) diff --git a/src/repo/repo.v b/src/repo/repo.v index 42ef3c9..f439f58 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -12,7 +12,8 @@ mut: pub: // Where to store repositories' files repos_dir string [required] - // Where packages are stored; each repository gets its own subdirectory + // Where packages are stored; each arch-repository gets its own + // subdirectory pkg_dir string [required] // The default architecture to use for a repository. Whenever a package of // arch "any" is added to a repo, it will also be added to this -- 2.40.1