CHANGELOG.md

@@ -7,9 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter)
 
-## [0.2.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.2.0)
+## Changed
-
-### Changed
 
 * Better config system
     * Support for both a config file & environment variables
@@ -19,7 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
     * All routes now return proper JSON where applicable & the correct status
       codes
 
-### Added
+## Added
 
 * Very basic build system
     * Build is triggered by separate cron container
@@ -34,12 +32,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Support for multiple repositories
 * Support for multiple architectures per repository
 
-### Fixed
+## Fixed
 
 * Each package can now only have one version in the repository at once
   (required by Pacman)
 * Packages with unknown fields in .PKGINFO are now allowed
-* Old packages are now properly removed
 
 ## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0)
 

README.md

@@ -1,17 +1,10 @@
 # Vieter
 
-## Documentation
+Vieter is a re-implementation of the Pieter project. The goal is to create a
-
+simple PKGBUILD-based build system, combined with a self-hosted Arch
-I host documentation for Vieter over at https://rustybever.be/docs/vieter/.
+repository. This would allow me to periodically re-build AUR packages (or
-
+PKGBUILDs I created myself), & make sure I never have to compile anything on my
-## Overview
+own systems, making my updates a lot quicker.
-
-Vieter is a restart of the Pieter project. The goal is to create a simple,
-lightweight self-hostable Arch repository server, paired with a system that
-periodically builds & publishes select Arch packages. This would allow me to
-build AUR packages (or PKGBUILDs I created myself) "in the cloud" & make sure I
-never have to compile anything on my own systems, making my updates a lot
-quicker.
 
 ## Why V?
 
@@ -27,26 +20,27 @@ V standard library, and therefore the compiler. The source code for this fork
 can be found [here](https://git.rustybever.be/Chewing_Bever/vieter-v). You can
 obtain this modified version of the compiler by running `make v`, which will
 clone & build the compiler. Afterwards, all make commands that require the V
-compiler will use this new binary. I try to keep this fork as up to date with
+compiler will use this new binary.
-upstream as possible.
 
 ## Features
 
-* Arch repository server
+The project will consist of a server-agent model, where one or more builder
-    * Support for multiple repositories & multiple architectures
+nodes can register with the server. These agents communicate with the Docker
-    * Endpoints for publishing new packages
+daemon to start builds, which are then uploaded to the server's repository. The
-    * API for managing repositories to build
+server also allows for non-agents to upload packages, as long as they have the
-* Build system
+required secrets. This allows me to also develop non-git packages, such as my
-    * Periodic rebuilding of packages
+terminal, & upload them to the servers using CI.
-    * Prevent unnecessary rebuilds
 
-## Building
+## Directory Structure
 
-In order to build Vieter, you'll need a couple of libraries:
+The data directory consists of three main directories:
 
-* gc
+* `downloads` - This is where packages are initially downloaded. Because vieter
-* libarchive
+  moves files from this folder to the `pkgs` folder, these two folders should
-* openssl
+  best be on the same drive
-
+* `pkgs` - This is where approved package files are stored.
-Before building Vieter, you'll have to build the compiler using `make v`.
+* `repos` - Each repository gets a subfolder here. The subfolder contains the
-Afterwards, run `make` to build the debug binary.
+  uncompressed contents of the db file.
+    * Each repo subdirectory contains the compressed db & files archive for the
+      repository, alongside a directory called `files` which contains the
+      uncompressed contents.

src/build/build.v

@@ -93,7 +93,7 @@ fn build(conf Config) ? {
 			'source PKGBUILD',
 			// The build container checks whether the package is already
 			// present on the server
-			'curl --head --fail $conf.address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
+			'curl --head --fail $conf.address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel-${build_arch}.pkg.tar.zst && exit 0',
 			'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/$repo.repo/publish; done',
 		]
 

src/main.v

@@ -11,7 +11,7 @@ fn main() {
 	mut app := cli.Command{
 		name: 'vieter'
 		description: 'Vieter is a lightweight implementation of an Arch repository server.'
-		version: '0.2.0'
+		version: '0.1.0'
 		flags: [
 			cli.Flag{
 				flag: cli.FlagType.string

src/repo/repo.v

@@ -12,12 +12,11 @@ mut:
 pub:
 	// Where to store repositories' files
 	repos_dir string [required]
-	// Where packages are stored; each arch-repository gets its own
+	// Where packages are stored; each architecture gets its own subdirectory
-	// subdirectory
 	pkg_dir string [required]
-	// The default architecture to use for a repository. Whenever a package of
+	// The default architecture to use for a repository. In reality, this value
-	// arch "any" is added to a repo, it will also be added to this
+	// is only required when a package with architecture "any" is added as the
-	// architecture.
+	// first package of a repository.
 	default_arch string [required]
 }
 
@@ -46,8 +45,8 @@ pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupMana
 
 // add_pkg_from_path adds a package to a given repo, given the file path to the
 // pkg archive. It's a wrapper around add_pkg_in_repo that parses the archive
-// file, passes the result to add_pkg_in_repo, and hard links the archive to
+// file, passes the result to add_pkg_in_repo, and moves the archive to
-// the right subdirectories in r.pkg_dir if it was successfully added.
+// r.pkg_dir if it was successfully added.
 pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult {
 	pkg := package.read_pkg_archive(pkg_path) or {
 		return error('Failed to read package file: $err.msg')
@@ -56,22 +55,17 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re
 	added := r.add_pkg_in_repo(repo, pkg) ?
 
 	// If the add was successful, we move the file to the packages directory
-	for arch in added {
+	if added {
-		repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch))
+		dest_path := os.real_path(os.join_path_single(r.pkg_dir, pkg.filename()))
-		dest_path := os.join_path_single(repo_pkg_path, pkg.filename())
 
-		os.mkdir_all(repo_pkg_path) ?
+		// Only move the file if it's not already in the package directory
-
+		if dest_path != os.real_path(pkg_path) {
-		// We create hard links so that "any" arch packages aren't stored
+			os.mv(pkg_path, dest_path) ?
-		// multiple times
+		}
-		os.link(pkg_path, dest_path) ?
 	}
 
-	// After linking, we can remove the original file
-	os.rm(pkg_path) ?
-
 	return RepoAddResult{
-		added: added.len > 0
+		added: added
 		pkg: &pkg
 	}
 }
@@ -79,32 +73,26 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re
 // add_pkg_in_repo adds a package to a given repo. This function is responsible
 // for inspecting the package architecture. If said architecture is 'any', the
 // package is added to each arch-repository within the given repo. A package of
-// architecture 'any' is always added to the arch-repo defined by
+// architecture 'any' will always be added to the arch-repo defined by
-// r.default_arch. If this arch-repo doesn't exist yet, it is created. If the
+// r.default_arch. If this arch-repo doesn't exist yet, it will be created. If
-// architecture isn't 'any', the package is only added to the specific
+// the architecture isn't 'any', the package is only added to the specific
 // architecture.
-fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]string {
+fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool {
-	// A package not of arch 'any' can be handled easily by adding it to the
+	// A package without arch 'any' can be handled without any further checks
-	// respective repo
 	if pkg.info.arch != 'any' {
-		if r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) ? {
+		return r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)
-			return [pkg.info.arch]
-		} else {
-			return []
-		}
 	}
 
-	mut arch_repos := []string{}
-
-	// If it is an "any" package, the package gets added to every currently
-	// present arch-repo. It will always get added to the r.default_arch repo,
-	// even if no or multiple others are present.
 	repo_dir := os.join_path_single(r.repos_dir, repo)
 
+	mut arch_repos := []string{}
+
 	// If this is the first package that's added to the repo, the directory
 	// won't exist yet
 	if os.exists(repo_dir) {
-		arch_repos = os.ls(repo_dir) ?
+		// We get a listing of all currently present arch-repos in the given repo
+		arch_repos = os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir,
+			it)))
 	}
 
 	// The default_arch should always be updated when a package with arch 'any'
@@ -113,14 +101,12 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin
 		arch_repos << r.default_arch
 	}
 
-	mut added := []string{}
+	mut added := false
 
 	// We add the package to each repository. If any of the repositories
 	// return true, the result of the function is also true.
 	for arch in arch_repos {
-		if r.add_pkg_in_arch_repo(repo, arch, pkg) ? {
+		added = added || r.add_pkg_in_arch_repo(repo, arch, pkg) ?
-			added << arch
-		}
 	}
 
 	return added
@@ -134,8 +120,13 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin
 fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool {
 	pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version')
 
-	// Remove the previous version of the package, if present
+	// We can't add the same package twice
-	r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false) ?
+	if os.exists(pkg_dir) {
+		return false
+	}
+
+	// We remove the older package version first, if present
+	r.remove_pkg_from_arch_repo(repo, arch, pkg, false) ?
 
 	os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') }
 
@@ -158,7 +149,7 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac
 // remove_pkg_from_arch_repo removes a package from an arch-repo's database. It
 // returns false if the package wasn't present in the database. It also
 // optionally re-syncs the repo archives.
-fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool {
+fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg &package.Pkg, sync bool) ?bool {
 	repo_dir := os.join_path(r.repos_dir, repo, arch)
 
 	// If the repository doesn't exist yet, the result is automatically false
@@ -174,24 +165,13 @@ fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg
 		// not the version.
 		name := d.split('-')#[..-2].join('-')
 
-		if name == pkg_name {
+		if name == pkg.info.name {
 			// We lock the mutex here to prevent other routines from creating a
 			// new archive while we remove an entry
 			lock r.mutex {
 				os.rmdir_all(os.join_path_single(repo_dir, d)) ?
 			}
 
-			// Also remove the package archive
-			repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch)
-
-			archives := os.ls(repo_pkg_dir) ?.filter(it.split('-')#[..-3].join('-') == name)
-
-			for archive_name in archives {
-				full_path := os.join_path_single(repo_pkg_dir, archive_name)
-				os.rm(full_path) ?
-			}
-
-			// Sync the db archives if requested
 			if sync {
 				r.sync(repo, arch) ?
 			}

src/server/routes.v

@@ -22,9 +22,7 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re
 
 	db_exts := ['.db', '.files', '.db.tar.gz', '.files.tar.gz']
 
-	// There's no point in having the ability to serve db archives with wrong
+	if db_exts.any(filename.ends_with(it)) {
-	// filenames
-	if db_exts.any(filename == '$repo$it') {
 		full_path = os.join_path(app.repo.repos_dir, repo, arch, filename)
 
 		// repo-add does this using symlinks, but we just change the requested
@@ -32,14 +30,8 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re
 		if !full_path.ends_with('.tar.gz') {
 			full_path += '.tar.gz'
 		}
-	} else if filename.contains('.pkg') {
+	} else {
-		full_path = os.join_path(app.repo.pkg_dir, repo, arch, filename)
+		full_path = os.join_path_single(app.repo.pkg_dir, filename)
-	}
-	// Default behavior is to return the desc file for the package, if present.
-	// This can then also be used by the build system to properly check whether
-	// a package is present in an arch-repo.
-	else {
-		full_path = os.join_path(app.repo.repos_dir, repo, arch, filename, 'desc')
 	}
 
 	// Scuffed way to respond to HEAD requests
@@ -97,7 +89,7 @@ fn (mut app App) put_package(repo string) web.Result {
 	if !res.added {
 		os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }
 
-		app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo'.")
+		app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo ($res.pkg.info.arch)'.")
 
 		return app.json(http.Status.bad_request, new_response('File already exists.'))
 	}

test.py

@@ -38,7 +38,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len):
     Generates a random .PKGINFO
     """
     name = "-".join(random_words(words, name_min_len, name_max_len))
-    ver = "0.1.0-3"  # doesn't matter what it is anyways
+    ver = "0.1.0-1"  # doesn't matter what it is anyways
 
     # TODO add random dependencies (all types)
 
@@ -46,7 +46,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len):
         "pkgname": name,
         "pkgbase": name,
         "pkgver": ver,
-        "arch": "x86_64"
+        "arch": "any"
     }
 
     return "\n".join(f"{key} = {value}" for key, value in data.items())