Merge pull request 'Some final stuff before 0.5.0' (#323) from Chewing_Bever/vieter:final-stuff into dev

Reviewed-on: vieter-v/vieter#323

CHANGELOG.md

@@ -7,12 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
 
+### Added
+
+* CLI commands for removing packages, arch-repos & repositories
+
 ## [0.5.0-rc.2](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.2)
 
 ### Added
 
 * API route for removing logs & accompanying CLI command
 * Daemon for periodically removing old logs
+* CLI flag to filter logs by specific exit codes
 
 ### Changed
 

Makefile

@@ -3,7 +3,7 @@ SRC_DIR := src
 SOURCES != find '$(SRC_DIR)' -iname '*.v'
 
 V_PATH ?= v
-V := $(V_PATH) -showcc -gc boehm -W -d use_openssl
+V := $(V_PATH) -showcc -gc boehm -W -d use_openssl -skip-unused
 
 all: vieter
 

docs/content/configuration.md

@@ -59,10 +59,9 @@ configuration variable required for each command.
       ([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the
       image used for the Vieter CI builds.
 * `max_log_age`: maximum age of logs (in days). Logs older than this will get
-  cleaned by the log removal daemon. If set to a negative value, no logs are
+  cleaned by the log removal daemon. If set to zero, no logs are ever removed.
-  ever removed. The age of logs is determined by the time the build was
+  The age of logs is determined by the time the build was started.
-  started.
+    * Default: `0`
-    * Default: `-1`
 * `log_removal_schedule`: cron schedule defining when to clean old logs.
     * Default: `0 0` (every day at midnight)
 

docs/content/other/_index.md

@@ -1,3 +0,0 @@
----
-weight: 100
----

docs/content/other/builds-in-depth.md

@@ -1,81 +0,0 @@
-# Builds In-depth
-
-For those interested, this page describes how the build system works
-internally.
-
-## Builder image
-
-Every cron daemon perodically creates a builder image that is then used as a
-base for all builds. This is done to prevent build containers having to pull
-down a bunch of updates when they update their system.
-
-The build container is created by running the following commands inside a
-container started from the image defined in `base_image`:
-
-```sh
-# Update repos & install required packages
-pacman -Syu --needed --noconfirm base-devel git
-# Add a non-root user to run makepkg
-groupadd -g 1000 builder
-useradd -mg builder builder
-# Make sure they can use sudo without a password
-echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
-# Create the directory for the builds & make it writeable for the
-# build user
-mkdir /build
-chown -R builder:builder /build
-```
-
-This script updates the packages to their latest versions & creates a non-root
-user to use when running `makepkg`.
-
-This script is base64-encoded & passed to the container as an environment
-variable. The container's entrypoint is set to `/bin/sh -c` & its command
-argument to `echo $BUILD_SCRIPT | base64 -d | /bin/sh -e`, with the
-`BUILD_SCRIPT` environment variable containing the base64-encoded script.
-
-Once the container exits, a new Docker image is created from it. This image is
-then used as the base for any builds.
-
-## Running builds
-
-Each build has its own Docker container, using the builder image as its base.
-The same base64-based technique as above is used, just with a different script.
-To make the build logs more clear, each command is appended by an echo command
-printing the next command to stdout.
-
-Given the Git repository URL is `https://examplerepo.com` with branch `main`,
-the URL of the Vieter server is `https://example.com` and `vieter` is the
-repository we wish to publish to, we get the following script:
-
-```sh
-echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf'
-echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
-echo -e '+ pacman -Syu --needed --noconfirm'
-pacman -Syu --needed --noconfirm
-echo -e '+ su builder'
-su builder
-echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo'
-git clone --single-branch --depth 1 --branch main https://examplerepo.com repo
-echo -e '+ cd repo'
-cd repo
-echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'
-makepkg --nobuild --syncdeps --needed --noconfirm
-echo -e '+ source PKGBUILD'
-source PKGBUILD
-echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0'
-curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
-echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
-[ "$(id -u)" == 0 ] && exit 0
-echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
-MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done
-```
-
-This script:
-
-1. Adds the target repository as a repository in the build container
-2. Updates mirrors & packages
-3. Clones the Git repository
-4. Runs `makepkg` without building to calculate `pkgver`
-5. Checks whether the package version is already present on the server
-6. If not, run `makepkg` & publish any generated package archives to the server

src/client/logs.v

@@ -1,28 +1,27 @@
 module client
 
 import models { BuildLog, BuildLogFilter }
-import net.http { Method }
 import web.response { Response }
 import time
 
 // get_build_logs returns all build logs.
 pub fn (c &Client) get_build_logs(filter BuildLogFilter) ![]BuildLog {
 	params := models.params_from(filter)
-	data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)!
+	data := c.send_request<[]BuildLog>(.get, '/api/v1/logs', params)!
 
 	return data.data
 }
 
 // get_build_log returns a specific build log.
 pub fn (c &Client) get_build_log(id int) !BuildLog {
-	data := c.send_request<BuildLog>(Method.get, '/api/v1/logs/$id', {})!
+	data := c.send_request<BuildLog>(.get, '/api/v1/logs/$id', {})!
 
 	return data.data
 }
 
 // get_build_log_content returns the contents of the build log file.
 pub fn (c &Client) get_build_log_content(id int) !string {
-	data := c.send_request_raw_response(Method.get, '/api/v1/logs/$id/content', {}, '')!
+	data := c.send_request_raw_response(.get, '/api/v1/logs/$id/content', {}, '')!
 
 	return data
 }
@@ -37,7 +36,7 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t
 		'exitCode':  exit_code.str()
 	}
 
-	data := c.send_request_with_body<int>(Method.post, '/api/v1/logs', params, content)!
+	data := c.send_request_with_body<int>(.post, '/api/v1/logs', params, content)!
 
 	return data
 }

src/client/repos.v

@@ -0,0 +1,16 @@
+module client
+
+// remove_repo removes an entire repository.
+pub fn (c &Client) remove_repo(repo string) ! {
+	c.send_request<string>(.delete, '/$repo', {})!
+}
+
+// remove_arch_repo removes an entire arch-repo.
+pub fn (c &Client) remove_arch_repo(repo string, arch string) ! {
+	c.send_request<string>(.delete, '/$repo/$arch', {})!
+}
+
+// remove_package removes a single package from the given arch-repo.
+pub fn (c &Client) remove_package(repo string, arch string, pkgname string) ! {
+	c.send_request<string>(.delete, '/$repo/$arch/$pkgname', {})!
+}

src/client/targets.v

@@ -1,12 +1,11 @@
 module client
 
 import models { Target, TargetFilter }
-import net.http { Method }
 
 // get_targets returns a list of targets, given a filter object.
 pub fn (c &Client) get_targets(filter TargetFilter) ![]Target {
 	params := models.params_from(filter)
-	data := c.send_request<[]Target>(Method.get, '/api/v1/targets', params)!
+	data := c.send_request<[]Target>(.get, '/api/v1/targets', params)!
 
 	return data.data
 }
@@ -34,7 +33,7 @@ pub fn (c &Client) get_all_targets() ![]Target {
 
 // get_target returns the target for a specific id.
 pub fn (c &Client) get_target(id int) !Target {
-	data := c.send_request<Target>(Method.get, '/api/v1/targets/$id', {})!
+	data := c.send_request<Target>(.get, '/api/v1/targets/$id', {})!
 
 	return data.data
 }
@@ -51,14 +50,14 @@ pub struct NewTarget {
 // add_target adds a new target to the server.
 pub fn (c &Client) add_target(t NewTarget) !int {
 	params := models.params_from<NewTarget>(t)
-	data := c.send_request<int>(Method.post, '/api/v1/targets', params)!
+	data := c.send_request<int>(.post, '/api/v1/targets', params)!
 
 	return data.data
 }
 
 // remove_target removes the target with the given id from the server.
 pub fn (c &Client) remove_target(id int) !string {
-	data := c.send_request<string>(Method.delete, '/api/v1/targets/$id', {})!
+	data := c.send_request<string>(.delete, '/api/v1/targets/$id', {})!
 
 	return data.data
 }
@@ -66,7 +65,7 @@ pub fn (c &Client) remove_target(id int) !string {
 // patch_target sends a PATCH request to the given target with the params as
 // payload.
 pub fn (c &Client) patch_target(id int, params map[string]string) !string {
-	data := c.send_request<string>(Method.patch, '/api/v1/targets/$id', params)!
+	data := c.send_request<string>(.patch, '/api/v1/targets/$id', params)!
 
 	return data.data
 }

src/console/logs/logs.v

@@ -24,11 +24,13 @@ pub fn cmd() cli.Command {
 				flags: [
 					cli.Flag{
 						name: 'limit'
+						abbrev: 'l'
 						description: 'How many results to return.'
 						flag: cli.FlagType.int
 					},
 					cli.Flag{
 						name: 'offset'
+						abbrev: 'o'
 						description: 'Minimum index to return.'
 						flag: cli.FlagType.int
 					},
@@ -39,16 +41,18 @@ pub fn cmd() cli.Command {
 					},
 					cli.Flag{
 						name: 'today'
-						description: 'Only list logs started today.'
+						abbrev: 't'
+						description: 'Only list logs started today. This flag overwrites any other date-related flag.'
 						flag: cli.FlagType.bool
 					},
 					cli.Flag{
 						name: 'failed'
-						description: 'Only list logs with non-zero exit codes.'
+						description: 'Only list logs with non-zero exit codes. This flag overwrites the --code flag.'
 						flag: cli.FlagType.bool
 					},
 					cli.Flag{
 						name: 'day'
+						abbrev: 'd'
 						description: 'Only list logs started on this day. (format: YYYY-MM-DD)'
 						flag: cli.FlagType.string
 					},
@@ -62,6 +66,11 @@ pub fn cmd() cli.Command {
 						description: 'Only list logs started after this timestamp. (format: YYYY-MM-DD HH:mm:ss)'
 						flag: cli.FlagType.string
 					},
+					cli.Flag{
+						name: 'code'
+						description: 'Only return logs with the given exit code. Prepend with `!` to exclude instead of include. Can be specified multiple times.'
+						flag: cli.FlagType.string_array
+					},
 				]
 				execute: fn (cmd cli.Command) ! {
 					config_file := cmd.flags.get_string('config-file')!
@@ -131,6 +140,8 @@ pub fn cmd() cli.Command {
 						filter.exit_codes = [
 							'!0',
 						]
+					} else {
+						filter.exit_codes = cmd.flags.get_strings('code')!
 					}
 
 					raw := cmd.flags.get_bool('raw')!

src/console/repos/repos.v

@@ -0,0 +1,52 @@
+module repos
+
+import cli
+import conf as vconf
+import client
+
+struct Config {
+	address string [required]
+	api_key string [required]
+}
+
+// cmd returns the cli module that handles modifying the repository contents.
+pub fn cmd() cli.Command {
+	return cli.Command{
+		name: 'repos'
+		description: 'Interact with the repositories & packages stored on the server.'
+		commands: [
+			cli.Command{
+				name: 'remove'
+				required_args: 1
+				usage: 'repo [arch [pkgname]]'
+				description: 'Remove a repo, arch-repo, or package from the server.'
+				flags: [
+					cli.Flag{
+						name: 'force'
+						flag: cli.FlagType.bool
+					},
+				]
+				execute: fn (cmd cli.Command) ! {
+					config_file := cmd.flags.get_string('config-file')!
+					conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
+
+					if cmd.args.len < 3 {
+						if !cmd.flags.get_bool('force')! {
+							return error('Removing an arch-repo or repository is a very destructive command. If you really do wish to perform this operation, explicitely add the --force flag.')
+						}
+					}
+
+					client := client.new(conf.address, conf.api_key)
+
+					if cmd.args.len == 1 {
+						client.remove_repo(cmd.args[0])!
+					} else if cmd.args.len == 2 {
+						client.remove_arch_repo(cmd.args[0], cmd.args[1])!
+					} else {
+						client.remove_package(cmd.args[0], cmd.args[1], cmd.args[2])!
+					}
+				}
+			},
+		]
+	}
+}

src/console/targets/targets.v

@@ -25,11 +25,13 @@ pub fn cmd() cli.Command {
 				flags: [
 					cli.Flag{
 						name: 'limit'
+						abbrev: 'l'
 						description: 'How many results to return.'
 						flag: cli.FlagType.int
 					},
 					cli.Flag{
 						name: 'offset'
+						abbrev: 'o'
 						description: 'Minimum index to return.'
 						flag: cli.FlagType.int
 					},

src/main.v

@@ -8,6 +8,7 @@ import console.logs
 import console.schedule
 import console.man
 import console.aur
+import console.repos
 import cron
 import agent
 
@@ -48,6 +49,7 @@ fn main() {
 			man.cmd(),
 			aur.cmd(),
 			agent.cmd(),
+			repos.cmd(),
 		]
 	}
 	app.setup()

src/server/api_jobs.v

@@ -4,7 +4,7 @@ import web
 import web.response { new_data_response, new_response }
 
 // v1_poll_job_queue allows agents to poll for new build jobs.
-['/api/v1/jobs/poll'; auth; get]
+['/api/v1/jobs/poll'; auth; get; markused]
 fn (mut app App) v1_poll_job_queue() web.Result {
 	arch := app.query['arch'] or {
 		return app.json(.bad_request, new_response('Missing arch query arg.'))
@@ -21,7 +21,7 @@ fn (mut app App) v1_poll_job_queue() web.Result {
 }
 
 // v1_queue_job allows queueing a new one-time build job for the given target.
-['/api/v1/jobs/queue'; auth; post]
+['/api/v1/jobs/queue'; auth; markused; post]
 fn (mut app App) v1_queue_job() web.Result {
 	target_id := app.query['target'] or {
 		return app.json(.bad_request, new_response('Missing target query arg.'))

src/server/api_logs.v

@@ -11,7 +11,7 @@ import models { BuildLog, BuildLogFilter }
 
 // v1_get_logs returns all build logs in the database. A 'target' query param can
 // optionally be added to limit the list of build logs to that repository.
-['/api/v1/logs'; auth; get]
+['/api/v1/logs'; auth; get; markused]
 fn (mut app App) v1_get_logs() web.Result {
 	filter := models.from_params<BuildLogFilter>(app.query) or {
 		return app.json(.bad_request, new_response('Invalid query parameters.'))
@@ -22,7 +22,7 @@ fn (mut app App) v1_get_logs() web.Result {
 }
 
 // v1_get_single_log returns the build log with the given id.
-['/api/v1/logs/:id'; auth; get]
+['/api/v1/logs/:id'; auth; get; markused]
 fn (mut app App) v1_get_single_log(id int) web.Result {
 	log := app.db.get_build_log(id) or { return app.status(.not_found) }
 
@@ -30,7 +30,7 @@ fn (mut app App) v1_get_single_log(id int) web.Result {
 }
 
 // v1_get_log_content returns the actual build log file for the given id.
-['/api/v1/logs/:id/content'; auth; get]
+['/api/v1/logs/:id/content'; auth; get; markused]
 fn (mut app App) v1_get_log_content(id int) web.Result {
 	log := app.db.get_build_log(id) or { return app.status(.not_found) }
 	file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
@@ -50,7 +50,7 @@ fn parse_query_time(query string) !time.Time {
 }
 
 // v1_post_log adds a new log to the database.
-['/api/v1/logs'; auth; post]
+['/api/v1/logs'; auth; markused; post]
 fn (mut app App) v1_post_log() web.Result {
 	// Parse query params
 	start_time_int := app.query['startTime'].int()
@@ -121,7 +121,7 @@ fn (mut app App) v1_post_log() web.Result {
 }
 
 // v1_delete_log allows removing a build log from the system.
-['/api/v1/logs/:id'; auth; delete]
+['/api/v1/logs/:id'; auth; delete; markused]
 fn (mut app App) v1_delete_log(id int) web.Result {
 	log := app.db.get_build_log(id) or { return app.status(.not_found) }
 	full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())

src/server/api_targets.v

@@ -6,7 +6,7 @@ import db
 import models { Target, TargetArch, TargetFilter }
 
 // v1_get_targets returns the current list of targets.
-['/api/v1/targets'; auth; get]
+['/api/v1/targets'; auth; get; markused]
 fn (mut app App) v1_get_targets() web.Result {
 	filter := models.from_params<TargetFilter>(app.query) or {
 		return app.json(.bad_request, new_response('Invalid query parameters.'))
@@ -17,7 +17,7 @@ fn (mut app App) v1_get_targets() web.Result {
 }
 
 // v1_get_single_target returns the information for a single target.
-['/api/v1/targets/:id'; auth; get]
+['/api/v1/targets/:id'; auth; get; markused]
 fn (mut app App) v1_get_single_target(id int) web.Result {
 	target := app.db.get_target(id) or { return app.status(.not_found) }
 
@@ -25,7 +25,7 @@ fn (mut app App) v1_get_single_target(id int) web.Result {
 }
 
 // v1_post_target creates a new target from the provided query string.
-['/api/v1/targets'; auth; post]
+['/api/v1/targets'; auth; markused; post]
 fn (mut app App) v1_post_target() web.Result {
 	mut params := app.query.clone()
 
@@ -55,7 +55,7 @@ fn (mut app App) v1_post_target() web.Result {
 }
 
 // v1_delete_target removes a given target from the server's list.
-['/api/v1/targets/:id'; auth; delete]
+['/api/v1/targets/:id'; auth; delete; markused]
 fn (mut app App) v1_delete_target(id int) web.Result {
 	app.db.delete_target(id)
 	app.job_queue.invalidate(id)
@@ -64,7 +64,7 @@ fn (mut app App) v1_delete_target(id int) web.Result {
 }
 
 // v1_patch_target updates a target's data with the given query params.
-['/api/v1/targets/:id'; auth; patch]
+['/api/v1/targets/:id'; auth; markused; patch]
 fn (mut app App) v1_patch_target(id int) web.Result {
 	app.db.update_target(id, app.query)
 

src/server/cli.v

@@ -13,7 +13,7 @@ pub:
 	default_arch         string
 	global_schedule      string = '0 3'
 	base_image           string = 'archlinux:base-devel'
-	max_log_age          int    = -1
+	max_log_age          int    [empty_default]
 	log_removal_schedule string = '0 0'
 }
 

src/server/repo.v

@@ -10,7 +10,7 @@ import web.response { new_data_response, new_response }
 
 // healthcheck just returns a string, but can be used to quickly check if the
 // server is still responsive.
-['/health'; get]
+['/health'; get; markused]
 pub fn (mut app App) healthcheck() web.Result {
 	return app.json(.ok, new_response('Healthy.'))
 }
@@ -18,7 +18,7 @@ pub fn (mut app App) healthcheck() web.Result {
 // get_repo_file handles all Pacman-related routes. It returns both the
 // repository's archives, but also package archives or the contents of a
 // package's desc file.
-['/:repo/:arch/:filename'; get; head]
+['/:repo/:arch/:filename'; get; head; markused]
 fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result {
 	mut full_path := ''
 
@@ -48,7 +48,7 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re
 }
 
 // put_package handles publishing a package to a repository.
-['/:repo/publish'; auth; post]
+['/:repo/publish'; auth; markused; post]
 fn (mut app App) put_package(repo string) web.Result {
 	// api is a reserved keyword for api routes & should never be allowed to be
 	// a repository.

src/server/repo_remove.v

@@ -3,7 +3,7 @@ module server
 import web
 
 // delete_package tries to remove the given package.
-['/:repo/:arch/:pkg'; auth; delete]
+['/:repo/:arch/:pkg'; auth; delete; markused]
 fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result {
 	res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or {
 		app.lerror('Error while deleting package: $err.msg()')
@@ -23,7 +23,7 @@ fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result
 }
 
 // delete_arch_repo tries to remove the given arch-repo.
-['/:repo/:arch'; auth; delete]
+['/:repo/:arch'; auth; delete; markused]
 fn (mut app App) delete_arch_repo(repo string, arch string) web.Result {
 	res := app.repo.remove_arch_repo(repo, arch) or {
 		app.lerror('Error while deleting arch-repo: $err.msg()')
@@ -43,7 +43,7 @@ fn (mut app App) delete_arch_repo(repo string, arch string) web.Result {
 }
 
 // delete_repo tries to remove the given repo.
-['/:repo'; auth; delete]
+['/:repo'; auth; delete; markused]
 fn (mut app App) delete_repo(repo string) web.Result {
 	res := app.repo.remove_repo(repo) or {
 		app.lerror('Error while deleting repo: $err.msg()')

src/web/parse.v

@@ -5,7 +5,7 @@ import net.http
 
 // Method attributes that should be ignored when parsing, as they're used
 // elsewhere.
-const attrs_to_ignore = ['auth']
+const attrs_to_ignore = ['auth', 'markused']
 
 // Parsing function attributes for methods and path.
 fn parse_attrs(name string, attrs []string) !([]http.Method, string) {