diff --git a/CHANGELOG.md b/CHANGELOG.md index 72c5440..9a651d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev) -* Metrics endpoint for Prometheus integration - -## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0) - ### Added -* CLI commands for removing packages, arch-repos & repositories +* Metrics endpoint for Prometheus integration ## [0.5.0-rc.2](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.2) @@ -21,7 +17,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * API route for removing logs & accompanying CLI command * Daemon for periodically removing old logs -* CLI flag to filter logs by specific exit codes ### Changed diff --git a/Makefile b/Makefile index 4bd1edc..e716807 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' V_PATH ?= v -V := $(V_PATH) -showcc -gc boehm -W -d use_openssl -skip-unused +V := $(V_PATH) -showcc -gc boehm -W -d use_openssl all: vieter diff --git a/PKGBUILD b/PKGBUILD index bf9c621..5e9530a 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -3,7 +3,7 @@ pkgbase='vieter' pkgname='vieter' -pkgver='0.5.0' +pkgver='0.5.0_rc.2' pkgrel=1 pkgdesc="Lightweight Arch repository server & package build system" depends=('glibc' 'openssl' 'libarchive' 'sqlite') diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 612c505..45c5de6 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -59,9 +59,10 @@ configuration variable required for each command. ([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the image used for the Vieter CI builds. * `max_log_age`: maximum age of logs (in days). Logs older than this will get - cleaned by the log removal daemon. If set to zero, no logs are ever removed. - The age of logs is determined by the time the build was started. - * Default: `0` + cleaned by the log removal daemon. If set to a negative value, no logs are + ever removed. The age of logs is determined by the time the build was + started. + * Default: `-1` * `log_removal_schedule`: cron schedule defining when to clean old logs. * Default: `0 0` (every day at midnight) diff --git a/docs/content/other/_index.md b/docs/content/other/_index.md new file mode 100644 index 0000000..394456b --- /dev/null +++ b/docs/content/other/_index.md @@ -0,0 +1,3 @@ +--- +weight: 100 +--- diff --git a/docs/content/other/builds-in-depth.md b/docs/content/other/builds-in-depth.md new file mode 100644 index 0000000..d8df6ec --- /dev/null +++ b/docs/content/other/builds-in-depth.md @@ -0,0 +1,81 @@ +# Builds In-depth + +For those interested, this page describes how the build system works +internally. + +## Builder image + +Every cron daemon perodically creates a builder image that is then used as a +base for all builds. This is done to prevent build containers having to pull +down a bunch of updates when they update their system. + +The build container is created by running the following commands inside a +container started from the image defined in `base_image`: + +```sh +# Update repos & install required packages +pacman -Syu --needed --noconfirm base-devel git +# Add a non-root user to run makepkg +groupadd -g 1000 builder +useradd -mg builder builder +# Make sure they can use sudo without a password +echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers +# Create the directory for the builds & make it writeable for the +# build user +mkdir /build +chown -R builder:builder /build +``` + +This script updates the packages to their latest versions & creates a non-root +user to use when running `makepkg`. + +This script is base64-encoded & passed to the container as an environment +variable. The container's entrypoint is set to `/bin/sh -c` & its command +argument to `echo $BUILD_SCRIPT | base64 -d | /bin/sh -e`, with the +`BUILD_SCRIPT` environment variable containing the base64-encoded script. + +Once the container exits, a new Docker image is created from it. This image is +then used as the base for any builds. + +## Running builds + +Each build has its own Docker container, using the builder image as its base. +The same base64-based technique as above is used, just with a different script. +To make the build logs more clear, each command is appended by an echo command +printing the next command to stdout. + +Given the Git repository URL is `https://examplerepo.com` with branch `main`, +the URL of the Vieter server is `https://example.com` and `vieter` is the +repository we wish to publish to, we get the following script: + +```sh +echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' +echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf +echo -e '+ pacman -Syu --needed --noconfirm' +pacman -Syu --needed --noconfirm +echo -e '+ su builder' +su builder +echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo' +git clone --single-branch --depth 1 --branch main https://examplerepo.com repo +echo -e '+ cd repo' +cd repo +echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' +makepkg --nobuild --syncdeps --needed --noconfirm +echo -e '+ source PKGBUILD' +source PKGBUILD +echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' +curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 +echo -e '+ [ "$(id -u)" == 0 ] && exit 0' +[ "$(id -u)" == 0 ] && exit 0 +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done +``` + +This script: + +1. Adds the target repository as a repository in the build container +2. Updates mirrors & packages +3. Clones the Git repository +4. Runs `makepkg` without building to calculate `pkgver` +5. Checks whether the package version is already present on the server +6. If not, run `makepkg` & publish any generated package archives to the server diff --git a/src/client/logs.v b/src/client/logs.v index 6553837..2ddb2e2 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -1,27 +1,28 @@ module client import models { BuildLog, BuildLogFilter } +import net.http { Method } import web.response { Response } import time // get_build_logs returns all build logs. pub fn (c &Client) get_build_logs(filter BuildLogFilter) ![]BuildLog { params := models.params_from(filter) - data := c.send_request<[]BuildLog>(.get, '/api/v1/logs', params)! + data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)! return data.data } // get_build_log returns a specific build log. pub fn (c &Client) get_build_log(id int) !BuildLog { - data := c.send_request(.get, '/api/v1/logs/$id', {})! + data := c.send_request(Method.get, '/api/v1/logs/$id', {})! return data.data } // get_build_log_content returns the contents of the build log file. pub fn (c &Client) get_build_log_content(id int) !string { - data := c.send_request_raw_response(.get, '/api/v1/logs/$id/content', {}, '')! + data := c.send_request_raw_response(Method.get, '/api/v1/logs/$id/content', {}, '')! return data } @@ -36,7 +37,7 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t 'exitCode': exit_code.str() } - data := c.send_request_with_body(.post, '/api/v1/logs', params, content)! + data := c.send_request_with_body(Method.post, '/api/v1/logs', params, content)! return data } diff --git a/src/client/repos.v b/src/client/repos.v deleted file mode 100644 index 9644e9b..0000000 --- a/src/client/repos.v +++ /dev/null @@ -1,16 +0,0 @@ -module client - -// remove_repo removes an entire repository. -pub fn (c &Client) remove_repo(repo string) ! { - c.send_request(.delete, '/$repo', {})! -} - -// remove_arch_repo removes an entire arch-repo. -pub fn (c &Client) remove_arch_repo(repo string, arch string) ! { - c.send_request(.delete, '/$repo/$arch', {})! -} - -// remove_package removes a single package from the given arch-repo. -pub fn (c &Client) remove_package(repo string, arch string, pkgname string) ! { - c.send_request(.delete, '/$repo/$arch/$pkgname', {})! -} diff --git a/src/client/targets.v b/src/client/targets.v index 565832e..da6a9e4 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -1,11 +1,12 @@ module client import models { Target, TargetFilter } +import net.http { Method } // get_targets returns a list of targets, given a filter object. pub fn (c &Client) get_targets(filter TargetFilter) ![]Target { params := models.params_from(filter) - data := c.send_request<[]Target>(.get, '/api/v1/targets', params)! + data := c.send_request<[]Target>(Method.get, '/api/v1/targets', params)! return data.data } @@ -33,7 +34,7 @@ pub fn (c &Client) get_all_targets() ![]Target { // get_target returns the target for a specific id. pub fn (c &Client) get_target(id int) !Target { - data := c.send_request(.get, '/api/v1/targets/$id', {})! + data := c.send_request(Method.get, '/api/v1/targets/$id', {})! return data.data } @@ -50,14 +51,14 @@ pub struct NewTarget { // add_target adds a new target to the server. pub fn (c &Client) add_target(t NewTarget) !int { params := models.params_from(t) - data := c.send_request(.post, '/api/v1/targets', params)! + data := c.send_request(Method.post, '/api/v1/targets', params)! return data.data } // remove_target removes the target with the given id from the server. pub fn (c &Client) remove_target(id int) !string { - data := c.send_request(.delete, '/api/v1/targets/$id', {})! + data := c.send_request(Method.delete, '/api/v1/targets/$id', {})! return data.data } @@ -65,7 +66,7 @@ pub fn (c &Client) remove_target(id int) !string { // patch_target sends a PATCH request to the given target with the params as // payload. pub fn (c &Client) patch_target(id int, params map[string]string) !string { - data := c.send_request(.patch, '/api/v1/targets/$id', params)! + data := c.send_request(Method.patch, '/api/v1/targets/$id', params)! return data.data } diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 35ce4d7..19c46f6 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -24,13 +24,11 @@ pub fn cmd() cli.Command { flags: [ cli.Flag{ name: 'limit' - abbrev: 'l' description: 'How many results to return.' flag: cli.FlagType.int }, cli.Flag{ name: 'offset' - abbrev: 'o' description: 'Minimum index to return.' flag: cli.FlagType.int }, @@ -41,18 +39,16 @@ pub fn cmd() cli.Command { }, cli.Flag{ name: 'today' - abbrev: 't' - description: 'Only list logs started today. This flag overwrites any other date-related flag.' + description: 'Only list logs started today.' flag: cli.FlagType.bool }, cli.Flag{ name: 'failed' - description: 'Only list logs with non-zero exit codes. This flag overwrites the --code flag.' + description: 'Only list logs with non-zero exit codes.' flag: cli.FlagType.bool }, cli.Flag{ name: 'day' - abbrev: 'd' description: 'Only list logs started on this day. (format: YYYY-MM-DD)' flag: cli.FlagType.string }, @@ -66,11 +62,6 @@ pub fn cmd() cli.Command { description: 'Only list logs started after this timestamp. (format: YYYY-MM-DD HH:mm:ss)' flag: cli.FlagType.string }, - cli.Flag{ - name: 'code' - description: 'Only return logs with the given exit code. Prepend with `!` to exclude instead of include. Can be specified multiple times.' - flag: cli.FlagType.string_array - }, ] execute: fn (cmd cli.Command) ! { config_file := cmd.flags.get_string('config-file')! @@ -140,8 +131,6 @@ pub fn cmd() cli.Command { filter.exit_codes = [ '!0', ] - } else { - filter.exit_codes = cmd.flags.get_strings('code')! } raw := cmd.flags.get_bool('raw')! diff --git a/src/console/repos/repos.v b/src/console/repos/repos.v deleted file mode 100644 index 729208e..0000000 --- a/src/console/repos/repos.v +++ /dev/null @@ -1,52 +0,0 @@ -module repos - -import cli -import conf as vconf -import client - -struct Config { - address string [required] - api_key string [required] -} - -// cmd returns the cli module that handles modifying the repository contents. -pub fn cmd() cli.Command { - return cli.Command{ - name: 'repos' - description: 'Interact with the repositories & packages stored on the server.' - commands: [ - cli.Command{ - name: 'remove' - required_args: 1 - usage: 'repo [arch [pkgname]]' - description: 'Remove a repo, arch-repo, or package from the server.' - flags: [ - cli.Flag{ - name: 'force' - flag: cli.FlagType.bool - }, - ] - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - - if cmd.args.len < 3 { - if !cmd.flags.get_bool('force')! { - return error('Removing an arch-repo or repository is a very destructive command. If you really do wish to perform this operation, explicitely add the --force flag.') - } - } - - client := client.new(conf.address, conf.api_key) - - if cmd.args.len == 1 { - client.remove_repo(cmd.args[0])! - } else if cmd.args.len == 2 { - client.remove_arch_repo(cmd.args[0], cmd.args[1])! - } else { - client.remove_package(cmd.args[0], cmd.args[1], cmd.args[2])! - } - } - }, - ] - } -} diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 3c0d755..94deebd 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -25,13 +25,11 @@ pub fn cmd() cli.Command { flags: [ cli.Flag{ name: 'limit' - abbrev: 'l' description: 'How many results to return.' flag: cli.FlagType.int }, cli.Flag{ name: 'offset' - abbrev: 'o' description: 'Minimum index to return.' flag: cli.FlagType.int }, diff --git a/src/main.v b/src/main.v index 1c8b816..eda38e7 100644 --- a/src/main.v +++ b/src/main.v @@ -8,7 +8,6 @@ import console.logs import console.schedule import console.man import console.aur -import console.repos import cron import agent @@ -21,7 +20,7 @@ fn main() { mut app := cli.Command{ name: 'vieter' description: 'Vieter is a lightweight implementation of an Arch repository server.' - version: '0.5.0' + version: '0.5.0-rc.2' posix_mode: true flags: [ cli.Flag{ @@ -49,7 +48,6 @@ fn main() { man.cmd(), aur.cmd(), agent.cmd(), - repos.cmd(), ] } app.setup() diff --git a/src/server/api_jobs.v b/src/server/api_jobs.v index 62bcb27..7795351 100644 --- a/src/server/api_jobs.v +++ b/src/server/api_jobs.v @@ -4,7 +4,7 @@ import web import web.response { new_data_response, new_response } // v1_poll_job_queue allows agents to poll for new build jobs. -['/api/v1/jobs/poll'; auth; get; markused] +['/api/v1/jobs/poll'; auth; get] fn (mut app App) v1_poll_job_queue() web.Result { arch := app.query['arch'] or { return app.json(.bad_request, new_response('Missing arch query arg.')) @@ -21,7 +21,7 @@ fn (mut app App) v1_poll_job_queue() web.Result { } // v1_queue_job allows queueing a new one-time build job for the given target. -['/api/v1/jobs/queue'; auth; markused; post] +['/api/v1/jobs/queue'; auth; post] fn (mut app App) v1_queue_job() web.Result { target_id := app.query['target'] or { return app.json(.bad_request, new_response('Missing target query arg.')) diff --git a/src/server/api_logs.v b/src/server/api_logs.v index 3db4204..13b50b9 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -11,7 +11,7 @@ import models { BuildLog, BuildLogFilter } // v1_get_logs returns all build logs in the database. A 'target' query param can // optionally be added to limit the list of build logs to that repository. -['/api/v1/logs'; auth; get; markused] +['/api/v1/logs'; auth; get] fn (mut app App) v1_get_logs() web.Result { filter := models.from_params(app.query) or { return app.json(.bad_request, new_response('Invalid query parameters.')) @@ -22,7 +22,7 @@ fn (mut app App) v1_get_logs() web.Result { } // v1_get_single_log returns the build log with the given id. -['/api/v1/logs/:id'; auth; get; markused] +['/api/v1/logs/:id'; auth; get] fn (mut app App) v1_get_single_log(id int) web.Result { log := app.db.get_build_log(id) or { return app.status(.not_found) } @@ -30,7 +30,7 @@ fn (mut app App) v1_get_single_log(id int) web.Result { } // v1_get_log_content returns the actual build log file for the given id. -['/api/v1/logs/:id/content'; auth; get; markused] +['/api/v1/logs/:id/content'; auth; get] fn (mut app App) v1_get_log_content(id int) web.Result { log := app.db.get_build_log(id) or { return app.status(.not_found) } file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') @@ -50,7 +50,7 @@ fn parse_query_time(query string) !time.Time { } // v1_post_log adds a new log to the database. -['/api/v1/logs'; auth; markused; post] +['/api/v1/logs'; auth; post] fn (mut app App) v1_post_log() web.Result { // Parse query params start_time_int := app.query['startTime'].int() @@ -121,7 +121,7 @@ fn (mut app App) v1_post_log() web.Result { } // v1_delete_log allows removing a build log from the system. -['/api/v1/logs/:id'; auth; delete; markused] +['/api/v1/logs/:id'; auth; delete] fn (mut app App) v1_delete_log(id int) web.Result { log := app.db.get_build_log(id) or { return app.status(.not_found) } full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) diff --git a/src/server/api_targets.v b/src/server/api_targets.v index 4bb7d12..cd5cb0a 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -6,7 +6,7 @@ import db import models { Target, TargetArch, TargetFilter } // v1_get_targets returns the current list of targets. -['/api/v1/targets'; auth; get; markused] +['/api/v1/targets'; auth; get] fn (mut app App) v1_get_targets() web.Result { filter := models.from_params(app.query) or { return app.json(.bad_request, new_response('Invalid query parameters.')) @@ -17,7 +17,7 @@ fn (mut app App) v1_get_targets() web.Result { } // v1_get_single_target returns the information for a single target. -['/api/v1/targets/:id'; auth; get; markused] +['/api/v1/targets/:id'; auth; get] fn (mut app App) v1_get_single_target(id int) web.Result { target := app.db.get_target(id) or { return app.status(.not_found) } @@ -25,7 +25,7 @@ fn (mut app App) v1_get_single_target(id int) web.Result { } // v1_post_target creates a new target from the provided query string. -['/api/v1/targets'; auth; markused; post] +['/api/v1/targets'; auth; post] fn (mut app App) v1_post_target() web.Result { mut params := app.query.clone() @@ -55,7 +55,7 @@ fn (mut app App) v1_post_target() web.Result { } // v1_delete_target removes a given target from the server's list. -['/api/v1/targets/:id'; auth; delete; markused] +['/api/v1/targets/:id'; auth; delete] fn (mut app App) v1_delete_target(id int) web.Result { app.db.delete_target(id) app.job_queue.invalidate(id) @@ -64,7 +64,7 @@ fn (mut app App) v1_delete_target(id int) web.Result { } // v1_patch_target updates a target's data with the given query params. -['/api/v1/targets/:id'; auth; markused; patch] +['/api/v1/targets/:id'; auth; patch] fn (mut app App) v1_patch_target(id int) web.Result { app.db.update_target(id, app.query) diff --git a/src/server/cli.v b/src/server/cli.v index 9a8b144..b259c89 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -13,7 +13,7 @@ pub: default_arch string global_schedule string = '0 3' base_image string = 'archlinux:base-devel' - max_log_age int [empty_default] + max_log_age int = -1 log_removal_schedule string = '0 0' collect_metrics bool [empty_default] } diff --git a/src/server/repo.v b/src/server/repo.v index 38d07fe..06ab72e 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -10,7 +10,7 @@ import web.response { new_data_response, new_response } // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. -['/health'; get; markused] +['/health'; get] pub fn (mut app App) healthcheck() web.Result { return app.json(.ok, new_response('Healthy.')) } @@ -18,7 +18,7 @@ pub fn (mut app App) healthcheck() web.Result { // get_repo_file handles all Pacman-related routes. It returns both the // repository's archives, but also package archives or the contents of a // package's desc file. -['/:repo/:arch/:filename'; get; head; markused] +['/:repo/:arch/:filename'; get; head] fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result { mut full_path := '' @@ -48,7 +48,7 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re } // put_package handles publishing a package to a repository. -['/:repo/publish'; auth; markused; post] +['/:repo/publish'; auth; post] fn (mut app App) put_package(repo string) web.Result { // api is a reserved keyword for api routes & should never be allowed to be // a repository. diff --git a/src/server/repo_remove.v b/src/server/repo_remove.v index 9e6d747..694f085 100644 --- a/src/server/repo_remove.v +++ b/src/server/repo_remove.v @@ -3,7 +3,7 @@ module server import web // delete_package tries to remove the given package. -['/:repo/:arch/:pkg'; auth; delete; markused] +['/:repo/:arch/:pkg'; auth; delete] fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result { res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { app.lerror('Error while deleting package: $err.msg()') @@ -23,7 +23,7 @@ fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result } // delete_arch_repo tries to remove the given arch-repo. -['/:repo/:arch'; auth; delete; markused] +['/:repo/:arch'; auth; delete] fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { res := app.repo.remove_arch_repo(repo, arch) or { app.lerror('Error while deleting arch-repo: $err.msg()') @@ -43,7 +43,7 @@ fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { } // delete_repo tries to remove the given repo. -['/:repo'; auth; delete; markused] +['/:repo'; auth; delete] fn (mut app App) delete_repo(repo string) web.Result { res := app.repo.remove_repo(repo) or { app.lerror('Error while deleting repo: $err.msg()') diff --git a/src/web/parse.v b/src/web/parse.v index 889944b..7af635f 100644 --- a/src/web/parse.v +++ b/src/web/parse.v @@ -5,7 +5,7 @@ import net.http // Method attributes that should be ignored when parsing, as they're used // elsewhere. -const attrs_to_ignore = ['auth', 'markused'] +const attrs_to_ignore = ['auth'] // Parsing function attributes for methods and path. fn parse_attrs(name string, attrs []string) !([]http.Method, string) {