diff --git a/CHANGELOG.md b/CHANGELOG.md index e615698b..0e4e228b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev) +## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0) + +### Added + +* CLI commands for removing packages, arch-repos & repositories + ## [0.5.0-rc.2](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.2) ### Added * API route for removing logs & accompanying CLI command * Daemon for periodically removing old logs +* CLI flag to filter logs by specific exit codes ### Changed diff --git a/Makefile b/Makefile index e7168076..4bd1edc1 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' V_PATH ?= v -V := $(V_PATH) -showcc -gc boehm -W -d use_openssl +V := $(V_PATH) -showcc -gc boehm -W -d use_openssl -skip-unused all: vieter diff --git a/PKGBUILD b/PKGBUILD index 5e9530a0..bf9c6213 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -3,7 +3,7 @@ pkgbase='vieter' pkgname='vieter' -pkgver='0.5.0_rc.2' +pkgver='0.5.0' pkgrel=1 pkgdesc="Lightweight Arch repository server & package build system" depends=('glibc' 'openssl' 'libarchive' 'sqlite') diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 45c5de6f..612c5053 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -59,10 +59,9 @@ configuration variable required for each command. ([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the image used for the Vieter CI builds. * `max_log_age`: maximum age of logs (in days). Logs older than this will get - cleaned by the log removal daemon. If set to a negative value, no logs are - ever removed. The age of logs is determined by the time the build was - started. - * Default: `-1` + cleaned by the log removal daemon. If set to zero, no logs are ever removed. + The age of logs is determined by the time the build was started. + * Default: `0` * `log_removal_schedule`: cron schedule defining when to clean old logs. * Default: `0 0` (every day at midnight) diff --git a/docs/content/other/_index.md b/docs/content/other/_index.md deleted file mode 100644 index 394456b2..00000000 --- a/docs/content/other/_index.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -weight: 100 ---- diff --git a/docs/content/other/builds-in-depth.md b/docs/content/other/builds-in-depth.md deleted file mode 100644 index d8df6ec7..00000000 --- a/docs/content/other/builds-in-depth.md +++ /dev/null @@ -1,81 +0,0 @@ -# Builds In-depth - -For those interested, this page describes how the build system works -internally. - -## Builder image - -Every cron daemon perodically creates a builder image that is then used as a -base for all builds. This is done to prevent build containers having to pull -down a bunch of updates when they update their system. - -The build container is created by running the following commands inside a -container started from the image defined in `base_image`: - -```sh -# Update repos & install required packages -pacman -Syu --needed --noconfirm base-devel git -# Add a non-root user to run makepkg -groupadd -g 1000 builder -useradd -mg builder builder -# Make sure they can use sudo without a password -echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers -# Create the directory for the builds & make it writeable for the -# build user -mkdir /build -chown -R builder:builder /build -``` - -This script updates the packages to their latest versions & creates a non-root -user to use when running `makepkg`. - -This script is base64-encoded & passed to the container as an environment -variable. The container's entrypoint is set to `/bin/sh -c` & its command -argument to `echo $BUILD_SCRIPT | base64 -d | /bin/sh -e`, with the -`BUILD_SCRIPT` environment variable containing the base64-encoded script. - -Once the container exits, a new Docker image is created from it. This image is -then used as the base for any builds. - -## Running builds - -Each build has its own Docker container, using the builder image as its base. -The same base64-based technique as above is used, just with a different script. -To make the build logs more clear, each command is appended by an echo command -printing the next command to stdout. - -Given the Git repository URL is `https://examplerepo.com` with branch `main`, -the URL of the Vieter server is `https://example.com` and `vieter` is the -repository we wish to publish to, we get the following script: - -```sh -echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' -echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf -echo -e '+ pacman -Syu --needed --noconfirm' -pacman -Syu --needed --noconfirm -echo -e '+ su builder' -su builder -echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo' -git clone --single-branch --depth 1 --branch main https://examplerepo.com repo -echo -e '+ cd repo' -cd repo -echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' -makepkg --nobuild --syncdeps --needed --noconfirm -echo -e '+ source PKGBUILD' -source PKGBUILD -echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' -curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 -echo -e '+ [ "$(id -u)" == 0 ] && exit 0' -[ "$(id -u)" == 0 ] && exit 0 -echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' -MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done -``` - -This script: - -1. Adds the target repository as a repository in the build container -2. Updates mirrors & packages -3. Clones the Git repository -4. Runs `makepkg` without building to calculate `pkgver` -5. Checks whether the package version is already present on the server -6. If not, run `makepkg` & publish any generated package archives to the server diff --git a/src/client/logs.v b/src/client/logs.v index 2ddb2e26..65538377 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -1,28 +1,27 @@ module client import models { BuildLog, BuildLogFilter } -import net.http { Method } import web.response { Response } import time // get_build_logs returns all build logs. pub fn (c &Client) get_build_logs(filter BuildLogFilter) ![]BuildLog { params := models.params_from(filter) - data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)! + data := c.send_request<[]BuildLog>(.get, '/api/v1/logs', params)! return data.data } // get_build_log returns a specific build log. pub fn (c &Client) get_build_log(id int) !BuildLog { - data := c.send_request(Method.get, '/api/v1/logs/$id', {})! + data := c.send_request(.get, '/api/v1/logs/$id', {})! return data.data } // get_build_log_content returns the contents of the build log file. pub fn (c &Client) get_build_log_content(id int) !string { - data := c.send_request_raw_response(Method.get, '/api/v1/logs/$id/content', {}, '')! + data := c.send_request_raw_response(.get, '/api/v1/logs/$id/content', {}, '')! return data } @@ -37,7 +36,7 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t 'exitCode': exit_code.str() } - data := c.send_request_with_body(Method.post, '/api/v1/logs', params, content)! + data := c.send_request_with_body(.post, '/api/v1/logs', params, content)! return data } diff --git a/src/client/repos.v b/src/client/repos.v new file mode 100644 index 00000000..9644e9ba --- /dev/null +++ b/src/client/repos.v @@ -0,0 +1,16 @@ +module client + +// remove_repo removes an entire repository. +pub fn (c &Client) remove_repo(repo string) ! { + c.send_request(.delete, '/$repo', {})! +} + +// remove_arch_repo removes an entire arch-repo. +pub fn (c &Client) remove_arch_repo(repo string, arch string) ! { + c.send_request(.delete, '/$repo/$arch', {})! +} + +// remove_package removes a single package from the given arch-repo. +pub fn (c &Client) remove_package(repo string, arch string, pkgname string) ! { + c.send_request(.delete, '/$repo/$arch/$pkgname', {})! +} diff --git a/src/client/targets.v b/src/client/targets.v index da6a9e42..565832ea 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -1,12 +1,11 @@ module client import models { Target, TargetFilter } -import net.http { Method } // get_targets returns a list of targets, given a filter object. pub fn (c &Client) get_targets(filter TargetFilter) ![]Target { params := models.params_from(filter) - data := c.send_request<[]Target>(Method.get, '/api/v1/targets', params)! + data := c.send_request<[]Target>(.get, '/api/v1/targets', params)! return data.data } @@ -34,7 +33,7 @@ pub fn (c &Client) get_all_targets() ![]Target { // get_target returns the target for a specific id. pub fn (c &Client) get_target(id int) !Target { - data := c.send_request(Method.get, '/api/v1/targets/$id', {})! + data := c.send_request(.get, '/api/v1/targets/$id', {})! return data.data } @@ -51,14 +50,14 @@ pub struct NewTarget { // add_target adds a new target to the server. pub fn (c &Client) add_target(t NewTarget) !int { params := models.params_from(t) - data := c.send_request(Method.post, '/api/v1/targets', params)! + data := c.send_request(.post, '/api/v1/targets', params)! return data.data } // remove_target removes the target with the given id from the server. pub fn (c &Client) remove_target(id int) !string { - data := c.send_request(Method.delete, '/api/v1/targets/$id', {})! + data := c.send_request(.delete, '/api/v1/targets/$id', {})! return data.data } @@ -66,7 +65,7 @@ pub fn (c &Client) remove_target(id int) !string { // patch_target sends a PATCH request to the given target with the params as // payload. pub fn (c &Client) patch_target(id int, params map[string]string) !string { - data := c.send_request(Method.patch, '/api/v1/targets/$id', params)! + data := c.send_request(.patch, '/api/v1/targets/$id', params)! return data.data } diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 19c46f64..35ce4d79 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -24,11 +24,13 @@ pub fn cmd() cli.Command { flags: [ cli.Flag{ name: 'limit' + abbrev: 'l' description: 'How many results to return.' flag: cli.FlagType.int }, cli.Flag{ name: 'offset' + abbrev: 'o' description: 'Minimum index to return.' flag: cli.FlagType.int }, @@ -39,16 +41,18 @@ pub fn cmd() cli.Command { }, cli.Flag{ name: 'today' - description: 'Only list logs started today.' + abbrev: 't' + description: 'Only list logs started today. This flag overwrites any other date-related flag.' flag: cli.FlagType.bool }, cli.Flag{ name: 'failed' - description: 'Only list logs with non-zero exit codes.' + description: 'Only list logs with non-zero exit codes. This flag overwrites the --code flag.' flag: cli.FlagType.bool }, cli.Flag{ name: 'day' + abbrev: 'd' description: 'Only list logs started on this day. (format: YYYY-MM-DD)' flag: cli.FlagType.string }, @@ -62,6 +66,11 @@ pub fn cmd() cli.Command { description: 'Only list logs started after this timestamp. (format: YYYY-MM-DD HH:mm:ss)' flag: cli.FlagType.string }, + cli.Flag{ + name: 'code' + description: 'Only return logs with the given exit code. Prepend with `!` to exclude instead of include. Can be specified multiple times.' + flag: cli.FlagType.string_array + }, ] execute: fn (cmd cli.Command) ! { config_file := cmd.flags.get_string('config-file')! @@ -131,6 +140,8 @@ pub fn cmd() cli.Command { filter.exit_codes = [ '!0', ] + } else { + filter.exit_codes = cmd.flags.get_strings('code')! } raw := cmd.flags.get_bool('raw')! diff --git a/src/console/repos/repos.v b/src/console/repos/repos.v new file mode 100644 index 00000000..729208e4 --- /dev/null +++ b/src/console/repos/repos.v @@ -0,0 +1,52 @@ +module repos + +import cli +import conf as vconf +import client + +struct Config { + address string [required] + api_key string [required] +} + +// cmd returns the cli module that handles modifying the repository contents. +pub fn cmd() cli.Command { + return cli.Command{ + name: 'repos' + description: 'Interact with the repositories & packages stored on the server.' + commands: [ + cli.Command{ + name: 'remove' + required_args: 1 + usage: 'repo [arch [pkgname]]' + description: 'Remove a repo, arch-repo, or package from the server.' + flags: [ + cli.Flag{ + name: 'force' + flag: cli.FlagType.bool + }, + ] + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + + if cmd.args.len < 3 { + if !cmd.flags.get_bool('force')! { + return error('Removing an arch-repo or repository is a very destructive command. If you really do wish to perform this operation, explicitely add the --force flag.') + } + } + + client := client.new(conf.address, conf.api_key) + + if cmd.args.len == 1 { + client.remove_repo(cmd.args[0])! + } else if cmd.args.len == 2 { + client.remove_arch_repo(cmd.args[0], cmd.args[1])! + } else { + client.remove_package(cmd.args[0], cmd.args[1], cmd.args[2])! + } + } + }, + ] + } +} diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 94deebd8..3c0d755a 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -25,11 +25,13 @@ pub fn cmd() cli.Command { flags: [ cli.Flag{ name: 'limit' + abbrev: 'l' description: 'How many results to return.' flag: cli.FlagType.int }, cli.Flag{ name: 'offset' + abbrev: 'o' description: 'Minimum index to return.' flag: cli.FlagType.int }, diff --git a/src/main.v b/src/main.v index eda38e75..1c8b816f 100644 --- a/src/main.v +++ b/src/main.v @@ -8,6 +8,7 @@ import console.logs import console.schedule import console.man import console.aur +import console.repos import cron import agent @@ -20,7 +21,7 @@ fn main() { mut app := cli.Command{ name: 'vieter' description: 'Vieter is a lightweight implementation of an Arch repository server.' - version: '0.5.0-rc.2' + version: '0.5.0' posix_mode: true flags: [ cli.Flag{ @@ -48,6 +49,7 @@ fn main() { man.cmd(), aur.cmd(), agent.cmd(), + repos.cmd(), ] } app.setup() diff --git a/src/server/api_jobs.v b/src/server/api_jobs.v index 77953514..62bcb275 100644 --- a/src/server/api_jobs.v +++ b/src/server/api_jobs.v @@ -4,7 +4,7 @@ import web import web.response { new_data_response, new_response } // v1_poll_job_queue allows agents to poll for new build jobs. -['/api/v1/jobs/poll'; auth; get] +['/api/v1/jobs/poll'; auth; get; markused] fn (mut app App) v1_poll_job_queue() web.Result { arch := app.query['arch'] or { return app.json(.bad_request, new_response('Missing arch query arg.')) @@ -21,7 +21,7 @@ fn (mut app App) v1_poll_job_queue() web.Result { } // v1_queue_job allows queueing a new one-time build job for the given target. -['/api/v1/jobs/queue'; auth; post] +['/api/v1/jobs/queue'; auth; markused; post] fn (mut app App) v1_queue_job() web.Result { target_id := app.query['target'] or { return app.json(.bad_request, new_response('Missing target query arg.')) diff --git a/src/server/api_logs.v b/src/server/api_logs.v index 13b50b94..3db4204d 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -11,7 +11,7 @@ import models { BuildLog, BuildLogFilter } // v1_get_logs returns all build logs in the database. A 'target' query param can // optionally be added to limit the list of build logs to that repository. -['/api/v1/logs'; auth; get] +['/api/v1/logs'; auth; get; markused] fn (mut app App) v1_get_logs() web.Result { filter := models.from_params(app.query) or { return app.json(.bad_request, new_response('Invalid query parameters.')) @@ -22,7 +22,7 @@ fn (mut app App) v1_get_logs() web.Result { } // v1_get_single_log returns the build log with the given id. -['/api/v1/logs/:id'; auth; get] +['/api/v1/logs/:id'; auth; get; markused] fn (mut app App) v1_get_single_log(id int) web.Result { log := app.db.get_build_log(id) or { return app.status(.not_found) } @@ -30,7 +30,7 @@ fn (mut app App) v1_get_single_log(id int) web.Result { } // v1_get_log_content returns the actual build log file for the given id. -['/api/v1/logs/:id/content'; auth; get] +['/api/v1/logs/:id/content'; auth; get; markused] fn (mut app App) v1_get_log_content(id int) web.Result { log := app.db.get_build_log(id) or { return app.status(.not_found) } file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') @@ -50,7 +50,7 @@ fn parse_query_time(query string) !time.Time { } // v1_post_log adds a new log to the database. -['/api/v1/logs'; auth; post] +['/api/v1/logs'; auth; markused; post] fn (mut app App) v1_post_log() web.Result { // Parse query params start_time_int := app.query['startTime'].int() @@ -121,7 +121,7 @@ fn (mut app App) v1_post_log() web.Result { } // v1_delete_log allows removing a build log from the system. -['/api/v1/logs/:id'; auth; delete] +['/api/v1/logs/:id'; auth; delete; markused] fn (mut app App) v1_delete_log(id int) web.Result { log := app.db.get_build_log(id) or { return app.status(.not_found) } full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) diff --git a/src/server/api_targets.v b/src/server/api_targets.v index cd5cb0a0..4bb7d128 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -6,7 +6,7 @@ import db import models { Target, TargetArch, TargetFilter } // v1_get_targets returns the current list of targets. -['/api/v1/targets'; auth; get] +['/api/v1/targets'; auth; get; markused] fn (mut app App) v1_get_targets() web.Result { filter := models.from_params(app.query) or { return app.json(.bad_request, new_response('Invalid query parameters.')) @@ -17,7 +17,7 @@ fn (mut app App) v1_get_targets() web.Result { } // v1_get_single_target returns the information for a single target. -['/api/v1/targets/:id'; auth; get] +['/api/v1/targets/:id'; auth; get; markused] fn (mut app App) v1_get_single_target(id int) web.Result { target := app.db.get_target(id) or { return app.status(.not_found) } @@ -25,7 +25,7 @@ fn (mut app App) v1_get_single_target(id int) web.Result { } // v1_post_target creates a new target from the provided query string. -['/api/v1/targets'; auth; post] +['/api/v1/targets'; auth; markused; post] fn (mut app App) v1_post_target() web.Result { mut params := app.query.clone() @@ -55,7 +55,7 @@ fn (mut app App) v1_post_target() web.Result { } // v1_delete_target removes a given target from the server's list. -['/api/v1/targets/:id'; auth; delete] +['/api/v1/targets/:id'; auth; delete; markused] fn (mut app App) v1_delete_target(id int) web.Result { app.db.delete_target(id) app.job_queue.invalidate(id) @@ -64,7 +64,7 @@ fn (mut app App) v1_delete_target(id int) web.Result { } // v1_patch_target updates a target's data with the given query params. -['/api/v1/targets/:id'; auth; patch] +['/api/v1/targets/:id'; auth; markused; patch] fn (mut app App) v1_patch_target(id int) web.Result { app.db.update_target(id, app.query) diff --git a/src/server/cli.v b/src/server/cli.v index 795f7646..21fb15e2 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -13,7 +13,7 @@ pub: default_arch string global_schedule string = '0 3' base_image string = 'archlinux:base-devel' - max_log_age int = -1 + max_log_age int [empty_default] log_removal_schedule string = '0 0' } diff --git a/src/server/repo.v b/src/server/repo.v index 06ab72e6..38d07fe3 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -10,7 +10,7 @@ import web.response { new_data_response, new_response } // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. -['/health'; get] +['/health'; get; markused] pub fn (mut app App) healthcheck() web.Result { return app.json(.ok, new_response('Healthy.')) } @@ -18,7 +18,7 @@ pub fn (mut app App) healthcheck() web.Result { // get_repo_file handles all Pacman-related routes. It returns both the // repository's archives, but also package archives or the contents of a // package's desc file. -['/:repo/:arch/:filename'; get; head] +['/:repo/:arch/:filename'; get; head; markused] fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result { mut full_path := '' @@ -48,7 +48,7 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re } // put_package handles publishing a package to a repository. -['/:repo/publish'; auth; post] +['/:repo/publish'; auth; markused; post] fn (mut app App) put_package(repo string) web.Result { // api is a reserved keyword for api routes & should never be allowed to be // a repository. diff --git a/src/server/repo_remove.v b/src/server/repo_remove.v index 694f0854..9e6d747b 100644 --- a/src/server/repo_remove.v +++ b/src/server/repo_remove.v @@ -3,7 +3,7 @@ module server import web // delete_package tries to remove the given package. -['/:repo/:arch/:pkg'; auth; delete] +['/:repo/:arch/:pkg'; auth; delete; markused] fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result { res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { app.lerror('Error while deleting package: $err.msg()') @@ -23,7 +23,7 @@ fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result } // delete_arch_repo tries to remove the given arch-repo. -['/:repo/:arch'; auth; delete] +['/:repo/:arch'; auth; delete; markused] fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { res := app.repo.remove_arch_repo(repo, arch) or { app.lerror('Error while deleting arch-repo: $err.msg()') @@ -43,7 +43,7 @@ fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { } // delete_repo tries to remove the given repo. -['/:repo'; auth; delete] +['/:repo'; auth; delete; markused] fn (mut app App) delete_repo(repo string) web.Result { res := app.repo.remove_repo(repo) or { app.lerror('Error while deleting repo: $err.msg()') diff --git a/src/web/parse.v b/src/web/parse.v index 7af635fb..889944b4 100644 --- a/src/web/parse.v +++ b/src/web/parse.v @@ -5,7 +5,7 @@ import net.http // Method attributes that should be ignored when parsing, as they're used // elsewhere. -const attrs_to_ignore = ['auth'] +const attrs_to_ignore = ['auth', 'markused'] // Parsing function attributes for methods and path. fn parse_attrs(name string, attrs []string) !([]http.Method, string) {