Merge pull request 'Some final stuff before 0.5.0' (#323) from Chewing_Bever/vieter:final-stuff into dev

Reviewed-on: vieter-v/vieter#323
web-stuff
Jef Roosens 2022-12-28 22:42:34 +01:00
commit 6738f8de67
19 changed files with 123 additions and 122 deletions

View File

@ -7,12 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
### Added
* CLI commands for removing packages, arch-repos & repositories
## [0.5.0-rc.2](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.2)
### Added
* API route for removing logs & accompanying CLI command
* Daemon for periodically removing old logs
* CLI flag to filter logs by specific exit codes
### Changed

View File

@ -3,7 +3,7 @@ SRC_DIR := src
SOURCES != find '$(SRC_DIR)' -iname '*.v'
V_PATH ?= v
V := $(V_PATH) -showcc -gc boehm -W -d use_openssl
V := $(V_PATH) -showcc -gc boehm -W -d use_openssl -skip-unused
all: vieter

View File

@ -59,10 +59,9 @@ configuration variable required for each command.
([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the
image used for the Vieter CI builds.
* `max_log_age`: maximum age of logs (in days). Logs older than this will get
cleaned by the log removal daemon. If set to a negative value, no logs are
ever removed. The age of logs is determined by the time the build was
started.
* Default: `-1`
cleaned by the log removal daemon. If set to zero, no logs are ever removed.
The age of logs is determined by the time the build was started.
* Default: `0`
* `log_removal_schedule`: cron schedule defining when to clean old logs.
* Default: `0 0` (every day at midnight)

View File

@ -1,3 +0,0 @@
---
weight: 100
---

View File

@ -1,81 +0,0 @@
# Builds In-depth
For those interested, this page describes how the build system works
internally.
## Builder image
Every cron daemon perodically creates a builder image that is then used as a
base for all builds. This is done to prevent build containers having to pull
down a bunch of updates when they update their system.
The build container is created by running the following commands inside a
container started from the image defined in `base_image`:
```sh
# Update repos & install required packages
pacman -Syu --needed --noconfirm base-devel git
# Add a non-root user to run makepkg
groupadd -g 1000 builder
useradd -mg builder builder
# Make sure they can use sudo without a password
echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
# Create the directory for the builds & make it writeable for the
# build user
mkdir /build
chown -R builder:builder /build
```
This script updates the packages to their latest versions & creates a non-root
user to use when running `makepkg`.
This script is base64-encoded & passed to the container as an environment
variable. The container's entrypoint is set to `/bin/sh -c` & its command
argument to `echo $BUILD_SCRIPT | base64 -d | /bin/sh -e`, with the
`BUILD_SCRIPT` environment variable containing the base64-encoded script.
Once the container exits, a new Docker image is created from it. This image is
then used as the base for any builds.
## Running builds
Each build has its own Docker container, using the builder image as its base.
The same base64-based technique as above is used, just with a different script.
To make the build logs more clear, each command is appended by an echo command
printing the next command to stdout.
Given the Git repository URL is `https://examplerepo.com` with branch `main`,
the URL of the Vieter server is `https://example.com` and `vieter` is the
repository we wish to publish to, we get the following script:
```sh
echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf'
echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
echo -e '+ pacman -Syu --needed --noconfirm'
pacman -Syu --needed --noconfirm
echo -e '+ su builder'
su builder
echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo'
git clone --single-branch --depth 1 --branch main https://examplerepo.com repo
echo -e '+ cd repo'
cd repo
echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'
makepkg --nobuild --syncdeps --needed --noconfirm
echo -e '+ source PKGBUILD'
source PKGBUILD
echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0'
curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
[ "$(id -u)" == 0 ] && exit 0
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done
```
This script:
1. Adds the target repository as a repository in the build container
2. Updates mirrors & packages
3. Clones the Git repository
4. Runs `makepkg` without building to calculate `pkgver`
5. Checks whether the package version is already present on the server
6. If not, run `makepkg` & publish any generated package archives to the server

View File

@ -1,28 +1,27 @@
module client
import models { BuildLog, BuildLogFilter }
import net.http { Method }
import web.response { Response }
import time
// get_build_logs returns all build logs.
pub fn (c &Client) get_build_logs(filter BuildLogFilter) ![]BuildLog {
params := models.params_from(filter)
data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)!
data := c.send_request<[]BuildLog>(.get, '/api/v1/logs', params)!
return data.data
}
// get_build_log returns a specific build log.
pub fn (c &Client) get_build_log(id int) !BuildLog {
data := c.send_request<BuildLog>(Method.get, '/api/v1/logs/$id', {})!
data := c.send_request<BuildLog>(.get, '/api/v1/logs/$id', {})!
return data.data
}
// get_build_log_content returns the contents of the build log file.
pub fn (c &Client) get_build_log_content(id int) !string {
data := c.send_request_raw_response(Method.get, '/api/v1/logs/$id/content', {}, '')!
data := c.send_request_raw_response(.get, '/api/v1/logs/$id/content', {}, '')!
return data
}
@ -37,7 +36,7 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t
'exitCode': exit_code.str()
}
data := c.send_request_with_body<int>(Method.post, '/api/v1/logs', params, content)!
data := c.send_request_with_body<int>(.post, '/api/v1/logs', params, content)!
return data
}

16
src/client/repos.v 100644
View File

@ -0,0 +1,16 @@
module client
// remove_repo removes an entire repository.
pub fn (c &Client) remove_repo(repo string) ! {
c.send_request<string>(.delete, '/$repo', {})!
}
// remove_arch_repo removes an entire arch-repo.
pub fn (c &Client) remove_arch_repo(repo string, arch string) ! {
c.send_request<string>(.delete, '/$repo/$arch', {})!
}
// remove_package removes a single package from the given arch-repo.
pub fn (c &Client) remove_package(repo string, arch string, pkgname string) ! {
c.send_request<string>(.delete, '/$repo/$arch/$pkgname', {})!
}

View File

@ -1,12 +1,11 @@
module client
import models { Target, TargetFilter }
import net.http { Method }
// get_targets returns a list of targets, given a filter object.
pub fn (c &Client) get_targets(filter TargetFilter) ![]Target {
params := models.params_from(filter)
data := c.send_request<[]Target>(Method.get, '/api/v1/targets', params)!
data := c.send_request<[]Target>(.get, '/api/v1/targets', params)!
return data.data
}
@ -34,7 +33,7 @@ pub fn (c &Client) get_all_targets() ![]Target {
// get_target returns the target for a specific id.
pub fn (c &Client) get_target(id int) !Target {
data := c.send_request<Target>(Method.get, '/api/v1/targets/$id', {})!
data := c.send_request<Target>(.get, '/api/v1/targets/$id', {})!
return data.data
}
@ -51,14 +50,14 @@ pub struct NewTarget {
// add_target adds a new target to the server.
pub fn (c &Client) add_target(t NewTarget) !int {
params := models.params_from<NewTarget>(t)
data := c.send_request<int>(Method.post, '/api/v1/targets', params)!
data := c.send_request<int>(.post, '/api/v1/targets', params)!
return data.data
}
// remove_target removes the target with the given id from the server.
pub fn (c &Client) remove_target(id int) !string {
data := c.send_request<string>(Method.delete, '/api/v1/targets/$id', {})!
data := c.send_request<string>(.delete, '/api/v1/targets/$id', {})!
return data.data
}
@ -66,7 +65,7 @@ pub fn (c &Client) remove_target(id int) !string {
// patch_target sends a PATCH request to the given target with the params as
// payload.
pub fn (c &Client) patch_target(id int, params map[string]string) !string {
data := c.send_request<string>(Method.patch, '/api/v1/targets/$id', params)!
data := c.send_request<string>(.patch, '/api/v1/targets/$id', params)!
return data.data
}

View File

@ -24,11 +24,13 @@ pub fn cmd() cli.Command {
flags: [
cli.Flag{
name: 'limit'
abbrev: 'l'
description: 'How many results to return.'
flag: cli.FlagType.int
},
cli.Flag{
name: 'offset'
abbrev: 'o'
description: 'Minimum index to return.'
flag: cli.FlagType.int
},
@ -39,16 +41,18 @@ pub fn cmd() cli.Command {
},
cli.Flag{
name: 'today'
description: 'Only list logs started today.'
abbrev: 't'
description: 'Only list logs started today. This flag overwrites any other date-related flag.'
flag: cli.FlagType.bool
},
cli.Flag{
name: 'failed'
description: 'Only list logs with non-zero exit codes.'
description: 'Only list logs with non-zero exit codes. This flag overwrites the --code flag.'
flag: cli.FlagType.bool
},
cli.Flag{
name: 'day'
abbrev: 'd'
description: 'Only list logs started on this day. (format: YYYY-MM-DD)'
flag: cli.FlagType.string
},
@ -62,6 +66,11 @@ pub fn cmd() cli.Command {
description: 'Only list logs started after this timestamp. (format: YYYY-MM-DD HH:mm:ss)'
flag: cli.FlagType.string
},
cli.Flag{
name: 'code'
description: 'Only return logs with the given exit code. Prepend with `!` to exclude instead of include. Can be specified multiple times.'
flag: cli.FlagType.string_array
},
]
execute: fn (cmd cli.Command) ! {
config_file := cmd.flags.get_string('config-file')!
@ -131,6 +140,8 @@ pub fn cmd() cli.Command {
filter.exit_codes = [
'!0',
]
} else {
filter.exit_codes = cmd.flags.get_strings('code')!
}
raw := cmd.flags.get_bool('raw')!

View File

@ -0,0 +1,52 @@
module repos
import cli
import conf as vconf
import client
struct Config {
address string [required]
api_key string [required]
}
// cmd returns the cli module that handles modifying the repository contents.
pub fn cmd() cli.Command {
return cli.Command{
name: 'repos'
description: 'Interact with the repositories & packages stored on the server.'
commands: [
cli.Command{
name: 'remove'
required_args: 1
usage: 'repo [arch [pkgname]]'
description: 'Remove a repo, arch-repo, or package from the server.'
flags: [
cli.Flag{
name: 'force'
flag: cli.FlagType.bool
},
]
execute: fn (cmd cli.Command) ! {
config_file := cmd.flags.get_string('config-file')!
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
if cmd.args.len < 3 {
if !cmd.flags.get_bool('force')! {
return error('Removing an arch-repo or repository is a very destructive command. If you really do wish to perform this operation, explicitely add the --force flag.')
}
}
client := client.new(conf.address, conf.api_key)
if cmd.args.len == 1 {
client.remove_repo(cmd.args[0])!
} else if cmd.args.len == 2 {
client.remove_arch_repo(cmd.args[0], cmd.args[1])!
} else {
client.remove_package(cmd.args[0], cmd.args[1], cmd.args[2])!
}
}
},
]
}
}

View File

@ -25,11 +25,13 @@ pub fn cmd() cli.Command {
flags: [
cli.Flag{
name: 'limit'
abbrev: 'l'
description: 'How many results to return.'
flag: cli.FlagType.int
},
cli.Flag{
name: 'offset'
abbrev: 'o'
description: 'Minimum index to return.'
flag: cli.FlagType.int
},

View File

@ -8,6 +8,7 @@ import console.logs
import console.schedule
import console.man
import console.aur
import console.repos
import cron
import agent
@ -48,6 +49,7 @@ fn main() {
man.cmd(),
aur.cmd(),
agent.cmd(),
repos.cmd(),
]
}
app.setup()

View File

@ -4,7 +4,7 @@ import web
import web.response { new_data_response, new_response }
// v1_poll_job_queue allows agents to poll for new build jobs.
['/api/v1/jobs/poll'; auth; get]
['/api/v1/jobs/poll'; auth; get; markused]
fn (mut app App) v1_poll_job_queue() web.Result {
arch := app.query['arch'] or {
return app.json(.bad_request, new_response('Missing arch query arg.'))
@ -21,7 +21,7 @@ fn (mut app App) v1_poll_job_queue() web.Result {
}
// v1_queue_job allows queueing a new one-time build job for the given target.
['/api/v1/jobs/queue'; auth; post]
['/api/v1/jobs/queue'; auth; markused; post]
fn (mut app App) v1_queue_job() web.Result {
target_id := app.query['target'] or {
return app.json(.bad_request, new_response('Missing target query arg.'))

View File

@ -11,7 +11,7 @@ import models { BuildLog, BuildLogFilter }
// v1_get_logs returns all build logs in the database. A 'target' query param can
// optionally be added to limit the list of build logs to that repository.
['/api/v1/logs'; auth; get]
['/api/v1/logs'; auth; get; markused]
fn (mut app App) v1_get_logs() web.Result {
filter := models.from_params<BuildLogFilter>(app.query) or {
return app.json(.bad_request, new_response('Invalid query parameters.'))
@ -22,7 +22,7 @@ fn (mut app App) v1_get_logs() web.Result {
}
// v1_get_single_log returns the build log with the given id.
['/api/v1/logs/:id'; auth; get]
['/api/v1/logs/:id'; auth; get; markused]
fn (mut app App) v1_get_single_log(id int) web.Result {
log := app.db.get_build_log(id) or { return app.status(.not_found) }
@ -30,7 +30,7 @@ fn (mut app App) v1_get_single_log(id int) web.Result {
}
// v1_get_log_content returns the actual build log file for the given id.
['/api/v1/logs/:id/content'; auth; get]
['/api/v1/logs/:id/content'; auth; get; markused]
fn (mut app App) v1_get_log_content(id int) web.Result {
log := app.db.get_build_log(id) or { return app.status(.not_found) }
file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
@ -50,7 +50,7 @@ fn parse_query_time(query string) !time.Time {
}
// v1_post_log adds a new log to the database.
['/api/v1/logs'; auth; post]
['/api/v1/logs'; auth; markused; post]
fn (mut app App) v1_post_log() web.Result {
// Parse query params
start_time_int := app.query['startTime'].int()
@ -121,7 +121,7 @@ fn (mut app App) v1_post_log() web.Result {
}
// v1_delete_log allows removing a build log from the system.
['/api/v1/logs/:id'; auth; delete]
['/api/v1/logs/:id'; auth; delete; markused]
fn (mut app App) v1_delete_log(id int) web.Result {
log := app.db.get_build_log(id) or { return app.status(.not_found) }
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())

View File

@ -6,7 +6,7 @@ import db
import models { Target, TargetArch, TargetFilter }
// v1_get_targets returns the current list of targets.
['/api/v1/targets'; auth; get]
['/api/v1/targets'; auth; get; markused]
fn (mut app App) v1_get_targets() web.Result {
filter := models.from_params<TargetFilter>(app.query) or {
return app.json(.bad_request, new_response('Invalid query parameters.'))
@ -17,7 +17,7 @@ fn (mut app App) v1_get_targets() web.Result {
}
// v1_get_single_target returns the information for a single target.
['/api/v1/targets/:id'; auth; get]
['/api/v1/targets/:id'; auth; get; markused]
fn (mut app App) v1_get_single_target(id int) web.Result {
target := app.db.get_target(id) or { return app.status(.not_found) }
@ -25,7 +25,7 @@ fn (mut app App) v1_get_single_target(id int) web.Result {
}
// v1_post_target creates a new target from the provided query string.
['/api/v1/targets'; auth; post]
['/api/v1/targets'; auth; markused; post]
fn (mut app App) v1_post_target() web.Result {
mut params := app.query.clone()
@ -55,7 +55,7 @@ fn (mut app App) v1_post_target() web.Result {
}
// v1_delete_target removes a given target from the server's list.
['/api/v1/targets/:id'; auth; delete]
['/api/v1/targets/:id'; auth; delete; markused]
fn (mut app App) v1_delete_target(id int) web.Result {
app.db.delete_target(id)
app.job_queue.invalidate(id)
@ -64,7 +64,7 @@ fn (mut app App) v1_delete_target(id int) web.Result {
}
// v1_patch_target updates a target's data with the given query params.
['/api/v1/targets/:id'; auth; patch]
['/api/v1/targets/:id'; auth; markused; patch]
fn (mut app App) v1_patch_target(id int) web.Result {
app.db.update_target(id, app.query)

View File

@ -13,7 +13,7 @@ pub:
default_arch string
global_schedule string = '0 3'
base_image string = 'archlinux:base-devel'
max_log_age int = -1
max_log_age int [empty_default]
log_removal_schedule string = '0 0'
}

View File

@ -10,7 +10,7 @@ import web.response { new_data_response, new_response }
// healthcheck just returns a string, but can be used to quickly check if the
// server is still responsive.
['/health'; get]
['/health'; get; markused]
pub fn (mut app App) healthcheck() web.Result {
return app.json(.ok, new_response('Healthy.'))
}
@ -18,7 +18,7 @@ pub fn (mut app App) healthcheck() web.Result {
// get_repo_file handles all Pacman-related routes. It returns both the
// repository's archives, but also package archives or the contents of a
// package's desc file.
['/:repo/:arch/:filename'; get; head]
['/:repo/:arch/:filename'; get; head; markused]
fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result {
mut full_path := ''
@ -48,7 +48,7 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re
}
// put_package handles publishing a package to a repository.
['/:repo/publish'; auth; post]
['/:repo/publish'; auth; markused; post]
fn (mut app App) put_package(repo string) web.Result {
// api is a reserved keyword for api routes & should never be allowed to be
// a repository.

View File

@ -3,7 +3,7 @@ module server
import web
// delete_package tries to remove the given package.
['/:repo/:arch/:pkg'; auth; delete]
['/:repo/:arch/:pkg'; auth; delete; markused]
fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result {
res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or {
app.lerror('Error while deleting package: $err.msg()')
@ -23,7 +23,7 @@ fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result
}
// delete_arch_repo tries to remove the given arch-repo.
['/:repo/:arch'; auth; delete]
['/:repo/:arch'; auth; delete; markused]
fn (mut app App) delete_arch_repo(repo string, arch string) web.Result {
res := app.repo.remove_arch_repo(repo, arch) or {
app.lerror('Error while deleting arch-repo: $err.msg()')
@ -43,7 +43,7 @@ fn (mut app App) delete_arch_repo(repo string, arch string) web.Result {
}
// delete_repo tries to remove the given repo.
['/:repo'; auth; delete]
['/:repo'; auth; delete; markused]
fn (mut app App) delete_repo(repo string) web.Result {
res := app.repo.remove_repo(repo) or {
app.lerror('Error while deleting repo: $err.msg()')

View File

@ -5,7 +5,7 @@ import net.http
// Method attributes that should be ignored when parsing, as they're used
// elsewhere.
const attrs_to_ignore = ['auth']
const attrs_to_ignore = ['auth', 'markused']
// Parsing function attributes for methods and path.
fn parse_attrs(name string, attrs []string) !([]http.Method, string) {