jenspots
/
vieter
forked from vieter-v/vieter
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Merge pull request 'Release 0.5.0: release candidate 2' (#320) from release-0.5.0-rc.2 into main 

Reviewed-on: vieter-v/vieter#320
Jef Roosens 2022-12-22 00:15:57 +01:00
parent 8b72a9fc0f be3762835d
commit dc517c23c5
19 changed files with 335 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
CHANGELOG.md
View File

@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
## [0.5.0-rc.2](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.2)
### Added
* API route for removing logs & accompanying CLI command
* Daemon for periodically removing old logs
### Changed
* Use `--long-option` instead of `-long-option` for CLI
## [0.5.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.1)
### Added

2
PKGBUILD
View File

@ -3,7 +3,7 @@
pkgbase='vieter'
pkgname='vieter'
pkgver='0.5.0-rc.1'
pkgver='0.5.0_rc.2'
pkgrel=1
pkgdesc="Lightweight Arch repository server & package build system"
depends=('glibc' 'openssl' 'libarchive' 'sqlite')

78
docs/api/source/includes/_jobs.md 100644
View File

@ -0,0 +1,78 @@
# Jobs
<aside class="notice">
All routes in this section require authentication.
</aside>
## Manually schedule a job
```shell
curl \
-H 'X-Api-Key: secret' \
https://example.com/api/v1/jobs/queue?target=10&force&arch=x86_64
```
Manually schedule a job on the server.
### HTTP Request
`POST /api/v1/jobs/queue`
### Query Parameters
Parameter | Description
--------- | -----------
target | Id of target to schedule build for
arch | Architecture to build on
force | Whether it's a forced build (true if present)
## Poll for new jobs
<aside class="warning">
This endpoint is used by the agents and should not be used manually. It's just
here for completeness. Requests to this endpoint modify the build queue,
meaning manual requests can cause builds to be skipped.
</aside>
```shell
curl \
-H 'X-Api-Key: secret' \
https://example.com/api/v1/jobs/poll?arch=x86_64&max=2
```
> JSON output format
```json
{
"message": "",
"data": [
{
"target_id": 1,
"kind": "git",
"url": "https://aur.archlinux.org/discord-ptb.git",
"branch": "master",
"path": "",
"repo": "bur",
"base_image": "archlinux:base-devel",
"force": true
}
]
}
```
Poll the server for new builds.
### HTTP Request
`GET /api/v1/jobs/poll`
### Query Parameters
Parameter | Description
--------- | -----------
arch | For which architecture to receive jobs
max | How many jobs to receive at most

25
docs/api/source/includes/_logs.md
View File

@ -125,8 +125,8 @@ id | ID of requested log
<aside class="warning">
You should probably not use this endpoint, as it's used by the build system to
publish its logs.
This endpoint is used by the agents and should not be used manually unless you
know what you're doing. It's just here for completeness.
</aside>
@ -149,3 +149,24 @@ target | id of target this build is for
### Request body
Plaintext contents of the build log.
## Remove a build log
```shell
curl \
-XDELETE \
-H 'X-Api-Key: secret' \
https://example.com/api/v1/logs/1
```
Remove a build log from the server.
### HTTP Request
`DELETE /api/v1/logs/:id`
### URL Parameters
Parameter | Description
--------- | -----------
id | id of log to remove

13
docs/api/source/includes/_targets.md
View File

@ -27,6 +27,7 @@ curl \
"kind": "git",
"url": "https://aur.archlinux.org/discord-ptb.git",
"branch": "master",
"path" : "",
"repo": "bur",
"schedule": "",
"arch": [
@ -73,8 +74,9 @@ curl \
"kind": "git",
"url": "https://aur.archlinux.org/discord-ptb.git",
"branch": "master",
"path": "",
"repo": "bur",
"schedule": "0 3",
"schedule": "0 2",
"arch": [
{
"id": 1,
@ -124,6 +126,7 @@ Parameter | Description
kind | Kind of target to add; one of 'git', 'url'.
url | URL of the Git repository.
branch | Branch of the Git repository.
path | Subdirectory inside Git repository to use.
repo | Vieter repository to publish built packages to.
schedule | Cron build schedule (syntax explained [here](https://rustybever.be/docs/vieter/usage/builds/schedule/))
arch | Comma-separated list of architectures to build package on.
@ -149,12 +152,20 @@ Parameter | Description
kind | Kind of target; one of 'git', 'url'.
url | URL of the Git repository.
branch | Branch of the Git repository.
path | Subdirectory inside Git repository to use.
repo | Vieter repository to publish built packages to.
schedule | Cron build schedule
arch | Comma-separated list of architectures to build package on.
## Remove a target
```shell
curl \
-XDELETE \
-H 'X-Api-Key: secret' \
https://example.com/api/v1/targets/1
```
Remove a target from the server.
### HTTP Request

1
docs/api/source/index.html.md
View File

@ -11,6 +11,7 @@ includes:
- repository
- targets
- logs
- jobs
search: true

28
docs/content/configuration.md
View File

@ -32,11 +32,11 @@ configuration variable required for each command.
### `vieter server`
* `port`: HTTP port to run on
* Default: `8000`
* `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`,
`WARN`, `INFO` or `DEBUG`.
* Default: `WARN`
* `log_file`: log file to write logs to.
* Default: `vieter.log` (in the current directory)
* `pkg_dir`: where Vieter should store the actual package archives.
* `data_dir`: where Vieter stores the repositories, log file & database.
* `api_key`: the API key to use when authenticating requests.
@ -44,9 +44,27 @@ configuration variable required for each command.
* Packages with architecture `any` are always added to this architecture.
This prevents the server from being confused when an `any` package is
published as the very first package for a repository.
* Git repositories added without an `arch` value use this value instead.
* `port`: HTTP port to run on
* Default: `8000`
* Targets added without an `arch` value use this value instead.
* `global_schedule`: build schedule for any target that does not have a
schedule defined. For information about this syntax, see
[here](/usage/builds/schedule).
* Default: `0 3` (3AM every night)
* `base_image`: Docker image to use when building a package. Any Pacman-based
distro image should work, as long as `/etc/pacman.conf` is used &
`base-devel` exists in the repositories. Make sure that the image supports
the architecture of your cron daemon.
* Default: `archlinux:base-devel` (only works on `x86_64`). If you require
`aarch64` support, consider using
[`menci/archlinuxarm:base-devel`](https://hub.docker.com/r/menci/archlinuxarm)
([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the
image used for the Vieter CI builds.
* `max_log_age`: maximum age of logs (in days). Logs older than this will get
cleaned by the log removal daemon. If set to a negative value, no logs are
ever removed. The age of logs is determined by the time the build was
started.
* Default: `-1`
* `log_removal_schedule`: cron schedule defining when to clean old logs.
* Default: `0 0` (every day at midnight)
### `vieter cron`

36
docs/content/installation.md
View File

@ -23,15 +23,15 @@ guarantees about stability, so beware!
Thanks to the single-binary design of Vieter, this image can be used both for
the repository server, the cron daemon and the agent.
Below is an example compose file to set up both the repository server & the
cron daemon:
Below is a minimal compose file to set up both the repository server & a build
agent:
```yaml
version: '3'
services:
server:
image: 'chewingbever/vieter:dev'
image: 'chewingbever/vieter:0.5.0-rc.1'
restart: 'always'
environment:
@ -41,18 +41,19 @@ services:
- 'data:/data'
cron:
image: 'chewingbever/vieter:dev'
image: 'chewingbever/vieter:0.5.0-rc.1'
restart: 'always'
# Required to connect to the Docker daemon
user: root
command: 'vieter cron'
command: 'vieter agent'
environment:
- 'VIETER_API_KEY=secret'
# MUST be public URL of Vieter repository
- 'VIETER_ADDRESS=https://example.com'
- 'VIETER_DEFAULT_ARCH=x86_64'
# Architecture for which the agent builds
- 'VIETER_ARCH=x86_64'
- 'VIETER_MAX_CONCURRENT_BUILDS=2'
- 'VIETER_GLOBAL_SCHEDULE=0 3'
volumes:
- '/var/run/docker.sock:/var/run/docker.sock'
@ -63,14 +64,17 @@ volumes:
If you do not require the build system, the repository server can be used
independently as well.
Of course, Vieter allows a lot more configuration than this. This compose file
is meant as a starting point for setting up your installation.
{{< hint info >}}
**Note**
Builds are executed on the cron daemon's system using the host's Docker daemon.
A cron daemon on a specific architecture will only build packages for that
specific architecture. Therefore, if you wish to build packages for both
`x86_64` & `aarch64`, you'll have to deploy two cron daemons, one on each
architecture. Afterwards, any Git repositories enabled for those two
architectures will build on both.
Builds are executed on the agent's system using the host's Docker daemon. An
agent for a specific `arch` will only build packages for that specific
architecture. Therefore, if you wish to build packages for both `x86_64` &
`aarch64`, you'll have to deploy two agents, one on each architecture.
Afterwards, any Git repositories enabled for those two architectures will build
on both.
{{< /hint >}}
## Binary
@ -99,9 +103,9 @@ latest official release or `vieter-git` for the latest development release.
### AUR
If you prefer building the packages locally (or on your own Vieter instance),
there's the `[vieter](https://aur.archlinux.org/packages/vieter)` &
`[vieter-git](https://aur.archlinux.org/packages/vieter-git)` packages on the
AUR. These packages build using the `vlang-git` compiler package, so I can't
there's the [`vieter`](https://aur.archlinux.org/packages/vieter) &
[`vieter-git`](https://aur.archlinux.org/packages/vieter-git) packages on the
AUR. These packages build using the `vlang` compiler package, so I can't
guarantee that a compiler update won't temporarily break them.
## Building from source

23
docs/content/usage/builds/cleanup.md 100644
View File

@ -0,0 +1,23 @@
---
weight: 20
---
# Cleanup
Vieter stores the logs of every single package build. While this is great for
debugging why builds fail, it also causes an active or long-running Vieter
instance to accumulate thousands of logs.
To combat this, a log removal daemon can be enabled that periodically removes
old build logs. By starting your server with the `max_log_age` variable (see
[Configuration](/configuration#vieter-server)), a daemon will get enabled that
periodically removes logs older than this setting. By default, this will happen
every day at midnight, but this behavior can be changed using the
`log_removal_schedule` variable.
{{< hint info >}}
**Note**
The daemon will always run a removal of logs on startup. Therefore, it's
possible the daemon will be *very* active when first enabling this setting.
After the initial surge of logs to remove, it'll calm down again.
{{< /hint >}}

4
docs/content/usage/builds/schedule.md
View File

@ -1,3 +1,7 @@
---
weight: 10
---
# Cron schedule syntax
The Vieter cron daemon uses a subset of the cron expression syntax to schedule

5
src/client/logs.v
View File

@ -41,3 +41,8 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t
return data
}
// remove_build_log removes the build log with the given id from the server.
pub fn (c &Client) remove_build_log(id int) ! {
c.send_request<string>(.delete, '/api/v1/logs/$id', {})!
}

18
src/console/logs/logs.v
View File

@ -138,6 +138,18 @@ pub fn cmd() cli.Command {
list(conf, filter, raw)!
}
},
cli.Command{
name: 'remove'
required_args: 1
usage: 'id'
description: 'Remove a build log that matches the given id.'
execute: fn (cmd cli.Command) ! {
config_file := cmd.flags.get_string('config-file')!
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
remove(conf, cmd.args[0])!
}
},
cli.Command{
name: 'info'
required_args: 1
@ -204,3 +216,9 @@ fn content(conf Config, id int) ! {
println(content)
}
// remove removes a build log from the server's list.
fn remove(conf Config, id string) ! {
c := client.new(conf.address, conf.api_key)
c.remove_build_log(id.int())!
}

3
src/main.v
View File

@ -20,7 +20,8 @@ fn main() {
mut app := cli.Command{
name: 'vieter'
description: 'Vieter is a lightweight implementation of an Arch repository server.'
version: '0.5.0-rc.1'
version: '0.5.0-rc.2'
posix_mode: true
flags: [
cli.Flag{
flag: cli.FlagType.string

8
src/models/logs.v
View File

@ -1,6 +1,7 @@
module models
import time
import os
pub struct BuildLog {
pub mut:
@ -28,6 +29,13 @@ pub fn (bl &BuildLog) str() string {
return str
}
// path returns the path to the log file, relative to the logs directory
pub fn (bl &BuildLog) path() string {
filename := bl.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
return os.join_path(bl.target_id.str(), bl.arch, filename)
}
[params]
pub struct BuildLogFilter {
pub mut:

38
src/server/api_logs.v
View File

@ -86,7 +86,7 @@ fn (mut app App) v1_post_log() web.Result {
}
// Store log in db
log := BuildLog{
mut log := BuildLog{
target_id: target_id
start_time: start_time
end_time: end_time
@ -95,25 +95,20 @@ fn (mut app App) v1_post_log() web.Result {
}
// id of newly created log
log_id := app.db.add_build_log(log)
repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, target_id.str(), arch)
log.id = app.db.add_build_log(log)
log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
// Create the logs directory of it doesn't exist
if !os.exists(repo_logs_dir) {
os.mkdir_all(repo_logs_dir) or {
app.lerror("Couldn't create dir '$repo_logs_dir'.")
if !os.exists(os.dir(log_file_path)) {
os.mkdir_all(os.dir(log_file_path)) or {
app.lerror('Error while creating log file: $err.msg()')
return app.status(.internal_server_error)
}
}
// Stream log contents to correct file
file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
full_path := os.join_path_single(repo_logs_dir, file_name)
if length := app.req.header.get(.content_length) {
util.reader_to_file(mut app.reader, length.int(), full_path) or {
util.reader_to_file(mut app.reader, length.int(), log_file_path) or {
app.lerror('An error occured while receiving logs: $err.msg()')
return app.status(.internal_server_error)
@ -122,5 +117,22 @@ fn (mut app App) v1_post_log() web.Result {
return app.status(.length_required)
}
return app.json(.ok, new_data_response(log_id))
return app.json(.ok, new_data_response(log.id))
}
// v1_delete_log allows removing a build log from the system.
['/api/v1/logs/:id'; auth; delete]
fn (mut app App) v1_delete_log(id int) web.Result {
log := app.db.get_build_log(id) or { return app.status(.not_found) }
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
os.rm(full_path) or {
app.lerror('Failed to remove log file $full_path: $err.msg()')
return app.status(.internal_server_error)
}
app.db.delete_build_log(id)
return app.status(.ok)
}

18
src/server/cli.v
View File

@ -5,14 +5,16 @@ import conf as vconf
struct Config {
pub:
log_level string = 'WARN'
pkg_dir string
data_dir string
api_key string
default_arch string
global_schedule string = '0 3'
port int = 8000
base_image string = 'archlinux:base-devel'
port int = 8000
log_level string = 'WARN'
pkg_dir string
data_dir string
api_key string
default_arch string
global_schedule string = '0 3'
base_image string = 'archlinux:base-devel'
max_log_age int = -1
log_removal_schedule string = '0 0'
}
// cmd returns the cli submodule that handles starting the server

62
src/server/log_removal.v 100644
View File

@ -0,0 +1,62 @@
module server
import time
import models { BuildLog }
import os
import cron.expression { CronExpression }
const fallback_log_removal_frequency = 24 * time.hour
// log_removal_daemon removes old build logs every `log_removal_frequency`.
fn (mut app App) log_removal_daemon(schedule CronExpression) {
mut start_time := time.Time{}
for {
start_time = time.now()
mut too_old_timestamp := time.now().add_days(-app.conf.max_log_age)
app.linfo('Cleaning logs before $too_old_timestamp')
mut logs := []BuildLog{}
mut counter := 0
mut failed := u64(0)
// Remove old logs
for {
// The offset is used to skip logs that failed to remove. Besides
// this, we don't need to move the offset, because all previously
// oldest logs will have been removed.
logs = app.db.get_build_logs(before: too_old_timestamp, offset: failed, limit: 50)
for log in logs {
log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
os.rm(log_file_path) or {
app.lerror('Failed to remove log file $log_file_path: $err.msg()')
failed += 1
continue
}
app.db.delete_build_log(log.id)
counter += 1
}
if logs.len < 50 {
break
}
}
app.linfo('Cleaned $counter logs ($failed failed)')
// Sleep until the next cycle
next_time := schedule.next_from_now() or {
app.lerror("Log removal daemon couldn't calculate next time: $err.msg(); fallback to $server.fallback_log_removal_frequency")
start_time.add(server.fallback_log_removal_frequency)
}
time.sleep(next_time - time.now())
}
}

8
src/server/server.v
View File

@ -55,6 +55,10 @@ pub fn server(conf Config) ! {
util.exit_with_message(1, 'Invalid global cron expression: $err.msg()')
}
log_removal_ce := expression.parse_expression(conf.log_removal_schedule) or {
util.exit_with_message(1, 'Invalid log removal cron expression: $err.msg()')
}
// Configure logger
log_level := log.level_from_tag(conf.log_level) or {
util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
@ -108,5 +112,9 @@ pub fn server(conf Config) ! {
util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()')
}
if conf.max_log_age > 0 {
go app.log_removal_daemon(log_removal_ce)
}
web.run(app, conf.port)
}

1
vieter.toml
View File

@ -12,3 +12,4 @@ address = "http://localhost:8000"
api_update_frequency = 2
image_rebuild_frequency = 1
max_concurrent_builds = 3
max_log_age = 64