From 09c61143b0ea3f0981f50ff1ebabf450f497cefb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Dec 2022 20:23:25 +0100 Subject: [PATCH 1/2] docs: updated the docs --- docs/api/source/includes/_jobs.md | 51 +++++++++++++++++++++++++++++++ docs/api/source/index.html.md | 1 + docs/content/installation.md | 36 ++++++++++++---------- 3 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 docs/api/source/includes/_jobs.md diff --git a/docs/api/source/includes/_jobs.md b/docs/api/source/includes/_jobs.md new file mode 100644 index 0000000..c124781 --- /dev/null +++ b/docs/api/source/includes/_jobs.md @@ -0,0 +1,51 @@ +# Jobs + + + +## Manually schedule a job + +```shell +curl \ + -H 'X-Api-Key: secret' \ + https://example.com/api/v1/jobs/queue?target=10&force&arch=x86_64 +``` + +Manually schedule a job on the server. + +### HTTP Request + +`POST /api/v1/jobs/queue` + +### Query Parameters + +Parameter | Description +--------- | ----------- +target | Id of target to schedule build for +arch | Architecture to build on +force | Whether it's a forced build (true if present) + +## Poll for new jobs + + + +Poll the server for new builds. + +### HTTP Request + +`GET /api/v1/jobs/poll` + +### Query Parameters + +Parameter | Description +--------- | ----------- +arch | For which architecture to receive jobs +max | How many jobs to receive at most diff --git a/docs/api/source/index.html.md b/docs/api/source/index.html.md index 4bfddb8..f61e44a 100644 --- a/docs/api/source/index.html.md +++ b/docs/api/source/index.html.md @@ -11,6 +11,7 @@ includes: - repository - targets - logs + - jobs search: true diff --git a/docs/content/installation.md b/docs/content/installation.md index 21eda64..5b8e2d8 100644 --- a/docs/content/installation.md +++ b/docs/content/installation.md @@ -23,15 +23,15 @@ guarantees about stability, so beware! Thanks to the single-binary design of Vieter, this image can be used both for the repository server, the cron daemon and the agent. -Below is an example compose file to set up both the repository server & the -cron daemon: +Below is a minimal compose file to set up both the repository server & a build +agent: ```yaml version: '3' services: server: - image: 'chewingbever/vieter:dev' + image: 'chewingbever/vieter:0.5.0-rc.1' restart: 'always' environment: @@ -41,18 +41,19 @@ services: - 'data:/data' cron: - image: 'chewingbever/vieter:dev' + image: 'chewingbever/vieter:0.5.0-rc.1' restart: 'always' + # Required to connect to the Docker daemon user: root - command: 'vieter cron' + command: 'vieter agent' environment: - 'VIETER_API_KEY=secret' # MUST be public URL of Vieter repository - 'VIETER_ADDRESS=https://example.com' - - 'VIETER_DEFAULT_ARCH=x86_64' + # Architecture for which the agent builds + - 'VIETER_ARCH=x86_64' - 'VIETER_MAX_CONCURRENT_BUILDS=2' - - 'VIETER_GLOBAL_SCHEDULE=0 3' volumes: - '/var/run/docker.sock:/var/run/docker.sock' @@ -63,14 +64,17 @@ volumes: If you do not require the build system, the repository server can be used independently as well. +Of course, Vieter allows a lot more configuration than this. This compose file +is meant as a starting point for setting up your installation. + {{< hint info >}} **Note** -Builds are executed on the cron daemon's system using the host's Docker daemon. -A cron daemon on a specific architecture will only build packages for that -specific architecture. Therefore, if you wish to build packages for both -`x86_64` & `aarch64`, you'll have to deploy two cron daemons, one on each -architecture. Afterwards, any Git repositories enabled for those two -architectures will build on both. +Builds are executed on the agent's system using the host's Docker daemon. An +agent for a specific `arch` will only build packages for that specific +architecture. Therefore, if you wish to build packages for both `x86_64` & +`aarch64`, you'll have to deploy two agents, one on each architecture. +Afterwards, any Git repositories enabled for those two architectures will build +on both. {{< /hint >}} ## Binary @@ -99,9 +103,9 @@ latest official release or `vieter-git` for the latest development release. ### AUR If you prefer building the packages locally (or on your own Vieter instance), -there's the `[vieter](https://aur.archlinux.org/packages/vieter)` & -`[vieter-git](https://aur.archlinux.org/packages/vieter-git)` packages on the -AUR. These packages build using the `vlang-git` compiler package, so I can't +there's the [`vieter`](https://aur.archlinux.org/packages/vieter) & +[`vieter-git`](https://aur.archlinux.org/packages/vieter-git) packages on the +AUR. These packages build using the `vlang` compiler package, so I can't guarantee that a compiler update won't temporarily break them. ## Building from source From 26796f2228fe92fb32330ea8f8d0095532ec40b7 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 19 Dec 2022 09:47:53 +0100 Subject: [PATCH 2/2] feat(server): use cron schedule for log removal instead --- docs/content/configuration.md | 29 +++++++++++++++++++-------- docs/content/usage/builds/cleanup.md | 24 ++++++++++++++++++++++ docs/content/usage/builds/schedule.md | 4 ++++ src/server/cli.v | 19 +++++++++--------- src/server/log_removal.v | 13 +++++++++--- src/server/server.v | 6 +++++- vieter.toml | 1 + 7 files changed, 75 insertions(+), 21 deletions(-) create mode 100644 docs/content/usage/builds/cleanup.md diff --git a/docs/content/configuration.md b/docs/content/configuration.md index e59fe06..e974a58 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -32,11 +32,11 @@ configuration variable required for each command. ### `vieter server` +* `port`: HTTP port to run on + * Default: `8000` * `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`, `WARN`, `INFO` or `DEBUG`. * Default: `WARN` -* `log_file`: log file to write logs to. - * Default: `vieter.log` (in the current directory) * `pkg_dir`: where Vieter should store the actual package archives. * `data_dir`: where Vieter stores the repositories, log file & database. * `api_key`: the API key to use when authenticating requests. @@ -44,14 +44,27 @@ configuration variable required for each command. * Packages with architecture `any` are always added to this architecture. This prevents the server from being confused when an `any` package is published as the very first package for a repository. - * Git repositories added without an `arch` value use this value instead. -* `port`: HTTP port to run on - * Default: `8000` + * Targets added without an `arch` value use this value instead. +* `global_schedule`: build schedule for any target that does not have a + schedule defined. For information about this syntax, see + [here](/usage/builds/schedule). + * Default: `0 3` (3AM every night) +* `base_image`: Docker image to use when building a package. Any Pacman-based + distro image should work, as long as `/etc/pacman.conf` is used & + `base-devel` exists in the repositories. Make sure that the image supports + the architecture of your cron daemon. + * Default: `archlinux:base-devel` (only works on `x86_64`). If you require + `aarch64` support, consider using + [`menci/archlinuxarm:base-devel`](https://hub.docker.com/r/menci/archlinuxarm) + ([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the + image used for the Vieter CI builds. * `max_log_age`: maximum age of logs (in days). Logs older than this will get - cleaned by the log removal daemon every 24 hours. If set to a negative value, - no logs are ever removed. The age of logs is determined by the time the build - was started. + cleaned by the log removal daemon . If set to a negative value, no logs are + ever removed. The age of logs is determined by the time the build was + started. * Default: `-1` +* `log_removal_schedule`: cron schedule defining when to clean old logs. + * Default: `0 0` (every day at midnight) ### `vieter cron` diff --git a/docs/content/usage/builds/cleanup.md b/docs/content/usage/builds/cleanup.md new file mode 100644 index 0000000..ddeeb85 --- /dev/null +++ b/docs/content/usage/builds/cleanup.md @@ -0,0 +1,24 @@ +--- +weight: 20 +--- + +# Cleanup + +Vieter stores the logs of every single package build. While this is great for +debugging why builds fails, it also causes an active or long-running Vieter +instance to accumulate thousands of logs. + +To combat this, a log removal daemon can be enabled that periodically removes +old build logs. By starting your server with the `max_log_age` variable (see +[Configuration](/configuration#vieter-server) for more info), a daemon will +get enabled that periodically removes logs older than this setting. By default, +this will happen every day at midnight, but this behavior can be changed using +the `log_removal_schedule` variable. + +{{< hint info >}} +**Note** +The daemon will always run a removal of logs on startup. Therefore, it's +possible the daemon will be *very* active when first enabling this setting. +After the initial surge of logs to remove, it'll calm down again. +{{< /hint >}} + diff --git a/docs/content/usage/builds/schedule.md b/docs/content/usage/builds/schedule.md index de59e25..d3802fd 100644 --- a/docs/content/usage/builds/schedule.md +++ b/docs/content/usage/builds/schedule.md @@ -1,3 +1,7 @@ +--- +weight: 10 +--- + # Cron schedule syntax The Vieter cron daemon uses a subset of the cron expression syntax to schedule diff --git a/src/server/cli.v b/src/server/cli.v index 52bce1e..795f764 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -5,15 +5,16 @@ import conf as vconf struct Config { pub: - log_level string = 'WARN' - pkg_dir string - data_dir string - api_key string - default_arch string - global_schedule string = '0 3' - port int = 8000 - base_image string = 'archlinux:base-devel' - max_log_age int = -1 + port int = 8000 + log_level string = 'WARN' + pkg_dir string + data_dir string + api_key string + default_arch string + global_schedule string = '0 3' + base_image string = 'archlinux:base-devel' + max_log_age int = -1 + log_removal_schedule string = '0 0' } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/log_removal.v b/src/server/log_removal.v index f68c575..a901fea 100644 --- a/src/server/log_removal.v +++ b/src/server/log_removal.v @@ -3,11 +3,12 @@ module server import time import models { BuildLog } import os +import cron.expression { CronExpression } -const log_removal_frequency = 24 * time.hour +const fallback_log_removal_frequency = 24 * time.hour // log_removal_daemon removes old build logs every `log_removal_frequency`. -fn (mut app App) log_removal_daemon() { +fn (mut app App) log_removal_daemon(schedule CronExpression) { mut start_time := time.Time{} for { @@ -51,6 +52,12 @@ fn (mut app App) log_removal_daemon() { app.linfo('Cleaned $counter logs ($failed failed)') // Sleep until the next cycle - time.sleep(start_time.add_days(1) - time.now()) + next_time := schedule.next_from_now() or { + app.lerror("Log removal daemon couldn't calculate next time: $err.msg(); fallback to $server.fallback_log_removal_frequency") + + start_time.add(server.fallback_log_removal_frequency) + } + + time.sleep(next_time - time.now()) } } diff --git a/src/server/server.v b/src/server/server.v index bb59b84..178f657 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -55,6 +55,10 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Invalid global cron expression: $err.msg()') } + log_removal_ce := expression.parse_expression(conf.log_removal_schedule) or { + util.exit_with_message(1, 'Invalid log removal cron expression: $err.msg()') + } + // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -109,7 +113,7 @@ pub fn server(conf Config) ! { } if conf.max_log_age > 0 { - go app.log_removal_daemon() + go app.log_removal_daemon(log_removal_ce) } web.run(app, conf.port) diff --git a/vieter.toml b/vieter.toml index 1f839f0..3f63d47 100644 --- a/vieter.toml +++ b/vieter.toml @@ -13,3 +13,4 @@ api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 max_log_age = 64 +log_removal_schedule = '*/2 *'