From 26796f2228fe92fb32330ea8f8d0095532ec40b7 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 19 Dec 2022 09:47:53 +0100 Subject: [PATCH] feat(server): use cron schedule for log removal instead --- docs/content/configuration.md | 29 +++++++++++++++++++-------- docs/content/usage/builds/cleanup.md | 24 ++++++++++++++++++++++ docs/content/usage/builds/schedule.md | 4 ++++ src/server/cli.v | 19 +++++++++--------- src/server/log_removal.v | 13 +++++++++--- src/server/server.v | 6 +++++- vieter.toml | 1 + 7 files changed, 75 insertions(+), 21 deletions(-) create mode 100644 docs/content/usage/builds/cleanup.md diff --git a/docs/content/configuration.md b/docs/content/configuration.md index e59fe06..e974a58 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -32,11 +32,11 @@ configuration variable required for each command. ### `vieter server` +* `port`: HTTP port to run on + * Default: `8000` * `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`, `WARN`, `INFO` or `DEBUG`. * Default: `WARN` -* `log_file`: log file to write logs to. - * Default: `vieter.log` (in the current directory) * `pkg_dir`: where Vieter should store the actual package archives. * `data_dir`: where Vieter stores the repositories, log file & database. * `api_key`: the API key to use when authenticating requests. @@ -44,14 +44,27 @@ configuration variable required for each command. * Packages with architecture `any` are always added to this architecture. This prevents the server from being confused when an `any` package is published as the very first package for a repository. - * Git repositories added without an `arch` value use this value instead. -* `port`: HTTP port to run on - * Default: `8000` + * Targets added without an `arch` value use this value instead. +* `global_schedule`: build schedule for any target that does not have a + schedule defined. For information about this syntax, see + [here](/usage/builds/schedule). + * Default: `0 3` (3AM every night) +* `base_image`: Docker image to use when building a package. Any Pacman-based + distro image should work, as long as `/etc/pacman.conf` is used & + `base-devel` exists in the repositories. Make sure that the image supports + the architecture of your cron daemon. + * Default: `archlinux:base-devel` (only works on `x86_64`). If you require + `aarch64` support, consider using + [`menci/archlinuxarm:base-devel`](https://hub.docker.com/r/menci/archlinuxarm) + ([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the + image used for the Vieter CI builds. * `max_log_age`: maximum age of logs (in days). Logs older than this will get - cleaned by the log removal daemon every 24 hours. If set to a negative value, - no logs are ever removed. The age of logs is determined by the time the build - was started. + cleaned by the log removal daemon . If set to a negative value, no logs are + ever removed. The age of logs is determined by the time the build was + started. * Default: `-1` +* `log_removal_schedule`: cron schedule defining when to clean old logs. + * Default: `0 0` (every day at midnight) ### `vieter cron` diff --git a/docs/content/usage/builds/cleanup.md b/docs/content/usage/builds/cleanup.md new file mode 100644 index 0000000..ddeeb85 --- /dev/null +++ b/docs/content/usage/builds/cleanup.md @@ -0,0 +1,24 @@ +--- +weight: 20 +--- + +# Cleanup + +Vieter stores the logs of every single package build. While this is great for +debugging why builds fails, it also causes an active or long-running Vieter +instance to accumulate thousands of logs. + +To combat this, a log removal daemon can be enabled that periodically removes +old build logs. By starting your server with the `max_log_age` variable (see +[Configuration](/configuration#vieter-server) for more info), a daemon will +get enabled that periodically removes logs older than this setting. By default, +this will happen every day at midnight, but this behavior can be changed using +the `log_removal_schedule` variable. + +{{< hint info >}} +**Note** +The daemon will always run a removal of logs on startup. Therefore, it's +possible the daemon will be *very* active when first enabling this setting. +After the initial surge of logs to remove, it'll calm down again. +{{< /hint >}} + diff --git a/docs/content/usage/builds/schedule.md b/docs/content/usage/builds/schedule.md index de59e25..d3802fd 100644 --- a/docs/content/usage/builds/schedule.md +++ b/docs/content/usage/builds/schedule.md @@ -1,3 +1,7 @@ +--- +weight: 10 +--- + # Cron schedule syntax The Vieter cron daemon uses a subset of the cron expression syntax to schedule diff --git a/src/server/cli.v b/src/server/cli.v index 52bce1e..795f764 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -5,15 +5,16 @@ import conf as vconf struct Config { pub: - log_level string = 'WARN' - pkg_dir string - data_dir string - api_key string - default_arch string - global_schedule string = '0 3' - port int = 8000 - base_image string = 'archlinux:base-devel' - max_log_age int = -1 + port int = 8000 + log_level string = 'WARN' + pkg_dir string + data_dir string + api_key string + default_arch string + global_schedule string = '0 3' + base_image string = 'archlinux:base-devel' + max_log_age int = -1 + log_removal_schedule string = '0 0' } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/log_removal.v b/src/server/log_removal.v index f68c575..a901fea 100644 --- a/src/server/log_removal.v +++ b/src/server/log_removal.v @@ -3,11 +3,12 @@ module server import time import models { BuildLog } import os +import cron.expression { CronExpression } -const log_removal_frequency = 24 * time.hour +const fallback_log_removal_frequency = 24 * time.hour // log_removal_daemon removes old build logs every `log_removal_frequency`. -fn (mut app App) log_removal_daemon() { +fn (mut app App) log_removal_daemon(schedule CronExpression) { mut start_time := time.Time{} for { @@ -51,6 +52,12 @@ fn (mut app App) log_removal_daemon() { app.linfo('Cleaned $counter logs ($failed failed)') // Sleep until the next cycle - time.sleep(start_time.add_days(1) - time.now()) + next_time := schedule.next_from_now() or { + app.lerror("Log removal daemon couldn't calculate next time: $err.msg(); fallback to $server.fallback_log_removal_frequency") + + start_time.add(server.fallback_log_removal_frequency) + } + + time.sleep(next_time - time.now()) } } diff --git a/src/server/server.v b/src/server/server.v index bb59b84..178f657 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -55,6 +55,10 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Invalid global cron expression: $err.msg()') } + log_removal_ce := expression.parse_expression(conf.log_removal_schedule) or { + util.exit_with_message(1, 'Invalid log removal cron expression: $err.msg()') + } + // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -109,7 +113,7 @@ pub fn server(conf Config) ! { } if conf.max_log_age > 0 { - go app.log_removal_daemon() + go app.log_removal_daemon(log_removal_ce) } web.run(app, conf.port) diff --git a/vieter.toml b/vieter.toml index 1f839f0..3f63d47 100644 --- a/vieter.toml +++ b/vieter.toml @@ -13,3 +13,4 @@ api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 max_log_age = 64 +log_removal_schedule = '*/2 *'