From af409011e697d4c6b8ebf7e7ca262e184eb88041 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 17 Dec 2022 16:24:01 +0100 Subject: [PATCH 1/9] feat: add api & cli command to remove log --- src/client/logs.v | 7 +++++++ src/console/logs/logs.v | 18 ++++++++++++++++++ src/server/api_logs.v | 19 +++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/client/logs.v b/src/client/logs.v index 85063bc..e5969dd 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -41,3 +41,10 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t return data } + +// remove_build_log removes the build log with the given id from the server. +pub fn (c &Client) remove_build_log(id int) !string { + data := c.send_request(.delete, '/api/v1/logs/$id', {})! + + return data.data +} diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 3064a58..19c46f6 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -138,6 +138,18 @@ pub fn cmd() cli.Command { list(conf, filter, raw)! } }, + cli.Command{ + name: 'remove' + required_args: 1 + usage: 'id' + description: 'Remove a build log that matches the given id.' + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + + remove(conf, cmd.args[0])! + } + }, cli.Command{ name: 'info' required_args: 1 @@ -204,3 +216,9 @@ fn content(conf Config, id int) ! { println(content) } + +// remove removes a build log from the server's list. +fn remove(conf Config, id string) ! { + c := client.new(conf.address, conf.api_key) + c.remove_build_log(id.int())! +} diff --git a/src/server/api_logs.v b/src/server/api_logs.v index c7521dd..352266c 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -124,3 +124,22 @@ fn (mut app App) v1_post_log() web.Result { return app.json(.ok, new_data_response(log_id)) } + +// v1_delete_log allows removing a build log from the system. +['/api/v1/logs/:id'; auth; delete] +fn (mut app App) v1_delete_log(id int) web.Result { + log := app.db.get_build_log(id) or { return app.status(.not_found) } + file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), log.arch, + file_name) + + os.rm(full_path) or { + app.lerror('Failed to remove log file $full_path: $err.msg()') + + return app.status(.internal_server_error) + } + + app.db.delete_build_log(id) + + return app.status(.ok) +} -- 2.40.1 From a9ad3088bbfd0366ffae6e8174c23c0808a0a385 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 17 Dec 2022 17:11:19 +0100 Subject: [PATCH 2/9] feat(server): add log removal daemon --- docs/api/source/includes/_logs.md | 14 ++++++++ docs/content/configuration.md | 5 +++ src/server/cli.v | 1 + src/server/log_removal.v | 56 +++++++++++++++++++++++++++++++ src/server/server.v | 4 +++ vieter.toml | 1 + 6 files changed, 81 insertions(+) create mode 100644 src/server/log_removal.v diff --git a/docs/api/source/includes/_logs.md b/docs/api/source/includes/_logs.md index 1c14e71..ba6dada 100644 --- a/docs/api/source/includes/_logs.md +++ b/docs/api/source/includes/_logs.md @@ -149,3 +149,17 @@ target | id of target this build is for ### Request body Plaintext contents of the build log. + +## Remove a build log + +Remove a build log from the server. + +### HTTP Request + +`DELETE /api/v1/logs/:id` + +### URL Parameters + +Parameter | Description +--------- | ----------- +id | id of log to remove diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 95bf713..e59fe06 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -47,6 +47,11 @@ configuration variable required for each command. * Git repositories added without an `arch` value use this value instead. * `port`: HTTP port to run on * Default: `8000` +* `max_log_age`: maximum age of logs (in days). Logs older than this will get + cleaned by the log removal daemon every 24 hours. If set to a negative value, + no logs are ever removed. The age of logs is determined by the time the build + was started. + * Default: `-1` ### `vieter cron` diff --git a/src/server/cli.v b/src/server/cli.v index 2fede6c..52bce1e 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -13,6 +13,7 @@ pub: global_schedule string = '0 3' port int = 8000 base_image string = 'archlinux:base-devel' + max_log_age int = -1 } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/log_removal.v b/src/server/log_removal.v new file mode 100644 index 0000000..f68c575 --- /dev/null +++ b/src/server/log_removal.v @@ -0,0 +1,56 @@ +module server + +import time +import models { BuildLog } +import os + +const log_removal_frequency = 24 * time.hour + +// log_removal_daemon removes old build logs every `log_removal_frequency`. +fn (mut app App) log_removal_daemon() { + mut start_time := time.Time{} + + for { + start_time = time.now() + + mut too_old_timestamp := time.now().add_days(-app.conf.max_log_age) + + app.linfo('Cleaning logs before $too_old_timestamp') + + mut offset := u64(0) + mut logs := []BuildLog{} + mut counter := 0 + mut failed := 0 + + // Remove old logs + for { + logs = app.db.get_build_logs(before: too_old_timestamp, offset: offset, limit: 50) + + for log in logs { + file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), + log.arch, file_name) + os.rm(full_path) or { + app.lerror('Failed to remove log file $full_path: $err.msg()') + failed += 1 + + continue + } + app.db.delete_build_log(log.id) + + counter += 1 + } + + if logs.len < 50 { + break + } + + offset += 50 + } + + app.linfo('Cleaned $counter logs ($failed failed)') + + // Sleep until the next cycle + time.sleep(start_time.add_days(1) - time.now()) + } +} diff --git a/src/server/server.v b/src/server/server.v index 74b1f37..bb59b84 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -108,5 +108,9 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()') } + if conf.max_log_age > 0 { + go app.log_removal_daemon() + } + web.run(app, conf.port) } diff --git a/vieter.toml b/vieter.toml index 74a7397..1f839f0 100644 --- a/vieter.toml +++ b/vieter.toml @@ -12,3 +12,4 @@ address = "http://localhost:8000" api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 +max_log_age = 64 -- 2.40.1 From 09c61143b0ea3f0981f50ff1ebabf450f497cefb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Dec 2022 20:23:25 +0100 Subject: [PATCH 3/9] docs: updated the docs --- docs/api/source/includes/_jobs.md | 51 +++++++++++++++++++++++++++++++ docs/api/source/index.html.md | 1 + docs/content/installation.md | 36 ++++++++++++---------- 3 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 docs/api/source/includes/_jobs.md diff --git a/docs/api/source/includes/_jobs.md b/docs/api/source/includes/_jobs.md new file mode 100644 index 0000000..c124781 --- /dev/null +++ b/docs/api/source/includes/_jobs.md @@ -0,0 +1,51 @@ +# Jobs + + + +## Manually schedule a job + +```shell +curl \ + -H 'X-Api-Key: secret' \ + https://example.com/api/v1/jobs/queue?target=10&force&arch=x86_64 +``` + +Manually schedule a job on the server. + +### HTTP Request + +`POST /api/v1/jobs/queue` + +### Query Parameters + +Parameter | Description +--------- | ----------- +target | Id of target to schedule build for +arch | Architecture to build on +force | Whether it's a forced build (true if present) + +## Poll for new jobs + + + +Poll the server for new builds. + +### HTTP Request + +`GET /api/v1/jobs/poll` + +### Query Parameters + +Parameter | Description +--------- | ----------- +arch | For which architecture to receive jobs +max | How many jobs to receive at most diff --git a/docs/api/source/index.html.md b/docs/api/source/index.html.md index 4bfddb8..f61e44a 100644 --- a/docs/api/source/index.html.md +++ b/docs/api/source/index.html.md @@ -11,6 +11,7 @@ includes: - repository - targets - logs + - jobs search: true diff --git a/docs/content/installation.md b/docs/content/installation.md index 21eda64..5b8e2d8 100644 --- a/docs/content/installation.md +++ b/docs/content/installation.md @@ -23,15 +23,15 @@ guarantees about stability, so beware! Thanks to the single-binary design of Vieter, this image can be used both for the repository server, the cron daemon and the agent. -Below is an example compose file to set up both the repository server & the -cron daemon: +Below is a minimal compose file to set up both the repository server & a build +agent: ```yaml version: '3' services: server: - image: 'chewingbever/vieter:dev' + image: 'chewingbever/vieter:0.5.0-rc.1' restart: 'always' environment: @@ -41,18 +41,19 @@ services: - 'data:/data' cron: - image: 'chewingbever/vieter:dev' + image: 'chewingbever/vieter:0.5.0-rc.1' restart: 'always' + # Required to connect to the Docker daemon user: root - command: 'vieter cron' + command: 'vieter agent' environment: - 'VIETER_API_KEY=secret' # MUST be public URL of Vieter repository - 'VIETER_ADDRESS=https://example.com' - - 'VIETER_DEFAULT_ARCH=x86_64' + # Architecture for which the agent builds + - 'VIETER_ARCH=x86_64' - 'VIETER_MAX_CONCURRENT_BUILDS=2' - - 'VIETER_GLOBAL_SCHEDULE=0 3' volumes: - '/var/run/docker.sock:/var/run/docker.sock' @@ -63,14 +64,17 @@ volumes: If you do not require the build system, the repository server can be used independently as well. +Of course, Vieter allows a lot more configuration than this. This compose file +is meant as a starting point for setting up your installation. + {{< hint info >}} **Note** -Builds are executed on the cron daemon's system using the host's Docker daemon. -A cron daemon on a specific architecture will only build packages for that -specific architecture. Therefore, if you wish to build packages for both -`x86_64` & `aarch64`, you'll have to deploy two cron daemons, one on each -architecture. Afterwards, any Git repositories enabled for those two -architectures will build on both. +Builds are executed on the agent's system using the host's Docker daemon. An +agent for a specific `arch` will only build packages for that specific +architecture. Therefore, if you wish to build packages for both `x86_64` & +`aarch64`, you'll have to deploy two agents, one on each architecture. +Afterwards, any Git repositories enabled for those two architectures will build +on both. {{< /hint >}} ## Binary @@ -99,9 +103,9 @@ latest official release or `vieter-git` for the latest development release. ### AUR If you prefer building the packages locally (or on your own Vieter instance), -there's the `[vieter](https://aur.archlinux.org/packages/vieter)` & -`[vieter-git](https://aur.archlinux.org/packages/vieter-git)` packages on the -AUR. These packages build using the `vlang-git` compiler package, so I can't +there's the [`vieter`](https://aur.archlinux.org/packages/vieter) & +[`vieter-git`](https://aur.archlinux.org/packages/vieter-git) packages on the +AUR. These packages build using the `vlang` compiler package, so I can't guarantee that a compiler update won't temporarily break them. ## Building from source -- 2.40.1 From 26796f2228fe92fb32330ea8f8d0095532ec40b7 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 19 Dec 2022 09:47:53 +0100 Subject: [PATCH 4/9] feat(server): use cron schedule for log removal instead --- docs/content/configuration.md | 29 +++++++++++++++++++-------- docs/content/usage/builds/cleanup.md | 24 ++++++++++++++++++++++ docs/content/usage/builds/schedule.md | 4 ++++ src/server/cli.v | 19 +++++++++--------- src/server/log_removal.v | 13 +++++++++--- src/server/server.v | 6 +++++- vieter.toml | 1 + 7 files changed, 75 insertions(+), 21 deletions(-) create mode 100644 docs/content/usage/builds/cleanup.md diff --git a/docs/content/configuration.md b/docs/content/configuration.md index e59fe06..e974a58 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -32,11 +32,11 @@ configuration variable required for each command. ### `vieter server` +* `port`: HTTP port to run on + * Default: `8000` * `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`, `WARN`, `INFO` or `DEBUG`. * Default: `WARN` -* `log_file`: log file to write logs to. - * Default: `vieter.log` (in the current directory) * `pkg_dir`: where Vieter should store the actual package archives. * `data_dir`: where Vieter stores the repositories, log file & database. * `api_key`: the API key to use when authenticating requests. @@ -44,14 +44,27 @@ configuration variable required for each command. * Packages with architecture `any` are always added to this architecture. This prevents the server from being confused when an `any` package is published as the very first package for a repository. - * Git repositories added without an `arch` value use this value instead. -* `port`: HTTP port to run on - * Default: `8000` + * Targets added without an `arch` value use this value instead. +* `global_schedule`: build schedule for any target that does not have a + schedule defined. For information about this syntax, see + [here](/usage/builds/schedule). + * Default: `0 3` (3AM every night) +* `base_image`: Docker image to use when building a package. Any Pacman-based + distro image should work, as long as `/etc/pacman.conf` is used & + `base-devel` exists in the repositories. Make sure that the image supports + the architecture of your cron daemon. + * Default: `archlinux:base-devel` (only works on `x86_64`). If you require + `aarch64` support, consider using + [`menci/archlinuxarm:base-devel`](https://hub.docker.com/r/menci/archlinuxarm) + ([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the + image used for the Vieter CI builds. * `max_log_age`: maximum age of logs (in days). Logs older than this will get - cleaned by the log removal daemon every 24 hours. If set to a negative value, - no logs are ever removed. The age of logs is determined by the time the build - was started. + cleaned by the log removal daemon . If set to a negative value, no logs are + ever removed. The age of logs is determined by the time the build was + started. * Default: `-1` +* `log_removal_schedule`: cron schedule defining when to clean old logs. + * Default: `0 0` (every day at midnight) ### `vieter cron` diff --git a/docs/content/usage/builds/cleanup.md b/docs/content/usage/builds/cleanup.md new file mode 100644 index 0000000..ddeeb85 --- /dev/null +++ b/docs/content/usage/builds/cleanup.md @@ -0,0 +1,24 @@ +--- +weight: 20 +--- + +# Cleanup + +Vieter stores the logs of every single package build. While this is great for +debugging why builds fails, it also causes an active or long-running Vieter +instance to accumulate thousands of logs. + +To combat this, a log removal daemon can be enabled that periodically removes +old build logs. By starting your server with the `max_log_age` variable (see +[Configuration](/configuration#vieter-server) for more info), a daemon will +get enabled that periodically removes logs older than this setting. By default, +this will happen every day at midnight, but this behavior can be changed using +the `log_removal_schedule` variable. + +{{< hint info >}} +**Note** +The daemon will always run a removal of logs on startup. Therefore, it's +possible the daemon will be *very* active when first enabling this setting. +After the initial surge of logs to remove, it'll calm down again. +{{< /hint >}} + diff --git a/docs/content/usage/builds/schedule.md b/docs/content/usage/builds/schedule.md index de59e25..d3802fd 100644 --- a/docs/content/usage/builds/schedule.md +++ b/docs/content/usage/builds/schedule.md @@ -1,3 +1,7 @@ +--- +weight: 10 +--- + # Cron schedule syntax The Vieter cron daemon uses a subset of the cron expression syntax to schedule diff --git a/src/server/cli.v b/src/server/cli.v index 52bce1e..795f764 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -5,15 +5,16 @@ import conf as vconf struct Config { pub: - log_level string = 'WARN' - pkg_dir string - data_dir string - api_key string - default_arch string - global_schedule string = '0 3' - port int = 8000 - base_image string = 'archlinux:base-devel' - max_log_age int = -1 + port int = 8000 + log_level string = 'WARN' + pkg_dir string + data_dir string + api_key string + default_arch string + global_schedule string = '0 3' + base_image string = 'archlinux:base-devel' + max_log_age int = -1 + log_removal_schedule string = '0 0' } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/log_removal.v b/src/server/log_removal.v index f68c575..a901fea 100644 --- a/src/server/log_removal.v +++ b/src/server/log_removal.v @@ -3,11 +3,12 @@ module server import time import models { BuildLog } import os +import cron.expression { CronExpression } -const log_removal_frequency = 24 * time.hour +const fallback_log_removal_frequency = 24 * time.hour // log_removal_daemon removes old build logs every `log_removal_frequency`. -fn (mut app App) log_removal_daemon() { +fn (mut app App) log_removal_daemon(schedule CronExpression) { mut start_time := time.Time{} for { @@ -51,6 +52,12 @@ fn (mut app App) log_removal_daemon() { app.linfo('Cleaned $counter logs ($failed failed)') // Sleep until the next cycle - time.sleep(start_time.add_days(1) - time.now()) + next_time := schedule.next_from_now() or { + app.lerror("Log removal daemon couldn't calculate next time: $err.msg(); fallback to $server.fallback_log_removal_frequency") + + start_time.add(server.fallback_log_removal_frequency) + } + + time.sleep(next_time - time.now()) } } diff --git a/src/server/server.v b/src/server/server.v index bb59b84..178f657 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -55,6 +55,10 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Invalid global cron expression: $err.msg()') } + log_removal_ce := expression.parse_expression(conf.log_removal_schedule) or { + util.exit_with_message(1, 'Invalid log removal cron expression: $err.msg()') + } + // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -109,7 +113,7 @@ pub fn server(conf Config) ! { } if conf.max_log_age > 0 { - go app.log_removal_daemon() + go app.log_removal_daemon(log_removal_ce) } web.run(app, conf.port) diff --git a/vieter.toml b/vieter.toml index 1f839f0..3f63d47 100644 --- a/vieter.toml +++ b/vieter.toml @@ -13,3 +13,4 @@ api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 max_log_age = 64 +log_removal_schedule = '*/2 *' -- 2.40.1 From b66d1161edaf9291e973de5666fec15bf0a1b806 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 19 Dec 2022 11:24:22 +0100 Subject: [PATCH 5/9] docs: update docs some more --- docs/api/source/includes/_jobs.md | 29 +++++++++++++++++++++++++++- docs/api/source/includes/_logs.md | 11 +++++++++-- docs/api/source/includes/_targets.md | 13 ++++++++++++- vieter.toml | 1 - 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/docs/api/source/includes/_jobs.md b/docs/api/source/includes/_jobs.md index c124781..6c08d49 100644 --- a/docs/api/source/includes/_jobs.md +++ b/docs/api/source/includes/_jobs.md @@ -33,10 +33,37 @@ force | Whether it's a forced build (true if present) +```shell +curl \ + -H 'x-api-key: secret' \ + 'https://example.com/api/v1/jobs/poll?arch=x86_64&max=2' +``` + +> JSON output format + +```json +{ + "message": "", + "data": [ + { + "target_id": 1, + "kind": "git", + "url": "https://aur.archlinux.org/discord-ptb.git", + "branch": "master", + "path": "", + "repo": "bur", + "base_image": "archlinux:base-devel", + "force": true + } + ] +} +``` + Poll the server for new builds. ### HTTP Request diff --git a/docs/api/source/includes/_logs.md b/docs/api/source/includes/_logs.md index ba6dada..d6134b7 100644 --- a/docs/api/source/includes/_logs.md +++ b/docs/api/source/includes/_logs.md @@ -125,8 +125,8 @@ id | ID of requested log @@ -152,6 +152,13 @@ Plaintext contents of the build log. ## Remove a build log +```shell +curl \ + -XDELETE \ + -H 'X-Api-Key: secret' \ + https://example.com/api/v1/logs/1 +``` + Remove a build log from the server. ### HTTP Request diff --git a/docs/api/source/includes/_targets.md b/docs/api/source/includes/_targets.md index 93a4e86..b71da84 100644 --- a/docs/api/source/includes/_targets.md +++ b/docs/api/source/includes/_targets.md @@ -27,6 +27,7 @@ curl \ "kind": "git", "url": "https://aur.archlinux.org/discord-ptb.git", "branch": "master", + "path" : "", "repo": "bur", "schedule": "", "arch": [ @@ -73,8 +74,9 @@ curl \ "kind": "git", "url": "https://aur.archlinux.org/discord-ptb.git", "branch": "master", + "path": "", "repo": "bur", - "schedule": "0 3", + "schedule": "0 2", "arch": [ { "id": 1, @@ -124,6 +126,7 @@ Parameter | Description kind | Kind of target to add; one of 'git', 'url'. url | URL of the Git repository. branch | Branch of the Git repository. +path | Subdirectory inside Git repository to use. repo | Vieter repository to publish built packages to. schedule | Cron build schedule (syntax explained [here](https://rustybever.be/docs/vieter/usage/builds/schedule/)) arch | Comma-separated list of architectures to build package on. @@ -149,12 +152,20 @@ Parameter | Description kind | Kind of target; one of 'git', 'url'. url | URL of the Git repository. branch | Branch of the Git repository. +path | Subdirectory inside Git repository to use. repo | Vieter repository to publish built packages to. schedule | Cron build schedule arch | Comma-separated list of architectures to build package on. ## Remove a target +```shell +curl \ + -XDELETE \ + -H 'X-Api-Key: secret' \ + https://example.com/api/v1/targets/1 +``` + Remove a target from the server. ### HTTP Request diff --git a/vieter.toml b/vieter.toml index 3f63d47..1f839f0 100644 --- a/vieter.toml +++ b/vieter.toml @@ -13,4 +13,3 @@ api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 max_log_age = 64 -log_removal_schedule = '*/2 *' -- 2.40.1 From ab81eebd8791248cdd279b4588465b506e6f79c8 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 19 Dec 2022 11:58:35 +0100 Subject: [PATCH 6/9] refactor: some small changes before PR --- docs/api/source/includes/_jobs.md | 4 ++-- docs/content/configuration.md | 2 +- docs/content/usage/builds/cleanup.md | 11 +++++------ src/client/logs.v | 6 ++---- src/models/logs.v | 8 ++++++++ src/server/api_logs.v | 25 +++++++++---------------- src/server/log_removal.v | 9 ++++----- 7 files changed, 31 insertions(+), 34 deletions(-) diff --git a/docs/api/source/includes/_jobs.md b/docs/api/source/includes/_jobs.md index 6c08d49..a25309d 100644 --- a/docs/api/source/includes/_jobs.md +++ b/docs/api/source/includes/_jobs.md @@ -40,8 +40,8 @@ meaning manual requests can cause builds to be skipped. ```shell curl \ - -H 'x-api-key: secret' \ - 'https://example.com/api/v1/jobs/poll?arch=x86_64&max=2' + -H 'X-Api-Key: secret' \ + https://example.com/api/v1/jobs/poll?arch=x86_64&max=2 ``` > JSON output format diff --git a/docs/content/configuration.md b/docs/content/configuration.md index e974a58..45c5de6 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -59,7 +59,7 @@ configuration variable required for each command. ([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the image used for the Vieter CI builds. * `max_log_age`: maximum age of logs (in days). Logs older than this will get - cleaned by the log removal daemon . If set to a negative value, no logs are + cleaned by the log removal daemon. If set to a negative value, no logs are ever removed. The age of logs is determined by the time the build was started. * Default: `-1` diff --git a/docs/content/usage/builds/cleanup.md b/docs/content/usage/builds/cleanup.md index ddeeb85..724a75f 100644 --- a/docs/content/usage/builds/cleanup.md +++ b/docs/content/usage/builds/cleanup.md @@ -5,15 +5,15 @@ weight: 20 # Cleanup Vieter stores the logs of every single package build. While this is great for -debugging why builds fails, it also causes an active or long-running Vieter +debugging why builds fail, it also causes an active or long-running Vieter instance to accumulate thousands of logs. To combat this, a log removal daemon can be enabled that periodically removes old build logs. By starting your server with the `max_log_age` variable (see -[Configuration](/configuration#vieter-server) for more info), a daemon will -get enabled that periodically removes logs older than this setting. By default, -this will happen every day at midnight, but this behavior can be changed using -the `log_removal_schedule` variable. +[Configuration](/configuration#vieter-server)), a daemon will get enabled that +periodically removes logs older than this setting. By default, this will happen +every day at midnight, but this behavior can be changed using the +`log_removal_schedule` variable. {{< hint info >}} **Note** @@ -21,4 +21,3 @@ The daemon will always run a removal of logs on startup. Therefore, it's possible the daemon will be *very* active when first enabling this setting. After the initial surge of logs to remove, it'll calm down again. {{< /hint >}} - diff --git a/src/client/logs.v b/src/client/logs.v index e5969dd..2ddb2e2 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -43,8 +43,6 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t } // remove_build_log removes the build log with the given id from the server. -pub fn (c &Client) remove_build_log(id int) !string { - data := c.send_request(.delete, '/api/v1/logs/$id', {})! - - return data.data +pub fn (c &Client) remove_build_log(id int) ! { + c.send_request(.delete, '/api/v1/logs/$id', {})! } diff --git a/src/models/logs.v b/src/models/logs.v index 12907d8..66a3a0a 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -1,6 +1,7 @@ module models import time +import os pub struct BuildLog { pub mut: @@ -28,6 +29,13 @@ pub fn (bl &BuildLog) str() string { return str } +// path returns the path to the log file, relative to the logs directory +pub fn (bl &BuildLog) path() string { + filename := bl.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + + return os.join_path(bl.target_id.str(), bl.arch, filename) +} + [params] pub struct BuildLogFilter { pub mut: diff --git a/src/server/api_logs.v b/src/server/api_logs.v index 352266c..13b50b9 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -86,7 +86,7 @@ fn (mut app App) v1_post_log() web.Result { } // Store log in db - log := BuildLog{ + mut log := BuildLog{ target_id: target_id start_time: start_time end_time: end_time @@ -95,25 +95,20 @@ fn (mut app App) v1_post_log() web.Result { } // id of newly created log - log_id := app.db.add_build_log(log) - - repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, target_id.str(), arch) + log.id = app.db.add_build_log(log) + log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) // Create the logs directory of it doesn't exist - if !os.exists(repo_logs_dir) { - os.mkdir_all(repo_logs_dir) or { - app.lerror("Couldn't create dir '$repo_logs_dir'.") + if !os.exists(os.dir(log_file_path)) { + os.mkdir_all(os.dir(log_file_path)) or { + app.lerror('Error while creating log file: $err.msg()') return app.status(.internal_server_error) } } - // Stream log contents to correct file - file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss') - full_path := os.join_path_single(repo_logs_dir, file_name) - if length := app.req.header.get(.content_length) { - util.reader_to_file(mut app.reader, length.int(), full_path) or { + util.reader_to_file(mut app.reader, length.int(), log_file_path) or { app.lerror('An error occured while receiving logs: $err.msg()') return app.status(.internal_server_error) @@ -122,16 +117,14 @@ fn (mut app App) v1_post_log() web.Result { return app.status(.length_required) } - return app.json(.ok, new_data_response(log_id)) + return app.json(.ok, new_data_response(log.id)) } // v1_delete_log allows removing a build log from the system. ['/api/v1/logs/:id'; auth; delete] fn (mut app App) v1_delete_log(id int) web.Result { log := app.db.get_build_log(id) or { return app.status(.not_found) } - file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') - full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), log.arch, - file_name) + full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) os.rm(full_path) or { app.lerror('Failed to remove log file $full_path: $err.msg()') diff --git a/src/server/log_removal.v b/src/server/log_removal.v index a901fea..a0a5f78 100644 --- a/src/server/log_removal.v +++ b/src/server/log_removal.v @@ -28,11 +28,10 @@ fn (mut app App) log_removal_daemon(schedule CronExpression) { logs = app.db.get_build_logs(before: too_old_timestamp, offset: offset, limit: 50) for log in logs { - file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') - full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), - log.arch, file_name) - os.rm(full_path) or { - app.lerror('Failed to remove log file $full_path: $err.msg()') + log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) + + os.rm(log_file_path) or { + app.lerror('Failed to remove log file $log_file_path: $err.msg()') failed += 1 continue -- 2.40.1 From 2c9331668853994d6f1c24a04e50da573c320008 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 19 Dec 2022 12:43:46 +0100 Subject: [PATCH 7/9] fix: log removal daemon now properly cleans all old logs --- src/server/log_removal.v | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server/log_removal.v b/src/server/log_removal.v index a0a5f78..8e1a8c2 100644 --- a/src/server/log_removal.v +++ b/src/server/log_removal.v @@ -18,14 +18,16 @@ fn (mut app App) log_removal_daemon(schedule CronExpression) { app.linfo('Cleaning logs before $too_old_timestamp') - mut offset := u64(0) mut logs := []BuildLog{} mut counter := 0 - mut failed := 0 + mut failed := u64(0) // Remove old logs for { - logs = app.db.get_build_logs(before: too_old_timestamp, offset: offset, limit: 50) + // The offset is used to skip logs that failed to remove. Besides + // this, we don't need to move the offset, because all previously + // oldest logs will have been removed. + logs = app.db.get_build_logs(before: too_old_timestamp, offset: failed, limit: 50) for log in logs { log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) @@ -44,8 +46,6 @@ fn (mut app App) log_removal_daemon(schedule CronExpression) { if logs.len < 50 { break } - - offset += 50 } app.linfo('Cleaned $counter logs ($failed failed)') -- 2.40.1 From ab6da78738637ab2624c1290ee73761412baed56 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 21 Dec 2022 23:42:15 +0100 Subject: [PATCH 8/9] feat(cli): use posx-style long options --- CHANGELOG.md | 9 +++++++++ src/main.v | 1 + 2 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27d9096..97f7021 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev) +### Added + +* API route for removing logs & accompanying CLI command +* Daemon for periodically removing old logs + +### Changed + +* Use `--long-option` instead of `-long-option` for CLI + ## [0.5.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.1) ### Added diff --git a/src/main.v b/src/main.v index 1053c2f..8b5c362 100644 --- a/src/main.v +++ b/src/main.v @@ -21,6 +21,7 @@ fn main() { name: 'vieter' description: 'Vieter is a lightweight implementation of an Arch repository server.' version: '0.5.0-rc.1' + posix_mode: true flags: [ cli.Flag{ flag: cli.FlagType.string -- 2.40.1 From be3762835d29e8c9cf79f1110057eb6ed8436814 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 21 Dec 2022 23:45:42 +0100 Subject: [PATCH 9/9] chore: bump versions to 0.5.0-rc.2 --- CHANGELOG.md | 2 ++ PKGBUILD | 2 +- src/main.v | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97f7021..e615698 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev) +## [0.5.0-rc.2](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.2) + ### Added * API route for removing logs & accompanying CLI command diff --git a/PKGBUILD b/PKGBUILD index 94db654..5e9530a 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -3,7 +3,7 @@ pkgbase='vieter' pkgname='vieter' -pkgver='0.5.0-rc.1' +pkgver='0.5.0_rc.2' pkgrel=1 pkgdesc="Lightweight Arch repository server & package build system" depends=('glibc' 'openssl' 'libarchive' 'sqlite') diff --git a/src/main.v b/src/main.v index 8b5c362..eda38e7 100644 --- a/src/main.v +++ b/src/main.v @@ -20,7 +20,7 @@ fn main() { mut app := cli.Command{ name: 'vieter' description: 'Vieter is a lightweight implementation of an Arch repository server.' - version: '0.5.0-rc.1' + version: '0.5.0-rc.2' posix_mode: true flags: [ cli.Flag{ -- 2.40.1