From a9ad3088bbfd0366ffae6e8174c23c0808a0a385 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 17 Dec 2022 17:11:19 +0100 Subject: [PATCH] feat(server): add log removal daemon --- docs/api/source/includes/_logs.md | 14 ++++++++ docs/content/configuration.md | 5 +++ src/server/cli.v | 1 + src/server/log_removal.v | 56 +++++++++++++++++++++++++++++++ src/server/server.v | 4 +++ vieter.toml | 1 + 6 files changed, 81 insertions(+) create mode 100644 src/server/log_removal.v diff --git a/docs/api/source/includes/_logs.md b/docs/api/source/includes/_logs.md index 1c14e71..ba6dada 100644 --- a/docs/api/source/includes/_logs.md +++ b/docs/api/source/includes/_logs.md @@ -149,3 +149,17 @@ target | id of target this build is for ### Request body Plaintext contents of the build log. + +## Remove a build log + +Remove a build log from the server. + +### HTTP Request + +`DELETE /api/v1/logs/:id` + +### URL Parameters + +Parameter | Description +--------- | ----------- +id | id of log to remove diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 95bf713..e59fe06 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -47,6 +47,11 @@ configuration variable required for each command. * Git repositories added without an `arch` value use this value instead. * `port`: HTTP port to run on * Default: `8000` +* `max_log_age`: maximum age of logs (in days). Logs older than this will get + cleaned by the log removal daemon every 24 hours. If set to a negative value, + no logs are ever removed. The age of logs is determined by the time the build + was started. + * Default: `-1` ### `vieter cron` diff --git a/src/server/cli.v b/src/server/cli.v index 2fede6c..52bce1e 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -13,6 +13,7 @@ pub: global_schedule string = '0 3' port int = 8000 base_image string = 'archlinux:base-devel' + max_log_age int = -1 } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/log_removal.v b/src/server/log_removal.v new file mode 100644 index 0000000..f68c575 --- /dev/null +++ b/src/server/log_removal.v @@ -0,0 +1,56 @@ +module server + +import time +import models { BuildLog } +import os + +const log_removal_frequency = 24 * time.hour + +// log_removal_daemon removes old build logs every `log_removal_frequency`. +fn (mut app App) log_removal_daemon() { + mut start_time := time.Time{} + + for { + start_time = time.now() + + mut too_old_timestamp := time.now().add_days(-app.conf.max_log_age) + + app.linfo('Cleaning logs before $too_old_timestamp') + + mut offset := u64(0) + mut logs := []BuildLog{} + mut counter := 0 + mut failed := 0 + + // Remove old logs + for { + logs = app.db.get_build_logs(before: too_old_timestamp, offset: offset, limit: 50) + + for log in logs { + file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), + log.arch, file_name) + os.rm(full_path) or { + app.lerror('Failed to remove log file $full_path: $err.msg()') + failed += 1 + + continue + } + app.db.delete_build_log(log.id) + + counter += 1 + } + + if logs.len < 50 { + break + } + + offset += 50 + } + + app.linfo('Cleaned $counter logs ($failed failed)') + + // Sleep until the next cycle + time.sleep(start_time.add_days(1) - time.now()) + } +} diff --git a/src/server/server.v b/src/server/server.v index 74b1f37..bb59b84 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -108,5 +108,9 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()') } + if conf.max_log_age > 0 { + go app.log_removal_daemon() + } + web.run(app, conf.port) } diff --git a/vieter.toml b/vieter.toml index 74a7397..1f839f0 100644 --- a/vieter.toml +++ b/vieter.toml @@ -12,3 +12,4 @@ address = "http://localhost:8000" api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 +max_log_age = 64