From ab81eebd8791248cdd279b4588465b506e6f79c8 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 19 Dec 2022 11:58:35 +0100 Subject: [PATCH] refactor: some small changes before PR --- docs/api/source/includes/_jobs.md | 4 ++-- docs/content/configuration.md | 2 +- docs/content/usage/builds/cleanup.md | 11 +++++------ src/client/logs.v | 6 ++---- src/models/logs.v | 8 ++++++++ src/server/api_logs.v | 25 +++++++++---------------- src/server/log_removal.v | 9 ++++----- 7 files changed, 31 insertions(+), 34 deletions(-) diff --git a/docs/api/source/includes/_jobs.md b/docs/api/source/includes/_jobs.md index 6c08d49..a25309d 100644 --- a/docs/api/source/includes/_jobs.md +++ b/docs/api/source/includes/_jobs.md @@ -40,8 +40,8 @@ meaning manual requests can cause builds to be skipped. ```shell curl \ - -H 'x-api-key: secret' \ - 'https://example.com/api/v1/jobs/poll?arch=x86_64&max=2' + -H 'X-Api-Key: secret' \ + https://example.com/api/v1/jobs/poll?arch=x86_64&max=2 ``` > JSON output format diff --git a/docs/content/configuration.md b/docs/content/configuration.md index e974a58..45c5de6 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -59,7 +59,7 @@ configuration variable required for each command. ([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the image used for the Vieter CI builds. * `max_log_age`: maximum age of logs (in days). Logs older than this will get - cleaned by the log removal daemon . If set to a negative value, no logs are + cleaned by the log removal daemon. If set to a negative value, no logs are ever removed. The age of logs is determined by the time the build was started. * Default: `-1` diff --git a/docs/content/usage/builds/cleanup.md b/docs/content/usage/builds/cleanup.md index ddeeb85..724a75f 100644 --- a/docs/content/usage/builds/cleanup.md +++ b/docs/content/usage/builds/cleanup.md @@ -5,15 +5,15 @@ weight: 20 # Cleanup Vieter stores the logs of every single package build. While this is great for -debugging why builds fails, it also causes an active or long-running Vieter +debugging why builds fail, it also causes an active or long-running Vieter instance to accumulate thousands of logs. To combat this, a log removal daemon can be enabled that periodically removes old build logs. By starting your server with the `max_log_age` variable (see -[Configuration](/configuration#vieter-server) for more info), a daemon will -get enabled that periodically removes logs older than this setting. By default, -this will happen every day at midnight, but this behavior can be changed using -the `log_removal_schedule` variable. +[Configuration](/configuration#vieter-server)), a daemon will get enabled that +periodically removes logs older than this setting. By default, this will happen +every day at midnight, but this behavior can be changed using the +`log_removal_schedule` variable. {{< hint info >}} **Note** @@ -21,4 +21,3 @@ The daemon will always run a removal of logs on startup. Therefore, it's possible the daemon will be *very* active when first enabling this setting. After the initial surge of logs to remove, it'll calm down again. {{< /hint >}} - diff --git a/src/client/logs.v b/src/client/logs.v index e5969dd..2ddb2e2 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -43,8 +43,6 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t } // remove_build_log removes the build log with the given id from the server. -pub fn (c &Client) remove_build_log(id int) !string { - data := c.send_request(.delete, '/api/v1/logs/$id', {})! - - return data.data +pub fn (c &Client) remove_build_log(id int) ! { + c.send_request(.delete, '/api/v1/logs/$id', {})! } diff --git a/src/models/logs.v b/src/models/logs.v index 12907d8..66a3a0a 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -1,6 +1,7 @@ module models import time +import os pub struct BuildLog { pub mut: @@ -28,6 +29,13 @@ pub fn (bl &BuildLog) str() string { return str } +// path returns the path to the log file, relative to the logs directory +pub fn (bl &BuildLog) path() string { + filename := bl.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + + return os.join_path(bl.target_id.str(), bl.arch, filename) +} + [params] pub struct BuildLogFilter { pub mut: diff --git a/src/server/api_logs.v b/src/server/api_logs.v index 352266c..13b50b9 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -86,7 +86,7 @@ fn (mut app App) v1_post_log() web.Result { } // Store log in db - log := BuildLog{ + mut log := BuildLog{ target_id: target_id start_time: start_time end_time: end_time @@ -95,25 +95,20 @@ fn (mut app App) v1_post_log() web.Result { } // id of newly created log - log_id := app.db.add_build_log(log) - - repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, target_id.str(), arch) + log.id = app.db.add_build_log(log) + log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) // Create the logs directory of it doesn't exist - if !os.exists(repo_logs_dir) { - os.mkdir_all(repo_logs_dir) or { - app.lerror("Couldn't create dir '$repo_logs_dir'.") + if !os.exists(os.dir(log_file_path)) { + os.mkdir_all(os.dir(log_file_path)) or { + app.lerror('Error while creating log file: $err.msg()') return app.status(.internal_server_error) } } - // Stream log contents to correct file - file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss') - full_path := os.join_path_single(repo_logs_dir, file_name) - if length := app.req.header.get(.content_length) { - util.reader_to_file(mut app.reader, length.int(), full_path) or { + util.reader_to_file(mut app.reader, length.int(), log_file_path) or { app.lerror('An error occured while receiving logs: $err.msg()') return app.status(.internal_server_error) @@ -122,16 +117,14 @@ fn (mut app App) v1_post_log() web.Result { return app.status(.length_required) } - return app.json(.ok, new_data_response(log_id)) + return app.json(.ok, new_data_response(log.id)) } // v1_delete_log allows removing a build log from the system. ['/api/v1/logs/:id'; auth; delete] fn (mut app App) v1_delete_log(id int) web.Result { log := app.db.get_build_log(id) or { return app.status(.not_found) } - file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') - full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), log.arch, - file_name) + full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) os.rm(full_path) or { app.lerror('Failed to remove log file $full_path: $err.msg()') diff --git a/src/server/log_removal.v b/src/server/log_removal.v index a901fea..a0a5f78 100644 --- a/src/server/log_removal.v +++ b/src/server/log_removal.v @@ -28,11 +28,10 @@ fn (mut app App) log_removal_daemon(schedule CronExpression) { logs = app.db.get_build_logs(before: too_old_timestamp, offset: offset, limit: 50) for log in logs { - file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') - full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), - log.arch, file_name) - os.rm(full_path) or { - app.lerror('Failed to remove log file $full_path: $err.msg()') + log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) + + os.rm(log_file_path) or { + app.lerror('Failed to remove log file $log_file_path: $err.msg()') failed += 1 continue