refactor: some small changes before PR

Jef Roosens 2022-12-19 11:58:35 +01:00
parent b66d1161ed
commit ab81eebd87
Signed by untrusted user: Jef Roosens
GPG Key ID: B75D4F293C7052DB
7 changed files with 31 additions and 34 deletions

View File

@ -40,8 +40,8 @@ meaning manual requests can cause builds to be skipped.
```shell ```shell
curl \ curl \
-H 'x-api-key: secret' \ -H 'X-Api-Key: secret' \
'https://example.com/api/v1/jobs/poll?arch=x86_64&max=2' https://example.com/api/v1/jobs/poll?arch=x86_64&max=2
``` ```
> JSON output format > JSON output format

View File

@ -59,7 +59,7 @@ configuration variable required for each command.
([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the ([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the
image used for the Vieter CI builds. image used for the Vieter CI builds.
* `max_log_age`: maximum age of logs (in days). Logs older than this will get * `max_log_age`: maximum age of logs (in days). Logs older than this will get
cleaned by the log removal daemon . If set to a negative value, no logs are cleaned by the log removal daemon. If set to a negative value, no logs are
ever removed. The age of logs is determined by the time the build was ever removed. The age of logs is determined by the time the build was
started. started.
* Default: `-1` * Default: `-1`

View File

@ -5,15 +5,15 @@ weight: 20
# Cleanup # Cleanup
Vieter stores the logs of every single package build. While this is great for Vieter stores the logs of every single package build. While this is great for
debugging why builds fails, it also causes an active or long-running Vieter debugging why builds fail, it also causes an active or long-running Vieter
instance to accumulate thousands of logs. instance to accumulate thousands of logs.
To combat this, a log removal daemon can be enabled that periodically removes To combat this, a log removal daemon can be enabled that periodically removes
old build logs. By starting your server with the `max_log_age` variable (see old build logs. By starting your server with the `max_log_age` variable (see
[Configuration](/configuration#vieter-server) for more info), a daemon will [Configuration](/configuration#vieter-server)), a daemon will get enabled that
get enabled that periodically removes logs older than this setting. By default, periodically removes logs older than this setting. By default, this will happen
this will happen every day at midnight, but this behavior can be changed using every day at midnight, but this behavior can be changed using the
the `log_removal_schedule` variable. `log_removal_schedule` variable.
{{< hint info >}} {{< hint info >}}
**Note** **Note**
@ -21,4 +21,3 @@ The daemon will always run a removal of logs on startup. Therefore, it's
possible the daemon will be *very* active when first enabling this setting. possible the daemon will be *very* active when first enabling this setting.
After the initial surge of logs to remove, it'll calm down again. After the initial surge of logs to remove, it'll calm down again.
{{< /hint >}} {{< /hint >}}

View File

@ -43,8 +43,6 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t
} }
// remove_build_log removes the build log with the given id from the server. // remove_build_log removes the build log with the given id from the server.
pub fn (c &Client) remove_build_log(id int) !string { pub fn (c &Client) remove_build_log(id int) ! {
data := c.send_request<string>(.delete, '/api/v1/logs/$id', {})! c.send_request<string>(.delete, '/api/v1/logs/$id', {})!
return data.data
} }

View File

@ -1,6 +1,7 @@
module models module models
import time import time
import os
pub struct BuildLog { pub struct BuildLog {
pub mut: pub mut:
@ -28,6 +29,13 @@ pub fn (bl &BuildLog) str() string {
return str return str
} }
// path returns the path to the log file, relative to the logs directory
pub fn (bl &BuildLog) path() string {
filename := bl.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
return os.join_path(bl.target_id.str(), bl.arch, filename)
}
[params] [params]
pub struct BuildLogFilter { pub struct BuildLogFilter {
pub mut: pub mut:

View File

@ -86,7 +86,7 @@ fn (mut app App) v1_post_log() web.Result {
} }
// Store log in db // Store log in db
log := BuildLog{ mut log := BuildLog{
target_id: target_id target_id: target_id
start_time: start_time start_time: start_time
end_time: end_time end_time: end_time
@ -95,25 +95,20 @@ fn (mut app App) v1_post_log() web.Result {
} }
// id of newly created log // id of newly created log
log_id := app.db.add_build_log(log) log.id = app.db.add_build_log(log)
log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, target_id.str(), arch)
// Create the logs directory of it doesn't exist // Create the logs directory of it doesn't exist
if !os.exists(repo_logs_dir) { if !os.exists(os.dir(log_file_path)) {
os.mkdir_all(repo_logs_dir) or { os.mkdir_all(os.dir(log_file_path)) or {
app.lerror("Couldn't create dir '$repo_logs_dir'.") app.lerror('Error while creating log file: $err.msg()')
return app.status(.internal_server_error) return app.status(.internal_server_error)
} }
} }
// Stream log contents to correct file
file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
full_path := os.join_path_single(repo_logs_dir, file_name)
if length := app.req.header.get(.content_length) { if length := app.req.header.get(.content_length) {
util.reader_to_file(mut app.reader, length.int(), full_path) or { util.reader_to_file(mut app.reader, length.int(), log_file_path) or {
app.lerror('An error occured while receiving logs: $err.msg()') app.lerror('An error occured while receiving logs: $err.msg()')
return app.status(.internal_server_error) return app.status(.internal_server_error)
@ -122,16 +117,14 @@ fn (mut app App) v1_post_log() web.Result {
return app.status(.length_required) return app.status(.length_required)
} }
return app.json(.ok, new_data_response(log_id)) return app.json(.ok, new_data_response(log.id))
} }
// v1_delete_log allows removing a build log from the system. // v1_delete_log allows removing a build log from the system.
['/api/v1/logs/:id'; auth; delete] ['/api/v1/logs/:id'; auth; delete]
fn (mut app App) v1_delete_log(id int) web.Result { fn (mut app App) v1_delete_log(id int) web.Result {
log := app.db.get_build_log(id) or { return app.status(.not_found) } log := app.db.get_build_log(id) or { return app.status(.not_found) }
file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), log.arch,
file_name)
os.rm(full_path) or { os.rm(full_path) or {
app.lerror('Failed to remove log file $full_path: $err.msg()') app.lerror('Failed to remove log file $full_path: $err.msg()')

View File

@ -28,11 +28,10 @@ fn (mut app App) log_removal_daemon(schedule CronExpression) {
logs = app.db.get_build_logs(before: too_old_timestamp, offset: offset, limit: 50) logs = app.db.get_build_logs(before: too_old_timestamp, offset: offset, limit: 50)
for log in logs { for log in logs {
file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(),
log.arch, file_name) os.rm(log_file_path) or {
os.rm(full_path) or { app.lerror('Failed to remove log file $log_file_path: $err.msg()')
app.lerror('Failed to remove log file $full_path: $err.msg()')
failed += 1 failed += 1
continue continue