feat(server): add log removal daemon

main
Jef Roosens 2022-12-17 17:11:19 +01:00
parent af409011e6
commit a9ad3088bb
Signed by untrusted user: Jef Roosens
GPG Key ID: B75D4F293C7052DB
6 changed files with 81 additions and 0 deletions

View File

@ -149,3 +149,17 @@ target | id of target this build is for
### Request body
Plaintext contents of the build log.
## Remove a build log
Remove a build log from the server.
### HTTP Request
`DELETE /api/v1/logs/:id`
### URL Parameters
Parameter | Description
--------- | -----------
id | id of log to remove

View File

@ -47,6 +47,11 @@ configuration variable required for each command.
* Git repositories added without an `arch` value use this value instead.
* `port`: HTTP port to run on
* Default: `8000`
* `max_log_age`: maximum age of logs (in days). Logs older than this will get
cleaned by the log removal daemon every 24 hours. If set to a negative value,
no logs are ever removed. The age of logs is determined by the time the build
was started.
* Default: `-1`
### `vieter cron`

View File

@ -13,6 +13,7 @@ pub:
global_schedule string = '0 3'
port int = 8000
base_image string = 'archlinux:base-devel'
max_log_age int = -1
}
// cmd returns the cli submodule that handles starting the server

View File

@ -0,0 +1,56 @@
module server
import time
import models { BuildLog }
import os
const log_removal_frequency = 24 * time.hour
// log_removal_daemon removes old build logs every `log_removal_frequency`.
fn (mut app App) log_removal_daemon() {
mut start_time := time.Time{}
for {
start_time = time.now()
mut too_old_timestamp := time.now().add_days(-app.conf.max_log_age)
app.linfo('Cleaning logs before $too_old_timestamp')
mut offset := u64(0)
mut logs := []BuildLog{}
mut counter := 0
mut failed := 0
// Remove old logs
for {
logs = app.db.get_build_logs(before: too_old_timestamp, offset: offset, limit: 50)
for log in logs {
file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(),
log.arch, file_name)
os.rm(full_path) or {
app.lerror('Failed to remove log file $full_path: $err.msg()')
failed += 1
continue
}
app.db.delete_build_log(log.id)
counter += 1
}
if logs.len < 50 {
break
}
offset += 50
}
app.linfo('Cleaned $counter logs ($failed failed)')
// Sleep until the next cycle
time.sleep(start_time.add_days(1) - time.now())
}
}

View File

@ -108,5 +108,9 @@ pub fn server(conf Config) ! {
util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()')
}
if conf.max_log_age > 0 {
go app.log_removal_daemon()
}
web.run(app, conf.port)
}

View File

@ -12,3 +12,4 @@ address = "http://localhost:8000"
api_update_frequency = 2
image_rebuild_frequency = 1
max_concurrent_builds = 3
max_log_age = 64