forked from vieter-v/vieter
refactor: some small changes before PR
parent
b66d1161ed
commit
ab81eebd87
|
@ -40,8 +40,8 @@ meaning manual requests can cause builds to be skipped.
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
curl \
|
curl \
|
||||||
-H 'x-api-key: secret' \
|
-H 'X-Api-Key: secret' \
|
||||||
'https://example.com/api/v1/jobs/poll?arch=x86_64&max=2'
|
https://example.com/api/v1/jobs/poll?arch=x86_64&max=2
|
||||||
```
|
```
|
||||||
|
|
||||||
> JSON output format
|
> JSON output format
|
||||||
|
|
|
@ -59,7 +59,7 @@ configuration variable required for each command.
|
||||||
([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the
|
([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the
|
||||||
image used for the Vieter CI builds.
|
image used for the Vieter CI builds.
|
||||||
* `max_log_age`: maximum age of logs (in days). Logs older than this will get
|
* `max_log_age`: maximum age of logs (in days). Logs older than this will get
|
||||||
cleaned by the log removal daemon . If set to a negative value, no logs are
|
cleaned by the log removal daemon. If set to a negative value, no logs are
|
||||||
ever removed. The age of logs is determined by the time the build was
|
ever removed. The age of logs is determined by the time the build was
|
||||||
started.
|
started.
|
||||||
* Default: `-1`
|
* Default: `-1`
|
||||||
|
|
|
@ -5,15 +5,15 @@ weight: 20
|
||||||
# Cleanup
|
# Cleanup
|
||||||
|
|
||||||
Vieter stores the logs of every single package build. While this is great for
|
Vieter stores the logs of every single package build. While this is great for
|
||||||
debugging why builds fails, it also causes an active or long-running Vieter
|
debugging why builds fail, it also causes an active or long-running Vieter
|
||||||
instance to accumulate thousands of logs.
|
instance to accumulate thousands of logs.
|
||||||
|
|
||||||
To combat this, a log removal daemon can be enabled that periodically removes
|
To combat this, a log removal daemon can be enabled that periodically removes
|
||||||
old build logs. By starting your server with the `max_log_age` variable (see
|
old build logs. By starting your server with the `max_log_age` variable (see
|
||||||
[Configuration](/configuration#vieter-server) for more info), a daemon will
|
[Configuration](/configuration#vieter-server)), a daemon will get enabled that
|
||||||
get enabled that periodically removes logs older than this setting. By default,
|
periodically removes logs older than this setting. By default, this will happen
|
||||||
this will happen every day at midnight, but this behavior can be changed using
|
every day at midnight, but this behavior can be changed using the
|
||||||
the `log_removal_schedule` variable.
|
`log_removal_schedule` variable.
|
||||||
|
|
||||||
{{< hint info >}}
|
{{< hint info >}}
|
||||||
**Note**
|
**Note**
|
||||||
|
@ -21,4 +21,3 @@ The daemon will always run a removal of logs on startup. Therefore, it's
|
||||||
possible the daemon will be *very* active when first enabling this setting.
|
possible the daemon will be *very* active when first enabling this setting.
|
||||||
After the initial surge of logs to remove, it'll calm down again.
|
After the initial surge of logs to remove, it'll calm down again.
|
||||||
{{< /hint >}}
|
{{< /hint >}}
|
||||||
|
|
||||||
|
|
|
@ -43,8 +43,6 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t
|
||||||
}
|
}
|
||||||
|
|
||||||
// remove_build_log removes the build log with the given id from the server.
|
// remove_build_log removes the build log with the given id from the server.
|
||||||
pub fn (c &Client) remove_build_log(id int) !string {
|
pub fn (c &Client) remove_build_log(id int) ! {
|
||||||
data := c.send_request<string>(.delete, '/api/v1/logs/$id', {})!
|
c.send_request<string>(.delete, '/api/v1/logs/$id', {})!
|
||||||
|
|
||||||
return data.data
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
module models
|
module models
|
||||||
|
|
||||||
import time
|
import time
|
||||||
|
import os
|
||||||
|
|
||||||
pub struct BuildLog {
|
pub struct BuildLog {
|
||||||
pub mut:
|
pub mut:
|
||||||
|
@ -28,6 +29,13 @@ pub fn (bl &BuildLog) str() string {
|
||||||
return str
|
return str
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// path returns the path to the log file, relative to the logs directory
|
||||||
|
pub fn (bl &BuildLog) path() string {
|
||||||
|
filename := bl.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
|
||||||
|
|
||||||
|
return os.join_path(bl.target_id.str(), bl.arch, filename)
|
||||||
|
}
|
||||||
|
|
||||||
[params]
|
[params]
|
||||||
pub struct BuildLogFilter {
|
pub struct BuildLogFilter {
|
||||||
pub mut:
|
pub mut:
|
||||||
|
|
|
@ -86,7 +86,7 @@ fn (mut app App) v1_post_log() web.Result {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store log in db
|
// Store log in db
|
||||||
log := BuildLog{
|
mut log := BuildLog{
|
||||||
target_id: target_id
|
target_id: target_id
|
||||||
start_time: start_time
|
start_time: start_time
|
||||||
end_time: end_time
|
end_time: end_time
|
||||||
|
@ -95,25 +95,20 @@ fn (mut app App) v1_post_log() web.Result {
|
||||||
}
|
}
|
||||||
|
|
||||||
// id of newly created log
|
// id of newly created log
|
||||||
log_id := app.db.add_build_log(log)
|
log.id = app.db.add_build_log(log)
|
||||||
|
log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
|
||||||
repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, target_id.str(), arch)
|
|
||||||
|
|
||||||
// Create the logs directory of it doesn't exist
|
// Create the logs directory of it doesn't exist
|
||||||
if !os.exists(repo_logs_dir) {
|
if !os.exists(os.dir(log_file_path)) {
|
||||||
os.mkdir_all(repo_logs_dir) or {
|
os.mkdir_all(os.dir(log_file_path)) or {
|
||||||
app.lerror("Couldn't create dir '$repo_logs_dir'.")
|
app.lerror('Error while creating log file: $err.msg()')
|
||||||
|
|
||||||
return app.status(.internal_server_error)
|
return app.status(.internal_server_error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stream log contents to correct file
|
|
||||||
file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
|
|
||||||
full_path := os.join_path_single(repo_logs_dir, file_name)
|
|
||||||
|
|
||||||
if length := app.req.header.get(.content_length) {
|
if length := app.req.header.get(.content_length) {
|
||||||
util.reader_to_file(mut app.reader, length.int(), full_path) or {
|
util.reader_to_file(mut app.reader, length.int(), log_file_path) or {
|
||||||
app.lerror('An error occured while receiving logs: $err.msg()')
|
app.lerror('An error occured while receiving logs: $err.msg()')
|
||||||
|
|
||||||
return app.status(.internal_server_error)
|
return app.status(.internal_server_error)
|
||||||
|
@ -122,16 +117,14 @@ fn (mut app App) v1_post_log() web.Result {
|
||||||
return app.status(.length_required)
|
return app.status(.length_required)
|
||||||
}
|
}
|
||||||
|
|
||||||
return app.json(.ok, new_data_response(log_id))
|
return app.json(.ok, new_data_response(log.id))
|
||||||
}
|
}
|
||||||
|
|
||||||
// v1_delete_log allows removing a build log from the system.
|
// v1_delete_log allows removing a build log from the system.
|
||||||
['/api/v1/logs/:id'; auth; delete]
|
['/api/v1/logs/:id'; auth; delete]
|
||||||
fn (mut app App) v1_delete_log(id int) web.Result {
|
fn (mut app App) v1_delete_log(id int) web.Result {
|
||||||
log := app.db.get_build_log(id) or { return app.status(.not_found) }
|
log := app.db.get_build_log(id) or { return app.status(.not_found) }
|
||||||
file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
|
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
|
||||||
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), log.arch,
|
|
||||||
file_name)
|
|
||||||
|
|
||||||
os.rm(full_path) or {
|
os.rm(full_path) or {
|
||||||
app.lerror('Failed to remove log file $full_path: $err.msg()')
|
app.lerror('Failed to remove log file $full_path: $err.msg()')
|
||||||
|
|
|
@ -28,11 +28,10 @@ fn (mut app App) log_removal_daemon(schedule CronExpression) {
|
||||||
logs = app.db.get_build_logs(before: too_old_timestamp, offset: offset, limit: 50)
|
logs = app.db.get_build_logs(before: too_old_timestamp, offset: offset, limit: 50)
|
||||||
|
|
||||||
for log in logs {
|
for log in logs {
|
||||||
file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
|
log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
|
||||||
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(),
|
|
||||||
log.arch, file_name)
|
os.rm(log_file_path) or {
|
||||||
os.rm(full_path) or {
|
app.lerror('Failed to remove log file $log_file_path: $err.msg()')
|
||||||
app.lerror('Failed to remove log file $full_path: $err.msg()')
|
|
||||||
failed += 1
|
failed += 1
|
||||||
|
|
||||||
continue
|
continue
|
||||||
|
|
Loading…
Reference in New Issue