feat(server): add log removal daemon

docs/api/source/includes/_logs.md

@@ -149,3 +149,17 @@ target | id of target this build is for
 ### Request body
 
 Plaintext contents of the build log.
+
+## Remove a build log
+
+Remove a build log from the server.
+
+### HTTP Request
+
+`DELETE /api/v1/logs/:id`
+
+### URL Parameters
+
+Parameter | Description
+--------- | -----------
+id | id of log to remove

docs/content/configuration.md

@@ -47,6 +47,11 @@ configuration variable required for each command.
     * Git repositories added without an `arch` value use this value instead.
 * `port`: HTTP port to run on
     * Default: `8000`
+* `max_log_age`: maximum age of logs (in days). Logs older than this will get
+  cleaned by the log removal daemon every 24 hours. If set to a negative value,
+  no logs are ever removed. The age of logs is determined by the time the build
+  was started.
+    * Default: `-1`
 
 ### `vieter cron`
 

src/client/logs.v

@@ -41,3 +41,10 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t
 
 	return data
 }
+
+// remove_build_log removes the build log with the given id from the server.
+pub fn (c &Client) remove_build_log(id int) !string {
+	data := c.send_request<string>(.delete, '/api/v1/logs/$id', {})!
+
+	return data.data
+}

src/console/logs/logs.v

@@ -138,6 +138,18 @@ pub fn cmd() cli.Command {
 					list(conf, filter, raw)!
 				}
 			},
+			cli.Command{
+				name: 'remove'
+				required_args: 1
+				usage: 'id'
+				description: 'Remove a build log that matches the given id.'
+				execute: fn (cmd cli.Command) ! {
+					config_file := cmd.flags.get_string('config-file')!
+					conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
+
+					remove(conf, cmd.args[0])!
+				}
+			},
 			cli.Command{
 				name: 'info'
 				required_args: 1
@@ -204,3 +216,9 @@ fn content(conf Config, id int) ! {
 
 	println(content)
 }
+
+// remove removes a build log from the server's list.
+fn remove(conf Config, id string) ! {
+	c := client.new(conf.address, conf.api_key)
+	c.remove_build_log(id.int())!
+}

src/server/api_logs.v

@@ -124,3 +124,22 @@ fn (mut app App) v1_post_log() web.Result {
 
 	return app.json(.ok, new_data_response(log_id))
 }
+
+// v1_delete_log allows removing a build log from the system.
+['/api/v1/logs/:id'; auth; delete]
+fn (mut app App) v1_delete_log(id int) web.Result {
+	log := app.db.get_build_log(id) or { return app.status(.not_found) }
+	file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
+	full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), log.arch,
+		file_name)
+
+	os.rm(full_path) or {
+		app.lerror('Failed to remove log file $full_path: $err.msg()')
+
+		return app.status(.internal_server_error)
+	}
+
+	app.db.delete_build_log(id)
+
+	return app.status(.ok)
+}

src/server/cli.v

@@ -13,6 +13,7 @@ pub:
 	global_schedule string = '0 3'
 	port            int    = 8000
 	base_image      string = 'archlinux:base-devel'
+	max_log_age     int    = -1
 }
 
 // cmd returns the cli submodule that handles starting the server

src/server/log_removal.v

@@ -0,0 +1,56 @@
+module server
+
+import time
+import models { BuildLog }
+import os
+
+const log_removal_frequency = 24 * time.hour
+
+// log_removal_daemon removes old build logs every `log_removal_frequency`.
+fn (mut app App) log_removal_daemon() {
+	mut start_time := time.Time{}
+
+	for {
+		start_time = time.now()
+
+		mut too_old_timestamp := time.now().add_days(-app.conf.max_log_age)
+
+		app.linfo('Cleaning logs before $too_old_timestamp')
+
+		mut offset := u64(0)
+		mut logs := []BuildLog{}
+		mut counter := 0
+		mut failed := 0
+
+		// Remove old logs
+		for {
+			logs = app.db.get_build_logs(before: too_old_timestamp, offset: offset, limit: 50)
+
+			for log in logs {
+				file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
+				full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(),
+					log.arch, file_name)
+				os.rm(full_path) or {
+					app.lerror('Failed to remove log file $full_path: $err.msg()')
+					failed += 1
+
+					continue
+				}
+				app.db.delete_build_log(log.id)
+
+				counter += 1
+			}
+
+			if logs.len < 50 {
+				break
+			}
+
+			offset += 50
+		}
+
+		app.linfo('Cleaned $counter logs ($failed failed)')
+
+		// Sleep until the next cycle
+		time.sleep(start_time.add_days(1) - time.now())
+	}
+}

src/server/server.v

@@ -108,5 +108,9 @@ pub fn server(conf Config) ! {
 		util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()')
 	}
 
+	if conf.max_log_age > 0 {
+		go app.log_removal_daemon()
+	}
+
 	web.run(app, conf.port)
 }

vieter.toml

@@ -12,3 +12,4 @@ address = "http://localhost:8000"
 api_update_frequency = 2
 image_rebuild_frequency = 1
 max_concurrent_builds = 3
+max_log_age = 64