From 7ba1727a9fd6c1dee9301bc2a54bf9670e66f272 Mon Sep 17 00:00:00 2001
From: Chewing_Bever <roosensjef@gmail.com>
Date: Sun, 19 Feb 2023 16:35:07 +0100
Subject: [PATCH 01/10] fix(web): don't log new metric for every query param

---
 src/web/web.v | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/web/web.v b/src/web/web.v
index 5c612f3..ea8559c 100644
--- a/src/web/web.v
+++ b/src/web/web.v
@@ -331,10 +331,10 @@ fn handle_conn[T](mut conn net.TcpConn, mut app T, routes map[string]Route) {
 			app.logger.flush()
 		}
 
-		// Record how long request took to process
+		path := urllib.parse(app.req.url) or { urllib.URL{} }.path
 		labels := [
 			['method', app.req.method.str()]!,
-			['path', app.req.url]!,
+			['path', path]!,
 			// Not all methods properly set this value yet I think
 			['status', app.status.int().str()]!,
 		]

From 3b24ad0f2c06908661e1b8ea595f0f98716f0f42 Mon Sep 17 00:00:00 2001
From: Chewing_Bever <roosensjef@gmail.com>
Date: Sun, 19 Feb 2023 16:38:53 +0100
Subject: [PATCH 02/10] fix(web): don't log new metric for every query param

---
 src/web/web.v | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/web/web.v b/src/web/web.v
index 5c612f3..775354a 100644
--- a/src/web/web.v
+++ b/src/web/web.v
@@ -332,9 +332,10 @@ fn handle_conn[T](mut conn net.TcpConn, mut app T, routes map[string]Route) {
 		}
 
 		// Record how long request took to process
+		path := urllib.parse(app.req.url) or { urllib.URL{} }.path
 		labels := [
 			['method', app.req.method.str()]!,
-			['path', app.req.url]!,
+			['path', path]!,
 			// Not all methods properly set this value yet I think
 			['status', app.status.int().str()]!,
 		]

From 094634084b741cce218364c8a5d5a28b5aaa9434 Mon Sep 17 00:00:00 2001
From: Chewing_Bever <roosensjef@gmail.com>
Date: Wed, 5 Apr 2023 10:50:30 +0200
Subject: [PATCH 03/10] feat(agent): use worker thread approach

---
 src/agent/daemon.v | 44 ++++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/src/agent/daemon.v b/src/agent/daemon.v
index 4364a92..d49b45e 100644
--- a/src/agent/daemon.v
+++ b/src/agent/daemon.v
@@ -20,11 +20,13 @@ struct AgentDaemon {
 	client client.Client
 mut:
 	images ImageManager
-	// Which builds are currently running; length is conf.max_concurrent_builds
-	builds []BuildConfig
 	// Atomic variables used to detect when a build has finished; length is
-	// conf.max_concurrent_builds
+	// conf.max_concurrent_builds. This approach is used as the difference
+	// between a recently finished build and an empty build slot is important
+	// for knowing whether the agent is currently "active".
 	atomics []u64
+	// Channel used to send builds to worker threads
+	build_channel chan BuildConfig
 }
 
 // agent_init initializes a new agent
@@ -34,8 +36,8 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon {
 		client: client.new(conf.address, conf.api_key)
 		conf: conf
 		images: new_image_manager(conf.image_rebuild_frequency * 60)
-		builds: []BuildConfig{len: conf.max_concurrent_builds}
 		atomics: []u64{len: conf.max_concurrent_builds}
+		build_channel: chan BuildConfig{cap: conf.max_concurrent_builds}
 	}
 
 	return d
@@ -43,6 +45,11 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon {
 
 // run starts the actual agent daemon. This function will run forever.
 pub fn (mut d AgentDaemon) run() {
+	// Spawn worker threads
+	for builder_index in 0 .. d.conf.max_concurrent_builds {
+		spawn d.builder_thread(d.build_channel, builder_index)
+	}
+
 	// This is just so that the very first time the loop is ran, the jobs are
 	// always polled
 	mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency)
@@ -107,10 +114,10 @@ pub fn (mut d AgentDaemon) run() {
 				// It's technically still possible that the build image is
 				// removed in the very short period between building the
 				// builder image and starting a build container with it. If
-				// this happens, faith really just didn't want you to do this
+				// this happens, fate really just didn't want you to do this
 				// build.
 
-				d.start_build(config)
+				d.build_channel <- config
 				running++
 			}
 		}
@@ -147,22 +154,6 @@ fn (mut d AgentDaemon) update_atomics() (int, int) {
 	return finished, empty
 }
 
-// start_build starts a build for the given BuildConfig.
-fn (mut d AgentDaemon) start_build(config BuildConfig) bool {
-	for i in 0 .. d.atomics.len {
-		if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty {
-			stdatomic.store_u64(&d.atomics[i], agent.build_running)
-			d.builds[i] = config
-
-			spawn d.run_build(i, config)
-
-			return true
-		}
-	}
-
-	return false
-}
-
 // run_build actually starts the build process for a given target.
 fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) {
 	d.linfo('started build: ${config}')
@@ -195,3 +186,12 @@ fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) {
 
 	stdatomic.store_u64(&d.atomics[build_index], agent.build_done)
 }
+
+// builder_thread is a thread that constantly listens for builds to process
+fn (mut d AgentDaemon) builder_thread(ch chan BuildConfig, builder_index int) {
+	for {
+		build_config := <-ch or { break }
+
+		d.run_build(builder_index, build_config)
+	}
+}

From 7595eb7bbea77479d59fdbdf5782e3ca34b5b7c9 Mon Sep 17 00:00:00 2001
From: Chewing_Bever <roosensjef@gmail.com>
Date: Thu, 27 Apr 2023 13:23:58 +0200
Subject: [PATCH 04/10] fix: error when upload failed before all bytes received

---
 src/server/repo.v | 2 +-
 src/util/stream.v | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/server/repo.v b/src/server/repo.v
index 051a7c2..8f8270d 100644
--- a/src/server/repo.v
+++ b/src/server/repo.v
@@ -68,7 +68,7 @@ fn (mut app App) put_package(repo_ string) web.Result {
 		mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true })
 
 		util.reader_to_file(mut app.reader, length.int(), pkg_path) or {
-			app.lwarn("Failed to upload '${pkg_path}'")
+			app.lwarn("Failed to upload '${pkg_path}': ${err.msg()}")
 
 			return app.status(.internal_server_error)
 		}
diff --git a/src/util/stream.v b/src/util/stream.v
index 4b362fc..ef6e872 100644
--- a/src/util/stream.v
+++ b/src/util/stream.v
@@ -46,6 +46,10 @@ pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ! {
 			to_write = to_write - bytes_written
 		}
 	}
+
+	if bytes_left > 0 {
+		return error('Not all bytes were received.')
+	}
 }
 
 // match_array_in_array[T] returns how many elements of a2 overlap with a1. For

From ac3a89500bbbbf9a4b95278913b61e4490ff5558 Mon Sep 17 00:00:00 2001
From: Chewing_Bever <roosensjef@gmail.com>
Date: Tue, 2 May 2023 11:46:19 +0200
Subject: [PATCH 05/10] feat: add non-functional build timeout setting

---
 src/build/build.v             |  4 ++--
 src/build/queue.v             |  7 +++++--
 src/console/targets/build.v   |  4 ++--
 src/console/targets/targets.v |  9 ++++++++-
 src/models/builds.v           |  3 ++-
 src/models/targets.v          |  3 ++-
 src/server/cli.v              | 23 ++++++++++++-----------
 src/server/server.v           |  2 +-
 8 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/src/build/build.v b/src/build/build.v
index c69a613..756e8f6 100644
--- a/src/build/build.v
+++ b/src/build/build.v
@@ -94,8 +94,8 @@ pub:
 }
 
 // build_target builds the given target. Internally it calls `build_config`.
-pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool) !BuildResult {
-	config := target.as_build_config(base_image_id, force)
+pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool, timeout int) !BuildResult {
+	config := target.as_build_config(base_image_id, force, timeout)
 
 	return build_config(address, api_key, config)
 }
diff --git a/src/build/queue.v b/src/build/queue.v
index 73068ac..bc4db9d 100644
--- a/src/build/queue.v
+++ b/src/build/queue.v
@@ -33,6 +33,8 @@ pub struct BuildJobQueue {
 	default_schedule &cron.Expression
 	// Base image to use for targets without defined base image
 	default_base_image string
+	// After how many minutes a build should be forcefully cancelled
+	default_build_timeout int
 mut:
 	mutex shared util.Dummy
 	// For each architecture, a priority queue is tracked
@@ -44,10 +46,11 @@ mut:
 }
 
 // new_job_queue initializes a new job queue
-pub fn new_job_queue(default_schedule &cron.Expression, default_base_image string) BuildJobQueue {
+pub fn new_job_queue(default_schedule &cron.Expression, default_base_image string, default_build_timeout int) BuildJobQueue {
 	return BuildJobQueue{
 		default_schedule: unsafe { default_schedule }
 		default_base_image: default_base_image
+		default_build_timeout: default_build_timeout
 		invalidated: map[int]time.Time{}
 	}
 }
@@ -80,7 +83,7 @@ pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! {
 		mut job := BuildJob{
 			created: time.now()
 			single: input.single
-			config: input.target.as_build_config(q.default_base_image, input.force)
+			config: input.target.as_build_config(q.default_base_image, input.force, q.default_build_timeout)
 		}
 
 		if !input.now {
diff --git a/src/console/targets/build.v b/src/console/targets/build.v
index a59e6a1..93464af 100644
--- a/src/console/targets/build.v
+++ b/src/console/targets/build.v
@@ -6,7 +6,7 @@ import os
 import build
 
 // build locally builds the target with the given id.
-fn build_target(conf Config, target_id int, force bool) ! {
+fn build_target(conf Config, target_id int, force bool, timeout int) ! {
 	c := client.new(conf.address, conf.api_key)
 	target := c.get_target(target_id)!
 
@@ -16,7 +16,7 @@ fn build_target(conf Config, target_id int, force bool) ! {
 	image_id := build.create_build_image(conf.base_image)!
 
 	println('Running build...')
-	res := build.build_target(conf.address, conf.api_key, image_id, target, force)!
+	res := build.build_target(conf.address, conf.api_key, image_id, target, force, timeout)!
 
 	println('Removing build image...')
 
diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v
index 676fa0a..80fc36b 100644
--- a/src/console/targets/targets.v
+++ b/src/console/targets/targets.v
@@ -232,6 +232,12 @@ pub fn cmd() cli.Command {
 						description: 'Architecture to schedule build for. Required when using -remote.'
 						flag: cli.FlagType.string
 					},
+					cli.Flag{
+						name: 'timeout'
+						description: 'After how many minutes to cancel the build. Only applies to local builds.'
+						flag: cli.FlagType.int
+						default_value: ['60']
+					},
 				]
 				execute: fn (cmd cli.Command) ! {
 					config_file := cmd.flags.get_string('config-file')!
@@ -239,6 +245,7 @@ pub fn cmd() cli.Command {
 
 					remote := cmd.flags.get_bool('remote')!
 					force := cmd.flags.get_bool('force')!
+					timeout := cmd.flags.get_int('timeout')!
 					target_id := cmd.args[0].int()
 
 					if remote {
@@ -251,7 +258,7 @@ pub fn cmd() cli.Command {
 						c := client.new(conf_.address, conf_.api_key)
 						c.queue_job(target_id, arch, force)!
 					} else {
-						build_target(conf_, target_id, force)!
+						build_target(conf_, target_id, force, timeout)!
 					}
 				}
 			},
diff --git a/src/models/builds.v b/src/models/builds.v
index be2910c..6923115 100644
--- a/src/models/builds.v
+++ b/src/models/builds.v
@@ -10,9 +10,10 @@ pub:
 	repo       string
 	base_image string
 	force      bool
+	timeout    int
 }
 
 // str return a single-line string representation of a build log
 pub fn (c BuildConfig) str() string {
-	return '{ target: ${c.target_id}, kind: ${c.kind}, url: ${c.url}, branch: ${c.branch}, path: ${c.path}, repo: ${c.repo}, base_image: ${c.base_image}, force: ${c.force} }'
+	return '{ target: ${c.target_id}, kind: ${c.kind}, url: ${c.url}, branch: ${c.branch}, path: ${c.path}, repo: ${c.repo}, base_image: ${c.base_image}, force: ${c.force}, timeout: ${c.timeout} }'
 }
diff --git a/src/models/targets.v b/src/models/targets.v
index 3c0c9cf..14cc8a6 100644
--- a/src/models/targets.v
+++ b/src/models/targets.v
@@ -54,7 +54,7 @@ pub fn (t &Target) str() string {
 
 // as_build_config converts a Target into a BuildConfig, given some extra
 // needed information.
-pub fn (t &Target) as_build_config(base_image string, force bool) BuildConfig {
+pub fn (t &Target) as_build_config(base_image string, force bool, timeout int) BuildConfig {
 	return BuildConfig{
 		target_id: t.id
 		kind: t.kind
@@ -64,6 +64,7 @@ pub fn (t &Target) as_build_config(base_image string, force bool) BuildConfig {
 		repo: t.repo
 		base_image: base_image
 		force: force
+		timeout: timeout
 	}
 }
 
diff --git a/src/server/cli.v b/src/server/cli.v
index 08ad5f8..c24812d 100644
--- a/src/server/cli.v
+++ b/src/server/cli.v
@@ -5,17 +5,18 @@ import conf as vconf
 
 struct Config {
 pub:
-	port                 int    = 8000
-	log_level            string = 'WARN'
-	pkg_dir              string
-	data_dir             string
-	api_key              string
-	default_arch         string
-	global_schedule      string = '0 3'
-	base_image           string = 'archlinux:base-devel'
-	max_log_age          int    [empty_default]
-	log_removal_schedule string = '0 0'
-	collect_metrics      bool   [empty_default]
+	port                  int    = 8000
+	log_level             string = 'WARN'
+	pkg_dir               string
+	data_dir              string
+	api_key               string
+	default_arch          string
+	global_schedule       string = '0 3'
+	base_image            string = 'archlinux:base-devel'
+	max_log_age           int    [empty_default]
+	log_removal_schedule  string = '0 0'
+	collect_metrics       bool   [empty_default]
+	default_build_timeout int = 60
 }
 
 // cmd returns the cli submodule that handles starting the server
diff --git a/src/server/server.v b/src/server/server.v
index 4cccb27..e1516fa 100644
--- a/src/server/server.v
+++ b/src/server/server.v
@@ -108,7 +108,7 @@ pub fn server(conf Config) ! {
 		repo: repo_
 		db: db
 		collector: collector
-		job_queue: build.new_job_queue(global_ce, conf.base_image)
+		job_queue: build.new_job_queue(global_ce, conf.base_image, conf.default_build_timeout)
 	}
 	app.init_job_queue() or {
 		util.exit_with_message(1, 'Failed to inialize job queue: ${err.msg()}')

From afb38256ac7ab48900582b13becefcc2df99d79d Mon Sep 17 00:00:00 2001
From: Chewing_Bever <roosensjef@gmail.com>
Date: Tue, 2 May 2023 14:49:32 +0200
Subject: [PATCH 06/10] feat: implement build timeout

---
 src/build/build.v | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/build/build.v b/src/build/build.v
index 756e8f6..b864792 100644
--- a/src/build/build.v
+++ b/src/build/build.v
@@ -136,9 +136,17 @@ pub fn build_config(address string, api_key string, config BuildConfig) !BuildRe
 	dd.container_start(id)!
 
 	mut data := dd.container_inspect(id)!
+	start_time := time.now()
 
 	// This loop waits until the container has stopped, so we can remove it after
 	for data.state.running {
+		if time.now() - start_time > config.timeout * time.second {
+			dd.container_kill(id)!
+			dd.container_remove(id)!
+
+			return error('Build killed due to timeout (${config.timeout}s)')
+		}
+
 		time.sleep(1 * time.second)
 
 		data = dd.container_inspect(id)!

From b278ebd73fe2a5096acd5e2a164b325a4b981035 Mon Sep 17 00:00:00 2001
From: Chewing_Bever <roosensjef@gmail.com>
Date: Tue, 2 May 2023 14:51:49 +0200
Subject: [PATCH 07/10] fix: set default timeout to 60 minutes

---
 src/console/targets/targets.v | 2 +-
 src/server/cli.v              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v
index 80fc36b..f85c4c0 100644
--- a/src/console/targets/targets.v
+++ b/src/console/targets/targets.v
@@ -236,7 +236,7 @@ pub fn cmd() cli.Command {
 						name: 'timeout'
 						description: 'After how many minutes to cancel the build. Only applies to local builds.'
 						flag: cli.FlagType.int
-						default_value: ['60']
+						default_value: ['3600']
 					},
 				]
 				execute: fn (cmd cli.Command) ! {
diff --git a/src/server/cli.v b/src/server/cli.v
index c24812d..abb5fe3 100644
--- a/src/server/cli.v
+++ b/src/server/cli.v
@@ -16,7 +16,7 @@ pub:
 	max_log_age           int    [empty_default]
 	log_removal_schedule  string = '0 0'
 	collect_metrics       bool   [empty_default]
-	default_build_timeout int = 60
+	default_build_timeout int = 3600
 }
 
 // cmd returns the cli submodule that handles starting the server

From 076ee24b1b7170e8a10fe054ec0c994b6cd3e2ca Mon Sep 17 00:00:00 2001
From: Chewing_Bever <roosensjef@gmail.com>
Date: Thu, 4 May 2023 09:36:03 +0200
Subject: [PATCH 08/10] chore: update changelog

---
 CHANGELOG.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a95d6c1..9747a16 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,11 +12,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Metrics endpoint for Prometheus integration
 * Search in list of targets using API & CLI
 * Allow filtering targets by arch value
+* Configurable global timeout for builds
 
 ### Changed
 
 * Rewrote cron expression logic in C
 * Updated codebase to V commit after 0.3.3
+* Agents now use worker threads and no longer spawn a new thread for every
+  build
+
+### Fixed
+
+* Package upload now fails if TCP connection is closed before all bytes have
+  been received
 
 ### Removed
 

From 47c0f0405b536a1e28d5713554cd6fe3473baa0a Mon Sep 17 00:00:00 2001
From: Chewing_Bever <roosensjef@gmail.com>
Date: Mon, 17 Jul 2023 13:26:38 +0200
Subject: [PATCH 09/10] chore: update versions to 0.6.0

---
 CHANGELOG.md | 2 ++
 PKGBUILD     | 2 +-
 src/main.v   | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9747a16..871877e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
 
+## [0.6.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.6.0)
+
 ### Added
 
 * Metrics endpoint for Prometheus integration
diff --git a/PKGBUILD b/PKGBUILD
index 7438390..05a3e73 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -3,7 +3,7 @@
 
 pkgbase='vieter'
 pkgname='vieter'
-pkgver='0.5.0'
+pkgver='0.6.0'
 pkgrel=1
 pkgdesc="Lightweight Arch repository server & package build system"
 depends=('glibc' 'openssl' 'libarchive' 'sqlite')
diff --git a/src/main.v b/src/main.v
index ce9ec81..e3b8a1a 100644
--- a/src/main.v
+++ b/src/main.v
@@ -20,7 +20,7 @@ fn main() {
 	mut app := cli.Command{
 		name: 'vieter'
 		description: 'Vieter is a lightweight implementation of an Arch repository server.'
-		version: '0.5.0'
+		version: '0.6.0'
 		posix_mode: true
 		flags: [
 			cli.Flag{

From 1a992806faf5e3678af9eb5c9dd30939203451cf Mon Sep 17 00:00:00 2001
From: Chewing_Bever <roosensjef@gmail.com>
Date: Mon, 17 Jul 2023 13:53:20 +0200
Subject: [PATCH 10/10] fix: update PKGBUILD to use libvieter

---
 PKGBUILD | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/PKGBUILD b/PKGBUILD
index 05a3e73..e5cde95 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -11,13 +11,23 @@ makedepends=('git' 'vieter-vlang')
 arch=('x86_64' 'aarch64')
 url='https://git.rustybever.be/vieter-v/vieter'
 license=('AGPL3')
-source=("$pkgname::git+https://git.rustybever.be/vieter-v/vieter#tag=${pkgver//_/-}")
-md5sums=('SKIP')
+source=(
+    "$pkgname::git+https://git.rustybever.be/vieter-v/vieter#tag=${pkgver//_/-}"
+    "libvieter::git+https://git.rustybever.be/vieter-v/libvieter"
+)
+md5sums=('SKIP' 'SKIP')
 
 prepare() {
-    export VMODULES="$srcdir/.vmodules"
+    cd "${pkgname}"
 
-    cd "$pkgname/src" && v install
+    # Add the libvieter submodule
+    git submodule init
+    git config submodules.src/libvieter.url "${srcdir}/libvieter"
+    git -c protocol.file.allow=always submodule update
+
+    export VMODULES="${srcdir}/.vmodules"
+
+    cd src && v install
 }
 
 build() {