Merge pull request 'implementation of cron daemon' (#134) from cron into dev

Reviewed-on: Chewing_Bever/vieter#134

.gitignore

@@ -5,8 +5,8 @@ data/
 vieter
 dvieter
 pvieter
-dvieterctl
+suvieter
-vieterctl
+afvieter
 vieter.c
 
 # Ignore testing files
@@ -23,3 +23,6 @@ v/
 
 # gdb log file
 gdb.txt
+
+# Generated docs
+_docs/

.woodpecker/.arch.yml

@@ -10,6 +10,8 @@ pipeline:
   build:
     image: 'menci/archlinuxarm:base-devel'
     commands:
+      # Add the vieter repository so we can use the compiler
+      - echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
       # Update packages
       - pacman -Syu --noconfirm
       # Create non-root user to perform build & switch to their home

.woodpecker/.build.yml

@@ -9,22 +9,21 @@ matrix:
 platform: ${PLATFORM}
 
 pipeline:
-  # The default build isn't needed, as alpine switches to gcc for the compiler anyways
   debug:
     image: 'chewingbever/vlang:latest'
     pull: true
-    group: 'build'
     commands:
-      - make debug
+      - make
     when:
       event: push
+      branch:
+        exclude: [main, dev]
 
   prod:
     image: 'chewingbever/vlang:latest'
     pull: true
     environment:
       - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static
-    group: 'build'
     commands:
       - make prod
       # Make sure the binary is actually statically built

.woodpecker/.build_experimental.yml

@@ -0,0 +1,29 @@
+# These builds are not important for the project, but might be valuable for
+# fixing bugs in the V compiler.
+
+platform: linux/amd64
+branches:
+  exclude: [master, dev]
+
+pipeline:
+  autofree:
+    image: 'chewingbever/vlang:latest'
+    pull: true
+    group: 'build'
+    commands:
+      - make autofree
+      - readelf -d afvieter
+      - du -h afvieter
+    when:
+      event: push
+
+  skip-unused:
+    image: 'chewingbever/vlang:latest'
+    pull: true
+    group: 'build'
+    commands:
+      - make skip-unused
+      - readelf -d suvieter
+      - du -h suvieter
+    when:
+      event: push

.woodpecker/.lint.yml

@@ -7,7 +7,5 @@ pipeline:
   lint:
     image: 'chewingbever/vlang:latest'
     pull: true
-    group: lint
     commands:
       - make lint
-      - make vet

Makefile

@@ -7,6 +7,7 @@ V := $(V_PATH) -showcc -gc boehm
 
 all: vieter
 
+
 # =====COMPILATION=====
 # Regular binary
 vieter: $(SOURCES)
@@ -23,7 +24,7 @@ dvieter: $(SOURCES)
 # Run the debug build inside gdb
 .PHONY: gdb
 gdb: dvieter
-		gdb --args './dvieter -f vieter.toml server'
+		gdb --args ./dvieter -f vieter.toml server
 
 # Optimised production build
 .PHONY: prod
@@ -33,33 +34,34 @@ pvieter: $(SOURCES)
 
 # Only generate C code
 .PHONY: c
-c:
+c: $(SOURCES)
 	$(V) -o vieter.c $(SRC_DIR)
 
+
 # =====EXECUTION=====
 # Run the server in the default 'data' directory
 .PHONY: run
 run: vieter
-		./vieter -f vieter.toml server
+	./vieter -f vieter.toml server
 
 .PHONY: run-prod
 run-prod: prod
 	./pvieter -f vieter.toml server
 
+
 # =====OTHER=====
 .PHONY: lint
 lint:
 	$(V) fmt -verify $(SRC_DIR)
+	$(V) vet -W $(SRC_DIR)
+	$(V_PATH) missdoc -p $(SRC_DIR)
+	@ [ $$($(V_PATH) missdoc -p $(SRC_DIR) | wc -l) = 0 ]
 
 # Format the V codebase
 .PHONY: fmt
 fmt:
 	$(V) fmt -w $(SRC_DIR)
 
-.PHONY: vet
-vet:
-	$(V) vet -W $(SRC_DIR)
-
 .PHONY: test
 test:
 	$(V) test $(SRC_DIR)
@@ -71,5 +73,23 @@ v/v:
 	git clone --single-branch https://git.rustybever.be/Chewing_Bever/v v
 	make -C v
 
+.PHONY: clean
 clean:
-	rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst
+	rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs'
+
+.PHONY: api-docs
+api-docs:
+	rm -rf '$(SRC_DIR)/_docs'
+	cd '$(SRC_DIR)' && v doc -all -f html -m -readme .
+
+
+# =====EXPERIMENTAL=====
+.PHONY: autofree
+autofree: afvieter
+afvieter: $(SOURCES)
+	$(V_PATH) -showcc -autofree -o afvieter $(SRC_DIR)
+
+.PHONY: skip-unused
+skip-unused: suvieter
+suvieter: $(SOURCES)
+	$(V_PATH) -showcc -skip-unused -o suvieter $(SRC_DIR)

PKGBUILD

@@ -2,10 +2,10 @@
 
 pkgbase='vieter'
 pkgname='vieter'
-pkgver=0.2.0.r24.g9a56bd0
+pkgver=0.2.0.r25.g20112b8
 pkgrel=1
 depends=('glibc' 'openssl' 'libarchive' 'gc')
-makedepends=('git' 'gcc')
+makedepends=('git' 'gcc' 'vieter-v')
 arch=('x86_64' 'aarch64' 'armv7')
 url='https://git.rustybever.be/Chewing_Bever/vieter'
 license=('AGPL3')
@@ -20,10 +20,7 @@ pkgver() {
 build() {
     cd "$pkgname"
 
-    # Build the compiler
+    make prod
-    CFLAGS= make v
-
-    V_PATH=v/v make prod
 }
 
 package() {

README.md

@@ -2,7 +2,9 @@
 
 ## Documentation
 
-I host documentation for Vieter over at https://rustybever.be/docs/vieter/.
+I host documentation for Vieter over at https://rustybever.be/docs/vieter/. API
+documentation for the current codebase can be found at
+https://rustybever.be/api-docs/vieter/.
 
 ## Overview
 

src/build/build.v

@@ -10,7 +10,12 @@ const container_build_dir = '/build'
 
 const build_image_repo = 'vieter-build'
 
-fn create_build_image(base_image string) ?string {
+// create_build_image creates a builder image given some base image which can
+// then be used to build & package Arch images. It mostly just updates the
+// system, install some necessary packages & creates a non-root user to run
+// makepkg with. The base image should be some Linux distribution that uses
+// Pacman as its package manager.
+pub fn create_build_image(base_image string) ?string {
 	commands := [
 		// Update repos & install required packages
 		'pacman -Syu --needed --noconfirm base-devel git'
@@ -53,12 +58,13 @@ fn create_build_image(base_image string) ?string {
 			break
 		}
 
-		// Wait for 5 seconds
+		time.sleep(1 * time.second)
-		time.sleep(5000000000)
 	}
 
 	// Finally, we create the image from the container
 	// As the tag, we use the epoch value
+	// TODO also add the base image's name into the image name to prevent
+	// conflicts.
 	tag := time.sys_mono_now().str()
 	image := docker.create_image_from_container(id, 'vieter-build', tag) ?
 	docker.remove_container(id) ?
@@ -66,6 +72,55 @@ fn create_build_image(base_image string) ?string {
 	return image.id
 }
 
+// build_repo builds, packages & publishes a given Arch package based on the
+// provided GitRepo. The base image ID should be of an image previously created
+// by create_build_image.
+pub fn build_repo(address string, api_key string, base_image_id string, repo &git.GitRepo) ? {
+	build_arch := os.uname().machine
+
+	// TODO what to do with PKGBUILDs that build multiple packages?
+	commands := [
+		'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
+		'cd repo',
+		'makepkg --nobuild --nodeps',
+		'source PKGBUILD',
+		// The build container checks whether the package is already
+		// present on the server
+		'curl --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
+		'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done',
+	]
+
+	// We convert the list of commands into a base64 string, which then gets
+	// passed to the container as an env var
+	cmds_str := base64.encode_str(commands.join('\n'))
+
+	c := docker.NewContainer{
+		image: '$base_image_id'
+		env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$api_key']
+		entrypoint: ['/bin/sh', '-c']
+		cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e']
+		work_dir: '/build'
+		user: 'builder:builder'
+	}
+
+	id := docker.create_container(c) ?
+	docker.start_container(id) ?
+
+	// This loop waits until the container has stopped, so we can remove it after
+	for {
+		data := docker.inspect_container(id) ?
+
+		if !data.state.running {
+			break
+		}
+
+		time.sleep(1 * time.second)
+	}
+
+	docker.remove_container(id) ?
+}
+
+// build builds every Git repo in the server's list.
 fn build(conf Config) ? {
 	build_arch := os.uname().machine
 
@@ -85,47 +140,7 @@ fn build(conf Config) ? {
 	image_id := create_build_image(conf.base_image) ?
 
 	for repo in filtered_repos {
-		// TODO what to do with PKGBUILDs that build multiple packages?
+		build_repo(conf.address, conf.api_key, image_id, repo) ?
-		commands := [
-			'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
-			'cd repo',
-			'makepkg --nobuild --nodeps',
-			'source PKGBUILD',
-			// The build container checks whether the package is already
-			// present on the server
-			'curl --head --fail $conf.address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
-			'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/$repo.repo/publish; done',
-		]
-
-		// We convert the list of commands into a base64 string, which then gets
-		// passed to the container as an env var
-		cmds_str := base64.encode_str(commands.join('\n'))
-
-		c := docker.NewContainer{
-			image: '$image_id'
-			env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key']
-			entrypoint: ['/bin/sh', '-c']
-			cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e']
-			work_dir: '/build'
-			user: 'builder:builder'
-		}
-
-		id := docker.create_container(c) ?
-		docker.start_container(id) ?
-
-		// This loop waits until the container has stopped, so we can remove it after
-		for {
-			data := docker.inspect_container(id) ?
-
-			if !data.state.running {
-				break
-			}
-
-			// Wait for 5 seconds
-			time.sleep(5000000000)
-		}
-
-		docker.remove_container(id) ?
 	}
 
 	// Finally, we remove the builder image

src/cron/cli.v

@@ -5,11 +5,16 @@ import env
 
 struct Config {
 pub:
-	log_level  string = 'WARN'
+	log_level               string = 'WARN'
-	log_file   string = 'vieter.log'
+	log_file                string = 'vieter.log'
-	api_key    string
+	api_key                 string
-	address    string
+	address                 string
-	base_image string = 'archlinux:base-devel'
+	base_image              string = 'archlinux:base-devel'
+	max_concurrent_builds   int    = 1
+	api_update_frequency    int    = 15
+	image_rebuild_frequency int    = 1440
+	// Replicates the behavior of the original cron system
+	global_schedule string = '0 3'
 }
 
 // cmd returns the cli module that handles the cron daemon.

src/cron/cron.v

@@ -1,18 +1,29 @@
 module cron
 
-import git
+import log
-import time
+import cron.daemon
-
+import cron.expression
-struct ScheduledBuild {
-	repo      git.GitRepo
-	timestamp time.Time
-}
-
-fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool {
-	return r1.timestamp < r2.timestamp
-}
 
 // cron starts a cron daemon & starts periodically scheduling builds.
 pub fn cron(conf Config) ? {
-	println('WIP')
+	// Configure logger
+	log_level := log.level_from_tag(conf.log_level) or {
+		return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
+	}
+
+	mut logger := log.Log{
+		level: log_level
+	}
+
+	logger.set_full_logpath(conf.log_file)
+	logger.log_to_console_too()
+
+	ce := expression.parse_expression(conf.global_schedule) or {
+		return error('Error while parsing global cron expression: $err.msg()')
+	}
+
+	mut d := daemon.init_daemon(logger, conf.address, conf.api_key, conf.base_image, ce,
+		conf.max_concurrent_builds, conf.api_update_frequency, conf.image_rebuild_frequency) ?
+
+	d.run() ?
 }

src/cron/daemon/build.v

@@ -0,0 +1,83 @@
+module daemon
+
+import time
+import sync.stdatomic
+import build
+
+const build_empty = 0
+
+const build_running = 1
+
+const build_done = 2
+
+// clean_finished_builds removes finished builds from the build slots & returns
+// them.
+fn (mut d Daemon) clean_finished_builds() ?[]ScheduledBuild {
+	mut out := []ScheduledBuild{}
+
+	for i in 0 .. d.atomics.len {
+		if stdatomic.load_u64(&d.atomics[i]) == daemon.build_done {
+			stdatomic.store_u64(&d.atomics[i], daemon.build_empty)
+			out << d.builds[i]
+		}
+	}
+
+	return out
+}
+
+// update_builds starts as many builds as possible.
+fn (mut d Daemon) start_new_builds() ? {
+	now := time.now()
+
+	for d.queue.len() > 0 {
+		if d.queue.peek() ?.timestamp < now {
+			sb := d.queue.pop() ?
+
+			// If this build couldn't be scheduled, no more will be possible.
+			if !d.start_build(sb) {
+				d.queue.insert(sb)
+				break
+			}
+		} else {
+			break
+		}
+	}
+}
+
+// start_build starts a build for the given ScheduledBuild object.
+fn (mut d Daemon) start_build(sb ScheduledBuild) bool {
+	for i in 0 .. d.atomics.len {
+		if stdatomic.load_u64(&d.atomics[i]) == daemon.build_empty {
+			stdatomic.store_u64(&d.atomics[i], daemon.build_running)
+			d.builds[i] = sb
+
+			go d.run_build(i, sb)
+
+			return true
+		}
+	}
+
+	return false
+}
+
+// run_build actually starts the build process for a given repo.
+fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) ? {
+	d.linfo('started build: $sb.repo.url $sb.repo.branch')
+
+	build.build_repo(d.address, d.api_key, d.builder_images.last(), &sb.repo) ?
+
+	stdatomic.store_u64(&d.atomics[build_index], daemon.build_done)
+}
+
+// current_build_count returns how many builds are currently running.
+fn (mut d Daemon) current_build_count() int {
+	mut res := 0
+
+	for i in 0 .. d.atomics.len {
+		if stdatomic.load_u64(&d.atomics[i]) == daemon.build_running {
+			res += 1
+		}
+	}
+
+	return res
+}

src/cron/daemon/daemon.v

@@ -0,0 +1,223 @@
+module daemon
+
+import git
+import time
+import log
+import datatypes { MinHeap }
+import cron.expression { CronExpression, parse_expression }
+import math
+import build
+import docker
+
+struct ScheduledBuild {
+pub:
+	repo_id   string
+	repo      git.GitRepo
+	timestamp time.Time
+}
+
+// Overloaded operator for comparing ScheduledBuild objects
+fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool {
+	return r1.timestamp < r2.timestamp
+}
+
+pub struct Daemon {
+mut:
+	address                 string
+	api_key                 string
+	base_image              string
+	builder_images          []string
+	global_schedule         CronExpression
+	api_update_frequency    int
+	image_rebuild_frequency int
+	// Repos currently loaded from API.
+	repos_map map[string]git.GitRepo
+	// At what point to update the list of repositories.
+	api_update_timestamp  time.Time
+	image_build_timestamp time.Time
+	queue                 MinHeap<ScheduledBuild>
+	// Which builds are currently running
+	builds []ScheduledBuild
+	// Atomic variables used to detect when a build has finished; length is the
+	// same as builds
+	atomics []u64
+	logger  shared log.Log
+}
+
+// init_daemon initializes a new Daemon object. It renews the repositories &
+// populates the build queue for the first time.
+pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon {
+	mut d := Daemon{
+		address: address
+		api_key: api_key
+		base_image: base_image
+		global_schedule: global_schedule
+		api_update_frequency: api_update_frequency
+		image_rebuild_frequency: image_rebuild_frequency
+		atomics: []u64{len: max_concurrent_builds}
+		builds: []ScheduledBuild{len: max_concurrent_builds}
+		logger: logger
+	}
+
+	// Initialize the repos & queue
+	d.renew_repos() ?
+	d.renew_queue() ?
+	d.rebuild_base_image() ?
+
+	return d
+}
+
+// run starts the actual daemon process. It runs builds when possible &
+// periodically refreshes the list of repositories to ensure we stay in sync.
+pub fn (mut d Daemon) run() ? {
+	for {
+		finished_builds := d.clean_finished_builds() ?
+
+		// Update the API's contents if needed & renew the queue
+		if time.now() >= d.api_update_timestamp {
+			d.renew_repos() ?
+			d.renew_queue() ?
+		}
+		// The finished builds should only be rescheduled if the API contents
+		// haven't been renewed.
+		else {
+			for sb in finished_builds {
+				d.schedule_build(sb.repo_id, sb.repo) ?
+			}
+		}
+
+		// TODO remove old builder images.
+		// This issue is less trivial than it sounds, because a build could
+		// still be running when the image has to be rebuilt. That would
+		// prevent the image from being removed. Therefore, we will need to
+		// keep track of a list or something & remove an image once we have
+		// made sure it isn't being used anymore.
+		if time.now() >= d.image_build_timestamp {
+			d.rebuild_base_image() ?
+			// In theory, executing this function here allows an old builder
+			// image to exist for at most image_rebuild_frequency minutes.
+			d.clean_old_base_images()
+		}
+
+		// Schedules new builds when possible
+		d.start_new_builds() ?
+
+		// If there are builds currently running, the daemon should refresh
+		// every second to clean up any finished builds & start new ones.
+		mut delay := time.Duration(1 * time.second)
+
+		// Sleep either until we have to refresh the repos or when the next
+		// build has to start, with a minimum of 1 second.
+		if d.current_build_count() == 0 {
+			now := time.now()
+			delay = d.api_update_timestamp - now
+
+			if d.queue.len() > 0 {
+				time_until_next_job := d.queue.peek() ?.timestamp - now
+
+				delay = math.min(delay, time_until_next_job)
+			}
+		}
+
+		// We sleep for at least one second. This is to prevent the program
+		// from looping agressively when a cronjob can be scheduled, but
+		// there's no spots free for it to be started.
+		delay = math.max(delay, 1 * time.second)
+
+		d.ldebug('Sleeping for ${delay}...')
+
+		time.sleep(delay)
+	}
+}
+
+// schedule_build adds the next occurence of the given repo build to the queue.
+fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) ? {
+	ce := if repo.schedule != '' {
+		parse_expression(repo.schedule) or {
+			// TODO This shouldn't return an error if the expression is empty.
+			d.lerror("Error while parsing cron expression '$repo.schedule' ($repo_id): $err.msg()")
+
+			d.global_schedule
+		}
+	} else {
+		d.global_schedule
+	}
+
+	// A repo that can't be scheduled will just be skipped for now
+	timestamp := ce.next_from_now() ?
+
+	d.queue.insert(ScheduledBuild{
+		repo_id: repo_id
+		repo: repo
+		timestamp: timestamp
+	})
+}
+
+// renew_repos requests the newest list of Git repos from the server & replaces
+// the old one.
+fn (mut d Daemon) renew_repos() ? {
+	d.linfo('Renewing repos...')
+	mut new_repos := git.get_repos(d.address, d.api_key) ?
+
+	d.repos_map = new_repos.move()
+
+	d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency)
+}
+
+// renew_queue replaces the old queue with a new one that reflects the newest
+// values in repos_map.
+fn (mut d Daemon) renew_queue() ? {
+	d.linfo('Renewing queue...')
+	mut new_queue := MinHeap<ScheduledBuild>{}
+
+	// Move any jobs that should have already started from the old queue onto
+	// the new one
+	now := time.now()
+
+	// For some reason, using
+	// ```v
+	// for d.queue.len() > 0 && d.queue.peek() ?.timestamp < now {
+	//```
+	// here causes the function to prematurely just exit, without any errors or anything, very weird
+	// https://github.com/vlang/v/issues/14042
+	for d.queue.len() > 0 {
+		if d.queue.peek() ?.timestamp < now {
+			new_queue.insert(d.queue.pop() ?)
+		} else {
+			break
+		}
+	}
+
+	d.queue = new_queue
+
+	// For each repository in repos_map, parse their cron expression (or use
+	// the default one if not present) & add them to the queue
+	for id, repo in d.repos_map {
+		d.schedule_build(id, repo) ?
+	}
+}
+
+// rebuild_base_image recreates the builder image.
+fn (mut d Daemon) rebuild_base_image() ? {
+	d.linfo('Rebuilding builder image....')
+
+	d.builder_images << build.create_build_image(d.base_image) ?
+	d.image_build_timestamp = time.now().add_seconds(60 * d.image_rebuild_frequency)
+}
+
+// clean_old_base_images tries to remove any old but still present builder
+// images.
+fn (mut d Daemon) clean_old_base_images() {
+	mut i := 0
+
+	for i < d.builder_images.len - 1 {
+		// For each builder image, we try to remove it by calling the Docker
+		// API. If the function returns an error or false, that means the image
+		// wasn't deleted. Therefore, we move the index over. If the function
+		// returns true, the array's length has decreased by one so we don't
+		// move the index.
+		if !docker.remove_image(d.builder_images[i]) or { false } {
+			i += 1
+		}
+	}
+}

src/cron/daemon/log.v

@@ -0,0 +1,35 @@
+module daemon
+
+import log
+
+// log reate a log message with the given level
+pub fn (mut d Daemon) log(msg &string, level log.Level) {
+	lock d.logger {
+		d.logger.send_output(msg, level)
+	}
+}
+
+// lfatal create a log message with the fatal level
+pub fn (mut d Daemon) lfatal(msg &string) {
+	d.log(msg, log.Level.fatal)
+}
+
+// lerror create a log message with the error level
+pub fn (mut d Daemon) lerror(msg &string) {
+	d.log(msg, log.Level.error)
+}
+
+// lwarn create a log message with the warn level
+pub fn (mut d Daemon) lwarn(msg &string) {
+	d.log(msg, log.Level.warn)
+}
+
+// linfo create a log message with the info level
+pub fn (mut d Daemon) linfo(msg &string) {
+	d.log(msg, log.Level.info)
+}
+
+// ldebug create a log message with the debug level
+pub fn (mut d Daemon) ldebug(msg &string) {
+	d.log(msg, log.Level.debug)
+}

src/cron/expression/expression.v

@@ -1,8 +1,8 @@
-module cron
+module expression
 
 import time
 
-struct CronExpression {
+pub struct CronExpression {
 	minutes []int
 	hours   []int
 	days    []int
@@ -65,6 +65,7 @@ pub fn (ce &CronExpression) next(ref time.Time) ?time.Time {
 	if minute_index == ce.minutes.len && hour_index < ce.hours.len {
 		hour_index += 1
 	}
+
 	if hour_index == ce.hours.len && day_index < ce.days.len {
 		day_index += 1
 	}
@@ -114,7 +115,9 @@ pub fn (ce &CronExpression) next(ref time.Time) ?time.Time {
 	})
 }
 
-fn (ce &CronExpression) next_from_now() ?time.Time {
+// next_from_now returns the result of ce.next(ref) where ref is the result of
+// time.now().
+pub fn (ce &CronExpression) next_from_now() ?time.Time {
 	return ce.next(time.now())
 }
 
@@ -195,6 +198,8 @@ fn parse_range(s string, min int, max int, mut bitv []bool) ? {
 	}
 }
 
+// bitv_to_ints converts a bit vector into an array containing the
+// corresponding values.
 fn bitv_to_ints(bitv []bool, min int) []int {
 	mut out := []int{}
 
@@ -207,6 +212,8 @@ fn bitv_to_ints(bitv []bool, min int) []int {
 	return out
 }
 
+// parse_part parses a given part of a cron expression & returns the
+// corresponding array of ints.
 fn parse_part(s string, min int, max int) ?[]int {
 	mut bitv := []bool{len: max - min + 1, init: false}
 
@@ -219,7 +226,7 @@ fn parse_part(s string, min int, max int) ?[]int {
 
 // parse_expression parses an entire cron expression string into a
 // CronExpression object, if possible.
-fn parse_expression(exp string) ?CronExpression {
+pub fn parse_expression(exp string) ?CronExpression {
 	// The filter allows for multiple spaces between parts
 	mut parts := exp.split(' ').filter(it != '')
 
@@ -241,7 +248,7 @@ fn parse_expression(exp string) ?CronExpression {
 	// This for loop allows us to more clearly propagate the error to the user.
 	for i, min in mins {
 		part_results << parse_part(parts[i], min, maxs[i]) or {
-			return error('An error occurred with part $i: $err.msg')
+			return error('An error occurred with part $i: $err.msg()')
 		}
 	}
 

src/cron/expression/expression_parse_test.v

@@ -1,4 +1,4 @@
-module cron
+module expression
 
 // parse_range_error returns the returned error message. If the result is '',
 // that means the function didn't error.

src/cron/expression/expression_test.v

@@ -1,4 +1,4 @@
-module cron
+module expression
 
 import time { parse }
 

src/docker/docker.v

@@ -9,6 +9,8 @@ const socket = '/var/run/docker.sock'
 
 const buf_len = 1024
 
+// send writes a request to the Docker socket, waits for a response & returns
+// it.
 fn send(req &string) ?http.Response {
 	// Open a connection to the socket
 	mut s := unix.connect_stream(docker.socket) or {
@@ -28,8 +30,8 @@ fn send(req &string) ?http.Response {
 	s.wait_for_write() ?
 
 	mut c := 0
-	mut buf := []byte{len: docker.buf_len}
+	mut buf := []u8{len: docker.buf_len}
-	mut res := []byte{}
+	mut res := []u8{}
 
 	for {
 		c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') }
@@ -52,7 +54,7 @@ fn send(req &string) ?http.Response {
 
 	// We loop until we've encountered the end of the chunked response
 	// A chunked HTTP response always ends with '0\r\n\r\n'.
-	for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] {
+	for res.len < 5 || res#[-5..] != [u8(`0`), `\r`, `\n`, `\r`, `\n`] {
 		// Wait for the server to respond
 		s.wait_for_write() ?
 
@@ -72,12 +74,14 @@ fn send(req &string) ?http.Response {
 	return http.parse_response(res.bytestr())
 }
 
+// request_with_body sends a request to the Docker socket with the given body.
 fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response {
 	req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n'
 
 	return send(req)
 }
 
+// request sends a request to the Docker socket with an empty body.
 fn request(method string, url urllib.URL) ?http.Response {
 	req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n'
 

src/env/env.v

@@ -36,7 +36,7 @@ fn get_env_var(field_name string) ?string {
 
 	// Otherwise, we process the file
 	return os.read_file(env_file) or {
-		error('Failed to read file defined in $env_file_name: ${err.msg}.')
+		error('Failed to read file defined in $env_file_name: ${err.msg()}.')
 	}
 }
 
@@ -55,27 +55,41 @@ pub fn load<T>(path string) ?T {
 		$for field in T.fields {
 			s := doc.value(field.name)
 
-			// We currently only support strings
+			if s !is toml.Null {
-			if s.type_name() == 'string' {
+				$if field.typ is string {
-				res.$(field.name) = s.string()
+					res.$(field.name) = s.string()
+				} $else $if field.typ is int {
+					res.$(field.name) = s.int()
+				}
 			}
 		}
 	}
 
 	$for field in T.fields {
-		$if field.typ is string {
+		env_value := get_env_var(field.name) ?
-			env_value := get_env_var(field.name) ?
 
-			// The value of the env var will always be chosen over the config
+		// The value of an env var will always take precedence over the toml
-			// file
+		// file.
-			if env_value != '' {
+		if env_value != '' {
+			$if field.typ is string {
 				res.$(field.name) = env_value
+			} $else $if field.typ is int {
+				res.$(field.name) = env_value.int()
 			}
-			// If there's no value from the toml file either, we try to find a
+		}
-			// default value
+
-			else if res.$(field.name) == '' {
+		// Now, we check whether a value is present. If there isn't, that means
-				return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.")
+		// it isn't in the config file, nor is there a default or an env var.
-			}
+		mut has_value := false
+
+		$if field.typ is string {
+			has_value = res.$(field.name) != ''
+		} $else $if field.typ is int {
+			has_value = res.$(field.name) != 0
+		}
+
+		if !has_value {
+			return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.")
 		}
 	}
 	return res

src/git/cli.v

@@ -96,6 +96,8 @@ pub fn cmd() cli.Command {
 	}
 }
 
+// get_repo_id_by_prefix tries to find the repo with the given prefix in its
+// ID. If multiple or none are found, an error is raised.
 fn get_repo_id_by_prefix(conf Config, id_prefix string) ?string {
 	repos := get_repos(conf.address, conf.api_key) ?
 
@@ -118,6 +120,7 @@ fn get_repo_id_by_prefix(conf Config, id_prefix string) ?string {
 	return res[0]
 }
 
+// list prints out a list of all repositories.
 fn list(conf Config) ? {
 	repos := get_repos(conf.address, conf.api_key) ?
 
@@ -126,12 +129,14 @@ fn list(conf Config) ? {
 	}
 }
 
+// add adds a new repository to the server's list.
 fn add(conf Config, url string, branch string, repo string, arch []string) ? {
 	res := add_repo(conf.address, conf.api_key, url, branch, repo, arch) ?
 
 	println(res.message)
 }
 
+// remove removes a repository from the server's list.
 fn remove(conf Config, id_prefix string) ? {
 	id := get_repo_id_by_prefix(conf, id_prefix) ?
 	res := remove_repo(conf.address, conf.api_key, id) ?
@@ -139,6 +144,7 @@ fn remove(conf Config, id_prefix string) ? {
 	println(res.message)
 }
 
+// patch patches a given repository with the provided params.
 fn patch(conf Config, id_prefix string, params map[string]string) ? {
 	id := get_repo_id_by_prefix(conf, id_prefix) ?
 	res := patch_repo(conf.address, conf.api_key, id, params) ?

src/git/client.v

@@ -4,6 +4,9 @@ import json
 import response { Response }
 import net.http
 
+// send_request<T> is a convenience method for sending requests to the repos
+// API. It mostly does string manipulation to create a query string containing
+// the provided params.
 fn send_request<T>(method http.Method, address string, url string, api_key string, params map[string]string) ?Response<T> {
 	mut full_url := '$address$url'
 

src/git/git.v

@@ -14,6 +14,8 @@ pub mut:
 	arch []string
 	// Which repo the builder should publish packages to
 	repo string
+	// Cron schedule describing how frequently to build the repo.
+	schedule string [optional]
 }
 
 // patch_from_params patches a GitRepo from a map[string]string, usually
@@ -72,7 +74,7 @@ pub fn repo_from_params(params map[string]string) ?GitRepo {
 	// If we're creating a new GitRepo, we want all fields to be present before
 	// "patching".
 	$for field in GitRepo.fields {
-		if field.name !in params {
+		if field.name !in params && !field.attrs.contains('optional') {
 			return error('Missing parameter: ${field.name}.')
 		}
 	}

src/package/package.v

@@ -175,6 +175,7 @@ pub fn read_pkg_archive(pkg_path string) ?Pkg {
 	}
 }
 
+// format_entry returns a string properly formatted to be added to a desc file.
 fn format_entry(key string, value string) string {
 	return '\n%$key%\n$value\n'
 }

src/repo/repo.v

@@ -30,11 +30,11 @@ pub:
 // new creates a new RepoGroupManager & creates the directories as needed
 pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupManager {
 	if !os.is_dir(repos_dir) {
-		os.mkdir_all(repos_dir) or { return error('Failed to create repos directory: $err.msg') }
+		os.mkdir_all(repos_dir) or { return error('Failed to create repos directory: $err.msg()') }
 	}
 
 	if !os.is_dir(pkg_dir) {
-		os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg') }
+		os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg()') }
 	}
 
 	return RepoGroupManager{
@@ -50,7 +50,7 @@ pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupMana
 // the right subdirectories in r.pkg_dir if it was successfully added.
 pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult {
 	pkg := package.read_pkg_archive(pkg_path) or {
-		return error('Failed to read package file: $err.msg')
+		return error('Failed to read package file: $err.msg()')
 	}
 
 	added := r.add_pkg_in_repo(repo, pkg) ?

src/repo/sync.v

@@ -2,6 +2,8 @@ module repo
 
 import os
 
+// archive_add_entry writes a file to an archive, given its path & inner path
+// inside the archive.
 fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &string, inner_path &string) {
 	st := C.stat{}
 
@@ -19,7 +21,7 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri
 	}
 
 	// Write the file to the archive
-	buf := [8192]byte{}
+	buf := [8192]u8{}
 	mut len := C.read(fd, &buf, sizeof(buf))
 
 	for len > 0 {
@@ -29,7 +31,7 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri
 	}
 }
 
-// Re-generate the repo archive files
+// sync regenerates the repository archive files.
 fn (r &RepoGroupManager) sync(repo string, arch string) ? {
 	subrepo_path := os.join_path(r.repos_dir, repo, arch)
 

src/server/auth.v

@@ -2,6 +2,7 @@ module server
 
 import net.http
 
+// is_authorized checks whether the provided API key is correct.
 fn (mut app App) is_authorized() bool {
 	x_header := app.req.header.get_custom('X-Api-Key', http.HeaderQueryConfig{ exact: true }) or {
 		return false

src/server/git.v

@@ -8,6 +8,7 @@ import response { new_data_response, new_response }
 
 const repos_file = 'repos.json'
 
+// get_repos returns the current list of repos.
 ['/api/repos'; get]
 fn (mut app App) get_repos() web.Result {
 	if !app.is_authorized() {
@@ -16,7 +17,7 @@ fn (mut app App) get_repos() web.Result {
 
 	repos := rlock app.git_mutex {
 		git.read_repos(app.conf.repos_file) or {
-			app.lerror('Failed to read repos file: $err.msg')
+			app.lerror('Failed to read repos file: $err.msg()')
 
 			return app.status(http.Status.internal_server_error)
 		}
@@ -25,6 +26,7 @@ fn (mut app App) get_repos() web.Result {
 	return app.json(http.Status.ok, new_data_response(repos))
 }
 
+// get_single_repo returns the information for a single repo.
 ['/api/repos/:id'; get]
 fn (mut app App) get_single_repo(id string) web.Result {
 	if !app.is_authorized() {
@@ -48,6 +50,7 @@ fn (mut app App) get_single_repo(id string) web.Result {
 	return app.json(http.Status.ok, new_data_response(repo))
 }
 
+// post_repo creates a new repo from the provided query string.
 ['/api/repos'; post]
 fn (mut app App) post_repo() web.Result {
 	if !app.is_authorized() {
@@ -55,7 +58,7 @@ fn (mut app App) post_repo() web.Result {
 	}
 
 	new_repo := git.repo_from_params(app.query) or {
-		return app.json(http.Status.bad_request, new_response(err.msg))
+		return app.json(http.Status.bad_request, new_response(err.msg()))
 	}
 
 	id := rand.uuid_v4()
@@ -86,6 +89,7 @@ fn (mut app App) post_repo() web.Result {
 	return app.json(http.Status.ok, new_response('Repo added successfully.'))
 }
 
+// delete_repo removes a given repo from the server's list.
 ['/api/repos/:id'; delete]
 fn (mut app App) delete_repo(id string) web.Result {
 	if !app.is_authorized() {
@@ -113,6 +117,7 @@ fn (mut app App) delete_repo(id string) web.Result {
 	return app.json(http.Status.ok, new_response('Repo removed successfully.'))
 }
 
+// patch_repo updates a repo's data with the given query params.
 ['/api/repos/:id'; patch]
 fn (mut app App) patch_repo(id string) web.Result {
 	if !app.is_authorized() {

src/server/routes.v

@@ -16,6 +16,9 @@ pub fn (mut app App) healthcheck() web.Result {
 	return app.json(http.Status.ok, new_response('Healthy.'))
 }
 
+// get_repo_file handles all Pacman-related routes. It returns both the
+// repository's archives, but also package archives or the contents of a
+// package's desc file.
 ['/:repo/:arch/:filename'; get; head]
 fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result {
 	mut full_path := ''
@@ -54,6 +57,7 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re
 	return app.file(full_path)
 }
 
+// put_package handles publishing a package to a repository.
 ['/:repo/publish'; post]
 fn (mut app App) put_package(repo string) web.Result {
 	if !app.is_authorized() {
@@ -87,15 +91,15 @@ fn (mut app App) put_package(repo string) web.Result {
 	}
 
 	res := app.repo.add_pkg_from_path(repo, pkg_path) or {
-		app.lerror('Error while adding package: $err.msg')
+		app.lerror('Error while adding package: $err.msg()')
 
-		os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }
+		os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg()") }
 
 		return app.json(http.Status.internal_server_error, new_response('Failed to add package.'))
 	}
 
 	if !res.added {
-		os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }
+		os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg()") }
 
 		app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo'.")
 

src/server/server.v

@@ -45,7 +45,7 @@ pub fn server(conf Config) ? {
 
 	// This also creates the directories if needed
 	repo := repo.new(conf.repos_dir, conf.pkg_dir, conf.default_arch) or {
-		logger.error(err.msg)
+		logger.error(err.msg())
 		exit(1)
 	}
 

src/util/util.v

@@ -30,7 +30,7 @@ pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? {
 		file.close()
 	}
 
-	mut buf := []byte{len: util.reader_buf_size}
+	mut buf := []u8{len: util.reader_buf_size}
 	mut bytes_left := length
 
 	// Repeat as long as the stream still has data
@@ -60,7 +60,7 @@ pub fn hash_file(path &string) ?(string, string) {
 	mut sha256sum := sha256.new()
 
 	buf_size := int(1_000_000)
-	mut buf := []byte{len: buf_size}
+	mut buf := []u8{len: buf_size}
 	mut bytes_left := os.file_size(path)
 
 	for bytes_left > 0 {

src/web/parse.v

@@ -47,6 +47,7 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) {
 	return methods, path.to_lower()
 }
 
+// Extracts query parameters from a URL.
 fn parse_query_from_url(url urllib.URL) map[string]string {
 	mut query := map[string]string{}
 	for v in url.query().data {
@@ -55,6 +56,7 @@ fn parse_query_from_url(url urllib.URL) map[string]string {
 	return query
 }
 
+// Extract form data from an HTTP request.
 fn parse_form_from_request(request http.Request) ?(map[string]string, map[string][]http.FileData) {
 	mut form := map[string]string{}
 	mut files := map[string][]http.FileData{}

src/web/web.v

@@ -249,7 +249,7 @@ pub fn (mut ctx Context) file(f_path string) Result {
 
 	// ext := os.file_ext(f_path)
 	// data := os.read_file(f_path) or {
-	// 	eprint(err.msg)
+	// 	eprint(err.msg())
 	// 	ctx.server_error(500)
 	// 	return Result{}
 	// }
@@ -267,7 +267,7 @@ pub fn (mut ctx Context) file(f_path string) Result {
 	file_size := os.file_size(f_path)
 
 	file := os.open(f_path) or {
-		eprintln(err.msg)
+		eprintln(err.msg())
 		ctx.server_error(500)
 		return Result{}
 	}
@@ -285,7 +285,7 @@ pub fn (mut ctx Context) file(f_path string) Result {
 	resp.set_status(ctx.status)
 	send_string(mut ctx.conn, resp.bytestr()) or { return Result{} }
 
-	mut buf := []byte{len: 1_000_000}
+	mut buf := []u8{len: 1_000_000}
 	mut bytes_left := file_size
 
 	// Repeat as long as the stream still has data
@@ -361,7 +361,7 @@ interface DbInterface {
 // run runs the app
 [manualfree]
 pub fn run<T>(global_app &T, port int) {
-	mut l := net.listen_tcp(.ip6, ':$port') or { panic('failed to listen $err.code $err') }
+	mut l := net.listen_tcp(.ip6, ':$port') or { panic('failed to listen $err.code() $err') }
 
 	// Parsing methods attributes
 	mut routes := map[string]Route{}
@@ -393,7 +393,7 @@ pub fn run<T>(global_app &T, port int) {
 		request_app.Context = global_app.Context // copy the context ref that contains static files map etc
 		mut conn := l.accept() or {
 			// failures should not panic
-			eprintln('accept() failed with error: $err.msg')
+			eprintln('accept() failed with error: $err.msg()')
 			continue
 		}
 		go handle_conn<T>(mut conn, mut request_app, routes)

vieter.toml

@@ -8,3 +8,9 @@ repos_file = "data/repos.json"
 default_arch = "x86_64"
 
 address = "http://localhost:8000"
+
+global_schedule = '* *'
+api_update_frequency = 2
+image_rebuild_frequency = 1
+max_concurrent_builds = 3
+