Merge pull request 'Release 0.3.0-alpha.2' (#185) from release-0.3.0-alpha.2 into main

Reviewed-on: vieter/vieter#185

.woodpecker/.arch-rel.yml

@@ -0,0 +1,39 @@
+matrix:
+  PLATFORM:
+    - linux/amd64
+    - linux/arm64
+
+platform: ${PLATFORM}
+branches: [main]
+skip_clone: true
+
+pipeline:
+  build:
+    image: 'menci/archlinuxarm:base-devel'
+    commands:
+      # Add the vieter repository so we can use the compiler
+      - echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
+      # Update packages
+      - pacman -Syu --noconfirm
+      # Create non-root user to perform build & switch to their home
+      - groupadd -g 1000 builder
+      - useradd -mg builder builder
+      - chown -R builder:builder "$PWD"
+      - "echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers"
+      - su builder
+      # Due to a bug with the V compiler, we can't just use the PKGBUILD from
+      # inside the repo
+      - curl -OL "https://git.rustybever.be/vieter/vieter/raw/tag/$CI_COMMIT_TAG/PKGBUILD"
+      - makepkg -s --noconfirm --needed
+    when:
+      event: tag
+
+  publish:
+    image: 'curlimages/curl'
+    commands:
+      # Publish the package
+      - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done'
+    secrets:
+      - vieter_api_key
+    when:
+      event: tag

.woodpecker/.arch.yml

@@ -23,7 +23,7 @@ pipeline:
       - su builder
       # Due to a bug with the V compiler, we can't just use the PKGBUILD from
       # inside the repo
-      - curl -OL https://git.rustybever.be/vieter/vieter/raw/branch/dev/PKGBUILD
+      - curl -o PKGBUILD -L https://git.rustybever.be/vieter/vieter/raw/branch/dev/PKGBUILD.dev
       - makepkg -s --noconfirm --needed
     when:
       event: push

.woodpecker/.docker.yml

@@ -11,7 +11,9 @@ pipeline:
       - 'docker_password'
     settings:
       repo: 'chewingbever/vieter'
-      tag: 'dev'
+      tags:
+        - 'dev'
+        - ${CI_COMMIT_SHA}
       platforms: [ 'linux/arm64/v8', 'linux/amd64' ]
       build_args_from_env:
         - 'CI_COMMIT_SHA'

CHANGELOG.md

@@ -5,6 +5,33 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev)
+
+## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2)
+
+### Added
+
+* Web API for adding & querying build logs
+* CLI commands to access build logs API
+* Cron build logs are uploaded to above API
+* Proper ASCII table output in CLI
+* `vieter repos build id` command to run builds locally
+
+### Removed
+
+* `vieter build` command
+    * This command was used alongside cron for periodic builds, but this has
+      been replaced by `vieter cron`
+
+### Changed
+
+* `vieter build` command now only builds a single repository & uploads the
+  build logs
+* Official Arch packages are now split between `vieter` & `vieter-git`
+    * `vieter` is the latest release
+    * `vieter-git` is the latest commit on the dev branch
+* Full refactor of Docker socket code
+
 ## [0.3.0-alpha.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.1)
 
 ### Changed

Dockerfile

@@ -36,15 +36,8 @@ ENV PATH=/bin \
 
 COPY --from=builder /app/dumb-init /app/vieter /bin/
 
-HEALTHCHECK --interval=30s \
-    --timeout=3s \
-    --start-period=5s \
-    CMD /bin/wget --spider http://localhost:8000/health || exit 1
-
 RUN mkdir /data && \
-    chown -R www-data:www-data /data && \
+    chown -R www-data:www-data /data
-    mkdir -p '/var/spool/cron/crontabs' && \
-    echo '0 3 * * * /bin/vieter build' | crontab - 
 
 WORKDIR /data
 

PKGBUILD

@@ -1,23 +1,18 @@
+# vim: ft=bash
 # Maintainer: Jef Roosens
 
 pkgbase='vieter'
 pkgname='vieter'
-pkgver=0.2.0.r25.g20112b8
+pkgver='0.3.0_alpha.2'
 pkgrel=1
-depends=('glibc' 'openssl' 'libarchive' 'gc' 'sqlite')
+depends=('glibc' 'openssl' 'libarchive' 'sqlite')
-makedepends=('git' 'gcc' 'vieter-v')
+makedepends=('git' 'vieter-v')
 arch=('x86_64' 'aarch64')
 url='https://git.rustybever.be/vieter/vieter'
 license=('AGPL3')
-source=($pkgname::git+https://git.rustybever.be/vieter/vieter#branch=dev)
+source=("$pkgname::git+https://git.rustybever.be/vieter/vieter#tag=${pkgver//_/-}")
 md5sums=('SKIP')
 
-pkgver() {
-    cd "$pkgname"
-
-    git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g'
-}
-
 build() {
     cd "$pkgname"
 
@@ -28,5 +23,5 @@ package() {
     pkgdesc="Vieter is a lightweight implementation of an Arch repository server."
 
     install -dm755 "$pkgdir/usr/bin"
-    install -Dm755 "$pkgbase/pvieter" "$pkgdir/usr/bin/vieter"
+    install -Dm755 "$pkgname/pvieter" "$pkgdir/usr/bin/vieter"
 }

PKGBUILD.dev

@@ -0,0 +1,35 @@
+# vim: ft=bash
+# Maintainer: Jef Roosens
+
+pkgbase='vieter-git'
+pkgname='vieter-git'
+pkgver=0.2.0.r25.g20112b8
+pkgrel=1
+depends=('glibc' 'openssl' 'libarchive' 'sqlite')
+makedepends=('git' 'vieter-v')
+arch=('x86_64' 'aarch64')
+url='https://git.rustybever.be/vieter/vieter'
+license=('AGPL3')
+source=("$pkgname::git+https://git.rustybever.be/vieter/vieter#branch=dev")
+md5sums=('SKIP')
+provides=('vieter')
+conflicts=('vieter')
+
+pkgver() {
+    cd "$pkgname"
+
+    git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g'
+}
+
+build() {
+    cd "$pkgname"
+
+    make prod
+}
+
+package() {
+    pkgdesc="Vieter is a lightweight implementation of an Arch repository server."
+
+    install -dm755 "$pkgdir/usr/bin"
+    install -Dm755 "$pkgname/pvieter" "$pkgdir/usr/bin/vieter"
+}

README.md

@@ -55,3 +55,13 @@ clone my compiler in the `v` directory & build it. Afterwards, you can use this
 compiler with make by prepending all make commands with `V_PATH=v/v`. If you do
 encounter this issue, please let me know so I can update my mirror & the
 codebase to fix it!
+
+## Contributing
+
+If you wish to contribute to the project, please take note of the following:
+
+* Rebase instead of merging whenever possible, e.g. when updating your branch
+  with the dev branch.
+* Please follow the
+  [Conventional Commits](https://www.conventionalcommits.org/) style for your
+  commit messages.

docs/content/api.md

@@ -56,6 +56,11 @@ Vieter only supports uploading archives compressed using either gzip, zstd or
 xz at the moment.
 {{< /hint >}}
 
+### `GET /health`
+
+This endpoint's only use is to be used with healthchecks. It returns a JSON
+response with the message "Healthy.".
+
 ## API
 
 All API routes require the API key to provided using the `X-Api-Key` header.

docs/content/builder.md

@@ -15,7 +15,7 @@ repositories. After the image has been created, each repository returned by
 previously created image as a base. Each container goes through the following steps:
 
 1. The repository is cloned
-2. `makepkg --nobuild --nodeps` is ran to update the `pkgver` variable inside
+2. `makepkg --nobuild --syncdeps --needed --noconfirm` is ran to update the `pkgver` variable inside
    the `PKGBUILD` file
 3. A HEAD request is sent to the Vieter server to check whether the specific
    version of the package is already present. If it is, the container exits.

src/build/build.v

@@ -3,13 +3,15 @@ module build
 import docker
 import encoding.base64
 import time
-import git
 import os
 import db
+import strings
+import util
 
-const container_build_dir = '/build'
+const (
-
+	container_build_dir = '/build'
-const build_image_repo = 'vieter-build'
+	build_image_repo    = 'vieter-build'
+)
 
 // create_build_image creates a builder image given some base image which can
 // then be used to build & package Arch images. It mostly just updates the
@@ -17,6 +19,12 @@ const build_image_repo = 'vieter-build'
 // makepkg with. The base image should be some Linux distribution that uses
 // Pacman as its package manager.
 pub fn create_build_image(base_image string) ?string {
+	mut dd := docker.new_conn()?
+
+	defer {
+		dd.close() or {}
+	}
+
 	commands := [
 		// Update repos & install required packages
 		'pacman -Syu --needed --noconfirm base-devel git'
@@ -46,14 +54,15 @@ pub fn create_build_image(base_image string) ?string {
 	image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' }
 
 	// We pull the provided image
-	docker.pull_image(image_name, image_tag) ?
+	dd.pull_image(image_name, image_tag)?
 
-	id := docker.create_container(c) ?
+	id := dd.create_container(c)?.id
-	docker.start_container(id) ?
+	// id := docker.create_container(c)?
+	dd.start_container(id)?
 
 	// This loop waits until the container has stopped, so we can remove it after
 	for {
-		data := docker.inspect_container(id) ?
+		data := dd.inspect_container(id)?
 
 		if !data.state.running {
 			break
@@ -67,27 +76,41 @@ pub fn create_build_image(base_image string) ?string {
 	// TODO also add the base image's name into the image name to prevent
 	// conflicts.
 	tag := time.sys_mono_now().str()
-	image := docker.create_image_from_container(id, 'vieter-build', tag) ?
+	image := dd.create_image_from_container(id, 'vieter-build', tag)?
-	docker.remove_container(id) ?
+	dd.remove_container(id)?
 
 	return image.id
 }
 
+pub struct BuildResult {
+pub:
+	start_time time.Time
+	end_time   time.Time
+	exit_code  int
+	logs       string
+}
+
 // build_repo builds, packages & publishes a given Arch package based on the
 // provided GitRepo. The base image ID should be of an image previously created
-// by create_build_image.
+// by create_build_image. It returns the logs of the container.
-pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ? {
+pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ?BuildResult {
+	mut dd := docker.new_conn()?
+
+	defer {
+		dd.close() or {}
+	}
+
 	build_arch := os.uname().machine
 
 	// TODO what to do with PKGBUILDs that build multiple packages?
 	commands := [
 		'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
 		'cd repo',
-		'makepkg --nobuild --nodeps',
+		'makepkg --nobuild --syncdeps --needed --noconfirm',
 		'source PKGBUILD',
 		// The build container checks whether the package is already
 		// present on the server
-		'curl --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
+		'curl -s --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
 		'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done',
 	]
 
@@ -104,46 +127,30 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db
 		user: 'builder:builder'
 	}
 
-	id := docker.create_container(c) ?
+	id := dd.create_container(c)?.id
-	docker.start_container(id) ?
+	dd.start_container(id)?
+
+	mut data := dd.inspect_container(id)?
 
 	// This loop waits until the container has stopped, so we can remove it after
-	for {
+	for data.state.running {
-		data := docker.inspect_container(id) ?
-
-		if !data.state.running {
-			break
-		}
-
 		time.sleep(1 * time.second)
+
+		data = dd.inspect_container(id)?
 	}
 
-	docker.remove_container(id) ?
+	mut logs_stream := dd.get_container_logs(id)?
+
+	// Read in the entire stream
+	mut logs_builder := strings.new_builder(10 * 1024)
+	util.reader_to_writer(mut logs_stream, mut logs_builder)?
+
+	dd.remove_container(id)?
+
+	return BuildResult{
+		start_time: data.state.start_time
+		end_time: data.state.end_time
+		exit_code: data.state.exit_code
+		logs: logs_builder.str()
 	}
-
-// build builds every Git repo in the server's list.
-fn build(conf Config) ? {
-	build_arch := os.uname().machine
-
-	// We get the repos map from the Vieter instance
-	repos := git.get_repos(conf.address, conf.api_key) ?
-
-	// We filter out any repos that aren't allowed to be built on this
-	// architecture
-	filtered_repos := repos.filter(it.arch.map(it.value).contains(build_arch))
-
-	// No point in doing work if there's no repos present
-	if filtered_repos.len == 0 {
-		return
-	}
-
-	// First, we create a base image which has updated repos n stuff
-	image_id := create_build_image(conf.base_image) ?
-
-	for repo in filtered_repos {
-		build_repo(conf.address, conf.api_key, image_id, repo) ?
-	}
-
-	// Finally, we remove the builder image
-	docker.remove_image(image_id) ?
 }

src/build/cli.v

@@ -1,25 +0,0 @@
-module build
-
-import cli
-import env
-
-pub struct Config {
-pub:
-	api_key    string
-	address    string
-	base_image string = 'archlinux:base-devel'
-}
-
-// cmd returns the cli submodule that handles the build process
-pub fn cmd() cli.Command {
-	return cli.Command{
-		name: 'build'
-		description: 'Run the build process.'
-		execute: fn (cmd cli.Command) ? {
-			config_file := cmd.flags.get_string('config-file') ?
-			conf := env.load<Config>(config_file) ?
-
-			build(conf) ?
-		}
-	}
-}

src/client/client.v

@@ -0,0 +1,67 @@
+module client
+
+import net.http { Method }
+import net.urllib
+import response { Response }
+import json
+
+pub struct Client {
+pub:
+	address string
+	api_key string
+}
+
+// new creates a new Client instance.
+pub fn new(address string, api_key string) Client {
+	return Client{
+		address: address
+		api_key: api_key
+	}
+}
+
+// send_request_raw sends an HTTP request, returning the http.Response object.
+// It encodes the params so that they're safe to pass as HTTP query parameters.
+fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) ?http.Response {
+	mut full_url := '$c.address$url'
+
+	if params.len > 0 {
+		mut params_escaped := map[string]string{}
+
+		// Escape each query param
+		for k, v in params {
+			params_escaped[k] = urllib.query_escape(v)
+		}
+
+		params_str := params_escaped.keys().map('$it=${params[it]}').join('&')
+
+		full_url = '$full_url?$params_str'
+	}
+
+	mut req := http.new_request(method, full_url, body)?
+	req.add_custom_header('X-Api-Key', c.api_key)?
+
+	res := req.do()?
+
+	return res
+}
+
+// send_request<T> just calls send_request_with_body<T> with an empty body.
+fn (c &Client) send_request<T>(method Method, url string, params map[string]string) ?Response<T> {
+	return c.send_request_with_body<T>(method, url, params, '')
+}
+
+// send_request_with_body<T> calls send_request_raw_response & parses its
+// output as a Response<T> object.
+fn (c &Client) send_request_with_body<T>(method Method, url string, params map[string]string, body string) ?Response<T> {
+	res_text := c.send_request_raw_response(method, url, params, body)?
+	data := json.decode(Response<T>, res_text)?
+
+	return data
+}
+
+// send_request_raw_response returns the raw text response for an HTTP request.
+fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) ?string {
+	res := c.send_request_raw(method, url, params, body)?
+
+	return res.text
+}

src/client/git.v

@@ -0,0 +1,51 @@
+module client
+
+import db { GitRepo }
+import net.http { Method }
+import response { Response }
+
+// get_git_repos returns the current list of repos.
+pub fn (c &Client) get_git_repos() ?[]GitRepo {
+	data := c.send_request<[]GitRepo>(Method.get, '/api/repos', {})?
+
+	return data.data
+}
+
+// get_git_repo returns the repo for a specific ID.
+pub fn (c &Client) get_git_repo(id int) ?GitRepo {
+	data := c.send_request<GitRepo>(Method.get, '/api/repos/$id', {})?
+
+	return data.data
+}
+
+// add_git_repo adds a new repo to the server.
+pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []string) ?Response<string> {
+	mut params := {
+		'url':    url
+		'branch': branch
+		'repo':   repo
+	}
+
+	if arch.len > 0 {
+		params['arch'] = arch.join(',')
+	}
+
+	data := c.send_request<string>(Method.post, '/api/repos', params)?
+
+	return data
+}
+
+// remove_git_repo removes the repo with the given ID from the server.
+pub fn (c &Client) remove_git_repo(id int) ?Response<string> {
+	data := c.send_request<string>(Method.delete, '/api/repos/$id', {})?
+
+	return data
+}
+
+// patch_git_repo sends a PATCH request to the given repo with the params as
+// payload.
+pub fn (c &Client) patch_git_repo(id int, params map[string]string) ?Response<string> {
+	data := c.send_request<string>(Method.patch, '/api/repos/$id', params)?
+
+	return data
+}

src/client/logs.v

@@ -0,0 +1,53 @@
+module client
+
+import db { BuildLog }
+import net.http { Method }
+import response { Response }
+import time
+
+// get_build_logs returns all build logs.
+pub fn (c &Client) get_build_logs() ?Response<[]BuildLog> {
+	data := c.send_request<[]BuildLog>(Method.get, '/api/logs', {})?
+
+	return data
+}
+
+// get_build_logs_for_repo returns all build logs for a given repo.
+pub fn (c &Client) get_build_logs_for_repo(repo_id int) ?Response<[]BuildLog> {
+	params := {
+		'repo': repo_id.str()
+	}
+
+	data := c.send_request<[]BuildLog>(Method.get, '/api/logs', params)?
+
+	return data
+}
+
+// get_build_log returns a specific build log.
+pub fn (c &Client) get_build_log(id int) ?Response<BuildLog> {
+	data := c.send_request<BuildLog>(Method.get, '/api/logs/$id', {})?
+
+	return data
+}
+
+// get_build_log_content returns the contents of the build log file.
+pub fn (c &Client) get_build_log_content(id int) ?string {
+	data := c.send_request_raw_response(Method.get, '/api/logs/$id/content', {}, '')?
+
+	return data
+}
+
+// add_build_log adds a new build log to the server.
+pub fn (c &Client) add_build_log(repo_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response<string> {
+	params := {
+		'repo':      repo_id.str()
+		'startTime': start_time.str()
+		'endTime':   end_time.str()
+		'arch':      arch
+		'exitCode':  exit_code.str()
+	}
+
+	data := c.send_request_with_body<string>(Method.post, '/api/logs', params, content)?
+
+	return data
+}

src/console/console.v

@@ -0,0 +1,56 @@
+module console
+
+import arrays
+import strings
+
+// pretty_table converts a list of string data into a pretty table. Many thanks
+// to @hungrybluedev in the Vlang Discord for providing this code!
+// https://ptb.discord.com/channels/592103645835821068/592106336838352923/970278787143045192
+pub fn pretty_table(header []string, data [][]string) ?string {
+	column_count := header.len
+
+	mut column_widths := []int{len: column_count, init: header[it].len}
+
+	for values in data {
+		for col, value in values {
+			if column_widths[col] < value.len {
+				column_widths[col] = value.len
+			}
+		}
+	}
+
+	single_line_length := arrays.sum(column_widths)? + (column_count + 1) * 3 - 4
+
+	horizontal_line := '+' + strings.repeat(`-`, single_line_length) + '+'
+	mut buffer := strings.new_builder(data.len * single_line_length)
+
+	buffer.writeln(horizontal_line)
+
+	buffer.write_string('| ')
+	for col, head in header {
+		if col != 0 {
+			buffer.write_string(' | ')
+		}
+		buffer.write_string(head)
+		buffer.write_string(strings.repeat(` `, column_widths[col] - head.len))
+	}
+	buffer.writeln(' |')
+
+	buffer.writeln(horizontal_line)
+
+	for values in data {
+		buffer.write_string('| ')
+		for col, value in values {
+			if col != 0 {
+				buffer.write_string(' | ')
+			}
+			buffer.write_string(value)
+			buffer.write_string(strings.repeat(` `, column_widths[col] - value.len))
+		}
+		buffer.writeln(' |')
+	}
+
+	buffer.writeln(horizontal_line)
+
+	return buffer.str()
+}

src/console/git/build.v

@@ -0,0 +1,34 @@
+module git
+
+import client
+import docker
+import os
+import build
+
+// build builds every Git repo in the server's list.
+fn build(conf Config, repo_id int) ? {
+	c := client.new(conf.address, conf.api_key)
+	repo := c.get_git_repo(repo_id)?
+
+	build_arch := os.uname().machine
+
+	println('Creating base image...')
+	image_id := build.create_build_image(conf.base_image)?
+
+	println('Running build...')
+	res := build.build_repo(conf.address, conf.api_key, image_id, repo)?
+
+	println('Removing build image...')
+
+	mut dd := docker.new_conn()?
+
+	defer {
+		dd.close() or {}
+	}
+
+	dd.remove_image(image_id)?
+
+	println('Uploading logs to Vieter...')
+	c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code,
+		res.logs)?
+}

src/console/git/git.v

@@ -3,10 +3,13 @@ module git
 import cli
 import env
 import cron.expression { parse_expression }
+import client
+import console
 
 struct Config {
 	address    string [required]
 	api_key    string [required]
+	base_image string = 'archlinux:base-devel'
 }
 
 // cmd returns the cli submodule that handles the repos API interaction
@@ -110,6 +113,18 @@ pub fn cmd() cli.Command {
 					patch(conf, cmd.args[0], params)?
 				}
 			},
+			cli.Command{
+				name: 'build'
+				required_args: 1
+				usage: 'id'
+				description: 'Build the repo with the given id & publish it.'
+				execute: fn (cmd cli.Command) ? {
+					config_file := cmd.flags.get_string('config-file')?
+					conf := env.load<Config>(config_file)?
+
+					build(conf, cmd.args[0].int())?
+				}
+			},
 		]
 	}
 }
@@ -119,16 +134,17 @@ pub fn cmd() cli.Command {
 
 // list prints out a list of all repositories.
 fn list(conf Config) ? {
-	repos := get_repos(conf.address, conf.api_key) ?
+	c := client.new(conf.address, conf.api_key)
+	repos := c.get_git_repos()?
+	data := repos.map([it.id.str(), it.url, it.branch, it.repo])
 
-	for repo in repos {
+	println(console.pretty_table(['id', 'url', 'branch', 'repo'], data)?)
-		println('$repo.id\t$repo.url\t$repo.branch\t$repo.repo')
-	}
 }
 
 // add adds a new repository to the server's list.
 fn add(conf Config, url string, branch string, repo string) ? {
-	res := add_repo(conf.address, conf.api_key, url, branch, repo, []) ?
+	c := client.new(conf.address, conf.api_key)
+	res := c.add_git_repo(url, branch, repo, [])?
 
 	println(res.message)
 }
@@ -139,7 +155,8 @@ fn remove(conf Config, id string) ? {
 	id_int := id.int()
 
 	if id_int != 0 {
-		res := remove_repo(conf.address, conf.api_key, id_int) ?
+		c := client.new(conf.address, conf.api_key)
+		res := c.remove_git_repo(id_int)?
 		println(res.message)
 	}
 }
@@ -156,7 +173,8 @@ fn patch(conf Config, id string, params map[string]string) ? {
 
 	id_int := id.int()
 	if id_int != 0 {
-		res := patch_repo(conf.address, conf.api_key, id_int, params) ?
+		c := client.new(conf.address, conf.api_key)
+		res := c.patch_git_repo(id_int, params)?
 
 		println(res.message)
 	}
@@ -170,6 +188,7 @@ fn info(conf Config, id string) ? {
 		return
 	}
 
-	repo := get_repo(conf.address, conf.api_key, id_int) ?
+	c := client.new(conf.address, conf.api_key)
+	repo := c.get_git_repo(id_int)?
 	println(repo)
 }

src/console/logs/logs.v

@@ -0,0 +1,108 @@
+module logs
+
+import cli
+import env
+import client
+import db
+import console
+
+struct Config {
+	address string [required]
+	api_key string [required]
+}
+
+// cmd returns the cli module that handles the build repos API.
+pub fn cmd() cli.Command {
+	return cli.Command{
+		name: 'logs'
+		description: 'Interact with the build logs API.'
+		commands: [
+			cli.Command{
+				name: 'list'
+				description: 'List the build logs. If a repo ID is provided, only list the build logs for that repo.'
+				flags: [
+					cli.Flag{
+						name: 'repo'
+						description: 'ID of the Git repo to restrict list to.'
+						flag: cli.FlagType.int
+					},
+				]
+				execute: fn (cmd cli.Command) ? {
+					config_file := cmd.flags.get_string('config-file')?
+					conf := env.load<Config>(config_file)?
+
+					repo_id := cmd.flags.get_int('repo')?
+
+					if repo_id == 0 { list(conf)? } else { list_for_repo(conf, repo_id)? }
+				}
+			},
+			cli.Command{
+				name: 'info'
+				required_args: 1
+				usage: 'id'
+				description: 'Show all info for a specific build log.'
+				execute: fn (cmd cli.Command) ? {
+					config_file := cmd.flags.get_string('config-file')?
+					conf := env.load<Config>(config_file)?
+
+					id := cmd.args[0].int()
+					info(conf, id)?
+				}
+			},
+			cli.Command{
+				name: 'content'
+				required_args: 1
+				usage: 'id'
+				description: 'Output the content of a build log to stdout.'
+				execute: fn (cmd cli.Command) ? {
+					config_file := cmd.flags.get_string('config-file')?
+					conf := env.load<Config>(config_file)?
+
+					id := cmd.args[0].int()
+					content(conf, id)?
+				}
+			},
+		]
+	}
+}
+
+// print_log_list prints a list of logs.
+fn print_log_list(logs []db.BuildLog) ? {
+	data := logs.map([it.id.str(), it.repo_id.str(), it.start_time.str(),
+		it.exit_code.str()])
+
+	println(console.pretty_table(['id', 'repo', 'start time', 'exit code'], data)?)
+}
+
+// list prints a list of all build logs.
+fn list(conf Config) ? {
+	c := client.new(conf.address, conf.api_key)
+	logs := c.get_build_logs()?.data
+
+	print_log_list(logs)?
+}
+
+// list prints a list of all build logs for a given repo.
+fn list_for_repo(conf Config, repo_id int) ? {
+	c := client.new(conf.address, conf.api_key)
+	logs := c.get_build_logs_for_repo(repo_id)?.data
+
+	print_log_list(logs)?
+}
+
+// info print the detailed info for a given build log.
+fn info(conf Config, id int) ? {
+	c := client.new(conf.address, conf.api_key)
+	log := c.get_build_log(id)?.data
+
+	print(log)
+}
+
+// content outputs the contents of the log file for a given build log to
+// stdout.
+fn content(conf Config, id int) ? {
+	c := client.new(conf.address, conf.api_key)
+	content := c.get_build_log_content(id)?
+
+	println(content)
+}

src/cron/daemon/build.v

@@ -3,6 +3,7 @@ module daemon
 import time
 import sync.stdatomic
 import build
+import os
 
 const (
 	build_empty   = 0
@@ -77,13 +78,20 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) {
 	// 0 means success, 1 means failure
 	mut status := 0
 
-	build.build_repo(d.address, d.api_key, d.builder_images.last(), &sb.repo) or {
+	res := build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(),
+		&sb.repo) or {
 		d.ldebug('build_repo error: $err.msg()')
 		status = 1
+
+		build.BuildResult{}
 	}
 
 	if status == 0 {
-		d.linfo('finished build: $sb.repo.url $sb.repo.branch')
+		d.linfo('finished build: $sb.repo.url $sb.repo.branch; uploading logs...')
+
+		build_arch := os.uname().machine
+		d.client.add_build_log(sb.repo.id, res.start_time, res.end_time, build_arch, res.exit_code,
+			res.logs) or { d.lerror('Failed to upload logs for $sb.repo.url $sb.repo.arch') }
 	} else {
 		d.linfo('failed build: $sb.repo.url $sb.repo.branch')
 	}

src/cron/daemon/daemon.v

@@ -1,6 +1,5 @@
 module daemon
 
-import git
 import time
 import log
 import datatypes { MinHeap }
@@ -10,6 +9,7 @@ import build
 import docker
 import db
 import os
+import client
 
 const (
 	// How many seconds to wait before retrying to update API if failed
@@ -31,8 +31,7 @@ fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool {
 
 pub struct Daemon {
 mut:
-	address                 string
+	client                  client.Client
-	api_key                 string
 	base_image              string
 	builder_images          []string
 	global_schedule         CronExpression
@@ -56,8 +55,7 @@ mut:
 // populates the build queue for the first time.
 pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon {
 	mut d := Daemon{
-		address: address
+		client: client.new(address, api_key)
-		api_key: api_key
 		base_image: base_image
 		global_schedule: global_schedule
 		api_update_frequency: api_update_frequency
@@ -180,7 +178,7 @@ fn (mut d Daemon) schedule_build(repo db.GitRepo) {
 fn (mut d Daemon) renew_repos() {
 	d.linfo('Renewing repos...')
 
-	mut new_repos := git.get_repos(d.address, d.api_key) or {
+	mut new_repos := d.client.get_git_repos() or {
 		d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...')
 		d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout)
 
@@ -255,14 +253,21 @@ fn (mut d Daemon) rebuild_base_image() bool {
 fn (mut d Daemon) clean_old_base_images() {
 	mut i := 0
 
+	mut dd := docker.new_conn() or {
+		d.lerror('Failed to connect to Docker socket.')
+		return
+	}
+
+	defer {
+		dd.close() or {}
+	}
+
 	for i < d.builder_images.len - 1 {
 		// For each builder image, we try to remove it by calling the Docker
 		// API. If the function returns an error or false, that means the image
 		// wasn't deleted. Therefore, we move the index over. If the function
 		// returns true, the array's length has decreased by one so we don't
 		// move the index.
-		if !docker.remove_image(d.builder_images[i]) or { false } {
+		dd.remove_image(d.builder_images[i]) or { i += 1 }
-			i += 1
-		}
 	}
 }

src/db/db.v

@@ -12,6 +12,7 @@ pub fn init(db_path string) ?VieterDb {
 
 	sql conn {
 		create table GitRepo
+		create table BuildLog
 	}
 
 	return VieterDb{

src/db/git.v

@@ -94,7 +94,7 @@ pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo {
 	// If a select statement fails, it returns a zeroed object. By
 	// checking one of the required fields, we can see whether the query
 	// returned a result or not.
-	if res.url == '' {
+	if res.id == 0 {
 		return none
 	}
 
@@ -152,3 +152,11 @@ pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) {
 		}
 	}
 }
+
+// git_repo_exists is a utility function that checks whether a repo with the
+// given id exists.
+pub fn (db &VieterDb) git_repo_exists(repo_id int) bool {
+	db.get_git_repo(repo_id) or { return false }
+
+	return true
+}

src/db/logs.v

@@ -0,0 +1,74 @@
+module db
+
+import time
+
+pub struct BuildLog {
+pub:
+	id         int       [primary; sql: serial]
+	repo_id    int       [nonull]
+	start_time time.Time [nonull]
+	end_time   time.Time [nonull]
+	arch       string    [nonull]
+	exit_code  int       [nonull]
+}
+
+// str returns a string representation.
+pub fn (bl &BuildLog) str() string {
+	mut parts := [
+		'id: $bl.id',
+		'repo id: $bl.repo_id',
+		'start time: $bl.start_time',
+		'end time: $bl.end_time',
+		'arch: $bl.arch',
+		'exit code: $bl.exit_code',
+	]
+	str := parts.join('\n')
+
+	return str
+}
+
+// get_build_logs returns all BuildLog's in the database.
+pub fn (db &VieterDb) get_build_logs() []BuildLog {
+	res := sql db.conn {
+		select from BuildLog order by id
+	}
+
+	return res
+}
+
+// get_build_logs_for_repo returns all BuildLog's in the database for a given
+// repo.
+pub fn (db &VieterDb) get_build_logs_for_repo(repo_id int) []BuildLog {
+	res := sql db.conn {
+		select from BuildLog where repo_id == repo_id order by id
+	}
+
+	return res
+}
+
+// get_build_log tries to return a specific BuildLog.
+pub fn (db &VieterDb) get_build_log(id int) ?BuildLog {
+	res := sql db.conn {
+		select from BuildLog where id == id
+	}
+
+	if res.id == 0 {
+		return none
+	}
+
+	return res
+}
+
+// add_build_log inserts the given BuildLog into the database.
+pub fn (db &VieterDb) add_build_log(log BuildLog) {
+	sql db.conn {
+		insert log into BuildLog
+	}
+}
+
+// delete_build_log delete the BuildLog with the given ID from the database.
+pub fn (db &VieterDb) delete_build_log(id int) {
+	sql db.conn {
+		delete from BuildLog where id == id
+	}
+}

src/docker/README.md

@@ -0,0 +1,3 @@
+This module implements part of the Docker Engine API v1.41
+([documentation](https://docs.docker.com/engine/api/v1.41/)) using socket-based
+HTTP communication.

src/docker/containers.v

@@ -2,17 +2,11 @@ module docker
 
 import json
 import net.urllib
+import time
+import net.http { Method }
 
-struct Container {
+struct DockerError {
-	id    string   [json: Id]
+	message string
-	names []string [json: Names]
-}
-
-// containers returns a list of all currently running containers
-pub fn containers() ?[]Container {
-	res := request('GET', urllib.parse('/v1.41/containers/json') ?) ?
-
-	return json.decode([]Container, res.text) or {}
 }
 
 pub struct NewContainer {
@@ -25,54 +19,104 @@ pub struct NewContainer {
 }
 
 struct CreatedContainer {
+pub:
 	id       string   [json: Id]
+	warnings []string [json: Warnings]
 }
 
-// create_container creates a container defined by the given configuration. If
+// create_container creates a new container with the given config.
-// successful, it returns the ID of the newly created container.
+pub fn (mut d DockerConn) create_container(c NewContainer) ?CreatedContainer {
-pub fn create_container(c &NewContainer) ?string {
+	d.send_request_with_json(Method.post, urllib.parse('/v1.41/containers/create')?, c)?
-	res := request_with_json('POST', urllib.parse('/v1.41/containers/create') ?, c) ?
+	head, res := d.read_response()?
 
-	if res.status_code != 201 {
+	if head.status_code != 201 {
-		return error('Failed to create container.')
+		data := json.decode(DockerError, res)?
+
+		return error(data.message)
 	}
 
-	return json.decode(CreatedContainer, res.text) ?.id
+	data := json.decode(CreatedContainer, res)?
+
+	return data
 }
 
-// start_container starts a container with a given ID. It returns whether the
+// start_container starts the container with the given id.
-// container was started or not.
+pub fn (mut d DockerConn) start_container(id string) ? {
-pub fn start_container(id string) ?bool {
+	d.send_request(Method.post, urllib.parse('/v1.41/containers/$id/start')?)?
-	res := request('POST', urllib.parse('/v1.41/containers/$id/start') ?) ?
+	head, body := d.read_response()?
 
-	return res.status_code == 204
+	if head.status_code != 204 {
+		data := json.decode(DockerError, body)?
+
+		return error(data.message)
+	}
 }
 
 struct ContainerInspect {
-pub:
+pub mut:
 	state ContainerState [json: State]
 }
 
 struct ContainerState {
 pub:
 	running   bool   [json: Running]
+	status    string [json: Status]
+	exit_code int    [json: ExitCode]
+	// These use a rather specific format so they have to be parsed later
+	start_time_str string [json: StartedAt]
+	end_time_str   string [json: FinishedAt]
+pub mut:
+	start_time time.Time [skip]
+	end_time   time.Time [skip]
 }
 
-// inspect_container returns the result of inspecting a container with a given
+// inspect_container returns detailed information for a given container.
-// ID.
+pub fn (mut d DockerConn) inspect_container(id string) ?ContainerInspect {
-pub fn inspect_container(id string) ?ContainerInspect {
+	d.send_request(Method.get, urllib.parse('/v1.41/containers/$id/json')?)?
-	res := request('GET', urllib.parse('/v1.41/containers/$id/json') ?) ?
+	head, body := d.read_response()?
 
-	if res.status_code != 200 {
+	if head.status_code != 200 {
-		return error('Failed to inspect container.')
+		data := json.decode(DockerError, body)?
+
+		return error(data.message)
 	}
 
-	return json.decode(ContainerInspect, res.text) or {}
+	mut data := json.decode(ContainerInspect, body)?
+
+	data.state.start_time = time.parse_rfc3339(data.state.start_time_str)?
+
+	if data.state.status == 'exited' {
+		data.state.end_time = time.parse_rfc3339(data.state.end_time_str)?
 	}
 
-// remove_container removes a container with a given ID.
+	return data
-pub fn remove_container(id string) ?bool {
+}
-	res := request('DELETE', urllib.parse('/v1.41/containers/$id') ?) ?
+
-
+// remove_container removes the container with the given id.
-	return res.status_code == 204
+pub fn (mut d DockerConn) remove_container(id string) ? {
+	d.send_request(Method.delete, urllib.parse('/v1.41/containers/$id')?)?
+	head, body := d.read_response()?
+
+	if head.status_code != 204 {
+		data := json.decode(DockerError, body)?
+
+		return error(data.message)
+	}
+}
+
+// get_container_logs returns a reader object allowing access to the
+// container's logs.
+pub fn (mut d DockerConn) get_container_logs(id string) ?&StreamFormatReader {
+	d.send_request(Method.get, urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true')?)?
+	head := d.read_response_head()?
+
+	if head.status_code != 200 {
+		content_length := head.header.get(http.CommonHeader.content_length)?.int()
+		body := d.read_response_body(content_length)?
+		data := json.decode(DockerError, body)?
+
+		return error(data.message)
+	}
+
+	return d.get_stream_format_reader()
 }

src/docker/docker.v

@@ -1,97 +1,137 @@
 module docker
 
 import net.unix
-import net.urllib
+import io
 import net.http
+import strings
+import net.urllib
 import json
+import util
 
-const socket = '/var/run/docker.sock'
+const (
+	socket               = '/var/run/docker.sock'
+	buf_len              = 10 * 1024
+	http_separator       = [u8(`\r`), `\n`, `\r`, `\n`]
+	http_chunk_separator = [u8(`\r`), `\n`]
+)
 
-const buf_len = 1024
+pub struct DockerConn {
-
+mut:
-// send writes a request to the Docker socket, waits for a response & returns
+	socket &unix.StreamConn
-// it.
+	reader &io.BufferedReader
-fn send(req &string) ?http.Response {
-	// Open a connection to the socket
-	mut s := unix.connect_stream(docker.socket) or {
-		return error('Failed to connect to socket ${docker.socket}.')
 }
 
-	defer {
+// new_conn creates a new connection to the Docker daemon.
-		// This or is required because otherwise, the V compiler segfaults for
+pub fn new_conn() ?&DockerConn {
-		// some reason
+	s := unix.connect_stream(docker.socket)?
-		// https://github.com/vlang/v/issues/13534
+
-		s.close() or {}
+	d := &DockerConn{
+		socket: s
+		reader: io.new_buffered_reader(reader: s)
 	}
 
-	// Write the request to the socket
+	return d
-	s.write_string(req) or { return error('Failed to write request to socket ${docker.socket}.') }
+}
 
-	s.wait_for_write() ?
+// close closes the underlying socket connection.
+pub fn (mut d DockerConn) close() ? {
+	d.socket.close()?
+}
 
-	mut c := 0
+// send_request sends an HTTP request without body.
-	mut buf := []u8{len: docker.buf_len}
+pub fn (mut d DockerConn) send_request(method http.Method, url urllib.URL) ? {
+	req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n'
+
+	d.socket.write_string(req)?
+
+	// When starting a new request, the reader needs to be reset.
+	d.reader = io.new_buffered_reader(reader: d.socket)
+}
+
+// send_request_with_body sends an HTTP request with the given body.
+pub fn (mut d DockerConn) send_request_with_body(method http.Method, url urllib.URL, content_type string, body string) ? {
+	req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n'
+
+	d.socket.write_string(req)?
+
+	// When starting a new request, the reader needs to be reset.
+	d.reader = io.new_buffered_reader(reader: d.socket)
+}
+
+// send_request_with_json<T> is a convenience wrapper around
+// send_request_with_body that encodes the input as JSON.
+pub fn (mut d DockerConn) send_request_with_json<T>(method http.Method, url urllib.URL, data &T) ? {
+	body := json.encode(data)
+
+	return d.send_request_with_body(method, url, 'application/json', body)
+}
+
+// read_response_head consumes the socket's contents until it encounters
+// '\r\n\r\n', after which it parses the response as an HTTP response.
+// Importantly, this function never consumes the reader past the HTTP
+// separator, so the body can be read fully later on.
+pub fn (mut d DockerConn) read_response_head() ?http.Response {
 	mut res := []u8{}
 
-	for {
+	util.read_until_separator(mut d.reader, mut res, docker.http_separator)?
-		c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') }
-		res << buf[..c]
 
-		if c < docker.buf_len {
-			break
-		}
-	}
-
-	// After reading the first part of the response, we parse it into an HTTP
-	// response. If it isn't chunked, we return early with the data.
-	parsed := http.parse_response(res.bytestr()) or {
-		return error('Failed to parse HTTP response from socket ${docker.socket}.')
-	}
-
-	if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' {
-		return parsed
-	}
-
-	// We loop until we've encountered the end of the chunked response
-	// A chunked HTTP response always ends with '0\r\n\r\n'.
-	for res.len < 5 || res#[-5..] != [u8(`0`), `\r`, `\n`, `\r`, `\n`] {
-		// Wait for the server to respond
-		s.wait_for_write() ?
-
-		for {
-			c = s.read(mut buf) or {
-				return error('Failed to read data from socket ${docker.socket}.')
-			}
-			res << buf[..c]
-
-			if c < docker.buf_len {
-				break
-			}
-		}
-	}
-
-	// Decode chunked response
 	return http.parse_response(res.bytestr())
 }
 
-// request_with_body sends a request to the Docker socket with the given body.
+// read_response_body reads `length` bytes from the stream. It can be used when
-fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response {
+// the response encoding isn't chunked to fully read it.
-	req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n'
+pub fn (mut d DockerConn) read_response_body(length int) ?string {
-
+	if length == 0 {
-	return send(req)
+		return ''
 	}
 
-// request sends a request to the Docker socket with an empty body.
+	mut buf := []u8{len: docker.buf_len}
-fn request(method string, url urllib.URL) ?http.Response {
+	mut c := 0
-	req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n'
+	mut builder := strings.new_builder(docker.buf_len)
 
-	return send(req)
+	for builder.len < length {
+		c = d.reader.read(mut buf) or { break }
+
+		builder.write(buf[..c])?
 	}
 
-// request_with_json<T> sends a request to the Docker socket with a given JSON
+	return builder.str()
-// payload
+}
-pub fn request_with_json<T>(method string, url urllib.URL, data &T) ?http.Response {
+
-	body := json.encode(data)
+// read_response is a convenience function which always consumes the entire
-
+// response & returns it. It should only be used when we're certain that the
-	return request_with_body(method, url, 'application/json', body)
+// result isn't too large.
+pub fn (mut d DockerConn) read_response() ?(http.Response, string) {
+	head := d.read_response_head()?
+
+	if head.header.get(http.CommonHeader.transfer_encoding) or { '' } == 'chunked' {
+		mut builder := strings.new_builder(1024)
+		mut body := d.get_chunked_response_reader()
+
+		util.reader_to_writer(mut body, mut builder)?
+
+		return head, builder.str()
+	}
+
+	content_length := head.header.get(http.CommonHeader.content_length)?.int()
+	res := d.read_response_body(content_length)?
+
+	return head, res
+}
+
+// get_chunked_response_reader returns a ChunkedResponseReader using the socket
+// as reader.
+pub fn (mut d DockerConn) get_chunked_response_reader() &ChunkedResponseReader {
+	r := new_chunked_response_reader(d.reader)
+
+	return r
+}
+
+// get_stream_format_reader returns a StreamFormatReader using the socket as
+// reader.
+pub fn (mut d DockerConn) get_stream_format_reader() &StreamFormatReader {
+	r := new_chunked_response_reader(d.reader)
+	r2 := new_stream_format_reader(r)
+
+	return r2
 }

src/docker/images.v

@@ -1,6 +1,6 @@
 module docker
 
-import net.http
+import net.http { Method }
 import net.urllib
 import json
 
@@ -9,26 +9,53 @@ pub:
 	id string [json: Id]
 }
 
-// pull_image pulls tries to pull the image for the given image & tag
+// pull_image pulls the given image:tag.
-pub fn pull_image(image string, tag string) ?http.Response {
+pub fn (mut d DockerConn) pull_image(image string, tag string) ? {
-	return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?)
+	d.send_request(Method.post, urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?)?
+	head := d.read_response_head()?
+
+	if head.status_code != 200 {
+		content_length := head.header.get(http.CommonHeader.content_length)?.int()
+		body := d.read_response_body(content_length)?
+		data := json.decode(DockerError, body)?
+
+		return error(data.message)
 	}
 
-// create_image_from_container creates a new image from a container with the
+	// Keep reading the body until the pull has completed
-// given repo & tag, given the container's ID.
+	mut body := d.get_chunked_response_reader()
-pub fn create_image_from_container(id string, repo string, tag string) ?Image {
-	res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag') ?) ?
 
-	if res.status_code != 201 {
+	mut buf := []u8{len: 1024}
-		return error('Failed to create image from container.')
+
+	for {
+		body.read(mut buf) or { break }
+	}
 }
 
-	return json.decode(Image, res.text) or {}
+// create_image_from_container creates a new image from a container.
+pub fn (mut d DockerConn) create_image_from_container(id string, repo string, tag string) ?Image {
+	d.send_request(Method.post, urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag')?)?
+	head, body := d.read_response()?
+
+	if head.status_code != 201 {
+		data := json.decode(DockerError, body)?
+
+		return error(data.message)
 	}
 
-// remove_image removes the image with the given ID.
+	data := json.decode(Image, body)?
-pub fn remove_image(id string) ?bool {
-	res := request('DELETE', urllib.parse('/v1.41/images/$id') ?) ?
 
-	return res.status_code == 200
+	return data
+}
+
+// remove_image removes the image with the given id.
+pub fn (mut d DockerConn) remove_image(id string) ? {
+	d.send_request(Method.delete, urllib.parse('/v1.41/images/$id')?)?
+	head, body := d.read_response()?
+
+	if head.status_code != 200 {
+		data := json.decode(DockerError, body)?
+
+		return error(data.message)
+	}
 }

src/docker/stream.v

@@ -0,0 +1,135 @@
+module docker
+
+import io
+import util
+import encoding.binary
+import encoding.hex
+
+// ChunkedResponseReader parses an underlying HTTP chunked response, exposing
+// it as if it was a continuous stream of data.
+struct ChunkedResponseReader {
+mut:
+	reader              io.BufferedReader
+	bytes_left_in_chunk u64
+	started             bool
+}
+
+// new_chunked_response_reader creates a new ChunkedResponseReader on the heap
+// with the provided reader.
+pub fn new_chunked_response_reader(reader io.BufferedReader) &ChunkedResponseReader {
+	r := &ChunkedResponseReader{
+		reader: reader
+	}
+
+	return r
+}
+
+// read satisfies the io.Reader interface.
+pub fn (mut r ChunkedResponseReader) read(mut buf []u8) ?int {
+	if r.bytes_left_in_chunk == 0 {
+		// An io.BufferedReader always returns none if its stream has
+		// ended.
+		r.bytes_left_in_chunk = r.read_chunk_size()?
+	}
+
+	mut c := 0
+
+	// Make sure we don't read more than we can safely read. This is to avoid
+	// the underlying reader from becoming out of sync with our parsing:
+	if buf.len > r.bytes_left_in_chunk {
+		c = r.reader.read(mut buf[..r.bytes_left_in_chunk])?
+	} else {
+		c = r.reader.read(mut buf)?
+	}
+
+	r.bytes_left_in_chunk -= u64(c)
+
+	return c
+}
+
+// read_chunk_size advances the reader & reads the size of the next HTTP chunk.
+// This function should only be called if the previous chunk has been
+// completely consumed.
+fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 {
+	if r.started {
+		mut buf := []u8{len: 2}
+
+		// Each chunk ends with a `\r\n` which we want to skip first
+		r.reader.read(mut buf)?
+	}
+
+	r.started = true
+
+	mut res := []u8{}
+	util.read_until_separator(mut r.reader, mut res, http_chunk_separator)?
+
+	// The length of the next chunk is provided as a hexadecimal
+	mut num_data := hex.decode(res#[..-2].bytestr())?
+
+	for num_data.len < 8 {
+		num_data.insert(0, 0)
+	}
+
+	num := binary.big_endian_u64(num_data)
+
+	// This only occurs for the very last chunk, which always reports a size of
+	// 0.
+	if num == 0 {
+		return none
+	}
+
+	return num
+}
+
+// StreamFormatReader parses an underlying stream of Docker logs, removing the
+// header bytes.
+struct StreamFormatReader {
+mut:
+	reader              ChunkedResponseReader
+	bytes_left_in_chunk u32
+}
+
+// new_stream_format_reader creates a new StreamFormatReader using the given
+// reader.
+pub fn new_stream_format_reader(reader ChunkedResponseReader) &StreamFormatReader {
+	r := &StreamFormatReader{
+		reader: reader
+	}
+
+	return r
+}
+
+// read satisfies the io.Reader interface.
+pub fn (mut r StreamFormatReader) read(mut buf []u8) ?int {
+	if r.bytes_left_in_chunk == 0 {
+		r.bytes_left_in_chunk = r.read_chunk_size()?
+	}
+
+	mut c := 0
+
+	if buf.len > r.bytes_left_in_chunk {
+		c = r.reader.read(mut buf[..r.bytes_left_in_chunk])?
+	} else {
+		c = r.reader.read(mut buf)?
+	}
+
+	r.bytes_left_in_chunk -= u32(c)
+
+	return c
+}
+
+// read_chunk_size advances the reader & reads the header bytes for the length
+// of the next chunk.
+fn (mut r StreamFormatReader) read_chunk_size() ?u32 {
+	mut buf := []u8{len: 8}
+
+	r.reader.read(mut buf)?
+
+	num := binary.big_endian_u32(buf[4..])
+
+	if num == 0 {
+		return none
+	}
+
+	return num
+}

src/env/README.md

@@ -0,0 +1,7 @@
+This module provides a framework for parsing a configuration, defined as a
+struct, from both a TOML configuration file & environment variables. Some
+notable features are:
+
+* Overwrite values in config file using environment variables
+* Allow default values in config struct
+* Read environment variable value from file

src/env/env.v

@@ -3,13 +3,19 @@ module env
 import os
 import toml
 
+const (
 	// The prefix that every environment variable should have
-const prefix = 'VIETER_'
+	prefix      = 'VIETER_'
-
 	// The suffix an environment variable in order for it to be loaded from a file
 	// instead
-const file_suffix = '_FILE'
+	file_suffix = '_FILE'
+)
 
+// get_env_var tries to read the contents of the given environment variable. It
+// looks for either `${env.prefix}${field_name.to_upper()}` or
+// `${env.prefix}${field_name.to_upper()}${env.file_suffix}`, returning the
+// contents of the file instead if the latter. If both or neither exist, the
+// function returns an error.
 fn get_env_var(field_name string) ?string {
 	env_var_name := '$env.prefix$field_name.to_upper()'
 	env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix'

src/git/client.v

@@ -1,77 +0,0 @@
-module git
-
-import json
-import response { Response }
-import net.http
-import db
-
-// send_request<T> is a convenience method for sending requests to the repos
-// API. It mostly does string manipulation to create a query string containing
-// the provided params.
-fn send_request<T>(method http.Method, address string, url string, api_key string, params map[string]string) ?Response<T> {
-	mut full_url := '$address$url'
-
-	if params.len > 0 {
-		params_str := params.keys().map('$it=${params[it]}').join('&')
-
-		full_url = '$full_url?$params_str'
-	}
-
-	mut req := http.new_request(method, full_url, '') ?
-	req.add_custom_header('X-API-Key', api_key) ?
-
-	res := req.do() ?
-	data := json.decode(Response<T>, res.text) ?
-
-	return data
-}
-
-// get_repos returns the current list of repos.
-pub fn get_repos(address string, api_key string) ?[]db.GitRepo {
-	data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', api_key,
-		{}) ?
-
-	return data.data
-}
-
-// get_repo returns the repo for a specific ID.
-pub fn get_repo(address string, api_key string, id int) ?db.GitRepo {
-	data := send_request<db.GitRepo>(http.Method.get, address, '/api/repos/$id', api_key,
-		{}) ?
-
-	return data.data
-}
-
-// add_repo adds a new repo to the server.
-pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response<string> {
-	mut params := {
-		'url':    url
-		'branch': branch
-		'repo':   repo
-	}
-
-	if arch.len > 0 {
-		params['arch'] = arch.join(',')
-	}
-
-	data := send_request<string>(http.Method.post, address, '/api/repos', api_key, params) ?
-
-	return data
-}
-
-// remove_repo removes the repo with the given ID from the server.
-pub fn remove_repo(address string, api_key string, id int) ?Response<string> {
-	data := send_request<string>(http.Method.delete, address, '/api/repos/$id', api_key,
-		{}) ?
-
-	return data
-}
-
-// patch_repo sends a PATCH request to the given repo with the params as
-// payload.
-pub fn patch_repo(address string, api_key string, id int, params map[string]string) ?Response<string> {
-	data := send_request<string>(http.Method.patch, address, '/api/repos/$id', api_key,
-		params) ?
-
-	return data
-}

src/main.v

@@ -3,15 +3,15 @@ module main
 import os
 import server
 import cli
-import build
+import console.git
-import git
+import console.logs
 import cron
 
 fn main() {
 	mut app := cli.Command{
 		name: 'vieter'
 		description: 'Vieter is a lightweight implementation of an Arch repository server.'
-		version: '0.3.0-alpha.1'
+		version: '0.3.0-alpha.2'
 		flags: [
 			cli.Flag{
 				flag: cli.FlagType.string
@@ -24,12 +24,12 @@ fn main() {
 		]
 		commands: [
 			server.cmd(),
-			build.cmd(),
 			git.cmd(),
 			cron.cmd(),
+			logs.cmd(),
 		]
 	}
-
 	app.setup()
 	app.parse(os.args)
+	return
 }

src/server/logs.v

@@ -0,0 +1,136 @@
+module server
+
+import web
+import net.http
+import net.urllib
+import response { new_data_response, new_response }
+import db
+import time
+import os
+import util
+
+// get_logs returns all build logs in the database. A 'repo' query param can
+// optionally be added to limit the list of build logs to that repository.
+['/api/logs'; get]
+fn (mut app App) get_logs() web.Result {
+	if !app.is_authorized() {
+		return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
+	}
+
+	logs := if 'repo' in app.query {
+		app.db.get_build_logs_for_repo(app.query['repo'].int())
+	} else {
+		app.db.get_build_logs()
+	}
+
+	return app.json(http.Status.ok, new_data_response(logs))
+}
+
+// get_single_log returns the build log with the given id.
+['/api/logs/:id'; get]
+fn (mut app App) get_single_log(id int) web.Result {
+	if !app.is_authorized() {
+		return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
+	}
+
+	log := app.db.get_build_log(id) or { return app.not_found() }
+
+	return app.json(http.Status.ok, new_data_response(log))
+}
+
+// get_log_content returns the actual build log file for the given id.
+['/api/logs/:id/content'; get]
+fn (mut app App) get_log_content(id int) web.Result {
+	if !app.is_authorized() {
+		return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
+	}
+
+	log := app.db.get_build_log(id) or { return app.not_found() }
+	file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
+	full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.repo_id.str(), log.arch,
+		file_name)
+
+	return app.file(full_path)
+}
+
+// parse_query_time unescapes an HTTP query parameter & tries to parse it as a
+// time.Time struct.
+fn parse_query_time(query string) ?time.Time {
+	unescaped := urllib.query_unescape(query)?
+	t := time.parse(unescaped)?
+
+	return t
+}
+
+// post_log adds a new log to the database.
+['/api/logs'; post]
+fn (mut app App) post_log() web.Result {
+	if !app.is_authorized() {
+		return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
+	}
+
+	// Parse query params
+	start_time := parse_query_time(app.query['startTime']) or {
+		return app.json(http.Status.bad_request, new_response('Invalid or missing start time.'))
+	}
+
+	end_time := parse_query_time(app.query['endTime']) or {
+		return app.json(http.Status.bad_request, new_response('Invalid or missing end time.'))
+	}
+
+	if 'exitCode' !in app.query {
+		return app.json(http.Status.bad_request, new_response('Missing exit code.'))
+	}
+
+	exit_code := app.query['exitCode'].int()
+
+	if 'arch' !in app.query {
+		return app.json(http.Status.bad_request, new_response("Missing parameter 'arch'."))
+	}
+
+	arch := app.query['arch']
+
+	repo_id := app.query['repo'].int()
+
+	if !app.db.git_repo_exists(repo_id) {
+		return app.json(http.Status.bad_request, new_response('Unknown Git repo.'))
+	}
+
+	// Store log in db
+	log := db.BuildLog{
+		repo_id: repo_id
+		start_time: start_time
+		end_time: end_time
+		arch: arch
+		exit_code: exit_code
+	}
+
+	app.db.add_build_log(log)
+
+	repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo_id.str(), arch)
+
+	// Create the logs directory of it doesn't exist
+	if !os.exists(repo_logs_dir) {
+		os.mkdir_all(repo_logs_dir) or {
+			app.lerror("Couldn't create dir '$repo_logs_dir'.")
+
+			return app.json(http.Status.internal_server_error, new_response('An error occured while processing the request.'))
+		}
+	}
+
+	// Stream log contents to correct file
+	file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
+	full_path := os.join_path_single(repo_logs_dir, file_name)
+
+	if length := app.req.header.get(.content_length) {
+		util.reader_to_file(mut app.reader, length.int(), full_path) or {
+			app.lerror('An error occured while receiving logs: $err.msg()')
+
+			return app.json(http.Status.internal_server_error, new_response('Failed to upload logs.'))
+		}
+	} else {
+		return app.status(http.Status.length_required)
+	}
+
+	return app.json(http.Status.ok, new_response('Logs added successfully.'))
+}

src/server/server.v

@@ -12,6 +12,7 @@ const (
 	log_file_name = 'vieter.log'
 	repo_dir_name = 'repos'
 	db_file_name  = 'vieter.sqlite'
+	logs_dir_name = 'logs'
 )
 
 struct App {
@@ -37,6 +38,14 @@ pub fn server(conf Config) ? {
 
 	os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') }
 
+	logs_dir := os.join_path_single(conf.data_dir, server.logs_dir_name)
+
+	if !os.exists(logs_dir) {
+		os.mkdir(os.join_path_single(conf.data_dir, server.logs_dir_name)) or {
+			util.exit_with_message(1, 'Failed to create logs directory.')
+		}
+	}
+
 	mut logger := log.Log{
 		level: log_level
 	}

src/util/README.md

@@ -0,0 +1,2 @@
+This module defines a few useful functions used throughout the codebase that
+don't specifically fit inside a module.

src/util/stream.v

@@ -0,0 +1,95 @@
+// Functions for interacting with `io.Reader` & `io.Writer` objects.
+module util
+
+import io
+import os
+
+// reader_to_writer tries to consume the entire reader & write it to the writer.
+pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ? {
+	mut buf := []u8{len: 10 * 1024}
+
+	for {
+		bytes_read := reader.read(mut buf) or { break }
+		mut bytes_written := 0
+
+		for bytes_written < bytes_read {
+			c := writer.write(buf[bytes_written..bytes_read]) or { break }
+
+			bytes_written += c
+		}
+	}
+}
+
+// reader_to_file writes the contents of a BufferedReader to a file
+pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? {
+	mut file := os.create(path)?
+	defer {
+		file.close()
+	}
+
+	mut buf := []u8{len: reader_buf_size}
+	mut bytes_left := length
+
+	// Repeat as long as the stream still has data
+	for bytes_left > 0 {
+		// TODO check if just breaking here is safe
+		bytes_read := reader.read(mut buf) or { break }
+		bytes_left -= bytes_read
+
+		mut to_write := bytes_read
+
+		for to_write > 0 {
+			// TODO don't just loop infinitely here
+			bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue }
+			// file.flush()
+
+			to_write = to_write - bytes_written
+		}
+	}
+}
+
+// match_array_in_array<T> returns how many elements of a2 overlap with a1. For
+// example, if a1 = "abcd" & a2 = "cd", the result will be 2. If the match is
+// not at the end of a1, the result is 0.
+pub fn match_array_in_array<T>(a1 []T, a2 []T) int {
+	mut i := 0
+	mut match_len := 0
+
+	for i + match_len < a1.len {
+		if a1[i + match_len] == a2[match_len] {
+			match_len += 1
+		} else {
+			i += match_len + 1
+			match_len = 0
+		}
+	}
+
+	return match_len
+}
+
+// read_until_separator consumes an io.Reader until it encounters some
+// separator array. The data read is stored inside the provided res array.
+pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ? {
+	mut buf := []u8{len: sep.len}
+
+	for {
+		c := reader.read(mut buf)?
+		res << buf[..c]
+
+		match_len := match_array_in_array(buf[..c], sep)
+
+		if match_len == sep.len {
+			break
+		}
+
+		if match_len > 0 {
+			match_left := sep.len - match_len
+			c2 := reader.read(mut buf[..match_left])?
+			res << buf[..c2]
+
+			if buf[..c2] == sep[match_len..] {
+				break
+			}
+		}
+	}
+}

src/util/util.v

@@ -1,13 +1,13 @@
 module util
 
 import os
-import io
 import crypto.md5
 import crypto.sha256
 
-const reader_buf_size = 1_000_000
+const (
-
+	reader_buf_size = 1_000_000
-const prefixes = ['B', 'KB', 'MB', 'GB']
+	prefixes        = ['B', 'KB', 'MB', 'GB']
+)
 
 // Dummy struct to work around the fact that you can only share structs, maps &
 // arrays
@@ -23,34 +23,6 @@ pub fn exit_with_message(code int, msg string) {
 	exit(code)
 }
 
-// reader_to_file writes the contents of a BufferedReader to a file
-pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? {
-	mut file := os.create(path) ?
-	defer {
-		file.close()
-	}
-
-	mut buf := []u8{len: util.reader_buf_size}
-	mut bytes_left := length
-
-	// Repeat as long as the stream still has data
-	for bytes_left > 0 {
-		// TODO check if just breaking here is safe
-		bytes_read := reader.read(mut buf) or { break }
-		bytes_left -= bytes_read
-
-		mut to_write := bytes_read
-
-		for to_write > 0 {
-			// TODO don't just loop infinitely here
-			bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue }
-			// file.flush()
-
-			to_write = to_write - bytes_written
-		}
-	}
-}
-
 // hash_file returns the md5 & sha256 hash of a given file
 // TODO actually implement sha256
 pub fn hash_file(path &string) ?(string, string) {