Merge pull request 'basic builds' (#84) from basic-builds into dev

Reviewed-on: Chewing_Bever/vieter#84

.editorconfig

@@ -7,4 +7,5 @@ end_of_line = lf
 insert_final_newline = true
 
 [*.v]
-indent_style = space
+# vfmt wants it :(
+indent_style = tab

CHANGELOG.md

@@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter)
 
+## Added
+
+* Very basic build system
+    * Build is triggered by separate cron container
+    * Packages build on cron container's system
+    * Packages are always rebuilt, even if they haven't changed
+    * Hardcoded planning of builds
+    * Builds are sequential
+
+## Fixed
+
+* Each package can now only have one version in the repository at once
+  (required by Pacman)
+
 ## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0)
 
 ### Changed

Dockerfile.ci

@@ -36,11 +36,14 @@ HEALTHCHECK --interval=30s \
     CMD /bin/wget --spider http://localhost:8000/health || exit 1
 
 RUN mkdir /data && \
-    chown -R www-data:www-data /data
+    chown -R www-data:www-data /data && \
+    mkdir -p '/var/spool/cron/crontabs' && \
+    echo '0 3 * * * /bin/vieter build' >> /var/spool/cron/crontabs/www-data && \
+    chown www-data:www-data /var/spool/cron/crontabs/www-data
 
 WORKDIR /data
 
 USER www-data:www-data
 
 ENTRYPOINT ["/bin/dumb-init", "--"]
-CMD ["/bin/vieter"]
+CMD ["/bin/vieter", "server"]

Makefile

@@ -34,7 +34,7 @@ c:
 # Run the server in the default 'data' directory
 .PHONY: run
 run: vieter
-	 API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter
+	 API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter server
 
 .PHONY: run-prod
 run-prod: prod

src/build.v

@@ -0,0 +1,84 @@
+module main
+
+import docker
+import encoding.base64
+import rand
+import time
+import os
+import json
+import git
+
+const container_build_dir = '/build'
+
+fn build(key string, repo_dir string) ? {
+	server_url := os.getenv_opt('VIETER_ADDRESS') or {
+		exit_with_message(1, 'No Vieter server address was provided.')
+	}
+
+	// Read in the repos from a json file
+	filename := os.join_path_single(repo_dir, 'repos.json')
+	txt := os.read_file(filename) ?
+	repos := json.decode([]git.GitRepo, txt) ?
+
+	mut commands := [
+		// Update repos & install required packages
+		'pacman -Syu --needed --noconfirm base-devel git'
+		// Add a non-root user to run makepkg
+		'groupadd -g 1000 builder',
+		'useradd -mg builder builder'
+		// Make sure they can use sudo without a password
+		"echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers"
+		// Create the directory for the builds & make it writeable for the
+		// build user
+		'mkdir /build',
+		'chown -R builder:builder /build',
+	]
+
+	// Each repo gets a unique UUID to avoid naming conflicts when cloning
+	mut uuids := []string{}
+
+	for repo in repos {
+		mut uuid := rand.uuid_v4()
+
+		// Just to be sure we don't have any collisions
+		for uuids.contains(uuid) {
+			uuid = rand.uuid_v4()
+		}
+
+		uuids << uuid
+
+		commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'"
+		commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $server_url/publish; done\''
+	}
+
+	// We convert the list of commands into a base64 string, which then gets
+	// passed to the container as an env var
+	cmds_str := base64.encode_str(commands.join('\n'))
+
+	c := docker.NewContainer{
+		image: 'archlinux:latest'
+		env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$key']
+		entrypoint: ['/bin/sh', '-c']
+		cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e']
+	}
+
+	// First, we pull the latest archlinux image
+	docker.pull_image('archlinux', 'latest') ?
+
+	id := docker.create_container(c) ?
+	docker.start_container(id) ?
+
+	// This loop waits until the container has stopped, so we can remove it after
+	for {
+		data := docker.inspect_container(id) ?
+
+		if !data.state.running {
+			break
+		}
+
+		// Wait for 5 seconds
+		time.sleep(5000000000)
+	}
+
+	docker.remove_container(id) ?
+}

src/docker/containers.v

@@ -0,0 +1,76 @@
+module docker
+
+import json
+import net.urllib
+
+struct Container {
+	id    string   [json: Id]
+	names []string [json: Names]
+}
+
+// containers returns a list of all currently running containers
+pub fn containers() ?[]Container {
+	res := request('GET', urllib.parse('/v1.41/containers/json') ?) ?
+
+	return json.decode([]Container, res.text) or {}
+}
+
+pub struct NewContainer {
+	image      string   [json: Image]
+	entrypoint []string [json: Entrypoint]
+	cmd        []string [json: Cmd]
+	env        []string [json: Env]
+}
+
+struct CreatedContainer {
+	id string [json: Id]
+}
+
+// create_container creates a container defined by the given configuration. If
+// successful, it returns the ID of the newly created container.
+pub fn create_container(c &NewContainer) ?string {
+	res := request_with_json('POST', urllib.parse('/v1.41/containers/create') ?, c) ?
+
+	if res.status_code != 201 {
+		return error('Failed to create container.')
+	}
+
+	return json.decode(CreatedContainer, res.text) ?.id
+}
+
+// start_container starts a container with a given ID. It returns whether the
+// container was started or not.
+pub fn start_container(id string) ?bool {
+	res := request('POST', urllib.parse('/v1.41/containers/$id/start') ?) ?
+
+	return res.status_code == 204
+}
+
+struct ContainerInspect {
+pub:
+	state ContainerState [json: State]
+}
+
+struct ContainerState {
+pub:
+	running bool [json: Running]
+}
+
+// inspect_container returns the result of inspecting a container with a given
+// ID.
+pub fn inspect_container(id string) ?ContainerInspect {
+	res := request('GET', urllib.parse('/v1.41/containers/$id/json') ?) ?
+
+	if res.status_code != 200 {
+		return error('Failed to inspect container.')
+	}
+
+	return json.decode(ContainerInspect, res.text) or {}
+}
+
+// remove_container removes a container with a given ID.
+pub fn remove_container(id string) ?bool {
+	res := request('DELETE', urllib.parse('/v1.41/containers/$id') ?) ?
+
+	return res.status_code == 204
+}

src/docker/docker.v

@@ -0,0 +1,98 @@
+module docker
+
+import net.unix
+import net.urllib
+import net.http
+import json
+
+const socket = '/var/run/docker.sock'
+
+const buf_len = 1024
+
+fn send(req &string) ?http.Response {
+	// Open a connection to the socket
+	mut s := unix.connect_stream(docker.socket) or {
+		return error('Failed to connect to socket ${docker.socket}.')
+	}
+
+	defer {
+		// This or is required because otherwise, the V compiler segfaults for
+		// some reason
+		// https://github.com/vlang/v/issues/13534
+		s.close() or {}
+	}
+
+	// Write the request to the socket
+	s.write_string(req) or { return error('Failed to write request to socket ${docker.socket}.') }
+
+	s.wait_for_write() ?
+
+	mut c := 0
+	mut buf := []byte{len: docker.buf_len}
+	mut res := []byte{}
+
+	for {
+		c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') }
+		res << buf[..c]
+
+		if c < docker.buf_len {
+			break
+		}
+	}
+
+	// After reading the first part of the response, we parse it into an HTTP
+	// response. If it isn't chunked, we return early with the data.
+	parsed := http.parse_response(res.bytestr()) or {
+		return error('Failed to parse HTTP response from socket ${docker.socket}.')
+	}
+
+	if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' {
+		return parsed
+	}
+
+	// We loop until we've encountered the end of the chunked response
+	// A chunked HTTP response always ends with '0\r\n\r\n'.
+	for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] {
+		// Wait for the server to respond
+		s.wait_for_write() ?
+
+		for {
+			c = s.read(mut buf) or {
+				return error('Failed to read data from socket ${docker.socket}.')
+			}
+			res << buf[..c]
+
+			if c < docker.buf_len {
+				break
+			}
+		}
+	}
+
+	// Decode chunked response
+	return http.parse_response(res.bytestr())
+}
+
+fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response {
+	req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n'
+
+	return send(req)
+}
+
+fn request(method string, url urllib.URL) ?http.Response {
+	req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n'
+
+	return send(req)
+}
+
+// request_with_json<T> sends a request to the Docker socket with a given JSON
+// payload
+pub fn request_with_json<T>(method string, url urllib.URL, data &T) ?http.Response {
+	body := json.encode(data)
+
+	return request_with_body(method, url, 'application/json', body)
+}
+
+// pull_image pulls tries to pull the image for the given image & tag
+pub fn pull_image(image string, tag string) ?http.Response {
+	return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?)
+}

src/git.v

@@ -0,0 +1,7 @@
+module git
+
+pub struct GitRepo {
+pub:
+	url    string [required]
+	branch string [required]
+}

src/main.v

@@ -2,7 +2,6 @@ module main
 
 import web
 import os
-import log
 import io
 import repo
 
@@ -10,8 +9,6 @@ const port = 8000
 
 const buf_size = 1_000_000
 
-const db_name = 'pieter.db'
-
 struct App {
 	web.Context
 pub:
@@ -54,50 +51,18 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? {
 }
 
 fn main() {
-	// Configure logger
-	log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' }
-	log_level := log.level_from_tag(log_level_str) or {
-		exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
-	}
-	log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' }
-
-	mut logger := log.Log{
-		level: log_level
-	}
-
-	logger.set_full_logpath(log_file)
-	logger.log_to_console_too()
-
-	defer {
-		logger.info('Flushing log file')
-		logger.flush()
-		logger.close()
-	}
-
-	// Configure web server
 	key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') }
 	repo_dir := os.getenv_opt('REPO_DIR') or {
 		exit_with_message(1, 'No repo directory was configured.')
 	}
-	pkg_dir := os.getenv_opt('PKG_DIR') or {
+
-		exit_with_message(1, 'No package directory was configured.')
+	if os.args.len == 1 {
-	}
+		exit_with_message(1, 'No action provided.')
-	dl_dir := os.getenv_opt('DOWNLOAD_DIR') or {
-		exit_with_message(1, 'No download directory was configured.')
 	}
 
-	// This also creates the directories if needed
+	match os.args[1] {
-	repo := repo.new(repo_dir, pkg_dir) or {
+		'server' { server(key, repo_dir) }
-		logger.error(err.msg)
+		'build' { build(key, repo_dir) ? }
-		exit(1)
+		else { exit_with_message(1, 'Unknown action: ${os.args[1]}') }
 	}
-
-	os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') }
-
-	web.run(&App{
-		logger: logger
-		api_key: key
-		dl_dir: dl_dir
-		repo: repo
-	}, port)
 }

src/server.v

@@ -0,0 +1,51 @@
+module main
+
+import web
+import os
+import log
+import repo
+
+fn server(key string, repo_dir string) {
+	// Configure logger
+	log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' }
+	log_level := log.level_from_tag(log_level_str) or {
+		exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
+	}
+	log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' }
+
+	mut logger := log.Log{
+		level: log_level
+	}
+
+	logger.set_full_logpath(log_file)
+	logger.log_to_console_too()
+
+	defer {
+		logger.info('Flushing log file')
+		logger.flush()
+		logger.close()
+	}
+
+	// Configure web server
+	pkg_dir := os.getenv_opt('PKG_DIR') or {
+		exit_with_message(1, 'No package directory was configured.')
+	}
+	dl_dir := os.getenv_opt('DOWNLOAD_DIR') or {
+		exit_with_message(1, 'No download directory was configured.')
+	}
+
+	// This also creates the directories if needed
+	repo := repo.new(repo_dir, pkg_dir) or {
+		logger.error(err.msg)
+		exit(1)
+	}
+
+	os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') }
+
+	web.run(&App{
+		logger: logger
+		api_key: key
+		dl_dir: dl_dir
+		repo: repo
+	}, port)
+}