From 6f86033cd9aa5cfe875e91a98809a5ac54c96c44 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 17 Feb 2022 22:00:46 +0100 Subject: [PATCH 01/13] Currently broken start of docker wrapper [CI SKIP] --- Makefile | 2 +- src/build.v | 7 ++++++ src/docker/containers.v | 15 +++++++++++ src/docker/docker.v | 48 +++++++++++++++++++++++++++++++++++ src/main.v | 54 ++++++---------------------------------- src/server.v | 55 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 134 insertions(+), 47 deletions(-) create mode 100644 src/build.v create mode 100644 src/docker/containers.v create mode 100644 src/docker/docker.v create mode 100644 src/server.v diff --git a/Makefile b/Makefile index 7166c66..63d772f 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ c: # Run the server in the default 'data' directory .PHONY: run run: vieter - API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter server .PHONY: run-prod run-prod: prod diff --git a/src/build.v b/src/build.v new file mode 100644 index 0000000..e635cc0 --- /dev/null +++ b/src/build.v @@ -0,0 +1,7 @@ +module main + +import docker + +fn build() { + println(docker.containers() or { panic("yeet") }) +} diff --git a/src/docker/containers.v b/src/docker/containers.v new file mode 100644 index 0000000..5bdd39e --- /dev/null +++ b/src/docker/containers.v @@ -0,0 +1,15 @@ +module docker + +import json +import net.urllib + +struct Container { + id string + names []string +} + +pub fn containers() ?[]Container { + res := docker.get(urllib.parse('/containers/json') ?) ? + + return json.decode([]Container, res.text) +} diff --git a/src/docker/docker.v b/src/docker/docker.v new file mode 100644 index 0000000..a090f02 --- /dev/null +++ b/src/docker/docker.v @@ -0,0 +1,48 @@ +module docker + +import net.unix +import net.urllib +import net.http + +const socket = '/var/run/docker.sock' +const buf_len = 1024 + +fn request(method string, url urllib.URL) ?http.Response { + req := "$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n" + + // Open a connection to the socket + mut s := unix.connect_stream(socket) ? + + defer { + s.close() ? + } + + // Write the request to the socket + s.write_string(req) ? + + // Wait for the server to respond + s.wait_for_write() ? + + mut buf := []byte{len: buf_len} + mut res := []byte{} + + mut c := 0 + + for { + c = s.read(mut buf) or { + return error('Failed to read data from socket.') + } + res << buf[..c] + + if c < buf_len { + break + } + } + + // Decode chunked response + return http.parse_response(res.bytestr()) +} + +fn get(url urllib.URL) ?http.Response { + return request('GET', url) +} diff --git a/src/main.v b/src/main.v index cddd9ae..7b19d4b 100644 --- a/src/main.v +++ b/src/main.v @@ -2,7 +2,6 @@ module main import web import os -import log import io import repo @@ -54,50 +53,13 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } fn main() { - // Configure logger - log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } - log_level := log.level_from_tag(log_level_str) or { - exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') - } - log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } + if os.args.len == 1 { + exit_with_message(1, 'No action provided.') + } - mut logger := log.Log{ - level: log_level - } - - logger.set_full_logpath(log_file) - logger.log_to_console_too() - - defer { - logger.info('Flushing log file') - logger.flush() - logger.close() - } - - // Configure web server - key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } - repo_dir := os.getenv_opt('REPO_DIR') or { - exit_with_message(1, 'No repo directory was configured.') - } - pkg_dir := os.getenv_opt('PKG_DIR') or { - exit_with_message(1, 'No package directory was configured.') - } - dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { - exit_with_message(1, 'No download directory was configured.') - } - - // This also creates the directories if needed - repo := repo.new(repo_dir, pkg_dir) or { - logger.error(err.msg) - exit(1) - } - - os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } - - web.run(&App{ - logger: logger - api_key: key - dl_dir: dl_dir - repo: repo - }, port) + match os.args[1] { + 'server' { server() } + 'build' { build() } + else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } + } } diff --git a/src/server.v b/src/server.v new file mode 100644 index 0000000..0e6ba45 --- /dev/null +++ b/src/server.v @@ -0,0 +1,55 @@ +module main + +import web +import os +import log +import repo + +fn server() { + // Configure logger + log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } + log_level := log.level_from_tag(log_level_str) or { + exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + } + log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } + + mut logger := log.Log{ + level: log_level + } + + logger.set_full_logpath(log_file) + logger.log_to_console_too() + + defer { + logger.info('Flushing log file') + logger.flush() + logger.close() + } + + // Configure web server + key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } + repo_dir := os.getenv_opt('REPO_DIR') or { + exit_with_message(1, 'No repo directory was configured.') + } + pkg_dir := os.getenv_opt('PKG_DIR') or { + exit_with_message(1, 'No package directory was configured.') + } + dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { + exit_with_message(1, 'No download directory was configured.') + } + + // This also creates the directories if needed + repo := repo.new(repo_dir, pkg_dir) or { + logger.error(err.msg) + exit(1) + } + + os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } + + web.run(&App{ + logger: logger + api_key: key + dl_dir: dl_dir + repo: repo + }, port) +} From 57c4af0aafd7ea728f15ed8c70e9637d1fdf6a44 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 19 Feb 2022 21:41:26 +0100 Subject: [PATCH 02/13] Fix for segfault --- src/build.v | 2 +- src/docker/containers.v | 8 +++---- src/docker/docker.v | 51 +++++++++++++++++++++-------------------- src/main.v | 16 ++++++------- 4 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/build.v b/src/build.v index e635cc0..85633a4 100644 --- a/src/build.v +++ b/src/build.v @@ -3,5 +3,5 @@ module main import docker fn build() { - println(docker.containers() or { panic("yeet") }) + println(docker.containers() or { panic('yeet') }) } diff --git a/src/docker/containers.v b/src/docker/containers.v index 5bdd39e..53d2880 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -4,12 +4,12 @@ import json import net.urllib struct Container { - id string - names []string + id string + names []string } pub fn containers() ?[]Container { - res := docker.get(urllib.parse('/containers/json') ?) ? + res := get(urllib.parse('/containers/json') ?) ? - return json.decode([]Container, res.text) + return json.decode([]Container, res.text) or {} } diff --git a/src/docker/docker.v b/src/docker/docker.v index a090f02..7f9d5db 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -5,44 +5,45 @@ import net.urllib import net.http const socket = '/var/run/docker.sock' + const buf_len = 1024 fn request(method string, url urllib.URL) ?http.Response { - req := "$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n" + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' - // Open a connection to the socket - mut s := unix.connect_stream(socket) ? + // Open a connection to the socket + mut s := unix.connect_stream(docker.socket) ? - defer { - s.close() ? - } + defer { + // This or is required because otherwise, the V compiler segfaults for + // some reason + s.close() or {} + } - // Write the request to the socket - s.write_string(req) ? + // Write the request to the socket + s.write_string(req) ? - // Wait for the server to respond - s.wait_for_write() ? + // Wait for the server to respond + s.wait_for_write() ? - mut buf := []byte{len: buf_len} - mut res := []byte{} + mut buf := []byte{len: docker.buf_len} + mut res := []byte{} - mut c := 0 + mut c := 0 - for { - c = s.read(mut buf) or { - return error('Failed to read data from socket.') - } - res << buf[..c] + for { + c = s.read(mut buf) or { return error('Failed to read data from socket.') } + res << buf[..c] - if c < buf_len { - break - } - } + if c < docker.buf_len { + break + } + } - // Decode chunked response - return http.parse_response(res.bytestr()) + // Decode chunked response + return http.parse_response(res.bytestr()) } fn get(url urllib.URL) ?http.Response { - return request('GET', url) + return request('GET', url) } diff --git a/src/main.v b/src/main.v index 7b19d4b..5d8c072 100644 --- a/src/main.v +++ b/src/main.v @@ -53,13 +53,13 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } fn main() { - if os.args.len == 1 { - exit_with_message(1, 'No action provided.') - } + if os.args.len == 1 { + exit_with_message(1, 'No action provided.') + } - match os.args[1] { - 'server' { server() } - 'build' { build() } - else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } - } + match os.args[1] { + 'server' { server() } + 'build' { build() } + else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } + } } From e6a1d32f0e9aa62e650116da33b1e3b1bda40025 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 19 Feb 2022 22:25:52 +0100 Subject: [PATCH 03/13] Some experimenting with docker api --- .editorconfig | 3 ++- src/build.v | 2 +- src/docker/containers.v | 4 ++-- src/docker/docker.v | 41 ++++++++++++++++++++++++++++++++++++++--- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/.editorconfig b/.editorconfig index 355c6bf..630e4fa 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,4 +7,5 @@ end_of_line = lf insert_final_newline = true [*.v] -indent_style = space +# vfmt wants it :( +indent_style = tab diff --git a/src/build.v b/src/build.v index 85633a4..304aaf2 100644 --- a/src/build.v +++ b/src/build.v @@ -3,5 +3,5 @@ module main import docker fn build() { - println(docker.containers() or { panic('yeet') }) + println(docker.pull('archlinux', 'latest') or { panic('yeetus') }) } diff --git a/src/docker/containers.v b/src/docker/containers.v index 53d2880..f0a1223 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -4,8 +4,8 @@ import json import net.urllib struct Container { - id string - names []string + id string [json: Id] + names []string [json: Names] } pub fn containers() ?[]Container { diff --git a/src/docker/docker.v b/src/docker/docker.v index 7f9d5db..7f5c854 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -3,20 +3,20 @@ module docker import net.unix import net.urllib import net.http +import json const socket = '/var/run/docker.sock' const buf_len = 1024 -fn request(method string, url urllib.URL) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' - +fn send(req &string) ?http.Response { // Open a connection to the socket mut s := unix.connect_stream(docker.socket) ? defer { // This or is required because otherwise, the V compiler segfaults for // some reason + // https://github.com/vlang/v/issues/13534 s.close() or {} } @@ -42,8 +42,43 @@ fn request(method string, url urllib.URL) ?http.Response { // Decode chunked response return http.parse_response(res.bytestr()) + +} + +fn request_with_body(method string, url urllib.URL, body &string) ?http.Response { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Length: ${body.len}\n$body\n' + + return send(req) +} + +fn request(method string, url urllib.URL) ?http.Response { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' + + return send(req) +} + +pub fn request_with_json(method string, url urllib.URL, data T) ?http.Response { + body := json.encode(data) + println(body) + + return request_with_body(method, url, body) } fn get(url urllib.URL) ?http.Response { return request('GET', url) } + +struct ImagePull { + from_image string [json: fromImage] + tag string +} + +pub fn pull(image string, tag string) ?http.Response { + // data := ImagePull{ + // from_image: image + // tag: tag + // } + + // return request_with_json("POST", urllib.parse("/images/create") ?, data) + return request("POST", urllib.parse("/images/create?fromImage=$image&tag=$tag") ?) +} From 275227400fc84a1070c8f642c2752e2858dea271 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 12:35:10 +0100 Subject: [PATCH 04/13] Wait for chunked stream WIP [CI SKIP] --- src/build.v | 1 + src/docker/docker.v | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/build.v b/src/build.v index 304aaf2..f3834fd 100644 --- a/src/build.v +++ b/src/build.v @@ -4,4 +4,5 @@ import docker fn build() { println(docker.pull('archlinux', 'latest') or { panic('yeetus') }) + // println(docker.containers() or { panic('heet') }) } diff --git a/src/docker/docker.v b/src/docker/docker.v index 7f5c854..0fcb058 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -23,23 +23,27 @@ fn send(req &string) ?http.Response { // Write the request to the socket s.write_string(req) ? - // Wait for the server to respond - s.wait_for_write() ? - mut buf := []byte{len: docker.buf_len} mut res := []byte{} mut c := 0 - for { - c = s.read(mut buf) or { return error('Failed to read data from socket.') } - res << buf[..c] + for res.len < 5 && res#[-4..] != [0, '\r', `\n`, `\r`, `\n`] { + // Wait for the server to respond + s.wait_for_write() ? - if c < docker.buf_len { - break + for { + c = s.read(mut buf) or { return error('Failed to read data from socket.') } + res << buf[..c] + + if c < docker.buf_len { + break + } } } + println(res) + // Decode chunked response return http.parse_response(res.bytestr()) From fbba66caa5cdc301208a76994e48c41f2e5eb6b1 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 13:03:00 +0100 Subject: [PATCH 05/13] Docker wrapper now waits for chunked responses --- src/build.v | 2 +- src/docker/docker.v | 25 +++++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/build.v b/src/build.v index f3834fd..47c6710 100644 --- a/src/build.v +++ b/src/build.v @@ -3,6 +3,6 @@ module main import docker fn build() { - println(docker.pull('archlinux', 'latest') or { panic('yeetus') }) + println(docker.pull('nginx', 'latest') or { panic('yeetus') }) // println(docker.containers() or { panic('heet') }) } diff --git a/src/docker/docker.v b/src/docker/docker.v index 0fcb058..b134bc2 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -23,12 +23,31 @@ fn send(req &string) ?http.Response { // Write the request to the socket s.write_string(req) ? + + s.wait_for_write() ? + + mut c := 0 mut buf := []byte{len: docker.buf_len} mut res := []byte{} - mut c := 0 + for { + c = s.read(mut buf) or { return error('Failed to read data from socket.') } + res << buf[..c] - for res.len < 5 && res#[-4..] != [0, '\r', `\n`, `\r`, `\n`] { + if c < docker.buf_len { + break + } + } + + // If the response isn't a chunked reply, we return early + parsed := http.parse_response(res.bytestr()) ? + + if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' { + return parsed + } + + // We loop until we've encountered the end of the chunked response + for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] { // Wait for the server to respond s.wait_for_write() ? @@ -42,8 +61,6 @@ fn send(req &string) ?http.Response { } } - println(res) - // Decode chunked response return http.parse_response(res.bytestr()) From 5515e2dca5bc1bee22bd930263048f86090410d4 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 13:10:48 +0100 Subject: [PATCH 06/13] Formatting & some cleanup --- src/build.v | 2 +- src/docker/containers.v | 4 ++-- src/docker/docker.v | 15 +++------------ 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/build.v b/src/build.v index 47c6710..6b3007f 100644 --- a/src/build.v +++ b/src/build.v @@ -3,6 +3,6 @@ module main import docker fn build() { - println(docker.pull('nginx', 'latest') or { panic('yeetus') }) + println(docker.pull('nginx', 'latest') or { panic('yeetus') }) // println(docker.containers() or { panic('heet') }) } diff --git a/src/docker/containers.v b/src/docker/containers.v index f0a1223..8ac6a66 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -4,8 +4,8 @@ import json import net.urllib struct Container { - id string [json: Id] - names []string [json: Names] + id string [json: Id] + names []string [json: Names] } pub fn containers() ?[]Container { diff --git a/src/docker/docker.v b/src/docker/docker.v index b134bc2..8873a9c 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -23,7 +23,6 @@ fn send(req &string) ?http.Response { // Write the request to the socket s.write_string(req) ? - s.wait_for_write() ? mut c := 0 @@ -63,11 +62,10 @@ fn send(req &string) ?http.Response { // Decode chunked response return http.parse_response(res.bytestr()) - } fn request_with_body(method string, url urllib.URL, body &string) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Length: ${body.len}\n$body\n' + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Length: $body.len\n$body\n' return send(req) } @@ -80,7 +78,6 @@ fn request(method string, url urllib.URL) ?http.Response { pub fn request_with_json(method string, url urllib.URL, data T) ?http.Response { body := json.encode(data) - println(body) return request_with_body(method, url, body) } @@ -91,15 +88,9 @@ fn get(url urllib.URL) ?http.Response { struct ImagePull { from_image string [json: fromImage] - tag string + tag string } pub fn pull(image string, tag string) ?http.Response { - // data := ImagePull{ - // from_image: image - // tag: tag - // } - - // return request_with_json("POST", urllib.parse("/images/create") ?, data) - return request("POST", urllib.parse("/images/create?fromImage=$image&tag=$tag") ?) + return request('POST', urllib.parse('/images/create?fromImage=$image&tag=$tag') ?) } From 4f705f5fb5cb618eac3d9d484f850109d79e7cc2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 20:26:39 +0100 Subject: [PATCH 07/13] Working build example!! --- src/build.v | 60 ++++++++++++++++++++++++++++++++++++++++- src/docker/containers.v | 30 ++++++++++++++++++++- src/docker/docker.v | 17 +++--------- 3 files changed, 92 insertions(+), 15 deletions(-) diff --git a/src/build.v b/src/build.v index 6b3007f..f303635 100644 --- a/src/build.v +++ b/src/build.v @@ -1,8 +1,66 @@ module main import docker +import encoding.base64 +import rand + +const container_build_dir = '/build' + +struct GitRepo { + url string [required] + branch string [required] +} fn build() { - println(docker.pull('nginx', 'latest') or { panic('yeetus') }) + // println(docker.pull('nginx', 'latest') or { panic('yeetus') }) // println(docker.containers() or { panic('heet') }) + repos := [ + GitRepo{'https://git.rustybever.be/Chewing_Bever/st', 'master'} + GitRepo{'https://aur.archlinux.org/libxft-bgra.git', 'master'} + ] + mut uuids := []string{} + + mut commands := [ + // Update repos & install required packages + 'pacman -Syu --needed --noconfirm base-devel git' + // Add a non-root user to run makepkg + 'groupadd -g 1000 builder' + 'useradd -mg builder builder' + // Make sure they can use sudo without a password + "echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers" + // Create the directory for the builds & make it writeable for the + // build user + 'mkdir /build' + 'chown -R builder:builder /build' + // "su builder -c 'git clone https://git.rustybever.be/Chewing_Bever/st /build/st'" + // 'su builder -c \'cd /build/st && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" https://arch.r8r.be/publish; done\'' + ] + + for repo in repos { + mut uuid := rand.uuid_v4() + + // Just to be sure we don't have any collisions + for uuids.contains(uuid) { + uuid = rand.uuid_v4() + } + + uuids << uuid + + commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'" + commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" https://arch.r8r.be/publish; done\'' + } + println(commands) + + // We convert the list of commands into a base64 string + cmds_str := base64.encode_str(commands.join('\n')) + + c := docker.NewContainer{ + image: 'archlinux:latest' + env: ['BUILD_SCRIPT=$cmds_str'] + entrypoint: ['/bin/sh', '-c'] + cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] + } + + id := docker.create_container(c) or { panic('aaaahh') } + print(docker.start_container(id) or { panic('yikes') }) } diff --git a/src/docker/containers.v b/src/docker/containers.v index 8ac6a66..f7eb8c7 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -9,7 +9,35 @@ struct Container { } pub fn containers() ?[]Container { - res := get(urllib.parse('/containers/json') ?) ? + res := request('GET', urllib.parse('/containers/json') ?) ? return json.decode([]Container, res.text) or {} } + +pub struct NewContainer { + image string [json: Image] + entrypoint []string [json: Entrypoint] + cmd []string [json: Cmd] + env []string [json: Env] +} + +struct CreatedContainer { + id string [json: Id] +} + +pub fn create_container(c &NewContainer) ?string { + res := request_with_json('POST', urllib.parse('/containers/create') ?, c) ? + + if res.status_code != 201 { + return error('Failed to create container.') + } + + return json.decode(CreatedContainer, res.text) ?.id +} + +pub fn start_container(id string) ?bool { + res := request('POST', urllib.parse('/containers/$id/start') ?) ? + println(res) + + return res.status_code == 204 +} diff --git a/src/docker/docker.v b/src/docker/docker.v index 8873a9c..9eda41f 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -64,8 +64,8 @@ fn send(req &string) ?http.Response { return http.parse_response(res.bytestr()) } -fn request_with_body(method string, url urllib.URL, body &string) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Length: $body.len\n$body\n' +fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: ${content_type}\nContent-Length: $body.len\n\n$body\n\n' return send(req) } @@ -76,19 +76,10 @@ fn request(method string, url urllib.URL) ?http.Response { return send(req) } -pub fn request_with_json(method string, url urllib.URL, data T) ?http.Response { +pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Response { body := json.encode(data) - return request_with_body(method, url, body) -} - -fn get(url urllib.URL) ?http.Response { - return request('GET', url) -} - -struct ImagePull { - from_image string [json: fromImage] - tag string + return request_with_body(method, url, 'application/json', body) } pub fn pull(image string, tag string) ?http.Response { From 941b30e7d20556c4bb3f6de653fdf695ed107b03 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 21:09:06 +0100 Subject: [PATCH 08/13] Fully functional hardcoded build command --- src/build.v | 43 ++++++++++++++++++++++++++++------------- src/docker/containers.v | 27 +++++++++++++++++++++++++- src/docker/docker.v | 2 +- src/main.v | 11 +++++++---- src/server.v | 6 +----- 5 files changed, 65 insertions(+), 24 deletions(-) diff --git a/src/build.v b/src/build.v index f303635..d664c8e 100644 --- a/src/build.v +++ b/src/build.v @@ -3,6 +3,8 @@ module main import docker import encoding.base64 import rand +import time +import os const container_build_dir = '/build' @@ -11,15 +13,12 @@ struct GitRepo { branch string [required] } -fn build() { - // println(docker.pull('nginx', 'latest') or { panic('yeetus') }) - // println(docker.containers() or { panic('heet') }) +fn build(key string, repo_dir string) ? { + server_url := os.getenv_opt('VIETER_ADDRESS') or { exit_with_message(1, 'No Vieter server address was provided.') } repos := [ GitRepo{'https://git.rustybever.be/Chewing_Bever/st', 'master'} GitRepo{'https://aur.archlinux.org/libxft-bgra.git', 'master'} ] - mut uuids := []string{} - mut commands := [ // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' @@ -32,10 +31,11 @@ fn build() { // build user 'mkdir /build' 'chown -R builder:builder /build' - // "su builder -c 'git clone https://git.rustybever.be/Chewing_Bever/st /build/st'" - // 'su builder -c \'cd /build/st && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" https://arch.r8r.be/publish; done\'' ] + // Each repo gets a unique UUID to avoid naming conflicts when cloning + mut uuids := []string{} + for repo in repos { mut uuid := rand.uuid_v4() @@ -47,20 +47,37 @@ fn build() { uuids << uuid commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'" - commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" https://arch.r8r.be/publish; done\'' + commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $server_url/publish; done\'' } - println(commands) - // We convert the list of commands into a base64 string + // We convert the list of commands into a base64 string, which then gets + // passed to the container as an env var cmds_str := base64.encode_str(commands.join('\n')) c := docker.NewContainer{ image: 'archlinux:latest' - env: ['BUILD_SCRIPT=$cmds_str'] + env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$key'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] } - id := docker.create_container(c) or { panic('aaaahh') } - print(docker.start_container(id) or { panic('yikes') }) + // First, we pull the latest archlinux image + docker.pull_image('archlinux', 'latest') ? + + id := docker.create_container(c) ? + docker.start_container(id) ? + + // This loop waits until the container has stopped, so we can remove it after + for { + data := docker.inspect_container(id) ? + + if !data.state.running { + break + } + + // Wait for 5 seconds + time.sleep(5000000000) + } + + docker.remove_container(id) ? } diff --git a/src/docker/containers.v b/src/docker/containers.v index f7eb8c7..5952ef3 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -37,7 +37,32 @@ pub fn create_container(c &NewContainer) ?string { pub fn start_container(id string) ?bool { res := request('POST', urllib.parse('/containers/$id/start') ?) ? - println(res) + + return res.status_code == 204 +} + +struct ContainerInspect { +pub: + state ContainerState [json: State] +} + +struct ContainerState { +pub: + running bool [json: Running] +} + +pub fn inspect_container(id string) ?ContainerInspect { + res := request('GET', urllib.parse('/containers/$id/json') ?) ? + + if res.status_code != 200 { + return error("Failed to inspect container.") + } + + return json.decode(ContainerInspect, res.text) +} + +pub fn remove_container(id string) ?bool { + res := request('DELETE', urllib.parse('/containers/$id') ?) ? return res.status_code == 204 } diff --git a/src/docker/docker.v b/src/docker/docker.v index 9eda41f..75d03ac 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -82,6 +82,6 @@ pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Respon return request_with_body(method, url, 'application/json', body) } -pub fn pull(image string, tag string) ?http.Response { +pub fn pull_image(image string, tag string) ?http.Response { return request('POST', urllib.parse('/images/create?fromImage=$image&tag=$tag') ?) } diff --git a/src/main.v b/src/main.v index 5d8c072..cc66b59 100644 --- a/src/main.v +++ b/src/main.v @@ -9,8 +9,6 @@ const port = 8000 const buf_size = 1_000_000 -const db_name = 'pieter.db' - struct App { web.Context pub: @@ -53,13 +51,18 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } fn main() { + key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } + repo_dir := os.getenv_opt('REPO_DIR') or { + exit_with_message(1, 'No repo directory was configured.') + } + if os.args.len == 1 { exit_with_message(1, 'No action provided.') } match os.args[1] { - 'server' { server() } - 'build' { build() } + 'server' { server(key, repo_dir) } + 'build' { build(key, repo_dir) ? } else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } } } diff --git a/src/server.v b/src/server.v index 0e6ba45..e0f22ec 100644 --- a/src/server.v +++ b/src/server.v @@ -5,7 +5,7 @@ import os import log import repo -fn server() { +fn server(key string, repo_dir string) { // Configure logger log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } log_level := log.level_from_tag(log_level_str) or { @@ -27,10 +27,6 @@ fn server() { } // Configure web server - key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } - repo_dir := os.getenv_opt('REPO_DIR') or { - exit_with_message(1, 'No repo directory was configured.') - } pkg_dir := os.getenv_opt('PKG_DIR') or { exit_with_message(1, 'No package directory was configured.') } From 138386682d341bbc3d546665388bf7865f58ea72 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 21:19:31 +0100 Subject: [PATCH 09/13] Repos are now read from a json file --- src/build.v | 17 ++++++++--------- src/git.v | 7 +++++++ 2 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 src/git.v diff --git a/src/build.v b/src/build.v index d664c8e..49ec48f 100644 --- a/src/build.v +++ b/src/build.v @@ -5,20 +5,19 @@ import encoding.base64 import rand import time import os +import json +import git const container_build_dir = '/build' -struct GitRepo { - url string [required] - branch string [required] -} - fn build(key string, repo_dir string) ? { server_url := os.getenv_opt('VIETER_ADDRESS') or { exit_with_message(1, 'No Vieter server address was provided.') } - repos := [ - GitRepo{'https://git.rustybever.be/Chewing_Bever/st', 'master'} - GitRepo{'https://aur.archlinux.org/libxft-bgra.git', 'master'} - ] + + // Read in the repos from a json file + filename := os.join_path_single(repo_dir, 'repos.json') + txt := os.read_file(filename) ? + repos := json.decode([]git.GitRepo, txt) ? + mut commands := [ // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' diff --git a/src/git.v b/src/git.v new file mode 100644 index 0000000..097e2e5 --- /dev/null +++ b/src/git.v @@ -0,0 +1,7 @@ +module git + +pub struct GitRepo { +pub: + url string [required] + branch string [required] +} From 36693900a3db4d0943ac3725100ec1cf2260167f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 21:43:18 +0100 Subject: [PATCH 10/13] Added crontab to docker image --- Dockerfile.ci | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 24f2bef..e0dd5da 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -36,11 +36,14 @@ HEALTHCHECK --interval=30s \ CMD /bin/wget --spider http://localhost:8000/health || exit 1 RUN mkdir /data && \ - chown -R www-data:www-data /data + chown -R www-data:www-data /data && \ + mkdir -p '/var/spool/cron/crontabs' && \ + echo '0 3 * * * /bin/vieter build' >> /var/spool/cron/crontabs/www-data && \ + chown www-data:www-data /var/spool/cron/crontabs/www-data WORKDIR /data USER www-data:www-data ENTRYPOINT ["/bin/dumb-init", "--"] -CMD ["/bin/vieter"] +CMD ["/bin/vieter", "server"] From 9cc88e629f096fb39550602a0739a53e7d57aaf2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 22:15:10 +0100 Subject: [PATCH 11/13] Added some documentation; ran format --- src/build.v | 10 ++++++---- src/docker/containers.v | 18 +++++++++++++----- src/docker/docker.v | 5 ++++- src/git.v | 2 +- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/build.v b/src/build.v index 49ec48f..00cf9ed 100644 --- a/src/build.v +++ b/src/build.v @@ -11,7 +11,9 @@ import git const container_build_dir = '/build' fn build(key string, repo_dir string) ? { - server_url := os.getenv_opt('VIETER_ADDRESS') or { exit_with_message(1, 'No Vieter server address was provided.') } + server_url := os.getenv_opt('VIETER_ADDRESS') or { + exit_with_message(1, 'No Vieter server address was provided.') + } // Read in the repos from a json file filename := os.join_path_single(repo_dir, 'repos.json') @@ -22,14 +24,14 @@ fn build(key string, repo_dir string) ? { // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' // Add a non-root user to run makepkg - 'groupadd -g 1000 builder' + 'groupadd -g 1000 builder', 'useradd -mg builder builder' // Make sure they can use sudo without a password "echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers" // Create the directory for the builds & make it writeable for the // build user - 'mkdir /build' - 'chown -R builder:builder /build' + 'mkdir /build', + 'chown -R builder:builder /build', ] // Each repo gets a unique UUID to avoid naming conflicts when cloning diff --git a/src/docker/containers.v b/src/docker/containers.v index 5952ef3..44b31ac 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -8,6 +8,7 @@ struct Container { names []string [json: Names] } +// containers returns a list of all currently running containers pub fn containers() ?[]Container { res := request('GET', urllib.parse('/containers/json') ?) ? @@ -15,16 +16,18 @@ pub fn containers() ?[]Container { } pub struct NewContainer { - image string [json: Image] + image string [json: Image] entrypoint []string [json: Entrypoint] - cmd []string [json: Cmd] - env []string [json: Env] + cmd []string [json: Cmd] + env []string [json: Env] } struct CreatedContainer { id string [json: Id] } +// create_container creates a container defined by the given configuration. If +// successful, it returns the ID of the newly created container. pub fn create_container(c &NewContainer) ?string { res := request_with_json('POST', urllib.parse('/containers/create') ?, c) ? @@ -35,6 +38,8 @@ pub fn create_container(c &NewContainer) ?string { return json.decode(CreatedContainer, res.text) ?.id } +// start_container starts a container with a given ID. It returns whether the +// container was started or not. pub fn start_container(id string) ?bool { res := request('POST', urllib.parse('/containers/$id/start') ?) ? @@ -51,16 +56,19 @@ pub: running bool [json: Running] } +// inspect_container returns the result of inspecting a container with a given +// ID. pub fn inspect_container(id string) ?ContainerInspect { res := request('GET', urllib.parse('/containers/$id/json') ?) ? if res.status_code != 200 { - return error("Failed to inspect container.") + return error('Failed to inspect container.') } - return json.decode(ContainerInspect, res.text) + return json.decode(ContainerInspect, res.text) or {} } +// remove_container removes a container with a given ID. pub fn remove_container(id string) ?bool { res := request('DELETE', urllib.parse('/containers/$id') ?) ? diff --git a/src/docker/docker.v b/src/docker/docker.v index 75d03ac..b68a7b4 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -65,7 +65,7 @@ fn send(req &string) ?http.Response { } fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: ${content_type}\nContent-Length: $body.len\n\n$body\n\n' + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' return send(req) } @@ -76,12 +76,15 @@ fn request(method string, url urllib.URL) ?http.Response { return send(req) } +// request_with_json sends a request to the Docker socket with a given JSON +// payload pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Response { body := json.encode(data) return request_with_body(method, url, 'application/json', body) } +// pull_image pulls tries to pull the image for the given image & tag pub fn pull_image(image string, tag string) ?http.Response { return request('POST', urllib.parse('/images/create?fromImage=$image&tag=$tag') ?) } diff --git a/src/git.v b/src/git.v index 097e2e5..76d80d7 100644 --- a/src/git.v +++ b/src/git.v @@ -2,6 +2,6 @@ module git pub struct GitRepo { pub: - url string [required] + url string [required] branch string [required] } From 96416585bcc8b4da733cb9834bec919a4ebbd7ca Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 21 Feb 2022 17:18:14 +0100 Subject: [PATCH 12/13] Added some error messages; updated changelog --- CHANGELOG.md | 14 ++++++++++++++ src/docker/docker.v | 14 ++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3824088..1166377 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter) +## Added + +* Very basic build system + * Build is triggered by separate cron container + * Packages build on cron container's system + * Packages are always rebuilt, even if they haven't changed + * Hardcoded planning of builds + * Builds are sequential + +## Fixed + +* Each package can now only have one version in the repository at once + (required by Pacman) + ## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0) ### Changed diff --git a/src/docker/docker.v b/src/docker/docker.v index b68a7b4..9c93986 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -11,7 +11,7 @@ const buf_len = 1024 fn send(req &string) ?http.Response { // Open a connection to the socket - mut s := unix.connect_stream(docker.socket) ? + mut s := unix.connect_stream(docker.socket) or { return error('Failed to connect to socket ${docker.socket}.') } defer { // This or is required because otherwise, the V compiler segfaults for @@ -21,7 +21,7 @@ fn send(req &string) ?http.Response { } // Write the request to the socket - s.write_string(req) ? + s.write_string(req) or { return error('Failed to write request to socket ${docker.socket}.') } s.wait_for_write() ? @@ -30,7 +30,7 @@ fn send(req &string) ?http.Response { mut res := []byte{} for { - c = s.read(mut buf) or { return error('Failed to read data from socket.') } + c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } res << buf[..c] if c < docker.buf_len { @@ -38,20 +38,22 @@ fn send(req &string) ?http.Response { } } - // If the response isn't a chunked reply, we return early - parsed := http.parse_response(res.bytestr()) ? + // After reading the first part of the response, we parse it into an HTTP + // response. If it isn't chunked, we return early with the data. + parsed := http.parse_response(res.bytestr()) or { return error('Failed to parse HTTP response from socket ${docker.socket}.') } if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' { return parsed } // We loop until we've encountered the end of the chunked response + // A chunked HTTP response always ends with '0\r\n\r\n'. for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] { // Wait for the server to respond s.wait_for_write() ? for { - c = s.read(mut buf) or { return error('Failed to read data from socket.') } + c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } res << buf[..c] if c < docker.buf_len { From 0cdd4e7b4b27167fe6f26bc1d367c28c9149762e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 21 Feb 2022 17:49:10 +0100 Subject: [PATCH 13/13] Versioned API endpoints (closes #91) --- src/docker/containers.v | 10 +++++----- src/docker/docker.v | 14 ++++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/docker/containers.v b/src/docker/containers.v index 44b31ac..a6df345 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -10,7 +10,7 @@ struct Container { // containers returns a list of all currently running containers pub fn containers() ?[]Container { - res := request('GET', urllib.parse('/containers/json') ?) ? + res := request('GET', urllib.parse('/v1.41/containers/json') ?) ? return json.decode([]Container, res.text) or {} } @@ -29,7 +29,7 @@ struct CreatedContainer { // create_container creates a container defined by the given configuration. If // successful, it returns the ID of the newly created container. pub fn create_container(c &NewContainer) ?string { - res := request_with_json('POST', urllib.parse('/containers/create') ?, c) ? + res := request_with_json('POST', urllib.parse('/v1.41/containers/create') ?, c) ? if res.status_code != 201 { return error('Failed to create container.') @@ -41,7 +41,7 @@ pub fn create_container(c &NewContainer) ?string { // start_container starts a container with a given ID. It returns whether the // container was started or not. pub fn start_container(id string) ?bool { - res := request('POST', urllib.parse('/containers/$id/start') ?) ? + res := request('POST', urllib.parse('/v1.41/containers/$id/start') ?) ? return res.status_code == 204 } @@ -59,7 +59,7 @@ pub: // inspect_container returns the result of inspecting a container with a given // ID. pub fn inspect_container(id string) ?ContainerInspect { - res := request('GET', urllib.parse('/containers/$id/json') ?) ? + res := request('GET', urllib.parse('/v1.41/containers/$id/json') ?) ? if res.status_code != 200 { return error('Failed to inspect container.') @@ -70,7 +70,7 @@ pub fn inspect_container(id string) ?ContainerInspect { // remove_container removes a container with a given ID. pub fn remove_container(id string) ?bool { - res := request('DELETE', urllib.parse('/containers/$id') ?) ? + res := request('DELETE', urllib.parse('/v1.41/containers/$id') ?) ? return res.status_code == 204 } diff --git a/src/docker/docker.v b/src/docker/docker.v index 9c93986..e0dbf7d 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -11,7 +11,9 @@ const buf_len = 1024 fn send(req &string) ?http.Response { // Open a connection to the socket - mut s := unix.connect_stream(docker.socket) or { return error('Failed to connect to socket ${docker.socket}.') } + mut s := unix.connect_stream(docker.socket) or { + return error('Failed to connect to socket ${docker.socket}.') + } defer { // This or is required because otherwise, the V compiler segfaults for @@ -40,7 +42,9 @@ fn send(req &string) ?http.Response { // After reading the first part of the response, we parse it into an HTTP // response. If it isn't chunked, we return early with the data. - parsed := http.parse_response(res.bytestr()) or { return error('Failed to parse HTTP response from socket ${docker.socket}.') } + parsed := http.parse_response(res.bytestr()) or { + return error('Failed to parse HTTP response from socket ${docker.socket}.') + } if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' { return parsed @@ -53,7 +57,9 @@ fn send(req &string) ?http.Response { s.wait_for_write() ? for { - c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } + c = s.read(mut buf) or { + return error('Failed to read data from socket ${docker.socket}.') + } res << buf[..c] if c < docker.buf_len { @@ -88,5 +94,5 @@ pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Respon // pull_image pulls tries to pull the image for the given image & tag pub fn pull_image(image string, tag string) ?http.Response { - return request('POST', urllib.parse('/images/create?fromImage=$image&tag=$tag') ?) + return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) }