diff --git a/.editorconfig b/.editorconfig index 355c6bf..630e4fa 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,4 +7,5 @@ end_of_line = lf insert_final_newline = true [*.v] -indent_style = space +# vfmt wants it :( +indent_style = tab diff --git a/CHANGELOG.md b/CHANGELOG.md index 3824088..1166377 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter) +## Added + +* Very basic build system + * Build is triggered by separate cron container + * Packages build on cron container's system + * Packages are always rebuilt, even if they haven't changed + * Hardcoded planning of builds + * Builds are sequential + +## Fixed + +* Each package can now only have one version in the repository at once + (required by Pacman) + ## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0) ### Changed diff --git a/Dockerfile.ci b/Dockerfile.ci index 24f2bef..e0dd5da 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -36,11 +36,14 @@ HEALTHCHECK --interval=30s \ CMD /bin/wget --spider http://localhost:8000/health || exit 1 RUN mkdir /data && \ - chown -R www-data:www-data /data + chown -R www-data:www-data /data && \ + mkdir -p '/var/spool/cron/crontabs' && \ + echo '0 3 * * * /bin/vieter build' >> /var/spool/cron/crontabs/www-data && \ + chown www-data:www-data /var/spool/cron/crontabs/www-data WORKDIR /data USER www-data:www-data ENTRYPOINT ["/bin/dumb-init", "--"] -CMD ["/bin/vieter"] +CMD ["/bin/vieter", "server"] diff --git a/Makefile b/Makefile index d7efc45..d69e48e 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ c: # Run the server in the default 'data' directory .PHONY: run run: vieter - API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter server .PHONY: run-prod run-prod: prod diff --git a/src/build.v b/src/build.v new file mode 100644 index 0000000..00cf9ed --- /dev/null +++ b/src/build.v @@ -0,0 +1,84 @@ +module main + +import docker +import encoding.base64 +import rand +import time +import os +import json +import git + +const container_build_dir = '/build' + +fn build(key string, repo_dir string) ? { + server_url := os.getenv_opt('VIETER_ADDRESS') or { + exit_with_message(1, 'No Vieter server address was provided.') + } + + // Read in the repos from a json file + filename := os.join_path_single(repo_dir, 'repos.json') + txt := os.read_file(filename) ? + repos := json.decode([]git.GitRepo, txt) ? + + mut commands := [ + // Update repos & install required packages + 'pacman -Syu --needed --noconfirm base-devel git' + // Add a non-root user to run makepkg + 'groupadd -g 1000 builder', + 'useradd -mg builder builder' + // Make sure they can use sudo without a password + "echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers" + // Create the directory for the builds & make it writeable for the + // build user + 'mkdir /build', + 'chown -R builder:builder /build', + ] + + // Each repo gets a unique UUID to avoid naming conflicts when cloning + mut uuids := []string{} + + for repo in repos { + mut uuid := rand.uuid_v4() + + // Just to be sure we don't have any collisions + for uuids.contains(uuid) { + uuid = rand.uuid_v4() + } + + uuids << uuid + + commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'" + commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $server_url/publish; done\'' + } + + // We convert the list of commands into a base64 string, which then gets + // passed to the container as an env var + cmds_str := base64.encode_str(commands.join('\n')) + + c := docker.NewContainer{ + image: 'archlinux:latest' + env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$key'] + entrypoint: ['/bin/sh', '-c'] + cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] + } + + // First, we pull the latest archlinux image + docker.pull_image('archlinux', 'latest') ? + + id := docker.create_container(c) ? + docker.start_container(id) ? + + // This loop waits until the container has stopped, so we can remove it after + for { + data := docker.inspect_container(id) ? + + if !data.state.running { + break + } + + // Wait for 5 seconds + time.sleep(5000000000) + } + + docker.remove_container(id) ? +} diff --git a/src/docker/containers.v b/src/docker/containers.v new file mode 100644 index 0000000..a6df345 --- /dev/null +++ b/src/docker/containers.v @@ -0,0 +1,76 @@ +module docker + +import json +import net.urllib + +struct Container { + id string [json: Id] + names []string [json: Names] +} + +// containers returns a list of all currently running containers +pub fn containers() ?[]Container { + res := request('GET', urllib.parse('/v1.41/containers/json') ?) ? + + return json.decode([]Container, res.text) or {} +} + +pub struct NewContainer { + image string [json: Image] + entrypoint []string [json: Entrypoint] + cmd []string [json: Cmd] + env []string [json: Env] +} + +struct CreatedContainer { + id string [json: Id] +} + +// create_container creates a container defined by the given configuration. If +// successful, it returns the ID of the newly created container. +pub fn create_container(c &NewContainer) ?string { + res := request_with_json('POST', urllib.parse('/v1.41/containers/create') ?, c) ? + + if res.status_code != 201 { + return error('Failed to create container.') + } + + return json.decode(CreatedContainer, res.text) ?.id +} + +// start_container starts a container with a given ID. It returns whether the +// container was started or not. +pub fn start_container(id string) ?bool { + res := request('POST', urllib.parse('/v1.41/containers/$id/start') ?) ? + + return res.status_code == 204 +} + +struct ContainerInspect { +pub: + state ContainerState [json: State] +} + +struct ContainerState { +pub: + running bool [json: Running] +} + +// inspect_container returns the result of inspecting a container with a given +// ID. +pub fn inspect_container(id string) ?ContainerInspect { + res := request('GET', urllib.parse('/v1.41/containers/$id/json') ?) ? + + if res.status_code != 200 { + return error('Failed to inspect container.') + } + + return json.decode(ContainerInspect, res.text) or {} +} + +// remove_container removes a container with a given ID. +pub fn remove_container(id string) ?bool { + res := request('DELETE', urllib.parse('/v1.41/containers/$id') ?) ? + + return res.status_code == 204 +} diff --git a/src/docker/docker.v b/src/docker/docker.v new file mode 100644 index 0000000..e0dbf7d --- /dev/null +++ b/src/docker/docker.v @@ -0,0 +1,98 @@ +module docker + +import net.unix +import net.urllib +import net.http +import json + +const socket = '/var/run/docker.sock' + +const buf_len = 1024 + +fn send(req &string) ?http.Response { + // Open a connection to the socket + mut s := unix.connect_stream(docker.socket) or { + return error('Failed to connect to socket ${docker.socket}.') + } + + defer { + // This or is required because otherwise, the V compiler segfaults for + // some reason + // https://github.com/vlang/v/issues/13534 + s.close() or {} + } + + // Write the request to the socket + s.write_string(req) or { return error('Failed to write request to socket ${docker.socket}.') } + + s.wait_for_write() ? + + mut c := 0 + mut buf := []byte{len: docker.buf_len} + mut res := []byte{} + + for { + c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } + res << buf[..c] + + if c < docker.buf_len { + break + } + } + + // After reading the first part of the response, we parse it into an HTTP + // response. If it isn't chunked, we return early with the data. + parsed := http.parse_response(res.bytestr()) or { + return error('Failed to parse HTTP response from socket ${docker.socket}.') + } + + if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' { + return parsed + } + + // We loop until we've encountered the end of the chunked response + // A chunked HTTP response always ends with '0\r\n\r\n'. + for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] { + // Wait for the server to respond + s.wait_for_write() ? + + for { + c = s.read(mut buf) or { + return error('Failed to read data from socket ${docker.socket}.') + } + res << buf[..c] + + if c < docker.buf_len { + break + } + } + } + + // Decode chunked response + return http.parse_response(res.bytestr()) +} + +fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' + + return send(req) +} + +fn request(method string, url urllib.URL) ?http.Response { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' + + return send(req) +} + +// request_with_json sends a request to the Docker socket with a given JSON +// payload +pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Response { + body := json.encode(data) + + return request_with_body(method, url, 'application/json', body) +} + +// pull_image pulls tries to pull the image for the given image & tag +pub fn pull_image(image string, tag string) ?http.Response { + return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) +} diff --git a/src/git.v b/src/git.v new file mode 100644 index 0000000..76d80d7 --- /dev/null +++ b/src/git.v @@ -0,0 +1,7 @@ +module git + +pub struct GitRepo { +pub: + url string [required] + branch string [required] +} diff --git a/src/main.v b/src/main.v index cddd9ae..cc66b59 100644 --- a/src/main.v +++ b/src/main.v @@ -2,7 +2,6 @@ module main import web import os -import log import io import repo @@ -10,8 +9,6 @@ const port = 8000 const buf_size = 1_000_000 -const db_name = 'pieter.db' - struct App { web.Context pub: @@ -54,50 +51,18 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } fn main() { - // Configure logger - log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } - log_level := log.level_from_tag(log_level_str) or { - exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') - } - log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } - - mut logger := log.Log{ - level: log_level - } - - logger.set_full_logpath(log_file) - logger.log_to_console_too() - - defer { - logger.info('Flushing log file') - logger.flush() - logger.close() - } - - // Configure web server key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } repo_dir := os.getenv_opt('REPO_DIR') or { exit_with_message(1, 'No repo directory was configured.') } - pkg_dir := os.getenv_opt('PKG_DIR') or { - exit_with_message(1, 'No package directory was configured.') - } - dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { - exit_with_message(1, 'No download directory was configured.') + + if os.args.len == 1 { + exit_with_message(1, 'No action provided.') } - // This also creates the directories if needed - repo := repo.new(repo_dir, pkg_dir) or { - logger.error(err.msg) - exit(1) + match os.args[1] { + 'server' { server(key, repo_dir) } + 'build' { build(key, repo_dir) ? } + else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } } - - os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } - - web.run(&App{ - logger: logger - api_key: key - dl_dir: dl_dir - repo: repo - }, port) } diff --git a/src/server.v b/src/server.v new file mode 100644 index 0000000..e0f22ec --- /dev/null +++ b/src/server.v @@ -0,0 +1,51 @@ +module main + +import web +import os +import log +import repo + +fn server(key string, repo_dir string) { + // Configure logger + log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } + log_level := log.level_from_tag(log_level_str) or { + exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + } + log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } + + mut logger := log.Log{ + level: log_level + } + + logger.set_full_logpath(log_file) + logger.log_to_console_too() + + defer { + logger.info('Flushing log file') + logger.flush() + logger.close() + } + + // Configure web server + pkg_dir := os.getenv_opt('PKG_DIR') or { + exit_with_message(1, 'No package directory was configured.') + } + dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { + exit_with_message(1, 'No download directory was configured.') + } + + // This also creates the directories if needed + repo := repo.new(repo_dir, pkg_dir) or { + logger.error(err.msg) + exit(1) + } + + os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } + + web.run(&App{ + logger: logger + api_key: key + dl_dir: dl_dir + repo: repo + }, port) +}