From a4346aad393a7dafb3a6d466a4ad4da1bc951f5c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 4 Feb 2022 13:17:12 +0100 Subject: [PATCH 001/105] Switched to boehm garbage collector --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8674ca0b..1cf4d3fc 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ SOURCES != find '$(SRC_DIR)' -iname '*.v' V_RELEASE := weekly.2022.05 V_PATH ?= v-$(V_RELEASE)/v -V := $(V_PATH) -showcc +V := $(V_PATH) -showcc -gc boehm all: vieter From 3379db017d3ef80eb46ae93c67a1ae050c5cf7a3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 4 Feb 2022 13:28:03 +0100 Subject: [PATCH 002/105] Added garbage collector to builder image --- .woodpecker/.build.yml | 2 ++ Dockerfile.builder | 1 + 2 files changed, 3 insertions(+) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 66caa8c7..c81e10b6 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -6,6 +6,8 @@ matrix: # These checks already get performed on the feature branches platform: ${PLATFORM} +depends_on: + - builder pipeline: # The default build isn't needed, as alpine switches to gcc for the compiler anyways diff --git a/Dockerfile.builder b/Dockerfile.builder index 3e40d89d..e88d0a6b 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -18,6 +18,7 @@ RUN ln -s /opt/vlang/v /usr/bin/v && \ sqlite-static sqlite-dev \ libx11-dev glfw-dev freetype-dev \ libarchive-static libarchive-dev \ + gc-dev \ diffutils COPY patches ./patches From c8a10261c481d32f463818dc8240326f34ab1b45 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 13 Feb 2022 13:31:07 +0100 Subject: [PATCH 003/105] Bump v version to 2022.06 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1cf4d3fc..25cba312 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' -V_RELEASE := weekly.2022.05 +V_RELEASE := weekly.2022.06 V_PATH ?= v-$(V_RELEASE)/v V := $(V_PATH) -showcc -gc boehm From 022f8c4fbe0da6b1f0c1b2a7b6c913e7ce71c25d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 14 Feb 2022 22:28:18 +0100 Subject: [PATCH 004/105] Updated to weekly.2022.07 --- Makefile | 2 +- patches/patch.sh | 2 +- src/web/parse.v | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 25cba312..7166c665 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' -V_RELEASE := weekly.2022.06 +V_RELEASE := weekly.2022.07 V_PATH ?= v-$(V_RELEASE)/v V := $(V_PATH) -showcc -gc boehm diff --git a/patches/patch.sh b/patches/patch.sh index 98139769..89f21760 100755 --- a/patches/patch.sh +++ b/patches/patch.sh @@ -6,8 +6,8 @@ # Add parse_request_no_body cat parse_request_no_body.v >> "$1"/vlib/net/http/request.v +# weekly.2022.07 fixes the write function being private # Make sha256 functions public sed -i \ -e 's/\(fn (mut d Digest) checksum(\)/pub \1/' \ - -e 's/\(fn (mut d Digest) write(\)/pub \1/' \ "$1"/vlib/crypto/sha256/sha256.v diff --git a/src/web/parse.v b/src/web/parse.v index 554c9f07..b5925409 100644 --- a/src/web/parse.v +++ b/src/web/parse.v @@ -34,7 +34,7 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) { } if x.len > 0 { return IError(http.UnexpectedExtraAttributeError{ - msg: 'Encountered unexpected extra attributes: $x' + attributes: x }) } if methods.len == 0 { @@ -49,8 +49,8 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) { fn parse_query_from_url(url urllib.URL) map[string]string { mut query := map[string]string{} - for k, v in url.query().data { - query[k] = v.data[0] + for v in url.query().data { + query[v.key] = v.value } return query } From 6f86033cd9aa5cfe875e91a98809a5ac54c96c44 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 17 Feb 2022 22:00:46 +0100 Subject: [PATCH 005/105] Currently broken start of docker wrapper [CI SKIP] --- Makefile | 2 +- src/build.v | 7 ++++++ src/docker/containers.v | 15 +++++++++++ src/docker/docker.v | 48 +++++++++++++++++++++++++++++++++++ src/main.v | 54 ++++++---------------------------------- src/server.v | 55 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 134 insertions(+), 47 deletions(-) create mode 100644 src/build.v create mode 100644 src/docker/containers.v create mode 100644 src/docker/docker.v create mode 100644 src/server.v diff --git a/Makefile b/Makefile index 7166c665..63d772ff 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ c: # Run the server in the default 'data' directory .PHONY: run run: vieter - API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter server .PHONY: run-prod run-prod: prod diff --git a/src/build.v b/src/build.v new file mode 100644 index 00000000..e635cc08 --- /dev/null +++ b/src/build.v @@ -0,0 +1,7 @@ +module main + +import docker + +fn build() { + println(docker.containers() or { panic("yeet") }) +} diff --git a/src/docker/containers.v b/src/docker/containers.v new file mode 100644 index 00000000..5bdd39ed --- /dev/null +++ b/src/docker/containers.v @@ -0,0 +1,15 @@ +module docker + +import json +import net.urllib + +struct Container { + id string + names []string +} + +pub fn containers() ?[]Container { + res := docker.get(urllib.parse('/containers/json') ?) ? + + return json.decode([]Container, res.text) +} diff --git a/src/docker/docker.v b/src/docker/docker.v new file mode 100644 index 00000000..a090f025 --- /dev/null +++ b/src/docker/docker.v @@ -0,0 +1,48 @@ +module docker + +import net.unix +import net.urllib +import net.http + +const socket = '/var/run/docker.sock' +const buf_len = 1024 + +fn request(method string, url urllib.URL) ?http.Response { + req := "$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n" + + // Open a connection to the socket + mut s := unix.connect_stream(socket) ? + + defer { + s.close() ? + } + + // Write the request to the socket + s.write_string(req) ? + + // Wait for the server to respond + s.wait_for_write() ? + + mut buf := []byte{len: buf_len} + mut res := []byte{} + + mut c := 0 + + for { + c = s.read(mut buf) or { + return error('Failed to read data from socket.') + } + res << buf[..c] + + if c < buf_len { + break + } + } + + // Decode chunked response + return http.parse_response(res.bytestr()) +} + +fn get(url urllib.URL) ?http.Response { + return request('GET', url) +} diff --git a/src/main.v b/src/main.v index cddd9aec..7b19d4b6 100644 --- a/src/main.v +++ b/src/main.v @@ -2,7 +2,6 @@ module main import web import os -import log import io import repo @@ -54,50 +53,13 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } fn main() { - // Configure logger - log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } - log_level := log.level_from_tag(log_level_str) or { - exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') - } - log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } + if os.args.len == 1 { + exit_with_message(1, 'No action provided.') + } - mut logger := log.Log{ - level: log_level - } - - logger.set_full_logpath(log_file) - logger.log_to_console_too() - - defer { - logger.info('Flushing log file') - logger.flush() - logger.close() - } - - // Configure web server - key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } - repo_dir := os.getenv_opt('REPO_DIR') or { - exit_with_message(1, 'No repo directory was configured.') - } - pkg_dir := os.getenv_opt('PKG_DIR') or { - exit_with_message(1, 'No package directory was configured.') - } - dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { - exit_with_message(1, 'No download directory was configured.') - } - - // This also creates the directories if needed - repo := repo.new(repo_dir, pkg_dir) or { - logger.error(err.msg) - exit(1) - } - - os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } - - web.run(&App{ - logger: logger - api_key: key - dl_dir: dl_dir - repo: repo - }, port) + match os.args[1] { + 'server' { server() } + 'build' { build() } + else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } + } } diff --git a/src/server.v b/src/server.v new file mode 100644 index 00000000..0e6ba45a --- /dev/null +++ b/src/server.v @@ -0,0 +1,55 @@ +module main + +import web +import os +import log +import repo + +fn server() { + // Configure logger + log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } + log_level := log.level_from_tag(log_level_str) or { + exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + } + log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } + + mut logger := log.Log{ + level: log_level + } + + logger.set_full_logpath(log_file) + logger.log_to_console_too() + + defer { + logger.info('Flushing log file') + logger.flush() + logger.close() + } + + // Configure web server + key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } + repo_dir := os.getenv_opt('REPO_DIR') or { + exit_with_message(1, 'No repo directory was configured.') + } + pkg_dir := os.getenv_opt('PKG_DIR') or { + exit_with_message(1, 'No package directory was configured.') + } + dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { + exit_with_message(1, 'No download directory was configured.') + } + + // This also creates the directories if needed + repo := repo.new(repo_dir, pkg_dir) or { + logger.error(err.msg) + exit(1) + } + + os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } + + web.run(&App{ + logger: logger + api_key: key + dl_dir: dl_dir + repo: repo + }, port) +} From 79b47a76e24c6c7e79617a53185991cf67bb2c0e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 18 Feb 2022 21:59:33 +0100 Subject: [PATCH 006/105] Remove packages from db before adding new version --- src/repo/repo.v | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/repo/repo.v b/src/repo/repo.v index 554b744c..1bf2d0c8 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -74,6 +74,9 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool { return false } + // We remove the older package version first, if present + r.remove(pkg.info.name, false) ? + os.mkdir(pkg_dir) or { return error('Failed to create package directory.') } os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or { @@ -92,6 +95,31 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool { return true } +// remove removes a package from the database. It returns false if the package +// wasn't present in the database. +fn (r &Repo) remove(pkg_name string, sync bool) ?bool { + // We iterate over every directory in the repo dir + for d in os.ls(r.repo_dir) ? { + name := d.split('-')#[..-2].join('-') + + if name == pkg_name { + // We lock the mutex here to prevent other routines from creating a + // new archive while we removed an entry + lock r.mutex { + os.rmdir_all(os.join_path_single(r.repo_dir, d)) ? + } + + if sync { + r.sync() ? + } + + return true + } + } + + return false +} + // Returns the path where the given package's desc & files files are stored fn (r &Repo) pkg_path(pkg &package.Pkg) string { return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version') From 99f6dc69fdef38c27a6b2b4784165bd2d25baa27 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 18 Feb 2022 22:08:58 +0100 Subject: [PATCH 007/105] Fixed tiny linting error --- src/web/parse.v | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/parse.v b/src/web/parse.v index b5925409..2eeef5e9 100644 --- a/src/web/parse.v +++ b/src/web/parse.v @@ -34,7 +34,7 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) { } if x.len > 0 { return IError(http.UnexpectedExtraAttributeError{ - attributes: x + attributes: x }) } if methods.len == 0 { From 07bd0beb5876bee877a3be0a2fdfdba28a50eba7 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 19 Feb 2022 15:27:41 +0100 Subject: [PATCH 008/105] Removed builder & patches code; updated README --- .gitignore | 2 +- .woodpecker/.build.yml | 2 -- .woodpecker/.builder.yml | 18 ----------------- .woodpecker/.docker.yml | 1 - Dockerfile.builder | 36 --------------------------------- Makefile | 15 ++++++-------- README.md | 10 ++++----- patches/parse_request_no_body.v | 23 --------------------- patches/patch.sh | 13 ------------ 9 files changed, 12 insertions(+), 108 deletions(-) delete mode 100644 .woodpecker/.builder.yml delete mode 100644 Dockerfile.builder delete mode 100644 patches/parse_request_no_body.v delete mode 100755 patches/patch.sh diff --git a/.gitignore b/.gitignore index df4fcc98..ca2e2f8b 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,4 @@ libarchive-* test/ # V compiler directory -v-*/ +v/ diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index c81e10b6..66caa8c7 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -6,8 +6,6 @@ matrix: # These checks already get performed on the feature branches platform: ${PLATFORM} -depends_on: - - builder pipeline: # The default build isn't needed, as alpine switches to gcc for the compiler anyways diff --git a/.woodpecker/.builder.yml b/.woodpecker/.builder.yml deleted file mode 100644 index e94a8463..00000000 --- a/.woodpecker/.builder.yml +++ /dev/null @@ -1,18 +0,0 @@ -branches: dev -platform: linux/amd64 - -pipeline: - publish: - image: woodpeckerci/plugin-docker-buildx - secrets: [ docker_username, docker_password ] - settings: - repo: chewingbever/vlang - tag: latest - dockerfile: Dockerfile.builder - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] - when: - event: push - path: - - Makefile - - Dockerfile.builder - - patches/* diff --git a/.woodpecker/.docker.yml b/.woodpecker/.docker.yml index 9dfdf08d..6202fe6c 100644 --- a/.woodpecker/.docker.yml +++ b/.woodpecker/.docker.yml @@ -1,7 +1,6 @@ branches: [main, dev] platform: linux/amd64 depends_on: - - builder - build pipeline: diff --git a/Dockerfile.builder b/Dockerfile.builder deleted file mode 100644 index e88d0a6b..00000000 --- a/Dockerfile.builder +++ /dev/null @@ -1,36 +0,0 @@ -FROM alpine:3.12 - -ARG TARGETPLATFORM - -WORKDIR /opt/vlang - -ENV VVV /opt/vlang -ENV PATH /opt/vlang:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -ENV VFLAGS -cc gcc -ENV V_PATH /opt/vlang/v - -RUN ln -s /opt/vlang/v /usr/bin/v && \ - apk --no-cache add \ - git make gcc curl openssl \ - musl-dev \ - openssl-libs-static openssl-dev \ - zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \ - sqlite-static sqlite-dev \ - libx11-dev glfw-dev freetype-dev \ - libarchive-static libarchive-dev \ - gc-dev \ - diffutils - -COPY patches ./patches -COPY Makefile ./ - -RUN make v && \ - mv v-*/* /opt/vlang && \ - v -version - -RUN if [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \ - wget -O /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && \ - chmod +x /usr/local/bin/mc ; \ -fi - -CMD ["v"] diff --git a/Makefile b/Makefile index 7166c665..d7efc451 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,7 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' -V_RELEASE := weekly.2022.07 -V_PATH ?= v-$(V_RELEASE)/v +V_PATH ?= v/v V := $(V_PATH) -showcc -gc boehm all: vieter @@ -46,7 +45,6 @@ run-prod: prod watch: API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG $(V) watch run vieter - # =====OTHER===== .PHONY: lint lint: @@ -63,11 +61,10 @@ vet: # Build & patch the V compiler .PHONY: v -v: v-$(V_RELEASE)/v -v-$(V_RELEASE)/v: - curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf - - cd patches && sh patch.sh '../v-$(V_RELEASE)' - make -C 'v-$(V_RELEASE)' +v: v/v +v/v: + git clone --single-branch --branch patches https://git.rustybever.be/Chewing_Bever/vieter-v v + make -C v clean: - rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'v-$(V_RELEASE)' + rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' diff --git a/README.md b/README.md index fbfa259a..f269027d 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,11 @@ that. ### Custom Compiler Currently, this program only works with a very slightly modified version of the -V standard library, and therefore the compiler. The changes that are made to -the standard V release can be found in the [patches](/patches) directory. You -can obtain this modified version of the compiler by running `make v`, which -will download, patch & build the compiler. Afterwards, all make commands that -require the V compiler will use this new binary. +V standard library, and therefore the compiler. The source code for this fork +can be found [here](https://git.rustybever.be/Chewing_Bever/vieter-v). You can +obtain this modified version of the compiler by running `make v`, which will +clone & build the compiler. Afterwards, all make commands that require the V +compiler will use this new binary. ## Features diff --git a/patches/parse_request_no_body.v b/patches/parse_request_no_body.v deleted file mode 100644 index c00a51c6..00000000 --- a/patches/parse_request_no_body.v +++ /dev/null @@ -1,23 +0,0 @@ -// Parse the header of a raw HTTP request into a Request object -pub fn parse_request_head(mut reader io.BufferedReader) ?Request { - // request line - mut line := reader.read_line() ? - method, target, version := parse_request_line(line) ? - - // headers - mut header := new_header() - line = reader.read_line() ? - for line != '' { - key, value := parse_header(line) ? - header.add_custom(key, value) ? - line = reader.read_line() ? - } - header.coerce(canonicalize: true) - - return Request{ - method: method - url: target.str() - header: header - version: version - } -} diff --git a/patches/patch.sh b/patches/patch.sh deleted file mode 100755 index 89f21760..00000000 --- a/patches/patch.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env sh -# This file patches the downloaded V version -# Should be run from within the directory it's in, as it uses relative paths to the files used for patching. -# $1 is the path to the downloaded V version - -# Add parse_request_no_body -cat parse_request_no_body.v >> "$1"/vlib/net/http/request.v - -# weekly.2022.07 fixes the write function being private -# Make sha256 functions public -sed -i \ - -e 's/\(fn (mut d Digest) checksum(\)/pub \1/' \ - "$1"/vlib/crypto/sha256/sha256.v From 3b3a2f4fc1a5a238b4a9a4e816184ae598093187 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 19 Feb 2022 15:36:30 +0100 Subject: [PATCH 009/105] Fixed linting errors --- src/web/parse.v | 2 +- src/web/web.v | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/web/parse.v b/src/web/parse.v index b5925409..2eeef5e9 100644 --- a/src/web/parse.v +++ b/src/web/parse.v @@ -34,7 +34,7 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) { } if x.len > 0 { return IError(http.UnexpectedExtraAttributeError{ - attributes: x + attributes: x }) } if methods.len == 0 { diff --git a/src/web/web.v b/src/web/web.v index b1e8b92a..ad647f20 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -22,7 +22,7 @@ pub struct Result {} pub const ( methods_with_form = [http.Method.post, .put, .patch] headers_close = http.new_custom_header_from_map({ - 'Server': 'VWeb' + 'Server': 'VWeb' http.CommonHeader.connection.str(): 'close' }) or { panic('should never fail') } From 57c4af0aafd7ea728f15ed8c70e9637d1fdf6a44 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 19 Feb 2022 21:41:26 +0100 Subject: [PATCH 010/105] Fix for segfault --- src/build.v | 2 +- src/docker/containers.v | 8 +++---- src/docker/docker.v | 51 +++++++++++++++++++++-------------------- src/main.v | 16 ++++++------- 4 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/build.v b/src/build.v index e635cc08..85633a49 100644 --- a/src/build.v +++ b/src/build.v @@ -3,5 +3,5 @@ module main import docker fn build() { - println(docker.containers() or { panic("yeet") }) + println(docker.containers() or { panic('yeet') }) } diff --git a/src/docker/containers.v b/src/docker/containers.v index 5bdd39ed..53d28807 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -4,12 +4,12 @@ import json import net.urllib struct Container { - id string - names []string + id string + names []string } pub fn containers() ?[]Container { - res := docker.get(urllib.parse('/containers/json') ?) ? + res := get(urllib.parse('/containers/json') ?) ? - return json.decode([]Container, res.text) + return json.decode([]Container, res.text) or {} } diff --git a/src/docker/docker.v b/src/docker/docker.v index a090f025..7f9d5db3 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -5,44 +5,45 @@ import net.urllib import net.http const socket = '/var/run/docker.sock' + const buf_len = 1024 fn request(method string, url urllib.URL) ?http.Response { - req := "$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n" + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' - // Open a connection to the socket - mut s := unix.connect_stream(socket) ? + // Open a connection to the socket + mut s := unix.connect_stream(docker.socket) ? - defer { - s.close() ? - } + defer { + // This or is required because otherwise, the V compiler segfaults for + // some reason + s.close() or {} + } - // Write the request to the socket - s.write_string(req) ? + // Write the request to the socket + s.write_string(req) ? - // Wait for the server to respond - s.wait_for_write() ? + // Wait for the server to respond + s.wait_for_write() ? - mut buf := []byte{len: buf_len} - mut res := []byte{} + mut buf := []byte{len: docker.buf_len} + mut res := []byte{} - mut c := 0 + mut c := 0 - for { - c = s.read(mut buf) or { - return error('Failed to read data from socket.') - } - res << buf[..c] + for { + c = s.read(mut buf) or { return error('Failed to read data from socket.') } + res << buf[..c] - if c < buf_len { - break - } - } + if c < docker.buf_len { + break + } + } - // Decode chunked response - return http.parse_response(res.bytestr()) + // Decode chunked response + return http.parse_response(res.bytestr()) } fn get(url urllib.URL) ?http.Response { - return request('GET', url) + return request('GET', url) } diff --git a/src/main.v b/src/main.v index 7b19d4b6..5d8c072c 100644 --- a/src/main.v +++ b/src/main.v @@ -53,13 +53,13 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } fn main() { - if os.args.len == 1 { - exit_with_message(1, 'No action provided.') - } + if os.args.len == 1 { + exit_with_message(1, 'No action provided.') + } - match os.args[1] { - 'server' { server() } - 'build' { build() } - else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } - } + match os.args[1] { + 'server' { server() } + 'build' { build() } + else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } + } } From e6a1d32f0e9aa62e650116da33b1e3b1bda40025 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 19 Feb 2022 22:25:52 +0100 Subject: [PATCH 011/105] Some experimenting with docker api --- .editorconfig | 3 ++- src/build.v | 2 +- src/docker/containers.v | 4 ++-- src/docker/docker.v | 41 ++++++++++++++++++++++++++++++++++++++--- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/.editorconfig b/.editorconfig index 355c6bf4..630e4fa7 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,4 +7,5 @@ end_of_line = lf insert_final_newline = true [*.v] -indent_style = space +# vfmt wants it :( +indent_style = tab diff --git a/src/build.v b/src/build.v index 85633a49..304aaf2a 100644 --- a/src/build.v +++ b/src/build.v @@ -3,5 +3,5 @@ module main import docker fn build() { - println(docker.containers() or { panic('yeet') }) + println(docker.pull('archlinux', 'latest') or { panic('yeetus') }) } diff --git a/src/docker/containers.v b/src/docker/containers.v index 53d28807..f0a12234 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -4,8 +4,8 @@ import json import net.urllib struct Container { - id string - names []string + id string [json: Id] + names []string [json: Names] } pub fn containers() ?[]Container { diff --git a/src/docker/docker.v b/src/docker/docker.v index 7f9d5db3..7f5c8546 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -3,20 +3,20 @@ module docker import net.unix import net.urllib import net.http +import json const socket = '/var/run/docker.sock' const buf_len = 1024 -fn request(method string, url urllib.URL) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' - +fn send(req &string) ?http.Response { // Open a connection to the socket mut s := unix.connect_stream(docker.socket) ? defer { // This or is required because otherwise, the V compiler segfaults for // some reason + // https://github.com/vlang/v/issues/13534 s.close() or {} } @@ -42,8 +42,43 @@ fn request(method string, url urllib.URL) ?http.Response { // Decode chunked response return http.parse_response(res.bytestr()) + +} + +fn request_with_body(method string, url urllib.URL, body &string) ?http.Response { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Length: ${body.len}\n$body\n' + + return send(req) +} + +fn request(method string, url urllib.URL) ?http.Response { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' + + return send(req) +} + +pub fn request_with_json(method string, url urllib.URL, data T) ?http.Response { + body := json.encode(data) + println(body) + + return request_with_body(method, url, body) } fn get(url urllib.URL) ?http.Response { return request('GET', url) } + +struct ImagePull { + from_image string [json: fromImage] + tag string +} + +pub fn pull(image string, tag string) ?http.Response { + // data := ImagePull{ + // from_image: image + // tag: tag + // } + + // return request_with_json("POST", urllib.parse("/images/create") ?, data) + return request("POST", urllib.parse("/images/create?fromImage=$image&tag=$tag") ?) +} From 275227400fc84a1070c8f642c2752e2858dea271 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 12:35:10 +0100 Subject: [PATCH 012/105] Wait for chunked stream WIP [CI SKIP] --- src/build.v | 1 + src/docker/docker.v | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/build.v b/src/build.v index 304aaf2a..f3834fdc 100644 --- a/src/build.v +++ b/src/build.v @@ -4,4 +4,5 @@ import docker fn build() { println(docker.pull('archlinux', 'latest') or { panic('yeetus') }) + // println(docker.containers() or { panic('heet') }) } diff --git a/src/docker/docker.v b/src/docker/docker.v index 7f5c8546..0fcb0583 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -23,23 +23,27 @@ fn send(req &string) ?http.Response { // Write the request to the socket s.write_string(req) ? - // Wait for the server to respond - s.wait_for_write() ? - mut buf := []byte{len: docker.buf_len} mut res := []byte{} mut c := 0 - for { - c = s.read(mut buf) or { return error('Failed to read data from socket.') } - res << buf[..c] + for res.len < 5 && res#[-4..] != [0, '\r', `\n`, `\r`, `\n`] { + // Wait for the server to respond + s.wait_for_write() ? - if c < docker.buf_len { - break + for { + c = s.read(mut buf) or { return error('Failed to read data from socket.') } + res << buf[..c] + + if c < docker.buf_len { + break + } } } + println(res) + // Decode chunked response return http.parse_response(res.bytestr()) From fbba66caa5cdc301208a76994e48c41f2e5eb6b1 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 13:03:00 +0100 Subject: [PATCH 013/105] Docker wrapper now waits for chunked responses --- src/build.v | 2 +- src/docker/docker.v | 25 +++++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/build.v b/src/build.v index f3834fdc..47c6710a 100644 --- a/src/build.v +++ b/src/build.v @@ -3,6 +3,6 @@ module main import docker fn build() { - println(docker.pull('archlinux', 'latest') or { panic('yeetus') }) + println(docker.pull('nginx', 'latest') or { panic('yeetus') }) // println(docker.containers() or { panic('heet') }) } diff --git a/src/docker/docker.v b/src/docker/docker.v index 0fcb0583..b134bc2e 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -23,12 +23,31 @@ fn send(req &string) ?http.Response { // Write the request to the socket s.write_string(req) ? + + s.wait_for_write() ? + + mut c := 0 mut buf := []byte{len: docker.buf_len} mut res := []byte{} - mut c := 0 + for { + c = s.read(mut buf) or { return error('Failed to read data from socket.') } + res << buf[..c] - for res.len < 5 && res#[-4..] != [0, '\r', `\n`, `\r`, `\n`] { + if c < docker.buf_len { + break + } + } + + // If the response isn't a chunked reply, we return early + parsed := http.parse_response(res.bytestr()) ? + + if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' { + return parsed + } + + // We loop until we've encountered the end of the chunked response + for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] { // Wait for the server to respond s.wait_for_write() ? @@ -42,8 +61,6 @@ fn send(req &string) ?http.Response { } } - println(res) - // Decode chunked response return http.parse_response(res.bytestr()) From 5515e2dca5bc1bee22bd930263048f86090410d4 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 13:10:48 +0100 Subject: [PATCH 014/105] Formatting & some cleanup --- src/build.v | 2 +- src/docker/containers.v | 4 ++-- src/docker/docker.v | 15 +++------------ 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/build.v b/src/build.v index 47c6710a..6b3007fa 100644 --- a/src/build.v +++ b/src/build.v @@ -3,6 +3,6 @@ module main import docker fn build() { - println(docker.pull('nginx', 'latest') or { panic('yeetus') }) + println(docker.pull('nginx', 'latest') or { panic('yeetus') }) // println(docker.containers() or { panic('heet') }) } diff --git a/src/docker/containers.v b/src/docker/containers.v index f0a12234..8ac6a660 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -4,8 +4,8 @@ import json import net.urllib struct Container { - id string [json: Id] - names []string [json: Names] + id string [json: Id] + names []string [json: Names] } pub fn containers() ?[]Container { diff --git a/src/docker/docker.v b/src/docker/docker.v index b134bc2e..8873a9ca 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -23,7 +23,6 @@ fn send(req &string) ?http.Response { // Write the request to the socket s.write_string(req) ? - s.wait_for_write() ? mut c := 0 @@ -63,11 +62,10 @@ fn send(req &string) ?http.Response { // Decode chunked response return http.parse_response(res.bytestr()) - } fn request_with_body(method string, url urllib.URL, body &string) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Length: ${body.len}\n$body\n' + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Length: $body.len\n$body\n' return send(req) } @@ -80,7 +78,6 @@ fn request(method string, url urllib.URL) ?http.Response { pub fn request_with_json(method string, url urllib.URL, data T) ?http.Response { body := json.encode(data) - println(body) return request_with_body(method, url, body) } @@ -91,15 +88,9 @@ fn get(url urllib.URL) ?http.Response { struct ImagePull { from_image string [json: fromImage] - tag string + tag string } pub fn pull(image string, tag string) ?http.Response { - // data := ImagePull{ - // from_image: image - // tag: tag - // } - - // return request_with_json("POST", urllib.parse("/images/create") ?, data) - return request("POST", urllib.parse("/images/create?fromImage=$image&tag=$tag") ?) + return request('POST', urllib.parse('/images/create?fromImage=$image&tag=$tag') ?) } From 4f705f5fb5cb618eac3d9d484f850109d79e7cc2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 20:26:39 +0100 Subject: [PATCH 015/105] Working build example!! --- src/build.v | 60 ++++++++++++++++++++++++++++++++++++++++- src/docker/containers.v | 30 ++++++++++++++++++++- src/docker/docker.v | 17 +++--------- 3 files changed, 92 insertions(+), 15 deletions(-) diff --git a/src/build.v b/src/build.v index 6b3007fa..f303635a 100644 --- a/src/build.v +++ b/src/build.v @@ -1,8 +1,66 @@ module main import docker +import encoding.base64 +import rand + +const container_build_dir = '/build' + +struct GitRepo { + url string [required] + branch string [required] +} fn build() { - println(docker.pull('nginx', 'latest') or { panic('yeetus') }) + // println(docker.pull('nginx', 'latest') or { panic('yeetus') }) // println(docker.containers() or { panic('heet') }) + repos := [ + GitRepo{'https://git.rustybever.be/Chewing_Bever/st', 'master'} + GitRepo{'https://aur.archlinux.org/libxft-bgra.git', 'master'} + ] + mut uuids := []string{} + + mut commands := [ + // Update repos & install required packages + 'pacman -Syu --needed --noconfirm base-devel git' + // Add a non-root user to run makepkg + 'groupadd -g 1000 builder' + 'useradd -mg builder builder' + // Make sure they can use sudo without a password + "echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers" + // Create the directory for the builds & make it writeable for the + // build user + 'mkdir /build' + 'chown -R builder:builder /build' + // "su builder -c 'git clone https://git.rustybever.be/Chewing_Bever/st /build/st'" + // 'su builder -c \'cd /build/st && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" https://arch.r8r.be/publish; done\'' + ] + + for repo in repos { + mut uuid := rand.uuid_v4() + + // Just to be sure we don't have any collisions + for uuids.contains(uuid) { + uuid = rand.uuid_v4() + } + + uuids << uuid + + commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'" + commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" https://arch.r8r.be/publish; done\'' + } + println(commands) + + // We convert the list of commands into a base64 string + cmds_str := base64.encode_str(commands.join('\n')) + + c := docker.NewContainer{ + image: 'archlinux:latest' + env: ['BUILD_SCRIPT=$cmds_str'] + entrypoint: ['/bin/sh', '-c'] + cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] + } + + id := docker.create_container(c) or { panic('aaaahh') } + print(docker.start_container(id) or { panic('yikes') }) } diff --git a/src/docker/containers.v b/src/docker/containers.v index 8ac6a660..f7eb8c78 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -9,7 +9,35 @@ struct Container { } pub fn containers() ?[]Container { - res := get(urllib.parse('/containers/json') ?) ? + res := request('GET', urllib.parse('/containers/json') ?) ? return json.decode([]Container, res.text) or {} } + +pub struct NewContainer { + image string [json: Image] + entrypoint []string [json: Entrypoint] + cmd []string [json: Cmd] + env []string [json: Env] +} + +struct CreatedContainer { + id string [json: Id] +} + +pub fn create_container(c &NewContainer) ?string { + res := request_with_json('POST', urllib.parse('/containers/create') ?, c) ? + + if res.status_code != 201 { + return error('Failed to create container.') + } + + return json.decode(CreatedContainer, res.text) ?.id +} + +pub fn start_container(id string) ?bool { + res := request('POST', urllib.parse('/containers/$id/start') ?) ? + println(res) + + return res.status_code == 204 +} diff --git a/src/docker/docker.v b/src/docker/docker.v index 8873a9ca..9eda41f4 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -64,8 +64,8 @@ fn send(req &string) ?http.Response { return http.parse_response(res.bytestr()) } -fn request_with_body(method string, url urllib.URL, body &string) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Length: $body.len\n$body\n' +fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: ${content_type}\nContent-Length: $body.len\n\n$body\n\n' return send(req) } @@ -76,19 +76,10 @@ fn request(method string, url urllib.URL) ?http.Response { return send(req) } -pub fn request_with_json(method string, url urllib.URL, data T) ?http.Response { +pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Response { body := json.encode(data) - return request_with_body(method, url, body) -} - -fn get(url urllib.URL) ?http.Response { - return request('GET', url) -} - -struct ImagePull { - from_image string [json: fromImage] - tag string + return request_with_body(method, url, 'application/json', body) } pub fn pull(image string, tag string) ?http.Response { From 941b30e7d20556c4bb3f6de653fdf695ed107b03 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 21:09:06 +0100 Subject: [PATCH 016/105] Fully functional hardcoded build command --- src/build.v | 43 ++++++++++++++++++++++++++++------------- src/docker/containers.v | 27 +++++++++++++++++++++++++- src/docker/docker.v | 2 +- src/main.v | 11 +++++++---- src/server.v | 6 +----- 5 files changed, 65 insertions(+), 24 deletions(-) diff --git a/src/build.v b/src/build.v index f303635a..d664c8ef 100644 --- a/src/build.v +++ b/src/build.v @@ -3,6 +3,8 @@ module main import docker import encoding.base64 import rand +import time +import os const container_build_dir = '/build' @@ -11,15 +13,12 @@ struct GitRepo { branch string [required] } -fn build() { - // println(docker.pull('nginx', 'latest') or { panic('yeetus') }) - // println(docker.containers() or { panic('heet') }) +fn build(key string, repo_dir string) ? { + server_url := os.getenv_opt('VIETER_ADDRESS') or { exit_with_message(1, 'No Vieter server address was provided.') } repos := [ GitRepo{'https://git.rustybever.be/Chewing_Bever/st', 'master'} GitRepo{'https://aur.archlinux.org/libxft-bgra.git', 'master'} ] - mut uuids := []string{} - mut commands := [ // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' @@ -32,10 +31,11 @@ fn build() { // build user 'mkdir /build' 'chown -R builder:builder /build' - // "su builder -c 'git clone https://git.rustybever.be/Chewing_Bever/st /build/st'" - // 'su builder -c \'cd /build/st && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" https://arch.r8r.be/publish; done\'' ] + // Each repo gets a unique UUID to avoid naming conflicts when cloning + mut uuids := []string{} + for repo in repos { mut uuid := rand.uuid_v4() @@ -47,20 +47,37 @@ fn build() { uuids << uuid commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'" - commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" https://arch.r8r.be/publish; done\'' + commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $server_url/publish; done\'' } - println(commands) - // We convert the list of commands into a base64 string + // We convert the list of commands into a base64 string, which then gets + // passed to the container as an env var cmds_str := base64.encode_str(commands.join('\n')) c := docker.NewContainer{ image: 'archlinux:latest' - env: ['BUILD_SCRIPT=$cmds_str'] + env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$key'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] } - id := docker.create_container(c) or { panic('aaaahh') } - print(docker.start_container(id) or { panic('yikes') }) + // First, we pull the latest archlinux image + docker.pull_image('archlinux', 'latest') ? + + id := docker.create_container(c) ? + docker.start_container(id) ? + + // This loop waits until the container has stopped, so we can remove it after + for { + data := docker.inspect_container(id) ? + + if !data.state.running { + break + } + + // Wait for 5 seconds + time.sleep(5000000000) + } + + docker.remove_container(id) ? } diff --git a/src/docker/containers.v b/src/docker/containers.v index f7eb8c78..5952ef3c 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -37,7 +37,32 @@ pub fn create_container(c &NewContainer) ?string { pub fn start_container(id string) ?bool { res := request('POST', urllib.parse('/containers/$id/start') ?) ? - println(res) + + return res.status_code == 204 +} + +struct ContainerInspect { +pub: + state ContainerState [json: State] +} + +struct ContainerState { +pub: + running bool [json: Running] +} + +pub fn inspect_container(id string) ?ContainerInspect { + res := request('GET', urllib.parse('/containers/$id/json') ?) ? + + if res.status_code != 200 { + return error("Failed to inspect container.") + } + + return json.decode(ContainerInspect, res.text) +} + +pub fn remove_container(id string) ?bool { + res := request('DELETE', urllib.parse('/containers/$id') ?) ? return res.status_code == 204 } diff --git a/src/docker/docker.v b/src/docker/docker.v index 9eda41f4..75d03ac0 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -82,6 +82,6 @@ pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Respon return request_with_body(method, url, 'application/json', body) } -pub fn pull(image string, tag string) ?http.Response { +pub fn pull_image(image string, tag string) ?http.Response { return request('POST', urllib.parse('/images/create?fromImage=$image&tag=$tag') ?) } diff --git a/src/main.v b/src/main.v index 5d8c072c..cc66b59a 100644 --- a/src/main.v +++ b/src/main.v @@ -9,8 +9,6 @@ const port = 8000 const buf_size = 1_000_000 -const db_name = 'pieter.db' - struct App { web.Context pub: @@ -53,13 +51,18 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } fn main() { + key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } + repo_dir := os.getenv_opt('REPO_DIR') or { + exit_with_message(1, 'No repo directory was configured.') + } + if os.args.len == 1 { exit_with_message(1, 'No action provided.') } match os.args[1] { - 'server' { server() } - 'build' { build() } + 'server' { server(key, repo_dir) } + 'build' { build(key, repo_dir) ? } else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } } } diff --git a/src/server.v b/src/server.v index 0e6ba45a..e0f22ecf 100644 --- a/src/server.v +++ b/src/server.v @@ -5,7 +5,7 @@ import os import log import repo -fn server() { +fn server(key string, repo_dir string) { // Configure logger log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } log_level := log.level_from_tag(log_level_str) or { @@ -27,10 +27,6 @@ fn server() { } // Configure web server - key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } - repo_dir := os.getenv_opt('REPO_DIR') or { - exit_with_message(1, 'No repo directory was configured.') - } pkg_dir := os.getenv_opt('PKG_DIR') or { exit_with_message(1, 'No package directory was configured.') } From 138386682d341bbc3d546665388bf7865f58ea72 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 21:19:31 +0100 Subject: [PATCH 017/105] Repos are now read from a json file --- src/build.v | 17 ++++++++--------- src/git.v | 7 +++++++ 2 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 src/git.v diff --git a/src/build.v b/src/build.v index d664c8ef..49ec48f5 100644 --- a/src/build.v +++ b/src/build.v @@ -5,20 +5,19 @@ import encoding.base64 import rand import time import os +import json +import git const container_build_dir = '/build' -struct GitRepo { - url string [required] - branch string [required] -} - fn build(key string, repo_dir string) ? { server_url := os.getenv_opt('VIETER_ADDRESS') or { exit_with_message(1, 'No Vieter server address was provided.') } - repos := [ - GitRepo{'https://git.rustybever.be/Chewing_Bever/st', 'master'} - GitRepo{'https://aur.archlinux.org/libxft-bgra.git', 'master'} - ] + + // Read in the repos from a json file + filename := os.join_path_single(repo_dir, 'repos.json') + txt := os.read_file(filename) ? + repos := json.decode([]git.GitRepo, txt) ? + mut commands := [ // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' diff --git a/src/git.v b/src/git.v new file mode 100644 index 00000000..097e2e5c --- /dev/null +++ b/src/git.v @@ -0,0 +1,7 @@ +module git + +pub struct GitRepo { +pub: + url string [required] + branch string [required] +} From 36693900a3db4d0943ac3725100ec1cf2260167f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 21:43:18 +0100 Subject: [PATCH 018/105] Added crontab to docker image --- Dockerfile.ci | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 24f2bef5..e0dd5dab 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -36,11 +36,14 @@ HEALTHCHECK --interval=30s \ CMD /bin/wget --spider http://localhost:8000/health || exit 1 RUN mkdir /data && \ - chown -R www-data:www-data /data + chown -R www-data:www-data /data && \ + mkdir -p '/var/spool/cron/crontabs' && \ + echo '0 3 * * * /bin/vieter build' >> /var/spool/cron/crontabs/www-data && \ + chown www-data:www-data /var/spool/cron/crontabs/www-data WORKDIR /data USER www-data:www-data ENTRYPOINT ["/bin/dumb-init", "--"] -CMD ["/bin/vieter"] +CMD ["/bin/vieter", "server"] From 9cc88e629f096fb39550602a0739a53e7d57aaf2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 20 Feb 2022 22:15:10 +0100 Subject: [PATCH 019/105] Added some documentation; ran format --- src/build.v | 10 ++++++---- src/docker/containers.v | 18 +++++++++++++----- src/docker/docker.v | 5 ++++- src/git.v | 2 +- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/build.v b/src/build.v index 49ec48f5..00cf9ed7 100644 --- a/src/build.v +++ b/src/build.v @@ -11,7 +11,9 @@ import git const container_build_dir = '/build' fn build(key string, repo_dir string) ? { - server_url := os.getenv_opt('VIETER_ADDRESS') or { exit_with_message(1, 'No Vieter server address was provided.') } + server_url := os.getenv_opt('VIETER_ADDRESS') or { + exit_with_message(1, 'No Vieter server address was provided.') + } // Read in the repos from a json file filename := os.join_path_single(repo_dir, 'repos.json') @@ -22,14 +24,14 @@ fn build(key string, repo_dir string) ? { // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' // Add a non-root user to run makepkg - 'groupadd -g 1000 builder' + 'groupadd -g 1000 builder', 'useradd -mg builder builder' // Make sure they can use sudo without a password "echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers" // Create the directory for the builds & make it writeable for the // build user - 'mkdir /build' - 'chown -R builder:builder /build' + 'mkdir /build', + 'chown -R builder:builder /build', ] // Each repo gets a unique UUID to avoid naming conflicts when cloning diff --git a/src/docker/containers.v b/src/docker/containers.v index 5952ef3c..44b31acc 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -8,6 +8,7 @@ struct Container { names []string [json: Names] } +// containers returns a list of all currently running containers pub fn containers() ?[]Container { res := request('GET', urllib.parse('/containers/json') ?) ? @@ -15,16 +16,18 @@ pub fn containers() ?[]Container { } pub struct NewContainer { - image string [json: Image] + image string [json: Image] entrypoint []string [json: Entrypoint] - cmd []string [json: Cmd] - env []string [json: Env] + cmd []string [json: Cmd] + env []string [json: Env] } struct CreatedContainer { id string [json: Id] } +// create_container creates a container defined by the given configuration. If +// successful, it returns the ID of the newly created container. pub fn create_container(c &NewContainer) ?string { res := request_with_json('POST', urllib.parse('/containers/create') ?, c) ? @@ -35,6 +38,8 @@ pub fn create_container(c &NewContainer) ?string { return json.decode(CreatedContainer, res.text) ?.id } +// start_container starts a container with a given ID. It returns whether the +// container was started or not. pub fn start_container(id string) ?bool { res := request('POST', urllib.parse('/containers/$id/start') ?) ? @@ -51,16 +56,19 @@ pub: running bool [json: Running] } +// inspect_container returns the result of inspecting a container with a given +// ID. pub fn inspect_container(id string) ?ContainerInspect { res := request('GET', urllib.parse('/containers/$id/json') ?) ? if res.status_code != 200 { - return error("Failed to inspect container.") + return error('Failed to inspect container.') } - return json.decode(ContainerInspect, res.text) + return json.decode(ContainerInspect, res.text) or {} } +// remove_container removes a container with a given ID. pub fn remove_container(id string) ?bool { res := request('DELETE', urllib.parse('/containers/$id') ?) ? diff --git a/src/docker/docker.v b/src/docker/docker.v index 75d03ac0..b68a7b42 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -65,7 +65,7 @@ fn send(req &string) ?http.Response { } fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: ${content_type}\nContent-Length: $body.len\n\n$body\n\n' + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' return send(req) } @@ -76,12 +76,15 @@ fn request(method string, url urllib.URL) ?http.Response { return send(req) } +// request_with_json sends a request to the Docker socket with a given JSON +// payload pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Response { body := json.encode(data) return request_with_body(method, url, 'application/json', body) } +// pull_image pulls tries to pull the image for the given image & tag pub fn pull_image(image string, tag string) ?http.Response { return request('POST', urllib.parse('/images/create?fromImage=$image&tag=$tag') ?) } diff --git a/src/git.v b/src/git.v index 097e2e5c..76d80d73 100644 --- a/src/git.v +++ b/src/git.v @@ -2,6 +2,6 @@ module git pub struct GitRepo { pub: - url string [required] + url string [required] branch string [required] } From 96416585bcc8b4da733cb9834bec919a4ebbd7ca Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 21 Feb 2022 17:18:14 +0100 Subject: [PATCH 020/105] Added some error messages; updated changelog --- CHANGELOG.md | 14 ++++++++++++++ src/docker/docker.v | 14 ++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3824088c..11663779 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter) +## Added + +* Very basic build system + * Build is triggered by separate cron container + * Packages build on cron container's system + * Packages are always rebuilt, even if they haven't changed + * Hardcoded planning of builds + * Builds are sequential + +## Fixed + +* Each package can now only have one version in the repository at once + (required by Pacman) + ## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0) ### Changed diff --git a/src/docker/docker.v b/src/docker/docker.v index b68a7b42..9c93986c 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -11,7 +11,7 @@ const buf_len = 1024 fn send(req &string) ?http.Response { // Open a connection to the socket - mut s := unix.connect_stream(docker.socket) ? + mut s := unix.connect_stream(docker.socket) or { return error('Failed to connect to socket ${docker.socket}.') } defer { // This or is required because otherwise, the V compiler segfaults for @@ -21,7 +21,7 @@ fn send(req &string) ?http.Response { } // Write the request to the socket - s.write_string(req) ? + s.write_string(req) or { return error('Failed to write request to socket ${docker.socket}.') } s.wait_for_write() ? @@ -30,7 +30,7 @@ fn send(req &string) ?http.Response { mut res := []byte{} for { - c = s.read(mut buf) or { return error('Failed to read data from socket.') } + c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } res << buf[..c] if c < docker.buf_len { @@ -38,20 +38,22 @@ fn send(req &string) ?http.Response { } } - // If the response isn't a chunked reply, we return early - parsed := http.parse_response(res.bytestr()) ? + // After reading the first part of the response, we parse it into an HTTP + // response. If it isn't chunked, we return early with the data. + parsed := http.parse_response(res.bytestr()) or { return error('Failed to parse HTTP response from socket ${docker.socket}.') } if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' { return parsed } // We loop until we've encountered the end of the chunked response + // A chunked HTTP response always ends with '0\r\n\r\n'. for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] { // Wait for the server to respond s.wait_for_write() ? for { - c = s.read(mut buf) or { return error('Failed to read data from socket.') } + c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } res << buf[..c] if c < docker.buf_len { From 0cdd4e7b4b27167fe6f26bc1d367c28c9149762e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 21 Feb 2022 17:49:10 +0100 Subject: [PATCH 021/105] Versioned API endpoints (closes #91) --- src/docker/containers.v | 10 +++++----- src/docker/docker.v | 14 ++++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/docker/containers.v b/src/docker/containers.v index 44b31acc..a6df3459 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -10,7 +10,7 @@ struct Container { // containers returns a list of all currently running containers pub fn containers() ?[]Container { - res := request('GET', urllib.parse('/containers/json') ?) ? + res := request('GET', urllib.parse('/v1.41/containers/json') ?) ? return json.decode([]Container, res.text) or {} } @@ -29,7 +29,7 @@ struct CreatedContainer { // create_container creates a container defined by the given configuration. If // successful, it returns the ID of the newly created container. pub fn create_container(c &NewContainer) ?string { - res := request_with_json('POST', urllib.parse('/containers/create') ?, c) ? + res := request_with_json('POST', urllib.parse('/v1.41/containers/create') ?, c) ? if res.status_code != 201 { return error('Failed to create container.') @@ -41,7 +41,7 @@ pub fn create_container(c &NewContainer) ?string { // start_container starts a container with a given ID. It returns whether the // container was started or not. pub fn start_container(id string) ?bool { - res := request('POST', urllib.parse('/containers/$id/start') ?) ? + res := request('POST', urllib.parse('/v1.41/containers/$id/start') ?) ? return res.status_code == 204 } @@ -59,7 +59,7 @@ pub: // inspect_container returns the result of inspecting a container with a given // ID. pub fn inspect_container(id string) ?ContainerInspect { - res := request('GET', urllib.parse('/containers/$id/json') ?) ? + res := request('GET', urllib.parse('/v1.41/containers/$id/json') ?) ? if res.status_code != 200 { return error('Failed to inspect container.') @@ -70,7 +70,7 @@ pub fn inspect_container(id string) ?ContainerInspect { // remove_container removes a container with a given ID. pub fn remove_container(id string) ?bool { - res := request('DELETE', urllib.parse('/containers/$id') ?) ? + res := request('DELETE', urllib.parse('/v1.41/containers/$id') ?) ? return res.status_code == 204 } diff --git a/src/docker/docker.v b/src/docker/docker.v index 9c93986c..e0dbf7d7 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -11,7 +11,9 @@ const buf_len = 1024 fn send(req &string) ?http.Response { // Open a connection to the socket - mut s := unix.connect_stream(docker.socket) or { return error('Failed to connect to socket ${docker.socket}.') } + mut s := unix.connect_stream(docker.socket) or { + return error('Failed to connect to socket ${docker.socket}.') + } defer { // This or is required because otherwise, the V compiler segfaults for @@ -40,7 +42,9 @@ fn send(req &string) ?http.Response { // After reading the first part of the response, we parse it into an HTTP // response. If it isn't chunked, we return early with the data. - parsed := http.parse_response(res.bytestr()) or { return error('Failed to parse HTTP response from socket ${docker.socket}.') } + parsed := http.parse_response(res.bytestr()) or { + return error('Failed to parse HTTP response from socket ${docker.socket}.') + } if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' { return parsed @@ -53,7 +57,9 @@ fn send(req &string) ?http.Response { s.wait_for_write() ? for { - c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } + c = s.read(mut buf) or { + return error('Failed to read data from socket ${docker.socket}.') + } res << buf[..c] if c < docker.buf_len { @@ -88,5 +94,5 @@ pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Respon // pull_image pulls tries to pull the image for the given image & tag pub fn pull_image(image string, tag string) ?http.Response { - return request('POST', urllib.parse('/images/create?fromImage=$image&tag=$tag') ?) + return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) } From 1d434db166918a19fee18c7d5e66c9c87e2ea9dd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 21 Feb 2022 20:40:13 +0100 Subject: [PATCH 022/105] Wrote a proper env file system --- CHANGELOG.md | 3 ++ Makefile | 19 +++++++----- src/auth.v | 2 +- src/build.v | 13 ++++----- src/env.v | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/main.v | 24 ++-------------- src/routes.v | 4 +-- src/server.v | 38 +++++++++++++----------- 8 files changed, 128 insertions(+), 56 deletions(-) create mode 100644 src/env.v diff --git a/CHANGELOG.md b/CHANGELOG.md index 11663779..238fd004 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Packages are always rebuilt, even if they haven't changed * Hardcoded planning of builds * Builds are sequential +* Better environment variable support + * Each env var can now be provided from a file by appending it with `_FILE` + & passing the path to the file as value ## Fixed diff --git a/Makefile b/Makefile index d69e48e8..94b079bc 100644 --- a/Makefile +++ b/Makefile @@ -34,16 +34,21 @@ c: # Run the server in the default 'data' directory .PHONY: run run: vieter - API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter server + VIETER_API_KEY=test \ + VIETER_DOWNLOAD_DIR=data/downloads \ + VIETER_REPO_DIR=data/repo \ + VIETER_PKG_DIR=data/pkgs \ + VIETER_LOG_LEVEL=DEBUG \ + ./vieter server .PHONY: run-prod run-prod: prod - API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./pvieter - -# Same as run, but restart when the source code changes -.PHONY: watch -watch: - API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG $(V) watch run vieter + VIETER_API_KEY=test \ + VIETER_DOWNLOAD_DIR=data/downloads \ + VIETER_REPO_DIR=data/repo \ + VIETER_PKG_DIR=data/pkgs \ + VIETER_LOG_LEVEL=DEBUG \ + ./pvieter server # =====OTHER===== .PHONY: lint diff --git a/src/auth.v b/src/auth.v index eab63c85..942bddad 100644 --- a/src/auth.v +++ b/src/auth.v @@ -7,5 +7,5 @@ fn (mut app App) is_authorized() bool { return false } - return x_header.trim_space() == app.api_key + return x_header.trim_space() == app.conf.api_key } diff --git a/src/build.v b/src/build.v index 00cf9ed7..0acca8fc 100644 --- a/src/build.v +++ b/src/build.v @@ -7,16 +7,15 @@ import time import os import json import git +import env const container_build_dir = '/build' -fn build(key string, repo_dir string) ? { - server_url := os.getenv_opt('VIETER_ADDRESS') or { - exit_with_message(1, 'No Vieter server address was provided.') - } +fn build() ? { + conf := env.load() ? // Read in the repos from a json file - filename := os.join_path_single(repo_dir, 'repos.json') + filename := os.join_path_single(conf.repo_dir, 'repos.json') txt := os.read_file(filename) ? repos := json.decode([]git.GitRepo, txt) ? @@ -48,7 +47,7 @@ fn build(key string, repo_dir string) ? { uuids << uuid commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'" - commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $server_url/publish; done\'' + commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\'' } // We convert the list of commands into a base64 string, which then gets @@ -57,7 +56,7 @@ fn build(key string, repo_dir string) ? { c := docker.NewContainer{ image: 'archlinux:latest' - env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$key'] + env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] } diff --git a/src/env.v b/src/env.v new file mode 100644 index 00000000..804d37c8 --- /dev/null +++ b/src/env.v @@ -0,0 +1,81 @@ +module env + +import os + +// The prefix that every environment variable should have +const prefix = 'VIETER_' + +// The suffix an environment variable in order for it to be loaded from a file +// instead +const file_suffix = '_FILE' + +pub struct ServerConfig { +pub: + log_level string [default: WARN] + log_file string [default: 'vieter.log'] + pkg_dir string + download_dir string + api_key string + repo_dir string +} + +pub struct BuildConfig { +pub: + api_key string + repo_dir string + address string +} + +fn get_env_var(field_name string) ?string { + env_var_name := '${prefix}${field_name.to_upper()}' + env_file_name := '${prefix}${field_name.to_upper()}${file_suffix}' + env_var := os.getenv(env_var_name) + env_file := os.getenv(env_file_name) + + // If both aren't set, we report them missing + if env_var == '' && env_file == '' { + return error('Either $env_var_name or $env_file_name is required.') + } + + // If they're both set, we report a conflict + if env_var != '' && env_file != '' { + return error('Only one of $env_var_name or $env_file_name can be defined.') + } + + // If it's the env var itself, we return it + if env_var != '' { + return env_var + } + + // Otherwise, we process the file + return os.read_file(env_file) or { + error('Failed to read file defined in $env_file_name: ${err.msg}.') + } +} + +// load attempts to create the given type from environment variables. +pub fn load() ?T { + res := T{} + + $for field in T.fields { + res.$(field.name) = get_env_var(field.name) or { + // We use the default instead, if it's present + mut default := '' + + for attr in field.attrs { + if attr.starts_with('default: ') { + default = attr[9..] + break + } + } + + if default == '' { + return err + } + + default + } + } + + return res +} diff --git a/src/main.v b/src/main.v index cc66b59a..2472ac0b 100644 --- a/src/main.v +++ b/src/main.v @@ -1,22 +1,7 @@ module main -import web import os import io -import repo - -const port = 8000 - -const buf_size = 1_000_000 - -struct App { - web.Context -pub: - api_key string [required; web_global] - dl_dir string [required; web_global] -pub mut: - repo repo.Repo [required; web_global] -} [noreturn] fn exit_with_message(code int, msg string) { @@ -51,18 +36,13 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } fn main() { - key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } - repo_dir := os.getenv_opt('REPO_DIR') or { - exit_with_message(1, 'No repo directory was configured.') - } - if os.args.len == 1 { exit_with_message(1, 'No action provided.') } match os.args[1] { - 'server' { server(key, repo_dir) } - 'build' { build(key, repo_dir) ? } + 'server' { server() ? } + 'build' { build() ? } else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } } } diff --git a/src/routes.v b/src/routes.v index 8b7ddebe..b152a9f3 100644 --- a/src/routes.v +++ b/src/routes.v @@ -58,10 +58,10 @@ fn (mut app App) put_package() web.Result { if length := app.req.header.get(.content_length) { // Generate a random filename for the temp file - pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) + pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4()) for os.exists(pkg_path) { - pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) + pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4()) } app.ldebug("Uploading $length bytes (${pretty_bytes(length.int())}) to '$pkg_path'.") diff --git a/src/server.v b/src/server.v index e0f22ecf..d5c5ab2c 100644 --- a/src/server.v +++ b/src/server.v @@ -4,20 +4,33 @@ import web import os import log import repo +import env + +const port = 8000 + +const buf_size = 1_000_000 + +struct App { + web.Context +pub: + conf env.ServerConfig [required: web_global] +pub mut: + repo repo.Repo [required; web_global] +} + +fn server() ? { + conf := env.load() ? -fn server(key string, repo_dir string) { // Configure logger - log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } - log_level := log.level_from_tag(log_level_str) or { + log_level := log.level_from_tag(conf.log_level) or { exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') } - log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } mut logger := log.Log{ level: log_level } - logger.set_full_logpath(log_file) + logger.set_full_logpath(conf.log_file) logger.log_to_console_too() defer { @@ -26,26 +39,17 @@ fn server(key string, repo_dir string) { logger.close() } - // Configure web server - pkg_dir := os.getenv_opt('PKG_DIR') or { - exit_with_message(1, 'No package directory was configured.') - } - dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { - exit_with_message(1, 'No download directory was configured.') - } - // This also creates the directories if needed - repo := repo.new(repo_dir, pkg_dir) or { + repo := repo.new(conf.repo_dir, conf.pkg_dir) or { logger.error(err.msg) exit(1) } - os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } + os.mkdir_all(conf.download_dir) or { exit_with_message(1, 'Failed to create download directory.') } web.run(&App{ logger: logger - api_key: key - dl_dir: dl_dir + conf: conf repo: repo }, port) } From 92ad0c51eb5ea0ef75a90bcf46d6d00b94e44007 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 21 Feb 2022 20:51:41 +0100 Subject: [PATCH 023/105] Split off server into own module --- src/env.v | 19 +++++++++--------- src/main.v | 41 +++++---------------------------------- src/{ => server}/auth.v | 2 +- src/{ => server}/routes.v | 7 ++++--- src/{ => server}/server.v | 15 +++++++------- src/util.v | 35 +++++++++++++++++++++++++++++++++ 6 files changed, 62 insertions(+), 57 deletions(-) rename src/{ => server}/auth.v (94%) rename src/{ => server}/routes.v (95%) rename src/{ => server}/server.v (72%) diff --git a/src/env.v b/src/env.v index 804d37c8..2cbcf634 100644 --- a/src/env.v +++ b/src/env.v @@ -11,24 +11,24 @@ const file_suffix = '_FILE' pub struct ServerConfig { pub: - log_level string [default: WARN] - log_file string [default: 'vieter.log'] - pkg_dir string + log_level string [default: WARN] + log_file string [default: 'vieter.log'] + pkg_dir string download_dir string - api_key string - repo_dir string + api_key string + repo_dir string } pub struct BuildConfig { pub: - api_key string + api_key string repo_dir string - address string + address string } fn get_env_var(field_name string) ?string { - env_var_name := '${prefix}${field_name.to_upper()}' - env_file_name := '${prefix}${field_name.to_upper()}${file_suffix}' + env_var_name := '$env.prefix$field_name.to_upper()' + env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix' env_var := os.getenv(env_var_name) env_file := os.getenv(env_file_name) @@ -76,6 +76,5 @@ pub fn load() ?T { default } } - return res } diff --git a/src/main.v b/src/main.v index 2472ac0b..156f0a39 100644 --- a/src/main.v +++ b/src/main.v @@ -1,48 +1,17 @@ module main import os -import io - -[noreturn] -fn exit_with_message(code int, msg string) { - eprintln(msg) - exit(code) -} - -fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { - mut file := os.create(path) ? - defer { - file.close() - } - - mut buf := []byte{len: buf_size} - mut bytes_left := length - - // Repeat as long as the stream still has data - for bytes_left > 0 { - // TODO check if just breaking here is safe - bytes_read := reader.read(mut buf) or { break } - bytes_left -= bytes_read - - mut to_write := bytes_read - - for to_write > 0 { - // TODO don't just loop infinitely here - bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue } - - to_write = to_write - bytes_written - } - } -} +import server +import util fn main() { if os.args.len == 1 { - exit_with_message(1, 'No action provided.') + util.exit_with_message(1, 'No action provided.') } match os.args[1] { - 'server' { server() ? } + 'server' { server.server() ? } 'build' { build() ? } - else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } + else { util.exit_with_message(1, 'Unknown action: ${os.args[1]}') } } } diff --git a/src/auth.v b/src/server/auth.v similarity index 94% rename from src/auth.v rename to src/server/auth.v index 942bddad..8bc9d55d 100644 --- a/src/auth.v +++ b/src/server/auth.v @@ -1,4 +1,4 @@ -module main +module server import net.http diff --git a/src/routes.v b/src/server/routes.v similarity index 95% rename from src/routes.v rename to src/server/routes.v index b152a9f3..872894e6 100644 --- a/src/routes.v +++ b/src/server/routes.v @@ -1,10 +1,11 @@ -module main +module server import web import os import repo import time import rand +import util const prefixes = ['B', 'KB', 'MB', 'GB'] @@ -18,7 +19,7 @@ fn pretty_bytes(bytes int) string { n /= 1024 } - return '${n:.2}${prefixes[i]}' + return '${n:.2}${server.prefixes[i]}' } fn is_pkg_name(s string) bool { @@ -69,7 +70,7 @@ fn (mut app App) put_package() web.Result { // This is used to time how long it takes to upload a file mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) - reader_to_file(mut app.reader, length.int(), pkg_path) or { + util.reader_to_file(mut app.reader, length.int(), pkg_path) or { app.lwarn("Failed to upload '$pkg_path'") return app.text('Failed to upload file.') diff --git a/src/server.v b/src/server/server.v similarity index 72% rename from src/server.v rename to src/server/server.v index d5c5ab2c..3b16c637 100644 --- a/src/server.v +++ b/src/server/server.v @@ -1,15 +1,14 @@ -module main +module server import web import os import log import repo import env +import util const port = 8000 -const buf_size = 1_000_000 - struct App { web.Context pub: @@ -18,12 +17,12 @@ pub mut: repo repo.Repo [required; web_global] } -fn server() ? { +pub fn server() ? { conf := env.load() ? // Configure logger log_level := log.level_from_tag(conf.log_level) or { - exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') } mut logger := log.Log{ @@ -45,11 +44,13 @@ fn server() ? { exit(1) } - os.mkdir_all(conf.download_dir) or { exit_with_message(1, 'Failed to create download directory.') } + os.mkdir_all(conf.download_dir) or { + util.exit_with_message(1, 'Failed to create download directory.') + } web.run(&App{ logger: logger conf: conf repo: repo - }, port) + }, server.port) } diff --git a/src/util.v b/src/util.v index f81a2561..d0fa9841 100644 --- a/src/util.v +++ b/src/util.v @@ -1,9 +1,44 @@ module util import os +import io import crypto.md5 import crypto.sha256 +const reader_buf_size = 1_000_000 + +[noreturn] +pub fn exit_with_message(code int, msg string) { + eprintln(msg) + exit(code) +} + +pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { + mut file := os.create(path) ? + defer { + file.close() + } + + mut buf := []byte{len: util.reader_buf_size} + mut bytes_left := length + + // Repeat as long as the stream still has data + for bytes_left > 0 { + // TODO check if just breaking here is safe + bytes_read := reader.read(mut buf) or { break } + bytes_left -= bytes_read + + mut to_write := bytes_read + + for to_write > 0 { + // TODO don't just loop infinitely here + bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue } + + to_write = to_write - bytes_written + } + } +} + // hash_file returns the md5 & sha256 hash of a given file // TODO actually implement sha256 pub fn hash_file(path &string) ?(string, string) { From e13252d36826004f298a96c2ed4bb8e9df4ea61c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 21 Feb 2022 22:22:36 +0100 Subject: [PATCH 024/105] Initial part of repos API (SEGFAULTS) [CI SKIP] --- .gitignore | 3 ++ Makefile | 13 +++++- src/build.v | 4 +- src/env.v | 6 ++- src/git.v | 7 ---- src/repo/repo.v | 9 +--- src/server/git.v | 100 ++++++++++++++++++++++++++++++++++++++++++++ src/server/routes.v | 21 +--------- src/server/server.v | 4 +- src/util.v | 22 ++++++++++ 10 files changed, 150 insertions(+), 39 deletions(-) delete mode 100644 src/git.v create mode 100644 src/server/git.v diff --git a/.gitignore b/.gitignore index ca2e2f8b..3a6b11be 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ test/ # V compiler directory v/ + +# gdb log file +gdb.txt diff --git a/Makefile b/Makefile index 94b079bc..b1b91f2b 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,17 @@ vieter: $(SOURCES) .PHONY: debug debug: dvieter dvieter: $(SOURCES) - $(V) -keepc -cg -cc gcc -o dvieter $(SRC_DIR) + $(V_PATH) -showcc -keepc -cg -o dvieter $(SRC_DIR) + +.PHONY: gdb +gdb: dvieter + VIETER_API_KEY=test \ + VIETER_DOWNLOAD_DIR=data/downloads \ + VIETER_REPO_DIR=data/repo \ + VIETER_PKG_DIR=data/pkgs \ + VIETER_LOG_LEVEL=DEBUG \ + VIETER_REPOS_FILE=data/repos.json \ + gdb --args ./dvieter # Optimised production build .PHONY: prod @@ -39,6 +49,7 @@ run: vieter VIETER_REPO_DIR=data/repo \ VIETER_PKG_DIR=data/pkgs \ VIETER_LOG_LEVEL=DEBUG \ + VIETER_REPOS_FILE=data/repos.json \ ./vieter server .PHONY: run-prod diff --git a/src/build.v b/src/build.v index 0acca8fc..8d6af140 100644 --- a/src/build.v +++ b/src/build.v @@ -6,7 +6,7 @@ import rand import time import os import json -import git +import server import env const container_build_dir = '/build' @@ -17,7 +17,7 @@ fn build() ? { // Read in the repos from a json file filename := os.join_path_single(conf.repo_dir, 'repos.json') txt := os.read_file(filename) ? - repos := json.decode([]git.GitRepo, txt) ? + repos := json.decode([]server.GitRepo, txt) ? mut commands := [ // Update repos & install required packages diff --git a/src/env.v b/src/env.v index 2cbcf634..1e601bd7 100644 --- a/src/env.v +++ b/src/env.v @@ -17,6 +17,7 @@ pub: download_dir string api_key string repo_dir string + repos_file string } pub struct BuildConfig { @@ -42,7 +43,10 @@ fn get_env_var(field_name string) ?string { return error('Only one of $env_var_name or $env_file_name can be defined.') } - // If it's the env var itself, we return it + // If it's the env var itself, we return it. + // I'm pretty sure this also prevents variable ending in _FILE (e.g. + // VIETER_LOG_FILE) from being mistakingely read as an _FILE suffixed env + // var. if env_var != '' { return env_var } diff --git a/src/git.v b/src/git.v deleted file mode 100644 index 76d80d73..00000000 --- a/src/git.v +++ /dev/null @@ -1,7 +0,0 @@ -module git - -pub struct GitRepo { -pub: - url string [required] - branch string [required] -} diff --git a/src/repo/repo.v b/src/repo/repo.v index 1bf2d0c8..f1419aca 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -2,17 +2,12 @@ module repo import os import package - -// Dummy struct to work around the fact that you can only share structs, maps & -// arrays -pub struct Dummy { - x int -} +import util // This struct manages a single repository. pub struct Repo { mut: - mutex shared Dummy + mutex shared util.Dummy pub: // Where to store repository files repo_dir string [required] diff --git a/src/server/git.v b/src/server/git.v new file mode 100644 index 00000000..6a1d9296 --- /dev/null +++ b/src/server/git.v @@ -0,0 +1,100 @@ +module server + +import web +import os +import json + +const repos_file = 'repos.json' + +pub struct GitRepo { +pub: + url string [required] + branch string [required] +} + +fn read_repos(path string) ?[]GitRepo { + if !os.exists(path) { + mut f := os.create(path) ? + + defer { + f.close() + } + + f.write_string('{}') ? + + return [] + } + + content := os.read_file(path) ? + return json.decode([]GitRepo, content) +} + +fn write_repos(path string, repos []GitRepo) ? { + mut f := os.create(path) ? + + defer { + f.close() + } + + dump(repos) + value := json.encode(repos) + f.write_string(value) ? +} + +['/api/repos'; get] +pub fn (mut app App) get_repos() web.Result { + if !app.is_authorized() { + return app.text('Unauthorized.') + } + + repos := rlock app.git_mutex { + read_repos(app.conf.repos_file) or { + app.lerror('Failed to read repos file.') + + return app.server_error(500) + } + } + + return app.json(repos) +} + +['/api/repos'; post] +pub fn (mut app App) post_repo() web.Result { + if !app.is_authorized() { + return app.text('Unauthorized.') + } + + if !('url' in app.query && 'branch' in app.query) { + return app.server_error(400) + } + + new_repo := GitRepo{ + url: app.query['url'] + branch: app.query['branch'] + } + + mut repos := rlock app.git_mutex { + read_repos(app.conf.repos_file) or { + app.lerror('Failed to read repos file.') + + return app.server_error(500) + } + } + + // We need to check for duplicates + for r in repos { + if r == new_repo { + return app.text('Duplicate repository.') + } + } + + repos << new_repo + + lock app.git_mutex { + write_repos(app.conf.repos_file, repos) or { + return app.server_error(500) + } + } + + return app.ok('Repo added successfully.') +} diff --git a/src/server/routes.v b/src/server/routes.v index 872894e6..7a0ee387 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -7,25 +7,6 @@ import time import rand import util -const prefixes = ['B', 'KB', 'MB', 'GB'] - -// pretty_bytes converts a byte count to human-readable version -fn pretty_bytes(bytes int) string { - mut i := 0 - mut n := f32(bytes) - - for n >= 1024 { - i++ - n /= 1024 - } - - return '${n:.2}${server.prefixes[i]}' -} - -fn is_pkg_name(s string) bool { - return s.contains('.pkg') -} - // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. ['/health'; get] @@ -65,7 +46,7 @@ fn (mut app App) put_package() web.Result { pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4()) } - app.ldebug("Uploading $length bytes (${pretty_bytes(length.int())}) to '$pkg_path'.") + app.ldebug("Uploading $length bytes (${util.pretty_bytes(length.int())}) to '$pkg_path'.") // This is used to time how long it takes to upload a file mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) diff --git a/src/server/server.v b/src/server/server.v index 3b16c637..ebb9f3bb 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -12,9 +12,11 @@ const port = 8000 struct App { web.Context pub: - conf env.ServerConfig [required: web_global] + conf env.ServerConfig [required; web_global] pub mut: repo repo.Repo [required; web_global] + // This is used to claim the file lock on the repos file + git_mutex shared util.Dummy } pub fn server() ? { diff --git a/src/util.v b/src/util.v index d0fa9841..65d5294f 100644 --- a/src/util.v +++ b/src/util.v @@ -7,6 +7,14 @@ import crypto.sha256 const reader_buf_size = 1_000_000 +const prefixes = ['B', 'KB', 'MB', 'GB'] + +// Dummy struct to work around the fact that you can only share structs, maps & +// arrays +pub struct Dummy { + x int +} + [noreturn] pub fn exit_with_message(code int, msg string) { eprintln(msg) @@ -67,3 +75,17 @@ pub fn hash_file(path &string) ?(string, string) { return md5sum.checksum().hex(), sha256sum.checksum().hex() } + +// pretty_bytes converts a byte count to human-readable version +pub fn pretty_bytes(bytes int) string { + mut i := 0 + mut n := f32(bytes) + + for n >= 1024 { + i++ + n /= 1024 + } + + return '${n:.2}${util.prefixes[i]}' +} + From 398758a72790f8ebdf9e1b5276e8a0ed8d5b57a7 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 21 Feb 2022 22:32:54 +0100 Subject: [PATCH 025/105] Fixed segfault All together now: thank you spytheman --- src/env.v | 2 +- src/server/git.v | 10 ++++------ src/util.v | 1 - 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/env.v b/src/env.v index 1e601bd7..26919f84 100644 --- a/src/env.v +++ b/src/env.v @@ -17,7 +17,7 @@ pub: download_dir string api_key string repo_dir string - repos_file string + repos_file string } pub struct BuildConfig { diff --git a/src/server/git.v b/src/server/git.v index 6a1d9296..8c569e66 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -20,13 +20,14 @@ fn read_repos(path string) ?[]GitRepo { f.close() } - f.write_string('{}') ? + f.write_string('[]') ? return [] } content := os.read_file(path) ? - return json.decode([]GitRepo, content) + res := json.decode([]GitRepo, content) ? + return res } fn write_repos(path string, repos []GitRepo) ? { @@ -36,7 +37,6 @@ fn write_repos(path string, repos []GitRepo) ? { f.close() } - dump(repos) value := json.encode(repos) f.write_string(value) ? } @@ -91,9 +91,7 @@ pub fn (mut app App) post_repo() web.Result { repos << new_repo lock app.git_mutex { - write_repos(app.conf.repos_file, repos) or { - return app.server_error(500) - } + write_repos(app.conf.repos_file, repos) or { return app.server_error(500) } } return app.ok('Repo added successfully.') diff --git a/src/util.v b/src/util.v index 65d5294f..24d33fca 100644 --- a/src/util.v +++ b/src/util.v @@ -88,4 +88,3 @@ pub fn pretty_bytes(bytes int) string { return '${n:.2}${util.prefixes[i]}' } - From fe98112f796d39bb14175386f6e0a0b29e94500b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 21 Feb 2022 22:40:59 +0100 Subject: [PATCH 026/105] Added repos delete route --- src/server/git.v | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/server/git.v b/src/server/git.v index 8c569e66..d506c7e9 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -96,3 +96,34 @@ pub fn (mut app App) post_repo() web.Result { return app.ok('Repo added successfully.') } + +['/api/repos'; delete] +pub fn (mut app App) delete_repo() web.Result { + if !app.is_authorized() { + return app.text('Unauthorized.') + } + + if !('url' in app.query && 'branch' in app.query) { + return app.server_error(400) + } + + repo_to_remove := GitRepo{ + url: app.query['url'] + branch: app.query['branch'] + } + + mut repos := rlock app.git_mutex { + read_repos(app.conf.repos_file) or { + app.lerror('Failed to read repos file.') + + return app.server_error(500) + } + } + filtered := repos.filter(it != repo_to_remove) + + lock app.git_mutex { + write_repos(app.conf.repos_file, filtered) or { return app.server_error(500) } + } + + return app.ok('Repo removed successfully.') +} From 6194a3f4086bf97009d6c83c5eb687032bb9e924 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 21 Feb 2022 22:58:05 +0100 Subject: [PATCH 027/105] Builder now gets list of repos from server --- CHANGELOG.md | 1 + src/build.v | 11 +++++++---- src/env.v | 1 - 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 238fd004..edae68bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Better environment variable support * Each env var can now be provided from a file by appending it with `_FILE` & passing the path to the file as value +* API for managing Git repositories to build ## Fixed diff --git a/src/build.v b/src/build.v index 8d6af140..ab72a592 100644 --- a/src/build.v +++ b/src/build.v @@ -8,16 +8,19 @@ import os import json import server import env +import net.http const container_build_dir = '/build' fn build() ? { conf := env.load() ? - // Read in the repos from a json file - filename := os.join_path_single(conf.repo_dir, 'repos.json') - txt := os.read_file(filename) ? - repos := json.decode([]server.GitRepo, txt) ? + // We get the repos list from the Vieter instance + mut req := http.new_request(http.Method.get, '${conf.address}/api/repos', '') ? + req.add_custom_header('X-Api-Key', conf.api_key) ? + + res := req.do() ? + repos := json.decode([]server.GitRepo, res.text) ? mut commands := [ // Update repos & install required packages diff --git a/src/env.v b/src/env.v index 26919f84..8b92d8dc 100644 --- a/src/env.v +++ b/src/env.v @@ -23,7 +23,6 @@ pub: pub struct BuildConfig { pub: api_key string - repo_dir string address string } From 27f59c6664e7b4fbb15e3b49735c55417f9f7605 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 22 Feb 2022 08:14:20 +0100 Subject: [PATCH 028/105] Updated CI Dockerfile; fixed formatting & vet --- Dockerfile.ci | 7 ++++--- Makefile | 3 +++ src/build.v | 3 +-- src/env.v | 9 ++++++--- src/server/git.v | 6 +++--- src/server/server.v | 1 + src/util.v | 3 +++ 7 files changed, 21 insertions(+), 11 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index e0dd5dab..a062d951 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -24,9 +24,10 @@ RUN curl --fail \ FROM busybox:1.35.0 ENV PATH=/bin \ - REPO_DIR=/data/repo \ - PKG_DIR=/data/pkgs \ - DOWNLOAD_DIR=/data/downloads + VIETER_REPO_DIR=/data/repo \ + VIETER_PKG_DIR=/data/pkgs \ + VIETER_DOWNLOAD_DIR=/data/downloads \ + VIETER_REPOS_FILE=/data/repos.json COPY --from=builder /app/dumb-init /app/vieter /bin/ diff --git a/Makefile b/Makefile index b1b91f2b..062a47f8 100644 --- a/Makefile +++ b/Makefile @@ -13,11 +13,14 @@ vieter: $(SOURCES) $(V) -g -o vieter $(SRC_DIR) # Debug build using gcc +# The debug build can't use the boehm garbage collector, as that is +# multi-threaded and causes issues when running vieter inside gdb. .PHONY: debug debug: dvieter dvieter: $(SOURCES) $(V_PATH) -showcc -keepc -cg -o dvieter $(SRC_DIR) +# Run the debug build inside gdb .PHONY: gdb gdb: dvieter VIETER_API_KEY=test \ diff --git a/src/build.v b/src/build.v index ab72a592..fc1fe6ff 100644 --- a/src/build.v +++ b/src/build.v @@ -4,7 +4,6 @@ import docker import encoding.base64 import rand import time -import os import json import server import env @@ -16,7 +15,7 @@ fn build() ? { conf := env.load() ? // We get the repos list from the Vieter instance - mut req := http.new_request(http.Method.get, '${conf.address}/api/repos', '') ? + mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? req.add_custom_header('X-Api-Key', conf.api_key) ? res := req.do() ? diff --git a/src/env.v b/src/env.v index 8b92d8dc..bd544cfc 100644 --- a/src/env.v +++ b/src/env.v @@ -22,8 +22,8 @@ pub: pub struct BuildConfig { pub: - api_key string - address string + api_key string + address string } fn get_env_var(field_name string) ?string { @@ -56,7 +56,10 @@ fn get_env_var(field_name string) ?string { } } -// load attempts to create the given type from environment variables. +// load attempts to create the given type from environment variables. For +// each field, the corresponding env var is its name in uppercase prepended +// with the hardcoded prefix. If this one isn't present, it looks for the env +// var with the file_suffix suffix. pub fn load() ?T { res := T{} diff --git a/src/server/git.v b/src/server/git.v index d506c7e9..0147d878 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -42,7 +42,7 @@ fn write_repos(path string, repos []GitRepo) ? { } ['/api/repos'; get] -pub fn (mut app App) get_repos() web.Result { +fn (mut app App) get_repos() web.Result { if !app.is_authorized() { return app.text('Unauthorized.') } @@ -59,7 +59,7 @@ pub fn (mut app App) get_repos() web.Result { } ['/api/repos'; post] -pub fn (mut app App) post_repo() web.Result { +fn (mut app App) post_repo() web.Result { if !app.is_authorized() { return app.text('Unauthorized.') } @@ -98,7 +98,7 @@ pub fn (mut app App) post_repo() web.Result { } ['/api/repos'; delete] -pub fn (mut app App) delete_repo() web.Result { +fn (mut app App) delete_repo() web.Result { if !app.is_authorized() { return app.text('Unauthorized.') } diff --git a/src/server/server.v b/src/server/server.v index ebb9f3bb..4b31b99e 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -19,6 +19,7 @@ pub mut: git_mutex shared util.Dummy } +// server starts the web server & starts listening for requests pub fn server() ? { conf := env.load() ? diff --git a/src/util.v b/src/util.v index 24d33fca..49c9d223 100644 --- a/src/util.v +++ b/src/util.v @@ -15,12 +15,15 @@ pub struct Dummy { x int } +// exit_with_message exits the program with a given status code after having +// first printed a specific message to STDERR [noreturn] pub fn exit_with_message(code int, msg string) { eprintln(msg) exit(code) } +// reader_to_file writes the contents of a BufferedReader to a file pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { mut file := os.create(path) ? defer { From 9f46c2723227baae46b0d0d326d05e93a7dda9e7 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 22 Feb 2022 10:11:18 +0100 Subject: [PATCH 029/105] Very basic CLI to update repos --- cli.v | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 cli.v diff --git a/cli.v b/cli.v new file mode 100644 index 00000000..256b8564 --- /dev/null +++ b/cli.v @@ -0,0 +1,84 @@ +import os +import toml +import net.http + +struct Config { + address string [required] + api_key string [required] +} + +fn list(conf Config) ? { + mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? + req.add_custom_header('X-API-Key', conf.api_key) ? + + res := req.do() ? + + println(res.text) +} + +fn add(conf Config, args []string) ? { + if args.len < 2 { + eprintln('Not enough arguments.') + exit(1) + } + + if args.len > 2 { + eprintln('Too many arguments.') + exit(1) + } + + mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=${args[0]}&branch=${args[1]}', '') ? + req.add_custom_header('X-API-Key', conf.api_key) ? + + res := req.do() ? + + println(res.text) +} + +fn remove(conf Config, args []string) ? { + if args.len < 2 { + eprintln('Not enough arguments.') + exit(1) + } + + if args.len > 2 { + eprintln('Too many arguments.') + exit(1) + } + + mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=${args[0]}&branch=${args[1]}', '') ? + req.add_custom_header('X-API-Key', conf.api_key) ? + + res := req.do() ? + + println(res.text) +} + +fn main() { + conf_path := os.expand_tilde_to_home('~/.vieterrc') + + if !os.is_file(conf_path) { + exit(1) + } + + conf := toml.parse_file(conf_path) ?.reflect() + + args := os.args[1..] + + if args.len == 0 { + eprintln('No action provided.') + exit(1) + } + + action := args[0] + + match action { + 'list' { list(conf) ? } + 'add' { add(conf, args[1..]) ? } + 'remove' { remove(conf, args[1..]) ? } + else { + eprintln("Invalid action '$action'.") + exit(1) + } + } +} From f6b0e29552d3a1a21c90864bcf9d6b64b4ed0cb5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 22 Feb 2022 18:04:54 +0100 Subject: [PATCH 030/105] Merged dockerfiles --- .gitignore | 2 ++ .woodpecker/.docker.yml | 2 -- Dockerfile | 49 +++++++++++++++++++++++++++++++++++----- Dockerfile.ci | 50 ----------------------------------------- Makefile | 11 +++++++++ 5 files changed, 56 insertions(+), 58 deletions(-) delete mode 100644 Dockerfile.ci diff --git a/.gitignore b/.gitignore index 3a6b11be..7847b3fb 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ data/ vieter dvieter pvieter +dvieterctl +vieterctl vieter.c # Ignore testing files diff --git a/.woodpecker/.docker.yml b/.woodpecker/.docker.yml index 6202fe6c..b2f08ca9 100644 --- a/.woodpecker/.docker.yml +++ b/.woodpecker/.docker.yml @@ -9,7 +9,6 @@ pipeline: secrets: [ docker_username, docker_password ] settings: repo: chewingbever/vieter - dockerfile: Dockerfile.ci tag: dev platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] build_args_from_env: @@ -23,7 +22,6 @@ pipeline: secrets: [ docker_username, docker_password ] settings: repo: chewingbever/vieter - dockerfile: Dockerfile.ci auto_tag: true platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] build_args_from_env: diff --git a/Dockerfile b/Dockerfile index 5564e34a..8b625217 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,56 @@ FROM chewingbever/vlang:latest AS builder +ARG TARGETPLATFORM +ARG CI_COMMIT_SHA +ARG DI_VER=1.2.5 + WORKDIR /app +# Build dumb-init +RUN curl -Lo - "https://github.com/Yelp/dumb-init/archive/refs/tags/v${DI_VER}.tar.gz" | tar -xzf - && \ + cd "dumb-init-${DI_VER}" && \ + make SHELL=/bin/sh && \ + mv dumb-init .. && \ + cd .. + # Copy over source code & build production binary COPY src ./src COPY Makefile ./ -ENV LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' -RUN v -o pvieter -cflags "-O3" src +RUN if [ -n "${CI_COMMIT_SHA}" ]; then \ + curl --fail \ + -o vieter \ + "https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \ + chmod +x vieter ; \ + else \ + LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' make prod && \ + mv pvieter vieter ; \ + fi -FROM alpine:3.15 +FROM busybox:1.35.0 -ENV REPO_DIR=/data +ENV PATH=/bin \ + VIETER_REPO_DIR=/data/repo \ + VIETER_PKG_DIR=/data/pkgs \ + VIETER_DOWNLOAD_DIR=/data/downloads \ + VIETER_REPOS_FILE=/data/repos.json -COPY --from=builder /app/pvieter /usr/local/bin/vieter +COPY --from=builder /app/dumb-init /app/vieter /bin/ -ENTRYPOINT [ "/usr/local/bin/vieter" ] +HEALTHCHECK --interval=30s \ + --timeout=3s \ + --start-period=5s \ + CMD /bin/wget --spider http://localhost:8000/health || exit 1 + +RUN mkdir /data && \ + chown -R www-data:www-data /data && \ + mkdir -p '/var/spool/cron/crontabs' && \ + echo '0 3 * * * /bin/vieter build' | crontab - + +WORKDIR /data + +USER www-data:www-data + +ENTRYPOINT ["/bin/dumb-init", "--"] +CMD ["/bin/vieter", "server"] diff --git a/Dockerfile.ci b/Dockerfile.ci deleted file mode 100644 index a062d951..00000000 --- a/Dockerfile.ci +++ /dev/null @@ -1,50 +0,0 @@ -# vim: ft=dockerfile -# This image just has the required tools to download the binaries -FROM chewingbever/vlang:latest AS builder - -ARG TARGETPLATFORM -ARG CI_COMMIT_SHA -ARG DI_VER=1.2.5 - -WORKDIR /app - -# Build dumb-init -RUN curl -Lo - "https://github.com/Yelp/dumb-init/archive/refs/tags/v${DI_VER}.tar.gz" | tar -xzf - && \ - cd "dumb-init-${DI_VER}" && \ - make SHELL=/bin/sh && \ - mv dumb-init .. && \ - cd .. - -RUN curl --fail \ - -o vieter \ - "https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \ - chmod +x vieter - - -FROM busybox:1.35.0 - -ENV PATH=/bin \ - VIETER_REPO_DIR=/data/repo \ - VIETER_PKG_DIR=/data/pkgs \ - VIETER_DOWNLOAD_DIR=/data/downloads \ - VIETER_REPOS_FILE=/data/repos.json - -COPY --from=builder /app/dumb-init /app/vieter /bin/ - -HEALTHCHECK --interval=30s \ - --timeout=3s \ - --start-period=5s \ - CMD /bin/wget --spider http://localhost:8000/health || exit 1 - -RUN mkdir /data && \ - chown -R www-data:www-data /data && \ - mkdir -p '/var/spool/cron/crontabs' && \ - echo '0 3 * * * /bin/vieter build' >> /var/spool/cron/crontabs/www-data && \ - chown www-data:www-data /var/spool/cron/crontabs/www-data - -WORKDIR /data - -USER www-data:www-data - -ENTRYPOINT ["/bin/dumb-init", "--"] -CMD ["/bin/vieter", "server"] diff --git a/Makefile b/Makefile index 062a47f8..bd629c93 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,17 @@ pvieter: $(SOURCES) c: $(V) -o vieter.c $(SRC_DIR) +# Build the CLI tool +.PHONY: cli +cli: dvieterctl +dvieterctl: cli.v + $(V_PATH) -showcc -o dvieterctl cli.v + +.PHONY: cli-prod +cli-prod: vieterctl +vieterctl: cli.v +cli-prod: + $(V_PATH) -showcc -o vieterctl -prod cli.v # =====EXECUTION===== # Run the server in the default 'data' directory From 098f89909d3d80f7a1bfdf0f0277565ecce21a64 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 22 Feb 2022 18:42:09 +0100 Subject: [PATCH 031/105] Added cli to CI builds --- .woodpecker/.build.yml | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 66caa8c7..5bfe4293 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -36,18 +36,49 @@ pipeline: when: event: push + cli: + image: 'chewingbever/vlang:latest' + environment: + - LDFLAGS=-static + commands: + - make cli-prod + # Make sure the binary is actually statically built + - readelf -d vieterctl + - du -h vieterctl + - '[ "$(readelf -d vieterctl | grep NEEDED | wc -l)" = 0 ]' + # This removes so much, it's amazing + - strip -s vieterctl + - du -h vieterctl + when: + event: push + upload: image: 'chewingbever/vlang:latest' secrets: [ s3_username, s3_password ] commands: # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f - export URL=s3.rustybever.be - - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')" - export DATE="$(date -R --utc)" - export CONTENT_TYPE='application/zstd' + + - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')" - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH" - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64` + - > + curl + --silent + -XPUT + -T pvieter + -H "Host: $URL" + -H "Date: $DATE" + -H "Content-Type: $CONTENT_TYPE" + -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" + https://$URL$OBJ_PATH + # Also update the CLI tool + - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieterctl-$(echo '${PLATFORM}' | sed 's:/:-:g')" + - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH" + - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64` - > curl --silent From 13e04cd615c2450660f45ac0a43078a42a52a07c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 22 Feb 2022 18:48:29 +0100 Subject: [PATCH 032/105] Wrong filename in CI --- .woodpecker/.build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 5bfe4293..4cddc6ad 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -83,7 +83,7 @@ pipeline: curl --silent -XPUT - -T pvieter + -T vieterctl -H "Host: $URL" -H "Date: $DATE" -H "Content-Type: $CONTENT_TYPE" From d5c1366d650be1d93b97e9269573be52c7607dbf Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 22 Feb 2022 20:46:28 +0100 Subject: [PATCH 033/105] Failed attempt at writing PKGBUILD [CI SKIP] --- CHANGELOG.md | 11 ++++++++--- Makefile | 4 ++-- PKGBUILD | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 PKGBUILD diff --git a/CHANGELOG.md b/CHANGELOG.md index edae68bd..9eaf477f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter) +## Changed + +* Better environment variable support + * Each env var can now be provided from a file by appending it with `_FILE` + & passing the path to the file as value + ## Added * Very basic build system @@ -15,10 +21,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Packages are always rebuilt, even if they haven't changed * Hardcoded planning of builds * Builds are sequential -* Better environment variable support - * Each env var can now be provided from a file by appending it with `_FILE` - & passing the path to the file as value * API for managing Git repositories to build +* CLI to list, add & remove Git repos to build +* Published packages on my Vieter instance ## Fixed diff --git a/Makefile b/Makefile index bd629c93..9debf8cf 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ c: .PHONY: cli cli: dvieterctl dvieterctl: cli.v - $(V_PATH) -showcc -o dvieterctl cli.v + $(V_PATH) -showcc -g -o dvieterctl cli.v .PHONY: cli-prod cli-prod: vieterctl @@ -97,4 +97,4 @@ v/v: make -C v clean: - rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' + rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' diff --git a/PKGBUILD b/PKGBUILD new file mode 100644 index 00000000..a8426dca --- /dev/null +++ b/PKGBUILD @@ -0,0 +1,40 @@ +# Maintainer: Jef Roosens + +pkgbase='vieter' +pkgname=('vieter' 'vieterctl') +pkgver=0.1.0.rc1.r45.g6d3ff8a +pkgrel=1 +depends=('openssl' 'libarchive' 'gc') +arch=('x86_64' 'aarch64' 'armv7') +url='https://git.rustybever.be/Chewing_Bever/vieter' +license=('AGPL3') +source=($pkgname::git+https://git.rustybever.be/Chewing_Bever/vieter#branch=dev) +md5sums=('SKIP') + +pkgver() { + cd "$pkgname" + git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g' +} + +build() { + cd "$pkgname" + + # Build the compiler + make v + + # Build the server & the CLI tool + make prod + make cli-prod +} + +package_vieter() { + install -dm755 "$pkgdir/usr/bin" + + install -Dm755 "$pkgname/pvieter" "$pkgdir/usr/bin/vieter" +} + +package_vieterctl() { + install -dm755 "$pkgdir/usr/bin" + + install -Dm755 "$pkgname/vieterctl" "$pkgdir/usr/bin/vieterctl" +} From 00804bf11500bba0d4bc8348c577310f2738380c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 24 Feb 2022 14:00:40 +0100 Subject: [PATCH 034/105] Fix for broken PKGBUILD --- Makefile | 2 +- PKGBUILD | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 9debf8cf..7b0fb1b1 100644 --- a/Makefile +++ b/Makefile @@ -97,4 +97,4 @@ v/v: make -C v clean: - rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' + rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' diff --git a/PKGBUILD b/PKGBUILD index a8426dca..8926a635 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -4,7 +4,7 @@ pkgbase='vieter' pkgname=('vieter' 'vieterctl') pkgver=0.1.0.rc1.r45.g6d3ff8a pkgrel=1 -depends=('openssl' 'libarchive' 'gc') +depends=('glibc' 'openssl' 'libarchive' 'gc') arch=('x86_64' 'aarch64' 'armv7') url='https://git.rustybever.be/Chewing_Bever/vieter' license=('AGPL3') @@ -20,7 +20,7 @@ build() { cd "$pkgname" # Build the compiler - make v + CFLAGS= make v # Build the server & the CLI tool make prod @@ -30,11 +30,11 @@ build() { package_vieter() { install -dm755 "$pkgdir/usr/bin" - install -Dm755 "$pkgname/pvieter" "$pkgdir/usr/bin/vieter" + install -Dm755 "$pkgbase/pvieter" "$pkgdir/usr/bin/vieter" } package_vieterctl() { install -dm755 "$pkgdir/usr/bin" - install -Dm755 "$pkgname/vieterctl" "$pkgdir/usr/bin/vieterctl" + install -Dm755 "$pkgbase/vieterctl" "$pkgdir/usr/bin/vieterctl" } From aa25290f4f98906b54d1c3c823ab1616b79ab2d7 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 24 Feb 2022 14:04:29 +0100 Subject: [PATCH 035/105] Added CI step to publish dev arch packages --- .woodpecker/.arch.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .woodpecker/.arch.yml diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml new file mode 100644 index 00000000..03f5d29d --- /dev/null +++ b/.woodpecker/.arch.yml @@ -0,0 +1,25 @@ +platform: linux/amd64 +branches: [dev] + +pipeline: + build: + image: 'archlinux:latest' + commands: + # Update packages + - pacman -Syu --needed --noconfirm base-devel + # Create non-root user to perform build & switch to their home + - groupadd -g 1000 builder + - useradd -mg builder builder + - chown -R builder:builder "$PWD" + - "echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers" + - su builder + # Build the package + - makepkg -s --noconfirm --needed + + publish: + image: 'archlinux:latest' + commands: + # Publish the package + - 'curl -F "file=@$(ls *.pkg*)" -H "X-API-KEY: $VIETER_API_KEY" https://pkgs.rustybever.be/api/publish' + secrets: + - vieter_api_key From 9f8765387701380a39f56afaadd5d8cb648b9a71 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 24 Feb 2022 14:18:25 +0100 Subject: [PATCH 036/105] Added makedepends to PKGBUILD --- PKGBUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/PKGBUILD b/PKGBUILD index 8926a635..7d32df83 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -5,6 +5,7 @@ pkgname=('vieter' 'vieterctl') pkgver=0.1.0.rc1.r45.g6d3ff8a pkgrel=1 depends=('glibc' 'openssl' 'libarchive' 'gc') +makedepends=('git' 'gcc') arch=('x86_64' 'aarch64' 'armv7') url='https://git.rustybever.be/Chewing_Bever/vieter' license=('AGPL3') From 2cf0286413277370bd123b3eb94f577b8799ccb2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 24 Feb 2022 14:24:57 +0100 Subject: [PATCH 037/105] This should fix the arch publish --- .woodpecker/.arch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index 03f5d29d..0417dbe8 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -20,6 +20,6 @@ pipeline: image: 'archlinux:latest' commands: # Publish the package - - 'curl -F "file=@$(ls *.pkg*)" -H "X-API-KEY: $VIETER_API_KEY" https://pkgs.rustybever.be/api/publish' + - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "${pkg}" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/publish; done' secrets: - vieter_api_key From 6276fbe0d3f7cb54f3efa16afe8ef0f47190eccc Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 24 Feb 2022 22:06:14 +0100 Subject: [PATCH 038/105] Added arch package descriptions --- PKGBUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/PKGBUILD b/PKGBUILD index 7d32df83..767c07c1 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -29,12 +29,14 @@ build() { } package_vieter() { + pkgdesc="Vieter is a lightweight implementation of an Arch repository server." install -dm755 "$pkgdir/usr/bin" install -Dm755 "$pkgbase/pvieter" "$pkgdir/usr/bin/vieter" } package_vieterctl() { + pkgdesc="Allows you to configure a Vieter server's list of Git repositories." install -dm755 "$pkgdir/usr/bin" install -Dm755 "$pkgbase/vieterctl" "$pkgdir/usr/bin/vieterctl" From 13a2ced6f9f126200b44fbb7a0eac3085dfd6ab0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 24 Feb 2022 22:13:23 +0100 Subject: [PATCH 039/105] Possible fix for CI arch build --- .woodpecker/.arch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index 0417dbe8..d6463532 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -20,6 +20,6 @@ pipeline: image: 'archlinux:latest' commands: # Publish the package - - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "${pkg}" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/publish; done' + - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/publish; done' secrets: - vieter_api_key From 540574b3c3c4d71df804e2ecd42028273f95e28d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 25 Feb 2022 20:52:30 +0100 Subject: [PATCH 040/105] Split builds into separate containers; made makepkg parallel --- src/build.v | 99 ++++++++++++++++++++++++++++++--------------- src/docker/docker.v | 5 --- src/docker/images.v | 31 ++++++++++++++ 3 files changed, 98 insertions(+), 37 deletions(-) create mode 100644 src/docker/images.v diff --git a/src/build.v b/src/build.v index fc1fe6ff..d299d39a 100644 --- a/src/build.v +++ b/src/build.v @@ -10,18 +10,10 @@ import env import net.http const container_build_dir = '/build' +const build_image_repo = 'vieter-build' -fn build() ? { - conf := env.load() ? - - // We get the repos list from the Vieter instance - mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? - req.add_custom_header('X-Api-Key', conf.api_key) ? - - res := req.do() ? - repos := json.decode([]server.GitRepo, res.text) ? - - mut commands := [ +fn create_build_image() ?string { + commands := [ // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' // Add a non-root user to run makepkg @@ -34,31 +26,11 @@ fn build() ? { 'mkdir /build', 'chown -R builder:builder /build', ] - - // Each repo gets a unique UUID to avoid naming conflicts when cloning - mut uuids := []string{} - - for repo in repos { - mut uuid := rand.uuid_v4() - - // Just to be sure we don't have any collisions - for uuids.contains(uuid) { - uuid = rand.uuid_v4() - } - - uuids << uuid - - commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'" - commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\'' - } - - // We convert the list of commands into a base64 string, which then gets - // passed to the container as an env var cmds_str := base64.encode_str(commands.join('\n')) c := docker.NewContainer{ image: 'archlinux:latest' - env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key'] + env: ['BUILD_SCRIPT=$cmds_str'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] } @@ -81,5 +53,68 @@ fn build() ? { time.sleep(5000000000) } + // Finally, we create the image from the container + // As the tag, we use the epoch value + tag := time.sys_mono_now().str() + image := docker.create_image_from_container(id, 'vieter-build', tag) ? docker.remove_container(id) ? + + return image.id +} + +fn build() ? { + conf := env.load() ? + + // We get the repos list from the Vieter instance + mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? + req.add_custom_header('X-Api-Key', conf.api_key) ? + + res := req.do() ? + repos := json.decode([]server.GitRepo, res.text) ? + + // No point in doing work if there's no repos present + if repos.len == 0 { + return + } + + // First, we create a base image which has updated repos n stuff + image_id := create_build_image() ? + + for repo in repos { + commands := [ + "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/repo'" + 'su builder -c \'cd /build/repo && MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\'' + ] + + // We convert the list of commands into a base64 string, which then gets + // passed to the container as an env var + cmds_str := base64.encode_str(commands.join('\n')) + + c := docker.NewContainer{ + image: '$image_id' + env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key'] + entrypoint: ['/bin/sh', '-c'] + cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] + } + + id := docker.create_container(c) ? + docker.start_container(id) ? + + // This loop waits until the container has stopped, so we can remove it after + for { + data := docker.inspect_container(id) ? + + if !data.state.running { + break + } + + // Wait for 5 seconds + time.sleep(5000000000) + } + + docker.remove_container(id) ? + } + + // Finally, we remove the builder image + docker.remove_image(image_id) ? } diff --git a/src/docker/docker.v b/src/docker/docker.v index e0dbf7d7..a6f76409 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -91,8 +91,3 @@ pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Respon return request_with_body(method, url, 'application/json', body) } - -// pull_image pulls tries to pull the image for the given image & tag -pub fn pull_image(image string, tag string) ?http.Response { - return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) -} diff --git a/src/docker/images.v b/src/docker/images.v new file mode 100644 index 00000000..51a315dd --- /dev/null +++ b/src/docker/images.v @@ -0,0 +1,31 @@ +module docker + +import net.http +import net.urllib +import json + +struct Image { +pub: + id string [json: Id] +} + +// pull_image pulls tries to pull the image for the given image & tag +pub fn pull_image(image string, tag string) ?http.Response { + return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) +} + +pub fn create_image_from_container(id string, repo string, tag string) ?Image { + res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag') ?) ? + + if res.status_code != 201 { + return error('Failed to create image from container.') + } + + return json.decode(Image, res.text) or {} +} + +pub fn remove_image(id string) ?bool { + res := request('DELETE', urllib.parse('/v1.41/images/$id') ?) ? + + return res.status_code == 200 +} From a7cb08f93d96dcc7cfe055f4d5dea1fa5c74f3b2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 25 Feb 2022 21:47:28 +0100 Subject: [PATCH 041/105] Containers now check whether package should be rebuilt --- src/build.v | 16 ++++++++++++---- src/docker/containers.v | 2 ++ src/server/routes.v | 12 +++++++++++- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/build.v b/src/build.v index d299d39a..167b492f 100644 --- a/src/build.v +++ b/src/build.v @@ -81,9 +81,15 @@ fn build() ? { image_id := create_build_image() ? for repo in repos { + // TODO what to do with PKGBUILDs that build multiple packages? commands := [ - "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/repo'" - 'su builder -c \'cd /build/repo && MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\'' + "git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo" + 'cd repo' + "makepkg --nobuild --nodeps" + 'source PKGBUILD' + // The build container checks whether the package is already present on the server + "curl --head --fail $conf.address/\$pkgname-\$pkgver-\$pkgrel-\$(uname -m).pkg.tar.zst && exit 0" + 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done' ] // We convert the list of commands into a base64 string, which then gets @@ -94,7 +100,9 @@ fn build() ? { image: '$image_id' env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key'] entrypoint: ['/bin/sh', '-c'] - cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] + cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e'] + work_dir: '/build' + user: 'builder:builder' } id := docker.create_container(c) ? @@ -112,7 +120,7 @@ fn build() ? { time.sleep(5000000000) } - docker.remove_container(id) ? + // docker.remove_container(id) ? } // Finally, we remove the builder image diff --git a/src/docker/containers.v b/src/docker/containers.v index a6df3459..37ff5162 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -20,6 +20,8 @@ pub struct NewContainer { entrypoint []string [json: Entrypoint] cmd []string [json: Cmd] env []string [json: Env] + work_dir string [json: WorkingDir] + user string [json: User] } struct CreatedContainer { diff --git a/src/server/routes.v b/src/server/routes.v index 7a0ee387..0090666e 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -6,6 +6,7 @@ import repo import time import rand import util +import net.http // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. @@ -15,7 +16,7 @@ pub fn (mut app App) healthcheck() web.Result { } // get_root handles a GET request for a file on the root -['/:filename'; get] +['/:filename'; get; head] fn (mut app App) get_root(filename string) web.Result { mut full_path := '' @@ -27,6 +28,15 @@ fn (mut app App) get_root(filename string) web.Result { full_path = os.join_path_single(app.repo.pkg_dir, filename) } + // Scuffed way to respond to HEAD requests + if app.req.method == http.Method.head { + if os.exists(full_path) { + return app.ok('') + } + + return app.not_found() + } + return app.file(full_path) } From 732b53b6cec8039b018e157bb30b672c602aa786 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 25 Feb 2022 21:50:07 +0100 Subject: [PATCH 042/105] Forgot to uncomment line --- src/build.v | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build.v b/src/build.v index 167b492f..d1fb21d8 100644 --- a/src/build.v +++ b/src/build.v @@ -120,7 +120,7 @@ fn build() ? { time.sleep(5000000000) } - // docker.remove_container(id) ? + docker.remove_container(id) ? } // Finally, we remove the builder image From 4a4362c1385934a3cdc5404199bfcc6e4e705a73 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 25 Feb 2022 21:54:16 +0100 Subject: [PATCH 043/105] Fixed linting errors --- src/build.v | 14 +++++++------- src/docker/containers.v | 4 ++-- src/docker/images.v | 3 +++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/build.v b/src/build.v index d1fb21d8..51de64c4 100644 --- a/src/build.v +++ b/src/build.v @@ -2,7 +2,6 @@ module main import docker import encoding.base64 -import rand import time import json import server @@ -10,6 +9,7 @@ import env import net.http const container_build_dir = '/build' + const build_image_repo = 'vieter-build' fn create_build_image() ?string { @@ -83,13 +83,13 @@ fn build() ? { for repo in repos { // TODO what to do with PKGBUILDs that build multiple packages? commands := [ - "git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo" - 'cd repo' - "makepkg --nobuild --nodeps" - 'source PKGBUILD' + 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', + 'cd repo', + 'makepkg --nobuild --nodeps', + 'source PKGBUILD', // The build container checks whether the package is already present on the server - "curl --head --fail $conf.address/\$pkgname-\$pkgver-\$pkgrel-\$(uname -m).pkg.tar.zst && exit 0" - 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done' + 'curl --head --fail $conf.address/\$pkgname-\$pkgver-\$pkgrel-\$(uname -m).pkg.tar.zst && exit 0', + 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done', ] // We convert the list of commands into a base64 string, which then gets diff --git a/src/docker/containers.v b/src/docker/containers.v index 37ff5162..d0f5a4d7 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -20,8 +20,8 @@ pub struct NewContainer { entrypoint []string [json: Entrypoint] cmd []string [json: Cmd] env []string [json: Env] - work_dir string [json: WorkingDir] - user string [json: User] + work_dir string [json: WorkingDir] + user string [json: User] } struct CreatedContainer { diff --git a/src/docker/images.v b/src/docker/images.v index 51a315dd..e94ceca2 100644 --- a/src/docker/images.v +++ b/src/docker/images.v @@ -14,6 +14,8 @@ pub fn pull_image(image string, tag string) ?http.Response { return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) } +// create_image_from_container creates a new image from a container with the +// given repo & tag, given the container's ID. pub fn create_image_from_container(id string, repo string, tag string) ?Image { res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag') ?) ? @@ -24,6 +26,7 @@ pub fn create_image_from_container(id string, repo string, tag string) ?Image { return json.decode(Image, res.text) or {} } +// remove_image removes the image with the given ID. pub fn remove_image(id string) ?bool { res := request('DELETE', urllib.parse('/v1.41/images/$id') ?) ? From 88e048fbe2ef908123b482b8082709769d92ee67 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 25 Feb 2022 22:07:03 +0100 Subject: [PATCH 044/105] Updated deploy webhooks [CI SKIP] --- .woodpecker/.deploy.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.woodpecker/.deploy.yml b/.woodpecker/.deploy.yml index 56e6c2f9..8cf39071 100644 --- a/.woodpecker/.deploy.yml +++ b/.woodpecker/.deploy.yml @@ -10,6 +10,8 @@ pipeline: webhook: image: chewingbever/vlang:latest secrets: - - webhook + - webhook_1 + - webhook_2 commands: - - curl -XPOST -s "$WEBHOOK" + - curl -XPOST -s "$WEBHOOK_1" + - curl -XPOST -s "$WEBHOOK_2" From 013ce511d774dc60d78423fd2dec954ab5cbd1b2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 15:10:45 +0200 Subject: [PATCH 045/105] Loosed up SRCINFO field requirements (fixes #111) --- src/package.v | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/package.v b/src/package.v index 4e3b97f8..d643ff1e 100644 --- a/src/package.v +++ b/src/package.v @@ -72,14 +72,11 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { 'pkgbase' { pkg_info.base = value } 'pkgver' { pkg_info.version = value } 'pkgdesc' { pkg_info.description = value } - 'csize' { continue } 'size' { pkg_info.size = value.int() } 'url' { pkg_info.url = value } 'arch' { pkg_info.arch = value } 'builddate' { pkg_info.build_date = value.int() } 'packager' { pkg_info.packager = value } - 'md5sum' { continue } - 'sha256sum' { continue } 'pgpsig' { pkg_info.pgpsig = value } 'pgpsigsize' { pkg_info.pgpsigsize = value.int() } // Array values @@ -92,7 +89,10 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { 'optdepend' { pkg_info.optdepends << value } 'makedepend' { pkg_info.makedepends << value } 'checkdepend' { pkg_info.checkdepends << value } - else { return error("Invalid key '$key'.") } + // There's no real point in trying to exactly manage which fields + // are allowed, so we just ignore any we don't explicitely need for + // in the db file + else { continue } } } From a47cace2964740833916e181084273cec8381a48 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 16:33:06 +0200 Subject: [PATCH 046/105] Very alpha support for multiple & multi-arch repos --- Makefile | 4 +- src/env.v | 2 +- src/package.v | 2 +- src/repo/repo.v | 93 ++++++++++++++++++++++++++++++--------------- src/repo/sync.v | 15 ++++---- src/server/routes.v | 8 ++-- src/server/server.v | 4 +- test.py | 2 +- 8 files changed, 83 insertions(+), 47 deletions(-) diff --git a/Makefile b/Makefile index 7b0fb1b1..9996cc3c 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ dvieter: $(SOURCES) gdb: dvieter VIETER_API_KEY=test \ VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ + VIETER_DATA_DIR=data/repo \ VIETER_PKG_DIR=data/pkgs \ VIETER_LOG_LEVEL=DEBUG \ VIETER_REPOS_FILE=data/repos.json \ @@ -60,7 +60,7 @@ cli-prod: run: vieter VIETER_API_KEY=test \ VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ + VIETER_DATA_DIR=data/repo \ VIETER_PKG_DIR=data/pkgs \ VIETER_LOG_LEVEL=DEBUG \ VIETER_REPOS_FILE=data/repos.json \ diff --git a/src/env.v b/src/env.v index bd544cfc..3e71a091 100644 --- a/src/env.v +++ b/src/env.v @@ -16,7 +16,7 @@ pub: pkg_dir string download_dir string api_key string - repo_dir string + data_dir string repos_file string } diff --git a/src/package.v b/src/package.v index d643ff1e..e0eeac28 100644 --- a/src/package.v +++ b/src/package.v @@ -101,7 +101,7 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { // read_pkg extracts the file list & .PKGINFO contents from an archive // NOTE: this command currently only supports zstd-compressed tarballs -pub fn read_pkg(pkg_path string) ?Pkg { +pub fn read_pkg_archive(pkg_path string) ?Pkg { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") } diff --git a/src/repo/repo.v b/src/repo/repo.v index f1419aca..d9ae06e3 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -4,15 +4,20 @@ import os import package import util -// This struct manages a single repository. -pub struct Repo { +// Manages a group of repositories. Each repository contains one or more +// arch-repositories, each of which represent a specific architecture. +pub struct RepoGroupManager { mut: mutex shared util.Dummy pub: - // Where to store repository files - repo_dir string [required] - // Where to find packages; packages are expected to all be in the same directory + // Where to store repositories' files + data_dir string [required] + // Where packages are stored; each architecture gets its own subdirectory pkg_dir string [required] + // The default architecture to use for a repository. In reality, this value + // is only required when a package with architecture "any" is added as the + // first package to a repository. + default_arch string [required] } pub struct RepoAddResult { @@ -21,28 +26,29 @@ pub: pkg &package.Pkg [required] } -// new creates a new Repo & creates the directories as needed -pub fn new(repo_dir string, pkg_dir string) ?Repo { - if !os.is_dir(repo_dir) { - os.mkdir_all(repo_dir) or { return error('Failed to create repo directory: $err.msg') } +// new creates a new RepoGroupManager & creates the directories as needed +pub fn new(data_dir string, pkg_dir string, default_arch string) ?RepoGroupManager { + if !os.is_dir(data_dir) { + os.mkdir_all(data_dir) or { return error('Failed to create repo directory: $err.msg') } } if !os.is_dir(pkg_dir) { os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg') } } - return Repo{ - repo_dir: repo_dir + return RepoGroupManager{ + data_dir: data_dir pkg_dir: pkg_dir + default_arch: default_arch } } // add_from_path adds a package from an arbitrary path & moves it into the pkgs // directory if necessary. -pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult { - pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') } +pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult { + pkg := package.read_pkg_archive(pkg_path) or { return error('Failed to read package file: $err.msg') } - added := r.add(pkg) ? + added := r.add_pkg_in_repo(repo, pkg) ? // If the add was successful, we move the file to the packages directory if added { @@ -60,9 +66,33 @@ pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult { } } -// add adds a given Pkg to the repository -fn (r &Repo) add(pkg &package.Pkg) ?bool { - pkg_dir := r.pkg_path(pkg) +fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { + if pkg.info.arch == "any" { + repo_dir := os.join_path_single(r.data_dir, repo) + + // We get a listing of all currently present arch-repos in the given repo + mut arch_repos := os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, it))) + + if arch_repos.len == 0 { + arch_repos << r.default_arch + } + + for arch in arch_repos { + r.add_pkg_in_arch_repo(repo, arch, pkg) ? + } + }else{ + r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) ? + } + + // TODO properly handle this + return true +} + +// add_pkg_in_repo adds the given package to the specified repo. A repo is an +// arbitrary subdirectory of r.repo_dir, but in practice, it will always be an +// architecture-specific version of some sub-repository. +fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool { + pkg_dir := os.join_path(r.data_dir, repo, arch, '$pkg.info.name-$pkg.info.version') // We can't add the same package twice if os.exists(pkg_dir) { @@ -70,9 +100,9 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool { } // We remove the older package version first, if present - r.remove(pkg.info.name, false) ? + r.remove_pkg_from_arch_repo(repo, arch, pkg, false) ? - os.mkdir(pkg_dir) or { return error('Failed to create package directory.') } + os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') } os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or { os.rmdir_all(pkg_dir) ? @@ -85,27 +115,35 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool { return error('Failed to write files file.') } - r.sync() ? + r.sync(repo, arch) ? return true } // remove removes a package from the database. It returns false if the package // wasn't present in the database. -fn (r &Repo) remove(pkg_name string, sync bool) ?bool { +fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg &package.Pkg, sync bool) ?bool { + repo_dir := os.join_path(r.data_dir, repo, arch) + + // If the repository doesn't exist yet, the result is automatically false + if !os.exists(repo_dir) { + return false + } + // We iterate over every directory in the repo dir - for d in os.ls(r.repo_dir) ? { + // TODO filter so we only check directories + for d in os.ls(repo_dir) ? { name := d.split('-')#[..-2].join('-') - if name == pkg_name { + if name == pkg.info.name { // We lock the mutex here to prevent other routines from creating a // new archive while we removed an entry lock r.mutex { - os.rmdir_all(os.join_path_single(r.repo_dir, d)) ? + os.rmdir_all(os.join_path_single(repo_dir, d)) ? } if sync { - r.sync() ? + r.sync(repo, arch) ? } return true @@ -114,8 +152,3 @@ fn (r &Repo) remove(pkg_name string, sync bool) ?bool { return false } - -// Returns the path where the given package's desc & files files are stored -fn (r &Repo) pkg_path(pkg &package.Pkg) string { - return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version') -} diff --git a/src/repo/sync.v b/src/repo/sync.v index d6080e07..c0c0de95 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -30,8 +30,9 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri } // Re-generate the repo archive files -fn (r &Repo) sync() ? { - // TODO also write files archive +fn (r &RepoGroupManager) sync(repo string, arch string) ? { + subrepo_path := os.join_path(r.data_dir, repo, arch) + lock r.mutex { a_db := C.archive_write_new() a_files := C.archive_write_new() @@ -44,18 +45,18 @@ fn (r &Repo) sync() ? { C.archive_write_add_filter_gzip(a_files) C.archive_write_set_format_pax_restricted(a_files) - db_path := os.join_path_single(r.repo_dir, 'vieter.db.tar.gz') - files_path := os.join_path_single(r.repo_dir, 'vieter.files.tar.gz') + db_path := os.join_path_single(subrepo_path, 'vieter.db.tar.gz') + files_path := os.join_path_single(subrepo_path, 'vieter.files.tar.gz') C.archive_write_open_filename(a_db, &char(db_path.str)) C.archive_write_open_filename(a_files, &char(files_path.str)) // Iterate over each directory - for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, + for d in os.ls(subrepo_path) ?.filter(os.is_dir(os.join_path_single(subrepo_path, it))) { // desc mut inner_path := os.join_path_single(d, 'desc') - mut actual_path := os.join_path_single(r.repo_dir, inner_path) + mut actual_path := os.join_path_single(subrepo_path, inner_path) archive_add_entry(a_db, entry, actual_path, inner_path) archive_add_entry(a_files, entry, actual_path, inner_path) @@ -64,7 +65,7 @@ fn (r &Repo) sync() ? { // files inner_path = os.join_path_single(d, 'files') - actual_path = os.join_path_single(r.repo_dir, inner_path) + actual_path = os.join_path_single(subrepo_path, inner_path) archive_add_entry(a_files, entry, actual_path, inner_path) diff --git a/src/server/routes.v b/src/server/routes.v index 0090666e..9d92192a 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -8,6 +8,8 @@ import rand import util import net.http +const default_repo = "vieter" + // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. ['/health'; get] @@ -21,9 +23,9 @@ fn (mut app App) get_root(filename string) web.Result { mut full_path := '' if filename.ends_with('.db') || filename.ends_with('.files') { - full_path = os.join_path_single(app.repo.repo_dir, '${filename}.tar.gz') + full_path = os.join_path_single(app.repo.data_dir, '${filename}.tar.gz') } else if filename.ends_with('.db.tar.gz') || filename.ends_with('.files.tar.gz') { - full_path = os.join_path_single(app.repo.repo_dir, '$filename') + full_path = os.join_path_single(app.repo.data_dir, '$filename') } else { full_path = os.join_path_single(app.repo.pkg_dir, filename) } @@ -74,7 +76,7 @@ fn (mut app App) put_package() web.Result { return app.text("Content-Type header isn't set.") } - res := app.repo.add_from_path(pkg_path) or { + res := app.repo.add_pkg_from_path(default_repo, pkg_path) or { app.lerror('Error while adding package: $err.msg') os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } diff --git a/src/server/server.v b/src/server/server.v index 4b31b99e..27321ed1 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -14,7 +14,7 @@ struct App { pub: conf env.ServerConfig [required; web_global] pub mut: - repo repo.Repo [required; web_global] + repo repo.RepoGroupManager [required; web_global] // This is used to claim the file lock on the repos file git_mutex shared util.Dummy } @@ -42,7 +42,7 @@ pub fn server() ? { } // This also creates the directories if needed - repo := repo.new(conf.repo_dir, conf.pkg_dir) or { + repo := repo.new(conf.data_dir, conf.pkg_dir, "x86_64") or { logger.error(err.msg) exit(1) } diff --git a/test.py b/test.py index 5721310b..cb817b7e 100644 --- a/test.py +++ b/test.py @@ -46,7 +46,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len): "pkgname": name, "pkgbase": name, "pkgver": ver, - "arch": "x86_64" + "arch": "any" } return "\n".join(f"{key} = {value}" for key, value in data.items()) From 014ade50929d22224c78cd493f7f4151895387cc Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 17:00:11 +0200 Subject: [PATCH 047/105] Updated routes for multi-repo setup (untested) --- src/repo/repo.v | 8 ++++++-- src/server/routes.v | 27 +++++++++++++++------------ test.py | 2 +- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/repo/repo.v b/src/repo/repo.v index d9ae06e3..bf5ab024 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -70,8 +70,12 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { if pkg.info.arch == "any" { repo_dir := os.join_path_single(r.data_dir, repo) - // We get a listing of all currently present arch-repos in the given repo - mut arch_repos := os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, it))) + mut arch_repos := []string{} + + if os.exists(repo_dir) { + // We get a listing of all currently present arch-repos in the given repo + arch_repos = os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, it))) + } if arch_repos.len == 0 { arch_repos << r.default_arch diff --git a/src/server/routes.v b/src/server/routes.v index 9d92192a..55964dbf 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -8,8 +8,6 @@ import rand import util import net.http -const default_repo = "vieter" - // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. ['/health'; get] @@ -17,15 +15,20 @@ pub fn (mut app App) healthcheck() web.Result { return app.text('Healthy') } -// get_root handles a GET request for a file on the root -['/:filename'; get; head] -fn (mut app App) get_root(filename string) web.Result { +['/:repo/:arch/:filename'; get; head] +fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result { mut full_path := '' - if filename.ends_with('.db') || filename.ends_with('.files') { - full_path = os.join_path_single(app.repo.data_dir, '${filename}.tar.gz') - } else if filename.ends_with('.db.tar.gz') || filename.ends_with('.files.tar.gz') { - full_path = os.join_path_single(app.repo.data_dir, '$filename') + db_exts := ['.db', '.files', '.db.tar.gz', '.files.tar.gz'] + + if db_exts.any(filename.ends_with(it)) { + full_path = os.join_path(app.repo.data_dir, repo, arch, filename) + + // repo-add does this using symlinks, but we just change the requested + // path + if !full_path.ends_with('.tar.gz') { + full_path += '.tar.gz' + } } else { full_path = os.join_path_single(app.repo.pkg_dir, filename) } @@ -42,8 +45,8 @@ fn (mut app App) get_root(filename string) web.Result { return app.file(full_path) } -['/publish'; post] -fn (mut app App) put_package() web.Result { +['/:repo/publish'; post] +fn (mut app App) put_package(repo string) web.Result { if !app.is_authorized() { return app.text('Unauthorized.') } @@ -76,7 +79,7 @@ fn (mut app App) put_package() web.Result { return app.text("Content-Type header isn't set.") } - res := app.repo.add_pkg_from_path(default_repo, pkg_path) or { + res := app.repo.add_pkg_from_path(repo, pkg_path) or { app.lerror('Error while adding package: $err.msg') os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } diff --git a/test.py b/test.py index cb817b7e..9b0116ec 100644 --- a/test.py +++ b/test.py @@ -97,7 +97,7 @@ async def upload_random_package(tar_path, sem): async with sem: with open(tar_path, 'rb') as f: async with aiohttp.ClientSession() as s: - async with s.post("http://localhost:8000/publish", data=f.read(), headers={"x-api-key": "test"}) as r: + async with s.post("http://localhost:8000/vieter/publish", data=f.read(), headers={"x-api-key": "test"}) as r: return await check_output(r) From cb2ba862000b3b492c1cd4e80bc8aebbae12aa47 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 18:22:30 +0200 Subject: [PATCH 048/105] Updated logging for multi-repo setup --- CHANGELOG.md | 6 +++++- src/{archive.v => archive.c.v} | 0 src/repo/repo.v | 17 +++++++++-------- src/server/routes.v | 5 +++-- test.py | 4 ++-- 5 files changed, 19 insertions(+), 13 deletions(-) rename src/{archive.v => archive.c.v} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eaf477f..754f04e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,17 +18,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Very basic build system * Build is triggered by separate cron container * Packages build on cron container's system - * Packages are always rebuilt, even if they haven't changed + * A HEAD request is used to determine whether a package should be rebuilt + or not * Hardcoded planning of builds * Builds are sequential * API for managing Git repositories to build * CLI to list, add & remove Git repos to build * Published packages on my Vieter instance +* Support for multiple repositories +* Support for multiple architectures per repository ## Fixed * Each package can now only have one version in the repository at once (required by Pacman) +* Packages with unknown fields in .PKGINFO are now allowed ## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0) diff --git a/src/archive.v b/src/archive.c.v similarity index 100% rename from src/archive.v rename to src/archive.c.v diff --git a/src/repo/repo.v b/src/repo/repo.v index bf5ab024..e1c9fdbd 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -81,15 +81,16 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { arch_repos << r.default_arch } - for arch in arch_repos { - r.add_pkg_in_arch_repo(repo, arch, pkg) ? - } - }else{ - r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) ? - } + mut added := false - // TODO properly handle this - return true + for arch in arch_repos { + added = added || r.add_pkg_in_arch_repo(repo, arch, pkg) ? + } + + return added + }else{ + return r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) + } } // add_pkg_in_repo adds the given package to the specified repo. A repo is an diff --git a/src/server/routes.v b/src/server/routes.v index 55964dbf..0f697f9f 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -86,15 +86,16 @@ fn (mut app App) put_package(repo string) web.Result { return app.text('Failed to add package.') } + if !res.added { os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } - app.lwarn("Duplicate package '$res.pkg.full_name()'.") + app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo ($res.pkg.info.arch)'.") return app.text('File already exists.') } - app.linfo("Added '$res.pkg.full_name()' to repository.") + app.linfo("Added '$res.pkg.full_name()' to repo '$repo ($res.pkg.info.arch)'.") return app.text('Package added successfully.') } diff --git a/test.py b/test.py index 9b0116ec..f1668d7b 100644 --- a/test.py +++ b/test.py @@ -46,7 +46,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len): "pkgname": name, "pkgbase": name, "pkgver": ver, - "arch": "any" + "arch": "x86_64" } return "\n".join(f"{key} = {value}" for key, value in data.items()) @@ -97,7 +97,7 @@ async def upload_random_package(tar_path, sem): async with sem: with open(tar_path, 'rb') as f: async with aiohttp.ClientSession() as s: - async with s.post("http://localhost:8000/vieter/publish", data=f.read(), headers={"x-api-key": "test"}) as r: + async with s.post("http://localhost:8000/vieter2/publish", data=f.read(), headers={"x-api-key": "test"}) as r: return await check_output(r) From 0dd4534e208cf84c3f1ecda18e2213295c634a63 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 19:54:49 +0200 Subject: [PATCH 049/105] Added extra comments; made linter happy --- src/package.v | 4 +-- src/repo/repo.v | 84 ++++++++++++++++++++++++++++----------------- src/server/server.v | 2 +- 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/src/package.v b/src/package.v index e0eeac28..8bd30d04 100644 --- a/src/package.v +++ b/src/package.v @@ -99,8 +99,8 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { return pkg_info } -// read_pkg extracts the file list & .PKGINFO contents from an archive -// NOTE: this command currently only supports zstd-compressed tarballs +// read_pkg_archive extracts the file list & .PKGINFO contents from an archive +// NOTE: this command only supports zstd- & gzip-compressed tarballs pub fn read_pkg_archive(pkg_path string) ?Pkg { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") diff --git a/src/repo/repo.v b/src/repo/repo.v index e1c9fdbd..4e4fa34b 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -16,7 +16,7 @@ pub: pkg_dir string [required] // The default architecture to use for a repository. In reality, this value // is only required when a package with architecture "any" is added as the - // first package to a repository. + // first package of a repository. default_arch string [required] } @@ -43,10 +43,14 @@ pub fn new(data_dir string, pkg_dir string, default_arch string) ?RepoGroupManag } } -// add_from_path adds a package from an arbitrary path & moves it into the pkgs -// directory if necessary. +// add_pkg_from_path adds a package to a given repo, given the file path to the +// pkg archive. It's a wrapper around add_pkg_in_repo that parses the archive +// file, passes the result to add_pkg_in_repo, and moves the archive to +// r.pkg_dir if it was successfully added. pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult { - pkg := package.read_pkg_archive(pkg_path) or { return error('Failed to read package file: $err.msg') } + pkg := package.read_pkg_archive(pkg_path) or { + return error('Failed to read package file: $err.msg') + } added := r.add_pkg_in_repo(repo, pkg) ? @@ -66,36 +70,48 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re } } +// add_pkg_in_repo adds a package to a given repo. This function is responsible +// for inspecting the package architecture. If said architecture is 'any', the +// package is added to each arch-repository within the given repo. If none +// exist, one is created for provided r.default_arch value. If the architecture +// isn't 'any', the package is only added to the specific architecture. fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { - if pkg.info.arch == "any" { - repo_dir := os.join_path_single(r.data_dir, repo) - - mut arch_repos := []string{} - - if os.exists(repo_dir) { - // We get a listing of all currently present arch-repos in the given repo - arch_repos = os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, it))) - } - - if arch_repos.len == 0 { - arch_repos << r.default_arch - } - - mut added := false - - for arch in arch_repos { - added = added || r.add_pkg_in_arch_repo(repo, arch, pkg) ? - } - - return added - }else{ + if pkg.info.arch != 'any' { return r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) } + + repo_dir := os.join_path_single(r.data_dir, repo) + + mut arch_repos := []string{} + + // If this is the first package to be added to the repository, it won't + // contain any arch-repos yet. + if os.exists(repo_dir) { + // We get a listing of all currently present arch-repos in the given repo + arch_repos = os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, + it))) + } + + if arch_repos.len == 0 { + arch_repos << r.default_arch + } + + mut added := false + + // We add the package to each repository. If any of the repositories + // return true, the result of the function is also true. + for arch in arch_repos { + added = added || r.add_pkg_in_arch_repo(repo, arch, pkg) ? + } + + return added } -// add_pkg_in_repo adds the given package to the specified repo. A repo is an -// arbitrary subdirectory of r.repo_dir, but in practice, it will always be an -// architecture-specific version of some sub-repository. +// add_pkg_in_arch_repo is the function that actually adds a package to a given +// arch-repo. It records the package's data in the arch-repo's desc & files +// files, and afterwards updates the db & files archives to reflect these +// changes. The function returns false if the package was already present in +// the repo, and true otherwise. fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool { pkg_dir := os.join_path(r.data_dir, repo, arch, '$pkg.info.name-$pkg.info.version') @@ -125,8 +141,9 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac return true } -// remove removes a package from the database. It returns false if the package -// wasn't present in the database. +// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It +// returns false if the package wasn't present in the database. It also +// optionally re-syncs the repo archives. fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg &package.Pkg, sync bool) ?bool { repo_dir := os.join_path(r.data_dir, repo, arch) @@ -138,11 +155,14 @@ fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg // We iterate over every directory in the repo dir // TODO filter so we only check directories for d in os.ls(repo_dir) ? { + // Because a repository only allows a single version of each package, + // we need only compare whether the name of the package is the same, + // not the version. name := d.split('-')#[..-2].join('-') if name == pkg.info.name { // We lock the mutex here to prevent other routines from creating a - // new archive while we removed an entry + // new archive while we remove an entry lock r.mutex { os.rmdir_all(os.join_path_single(repo_dir, d)) ? } diff --git a/src/server/server.v b/src/server/server.v index 27321ed1..db2c9aef 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -42,7 +42,7 @@ pub fn server() ? { } // This also creates the directories if needed - repo := repo.new(conf.data_dir, conf.pkg_dir, "x86_64") or { + repo := repo.new(conf.data_dir, conf.pkg_dir, 'x86_64') or { logger.error(err.msg) exit(1) } From aa632c7cd92a1b930929a6a6ce0bc6d3cb8a3676 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 20:26:58 +0200 Subject: [PATCH 050/105] Switched to correct filenames --- src/repo/sync.v | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/repo/sync.v b/src/repo/sync.v index c0c0de95..afb8a4bc 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -45,8 +45,8 @@ fn (r &RepoGroupManager) sync(repo string, arch string) ? { C.archive_write_add_filter_gzip(a_files) C.archive_write_set_format_pax_restricted(a_files) - db_path := os.join_path_single(subrepo_path, 'vieter.db.tar.gz') - files_path := os.join_path_single(subrepo_path, 'vieter.files.tar.gz') + db_path := os.join_path_single(subrepo_path, '${repo}.db.tar.gz') + files_path := os.join_path_single(subrepo_path, '${repo}.files.tar.gz') C.archive_write_open_filename(a_db, &char(db_path.str)) C.archive_write_open_filename(a_files, &char(files_path.str)) From 4139c50780cf14408b6b34a5eacb8920066d0f6c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 23:30:38 +0200 Subject: [PATCH 051/105] Small changes to Makefile --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9996cc3c..7e3c7ea8 100644 --- a/Makefile +++ b/Makefile @@ -70,9 +70,10 @@ run: vieter run-prod: prod VIETER_API_KEY=test \ VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ + VIETER_DATA_DIR=data/repo \ VIETER_PKG_DIR=data/pkgs \ VIETER_LOG_LEVEL=DEBUG \ + VIETER_REPOS_FILE=data/repos.json \ ./pvieter server # =====OTHER===== From 56cb23cc7e16a1444927b245d5c92f114f97cc98 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 27 Mar 2022 23:33:33 +0200 Subject: [PATCH 052/105] Just some changes to poke CI --- .editorconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 630e4fa7..e23a3c76 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,4 +1,3 @@ -# top-most EditorConfig file root = true # Unix-style newlines with a newline ending every file From a4a71a27974b82ef3a4d8166bdc0272a31c0a841 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 28 Mar 2022 10:19:57 +0200 Subject: [PATCH 053/105] First part of RESTful API (not correct yet) [CI SKIP] --- src/server/git.v | 135 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 103 insertions(+), 32 deletions(-) diff --git a/src/server/git.v b/src/server/git.v index 0147d878..967613f1 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -3,16 +3,44 @@ module server import web import os import json - -const repos_file = 'repos.json' +import rand pub struct GitRepo { -pub: - url string [required] - branch string [required] +pub mut: + // URL of the Git repository + url string + // Branch of the Git repository to use + branch string + // On which architectures the package is allowed to be built. In reality, + // this controls which builders will periodically build the image. + arch []string } -fn read_repos(path string) ?[]GitRepo { +fn (mut r GitRepo) patch_from_params(params &map[string]string) ? { + $for field in GitRepo.fields { + if field.name in params { + $if field.typ is string { + r.$(field.name) = params[field.name] + // This specific type check is needed for the compiler to ensure + // our types are correct + } $else $if field.typ is []string { + r.$(field.name) = params[field.name].split(',') + } + }else{ + return error('Missing parameter: ${field.name}.') + } + } +} + +fn repo_from_params(params &map[string]string) ?GitRepo { + mut repo := GitRepo{} + + repo.patch_from_params(params) ? + + return repo +} + +fn read_repos(path string) ?map[string]GitRepo { if !os.exists(path) { mut f := os.create(path) ? @@ -20,17 +48,17 @@ fn read_repos(path string) ?[]GitRepo { f.close() } - f.write_string('[]') ? + f.write_string('{}') ? - return [] + return {} } content := os.read_file(path) ? - res := json.decode([]GitRepo, content) ? + res := json.decode(map[string]GitRepo, content) ? return res } -fn write_repos(path string, repos []GitRepo) ? { +fn write_repos(path string, repos &map[string]GitRepo) ? { mut f := os.create(path) ? defer { @@ -58,20 +86,40 @@ fn (mut app App) get_repos() web.Result { return app.json(repos) } +['/api/repos/:id'; get] +fn (mut app App) get_single_repo(id string) web.Result { + if !app.is_authorized() { + return app.text('Unauthorized.') + } + + repos := rlock app.git_mutex { + read_repos(app.conf.repos_file) or { + app.lerror('Failed to read repos file.') + + return app.server_error(500) + } + } + + if id !in repos { + return app.not_found() + } + + repo := repos[id] + + return app.json(repo) +} + ['/api/repos'; post] fn (mut app App) post_repo() web.Result { if !app.is_authorized() { return app.text('Unauthorized.') } - if !('url' in app.query && 'branch' in app.query) { + new_repo := repo_from_params(&app.query) or { return app.server_error(400) } - new_repo := GitRepo{ - url: app.query['url'] - branch: app.query['branch'] - } + id := rand.uuid_v4() mut repos := rlock app.git_mutex { read_repos(app.conf.repos_file) or { @@ -82,36 +130,27 @@ fn (mut app App) post_repo() web.Result { } // We need to check for duplicates - for r in repos { - if r == new_repo { + for _, repo in repos { + if repo == new_repo { return app.text('Duplicate repository.') } } - repos << new_repo + repos[id] = new_repo lock app.git_mutex { - write_repos(app.conf.repos_file, repos) or { return app.server_error(500) } + write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } } return app.ok('Repo added successfully.') } -['/api/repos'; delete] -fn (mut app App) delete_repo() web.Result { +['/api/repos/:id'; delete] +fn (mut app App) delete_repo(id string) web.Result { if !app.is_authorized() { return app.text('Unauthorized.') } - if !('url' in app.query && 'branch' in app.query) { - return app.server_error(400) - } - - repo_to_remove := GitRepo{ - url: app.query['url'] - branch: app.query['branch'] - } - mut repos := rlock app.git_mutex { read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') @@ -119,11 +158,43 @@ fn (mut app App) delete_repo() web.Result { return app.server_error(500) } } - filtered := repos.filter(it != repo_to_remove) + + if id !in repos { + return app.not_found() + } + + repos.delete(id) lock app.git_mutex { - write_repos(app.conf.repos_file, filtered) or { return app.server_error(500) } + write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } } return app.ok('Repo removed successfully.') } + +['/api/repos/:id'; patch] +fn (mut app App) patch_repo(id string) web.Result { + if !app.is_authorized() { + return app.text('Unauthorized.') + } + + mut repos := rlock app.git_mutex { + read_repos(app.conf.repos_file) or { + app.lerror('Failed to read repos file.') + + return app.server_error(500) + } + } + + if id !in repos { + return app.not_found() + } + + repos[id].patch_from_params(&app.query) + + lock app.git_mutex { + write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } + } + + return app.ok('Repo updated successfully.') +} From 08821725f921d1e2a1994c6fe03e3007e03e9a9d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 28 Mar 2022 13:34:22 +0200 Subject: [PATCH 054/105] Added proper constraint for creating repo --- src/server/git.v | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/server/git.v b/src/server/git.v index 967613f1..684e75a8 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -16,7 +16,7 @@ pub mut: arch []string } -fn (mut r GitRepo) patch_from_params(params &map[string]string) ? { +fn (mut r GitRepo) patch_from_params(params map[string]string) { $for field in GitRepo.fields { if field.name in params { $if field.typ is string { @@ -26,16 +26,22 @@ fn (mut r GitRepo) patch_from_params(params &map[string]string) ? { } $else $if field.typ is []string { r.$(field.name) = params[field.name].split(',') } - }else{ - return error('Missing parameter: ${field.name}.') } } } -fn repo_from_params(params &map[string]string) ?GitRepo { +fn repo_from_params(params map[string]string) ?GitRepo { mut repo := GitRepo{} - repo.patch_from_params(params) ? + // If we're creating a new GitRepo, we want all fields to be present before + // "patching". + $for field in GitRepo.fields { + if field.name !in params { + return error('Missing parameter: ${field.name}.') + } + } + + repo.patch_from_params(params) return repo } @@ -115,8 +121,9 @@ fn (mut app App) post_repo() web.Result { return app.text('Unauthorized.') } - new_repo := repo_from_params(&app.query) or { - return app.server_error(400) + new_repo := repo_from_params(app.query) or { + app.set_status(400, err.msg) + return app.ok("") } id := rand.uuid_v4() @@ -190,7 +197,7 @@ fn (mut app App) patch_repo(id string) web.Result { return app.not_found() } - repos[id].patch_from_params(&app.query) + repos[id].patch_from_params(app.query) lock app.git_mutex { write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } From 0a6be87970fd7553eb664800c07e3822c9844564 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 28 Mar 2022 14:24:26 +0200 Subject: [PATCH 055/105] Start of web framework revamp --- src/server/git.v | 37 ++++---- src/server/response.v | 27 ++++++ src/server/routes.v | 18 ++-- src/web/web.v | 204 ++++-------------------------------------- 4 files changed, 70 insertions(+), 216 deletions(-) create mode 100644 src/server/response.v diff --git a/src/server/git.v b/src/server/git.v index 684e75a8..0743ecc3 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -78,31 +78,31 @@ fn write_repos(path string, repos &map[string]GitRepo) ? { ['/api/repos'; get] fn (mut app App) get_repos() web.Result { if !app.is_authorized() { - return app.text('Unauthorized.') + return app.json(401, new_response('Unauthorized.')) } repos := rlock app.git_mutex { read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') - return app.server_error(500) + return app.status(500) } } - return app.json(repos) + return app.json(200, new_data_response(repos)) } ['/api/repos/:id'; get] fn (mut app App) get_single_repo(id string) web.Result { if !app.is_authorized() { - return app.text('Unauthorized.') + return app.json(401, new_response('Unauthorized.')) } repos := rlock app.git_mutex { read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') - return app.server_error(500) + return app.status(500) } } @@ -112,18 +112,17 @@ fn (mut app App) get_single_repo(id string) web.Result { repo := repos[id] - return app.json(repo) + return app.json(200, new_data_response(repo)) } ['/api/repos'; post] fn (mut app App) post_repo() web.Result { if !app.is_authorized() { - return app.text('Unauthorized.') + return app.json(401, new_response('Unauthorized.')) } new_repo := repo_from_params(app.query) or { - app.set_status(400, err.msg) - return app.ok("") + return app.json(400, new_response(err.msg)) } id := rand.uuid_v4() @@ -132,37 +131,37 @@ fn (mut app App) post_repo() web.Result { read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') - return app.server_error(500) + return app.status(500) } } // We need to check for duplicates for _, repo in repos { if repo == new_repo { - return app.text('Duplicate repository.') + return app.json(400, new_response('Duplicate repository.')) } } repos[id] = new_repo lock app.git_mutex { - write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } + write_repos(app.conf.repos_file, &repos) or { return app.status(500) } } - return app.ok('Repo added successfully.') + return app.json(200, new_response('Repo added successfully.')) } ['/api/repos/:id'; delete] fn (mut app App) delete_repo(id string) web.Result { if !app.is_authorized() { - return app.text('Unauthorized.') + return app.json(401, new_response('Unauthorized.')) } mut repos := rlock app.git_mutex { read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') - return app.server_error(500) + return app.status(500) } } @@ -176,20 +175,20 @@ fn (mut app App) delete_repo(id string) web.Result { write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } } - return app.ok('Repo removed successfully.') + return app.json(200, new_response('Repo removed successfully.')) } ['/api/repos/:id'; patch] fn (mut app App) patch_repo(id string) web.Result { if !app.is_authorized() { - return app.text('Unauthorized.') + return app.json(401, new_response('Unauthorized.')) } mut repos := rlock app.git_mutex { read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') - return app.server_error(500) + return app.status(500) } } @@ -203,5 +202,5 @@ fn (mut app App) patch_repo(id string) web.Result { write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } } - return app.ok('Repo updated successfully.') + return app.json(200, new_response('Repo updated successfully.')) } diff --git a/src/server/response.v b/src/server/response.v new file mode 100644 index 00000000..376493f2 --- /dev/null +++ b/src/server/response.v @@ -0,0 +1,27 @@ +module server + +struct Response { + message string + data T +} + +fn new_response(message string) Response { + return Response{ + message: message + data: "" + } +} + +fn new_data_response(data T) Response { + return Response{ + message: "" + data: data + } +} + +fn new_full_response(message string, data T) Response { + return Response{ + message: message + data: data + } +} diff --git a/src/server/routes.v b/src/server/routes.v index 0090666e..ef5d5a55 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -12,7 +12,7 @@ import net.http // server is still responsive. ['/health'; get] pub fn (mut app App) healthcheck() web.Result { - return app.text('Healthy') + return app.json(200, new_response('Healthy.')) } // get_root handles a GET request for a file on the root @@ -31,7 +31,7 @@ fn (mut app App) get_root(filename string) web.Result { // Scuffed way to respond to HEAD requests if app.req.method == http.Method.head { if os.exists(full_path) { - return app.ok('') + return app.status(200) } return app.not_found() @@ -43,7 +43,7 @@ fn (mut app App) get_root(filename string) web.Result { ['/publish'; post] fn (mut app App) put_package() web.Result { if !app.is_authorized() { - return app.text('Unauthorized.') + return app.json(401, new_response('Unauthorized.')) } mut pkg_path := '' @@ -64,14 +64,16 @@ fn (mut app App) put_package() web.Result { util.reader_to_file(mut app.reader, length.int(), pkg_path) or { app.lwarn("Failed to upload '$pkg_path'") - return app.text('Failed to upload file.') + return app.json(500, new_response('Failed to upload file.')) } sw.stop() app.ldebug("Upload of '$pkg_path' completed in ${sw.elapsed().seconds():.3}s.") } else { app.lwarn('Tried to upload package without specifying a Content-Length.') - return app.text("Content-Type header isn't set.") + + // length required + return app.status(411) } res := app.repo.add_from_path(pkg_path) or { @@ -79,17 +81,17 @@ fn (mut app App) put_package() web.Result { os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } - return app.text('Failed to add package.') + return app.json(500, new_response('Failed to add package.')) } if !res.added { os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } app.lwarn("Duplicate package '$res.pkg.full_name()'.") - return app.text('File already exists.') + return app.json(400, new_response('File already exists.')) } app.linfo("Added '$res.pkg.full_name()' to repository.") - return app.text('Package added successfully.') + return app.json(200, new_response('Package added successfully.')) } diff --git a/src/web/web.v b/src/web/web.v index ad647f20..4e931899 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -12,9 +12,6 @@ import time import json import log -// A type which don't get filtered inside templates -pub type RawHtml = string - // A dummy structure that returns from routes to indicate that you actually sent something to a user [noinit] pub struct Result {} @@ -142,7 +139,7 @@ pub const ( pub struct Context { mut: content_type string = 'text/plain' - status string = '200 OK' + status int = 200 pub: // HTTP Request req http.Request @@ -186,24 +183,14 @@ struct Route { path string } -// Defining this method is optional. -// init_server is called at server start. -// You can use it for initializing globals. -pub fn (ctx Context) init_server() { - eprintln('init_server() has been deprecated, please init your web app in `fn main()`') -} - // Defining this method is optional. // before_request is called before every request (aka middleware). // Probably you can use it for check user session cookie or add header. pub fn (ctx Context) before_request() {} -pub struct Cookie { - name string - value string - expires time.Time - secure bool - http_only bool +// send_string +fn send_string(mut conn net.TcpConn, s string) ? { + conn.write(s.bytes()) ? } // send_response_to_client sends a response to the client @@ -225,33 +212,24 @@ pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bo text: res } resp.set_version(.v1_1) - resp.set_status(http.status_from_int(ctx.status.int())) + resp.set_status(http.status_from_int(ctx.status)) send_string(mut ctx.conn, resp.bytestr()) or { return false } return true } -// html HTTP_OK with s as payload with content-type `text/html` -pub fn (mut ctx Context) html(s string) Result { - ctx.send_response_to_client('text/html', s) - return Result{} -} +pub fn (mut ctx Context) text(status int, s string) Result { + ctx.status = status -// text HTTP_OK with s as payload with content-type `text/plain` -pub fn (mut ctx Context) text(s string) Result { ctx.send_response_to_client('text/plain', s) + return Result{} } // json HTTP_OK with json_s as payload with content-type `application/json` -pub fn (mut ctx Context) json(j T) Result { - json_s := json.encode(j) - ctx.send_response_to_client('application/json', json_s) - return Result{} -} +pub fn (mut ctx Context) json(status int, j T) Result { + ctx.status = status -// json_pretty Response HTTP_OK with a pretty-printed JSON result -pub fn (mut ctx Context) json_pretty(j T) Result { - json_s := json.encode_pretty(j) + json_s := json.encode(j) ctx.send_response_to_client('application/json', json_s) return Result{} } @@ -302,7 +280,7 @@ pub fn (mut ctx Context) file(f_path string) Result { header: header.join(web.headers_close) } resp.set_version(.v1_1) - resp.set_status(http.status_from_int(ctx.status.int())) + resp.set_status(http.status_from_int(ctx.status)) send_string(mut ctx.conn, resp.bytestr()) or { return Result{} } mut buf := []byte{len: 1_000_000} @@ -328,10 +306,8 @@ pub fn (mut ctx Context) file(f_path string) Result { return Result{} } -// ok Response HTTP_OK with s as payload -pub fn (mut ctx Context) ok(s string) Result { - ctx.send_response_to_client(ctx.content_type, s) - return Result{} +pub fn (mut ctx Context) status(status int) Result { + return ctx.text(status, '') } // server_error Response a server error @@ -361,64 +337,7 @@ pub fn (mut ctx Context) redirect(url string) Result { // not_found Send an not_found response pub fn (mut ctx Context) not_found() Result { - if ctx.done { - return Result{} - } - ctx.done = true - send_string(mut ctx.conn, web.http_404.bytestr()) or {} - return Result{} -} - -// set_cookie Sets a cookie -pub fn (mut ctx Context) set_cookie(cookie Cookie) { - mut cookie_data := []string{} - mut secure := if cookie.secure { 'Secure;' } else { '' } - secure += if cookie.http_only { ' HttpOnly' } else { ' ' } - cookie_data << secure - if cookie.expires.unix > 0 { - cookie_data << 'expires=$cookie.expires.utc_string()' - } - data := cookie_data.join(' ') - ctx.add_header('Set-Cookie', '$cookie.name=$cookie.value; $data') -} - -// set_content_type Sets the response content type -pub fn (mut ctx Context) set_content_type(typ string) { - ctx.content_type = typ -} - -// set_cookie_with_expire_date Sets a cookie with a `expire_data` -pub fn (mut ctx Context) set_cookie_with_expire_date(key string, val string, expire_date time.Time) { - ctx.add_header('Set-Cookie', '$key=$val; Secure; HttpOnly; expires=$expire_date.utc_string()') -} - -// get_cookie Gets a cookie by a key -pub fn (ctx &Context) get_cookie(key string) ?string { // TODO refactor - mut cookie_header := ctx.get_header('cookie') - if cookie_header == '' { - cookie_header = ctx.get_header('Cookie') - } - cookie_header = ' ' + cookie_header - // println('cookie_header="$cookie_header"') - // println(ctx.req.header) - cookie := if cookie_header.contains(';') { - cookie_header.find_between(' $key=', ';') - } else { - cookie_header.find_between(' $key=', '\r') - } - if cookie != '' { - return cookie.trim_space() - } - return error('Cookie not found') -} - -// set_status Sets the response status -pub fn (mut ctx Context) set_status(code int, desc string) { - if code < 100 || code > 599 { - ctx.status = '500 Internal Server Error' - } else { - ctx.status = '$code $desc' - } + return ctx.status(404) } // add_header Adds an header to the response with key and val @@ -560,12 +479,6 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { // Calling middleware... app.before_request() - // Static handling - if serve_if_static(mut app, url) { - // successfully served a static file - return - } - // Route matching $for method in T.methods { $if method.return_type is Result { @@ -661,83 +574,6 @@ fn route_matches(url_words []string, route_words []string) ?[]string { return params } -// serve_if_static checks if request is for a static file and serves it -// returns true if we served a static file, false otherwise -[manualfree] -fn serve_if_static(mut app T, url urllib.URL) bool { - // TODO: handle url parameters properly - for now, ignore them - static_file := app.static_files[url.path] - mime_type := app.static_mime_types[url.path] - if static_file == '' || mime_type == '' { - return false - } - data := os.read_file(static_file) or { - send_string(mut app.conn, web.http_404.bytestr()) or {} - return true - } - app.send_response_to_client(mime_type, data) - unsafe { data.free() } - return true -} - -// scan_static_directory makes a static route for each file in a directory -fn (mut ctx Context) scan_static_directory(directory_path string, mount_path string) { - files := os.ls(directory_path) or { panic(err) } - if files.len > 0 { - for file in files { - full_path := os.join_path(directory_path, file) - if os.is_dir(full_path) { - ctx.scan_static_directory(full_path, mount_path + '/' + file) - } else if file.contains('.') && !file.starts_with('.') && !file.ends_with('.') { - ext := os.file_ext(file) - // Rudimentary guard against adding files not in mime_types. - // Use serve_static directly to add non-standard mime types. - if ext in web.mime_types { - ctx.serve_static(mount_path + '/' + file, full_path) - } - } - } - } -} - -// handle_static Handles a directory static -// If `root` is set the mount path for the dir will be in '/' -pub fn (mut ctx Context) handle_static(directory_path string, root bool) bool { - if ctx.done || !os.exists(directory_path) { - return false - } - dir_path := directory_path.trim_space().trim_right('/') - mut mount_path := '' - if dir_path != '.' && os.is_dir(dir_path) && !root { - // Mount point hygene, "./assets" => "/assets". - mount_path = '/' + dir_path.trim_left('.').trim('/') - } - ctx.scan_static_directory(dir_path, mount_path) - return true -} - -// mount_static_folder_at - makes all static files in `directory_path` and inside it, available at http://server/mount_path -// For example: suppose you have called .mount_static_folder_at('/var/share/myassets', '/assets'), -// and you have a file /var/share/myassets/main.css . -// => That file will be available at URL: http://server/assets/main.css . -pub fn (mut ctx Context) mount_static_folder_at(directory_path string, mount_path string) bool { - if ctx.done || mount_path.len < 1 || mount_path[0] != `/` || !os.exists(directory_path) { - return false - } - dir_path := directory_path.trim_right('/') - ctx.scan_static_directory(dir_path, mount_path[1..]) - return true -} - -// serve_static Serves a file static -// `url` is the access path on the site, `file_path` is the real path to the file, `mime_type` is the file type -pub fn (mut ctx Context) serve_static(url string, file_path string) { - ctx.static_files[url] = file_path - // ctx.static_mime_types[url] = mime_type - ext := os.file_ext(file_path) - ctx.static_mime_types[url] = web.mime_types[ext] -} - // ip Returns the ip address from the current user pub fn (ctx &Context) ip() string { mut ip := ctx.req.header.get(.x_forwarded_for) or { '' } @@ -760,16 +596,6 @@ pub fn (mut ctx Context) error(s string) { ctx.form_error = s } -// not_found Returns an empty result -pub fn not_found() Result { - return Result{} -} - -// send_string -fn send_string(mut conn net.TcpConn, s string) ? { - conn.write(s.bytes()) ? -} - // filter Do not delete. // It used by `vlib/v/gen/c/str_intp.v:130` for string interpolation inside web templates // TODO: move it to template render From e5a630e9907c61f2fa4af116659eed595671cc15 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 28 Mar 2022 14:44:23 +0200 Subject: [PATCH 056/105] Switched to net.http.Status for status codes --- src/server/git.v | 37 +++++++++++++++++++------------------ src/server/routes.v | 16 ++++++++-------- src/web/web.v | 14 +++++++------- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/src/server/git.v b/src/server/git.v index 0743ecc3..52cd262c 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -4,6 +4,7 @@ import web import os import json import rand +import net.http pub struct GitRepo { pub mut: @@ -78,31 +79,31 @@ fn write_repos(path string, repos &map[string]GitRepo) ? { ['/api/repos'; get] fn (mut app App) get_repos() web.Result { if !app.is_authorized() { - return app.json(401, new_response('Unauthorized.')) + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } repos := rlock app.git_mutex { read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') - return app.status(500) + return app.status(http.Status.internal_server_error) } } - return app.json(200, new_data_response(repos)) + return app.json(http.Status.ok, new_data_response(repos)) } ['/api/repos/:id'; get] fn (mut app App) get_single_repo(id string) web.Result { if !app.is_authorized() { - return app.json(401, new_response('Unauthorized.')) + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } repos := rlock app.git_mutex { read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') - return app.status(500) + return app.status(http.Status.internal_server_error) } } @@ -112,17 +113,17 @@ fn (mut app App) get_single_repo(id string) web.Result { repo := repos[id] - return app.json(200, new_data_response(repo)) + return app.json(http.Status.ok, new_data_response(repo)) } ['/api/repos'; post] fn (mut app App) post_repo() web.Result { if !app.is_authorized() { - return app.json(401, new_response('Unauthorized.')) + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } new_repo := repo_from_params(app.query) or { - return app.json(400, new_response(err.msg)) + return app.json(http.Status.bad_request, new_response(err.msg)) } id := rand.uuid_v4() @@ -131,37 +132,37 @@ fn (mut app App) post_repo() web.Result { read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') - return app.status(500) + return app.status(http.Status.internal_server_error) } } // We need to check for duplicates for _, repo in repos { if repo == new_repo { - return app.json(400, new_response('Duplicate repository.')) + return app.json(http.Status.bad_request, new_response('Duplicate repository.')) } } repos[id] = new_repo lock app.git_mutex { - write_repos(app.conf.repos_file, &repos) or { return app.status(500) } + write_repos(app.conf.repos_file, &repos) or { return app.status(http.Status.internal_server_error) } } - return app.json(200, new_response('Repo added successfully.')) + return app.json(http.Status.ok, new_response('Repo added successfully.')) } ['/api/repos/:id'; delete] fn (mut app App) delete_repo(id string) web.Result { if !app.is_authorized() { - return app.json(401, new_response('Unauthorized.')) + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } mut repos := rlock app.git_mutex { read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') - return app.status(500) + return app.status(http.Status.internal_server_error) } } @@ -175,20 +176,20 @@ fn (mut app App) delete_repo(id string) web.Result { write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } } - return app.json(200, new_response('Repo removed successfully.')) + return app.json(http.Status.ok, new_response('Repo removed successfully.')) } ['/api/repos/:id'; patch] fn (mut app App) patch_repo(id string) web.Result { if !app.is_authorized() { - return app.json(401, new_response('Unauthorized.')) + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } mut repos := rlock app.git_mutex { read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') - return app.status(500) + return app.status(http.Status.internal_server_error) } } @@ -202,5 +203,5 @@ fn (mut app App) patch_repo(id string) web.Result { write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } } - return app.json(200, new_response('Repo updated successfully.')) + return app.json(http.Status.ok, new_response('Repo updated successfully.')) } diff --git a/src/server/routes.v b/src/server/routes.v index ef5d5a55..0e7d72c8 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -12,7 +12,7 @@ import net.http // server is still responsive. ['/health'; get] pub fn (mut app App) healthcheck() web.Result { - return app.json(200, new_response('Healthy.')) + return app.json(http.Status.ok, new_response('Healthy.')) } // get_root handles a GET request for a file on the root @@ -31,7 +31,7 @@ fn (mut app App) get_root(filename string) web.Result { // Scuffed way to respond to HEAD requests if app.req.method == http.Method.head { if os.exists(full_path) { - return app.status(200) + return app.status(http.Status.ok) } return app.not_found() @@ -43,7 +43,7 @@ fn (mut app App) get_root(filename string) web.Result { ['/publish'; post] fn (mut app App) put_package() web.Result { if !app.is_authorized() { - return app.json(401, new_response('Unauthorized.')) + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } mut pkg_path := '' @@ -64,7 +64,7 @@ fn (mut app App) put_package() web.Result { util.reader_to_file(mut app.reader, length.int(), pkg_path) or { app.lwarn("Failed to upload '$pkg_path'") - return app.json(500, new_response('Failed to upload file.')) + return app.json(http.Status.internal_server_error, new_response('Failed to upload file.')) } sw.stop() @@ -73,7 +73,7 @@ fn (mut app App) put_package() web.Result { app.lwarn('Tried to upload package without specifying a Content-Length.') // length required - return app.status(411) + return app.status(http.Status.length_required) } res := app.repo.add_from_path(pkg_path) or { @@ -81,17 +81,17 @@ fn (mut app App) put_package() web.Result { os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } - return app.json(500, new_response('Failed to add package.')) + return app.json(http.Status.internal_server_error, new_response('Failed to add package.')) } if !res.added { os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } app.lwarn("Duplicate package '$res.pkg.full_name()'.") - return app.json(400, new_response('File already exists.')) + return app.json(http.Status.bad_request, new_response('File already exists.')) } app.linfo("Added '$res.pkg.full_name()' to repository.") - return app.json(200, new_response('Package added successfully.')) + return app.json(http.Status.ok, new_response('Package added successfully.')) } diff --git a/src/web/web.v b/src/web/web.v index 4e931899..23ae7462 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -139,7 +139,7 @@ pub const ( pub struct Context { mut: content_type string = 'text/plain' - status int = 200 + status http.Status = http.Status.ok pub: // HTTP Request req http.Request @@ -212,12 +212,12 @@ pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bo text: res } resp.set_version(.v1_1) - resp.set_status(http.status_from_int(ctx.status)) + resp.set_status(ctx.status) send_string(mut ctx.conn, resp.bytestr()) or { return false } return true } -pub fn (mut ctx Context) text(status int, s string) Result { +pub fn (mut ctx Context) text(status http.Status, s string) Result { ctx.status = status ctx.send_response_to_client('text/plain', s) @@ -226,7 +226,7 @@ pub fn (mut ctx Context) text(status int, s string) Result { } // json HTTP_OK with json_s as payload with content-type `application/json` -pub fn (mut ctx Context) json(status int, j T) Result { +pub fn (mut ctx Context) json(status http.Status, j T) Result { ctx.status = status json_s := json.encode(j) @@ -280,7 +280,7 @@ pub fn (mut ctx Context) file(f_path string) Result { header: header.join(web.headers_close) } resp.set_version(.v1_1) - resp.set_status(http.status_from_int(ctx.status)) + resp.set_status(ctx.status) send_string(mut ctx.conn, resp.bytestr()) or { return Result{} } mut buf := []byte{len: 1_000_000} @@ -306,7 +306,7 @@ pub fn (mut ctx Context) file(f_path string) Result { return Result{} } -pub fn (mut ctx Context) status(status int) Result { +pub fn (mut ctx Context) status(status http.Status) Result { return ctx.text(status, '') } @@ -337,7 +337,7 @@ pub fn (mut ctx Context) redirect(url string) Result { // not_found Send an not_found response pub fn (mut ctx Context) not_found() Result { - return ctx.status(404) + return ctx.status(http.Status.not_found) } // add_header Adds an header to the response with key and val From 148ec3ab47a496d63055864b7f15a3323b82c214 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 1 Apr 2022 21:33:55 +0200 Subject: [PATCH 057/105] Updated changelog --- CHANGELOG.md | 3 +++ src/web/web.v | 1 + 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eaf477f..f95351fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Better environment variable support * Each env var can now be provided from a file by appending it with `_FILE` & passing the path to the file as value +* Revamped web framework + * All routes now return proper JSON where applicable & the correct status + codes ## Added diff --git a/src/web/web.v b/src/web/web.v index 23ae7462..6e07aea2 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -231,6 +231,7 @@ pub fn (mut ctx Context) json(status http.Status, j T) Result { json_s := json.encode(j) ctx.send_response_to_client('application/json', json_s) + return Result{} } From 3a6effad807b5bd7b0acd9c97699555ad1f3dffa Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 1 Apr 2022 21:34:58 +0200 Subject: [PATCH 058/105] Ran vfmt --- src/server/git.v | 13 +++++++------ src/server/response.v | 6 +++--- src/web/web.v | 4 ++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/server/git.v b/src/server/git.v index 52cd262c..8862af88 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -9,12 +9,12 @@ import net.http pub struct GitRepo { pub mut: // URL of the Git repository - url string + url string // Branch of the Git repository to use branch string // On which architectures the package is allowed to be built. In reality, // this controls which builders will periodically build the image. - arch []string + arch []string } fn (mut r GitRepo) patch_from_params(params map[string]string) { @@ -22,8 +22,8 @@ fn (mut r GitRepo) patch_from_params(params map[string]string) { if field.name in params { $if field.typ is string { r.$(field.name) = params[field.name] - // This specific type check is needed for the compiler to ensure - // our types are correct + // This specific type check is needed for the compiler to ensure + // our types are correct } $else $if field.typ is []string { r.$(field.name) = params[field.name].split(',') } @@ -41,7 +41,6 @@ fn repo_from_params(params map[string]string) ?GitRepo { return error('Missing parameter: ${field.name}.') } } - repo.patch_from_params(params) return repo @@ -146,7 +145,9 @@ fn (mut app App) post_repo() web.Result { repos[id] = new_repo lock app.git_mutex { - write_repos(app.conf.repos_file, &repos) or { return app.status(http.Status.internal_server_error) } + write_repos(app.conf.repos_file, &repos) or { + return app.status(http.Status.internal_server_error) + } } return app.json(http.Status.ok, new_response('Repo added successfully.')) diff --git a/src/server/response.v b/src/server/response.v index 376493f2..354e7da7 100644 --- a/src/server/response.v +++ b/src/server/response.v @@ -2,19 +2,19 @@ module server struct Response { message string - data T + data T } fn new_response(message string) Response { return Response{ message: message - data: "" + data: '' } } fn new_data_response(data T) Response { return Response{ - message: "" + message: '' data: data } } diff --git a/src/web/web.v b/src/web/web.v index 6e07aea2..eb8d5df7 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -138,8 +138,8 @@ pub const ( // It has fields for the query, form, files. pub struct Context { mut: - content_type string = 'text/plain' - status http.Status = http.Status.ok + content_type string = 'text/plain' + status http.Status = http.Status.ok pub: // HTTP Request req http.Request From fe247748480c722f0fbb41f5bf2203972af7e851 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 1 Apr 2022 21:50:00 +0200 Subject: [PATCH 059/105] Added some docs --- src/web/web.v | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/web/web.v b/src/web/web.v index eb8d5df7..000c6a69 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -217,6 +217,7 @@ pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bo return true } +// text responds to a request with some plaintext. pub fn (mut ctx Context) text(status http.Status, s string) Result { ctx.status = status @@ -307,6 +308,8 @@ pub fn (mut ctx Context) file(f_path string) Result { return Result{} } +// status responds with an empty textual response, essentially only returning +// the given status code. pub fn (mut ctx Context) status(status http.Status) Result { return ctx.text(status, '') } From 56517c0ff011d5d88adeeba406f2596fca9e4b0b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 1 Apr 2022 23:15:30 +0200 Subject: [PATCH 060/105] Removed arm/v7 from CI --- .woodpecker/.build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 4cddc6ad..81fa32fd 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -2,7 +2,8 @@ matrix: PLATFORM: - linux/amd64 - linux/arm64 - - linux/arm/v7 + # I just don't have a performant enough runner for this platform + # - linux/arm/v7 # These checks already get performed on the feature branches platform: ${PLATFORM} From 5b919ceeaeec6a1b1520f08c9109f1e4448f0ac0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 16:52:31 +0200 Subject: [PATCH 061/105] Switched to cli module; merged cli & vieter into single binary --- src/{ => build}/build.v | 10 ++++----- src/build/cli.v | 17 ++++++++++++++ src/env.v | 25 ++++++++++++++------- src/git/cli.v | 37 ++++++++++++++++++++++++++++++ src/git/git.v | 40 +++++++++++++++++++++++++++++++++ src/main.v | 32 ++++++++++++++++++++------ src/server/cli.v | 16 +++++++++++++ src/server/git.v | 50 +++++++---------------------------------- src/server/server.v | 5 ++--- 9 files changed, 167 insertions(+), 65 deletions(-) rename src/{ => build}/build.v (96%) create mode 100644 src/build/cli.v create mode 100644 src/git/cli.v create mode 100644 src/git/git.v create mode 100644 src/server/cli.v diff --git a/src/build.v b/src/build/build.v similarity index 96% rename from src/build.v rename to src/build/build.v index 51de64c4..f2ff9af9 100644 --- a/src/build.v +++ b/src/build/build.v @@ -1,4 +1,4 @@ -module main +module build import docker import encoding.base64 @@ -7,6 +7,8 @@ import json import server import env import net.http +import cli +import git const container_build_dir = '/build' @@ -62,15 +64,13 @@ fn create_build_image() ?string { return image.id } -fn build() ? { - conf := env.load() ? - +fn build(conf env.BuildConfig) ? { // We get the repos list from the Vieter instance mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? req.add_custom_header('X-Api-Key', conf.api_key) ? res := req.do() ? - repos := json.decode([]server.GitRepo, res.text) ? + repos := json.decode([]git.GitRepo, res.text) ? // No point in doing work if there's no repos present if repos.len == 0 { diff --git a/src/build/cli.v b/src/build/cli.v new file mode 100644 index 00000000..e1f6cb54 --- /dev/null +++ b/src/build/cli.v @@ -0,0 +1,17 @@ +module build + +import cli +import env + +pub fn cmd() cli.Command { + return cli.Command{ + name: 'build' + description: 'Run the build process.' + execute: fn (cmd cli.Command) ? { + conf := env.load() ? + + build(conf) ? + } + } +} + diff --git a/src/env.v b/src/env.v index bd544cfc..11709ce5 100644 --- a/src/env.v +++ b/src/env.v @@ -56,15 +56,9 @@ fn get_env_var(field_name string) ?string { } } -// load attempts to create the given type from environment variables. For -// each field, the corresponding env var is its name in uppercase prepended -// with the hardcoded prefix. If this one isn't present, it looks for the env -// var with the file_suffix suffix. -pub fn load() ?T { - res := T{} - +fn load_in_place(mut o T) ? { $for field in T.fields { - res.$(field.name) = get_env_var(field.name) or { + o.$(field.name) = get_env_var(field.name) or { // We use the default instead, if it's present mut default := '' @@ -82,5 +76,20 @@ pub fn load() ?T { default } } +} + +// load attempts to create the given type from environment variables. For +// each field, the corresponding env var is its name in uppercase prepended +// with the hardcoded prefix. If this one isn't present, it looks for the env +// var with the file_suffix suffix. +pub fn load() ?T { + mut res := T{} + + load_in_place(mut res) ? + return res } + +pub fn load_with_file(path string) ?T { + +} diff --git a/src/git/cli.v b/src/git/cli.v new file mode 100644 index 00000000..fccb9b08 --- /dev/null +++ b/src/git/cli.v @@ -0,0 +1,37 @@ +module git + +import cli +import env +import net.http + +struct Config { + address string [required] + api_key string [required] +} + +pub fn cmd() cli.Command { + return cli.Command{ + name: 'repos' + description: 'Interact with the repos API.' + commands: [ + cli.Command{ + name: 'list' + description: 'List the current repos.' + execute: fn (cmd cli.Command) ? { + conf := env.load() ? + + list(conf) ? + } + } + ] + } +} + +fn list(conf Config) ? { + mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? + req.add_custom_header('X-API-Key', conf.api_key) ? + + res := req.do() ? + + println(res.text) +} diff --git a/src/git/git.v b/src/git/git.v new file mode 100644 index 00000000..6daf25be --- /dev/null +++ b/src/git/git.v @@ -0,0 +1,40 @@ +module git + +import os +import json + +pub struct GitRepo { +pub: + url string [required] + branch string [required] +} + +pub fn read_repos(path string) ?[]GitRepo { + if !os.exists(path) { + mut f := os.create(path) ? + + defer { + f.close() + } + + f.write_string('[]') ? + + return [] + } + + content := os.read_file(path) ? + res := json.decode([]GitRepo, content) ? + return res +} + +pub fn write_repos(path string, repos []GitRepo) ? { + mut f := os.create(path) ? + + defer { + f.close() + } + + value := json.encode(repos) + f.write_string(value) ? +} + diff --git a/src/main.v b/src/main.v index 156f0a39..31b6bcbc 100644 --- a/src/main.v +++ b/src/main.v @@ -3,15 +3,33 @@ module main import os import server import util +import cli +import env +import build +import git fn main() { - if os.args.len == 1 { - util.exit_with_message(1, 'No action provided.') + mut app := cli.Command{ + name: 'vieter' + description: 'Arch repository server' + version: '0.1.0' + flags: [ + cli.Flag{ + flag: cli.FlagType.string + name: 'config-file' + abbrev: 'f' + description: 'Location of Vieter config file; defaults to ~/.vieterrc.' + global: true + default_value: [os.expand_tilde_to_home('~/.vieterrc')] + } + ] + commands: [ + server.cmd(), + build.cmd(), + git.cmd() + ] } - match os.args[1] { - 'server' { server.server() ? } - 'build' { build() ? } - else { util.exit_with_message(1, 'Unknown action: ${os.args[1]}') } - } + app.setup() + app.parse(os.args) } diff --git a/src/server/cli.v b/src/server/cli.v new file mode 100644 index 00000000..944f05bc --- /dev/null +++ b/src/server/cli.v @@ -0,0 +1,16 @@ +module server + +import cli +import env + +pub fn cmd() cli.Command { + return cli.Command{ + name: 'server' + description: 'Start the Vieter server' + execute: fn (cmd cli.Command) ? { + conf := env.load() ? + + server.server(conf) ? + } + } +} diff --git a/src/server/git.v b/src/server/git.v index 0147d878..2e014db7 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -3,44 +3,10 @@ module server import web import os import json +import git const repos_file = 'repos.json' -pub struct GitRepo { -pub: - url string [required] - branch string [required] -} - -fn read_repos(path string) ?[]GitRepo { - if !os.exists(path) { - mut f := os.create(path) ? - - defer { - f.close() - } - - f.write_string('[]') ? - - return [] - } - - content := os.read_file(path) ? - res := json.decode([]GitRepo, content) ? - return res -} - -fn write_repos(path string, repos []GitRepo) ? { - mut f := os.create(path) ? - - defer { - f.close() - } - - value := json.encode(repos) - f.write_string(value) ? -} - ['/api/repos'; get] fn (mut app App) get_repos() web.Result { if !app.is_authorized() { @@ -48,7 +14,7 @@ fn (mut app App) get_repos() web.Result { } repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { + git.read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') return app.server_error(500) @@ -68,13 +34,13 @@ fn (mut app App) post_repo() web.Result { return app.server_error(400) } - new_repo := GitRepo{ + new_repo := git.GitRepo{ url: app.query['url'] branch: app.query['branch'] } mut repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { + git.read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') return app.server_error(500) @@ -91,7 +57,7 @@ fn (mut app App) post_repo() web.Result { repos << new_repo lock app.git_mutex { - write_repos(app.conf.repos_file, repos) or { return app.server_error(500) } + git.write_repos(app.conf.repos_file, repos) or { return app.server_error(500) } } return app.ok('Repo added successfully.') @@ -107,13 +73,13 @@ fn (mut app App) delete_repo() web.Result { return app.server_error(400) } - repo_to_remove := GitRepo{ + repo_to_remove := git.GitRepo{ url: app.query['url'] branch: app.query['branch'] } mut repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { + git.read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') return app.server_error(500) @@ -122,7 +88,7 @@ fn (mut app App) delete_repo() web.Result { filtered := repos.filter(it != repo_to_remove) lock app.git_mutex { - write_repos(app.conf.repos_file, filtered) or { return app.server_error(500) } + git.write_repos(app.conf.repos_file, filtered) or { return app.server_error(500) } } return app.ok('Repo removed successfully.') diff --git a/src/server/server.v b/src/server/server.v index 4b31b99e..7f129e93 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -6,6 +6,7 @@ import log import repo import env import util +import cli const port = 8000 @@ -20,9 +21,7 @@ pub mut: } // server starts the web server & starts listening for requests -pub fn server() ? { - conf := env.load() ? - +pub fn server(conf env.ServerConfig) ? { // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') From b90e6cf6b4f1d87e90de36d964195882a51af5b3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 16:57:27 +0200 Subject: [PATCH 062/105] Updatd config files; ran formatter --- .woodpecker/.build.yml | 31 ------------------------------- Makefile | 12 ------------ PKGBUILD | 13 ++----------- src/build/cli.v | 1 - src/env.v | 1 - src/git/cli.v | 16 ++++++++-------- src/git/git.v | 1 - src/main.v | 4 ++-- src/server/cli.v | 2 +- 9 files changed, 13 insertions(+), 68 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 4cddc6ad..a4772b36 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -36,22 +36,6 @@ pipeline: when: event: push - cli: - image: 'chewingbever/vlang:latest' - environment: - - LDFLAGS=-static - commands: - - make cli-prod - # Make sure the binary is actually statically built - - readelf -d vieterctl - - du -h vieterctl - - '[ "$(readelf -d vieterctl | grep NEEDED | wc -l)" = 0 ]' - # This removes so much, it's amazing - - strip -s vieterctl - - du -h vieterctl - when: - event: push - upload: image: 'chewingbever/vlang:latest' secrets: [ s3_username, s3_password ] @@ -74,20 +58,5 @@ pipeline: -H "Content-Type: $CONTENT_TYPE" -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" https://$URL$OBJ_PATH - - # Also update the CLI tool - - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieterctl-$(echo '${PLATFORM}' | sed 's:/:-:g')" - - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH" - - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64` - - > - curl - --silent - -XPUT - -T vieterctl - -H "Host: $URL" - -H "Date: $DATE" - -H "Content-Type: $CONTENT_TYPE" - -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" - https://$URL$OBJ_PATH when: event: push diff --git a/Makefile b/Makefile index 7b0fb1b1..2238585d 100644 --- a/Makefile +++ b/Makefile @@ -42,18 +42,6 @@ pvieter: $(SOURCES) c: $(V) -o vieter.c $(SRC_DIR) -# Build the CLI tool -.PHONY: cli -cli: dvieterctl -dvieterctl: cli.v - $(V_PATH) -showcc -g -o dvieterctl cli.v - -.PHONY: cli-prod -cli-prod: vieterctl -vieterctl: cli.v -cli-prod: - $(V_PATH) -showcc -o vieterctl -prod cli.v - # =====EXECUTION===== # Run the server in the default 'data' directory .PHONY: run diff --git a/PKGBUILD b/PKGBUILD index 767c07c1..cb0c24a0 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -1,7 +1,7 @@ # Maintainer: Jef Roosens pkgbase='vieter' -pkgname=('vieter' 'vieterctl') +pkgname='vieter' pkgver=0.1.0.rc1.r45.g6d3ff8a pkgrel=1 depends=('glibc' 'openssl' 'libarchive' 'gc') @@ -23,21 +23,12 @@ build() { # Build the compiler CFLAGS= make v - # Build the server & the CLI tool make prod - make cli-prod } -package_vieter() { +package() { pkgdesc="Vieter is a lightweight implementation of an Arch repository server." install -dm755 "$pkgdir/usr/bin" install -Dm755 "$pkgbase/pvieter" "$pkgdir/usr/bin/vieter" } - -package_vieterctl() { - pkgdesc="Allows you to configure a Vieter server's list of Git repositories." - install -dm755 "$pkgdir/usr/bin" - - install -Dm755 "$pkgbase/vieterctl" "$pkgdir/usr/bin/vieterctl" -} diff --git a/src/build/cli.v b/src/build/cli.v index e1f6cb54..bb3c0161 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -14,4 +14,3 @@ pub fn cmd() cli.Command { } } } - diff --git a/src/env.v b/src/env.v index 11709ce5..3babd55e 100644 --- a/src/env.v +++ b/src/env.v @@ -91,5 +91,4 @@ pub fn load() ?T { } pub fn load_with_file(path string) ?T { - } diff --git a/src/git/cli.v b/src/git/cli.v index fccb9b08..a91f5e29 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -14,15 +14,15 @@ pub fn cmd() cli.Command { name: 'repos' description: 'Interact with the repos API.' commands: [ - cli.Command{ - name: 'list' - description: 'List the current repos.' - execute: fn (cmd cli.Command) ? { - conf := env.load() ? + cli.Command{ + name: 'list' + description: 'List the current repos.' + execute: fn (cmd cli.Command) ? { + conf := env.load() ? - list(conf) ? - } - } + list(conf) ? + } + }, ] } } diff --git a/src/git/git.v b/src/git/git.v index 6daf25be..597aa3ca 100644 --- a/src/git/git.v +++ b/src/git/git.v @@ -37,4 +37,3 @@ pub fn write_repos(path string, repos []GitRepo) ? { value := json.encode(repos) f.write_string(value) ? } - diff --git a/src/main.v b/src/main.v index 31b6bcbc..18a822ba 100644 --- a/src/main.v +++ b/src/main.v @@ -21,12 +21,12 @@ fn main() { description: 'Location of Vieter config file; defaults to ~/.vieterrc.' global: true default_value: [os.expand_tilde_to_home('~/.vieterrc')] - } + }, ] commands: [ server.cmd(), build.cmd(), - git.cmd() + git.cmd(), ] } diff --git a/src/server/cli.v b/src/server/cli.v index 944f05bc..2b51c29a 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -10,7 +10,7 @@ pub fn cmd() cli.Command { execute: fn (cmd cli.Command) ? { conf := env.load() ? - server.server(conf) ? + server(conf) ? } } } From 9dd02426ff429e98b8c39f48c6b161fdf786854e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 16:58:15 +0200 Subject: [PATCH 063/105] Removd armv7 from ci --- .woodpecker/.build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index a4772b36..16e5a69b 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -2,7 +2,6 @@ matrix: PLATFORM: - linux/amd64 - linux/arm64 - - linux/arm/v7 # These checks already get performed on the feature branches platform: ${PLATFORM} From 21ef262edeeb159f855ae4cd2083e6bce11090ae Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 17:16:27 +0200 Subject: [PATCH 064/105] Migrated rest of cli commands --- cli.v | 84 --------------------------------------------------- src/git/cli.v | 40 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 84 deletions(-) delete mode 100644 cli.v diff --git a/cli.v b/cli.v deleted file mode 100644 index 256b8564..00000000 --- a/cli.v +++ /dev/null @@ -1,84 +0,0 @@ -import os -import toml -import net.http - -struct Config { - address string [required] - api_key string [required] -} - -fn list(conf Config) ? { - mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? - req.add_custom_header('X-API-Key', conf.api_key) ? - - res := req.do() ? - - println(res.text) -} - -fn add(conf Config, args []string) ? { - if args.len < 2 { - eprintln('Not enough arguments.') - exit(1) - } - - if args.len > 2 { - eprintln('Too many arguments.') - exit(1) - } - - mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=${args[0]}&branch=${args[1]}', '') ? - req.add_custom_header('X-API-Key', conf.api_key) ? - - res := req.do() ? - - println(res.text) -} - -fn remove(conf Config, args []string) ? { - if args.len < 2 { - eprintln('Not enough arguments.') - exit(1) - } - - if args.len > 2 { - eprintln('Too many arguments.') - exit(1) - } - - mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=${args[0]}&branch=${args[1]}', '') ? - req.add_custom_header('X-API-Key', conf.api_key) ? - - res := req.do() ? - - println(res.text) -} - -fn main() { - conf_path := os.expand_tilde_to_home('~/.vieterrc') - - if !os.is_file(conf_path) { - exit(1) - } - - conf := toml.parse_file(conf_path) ?.reflect() - - args := os.args[1..] - - if args.len == 0 { - eprintln('No action provided.') - exit(1) - } - - action := args[0] - - match action { - 'list' { list(conf) ? } - 'add' { add(conf, args[1..]) ? } - 'remove' { remove(conf, args[1..]) ? } - else { - eprintln("Invalid action '$action'.") - exit(1) - } - } -} diff --git a/src/git/cli.v b/src/git/cli.v index a91f5e29..2b164e1c 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -23,6 +23,28 @@ pub fn cmd() cli.Command { list(conf) ? } }, + cli.Command{ + name: 'add' + required_args: 2 + usage: 'url branch' + description: 'Add a new repository.' + execute: fn (cmd cli.Command) ? { + conf := env.load() ? + + add(conf, cmd.args[0], cmd.args[1]) ? + } + }, + cli.Command{ + name: 'remove' + required_args: 2 + usage: 'url branch' + description: 'Remove a repository.' + execute: fn (cmd cli.Command) ? { + conf := env.load() ? + + remove(conf, cmd.args[0], cmd.args[1]) ? + } + }, ] } } @@ -35,3 +57,21 @@ fn list(conf Config) ? { println(res.text) } + +fn add(conf Config, url string, branch string) ? { + mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=$url&branch=$branch', '') ? + req.add_custom_header('X-API-Key', conf.api_key) ? + + res := req.do() ? + + println(res.text) +} + +fn remove(conf Config, url string, branch string) ? { + mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=$url&branch=$branch', '') ? + req.add_custom_header('X-API-Key', conf.api_key) ? + + res := req.do() ? + + println(res.text) +} From e9d7858380162b3cf36683e91f73e1671f80d368 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 17:51:06 +0200 Subject: [PATCH 065/105] env module now properly supports config file --- src/build/cli.v | 3 ++- src/env.v | 63 ++++++++++++++++++++++++++++-------------------- src/git/cli.v | 15 ++++++++---- src/server/cli.v | 3 ++- 4 files changed, 51 insertions(+), 33 deletions(-) diff --git a/src/build/cli.v b/src/build/cli.v index bb3c0161..a32a6e74 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -8,7 +8,8 @@ pub fn cmd() cli.Command { name: 'build' description: 'Run the build process.' execute: fn (cmd cli.Command) ? { - conf := env.load() ? + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? build(conf) ? } diff --git a/src/env.v b/src/env.v index 3babd55e..62678e69 100644 --- a/src/env.v +++ b/src/env.v @@ -1,6 +1,7 @@ module env import os +import toml // The prefix that every environment variable should have const prefix = 'VIETER_' @@ -32,9 +33,9 @@ fn get_env_var(field_name string) ?string { env_var := os.getenv(env_var_name) env_file := os.getenv(env_file_name) - // If both aren't set, we report them missing + // If both are missing, we return an empty string if env_var == '' && env_file == '' { - return error('Either $env_var_name or $env_file_name is required.') + return '' } // If they're both set, we report a conflict @@ -56,37 +57,47 @@ fn get_env_var(field_name string) ?string { } } -fn load_in_place(mut o T) ? { - $for field in T.fields { - o.$(field.name) = get_env_var(field.name) or { - // We use the default instead, if it's present - mut default := '' - - for attr in field.attrs { - if attr.starts_with('default: ') { - default = attr[9..] - break - } - } - - if default == '' { - return err - } - - default - } - } -} - // load attempts to create the given type from environment variables. For // each field, the corresponding env var is its name in uppercase prepended // with the hardcoded prefix. If this one isn't present, it looks for the env // var with the file_suffix suffix. -pub fn load() ?T { +pub fn load(path string) ?T { mut res := T{} - load_in_place(mut res) ? + if os.exists(path) { + res = toml.parse_file(path) ?.reflect() + } + $for field in T.fields { + $if field.typ is string { + env_value := get_env_var(field.name) ? + + // The value of the env var will always be chosen over the config + // file + if env_value != '' { + res.$(field.name) = env_value + } + // If there's no value from the toml file either, we try to find a + // default value + else if res.$(field.name) == '' { + // We use the default instead, if it's present + mut default := '' + + for attr in field.attrs { + if attr.starts_with('default: ') { + default = attr[9..] + break + } + } + + if default == '' { + return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.") + } + + res.$(field.name) = default + } + } + } return res } diff --git a/src/git/cli.v b/src/git/cli.v index 2b164e1c..60306125 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -18,7 +18,8 @@ pub fn cmd() cli.Command { name: 'list' description: 'List the current repos.' execute: fn (cmd cli.Command) ? { - conf := env.load() ? + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? list(conf) ? } @@ -29,7 +30,8 @@ pub fn cmd() cli.Command { usage: 'url branch' description: 'Add a new repository.' execute: fn (cmd cli.Command) ? { - conf := env.load() ? + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? add(conf, cmd.args[0], cmd.args[1]) ? } @@ -40,7 +42,8 @@ pub fn cmd() cli.Command { usage: 'url branch' description: 'Remove a repository.' execute: fn (cmd cli.Command) ? { - conf := env.load() ? + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? remove(conf, cmd.args[0], cmd.args[1]) ? } @@ -59,7 +62,8 @@ fn list(conf Config) ? { } fn add(conf Config, url string, branch string) ? { - mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=$url&branch=$branch', '') ? + mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=$url&branch=$branch', + '') ? req.add_custom_header('X-API-Key', conf.api_key) ? res := req.do() ? @@ -68,7 +72,8 @@ fn add(conf Config, url string, branch string) ? { } fn remove(conf Config, url string, branch string) ? { - mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=$url&branch=$branch', '') ? + mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=$url&branch=$branch', + '') ? req.add_custom_header('X-API-Key', conf.api_key) ? res := req.do() ? diff --git a/src/server/cli.v b/src/server/cli.v index 2b51c29a..367878ae 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -8,7 +8,8 @@ pub fn cmd() cli.Command { name: 'server' description: 'Start the Vieter server' execute: fn (cmd cli.Command) ? { - conf := env.load() ? + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? server(conf) ? } From 75dfc5267b59886a09d52ed5ca4820ed3e93f428 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 17:57:05 +0200 Subject: [PATCH 066/105] Made vet happy --- src/build/build.v | 4 +--- src/build/cli.v | 1 + src/env.v | 3 --- src/git/cli.v | 1 + src/git/git.v | 2 ++ src/main.v | 2 -- src/server/cli.v | 1 + src/server/git.v | 2 -- src/server/server.v | 1 - 9 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index f2ff9af9..6f2b0a84 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -3,12 +3,10 @@ module build import docker import encoding.base64 import time -import json -import server import env import net.http -import cli import git +import json const container_build_dir = '/build' diff --git a/src/build/cli.v b/src/build/cli.v index a32a6e74..463d2ca3 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -3,6 +3,7 @@ module build import cli import env +// cmd returns the cli submodule that handles the build process pub fn cmd() cli.Command { return cli.Command{ name: 'build' diff --git a/src/env.v b/src/env.v index 62678e69..58c68036 100644 --- a/src/env.v +++ b/src/env.v @@ -100,6 +100,3 @@ pub fn load(path string) ?T { } return res } - -pub fn load_with_file(path string) ?T { -} diff --git a/src/git/cli.v b/src/git/cli.v index 60306125..17fa9847 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -9,6 +9,7 @@ struct Config { api_key string [required] } +// cmd returns the cli submodule that handles the repos API interaction pub fn cmd() cli.Command { return cli.Command{ name: 'repos' diff --git a/src/git/git.v b/src/git/git.v index 597aa3ca..913bc39d 100644 --- a/src/git/git.v +++ b/src/git/git.v @@ -9,6 +9,7 @@ pub: branch string [required] } +// read_repos reads the given JSON file & parses it as a list of Git repos pub fn read_repos(path string) ?[]GitRepo { if !os.exists(path) { mut f := os.create(path) ? @@ -27,6 +28,7 @@ pub fn read_repos(path string) ?[]GitRepo { return res } +// write_repos writes a list of repositories back to a given file pub fn write_repos(path string, repos []GitRepo) ? { mut f := os.create(path) ? diff --git a/src/main.v b/src/main.v index 18a822ba..11bc77b8 100644 --- a/src/main.v +++ b/src/main.v @@ -2,9 +2,7 @@ module main import os import server -import util import cli -import env import build import git diff --git a/src/server/cli.v b/src/server/cli.v index 367878ae..6e676867 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -3,6 +3,7 @@ module server import cli import env +// cmd returns the cli submodule that handles starting the server pub fn cmd() cli.Command { return cli.Command{ name: 'server' diff --git a/src/server/git.v b/src/server/git.v index 2e014db7..3ec8eeb8 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -1,8 +1,6 @@ module server import web -import os -import json import git const repos_file = 'repos.json' diff --git a/src/server/server.v b/src/server/server.v index 7f129e93..0738bc4f 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -6,7 +6,6 @@ import log import repo import env import util -import cli const port = 8000 From b70be0574e24c5a0adbb5a4826d5ceb8f2ca9bc3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 18:02:57 +0200 Subject: [PATCH 067/105] Small change to documentation --- src/env.v | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/env.v b/src/env.v index 58c68036..6abd6749 100644 --- a/src/env.v +++ b/src/env.v @@ -57,10 +57,10 @@ fn get_env_var(field_name string) ?string { } } -// load attempts to create the given type from environment variables. For -// each field, the corresponding env var is its name in uppercase prepended -// with the hardcoded prefix. If this one isn't present, it looks for the env -// var with the file_suffix suffix. +// load attempts to create an object of type T from the given path to a toml +// file & environment variables. For each field, it will select either a value +// given from an environment variable, a value defined in the config file or a +// configured default if present, in that order. pub fn load(path string) ?T { mut res := T{} From 2aa2aa143ca92a1f93e29a9c7bed369d62516a48 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 18:17:33 +0200 Subject: [PATCH 068/105] Switched to proper default values system --- Makefile | 23 +++-------------------- src/env.v | 33 +++++++++++++++------------------ vieter.toml | 8 ++++++++ 3 files changed, 26 insertions(+), 38 deletions(-) create mode 100644 vieter.toml diff --git a/Makefile b/Makefile index 2238585d..76ab7b58 100644 --- a/Makefile +++ b/Makefile @@ -23,13 +23,7 @@ dvieter: $(SOURCES) # Run the debug build inside gdb .PHONY: gdb gdb: dvieter - VIETER_API_KEY=test \ - VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ - VIETER_PKG_DIR=data/pkgs \ - VIETER_LOG_LEVEL=DEBUG \ - VIETER_REPOS_FILE=data/repos.json \ - gdb --args ./dvieter + gdb --args './dvieter -f vieter.toml server' # Optimised production build .PHONY: prod @@ -46,22 +40,11 @@ c: # Run the server in the default 'data' directory .PHONY: run run: vieter - VIETER_API_KEY=test \ - VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ - VIETER_PKG_DIR=data/pkgs \ - VIETER_LOG_LEVEL=DEBUG \ - VIETER_REPOS_FILE=data/repos.json \ - ./vieter server + ./vieter -f vieter.toml server .PHONY: run-prod run-prod: prod - VIETER_API_KEY=test \ - VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ - VIETER_PKG_DIR=data/pkgs \ - VIETER_LOG_LEVEL=DEBUG \ - ./pvieter server + ./pvieter -f vieter.toml server # =====OTHER===== .PHONY: lint diff --git a/src/env.v b/src/env.v index 6abd6749..b7337919 100644 --- a/src/env.v +++ b/src/env.v @@ -12,8 +12,8 @@ const file_suffix = '_FILE' pub struct ServerConfig { pub: - log_level string [default: WARN] - log_file string [default: 'vieter.log'] + log_level string = "WARN" + log_file string = "vieter.log" pkg_dir string download_dir string api_key string @@ -65,7 +65,18 @@ pub fn load(path string) ?T { mut res := T{} if os.exists(path) { - res = toml.parse_file(path) ?.reflect() + // We don't use reflect here because reflect also sets any fields not + // in the toml back to "empty", which we don't want + doc := toml.parse_file(path) ? + + $for field in T.fields { + s := doc.value(field.name) + + // We currently only support strings + if s.type_name() == "string" { + res.$(field.name) = s.string() + } + } } $for field in T.fields { @@ -80,21 +91,7 @@ pub fn load(path string) ?T { // If there's no value from the toml file either, we try to find a // default value else if res.$(field.name) == '' { - // We use the default instead, if it's present - mut default := '' - - for attr in field.attrs { - if attr.starts_with('default: ') { - default = attr[9..] - break - } - } - - if default == '' { - return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.") - } - - res.$(field.name) = default + return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.") } } } diff --git a/vieter.toml b/vieter.toml new file mode 100644 index 00000000..3f953987 --- /dev/null +++ b/vieter.toml @@ -0,0 +1,8 @@ +# This file contains settings used during development +api_key = "test" +download_dir = "data/downloads" +repo_dir = "data/repo" +pkg_dir = "data/pkgs" +# log_level = "DEBUG" +repos_file = "data/repos.json" + From c656e672e27ae789efec5ec7d34111a0fccf88ad Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 18:20:14 +0200 Subject: [PATCH 069/105] Moved config structs to more logical location --- src/build/build.v | 3 +-- src/build/cli.v | 8 +++++++- src/env.v | 19 +------------------ src/server/cli.v | 13 ++++++++++++- src/server/server.v | 5 ++--- 5 files changed, 23 insertions(+), 25 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 6f2b0a84..934627f3 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -3,7 +3,6 @@ module build import docker import encoding.base64 import time -import env import net.http import git import json @@ -62,7 +61,7 @@ fn create_build_image() ?string { return image.id } -fn build(conf env.BuildConfig) ? { +fn build(conf Config) ? { // We get the repos list from the Vieter instance mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? req.add_custom_header('X-Api-Key', conf.api_key) ? diff --git a/src/build/cli.v b/src/build/cli.v index 463d2ca3..c95d1caa 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -3,6 +3,12 @@ module build import cli import env +pub struct Config { +pub: + api_key string + address string +} + // cmd returns the cli submodule that handles the build process pub fn cmd() cli.Command { return cli.Command{ @@ -10,7 +16,7 @@ pub fn cmd() cli.Command { description: 'Run the build process.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? + conf := env.load(config_file) ? build(conf) ? } diff --git a/src/env.v b/src/env.v index b7337919..7b207654 100644 --- a/src/env.v +++ b/src/env.v @@ -10,23 +10,6 @@ const prefix = 'VIETER_' // instead const file_suffix = '_FILE' -pub struct ServerConfig { -pub: - log_level string = "WARN" - log_file string = "vieter.log" - pkg_dir string - download_dir string - api_key string - repo_dir string - repos_file string -} - -pub struct BuildConfig { -pub: - api_key string - address string -} - fn get_env_var(field_name string) ?string { env_var_name := '$env.prefix$field_name.to_upper()' env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix' @@ -73,7 +56,7 @@ pub fn load(path string) ?T { s := doc.value(field.name) // We currently only support strings - if s.type_name() == "string" { + if s.type_name() == 'string' { res.$(field.name) = s.string() } } diff --git a/src/server/cli.v b/src/server/cli.v index 6e676867..2383d9f6 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -3,6 +3,17 @@ module server import cli import env +struct Config { +pub: + log_level string = 'WARN' + log_file string = 'vieter.log' + pkg_dir string + download_dir string + api_key string + repo_dir string + repos_file string +} + // cmd returns the cli submodule that handles starting the server pub fn cmd() cli.Command { return cli.Command{ @@ -10,7 +21,7 @@ pub fn cmd() cli.Command { description: 'Start the Vieter server' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? + conf := env.load(config_file) ? server(conf) ? } diff --git a/src/server/server.v b/src/server/server.v index 0738bc4f..ab2e46bc 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -4,7 +4,6 @@ import web import os import log import repo -import env import util const port = 8000 @@ -12,7 +11,7 @@ const port = 8000 struct App { web.Context pub: - conf env.ServerConfig [required; web_global] + conf Config [required; web_global] pub mut: repo repo.Repo [required; web_global] // This is used to claim the file lock on the repos file @@ -20,7 +19,7 @@ pub mut: } // server starts the web server & starts listening for requests -pub fn server(conf env.ServerConfig) ? { +pub fn server(conf Config) ? { // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') From dff531d866e532d5d92ea3baffd146570e64123c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 19:51:54 +0200 Subject: [PATCH 070/105] Some final adjustments before merge --- src/env.v | 2 +- src/main.v | 2 +- src/server/cli.v | 2 +- vieter.toml | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/env.v b/src/env.v index 7b207654..cbde67e1 100644 --- a/src/env.v +++ b/src/env.v @@ -49,7 +49,7 @@ pub fn load(path string) ?T { if os.exists(path) { // We don't use reflect here because reflect also sets any fields not - // in the toml back to "empty", which we don't want + // in the toml back to their zero value, which we don't want doc := toml.parse_file(path) ? $for field in T.fields { diff --git a/src/main.v b/src/main.v index 11bc77b8..c77e5519 100644 --- a/src/main.v +++ b/src/main.v @@ -9,7 +9,7 @@ import git fn main() { mut app := cli.Command{ name: 'vieter' - description: 'Arch repository server' + description: 'Vieter is a lightweight implementation of an Arch repository server.' version: '0.1.0' flags: [ cli.Flag{ diff --git a/src/server/cli.v b/src/server/cli.v index 2383d9f6..f0dc0b12 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -18,7 +18,7 @@ pub: pub fn cmd() cli.Command { return cli.Command{ name: 'server' - description: 'Start the Vieter server' + description: 'Start the Vieter server.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file') ? conf := env.load(config_file) ? diff --git a/vieter.toml b/vieter.toml index 3f953987..c17c3c30 100644 --- a/vieter.toml +++ b/vieter.toml @@ -6,3 +6,4 @@ pkg_dir = "data/pkgs" # log_level = "DEBUG" repos_file = "data/repos.json" +address = "http://localhost:8000" From 4c3f322e6f064a4c290a1a07f324973db0c3e206 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 20:03:26 +0200 Subject: [PATCH 071/105] Removed arm/v7 from docker release builds --- .woodpecker/.docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.woodpecker/.docker.yml b/.woodpecker/.docker.yml index b2f08ca9..9b605f3f 100644 --- a/.woodpecker/.docker.yml +++ b/.woodpecker/.docker.yml @@ -10,7 +10,7 @@ pipeline: settings: repo: chewingbever/vieter tag: dev - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm64/v8, linux/amd64 ] build_args_from_env: - CI_COMMIT_SHA when: @@ -23,7 +23,7 @@ pipeline: settings: repo: chewingbever/vieter auto_tag: true - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm64/v8, linux/amd64 ] build_args_from_env: - CI_COMMIT_SHA when: From d7f6c87053fda009c614521ec63c532b6b9a1af8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 21:08:50 +0200 Subject: [PATCH 072/105] Added some comments --- src/git/git.v | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/git/git.v b/src/git/git.v index 3ad35fdc..c5390b60 100644 --- a/src/git/git.v +++ b/src/git/git.v @@ -14,6 +14,8 @@ pub mut: arch []string } +// patch_from_params patches a GitRepo from a map[string]string, usually +// provided from a web.App's params pub fn (mut r GitRepo) patch_from_params(params map[string]string) { $for field in GitRepo.fields { if field.name in params { @@ -28,6 +30,7 @@ pub fn (mut r GitRepo) patch_from_params(params map[string]string) { } } +// read_repos reads the provided path & parses it into a map of GitRepo's. pub fn read_repos(path string) ?map[string]GitRepo { if !os.exists(path) { mut f := os.create(path) ? @@ -47,6 +50,7 @@ pub fn read_repos(path string) ?map[string]GitRepo { return res } +// write_repos writes a map of GitRepo's back to disk given the provided path. pub fn write_repos(path string, repos &map[string]GitRepo) ? { mut f := os.create(path) ? @@ -58,6 +62,8 @@ pub fn write_repos(path string, repos &map[string]GitRepo) ? { f.write_string(value) ? } +// repo_from_params creates a GitRepo from a map[string]string, usually +// provided from a web.App's params pub fn repo_from_params(params map[string]string) ?GitRepo { mut repo := GitRepo{} From f44ce1c17f572bb788e2002c877fc0678d71344c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 22:41:19 +0200 Subject: [PATCH 073/105] Started work on better repos cli --- src/git/cli.v | 16 +++++++++++----- src/response.v | 27 +++++++++++++++++++++++++++ src/server/git.v | 3 ++- src/server/response.v | 27 --------------------------- src/server/routes.v | 1 + 5 files changed, 41 insertions(+), 33 deletions(-) create mode 100644 src/response.v delete mode 100644 src/server/response.v diff --git a/src/git/cli.v b/src/git/cli.v index 17fa9847..586b5ba0 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -3,6 +3,9 @@ module git import cli import env import net.http +import json +import git +import response struct Config { address string [required] @@ -28,13 +31,13 @@ pub fn cmd() cli.Command { cli.Command{ name: 'add' required_args: 2 - usage: 'url branch' + usage: 'url branch arch...' description: 'Add a new repository.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file') ? conf := env.load(config_file) ? - add(conf, cmd.args[0], cmd.args[1]) ? + add(conf, cmd.args[0], cmd.args[1], cmd.args[2..]) ? } }, cli.Command{ @@ -58,12 +61,15 @@ fn list(conf Config) ? { req.add_custom_header('X-API-Key', conf.api_key) ? res := req.do() ? + data := json.decode(response.Response, res.text) ? - println(res.text) + for id, details in data.data { + println("${id[..8]}\t$details.url\t$details.branch\t$details.arch") + } } -fn add(conf Config, url string, branch string) ? { - mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=$url&branch=$branch', +fn add(conf Config, url string, branch string, arch []string) ? { + mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=$url&branch=$branch&arch=${arch.join(',')}', '') ? req.add_custom_header('X-API-Key', conf.api_key) ? diff --git a/src/response.v b/src/response.v new file mode 100644 index 00000000..1618fcf6 --- /dev/null +++ b/src/response.v @@ -0,0 +1,27 @@ +module response + +pub struct Response { + message string + data T +} + +pub fn new_response(message string) Response { + return Response{ + message: message + data: '' + } +} + +pub fn new_data_response(data T) Response { + return Response{ + message: '' + data: data + } +} + +pub fn new_full_response(message string, data T) Response { + return Response{ + message: message + data: data + } +} diff --git a/src/server/git.v b/src/server/git.v index 7d177d7d..359d6ced 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -4,6 +4,7 @@ import web import git import net.http import rand +import response { new_response, new_data_response } const repos_file = 'repos.json' @@ -15,7 +16,7 @@ fn (mut app App) get_repos() web.Result { repos := rlock app.git_mutex { git.read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file.') + app.lerror('Failed to read repos file: $err.msg') return app.status(http.Status.internal_server_error) } diff --git a/src/server/response.v b/src/server/response.v deleted file mode 100644 index 354e7da7..00000000 --- a/src/server/response.v +++ /dev/null @@ -1,27 +0,0 @@ -module server - -struct Response { - message string - data T -} - -fn new_response(message string) Response { - return Response{ - message: message - data: '' - } -} - -fn new_data_response(data T) Response { - return Response{ - message: '' - data: data - } -} - -fn new_full_response(message string, data T) Response { - return Response{ - message: message - data: data - } -} diff --git a/src/server/routes.v b/src/server/routes.v index 0e7d72c8..71da02a3 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -7,6 +7,7 @@ import time import rand import util import net.http +import response { new_response, new_data_response } // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. From 7eb0aa76e179d041e817ccbdb0632fad93256e82 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 11:54:20 +0200 Subject: [PATCH 074/105] CLI tool can now work with new repo UUIDs --- src/git/cli.v | 42 ++++++++++++++++++++++++++++++++++-------- vieter.toml | 2 +- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/git/cli.v b/src/git/cli.v index 586b5ba0..c4d4d800 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -42,28 +42,34 @@ pub fn cmd() cli.Command { }, cli.Command{ name: 'remove' - required_args: 2 - usage: 'url branch' - description: 'Remove a repository.' + required_args: 1 + usage: 'id' + description: 'Remove a repository that matches the given ID prefix.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file') ? conf := env.load(config_file) ? - remove(conf, cmd.args[0], cmd.args[1]) ? + remove(conf, cmd.args[0]) ? } }, ] } } -fn list(conf Config) ? { +fn get_repos(conf Config) ?map[string]git.GitRepo { mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? req.add_custom_header('X-API-Key', conf.api_key) ? res := req.do() ? data := json.decode(response.Response, res.text) ? - for id, details in data.data { + return data.data +} + +fn list(conf Config) ? { + repos := get_repos(conf) ? + + for id, details in repos { println("${id[..8]}\t$details.url\t$details.branch\t$details.arch") } } @@ -78,8 +84,28 @@ fn add(conf Config, url string, branch string, arch []string) ? { println(res.text) } -fn remove(conf Config, url string, branch string) ? { - mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=$url&branch=$branch', +fn remove(conf Config, id_prefix string) ? { + repos := get_repos(conf) ? + + mut to_remove := []string{} + + for id, _ in repos { + if id.starts_with(id_prefix) { + to_remove << id + } + } + + if to_remove.len == 0 { + eprintln("No repo found for given prefix.") + exit(1) + } + + if to_remove.len > 1 { + eprintln("Multiple repos found for given prefix.") + exit(1) + } + + mut req := http.new_request(http.Method.delete, '$conf.address/api/repos/${to_remove[0]}', '') ? req.add_custom_header('X-API-Key', conf.api_key) ? diff --git a/vieter.toml b/vieter.toml index c17c3c30..75929423 100644 --- a/vieter.toml +++ b/vieter.toml @@ -3,7 +3,7 @@ api_key = "test" download_dir = "data/downloads" repo_dir = "data/repo" pkg_dir = "data/pkgs" -# log_level = "DEBUG" +log_level = "DEBUG" repos_file = "data/repos.json" address = "http://localhost:8000" From b31b4cbd7a9fc1b64061c253e4260f66d56e0430 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 11:58:05 +0200 Subject: [PATCH 075/105] Pleased vfmt & vet --- src/git/cli.v | 11 +++++------ src/response.v | 6 ++++++ src/server/git.v | 2 +- src/server/routes.v | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/git/cli.v b/src/git/cli.v index c4d4d800..7e41b1c3 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -4,7 +4,6 @@ import cli import env import net.http import json -import git import response struct Config { @@ -56,12 +55,12 @@ pub fn cmd() cli.Command { } } -fn get_repos(conf Config) ?map[string]git.GitRepo { +fn get_repos(conf Config) ?map[string]GitRepo { mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? req.add_custom_header('X-API-Key', conf.api_key) ? res := req.do() ? - data := json.decode(response.Response, res.text) ? + data := json.decode(response.Response, res.text) ? return data.data } @@ -70,7 +69,7 @@ fn list(conf Config) ? { repos := get_repos(conf) ? for id, details in repos { - println("${id[..8]}\t$details.url\t$details.branch\t$details.arch") + println('${id[..8]}\t$details.url\t$details.branch\t$details.arch') } } @@ -96,12 +95,12 @@ fn remove(conf Config, id_prefix string) ? { } if to_remove.len == 0 { - eprintln("No repo found for given prefix.") + eprintln('No repo found for given prefix.') exit(1) } if to_remove.len > 1 { - eprintln("Multiple repos found for given prefix.") + eprintln('Multiple repos found for given prefix.') exit(1) } diff --git a/src/response.v b/src/response.v index 1618fcf6..7e268b13 100644 --- a/src/response.v +++ b/src/response.v @@ -5,6 +5,8 @@ pub struct Response { data T } +// new_response constructs a new Response object with the given message +// & an empty data field. pub fn new_response(message string) Response { return Response{ message: message @@ -12,6 +14,8 @@ pub fn new_response(message string) Response { } } +// new_data_response constructs a new Response object with the given data +// & an empty message field. pub fn new_data_response(data T) Response { return Response{ message: '' @@ -19,6 +23,8 @@ pub fn new_data_response(data T) Response { } } +// new_full_response constructs a new Response object with the given +// message & data. pub fn new_full_response(message string, data T) Response { return Response{ message: message diff --git a/src/server/git.v b/src/server/git.v index 359d6ced..2a682d8d 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -4,7 +4,7 @@ import web import git import net.http import rand -import response { new_response, new_data_response } +import response { new_data_response, new_response } const repos_file = 'repos.json' diff --git a/src/server/routes.v b/src/server/routes.v index 71da02a3..07279cbe 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -7,7 +7,7 @@ import time import rand import util import net.http -import response { new_response, new_data_response } +import response { new_response } // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. From fc1d4480dc220e6f6245b6d6add8a24550f39493 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 12:07:56 +0200 Subject: [PATCH 076/105] Split Git client code into separate module --- src/git/cli.v | 33 ++++++--------------------------- src/git/client.v | 36 ++++++++++++++++++++++++++++++++++++ src/response.v | 1 + 3 files changed, 43 insertions(+), 27 deletions(-) create mode 100644 src/git/client.v diff --git a/src/git/cli.v b/src/git/cli.v index 7e41b1c3..4a066d51 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -2,9 +2,6 @@ module git import cli import env -import net.http -import json -import response struct Config { address string [required] @@ -55,18 +52,8 @@ pub fn cmd() cli.Command { } } -fn get_repos(conf Config) ?map[string]GitRepo { - mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? - req.add_custom_header('X-API-Key', conf.api_key) ? - - res := req.do() ? - data := json.decode(response.Response, res.text) ? - - return data.data -} - fn list(conf Config) ? { - repos := get_repos(conf) ? + repos := get_repos(conf.address, conf.api_key) ? for id, details in repos { println('${id[..8]}\t$details.url\t$details.branch\t$details.arch') @@ -74,17 +61,13 @@ fn list(conf Config) ? { } fn add(conf Config, url string, branch string, arch []string) ? { - mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=$url&branch=$branch&arch=${arch.join(',')}', - '') ? - req.add_custom_header('X-API-Key', conf.api_key) ? + res := add_repo(conf.address, conf.api_key, url, branch, arch) ? - res := req.do() ? - - println(res.text) + println(res.message) } fn remove(conf Config, id_prefix string) ? { - repos := get_repos(conf) ? + repos := get_repos(conf.address, conf.api_key) ? mut to_remove := []string{} @@ -104,11 +87,7 @@ fn remove(conf Config, id_prefix string) ? { exit(1) } - mut req := http.new_request(http.Method.delete, '$conf.address/api/repos/${to_remove[0]}', - '') ? - req.add_custom_header('X-API-Key', conf.api_key) ? + res := remove_repo(conf.address, conf.api_key, to_remove[0]) ? - res := req.do() ? - - println(res.text) + println(res.message) } diff --git a/src/git/client.v b/src/git/client.v new file mode 100644 index 00000000..3da1a616 --- /dev/null +++ b/src/git/client.v @@ -0,0 +1,36 @@ +module git + +import json +import response { Response } +import net.http + +fn get_repos(address string, api_key string) ?map[string]GitRepo { + mut req := http.new_request(http.Method.get, '$address/api/repos', '') ? + req.add_custom_header('X-API-Key', api_key) ? + + res := req.do() ? + data := json.decode(Response, res.text) ? + + return data.data +} + +fn add_repo(address string, api_key string, url string, branch string, arch []string) ?Response { + mut req := http.new_request(http.Method.post, '$address/api/repos?url=$url&branch=$branch&arch=${arch.join(',')}', + '') ? + req.add_custom_header('X-API-Key', api_key) ? + + res := req.do() ? + data := json.decode(Response, res.text) ? + + return data +} + +fn remove_repo(address string, api_key string, id string) ?Response { + mut req := http.new_request(http.Method.delete, '$address/api/repos/$id', '') ? + req.add_custom_header('X-API-Key', api_key) ? + + res := req.do() ? + data := json.decode(Response, res.text) ? + + return data +} diff --git a/src/response.v b/src/response.v index 7e268b13..a06a589c 100644 --- a/src/response.v +++ b/src/response.v @@ -1,6 +1,7 @@ module response pub struct Response { +pub: message string data T } From 78310fa1e425652fb0f2f2e29865a6f2b9310cf0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 12:10:37 +0200 Subject: [PATCH 077/105] Builder now uses new Git repos API --- src/build/build.v | 10 ++-------- src/git/client.v | 9 ++++++--- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 934627f3..c42c98d8 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -3,9 +3,7 @@ module build import docker import encoding.base64 import time -import net.http import git -import json const container_build_dir = '/build' @@ -63,11 +61,7 @@ fn create_build_image() ?string { fn build(conf Config) ? { // We get the repos list from the Vieter instance - mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? - req.add_custom_header('X-Api-Key', conf.api_key) ? - - res := req.do() ? - repos := json.decode([]git.GitRepo, res.text) ? + repos := git.get_repos(conf.address, conf.api_key) ? // No point in doing work if there's no repos present if repos.len == 0 { @@ -77,7 +71,7 @@ fn build(conf Config) ? { // First, we create a base image which has updated repos n stuff image_id := create_build_image() ? - for repo in repos { + for _, repo in repos { // TODO what to do with PKGBUILDs that build multiple packages? commands := [ 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', diff --git a/src/git/client.v b/src/git/client.v index 3da1a616..97fe9fb5 100644 --- a/src/git/client.v +++ b/src/git/client.v @@ -4,7 +4,8 @@ import json import response { Response } import net.http -fn get_repos(address string, api_key string) ?map[string]GitRepo { +// get_repos returns the current list of repos. +pub fn get_repos(address string, api_key string) ?map[string]GitRepo { mut req := http.new_request(http.Method.get, '$address/api/repos', '') ? req.add_custom_header('X-API-Key', api_key) ? @@ -14,7 +15,8 @@ fn get_repos(address string, api_key string) ?map[string]GitRepo { return data.data } -fn add_repo(address string, api_key string, url string, branch string, arch []string) ?Response { +// add_repo adds a new repo to the server. +pub fn add_repo(address string, api_key string, url string, branch string, arch []string) ?Response { mut req := http.new_request(http.Method.post, '$address/api/repos?url=$url&branch=$branch&arch=${arch.join(',')}', '') ? req.add_custom_header('X-API-Key', api_key) ? @@ -25,7 +27,8 @@ fn add_repo(address string, api_key string, url string, branch string, arch []st return data } -fn remove_repo(address string, api_key string, id string) ?Response { +// remove_repo removes the repo with the given ID from the server. +pub fn remove_repo(address string, api_key string, id string) ?Response { mut req := http.new_request(http.Method.delete, '$address/api/repos/$id', '') ? req.add_custom_header('X-API-Key', api_key) ? From 7895240e9b538d5544bf063bad549fcebc8d10cb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 12:22:41 +0200 Subject: [PATCH 078/105] Updatd CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f95351fc..96f7dcf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Changed -* Better environment variable support +* Better config system + * Support for both a config file & environment variables * Each env var can now be provided from a file by appending it with `_FILE` & passing the path to the file as value * Revamped web framework @@ -21,7 +22,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Very basic build system * Build is triggered by separate cron container * Packages build on cron container's system - * Packages are always rebuilt, even if they haven't changed * Hardcoded planning of builds * Builds are sequential * API for managing Git repositories to build From 9d8491a77a3fee0d6a2a325fec98a98b065a20da Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 13:47:06 +0200 Subject: [PATCH 079/105] Changed behavior for default_arch variable --- src/repo/repo.v | 17 +++++++++++------ src/server/cli.v | 1 + src/server/routes.v | 4 ---- src/server/server.v | 7 ++++++- test.py | 4 ++-- vieter.toml | 1 + 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/repo/repo.v b/src/repo/repo.v index 4e4fa34b..334c8942 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -72,10 +72,13 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re // add_pkg_in_repo adds a package to a given repo. This function is responsible // for inspecting the package architecture. If said architecture is 'any', the -// package is added to each arch-repository within the given repo. If none -// exist, one is created for provided r.default_arch value. If the architecture -// isn't 'any', the package is only added to the specific architecture. +// package is added to each arch-repository within the given repo. A package of +// architecture 'any' will always be added to the arch-repo defined by +// r.default_arch. If this arch-repo doesn't exist yet, it will be created. If +// the architecture isn't 'any', the package is only added to the specific +// architecture. fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { + // A package without arch 'any' can be handled without any further checks if pkg.info.arch != 'any' { return r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) } @@ -84,15 +87,17 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { mut arch_repos := []string{} - // If this is the first package to be added to the repository, it won't - // contain any arch-repos yet. + // If this is the first package that's added to the repo, the directory + // won't exist yet if os.exists(repo_dir) { // We get a listing of all currently present arch-repos in the given repo arch_repos = os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, it))) } - if arch_repos.len == 0 { + // The default_arch should always be updated when a package with arch 'any' + // is added. + if !arch_repos.contains(r.default_arch) { arch_repos << r.default_arch } diff --git a/src/server/cli.v b/src/server/cli.v index 9820cf8a..b3f37e30 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -12,6 +12,7 @@ pub: api_key string data_dir string repos_file string + default_arch string } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/routes.v b/src/server/routes.v index 55f6f129..8c8cdff2 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -58,10 +58,6 @@ fn (mut app App) put_package(repo string) web.Result { // Generate a random filename for the temp file pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4()) - for os.exists(pkg_path) { - pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4()) - } - app.ldebug("Uploading $length bytes (${util.pretty_bytes(length.int())}) to '$pkg_path'.") // This is used to time how long it takes to upload a file diff --git a/src/server/server.v b/src/server/server.v index 7b59896a..f93a5aea 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -20,6 +20,11 @@ pub mut: // server starts the web server & starts listening for requests pub fn server(conf Config) ? { + // Prevent using 'any' as the default arch + if conf.default_arch == 'any' { + util.exit_with_message(1, "'any' is not allowed as the value for default_arch.") + } + // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -39,7 +44,7 @@ pub fn server(conf Config) ? { } // This also creates the directories if needed - repo := repo.new(conf.data_dir, conf.pkg_dir, 'x86_64') or { + repo := repo.new(conf.data_dir, conf.pkg_dir, conf.default_arch) or { logger.error(err.msg) exit(1) } diff --git a/test.py b/test.py index f1668d7b..9b0116ec 100644 --- a/test.py +++ b/test.py @@ -46,7 +46,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len): "pkgname": name, "pkgbase": name, "pkgver": ver, - "arch": "x86_64" + "arch": "any" } return "\n".join(f"{key} = {value}" for key, value in data.items()) @@ -97,7 +97,7 @@ async def upload_random_package(tar_path, sem): async with sem: with open(tar_path, 'rb') as f: async with aiohttp.ClientSession() as s: - async with s.post("http://localhost:8000/vieter2/publish", data=f.read(), headers={"x-api-key": "test"}) as r: + async with s.post("http://localhost:8000/vieter/publish", data=f.read(), headers={"x-api-key": "test"}) as r: return await check_output(r) diff --git a/vieter.toml b/vieter.toml index f3904b6f..10b3855d 100644 --- a/vieter.toml +++ b/vieter.toml @@ -5,5 +5,6 @@ data_dir = "data" pkg_dir = "data/pkgs" log_level = "DEBUG" repos_file = "data/repos.json" +default_arch = "x86_64" address = "http://localhost:8000" From 7eab7afa982ea24232460374210e5d2576b9288d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 14:28:21 +0200 Subject: [PATCH 080/105] Expanded git repos api to store repository to publish to --- src/git/cli.v | 12 ++++++------ src/git/client.v | 43 +++++++++++++++++++++++++++---------------- src/git/git.v | 2 ++ 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/src/git/cli.v b/src/git/cli.v index 4a066d51..376592d8 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -26,14 +26,14 @@ pub fn cmd() cli.Command { }, cli.Command{ name: 'add' - required_args: 2 - usage: 'url branch arch...' + required_args: 4 + usage: 'url branch repo arch...' description: 'Add a new repository.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file') ? conf := env.load(config_file) ? - add(conf, cmd.args[0], cmd.args[1], cmd.args[2..]) ? + add(conf, cmd.args[0], cmd.args[1], cmd.args[2], cmd.args[3..]) ? } }, cli.Command{ @@ -56,12 +56,12 @@ fn list(conf Config) ? { repos := get_repos(conf.address, conf.api_key) ? for id, details in repos { - println('${id[..8]}\t$details.url\t$details.branch\t$details.arch') + println('${id[..8]}\t$details.url\t$details.branch\t$details.repo\t$details.arch') } } -fn add(conf Config, url string, branch string, arch []string) ? { - res := add_repo(conf.address, conf.api_key, url, branch, arch) ? +fn add(conf Config, url string, branch string, repo string, arch []string) ? { + res := add_repo(conf.address, conf.api_key, url, branch, repo, arch) ? println(res.message) } diff --git a/src/git/client.v b/src/git/client.v index 97fe9fb5..7e1e55b0 100644 --- a/src/git/client.v +++ b/src/git/client.v @@ -4,36 +4,47 @@ import json import response { Response } import net.http -// get_repos returns the current list of repos. -pub fn get_repos(address string, api_key string) ?map[string]GitRepo { - mut req := http.new_request(http.Method.get, '$address/api/repos', '') ? +fn send_request(method http.Method, address string, url string, api_key string, params map[string]string) ?Response { + mut full_url := '$address$url' + + if params.len > 0 { + params_str := params.keys().map('$it=${params[it]}').join('&') + + full_url = "$full_url?$params_str" + } + + mut req := http.new_request(method, full_url, '') ? req.add_custom_header('X-API-Key', api_key) ? res := req.do() ? - data := json.decode(Response, res.text) ? + data := json.decode(Response, res.text) ? + + return data +} + +// get_repos returns the current list of repos. +pub fn get_repos(address string, api_key string) ?map[string]GitRepo { + data := send_request(http.Method.get, address, '/api/repos', api_key, {}) ? return data.data } // add_repo adds a new repo to the server. -pub fn add_repo(address string, api_key string, url string, branch string, arch []string) ?Response { - mut req := http.new_request(http.Method.post, '$address/api/repos?url=$url&branch=$branch&arch=${arch.join(',')}', - '') ? - req.add_custom_header('X-API-Key', api_key) ? - - res := req.do() ? - data := json.decode(Response, res.text) ? +pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response { + params := { + 'url': url + 'branch': branch + 'repo': repo + 'arch': arch.join(',') + } + data := send_request(http.Method.post, address, '/api/repos', api_key, params) ? return data } // remove_repo removes the repo with the given ID from the server. pub fn remove_repo(address string, api_key string, id string) ?Response { - mut req := http.new_request(http.Method.delete, '$address/api/repos/$id', '') ? - req.add_custom_header('X-API-Key', api_key) ? - - res := req.do() ? - data := json.decode(Response, res.text) ? + data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, {}) ? return data } diff --git a/src/git/git.v b/src/git/git.v index c5390b60..eaec8959 100644 --- a/src/git/git.v +++ b/src/git/git.v @@ -12,6 +12,8 @@ pub mut: // On which architectures the package is allowed to be built. In reality, // this controls which builders will periodically build the image. arch []string + // Which repo the builder should publish packages to + repo string } // patch_from_params patches a GitRepo from a map[string]string, usually From 8ecac2ff28895151a2cc1adc0d4c306d3ef71581 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 14:40:49 +0200 Subject: [PATCH 081/105] Updated build system to respect repo arch settings --- src/build/build.v | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index c42c98d8..31e7c3f3 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -4,11 +4,14 @@ import docker import encoding.base64 import time import git +import os const container_build_dir = '/build' const build_image_repo = 'vieter-build' +const base_image = 'archlinux:latest' + fn create_build_image() ?string { commands := [ // Update repos & install required packages @@ -26,7 +29,7 @@ fn create_build_image() ?string { cmds_str := base64.encode_str(commands.join('\n')) c := docker.NewContainer{ - image: 'archlinux:latest' + image: base_image env: ['BUILD_SCRIPT=$cmds_str'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] @@ -60,27 +63,34 @@ fn create_build_image() ?string { } fn build(conf Config) ? { - // We get the repos list from the Vieter instance - repos := git.get_repos(conf.address, conf.api_key) ? + build_arch := os.uname().machine + + // We get the repos map from the Vieter instance + repos_map := git.get_repos(conf.address, conf.api_key) ? + + // We filter out any repos that aren't allowed to be built on this + // architecture + filtered_repos := repos_map.keys().map(repos_map[it]).filter(it.arch.contains(build_arch)) // No point in doing work if there's no repos present - if repos.len == 0 { + if filtered_repos.len == 0 { return } // First, we create a base image which has updated repos n stuff image_id := create_build_image() ? - for _, repo in repos { + for repo in filtered_repos { // TODO what to do with PKGBUILDs that build multiple packages? commands := [ 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', 'cd repo', 'makepkg --nobuild --nodeps', 'source PKGBUILD', - // The build container checks whether the package is already present on the server - 'curl --head --fail $conf.address/\$pkgname-\$pkgver-\$pkgrel-\$(uname -m).pkg.tar.zst && exit 0', - 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done', + // The build container checks whether the package is already + // present on the server + 'curl --head --fail $conf.address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel-${build_arch}.pkg.tar.zst && exit 0', + 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/$repo.repo/publish; done', ] // We convert the list of commands into a base64 string, which then gets From d11ad78bff759d050029cc9641c8d3662d52832a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 14:43:41 +0200 Subject: [PATCH 082/105] Updated CI to use new system --- .woodpecker/.arch.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index d6463532..23c34083 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -3,10 +3,10 @@ branches: [dev] pipeline: build: - image: 'archlinux:latest' + image: 'archlinux:base-devel' commands: # Update packages - - pacman -Syu --needed --noconfirm base-devel + - pacman -Syu # Create non-root user to perform build & switch to their home - groupadd -g 1000 builder - useradd -mg builder builder @@ -17,9 +17,9 @@ pipeline: - makepkg -s --noconfirm --needed publish: - image: 'archlinux:latest' + image: 'curlimages/curl:latest' commands: # Publish the package - - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/publish; done' + - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done' secrets: - vieter_api_key From ab72c800c360c996a7f4beb1321c718105056b1f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 14:50:07 +0200 Subject: [PATCH 083/105] Ran vfmt --- src/build/build.v | 2 +- src/git/client.v | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 31e7c3f3..4270e9d7 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -29,7 +29,7 @@ fn create_build_image() ?string { cmds_str := base64.encode_str(commands.join('\n')) c := docker.NewContainer{ - image: base_image + image: build.base_image env: ['BUILD_SCRIPT=$cmds_str'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] diff --git a/src/git/client.v b/src/git/client.v index 7e1e55b0..92d27eea 100644 --- a/src/git/client.v +++ b/src/git/client.v @@ -10,7 +10,7 @@ fn send_request(method http.Method, address string, url string, api_key strin if params.len > 0 { params_str := params.keys().map('$it=${params[it]}').join('&') - full_url = "$full_url?$params_str" + full_url = '$full_url?$params_str' } mut req := http.new_request(method, full_url, '') ? @@ -24,7 +24,8 @@ fn send_request(method http.Method, address string, url string, api_key strin // get_repos returns the current list of repos. pub fn get_repos(address string, api_key string) ?map[string]GitRepo { - data := send_request(http.Method.get, address, '/api/repos', api_key, {}) ? + data := send_request(http.Method.get, address, '/api/repos', api_key, + {}) ? return data.data } @@ -32,10 +33,10 @@ pub fn get_repos(address string, api_key string) ?map[string]GitRepo { // add_repo adds a new repo to the server. pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response { params := { - 'url': url + 'url': url 'branch': branch - 'repo': repo - 'arch': arch.join(',') + 'repo': repo + 'arch': arch.join(',') } data := send_request(http.Method.post, address, '/api/repos', api_key, params) ? @@ -44,7 +45,8 @@ pub fn add_repo(address string, api_key string, url string, branch string, repo // remove_repo removes the repo with the given ID from the server. pub fn remove_repo(address string, api_key string, id string) ?Response { - data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, {}) ? + data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, + {}) ? return data } From c6d176f426738c7282757927b2a516f5fe136e26 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 15:08:27 +0200 Subject: [PATCH 084/105] Added patch support to repos cli --- src/git/cli.v | 96 +++++++++++++++++++++++++++++++++++++----------- src/git/client.v | 9 +++++ 2 files changed, 84 insertions(+), 21 deletions(-) diff --git a/src/git/cli.v b/src/git/cli.v index 376592d8..a75f8a02 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -48,10 +48,76 @@ pub fn cmd() cli.Command { remove(conf, cmd.args[0]) ? } }, + cli.Command{ + name: 'edit' + required_args: 1 + usage: 'id' + description: 'Edit the repository that matches the given ID prefix.' + flags: [ + cli.Flag{ + name: 'url' + description: 'URL of the Git repository.' + flag: cli.FlagType.string + }, + cli.Flag{ + name: 'branch' + description: 'Branch of the Git repository.' + flag: cli.FlagType.string + }, + cli.Flag{ + name: 'repo' + description: 'Repo to publish builds to.' + flag: cli.FlagType.string + }, + cli.Flag{ + name: 'arch' + description: 'Comma-separated list of architectures to build on.' + flag: cli.FlagType.string + }, + ] + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? + + found := cmd.flags.get_all_found() + + mut params := map[string]string{} + + for f in found { + if f.name != 'config-file' { + params[f.name] = f.get_string() ? + } + } + + patch(conf, cmd.args[0], params) ? + } + }, ] } } +fn get_repo_id_by_prefix(conf Config, id_prefix string) ?string { + repos := get_repos(conf.address, conf.api_key) ? + + mut res := []string{} + + for id, _ in repos { + if id.starts_with(id_prefix) { + res << id + } + } + + if res.len == 0 { + eprintln('No repo found for given prefix.') + } + + if res.len > 1 { + eprintln('Multiple repos found for given prefix.') + } + + return res[0] +} + fn list(conf Config) ? { repos := get_repos(conf.address, conf.api_key) ? @@ -67,27 +133,15 @@ fn add(conf Config, url string, branch string, repo string, arch []string) ? { } fn remove(conf Config, id_prefix string) ? { - repos := get_repos(conf.address, conf.api_key) ? - - mut to_remove := []string{} - - for id, _ in repos { - if id.starts_with(id_prefix) { - to_remove << id - } - } - - if to_remove.len == 0 { - eprintln('No repo found for given prefix.') - exit(1) - } - - if to_remove.len > 1 { - eprintln('Multiple repos found for given prefix.') - exit(1) - } - - res := remove_repo(conf.address, conf.api_key, to_remove[0]) ? + id := get_repo_id_by_prefix(conf, id_prefix) ? + res := remove_repo(conf.address, conf.api_key, id) ? + + println(res.message) +} + +fn patch(conf Config, id_prefix string, params map[string]string) ? { + id := get_repo_id_by_prefix(conf, id_prefix) ? + res := patch_repo(conf.address, conf.api_key, id, params) ? println(res.message) } diff --git a/src/git/client.v b/src/git/client.v index 92d27eea..e4a39acd 100644 --- a/src/git/client.v +++ b/src/git/client.v @@ -50,3 +50,12 @@ pub fn remove_repo(address string, api_key string, id string) ?Response return data } + +// patch_repo sends a PATCH request to the given repo with the params as +// payload. +pub fn patch_repo(address string, api_key string, id string, params map[string]string) ?Response { + data := send_request(http.Method.patch, address, '/api/repos/$id', api_key, + params) ? + + return data +} From a5ac0b69562da2d9558fcc4d19860a5e816ef16e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 15:12:48 +0200 Subject: [PATCH 085/105] Fixed dumb mistake in repos cli --- src/git/cli.v | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/git/cli.v b/src/git/cli.v index a75f8a02..463f1ba1 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -108,11 +108,11 @@ fn get_repo_id_by_prefix(conf Config, id_prefix string) ?string { } if res.len == 0 { - eprintln('No repo found for given prefix.') + return error('No repo found for given prefix.') } if res.len > 1 { - eprintln('Multiple repos found for given prefix.') + return error('Multiple repos found for given prefix.') } return res[0] From 3b555efa916179e0cbd885d5a229151b67f761cc Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 15:21:27 +0200 Subject: [PATCH 086/105] Renamed data_dir to repos_dir --- Dockerfile | 2 +- src/repo/repo.v | 16 ++++++++-------- src/repo/sync.v | 2 +- src/server/cli.v | 2 +- src/server/routes.v | 2 +- src/server/server.v | 2 +- vieter.toml | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8b625217..58087ad2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN if [ -n "${CI_COMMIT_SHA}" ]; then \ FROM busybox:1.35.0 ENV PATH=/bin \ - VIETER_REPO_DIR=/data/repo \ + VIETER_REPOS_DIR=/data/repos \ VIETER_PKG_DIR=/data/pkgs \ VIETER_DOWNLOAD_DIR=/data/downloads \ VIETER_REPOS_FILE=/data/repos.json diff --git a/src/repo/repo.v b/src/repo/repo.v index 334c8942..228b17e7 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -11,7 +11,7 @@ mut: mutex shared util.Dummy pub: // Where to store repositories' files - data_dir string [required] + repos_dir string [required] // Where packages are stored; each architecture gets its own subdirectory pkg_dir string [required] // The default architecture to use for a repository. In reality, this value @@ -27,9 +27,9 @@ pub: } // new creates a new RepoGroupManager & creates the directories as needed -pub fn new(data_dir string, pkg_dir string, default_arch string) ?RepoGroupManager { - if !os.is_dir(data_dir) { - os.mkdir_all(data_dir) or { return error('Failed to create repo directory: $err.msg') } +pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupManager { + if !os.is_dir(repos_dir) { + os.mkdir_all(repos_dir) or { return error('Failed to create repos directory: $err.msg') } } if !os.is_dir(pkg_dir) { @@ -37,7 +37,7 @@ pub fn new(data_dir string, pkg_dir string, default_arch string) ?RepoGroupManag } return RepoGroupManager{ - data_dir: data_dir + repos_dir: repos_dir pkg_dir: pkg_dir default_arch: default_arch } @@ -83,7 +83,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { return r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) } - repo_dir := os.join_path_single(r.data_dir, repo) + repo_dir := os.join_path_single(r.repos_dir, repo) mut arch_repos := []string{} @@ -118,7 +118,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { // changes. The function returns false if the package was already present in // the repo, and true otherwise. fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool { - pkg_dir := os.join_path(r.data_dir, repo, arch, '$pkg.info.name-$pkg.info.version') + pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version') // We can't add the same package twice if os.exists(pkg_dir) { @@ -150,7 +150,7 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac // returns false if the package wasn't present in the database. It also // optionally re-syncs the repo archives. fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg &package.Pkg, sync bool) ?bool { - repo_dir := os.join_path(r.data_dir, repo, arch) + repo_dir := os.join_path(r.repos_dir, repo, arch) // If the repository doesn't exist yet, the result is automatically false if !os.exists(repo_dir) { diff --git a/src/repo/sync.v b/src/repo/sync.v index afb8a4bc..e2b7aac7 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -31,7 +31,7 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri // Re-generate the repo archive files fn (r &RepoGroupManager) sync(repo string, arch string) ? { - subrepo_path := os.join_path(r.data_dir, repo, arch) + subrepo_path := os.join_path(r.repos_dir, repo, arch) lock r.mutex { a_db := C.archive_write_new() diff --git a/src/server/cli.v b/src/server/cli.v index b3f37e30..ae9e89ce 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -10,7 +10,7 @@ pub: pkg_dir string download_dir string api_key string - data_dir string + repos_dir string repos_file string default_arch string } diff --git a/src/server/routes.v b/src/server/routes.v index 8c8cdff2..a9264256 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -23,7 +23,7 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re db_exts := ['.db', '.files', '.db.tar.gz', '.files.tar.gz'] if db_exts.any(filename.ends_with(it)) { - full_path = os.join_path(app.repo.data_dir, repo, arch, filename) + full_path = os.join_path(app.repo.repos_dir, repo, arch, filename) // repo-add does this using symlinks, but we just change the requested // path diff --git a/src/server/server.v b/src/server/server.v index f93a5aea..5bf9a87e 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -44,7 +44,7 @@ pub fn server(conf Config) ? { } // This also creates the directories if needed - repo := repo.new(conf.data_dir, conf.pkg_dir, conf.default_arch) or { + repo := repo.new(conf.repos_dir, conf.pkg_dir, conf.default_arch) or { logger.error(err.msg) exit(1) } diff --git a/vieter.toml b/vieter.toml index 10b3855d..8e0447b2 100644 --- a/vieter.toml +++ b/vieter.toml @@ -1,7 +1,7 @@ # This file contains settings used during development api_key = "test" download_dir = "data/downloads" -data_dir = "data" +repos_dir = "data/repos" pkg_dir = "data/pkgs" log_level = "DEBUG" repos_file = "data/repos.json" From 97f35322042c655b39483ec6fe36f2d2b2640561 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 16:11:05 +0200 Subject: [PATCH 087/105] Ran vfmt --- src/server/cli.v | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/cli.v b/src/server/cli.v index ae9e89ce..bea223d4 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -10,7 +10,7 @@ pub: pkg_dir string download_dir string api_key string - repos_dir string + repos_dir string repos_file string default_arch string } From 8fd2b8c7ecbb3337cfba9166976d84730eb76e50 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 16:23:52 +0200 Subject: [PATCH 088/105] Switched arch build curl back to other image --- .woodpecker/.arch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index 23c34083..bb5dae09 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -17,7 +17,7 @@ pipeline: - makepkg -s --noconfirm --needed publish: - image: 'curlimages/curl:latest' + image: 'archlinux:base-devel' commands: # Publish the package - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done' From 690ca90b45a67d10aff6ece82cd4f37b3f80963d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 16:24:51 +0200 Subject: [PATCH 089/105] Maybe it needs some quotes? --- .woodpecker/.arch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index bb5dae09..76e8dc20 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -6,7 +6,7 @@ pipeline: image: 'archlinux:base-devel' commands: # Update packages - - pacman -Syu + - 'pacman -Syu' # Create non-root user to perform build & switch to their home - groupadd -g 1000 builder - useradd -mg builder builder From 0bfe28dbc96295b5015a99ad8f7f0e240a6d9996 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 16:26:06 +0200 Subject: [PATCH 090/105] Added --noconfirm flag to pacman --- .woodpecker/.arch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index 76e8dc20..3fdf1025 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -6,7 +6,7 @@ pipeline: image: 'archlinux:base-devel' commands: # Update packages - - 'pacman -Syu' + - pacman -Syu --noconfirm # Create non-root user to perform build & switch to their home - groupadd -g 1000 builder - useradd -mg builder builder From c3ac00f24d930e2db9c4b122cf3c9f34dc2e05e2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 21:29:08 +0200 Subject: [PATCH 091/105] Added support for xz-compressed packages --- src/archive.c.v | 3 +++ src/package.v | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/archive.c.v b/src/archive.c.v index 8d1314fd..1f0d1dd5 100644 --- a/src/archive.c.v +++ b/src/archive.c.v @@ -15,6 +15,9 @@ fn C.archive_read_support_filter_zstd(&C.archive) // Configure the archive to work with gzip compression fn C.archive_read_support_filter_gzip(&C.archive) +// Configure the archive to work with xz compression +fn C.archive_read_support_filter_xz(&C.archive) + // Configure the archive to work with a tarball content fn C.archive_read_support_format_tar(&C.archive) diff --git a/src/package.v b/src/package.v index 8bd30d04..41251a6c 100644 --- a/src/package.v +++ b/src/package.v @@ -100,7 +100,7 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { } // read_pkg_archive extracts the file list & .PKGINFO contents from an archive -// NOTE: this command only supports zstd- & gzip-compressed tarballs +// NOTE: this command only supports zstd-, xz- & gzip-compressed tarballs. pub fn read_pkg_archive(pkg_path string) ?Pkg { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") @@ -112,6 +112,7 @@ pub fn read_pkg_archive(pkg_path string) ?Pkg { // Sinds 2020, all newly built Arch packages use zstd C.archive_read_support_filter_zstd(a) C.archive_read_support_filter_gzip(a) + C.archive_read_support_filter_xz(a) // The content should always be a tarball C.archive_read_support_format_tar(a) From 0a74f100520f91e6465318ebfe29087b1f2e360e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 21:36:33 +0200 Subject: [PATCH 092/105] This is to tickle the CI --- PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PKGBUILD b/PKGBUILD index cb0c24a0..5011ab16 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -2,7 +2,7 @@ pkgbase='vieter' pkgname='vieter' -pkgver=0.1.0.rc1.r45.g6d3ff8a +pkgver=0.1.0.rc1.r117.gc3ac00f pkgrel=1 depends=('glibc' 'openssl' 'libarchive' 'gc') makedepends=('git' 'gcc') From 0617c9b9048d055eb02b8e0be929cc159d21719c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 22:00:38 +0200 Subject: [PATCH 093/105] forgot to add part for creating filename --- src/package.v | 1 + 1 file changed, 1 insertion(+) diff --git a/src/package.v b/src/package.v index 41251a6c..a6be6360 100644 --- a/src/package.v +++ b/src/package.v @@ -191,6 +191,7 @@ pub fn (pkg &Pkg) filename() string { ext := match pkg.compression { 0 { '.tar' } 1 { '.tar.gz' } + 6 { '.tar.xz' } 14 { '.tar.zst' } else { panic("Another compression code shouldn't be possible. Faulty code: $pkg.compression") } } From ffd83eeb18fce475210166919af7ee5a701fce50 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 22:01:59 +0200 Subject: [PATCH 094/105] Vieter releases are now also aarch64 --- .woodpecker/.arch.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index 3fdf1025..ab3c6ea4 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -1,9 +1,14 @@ -platform: linux/amd64 +matrix: + PLATFORM: + - linux/amd64 + - linux/arm64 + +platform: ${PLATFORM} branches: [dev] pipeline: build: - image: 'archlinux:base-devel' + image: 'menci/archlinuxarm:base-devel' commands: # Update packages - pacman -Syu --noconfirm @@ -17,7 +22,7 @@ pipeline: - makepkg -s --noconfirm --needed publish: - image: 'archlinux:base-devel' + image: 'curlimages/curl' commands: # Publish the package - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done' From fb4710fc2a1f5d8ffc7af493f72ce79efeb90b06 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 7 Apr 2022 22:23:10 +0200 Subject: [PATCH 095/105] Remove deploy step Because of the bug that causes a panic, this CI run is also required to properly deploy the newest vieter version for aarch64 --- .woodpecker/.deploy.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .woodpecker/.deploy.yml diff --git a/.woodpecker/.deploy.yml b/.woodpecker/.deploy.yml deleted file mode 100644 index 8cf39071..00000000 --- a/.woodpecker/.deploy.yml +++ /dev/null @@ -1,17 +0,0 @@ -# Deploys the newest development image to my server -branches: [dev] -platform: linux/amd64 -depends_on: - - docker - -skip_clone: true - -pipeline: - webhook: - image: chewingbever/vlang:latest - secrets: - - webhook_1 - - webhook_2 - commands: - - curl -XPOST -s "$WEBHOOK_1" - - curl -XPOST -s "$WEBHOOK_2" From 8b2900d7f33d12c46ae8f3e9d792c7f1d23c1e1c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 8 Apr 2022 13:22:29 +0200 Subject: [PATCH 096/105] Added option to specify base build image to use --- src/build/build.v | 18 +++++++++++------- src/build/cli.v | 1 + 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 4270e9d7..cf70894f 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -10,9 +10,7 @@ const container_build_dir = '/build' const build_image_repo = 'vieter-build' -const base_image = 'archlinux:latest' - -fn create_build_image() ?string { +fn create_build_image(base_image string) ?string { commands := [ // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' @@ -29,14 +27,20 @@ fn create_build_image() ?string { cmds_str := base64.encode_str(commands.join('\n')) c := docker.NewContainer{ - image: build.base_image + image: base_image env: ['BUILD_SCRIPT=$cmds_str'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] } - // First, we pull the latest archlinux image - docker.pull_image('archlinux', 'latest') ? + // This check is needed so the user can pass "archlinux" without passing a + // tag & make it still work + image_parts := base_image.split_nth(':', 2) + image_name := image_parts[0] + image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' } + + // We pull the provided image + docker.pull_image(image_name, image_tag) ? id := docker.create_container(c) ? docker.start_container(id) ? @@ -78,7 +82,7 @@ fn build(conf Config) ? { } // First, we create a base image which has updated repos n stuff - image_id := create_build_image() ? + image_id := create_build_image(conf.base_image) ? for repo in filtered_repos { // TODO what to do with PKGBUILDs that build multiple packages? diff --git a/src/build/cli.v b/src/build/cli.v index c95d1caa..7cdcf834 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -7,6 +7,7 @@ pub struct Config { pub: api_key string address string + base_image string = 'archlinux:base-devel' } // cmd returns the cli submodule that handles the build process From ebe01c2d449cae1198a60e0e50dc120d1274e86b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 9 Apr 2022 12:23:54 +0200 Subject: [PATCH 097/105] Added route to request desc file; updated builder for new route (fixes #118) --- src/build/build.v | 2 +- src/server/routes.v | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index cf70894f..942ce8a8 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -93,7 +93,7 @@ fn build(conf Config) ? { 'source PKGBUILD', // The build container checks whether the package is already // present on the server - 'curl --head --fail $conf.address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel-${build_arch}.pkg.tar.zst && exit 0', + 'curl --head --fail $conf.address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/$repo.repo/publish; done', ] diff --git a/src/server/routes.v b/src/server/routes.v index a9264256..f259af41 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -22,7 +22,9 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re db_exts := ['.db', '.files', '.db.tar.gz', '.files.tar.gz'] - if db_exts.any(filename.ends_with(it)) { + // There's no point in having the ability to serve db archives with wrong + // filenames + if db_exts.any(filename == '$repo$it') { full_path = os.join_path(app.repo.repos_dir, repo, arch, filename) // repo-add does this using symlinks, but we just change the requested @@ -30,8 +32,14 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re if !full_path.ends_with('.tar.gz') { full_path += '.tar.gz' } - } else { + } else if filename.contains('.pkg') { full_path = os.join_path_single(app.repo.pkg_dir, filename) + + // Default behavior is to return the desc file for the package, if present. + // This can then also be used by the build system to properly check whether + // a package is present in an arch-repo. + } else { + full_path = os.join_path(app.repo.repos_dir, repo, arch, filename, 'desc') } // Scuffed way to respond to HEAD requests From cec5ecce7f936e118fb2bca41006fa8daa390f59 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 9 Apr 2022 17:41:41 +0200 Subject: [PATCH 098/105] Each repo now has its own subdir in pkg_dir --- src/build/cli.v | 4 ++-- src/repo/repo.v | 6 ++++-- src/server/routes.v | 8 ++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/build/cli.v b/src/build/cli.v index 7cdcf834..01313960 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -5,8 +5,8 @@ import env pub struct Config { pub: - api_key string - address string + api_key string + address string base_image string = 'archlinux:base-devel' } diff --git a/src/repo/repo.v b/src/repo/repo.v index 228b17e7..9c2b0ec0 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -12,7 +12,7 @@ mut: pub: // Where to store repositories' files repos_dir string [required] - // Where packages are stored; each architecture gets its own subdirectory + // Where packages are stored; each repository gets its own subdirectory pkg_dir string [required] // The default architecture to use for a repository. In reality, this value // is only required when a package with architecture "any" is added as the @@ -56,10 +56,12 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re // If the add was successful, we move the file to the packages directory if added { - dest_path := os.real_path(os.join_path_single(r.pkg_dir, pkg.filename())) + repo_pkg_path := os.real_path(os.join_path_single(r.pkg_dir, repo)) + dest_path := os.join_path_single(repo_pkg_path, pkg.filename()) // Only move the file if it's not already in the package directory if dest_path != os.real_path(pkg_path) { + os.mkdir_all(repo_pkg_path) ? os.mv(pkg_path, dest_path) ? } } diff --git a/src/server/routes.v b/src/server/routes.v index f259af41..c449d7dc 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -33,11 +33,11 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re full_path += '.tar.gz' } } else if filename.contains('.pkg') { - full_path = os.join_path_single(app.repo.pkg_dir, filename) + full_path = os.join_path(app.repo.pkg_dir, repo, filename) - // Default behavior is to return the desc file for the package, if present. - // This can then also be used by the build system to properly check whether - // a package is present in an arch-repo. + // Default behavior is to return the desc file for the package, if present. + // This can then also be used by the build system to properly check whether + // a package is present in an arch-repo. } else { full_path = os.join_path(app.repo.repos_dir, repo, arch, filename, 'desc') } From a65207f961c022898b631681a466088726518d65 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 9 Apr 2022 21:08:54 +0200 Subject: [PATCH 099/105] Solved the "removing old packages" problem (I think) --- src/repo/repo.v | 87 +++++++++++++++++++++++++++------------------ src/server/routes.v | 12 +++---- test.py | 4 +-- 3 files changed, 60 insertions(+), 43 deletions(-) diff --git a/src/repo/repo.v b/src/repo/repo.v index 9c2b0ec0..42ef3c95 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -14,9 +14,9 @@ pub: repos_dir string [required] // Where packages are stored; each repository gets its own subdirectory pkg_dir string [required] - // The default architecture to use for a repository. In reality, this value - // is only required when a package with architecture "any" is added as the - // first package of a repository. + // The default architecture to use for a repository. Whenever a package of + // arch "any" is added to a repo, it will also be added to this + // architecture. default_arch string [required] } @@ -45,8 +45,8 @@ pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupMana // add_pkg_from_path adds a package to a given repo, given the file path to the // pkg archive. It's a wrapper around add_pkg_in_repo that parses the archive -// file, passes the result to add_pkg_in_repo, and moves the archive to -// r.pkg_dir if it was successfully added. +// file, passes the result to add_pkg_in_repo, and hard links the archive to +// the right subdirectories in r.pkg_dir if it was successfully added. pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult { pkg := package.read_pkg_archive(pkg_path) or { return error('Failed to read package file: $err.msg') @@ -55,19 +55,22 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re added := r.add_pkg_in_repo(repo, pkg) ? // If the add was successful, we move the file to the packages directory - if added { - repo_pkg_path := os.real_path(os.join_path_single(r.pkg_dir, repo)) + for arch in added { + repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch)) dest_path := os.join_path_single(repo_pkg_path, pkg.filename()) - // Only move the file if it's not already in the package directory - if dest_path != os.real_path(pkg_path) { - os.mkdir_all(repo_pkg_path) ? - os.mv(pkg_path, dest_path) ? - } + os.mkdir_all(repo_pkg_path) ? + + // We create hard links so that "any" arch packages aren't stored + // multiple times + os.link(pkg_path, dest_path) ? } + // After linking, we can remove the original file + os.rm(pkg_path) ? + return RepoAddResult{ - added: added + added: added.len > 0 pkg: &pkg } } @@ -75,26 +78,32 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re // add_pkg_in_repo adds a package to a given repo. This function is responsible // for inspecting the package architecture. If said architecture is 'any', the // package is added to each arch-repository within the given repo. A package of -// architecture 'any' will always be added to the arch-repo defined by -// r.default_arch. If this arch-repo doesn't exist yet, it will be created. If -// the architecture isn't 'any', the package is only added to the specific +// architecture 'any' is always added to the arch-repo defined by +// r.default_arch. If this arch-repo doesn't exist yet, it is created. If the +// architecture isn't 'any', the package is only added to the specific // architecture. -fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { - // A package without arch 'any' can be handled without any further checks +fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]string { + // A package not of arch 'any' can be handled easily by adding it to the + // respective repo if pkg.info.arch != 'any' { - return r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) + if r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) ? { + return [pkg.info.arch] + } else { + return [] + } } - repo_dir := os.join_path_single(r.repos_dir, repo) - mut arch_repos := []string{} + // If it is an "any" package, the package gets added to every currently + // present arch-repo. It will always get added to the r.default_arch repo, + // even if no or multiple others are present. + repo_dir := os.join_path_single(r.repos_dir, repo) + // If this is the first package that's added to the repo, the directory // won't exist yet if os.exists(repo_dir) { - // We get a listing of all currently present arch-repos in the given repo - arch_repos = os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, - it))) + arch_repos = os.ls(repo_dir) ? } // The default_arch should always be updated when a package with arch 'any' @@ -103,12 +112,14 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { arch_repos << r.default_arch } - mut added := false + mut added := []string{} // We add the package to each repository. If any of the repositories // return true, the result of the function is also true. for arch in arch_repos { - added = added || r.add_pkg_in_arch_repo(repo, arch, pkg) ? + if r.add_pkg_in_arch_repo(repo, arch, pkg) ? { + added << arch + } } return added @@ -122,13 +133,8 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool { fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool { pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version') - // We can't add the same package twice - if os.exists(pkg_dir) { - return false - } - - // We remove the older package version first, if present - r.remove_pkg_from_arch_repo(repo, arch, pkg, false) ? + // Remove the previous version of the package, if present + r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false) ? os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') } @@ -151,7 +157,7 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac // remove_pkg_from_arch_repo removes a package from an arch-repo's database. It // returns false if the package wasn't present in the database. It also // optionally re-syncs the repo archives. -fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg &package.Pkg, sync bool) ?bool { +fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool { repo_dir := os.join_path(r.repos_dir, repo, arch) // If the repository doesn't exist yet, the result is automatically false @@ -167,13 +173,24 @@ fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg // not the version. name := d.split('-')#[..-2].join('-') - if name == pkg.info.name { + if name == pkg_name { // We lock the mutex here to prevent other routines from creating a // new archive while we remove an entry lock r.mutex { os.rmdir_all(os.join_path_single(repo_dir, d)) ? } + // Also remove the package archive + repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch) + + archives := os.ls(repo_pkg_dir) ?.filter(it.split('-')#[..-3].join('-') == name) + + for archive_name in archives { + full_path := os.join_path_single(repo_pkg_dir, archive_name) + os.rm(full_path) ? + } + + // Sync the db archives if requested if sync { r.sync(repo, arch) ? } diff --git a/src/server/routes.v b/src/server/routes.v index c449d7dc..ccc0e0e9 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -33,12 +33,12 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re full_path += '.tar.gz' } } else if filename.contains('.pkg') { - full_path = os.join_path(app.repo.pkg_dir, repo, filename) - - // Default behavior is to return the desc file for the package, if present. - // This can then also be used by the build system to properly check whether - // a package is present in an arch-repo. - } else { + full_path = os.join_path(app.repo.pkg_dir, repo, arch, filename) + } + // Default behavior is to return the desc file for the package, if present. + // This can then also be used by the build system to properly check whether + // a package is present in an arch-repo. + else { full_path = os.join_path(app.repo.repos_dir, repo, arch, filename, 'desc') } diff --git a/test.py b/test.py index 9b0116ec..46dac52a 100644 --- a/test.py +++ b/test.py @@ -38,7 +38,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len): Generates a random .PKGINFO """ name = "-".join(random_words(words, name_min_len, name_max_len)) - ver = "0.1.0-1" # doesn't matter what it is anyways + ver = "0.1.0-3" # doesn't matter what it is anyways # TODO add random dependencies (all types) @@ -46,7 +46,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len): "pkgname": name, "pkgbase": name, "pkgver": ver, - "arch": "any" + "arch": "x86_64" } return "\n".join(f"{key} = {value}" for key, value in data.items()) From 62bee78955ea205f587c4b2cfe8c7a31572a16b6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 9 Apr 2022 21:17:22 +0200 Subject: [PATCH 100/105] Updated changelog --- CHANGELOG.md | 1 + src/repo/repo.v | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1daaec4..66ff9d3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Each package can now only have one version in the repository at once (required by Pacman) * Packages with unknown fields in .PKGINFO are now allowed +* Old packages are now properly removed ## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0) diff --git a/src/repo/repo.v b/src/repo/repo.v index 42ef3c95..f439f581 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -12,7 +12,8 @@ mut: pub: // Where to store repositories' files repos_dir string [required] - // Where packages are stored; each repository gets its own subdirectory + // Where packages are stored; each arch-repository gets its own + // subdirectory pkg_dir string [required] // The default architecture to use for a repository. Whenever a package of // arch "any" is added to a repo, it will also be added to this From 2d01c5374bd8a6858cd6dad96a33d27f2bbe0ad8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 9 Apr 2022 21:53:54 +0200 Subject: [PATCH 101/105] Updated README (Closes #68) [CI SKIP] --- README.md | 52 +++++++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index f269027d..f83c3dc5 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,17 @@ # Vieter -Vieter is a re-implementation of the Pieter project. The goal is to create a -simple PKGBUILD-based build system, combined with a self-hosted Arch -repository. This would allow me to periodically re-build AUR packages (or -PKGBUILDs I created myself), & make sure I never have to compile anything on my -own systems, making my updates a lot quicker. +## Documentation + +I host documentation for Vieter over at https://rustybever.be/docs/vieter. + +## Overview + +Vieter is a restart of the Pieter project. The goal is to create a simple, +lightweight self-hostable Arch repository server, paired with a system that +periodically builds & publishes select Arch packages. This would allow me to +build AUR packages (or PKGBUILDs I created myself) "in the cloud" & make sure I +never have to compile anything on my own systems, making my updates a lot +quicker. ## Why V? @@ -20,27 +27,26 @@ V standard library, and therefore the compiler. The source code for this fork can be found [here](https://git.rustybever.be/Chewing_Bever/vieter-v). You can obtain this modified version of the compiler by running `make v`, which will clone & build the compiler. Afterwards, all make commands that require the V -compiler will use this new binary. +compiler will use this new binary. I try to keep this fork as up to date with +upstream as possible. ## Features -The project will consist of a server-agent model, where one or more builder -nodes can register with the server. These agents communicate with the Docker -daemon to start builds, which are then uploaded to the server's repository. The -server also allows for non-agents to upload packages, as long as they have the -required secrets. This allows me to also develop non-git packages, such as my -terminal, & upload them to the servers using CI. +* Arch repository server + * Support for multiple repositories & multiple architectures + * Endpoints for publishing new packages + * API for managing repositories to build +* Build system + * Periodic rebuilding of packages + * Prevent unnecessary rebuilds -## Directory Structure +## Building -The data directory consists of three main directories: +In order to build Vieter, you'll need a couple of libraries: -* `downloads` - This is where packages are initially downloaded. Because vieter - moves files from this folder to the `pkgs` folder, these two folders should - best be on the same drive -* `pkgs` - This is where approved package files are stored. -* `repos` - Each repository gets a subfolder here. The subfolder contains the - uncompressed contents of the db file. - * Each repo subdirectory contains the compressed db & files archive for the - repository, alongside a directory called `files` which contains the - uncompressed contents. +* gc +* libarchive +* openssl + +Before building Vieter, you'll have to build the compiler using `make v`. +Afterwards, run `make` to build the debug binary. From ff9ff9bf9971125e53d25a28aa3096d06b44a836 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 9 Apr 2022 22:09:38 +0200 Subject: [PATCH 102/105] Forgot trailing slash [CI SKIP] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f83c3dc5..cd78f744 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Documentation -I host documentation for Vieter over at https://rustybever.be/docs/vieter. +I host documentation for Vieter over at https://rustybever.be/docs/vieter/. ## Overview From aacadca7a8533133bb277997639209560ac44ef2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 9 Apr 2022 22:13:21 +0200 Subject: [PATCH 103/105] Bumped version in cli --- src/main.v | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.v b/src/main.v index c77e5519..30253890 100644 --- a/src/main.v +++ b/src/main.v @@ -10,7 +10,7 @@ fn main() { mut app := cli.Command{ name: 'vieter' description: 'Vieter is a lightweight implementation of an Arch repository server.' - version: '0.1.0' + version: '0.2.0' flags: [ cli.Flag{ flag: cli.FlagType.string From 7e142cb6c76fa58c7e5cafc94dcd2b436503a4ef Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 9 Apr 2022 22:41:38 +0200 Subject: [PATCH 104/105] Updated CHANGELOG [CI SKIP] --- CHANGELOG.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66ff9d3f..8738952a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter) -## Changed +## [0.2.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.2.0) + +### Changed * Better config system * Support for both a config file & environment variables @@ -17,7 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * All routes now return proper JSON where applicable & the correct status codes -## Added +### Added * Very basic build system * Build is triggered by separate cron container @@ -32,7 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Support for multiple repositories * Support for multiple architectures per repository -## Fixed +### Fixed * Each package can now only have one version in the repository at once (required by Pacman) From 3b15066329315906270e3dac88955b770cfb760f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 11 Apr 2022 09:36:40 +0200 Subject: [PATCH 105/105] Removed unnecessary log output --- src/server/routes.v | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/routes.v b/src/server/routes.v index ccc0e0e9..138f2532 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -97,7 +97,7 @@ fn (mut app App) put_package(repo string) web.Result { if !res.added { os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } - app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo ($res.pkg.info.arch)'.") + app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo'.") return app.json(http.Status.bad_request, new_response('File already exists.')) }