From 5b919ceeaeec6a1b1520f08c9109f1e4448f0ac0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 16:52:31 +0200 Subject: [PATCH 01/11] Switched to cli module; merged cli & vieter into single binary --- src/{ => build}/build.v | 10 ++++----- src/build/cli.v | 17 ++++++++++++++ src/env.v | 25 ++++++++++++++------- src/git/cli.v | 37 ++++++++++++++++++++++++++++++ src/git/git.v | 40 +++++++++++++++++++++++++++++++++ src/main.v | 32 ++++++++++++++++++++------ src/server/cli.v | 16 +++++++++++++ src/server/git.v | 50 +++++++---------------------------------- src/server/server.v | 5 ++--- 9 files changed, 167 insertions(+), 65 deletions(-) rename src/{ => build}/build.v (96%) create mode 100644 src/build/cli.v create mode 100644 src/git/cli.v create mode 100644 src/git/git.v create mode 100644 src/server/cli.v diff --git a/src/build.v b/src/build/build.v similarity index 96% rename from src/build.v rename to src/build/build.v index 51de64c..f2ff9af 100644 --- a/src/build.v +++ b/src/build/build.v @@ -1,4 +1,4 @@ -module main +module build import docker import encoding.base64 @@ -7,6 +7,8 @@ import json import server import env import net.http +import cli +import git const container_build_dir = '/build' @@ -62,15 +64,13 @@ fn create_build_image() ?string { return image.id } -fn build() ? { - conf := env.load() ? - +fn build(conf env.BuildConfig) ? { // We get the repos list from the Vieter instance mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? req.add_custom_header('X-Api-Key', conf.api_key) ? res := req.do() ? - repos := json.decode([]server.GitRepo, res.text) ? + repos := json.decode([]git.GitRepo, res.text) ? // No point in doing work if there's no repos present if repos.len == 0 { diff --git a/src/build/cli.v b/src/build/cli.v new file mode 100644 index 0000000..e1f6cb5 --- /dev/null +++ b/src/build/cli.v @@ -0,0 +1,17 @@ +module build + +import cli +import env + +pub fn cmd() cli.Command { + return cli.Command{ + name: 'build' + description: 'Run the build process.' + execute: fn (cmd cli.Command) ? { + conf := env.load() ? + + build(conf) ? + } + } +} + diff --git a/src/env.v b/src/env.v index bd544cf..11709ce 100644 --- a/src/env.v +++ b/src/env.v @@ -56,15 +56,9 @@ fn get_env_var(field_name string) ?string { } } -// load attempts to create the given type from environment variables. For -// each field, the corresponding env var is its name in uppercase prepended -// with the hardcoded prefix. If this one isn't present, it looks for the env -// var with the file_suffix suffix. -pub fn load() ?T { - res := T{} - +fn load_in_place(mut o T) ? { $for field in T.fields { - res.$(field.name) = get_env_var(field.name) or { + o.$(field.name) = get_env_var(field.name) or { // We use the default instead, if it's present mut default := '' @@ -82,5 +76,20 @@ pub fn load() ?T { default } } +} + +// load attempts to create the given type from environment variables. For +// each field, the corresponding env var is its name in uppercase prepended +// with the hardcoded prefix. If this one isn't present, it looks for the env +// var with the file_suffix suffix. +pub fn load() ?T { + mut res := T{} + + load_in_place(mut res) ? + return res } + +pub fn load_with_file(path string) ?T { + +} diff --git a/src/git/cli.v b/src/git/cli.v new file mode 100644 index 0000000..fccb9b0 --- /dev/null +++ b/src/git/cli.v @@ -0,0 +1,37 @@ +module git + +import cli +import env +import net.http + +struct Config { + address string [required] + api_key string [required] +} + +pub fn cmd() cli.Command { + return cli.Command{ + name: 'repos' + description: 'Interact with the repos API.' + commands: [ + cli.Command{ + name: 'list' + description: 'List the current repos.' + execute: fn (cmd cli.Command) ? { + conf := env.load() ? + + list(conf) ? + } + } + ] + } +} + +fn list(conf Config) ? { + mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? + req.add_custom_header('X-API-Key', conf.api_key) ? + + res := req.do() ? + + println(res.text) +} diff --git a/src/git/git.v b/src/git/git.v new file mode 100644 index 0000000..6daf25b --- /dev/null +++ b/src/git/git.v @@ -0,0 +1,40 @@ +module git + +import os +import json + +pub struct GitRepo { +pub: + url string [required] + branch string [required] +} + +pub fn read_repos(path string) ?[]GitRepo { + if !os.exists(path) { + mut f := os.create(path) ? + + defer { + f.close() + } + + f.write_string('[]') ? + + return [] + } + + content := os.read_file(path) ? + res := json.decode([]GitRepo, content) ? + return res +} + +pub fn write_repos(path string, repos []GitRepo) ? { + mut f := os.create(path) ? + + defer { + f.close() + } + + value := json.encode(repos) + f.write_string(value) ? +} + diff --git a/src/main.v b/src/main.v index 156f0a3..31b6bcb 100644 --- a/src/main.v +++ b/src/main.v @@ -3,15 +3,33 @@ module main import os import server import util +import cli +import env +import build +import git fn main() { - if os.args.len == 1 { - util.exit_with_message(1, 'No action provided.') + mut app := cli.Command{ + name: 'vieter' + description: 'Arch repository server' + version: '0.1.0' + flags: [ + cli.Flag{ + flag: cli.FlagType.string + name: 'config-file' + abbrev: 'f' + description: 'Location of Vieter config file; defaults to ~/.vieterrc.' + global: true + default_value: [os.expand_tilde_to_home('~/.vieterrc')] + } + ] + commands: [ + server.cmd(), + build.cmd(), + git.cmd() + ] } - match os.args[1] { - 'server' { server.server() ? } - 'build' { build() ? } - else { util.exit_with_message(1, 'Unknown action: ${os.args[1]}') } - } + app.setup() + app.parse(os.args) } diff --git a/src/server/cli.v b/src/server/cli.v new file mode 100644 index 0000000..944f05b --- /dev/null +++ b/src/server/cli.v @@ -0,0 +1,16 @@ +module server + +import cli +import env + +pub fn cmd() cli.Command { + return cli.Command{ + name: 'server' + description: 'Start the Vieter server' + execute: fn (cmd cli.Command) ? { + conf := env.load() ? + + server.server(conf) ? + } + } +} diff --git a/src/server/git.v b/src/server/git.v index 0147d87..2e014db 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -3,44 +3,10 @@ module server import web import os import json +import git const repos_file = 'repos.json' -pub struct GitRepo { -pub: - url string [required] - branch string [required] -} - -fn read_repos(path string) ?[]GitRepo { - if !os.exists(path) { - mut f := os.create(path) ? - - defer { - f.close() - } - - f.write_string('[]') ? - - return [] - } - - content := os.read_file(path) ? - res := json.decode([]GitRepo, content) ? - return res -} - -fn write_repos(path string, repos []GitRepo) ? { - mut f := os.create(path) ? - - defer { - f.close() - } - - value := json.encode(repos) - f.write_string(value) ? -} - ['/api/repos'; get] fn (mut app App) get_repos() web.Result { if !app.is_authorized() { @@ -48,7 +14,7 @@ fn (mut app App) get_repos() web.Result { } repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { + git.read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') return app.server_error(500) @@ -68,13 +34,13 @@ fn (mut app App) post_repo() web.Result { return app.server_error(400) } - new_repo := GitRepo{ + new_repo := git.GitRepo{ url: app.query['url'] branch: app.query['branch'] } mut repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { + git.read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') return app.server_error(500) @@ -91,7 +57,7 @@ fn (mut app App) post_repo() web.Result { repos << new_repo lock app.git_mutex { - write_repos(app.conf.repos_file, repos) or { return app.server_error(500) } + git.write_repos(app.conf.repos_file, repos) or { return app.server_error(500) } } return app.ok('Repo added successfully.') @@ -107,13 +73,13 @@ fn (mut app App) delete_repo() web.Result { return app.server_error(400) } - repo_to_remove := GitRepo{ + repo_to_remove := git.GitRepo{ url: app.query['url'] branch: app.query['branch'] } mut repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { + git.read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') return app.server_error(500) @@ -122,7 +88,7 @@ fn (mut app App) delete_repo() web.Result { filtered := repos.filter(it != repo_to_remove) lock app.git_mutex { - write_repos(app.conf.repos_file, filtered) or { return app.server_error(500) } + git.write_repos(app.conf.repos_file, filtered) or { return app.server_error(500) } } return app.ok('Repo removed successfully.') diff --git a/src/server/server.v b/src/server/server.v index 4b31b99..7f129e9 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -6,6 +6,7 @@ import log import repo import env import util +import cli const port = 8000 @@ -20,9 +21,7 @@ pub mut: } // server starts the web server & starts listening for requests -pub fn server() ? { - conf := env.load() ? - +pub fn server(conf env.ServerConfig) ? { // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') From b90e6cf6b4f1d87e90de36d964195882a51af5b3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 16:57:27 +0200 Subject: [PATCH 02/11] Updatd config files; ran formatter --- .woodpecker/.build.yml | 31 ------------------------------- Makefile | 12 ------------ PKGBUILD | 13 ++----------- src/build/cli.v | 1 - src/env.v | 1 - src/git/cli.v | 16 ++++++++-------- src/git/git.v | 1 - src/main.v | 4 ++-- src/server/cli.v | 2 +- 9 files changed, 13 insertions(+), 68 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 4cddc6a..a4772b3 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -36,22 +36,6 @@ pipeline: when: event: push - cli: - image: 'chewingbever/vlang:latest' - environment: - - LDFLAGS=-static - commands: - - make cli-prod - # Make sure the binary is actually statically built - - readelf -d vieterctl - - du -h vieterctl - - '[ "$(readelf -d vieterctl | grep NEEDED | wc -l)" = 0 ]' - # This removes so much, it's amazing - - strip -s vieterctl - - du -h vieterctl - when: - event: push - upload: image: 'chewingbever/vlang:latest' secrets: [ s3_username, s3_password ] @@ -74,20 +58,5 @@ pipeline: -H "Content-Type: $CONTENT_TYPE" -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" https://$URL$OBJ_PATH - - # Also update the CLI tool - - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieterctl-$(echo '${PLATFORM}' | sed 's:/:-:g')" - - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH" - - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64` - - > - curl - --silent - -XPUT - -T vieterctl - -H "Host: $URL" - -H "Date: $DATE" - -H "Content-Type: $CONTENT_TYPE" - -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" - https://$URL$OBJ_PATH when: event: push diff --git a/Makefile b/Makefile index 7b0fb1b..2238585 100644 --- a/Makefile +++ b/Makefile @@ -42,18 +42,6 @@ pvieter: $(SOURCES) c: $(V) -o vieter.c $(SRC_DIR) -# Build the CLI tool -.PHONY: cli -cli: dvieterctl -dvieterctl: cli.v - $(V_PATH) -showcc -g -o dvieterctl cli.v - -.PHONY: cli-prod -cli-prod: vieterctl -vieterctl: cli.v -cli-prod: - $(V_PATH) -showcc -o vieterctl -prod cli.v - # =====EXECUTION===== # Run the server in the default 'data' directory .PHONY: run diff --git a/PKGBUILD b/PKGBUILD index 767c07c..cb0c24a 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -1,7 +1,7 @@ # Maintainer: Jef Roosens pkgbase='vieter' -pkgname=('vieter' 'vieterctl') +pkgname='vieter' pkgver=0.1.0.rc1.r45.g6d3ff8a pkgrel=1 depends=('glibc' 'openssl' 'libarchive' 'gc') @@ -23,21 +23,12 @@ build() { # Build the compiler CFLAGS= make v - # Build the server & the CLI tool make prod - make cli-prod } -package_vieter() { +package() { pkgdesc="Vieter is a lightweight implementation of an Arch repository server." install -dm755 "$pkgdir/usr/bin" install -Dm755 "$pkgbase/pvieter" "$pkgdir/usr/bin/vieter" } - -package_vieterctl() { - pkgdesc="Allows you to configure a Vieter server's list of Git repositories." - install -dm755 "$pkgdir/usr/bin" - - install -Dm755 "$pkgbase/vieterctl" "$pkgdir/usr/bin/vieterctl" -} diff --git a/src/build/cli.v b/src/build/cli.v index e1f6cb5..bb3c016 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -14,4 +14,3 @@ pub fn cmd() cli.Command { } } } - diff --git a/src/env.v b/src/env.v index 11709ce..3babd55 100644 --- a/src/env.v +++ b/src/env.v @@ -91,5 +91,4 @@ pub fn load() ?T { } pub fn load_with_file(path string) ?T { - } diff --git a/src/git/cli.v b/src/git/cli.v index fccb9b0..a91f5e2 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -14,15 +14,15 @@ pub fn cmd() cli.Command { name: 'repos' description: 'Interact with the repos API.' commands: [ - cli.Command{ - name: 'list' - description: 'List the current repos.' - execute: fn (cmd cli.Command) ? { - conf := env.load() ? + cli.Command{ + name: 'list' + description: 'List the current repos.' + execute: fn (cmd cli.Command) ? { + conf := env.load() ? - list(conf) ? - } - } + list(conf) ? + } + }, ] } } diff --git a/src/git/git.v b/src/git/git.v index 6daf25b..597aa3c 100644 --- a/src/git/git.v +++ b/src/git/git.v @@ -37,4 +37,3 @@ pub fn write_repos(path string, repos []GitRepo) ? { value := json.encode(repos) f.write_string(value) ? } - diff --git a/src/main.v b/src/main.v index 31b6bcb..18a822b 100644 --- a/src/main.v +++ b/src/main.v @@ -21,12 +21,12 @@ fn main() { description: 'Location of Vieter config file; defaults to ~/.vieterrc.' global: true default_value: [os.expand_tilde_to_home('~/.vieterrc')] - } + }, ] commands: [ server.cmd(), build.cmd(), - git.cmd() + git.cmd(), ] } diff --git a/src/server/cli.v b/src/server/cli.v index 944f05b..2b51c29 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -10,7 +10,7 @@ pub fn cmd() cli.Command { execute: fn (cmd cli.Command) ? { conf := env.load() ? - server.server(conf) ? + server(conf) ? } } } From 9dd02426ff429e98b8c39f48c6b161fdf786854e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 16:58:15 +0200 Subject: [PATCH 03/11] Removd armv7 from ci --- .woodpecker/.build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index a4772b3..16e5a69 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -2,7 +2,6 @@ matrix: PLATFORM: - linux/amd64 - linux/arm64 - - linux/arm/v7 # These checks already get performed on the feature branches platform: ${PLATFORM} From 21ef262edeeb159f855ae4cd2083e6bce11090ae Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 17:16:27 +0200 Subject: [PATCH 04/11] Migrated rest of cli commands --- cli.v | 84 --------------------------------------------------- src/git/cli.v | 40 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 84 deletions(-) delete mode 100644 cli.v diff --git a/cli.v b/cli.v deleted file mode 100644 index 256b856..0000000 --- a/cli.v +++ /dev/null @@ -1,84 +0,0 @@ -import os -import toml -import net.http - -struct Config { - address string [required] - api_key string [required] -} - -fn list(conf Config) ? { - mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? - req.add_custom_header('X-API-Key', conf.api_key) ? - - res := req.do() ? - - println(res.text) -} - -fn add(conf Config, args []string) ? { - if args.len < 2 { - eprintln('Not enough arguments.') - exit(1) - } - - if args.len > 2 { - eprintln('Too many arguments.') - exit(1) - } - - mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=${args[0]}&branch=${args[1]}', '') ? - req.add_custom_header('X-API-Key', conf.api_key) ? - - res := req.do() ? - - println(res.text) -} - -fn remove(conf Config, args []string) ? { - if args.len < 2 { - eprintln('Not enough arguments.') - exit(1) - } - - if args.len > 2 { - eprintln('Too many arguments.') - exit(1) - } - - mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=${args[0]}&branch=${args[1]}', '') ? - req.add_custom_header('X-API-Key', conf.api_key) ? - - res := req.do() ? - - println(res.text) -} - -fn main() { - conf_path := os.expand_tilde_to_home('~/.vieterrc') - - if !os.is_file(conf_path) { - exit(1) - } - - conf := toml.parse_file(conf_path) ?.reflect() - - args := os.args[1..] - - if args.len == 0 { - eprintln('No action provided.') - exit(1) - } - - action := args[0] - - match action { - 'list' { list(conf) ? } - 'add' { add(conf, args[1..]) ? } - 'remove' { remove(conf, args[1..]) ? } - else { - eprintln("Invalid action '$action'.") - exit(1) - } - } -} diff --git a/src/git/cli.v b/src/git/cli.v index a91f5e2..2b164e1 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -23,6 +23,28 @@ pub fn cmd() cli.Command { list(conf) ? } }, + cli.Command{ + name: 'add' + required_args: 2 + usage: 'url branch' + description: 'Add a new repository.' + execute: fn (cmd cli.Command) ? { + conf := env.load() ? + + add(conf, cmd.args[0], cmd.args[1]) ? + } + }, + cli.Command{ + name: 'remove' + required_args: 2 + usage: 'url branch' + description: 'Remove a repository.' + execute: fn (cmd cli.Command) ? { + conf := env.load() ? + + remove(conf, cmd.args[0], cmd.args[1]) ? + } + }, ] } } @@ -35,3 +57,21 @@ fn list(conf Config) ? { println(res.text) } + +fn add(conf Config, url string, branch string) ? { + mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=$url&branch=$branch', '') ? + req.add_custom_header('X-API-Key', conf.api_key) ? + + res := req.do() ? + + println(res.text) +} + +fn remove(conf Config, url string, branch string) ? { + mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=$url&branch=$branch', '') ? + req.add_custom_header('X-API-Key', conf.api_key) ? + + res := req.do() ? + + println(res.text) +} From e9d7858380162b3cf36683e91f73e1671f80d368 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 17:51:06 +0200 Subject: [PATCH 05/11] env module now properly supports config file --- src/build/cli.v | 3 ++- src/env.v | 63 ++++++++++++++++++++++++++++-------------------- src/git/cli.v | 15 ++++++++---- src/server/cli.v | 3 ++- 4 files changed, 51 insertions(+), 33 deletions(-) diff --git a/src/build/cli.v b/src/build/cli.v index bb3c016..a32a6e7 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -8,7 +8,8 @@ pub fn cmd() cli.Command { name: 'build' description: 'Run the build process.' execute: fn (cmd cli.Command) ? { - conf := env.load() ? + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? build(conf) ? } diff --git a/src/env.v b/src/env.v index 3babd55..62678e6 100644 --- a/src/env.v +++ b/src/env.v @@ -1,6 +1,7 @@ module env import os +import toml // The prefix that every environment variable should have const prefix = 'VIETER_' @@ -32,9 +33,9 @@ fn get_env_var(field_name string) ?string { env_var := os.getenv(env_var_name) env_file := os.getenv(env_file_name) - // If both aren't set, we report them missing + // If both are missing, we return an empty string if env_var == '' && env_file == '' { - return error('Either $env_var_name or $env_file_name is required.') + return '' } // If they're both set, we report a conflict @@ -56,37 +57,47 @@ fn get_env_var(field_name string) ?string { } } -fn load_in_place(mut o T) ? { - $for field in T.fields { - o.$(field.name) = get_env_var(field.name) or { - // We use the default instead, if it's present - mut default := '' - - for attr in field.attrs { - if attr.starts_with('default: ') { - default = attr[9..] - break - } - } - - if default == '' { - return err - } - - default - } - } -} - // load attempts to create the given type from environment variables. For // each field, the corresponding env var is its name in uppercase prepended // with the hardcoded prefix. If this one isn't present, it looks for the env // var with the file_suffix suffix. -pub fn load() ?T { +pub fn load(path string) ?T { mut res := T{} - load_in_place(mut res) ? + if os.exists(path) { + res = toml.parse_file(path) ?.reflect() + } + $for field in T.fields { + $if field.typ is string { + env_value := get_env_var(field.name) ? + + // The value of the env var will always be chosen over the config + // file + if env_value != '' { + res.$(field.name) = env_value + } + // If there's no value from the toml file either, we try to find a + // default value + else if res.$(field.name) == '' { + // We use the default instead, if it's present + mut default := '' + + for attr in field.attrs { + if attr.starts_with('default: ') { + default = attr[9..] + break + } + } + + if default == '' { + return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.") + } + + res.$(field.name) = default + } + } + } return res } diff --git a/src/git/cli.v b/src/git/cli.v index 2b164e1..6030612 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -18,7 +18,8 @@ pub fn cmd() cli.Command { name: 'list' description: 'List the current repos.' execute: fn (cmd cli.Command) ? { - conf := env.load() ? + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? list(conf) ? } @@ -29,7 +30,8 @@ pub fn cmd() cli.Command { usage: 'url branch' description: 'Add a new repository.' execute: fn (cmd cli.Command) ? { - conf := env.load() ? + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? add(conf, cmd.args[0], cmd.args[1]) ? } @@ -40,7 +42,8 @@ pub fn cmd() cli.Command { usage: 'url branch' description: 'Remove a repository.' execute: fn (cmd cli.Command) ? { - conf := env.load() ? + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? remove(conf, cmd.args[0], cmd.args[1]) ? } @@ -59,7 +62,8 @@ fn list(conf Config) ? { } fn add(conf Config, url string, branch string) ? { - mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=$url&branch=$branch', '') ? + mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=$url&branch=$branch', + '') ? req.add_custom_header('X-API-Key', conf.api_key) ? res := req.do() ? @@ -68,7 +72,8 @@ fn add(conf Config, url string, branch string) ? { } fn remove(conf Config, url string, branch string) ? { - mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=$url&branch=$branch', '') ? + mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=$url&branch=$branch', + '') ? req.add_custom_header('X-API-Key', conf.api_key) ? res := req.do() ? diff --git a/src/server/cli.v b/src/server/cli.v index 2b51c29..367878a 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -8,7 +8,8 @@ pub fn cmd() cli.Command { name: 'server' description: 'Start the Vieter server' execute: fn (cmd cli.Command) ? { - conf := env.load() ? + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? server(conf) ? } From 75dfc5267b59886a09d52ed5ca4820ed3e93f428 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 17:57:05 +0200 Subject: [PATCH 06/11] Made vet happy --- src/build/build.v | 4 +--- src/build/cli.v | 1 + src/env.v | 3 --- src/git/cli.v | 1 + src/git/git.v | 2 ++ src/main.v | 2 -- src/server/cli.v | 1 + src/server/git.v | 2 -- src/server/server.v | 1 - 9 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index f2ff9af..6f2b0a8 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -3,12 +3,10 @@ module build import docker import encoding.base64 import time -import json -import server import env import net.http -import cli import git +import json const container_build_dir = '/build' diff --git a/src/build/cli.v b/src/build/cli.v index a32a6e7..463d2ca 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -3,6 +3,7 @@ module build import cli import env +// cmd returns the cli submodule that handles the build process pub fn cmd() cli.Command { return cli.Command{ name: 'build' diff --git a/src/env.v b/src/env.v index 62678e6..58c6803 100644 --- a/src/env.v +++ b/src/env.v @@ -100,6 +100,3 @@ pub fn load(path string) ?T { } return res } - -pub fn load_with_file(path string) ?T { -} diff --git a/src/git/cli.v b/src/git/cli.v index 6030612..17fa984 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -9,6 +9,7 @@ struct Config { api_key string [required] } +// cmd returns the cli submodule that handles the repos API interaction pub fn cmd() cli.Command { return cli.Command{ name: 'repos' diff --git a/src/git/git.v b/src/git/git.v index 597aa3c..913bc39 100644 --- a/src/git/git.v +++ b/src/git/git.v @@ -9,6 +9,7 @@ pub: branch string [required] } +// read_repos reads the given JSON file & parses it as a list of Git repos pub fn read_repos(path string) ?[]GitRepo { if !os.exists(path) { mut f := os.create(path) ? @@ -27,6 +28,7 @@ pub fn read_repos(path string) ?[]GitRepo { return res } +// write_repos writes a list of repositories back to a given file pub fn write_repos(path string, repos []GitRepo) ? { mut f := os.create(path) ? diff --git a/src/main.v b/src/main.v index 18a822b..11bc77b 100644 --- a/src/main.v +++ b/src/main.v @@ -2,9 +2,7 @@ module main import os import server -import util import cli -import env import build import git diff --git a/src/server/cli.v b/src/server/cli.v index 367878a..6e67686 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -3,6 +3,7 @@ module server import cli import env +// cmd returns the cli submodule that handles starting the server pub fn cmd() cli.Command { return cli.Command{ name: 'server' diff --git a/src/server/git.v b/src/server/git.v index 2e014db..3ec8eeb 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -1,8 +1,6 @@ module server import web -import os -import json import git const repos_file = 'repos.json' diff --git a/src/server/server.v b/src/server/server.v index 7f129e9..0738bc4 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -6,7 +6,6 @@ import log import repo import env import util -import cli const port = 8000 From b70be0574e24c5a0adbb5a4826d5ceb8f2ca9bc3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 18:02:57 +0200 Subject: [PATCH 07/11] Small change to documentation --- src/env.v | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/env.v b/src/env.v index 58c6803..6abd674 100644 --- a/src/env.v +++ b/src/env.v @@ -57,10 +57,10 @@ fn get_env_var(field_name string) ?string { } } -// load attempts to create the given type from environment variables. For -// each field, the corresponding env var is its name in uppercase prepended -// with the hardcoded prefix. If this one isn't present, it looks for the env -// var with the file_suffix suffix. +// load attempts to create an object of type T from the given path to a toml +// file & environment variables. For each field, it will select either a value +// given from an environment variable, a value defined in the config file or a +// configured default if present, in that order. pub fn load(path string) ?T { mut res := T{} From 2aa2aa143ca92a1f93e29a9c7bed369d62516a48 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 18:17:33 +0200 Subject: [PATCH 08/11] Switched to proper default values system --- Makefile | 23 +++-------------------- src/env.v | 33 +++++++++++++++------------------ vieter.toml | 8 ++++++++ 3 files changed, 26 insertions(+), 38 deletions(-) create mode 100644 vieter.toml diff --git a/Makefile b/Makefile index 2238585..76ab7b5 100644 --- a/Makefile +++ b/Makefile @@ -23,13 +23,7 @@ dvieter: $(SOURCES) # Run the debug build inside gdb .PHONY: gdb gdb: dvieter - VIETER_API_KEY=test \ - VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ - VIETER_PKG_DIR=data/pkgs \ - VIETER_LOG_LEVEL=DEBUG \ - VIETER_REPOS_FILE=data/repos.json \ - gdb --args ./dvieter + gdb --args './dvieter -f vieter.toml server' # Optimised production build .PHONY: prod @@ -46,22 +40,11 @@ c: # Run the server in the default 'data' directory .PHONY: run run: vieter - VIETER_API_KEY=test \ - VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ - VIETER_PKG_DIR=data/pkgs \ - VIETER_LOG_LEVEL=DEBUG \ - VIETER_REPOS_FILE=data/repos.json \ - ./vieter server + ./vieter -f vieter.toml server .PHONY: run-prod run-prod: prod - VIETER_API_KEY=test \ - VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ - VIETER_PKG_DIR=data/pkgs \ - VIETER_LOG_LEVEL=DEBUG \ - ./pvieter server + ./pvieter -f vieter.toml server # =====OTHER===== .PHONY: lint diff --git a/src/env.v b/src/env.v index 6abd674..b733791 100644 --- a/src/env.v +++ b/src/env.v @@ -12,8 +12,8 @@ const file_suffix = '_FILE' pub struct ServerConfig { pub: - log_level string [default: WARN] - log_file string [default: 'vieter.log'] + log_level string = "WARN" + log_file string = "vieter.log" pkg_dir string download_dir string api_key string @@ -65,7 +65,18 @@ pub fn load(path string) ?T { mut res := T{} if os.exists(path) { - res = toml.parse_file(path) ?.reflect() + // We don't use reflect here because reflect also sets any fields not + // in the toml back to "empty", which we don't want + doc := toml.parse_file(path) ? + + $for field in T.fields { + s := doc.value(field.name) + + // We currently only support strings + if s.type_name() == "string" { + res.$(field.name) = s.string() + } + } } $for field in T.fields { @@ -80,21 +91,7 @@ pub fn load(path string) ?T { // If there's no value from the toml file either, we try to find a // default value else if res.$(field.name) == '' { - // We use the default instead, if it's present - mut default := '' - - for attr in field.attrs { - if attr.starts_with('default: ') { - default = attr[9..] - break - } - } - - if default == '' { - return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.") - } - - res.$(field.name) = default + return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.") } } } diff --git a/vieter.toml b/vieter.toml new file mode 100644 index 0000000..3f95398 --- /dev/null +++ b/vieter.toml @@ -0,0 +1,8 @@ +# This file contains settings used during development +api_key = "test" +download_dir = "data/downloads" +repo_dir = "data/repo" +pkg_dir = "data/pkgs" +# log_level = "DEBUG" +repos_file = "data/repos.json" + From c656e672e27ae789efec5ec7d34111a0fccf88ad Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 18:20:14 +0200 Subject: [PATCH 09/11] Moved config structs to more logical location --- src/build/build.v | 3 +-- src/build/cli.v | 8 +++++++- src/env.v | 19 +------------------ src/server/cli.v | 13 ++++++++++++- src/server/server.v | 5 ++--- 5 files changed, 23 insertions(+), 25 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 6f2b0a8..934627f 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -3,7 +3,6 @@ module build import docker import encoding.base64 import time -import env import net.http import git import json @@ -62,7 +61,7 @@ fn create_build_image() ?string { return image.id } -fn build(conf env.BuildConfig) ? { +fn build(conf Config) ? { // We get the repos list from the Vieter instance mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? req.add_custom_header('X-Api-Key', conf.api_key) ? diff --git a/src/build/cli.v b/src/build/cli.v index 463d2ca..c95d1ca 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -3,6 +3,12 @@ module build import cli import env +pub struct Config { +pub: + api_key string + address string +} + // cmd returns the cli submodule that handles the build process pub fn cmd() cli.Command { return cli.Command{ @@ -10,7 +16,7 @@ pub fn cmd() cli.Command { description: 'Run the build process.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? + conf := env.load(config_file) ? build(conf) ? } diff --git a/src/env.v b/src/env.v index b733791..7b20765 100644 --- a/src/env.v +++ b/src/env.v @@ -10,23 +10,6 @@ const prefix = 'VIETER_' // instead const file_suffix = '_FILE' -pub struct ServerConfig { -pub: - log_level string = "WARN" - log_file string = "vieter.log" - pkg_dir string - download_dir string - api_key string - repo_dir string - repos_file string -} - -pub struct BuildConfig { -pub: - api_key string - address string -} - fn get_env_var(field_name string) ?string { env_var_name := '$env.prefix$field_name.to_upper()' env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix' @@ -73,7 +56,7 @@ pub fn load(path string) ?T { s := doc.value(field.name) // We currently only support strings - if s.type_name() == "string" { + if s.type_name() == 'string' { res.$(field.name) = s.string() } } diff --git a/src/server/cli.v b/src/server/cli.v index 6e67686..2383d9f 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -3,6 +3,17 @@ module server import cli import env +struct Config { +pub: + log_level string = 'WARN' + log_file string = 'vieter.log' + pkg_dir string + download_dir string + api_key string + repo_dir string + repos_file string +} + // cmd returns the cli submodule that handles starting the server pub fn cmd() cli.Command { return cli.Command{ @@ -10,7 +21,7 @@ pub fn cmd() cli.Command { description: 'Start the Vieter server' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? + conf := env.load(config_file) ? server(conf) ? } diff --git a/src/server/server.v b/src/server/server.v index 0738bc4..ab2e46b 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -4,7 +4,6 @@ import web import os import log import repo -import env import util const port = 8000 @@ -12,7 +11,7 @@ const port = 8000 struct App { web.Context pub: - conf env.ServerConfig [required; web_global] + conf Config [required; web_global] pub mut: repo repo.Repo [required; web_global] // This is used to claim the file lock on the repos file @@ -20,7 +19,7 @@ pub mut: } // server starts the web server & starts listening for requests -pub fn server(conf env.ServerConfig) ? { +pub fn server(conf Config) ? { // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') From dff531d866e532d5d92ea3baffd146570e64123c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 19:51:54 +0200 Subject: [PATCH 10/11] Some final adjustments before merge --- src/env.v | 2 +- src/main.v | 2 +- src/server/cli.v | 2 +- vieter.toml | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/env.v b/src/env.v index 7b20765..cbde67e 100644 --- a/src/env.v +++ b/src/env.v @@ -49,7 +49,7 @@ pub fn load(path string) ?T { if os.exists(path) { // We don't use reflect here because reflect also sets any fields not - // in the toml back to "empty", which we don't want + // in the toml back to their zero value, which we don't want doc := toml.parse_file(path) ? $for field in T.fields { diff --git a/src/main.v b/src/main.v index 11bc77b..c77e551 100644 --- a/src/main.v +++ b/src/main.v @@ -9,7 +9,7 @@ import git fn main() { mut app := cli.Command{ name: 'vieter' - description: 'Arch repository server' + description: 'Vieter is a lightweight implementation of an Arch repository server.' version: '0.1.0' flags: [ cli.Flag{ diff --git a/src/server/cli.v b/src/server/cli.v index 2383d9f..f0dc0b1 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -18,7 +18,7 @@ pub: pub fn cmd() cli.Command { return cli.Command{ name: 'server' - description: 'Start the Vieter server' + description: 'Start the Vieter server.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file') ? conf := env.load(config_file) ? diff --git a/vieter.toml b/vieter.toml index 3f95398..c17c3c3 100644 --- a/vieter.toml +++ b/vieter.toml @@ -6,3 +6,4 @@ pkg_dir = "data/pkgs" # log_level = "DEBUG" repos_file = "data/repos.json" +address = "http://localhost:8000" From 4c3f322e6f064a4c290a1a07f324973db0c3e206 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 6 Apr 2022 20:03:26 +0200 Subject: [PATCH 11/11] Removed arm/v7 from docker release builds --- .woodpecker/.docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.woodpecker/.docker.yml b/.woodpecker/.docker.yml index b2f08ca..9b605f3 100644 --- a/.woodpecker/.docker.yml +++ b/.woodpecker/.docker.yml @@ -10,7 +10,7 @@ pipeline: settings: repo: chewingbever/vieter tag: dev - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm64/v8, linux/amd64 ] build_args_from_env: - CI_COMMIT_SHA when: @@ -23,7 +23,7 @@ pipeline: settings: repo: chewingbever/vieter auto_tag: true - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm64/v8, linux/amd64 ] build_args_from_env: - CI_COMMIT_SHA when: