diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 81fa32f..e68c4c9 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -37,22 +37,6 @@ pipeline: when: event: push - cli: - image: 'chewingbever/vlang:latest' - environment: - - LDFLAGS=-static - commands: - - make cli-prod - # Make sure the binary is actually statically built - - readelf -d vieterctl - - du -h vieterctl - - '[ "$(readelf -d vieterctl | grep NEEDED | wc -l)" = 0 ]' - # This removes so much, it's amazing - - strip -s vieterctl - - du -h vieterctl - when: - event: push - upload: image: 'chewingbever/vlang:latest' secrets: [ s3_username, s3_password ] @@ -75,20 +59,5 @@ pipeline: -H "Content-Type: $CONTENT_TYPE" -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" https://$URL$OBJ_PATH - - # Also update the CLI tool - - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieterctl-$(echo '${PLATFORM}' | sed 's:/:-:g')" - - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH" - - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64` - - > - curl - --silent - -XPUT - -T vieterctl - -H "Host: $URL" - -H "Date: $DATE" - -H "Content-Type: $CONTENT_TYPE" - -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" - https://$URL$OBJ_PATH when: event: push diff --git a/.woodpecker/.docker.yml b/.woodpecker/.docker.yml index b2f08ca..9b605f3 100644 --- a/.woodpecker/.docker.yml +++ b/.woodpecker/.docker.yml @@ -10,7 +10,7 @@ pipeline: settings: repo: chewingbever/vieter tag: dev - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm64/v8, linux/amd64 ] build_args_from_env: - CI_COMMIT_SHA when: @@ -23,7 +23,7 @@ pipeline: settings: repo: chewingbever/vieter auto_tag: true - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm64/v8, linux/amd64 ] build_args_from_env: - CI_COMMIT_SHA when: diff --git a/Makefile b/Makefile index 7b0fb1b..76ab7b5 100644 --- a/Makefile +++ b/Makefile @@ -23,13 +23,7 @@ dvieter: $(SOURCES) # Run the debug build inside gdb .PHONY: gdb gdb: dvieter - VIETER_API_KEY=test \ - VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ - VIETER_PKG_DIR=data/pkgs \ - VIETER_LOG_LEVEL=DEBUG \ - VIETER_REPOS_FILE=data/repos.json \ - gdb --args ./dvieter + gdb --args './dvieter -f vieter.toml server' # Optimised production build .PHONY: prod @@ -42,38 +36,15 @@ pvieter: $(SOURCES) c: $(V) -o vieter.c $(SRC_DIR) -# Build the CLI tool -.PHONY: cli -cli: dvieterctl -dvieterctl: cli.v - $(V_PATH) -showcc -g -o dvieterctl cli.v - -.PHONY: cli-prod -cli-prod: vieterctl -vieterctl: cli.v -cli-prod: - $(V_PATH) -showcc -o vieterctl -prod cli.v - # =====EXECUTION===== # Run the server in the default 'data' directory .PHONY: run run: vieter - VIETER_API_KEY=test \ - VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ - VIETER_PKG_DIR=data/pkgs \ - VIETER_LOG_LEVEL=DEBUG \ - VIETER_REPOS_FILE=data/repos.json \ - ./vieter server + ./vieter -f vieter.toml server .PHONY: run-prod run-prod: prod - VIETER_API_KEY=test \ - VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ - VIETER_PKG_DIR=data/pkgs \ - VIETER_LOG_LEVEL=DEBUG \ - ./pvieter server + ./pvieter -f vieter.toml server # =====OTHER===== .PHONY: lint diff --git a/PKGBUILD b/PKGBUILD index 767c07c..cb0c24a 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -1,7 +1,7 @@ # Maintainer: Jef Roosens pkgbase='vieter' -pkgname=('vieter' 'vieterctl') +pkgname='vieter' pkgver=0.1.0.rc1.r45.g6d3ff8a pkgrel=1 depends=('glibc' 'openssl' 'libarchive' 'gc') @@ -23,21 +23,12 @@ build() { # Build the compiler CFLAGS= make v - # Build the server & the CLI tool make prod - make cli-prod } -package_vieter() { +package() { pkgdesc="Vieter is a lightweight implementation of an Arch repository server." install -dm755 "$pkgdir/usr/bin" install -Dm755 "$pkgbase/pvieter" "$pkgdir/usr/bin/vieter" } - -package_vieterctl() { - pkgdesc="Allows you to configure a Vieter server's list of Git repositories." - install -dm755 "$pkgdir/usr/bin" - - install -Dm755 "$pkgbase/vieterctl" "$pkgdir/usr/bin/vieterctl" -} diff --git a/cli.v b/cli.v deleted file mode 100644 index 256b856..0000000 --- a/cli.v +++ /dev/null @@ -1,84 +0,0 @@ -import os -import toml -import net.http - -struct Config { - address string [required] - api_key string [required] -} - -fn list(conf Config) ? { - mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? - req.add_custom_header('X-API-Key', conf.api_key) ? - - res := req.do() ? - - println(res.text) -} - -fn add(conf Config, args []string) ? { - if args.len < 2 { - eprintln('Not enough arguments.') - exit(1) - } - - if args.len > 2 { - eprintln('Too many arguments.') - exit(1) - } - - mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=${args[0]}&branch=${args[1]}', '') ? - req.add_custom_header('X-API-Key', conf.api_key) ? - - res := req.do() ? - - println(res.text) -} - -fn remove(conf Config, args []string) ? { - if args.len < 2 { - eprintln('Not enough arguments.') - exit(1) - } - - if args.len > 2 { - eprintln('Too many arguments.') - exit(1) - } - - mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=${args[0]}&branch=${args[1]}', '') ? - req.add_custom_header('X-API-Key', conf.api_key) ? - - res := req.do() ? - - println(res.text) -} - -fn main() { - conf_path := os.expand_tilde_to_home('~/.vieterrc') - - if !os.is_file(conf_path) { - exit(1) - } - - conf := toml.parse_file(conf_path) ?.reflect() - - args := os.args[1..] - - if args.len == 0 { - eprintln('No action provided.') - exit(1) - } - - action := args[0] - - match action { - 'list' { list(conf) ? } - 'add' { add(conf, args[1..]) ? } - 'remove' { remove(conf, args[1..]) ? } - else { - eprintln("Invalid action '$action'.") - exit(1) - } - } -} diff --git a/src/build.v b/src/build/build.v similarity index 95% rename from src/build.v rename to src/build/build.v index 51de64c..934627f 100644 --- a/src/build.v +++ b/src/build/build.v @@ -1,12 +1,11 @@ -module main +module build import docker import encoding.base64 import time -import json -import server -import env import net.http +import git +import json const container_build_dir = '/build' @@ -62,15 +61,13 @@ fn create_build_image() ?string { return image.id } -fn build() ? { - conf := env.load() ? - +fn build(conf Config) ? { // We get the repos list from the Vieter instance mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? req.add_custom_header('X-Api-Key', conf.api_key) ? res := req.do() ? - repos := json.decode([]server.GitRepo, res.text) ? + repos := json.decode([]git.GitRepo, res.text) ? // No point in doing work if there's no repos present if repos.len == 0 { diff --git a/src/build/cli.v b/src/build/cli.v new file mode 100644 index 0000000..c95d1ca --- /dev/null +++ b/src/build/cli.v @@ -0,0 +1,24 @@ +module build + +import cli +import env + +pub struct Config { +pub: + api_key string + address string +} + +// cmd returns the cli submodule that handles the build process +pub fn cmd() cli.Command { + return cli.Command{ + name: 'build' + description: 'Run the build process.' + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? + + build(conf) ? + } + } +} diff --git a/src/env.v b/src/env.v index bd544cf..cbde67e 100644 --- a/src/env.v +++ b/src/env.v @@ -1,6 +1,7 @@ module env import os +import toml // The prefix that every environment variable should have const prefix = 'VIETER_' @@ -9,32 +10,15 @@ const prefix = 'VIETER_' // instead const file_suffix = '_FILE' -pub struct ServerConfig { -pub: - log_level string [default: WARN] - log_file string [default: 'vieter.log'] - pkg_dir string - download_dir string - api_key string - repo_dir string - repos_file string -} - -pub struct BuildConfig { -pub: - api_key string - address string -} - fn get_env_var(field_name string) ?string { env_var_name := '$env.prefix$field_name.to_upper()' env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix' env_var := os.getenv(env_var_name) env_file := os.getenv(env_file_name) - // If both aren't set, we report them missing + // If both are missing, we return an empty string if env_var == '' && env_file == '' { - return error('Either $env_var_name or $env_file_name is required.') + return '' } // If they're both set, we report a conflict @@ -56,30 +40,42 @@ fn get_env_var(field_name string) ?string { } } -// load attempts to create the given type from environment variables. For -// each field, the corresponding env var is its name in uppercase prepended -// with the hardcoded prefix. If this one isn't present, it looks for the env -// var with the file_suffix suffix. -pub fn load() ?T { - res := T{} +// load attempts to create an object of type T from the given path to a toml +// file & environment variables. For each field, it will select either a value +// given from an environment variable, a value defined in the config file or a +// configured default if present, in that order. +pub fn load(path string) ?T { + mut res := T{} + + if os.exists(path) { + // We don't use reflect here because reflect also sets any fields not + // in the toml back to their zero value, which we don't want + doc := toml.parse_file(path) ? + + $for field in T.fields { + s := doc.value(field.name) + + // We currently only support strings + if s.type_name() == 'string' { + res.$(field.name) = s.string() + } + } + } $for field in T.fields { - res.$(field.name) = get_env_var(field.name) or { - // We use the default instead, if it's present - mut default := '' + $if field.typ is string { + env_value := get_env_var(field.name) ? - for attr in field.attrs { - if attr.starts_with('default: ') { - default = attr[9..] - break - } + // The value of the env var will always be chosen over the config + // file + if env_value != '' { + res.$(field.name) = env_value } - - if default == '' { - return err + // If there's no value from the toml file either, we try to find a + // default value + else if res.$(field.name) == '' { + return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.") } - - default } } return res diff --git a/src/git/cli.v b/src/git/cli.v new file mode 100644 index 0000000..17fa984 --- /dev/null +++ b/src/git/cli.v @@ -0,0 +1,83 @@ +module git + +import cli +import env +import net.http + +struct Config { + address string [required] + api_key string [required] +} + +// cmd returns the cli submodule that handles the repos API interaction +pub fn cmd() cli.Command { + return cli.Command{ + name: 'repos' + description: 'Interact with the repos API.' + commands: [ + cli.Command{ + name: 'list' + description: 'List the current repos.' + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? + + list(conf) ? + } + }, + cli.Command{ + name: 'add' + required_args: 2 + usage: 'url branch' + description: 'Add a new repository.' + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? + + add(conf, cmd.args[0], cmd.args[1]) ? + } + }, + cli.Command{ + name: 'remove' + required_args: 2 + usage: 'url branch' + description: 'Remove a repository.' + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? + + remove(conf, cmd.args[0], cmd.args[1]) ? + } + }, + ] + } +} + +fn list(conf Config) ? { + mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? + req.add_custom_header('X-API-Key', conf.api_key) ? + + res := req.do() ? + + println(res.text) +} + +fn add(conf Config, url string, branch string) ? { + mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=$url&branch=$branch', + '') ? + req.add_custom_header('X-API-Key', conf.api_key) ? + + res := req.do() ? + + println(res.text) +} + +fn remove(conf Config, url string, branch string) ? { + mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=$url&branch=$branch', + '') ? + req.add_custom_header('X-API-Key', conf.api_key) ? + + res := req.do() ? + + println(res.text) +} diff --git a/src/git/git.v b/src/git/git.v new file mode 100644 index 0000000..3ad35fd --- /dev/null +++ b/src/git/git.v @@ -0,0 +1,74 @@ +module git + +import os +import json + +pub struct GitRepo { +pub mut: + // URL of the Git repository + url string + // Branch of the Git repository to use + branch string + // On which architectures the package is allowed to be built. In reality, + // this controls which builders will periodically build the image. + arch []string +} + +pub fn (mut r GitRepo) patch_from_params(params map[string]string) { + $for field in GitRepo.fields { + if field.name in params { + $if field.typ is string { + r.$(field.name) = params[field.name] + // This specific type check is needed for the compiler to ensure + // our types are correct + } $else $if field.typ is []string { + r.$(field.name) = params[field.name].split(',') + } + } + } +} + +pub fn read_repos(path string) ?map[string]GitRepo { + if !os.exists(path) { + mut f := os.create(path) ? + + defer { + f.close() + } + + f.write_string('{}') ? + + return {} + } + + content := os.read_file(path) ? + res := json.decode(map[string]GitRepo, content) ? + + return res +} + +pub fn write_repos(path string, repos &map[string]GitRepo) ? { + mut f := os.create(path) ? + + defer { + f.close() + } + + value := json.encode(repos) + f.write_string(value) ? +} + +pub fn repo_from_params(params map[string]string) ?GitRepo { + mut repo := GitRepo{} + + // If we're creating a new GitRepo, we want all fields to be present before + // "patching". + $for field in GitRepo.fields { + if field.name !in params { + return error('Missing parameter: ${field.name}.') + } + } + repo.patch_from_params(params) + + return repo +} diff --git a/src/main.v b/src/main.v index 156f0a3..c77e551 100644 --- a/src/main.v +++ b/src/main.v @@ -2,16 +2,32 @@ module main import os import server -import util +import cli +import build +import git fn main() { - if os.args.len == 1 { - util.exit_with_message(1, 'No action provided.') + mut app := cli.Command{ + name: 'vieter' + description: 'Vieter is a lightweight implementation of an Arch repository server.' + version: '0.1.0' + flags: [ + cli.Flag{ + flag: cli.FlagType.string + name: 'config-file' + abbrev: 'f' + description: 'Location of Vieter config file; defaults to ~/.vieterrc.' + global: true + default_value: [os.expand_tilde_to_home('~/.vieterrc')] + }, + ] + commands: [ + server.cmd(), + build.cmd(), + git.cmd(), + ] } - match os.args[1] { - 'server' { server.server() ? } - 'build' { build() ? } - else { util.exit_with_message(1, 'Unknown action: ${os.args[1]}') } - } + app.setup() + app.parse(os.args) } diff --git a/src/server/cli.v b/src/server/cli.v new file mode 100644 index 0000000..f0dc0b1 --- /dev/null +++ b/src/server/cli.v @@ -0,0 +1,29 @@ +module server + +import cli +import env + +struct Config { +pub: + log_level string = 'WARN' + log_file string = 'vieter.log' + pkg_dir string + download_dir string + api_key string + repo_dir string + repos_file string +} + +// cmd returns the cli submodule that handles starting the server +pub fn cmd() cli.Command { + return cli.Command{ + name: 'server' + description: 'Start the Vieter server.' + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? + + server(conf) ? + } + } +} diff --git a/src/server/git.v b/src/server/git.v index 8862af8..7d177d7 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -1,79 +1,11 @@ module server import web -import os -import json -import rand +import git import net.http +import rand -pub struct GitRepo { -pub mut: - // URL of the Git repository - url string - // Branch of the Git repository to use - branch string - // On which architectures the package is allowed to be built. In reality, - // this controls which builders will periodically build the image. - arch []string -} - -fn (mut r GitRepo) patch_from_params(params map[string]string) { - $for field in GitRepo.fields { - if field.name in params { - $if field.typ is string { - r.$(field.name) = params[field.name] - // This specific type check is needed for the compiler to ensure - // our types are correct - } $else $if field.typ is []string { - r.$(field.name) = params[field.name].split(',') - } - } - } -} - -fn repo_from_params(params map[string]string) ?GitRepo { - mut repo := GitRepo{} - - // If we're creating a new GitRepo, we want all fields to be present before - // "patching". - $for field in GitRepo.fields { - if field.name !in params { - return error('Missing parameter: ${field.name}.') - } - } - repo.patch_from_params(params) - - return repo -} - -fn read_repos(path string) ?map[string]GitRepo { - if !os.exists(path) { - mut f := os.create(path) ? - - defer { - f.close() - } - - f.write_string('{}') ? - - return {} - } - - content := os.read_file(path) ? - res := json.decode(map[string]GitRepo, content) ? - return res -} - -fn write_repos(path string, repos &map[string]GitRepo) ? { - mut f := os.create(path) ? - - defer { - f.close() - } - - value := json.encode(repos) - f.write_string(value) ? -} +const repos_file = 'repos.json' ['/api/repos'; get] fn (mut app App) get_repos() web.Result { @@ -82,7 +14,7 @@ fn (mut app App) get_repos() web.Result { } repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { + git.read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') return app.status(http.Status.internal_server_error) @@ -99,7 +31,7 @@ fn (mut app App) get_single_repo(id string) web.Result { } repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { + git.read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') return app.status(http.Status.internal_server_error) @@ -121,14 +53,14 @@ fn (mut app App) post_repo() web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - new_repo := repo_from_params(app.query) or { + new_repo := git.repo_from_params(app.query) or { return app.json(http.Status.bad_request, new_response(err.msg)) } id := rand.uuid_v4() mut repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { + git.read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') return app.status(http.Status.internal_server_error) @@ -145,7 +77,7 @@ fn (mut app App) post_repo() web.Result { repos[id] = new_repo lock app.git_mutex { - write_repos(app.conf.repos_file, &repos) or { + git.write_repos(app.conf.repos_file, &repos) or { return app.status(http.Status.internal_server_error) } } @@ -160,7 +92,7 @@ fn (mut app App) delete_repo(id string) web.Result { } mut repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { + git.read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') return app.status(http.Status.internal_server_error) @@ -174,7 +106,7 @@ fn (mut app App) delete_repo(id string) web.Result { repos.delete(id) lock app.git_mutex { - write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } + git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } } return app.json(http.Status.ok, new_response('Repo removed successfully.')) @@ -187,7 +119,7 @@ fn (mut app App) patch_repo(id string) web.Result { } mut repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { + git.read_repos(app.conf.repos_file) or { app.lerror('Failed to read repos file.') return app.status(http.Status.internal_server_error) @@ -201,7 +133,7 @@ fn (mut app App) patch_repo(id string) web.Result { repos[id].patch_from_params(app.query) lock app.git_mutex { - write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } + git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } } return app.json(http.Status.ok, new_response('Repo updated successfully.')) diff --git a/src/server/server.v b/src/server/server.v index 4b31b99..ab2e46b 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -4,7 +4,6 @@ import web import os import log import repo -import env import util const port = 8000 @@ -12,7 +11,7 @@ const port = 8000 struct App { web.Context pub: - conf env.ServerConfig [required; web_global] + conf Config [required; web_global] pub mut: repo repo.Repo [required; web_global] // This is used to claim the file lock on the repos file @@ -20,9 +19,7 @@ pub mut: } // server starts the web server & starts listening for requests -pub fn server() ? { - conf := env.load() ? - +pub fn server(conf Config) ? { // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') diff --git a/vieter.toml b/vieter.toml new file mode 100644 index 0000000..c17c3c3 --- /dev/null +++ b/vieter.toml @@ -0,0 +1,9 @@ +# This file contains settings used during development +api_key = "test" +download_dir = "data/downloads" +repo_dir = "data/repo" +pkg_dir = "data/pkgs" +# log_level = "DEBUG" +repos_file = "data/repos.json" + +address = "http://localhost:8000"