Merge branch 'dev' into multi-arch-repos-v2

.woodpecker/.build.yml

@@ -2,7 +2,6 @@ matrix:
   PLATFORM:
     - linux/amd64
     - linux/arm64
-    - linux/arm/v7
 
 # These checks already get performed on the feature branches
 platform: ${PLATFORM}
@@ -36,22 +35,6 @@ pipeline:
     when:
       event: push
 
-  cli:
-    image: 'chewingbever/vlang:latest'
-    environment:
-      - LDFLAGS=-static
-    commands:
-      - make cli-prod
-      # Make sure the binary is actually statically built
-      - readelf -d vieterctl
-      - du -h vieterctl
-      - '[ "$(readelf -d vieterctl | grep NEEDED | wc -l)" = 0 ]'
-      # This removes so much, it's amazing
-      - strip -s vieterctl
-      - du -h vieterctl
-    when:
-      event: push
-
   upload:
     image: 'chewingbever/vlang:latest'
     secrets: [ s3_username, s3_password ]
@@ -74,20 +57,5 @@ pipeline:
         -H "Content-Type: $CONTENT_TYPE"
         -H "Authorization: AWS $S3_USERNAME:$SIGNATURE"
         https://$URL$OBJ_PATH
-
-      # Also update the CLI tool
-      - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieterctl-$(echo '${PLATFORM}' | sed 's:/:-:g')"
-      - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH"
-      - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64`
-      - >
-        curl 
-        --silent
-        -XPUT
-        -T vieterctl
-        -H "Host: $URL"
-        -H "Date: $DATE"
-        -H "Content-Type: $CONTENT_TYPE"
-        -H "Authorization: AWS $S3_USERNAME:$SIGNATURE"
-        https://$URL$OBJ_PATH
     when:
       event: push

.woodpecker/.docker.yml

@@ -10,7 +10,7 @@ pipeline:
     settings:
       repo: chewingbever/vieter
       tag: dev
-      platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
+      platforms: [ linux/arm64/v8, linux/amd64 ]
       build_args_from_env:
         - CI_COMMIT_SHA
     when:
@@ -23,7 +23,7 @@ pipeline:
     settings:
       repo: chewingbever/vieter
       auto_tag: true
-      platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
+      platforms: [ linux/arm64/v8, linux/amd64 ]
       build_args_from_env:
         - CI_COMMIT_SHA
     when:

Makefile

@@ -23,13 +23,7 @@ dvieter: $(SOURCES)
 # Run the debug build inside gdb
 .PHONY: gdb
 gdb: dvieter
-	 VIETER_API_KEY=test \
+		gdb --args './dvieter -f vieter.toml server'
-		VIETER_DOWNLOAD_DIR=data/downloads \
-		VIETER_DATA_DIR=data/repo \
-		VIETER_PKG_DIR=data/pkgs \
-		VIETER_LOG_LEVEL=DEBUG \
-		VIETER_REPOS_FILE=data/repos.json \
-		gdb --args ./dvieter
 
 # Optimised production build
 .PHONY: prod
@@ -42,39 +36,15 @@ pvieter: $(SOURCES)
 c:
 	$(V) -o vieter.c $(SRC_DIR)
 
-# Build the CLI tool
-.PHONY: cli
-cli: dvieterctl
-dvieterctl: cli.v
-	$(V_PATH) -showcc -g -o dvieterctl cli.v
-
-.PHONY: cli-prod
-cli-prod: vieterctl
-vieterctl: cli.v
-cli-prod:
-	$(V_PATH) -showcc -o vieterctl -prod cli.v
-
 # =====EXECUTION=====
 # Run the server in the default 'data' directory
 .PHONY: run
 run: vieter
-	 VIETER_API_KEY=test \
+		./vieter -f vieter.toml server
-		VIETER_DOWNLOAD_DIR=data/downloads \
-		VIETER_DATA_DIR=data/repo \
-		VIETER_PKG_DIR=data/pkgs \
-		VIETER_LOG_LEVEL=DEBUG \
-		VIETER_REPOS_FILE=data/repos.json \
-		./vieter server
 
 .PHONY: run-prod
 run-prod: prod
-	VIETER_API_KEY=test \
+	./pvieter -f vieter.toml server
-		VIETER_DOWNLOAD_DIR=data/downloads \
-		VIETER_DATA_DIR=data/repo \
-		VIETER_PKG_DIR=data/pkgs \
-		VIETER_LOG_LEVEL=DEBUG \
-		VIETER_REPOS_FILE=data/repos.json \
-	./pvieter server
 
 # =====OTHER=====
 .PHONY: lint

PKGBUILD

@@ -1,7 +1,7 @@
 # Maintainer: Jef Roosens
 
 pkgbase='vieter'
-pkgname=('vieter' 'vieterctl')
+pkgname='vieter'
 pkgver=0.1.0.rc1.r45.g6d3ff8a
 pkgrel=1
 depends=('glibc' 'openssl' 'libarchive' 'gc')
@@ -23,21 +23,12 @@ build() {
     # Build the compiler
     CFLAGS= make v
 
-    # Build the server & the CLI tool
     make prod
-    make cli-prod
 }
 
-package_vieter() {
+package() {
     pkgdesc="Vieter is a lightweight implementation of an Arch repository server."
     install -dm755 "$pkgdir/usr/bin"
 
     install -Dm755 "$pkgbase/pvieter" "$pkgdir/usr/bin/vieter"
 }
-
-package_vieterctl() {
-    pkgdesc="Allows you to configure a Vieter server's list of Git repositories."
-    install -dm755 "$pkgdir/usr/bin"
-
-    install -Dm755 "$pkgbase/vieterctl" "$pkgdir/usr/bin/vieterctl"
-}

cli.v

@@ -1,84 +0,0 @@
-import os
-import toml
-import net.http
-
-struct Config {
-	address string [required]
-	api_key string [required]
-}
-
-fn list(conf Config) ? {
-	mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ?
-	req.add_custom_header('X-API-Key', conf.api_key) ?
-
-	res := req.do() ?
-
-	println(res.text)
-}
-
-fn add(conf Config, args []string) ? {
-	if args.len < 2 {
-		eprintln('Not enough arguments.')
-		exit(1)
-	}
-
-	if args.len > 2 {
-		eprintln('Too many arguments.')
-		exit(1)
-	}
-
-	mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=${args[0]}&branch=${args[1]}', '') ?
-	req.add_custom_header('X-API-Key', conf.api_key) ?
-
-	res := req.do() ?
-
-	println(res.text)
-}
-
-fn remove(conf Config, args []string) ? {
-	if args.len < 2 {
-		eprintln('Not enough arguments.')
-		exit(1)
-	}
-
-	if args.len > 2 {
-		eprintln('Too many arguments.')
-		exit(1)
-	}
-
-	mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=${args[0]}&branch=${args[1]}', '') ?
-	req.add_custom_header('X-API-Key', conf.api_key) ?
-
-	res := req.do() ?
-
-	println(res.text)
-}
-
-fn main() {
-	conf_path := os.expand_tilde_to_home('~/.vieterrc')
-
-	if !os.is_file(conf_path) {
-		exit(1)
-	}
-
-	conf := toml.parse_file(conf_path) ?.reflect<Config>()
-
-	args := os.args[1..]
-
-	if args.len == 0 {
-		eprintln('No action provided.')
-		exit(1)
-	}
-
-	action := args[0]
-
-	match action {
-		'list' { list(conf) ? }
-		'add' { add(conf, args[1..]) ? }
-		'remove' { remove(conf, args[1..]) ? }
-		else {
-			eprintln("Invalid action '$action'.")
-			exit(1)
-		}
-	}
-}

src/build/build.v

@@ -1,12 +1,11 @@
-module main
+module build
 
 import docker
 import encoding.base64
 import time
-import json
-import server
-import env
 import net.http
+import git
+import json
 
 const container_build_dir = '/build'
 
@@ -62,15 +61,13 @@ fn create_build_image() ?string {
 	return image.id
 }
 
-fn build() ? {
+fn build(conf Config) ? {
-	conf := env.load<env.BuildConfig>() ?
-
 	// We get the repos list from the Vieter instance
 	mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ?
 	req.add_custom_header('X-Api-Key', conf.api_key) ?
 
 	res := req.do() ?
-	repos := json.decode([]server.GitRepo, res.text) ?
+	repos := json.decode([]git.GitRepo, res.text) ?
 
 	// No point in doing work if there's no repos present
 	if repos.len == 0 {

src/build/cli.v

@@ -0,0 +1,24 @@
+module build
+
+import cli
+import env
+
+pub struct Config {
+pub:
+	api_key string
+	address string
+}
+
+// cmd returns the cli submodule that handles the build process
+pub fn cmd() cli.Command {
+	return cli.Command{
+		name: 'build'
+		description: 'Run the build process.'
+		execute: fn (cmd cli.Command) ? {
+			config_file := cmd.flags.get_string('config-file') ?
+			conf := env.load<Config>(config_file) ?
+
+			build(conf) ?
+		}
+	}
+}

src/env.v

@@ -1,6 +1,7 @@
 module env
 
 import os
+import toml
 
 // The prefix that every environment variable should have
 const prefix = 'VIETER_'
@@ -9,32 +10,15 @@ const prefix = 'VIETER_'
 // instead
 const file_suffix = '_FILE'
 
-pub struct ServerConfig {
-pub:
-	log_level    string [default: WARN]
-	log_file     string [default: 'vieter.log']
-	pkg_dir      string
-	download_dir string
-	api_key      string
-	data_dir     string
-	repos_file   string
-}
-
-pub struct BuildConfig {
-pub:
-	api_key string
-	address string
-}
-
 fn get_env_var(field_name string) ?string {
 	env_var_name := '$env.prefix$field_name.to_upper()'
 	env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix'
 	env_var := os.getenv(env_var_name)
 	env_file := os.getenv(env_file_name)
 
-	// If both aren't set, we report them missing
+	// If both are missing, we return an empty string
 	if env_var == '' && env_file == '' {
-		return error('Either $env_var_name or $env_file_name is required.')
+		return ''
 	}
 
 	// If they're both set, we report a conflict
@@ -56,30 +40,42 @@ fn get_env_var(field_name string) ?string {
 	}
 }
 
-// load<T> attempts to create the given type from environment variables. For
+// load<T> attempts to create an object of type T from the given path to a toml
-// each field, the corresponding env var is its name in uppercase prepended
+// file & environment variables. For each field, it will select either a value
-// with the hardcoded prefix. If this one isn't present, it looks for the env
+// given from an environment variable, a value defined in the config file or a
-// var with the file_suffix suffix.
+// configured default if present, in that order.
-pub fn load<T>() ?T {
+pub fn load<T>(path string) ?T {
-	res := T{}
+	mut res := T{}
+
+	if os.exists(path) {
+		// We don't use reflect here because reflect also sets any fields not
+		// in the toml back to their zero value, which we don't want
+		doc := toml.parse_file(path) ?
+
+		$for field in T.fields {
+			s := doc.value(field.name)
+
+			// We currently only support strings
+			if s.type_name() == 'string' {
+				res.$(field.name) = s.string()
+			}
+		}
+	}
 
 	$for field in T.fields {
-		res.$(field.name) = get_env_var(field.name) or {
+		$if field.typ is string {
-			// We use the default instead, if it's present
+			env_value := get_env_var(field.name) ?
-			mut default := ''
 
-			for attr in field.attrs {
+			// The value of the env var will always be chosen over the config
-				if attr.starts_with('default: ') {
+			// file
-					default = attr[9..]
+			if env_value != '' {
-					break
+				res.$(field.name) = env_value
-				}
 			}
-
+			// If there's no value from the toml file either, we try to find a
-			if default == '' {
+			// default value
-				return err
+			else if res.$(field.name) == '' {
+				return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.")
 			}
-
-			default
 		}
 	}
 	return res

src/git/cli.v

@@ -0,0 +1,83 @@
+module git
+
+import cli
+import env
+import net.http
+
+struct Config {
+	address string [required]
+	api_key string [required]
+}
+
+// cmd returns the cli submodule that handles the repos API interaction
+pub fn cmd() cli.Command {
+	return cli.Command{
+		name: 'repos'
+		description: 'Interact with the repos API.'
+		commands: [
+			cli.Command{
+				name: 'list'
+				description: 'List the current repos.'
+				execute: fn (cmd cli.Command) ? {
+					config_file := cmd.flags.get_string('config-file') ?
+					conf := env.load<Config>(config_file) ?
+
+					list(conf) ?
+				}
+			},
+			cli.Command{
+				name: 'add'
+				required_args: 2
+				usage: 'url branch'
+				description: 'Add a new repository.'
+				execute: fn (cmd cli.Command) ? {
+					config_file := cmd.flags.get_string('config-file') ?
+					conf := env.load<Config>(config_file) ?
+
+					add(conf, cmd.args[0], cmd.args[1]) ?
+				}
+			},
+			cli.Command{
+				name: 'remove'
+				required_args: 2
+				usage: 'url branch'
+				description: 'Remove a repository.'
+				execute: fn (cmd cli.Command) ? {
+					config_file := cmd.flags.get_string('config-file') ?
+					conf := env.load<Config>(config_file) ?
+
+					remove(conf, cmd.args[0], cmd.args[1]) ?
+				}
+			},
+		]
+	}
+}
+
+fn list(conf Config) ? {
+	mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ?
+	req.add_custom_header('X-API-Key', conf.api_key) ?
+
+	res := req.do() ?
+
+	println(res.text)
+}
+
+fn add(conf Config, url string, branch string) ? {
+	mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=$url&branch=$branch',
+		'') ?
+	req.add_custom_header('X-API-Key', conf.api_key) ?
+
+	res := req.do() ?
+
+	println(res.text)
+}
+
+fn remove(conf Config, url string, branch string) ? {
+	mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=$url&branch=$branch',
+		'') ?
+	req.add_custom_header('X-API-Key', conf.api_key) ?
+
+	res := req.do() ?
+
+	println(res.text)
+}

src/git/git.v

@@ -0,0 +1,41 @@
+module git
+
+import os
+import json
+
+pub struct GitRepo {
+pub:
+	url    string [required]
+	branch string [required]
+}
+
+// read_repos reads the given JSON file & parses it as a list of Git repos
+pub fn read_repos(path string) ?[]GitRepo {
+	if !os.exists(path) {
+		mut f := os.create(path) ?
+
+		defer {
+			f.close()
+		}
+
+		f.write_string('[]') ?
+
+		return []
+	}
+
+	content := os.read_file(path) ?
+	res := json.decode([]GitRepo, content) ?
+	return res
+}
+
+// write_repos writes a list of repositories back to a given file
+pub fn write_repos(path string, repos []GitRepo) ? {
+	mut f := os.create(path) ?
+
+	defer {
+		f.close()
+	}
+
+	value := json.encode(repos)
+	f.write_string(value) ?
+}

src/main.v

@@ -2,16 +2,32 @@ module main
 
 import os
 import server
-import util
+import cli
+import build
+import git
 
 fn main() {
-	if os.args.len == 1 {
+	mut app := cli.Command{
-		util.exit_with_message(1, 'No action provided.')
+		name: 'vieter'
+		description: 'Vieter is a lightweight implementation of an Arch repository server.'
+		version: '0.1.0'
+		flags: [
+			cli.Flag{
+				flag: cli.FlagType.string
+				name: 'config-file'
+				abbrev: 'f'
+				description: 'Location of Vieter config file; defaults to ~/.vieterrc.'
+				global: true
+				default_value: [os.expand_tilde_to_home('~/.vieterrc')]
+			},
+		]
+		commands: [
+			server.cmd(),
+			build.cmd(),
+			git.cmd(),
+		]
 	}
 
-	match os.args[1] {
+	app.setup()
-		'server' { server.server() ? }
+	app.parse(os.args)
-		'build' { build() ? }
-		else { util.exit_with_message(1, 'Unknown action: ${os.args[1]}') }
-	}
 }

src/server/cli.v

@@ -0,0 +1,29 @@
+module server
+
+import cli
+import env
+
+struct Config {
+pub:
+	log_level    string = 'WARN'
+	log_file     string = 'vieter.log'
+	pkg_dir      string
+	download_dir string
+	api_key      string
+	data_dir     string
+	repos_file   string
+}
+
+// cmd returns the cli submodule that handles starting the server
+pub fn cmd() cli.Command {
+	return cli.Command{
+		name: 'server'
+		description: 'Start the Vieter server.'
+		execute: fn (cmd cli.Command) ? {
+			config_file := cmd.flags.get_string('config-file') ?
+			conf := env.load<Config>(config_file) ?
+
+			server(conf) ?
+		}
+	}
+}

src/server/git.v

@@ -1,46 +1,10 @@
 module server
 
 import web
-import os
+import git
-import json
 
 const repos_file = 'repos.json'
 
-pub struct GitRepo {
-pub:
-	url    string [required]
-	branch string [required]
-}
-
-fn read_repos(path string) ?[]GitRepo {
-	if !os.exists(path) {
-		mut f := os.create(path) ?
-
-		defer {
-			f.close()
-		}
-
-		f.write_string('[]') ?
-
-		return []
-	}
-
-	content := os.read_file(path) ?
-	res := json.decode([]GitRepo, content) ?
-	return res
-}
-
-fn write_repos(path string, repos []GitRepo) ? {
-	mut f := os.create(path) ?
-
-	defer {
-		f.close()
-	}
-
-	value := json.encode(repos)
-	f.write_string(value) ?
-}
-
 ['/api/repos'; get]
 fn (mut app App) get_repos() web.Result {
 	if !app.is_authorized() {
@@ -48,7 +12,7 @@ fn (mut app App) get_repos() web.Result {
 	}
 
 	repos := rlock app.git_mutex {
-		read_repos(app.conf.repos_file) or {
+		git.read_repos(app.conf.repos_file) or {
 			app.lerror('Failed to read repos file.')
 
 			return app.server_error(500)
@@ -68,13 +32,13 @@ fn (mut app App) post_repo() web.Result {
 		return app.server_error(400)
 	}
 
-	new_repo := GitRepo{
+	new_repo := git.GitRepo{
 		url: app.query['url']
 		branch: app.query['branch']
 	}
 
 	mut repos := rlock app.git_mutex {
-		read_repos(app.conf.repos_file) or {
+		git.read_repos(app.conf.repos_file) or {
 			app.lerror('Failed to read repos file.')
 
 			return app.server_error(500)
@@ -91,7 +55,7 @@ fn (mut app App) post_repo() web.Result {
 	repos << new_repo
 
 	lock app.git_mutex {
-		write_repos(app.conf.repos_file, repos) or { return app.server_error(500) }
+		git.write_repos(app.conf.repos_file, repos) or { return app.server_error(500) }
 	}
 
 	return app.ok('Repo added successfully.')
@@ -107,13 +71,13 @@ fn (mut app App) delete_repo() web.Result {
 		return app.server_error(400)
 	}
 
-	repo_to_remove := GitRepo{
+	repo_to_remove := git.GitRepo{
 		url: app.query['url']
 		branch: app.query['branch']
 	}
 
 	mut repos := rlock app.git_mutex {
-		read_repos(app.conf.repos_file) or {
+		git.read_repos(app.conf.repos_file) or {
 			app.lerror('Failed to read repos file.')
 
 			return app.server_error(500)
@@ -122,7 +86,7 @@ fn (mut app App) delete_repo() web.Result {
 	filtered := repos.filter(it != repo_to_remove)
 
 	lock app.git_mutex {
-		write_repos(app.conf.repos_file, filtered) or { return app.server_error(500) }
+		git.write_repos(app.conf.repos_file, filtered) or { return app.server_error(500) }
 	}
 
 	return app.ok('Repo removed successfully.')

src/server/server.v

@@ -4,7 +4,6 @@ import web
 import os
 import log
 import repo
-import env
 import util
 
 const port = 8000
@@ -12,7 +11,7 @@ const port = 8000
 struct App {
 	web.Context
 pub:
-	conf env.ServerConfig [required; web_global]
+	conf Config [required; web_global]
 pub mut:
 	repo repo.RepoGroupManager [required; web_global]
 	// This is used to claim the file lock on the repos file
@@ -20,9 +19,7 @@ pub mut:
 }
 
 // server starts the web server & starts listening for requests
-pub fn server() ? {
+pub fn server(conf Config) ? {
-	conf := env.load<env.ServerConfig>() ?
-
 	// Configure logger
 	log_level := log.level_from_tag(conf.log_level) or {
 		util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')

vieter.toml

@@ -0,0 +1,9 @@
+# This file contains settings used during development
+api_key = "test"
+download_dir = "data/downloads"
+data_dir = "data"
+pkg_dir = "data/pkgs"
+# log_level = "DEBUG"
+repos_file = "data/repos.json"
+
+address = "http://localhost:8000"