Wrote a proper env file system

main^2
Jef Roosens 2022-02-21 20:40:13 +01:00
parent d6e71e9a1c
commit 1d434db166
Signed by: Jef Roosens
GPG Key ID: 955C0660072F691F
8 changed files with 128 additions and 56 deletions

View File

@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* Packages are always rebuilt, even if they haven't changed * Packages are always rebuilt, even if they haven't changed
* Hardcoded planning of builds * Hardcoded planning of builds
* Builds are sequential * Builds are sequential
* Better environment variable support
* Each env var can now be provided from a file by appending it with `_FILE`
& passing the path to the file as value
## Fixed ## Fixed

View File

@ -34,16 +34,21 @@ c:
# Run the server in the default 'data' directory # Run the server in the default 'data' directory
.PHONY: run .PHONY: run
run: vieter run: vieter
API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter server VIETER_API_KEY=test \
VIETER_DOWNLOAD_DIR=data/downloads \
VIETER_REPO_DIR=data/repo \
VIETER_PKG_DIR=data/pkgs \
VIETER_LOG_LEVEL=DEBUG \
./vieter server
.PHONY: run-prod .PHONY: run-prod
run-prod: prod run-prod: prod
API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./pvieter VIETER_API_KEY=test \
VIETER_DOWNLOAD_DIR=data/downloads \
# Same as run, but restart when the source code changes VIETER_REPO_DIR=data/repo \
.PHONY: watch VIETER_PKG_DIR=data/pkgs \
watch: VIETER_LOG_LEVEL=DEBUG \
API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG $(V) watch run vieter ./pvieter server
# =====OTHER===== # =====OTHER=====
.PHONY: lint .PHONY: lint

View File

@ -7,5 +7,5 @@ fn (mut app App) is_authorized() bool {
return false return false
} }
return x_header.trim_space() == app.api_key return x_header.trim_space() == app.conf.api_key
} }

View File

@ -7,16 +7,15 @@ import time
import os import os
import json import json
import git import git
import env
const container_build_dir = '/build' const container_build_dir = '/build'
fn build(key string, repo_dir string) ? { fn build() ? {
server_url := os.getenv_opt('VIETER_ADDRESS') or { conf := env.load<env.BuildConfig>() ?
exit_with_message(1, 'No Vieter server address was provided.')
}
// Read in the repos from a json file // Read in the repos from a json file
filename := os.join_path_single(repo_dir, 'repos.json') filename := os.join_path_single(conf.repo_dir, 'repos.json')
txt := os.read_file(filename) ? txt := os.read_file(filename) ?
repos := json.decode([]git.GitRepo, txt) ? repos := json.decode([]git.GitRepo, txt) ?
@ -48,7 +47,7 @@ fn build(key string, repo_dir string) ? {
uuids << uuid uuids << uuid
commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'" commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'"
commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $server_url/publish; done\'' commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\''
} }
// We convert the list of commands into a base64 string, which then gets // We convert the list of commands into a base64 string, which then gets
@ -57,7 +56,7 @@ fn build(key string, repo_dir string) ? {
c := docker.NewContainer{ c := docker.NewContainer{
image: 'archlinux:latest' image: 'archlinux:latest'
env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$key'] env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key']
entrypoint: ['/bin/sh', '-c'] entrypoint: ['/bin/sh', '-c']
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e']
} }

81
src/env.v 100644
View File

@ -0,0 +1,81 @@
module env
import os
// The prefix that every environment variable should have
const prefix = 'VIETER_'
// The suffix an environment variable in order for it to be loaded from a file
// instead
const file_suffix = '_FILE'
pub struct ServerConfig {
pub:
log_level string [default: WARN]
log_file string [default: 'vieter.log']
pkg_dir string
download_dir string
api_key string
repo_dir string
}
pub struct BuildConfig {
pub:
api_key string
repo_dir string
address string
}
fn get_env_var(field_name string) ?string {
env_var_name := '${prefix}${field_name.to_upper()}'
env_file_name := '${prefix}${field_name.to_upper()}${file_suffix}'
env_var := os.getenv(env_var_name)
env_file := os.getenv(env_file_name)
// If both aren't set, we report them missing
if env_var == '' && env_file == '' {
return error('Either $env_var_name or $env_file_name is required.')
}
// If they're both set, we report a conflict
if env_var != '' && env_file != '' {
return error('Only one of $env_var_name or $env_file_name can be defined.')
}
// If it's the env var itself, we return it
if env_var != '' {
return env_var
}
// Otherwise, we process the file
return os.read_file(env_file) or {
error('Failed to read file defined in $env_file_name: ${err.msg}.')
}
}
// load attempts to create the given type from environment variables.
pub fn load<T>() ?T {
res := T{}
$for field in T.fields {
res.$(field.name) = get_env_var(field.name) or {
// We use the default instead, if it's present
mut default := ''
for attr in field.attrs {
if attr.starts_with('default: ') {
default = attr[9..]
break
}
}
if default == '' {
return err
}
default
}
}
return res
}

View File

@ -1,22 +1,7 @@
module main module main
import web
import os import os
import io import io
import repo
const port = 8000
const buf_size = 1_000_000
struct App {
web.Context
pub:
api_key string [required; web_global]
dl_dir string [required; web_global]
pub mut:
repo repo.Repo [required; web_global]
}
[noreturn] [noreturn]
fn exit_with_message(code int, msg string) { fn exit_with_message(code int, msg string) {
@ -51,18 +36,13 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? {
} }
fn main() { fn main() {
key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') }
repo_dir := os.getenv_opt('REPO_DIR') or {
exit_with_message(1, 'No repo directory was configured.')
}
if os.args.len == 1 { if os.args.len == 1 {
exit_with_message(1, 'No action provided.') exit_with_message(1, 'No action provided.')
} }
match os.args[1] { match os.args[1] {
'server' { server(key, repo_dir) } 'server' { server() ? }
'build' { build(key, repo_dir) ? } 'build' { build() ? }
else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } else { exit_with_message(1, 'Unknown action: ${os.args[1]}') }
} }
} }

View File

@ -58,10 +58,10 @@ fn (mut app App) put_package() web.Result {
if length := app.req.header.get(.content_length) { if length := app.req.header.get(.content_length) {
// Generate a random filename for the temp file // Generate a random filename for the temp file
pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4())
for os.exists(pkg_path) { for os.exists(pkg_path) {
pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4())
} }
app.ldebug("Uploading $length bytes (${pretty_bytes(length.int())}) to '$pkg_path'.") app.ldebug("Uploading $length bytes (${pretty_bytes(length.int())}) to '$pkg_path'.")

View File

@ -4,20 +4,33 @@ import web
import os import os
import log import log
import repo import repo
import env
const port = 8000
const buf_size = 1_000_000
struct App {
web.Context
pub:
conf env.ServerConfig [required: web_global]
pub mut:
repo repo.Repo [required; web_global]
}
fn server() ? {
conf := env.load<env.ServerConfig>() ?
fn server(key string, repo_dir string) {
// Configure logger // Configure logger
log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } log_level := log.level_from_tag(conf.log_level) or {
log_level := log.level_from_tag(log_level_str) or {
exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
} }
log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' }
mut logger := log.Log{ mut logger := log.Log{
level: log_level level: log_level
} }
logger.set_full_logpath(log_file) logger.set_full_logpath(conf.log_file)
logger.log_to_console_too() logger.log_to_console_too()
defer { defer {
@ -26,26 +39,17 @@ fn server(key string, repo_dir string) {
logger.close() logger.close()
} }
// Configure web server
pkg_dir := os.getenv_opt('PKG_DIR') or {
exit_with_message(1, 'No package directory was configured.')
}
dl_dir := os.getenv_opt('DOWNLOAD_DIR') or {
exit_with_message(1, 'No download directory was configured.')
}
// This also creates the directories if needed // This also creates the directories if needed
repo := repo.new(repo_dir, pkg_dir) or { repo := repo.new(conf.repo_dir, conf.pkg_dir) or {
logger.error(err.msg) logger.error(err.msg)
exit(1) exit(1)
} }
os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } os.mkdir_all(conf.download_dir) or { exit_with_message(1, 'Failed to create download directory.') }
web.run(&App{ web.run(&App{
logger: logger logger: logger
api_key: key conf: conf
dl_dir: dl_dir
repo: repo repo: repo
}, port) }, port)
} }