diff --git a/.gitignore b/.gitignore index 3a6b11b..ca2e2f8 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,3 @@ test/ # V compiler directory v/ - -# gdb log file -gdb.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index edae68b..1166377 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,10 +15,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Packages are always rebuilt, even if they haven't changed * Hardcoded planning of builds * Builds are sequential -* Better environment variable support - * Each env var can now be provided from a file by appending it with `_FILE` - & passing the path to the file as value -* API for managing Git repositories to build ## Fixed diff --git a/Dockerfile.ci b/Dockerfile.ci index a062d95..e0dd5da 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -24,10 +24,9 @@ RUN curl --fail \ FROM busybox:1.35.0 ENV PATH=/bin \ - VIETER_REPO_DIR=/data/repo \ - VIETER_PKG_DIR=/data/pkgs \ - VIETER_DOWNLOAD_DIR=/data/downloads \ - VIETER_REPOS_FILE=/data/repos.json + REPO_DIR=/data/repo \ + PKG_DIR=/data/pkgs \ + DOWNLOAD_DIR=/data/downloads COPY --from=builder /app/dumb-init /app/vieter /bin/ diff --git a/Makefile b/Makefile index 062a47f..d69e48e 100644 --- a/Makefile +++ b/Makefile @@ -13,23 +13,10 @@ vieter: $(SOURCES) $(V) -g -o vieter $(SRC_DIR) # Debug build using gcc -# The debug build can't use the boehm garbage collector, as that is -# multi-threaded and causes issues when running vieter inside gdb. .PHONY: debug debug: dvieter dvieter: $(SOURCES) - $(V_PATH) -showcc -keepc -cg -o dvieter $(SRC_DIR) - -# Run the debug build inside gdb -.PHONY: gdb -gdb: dvieter - VIETER_API_KEY=test \ - VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ - VIETER_PKG_DIR=data/pkgs \ - VIETER_LOG_LEVEL=DEBUG \ - VIETER_REPOS_FILE=data/repos.json \ - gdb --args ./dvieter + $(V) -keepc -cg -cc gcc -o dvieter $(SRC_DIR) # Optimised production build .PHONY: prod @@ -47,22 +34,16 @@ c: # Run the server in the default 'data' directory .PHONY: run run: vieter - VIETER_API_KEY=test \ - VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ - VIETER_PKG_DIR=data/pkgs \ - VIETER_LOG_LEVEL=DEBUG \ - VIETER_REPOS_FILE=data/repos.json \ - ./vieter server + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter server .PHONY: run-prod run-prod: prod - VIETER_API_KEY=test \ - VIETER_DOWNLOAD_DIR=data/downloads \ - VIETER_REPO_DIR=data/repo \ - VIETER_PKG_DIR=data/pkgs \ - VIETER_LOG_LEVEL=DEBUG \ - ./pvieter server + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./pvieter + +# Same as run, but restart when the source code changes +.PHONY: watch +watch: + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG $(V) watch run vieter # =====OTHER===== .PHONY: lint diff --git a/src/server/auth.v b/src/auth.v similarity index 73% rename from src/server/auth.v rename to src/auth.v index 8bc9d55..eab63c8 100644 --- a/src/server/auth.v +++ b/src/auth.v @@ -1,4 +1,4 @@ -module server +module main import net.http @@ -7,5 +7,5 @@ fn (mut app App) is_authorized() bool { return false } - return x_header.trim_space() == app.conf.api_key + return x_header.trim_space() == app.api_key } diff --git a/src/build.v b/src/build.v index fc1fe6f..00cf9ed 100644 --- a/src/build.v +++ b/src/build.v @@ -4,22 +4,21 @@ import docker import encoding.base64 import rand import time +import os import json -import server -import env -import net.http +import git const container_build_dir = '/build' -fn build() ? { - conf := env.load() ? +fn build(key string, repo_dir string) ? { + server_url := os.getenv_opt('VIETER_ADDRESS') or { + exit_with_message(1, 'No Vieter server address was provided.') + } - // We get the repos list from the Vieter instance - mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? - req.add_custom_header('X-Api-Key', conf.api_key) ? - - res := req.do() ? - repos := json.decode([]server.GitRepo, res.text) ? + // Read in the repos from a json file + filename := os.join_path_single(repo_dir, 'repos.json') + txt := os.read_file(filename) ? + repos := json.decode([]git.GitRepo, txt) ? mut commands := [ // Update repos & install required packages @@ -49,7 +48,7 @@ fn build() ? { uuids << uuid commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'" - commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\'' + commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $server_url/publish; done\'' } // We convert the list of commands into a base64 string, which then gets @@ -58,7 +57,7 @@ fn build() ? { c := docker.NewContainer{ image: 'archlinux:latest' - env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key'] + env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$key'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] } diff --git a/src/env.v b/src/env.v deleted file mode 100644 index bd544cf..0000000 --- a/src/env.v +++ /dev/null @@ -1,86 +0,0 @@ -module env - -import os - -// The prefix that every environment variable should have -const prefix = 'VIETER_' - -// The suffix an environment variable in order for it to be loaded from a file -// instead -const file_suffix = '_FILE' - -pub struct ServerConfig { -pub: - log_level string [default: WARN] - log_file string [default: 'vieter.log'] - pkg_dir string - download_dir string - api_key string - repo_dir string - repos_file string -} - -pub struct BuildConfig { -pub: - api_key string - address string -} - -fn get_env_var(field_name string) ?string { - env_var_name := '$env.prefix$field_name.to_upper()' - env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix' - env_var := os.getenv(env_var_name) - env_file := os.getenv(env_file_name) - - // If both aren't set, we report them missing - if env_var == '' && env_file == '' { - return error('Either $env_var_name or $env_file_name is required.') - } - - // If they're both set, we report a conflict - if env_var != '' && env_file != '' { - return error('Only one of $env_var_name or $env_file_name can be defined.') - } - - // If it's the env var itself, we return it. - // I'm pretty sure this also prevents variable ending in _FILE (e.g. - // VIETER_LOG_FILE) from being mistakingely read as an _FILE suffixed env - // var. - if env_var != '' { - return env_var - } - - // Otherwise, we process the file - return os.read_file(env_file) or { - error('Failed to read file defined in $env_file_name: ${err.msg}.') - } -} - -// load attempts to create the given type from environment variables. For -// each field, the corresponding env var is its name in uppercase prepended -// with the hardcoded prefix. If this one isn't present, it looks for the env -// var with the file_suffix suffix. -pub fn load() ?T { - res := T{} - - $for field in T.fields { - res.$(field.name) = get_env_var(field.name) or { - // We use the default instead, if it's present - mut default := '' - - for attr in field.attrs { - if attr.starts_with('default: ') { - default = attr[9..] - break - } - } - - if default == '' { - return err - } - - default - } - } - return res -} diff --git a/src/git.v b/src/git.v new file mode 100644 index 0000000..76d80d7 --- /dev/null +++ b/src/git.v @@ -0,0 +1,7 @@ +module git + +pub struct GitRepo { +pub: + url string [required] + branch string [required] +} diff --git a/src/main.v b/src/main.v index 156f0a3..cc66b59 100644 --- a/src/main.v +++ b/src/main.v @@ -1,17 +1,68 @@ module main +import web import os -import server -import util +import io +import repo + +const port = 8000 + +const buf_size = 1_000_000 + +struct App { + web.Context +pub: + api_key string [required; web_global] + dl_dir string [required; web_global] +pub mut: + repo repo.Repo [required; web_global] +} + +[noreturn] +fn exit_with_message(code int, msg string) { + eprintln(msg) + exit(code) +} + +fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { + mut file := os.create(path) ? + defer { + file.close() + } + + mut buf := []byte{len: buf_size} + mut bytes_left := length + + // Repeat as long as the stream still has data + for bytes_left > 0 { + // TODO check if just breaking here is safe + bytes_read := reader.read(mut buf) or { break } + bytes_left -= bytes_read + + mut to_write := bytes_read + + for to_write > 0 { + // TODO don't just loop infinitely here + bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue } + + to_write = to_write - bytes_written + } + } +} fn main() { + key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } + repo_dir := os.getenv_opt('REPO_DIR') or { + exit_with_message(1, 'No repo directory was configured.') + } + if os.args.len == 1 { - util.exit_with_message(1, 'No action provided.') + exit_with_message(1, 'No action provided.') } match os.args[1] { - 'server' { server.server() ? } - 'build' { build() ? } - else { util.exit_with_message(1, 'Unknown action: ${os.args[1]}') } + 'server' { server(key, repo_dir) } + 'build' { build(key, repo_dir) ? } + else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } } } diff --git a/src/repo/repo.v b/src/repo/repo.v index f1419ac..1bf2d0c 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -2,12 +2,17 @@ module repo import os import package -import util + +// Dummy struct to work around the fact that you can only share structs, maps & +// arrays +pub struct Dummy { + x int +} // This struct manages a single repository. pub struct Repo { mut: - mutex shared util.Dummy + mutex shared Dummy pub: // Where to store repository files repo_dir string [required] diff --git a/src/server/routes.v b/src/routes.v similarity index 78% rename from src/server/routes.v rename to src/routes.v index 7a0ee38..8b7ddeb 100644 --- a/src/server/routes.v +++ b/src/routes.v @@ -1,11 +1,29 @@ -module server +module main import web import os import repo import time import rand -import util + +const prefixes = ['B', 'KB', 'MB', 'GB'] + +// pretty_bytes converts a byte count to human-readable version +fn pretty_bytes(bytes int) string { + mut i := 0 + mut n := f32(bytes) + + for n >= 1024 { + i++ + n /= 1024 + } + + return '${n:.2}${prefixes[i]}' +} + +fn is_pkg_name(s string) bool { + return s.contains('.pkg') +} // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. @@ -40,18 +58,18 @@ fn (mut app App) put_package() web.Result { if length := app.req.header.get(.content_length) { // Generate a random filename for the temp file - pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4()) + pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) for os.exists(pkg_path) { - pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4()) + pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) } - app.ldebug("Uploading $length bytes (${util.pretty_bytes(length.int())}) to '$pkg_path'.") + app.ldebug("Uploading $length bytes (${pretty_bytes(length.int())}) to '$pkg_path'.") // This is used to time how long it takes to upload a file mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) - util.reader_to_file(mut app.reader, length.int(), pkg_path) or { + reader_to_file(mut app.reader, length.int(), pkg_path) or { app.lwarn("Failed to upload '$pkg_path'") return app.text('Failed to upload file.') diff --git a/src/server.v b/src/server.v new file mode 100644 index 0000000..e0f22ec --- /dev/null +++ b/src/server.v @@ -0,0 +1,51 @@ +module main + +import web +import os +import log +import repo + +fn server(key string, repo_dir string) { + // Configure logger + log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } + log_level := log.level_from_tag(log_level_str) or { + exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + } + log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } + + mut logger := log.Log{ + level: log_level + } + + logger.set_full_logpath(log_file) + logger.log_to_console_too() + + defer { + logger.info('Flushing log file') + logger.flush() + logger.close() + } + + // Configure web server + pkg_dir := os.getenv_opt('PKG_DIR') or { + exit_with_message(1, 'No package directory was configured.') + } + dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { + exit_with_message(1, 'No download directory was configured.') + } + + // This also creates the directories if needed + repo := repo.new(repo_dir, pkg_dir) or { + logger.error(err.msg) + exit(1) + } + + os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } + + web.run(&App{ + logger: logger + api_key: key + dl_dir: dl_dir + repo: repo + }, port) +} diff --git a/src/server/git.v b/src/server/git.v deleted file mode 100644 index 0147d87..0000000 --- a/src/server/git.v +++ /dev/null @@ -1,129 +0,0 @@ -module server - -import web -import os -import json - -const repos_file = 'repos.json' - -pub struct GitRepo { -pub: - url string [required] - branch string [required] -} - -fn read_repos(path string) ?[]GitRepo { - if !os.exists(path) { - mut f := os.create(path) ? - - defer { - f.close() - } - - f.write_string('[]') ? - - return [] - } - - content := os.read_file(path) ? - res := json.decode([]GitRepo, content) ? - return res -} - -fn write_repos(path string, repos []GitRepo) ? { - mut f := os.create(path) ? - - defer { - f.close() - } - - value := json.encode(repos) - f.write_string(value) ? -} - -['/api/repos'; get] -fn (mut app App) get_repos() web.Result { - if !app.is_authorized() { - return app.text('Unauthorized.') - } - - repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file.') - - return app.server_error(500) - } - } - - return app.json(repos) -} - -['/api/repos'; post] -fn (mut app App) post_repo() web.Result { - if !app.is_authorized() { - return app.text('Unauthorized.') - } - - if !('url' in app.query && 'branch' in app.query) { - return app.server_error(400) - } - - new_repo := GitRepo{ - url: app.query['url'] - branch: app.query['branch'] - } - - mut repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file.') - - return app.server_error(500) - } - } - - // We need to check for duplicates - for r in repos { - if r == new_repo { - return app.text('Duplicate repository.') - } - } - - repos << new_repo - - lock app.git_mutex { - write_repos(app.conf.repos_file, repos) or { return app.server_error(500) } - } - - return app.ok('Repo added successfully.') -} - -['/api/repos'; delete] -fn (mut app App) delete_repo() web.Result { - if !app.is_authorized() { - return app.text('Unauthorized.') - } - - if !('url' in app.query && 'branch' in app.query) { - return app.server_error(400) - } - - repo_to_remove := GitRepo{ - url: app.query['url'] - branch: app.query['branch'] - } - - mut repos := rlock app.git_mutex { - read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file.') - - return app.server_error(500) - } - } - filtered := repos.filter(it != repo_to_remove) - - lock app.git_mutex { - write_repos(app.conf.repos_file, filtered) or { return app.server_error(500) } - } - - return app.ok('Repo removed successfully.') -} diff --git a/src/server/server.v b/src/server/server.v deleted file mode 100644 index 4b31b99..0000000 --- a/src/server/server.v +++ /dev/null @@ -1,59 +0,0 @@ -module server - -import web -import os -import log -import repo -import env -import util - -const port = 8000 - -struct App { - web.Context -pub: - conf env.ServerConfig [required; web_global] -pub mut: - repo repo.Repo [required; web_global] - // This is used to claim the file lock on the repos file - git_mutex shared util.Dummy -} - -// server starts the web server & starts listening for requests -pub fn server() ? { - conf := env.load() ? - - // Configure logger - log_level := log.level_from_tag(conf.log_level) or { - util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') - } - - mut logger := log.Log{ - level: log_level - } - - logger.set_full_logpath(conf.log_file) - logger.log_to_console_too() - - defer { - logger.info('Flushing log file') - logger.flush() - logger.close() - } - - // This also creates the directories if needed - repo := repo.new(conf.repo_dir, conf.pkg_dir) or { - logger.error(err.msg) - exit(1) - } - - os.mkdir_all(conf.download_dir) or { - util.exit_with_message(1, 'Failed to create download directory.') - } - - web.run(&App{ - logger: logger - conf: conf - repo: repo - }, server.port) -} diff --git a/src/util.v b/src/util.v index 49c9d22..f81a256 100644 --- a/src/util.v +++ b/src/util.v @@ -1,55 +1,9 @@ module util import os -import io import crypto.md5 import crypto.sha256 -const reader_buf_size = 1_000_000 - -const prefixes = ['B', 'KB', 'MB', 'GB'] - -// Dummy struct to work around the fact that you can only share structs, maps & -// arrays -pub struct Dummy { - x int -} - -// exit_with_message exits the program with a given status code after having -// first printed a specific message to STDERR -[noreturn] -pub fn exit_with_message(code int, msg string) { - eprintln(msg) - exit(code) -} - -// reader_to_file writes the contents of a BufferedReader to a file -pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { - mut file := os.create(path) ? - defer { - file.close() - } - - mut buf := []byte{len: util.reader_buf_size} - mut bytes_left := length - - // Repeat as long as the stream still has data - for bytes_left > 0 { - // TODO check if just breaking here is safe - bytes_read := reader.read(mut buf) or { break } - bytes_left -= bytes_read - - mut to_write := bytes_read - - for to_write > 0 { - // TODO don't just loop infinitely here - bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue } - - to_write = to_write - bytes_written - } - } -} - // hash_file returns the md5 & sha256 hash of a given file // TODO actually implement sha256 pub fn hash_file(path &string) ?(string, string) { @@ -78,16 +32,3 @@ pub fn hash_file(path &string) ?(string, string) { return md5sum.checksum().hex(), sha256sum.checksum().hex() } - -// pretty_bytes converts a byte count to human-readable version -pub fn pretty_bytes(bytes int) string { - mut i := 0 - mut n := f32(bytes) - - for n >= 1024 { - i++ - n /= 1024 - } - - return '${n:.2}${util.prefixes[i]}' -}