diff --git a/.gitignore b/.gitignore index 6a06eb2..7847b3f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,8 @@ data/ vieter dvieter pvieter -suvieter -afvieter +dvieterctl +vieterctl vieter.c # Ignore testing files @@ -23,6 +23,3 @@ v/ # gdb log file gdb.txt - -# Generated docs -_docs/ diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index e37dc1a..ab3c6ea 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -10,8 +10,6 @@ pipeline: build: image: 'menci/archlinuxarm:base-devel' commands: - # Add the vieter repository so we can use the compiler - - echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf # Update packages - pacman -Syu --noconfirm # Create non-root user to perform build & switch to their home diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index e7341fd..e68c4c9 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -9,21 +9,22 @@ matrix: platform: ${PLATFORM} pipeline: + # The default build isn't needed, as alpine switches to gcc for the compiler anyways debug: image: 'chewingbever/vlang:latest' pull: true + group: 'build' commands: - - make + - make debug when: event: push - branch: - exclude: [main, dev] prod: image: 'chewingbever/vlang:latest' pull: true environment: - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static + group: 'build' commands: - make prod # Make sure the binary is actually statically built diff --git a/.woodpecker/.build_experimental.yml b/.woodpecker/.build_experimental.yml deleted file mode 100644 index 0129d2b..0000000 --- a/.woodpecker/.build_experimental.yml +++ /dev/null @@ -1,29 +0,0 @@ -# These builds are not important for the project, but might be valuable for -# fixing bugs in the V compiler. - -platform: linux/amd64 -branches: - exclude: [master, dev] - -pipeline: - autofree: - image: 'chewingbever/vlang:latest' - pull: true - group: 'build' - commands: - - make autofree - - readelf -d afvieter - - du -h afvieter - when: - event: push - - skip-unused: - image: 'chewingbever/vlang:latest' - pull: true - group: 'build' - commands: - - make skip-unused - - readelf -d suvieter - - du -h suvieter - when: - event: push diff --git a/.woodpecker/.lint.yml b/.woodpecker/.lint.yml index b1c16fd..ce000cd 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/.lint.yml @@ -7,5 +7,7 @@ pipeline: lint: image: 'chewingbever/vlang:latest' pull: true + group: lint commands: - make lint + - make vet diff --git a/Makefile b/Makefile index 041bafc..faae1a6 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,6 @@ V := $(V_PATH) -showcc -gc boehm all: vieter - # =====COMPILATION===== # Regular binary vieter: $(SOURCES) @@ -24,7 +23,7 @@ dvieter: $(SOURCES) # Run the debug build inside gdb .PHONY: gdb gdb: dvieter - gdb --args ./dvieter -f vieter.toml server + gdb --args './dvieter -f vieter.toml server' # Optimised production build .PHONY: prod @@ -34,34 +33,33 @@ pvieter: $(SOURCES) # Only generate C code .PHONY: c -c: $(SOURCES) +c: $(V) -o vieter.c $(SRC_DIR) - # =====EXECUTION===== # Run the server in the default 'data' directory .PHONY: run run: vieter - ./vieter -f vieter.toml server + ./vieter -f vieter.toml server .PHONY: run-prod run-prod: prod ./pvieter -f vieter.toml server - # =====OTHER===== .PHONY: lint lint: $(V) fmt -verify $(SRC_DIR) - $(V) vet -W $(SRC_DIR) - $(V_PATH) missdoc -p $(SRC_DIR) - @ [ $$($(V_PATH) missdoc -p $(SRC_DIR) | wc -l) = 0 ] # Format the V codebase .PHONY: fmt fmt: $(V) fmt -w $(SRC_DIR) +.PHONY: vet +vet: + $(V) vet -W $(SRC_DIR) + .PHONY: test test: $(V) test $(SRC_DIR) @@ -73,23 +71,5 @@ v/v: git clone --single-branch https://git.rustybever.be/Chewing_Bever/v v make -C v -.PHONY: clean clean: - rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' - -.PHONY: api-docs -api-docs: - rm -rf '$(SRC_DIR)/_docs' - cd '$(SRC_DIR)' && v doc -all -f html -m -readme . - - -# =====EXPERIMENTAL===== -.PHONY: autofree -autofree: afvieter -afvieter: $(SOURCES) - $(V_PATH) -showcc -autofree -o afvieter $(SRC_DIR) - -.PHONY: skip-unused -skip-unused: suvieter -suvieter: $(SOURCES) - $(V_PATH) -showcc -skip-unused -o suvieter $(SRC_DIR) + rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst diff --git a/PKGBUILD b/PKGBUILD index 3f8c480..0c558b4 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -2,10 +2,10 @@ pkgbase='vieter' pkgname='vieter' -pkgver=0.2.0.r25.g20112b8 +pkgver=0.2.0.r24.g9a56bd0 pkgrel=1 depends=('glibc' 'openssl' 'libarchive' 'gc') -makedepends=('git' 'gcc' 'vieter-v') +makedepends=('git' 'gcc') arch=('x86_64' 'aarch64' 'armv7') url='https://git.rustybever.be/Chewing_Bever/vieter' license=('AGPL3') @@ -20,7 +20,10 @@ pkgver() { build() { cd "$pkgname" - make prod + # Build the compiler + CFLAGS= make v + + V_PATH=v/v make prod } package() { diff --git a/README.md b/README.md index 08f1e75..96b104d 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,7 @@ ## Documentation -I host documentation for Vieter over at https://rustybever.be/docs/vieter/. API -documentation for the current codebase can be found at -https://rustybever.be/api-docs/vieter/. +I host documentation for Vieter over at https://rustybever.be/docs/vieter/. ## Overview diff --git a/src/build/build.v b/src/build/build.v index bc604fa..942ce8a 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -10,12 +10,7 @@ const container_build_dir = '/build' const build_image_repo = 'vieter-build' -// create_build_image creates a builder image given some base image which can -// then be used to build & package Arch images. It mostly just updates the -// system, install some necessary packages & creates a non-root user to run -// makepkg with. The base image should be some Linux distribution that uses -// Pacman as its package manager. -pub fn create_build_image(base_image string) ?string { +fn create_build_image(base_image string) ?string { commands := [ // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' @@ -58,13 +53,12 @@ pub fn create_build_image(base_image string) ?string { break } - time.sleep(1 * time.second) + // Wait for 5 seconds + time.sleep(5000000000) } // Finally, we create the image from the container // As the tag, we use the epoch value - // TODO also add the base image's name into the image name to prevent - // conflicts. tag := time.sys_mono_now().str() image := docker.create_image_from_container(id, 'vieter-build', tag) ? docker.remove_container(id) ? @@ -72,55 +66,6 @@ pub fn create_build_image(base_image string) ?string { return image.id } -// build_repo builds, packages & publishes a given Arch package based on the -// provided GitRepo. The base image ID should be of an image previously created -// by create_build_image. -pub fn build_repo(address string, api_key string, base_image_id string, repo &git.GitRepo) ? { - build_arch := os.uname().machine - - // TODO what to do with PKGBUILDs that build multiple packages? - commands := [ - 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', - 'cd repo', - 'makepkg --nobuild --nodeps', - 'source PKGBUILD', - // The build container checks whether the package is already - // present on the server - 'curl --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', - 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done', - ] - - // We convert the list of commands into a base64 string, which then gets - // passed to the container as an env var - cmds_str := base64.encode_str(commands.join('\n')) - - c := docker.NewContainer{ - image: '$base_image_id' - env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$api_key'] - entrypoint: ['/bin/sh', '-c'] - cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e'] - work_dir: '/build' - user: 'builder:builder' - } - - id := docker.create_container(c) ? - docker.start_container(id) ? - - // This loop waits until the container has stopped, so we can remove it after - for { - data := docker.inspect_container(id) ? - - if !data.state.running { - break - } - - time.sleep(1 * time.second) - } - - docker.remove_container(id) ? -} - -// build builds every Git repo in the server's list. fn build(conf Config) ? { build_arch := os.uname().machine @@ -140,7 +85,47 @@ fn build(conf Config) ? { image_id := create_build_image(conf.base_image) ? for repo in filtered_repos { - build_repo(conf.address, conf.api_key, image_id, repo) ? + // TODO what to do with PKGBUILDs that build multiple packages? + commands := [ + 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', + 'cd repo', + 'makepkg --nobuild --nodeps', + 'source PKGBUILD', + // The build container checks whether the package is already + // present on the server + 'curl --head --fail $conf.address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/$repo.repo/publish; done', + ] + + // We convert the list of commands into a base64 string, which then gets + // passed to the container as an env var + cmds_str := base64.encode_str(commands.join('\n')) + + c := docker.NewContainer{ + image: '$image_id' + env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key'] + entrypoint: ['/bin/sh', '-c'] + cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e'] + work_dir: '/build' + user: 'builder:builder' + } + + id := docker.create_container(c) ? + docker.start_container(id) ? + + // This loop waits until the container has stopped, so we can remove it after + for { + data := docker.inspect_container(id) ? + + if !data.state.running { + break + } + + // Wait for 5 seconds + time.sleep(5000000000) + } + + docker.remove_container(id) ? } // Finally, we remove the builder image diff --git a/src/cron/cli.v b/src/cron/cli.v index 24cbe2c..8e6b0f1 100644 --- a/src/cron/cli.v +++ b/src/cron/cli.v @@ -5,16 +5,11 @@ import env struct Config { pub: - log_level string = 'WARN' - log_file string = 'vieter.log' - api_key string - address string - base_image string = 'archlinux:base-devel' - max_concurrent_builds int = 1 - api_update_frequency int = 15 - image_rebuild_frequency int = 1440 - // Replicates the behavior of the original cron system - global_schedule string = '0 3' + log_level string = 'WARN' + log_file string = 'vieter.log' + api_key string + address string + base_image string = 'archlinux:base-devel' } // cmd returns the cli module that handles the cron daemon. diff --git a/src/cron/cron.v b/src/cron/cron.v index 49a379e..3ba9d0f 100644 --- a/src/cron/cron.v +++ b/src/cron/cron.v @@ -1,29 +1,18 @@ module cron -import log -import cron.daemon -import cron.expression +import git +import time + +struct ScheduledBuild { + repo git.GitRepo + timestamp time.Time +} + +fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool { + return r1.timestamp < r2.timestamp +} // cron starts a cron daemon & starts periodically scheduling builds. pub fn cron(conf Config) ? { - // Configure logger - log_level := log.level_from_tag(conf.log_level) or { - return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') - } - - mut logger := log.Log{ - level: log_level - } - - logger.set_full_logpath(conf.log_file) - logger.log_to_console_too() - - ce := expression.parse_expression(conf.global_schedule) or { - return error('Error while parsing global cron expression: $err.msg()') - } - - mut d := daemon.init_daemon(logger, conf.address, conf.api_key, conf.base_image, ce, - conf.max_concurrent_builds, conf.api_update_frequency, conf.image_rebuild_frequency) ? - - d.run() ? + println('WIP') } diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v deleted file mode 100644 index ec8be4d..0000000 --- a/src/cron/daemon/build.v +++ /dev/null @@ -1,83 +0,0 @@ -module daemon - -import time -import sync.stdatomic -import build - -const build_empty = 0 - -const build_running = 1 - -const build_done = 2 - -// clean_finished_builds removes finished builds from the build slots & returns -// them. -fn (mut d Daemon) clean_finished_builds() ?[]ScheduledBuild { - mut out := []ScheduledBuild{} - - for i in 0 .. d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == daemon.build_done { - stdatomic.store_u64(&d.atomics[i], daemon.build_empty) - out << d.builds[i] - } - } - - return out -} - -// update_builds starts as many builds as possible. -fn (mut d Daemon) start_new_builds() ? { - now := time.now() - - for d.queue.len() > 0 { - if d.queue.peek() ?.timestamp < now { - sb := d.queue.pop() ? - - // If this build couldn't be scheduled, no more will be possible. - if !d.start_build(sb) { - d.queue.insert(sb) - break - } - } else { - break - } - } -} - -// start_build starts a build for the given ScheduledBuild object. -fn (mut d Daemon) start_build(sb ScheduledBuild) bool { - for i in 0 .. d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == daemon.build_empty { - stdatomic.store_u64(&d.atomics[i], daemon.build_running) - d.builds[i] = sb - - go d.run_build(i, sb) - - return true - } - } - - return false -} - -// run_build actually starts the build process for a given repo. -fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) ? { - d.linfo('started build: $sb.repo.url $sb.repo.branch') - - build.build_repo(d.address, d.api_key, d.builder_images.last(), &sb.repo) ? - - stdatomic.store_u64(&d.atomics[build_index], daemon.build_done) -} - -// current_build_count returns how many builds are currently running. -fn (mut d Daemon) current_build_count() int { - mut res := 0 - - for i in 0 .. d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == daemon.build_running { - res += 1 - } - } - - return res -} diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v deleted file mode 100644 index 729e94b..0000000 --- a/src/cron/daemon/daemon.v +++ /dev/null @@ -1,223 +0,0 @@ -module daemon - -import git -import time -import log -import datatypes { MinHeap } -import cron.expression { CronExpression, parse_expression } -import math -import build -import docker - -struct ScheduledBuild { -pub: - repo_id string - repo git.GitRepo - timestamp time.Time -} - -// Overloaded operator for comparing ScheduledBuild objects -fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool { - return r1.timestamp < r2.timestamp -} - -pub struct Daemon { -mut: - address string - api_key string - base_image string - builder_images []string - global_schedule CronExpression - api_update_frequency int - image_rebuild_frequency int - // Repos currently loaded from API. - repos_map map[string]git.GitRepo - // At what point to update the list of repositories. - api_update_timestamp time.Time - image_build_timestamp time.Time - queue MinHeap - // Which builds are currently running - builds []ScheduledBuild - // Atomic variables used to detect when a build has finished; length is the - // same as builds - atomics []u64 - logger shared log.Log -} - -// init_daemon initializes a new Daemon object. It renews the repositories & -// populates the build queue for the first time. -pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon { - mut d := Daemon{ - address: address - api_key: api_key - base_image: base_image - global_schedule: global_schedule - api_update_frequency: api_update_frequency - image_rebuild_frequency: image_rebuild_frequency - atomics: []u64{len: max_concurrent_builds} - builds: []ScheduledBuild{len: max_concurrent_builds} - logger: logger - } - - // Initialize the repos & queue - d.renew_repos() ? - d.renew_queue() ? - d.rebuild_base_image() ? - - return d -} - -// run starts the actual daemon process. It runs builds when possible & -// periodically refreshes the list of repositories to ensure we stay in sync. -pub fn (mut d Daemon) run() ? { - for { - finished_builds := d.clean_finished_builds() ? - - // Update the API's contents if needed & renew the queue - if time.now() >= d.api_update_timestamp { - d.renew_repos() ? - d.renew_queue() ? - } - // The finished builds should only be rescheduled if the API contents - // haven't been renewed. - else { - for sb in finished_builds { - d.schedule_build(sb.repo_id, sb.repo) ? - } - } - - // TODO remove old builder images. - // This issue is less trivial than it sounds, because a build could - // still be running when the image has to be rebuilt. That would - // prevent the image from being removed. Therefore, we will need to - // keep track of a list or something & remove an image once we have - // made sure it isn't being used anymore. - if time.now() >= d.image_build_timestamp { - d.rebuild_base_image() ? - // In theory, executing this function here allows an old builder - // image to exist for at most image_rebuild_frequency minutes. - d.clean_old_base_images() - } - - // Schedules new builds when possible - d.start_new_builds() ? - - // If there are builds currently running, the daemon should refresh - // every second to clean up any finished builds & start new ones. - mut delay := time.Duration(1 * time.second) - - // Sleep either until we have to refresh the repos or when the next - // build has to start, with a minimum of 1 second. - if d.current_build_count() == 0 { - now := time.now() - delay = d.api_update_timestamp - now - - if d.queue.len() > 0 { - time_until_next_job := d.queue.peek() ?.timestamp - now - - delay = math.min(delay, time_until_next_job) - } - } - - // We sleep for at least one second. This is to prevent the program - // from looping agressively when a cronjob can be scheduled, but - // there's no spots free for it to be started. - delay = math.max(delay, 1 * time.second) - - d.ldebug('Sleeping for ${delay}...') - - time.sleep(delay) - } -} - -// schedule_build adds the next occurence of the given repo build to the queue. -fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) ? { - ce := if repo.schedule != '' { - parse_expression(repo.schedule) or { - // TODO This shouldn't return an error if the expression is empty. - d.lerror("Error while parsing cron expression '$repo.schedule' ($repo_id): $err.msg()") - - d.global_schedule - } - } else { - d.global_schedule - } - - // A repo that can't be scheduled will just be skipped for now - timestamp := ce.next_from_now() ? - - d.queue.insert(ScheduledBuild{ - repo_id: repo_id - repo: repo - timestamp: timestamp - }) -} - -// renew_repos requests the newest list of Git repos from the server & replaces -// the old one. -fn (mut d Daemon) renew_repos() ? { - d.linfo('Renewing repos...') - mut new_repos := git.get_repos(d.address, d.api_key) ? - - d.repos_map = new_repos.move() - - d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency) -} - -// renew_queue replaces the old queue with a new one that reflects the newest -// values in repos_map. -fn (mut d Daemon) renew_queue() ? { - d.linfo('Renewing queue...') - mut new_queue := MinHeap{} - - // Move any jobs that should have already started from the old queue onto - // the new one - now := time.now() - - // For some reason, using - // ```v - // for d.queue.len() > 0 && d.queue.peek() ?.timestamp < now { - //``` - // here causes the function to prematurely just exit, without any errors or anything, very weird - // https://github.com/vlang/v/issues/14042 - for d.queue.len() > 0 { - if d.queue.peek() ?.timestamp < now { - new_queue.insert(d.queue.pop() ?) - } else { - break - } - } - - d.queue = new_queue - - // For each repository in repos_map, parse their cron expression (or use - // the default one if not present) & add them to the queue - for id, repo in d.repos_map { - d.schedule_build(id, repo) ? - } -} - -// rebuild_base_image recreates the builder image. -fn (mut d Daemon) rebuild_base_image() ? { - d.linfo('Rebuilding builder image....') - - d.builder_images << build.create_build_image(d.base_image) ? - d.image_build_timestamp = time.now().add_seconds(60 * d.image_rebuild_frequency) -} - -// clean_old_base_images tries to remove any old but still present builder -// images. -fn (mut d Daemon) clean_old_base_images() { - mut i := 0 - - for i < d.builder_images.len - 1 { - // For each builder image, we try to remove it by calling the Docker - // API. If the function returns an error or false, that means the image - // wasn't deleted. Therefore, we move the index over. If the function - // returns true, the array's length has decreased by one so we don't - // move the index. - if !docker.remove_image(d.builder_images[i]) or { false } { - i += 1 - } - } -} diff --git a/src/cron/daemon/log.v b/src/cron/daemon/log.v deleted file mode 100644 index 003898b..0000000 --- a/src/cron/daemon/log.v +++ /dev/null @@ -1,35 +0,0 @@ -module daemon - -import log - -// log reate a log message with the given level -pub fn (mut d Daemon) log(msg &string, level log.Level) { - lock d.logger { - d.logger.send_output(msg, level) - } -} - -// lfatal create a log message with the fatal level -pub fn (mut d Daemon) lfatal(msg &string) { - d.log(msg, log.Level.fatal) -} - -// lerror create a log message with the error level -pub fn (mut d Daemon) lerror(msg &string) { - d.log(msg, log.Level.error) -} - -// lwarn create a log message with the warn level -pub fn (mut d Daemon) lwarn(msg &string) { - d.log(msg, log.Level.warn) -} - -// linfo create a log message with the info level -pub fn (mut d Daemon) linfo(msg &string) { - d.log(msg, log.Level.info) -} - -// ldebug create a log message with the debug level -pub fn (mut d Daemon) ldebug(msg &string) { - d.log(msg, log.Level.debug) -} diff --git a/src/cron/expression/expression.v b/src/cron/expression.v similarity index 93% rename from src/cron/expression/expression.v rename to src/cron/expression.v index 124337f..0a35541 100644 --- a/src/cron/expression/expression.v +++ b/src/cron/expression.v @@ -1,8 +1,8 @@ -module expression +module cron import time -pub struct CronExpression { +struct CronExpression { minutes []int hours []int days []int @@ -65,7 +65,6 @@ pub fn (ce &CronExpression) next(ref time.Time) ?time.Time { if minute_index == ce.minutes.len && hour_index < ce.hours.len { hour_index += 1 } - if hour_index == ce.hours.len && day_index < ce.days.len { day_index += 1 } @@ -115,9 +114,7 @@ pub fn (ce &CronExpression) next(ref time.Time) ?time.Time { }) } -// next_from_now returns the result of ce.next(ref) where ref is the result of -// time.now(). -pub fn (ce &CronExpression) next_from_now() ?time.Time { +fn (ce &CronExpression) next_from_now() ?time.Time { return ce.next(time.now()) } @@ -198,8 +195,6 @@ fn parse_range(s string, min int, max int, mut bitv []bool) ? { } } -// bitv_to_ints converts a bit vector into an array containing the -// corresponding values. fn bitv_to_ints(bitv []bool, min int) []int { mut out := []int{} @@ -212,8 +207,6 @@ fn bitv_to_ints(bitv []bool, min int) []int { return out } -// parse_part parses a given part of a cron expression & returns the -// corresponding array of ints. fn parse_part(s string, min int, max int) ?[]int { mut bitv := []bool{len: max - min + 1, init: false} @@ -226,7 +219,7 @@ fn parse_part(s string, min int, max int) ?[]int { // parse_expression parses an entire cron expression string into a // CronExpression object, if possible. -pub fn parse_expression(exp string) ?CronExpression { +fn parse_expression(exp string) ?CronExpression { // The filter allows for multiple spaces between parts mut parts := exp.split(' ').filter(it != '') @@ -248,7 +241,7 @@ pub fn parse_expression(exp string) ?CronExpression { // This for loop allows us to more clearly propagate the error to the user. for i, min in mins { part_results << parse_part(parts[i], min, maxs[i]) or { - return error('An error occurred with part $i: $err.msg()') + return error('An error occurred with part $i: $err.msg') } } diff --git a/src/cron/expression/expression_parse_test.v b/src/cron/expression_parse_test.v similarity index 99% rename from src/cron/expression/expression_parse_test.v rename to src/cron/expression_parse_test.v index 18531c0..8f3ac38 100644 --- a/src/cron/expression/expression_parse_test.v +++ b/src/cron/expression_parse_test.v @@ -1,4 +1,4 @@ -module expression +module cron // parse_range_error returns the returned error message. If the result is '', // that means the function didn't error. diff --git a/src/cron/expression/expression_test.v b/src/cron/expression_test.v similarity index 97% rename from src/cron/expression/expression_test.v rename to src/cron/expression_test.v index ef0283a..0be9a64 100644 --- a/src/cron/expression/expression_test.v +++ b/src/cron/expression_test.v @@ -1,4 +1,4 @@ -module expression +module cron import time { parse } diff --git a/src/docker/docker.v b/src/docker/docker.v index 5deef83..a6f7640 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -9,8 +9,6 @@ const socket = '/var/run/docker.sock' const buf_len = 1024 -// send writes a request to the Docker socket, waits for a response & returns -// it. fn send(req &string) ?http.Response { // Open a connection to the socket mut s := unix.connect_stream(docker.socket) or { @@ -30,8 +28,8 @@ fn send(req &string) ?http.Response { s.wait_for_write() ? mut c := 0 - mut buf := []u8{len: docker.buf_len} - mut res := []u8{} + mut buf := []byte{len: docker.buf_len} + mut res := []byte{} for { c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } @@ -54,7 +52,7 @@ fn send(req &string) ?http.Response { // We loop until we've encountered the end of the chunked response // A chunked HTTP response always ends with '0\r\n\r\n'. - for res.len < 5 || res#[-5..] != [u8(`0`), `\r`, `\n`, `\r`, `\n`] { + for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] { // Wait for the server to respond s.wait_for_write() ? @@ -74,14 +72,12 @@ fn send(req &string) ?http.Response { return http.parse_response(res.bytestr()) } -// request_with_body sends a request to the Docker socket with the given body. fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response { req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' return send(req) } -// request sends a request to the Docker socket with an empty body. fn request(method string, url urllib.URL) ?http.Response { req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' diff --git a/src/env/env.v b/src/env/env.v index b2b5f44..cbde67e 100644 --- a/src/env/env.v +++ b/src/env/env.v @@ -36,7 +36,7 @@ fn get_env_var(field_name string) ?string { // Otherwise, we process the file return os.read_file(env_file) or { - error('Failed to read file defined in $env_file_name: ${err.msg()}.') + error('Failed to read file defined in $env_file_name: ${err.msg}.') } } @@ -55,41 +55,27 @@ pub fn load(path string) ?T { $for field in T.fields { s := doc.value(field.name) - if s !is toml.Null { - $if field.typ is string { - res.$(field.name) = s.string() - } $else $if field.typ is int { - res.$(field.name) = s.int() - } + // We currently only support strings + if s.type_name() == 'string' { + res.$(field.name) = s.string() } } } $for field in T.fields { - env_value := get_env_var(field.name) ? - - // The value of an env var will always take precedence over the toml - // file. - if env_value != '' { - $if field.typ is string { - res.$(field.name) = env_value - } $else $if field.typ is int { - res.$(field.name) = env_value.int() - } - } - - // Now, we check whether a value is present. If there isn't, that means - // it isn't in the config file, nor is there a default or an env var. - mut has_value := false - $if field.typ is string { - has_value = res.$(field.name) != '' - } $else $if field.typ is int { - has_value = res.$(field.name) != 0 - } + env_value := get_env_var(field.name) ? - if !has_value { - return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.") + // The value of the env var will always be chosen over the config + // file + if env_value != '' { + res.$(field.name) = env_value + } + // If there's no value from the toml file either, we try to find a + // default value + else if res.$(field.name) == '' { + return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.") + } } } return res diff --git a/src/git/cli.v b/src/git/cli.v index 53527d5..463f1ba 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -96,8 +96,6 @@ pub fn cmd() cli.Command { } } -// get_repo_id_by_prefix tries to find the repo with the given prefix in its -// ID. If multiple or none are found, an error is raised. fn get_repo_id_by_prefix(conf Config, id_prefix string) ?string { repos := get_repos(conf.address, conf.api_key) ? @@ -120,7 +118,6 @@ fn get_repo_id_by_prefix(conf Config, id_prefix string) ?string { return res[0] } -// list prints out a list of all repositories. fn list(conf Config) ? { repos := get_repos(conf.address, conf.api_key) ? @@ -129,14 +126,12 @@ fn list(conf Config) ? { } } -// add adds a new repository to the server's list. fn add(conf Config, url string, branch string, repo string, arch []string) ? { res := add_repo(conf.address, conf.api_key, url, branch, repo, arch) ? println(res.message) } -// remove removes a repository from the server's list. fn remove(conf Config, id_prefix string) ? { id := get_repo_id_by_prefix(conf, id_prefix) ? res := remove_repo(conf.address, conf.api_key, id) ? @@ -144,7 +139,6 @@ fn remove(conf Config, id_prefix string) ? { println(res.message) } -// patch patches a given repository with the provided params. fn patch(conf Config, id_prefix string, params map[string]string) ? { id := get_repo_id_by_prefix(conf, id_prefix) ? res := patch_repo(conf.address, conf.api_key, id, params) ? diff --git a/src/git/client.v b/src/git/client.v index a43c9ca..e4a39ac 100644 --- a/src/git/client.v +++ b/src/git/client.v @@ -4,9 +4,6 @@ import json import response { Response } import net.http -// send_request is a convenience method for sending requests to the repos -// API. It mostly does string manipulation to create a query string containing -// the provided params. fn send_request(method http.Method, address string, url string, api_key string, params map[string]string) ?Response { mut full_url := '$address$url' diff --git a/src/git/git.v b/src/git/git.v index 2023f34..eaec895 100644 --- a/src/git/git.v +++ b/src/git/git.v @@ -14,8 +14,6 @@ pub mut: arch []string // Which repo the builder should publish packages to repo string - // Cron schedule describing how frequently to build the repo. - schedule string [optional] } // patch_from_params patches a GitRepo from a map[string]string, usually @@ -74,7 +72,7 @@ pub fn repo_from_params(params map[string]string) ?GitRepo { // If we're creating a new GitRepo, we want all fields to be present before // "patching". $for field in GitRepo.fields { - if field.name !in params && !field.attrs.contains('optional') { + if field.name !in params { return error('Missing parameter: ${field.name}.') } } diff --git a/src/package/package.v b/src/package/package.v index a1042b5..a6be636 100644 --- a/src/package/package.v +++ b/src/package/package.v @@ -175,7 +175,6 @@ pub fn read_pkg_archive(pkg_path string) ?Pkg { } } -// format_entry returns a string properly formatted to be added to a desc file. fn format_entry(key string, value string) string { return '\n%$key%\n$value\n' } diff --git a/src/repo/repo.v b/src/repo/repo.v index e27e232..f439f58 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -30,11 +30,11 @@ pub: // new creates a new RepoGroupManager & creates the directories as needed pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupManager { if !os.is_dir(repos_dir) { - os.mkdir_all(repos_dir) or { return error('Failed to create repos directory: $err.msg()') } + os.mkdir_all(repos_dir) or { return error('Failed to create repos directory: $err.msg') } } if !os.is_dir(pkg_dir) { - os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg()') } + os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg') } } return RepoGroupManager{ @@ -50,7 +50,7 @@ pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupMana // the right subdirectories in r.pkg_dir if it was successfully added. pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult { pkg := package.read_pkg_archive(pkg_path) or { - return error('Failed to read package file: $err.msg()') + return error('Failed to read package file: $err.msg') } added := r.add_pkg_in_repo(repo, pkg) ? diff --git a/src/repo/sync.v b/src/repo/sync.v index 9c5e7ed..e2b7aac 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -2,8 +2,6 @@ module repo import os -// archive_add_entry writes a file to an archive, given its path & inner path -// inside the archive. fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &string, inner_path &string) { st := C.stat{} @@ -21,7 +19,7 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri } // Write the file to the archive - buf := [8192]u8{} + buf := [8192]byte{} mut len := C.read(fd, &buf, sizeof(buf)) for len > 0 { @@ -31,7 +29,7 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri } } -// sync regenerates the repository archive files. +// Re-generate the repo archive files fn (r &RepoGroupManager) sync(repo string, arch string) ? { subrepo_path := os.join_path(r.repos_dir, repo, arch) diff --git a/src/server/auth.v b/src/server/auth.v index 7c8a676..8bc9d55 100644 --- a/src/server/auth.v +++ b/src/server/auth.v @@ -2,7 +2,6 @@ module server import net.http -// is_authorized checks whether the provided API key is correct. fn (mut app App) is_authorized() bool { x_header := app.req.header.get_custom('X-Api-Key', http.HeaderQueryConfig{ exact: true }) or { return false diff --git a/src/server/git.v b/src/server/git.v index 0cba17c..2a682d8 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -8,7 +8,6 @@ import response { new_data_response, new_response } const repos_file = 'repos.json' -// get_repos returns the current list of repos. ['/api/repos'; get] fn (mut app App) get_repos() web.Result { if !app.is_authorized() { @@ -17,7 +16,7 @@ fn (mut app App) get_repos() web.Result { repos := rlock app.git_mutex { git.read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file: $err.msg()') + app.lerror('Failed to read repos file: $err.msg') return app.status(http.Status.internal_server_error) } @@ -26,7 +25,6 @@ fn (mut app App) get_repos() web.Result { return app.json(http.Status.ok, new_data_response(repos)) } -// get_single_repo returns the information for a single repo. ['/api/repos/:id'; get] fn (mut app App) get_single_repo(id string) web.Result { if !app.is_authorized() { @@ -50,7 +48,6 @@ fn (mut app App) get_single_repo(id string) web.Result { return app.json(http.Status.ok, new_data_response(repo)) } -// post_repo creates a new repo from the provided query string. ['/api/repos'; post] fn (mut app App) post_repo() web.Result { if !app.is_authorized() { @@ -58,7 +55,7 @@ fn (mut app App) post_repo() web.Result { } new_repo := git.repo_from_params(app.query) or { - return app.json(http.Status.bad_request, new_response(err.msg())) + return app.json(http.Status.bad_request, new_response(err.msg)) } id := rand.uuid_v4() @@ -89,7 +86,6 @@ fn (mut app App) post_repo() web.Result { return app.json(http.Status.ok, new_response('Repo added successfully.')) } -// delete_repo removes a given repo from the server's list. ['/api/repos/:id'; delete] fn (mut app App) delete_repo(id string) web.Result { if !app.is_authorized() { @@ -117,7 +113,6 @@ fn (mut app App) delete_repo(id string) web.Result { return app.json(http.Status.ok, new_response('Repo removed successfully.')) } -// patch_repo updates a repo's data with the given query params. ['/api/repos/:id'; patch] fn (mut app App) patch_repo(id string) web.Result { if !app.is_authorized() { diff --git a/src/server/routes.v b/src/server/routes.v index f27afb4..138f253 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -16,9 +16,6 @@ pub fn (mut app App) healthcheck() web.Result { return app.json(http.Status.ok, new_response('Healthy.')) } -// get_repo_file handles all Pacman-related routes. It returns both the -// repository's archives, but also package archives or the contents of a -// package's desc file. ['/:repo/:arch/:filename'; get; head] fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result { mut full_path := '' @@ -57,7 +54,6 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re return app.file(full_path) } -// put_package handles publishing a package to a repository. ['/:repo/publish'; post] fn (mut app App) put_package(repo string) web.Result { if !app.is_authorized() { @@ -91,15 +87,15 @@ fn (mut app App) put_package(repo string) web.Result { } res := app.repo.add_pkg_from_path(repo, pkg_path) or { - app.lerror('Error while adding package: $err.msg()') + app.lerror('Error while adding package: $err.msg') - os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg()") } + os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } return app.json(http.Status.internal_server_error, new_response('Failed to add package.')) } if !res.added { - os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg()") } + os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo'.") diff --git a/src/server/server.v b/src/server/server.v index c4317c5..5bf9a87 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -45,7 +45,7 @@ pub fn server(conf Config) ? { // This also creates the directories if needed repo := repo.new(conf.repos_dir, conf.pkg_dir, conf.default_arch) or { - logger.error(err.msg()) + logger.error(err.msg) exit(1) } diff --git a/src/util/util.v b/src/util/util.v index c1af30e..228f584 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -30,7 +30,7 @@ pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { file.close() } - mut buf := []u8{len: util.reader_buf_size} + mut buf := []byte{len: util.reader_buf_size} mut bytes_left := length // Repeat as long as the stream still has data @@ -60,7 +60,7 @@ pub fn hash_file(path &string) ?(string, string) { mut sha256sum := sha256.new() buf_size := int(1_000_000) - mut buf := []u8{len: buf_size} + mut buf := []byte{len: buf_size} mut bytes_left := os.file_size(path) for bytes_left > 0 { diff --git a/src/v.mod b/src/v.mod deleted file mode 100644 index e69de29..0000000 diff --git a/src/web/parse.v b/src/web/parse.v index a095f0c..2eeef5e 100644 --- a/src/web/parse.v +++ b/src/web/parse.v @@ -47,7 +47,6 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) { return methods, path.to_lower() } -// Extracts query parameters from a URL. fn parse_query_from_url(url urllib.URL) map[string]string { mut query := map[string]string{} for v in url.query().data { @@ -56,7 +55,6 @@ fn parse_query_from_url(url urllib.URL) map[string]string { return query } -// Extract form data from an HTTP request. fn parse_form_from_request(request http.Request) ?(map[string]string, map[string][]http.FileData) { mut form := map[string]string{} mut files := map[string][]http.FileData{} diff --git a/src/web/web.v b/src/web/web.v index 3e7b047..000c6a6 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -249,7 +249,7 @@ pub fn (mut ctx Context) file(f_path string) Result { // ext := os.file_ext(f_path) // data := os.read_file(f_path) or { - // eprint(err.msg()) + // eprint(err.msg) // ctx.server_error(500) // return Result{} // } @@ -267,7 +267,7 @@ pub fn (mut ctx Context) file(f_path string) Result { file_size := os.file_size(f_path) file := os.open(f_path) or { - eprintln(err.msg()) + eprintln(err.msg) ctx.server_error(500) return Result{} } @@ -285,7 +285,7 @@ pub fn (mut ctx Context) file(f_path string) Result { resp.set_status(ctx.status) send_string(mut ctx.conn, resp.bytestr()) or { return Result{} } - mut buf := []u8{len: 1_000_000} + mut buf := []byte{len: 1_000_000} mut bytes_left := file_size // Repeat as long as the stream still has data @@ -361,7 +361,7 @@ interface DbInterface { // run runs the app [manualfree] pub fn run(global_app &T, port int) { - mut l := net.listen_tcp(.ip6, ':$port') or { panic('failed to listen $err.code() $err') } + mut l := net.listen_tcp(.ip6, ':$port') or { panic('failed to listen $err.code $err') } // Parsing methods attributes mut routes := map[string]Route{} @@ -393,7 +393,7 @@ pub fn run(global_app &T, port int) { request_app.Context = global_app.Context // copy the context ref that contains static files map etc mut conn := l.accept() or { // failures should not panic - eprintln('accept() failed with error: $err.msg()') + eprintln('accept() failed with error: $err.msg') continue } go handle_conn(mut conn, mut request_app, routes) diff --git a/vieter.toml b/vieter.toml index fc86d77..8e0447b 100644 --- a/vieter.toml +++ b/vieter.toml @@ -8,9 +8,3 @@ repos_file = "data/repos.json" default_arch = "x86_64" address = "http://localhost:8000" - -global_schedule = '* *' -api_update_frequency = 2 -image_rebuild_frequency = 1 -max_concurrent_builds = 3 -