diff --git a/.gitignore b/.gitignore index aaec9ef..a2804fe 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,6 @@ gdb.txt # Generated docs _docs/ -docs/resources/_gen/ /man/ # VLS logs diff --git a/.woodpecker/arch-rel.yml b/.woodpecker/arch-rel.yml index f727486..f5f228e 100644 --- a/.woodpecker/arch-rel.yml +++ b/.woodpecker/arch-rel.yml @@ -9,7 +9,7 @@ skip_clone: true pipeline: build: - image: 'git.rustybever.be/vieter-v/vieter-builder' + image: 'menci/archlinuxarm:base-devel' commands: # Add the vieter repository so we can use the compiler - echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf diff --git a/.woodpecker/arch.yml b/.woodpecker/arch.yml index f5f8432..8f1a6ff 100644 --- a/.woodpecker/arch.yml +++ b/.woodpecker/arch.yml @@ -9,7 +9,7 @@ skip_clone: true pipeline: build: - image: 'git.rustybever.be/vieter-v/vieter-builder' + image: 'menci/archlinuxarm:base-devel' commands: # Add the vieter repository so we can use the compiler - echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf diff --git a/.woodpecker/build.yml b/.woodpecker/build.yml index f10e2a5..9ee8085 100644 --- a/.woodpecker/build.yml +++ b/.woodpecker/build.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' + - &vlang_image 'chewingbever/vlang:0.3' matrix: PLATFORM: diff --git a/.woodpecker/docs.yml b/.woodpecker/docs.yml index cf4874e..048b1ad 100644 --- a/.woodpecker/docs.yml +++ b/.woodpecker/docs.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' + - &vlang_image 'chewingbever/vlang:0.3' platform: 'linux/amd64' branches: diff --git a/.woodpecker/gitea.yml b/.woodpecker/gitea.yml index 9034f33..8e3b9d4 100644 --- a/.woodpecker/gitea.yml +++ b/.woodpecker/gitea.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' + - &vlang_image 'chewingbever/vlang:0.3' platform: 'linux/amd64' branches: [ 'main' ] diff --git a/.woodpecker/lint.yml b/.woodpecker/lint.yml index ec64d13..c80ce33 100644 --- a/.woodpecker/lint.yml +++ b/.woodpecker/lint.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' + - &vlang_image 'chewingbever/vlang:0.3' # These checks already get performed on the feature branches branches: @@ -7,21 +7,10 @@ branches: platform: 'linux/amd64' pipeline: - # vfmt seems to get confused if these aren't present - install-modules: - image: *vlang_image - pull: true - commands: - - export VMODULES=$PWD/.vmodules - - 'cd src && v install' - when: - event: [pull_request] - lint: image: *vlang_image pull: true commands: - - export VMODULES=$PWD/.vmodules - make lint when: - event: [pull_request] + event: [ pull_request ] diff --git a/.woodpecker/man.yml b/.woodpecker/man.yml index 8c6ca06..86a1bd8 100644 --- a/.woodpecker/man.yml +++ b/.woodpecker/man.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' + - &vlang_image 'chewingbever/vlang:0.3' platform: 'linux/amd64' branches: diff --git a/.woodpecker/test.yml b/.woodpecker/test.yml index 39cb9f9..08b7534 100644 --- a/.woodpecker/test.yml +++ b/.woodpecker/test.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' + - &vlang_image 'chewingbever/vlang:0.3' matrix: PLATFORM: diff --git a/CHANGELOG.md b/CHANGELOG.md index 27d9096..3cd39c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,33 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev) -## [0.5.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.1) - -### Added - -* Allow specifying subdirectory inside Git repository -* Added option to deploy using agent-server architecture instead of cron daemon -* Allow scheduling builds on the server from the CLI tool instead of building - them locally -* Allow force-building packages, meaning the build won't check if the - repository is already up to date - -### Changed - -* Migrated codebase to V 0.3.2 -* Cron expression parser now uses bitfields instead of bool arrays - -### Fixed - -* Arch value for target is now properly set if not provided -* Allow NULL values for branch in database -* Endpoint for adding targets now returns the correct id -* CLI now correctly errors and doesn't error when sending requests -* Fixed possible infinite loop when removing old build images -* Check whether build image still exists before starting build -* Don't run makepkg `prepare()` function twice -* Don't buffer stdout in Docker containers - ## [0.4.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.4.0) ### Added diff --git a/Dockerfile b/Dockerfile index a27ad44..7aed917 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM git.rustybever.be/chewing_bever/vlang:0.3.2 AS builder +FROM chewingbever/vlang:0.3 AS builder ARG TARGETPLATFORM ARG CI_COMMIT_SHA @@ -23,7 +23,6 @@ RUN if [ -n "${CI_COMMIT_SHA}" ]; then \ "https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \ chmod +x vieter ; \ else \ - cd src && v install && cd .. && \ LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static' make prod && \ mv pvieter vieter ; \ fi diff --git a/Makefile b/Makefile index e716807..69bd795 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' V_PATH ?= v -V := $(V_PATH) -showcc -gc boehm -W -d use_openssl +V := $(V_PATH) -showcc -gc boehm all: vieter @@ -92,9 +92,9 @@ clean: .PHONY: autofree autofree: afvieter afvieter: $(SOURCES) - $(V) -showcc -autofree -o afvieter $(SRC_DIR) + $(V_PATH) -showcc -autofree -o afvieter $(SRC_DIR) .PHONY: skip-unused skip-unused: suvieter suvieter: $(SOURCES) - $(V) -skip-unused -o suvieter $(SRC_DIR) + $(V_PATH) -showcc -skip-unused -o suvieter $(SRC_DIR) diff --git a/PKGBUILD b/PKGBUILD index 94db654..b600ba0 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -3,7 +3,7 @@ pkgbase='vieter' pkgname='vieter' -pkgver='0.5.0-rc.1' +pkgver='0.4.0' pkgrel=1 pkgdesc="Lightweight Arch repository server & package build system" depends=('glibc' 'openssl' 'libarchive' 'sqlite') diff --git a/README.md b/README.md index 637d4c1..b9fff69 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,7 @@ quicker. I chose [V](https://vlang.io/) as I've been very intrigued by this language for a while now. I wanted a fast language that I could code while relaxing, without having to exert too much mental effort & V seemed like the right choice for -that. Sadly, this didn't quite turn out the way I expected, but I'm sticking -with it anyways ;p +that. ## Features @@ -50,7 +49,7 @@ update`. I used to maintain a mirror that tracked the latest master, but nowadays, I maintain a Docker image containing the specific compiler version that Vieter -builds with. Currently, this is V 0.3.2. +builds with. Currently, this is V 0.3. ## Contributing diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 95bf713..af941a2 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -17,7 +17,7 @@ If a variable is both present in the config file & as an environment variable, the value in the environment variable is used. {{< hint info >}} -**Note** +**Note** All environment variables can also be provided from a file by appending them with `_FILE`. This for example allows you to provide the API key from a Docker secrets file. @@ -97,25 +97,3 @@ configuration variable required for each command. build`. * Default: `archlinux:base-devel` -### `vieter agent` - -* `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`, - `WARN`, `INFO` or `DEBUG`. - * Default: `WARN` -* `address`: *public* URL of the Vieter repository server to build for. From - this server jobs are retrieved. All built packages are published to this - server. -* `api_key`: API key of the above server. -* `data_dir`: directory to store log file in. -* `max_concurrent_builds`: how many builds to run at the same time. - * Default: `1` -* `polling_frequency`: how often (in seconds) to poll the server for new - builds. Note that the agent might poll more frequently when it's actively - processing builds. -* `image_rebuild_frequency`: Vieter periodically builds images that are then - used as a basis for running build containers. This is to prevent each build - from downloading an entire repository worth of dependencies. This setting - defines how frequently (in minutes) to rebuild these images. - * Default: `1440` (every 24 hours) -* `arch`: architecture for which this agent should pull down builds (e.g. - `x86_64`) diff --git a/docs/content/installation.md b/docs/content/installation.md index 21eda64..87b9cba 100644 --- a/docs/content/installation.md +++ b/docs/content/installation.md @@ -21,7 +21,7 @@ branch. This branch will be the most up to date, but does not give any guarantees about stability, so beware! Thanks to the single-binary design of Vieter, this image can be used both for -the repository server, the cron daemon and the agent. +the repository server & the cron daemon. Below is an example compose file to set up both the repository server & the cron daemon: @@ -76,7 +76,7 @@ architectures will build on both. ## Binary On the -[releases](https://git.rustybever.be/vieter-v/vieter/releases) +[releases](https://git.rustybever.be/vieter/vieter/releases) page, you can find statically compiled binaries for all released versions. This is the same binary as used inside the Docker images. @@ -106,5 +106,5 @@ guarantee that a compiler update won't temporarily break them. ## Building from source -The project [README](https://git.rustybever.be/vieter-v/vieter#building) -contains instructions for building Vieter from source. +The project [README](https://git.rustybever.be/vieter/vieter#building) contains +instructions for building Vieter from source. diff --git a/docs/content/usage/builds/schedule.md b/docs/content/usage/builds/schedule.md index de59e25..38f76a4 100644 --- a/docs/content/usage/builds/schedule.md +++ b/docs/content/usage/builds/schedule.md @@ -37,6 +37,6 @@ Each section can consist of as many of these parts as necessary. ## CLI tool The Vieter binary contains a command that shows you the next matching times for -a given expression. This can be useful for understanding the syntax. For more +a given expression. This can be useful to understand the syntax. For more information, see [vieter-schedule(1)](https://rustybever.be/man/vieter/vieter-schedule.1.html). diff --git a/src/agent/agent.v b/src/agent/agent.v deleted file mode 100644 index 69b9947..0000000 --- a/src/agent/agent.v +++ /dev/null @@ -1,27 +0,0 @@ -module agent - -import log -import os -import util - -const log_file_name = 'vieter.agent.log' - -// agent starts an agent service -pub fn agent(conf Config) ! { - log_level := log.level_from_tag(conf.log_level) or { - return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') - } - - mut logger := log.Log{ - level: log_level - } - - os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') } - - log_file := os.join_path_single(conf.data_dir, agent.log_file_name) - logger.set_full_logpath(log_file) - logger.log_to_console_too() - - mut d := agent_init(logger, conf) - d.run() -} diff --git a/src/agent/cli.v b/src/agent/cli.v deleted file mode 100644 index 1535e17..0000000 --- a/src/agent/cli.v +++ /dev/null @@ -1,31 +0,0 @@ -module agent - -import cli -import conf as vconf - -struct Config { -pub: - log_level string = 'WARN' - // Architecture that the agent represents - arch string - api_key string - address string - data_dir string - max_concurrent_builds int = 1 - polling_frequency int = 30 - image_rebuild_frequency int = 1440 -} - -// cmd returns the cli module that handles the cron daemon. -pub fn cmd() cli.Command { - return cli.Command{ - name: 'agent' - description: 'Start an agent daemon.' - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - - agent(conf)! - } - } -} diff --git a/src/agent/daemon.v b/src/agent/daemon.v deleted file mode 100644 index 62f36c2..0000000 --- a/src/agent/daemon.v +++ /dev/null @@ -1,197 +0,0 @@ -module agent - -import log -import sync.stdatomic -import build -import models { BuildConfig } -import client -import time -import os - -const ( - build_empty = 0 - build_running = 1 - build_done = 2 -) - -struct AgentDaemon { - logger shared log.Log - conf Config - client client.Client -mut: - images ImageManager - // Which builds are currently running; length is conf.max_concurrent_builds - builds []BuildConfig - // Atomic variables used to detect when a build has finished; length is - // conf.max_concurrent_builds - atomics []u64 -} - -// agent_init initializes a new agent -fn agent_init(logger log.Log, conf Config) AgentDaemon { - mut d := AgentDaemon{ - logger: logger - client: client.new(conf.address, conf.api_key) - conf: conf - images: new_image_manager(conf.image_rebuild_frequency * 60) - builds: []BuildConfig{len: conf.max_concurrent_builds} - atomics: []u64{len: conf.max_concurrent_builds} - } - - return d -} - -// run starts the actual agent daemon. This function will run forever. -pub fn (mut d AgentDaemon) run() { - // This is just so that the very first time the loop is ran, the jobs are - // always polled - mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency) - mut sleep_time := 0 * time.second - mut finished, mut empty, mut running := 0, 0, 0 - - for { - if sleep_time > 0 { - d.ldebug('Sleeping for $sleep_time') - time.sleep(sleep_time) - } - - finished, empty = d.update_atomics() - running = d.conf.max_concurrent_builds - finished - empty - - // No new finished builds and no free slots, so there's nothing to be - // done - if finished + empty == 0 { - sleep_time = 1 * time.second - continue - } - - // Builds have finished, so old builder images might have freed up. - // TODO this might query the docker daemon too frequently. - if finished > 0 { - d.images.clean_old_images() - } - - // The agent will always poll for new jobs after at most - // `polling_frequency` seconds. However, when jobs have finished, the - // agent will also poll for new jobs. This is because jobs are often - // clustered together (especially when mostly using the global cron - // schedule), so there's a much higher chance jobs are available. - if finished > 0 || time.now() >= last_poll_time.add_seconds(d.conf.polling_frequency) { - d.ldebug('Polling for new jobs') - - new_configs := d.client.poll_jobs(d.conf.arch, finished + empty) or { - d.lerror('Failed to poll jobs: $err.msg()') - - // TODO pick a better delay here - sleep_time = 5 * time.second - continue - } - - d.ldebug('Received $new_configs.len jobs') - - last_poll_time = time.now() - - for config in new_configs { - // Make sure a recent build base image is available for - // building the config - if !d.images.up_to_date(config.base_image) { - d.linfo('Building builder image from base image $config.base_image') - - // TODO handle this better than to just skip the config - d.images.refresh_image(config.base_image) or { - d.lerror(err.msg()) - continue - } - } - - // It's technically still possible that the build image is - // removed in the very short period between building the - // builder image and starting a build container with it. If - // this happens, faith really just didn't want you to do this - // build. - - d.start_build(config) - running++ - } - } - - // The agent is not doing anything, so we just wait until the next poll - // time - if running == 0 { - sleep_time = last_poll_time.add_seconds(d.conf.polling_frequency) - time.now() - } else { - sleep_time = 1 * time.second - } - } -} - -// update_atomics checks for each build whether it's completed, and sets it to -// empty again if so. The return value is a tuple `(finished, empty)` where -// `finished` is how many builds were just finished and thus set to empty, and -// `empty` is how many build slots were already empty. The amount of running -// builds can then be calculated by substracting these two values from the -// total allowed concurrent builds. -fn (mut d AgentDaemon) update_atomics() (int, int) { - mut finished := 0 - mut empty := 0 - - for i in 0 .. d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == agent.build_done { - stdatomic.store_u64(&d.atomics[i], agent.build_empty) - finished++ - } else if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { - empty++ - } - } - - return finished, empty -} - -// start_build starts a build for the given BuildConfig. -fn (mut d AgentDaemon) start_build(config BuildConfig) bool { - for i in 0 .. d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { - stdatomic.store_u64(&d.atomics[i], agent.build_running) - d.builds[i] = config - - go d.run_build(i, config) - - return true - } - } - - return false -} - -// run_build actually starts the build process for a given target. -fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) { - d.linfo('started build: $config') - - // 0 means success, 1 means failure - mut status := 0 - - new_config := BuildConfig{ - ...config - base_image: d.images.get(config.base_image) - } - - res := build.build_config(d.client.address, d.client.api_key, new_config) or { - d.ldebug('build_config error: $err.msg()') - status = 1 - - build.BuildResult{} - } - - if status == 0 { - d.linfo('Uploading build logs for $config') - - // TODO use the arch value here - build_arch := os.uname().machine - d.client.add_build_log(config.target_id, res.start_time, res.end_time, build_arch, - res.exit_code, res.logs) or { d.lerror('Failed to upload logs for $config') } - } else { - d.lwarn('an error occurred during build: $config') - } - - stdatomic.store_u64(&d.atomics[build_index], agent.build_done) -} diff --git a/src/agent/images.v b/src/agent/images.v deleted file mode 100644 index 1fec567..0000000 --- a/src/agent/images.v +++ /dev/null @@ -1,119 +0,0 @@ -module agent - -import time -import docker -import build - -// An ImageManager is a utility that creates builder images from given base -// images, updating these builder images if they've become too old. This -// structure can manage images from any number of base images, paving the way -// for configurable base images per target/repository. -struct ImageManager { - max_image_age int [required] -mut: - // For each base image, one or more builder images can exist at the same - // time - images map[string][]string [required] - // For each base image, we track when its newest image was built - timestamps map[string]time.Time [required] -} - -// new_image_manager initializes a new image manager. -fn new_image_manager(max_image_age int) ImageManager { - return ImageManager{ - max_image_age: max_image_age - images: map[string][]string{} - timestamps: map[string]time.Time{} - } -} - -// get returns the name of the newest image for the given base image. Note that -// this function should only be called *after* a first call to `refresh_image`. -pub fn (m &ImageManager) get(base_image string) string { - return m.images[base_image].last() -} - -// up_to_date returns true if the last known builder image exists and is up to -// date. If this function returns true, the last builder image may be used to -// perform a build. -pub fn (mut m ImageManager) up_to_date(base_image string) bool { - if base_image !in m.timestamps - || m.timestamps[base_image].add_seconds(m.max_image_age) <= time.now() { - return false - } - - // It's possible the image has been removed by some external event, so we - // check whether it actually exists as well. - mut dd := docker.new_conn() or { return false } - - defer { - dd.close() or {} - } - - dd.image_inspect(m.images[base_image].last()) or { - // Image doesn't exist, so we stop tracking it - if err.code() == 404 { - m.images[base_image].delete_last() - m.timestamps.delete(base_image) - } - - // If the inspect fails, it's either because the image doesn't exist or - // because of some other error. Either way, we can't know *for certain* - // that the image exists, so we return false. - return false - } - - return true -} - -// refresh_image builds a new builder image from the given base image. This -// function should only be called if `up_to_date` returned false. -fn (mut m ImageManager) refresh_image(base_image string) ! { - // TODO use better image tags for built images - new_image := build.create_build_image(base_image) or { - return error('Failed to build builder image from base image $base_image') - } - - m.images[base_image] << new_image - m.timestamps[base_image] = time.now() -} - -// clean_old_images removes all older builder images that are no longer in use. -// The function will always leave at least one builder image, namely the newest -// one. -fn (mut m ImageManager) clean_old_images() { - mut dd := docker.new_conn() or { return } - - defer { - dd.close() or {} - } - - mut i := 0 - - for image in m.images.keys() { - i = 0 - - for i < m.images[image].len - 1 { - // For each builder image, we try to remove it by calling the Docker - // API. If the function returns an error or false, that means the image - // wasn't deleted. Therefore, we move the index over. If the function - // returns true, the array's length has decreased by one so we don't - // move the index. - dd.remove_image(m.images[image][i]) or { - // The image was removed by an external event - if err.code() == 404 { - m.images[image].delete(i) - } - // The image couldn't be removed, so we need to keep track of - // it - else { - i += 1 - } - - continue - } - - m.images[image].delete(i) - } - } -} diff --git a/src/agent/log.v b/src/agent/log.v deleted file mode 100644 index cd59207..0000000 --- a/src/agent/log.v +++ /dev/null @@ -1,35 +0,0 @@ -module agent - -import log - -// log a message with the given level -pub fn (mut d AgentDaemon) log(msg string, level log.Level) { - lock d.logger { - d.logger.send_output(msg, level) - } -} - -// lfatal create a log message with the fatal level -pub fn (mut d AgentDaemon) lfatal(msg string) { - d.log(msg, log.Level.fatal) -} - -// lerror create a log message with the error level -pub fn (mut d AgentDaemon) lerror(msg string) { - d.log(msg, log.Level.error) -} - -// lwarn create a log message with the warn level -pub fn (mut d AgentDaemon) lwarn(msg string) { - d.log(msg, log.Level.warn) -} - -// linfo create a log message with the info level -pub fn (mut d AgentDaemon) linfo(msg string) { - d.log(msg, log.Level.info) -} - -// ldebug create a log message with the debug level -pub fn (mut d AgentDaemon) ldebug(msg string) { - d.log(msg, log.Level.debug) -} diff --git a/src/build/build.v b/src/build/build.v index 712c93b..2ad70a6 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -1,12 +1,12 @@ module build -import docker +import vieter_v.docker import encoding.base64 import time import os import strings import util -import models { BuildConfig, Target } +import models { Target } const ( container_build_dir = '/build' @@ -21,8 +21,8 @@ const ( // system, install some necessary packages & creates a non-root user to run // makepkg with. The base image should be some Linux distribution that uses // Pacman as its package manager. -pub fn create_build_image(base_image string) !string { - mut dd := docker.new_conn()! +pub fn create_build_image(base_image string) ?string { + mut dd := docker.new_conn()? defer { dd.close() or {} @@ -57,15 +57,15 @@ pub fn create_build_image(base_image string) !string { image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' } // We pull the provided image - dd.pull_image(image_name, image_tag)! + dd.pull_image(image_name, image_tag)? - id := dd.container_create(c)!.id - // id := docker.create_container(c)! - dd.container_start(id)! + id := dd.container_create(c)?.id + // id := docker.create_container(c)? + dd.container_start(id)? // This loop waits until the container has stopped, so we can remove it after for { - data := dd.container_inspect(id)! + data := dd.container_inspect(id)? if !data.state.running { break @@ -79,8 +79,8 @@ pub fn create_build_image(base_image string) !string { // TODO also add the base image's name into the image name to prevent // conflicts. tag := time.sys_mono_now().str() - image := dd.create_image_from_container(id, 'vieter-build', tag)! - dd.container_remove(id)! + image := dd.create_image_from_container(id, 'vieter-build', tag)? + dd.container_remove(id)? return image.id } @@ -93,32 +93,25 @@ pub: logs string } -// build_target builds the given target. Internally it calls `build_config`. -pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool) !BuildResult { - config := target.as_build_config(base_image_id, force) - - return build_config(address, api_key, config) -} - -// build_config builds, packages & publishes a given Arch package based on the +// build_target builds, packages & publishes a given Arch package based on the // provided target. The base image ID should be of an image previously created // by create_build_image. It returns the logs of the container. -pub fn build_config(address string, api_key string, config BuildConfig) !BuildResult { - mut dd := docker.new_conn()! +pub fn build_target(address string, api_key string, base_image_id string, target &Target) ?BuildResult { + mut dd := docker.new_conn()? defer { dd.close() or {} } build_arch := os.uname().machine - build_script := create_build_script(address, config, build_arch) + build_script := create_build_script(address, target, build_arch) // We convert the build script into a base64 string, which then gets passed // to the container as an env var base64_script := base64.encode_str(build_script) c := docker.NewContainer{ - image: '$config.base_image' + image: '$base_image_id' env: [ 'BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key', @@ -132,25 +125,25 @@ pub fn build_config(address string, api_key string, config BuildConfig) !BuildRe user: '0:0' } - id := dd.container_create(c)!.id - dd.container_start(id)! + id := dd.container_create(c)?.id + dd.container_start(id)? - mut data := dd.container_inspect(id)! + mut data := dd.container_inspect(id)? // This loop waits until the container has stopped, so we can remove it after for data.state.running { time.sleep(1 * time.second) - data = dd.container_inspect(id)! + data = dd.container_inspect(id)? } - mut logs_stream := dd.container_get_logs(id)! + mut logs_stream := dd.container_get_logs(id)? // Read in the entire stream mut logs_builder := strings.new_builder(10 * 1024) - util.reader_to_writer(mut logs_stream, mut logs_builder)! + util.reader_to_writer(mut logs_stream, mut logs_builder)? - dd.container_remove(id)! + dd.container_remove(id)? return BuildResult{ start_time: data.state.start_time diff --git a/src/build/scripts/git.sh b/src/build/build_script_git.sh similarity index 75% rename from src/build/scripts/git.sh rename to src/build/build_script_git.sh index 2644243..73e0965 100644 --- a/src/build/scripts/git.sh +++ b/src/build/build_script_git.sh @@ -16,5 +16,5 @@ echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkg curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 echo -e '+ [ "$(id -u)" == 0 ] && exit 0' [ "$(id -u)" == 0 ] && exit 0 -echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' -MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/scripts/git_branch.sh b/src/build/build_script_git_branch.sh similarity index 75% rename from src/build/scripts/git_branch.sh rename to src/build/build_script_git_branch.sh index 9f36bdc..be1ff4f 100644 --- a/src/build/scripts/git_branch.sh +++ b/src/build/build_script_git_branch.sh @@ -16,5 +16,5 @@ echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkg curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 echo -e '+ [ "$(id -u)" == 0 ] && exit 0' [ "$(id -u)" == 0 ] && exit 0 -echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' -MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/scripts/url.sh b/src/build/build_script_url.sh similarity index 75% rename from src/build/scripts/url.sh rename to src/build/build_script_url.sh index 2d27de7..3bc97e1 100644 --- a/src/build/scripts/url.sh +++ b/src/build/build_script_url.sh @@ -18,5 +18,5 @@ echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkg curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 echo -e '+ [ "$(id -u)" == 0 ] && exit 0' [ "$(id -u)" == 0 ] && exit 0 -echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' -MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/queue.v b/src/build/queue.v deleted file mode 100644 index e74529c..0000000 --- a/src/build/queue.v +++ /dev/null @@ -1,220 +0,0 @@ -module build - -import models { BuildConfig, Target } -import cron.expression { CronExpression, parse_expression } -import time -import datatypes { MinHeap } -import util - -struct BuildJob { -pub mut: - // Time at which this build job was created/queued - created time.Time - // Next timestamp from which point this job is allowed to be executed - timestamp time.Time - // Required for calculating next timestamp after having pop'ed a job - ce CronExpression - // Actual build config sent to the agent - config BuildConfig - // Whether this is a one-time job - single bool -} - -// Allows BuildJob structs to be sorted according to their timestamp in -// MinHeaps -fn (r1 BuildJob) < (r2 BuildJob) bool { - return r1.timestamp < r2.timestamp -} - -// The build job queue is responsible for managing the list of scheduled builds -// for each architecture. Agents receive jobs from this queue. -pub struct BuildJobQueue { - // Schedule to use for targets without explicitely defined cron expression - default_schedule CronExpression - // Base image to use for targets without defined base image - default_base_image string -mut: - mutex shared util.Dummy - // For each architecture, a priority queue is tracked - queues map[string]MinHeap - // When a target is removed from the server or edited, its previous build - // configs will be invalid. This map allows for those to be simply skipped - // by ignoring any build configs created before this timestamp. - invalidated map[int]time.Time -} - -// new_job_queue initializes a new job queue -pub fn new_job_queue(default_schedule CronExpression, default_base_image string) BuildJobQueue { - return BuildJobQueue{ - default_schedule: default_schedule - default_base_image: default_base_image - invalidated: map[int]time.Time{} - } -} - -// insert_all executes insert for each architecture of the given Target. -pub fn (mut q BuildJobQueue) insert_all(target Target) ! { - for arch in target.arch { - q.insert(target: target, arch: arch.value)! - } -} - -[params] -pub struct InsertConfig { - target Target [required] - arch string [required] - single bool - force bool - now bool -} - -// insert a new target's job into the queue for the given architecture. This -// job will then be endlessly rescheduled after being pop'ed, unless removed -// explicitely. -pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! { - lock q.mutex { - if input.arch !in q.queues { - q.queues[input.arch] = MinHeap{} - } - - mut job := BuildJob{ - created: time.now() - single: input.single - config: input.target.as_build_config(q.default_base_image, input.force) - } - - if !input.now { - ce := if input.target.schedule != '' { - parse_expression(input.target.schedule) or { - return error("Error while parsing cron expression '$input.target.schedule' (id $input.target.id): $err.msg()") - } - } else { - q.default_schedule - } - - job.timestamp = ce.next_from_now()! - job.ce = ce - } else { - job.timestamp = time.now() - } - - q.queues[input.arch].insert(job) - } -} - -// reschedule the given job by calculating the next timestamp and re-adding it -// to its respective queue. This function is called by the pop functions -// *after* having pop'ed the job. -fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) ! { - new_timestamp := job.ce.next_from_now()! - - new_job := BuildJob{ - ...job - created: time.now() - timestamp: new_timestamp - } - - q.queues[arch].insert(new_job) -} - -// pop_invalid pops all invalid jobs. -fn (mut q BuildJobQueue) pop_invalid(arch string) { - for { - job := q.queues[arch].peek() or { return } - - if job.config.target_id in q.invalidated - && job.created < q.invalidated[job.config.target_id] { - // This pop *should* never fail according to the source code - q.queues[arch].pop() or {} - } else { - break - } - } -} - -// peek shows the first job for the given architecture that's ready to be -// executed, if present. -pub fn (mut q BuildJobQueue) peek(arch string) ?BuildJob { - // Even peek requires a write lock, because pop_invalid can modify the data - // structure - lock q.mutex { - if arch !in q.queues { - return none - } - - q.pop_invalid(arch) - job := q.queues[arch].peek()? - - if job.timestamp < time.now() { - return job - } - } - - return none -} - -// pop removes the first job for the given architecture that's ready to be -// executed from the queue and returns it, if present. -pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob { - lock q.mutex { - if arch !in q.queues { - return none - } - - q.pop_invalid(arch) - mut job := q.queues[arch].peek()? - - if job.timestamp < time.now() { - job = q.queues[arch].pop()? - - if !job.single { - // TODO how do we handle this properly? Is it even possible for a - // cron expression to not return a next time if it's already been - // used before? - q.reschedule(job, arch) or {} - } - - return job - } - } - - return none -} - -// pop_n tries to pop at most n available jobs for the given architecture. -pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { - lock q.mutex { - if arch !in q.queues { - return [] - } - - mut out := []BuildJob{} - - for out.len < n { - q.pop_invalid(arch) - mut job := q.queues[arch].peek() or { break } - - if job.timestamp < time.now() { - job = q.queues[arch].pop() or { break } - - if !job.single { - // TODO idem - q.reschedule(job, arch) or {} - } - - out << job - } else { - break - } - } - - return out - } - - return [] -} - -// invalidate a target's old build jobs. -pub fn (mut q BuildJobQueue) invalidate(target_id int) { - q.invalidated[target_id] = time.now() -} diff --git a/src/build/scripts/git_path.sh b/src/build/scripts/git_path.sh deleted file mode 100644 index 65b7fb9..0000000 --- a/src/build/scripts/git_path.sh +++ /dev/null @@ -1,20 +0,0 @@ -echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' -echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf -echo -e '+ pacman -Syu --needed --noconfirm' -pacman -Syu --needed --noconfirm -echo -e '+ su builder' -su builder -echo -e '+ git clone --single-branch --depth 1 '\''https://examplerepo.com'\'' repo' -git clone --single-branch --depth 1 'https://examplerepo.com' repo -echo -e '+ cd '\''repo/example/path'\''' -cd 'repo/example/path' -echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' -makepkg --nobuild --syncdeps --needed --noconfirm -echo -e '+ source PKGBUILD' -source PKGBUILD -echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' -curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 -echo -e '+ [ "$(id -u)" == 0 ] && exit 0' -[ "$(id -u)" == 0 ] && exit 0 -echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' -MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/scripts/git_path_spaces.sh b/src/build/scripts/git_path_spaces.sh deleted file mode 100644 index b632b91..0000000 --- a/src/build/scripts/git_path_spaces.sh +++ /dev/null @@ -1,20 +0,0 @@ -echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' -echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf -echo -e '+ pacman -Syu --needed --noconfirm' -pacman -Syu --needed --noconfirm -echo -e '+ su builder' -su builder -echo -e '+ git clone --single-branch --depth 1 '\''https://examplerepo.com'\'' repo' -git clone --single-branch --depth 1 'https://examplerepo.com' repo -echo -e '+ cd '\''repo/example/path with spaces'\''' -cd 'repo/example/path with spaces' -echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' -makepkg --nobuild --syncdeps --needed --noconfirm -echo -e '+ source PKGBUILD' -source PKGBUILD -echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' -curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 -echo -e '+ [ "$(id -u)" == 0 ] && exit 0' -[ "$(id -u)" == 0 ] && exit 0 -echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' -MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/shell.v b/src/build/shell.v index 16f93b5..e573d53 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -1,6 +1,6 @@ module build -import models { BuildConfig } +import models { Target } // escape_shell_string escapes any characters that could be interpreted // incorrectly by a shell. The resulting value should be safe to use inside an @@ -23,13 +23,13 @@ pub fn echo_commands(cmds []string) []string { } // create_build_script generates a shell script that builds a given Target. -fn create_build_script(address string, config BuildConfig, build_arch string) string { - repo_url := '$address/$config.repo' +fn create_build_script(address string, target &Target, build_arch string) string { + repo_url := '$address/$target.repo' mut commands := [ // This will later be replaced by a proper setting for changing the // mirrorlist - "echo -e '[$config.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" + "echo -e '[$target.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" // We need to update the package list of the repo we just added above. // This should however not pull in a lot of packages as long as the // builder image is rebuilt frequently. @@ -38,22 +38,22 @@ fn create_build_script(address string, config BuildConfig, build_arch string) st 'su builder', ] - commands << match config.kind { + commands << match target.kind { 'git' { - if config.branch == '' { + if target.branch == '' { [ - "git clone --single-branch --depth 1 '$config.url' repo", + "git clone --single-branch --depth 1 '$target.url' repo", ] } else { [ - "git clone --single-branch --depth 1 --branch $config.branch '$config.url' repo", + "git clone --single-branch --depth 1 --branch $target.branch '$target.url' repo", ] } } 'url' { [ 'mkdir repo', - "curl -o repo/PKGBUILD -L '$config.url'", + "curl -o repo/PKGBUILD -L '$target.url'", ] } else { @@ -61,32 +61,19 @@ fn create_build_script(address string, config BuildConfig, build_arch string) st } } - commands << if config.path != '' { - "cd 'repo/$config.path'" - } else { - 'cd repo' - } - commands << [ + 'cd repo', 'makepkg --nobuild --syncdeps --needed --noconfirm', 'source PKGBUILD', - ] - - if !config.force { // The build container checks whether the package is already present on // the server. - commands << [ - 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', - // If the above curl command succeeds, we don't need to rebuild the - // package. However, because we're in a su shell, the exit command will - // drop us back into the root shell. Therefore, we must check whether - // we're in root so we don't proceed. - '[ "\$(id -u)" == 0 ] && exit 0', - ] - } - - commands << [ - 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', + 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + // If the above curl command succeeds, we don't need to rebuild the + // package. However, because we're in a su shell, the exit command will + // drop us back into the root shell. Therefore, we must check whether + // we're in root so we don't proceed. + '[ "\$(id -u)" == 0 ] && exit 0', + 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', ] return echo_commands(commands).join('\n') diff --git a/src/build/shell_test.v b/src/build/shell_test.v index e23d964..341df88 100644 --- a/src/build/shell_test.v +++ b/src/build/shell_test.v @@ -1,75 +1,43 @@ module build -import models { BuildConfig } - -fn test_create_build_script_git() { - config := BuildConfig{ - target_id: 1 - kind: 'git' - url: 'https://examplerepo.com' - repo: 'vieter' - base_image: 'not-used:latest' - } - - build_script := create_build_script('https://example.com', config, 'x86_64') - expected := $embed_file('scripts/git.sh') - - assert build_script == expected.to_string().trim_space() -} - -fn test_create_build_script_git_path() { - mut config := BuildConfig{ - target_id: 1 - kind: 'git' - url: 'https://examplerepo.com' - repo: 'vieter' - path: 'example/path' - base_image: 'not-used:latest' - } - - mut build_script := create_build_script('https://example.com', config, 'x86_64') - mut expected := $embed_file('scripts/git_path.sh') - - assert build_script == expected.to_string().trim_space() - - config = BuildConfig{ - ...config - path: 'example/path with spaces' - } - - build_script = create_build_script('https://example.com', config, 'x86_64') - expected = $embed_file('scripts/git_path_spaces.sh') - - assert build_script == expected.to_string().trim_space() -} +import models { Target } fn test_create_build_script_git_branch() { - config := BuildConfig{ - target_id: 1 + target := Target{ + id: 1 kind: 'git' url: 'https://examplerepo.com' branch: 'main' repo: 'vieter' - base_image: 'not-used:latest' } + build_script := create_build_script('https://example.com', target, 'x86_64') + expected := $embed_file('build_script_git_branch.sh') - build_script := create_build_script('https://example.com', config, 'x86_64') - expected := $embed_file('scripts/git_branch.sh') + assert build_script == expected.to_string().trim_space() +} + +fn test_create_build_script_git() { + target := Target{ + id: 1 + kind: 'git' + url: 'https://examplerepo.com' + repo: 'vieter' + } + build_script := create_build_script('https://example.com', target, 'x86_64') + expected := $embed_file('build_script_git.sh') assert build_script == expected.to_string().trim_space() } fn test_create_build_script_url() { - config := BuildConfig{ - target_id: 1 + target := Target{ + id: 1 kind: 'url' url: 'https://examplerepo.com' repo: 'vieter' - base_image: 'not-used:latest' } - - build_script := create_build_script('https://example.com', config, 'x86_64') - expected := $embed_file('scripts/url.sh') + build_script := create_build_script('https://example.com', target, 'x86_64') + expected := $embed_file('build_script_url.sh') assert build_script == expected.to_string().trim_space() } diff --git a/src/client/client.v b/src/client/client.v index cce4e70..d68ff18 100644 --- a/src/client/client.v +++ b/src/client/client.v @@ -2,7 +2,7 @@ module client import net.http { Method } import net.urllib -import web.response { Response, new_data_response } +import web.response { Response } import json pub struct Client { @@ -21,7 +21,7 @@ pub fn new(address string, api_key string) Client { // send_request_raw sends an HTTP request, returning the http.Response object. // It encodes the params so that they're safe to pass as HTTP query parameters. -fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) !http.Response { +fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) ?http.Response { mut full_url := '$c.address$url' if params.len > 0 { @@ -38,53 +38,31 @@ fn (c &Client) send_request_raw(method Method, url string, params map[string]str full_url = '$full_url?$params_str' } - // Looking at the source code, this function doesn't actually fail, so I'm - // not sure why it returns an optional - mut req := http.new_request(method, full_url, body) or { return error('') } - req.add_custom_header('X-Api-Key', c.api_key)! + mut req := http.new_request(method, full_url, body)? + req.add_custom_header('X-Api-Key', c.api_key)? - res := req.do()! + res := req.do()? return res } // send_request just calls send_request_with_body with an empty body. -fn (c &Client) send_request(method Method, url string, params map[string]string) !Response { +fn (c &Client) send_request(method Method, url string, params map[string]string) ?Response { return c.send_request_with_body(method, url, params, '') } // send_request_with_body calls send_request_raw_response & parses its // output as a Response object. -fn (c &Client) send_request_with_body(method Method, url string, params map[string]string, body string) !Response { - res := c.send_request_raw(method, url, params, body)! - status := res.status() - - // Non-successful requests are expected to return either an empty body or - // Response - if status.is_error() { - // A non-successful status call will have an empty body - if res.body == '' { - return error('Error $res.status_code ($status.str()): (empty response)') - } - - data := json.decode(Response, res.body)! - - return error('Status $res.status_code ($status.str()): $data.message') - } - - // Just return an empty successful response - if res.body == '' { - return new_data_response(T{}) - } - - data := json.decode(Response, res.body)! +fn (c &Client) send_request_with_body(method Method, url string, params map[string]string, body string) ?Response { + res_text := c.send_request_raw_response(method, url, params, body)? + data := json.decode(Response, res_text)? return data } // send_request_raw_response returns the raw text response for an HTTP request. -fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) !string { - res := c.send_request_raw(method, url, params, body)! +fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) ?string { + res := c.send_request_raw(method, url, params, body)? return res.body } diff --git a/src/client/jobs.v b/src/client/jobs.v deleted file mode 100644 index 784639e..0000000 --- a/src/client/jobs.v +++ /dev/null @@ -1,23 +0,0 @@ -module client - -import models { BuildConfig } - -// poll_jobs requests a list of new build jobs from the server. -pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig { - data := c.send_request<[]BuildConfig>(.get, '/api/v1/jobs/poll', { - 'arch': arch - 'max': max.str() - })! - - return data.data -} - -// queue_job adds a new one-time build job for the given target to the job -// queue. -pub fn (c &Client) queue_job(target_id int, arch string, force bool) ! { - c.send_request(.post, '/api/v1/jobs/queue', { - 'target': target_id.str() - 'arch': arch - 'force': force.str() - })! -} diff --git a/src/client/logs.v b/src/client/logs.v index 85063bc..b414245 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -6,29 +6,40 @@ import web.response { Response } import time // get_build_logs returns all build logs. -pub fn (c &Client) get_build_logs(filter BuildLogFilter) ![]BuildLog { +pub fn (c &Client) get_build_logs(filter BuildLogFilter) ?Response<[]BuildLog> { params := models.params_from(filter) - data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)! + data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)? - return data.data + return data +} + +// get_build_logs_for_target returns all build logs for a given target. +pub fn (c &Client) get_build_logs_for_target(target_id int) ?Response<[]BuildLog> { + params := { + 'repo': target_id.str() + } + + data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)? + + return data } // get_build_log returns a specific build log. -pub fn (c &Client) get_build_log(id int) !BuildLog { - data := c.send_request(Method.get, '/api/v1/logs/$id', {})! +pub fn (c &Client) get_build_log(id int) ?Response { + data := c.send_request(Method.get, '/api/v1/logs/$id', {})? - return data.data + return data } // get_build_log_content returns the contents of the build log file. -pub fn (c &Client) get_build_log_content(id int) !string { - data := c.send_request_raw_response(Method.get, '/api/v1/logs/$id/content', {}, '')! +pub fn (c &Client) get_build_log_content(id int) ?string { + data := c.send_request_raw_response(Method.get, '/api/v1/logs/$id/content', {}, '')? return data } // add_build_log adds a new build log to the server. -pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) !Response { +pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { params := { 'target': target_id.str() 'startTime': start_time.unix_time().str() @@ -37,7 +48,7 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t 'exitCode': exit_code.str() } - data := c.send_request_with_body(Method.post, '/api/v1/logs', params, content)! + data := c.send_request_with_body(Method.post, '/api/v1/logs', params, content)? return data } diff --git a/src/client/targets.v b/src/client/targets.v index da6a9e4..c5e44fe 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -2,23 +2,24 @@ module client import models { Target, TargetFilter } import net.http { Method } +import web.response { Response } // get_targets returns a list of targets, given a filter object. -pub fn (c &Client) get_targets(filter TargetFilter) ![]Target { +pub fn (c &Client) get_targets(filter TargetFilter) ?[]Target { params := models.params_from(filter) - data := c.send_request<[]Target>(Method.get, '/api/v1/targets', params)! + data := c.send_request<[]Target>(Method.get, '/api/v1/targets', params)? return data.data } // get_all_targets retrieves *all* targs from the API using the default // limit. -pub fn (c &Client) get_all_targets() ![]Target { +pub fn (c &Client) get_all_targets() ?[]Target { mut targets := []Target{} mut offset := u64(0) for { - sub_targets := c.get_targets(offset: offset)! + sub_targets := c.get_targets(offset: offset)? if sub_targets.len == 0 { break @@ -33,8 +34,8 @@ pub fn (c &Client) get_all_targets() ![]Target { } // get_target returns the target for a specific id. -pub fn (c &Client) get_target(id int) !Target { - data := c.send_request(Method.get, '/api/v1/targets/$id', {})! +pub fn (c &Client) get_target(id int) ?Target { + data := c.send_request(Method.get, '/api/v1/targets/$id', {})? return data.data } @@ -44,29 +45,28 @@ pub struct NewTarget { url string branch string repo string - path string arch []string } // add_target adds a new target to the server. -pub fn (c &Client) add_target(t NewTarget) !int { +pub fn (c &Client) add_target(t NewTarget) ?Response { params := models.params_from(t) - data := c.send_request(Method.post, '/api/v1/targets', params)! + data := c.send_request(Method.post, '/api/v1/targets', params)? - return data.data + return data } // remove_target removes the target with the given id from the server. -pub fn (c &Client) remove_target(id int) !string { - data := c.send_request(Method.delete, '/api/v1/targets/$id', {})! +pub fn (c &Client) remove_target(id int) ?Response { + data := c.send_request(Method.delete, '/api/v1/targets/$id', {})? - return data.data + return data } // patch_target sends a PATCH request to the given target with the params as // payload. -pub fn (c &Client) patch_target(id int, params map[string]string) !string { - data := c.send_request(Method.patch, '/api/v1/targets/$id', params)! +pub fn (c &Client) patch_target(id int, params map[string]string) ?Response { + data := c.send_request(Method.patch, '/api/v1/targets/$id', params)? - return data.data + return data } diff --git a/src/console/aur/aur.v b/src/console/aur/aur.v index a6a3324..c98f8e6 100644 --- a/src/console/aur/aur.v +++ b/src/console/aur/aur.v @@ -3,8 +3,8 @@ module aur import cli import console import client -import aur -import conf as vconf +import vieter_v.aur +import vieter_v.conf as vconf struct Config { address string [required] @@ -21,12 +21,12 @@ pub fn cmd() cli.Command { name: 'search' description: 'Search for packages.' required_args: 1 - execute: fn (cmd cli.Command) ! { + execute: fn (cmd cli.Command) ? { c := aur.new() - pkgs := c.search(cmd.args[0])! + pkgs := c.search(cmd.args[0])? data := pkgs.map([it.name, it.description]) - println(console.pretty_table(['name', 'description'], data)!) + println(console.pretty_table(['name', 'description'], data)?) } }, cli.Command{ @@ -34,12 +34,12 @@ pub fn cmd() cli.Command { usage: 'repo pkg-name [pkg-name...]' description: 'Add the given AUR package(s) to Vieter. Non-existent packages will be silently ignored.' required_args: 2 - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? c := aur.new() - pkgs := c.info(cmd.args[1..])! + pkgs := c.info(cmd.args[1..])? vc := client.new(conf.address, conf.api_key) diff --git a/src/console/console.v b/src/console/console.v index 5c40de8..caf4cca 100644 --- a/src/console/console.v +++ b/src/console/console.v @@ -13,7 +13,7 @@ pub fn tabbed_table(data [][]string) string { // pretty_table converts a list of string data into a pretty table. Many thanks // to @hungrybluedev in the Vlang Discord for providing this code! // https://ptb.discord.com/channels/592103645835821068/592106336838352923/970278787143045192 -pub fn pretty_table(header []string, data [][]string) !string { +pub fn pretty_table(header []string, data [][]string) ?string { column_count := header.len mut column_widths := []int{len: column_count, init: header[it].len} @@ -26,7 +26,7 @@ pub fn pretty_table(header []string, data [][]string) !string { } } - single_line_length := arrays.sum(column_widths)! + (column_count + 1) * 3 - 4 + single_line_length := arrays.sum(column_widths)? + (column_count + 1) * 3 - 4 horizontal_line := '+' + strings.repeat(`-`, single_line_length) + '+' mut buffer := strings.new_builder(data.len * single_line_length) @@ -64,12 +64,12 @@ pub fn pretty_table(header []string, data [][]string) !string { // export_man_pages recursively generates all man pages for the given // cli.Command & writes them to the given directory. -pub fn export_man_pages(cmd cli.Command, path string) ! { +pub fn export_man_pages(cmd cli.Command, path string) ? { man := cmd.manpage() os.write_file(os.join_path_single(path, cmd.full_name().replace(' ', '-') + '.1'), - man)! + man)? for sub_cmd in cmd.commands { - export_man_pages(sub_cmd, path)! + export_man_pages(sub_cmd, path)? } } diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 3064a58..41830c2 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -1,7 +1,7 @@ module logs import cli -import conf as vconf +import vieter_v.conf as vconf import client import console import time @@ -63,30 +63,30 @@ pub fn cmd() cli.Command { flag: cli.FlagType.string }, ] - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? mut filter := BuildLogFilter{} - limit := cmd.flags.get_int('limit')! + limit := cmd.flags.get_int('limit')? if limit != 0 { filter.limit = u64(limit) } - offset := cmd.flags.get_int('offset')! + offset := cmd.flags.get_int('offset')? if offset != 0 { filter.offset = u64(offset) } - target_id := cmd.flags.get_int('target')! + target_id := cmd.flags.get_int('target')? if target_id != 0 { filter.target = target_id } tz_offset := time.offset() - if cmd.flags.get_bool('today')! { + if cmd.flags.get_bool('today')? { today := time.now() filter.after = time.new_time(time.Time{ @@ -98,12 +98,12 @@ pub fn cmd() cli.Command { } // The -today flag overwrites any of the other date flags. else { - day_str := cmd.flags.get_string('day')! - before_str := cmd.flags.get_string('before')! - after_str := cmd.flags.get_string('after')! + day_str := cmd.flags.get_string('day')? + before_str := cmd.flags.get_string('before')? + after_str := cmd.flags.get_string('after')? if day_str != '' { - day := time.parse_rfc3339(day_str)! + day := time.parse_rfc3339(day_str)? day_utc := time.new_time(time.Time{ year: day.year month: day.month @@ -118,24 +118,24 @@ pub fn cmd() cli.Command { filter.before = day_utc.add_days(1) } else { if before_str != '' { - filter.before = time.parse(before_str)!.add_seconds(-tz_offset) + filter.before = time.parse(before_str)?.add_seconds(-tz_offset) } if after_str != '' { - filter.after = time.parse(after_str)!.add_seconds(-tz_offset) + filter.after = time.parse(after_str)?.add_seconds(-tz_offset) } } } - if cmd.flags.get_bool('failed')! { + if cmd.flags.get_bool('failed')? { filter.exit_codes = [ '!0', ] } - raw := cmd.flags.get_bool('raw')! + raw := cmd.flags.get_bool('raw')? - list(conf, filter, raw)! + list(conf, filter, raw)? } }, cli.Command{ @@ -143,12 +143,12 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Show all info for a specific build log.' - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? id := cmd.args[0].int() - info(conf, id)! + info(conf, id)? } }, cli.Command{ @@ -156,12 +156,12 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Output the content of a build log to stdout.' - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? id := cmd.args[0].int() - content(conf, id)! + content(conf, id)? } }, ] @@ -169,38 +169,46 @@ pub fn cmd() cli.Command { } // print_log_list prints a list of logs. -fn print_log_list(logs []BuildLog, raw bool) ! { +fn print_log_list(logs []BuildLog, raw bool) ? { data := logs.map([it.id.str(), it.target_id.str(), it.start_time.local().str(), it.exit_code.str()]) if raw { println(console.tabbed_table(data)) } else { - println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)!) + println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)?) } } // list prints a list of all build logs. -fn list(conf Config, filter BuildLogFilter, raw bool) ! { +fn list(conf Config, filter BuildLogFilter, raw bool) ? { c := client.new(conf.address, conf.api_key) - logs := c.get_build_logs(filter)! + logs := c.get_build_logs(filter)?.data - print_log_list(logs, raw)! + print_log_list(logs, raw)? +} + +// list prints a list of all build logs for a given target. +fn list_for_target(conf Config, target_id int, raw bool) ? { + c := client.new(conf.address, conf.api_key) + logs := c.get_build_logs_for_target(target_id)?.data + + print_log_list(logs, raw)? } // info print the detailed info for a given build log. -fn info(conf Config, id int) ! { +fn info(conf Config, id int) ? { c := client.new(conf.address, conf.api_key) - log := c.get_build_log(id)! + log := c.get_build_log(id)?.data print(log) } // content outputs the contents of the log file for a given build log to // stdout. -fn content(conf Config, id int) ! { +fn content(conf Config, id int) ? { c := client.new(conf.address, conf.api_key) - content := c.get_build_log_content(id)! + content := c.get_build_log_content(id)? println(content) } diff --git a/src/console/man/man.v b/src/console/man/man.v index 22cb5f7..d91a140 100644 --- a/src/console/man/man.v +++ b/src/console/man/man.v @@ -11,11 +11,11 @@ pub fn cmd() cli.Command { description: 'Generate all man pages & save them in the given directory.' usage: 'dir' required_args: 1 - execute: fn (cmd cli.Command) ! { + execute: fn (cmd cli.Command) ? { root := cmd.root() - os.mkdir_all(cmd.args[0])! + os.mkdir_all(cmd.args[0])? - console.export_man_pages(root, cmd.args[0])! + console.export_man_pages(root, cmd.args[0])? } } } diff --git a/src/console/schedule/schedule.v b/src/console/schedule/schedule.v index 7ce0516..8fceddd 100644 --- a/src/console/schedule/schedule.v +++ b/src/console/schedule/schedule.v @@ -18,11 +18,11 @@ pub fn cmd() cli.Command { default_value: ['5'] }, ] - execute: fn (cmd cli.Command) ! { - ce := parse_expression(cmd.args.join(' '))! - count := cmd.flags.get_int('count')! + execute: fn (cmd cli.Command) ? { + ce := parse_expression(cmd.args.join(' '))? + count := cmd.flags.get_int('count')? - for t in ce.next_n(time.now(), count)! { + for t in ce.next_n(time.now(), count)? { println(t) } } diff --git a/src/console/targets/build.v b/src/console/targets/build.v index e18077d..6337aa3 100644 --- a/src/console/targets/build.v +++ b/src/console/targets/build.v @@ -1,34 +1,34 @@ module targets import client -import docker +import vieter_v.docker import os import build // build locally builds the target with the given id. -fn build(conf Config, target_id int, force bool) ! { +fn build(conf Config, target_id int) ? { c := client.new(conf.address, conf.api_key) - target := c.get_target(target_id)! + target := c.get_target(target_id)? build_arch := os.uname().machine println('Creating base image...') - image_id := build.create_build_image(conf.base_image)! + image_id := build.create_build_image(conf.base_image)? println('Running build...') - res := build.build_target(conf.address, conf.api_key, image_id, target, force)! + res := build.build_target(conf.address, conf.api_key, image_id, target)? println('Removing build image...') - mut dd := docker.new_conn()! + mut dd := docker.new_conn()? defer { dd.close() or {} } - dd.remove_image(image_id)! + dd.remove_image(image_id)? println('Uploading logs to Vieter...') c.add_build_log(target.id, res.start_time, res.end_time, build_arch, res.exit_code, - res.logs)! + res.logs)? } diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 94deebd..5640011 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -1,7 +1,7 @@ module targets import cli -import conf as vconf +import vieter_v.conf as vconf import cron.expression { parse_expression } import client { NewTarget } import console @@ -13,7 +13,7 @@ struct Config { base_image string = 'archlinux:base-devel' } -// cmd returns the cli submodule that handles the targets API interaction +// cmd returns the cli submodule that handles the repos API interaction pub fn cmd() cli.Command { return cli.Command{ name: 'targets' @@ -39,30 +39,30 @@ pub fn cmd() cli.Command { flag: cli.FlagType.string }, ] - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? mut filter := TargetFilter{} - limit := cmd.flags.get_int('limit')! + limit := cmd.flags.get_int('limit')? if limit != 0 { filter.limit = u64(limit) } - offset := cmd.flags.get_int('offset')! + offset := cmd.flags.get_int('offset')? if offset != 0 { filter.offset = u64(offset) } - repo := cmd.flags.get_string('repo')! + repo := cmd.flags.get_string('repo')? if repo != '' { filter.repo = repo } - raw := cmd.flags.get_bool('raw')! + raw := cmd.flags.get_bool('raw')? - list(conf, filter, raw)! + list(conf, filter, raw)? } }, cli.Command{ @@ -82,27 +82,21 @@ pub fn cmd() cli.Command { description: "Which branch to clone; only applies to kind 'git'." flag: cli.FlagType.string }, - cli.Flag{ - name: 'path' - description: 'Subdirectory inside Git repository to use.' - flag: cli.FlagType.string - }, ] - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? t := NewTarget{ - kind: cmd.flags.get_string('kind')! + kind: cmd.flags.get_string('kind')? url: cmd.args[0] repo: cmd.args[1] branch: cmd.flags.get_string('branch') or { '' } - path: cmd.flags.get_string('path') or { '' } } - raw := cmd.flags.get_bool('raw')! + raw := cmd.flags.get_bool('raw')? - add(conf, t, raw)! + add(conf, t, raw)? } }, cli.Command{ @@ -110,11 +104,11 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Remove a target that matches the given id.' - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? - remove(conf, cmd.args[0])! + remove(conf, cmd.args[0])? } }, cli.Command{ @@ -122,11 +116,11 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Show detailed information for the target matching the id.' - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? - info(conf, cmd.args[0])! + info(conf, cmd.args[0])? } }, cli.Command{ @@ -165,15 +159,10 @@ pub fn cmd() cli.Command { description: 'Kind of target.' flag: cli.FlagType.string }, - cli.Flag{ - name: 'path' - description: 'Subdirectory inside Git repository to use.' - flag: cli.FlagType.string - }, ] - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? found := cmd.flags.get_all_found() @@ -181,11 +170,11 @@ pub fn cmd() cli.Command { for f in found { if f.name != 'config-file' { - params[f.name] = f.get_string()! + params[f.name] = f.get_string()? } } - patch(conf, cmd.args[0], params)! + patch(conf, cmd.args[0], params)? } }, cli.Command{ @@ -193,82 +182,58 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Build the target with the given id & publish it.' - flags: [ - cli.Flag{ - name: 'force' - description: 'Build the target without checking whether it needs to be renewed.' - flag: cli.FlagType.bool - }, - cli.Flag{ - name: 'remote' - description: 'Schedule the build on the server instead of running it locally.' - flag: cli.FlagType.bool - }, - cli.Flag{ - name: 'arch' - description: 'Architecture to schedule build for. Required when using -remote.' - flag: cli.FlagType.string - }, - ] - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? - remote := cmd.flags.get_bool('remote')! - force := cmd.flags.get_bool('force')! - target_id := cmd.args[0].int() - - if remote { - arch := cmd.flags.get_string('arch')! - - if arch == '' { - return error('When scheduling the build remotely, you have to specify an architecture.') - } - - c := client.new(conf.address, conf.api_key) - c.queue_job(target_id, arch, force)! - } else { - build(conf, target_id, force)! - } + build(conf, cmd.args[0].int())? } }, ] } } +// get_repo_by_prefix tries to find the repo with the given prefix in its +// ID. If multiple or none are found, an error is raised. + // list prints out a list of all repositories. -fn list(conf Config, filter TargetFilter, raw bool) ! { +fn list(conf Config, filter TargetFilter, raw bool) ? { c := client.new(conf.address, conf.api_key) - targets := c.get_targets(filter)! - data := targets.map([it.id.str(), it.kind, it.url, it.repo]) + repos := c.get_targets(filter)? + data := repos.map([it.id.str(), it.kind, it.url, it.repo]) if raw { println(console.tabbed_table(data)) } else { - println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)!) + println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?) } } -// add adds a new target to the server's list. -fn add(conf Config, t &NewTarget, raw bool) ! { +// add adds a new repository to the server's list. +fn add(conf Config, t &NewTarget, raw bool) ? { c := client.new(conf.address, conf.api_key) - target_id := c.add_target(t)! + res := c.add_target(t)? if raw { - println(target_id) + println(res.data) } else { - println('Target added with id $target_id') + println('Target added with id $res.data') } } -// remove removes a target from the server's list. -fn remove(conf Config, id string) ! { - c := client.new(conf.address, conf.api_key) - c.remove_target(id.int())! +// remove removes a repository from the server's list. +fn remove(conf Config, id string) ? { + id_int := id.int() + + if id_int != 0 { + c := client.new(conf.address, conf.api_key) + res := c.remove_target(id_int)? + println(res.message) + } } -// patch patches a given target with the provided params. -fn patch(conf Config, id string, params map[string]string) ! { +// patch patches a given repository with the provided params. +fn patch(conf Config, id string, params map[string]string) ? { // We check the cron expression first because it's useless to send an // invalid one to the server. if 'schedule' in params && params['schedule'] != '' { @@ -277,13 +242,24 @@ fn patch(conf Config, id string, params map[string]string) ! { } } - c := client.new(conf.address, conf.api_key) - c.patch_target(id.int(), params)! + id_int := id.int() + if id_int != 0 { + c := client.new(conf.address, conf.api_key) + res := c.patch_target(id_int, params)? + + println(res.message) + } } -// info shows detailed information for a given target. -fn info(conf Config, id string) ! { +// info shows detailed information for a given repo. +fn info(conf Config, id string) ? { + id_int := id.int() + + if id_int == 0 { + return + } + c := client.new(conf.address, conf.api_key) - target := c.get_target(id.int())! - println(target) + repo := c.get_target(id_int)? + println(repo) } diff --git a/src/cron/cli.v b/src/cron/cli.v index 16a3537..4d95833 100644 --- a/src/cron/cli.v +++ b/src/cron/cli.v @@ -1,7 +1,7 @@ module cron import cli -import conf as vconf +import vieter_v.conf as vconf struct Config { pub: @@ -22,11 +22,11 @@ pub fn cmd() cli.Command { return cli.Command{ name: 'cron' description: 'Start the cron service that periodically runs builds.' - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? - cron(conf)! + cron(conf)? } } } diff --git a/src/cron/cron.v b/src/cron/cron.v index f1d6b7b..5f128cf 100644 --- a/src/cron/cron.v +++ b/src/cron/cron.v @@ -8,7 +8,7 @@ import os const log_file_name = 'vieter.cron.log' // cron starts a cron daemon & starts periodically scheduling builds. -pub fn cron(conf Config) ! { +pub fn cron(conf Config) ? { // Configure logger log_level := log.level_from_tag(conf.log_level) or { return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -27,7 +27,7 @@ pub fn cron(conf Config) ! { } mut d := daemon.init_daemon(logger, conf.address, conf.api_key, conf.base_image, ce, - conf.max_concurrent_builds, conf.api_update_frequency, conf.image_rebuild_frequency)! + conf.max_concurrent_builds, conf.api_update_frequency, conf.image_rebuild_frequency)? d.run() } diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index 42edc92..beed9fc 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -79,7 +79,7 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { mut status := 0 res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(), - &sb.target, false) or { + &sb.target) or { d.ldebug('build_target error: $err.msg()') status = 1 diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 0d30a23..934d35a 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -6,7 +6,7 @@ import datatypes { MinHeap } import cron.expression { CronExpression, parse_expression } import math import build -import docker +import vieter_v.docker import os import client import models { Target } @@ -53,7 +53,7 @@ mut: // init_daemon initializes a new Daemon object. It renews the targets & // populates the build queue for the first time. -pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) !Daemon { +pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon { mut d := Daemon{ client: client.new(address, api_key) base_image: base_image @@ -207,7 +207,7 @@ fn (mut d Daemon) renew_queue() { // For some reason, using // ```v - // for d.queue.len() > 0 && d.queue.peek() !.timestamp < now { + // for d.queue.len() > 0 && d.queue.peek() ?.timestamp < now { //``` // here causes the function to prematurely just exit, without any errors or anything, very weird // https://github.com/vlang/v/issues/14042 diff --git a/src/cron/daemon/log.v b/src/cron/daemon/log.v index 95a50e7..003898b 100644 --- a/src/cron/daemon/log.v +++ b/src/cron/daemon/log.v @@ -3,33 +3,33 @@ module daemon import log // log reate a log message with the given level -pub fn (mut d Daemon) log(msg string, level log.Level) { +pub fn (mut d Daemon) log(msg &string, level log.Level) { lock d.logger { d.logger.send_output(msg, level) } } // lfatal create a log message with the fatal level -pub fn (mut d Daemon) lfatal(msg string) { +pub fn (mut d Daemon) lfatal(msg &string) { d.log(msg, log.Level.fatal) } // lerror create a log message with the error level -pub fn (mut d Daemon) lerror(msg string) { +pub fn (mut d Daemon) lerror(msg &string) { d.log(msg, log.Level.error) } // lwarn create a log message with the warn level -pub fn (mut d Daemon) lwarn(msg string) { +pub fn (mut d Daemon) lwarn(msg &string) { d.log(msg, log.Level.warn) } // linfo create a log message with the info level -pub fn (mut d Daemon) linfo(msg string) { +pub fn (mut d Daemon) linfo(msg &string) { d.log(msg, log.Level.info) } // ldebug create a log message with the debug level -pub fn (mut d Daemon) ldebug(msg string) { +pub fn (mut d Daemon) ldebug(msg &string) { d.log(msg, log.Level.debug) } diff --git a/src/cron/expression/expression.v b/src/cron/expression/expression.v index c3ff8c5..17d2dde 100644 --- a/src/cron/expression/expression.v +++ b/src/cron/expression/expression.v @@ -12,7 +12,7 @@ pub struct CronExpression { // next calculates the earliest time this cron expression is valid. It will // always pick a moment in the future, even if ref matches completely up to the // minute. This function conciously does not take gap years into account. -pub fn (ce &CronExpression) next(ref time.Time) !time.Time { +pub fn (ce &CronExpression) next(ref time.Time) ?time.Time { // If the given ref matches the next cron occurence up to the minute, it // will return that value. Because we always want to return a value in the // future, we artifically shift the ref 60 seconds to make sure we always @@ -117,20 +117,159 @@ pub fn (ce &CronExpression) next(ref time.Time) !time.Time { // next_from_now returns the result of ce.next(ref) where ref is the result of // time.now(). -pub fn (ce &CronExpression) next_from_now() !time.Time { +pub fn (ce &CronExpression) next_from_now() ?time.Time { return ce.next(time.now()) } // next_n returns the n next occurences of the expression, given a starting // time. -pub fn (ce &CronExpression) next_n(ref time.Time, n int) ![]time.Time { +pub fn (ce &CronExpression) next_n(ref time.Time, n int) ?[]time.Time { mut times := []time.Time{cap: n} - times << ce.next(ref)! + times << ce.next(ref)? for i in 1 .. n { - times << ce.next(times[i - 1])! + times << ce.next(times[i - 1])? } return times } + +// parse_range parses a given string into a range of sorted integers, if +// possible. +fn parse_range(s string, min int, max int, mut bitv []bool) ? { + mut start := min + mut end := max + mut interval := 1 + + exps := s.split('/') + + if exps.len > 2 { + return error('Invalid expression.') + } + + if exps[0] != '*' { + dash_parts := exps[0].split('-') + + if dash_parts.len > 2 { + return error('Invalid expression.') + } + + start = dash_parts[0].int() + + // The builtin parsing functions return zero if the string can't be + // parsed into a number, so we have to explicitely check whether they + // actually entered zero or if it's an invalid number. + if start == 0 && dash_parts[0] != '0' { + return error('Invalid number.') + } + + // Check whether the start value is out of range + if start < min || start > max { + return error('Out of range.') + } + + if dash_parts.len == 2 { + end = dash_parts[1].int() + + if end == 0 && dash_parts[1] != '0' { + return error('Invalid number.') + } + + if end < start || end > max { + return error('Out of range.') + } + } + } + + if exps.len > 1 { + interval = exps[1].int() + + // interval being zero is always invalid, but we want to check why + // it's invalid for better error messages. + if interval == 0 { + if exps[1] != '0' { + return error('Invalid number.') + } else { + return error('Step size zero not allowed.') + } + } + + if interval > max - min { + return error('Step size too large.') + } + } + // Here, s solely consists of a number, so that's the only value we + // should return. + else if exps[0] != '*' && !exps[0].contains('-') { + bitv[start - min] = true + return + } + + for start <= end { + bitv[start - min] = true + start += interval + } +} + +// bitv_to_ints converts a bit vector into an array containing the +// corresponding values. +fn bitv_to_ints(bitv []bool, min int) []int { + mut out := []int{} + + for i in 0 .. bitv.len { + if bitv[i] { + out << min + i + } + } + + return out +} + +// parse_part parses a given part of a cron expression & returns the +// corresponding array of ints. +fn parse_part(s string, min int, max int) ?[]int { + mut bitv := []bool{len: max - min + 1, init: false} + + for range in s.split(',') { + parse_range(range, min, max, mut bitv)? + } + + return bitv_to_ints(bitv, min) +} + +// parse_expression parses an entire cron expression string into a +// CronExpression object, if possible. +pub fn parse_expression(exp string) ?CronExpression { + // The filter allows for multiple spaces between parts + mut parts := exp.split(' ').filter(it != '') + + if parts.len < 2 || parts.len > 4 { + return error('Expression must contain between 2 and 4 space-separated parts.') + } + + // For ease of use, we allow the user to only specify as many parts as they + // need. + for parts.len < 4 { + parts << '*' + } + + mut part_results := [][]int{} + + mins := [0, 0, 1, 1] + maxs := [59, 23, 31, 12] + + // This for loop allows us to more clearly propagate the error to the user. + for i, min in mins { + part_results << parse_part(parts[i], min, maxs[i]) or { + return error('An error occurred with part $i: $err.msg()') + } + } + + return CronExpression{ + minutes: part_results[0] + hours: part_results[1] + days: part_results[2] + months: part_results[3] + } +} diff --git a/src/cron/expression/expression_parse.v b/src/cron/expression/expression_parse.v deleted file mode 100644 index 4aaec5b..0000000 --- a/src/cron/expression/expression_parse.v +++ /dev/null @@ -1,146 +0,0 @@ -module expression - -import bitfield - -// parse_range parses a given string into a range of sorted integers. Its -// result is a BitField with set bits for all numbers in the result. -fn parse_range(s string, min int, max int) !bitfield.BitField { - mut start := min - mut end := max - mut interval := 1 - mut bf := bitfield.new(max - min + 1) - - exps := s.split('/') - - if exps.len > 2 { - return error('Invalid expression.') - } - - if exps[0] != '*' { - dash_parts := exps[0].split('-') - - if dash_parts.len > 2 { - return error('Invalid expression.') - } - - start = dash_parts[0].int() - - // The builtin parsing functions return zero if the string can't be - // parsed into a number, so we have to explicitely check whether they - // actually entered zero or if it's an invalid number. - if start == 0 && dash_parts[0] != '0' { - return error('Invalid number.') - } - - // Check whether the start value is out of range - if start < min || start > max { - return error('Out of range.') - } - - if dash_parts.len == 2 { - end = dash_parts[1].int() - - if end == 0 && dash_parts[1] != '0' { - return error('Invalid number.') - } - - if end < start || end > max { - return error('Out of range.') - } - } - } - - if exps.len > 1 { - interval = exps[1].int() - - // interval being zero is always invalid, but we want to check why - // it's invalid for better error messages. - if interval == 0 { - if exps[1] != '0' { - return error('Invalid number.') - } else { - return error('Step size zero not allowed.') - } - } - - if interval > max - min { - return error('Step size too large.') - } - } - // Here, s solely consists of a number, so that's the only value we - // should return. - else if exps[0] != '*' && !exps[0].contains('-') { - bf.set_bit(start - min) - return bf - } - - for start <= end { - bf.set_bit(start - min) - start += interval - } - - return bf -} - -// bf_to_ints takes a BitField and converts it into the expected list of actual -// integers. -fn bf_to_ints(bf bitfield.BitField, min int) []int { - mut out := []int{} - - for i in 0 .. bf.get_size() { - if bf.get_bit(i) == 1 { - out << min + i - } - } - - return out -} - -// parse_part parses a given part of a cron expression & returns the -// corresponding array of ints. -fn parse_part(s string, min int, max int) ![]int { - mut bf := bitfield.new(max - min + 1) - - for range in s.split(',') { - bf2 := parse_range(range, min, max)! - bf = bitfield.bf_or(bf, bf2) - } - - return bf_to_ints(bf, min) -} - -// parse_expression parses an entire cron expression string into a -// CronExpression object, if possible. -pub fn parse_expression(exp string) !CronExpression { - // The filter allows for multiple spaces between parts - mut parts := exp.split(' ').filter(it != '') - - if parts.len < 2 || parts.len > 4 { - return error('Expression must contain between 2 and 4 space-separated parts.') - } - - // For ease of use, we allow the user to only specify as many parts as they - // need. - for parts.len < 4 { - parts << '*' - } - - mut part_results := [][]int{} - - mins := [0, 0, 1, 1] - maxs := [59, 23, 31, 12] - - // This for loop allows us to more clearly propagate the error to the user. - for i, min in mins { - part_results << parse_part(parts[i], min, maxs[i]) or { - return error('An error occurred with part $i: $err.msg()') - } - } - - return CronExpression{ - minutes: part_results[0] - hours: part_results[1] - days: part_results[2] - months: part_results[3] - } -} diff --git a/src/cron/expression/expression_parse_test.v b/src/cron/expression/expression_parse_test.v index 92e8291..4eebc49 100644 --- a/src/cron/expression/expression_parse_test.v +++ b/src/cron/expression/expression_parse_test.v @@ -3,87 +3,96 @@ module expression // parse_range_error returns the returned error message. If the result is '', // that means the function didn't error. fn parse_range_error(s string, min int, max int) string { - parse_range(s, min, max) or { return err.msg } + mut bitv := []bool{len: max - min + 1, init: false} + + parse_range(s, min, max, mut bitv) or { return err.msg } return '' } // =====parse_range===== -fn test_range_star_range() ! { - bf := parse_range('*', 0, 5)! +fn test_range_star_range() ? { + mut bitv := []bool{len: 6, init: false} + parse_range('*', 0, 5, mut bitv)? - assert bf_to_ints(bf, 0) == [0, 1, 2, 3, 4, 5] + assert bitv == [true, true, true, true, true, true] } -fn test_range_number() ! { - bf := parse_range('4', 0, 5)! +fn test_range_number() ? { + mut bitv := []bool{len: 6, init: false} + parse_range('4', 0, 5, mut bitv)? - assert bf_to_ints(bf, 0) == [4] + assert bitv_to_ints(bitv, 0) == [4] } -fn test_range_number_too_large() ! { +fn test_range_number_too_large() ? { assert parse_range_error('10', 0, 6) == 'Out of range.' } -fn test_range_number_too_small() ! { +fn test_range_number_too_small() ? { assert parse_range_error('0', 2, 6) == 'Out of range.' } -fn test_range_number_invalid() ! { +fn test_range_number_invalid() ? { assert parse_range_error('x', 0, 6) == 'Invalid number.' } -fn test_range_step_star_1() ! { - bf := parse_range('*/4', 0, 20)! +fn test_range_step_star_1() ? { + mut bitv := []bool{len: 21, init: false} + parse_range('*/4', 0, 20, mut bitv)? - assert bf_to_ints(bf, 0) == [0, 4, 8, 12, 16, 20] + assert bitv_to_ints(bitv, 0) == [0, 4, 8, 12, 16, 20] } -fn test_range_step_star_2() ! { - bf := parse_range('*/3', 1, 8)! +fn test_range_step_star_2() ? { + mut bitv := []bool{len: 8, init: false} + parse_range('*/3', 1, 8, mut bitv)? - assert bf_to_ints(bf, 1) == [1, 4, 7] + assert bitv_to_ints(bitv, 1) == [1, 4, 7] } -fn test_range_step_star_too_large() ! { +fn test_range_step_star_too_large() ? { assert parse_range_error('*/21', 0, 20) == 'Step size too large.' } -fn test_range_step_zero() ! { +fn test_range_step_zero() ? { assert parse_range_error('*/0', 0, 20) == 'Step size zero not allowed.' } -fn test_range_step_number() ! { - bf := parse_range('5/4', 2, 22)! +fn test_range_step_number() ? { + mut bitv := []bool{len: 21, init: false} + parse_range('5/4', 2, 22, mut bitv)? - assert bf_to_ints(bf, 2) == [5, 9, 13, 17, 21] + assert bitv_to_ints(bitv, 2) == [5, 9, 13, 17, 21] } -fn test_range_step_number_too_large() ! { +fn test_range_step_number_too_large() ? { assert parse_range_error('10/4', 0, 5) == 'Out of range.' } -fn test_range_step_number_too_small() ! { +fn test_range_step_number_too_small() ? { assert parse_range_error('2/4', 5, 10) == 'Out of range.' } -fn test_range_dash() ! { - bf := parse_range('4-8', 0, 9)! +fn test_range_dash() ? { + mut bitv := []bool{len: 10, init: false} + parse_range('4-8', 0, 9, mut bitv)? - assert bf_to_ints(bf, 0) == [4, 5, 6, 7, 8] + assert bitv_to_ints(bitv, 0) == [4, 5, 6, 7, 8] } -fn test_range_dash_step() ! { - bf := parse_range('4-8/2', 0, 9)! +fn test_range_dash_step() ? { + mut bitv := []bool{len: 10, init: false} + parse_range('4-8/2', 0, 9, mut bitv)? - assert bf_to_ints(bf, 0) == [4, 6, 8] + assert bitv_to_ints(bitv, 0) == [4, 6, 8] } // =====parse_part===== -fn test_part_single() ! { - assert parse_part('*', 0, 5)! == [0, 1, 2, 3, 4, 5] +fn test_part_single() ? { + assert parse_part('*', 0, 5)? == [0, 1, 2, 3, 4, 5] } -fn test_part_multiple() ! { - assert parse_part('*/2,2/3', 1, 8)! == [1, 2, 3, 5, 7, 8] +fn test_part_multiple() ? { + assert parse_part('*/2,2/3', 1, 8)? == [1, 2, 3, 5, 7, 8] } diff --git a/src/cron/expression/expression_test.v b/src/cron/expression/expression_test.v index 82bf959..9e25e92 100644 --- a/src/cron/expression/expression_test.v +++ b/src/cron/expression/expression_test.v @@ -2,12 +2,12 @@ module expression import time { parse } -fn util_test_time(exp string, t1_str string, t2_str string) ! { - ce := parse_expression(exp)! - t1 := parse(t1_str)! - t2 := parse(t2_str)! +fn util_test_time(exp string, t1_str string, t2_str string) ? { + ce := parse_expression(exp)? + t1 := parse(t1_str)? + t2 := parse(t2_str)? - t3 := ce.next(t1)! + t3 := ce.next(t1)? assert t2.year == t3.year assert t2.month == t3.month @@ -16,19 +16,19 @@ fn util_test_time(exp string, t1_str string, t2_str string) ! { assert t2.minute == t3.minute } -fn test_next_simple() ! { +fn test_next_simple() ? { // Very simple - util_test_time('0 3', '2002-01-01 00:00:00', '2002-01-01 03:00:00')! + util_test_time('0 3', '2002-01-01 00:00:00', '2002-01-01 03:00:00')? // Overlap to next day - util_test_time('0 3', '2002-01-01 03:00:00', '2002-01-02 03:00:00')! - util_test_time('0 3', '2002-01-01 04:00:00', '2002-01-02 03:00:00')! + util_test_time('0 3', '2002-01-01 03:00:00', '2002-01-02 03:00:00')? + util_test_time('0 3', '2002-01-01 04:00:00', '2002-01-02 03:00:00')? - util_test_time('0 3/4', '2002-01-01 04:00:00', '2002-01-01 07:00:00')! + util_test_time('0 3/4', '2002-01-01 04:00:00', '2002-01-01 07:00:00')? // Overlap to next month - util_test_time('0 3', '2002-11-31 04:00:00', '2002-12-01 03:00:00')! + util_test_time('0 3', '2002-11-31 04:00:00', '2002-12-01 03:00:00')? // Overlap to next year - util_test_time('0 3', '2002-12-31 04:00:00', '2003-01-01 03:00:00')! + util_test_time('0 3', '2002-12-31 04:00:00', '2003-01-01 03:00:00')? } diff --git a/src/db/db.v b/src/db/db.v index 98ee000..9459c05 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -17,21 +17,17 @@ const ( $embed_file('migrations/001-initial/up.sql'), $embed_file('migrations/002-rename-to-targets/up.sql'), $embed_file('migrations/003-target-url-type/up.sql'), - $embed_file('migrations/004-nullable-branch/up.sql'), - $embed_file('migrations/005-repo-path/up.sql'), ] migrations_down = [ $embed_file('migrations/001-initial/down.sql'), $embed_file('migrations/002-rename-to-targets/down.sql'), $embed_file('migrations/003-target-url-type/down.sql'), - $embed_file('migrations/004-nullable-branch/down.sql'), - $embed_file('migrations/005-repo-path/down.sql'), ] ) // init initializes a database & adds the correct tables. -pub fn init(db_path string) !VieterDb { - conn := sqlite.connect(db_path)! +pub fn init(db_path string) ?VieterDb { + conn := sqlite.connect(db_path)? sql conn { create table MigrationVersion @@ -64,7 +60,7 @@ pub fn init(db_path string) !VieterDb { res := conn.exec_none(part) if res != sqlite.sqlite_done { - return error('An error occurred while applying migration $version_num: SQLite error code $res') + return error('An error occurred while applying migration $version_num') } } diff --git a/src/db/logs.v b/src/db/logs.v index 2745467..923dde2 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -84,8 +84,6 @@ pub fn (db &VieterDb) add_build_log(log BuildLog) int { insert log into BuildLog } - // Here, this does work because a log doesn't contain any foreign keys, - // meaning the ORM only has to do a single add inserted_id := db.conn.last_id() as int return inserted_id diff --git a/src/db/migrations/004-nullable-branch/down.sql b/src/db/migrations/004-nullable-branch/down.sql deleted file mode 100644 index 2515593..0000000 --- a/src/db/migrations/004-nullable-branch/down.sql +++ /dev/null @@ -1,26 +0,0 @@ --- This down won't really work because it'll throw NOT NULL errors, but I'm --- just putting it here for future reference (still not sure whether I'm even - -- gonna use these) -PRAGMA foreign_keys=off; - -BEGIN TRANSACTION; - -ALTER TABLE Target RENAME TO _Target_old; - -CREATE TABLE Target ( - id INTEGER PRIMARY KEY, - url TEXT NOT NULL, - branch TEXT NOT NULL, - repo TEXT NOT NULL, - schedule TEXT, - kind TEXT NOT NULL DEFAULT 'git' -); - -INSERT INTO Target (id, url, branch, repo, schedule, kind) - SELECT id, url, branch, repo, schedule, kind FROM _Target_old; - -DROP TABLE _Target_old; - -COMMIT; - -PRAGMA foreign_keys=on; diff --git a/src/db/migrations/004-nullable-branch/up.sql b/src/db/migrations/004-nullable-branch/up.sql deleted file mode 100644 index 6333c37..0000000 --- a/src/db/migrations/004-nullable-branch/up.sql +++ /dev/null @@ -1,23 +0,0 @@ -PRAGMA foreign_keys=off; - -BEGIN TRANSACTION; - -ALTER TABLE Target RENAME TO _Target_old; - -CREATE TABLE Target ( - id INTEGER PRIMARY KEY, - url TEXT NOT NULL, - branch TEXT, - repo TEXT NOT NULL, - schedule TEXT, - kind TEXT NOT NULL DEFAULT 'git' -); - -INSERT INTO Target (id, url, branch, repo, schedule, kind) - SELECT id, url, branch, repo, schedule, kind FROM _Target_old; - -DROP TABLE _Target_old; - -COMMIT; - -PRAGMA foreign_keys=on; diff --git a/src/db/migrations/005-repo-path/down.sql b/src/db/migrations/005-repo-path/down.sql deleted file mode 100644 index 8a6f021..0000000 --- a/src/db/migrations/005-repo-path/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE Target DROP COLUMN path; diff --git a/src/db/migrations/005-repo-path/up.sql b/src/db/migrations/005-repo-path/up.sql deleted file mode 100644 index f7e5c29..0000000 --- a/src/db/migrations/005-repo-path/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE Target ADD COLUMN path TEXT; diff --git a/src/db/targets.v b/src/db/targets.v index 41e56df..a705ebb 100644 --- a/src/db/targets.v +++ b/src/db/targets.v @@ -38,17 +38,14 @@ pub fn (db &VieterDb) get_target(target_id int) ?Target { } // add_target inserts the given target into the database. -pub fn (db &VieterDb) add_target(target Target) int { +pub fn (db &VieterDb) add_target(repo Target) int { sql db.conn { - insert target into Target + insert repo into Target } - // ID of inserted target is the largest id - inserted_target := sql db.conn { - select from Target order by id desc limit 1 - } + inserted_id := db.conn.last_id() as int - return inserted_target.id + return inserted_id } // delete_target deletes the target with the given id from the database. diff --git a/src/main.v b/src/main.v index 1053c2f..fc09f7e 100644 --- a/src/main.v +++ b/src/main.v @@ -9,18 +9,12 @@ import console.schedule import console.man import console.aur import cron -import agent fn main() { - // Stop buffering output so logs always show up immediately - unsafe { - C.setbuf(C.stdout, 0) - } - mut app := cli.Command{ name: 'vieter' description: 'Vieter is a lightweight implementation of an Arch repository server.' - version: '0.5.0-rc.1' + version: '0.4.0' flags: [ cli.Flag{ flag: cli.FlagType.string @@ -46,7 +40,6 @@ fn main() { schedule.cmd(), man.cmd(), aur.cmd(), - agent.cmd(), ] } app.setup() diff --git a/src/models/builds.v b/src/models/builds.v deleted file mode 100644 index 926a53c..0000000 --- a/src/models/builds.v +++ /dev/null @@ -1,18 +0,0 @@ -module models - -pub struct BuildConfig { -pub: - target_id int - kind string - url string - branch string - path string - repo string - base_image string - force bool -} - -// str return a single-line string representation of a build log -pub fn (c BuildConfig) str() string { - return '{ target: $c.target_id, kind: $c.kind, url: $c.url, branch: $c.branch, path: $c.path, repo: $c.repo, base_image: $c.base_image, force: $c.force }' -} diff --git a/src/models/targets.v b/src/models/targets.v index af3cb0d..c8aa535 100644 --- a/src/models/targets.v +++ b/src/models/targets.v @@ -28,45 +28,27 @@ pub mut: repo string [nonull] // Cron schedule describing how frequently to build the repo. schedule string - // Subdirectory in the Git repository to cd into - path string // On which architectures the package is allowed to be built. In reality, - // this controls which agents will build this package when scheduled. + // this controls which builders will periodically build the image. arch []TargetArch [fkey: 'target_id'] } // str returns a string representation. -pub fn (t &Target) str() string { +pub fn (gr &Target) str() string { mut parts := [ - 'id: $t.id', - 'kind: $t.kind', - 'url: $t.url', - 'branch: $t.branch', - 'path: $t.path', - 'repo: $t.repo', - 'schedule: $t.schedule', - 'arch: ${t.arch.map(it.value).join(', ')}', + 'id: $gr.id', + 'kind: $gr.kind', + 'url: $gr.url', + 'branch: $gr.branch', + 'repo: $gr.repo', + 'schedule: $gr.schedule', + 'arch: ${gr.arch.map(it.value).join(', ')}', ] str := parts.join('\n') return str } -// as_build_config converts a Target into a BuildConfig, given some extra -// needed information. -pub fn (t &Target) as_build_config(base_image string, force bool) BuildConfig { - return BuildConfig{ - target_id: t.id - kind: t.kind - url: t.url - branch: t.branch - path: t.path - repo: t.repo - base_image: base_image - force: force - } -} - [params] pub struct TargetFilter { pub mut: diff --git a/src/package/README.md b/src/package/README.md deleted file mode 100644 index b2bcbd7..0000000 --- a/src/package/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# package - -This module handles both parsing the published Arch tarballs & the contents of -their `.PKGINFO` files, as well as generating the contents of the database -archives' `desc` & `files` files. diff --git a/src/package/format.v b/src/package/format.v deleted file mode 100644 index a81d327..0000000 --- a/src/package/format.v +++ /dev/null @@ -1,103 +0,0 @@ -module package - -// format_entry returns a string properly formatted to be added to a desc file. -[inline] -fn format_entry(key string, value string) string { - return '\n%$key%\n$value\n' -} - -// full_name returns the properly formatted name for the package, including -// version & architecture -pub fn (pkg &Pkg) full_name() string { - p := pkg.info - return '$p.name-$p.version-$p.arch' -} - -// filename returns the correct filename of the package file -pub fn (pkg &Pkg) filename() string { - ext := match pkg.compression { - 0 { '.tar' } - 1 { '.tar.gz' } - 6 { '.tar.xz' } - 14 { '.tar.zst' } - else { panic("Another compression code shouldn't be possible. Faulty code: $pkg.compression") } - } - - return '${pkg.full_name()}.pkg$ext' -} - -// to_desc returns a desc file valid string representation -pub fn (pkg &Pkg) to_desc() !string { - p := pkg.info - - // filename - mut desc := '%FILENAME%\n$pkg.filename()\n' - - desc += format_entry('NAME', p.name) - desc += format_entry('BASE', p.base) - desc += format_entry('VERSION', p.version) - - if p.description.len > 0 { - desc += format_entry('DESC', p.description) - } - - if p.groups.len > 0 { - desc += format_entry('GROUPS', p.groups.join_lines()) - } - - desc += format_entry('CSIZE', p.csize.str()) - desc += format_entry('ISIZE', p.size.str()) - - sha256sum := pkg.checksum()! - - desc += format_entry('SHA256SUM', sha256sum) - - // TODO add pgpsig stuff - - if p.url.len > 0 { - desc += format_entry('URL', p.url) - } - - if p.licenses.len > 0 { - desc += format_entry('LICENSE', p.licenses.join_lines()) - } - - desc += format_entry('ARCH', p.arch) - desc += format_entry('BUILDDATE', p.build_date.str()) - desc += format_entry('PACKAGER', p.packager) - - if p.replaces.len > 0 { - desc += format_entry('REPLACES', p.replaces.join_lines()) - } - - if p.conflicts.len > 0 { - desc += format_entry('CONFLICTS', p.conflicts.join_lines()) - } - - if p.provides.len > 0 { - desc += format_entry('PROVIDES', p.provides.join_lines()) - } - - if p.depends.len > 0 { - desc += format_entry('DEPENDS', p.depends.join_lines()) - } - - if p.optdepends.len > 0 { - desc += format_entry('OPTDEPENDS', p.optdepends.join_lines()) - } - - if p.makedepends.len > 0 { - desc += format_entry('MAKEDEPENDS', p.makedepends.join_lines()) - } - - if p.checkdepends.len > 0 { - desc += format_entry('CHECKDEPENDS', p.checkdepends.join_lines()) - } - - return '$desc\n' -} - -// to_files returns a files file valid string representation -pub fn (pkg &Pkg) to_files() string { - return '%FILES%\n$pkg.files.join_lines()\n' -} diff --git a/src/package/package.v b/src/package/package.v index 4518ffd..9eaf5a2 100644 --- a/src/package/package.v +++ b/src/package/package.v @@ -43,12 +43,12 @@ pub mut: } // checksum calculates the sha256 hash of the package -pub fn (p &Pkg) checksum() !string { +pub fn (p &Pkg) checksum() ?string { return util.hash_file(p.path) } // parse_pkg_info_string parses a PkgInfo object from a string -fn parse_pkg_info_string(pkg_info_str &string) !PkgInfo { +fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { mut pkg_info := PkgInfo{} // Iterate over the entire string @@ -101,7 +101,7 @@ fn parse_pkg_info_string(pkg_info_str &string) !PkgInfo { // read_pkg_archive extracts the file list & .PKGINFO contents from an archive // NOTE: this command only supports zstd-, xz- & gzip-compressed tarballs. -pub fn read_pkg_archive(pkg_path string) !Pkg { +pub fn read_pkg_archive(pkg_path string) ?Pkg { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") } @@ -159,7 +159,7 @@ pub fn read_pkg_archive(pkg_path string) !Pkg { pkg_text := unsafe { buf.vstring_with_len(size).clone() } - pkg_info = parse_pkg_info_string(pkg_text)! + pkg_info = parse_pkg_info_string(pkg_text)? } else { C.archive_read_data_skip(a) } @@ -174,3 +174,104 @@ pub fn read_pkg_archive(pkg_path string) !Pkg { compression: compression_code } } + +// format_entry returns a string properly formatted to be added to a desc file. +fn format_entry(key string, value string) string { + return '\n%$key%\n$value\n' +} + +// full_name returns the properly formatted name for the package, including +// version & architecture +pub fn (pkg &Pkg) full_name() string { + p := pkg.info + return '$p.name-$p.version-$p.arch' +} + +// filename returns the correct filename of the package file +pub fn (pkg &Pkg) filename() string { + ext := match pkg.compression { + 0 { '.tar' } + 1 { '.tar.gz' } + 6 { '.tar.xz' } + 14 { '.tar.zst' } + else { panic("Another compression code shouldn't be possible. Faulty code: $pkg.compression") } + } + + return '${pkg.full_name()}.pkg$ext' +} + +// to_desc returns a desc file valid string representation +pub fn (pkg &Pkg) to_desc() ?string { + p := pkg.info + + // filename + mut desc := '%FILENAME%\n$pkg.filename()\n' + + desc += format_entry('NAME', p.name) + desc += format_entry('BASE', p.base) + desc += format_entry('VERSION', p.version) + + if p.description.len > 0 { + desc += format_entry('DESC', p.description) + } + + if p.groups.len > 0 { + desc += format_entry('GROUPS', p.groups.join_lines()) + } + + desc += format_entry('CSIZE', p.csize.str()) + desc += format_entry('ISIZE', p.size.str()) + + sha256sum := pkg.checksum()? + + desc += format_entry('SHA256SUM', sha256sum) + + // TODO add pgpsig stuff + + if p.url.len > 0 { + desc += format_entry('URL', p.url) + } + + if p.licenses.len > 0 { + desc += format_entry('LICENSE', p.licenses.join_lines()) + } + + desc += format_entry('ARCH', p.arch) + desc += format_entry('BUILDDATE', p.build_date.str()) + desc += format_entry('PACKAGER', p.packager) + + if p.replaces.len > 0 { + desc += format_entry('REPLACES', p.replaces.join_lines()) + } + + if p.conflicts.len > 0 { + desc += format_entry('CONFLICTS', p.conflicts.join_lines()) + } + + if p.provides.len > 0 { + desc += format_entry('PROVIDES', p.provides.join_lines()) + } + + if p.depends.len > 0 { + desc += format_entry('DEPENDS', p.depends.join_lines()) + } + + if p.optdepends.len > 0 { + desc += format_entry('OPTDEPENDS', p.optdepends.join_lines()) + } + + if p.makedepends.len > 0 { + desc += format_entry('MAKEDEPENDS', p.makedepends.join_lines()) + } + + if p.checkdepends.len > 0 { + desc += format_entry('CHECKDEPENDS', p.checkdepends.join_lines()) + } + + return '$desc\n' +} + +// to_files returns a files file valid string representation +pub fn (pkg &Pkg) to_files() string { + return '%FILES%\n$pkg.files.join_lines()\n' +} diff --git a/src/repo/README.md b/src/repo/README.md deleted file mode 100644 index f06b1d3..0000000 --- a/src/repo/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# repo - -This module manages the contents of the various repositories stored within a -Vieter instance. - -## Terminology - -* Arch-repository (arch-repo): specific architecture of a given repository. This is what - Pacman actually uses as a repository, and contains its own `.db` & `.files` - files. -* Repository (repo): a collection of arch-repositories. A single repository can - contain packages of different architectures, with each package being stored - in that specific architecture' arch-repository. -* Repository group (repo-group): a collection of repositories. Each Vieter - instance consists of a single repository group, which manages all underlying - repositories & arch-repositories. - -## Arch-repository layout - -An arch-repository (aka a regular Pacman repository) consists of a directory -with the following files (`{repo}` should be replaced with the name of the -repository): - -* One or more package directories. These directories follow the naming scheme - `${pkgname}-${pkgver}-${pkgrel}`. Each of these directories contains two - files, `desc` & `files`. The `desc` file is a list of the package's metadata, - while `files` contains a list of all files that the package contains. The - latter is used when using `pacman -F`. -* `{repo}.db` & `{repo}.db.tar.gz`: the database file of the repository. This - is just a compressed tarball of all package directories, but only their - `desc` files. Both these files should have the same content (`repo-add` - creates a symlink, but Vieter just serves the same file for both routes) -* `{repo}.files` & `{repo}.files.tar.gz`: the same as the `.db` file, but this - also contains the `files` files, instead of just the `desc` files. - -## Filesystem layout - -The repository part of Vieter consists of two directories. One is the `repos` -directory inside the configured `data_dir`, while the other is the configured -`pkg_dir`. `repos` contains only the repository group, while `pkg_dir` contains -the actual package archives. `pkg_dir` is the directory that can take up a -significant amount of memory, while `repos` solely consists of small text -files. diff --git a/src/repo/add.v b/src/repo/add.v index 8ab3ae1..608ca50 100644 --- a/src/repo/add.v +++ b/src/repo/add.v @@ -29,7 +29,7 @@ pub: } // new creates a new RepoGroupManager & creates the directories as needed -pub fn new(repos_dir string, pkg_dir string, default_arch string) !RepoGroupManager { +pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupManager { if !os.is_dir(repos_dir) { os.mkdir_all(repos_dir) or { return error('Failed to create repos directory: $err.msg()') } } @@ -49,27 +49,27 @@ pub fn new(repos_dir string, pkg_dir string, default_arch string) !RepoGroupMana // pkg archive. It's a wrapper around add_pkg_in_repo that parses the archive // file, passes the result to add_pkg_in_repo, and hard links the archive to // the right subdirectories in r.pkg_dir if it was successfully added. -pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) !RepoAddResult { +pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult { pkg := package.read_pkg_archive(pkg_path) or { return error('Failed to read package file: $err.msg()') } - archs := r.add_pkg_in_repo(repo, pkg)! + archs := r.add_pkg_in_repo(repo, pkg)? // If the add was successful, we move the file to the packages directory for arch in archs { repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch)) dest_path := os.join_path_single(repo_pkg_path, pkg.filename()) - os.mkdir_all(repo_pkg_path)! + os.mkdir_all(repo_pkg_path)? // We create hard links so that "any" arch packages aren't stored // multiple times - os.link(pkg_path, dest_path)! + os.link(pkg_path, dest_path)? } // After linking, we can remove the original file - os.rm(pkg_path)! + os.rm(pkg_path)? return RepoAddResult{ name: pkg.info.name @@ -85,11 +85,11 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) !Re // r.default_arch. If this arch-repo doesn't exist yet, it is created. If the // architecture isn't 'any', the package is only added to the specific // architecture. -fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ![]string { +fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]string { // A package not of arch 'any' can be handled easily by adding it to the // respective repo if pkg.info.arch != 'any' { - r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)! + r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)? return [pkg.info.arch] } @@ -104,7 +104,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ![]strin // If this is the first package that's added to the repo, the directory // won't exist yet if os.exists(repo_dir) { - arch_repos = os.ls(repo_dir)! + arch_repos = os.ls(repo_dir)? } // The default_arch should always be updated when a package with arch 'any' @@ -118,7 +118,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ![]strin // not know which arch-repositories did succeed in adding the package, if // any. for arch in arch_repos { - r.add_pkg_in_arch_repo(repo, arch, pkg)! + r.add_pkg_in_arch_repo(repo, arch, pkg)? } return arch_repos @@ -128,24 +128,24 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ![]strin // arch-repo. It records the package's data in the arch-repo's desc & files // files, and afterwards updates the db & files archives to reflect these // changes. -fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ! { +fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ? { pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version') // Remove the previous version of the package, if present - r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false)! + r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false)? os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') } - os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()!) or { - os.rmdir_all(pkg_dir)! + os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()?) or { + os.rmdir_all(pkg_dir)? return error('Failed to write desc file.') } os.write_file(os.join_path_single(pkg_dir, 'files'), pkg.to_files()) or { - os.rmdir_all(pkg_dir)! + os.rmdir_all(pkg_dir)? return error('Failed to write files file.') } - r.sync(repo, arch)! + r.sync(repo, arch)? } diff --git a/src/repo/remove.v b/src/repo/remove.v index 63866a9..add921c 100644 --- a/src/repo/remove.v +++ b/src/repo/remove.v @@ -5,7 +5,7 @@ import os // remove_pkg_from_arch_repo removes a package from an arch-repo's database. It // returns false if the package wasn't present in the database. It also // optionally re-syncs the repo archives. -pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) !bool { +pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool { repo_dir := os.join_path(r.repos_dir, repo, arch) // If the repository doesn't exist yet, the result is automatically false @@ -15,7 +15,7 @@ pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, // We iterate over every directory in the repo dir // TODO filter so we only check directories - for d in os.ls(repo_dir)! { + for d in os.ls(repo_dir)? { // Because a repository only allows a single version of each package, // we need only compare whether the name of the package is the same, // not the version. @@ -25,22 +25,22 @@ pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, // We lock the mutex here to prevent other routines from creating a // new archive while we remove an entry lock r.mutex { - os.rmdir_all(os.join_path_single(repo_dir, d))! + os.rmdir_all(os.join_path_single(repo_dir, d))? } // Also remove the package archive repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch) - archives := os.ls(repo_pkg_dir)!.filter(it.split('-')#[..-3].join('-') == name) + archives := os.ls(repo_pkg_dir)?.filter(it.split('-')#[..-3].join('-') == name) for archive_name in archives { full_path := os.join_path_single(repo_pkg_dir, archive_name) - os.rm(full_path)! + os.rm(full_path)? } // Sync the db archives if requested if sync { - r.sync(repo, arch)! + r.sync(repo, arch)? } return true @@ -51,7 +51,7 @@ pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, } // remove_arch_repo removes an arch-repo & its packages. -pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) !bool { +pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) ?bool { repo_dir := os.join_path(r.repos_dir, repo, arch) // If the repository doesn't exist yet, the result is automatically false @@ -59,16 +59,16 @@ pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) !bool { return false } - os.rmdir_all(repo_dir)! + os.rmdir_all(repo_dir)? pkg_dir := os.join_path(r.pkg_dir, repo, arch) - os.rmdir_all(pkg_dir)! + os.rmdir_all(pkg_dir)? return true } // remove_repo removes a repo & its packages. -pub fn (r &RepoGroupManager) remove_repo(repo string) !bool { +pub fn (r &RepoGroupManager) remove_repo(repo string) ?bool { repo_dir := os.join_path_single(r.repos_dir, repo) // If the repository doesn't exist yet, the result is automatically false @@ -76,10 +76,10 @@ pub fn (r &RepoGroupManager) remove_repo(repo string) !bool { return false } - os.rmdir_all(repo_dir)! + os.rmdir_all(repo_dir)? pkg_dir := os.join_path_single(r.pkg_dir, repo) - os.rmdir_all(pkg_dir)! + os.rmdir_all(pkg_dir)? return true } diff --git a/src/repo/sync.v b/src/repo/sync.v index 9554748..73d21c8 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -32,7 +32,7 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri } // sync regenerates the repository archive files. -fn (r &RepoGroupManager) sync(repo string, arch string) ! { +fn (r &RepoGroupManager) sync(repo string, arch string) ? { subrepo_path := os.join_path(r.repos_dir, repo, arch) lock r.mutex { @@ -54,7 +54,7 @@ fn (r &RepoGroupManager) sync(repo string, arch string) ! { C.archive_write_open_filename(a_files, &char(files_path.str)) // Iterate over each directory - for d in os.ls(subrepo_path)!.filter(os.is_dir(os.join_path_single(subrepo_path, + for d in os.ls(subrepo_path)?.filter(os.is_dir(os.join_path_single(subrepo_path, it))) { // desc mut inner_path := os.join_path_single(d, 'desc') diff --git a/src/server/api_jobs.v b/src/server/api_jobs.v deleted file mode 100644 index 7795351..0000000 --- a/src/server/api_jobs.v +++ /dev/null @@ -1,49 +0,0 @@ -module server - -import web -import web.response { new_data_response, new_response } - -// v1_poll_job_queue allows agents to poll for new build jobs. -['/api/v1/jobs/poll'; auth; get] -fn (mut app App) v1_poll_job_queue() web.Result { - arch := app.query['arch'] or { - return app.json(.bad_request, new_response('Missing arch query arg.')) - } - - max_str := app.query['max'] or { - return app.json(.bad_request, new_response('Missing max query arg.')) - } - max := max_str.int() - - mut out := app.job_queue.pop_n(arch, max).map(it.config) - - return app.json(.ok, new_data_response(out)) -} - -// v1_queue_job allows queueing a new one-time build job for the given target. -['/api/v1/jobs/queue'; auth; post] -fn (mut app App) v1_queue_job() web.Result { - target_id := app.query['target'] or { - return app.json(.bad_request, new_response('Missing target query arg.')) - }.int() - - arch := app.query['arch'] or { - return app.json(.bad_request, new_response('Missing arch query arg.')) - } - - if arch == '' { - app.json(.bad_request, new_response('Empty arch query arg.')) - } - - force := 'force' in app.query - - target := app.db.get_target(target_id) or { - return app.json(.bad_request, new_response('Unknown target id.')) - } - - app.job_queue.insert(target: target, arch: arch, single: true, now: true, force: force) or { - return app.status(.internal_server_error) - } - - return app.status(.ok) -} diff --git a/src/server/api_logs.v b/src/server/api_logs.v index c7521dd..287755a 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -1,6 +1,7 @@ module server import web +import net.http import net.urllib import web.response { new_data_response, new_response } import db @@ -14,7 +15,7 @@ import models { BuildLog, BuildLogFilter } ['/api/v1/logs'; auth; get] fn (mut app App) v1_get_logs() web.Result { filter := models.from_params(app.query) or { - return app.json(.bad_request, new_response('Invalid query parameters.')) + return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } logs := app.db.get_build_logs(filter) @@ -24,7 +25,7 @@ fn (mut app App) v1_get_logs() web.Result { // v1_get_single_log returns the build log with the given id. ['/api/v1/logs/:id'; auth; get] fn (mut app App) v1_get_single_log(id int) web.Result { - log := app.db.get_build_log(id) or { return app.status(.not_found) } + log := app.db.get_build_log(id) or { return app.not_found() } return app.json(.ok, new_data_response(log)) } @@ -32,7 +33,7 @@ fn (mut app App) v1_get_single_log(id int) web.Result { // v1_get_log_content returns the actual build log file for the given id. ['/api/v1/logs/:id/content'; auth; get] fn (mut app App) v1_get_log_content(id int) web.Result { - log := app.db.get_build_log(id) or { return app.status(.not_found) } + log := app.db.get_build_log(id) or { return app.not_found() } file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), log.arch, file_name) @@ -42,9 +43,9 @@ fn (mut app App) v1_get_log_content(id int) web.Result { // parse_query_time unescapes an HTTP query parameter & tries to parse it as a // time.Time struct. -fn parse_query_time(query string) !time.Time { - unescaped := urllib.query_unescape(query)! - t := time.parse(unescaped)! +fn parse_query_time(query string) ?time.Time { + unescaped := urllib.query_unescape(query)? + t := time.parse(unescaped)? return t } @@ -56,25 +57,25 @@ fn (mut app App) v1_post_log() web.Result { start_time_int := app.query['startTime'].int() if start_time_int == 0 { - return app.json(.bad_request, new_response('Invalid or missing start time.')) + return app.json(http.Status.bad_request, new_response('Invalid or missing start time.')) } start_time := time.unix(start_time_int) end_time_int := app.query['endTime'].int() if end_time_int == 0 { - return app.json(.bad_request, new_response('Invalid or missing end time.')) + return app.json(http.Status.bad_request, new_response('Invalid or missing end time.')) } end_time := time.unix(end_time_int) if 'exitCode' !in app.query { - return app.json(.bad_request, new_response('Missing exit code.')) + return app.json(http.Status.bad_request, new_response('Missing exit code.')) } exit_code := app.query['exitCode'].int() if 'arch' !in app.query { - return app.json(.bad_request, new_response("Missing parameter 'arch'.")) + return app.json(http.Status.bad_request, new_response("Missing parameter 'arch'.")) } arch := app.query['arch'] @@ -82,7 +83,7 @@ fn (mut app App) v1_post_log() web.Result { target_id := app.query['target'].int() if !app.db.target_exists(target_id) { - return app.json(.bad_request, new_response('Unknown target.')) + return app.json(http.Status.bad_request, new_response('Unknown target.')) } // Store log in db @@ -104,7 +105,7 @@ fn (mut app App) v1_post_log() web.Result { os.mkdir_all(repo_logs_dir) or { app.lerror("Couldn't create dir '$repo_logs_dir'.") - return app.status(.internal_server_error) + return app.json(http.Status.internal_server_error, new_response('An error occured while processing the request.')) } } @@ -116,10 +117,10 @@ fn (mut app App) v1_post_log() web.Result { util.reader_to_file(mut app.reader, length.int(), full_path) or { app.lerror('An error occured while receiving logs: $err.msg()') - return app.status(.internal_server_error) + return app.json(http.Status.internal_server_error, new_response('Failed to upload logs.')) } } else { - return app.status(.length_required) + return app.status(http.Status.length_required) } return app.json(.ok, new_data_response(log_id)) diff --git a/src/server/api_targets.v b/src/server/api_targets.v index cd5cb0a..6f284af 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -1,6 +1,7 @@ module server import web +import net.http import web.response { new_data_response, new_response } import db import models { Target, TargetArch, TargetFilter } @@ -9,19 +10,19 @@ import models { Target, TargetArch, TargetFilter } ['/api/v1/targets'; auth; get] fn (mut app App) v1_get_targets() web.Result { filter := models.from_params(app.query) or { - return app.json(.bad_request, new_response('Invalid query parameters.')) + return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } - targets := app.db.get_targets(filter) + repos := app.db.get_targets(filter) - return app.json(.ok, new_data_response(targets)) + return app.json(.ok, new_data_response(repos)) } // v1_get_single_target returns the information for a single target. ['/api/v1/targets/:id'; auth; get] fn (mut app App) v1_get_single_target(id int) web.Result { - target := app.db.get_target(id) or { return app.status(.not_found) } + repo := app.db.get_target(id) or { return app.not_found() } - return app.json(.ok, new_data_response(target)) + return app.json(.ok, new_data_response(repo)) } // v1_post_target creates a new target from the provided query string. @@ -29,36 +30,30 @@ fn (mut app App) v1_get_single_target(id int) web.Result { fn (mut app App) v1_post_target() web.Result { mut params := app.query.clone() - // If a target is created without specifying the arch, we assume it's meant + // If a repo is created without specifying the arch, we assume it's meant // for the default architecture. - if 'arch' !in params || params['arch'] == '' { + if 'arch' !in params { params['arch'] = app.conf.default_arch } - mut new_target := models.from_params(params) or { - return app.json(.bad_request, new_response(err.msg())) + new_repo := models.from_params(params) or { + return app.json(http.Status.bad_request, new_response(err.msg())) } // Ensure someone doesn't submit an invalid kind - if new_target.kind !in models.valid_kinds { - return app.json(.bad_request, new_response('Invalid kind.')) + if new_repo.kind !in models.valid_kinds { + return app.json(http.Status.bad_request, new_response('Invalid kind.')) } - id := app.db.add_target(new_target) - new_target.id = id + id := app.db.add_target(new_repo) - // Add the target to the job queue - // TODO return better error here if it's the cron schedule that's incorrect - app.job_queue.insert_all(new_target) or { return app.status(.internal_server_error) } - - return app.json(.ok, new_data_response(id)) + return app.json(http.Status.ok, new_data_response(id)) } // v1_delete_target removes a given target from the server's list. ['/api/v1/targets/:id'; auth; delete] fn (mut app App) v1_delete_target(id int) web.Result { app.db.delete_target(id) - app.job_queue.invalidate(id) return app.status(.ok) } @@ -74,10 +69,5 @@ fn (mut app App) v1_patch_target(id int) web.Result { app.db.update_target_archs(id, arch_objs) } - target := app.db.get_target(id) or { return app.status(.internal_server_error) } - - app.job_queue.invalidate(id) - app.job_queue.insert_all(target) or { return app.status(.internal_server_error) } - - return app.json(.ok, new_data_response(target)) + return app.status(.ok) } diff --git a/src/server/cli.v b/src/server/cli.v index 2fede6c..6fd09c5 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -1,18 +1,16 @@ module server import cli -import conf as vconf +import vieter_v.conf as vconf struct Config { pub: - log_level string = 'WARN' - pkg_dir string - data_dir string - api_key string - default_arch string - global_schedule string = '0 3' - port int = 8000 - base_image string = 'archlinux:base-devel' + log_level string = 'WARN' + pkg_dir string + data_dir string + api_key string + default_arch string + port int = 8000 } // cmd returns the cli submodule that handles starting the server @@ -20,11 +18,11 @@ pub fn cmd() cli.Command { return cli.Command{ name: 'server' description: 'Start the Vieter server.' - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? - server(conf)! + server(conf)? } } } diff --git a/src/server/server.v b/src/server/server.v index 74b1f37..9903cea 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -6,8 +6,6 @@ import log import repo import util import db -import build { BuildJobQueue } -import cron.expression const ( log_file_name = 'vieter.log' @@ -22,39 +20,16 @@ pub: conf Config [required; web_global] pub mut: repo repo.RepoGroupManager [required; web_global] - // Keys are the various architectures for packages - job_queue BuildJobQueue [required; web_global] - db db.VieterDb -} - -// init_job_queue populates a fresh job queue with all the targets currently -// stored in the database. -fn (mut app App) init_job_queue() ! { - // Initialize build queues - mut targets := app.db.get_targets(limit: 25) - mut i := u64(0) - - for targets.len > 0 { - for target in targets { - app.job_queue.insert_all(target)! - } - - i += 25 - targets = app.db.get_targets(limit: 25, offset: i) - } + db db.VieterDb } // server starts the web server & starts listening for requests -pub fn server(conf Config) ! { +pub fn server(conf Config) ? { // Prevent using 'any' as the default arch if conf.default_arch == 'any' { util.exit_with_message(1, "'any' is not allowed as the value for default_arch.") } - global_ce := expression.parse_expression(conf.global_schedule) or { - util.exit_with_message(1, 'Invalid global cron expression: $err.msg()') - } - // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -96,17 +71,11 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Failed to initialize database: $err.msg()') } - mut app := &App{ + web.run(&App{ logger: logger api_key: conf.api_key conf: conf repo: repo db: db - job_queue: build.new_job_queue(global_ce, conf.base_image) - } - app.init_job_queue() or { - util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()') - } - - web.run(app, conf.port) + }, conf.port) } diff --git a/src/util/stream.v b/src/util/stream.v index 15cc618..06397aa 100644 --- a/src/util/stream.v +++ b/src/util/stream.v @@ -5,7 +5,7 @@ import io import os // reader_to_writer tries to consume the entire reader & write it to the writer. -pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ! { +pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ? { mut buf := []u8{len: 10 * 1024} for { @@ -21,8 +21,8 @@ pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ! { } // reader_to_file writes the contents of a BufferedReader to a file -pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ! { - mut file := os.create(path)! +pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { + mut file := os.create(path)? defer { file.close() } @@ -69,11 +69,11 @@ pub fn match_array_in_array(a1 []T, a2 []T) int { // read_until_separator consumes an io.Reader until it encounters some // separator array. The data read is stored inside the provided res array. -pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ! { +pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ? { mut buf := []u8{len: sep.len} for { - c := reader.read(mut buf)! + c := reader.read(mut buf)? res << buf[..c] match_len := match_array_in_array(buf[..c], sep) @@ -84,7 +84,7 @@ pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ! { if match_len > 0 { match_left := sep.len - match_len - c2 := reader.read(mut buf[..match_left])! + c2 := reader.read(mut buf[..match_left])? res << buf[..c2] if buf[..c2] == sep[match_len..] { diff --git a/src/util/util.v b/src/util/util.v index 213104c..4cd374e 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -23,7 +23,7 @@ pub fn exit_with_message(code int, msg string) { } // hash_file returns the sha256 hash of a given file -pub fn hash_file(path &string) !string { +pub fn hash_file(path &string) ?string { file := os.open(path) or { return error('Failed to open file.') } mut sha256sum := sha256.new() @@ -39,7 +39,7 @@ pub fn hash_file(path &string) !string { // This function never actually fails, but returns an option to follow // the Writer interface. - sha256sum.write(buf[..bytes_read])! + sha256sum.write(buf[..bytes_read])? } return sha256sum.checksum().hex() diff --git a/src/web/logging.v b/src/web/logging.v index 12b07d7..fc697ff 100644 --- a/src/web/logging.v +++ b/src/web/logging.v @@ -3,33 +3,33 @@ module web import log // log reate a log message with the given level -pub fn (mut ctx Context) log(msg string, level log.Level) { +pub fn (mut ctx Context) log(msg &string, level log.Level) { lock ctx.logger { ctx.logger.send_output(msg, level) } } // lfatal create a log message with the fatal level -pub fn (mut ctx Context) lfatal(msg string) { +pub fn (mut ctx Context) lfatal(msg &string) { ctx.log(msg, log.Level.fatal) } // lerror create a log message with the error level -pub fn (mut ctx Context) lerror(msg string) { +pub fn (mut ctx Context) lerror(msg &string) { ctx.log(msg, log.Level.error) } // lwarn create a log message with the warn level -pub fn (mut ctx Context) lwarn(msg string) { +pub fn (mut ctx Context) lwarn(msg &string) { ctx.log(msg, log.Level.warn) } // linfo create a log message with the info level -pub fn (mut ctx Context) linfo(msg string) { +pub fn (mut ctx Context) linfo(msg &string) { ctx.log(msg, log.Level.info) } // ldebug create a log message with the debug level -pub fn (mut ctx Context) ldebug(msg string) { +pub fn (mut ctx Context) ldebug(msg &string) { ctx.log(msg, log.Level.debug) } diff --git a/src/web/parse.v b/src/web/parse.v index 7af635f..ee7a72c 100644 --- a/src/web/parse.v +++ b/src/web/parse.v @@ -8,7 +8,7 @@ import net.http const attrs_to_ignore = ['auth'] // Parsing function attributes for methods and path. -fn parse_attrs(name string, attrs []string) !([]http.Method, string) { +fn parse_attrs(name string, attrs []string) ?([]http.Method, string) { if attrs.len == 0 { return [http.Method.get], '/$name' } @@ -61,7 +61,7 @@ fn parse_query_from_url(url urllib.URL) map[string]string { } // Extract form data from an HTTP request. -fn parse_form_from_request(request http.Request) !(map[string]string, map[string][]http.FileData) { +fn parse_form_from_request(request http.Request) ?(map[string]string, map[string][]http.FileData) { mut form := map[string]string{} mut files := map[string][]http.FileData{} if request.method in methods_with_form { diff --git a/src/web/web.v b/src/web/web.v index 565baff..1d1480f 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -24,7 +24,7 @@ pub: pub mut: // TCP connection to client. // But beware, do not store it for further use, after request processing web will close connection. - conn &net.TcpConn = unsafe { nil } + conn &net.TcpConn // Gives access to a shared logger object logger shared log.Log // time.ticks() from start of web connection handle. @@ -67,20 +67,20 @@ struct Route { pub fn (ctx Context) before_request() {} // send_string writes the given string to the TCP connection socket. -fn (mut ctx Context) send_string(s string) ! { - ctx.conn.write(s.bytes())! +fn (mut ctx Context) send_string(s string) ? { + ctx.conn.write(s.bytes())? } // send_reader reads at most `size` bytes from the given reader & writes them // to the TCP connection socket. Internally, a 10KB buffer is used, to avoid // having to store all bytes in memory at once. -fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ! { +fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ? { mut buf := []u8{len: 10_000} mut bytes_left := size // Repeat as long as the stream still has data for bytes_left > 0 { - bytes_read := reader.read(mut buf)! + bytes_read := reader.read(mut buf)? bytes_left -= u64(bytes_read) mut to_write := bytes_read @@ -96,20 +96,20 @@ fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ! { // send_custom_response sends the given http.Response to the client. It can be // used to overwrite the Context object & send a completely custom // http.Response instead. -fn (mut ctx Context) send_custom_response(resp &http.Response) ! { - ctx.send_string(resp.bytestr())! +fn (mut ctx Context) send_custom_response(resp &http.Response) ? { + ctx.send_string(resp.bytestr())? } // send_response_header constructs a valid HTTP response with an empty body & // sends it to the client. -pub fn (mut ctx Context) send_response_header() ! { +pub fn (mut ctx Context) send_response_header() ? { mut resp := http.new_response( header: ctx.header.join(headers_close) ) resp.header.add(.content_type, ctx.content_type) resp.set_status(ctx.status) - ctx.send_custom_response(resp)! + ctx.send_custom_response(resp)? } // send is a convenience function for sending the HTTP response with an empty @@ -260,6 +260,13 @@ pub fn (mut ctx Context) redirect(url string) Result { return Result{} } +// not_found Send an not_found response +pub fn (mut ctx Context) not_found() Result { + ctx.send_custom_response(http_404) or {} + + return Result{} +} + interface DbInterface { db voidptr } diff --git a/vieter.toml b/vieter.toml index 74a7397..d3922a4 100644 --- a/vieter.toml +++ b/vieter.toml @@ -4,11 +4,11 @@ data_dir = "data" pkg_dir = "data/pkgs" log_level = "DEBUG" default_arch = "x86_64" -arch = "x86_64" address = "http://localhost:8000" -# global_schedule = '* *' +global_schedule = '* *' api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 +