diff --git a/.gitignore b/.gitignore index a2804fe6..aaec9ef0 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ gdb.txt # Generated docs _docs/ +docs/resources/_gen/ /man/ # VLS logs diff --git a/.woodpecker/arch-rel.yml b/.woodpecker/arch-rel.yml index f5f228ef..f7274866 100644 --- a/.woodpecker/arch-rel.yml +++ b/.woodpecker/arch-rel.yml @@ -9,7 +9,7 @@ skip_clone: true pipeline: build: - image: 'menci/archlinuxarm:base-devel' + image: 'git.rustybever.be/vieter-v/vieter-builder' commands: # Add the vieter repository so we can use the compiler - echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf diff --git a/.woodpecker/arch.yml b/.woodpecker/arch.yml index 8f1a6ff0..f5f84323 100644 --- a/.woodpecker/arch.yml +++ b/.woodpecker/arch.yml @@ -9,7 +9,7 @@ skip_clone: true pipeline: build: - image: 'menci/archlinuxarm:base-devel' + image: 'git.rustybever.be/vieter-v/vieter-builder' commands: # Add the vieter repository so we can use the compiler - echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf diff --git a/.woodpecker/build.yml b/.woodpecker/build.yml index 9ee80853..f10e2a5a 100644 --- a/.woodpecker/build.yml +++ b/.woodpecker/build.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'chewingbever/vlang:0.3' + - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' matrix: PLATFORM: diff --git a/.woodpecker/docs.yml b/.woodpecker/docs.yml index 048b1ad9..cf4874e9 100644 --- a/.woodpecker/docs.yml +++ b/.woodpecker/docs.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'chewingbever/vlang:0.3' + - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' platform: 'linux/amd64' branches: diff --git a/.woodpecker/gitea.yml b/.woodpecker/gitea.yml index 8e3b9d44..9034f337 100644 --- a/.woodpecker/gitea.yml +++ b/.woodpecker/gitea.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'chewingbever/vlang:0.3' + - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' platform: 'linux/amd64' branches: [ 'main' ] diff --git a/.woodpecker/lint.yml b/.woodpecker/lint.yml index c80ce338..ec64d132 100644 --- a/.woodpecker/lint.yml +++ b/.woodpecker/lint.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'chewingbever/vlang:0.3' + - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' # These checks already get performed on the feature branches branches: @@ -7,10 +7,21 @@ branches: platform: 'linux/amd64' pipeline: + # vfmt seems to get confused if these aren't present + install-modules: + image: *vlang_image + pull: true + commands: + - export VMODULES=$PWD/.vmodules + - 'cd src && v install' + when: + event: [pull_request] + lint: image: *vlang_image pull: true commands: + - export VMODULES=$PWD/.vmodules - make lint when: - event: [ pull_request ] + event: [pull_request] diff --git a/.woodpecker/man.yml b/.woodpecker/man.yml index 86a1bd8c..8c6ca068 100644 --- a/.woodpecker/man.yml +++ b/.woodpecker/man.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'chewingbever/vlang:0.3' + - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' platform: 'linux/amd64' branches: diff --git a/.woodpecker/test.yml b/.woodpecker/test.yml index 08b7534f..39cb9f9b 100644 --- a/.woodpecker/test.yml +++ b/.woodpecker/test.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'chewingbever/vlang:0.3' + - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' matrix: PLATFORM: diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cd39c4a..27d90961 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev) +## [0.5.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.1) + +### Added + +* Allow specifying subdirectory inside Git repository +* Added option to deploy using agent-server architecture instead of cron daemon +* Allow scheduling builds on the server from the CLI tool instead of building + them locally +* Allow force-building packages, meaning the build won't check if the + repository is already up to date + +### Changed + +* Migrated codebase to V 0.3.2 +* Cron expression parser now uses bitfields instead of bool arrays + +### Fixed + +* Arch value for target is now properly set if not provided +* Allow NULL values for branch in database +* Endpoint for adding targets now returns the correct id +* CLI now correctly errors and doesn't error when sending requests +* Fixed possible infinite loop when removing old build images +* Check whether build image still exists before starting build +* Don't run makepkg `prepare()` function twice +* Don't buffer stdout in Docker containers + ## [0.4.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.4.0) ### Added diff --git a/Dockerfile b/Dockerfile index 7aed9179..a27ad446 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM chewingbever/vlang:0.3 AS builder +FROM git.rustybever.be/chewing_bever/vlang:0.3.2 AS builder ARG TARGETPLATFORM ARG CI_COMMIT_SHA @@ -23,6 +23,7 @@ RUN if [ -n "${CI_COMMIT_SHA}" ]; then \ "https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \ chmod +x vieter ; \ else \ + cd src && v install && cd .. && \ LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static' make prod && \ mv pvieter vieter ; \ fi diff --git a/Makefile b/Makefile index 69bd7957..e7168076 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' V_PATH ?= v -V := $(V_PATH) -showcc -gc boehm +V := $(V_PATH) -showcc -gc boehm -W -d use_openssl all: vieter @@ -92,9 +92,9 @@ clean: .PHONY: autofree autofree: afvieter afvieter: $(SOURCES) - $(V_PATH) -showcc -autofree -o afvieter $(SRC_DIR) + $(V) -showcc -autofree -o afvieter $(SRC_DIR) .PHONY: skip-unused skip-unused: suvieter suvieter: $(SOURCES) - $(V_PATH) -showcc -skip-unused -o suvieter $(SRC_DIR) + $(V) -skip-unused -o suvieter $(SRC_DIR) diff --git a/PKGBUILD b/PKGBUILD index b600ba0e..94db6546 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -3,7 +3,7 @@ pkgbase='vieter' pkgname='vieter' -pkgver='0.4.0' +pkgver='0.5.0-rc.1' pkgrel=1 pkgdesc="Lightweight Arch repository server & package build system" depends=('glibc' 'openssl' 'libarchive' 'sqlite') diff --git a/README.md b/README.md index b9fff692..637d4c15 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ quicker. I chose [V](https://vlang.io/) as I've been very intrigued by this language for a while now. I wanted a fast language that I could code while relaxing, without having to exert too much mental effort & V seemed like the right choice for -that. +that. Sadly, this didn't quite turn out the way I expected, but I'm sticking +with it anyways ;p ## Features @@ -49,7 +50,7 @@ update`. I used to maintain a mirror that tracked the latest master, but nowadays, I maintain a Docker image containing the specific compiler version that Vieter -builds with. Currently, this is V 0.3. +builds with. Currently, this is V 0.3.2. ## Contributing diff --git a/docs/content/configuration.md b/docs/content/configuration.md index af941a22..95bf7135 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -17,7 +17,7 @@ If a variable is both present in the config file & as an environment variable, the value in the environment variable is used. {{< hint info >}} -**Note** +**Note** All environment variables can also be provided from a file by appending them with `_FILE`. This for example allows you to provide the API key from a Docker secrets file. @@ -97,3 +97,25 @@ configuration variable required for each command. build`. * Default: `archlinux:base-devel` +### `vieter agent` + +* `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`, + `WARN`, `INFO` or `DEBUG`. + * Default: `WARN` +* `address`: *public* URL of the Vieter repository server to build for. From + this server jobs are retrieved. All built packages are published to this + server. +* `api_key`: API key of the above server. +* `data_dir`: directory to store log file in. +* `max_concurrent_builds`: how many builds to run at the same time. + * Default: `1` +* `polling_frequency`: how often (in seconds) to poll the server for new + builds. Note that the agent might poll more frequently when it's actively + processing builds. +* `image_rebuild_frequency`: Vieter periodically builds images that are then + used as a basis for running build containers. This is to prevent each build + from downloading an entire repository worth of dependencies. This setting + defines how frequently (in minutes) to rebuild these images. + * Default: `1440` (every 24 hours) +* `arch`: architecture for which this agent should pull down builds (e.g. + `x86_64`) diff --git a/docs/content/installation.md b/docs/content/installation.md index 87b9cba9..21eda643 100644 --- a/docs/content/installation.md +++ b/docs/content/installation.md @@ -21,7 +21,7 @@ branch. This branch will be the most up to date, but does not give any guarantees about stability, so beware! Thanks to the single-binary design of Vieter, this image can be used both for -the repository server & the cron daemon. +the repository server, the cron daemon and the agent. Below is an example compose file to set up both the repository server & the cron daemon: @@ -76,7 +76,7 @@ architectures will build on both. ## Binary On the -[releases](https://git.rustybever.be/vieter/vieter/releases) +[releases](https://git.rustybever.be/vieter-v/vieter/releases) page, you can find statically compiled binaries for all released versions. This is the same binary as used inside the Docker images. @@ -106,5 +106,5 @@ guarantee that a compiler update won't temporarily break them. ## Building from source -The project [README](https://git.rustybever.be/vieter/vieter#building) contains -instructions for building Vieter from source. +The project [README](https://git.rustybever.be/vieter-v/vieter#building) +contains instructions for building Vieter from source. diff --git a/docs/content/usage/builds/schedule.md b/docs/content/usage/builds/schedule.md index 38f76a40..de59e25f 100644 --- a/docs/content/usage/builds/schedule.md +++ b/docs/content/usage/builds/schedule.md @@ -37,6 +37,6 @@ Each section can consist of as many of these parts as necessary. ## CLI tool The Vieter binary contains a command that shows you the next matching times for -a given expression. This can be useful to understand the syntax. For more +a given expression. This can be useful for understanding the syntax. For more information, see [vieter-schedule(1)](https://rustybever.be/man/vieter/vieter-schedule.1.html). diff --git a/src/agent/agent.v b/src/agent/agent.v new file mode 100644 index 00000000..69b9947d --- /dev/null +++ b/src/agent/agent.v @@ -0,0 +1,27 @@ +module agent + +import log +import os +import util + +const log_file_name = 'vieter.agent.log' + +// agent starts an agent service +pub fn agent(conf Config) ! { + log_level := log.level_from_tag(conf.log_level) or { + return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + } + + mut logger := log.Log{ + level: log_level + } + + os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') } + + log_file := os.join_path_single(conf.data_dir, agent.log_file_name) + logger.set_full_logpath(log_file) + logger.log_to_console_too() + + mut d := agent_init(logger, conf) + d.run() +} diff --git a/src/agent/cli.v b/src/agent/cli.v new file mode 100644 index 00000000..1535e172 --- /dev/null +++ b/src/agent/cli.v @@ -0,0 +1,31 @@ +module agent + +import cli +import conf as vconf + +struct Config { +pub: + log_level string = 'WARN' + // Architecture that the agent represents + arch string + api_key string + address string + data_dir string + max_concurrent_builds int = 1 + polling_frequency int = 30 + image_rebuild_frequency int = 1440 +} + +// cmd returns the cli module that handles the cron daemon. +pub fn cmd() cli.Command { + return cli.Command{ + name: 'agent' + description: 'Start an agent daemon.' + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + + agent(conf)! + } + } +} diff --git a/src/agent/daemon.v b/src/agent/daemon.v new file mode 100644 index 00000000..62f36c21 --- /dev/null +++ b/src/agent/daemon.v @@ -0,0 +1,197 @@ +module agent + +import log +import sync.stdatomic +import build +import models { BuildConfig } +import client +import time +import os + +const ( + build_empty = 0 + build_running = 1 + build_done = 2 +) + +struct AgentDaemon { + logger shared log.Log + conf Config + client client.Client +mut: + images ImageManager + // Which builds are currently running; length is conf.max_concurrent_builds + builds []BuildConfig + // Atomic variables used to detect when a build has finished; length is + // conf.max_concurrent_builds + atomics []u64 +} + +// agent_init initializes a new agent +fn agent_init(logger log.Log, conf Config) AgentDaemon { + mut d := AgentDaemon{ + logger: logger + client: client.new(conf.address, conf.api_key) + conf: conf + images: new_image_manager(conf.image_rebuild_frequency * 60) + builds: []BuildConfig{len: conf.max_concurrent_builds} + atomics: []u64{len: conf.max_concurrent_builds} + } + + return d +} + +// run starts the actual agent daemon. This function will run forever. +pub fn (mut d AgentDaemon) run() { + // This is just so that the very first time the loop is ran, the jobs are + // always polled + mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency) + mut sleep_time := 0 * time.second + mut finished, mut empty, mut running := 0, 0, 0 + + for { + if sleep_time > 0 { + d.ldebug('Sleeping for $sleep_time') + time.sleep(sleep_time) + } + + finished, empty = d.update_atomics() + running = d.conf.max_concurrent_builds - finished - empty + + // No new finished builds and no free slots, so there's nothing to be + // done + if finished + empty == 0 { + sleep_time = 1 * time.second + continue + } + + // Builds have finished, so old builder images might have freed up. + // TODO this might query the docker daemon too frequently. + if finished > 0 { + d.images.clean_old_images() + } + + // The agent will always poll for new jobs after at most + // `polling_frequency` seconds. However, when jobs have finished, the + // agent will also poll for new jobs. This is because jobs are often + // clustered together (especially when mostly using the global cron + // schedule), so there's a much higher chance jobs are available. + if finished > 0 || time.now() >= last_poll_time.add_seconds(d.conf.polling_frequency) { + d.ldebug('Polling for new jobs') + + new_configs := d.client.poll_jobs(d.conf.arch, finished + empty) or { + d.lerror('Failed to poll jobs: $err.msg()') + + // TODO pick a better delay here + sleep_time = 5 * time.second + continue + } + + d.ldebug('Received $new_configs.len jobs') + + last_poll_time = time.now() + + for config in new_configs { + // Make sure a recent build base image is available for + // building the config + if !d.images.up_to_date(config.base_image) { + d.linfo('Building builder image from base image $config.base_image') + + // TODO handle this better than to just skip the config + d.images.refresh_image(config.base_image) or { + d.lerror(err.msg()) + continue + } + } + + // It's technically still possible that the build image is + // removed in the very short period between building the + // builder image and starting a build container with it. If + // this happens, faith really just didn't want you to do this + // build. + + d.start_build(config) + running++ + } + } + + // The agent is not doing anything, so we just wait until the next poll + // time + if running == 0 { + sleep_time = last_poll_time.add_seconds(d.conf.polling_frequency) - time.now() + } else { + sleep_time = 1 * time.second + } + } +} + +// update_atomics checks for each build whether it's completed, and sets it to +// empty again if so. The return value is a tuple `(finished, empty)` where +// `finished` is how many builds were just finished and thus set to empty, and +// `empty` is how many build slots were already empty. The amount of running +// builds can then be calculated by substracting these two values from the +// total allowed concurrent builds. +fn (mut d AgentDaemon) update_atomics() (int, int) { + mut finished := 0 + mut empty := 0 + + for i in 0 .. d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == agent.build_done { + stdatomic.store_u64(&d.atomics[i], agent.build_empty) + finished++ + } else if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { + empty++ + } + } + + return finished, empty +} + +// start_build starts a build for the given BuildConfig. +fn (mut d AgentDaemon) start_build(config BuildConfig) bool { + for i in 0 .. d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { + stdatomic.store_u64(&d.atomics[i], agent.build_running) + d.builds[i] = config + + go d.run_build(i, config) + + return true + } + } + + return false +} + +// run_build actually starts the build process for a given target. +fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) { + d.linfo('started build: $config') + + // 0 means success, 1 means failure + mut status := 0 + + new_config := BuildConfig{ + ...config + base_image: d.images.get(config.base_image) + } + + res := build.build_config(d.client.address, d.client.api_key, new_config) or { + d.ldebug('build_config error: $err.msg()') + status = 1 + + build.BuildResult{} + } + + if status == 0 { + d.linfo('Uploading build logs for $config') + + // TODO use the arch value here + build_arch := os.uname().machine + d.client.add_build_log(config.target_id, res.start_time, res.end_time, build_arch, + res.exit_code, res.logs) or { d.lerror('Failed to upload logs for $config') } + } else { + d.lwarn('an error occurred during build: $config') + } + + stdatomic.store_u64(&d.atomics[build_index], agent.build_done) +} diff --git a/src/agent/images.v b/src/agent/images.v new file mode 100644 index 00000000..1fec5672 --- /dev/null +++ b/src/agent/images.v @@ -0,0 +1,119 @@ +module agent + +import time +import docker +import build + +// An ImageManager is a utility that creates builder images from given base +// images, updating these builder images if they've become too old. This +// structure can manage images from any number of base images, paving the way +// for configurable base images per target/repository. +struct ImageManager { + max_image_age int [required] +mut: + // For each base image, one or more builder images can exist at the same + // time + images map[string][]string [required] + // For each base image, we track when its newest image was built + timestamps map[string]time.Time [required] +} + +// new_image_manager initializes a new image manager. +fn new_image_manager(max_image_age int) ImageManager { + return ImageManager{ + max_image_age: max_image_age + images: map[string][]string{} + timestamps: map[string]time.Time{} + } +} + +// get returns the name of the newest image for the given base image. Note that +// this function should only be called *after* a first call to `refresh_image`. +pub fn (m &ImageManager) get(base_image string) string { + return m.images[base_image].last() +} + +// up_to_date returns true if the last known builder image exists and is up to +// date. If this function returns true, the last builder image may be used to +// perform a build. +pub fn (mut m ImageManager) up_to_date(base_image string) bool { + if base_image !in m.timestamps + || m.timestamps[base_image].add_seconds(m.max_image_age) <= time.now() { + return false + } + + // It's possible the image has been removed by some external event, so we + // check whether it actually exists as well. + mut dd := docker.new_conn() or { return false } + + defer { + dd.close() or {} + } + + dd.image_inspect(m.images[base_image].last()) or { + // Image doesn't exist, so we stop tracking it + if err.code() == 404 { + m.images[base_image].delete_last() + m.timestamps.delete(base_image) + } + + // If the inspect fails, it's either because the image doesn't exist or + // because of some other error. Either way, we can't know *for certain* + // that the image exists, so we return false. + return false + } + + return true +} + +// refresh_image builds a new builder image from the given base image. This +// function should only be called if `up_to_date` returned false. +fn (mut m ImageManager) refresh_image(base_image string) ! { + // TODO use better image tags for built images + new_image := build.create_build_image(base_image) or { + return error('Failed to build builder image from base image $base_image') + } + + m.images[base_image] << new_image + m.timestamps[base_image] = time.now() +} + +// clean_old_images removes all older builder images that are no longer in use. +// The function will always leave at least one builder image, namely the newest +// one. +fn (mut m ImageManager) clean_old_images() { + mut dd := docker.new_conn() or { return } + + defer { + dd.close() or {} + } + + mut i := 0 + + for image in m.images.keys() { + i = 0 + + for i < m.images[image].len - 1 { + // For each builder image, we try to remove it by calling the Docker + // API. If the function returns an error or false, that means the image + // wasn't deleted. Therefore, we move the index over. If the function + // returns true, the array's length has decreased by one so we don't + // move the index. + dd.remove_image(m.images[image][i]) or { + // The image was removed by an external event + if err.code() == 404 { + m.images[image].delete(i) + } + // The image couldn't be removed, so we need to keep track of + // it + else { + i += 1 + } + + continue + } + + m.images[image].delete(i) + } + } +} diff --git a/src/agent/log.v b/src/agent/log.v new file mode 100644 index 00000000..cd592074 --- /dev/null +++ b/src/agent/log.v @@ -0,0 +1,35 @@ +module agent + +import log + +// log a message with the given level +pub fn (mut d AgentDaemon) log(msg string, level log.Level) { + lock d.logger { + d.logger.send_output(msg, level) + } +} + +// lfatal create a log message with the fatal level +pub fn (mut d AgentDaemon) lfatal(msg string) { + d.log(msg, log.Level.fatal) +} + +// lerror create a log message with the error level +pub fn (mut d AgentDaemon) lerror(msg string) { + d.log(msg, log.Level.error) +} + +// lwarn create a log message with the warn level +pub fn (mut d AgentDaemon) lwarn(msg string) { + d.log(msg, log.Level.warn) +} + +// linfo create a log message with the info level +pub fn (mut d AgentDaemon) linfo(msg string) { + d.log(msg, log.Level.info) +} + +// ldebug create a log message with the debug level +pub fn (mut d AgentDaemon) ldebug(msg string) { + d.log(msg, log.Level.debug) +} diff --git a/src/build/build.v b/src/build/build.v index 2ad70a6f..712c93b3 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -1,12 +1,12 @@ module build -import vieter_v.docker +import docker import encoding.base64 import time import os import strings import util -import models { Target } +import models { BuildConfig, Target } const ( container_build_dir = '/build' @@ -21,8 +21,8 @@ const ( // system, install some necessary packages & creates a non-root user to run // makepkg with. The base image should be some Linux distribution that uses // Pacman as its package manager. -pub fn create_build_image(base_image string) ?string { - mut dd := docker.new_conn()? +pub fn create_build_image(base_image string) !string { + mut dd := docker.new_conn()! defer { dd.close() or {} @@ -57,15 +57,15 @@ pub fn create_build_image(base_image string) ?string { image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' } // We pull the provided image - dd.pull_image(image_name, image_tag)? + dd.pull_image(image_name, image_tag)! - id := dd.container_create(c)?.id - // id := docker.create_container(c)? - dd.container_start(id)? + id := dd.container_create(c)!.id + // id := docker.create_container(c)! + dd.container_start(id)! // This loop waits until the container has stopped, so we can remove it after for { - data := dd.container_inspect(id)? + data := dd.container_inspect(id)! if !data.state.running { break @@ -79,8 +79,8 @@ pub fn create_build_image(base_image string) ?string { // TODO also add the base image's name into the image name to prevent // conflicts. tag := time.sys_mono_now().str() - image := dd.create_image_from_container(id, 'vieter-build', tag)? - dd.container_remove(id)? + image := dd.create_image_from_container(id, 'vieter-build', tag)! + dd.container_remove(id)! return image.id } @@ -93,25 +93,32 @@ pub: logs string } -// build_target builds, packages & publishes a given Arch package based on the +// build_target builds the given target. Internally it calls `build_config`. +pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool) !BuildResult { + config := target.as_build_config(base_image_id, force) + + return build_config(address, api_key, config) +} + +// build_config builds, packages & publishes a given Arch package based on the // provided target. The base image ID should be of an image previously created // by create_build_image. It returns the logs of the container. -pub fn build_target(address string, api_key string, base_image_id string, target &Target) ?BuildResult { - mut dd := docker.new_conn()? +pub fn build_config(address string, api_key string, config BuildConfig) !BuildResult { + mut dd := docker.new_conn()! defer { dd.close() or {} } build_arch := os.uname().machine - build_script := create_build_script(address, target, build_arch) + build_script := create_build_script(address, config, build_arch) // We convert the build script into a base64 string, which then gets passed // to the container as an env var base64_script := base64.encode_str(build_script) c := docker.NewContainer{ - image: '$base_image_id' + image: '$config.base_image' env: [ 'BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key', @@ -125,25 +132,25 @@ pub fn build_target(address string, api_key string, base_image_id string, target user: '0:0' } - id := dd.container_create(c)?.id - dd.container_start(id)? + id := dd.container_create(c)!.id + dd.container_start(id)! - mut data := dd.container_inspect(id)? + mut data := dd.container_inspect(id)! // This loop waits until the container has stopped, so we can remove it after for data.state.running { time.sleep(1 * time.second) - data = dd.container_inspect(id)? + data = dd.container_inspect(id)! } - mut logs_stream := dd.container_get_logs(id)? + mut logs_stream := dd.container_get_logs(id)! // Read in the entire stream mut logs_builder := strings.new_builder(10 * 1024) - util.reader_to_writer(mut logs_stream, mut logs_builder)? + util.reader_to_writer(mut logs_stream, mut logs_builder)! - dd.container_remove(id)? + dd.container_remove(id)! return BuildResult{ start_time: data.state.start_time diff --git a/src/build/queue.v b/src/build/queue.v new file mode 100644 index 00000000..e74529ca --- /dev/null +++ b/src/build/queue.v @@ -0,0 +1,220 @@ +module build + +import models { BuildConfig, Target } +import cron.expression { CronExpression, parse_expression } +import time +import datatypes { MinHeap } +import util + +struct BuildJob { +pub mut: + // Time at which this build job was created/queued + created time.Time + // Next timestamp from which point this job is allowed to be executed + timestamp time.Time + // Required for calculating next timestamp after having pop'ed a job + ce CronExpression + // Actual build config sent to the agent + config BuildConfig + // Whether this is a one-time job + single bool +} + +// Allows BuildJob structs to be sorted according to their timestamp in +// MinHeaps +fn (r1 BuildJob) < (r2 BuildJob) bool { + return r1.timestamp < r2.timestamp +} + +// The build job queue is responsible for managing the list of scheduled builds +// for each architecture. Agents receive jobs from this queue. +pub struct BuildJobQueue { + // Schedule to use for targets without explicitely defined cron expression + default_schedule CronExpression + // Base image to use for targets without defined base image + default_base_image string +mut: + mutex shared util.Dummy + // For each architecture, a priority queue is tracked + queues map[string]MinHeap + // When a target is removed from the server or edited, its previous build + // configs will be invalid. This map allows for those to be simply skipped + // by ignoring any build configs created before this timestamp. + invalidated map[int]time.Time +} + +// new_job_queue initializes a new job queue +pub fn new_job_queue(default_schedule CronExpression, default_base_image string) BuildJobQueue { + return BuildJobQueue{ + default_schedule: default_schedule + default_base_image: default_base_image + invalidated: map[int]time.Time{} + } +} + +// insert_all executes insert for each architecture of the given Target. +pub fn (mut q BuildJobQueue) insert_all(target Target) ! { + for arch in target.arch { + q.insert(target: target, arch: arch.value)! + } +} + +[params] +pub struct InsertConfig { + target Target [required] + arch string [required] + single bool + force bool + now bool +} + +// insert a new target's job into the queue for the given architecture. This +// job will then be endlessly rescheduled after being pop'ed, unless removed +// explicitely. +pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! { + lock q.mutex { + if input.arch !in q.queues { + q.queues[input.arch] = MinHeap{} + } + + mut job := BuildJob{ + created: time.now() + single: input.single + config: input.target.as_build_config(q.default_base_image, input.force) + } + + if !input.now { + ce := if input.target.schedule != '' { + parse_expression(input.target.schedule) or { + return error("Error while parsing cron expression '$input.target.schedule' (id $input.target.id): $err.msg()") + } + } else { + q.default_schedule + } + + job.timestamp = ce.next_from_now()! + job.ce = ce + } else { + job.timestamp = time.now() + } + + q.queues[input.arch].insert(job) + } +} + +// reschedule the given job by calculating the next timestamp and re-adding it +// to its respective queue. This function is called by the pop functions +// *after* having pop'ed the job. +fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) ! { + new_timestamp := job.ce.next_from_now()! + + new_job := BuildJob{ + ...job + created: time.now() + timestamp: new_timestamp + } + + q.queues[arch].insert(new_job) +} + +// pop_invalid pops all invalid jobs. +fn (mut q BuildJobQueue) pop_invalid(arch string) { + for { + job := q.queues[arch].peek() or { return } + + if job.config.target_id in q.invalidated + && job.created < q.invalidated[job.config.target_id] { + // This pop *should* never fail according to the source code + q.queues[arch].pop() or {} + } else { + break + } + } +} + +// peek shows the first job for the given architecture that's ready to be +// executed, if present. +pub fn (mut q BuildJobQueue) peek(arch string) ?BuildJob { + // Even peek requires a write lock, because pop_invalid can modify the data + // structure + lock q.mutex { + if arch !in q.queues { + return none + } + + q.pop_invalid(arch) + job := q.queues[arch].peek()? + + if job.timestamp < time.now() { + return job + } + } + + return none +} + +// pop removes the first job for the given architecture that's ready to be +// executed from the queue and returns it, if present. +pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob { + lock q.mutex { + if arch !in q.queues { + return none + } + + q.pop_invalid(arch) + mut job := q.queues[arch].peek()? + + if job.timestamp < time.now() { + job = q.queues[arch].pop()? + + if !job.single { + // TODO how do we handle this properly? Is it even possible for a + // cron expression to not return a next time if it's already been + // used before? + q.reschedule(job, arch) or {} + } + + return job + } + } + + return none +} + +// pop_n tries to pop at most n available jobs for the given architecture. +pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { + lock q.mutex { + if arch !in q.queues { + return [] + } + + mut out := []BuildJob{} + + for out.len < n { + q.pop_invalid(arch) + mut job := q.queues[arch].peek() or { break } + + if job.timestamp < time.now() { + job = q.queues[arch].pop() or { break } + + if !job.single { + // TODO idem + q.reschedule(job, arch) or {} + } + + out << job + } else { + break + } + } + + return out + } + + return [] +} + +// invalidate a target's old build jobs. +pub fn (mut q BuildJobQueue) invalidate(target_id int) { + q.invalidated[target_id] = time.now() +} diff --git a/src/build/build_script_git.sh b/src/build/scripts/git.sh similarity index 75% rename from src/build/build_script_git.sh rename to src/build/scripts/git.sh index 73e0965d..26442434 100644 --- a/src/build/build_script_git.sh +++ b/src/build/scripts/git.sh @@ -16,5 +16,5 @@ echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkg curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 echo -e '+ [ "$(id -u)" == 0 ] && exit 0' [ "$(id -u)" == 0 ] && exit 0 -echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' -MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/build_script_git_branch.sh b/src/build/scripts/git_branch.sh similarity index 75% rename from src/build/build_script_git_branch.sh rename to src/build/scripts/git_branch.sh index be1ff4fb..9f36bdc2 100644 --- a/src/build/build_script_git_branch.sh +++ b/src/build/scripts/git_branch.sh @@ -16,5 +16,5 @@ echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkg curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 echo -e '+ [ "$(id -u)" == 0 ] && exit 0' [ "$(id -u)" == 0 ] && exit 0 -echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' -MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/scripts/git_path.sh b/src/build/scripts/git_path.sh new file mode 100644 index 00000000..65b7fb9d --- /dev/null +++ b/src/build/scripts/git_path.sh @@ -0,0 +1,20 @@ +echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' +echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf +echo -e '+ pacman -Syu --needed --noconfirm' +pacman -Syu --needed --noconfirm +echo -e '+ su builder' +su builder +echo -e '+ git clone --single-branch --depth 1 '\''https://examplerepo.com'\'' repo' +git clone --single-branch --depth 1 'https://examplerepo.com' repo +echo -e '+ cd '\''repo/example/path'\''' +cd 'repo/example/path' +echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' +makepkg --nobuild --syncdeps --needed --noconfirm +echo -e '+ source PKGBUILD' +source PKGBUILD +echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' +curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 +echo -e '+ [ "$(id -u)" == 0 ] && exit 0' +[ "$(id -u)" == 0 ] && exit 0 +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/scripts/git_path_spaces.sh b/src/build/scripts/git_path_spaces.sh new file mode 100644 index 00000000..b632b911 --- /dev/null +++ b/src/build/scripts/git_path_spaces.sh @@ -0,0 +1,20 @@ +echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' +echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf +echo -e '+ pacman -Syu --needed --noconfirm' +pacman -Syu --needed --noconfirm +echo -e '+ su builder' +su builder +echo -e '+ git clone --single-branch --depth 1 '\''https://examplerepo.com'\'' repo' +git clone --single-branch --depth 1 'https://examplerepo.com' repo +echo -e '+ cd '\''repo/example/path with spaces'\''' +cd 'repo/example/path with spaces' +echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' +makepkg --nobuild --syncdeps --needed --noconfirm +echo -e '+ source PKGBUILD' +source PKGBUILD +echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' +curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 +echo -e '+ [ "$(id -u)" == 0 ] && exit 0' +[ "$(id -u)" == 0 ] && exit 0 +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/build_script_url.sh b/src/build/scripts/url.sh similarity index 75% rename from src/build/build_script_url.sh rename to src/build/scripts/url.sh index 3bc97e12..2d27de7e 100644 --- a/src/build/build_script_url.sh +++ b/src/build/scripts/url.sh @@ -18,5 +18,5 @@ echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkg curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 echo -e '+ [ "$(id -u)" == 0 ] && exit 0' [ "$(id -u)" == 0 ] && exit 0 -echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' -MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/shell.v b/src/build/shell.v index e573d53c..16f93b54 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -1,6 +1,6 @@ module build -import models { Target } +import models { BuildConfig } // escape_shell_string escapes any characters that could be interpreted // incorrectly by a shell. The resulting value should be safe to use inside an @@ -23,13 +23,13 @@ pub fn echo_commands(cmds []string) []string { } // create_build_script generates a shell script that builds a given Target. -fn create_build_script(address string, target &Target, build_arch string) string { - repo_url := '$address/$target.repo' +fn create_build_script(address string, config BuildConfig, build_arch string) string { + repo_url := '$address/$config.repo' mut commands := [ // This will later be replaced by a proper setting for changing the // mirrorlist - "echo -e '[$target.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" + "echo -e '[$config.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" // We need to update the package list of the repo we just added above. // This should however not pull in a lot of packages as long as the // builder image is rebuilt frequently. @@ -38,22 +38,22 @@ fn create_build_script(address string, target &Target, build_arch string) string 'su builder', ] - commands << match target.kind { + commands << match config.kind { 'git' { - if target.branch == '' { + if config.branch == '' { [ - "git clone --single-branch --depth 1 '$target.url' repo", + "git clone --single-branch --depth 1 '$config.url' repo", ] } else { [ - "git clone --single-branch --depth 1 --branch $target.branch '$target.url' repo", + "git clone --single-branch --depth 1 --branch $config.branch '$config.url' repo", ] } } 'url' { [ 'mkdir repo', - "curl -o repo/PKGBUILD -L '$target.url'", + "curl -o repo/PKGBUILD -L '$config.url'", ] } else { @@ -61,19 +61,32 @@ fn create_build_script(address string, target &Target, build_arch string) string } } + commands << if config.path != '' { + "cd 'repo/$config.path'" + } else { + 'cd repo' + } + commands << [ - 'cd repo', 'makepkg --nobuild --syncdeps --needed --noconfirm', 'source PKGBUILD', + ] + + if !config.force { // The build container checks whether the package is already present on // the server. - 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', - // If the above curl command succeeds, we don't need to rebuild the - // package. However, because we're in a su shell, the exit command will - // drop us back into the root shell. Therefore, we must check whether - // we're in root so we don't proceed. - '[ "\$(id -u)" == 0 ] && exit 0', - 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', + commands << [ + 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + // If the above curl command succeeds, we don't need to rebuild the + // package. However, because we're in a su shell, the exit command will + // drop us back into the root shell. Therefore, we must check whether + // we're in root so we don't proceed. + '[ "\$(id -u)" == 0 ] && exit 0', + ] + } + + commands << [ + 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', ] return echo_commands(commands).join('\n') diff --git a/src/build/shell_test.v b/src/build/shell_test.v index 341df888..e23d964d 100644 --- a/src/build/shell_test.v +++ b/src/build/shell_test.v @@ -1,43 +1,75 @@ module build -import models { Target } +import models { BuildConfig } -fn test_create_build_script_git_branch() { - target := Target{ - id: 1 +fn test_create_build_script_git() { + config := BuildConfig{ + target_id: 1 kind: 'git' url: 'https://examplerepo.com' - branch: 'main' repo: 'vieter' + base_image: 'not-used:latest' } - build_script := create_build_script('https://example.com', target, 'x86_64') - expected := $embed_file('build_script_git_branch.sh') + + build_script := create_build_script('https://example.com', config, 'x86_64') + expected := $embed_file('scripts/git.sh') assert build_script == expected.to_string().trim_space() } -fn test_create_build_script_git() { - target := Target{ - id: 1 +fn test_create_build_script_git_path() { + mut config := BuildConfig{ + target_id: 1 kind: 'git' url: 'https://examplerepo.com' repo: 'vieter' + path: 'example/path' + base_image: 'not-used:latest' } - build_script := create_build_script('https://example.com', target, 'x86_64') - expected := $embed_file('build_script_git.sh') + + mut build_script := create_build_script('https://example.com', config, 'x86_64') + mut expected := $embed_file('scripts/git_path.sh') + + assert build_script == expected.to_string().trim_space() + + config = BuildConfig{ + ...config + path: 'example/path with spaces' + } + + build_script = create_build_script('https://example.com', config, 'x86_64') + expected = $embed_file('scripts/git_path_spaces.sh') + + assert build_script == expected.to_string().trim_space() +} + +fn test_create_build_script_git_branch() { + config := BuildConfig{ + target_id: 1 + kind: 'git' + url: 'https://examplerepo.com' + branch: 'main' + repo: 'vieter' + base_image: 'not-used:latest' + } + + build_script := create_build_script('https://example.com', config, 'x86_64') + expected := $embed_file('scripts/git_branch.sh') assert build_script == expected.to_string().trim_space() } fn test_create_build_script_url() { - target := Target{ - id: 1 + config := BuildConfig{ + target_id: 1 kind: 'url' url: 'https://examplerepo.com' repo: 'vieter' + base_image: 'not-used:latest' } - build_script := create_build_script('https://example.com', target, 'x86_64') - expected := $embed_file('build_script_url.sh') + + build_script := create_build_script('https://example.com', config, 'x86_64') + expected := $embed_file('scripts/url.sh') assert build_script == expected.to_string().trim_space() } diff --git a/src/client/client.v b/src/client/client.v index d68ff18f..cce4e70b 100644 --- a/src/client/client.v +++ b/src/client/client.v @@ -2,7 +2,7 @@ module client import net.http { Method } import net.urllib -import web.response { Response } +import web.response { Response, new_data_response } import json pub struct Client { @@ -21,7 +21,7 @@ pub fn new(address string, api_key string) Client { // send_request_raw sends an HTTP request, returning the http.Response object. // It encodes the params so that they're safe to pass as HTTP query parameters. -fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) ?http.Response { +fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) !http.Response { mut full_url := '$c.address$url' if params.len > 0 { @@ -38,31 +38,53 @@ fn (c &Client) send_request_raw(method Method, url string, params map[string]str full_url = '$full_url?$params_str' } - mut req := http.new_request(method, full_url, body)? - req.add_custom_header('X-Api-Key', c.api_key)? + // Looking at the source code, this function doesn't actually fail, so I'm + // not sure why it returns an optional + mut req := http.new_request(method, full_url, body) or { return error('') } + req.add_custom_header('X-Api-Key', c.api_key)! - res := req.do()? + res := req.do()! return res } // send_request just calls send_request_with_body with an empty body. -fn (c &Client) send_request(method Method, url string, params map[string]string) ?Response { +fn (c &Client) send_request(method Method, url string, params map[string]string) !Response { return c.send_request_with_body(method, url, params, '') } // send_request_with_body calls send_request_raw_response & parses its // output as a Response object. -fn (c &Client) send_request_with_body(method Method, url string, params map[string]string, body string) ?Response { - res_text := c.send_request_raw_response(method, url, params, body)? - data := json.decode(Response, res_text)? +fn (c &Client) send_request_with_body(method Method, url string, params map[string]string, body string) !Response { + res := c.send_request_raw(method, url, params, body)! + status := res.status() + + // Non-successful requests are expected to return either an empty body or + // Response + if status.is_error() { + // A non-successful status call will have an empty body + if res.body == '' { + return error('Error $res.status_code ($status.str()): (empty response)') + } + + data := json.decode(Response, res.body)! + + return error('Status $res.status_code ($status.str()): $data.message') + } + + // Just return an empty successful response + if res.body == '' { + return new_data_response(T{}) + } + + data := json.decode(Response, res.body)! return data } // send_request_raw_response returns the raw text response for an HTTP request. -fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) ?string { - res := c.send_request_raw(method, url, params, body)? +fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) !string { + res := c.send_request_raw(method, url, params, body)! return res.body } diff --git a/src/client/jobs.v b/src/client/jobs.v new file mode 100644 index 00000000..784639e2 --- /dev/null +++ b/src/client/jobs.v @@ -0,0 +1,23 @@ +module client + +import models { BuildConfig } + +// poll_jobs requests a list of new build jobs from the server. +pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig { + data := c.send_request<[]BuildConfig>(.get, '/api/v1/jobs/poll', { + 'arch': arch + 'max': max.str() + })! + + return data.data +} + +// queue_job adds a new one-time build job for the given target to the job +// queue. +pub fn (c &Client) queue_job(target_id int, arch string, force bool) ! { + c.send_request(.post, '/api/v1/jobs/queue', { + 'target': target_id.str() + 'arch': arch + 'force': force.str() + })! +} diff --git a/src/client/logs.v b/src/client/logs.v index b4142450..85063bcd 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -6,40 +6,29 @@ import web.response { Response } import time // get_build_logs returns all build logs. -pub fn (c &Client) get_build_logs(filter BuildLogFilter) ?Response<[]BuildLog> { +pub fn (c &Client) get_build_logs(filter BuildLogFilter) ![]BuildLog { params := models.params_from(filter) - data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)? + data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)! - return data -} - -// get_build_logs_for_target returns all build logs for a given target. -pub fn (c &Client) get_build_logs_for_target(target_id int) ?Response<[]BuildLog> { - params := { - 'repo': target_id.str() - } - - data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)? - - return data + return data.data } // get_build_log returns a specific build log. -pub fn (c &Client) get_build_log(id int) ?Response { - data := c.send_request(Method.get, '/api/v1/logs/$id', {})? +pub fn (c &Client) get_build_log(id int) !BuildLog { + data := c.send_request(Method.get, '/api/v1/logs/$id', {})! - return data + return data.data } // get_build_log_content returns the contents of the build log file. -pub fn (c &Client) get_build_log_content(id int) ?string { - data := c.send_request_raw_response(Method.get, '/api/v1/logs/$id/content', {}, '')? +pub fn (c &Client) get_build_log_content(id int) !string { + data := c.send_request_raw_response(Method.get, '/api/v1/logs/$id/content', {}, '')! return data } // add_build_log adds a new build log to the server. -pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { +pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) !Response { params := { 'target': target_id.str() 'startTime': start_time.unix_time().str() @@ -48,7 +37,7 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t 'exitCode': exit_code.str() } - data := c.send_request_with_body(Method.post, '/api/v1/logs', params, content)? + data := c.send_request_with_body(Method.post, '/api/v1/logs', params, content)! return data } diff --git a/src/client/targets.v b/src/client/targets.v index c5e44fea..da6a9e42 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -2,24 +2,23 @@ module client import models { Target, TargetFilter } import net.http { Method } -import web.response { Response } // get_targets returns a list of targets, given a filter object. -pub fn (c &Client) get_targets(filter TargetFilter) ?[]Target { +pub fn (c &Client) get_targets(filter TargetFilter) ![]Target { params := models.params_from(filter) - data := c.send_request<[]Target>(Method.get, '/api/v1/targets', params)? + data := c.send_request<[]Target>(Method.get, '/api/v1/targets', params)! return data.data } // get_all_targets retrieves *all* targs from the API using the default // limit. -pub fn (c &Client) get_all_targets() ?[]Target { +pub fn (c &Client) get_all_targets() ![]Target { mut targets := []Target{} mut offset := u64(0) for { - sub_targets := c.get_targets(offset: offset)? + sub_targets := c.get_targets(offset: offset)! if sub_targets.len == 0 { break @@ -34,8 +33,8 @@ pub fn (c &Client) get_all_targets() ?[]Target { } // get_target returns the target for a specific id. -pub fn (c &Client) get_target(id int) ?Target { - data := c.send_request(Method.get, '/api/v1/targets/$id', {})? +pub fn (c &Client) get_target(id int) !Target { + data := c.send_request(Method.get, '/api/v1/targets/$id', {})! return data.data } @@ -45,28 +44,29 @@ pub struct NewTarget { url string branch string repo string + path string arch []string } // add_target adds a new target to the server. -pub fn (c &Client) add_target(t NewTarget) ?Response { +pub fn (c &Client) add_target(t NewTarget) !int { params := models.params_from(t) - data := c.send_request(Method.post, '/api/v1/targets', params)? + data := c.send_request(Method.post, '/api/v1/targets', params)! - return data + return data.data } // remove_target removes the target with the given id from the server. -pub fn (c &Client) remove_target(id int) ?Response { - data := c.send_request(Method.delete, '/api/v1/targets/$id', {})? +pub fn (c &Client) remove_target(id int) !string { + data := c.send_request(Method.delete, '/api/v1/targets/$id', {})! - return data + return data.data } // patch_target sends a PATCH request to the given target with the params as // payload. -pub fn (c &Client) patch_target(id int, params map[string]string) ?Response { - data := c.send_request(Method.patch, '/api/v1/targets/$id', params)? +pub fn (c &Client) patch_target(id int, params map[string]string) !string { + data := c.send_request(Method.patch, '/api/v1/targets/$id', params)! - return data + return data.data } diff --git a/src/console/aur/aur.v b/src/console/aur/aur.v index c98f8e68..a6a33242 100644 --- a/src/console/aur/aur.v +++ b/src/console/aur/aur.v @@ -3,8 +3,8 @@ module aur import cli import console import client -import vieter_v.aur -import vieter_v.conf as vconf +import aur +import conf as vconf struct Config { address string [required] @@ -21,12 +21,12 @@ pub fn cmd() cli.Command { name: 'search' description: 'Search for packages.' required_args: 1 - execute: fn (cmd cli.Command) ? { + execute: fn (cmd cli.Command) ! { c := aur.new() - pkgs := c.search(cmd.args[0])? + pkgs := c.search(cmd.args[0])! data := pkgs.map([it.name, it.description]) - println(console.pretty_table(['name', 'description'], data)?) + println(console.pretty_table(['name', 'description'], data)!) } }, cli.Command{ @@ -34,12 +34,12 @@ pub fn cmd() cli.Command { usage: 'repo pkg-name [pkg-name...]' description: 'Add the given AUR package(s) to Vieter. Non-existent packages will be silently ignored.' required_args: 2 - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! c := aur.new() - pkgs := c.info(cmd.args[1..])? + pkgs := c.info(cmd.args[1..])! vc := client.new(conf.address, conf.api_key) diff --git a/src/console/console.v b/src/console/console.v index caf4cca7..5c40de8c 100644 --- a/src/console/console.v +++ b/src/console/console.v @@ -13,7 +13,7 @@ pub fn tabbed_table(data [][]string) string { // pretty_table converts a list of string data into a pretty table. Many thanks // to @hungrybluedev in the Vlang Discord for providing this code! // https://ptb.discord.com/channels/592103645835821068/592106336838352923/970278787143045192 -pub fn pretty_table(header []string, data [][]string) ?string { +pub fn pretty_table(header []string, data [][]string) !string { column_count := header.len mut column_widths := []int{len: column_count, init: header[it].len} @@ -26,7 +26,7 @@ pub fn pretty_table(header []string, data [][]string) ?string { } } - single_line_length := arrays.sum(column_widths)? + (column_count + 1) * 3 - 4 + single_line_length := arrays.sum(column_widths)! + (column_count + 1) * 3 - 4 horizontal_line := '+' + strings.repeat(`-`, single_line_length) + '+' mut buffer := strings.new_builder(data.len * single_line_length) @@ -64,12 +64,12 @@ pub fn pretty_table(header []string, data [][]string) ?string { // export_man_pages recursively generates all man pages for the given // cli.Command & writes them to the given directory. -pub fn export_man_pages(cmd cli.Command, path string) ? { +pub fn export_man_pages(cmd cli.Command, path string) ! { man := cmd.manpage() os.write_file(os.join_path_single(path, cmd.full_name().replace(' ', '-') + '.1'), - man)? + man)! for sub_cmd in cmd.commands { - export_man_pages(sub_cmd, path)? + export_man_pages(sub_cmd, path)! } } diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 41830c25..3064a58d 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -1,7 +1,7 @@ module logs import cli -import vieter_v.conf as vconf +import conf as vconf import client import console import time @@ -63,30 +63,30 @@ pub fn cmd() cli.Command { flag: cli.FlagType.string }, ] - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! mut filter := BuildLogFilter{} - limit := cmd.flags.get_int('limit')? + limit := cmd.flags.get_int('limit')! if limit != 0 { filter.limit = u64(limit) } - offset := cmd.flags.get_int('offset')? + offset := cmd.flags.get_int('offset')! if offset != 0 { filter.offset = u64(offset) } - target_id := cmd.flags.get_int('target')? + target_id := cmd.flags.get_int('target')! if target_id != 0 { filter.target = target_id } tz_offset := time.offset() - if cmd.flags.get_bool('today')? { + if cmd.flags.get_bool('today')! { today := time.now() filter.after = time.new_time(time.Time{ @@ -98,12 +98,12 @@ pub fn cmd() cli.Command { } // The -today flag overwrites any of the other date flags. else { - day_str := cmd.flags.get_string('day')? - before_str := cmd.flags.get_string('before')? - after_str := cmd.flags.get_string('after')? + day_str := cmd.flags.get_string('day')! + before_str := cmd.flags.get_string('before')! + after_str := cmd.flags.get_string('after')! if day_str != '' { - day := time.parse_rfc3339(day_str)? + day := time.parse_rfc3339(day_str)! day_utc := time.new_time(time.Time{ year: day.year month: day.month @@ -118,24 +118,24 @@ pub fn cmd() cli.Command { filter.before = day_utc.add_days(1) } else { if before_str != '' { - filter.before = time.parse(before_str)?.add_seconds(-tz_offset) + filter.before = time.parse(before_str)!.add_seconds(-tz_offset) } if after_str != '' { - filter.after = time.parse(after_str)?.add_seconds(-tz_offset) + filter.after = time.parse(after_str)!.add_seconds(-tz_offset) } } } - if cmd.flags.get_bool('failed')? { + if cmd.flags.get_bool('failed')! { filter.exit_codes = [ '!0', ] } - raw := cmd.flags.get_bool('raw')? + raw := cmd.flags.get_bool('raw')! - list(conf, filter, raw)? + list(conf, filter, raw)! } }, cli.Command{ @@ -143,12 +143,12 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Show all info for a specific build log.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! id := cmd.args[0].int() - info(conf, id)? + info(conf, id)! } }, cli.Command{ @@ -156,12 +156,12 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Output the content of a build log to stdout.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! id := cmd.args[0].int() - content(conf, id)? + content(conf, id)! } }, ] @@ -169,46 +169,38 @@ pub fn cmd() cli.Command { } // print_log_list prints a list of logs. -fn print_log_list(logs []BuildLog, raw bool) ? { +fn print_log_list(logs []BuildLog, raw bool) ! { data := logs.map([it.id.str(), it.target_id.str(), it.start_time.local().str(), it.exit_code.str()]) if raw { println(console.tabbed_table(data)) } else { - println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)?) + println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)!) } } // list prints a list of all build logs. -fn list(conf Config, filter BuildLogFilter, raw bool) ? { +fn list(conf Config, filter BuildLogFilter, raw bool) ! { c := client.new(conf.address, conf.api_key) - logs := c.get_build_logs(filter)?.data + logs := c.get_build_logs(filter)! - print_log_list(logs, raw)? -} - -// list prints a list of all build logs for a given target. -fn list_for_target(conf Config, target_id int, raw bool) ? { - c := client.new(conf.address, conf.api_key) - logs := c.get_build_logs_for_target(target_id)?.data - - print_log_list(logs, raw)? + print_log_list(logs, raw)! } // info print the detailed info for a given build log. -fn info(conf Config, id int) ? { +fn info(conf Config, id int) ! { c := client.new(conf.address, conf.api_key) - log := c.get_build_log(id)?.data + log := c.get_build_log(id)! print(log) } // content outputs the contents of the log file for a given build log to // stdout. -fn content(conf Config, id int) ? { +fn content(conf Config, id int) ! { c := client.new(conf.address, conf.api_key) - content := c.get_build_log_content(id)? + content := c.get_build_log_content(id)! println(content) } diff --git a/src/console/man/man.v b/src/console/man/man.v index d91a140f..22cb5f7b 100644 --- a/src/console/man/man.v +++ b/src/console/man/man.v @@ -11,11 +11,11 @@ pub fn cmd() cli.Command { description: 'Generate all man pages & save them in the given directory.' usage: 'dir' required_args: 1 - execute: fn (cmd cli.Command) ? { + execute: fn (cmd cli.Command) ! { root := cmd.root() - os.mkdir_all(cmd.args[0])? + os.mkdir_all(cmd.args[0])! - console.export_man_pages(root, cmd.args[0])? + console.export_man_pages(root, cmd.args[0])! } } } diff --git a/src/console/schedule/schedule.v b/src/console/schedule/schedule.v index 8fceddd9..7ce0516e 100644 --- a/src/console/schedule/schedule.v +++ b/src/console/schedule/schedule.v @@ -18,11 +18,11 @@ pub fn cmd() cli.Command { default_value: ['5'] }, ] - execute: fn (cmd cli.Command) ? { - ce := parse_expression(cmd.args.join(' '))? - count := cmd.flags.get_int('count')? + execute: fn (cmd cli.Command) ! { + ce := parse_expression(cmd.args.join(' '))! + count := cmd.flags.get_int('count')! - for t in ce.next_n(time.now(), count)? { + for t in ce.next_n(time.now(), count)! { println(t) } } diff --git a/src/console/targets/build.v b/src/console/targets/build.v index 6337aa3e..e18077d7 100644 --- a/src/console/targets/build.v +++ b/src/console/targets/build.v @@ -1,34 +1,34 @@ module targets import client -import vieter_v.docker +import docker import os import build // build locally builds the target with the given id. -fn build(conf Config, target_id int) ? { +fn build(conf Config, target_id int, force bool) ! { c := client.new(conf.address, conf.api_key) - target := c.get_target(target_id)? + target := c.get_target(target_id)! build_arch := os.uname().machine println('Creating base image...') - image_id := build.create_build_image(conf.base_image)? + image_id := build.create_build_image(conf.base_image)! println('Running build...') - res := build.build_target(conf.address, conf.api_key, image_id, target)? + res := build.build_target(conf.address, conf.api_key, image_id, target, force)! println('Removing build image...') - mut dd := docker.new_conn()? + mut dd := docker.new_conn()! defer { dd.close() or {} } - dd.remove_image(image_id)? + dd.remove_image(image_id)! println('Uploading logs to Vieter...') c.add_build_log(target.id, res.start_time, res.end_time, build_arch, res.exit_code, - res.logs)? + res.logs)! } diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 56400117..94deebd8 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -1,7 +1,7 @@ module targets import cli -import vieter_v.conf as vconf +import conf as vconf import cron.expression { parse_expression } import client { NewTarget } import console @@ -13,7 +13,7 @@ struct Config { base_image string = 'archlinux:base-devel' } -// cmd returns the cli submodule that handles the repos API interaction +// cmd returns the cli submodule that handles the targets API interaction pub fn cmd() cli.Command { return cli.Command{ name: 'targets' @@ -39,30 +39,30 @@ pub fn cmd() cli.Command { flag: cli.FlagType.string }, ] - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! mut filter := TargetFilter{} - limit := cmd.flags.get_int('limit')? + limit := cmd.flags.get_int('limit')! if limit != 0 { filter.limit = u64(limit) } - offset := cmd.flags.get_int('offset')? + offset := cmd.flags.get_int('offset')! if offset != 0 { filter.offset = u64(offset) } - repo := cmd.flags.get_string('repo')? + repo := cmd.flags.get_string('repo')! if repo != '' { filter.repo = repo } - raw := cmd.flags.get_bool('raw')? + raw := cmd.flags.get_bool('raw')! - list(conf, filter, raw)? + list(conf, filter, raw)! } }, cli.Command{ @@ -82,21 +82,27 @@ pub fn cmd() cli.Command { description: "Which branch to clone; only applies to kind 'git'." flag: cli.FlagType.string }, + cli.Flag{ + name: 'path' + description: 'Subdirectory inside Git repository to use.' + flag: cli.FlagType.string + }, ] - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! t := NewTarget{ - kind: cmd.flags.get_string('kind')? + kind: cmd.flags.get_string('kind')! url: cmd.args[0] repo: cmd.args[1] branch: cmd.flags.get_string('branch') or { '' } + path: cmd.flags.get_string('path') or { '' } } - raw := cmd.flags.get_bool('raw')? + raw := cmd.flags.get_bool('raw')! - add(conf, t, raw)? + add(conf, t, raw)! } }, cli.Command{ @@ -104,11 +110,11 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Remove a target that matches the given id.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - remove(conf, cmd.args[0])? + remove(conf, cmd.args[0])! } }, cli.Command{ @@ -116,11 +122,11 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Show detailed information for the target matching the id.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - info(conf, cmd.args[0])? + info(conf, cmd.args[0])! } }, cli.Command{ @@ -159,10 +165,15 @@ pub fn cmd() cli.Command { description: 'Kind of target.' flag: cli.FlagType.string }, + cli.Flag{ + name: 'path' + description: 'Subdirectory inside Git repository to use.' + flag: cli.FlagType.string + }, ] - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! found := cmd.flags.get_all_found() @@ -170,11 +181,11 @@ pub fn cmd() cli.Command { for f in found { if f.name != 'config-file' { - params[f.name] = f.get_string()? + params[f.name] = f.get_string()! } } - patch(conf, cmd.args[0], params)? + patch(conf, cmd.args[0], params)! } }, cli.Command{ @@ -182,58 +193,82 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Build the target with the given id & publish it.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + flags: [ + cli.Flag{ + name: 'force' + description: 'Build the target without checking whether it needs to be renewed.' + flag: cli.FlagType.bool + }, + cli.Flag{ + name: 'remote' + description: 'Schedule the build on the server instead of running it locally.' + flag: cli.FlagType.bool + }, + cli.Flag{ + name: 'arch' + description: 'Architecture to schedule build for. Required when using -remote.' + flag: cli.FlagType.string + }, + ] + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - build(conf, cmd.args[0].int())? + remote := cmd.flags.get_bool('remote')! + force := cmd.flags.get_bool('force')! + target_id := cmd.args[0].int() + + if remote { + arch := cmd.flags.get_string('arch')! + + if arch == '' { + return error('When scheduling the build remotely, you have to specify an architecture.') + } + + c := client.new(conf.address, conf.api_key) + c.queue_job(target_id, arch, force)! + } else { + build(conf, target_id, force)! + } } }, ] } } -// get_repo_by_prefix tries to find the repo with the given prefix in its -// ID. If multiple or none are found, an error is raised. - // list prints out a list of all repositories. -fn list(conf Config, filter TargetFilter, raw bool) ? { +fn list(conf Config, filter TargetFilter, raw bool) ! { c := client.new(conf.address, conf.api_key) - repos := c.get_targets(filter)? - data := repos.map([it.id.str(), it.kind, it.url, it.repo]) + targets := c.get_targets(filter)! + data := targets.map([it.id.str(), it.kind, it.url, it.repo]) if raw { println(console.tabbed_table(data)) } else { - println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?) + println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)!) } } -// add adds a new repository to the server's list. -fn add(conf Config, t &NewTarget, raw bool) ? { +// add adds a new target to the server's list. +fn add(conf Config, t &NewTarget, raw bool) ! { c := client.new(conf.address, conf.api_key) - res := c.add_target(t)? + target_id := c.add_target(t)! if raw { - println(res.data) + println(target_id) } else { - println('Target added with id $res.data') + println('Target added with id $target_id') } } -// remove removes a repository from the server's list. -fn remove(conf Config, id string) ? { - id_int := id.int() - - if id_int != 0 { - c := client.new(conf.address, conf.api_key) - res := c.remove_target(id_int)? - println(res.message) - } +// remove removes a target from the server's list. +fn remove(conf Config, id string) ! { + c := client.new(conf.address, conf.api_key) + c.remove_target(id.int())! } -// patch patches a given repository with the provided params. -fn patch(conf Config, id string, params map[string]string) ? { +// patch patches a given target with the provided params. +fn patch(conf Config, id string, params map[string]string) ! { // We check the cron expression first because it's useless to send an // invalid one to the server. if 'schedule' in params && params['schedule'] != '' { @@ -242,24 +277,13 @@ fn patch(conf Config, id string, params map[string]string) ? { } } - id_int := id.int() - if id_int != 0 { - c := client.new(conf.address, conf.api_key) - res := c.patch_target(id_int, params)? - - println(res.message) - } -} - -// info shows detailed information for a given repo. -fn info(conf Config, id string) ? { - id_int := id.int() - - if id_int == 0 { - return - } - c := client.new(conf.address, conf.api_key) - repo := c.get_target(id_int)? - println(repo) + c.patch_target(id.int(), params)! +} + +// info shows detailed information for a given target. +fn info(conf Config, id string) ! { + c := client.new(conf.address, conf.api_key) + target := c.get_target(id.int())! + println(target) } diff --git a/src/cron/cli.v b/src/cron/cli.v index 4d958339..16a3537f 100644 --- a/src/cron/cli.v +++ b/src/cron/cli.v @@ -1,7 +1,7 @@ module cron import cli -import vieter_v.conf as vconf +import conf as vconf struct Config { pub: @@ -22,11 +22,11 @@ pub fn cmd() cli.Command { return cli.Command{ name: 'cron' description: 'Start the cron service that periodically runs builds.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - cron(conf)? + cron(conf)! } } } diff --git a/src/cron/cron.v b/src/cron/cron.v index 5f128cf0..f1d6b7bd 100644 --- a/src/cron/cron.v +++ b/src/cron/cron.v @@ -8,7 +8,7 @@ import os const log_file_name = 'vieter.cron.log' // cron starts a cron daemon & starts periodically scheduling builds. -pub fn cron(conf Config) ? { +pub fn cron(conf Config) ! { // Configure logger log_level := log.level_from_tag(conf.log_level) or { return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -27,7 +27,7 @@ pub fn cron(conf Config) ? { } mut d := daemon.init_daemon(logger, conf.address, conf.api_key, conf.base_image, ce, - conf.max_concurrent_builds, conf.api_update_frequency, conf.image_rebuild_frequency)? + conf.max_concurrent_builds, conf.api_update_frequency, conf.image_rebuild_frequency)! d.run() } diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index beed9fc8..42edc92b 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -79,7 +79,7 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { mut status := 0 res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(), - &sb.target) or { + &sb.target, false) or { d.ldebug('build_target error: $err.msg()') status = 1 diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 934d35a0..0d30a23b 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -6,7 +6,7 @@ import datatypes { MinHeap } import cron.expression { CronExpression, parse_expression } import math import build -import vieter_v.docker +import docker import os import client import models { Target } @@ -53,7 +53,7 @@ mut: // init_daemon initializes a new Daemon object. It renews the targets & // populates the build queue for the first time. -pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon { +pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) !Daemon { mut d := Daemon{ client: client.new(address, api_key) base_image: base_image @@ -207,7 +207,7 @@ fn (mut d Daemon) renew_queue() { // For some reason, using // ```v - // for d.queue.len() > 0 && d.queue.peek() ?.timestamp < now { + // for d.queue.len() > 0 && d.queue.peek() !.timestamp < now { //``` // here causes the function to prematurely just exit, without any errors or anything, very weird // https://github.com/vlang/v/issues/14042 diff --git a/src/cron/daemon/log.v b/src/cron/daemon/log.v index 003898b5..95a50e74 100644 --- a/src/cron/daemon/log.v +++ b/src/cron/daemon/log.v @@ -3,33 +3,33 @@ module daemon import log // log reate a log message with the given level -pub fn (mut d Daemon) log(msg &string, level log.Level) { +pub fn (mut d Daemon) log(msg string, level log.Level) { lock d.logger { d.logger.send_output(msg, level) } } // lfatal create a log message with the fatal level -pub fn (mut d Daemon) lfatal(msg &string) { +pub fn (mut d Daemon) lfatal(msg string) { d.log(msg, log.Level.fatal) } // lerror create a log message with the error level -pub fn (mut d Daemon) lerror(msg &string) { +pub fn (mut d Daemon) lerror(msg string) { d.log(msg, log.Level.error) } // lwarn create a log message with the warn level -pub fn (mut d Daemon) lwarn(msg &string) { +pub fn (mut d Daemon) lwarn(msg string) { d.log(msg, log.Level.warn) } // linfo create a log message with the info level -pub fn (mut d Daemon) linfo(msg &string) { +pub fn (mut d Daemon) linfo(msg string) { d.log(msg, log.Level.info) } // ldebug create a log message with the debug level -pub fn (mut d Daemon) ldebug(msg &string) { +pub fn (mut d Daemon) ldebug(msg string) { d.log(msg, log.Level.debug) } diff --git a/src/cron/expression/expression.v b/src/cron/expression/expression.v index 17d2dde2..c3ff8c58 100644 --- a/src/cron/expression/expression.v +++ b/src/cron/expression/expression.v @@ -12,7 +12,7 @@ pub struct CronExpression { // next calculates the earliest time this cron expression is valid. It will // always pick a moment in the future, even if ref matches completely up to the // minute. This function conciously does not take gap years into account. -pub fn (ce &CronExpression) next(ref time.Time) ?time.Time { +pub fn (ce &CronExpression) next(ref time.Time) !time.Time { // If the given ref matches the next cron occurence up to the minute, it // will return that value. Because we always want to return a value in the // future, we artifically shift the ref 60 seconds to make sure we always @@ -117,159 +117,20 @@ pub fn (ce &CronExpression) next(ref time.Time) ?time.Time { // next_from_now returns the result of ce.next(ref) where ref is the result of // time.now(). -pub fn (ce &CronExpression) next_from_now() ?time.Time { +pub fn (ce &CronExpression) next_from_now() !time.Time { return ce.next(time.now()) } // next_n returns the n next occurences of the expression, given a starting // time. -pub fn (ce &CronExpression) next_n(ref time.Time, n int) ?[]time.Time { +pub fn (ce &CronExpression) next_n(ref time.Time, n int) ![]time.Time { mut times := []time.Time{cap: n} - times << ce.next(ref)? + times << ce.next(ref)! for i in 1 .. n { - times << ce.next(times[i - 1])? + times << ce.next(times[i - 1])! } return times } - -// parse_range parses a given string into a range of sorted integers, if -// possible. -fn parse_range(s string, min int, max int, mut bitv []bool) ? { - mut start := min - mut end := max - mut interval := 1 - - exps := s.split('/') - - if exps.len > 2 { - return error('Invalid expression.') - } - - if exps[0] != '*' { - dash_parts := exps[0].split('-') - - if dash_parts.len > 2 { - return error('Invalid expression.') - } - - start = dash_parts[0].int() - - // The builtin parsing functions return zero if the string can't be - // parsed into a number, so we have to explicitely check whether they - // actually entered zero or if it's an invalid number. - if start == 0 && dash_parts[0] != '0' { - return error('Invalid number.') - } - - // Check whether the start value is out of range - if start < min || start > max { - return error('Out of range.') - } - - if dash_parts.len == 2 { - end = dash_parts[1].int() - - if end == 0 && dash_parts[1] != '0' { - return error('Invalid number.') - } - - if end < start || end > max { - return error('Out of range.') - } - } - } - - if exps.len > 1 { - interval = exps[1].int() - - // interval being zero is always invalid, but we want to check why - // it's invalid for better error messages. - if interval == 0 { - if exps[1] != '0' { - return error('Invalid number.') - } else { - return error('Step size zero not allowed.') - } - } - - if interval > max - min { - return error('Step size too large.') - } - } - // Here, s solely consists of a number, so that's the only value we - // should return. - else if exps[0] != '*' && !exps[0].contains('-') { - bitv[start - min] = true - return - } - - for start <= end { - bitv[start - min] = true - start += interval - } -} - -// bitv_to_ints converts a bit vector into an array containing the -// corresponding values. -fn bitv_to_ints(bitv []bool, min int) []int { - mut out := []int{} - - for i in 0 .. bitv.len { - if bitv[i] { - out << min + i - } - } - - return out -} - -// parse_part parses a given part of a cron expression & returns the -// corresponding array of ints. -fn parse_part(s string, min int, max int) ?[]int { - mut bitv := []bool{len: max - min + 1, init: false} - - for range in s.split(',') { - parse_range(range, min, max, mut bitv)? - } - - return bitv_to_ints(bitv, min) -} - -// parse_expression parses an entire cron expression string into a -// CronExpression object, if possible. -pub fn parse_expression(exp string) ?CronExpression { - // The filter allows for multiple spaces between parts - mut parts := exp.split(' ').filter(it != '') - - if parts.len < 2 || parts.len > 4 { - return error('Expression must contain between 2 and 4 space-separated parts.') - } - - // For ease of use, we allow the user to only specify as many parts as they - // need. - for parts.len < 4 { - parts << '*' - } - - mut part_results := [][]int{} - - mins := [0, 0, 1, 1] - maxs := [59, 23, 31, 12] - - // This for loop allows us to more clearly propagate the error to the user. - for i, min in mins { - part_results << parse_part(parts[i], min, maxs[i]) or { - return error('An error occurred with part $i: $err.msg()') - } - } - - return CronExpression{ - minutes: part_results[0] - hours: part_results[1] - days: part_results[2] - months: part_results[3] - } -} diff --git a/src/cron/expression/expression_parse.v b/src/cron/expression/expression_parse.v new file mode 100644 index 00000000..4aaec5bb --- /dev/null +++ b/src/cron/expression/expression_parse.v @@ -0,0 +1,146 @@ +module expression + +import bitfield + +// parse_range parses a given string into a range of sorted integers. Its +// result is a BitField with set bits for all numbers in the result. +fn parse_range(s string, min int, max int) !bitfield.BitField { + mut start := min + mut end := max + mut interval := 1 + mut bf := bitfield.new(max - min + 1) + + exps := s.split('/') + + if exps.len > 2 { + return error('Invalid expression.') + } + + if exps[0] != '*' { + dash_parts := exps[0].split('-') + + if dash_parts.len > 2 { + return error('Invalid expression.') + } + + start = dash_parts[0].int() + + // The builtin parsing functions return zero if the string can't be + // parsed into a number, so we have to explicitely check whether they + // actually entered zero or if it's an invalid number. + if start == 0 && dash_parts[0] != '0' { + return error('Invalid number.') + } + + // Check whether the start value is out of range + if start < min || start > max { + return error('Out of range.') + } + + if dash_parts.len == 2 { + end = dash_parts[1].int() + + if end == 0 && dash_parts[1] != '0' { + return error('Invalid number.') + } + + if end < start || end > max { + return error('Out of range.') + } + } + } + + if exps.len > 1 { + interval = exps[1].int() + + // interval being zero is always invalid, but we want to check why + // it's invalid for better error messages. + if interval == 0 { + if exps[1] != '0' { + return error('Invalid number.') + } else { + return error('Step size zero not allowed.') + } + } + + if interval > max - min { + return error('Step size too large.') + } + } + // Here, s solely consists of a number, so that's the only value we + // should return. + else if exps[0] != '*' && !exps[0].contains('-') { + bf.set_bit(start - min) + return bf + } + + for start <= end { + bf.set_bit(start - min) + start += interval + } + + return bf +} + +// bf_to_ints takes a BitField and converts it into the expected list of actual +// integers. +fn bf_to_ints(bf bitfield.BitField, min int) []int { + mut out := []int{} + + for i in 0 .. bf.get_size() { + if bf.get_bit(i) == 1 { + out << min + i + } + } + + return out +} + +// parse_part parses a given part of a cron expression & returns the +// corresponding array of ints. +fn parse_part(s string, min int, max int) ![]int { + mut bf := bitfield.new(max - min + 1) + + for range in s.split(',') { + bf2 := parse_range(range, min, max)! + bf = bitfield.bf_or(bf, bf2) + } + + return bf_to_ints(bf, min) +} + +// parse_expression parses an entire cron expression string into a +// CronExpression object, if possible. +pub fn parse_expression(exp string) !CronExpression { + // The filter allows for multiple spaces between parts + mut parts := exp.split(' ').filter(it != '') + + if parts.len < 2 || parts.len > 4 { + return error('Expression must contain between 2 and 4 space-separated parts.') + } + + // For ease of use, we allow the user to only specify as many parts as they + // need. + for parts.len < 4 { + parts << '*' + } + + mut part_results := [][]int{} + + mins := [0, 0, 1, 1] + maxs := [59, 23, 31, 12] + + // This for loop allows us to more clearly propagate the error to the user. + for i, min in mins { + part_results << parse_part(parts[i], min, maxs[i]) or { + return error('An error occurred with part $i: $err.msg()') + } + } + + return CronExpression{ + minutes: part_results[0] + hours: part_results[1] + days: part_results[2] + months: part_results[3] + } +} diff --git a/src/cron/expression/expression_parse_test.v b/src/cron/expression/expression_parse_test.v index 4eebc49e..92e82915 100644 --- a/src/cron/expression/expression_parse_test.v +++ b/src/cron/expression/expression_parse_test.v @@ -3,96 +3,87 @@ module expression // parse_range_error returns the returned error message. If the result is '', // that means the function didn't error. fn parse_range_error(s string, min int, max int) string { - mut bitv := []bool{len: max - min + 1, init: false} - - parse_range(s, min, max, mut bitv) or { return err.msg } + parse_range(s, min, max) or { return err.msg } return '' } // =====parse_range===== -fn test_range_star_range() ? { - mut bitv := []bool{len: 6, init: false} - parse_range('*', 0, 5, mut bitv)? +fn test_range_star_range() ! { + bf := parse_range('*', 0, 5)! - assert bitv == [true, true, true, true, true, true] + assert bf_to_ints(bf, 0) == [0, 1, 2, 3, 4, 5] } -fn test_range_number() ? { - mut bitv := []bool{len: 6, init: false} - parse_range('4', 0, 5, mut bitv)? +fn test_range_number() ! { + bf := parse_range('4', 0, 5)! - assert bitv_to_ints(bitv, 0) == [4] + assert bf_to_ints(bf, 0) == [4] } -fn test_range_number_too_large() ? { +fn test_range_number_too_large() ! { assert parse_range_error('10', 0, 6) == 'Out of range.' } -fn test_range_number_too_small() ? { +fn test_range_number_too_small() ! { assert parse_range_error('0', 2, 6) == 'Out of range.' } -fn test_range_number_invalid() ? { +fn test_range_number_invalid() ! { assert parse_range_error('x', 0, 6) == 'Invalid number.' } -fn test_range_step_star_1() ? { - mut bitv := []bool{len: 21, init: false} - parse_range('*/4', 0, 20, mut bitv)? +fn test_range_step_star_1() ! { + bf := parse_range('*/4', 0, 20)! - assert bitv_to_ints(bitv, 0) == [0, 4, 8, 12, 16, 20] + assert bf_to_ints(bf, 0) == [0, 4, 8, 12, 16, 20] } -fn test_range_step_star_2() ? { - mut bitv := []bool{len: 8, init: false} - parse_range('*/3', 1, 8, mut bitv)? +fn test_range_step_star_2() ! { + bf := parse_range('*/3', 1, 8)! - assert bitv_to_ints(bitv, 1) == [1, 4, 7] + assert bf_to_ints(bf, 1) == [1, 4, 7] } -fn test_range_step_star_too_large() ? { +fn test_range_step_star_too_large() ! { assert parse_range_error('*/21', 0, 20) == 'Step size too large.' } -fn test_range_step_zero() ? { +fn test_range_step_zero() ! { assert parse_range_error('*/0', 0, 20) == 'Step size zero not allowed.' } -fn test_range_step_number() ? { - mut bitv := []bool{len: 21, init: false} - parse_range('5/4', 2, 22, mut bitv)? +fn test_range_step_number() ! { + bf := parse_range('5/4', 2, 22)! - assert bitv_to_ints(bitv, 2) == [5, 9, 13, 17, 21] + assert bf_to_ints(bf, 2) == [5, 9, 13, 17, 21] } -fn test_range_step_number_too_large() ? { +fn test_range_step_number_too_large() ! { assert parse_range_error('10/4', 0, 5) == 'Out of range.' } -fn test_range_step_number_too_small() ? { +fn test_range_step_number_too_small() ! { assert parse_range_error('2/4', 5, 10) == 'Out of range.' } -fn test_range_dash() ? { - mut bitv := []bool{len: 10, init: false} - parse_range('4-8', 0, 9, mut bitv)? +fn test_range_dash() ! { + bf := parse_range('4-8', 0, 9)! - assert bitv_to_ints(bitv, 0) == [4, 5, 6, 7, 8] + assert bf_to_ints(bf, 0) == [4, 5, 6, 7, 8] } -fn test_range_dash_step() ? { - mut bitv := []bool{len: 10, init: false} - parse_range('4-8/2', 0, 9, mut bitv)? +fn test_range_dash_step() ! { + bf := parse_range('4-8/2', 0, 9)! - assert bitv_to_ints(bitv, 0) == [4, 6, 8] + assert bf_to_ints(bf, 0) == [4, 6, 8] } // =====parse_part===== -fn test_part_single() ? { - assert parse_part('*', 0, 5)? == [0, 1, 2, 3, 4, 5] +fn test_part_single() ! { + assert parse_part('*', 0, 5)! == [0, 1, 2, 3, 4, 5] } -fn test_part_multiple() ? { - assert parse_part('*/2,2/3', 1, 8)? == [1, 2, 3, 5, 7, 8] +fn test_part_multiple() ! { + assert parse_part('*/2,2/3', 1, 8)! == [1, 2, 3, 5, 7, 8] } diff --git a/src/cron/expression/expression_test.v b/src/cron/expression/expression_test.v index 9e25e924..82bf9598 100644 --- a/src/cron/expression/expression_test.v +++ b/src/cron/expression/expression_test.v @@ -2,12 +2,12 @@ module expression import time { parse } -fn util_test_time(exp string, t1_str string, t2_str string) ? { - ce := parse_expression(exp)? - t1 := parse(t1_str)? - t2 := parse(t2_str)? +fn util_test_time(exp string, t1_str string, t2_str string) ! { + ce := parse_expression(exp)! + t1 := parse(t1_str)! + t2 := parse(t2_str)! - t3 := ce.next(t1)? + t3 := ce.next(t1)! assert t2.year == t3.year assert t2.month == t3.month @@ -16,19 +16,19 @@ fn util_test_time(exp string, t1_str string, t2_str string) ? { assert t2.minute == t3.minute } -fn test_next_simple() ? { +fn test_next_simple() ! { // Very simple - util_test_time('0 3', '2002-01-01 00:00:00', '2002-01-01 03:00:00')? + util_test_time('0 3', '2002-01-01 00:00:00', '2002-01-01 03:00:00')! // Overlap to next day - util_test_time('0 3', '2002-01-01 03:00:00', '2002-01-02 03:00:00')? - util_test_time('0 3', '2002-01-01 04:00:00', '2002-01-02 03:00:00')? + util_test_time('0 3', '2002-01-01 03:00:00', '2002-01-02 03:00:00')! + util_test_time('0 3', '2002-01-01 04:00:00', '2002-01-02 03:00:00')! - util_test_time('0 3/4', '2002-01-01 04:00:00', '2002-01-01 07:00:00')? + util_test_time('0 3/4', '2002-01-01 04:00:00', '2002-01-01 07:00:00')! // Overlap to next month - util_test_time('0 3', '2002-11-31 04:00:00', '2002-12-01 03:00:00')? + util_test_time('0 3', '2002-11-31 04:00:00', '2002-12-01 03:00:00')! // Overlap to next year - util_test_time('0 3', '2002-12-31 04:00:00', '2003-01-01 03:00:00')? + util_test_time('0 3', '2002-12-31 04:00:00', '2003-01-01 03:00:00')! } diff --git a/src/db/db.v b/src/db/db.v index 9459c052..98ee0005 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -17,17 +17,21 @@ const ( $embed_file('migrations/001-initial/up.sql'), $embed_file('migrations/002-rename-to-targets/up.sql'), $embed_file('migrations/003-target-url-type/up.sql'), + $embed_file('migrations/004-nullable-branch/up.sql'), + $embed_file('migrations/005-repo-path/up.sql'), ] migrations_down = [ $embed_file('migrations/001-initial/down.sql'), $embed_file('migrations/002-rename-to-targets/down.sql'), $embed_file('migrations/003-target-url-type/down.sql'), + $embed_file('migrations/004-nullable-branch/down.sql'), + $embed_file('migrations/005-repo-path/down.sql'), ] ) // init initializes a database & adds the correct tables. -pub fn init(db_path string) ?VieterDb { - conn := sqlite.connect(db_path)? +pub fn init(db_path string) !VieterDb { + conn := sqlite.connect(db_path)! sql conn { create table MigrationVersion @@ -60,7 +64,7 @@ pub fn init(db_path string) ?VieterDb { res := conn.exec_none(part) if res != sqlite.sqlite_done { - return error('An error occurred while applying migration $version_num') + return error('An error occurred while applying migration $version_num: SQLite error code $res') } } diff --git a/src/db/logs.v b/src/db/logs.v index 923dde2d..27454674 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -84,6 +84,8 @@ pub fn (db &VieterDb) add_build_log(log BuildLog) int { insert log into BuildLog } + // Here, this does work because a log doesn't contain any foreign keys, + // meaning the ORM only has to do a single add inserted_id := db.conn.last_id() as int return inserted_id diff --git a/src/db/migrations/004-nullable-branch/down.sql b/src/db/migrations/004-nullable-branch/down.sql new file mode 100644 index 00000000..2515593e --- /dev/null +++ b/src/db/migrations/004-nullable-branch/down.sql @@ -0,0 +1,26 @@ +-- This down won't really work because it'll throw NOT NULL errors, but I'm +-- just putting it here for future reference (still not sure whether I'm even + -- gonna use these) +PRAGMA foreign_keys=off; + +BEGIN TRANSACTION; + +ALTER TABLE Target RENAME TO _Target_old; + +CREATE TABLE Target ( + id INTEGER PRIMARY KEY, + url TEXT NOT NULL, + branch TEXT NOT NULL, + repo TEXT NOT NULL, + schedule TEXT, + kind TEXT NOT NULL DEFAULT 'git' +); + +INSERT INTO Target (id, url, branch, repo, schedule, kind) + SELECT id, url, branch, repo, schedule, kind FROM _Target_old; + +DROP TABLE _Target_old; + +COMMIT; + +PRAGMA foreign_keys=on; diff --git a/src/db/migrations/004-nullable-branch/up.sql b/src/db/migrations/004-nullable-branch/up.sql new file mode 100644 index 00000000..6333c37f --- /dev/null +++ b/src/db/migrations/004-nullable-branch/up.sql @@ -0,0 +1,23 @@ +PRAGMA foreign_keys=off; + +BEGIN TRANSACTION; + +ALTER TABLE Target RENAME TO _Target_old; + +CREATE TABLE Target ( + id INTEGER PRIMARY KEY, + url TEXT NOT NULL, + branch TEXT, + repo TEXT NOT NULL, + schedule TEXT, + kind TEXT NOT NULL DEFAULT 'git' +); + +INSERT INTO Target (id, url, branch, repo, schedule, kind) + SELECT id, url, branch, repo, schedule, kind FROM _Target_old; + +DROP TABLE _Target_old; + +COMMIT; + +PRAGMA foreign_keys=on; diff --git a/src/db/migrations/005-repo-path/down.sql b/src/db/migrations/005-repo-path/down.sql new file mode 100644 index 00000000..8a6f021e --- /dev/null +++ b/src/db/migrations/005-repo-path/down.sql @@ -0,0 +1 @@ +ALTER TABLE Target DROP COLUMN path; diff --git a/src/db/migrations/005-repo-path/up.sql b/src/db/migrations/005-repo-path/up.sql new file mode 100644 index 00000000..f7e5c296 --- /dev/null +++ b/src/db/migrations/005-repo-path/up.sql @@ -0,0 +1 @@ +ALTER TABLE Target ADD COLUMN path TEXT; diff --git a/src/db/targets.v b/src/db/targets.v index a705ebb5..41e56dfe 100644 --- a/src/db/targets.v +++ b/src/db/targets.v @@ -38,14 +38,17 @@ pub fn (db &VieterDb) get_target(target_id int) ?Target { } // add_target inserts the given target into the database. -pub fn (db &VieterDb) add_target(repo Target) int { +pub fn (db &VieterDb) add_target(target Target) int { sql db.conn { - insert repo into Target + insert target into Target } - inserted_id := db.conn.last_id() as int + // ID of inserted target is the largest id + inserted_target := sql db.conn { + select from Target order by id desc limit 1 + } - return inserted_id + return inserted_target.id } // delete_target deletes the target with the given id from the database. diff --git a/src/main.v b/src/main.v index fc09f7e2..1053c2f4 100644 --- a/src/main.v +++ b/src/main.v @@ -9,12 +9,18 @@ import console.schedule import console.man import console.aur import cron +import agent fn main() { + // Stop buffering output so logs always show up immediately + unsafe { + C.setbuf(C.stdout, 0) + } + mut app := cli.Command{ name: 'vieter' description: 'Vieter is a lightweight implementation of an Arch repository server.' - version: '0.4.0' + version: '0.5.0-rc.1' flags: [ cli.Flag{ flag: cli.FlagType.string @@ -40,6 +46,7 @@ fn main() { schedule.cmd(), man.cmd(), aur.cmd(), + agent.cmd(), ] } app.setup() diff --git a/src/models/builds.v b/src/models/builds.v new file mode 100644 index 00000000..926a53c2 --- /dev/null +++ b/src/models/builds.v @@ -0,0 +1,18 @@ +module models + +pub struct BuildConfig { +pub: + target_id int + kind string + url string + branch string + path string + repo string + base_image string + force bool +} + +// str return a single-line string representation of a build log +pub fn (c BuildConfig) str() string { + return '{ target: $c.target_id, kind: $c.kind, url: $c.url, branch: $c.branch, path: $c.path, repo: $c.repo, base_image: $c.base_image, force: $c.force }' +} diff --git a/src/models/targets.v b/src/models/targets.v index c8aa5357..af3cb0df 100644 --- a/src/models/targets.v +++ b/src/models/targets.v @@ -28,27 +28,45 @@ pub mut: repo string [nonull] // Cron schedule describing how frequently to build the repo. schedule string + // Subdirectory in the Git repository to cd into + path string // On which architectures the package is allowed to be built. In reality, - // this controls which builders will periodically build the image. + // this controls which agents will build this package when scheduled. arch []TargetArch [fkey: 'target_id'] } // str returns a string representation. -pub fn (gr &Target) str() string { +pub fn (t &Target) str() string { mut parts := [ - 'id: $gr.id', - 'kind: $gr.kind', - 'url: $gr.url', - 'branch: $gr.branch', - 'repo: $gr.repo', - 'schedule: $gr.schedule', - 'arch: ${gr.arch.map(it.value).join(', ')}', + 'id: $t.id', + 'kind: $t.kind', + 'url: $t.url', + 'branch: $t.branch', + 'path: $t.path', + 'repo: $t.repo', + 'schedule: $t.schedule', + 'arch: ${t.arch.map(it.value).join(', ')}', ] str := parts.join('\n') return str } +// as_build_config converts a Target into a BuildConfig, given some extra +// needed information. +pub fn (t &Target) as_build_config(base_image string, force bool) BuildConfig { + return BuildConfig{ + target_id: t.id + kind: t.kind + url: t.url + branch: t.branch + path: t.path + repo: t.repo + base_image: base_image + force: force + } +} + [params] pub struct TargetFilter { pub mut: diff --git a/src/package/README.md b/src/package/README.md new file mode 100644 index 00000000..b2bcbd77 --- /dev/null +++ b/src/package/README.md @@ -0,0 +1,5 @@ +# package + +This module handles both parsing the published Arch tarballs & the contents of +their `.PKGINFO` files, as well as generating the contents of the database +archives' `desc` & `files` files. diff --git a/src/package/format.v b/src/package/format.v new file mode 100644 index 00000000..a81d3272 --- /dev/null +++ b/src/package/format.v @@ -0,0 +1,103 @@ +module package + +// format_entry returns a string properly formatted to be added to a desc file. +[inline] +fn format_entry(key string, value string) string { + return '\n%$key%\n$value\n' +} + +// full_name returns the properly formatted name for the package, including +// version & architecture +pub fn (pkg &Pkg) full_name() string { + p := pkg.info + return '$p.name-$p.version-$p.arch' +} + +// filename returns the correct filename of the package file +pub fn (pkg &Pkg) filename() string { + ext := match pkg.compression { + 0 { '.tar' } + 1 { '.tar.gz' } + 6 { '.tar.xz' } + 14 { '.tar.zst' } + else { panic("Another compression code shouldn't be possible. Faulty code: $pkg.compression") } + } + + return '${pkg.full_name()}.pkg$ext' +} + +// to_desc returns a desc file valid string representation +pub fn (pkg &Pkg) to_desc() !string { + p := pkg.info + + // filename + mut desc := '%FILENAME%\n$pkg.filename()\n' + + desc += format_entry('NAME', p.name) + desc += format_entry('BASE', p.base) + desc += format_entry('VERSION', p.version) + + if p.description.len > 0 { + desc += format_entry('DESC', p.description) + } + + if p.groups.len > 0 { + desc += format_entry('GROUPS', p.groups.join_lines()) + } + + desc += format_entry('CSIZE', p.csize.str()) + desc += format_entry('ISIZE', p.size.str()) + + sha256sum := pkg.checksum()! + + desc += format_entry('SHA256SUM', sha256sum) + + // TODO add pgpsig stuff + + if p.url.len > 0 { + desc += format_entry('URL', p.url) + } + + if p.licenses.len > 0 { + desc += format_entry('LICENSE', p.licenses.join_lines()) + } + + desc += format_entry('ARCH', p.arch) + desc += format_entry('BUILDDATE', p.build_date.str()) + desc += format_entry('PACKAGER', p.packager) + + if p.replaces.len > 0 { + desc += format_entry('REPLACES', p.replaces.join_lines()) + } + + if p.conflicts.len > 0 { + desc += format_entry('CONFLICTS', p.conflicts.join_lines()) + } + + if p.provides.len > 0 { + desc += format_entry('PROVIDES', p.provides.join_lines()) + } + + if p.depends.len > 0 { + desc += format_entry('DEPENDS', p.depends.join_lines()) + } + + if p.optdepends.len > 0 { + desc += format_entry('OPTDEPENDS', p.optdepends.join_lines()) + } + + if p.makedepends.len > 0 { + desc += format_entry('MAKEDEPENDS', p.makedepends.join_lines()) + } + + if p.checkdepends.len > 0 { + desc += format_entry('CHECKDEPENDS', p.checkdepends.join_lines()) + } + + return '$desc\n' +} + +// to_files returns a files file valid string representation +pub fn (pkg &Pkg) to_files() string { + return '%FILES%\n$pkg.files.join_lines()\n' +} diff --git a/src/package/package.v b/src/package/package.v index 9eaf5a23..4518ffde 100644 --- a/src/package/package.v +++ b/src/package/package.v @@ -43,12 +43,12 @@ pub mut: } // checksum calculates the sha256 hash of the package -pub fn (p &Pkg) checksum() ?string { +pub fn (p &Pkg) checksum() !string { return util.hash_file(p.path) } // parse_pkg_info_string parses a PkgInfo object from a string -fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { +fn parse_pkg_info_string(pkg_info_str &string) !PkgInfo { mut pkg_info := PkgInfo{} // Iterate over the entire string @@ -101,7 +101,7 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { // read_pkg_archive extracts the file list & .PKGINFO contents from an archive // NOTE: this command only supports zstd-, xz- & gzip-compressed tarballs. -pub fn read_pkg_archive(pkg_path string) ?Pkg { +pub fn read_pkg_archive(pkg_path string) !Pkg { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") } @@ -159,7 +159,7 @@ pub fn read_pkg_archive(pkg_path string) ?Pkg { pkg_text := unsafe { buf.vstring_with_len(size).clone() } - pkg_info = parse_pkg_info_string(pkg_text)? + pkg_info = parse_pkg_info_string(pkg_text)! } else { C.archive_read_data_skip(a) } @@ -174,104 +174,3 @@ pub fn read_pkg_archive(pkg_path string) ?Pkg { compression: compression_code } } - -// format_entry returns a string properly formatted to be added to a desc file. -fn format_entry(key string, value string) string { - return '\n%$key%\n$value\n' -} - -// full_name returns the properly formatted name for the package, including -// version & architecture -pub fn (pkg &Pkg) full_name() string { - p := pkg.info - return '$p.name-$p.version-$p.arch' -} - -// filename returns the correct filename of the package file -pub fn (pkg &Pkg) filename() string { - ext := match pkg.compression { - 0 { '.tar' } - 1 { '.tar.gz' } - 6 { '.tar.xz' } - 14 { '.tar.zst' } - else { panic("Another compression code shouldn't be possible. Faulty code: $pkg.compression") } - } - - return '${pkg.full_name()}.pkg$ext' -} - -// to_desc returns a desc file valid string representation -pub fn (pkg &Pkg) to_desc() ?string { - p := pkg.info - - // filename - mut desc := '%FILENAME%\n$pkg.filename()\n' - - desc += format_entry('NAME', p.name) - desc += format_entry('BASE', p.base) - desc += format_entry('VERSION', p.version) - - if p.description.len > 0 { - desc += format_entry('DESC', p.description) - } - - if p.groups.len > 0 { - desc += format_entry('GROUPS', p.groups.join_lines()) - } - - desc += format_entry('CSIZE', p.csize.str()) - desc += format_entry('ISIZE', p.size.str()) - - sha256sum := pkg.checksum()? - - desc += format_entry('SHA256SUM', sha256sum) - - // TODO add pgpsig stuff - - if p.url.len > 0 { - desc += format_entry('URL', p.url) - } - - if p.licenses.len > 0 { - desc += format_entry('LICENSE', p.licenses.join_lines()) - } - - desc += format_entry('ARCH', p.arch) - desc += format_entry('BUILDDATE', p.build_date.str()) - desc += format_entry('PACKAGER', p.packager) - - if p.replaces.len > 0 { - desc += format_entry('REPLACES', p.replaces.join_lines()) - } - - if p.conflicts.len > 0 { - desc += format_entry('CONFLICTS', p.conflicts.join_lines()) - } - - if p.provides.len > 0 { - desc += format_entry('PROVIDES', p.provides.join_lines()) - } - - if p.depends.len > 0 { - desc += format_entry('DEPENDS', p.depends.join_lines()) - } - - if p.optdepends.len > 0 { - desc += format_entry('OPTDEPENDS', p.optdepends.join_lines()) - } - - if p.makedepends.len > 0 { - desc += format_entry('MAKEDEPENDS', p.makedepends.join_lines()) - } - - if p.checkdepends.len > 0 { - desc += format_entry('CHECKDEPENDS', p.checkdepends.join_lines()) - } - - return '$desc\n' -} - -// to_files returns a files file valid string representation -pub fn (pkg &Pkg) to_files() string { - return '%FILES%\n$pkg.files.join_lines()\n' -} diff --git a/src/repo/README.md b/src/repo/README.md new file mode 100644 index 00000000..f06b1d38 --- /dev/null +++ b/src/repo/README.md @@ -0,0 +1,43 @@ +# repo + +This module manages the contents of the various repositories stored within a +Vieter instance. + +## Terminology + +* Arch-repository (arch-repo): specific architecture of a given repository. This is what + Pacman actually uses as a repository, and contains its own `.db` & `.files` + files. +* Repository (repo): a collection of arch-repositories. A single repository can + contain packages of different architectures, with each package being stored + in that specific architecture' arch-repository. +* Repository group (repo-group): a collection of repositories. Each Vieter + instance consists of a single repository group, which manages all underlying + repositories & arch-repositories. + +## Arch-repository layout + +An arch-repository (aka a regular Pacman repository) consists of a directory +with the following files (`{repo}` should be replaced with the name of the +repository): + +* One or more package directories. These directories follow the naming scheme + `${pkgname}-${pkgver}-${pkgrel}`. Each of these directories contains two + files, `desc` & `files`. The `desc` file is a list of the package's metadata, + while `files` contains a list of all files that the package contains. The + latter is used when using `pacman -F`. +* `{repo}.db` & `{repo}.db.tar.gz`: the database file of the repository. This + is just a compressed tarball of all package directories, but only their + `desc` files. Both these files should have the same content (`repo-add` + creates a symlink, but Vieter just serves the same file for both routes) +* `{repo}.files` & `{repo}.files.tar.gz`: the same as the `.db` file, but this + also contains the `files` files, instead of just the `desc` files. + +## Filesystem layout + +The repository part of Vieter consists of two directories. One is the `repos` +directory inside the configured `data_dir`, while the other is the configured +`pkg_dir`. `repos` contains only the repository group, while `pkg_dir` contains +the actual package archives. `pkg_dir` is the directory that can take up a +significant amount of memory, while `repos` solely consists of small text +files. diff --git a/src/repo/add.v b/src/repo/add.v index 608ca501..8ab3ae1f 100644 --- a/src/repo/add.v +++ b/src/repo/add.v @@ -29,7 +29,7 @@ pub: } // new creates a new RepoGroupManager & creates the directories as needed -pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupManager { +pub fn new(repos_dir string, pkg_dir string, default_arch string) !RepoGroupManager { if !os.is_dir(repos_dir) { os.mkdir_all(repos_dir) or { return error('Failed to create repos directory: $err.msg()') } } @@ -49,27 +49,27 @@ pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupMana // pkg archive. It's a wrapper around add_pkg_in_repo that parses the archive // file, passes the result to add_pkg_in_repo, and hard links the archive to // the right subdirectories in r.pkg_dir if it was successfully added. -pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult { +pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) !RepoAddResult { pkg := package.read_pkg_archive(pkg_path) or { return error('Failed to read package file: $err.msg()') } - archs := r.add_pkg_in_repo(repo, pkg)? + archs := r.add_pkg_in_repo(repo, pkg)! // If the add was successful, we move the file to the packages directory for arch in archs { repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch)) dest_path := os.join_path_single(repo_pkg_path, pkg.filename()) - os.mkdir_all(repo_pkg_path)? + os.mkdir_all(repo_pkg_path)! // We create hard links so that "any" arch packages aren't stored // multiple times - os.link(pkg_path, dest_path)? + os.link(pkg_path, dest_path)! } // After linking, we can remove the original file - os.rm(pkg_path)? + os.rm(pkg_path)! return RepoAddResult{ name: pkg.info.name @@ -85,11 +85,11 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re // r.default_arch. If this arch-repo doesn't exist yet, it is created. If the // architecture isn't 'any', the package is only added to the specific // architecture. -fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]string { +fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ![]string { // A package not of arch 'any' can be handled easily by adding it to the // respective repo if pkg.info.arch != 'any' { - r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)? + r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)! return [pkg.info.arch] } @@ -104,7 +104,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin // If this is the first package that's added to the repo, the directory // won't exist yet if os.exists(repo_dir) { - arch_repos = os.ls(repo_dir)? + arch_repos = os.ls(repo_dir)! } // The default_arch should always be updated when a package with arch 'any' @@ -118,7 +118,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin // not know which arch-repositories did succeed in adding the package, if // any. for arch in arch_repos { - r.add_pkg_in_arch_repo(repo, arch, pkg)? + r.add_pkg_in_arch_repo(repo, arch, pkg)! } return arch_repos @@ -128,24 +128,24 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin // arch-repo. It records the package's data in the arch-repo's desc & files // files, and afterwards updates the db & files archives to reflect these // changes. -fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ? { +fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ! { pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version') // Remove the previous version of the package, if present - r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false)? + r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false)! os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') } - os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()?) or { - os.rmdir_all(pkg_dir)? + os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()!) or { + os.rmdir_all(pkg_dir)! return error('Failed to write desc file.') } os.write_file(os.join_path_single(pkg_dir, 'files'), pkg.to_files()) or { - os.rmdir_all(pkg_dir)? + os.rmdir_all(pkg_dir)! return error('Failed to write files file.') } - r.sync(repo, arch)? + r.sync(repo, arch)! } diff --git a/src/repo/remove.v b/src/repo/remove.v index add921cb..63866a95 100644 --- a/src/repo/remove.v +++ b/src/repo/remove.v @@ -5,7 +5,7 @@ import os // remove_pkg_from_arch_repo removes a package from an arch-repo's database. It // returns false if the package wasn't present in the database. It also // optionally re-syncs the repo archives. -pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool { +pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) !bool { repo_dir := os.join_path(r.repos_dir, repo, arch) // If the repository doesn't exist yet, the result is automatically false @@ -15,7 +15,7 @@ pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, // We iterate over every directory in the repo dir // TODO filter so we only check directories - for d in os.ls(repo_dir)? { + for d in os.ls(repo_dir)! { // Because a repository only allows a single version of each package, // we need only compare whether the name of the package is the same, // not the version. @@ -25,22 +25,22 @@ pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, // We lock the mutex here to prevent other routines from creating a // new archive while we remove an entry lock r.mutex { - os.rmdir_all(os.join_path_single(repo_dir, d))? + os.rmdir_all(os.join_path_single(repo_dir, d))! } // Also remove the package archive repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch) - archives := os.ls(repo_pkg_dir)?.filter(it.split('-')#[..-3].join('-') == name) + archives := os.ls(repo_pkg_dir)!.filter(it.split('-')#[..-3].join('-') == name) for archive_name in archives { full_path := os.join_path_single(repo_pkg_dir, archive_name) - os.rm(full_path)? + os.rm(full_path)! } // Sync the db archives if requested if sync { - r.sync(repo, arch)? + r.sync(repo, arch)! } return true @@ -51,7 +51,7 @@ pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, } // remove_arch_repo removes an arch-repo & its packages. -pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) ?bool { +pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) !bool { repo_dir := os.join_path(r.repos_dir, repo, arch) // If the repository doesn't exist yet, the result is automatically false @@ -59,16 +59,16 @@ pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) ?bool { return false } - os.rmdir_all(repo_dir)? + os.rmdir_all(repo_dir)! pkg_dir := os.join_path(r.pkg_dir, repo, arch) - os.rmdir_all(pkg_dir)? + os.rmdir_all(pkg_dir)! return true } // remove_repo removes a repo & its packages. -pub fn (r &RepoGroupManager) remove_repo(repo string) ?bool { +pub fn (r &RepoGroupManager) remove_repo(repo string) !bool { repo_dir := os.join_path_single(r.repos_dir, repo) // If the repository doesn't exist yet, the result is automatically false @@ -76,10 +76,10 @@ pub fn (r &RepoGroupManager) remove_repo(repo string) ?bool { return false } - os.rmdir_all(repo_dir)? + os.rmdir_all(repo_dir)! pkg_dir := os.join_path_single(r.pkg_dir, repo) - os.rmdir_all(pkg_dir)? + os.rmdir_all(pkg_dir)! return true } diff --git a/src/repo/sync.v b/src/repo/sync.v index 73d21c88..95547484 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -32,7 +32,7 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri } // sync regenerates the repository archive files. -fn (r &RepoGroupManager) sync(repo string, arch string) ? { +fn (r &RepoGroupManager) sync(repo string, arch string) ! { subrepo_path := os.join_path(r.repos_dir, repo, arch) lock r.mutex { @@ -54,7 +54,7 @@ fn (r &RepoGroupManager) sync(repo string, arch string) ? { C.archive_write_open_filename(a_files, &char(files_path.str)) // Iterate over each directory - for d in os.ls(subrepo_path)?.filter(os.is_dir(os.join_path_single(subrepo_path, + for d in os.ls(subrepo_path)!.filter(os.is_dir(os.join_path_single(subrepo_path, it))) { // desc mut inner_path := os.join_path_single(d, 'desc') diff --git a/src/server/api_jobs.v b/src/server/api_jobs.v new file mode 100644 index 00000000..77953514 --- /dev/null +++ b/src/server/api_jobs.v @@ -0,0 +1,49 @@ +module server + +import web +import web.response { new_data_response, new_response } + +// v1_poll_job_queue allows agents to poll for new build jobs. +['/api/v1/jobs/poll'; auth; get] +fn (mut app App) v1_poll_job_queue() web.Result { + arch := app.query['arch'] or { + return app.json(.bad_request, new_response('Missing arch query arg.')) + } + + max_str := app.query['max'] or { + return app.json(.bad_request, new_response('Missing max query arg.')) + } + max := max_str.int() + + mut out := app.job_queue.pop_n(arch, max).map(it.config) + + return app.json(.ok, new_data_response(out)) +} + +// v1_queue_job allows queueing a new one-time build job for the given target. +['/api/v1/jobs/queue'; auth; post] +fn (mut app App) v1_queue_job() web.Result { + target_id := app.query['target'] or { + return app.json(.bad_request, new_response('Missing target query arg.')) + }.int() + + arch := app.query['arch'] or { + return app.json(.bad_request, new_response('Missing arch query arg.')) + } + + if arch == '' { + app.json(.bad_request, new_response('Empty arch query arg.')) + } + + force := 'force' in app.query + + target := app.db.get_target(target_id) or { + return app.json(.bad_request, new_response('Unknown target id.')) + } + + app.job_queue.insert(target: target, arch: arch, single: true, now: true, force: force) or { + return app.status(.internal_server_error) + } + + return app.status(.ok) +} diff --git a/src/server/api_logs.v b/src/server/api_logs.v index 287755a9..c7521dd5 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -1,7 +1,6 @@ module server import web -import net.http import net.urllib import web.response { new_data_response, new_response } import db @@ -15,7 +14,7 @@ import models { BuildLog, BuildLogFilter } ['/api/v1/logs'; auth; get] fn (mut app App) v1_get_logs() web.Result { filter := models.from_params(app.query) or { - return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) + return app.json(.bad_request, new_response('Invalid query parameters.')) } logs := app.db.get_build_logs(filter) @@ -25,7 +24,7 @@ fn (mut app App) v1_get_logs() web.Result { // v1_get_single_log returns the build log with the given id. ['/api/v1/logs/:id'; auth; get] fn (mut app App) v1_get_single_log(id int) web.Result { - log := app.db.get_build_log(id) or { return app.not_found() } + log := app.db.get_build_log(id) or { return app.status(.not_found) } return app.json(.ok, new_data_response(log)) } @@ -33,7 +32,7 @@ fn (mut app App) v1_get_single_log(id int) web.Result { // v1_get_log_content returns the actual build log file for the given id. ['/api/v1/logs/:id/content'; auth; get] fn (mut app App) v1_get_log_content(id int) web.Result { - log := app.db.get_build_log(id) or { return app.not_found() } + log := app.db.get_build_log(id) or { return app.status(.not_found) } file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), log.arch, file_name) @@ -43,9 +42,9 @@ fn (mut app App) v1_get_log_content(id int) web.Result { // parse_query_time unescapes an HTTP query parameter & tries to parse it as a // time.Time struct. -fn parse_query_time(query string) ?time.Time { - unescaped := urllib.query_unescape(query)? - t := time.parse(unescaped)? +fn parse_query_time(query string) !time.Time { + unescaped := urllib.query_unescape(query)! + t := time.parse(unescaped)! return t } @@ -57,25 +56,25 @@ fn (mut app App) v1_post_log() web.Result { start_time_int := app.query['startTime'].int() if start_time_int == 0 { - return app.json(http.Status.bad_request, new_response('Invalid or missing start time.')) + return app.json(.bad_request, new_response('Invalid or missing start time.')) } start_time := time.unix(start_time_int) end_time_int := app.query['endTime'].int() if end_time_int == 0 { - return app.json(http.Status.bad_request, new_response('Invalid or missing end time.')) + return app.json(.bad_request, new_response('Invalid or missing end time.')) } end_time := time.unix(end_time_int) if 'exitCode' !in app.query { - return app.json(http.Status.bad_request, new_response('Missing exit code.')) + return app.json(.bad_request, new_response('Missing exit code.')) } exit_code := app.query['exitCode'].int() if 'arch' !in app.query { - return app.json(http.Status.bad_request, new_response("Missing parameter 'arch'.")) + return app.json(.bad_request, new_response("Missing parameter 'arch'.")) } arch := app.query['arch'] @@ -83,7 +82,7 @@ fn (mut app App) v1_post_log() web.Result { target_id := app.query['target'].int() if !app.db.target_exists(target_id) { - return app.json(http.Status.bad_request, new_response('Unknown target.')) + return app.json(.bad_request, new_response('Unknown target.')) } // Store log in db @@ -105,7 +104,7 @@ fn (mut app App) v1_post_log() web.Result { os.mkdir_all(repo_logs_dir) or { app.lerror("Couldn't create dir '$repo_logs_dir'.") - return app.json(http.Status.internal_server_error, new_response('An error occured while processing the request.')) + return app.status(.internal_server_error) } } @@ -117,10 +116,10 @@ fn (mut app App) v1_post_log() web.Result { util.reader_to_file(mut app.reader, length.int(), full_path) or { app.lerror('An error occured while receiving logs: $err.msg()') - return app.json(http.Status.internal_server_error, new_response('Failed to upload logs.')) + return app.status(.internal_server_error) } } else { - return app.status(http.Status.length_required) + return app.status(.length_required) } return app.json(.ok, new_data_response(log_id)) diff --git a/src/server/api_targets.v b/src/server/api_targets.v index 6f284af6..cd5cb0a0 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -1,7 +1,6 @@ module server import web -import net.http import web.response { new_data_response, new_response } import db import models { Target, TargetArch, TargetFilter } @@ -10,19 +9,19 @@ import models { Target, TargetArch, TargetFilter } ['/api/v1/targets'; auth; get] fn (mut app App) v1_get_targets() web.Result { filter := models.from_params(app.query) or { - return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) + return app.json(.bad_request, new_response('Invalid query parameters.')) } - repos := app.db.get_targets(filter) + targets := app.db.get_targets(filter) - return app.json(.ok, new_data_response(repos)) + return app.json(.ok, new_data_response(targets)) } // v1_get_single_target returns the information for a single target. ['/api/v1/targets/:id'; auth; get] fn (mut app App) v1_get_single_target(id int) web.Result { - repo := app.db.get_target(id) or { return app.not_found() } + target := app.db.get_target(id) or { return app.status(.not_found) } - return app.json(.ok, new_data_response(repo)) + return app.json(.ok, new_data_response(target)) } // v1_post_target creates a new target from the provided query string. @@ -30,30 +29,36 @@ fn (mut app App) v1_get_single_target(id int) web.Result { fn (mut app App) v1_post_target() web.Result { mut params := app.query.clone() - // If a repo is created without specifying the arch, we assume it's meant + // If a target is created without specifying the arch, we assume it's meant // for the default architecture. - if 'arch' !in params { + if 'arch' !in params || params['arch'] == '' { params['arch'] = app.conf.default_arch } - new_repo := models.from_params(params) or { - return app.json(http.Status.bad_request, new_response(err.msg())) + mut new_target := models.from_params(params) or { + return app.json(.bad_request, new_response(err.msg())) } // Ensure someone doesn't submit an invalid kind - if new_repo.kind !in models.valid_kinds { - return app.json(http.Status.bad_request, new_response('Invalid kind.')) + if new_target.kind !in models.valid_kinds { + return app.json(.bad_request, new_response('Invalid kind.')) } - id := app.db.add_target(new_repo) + id := app.db.add_target(new_target) + new_target.id = id - return app.json(http.Status.ok, new_data_response(id)) + // Add the target to the job queue + // TODO return better error here if it's the cron schedule that's incorrect + app.job_queue.insert_all(new_target) or { return app.status(.internal_server_error) } + + return app.json(.ok, new_data_response(id)) } // v1_delete_target removes a given target from the server's list. ['/api/v1/targets/:id'; auth; delete] fn (mut app App) v1_delete_target(id int) web.Result { app.db.delete_target(id) + app.job_queue.invalidate(id) return app.status(.ok) } @@ -69,5 +74,10 @@ fn (mut app App) v1_patch_target(id int) web.Result { app.db.update_target_archs(id, arch_objs) } - return app.status(.ok) + target := app.db.get_target(id) or { return app.status(.internal_server_error) } + + app.job_queue.invalidate(id) + app.job_queue.insert_all(target) or { return app.status(.internal_server_error) } + + return app.json(.ok, new_data_response(target)) } diff --git a/src/server/cli.v b/src/server/cli.v index 6fd09c54..2fede6ce 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -1,16 +1,18 @@ module server import cli -import vieter_v.conf as vconf +import conf as vconf struct Config { pub: - log_level string = 'WARN' - pkg_dir string - data_dir string - api_key string - default_arch string - port int = 8000 + log_level string = 'WARN' + pkg_dir string + data_dir string + api_key string + default_arch string + global_schedule string = '0 3' + port int = 8000 + base_image string = 'archlinux:base-devel' } // cmd returns the cli submodule that handles starting the server @@ -18,11 +20,11 @@ pub fn cmd() cli.Command { return cli.Command{ name: 'server' description: 'Start the Vieter server.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - server(conf)? + server(conf)! } } } diff --git a/src/server/server.v b/src/server/server.v index 9903cea9..74b1f377 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -6,6 +6,8 @@ import log import repo import util import db +import build { BuildJobQueue } +import cron.expression const ( log_file_name = 'vieter.log' @@ -20,16 +22,39 @@ pub: conf Config [required; web_global] pub mut: repo repo.RepoGroupManager [required; web_global] - db db.VieterDb + // Keys are the various architectures for packages + job_queue BuildJobQueue [required; web_global] + db db.VieterDb +} + +// init_job_queue populates a fresh job queue with all the targets currently +// stored in the database. +fn (mut app App) init_job_queue() ! { + // Initialize build queues + mut targets := app.db.get_targets(limit: 25) + mut i := u64(0) + + for targets.len > 0 { + for target in targets { + app.job_queue.insert_all(target)! + } + + i += 25 + targets = app.db.get_targets(limit: 25, offset: i) + } } // server starts the web server & starts listening for requests -pub fn server(conf Config) ? { +pub fn server(conf Config) ! { // Prevent using 'any' as the default arch if conf.default_arch == 'any' { util.exit_with_message(1, "'any' is not allowed as the value for default_arch.") } + global_ce := expression.parse_expression(conf.global_schedule) or { + util.exit_with_message(1, 'Invalid global cron expression: $err.msg()') + } + // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -71,11 +96,17 @@ pub fn server(conf Config) ? { util.exit_with_message(1, 'Failed to initialize database: $err.msg()') } - web.run(&App{ + mut app := &App{ logger: logger api_key: conf.api_key conf: conf repo: repo db: db - }, conf.port) + job_queue: build.new_job_queue(global_ce, conf.base_image) + } + app.init_job_queue() or { + util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()') + } + + web.run(app, conf.port) } diff --git a/src/util/stream.v b/src/util/stream.v index 06397aa8..15cc6186 100644 --- a/src/util/stream.v +++ b/src/util/stream.v @@ -5,7 +5,7 @@ import io import os // reader_to_writer tries to consume the entire reader & write it to the writer. -pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ? { +pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ! { mut buf := []u8{len: 10 * 1024} for { @@ -21,8 +21,8 @@ pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ? { } // reader_to_file writes the contents of a BufferedReader to a file -pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { - mut file := os.create(path)? +pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ! { + mut file := os.create(path)! defer { file.close() } @@ -69,11 +69,11 @@ pub fn match_array_in_array(a1 []T, a2 []T) int { // read_until_separator consumes an io.Reader until it encounters some // separator array. The data read is stored inside the provided res array. -pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ? { +pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ! { mut buf := []u8{len: sep.len} for { - c := reader.read(mut buf)? + c := reader.read(mut buf)! res << buf[..c] match_len := match_array_in_array(buf[..c], sep) @@ -84,7 +84,7 @@ pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ? { if match_len > 0 { match_left := sep.len - match_len - c2 := reader.read(mut buf[..match_left])? + c2 := reader.read(mut buf[..match_left])! res << buf[..c2] if buf[..c2] == sep[match_len..] { diff --git a/src/util/util.v b/src/util/util.v index 4cd374e7..213104c3 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -23,7 +23,7 @@ pub fn exit_with_message(code int, msg string) { } // hash_file returns the sha256 hash of a given file -pub fn hash_file(path &string) ?string { +pub fn hash_file(path &string) !string { file := os.open(path) or { return error('Failed to open file.') } mut sha256sum := sha256.new() @@ -39,7 +39,7 @@ pub fn hash_file(path &string) ?string { // This function never actually fails, but returns an option to follow // the Writer interface. - sha256sum.write(buf[..bytes_read])? + sha256sum.write(buf[..bytes_read])! } return sha256sum.checksum().hex() diff --git a/src/web/logging.v b/src/web/logging.v index fc697ffc..12b07d7a 100644 --- a/src/web/logging.v +++ b/src/web/logging.v @@ -3,33 +3,33 @@ module web import log // log reate a log message with the given level -pub fn (mut ctx Context) log(msg &string, level log.Level) { +pub fn (mut ctx Context) log(msg string, level log.Level) { lock ctx.logger { ctx.logger.send_output(msg, level) } } // lfatal create a log message with the fatal level -pub fn (mut ctx Context) lfatal(msg &string) { +pub fn (mut ctx Context) lfatal(msg string) { ctx.log(msg, log.Level.fatal) } // lerror create a log message with the error level -pub fn (mut ctx Context) lerror(msg &string) { +pub fn (mut ctx Context) lerror(msg string) { ctx.log(msg, log.Level.error) } // lwarn create a log message with the warn level -pub fn (mut ctx Context) lwarn(msg &string) { +pub fn (mut ctx Context) lwarn(msg string) { ctx.log(msg, log.Level.warn) } // linfo create a log message with the info level -pub fn (mut ctx Context) linfo(msg &string) { +pub fn (mut ctx Context) linfo(msg string) { ctx.log(msg, log.Level.info) } // ldebug create a log message with the debug level -pub fn (mut ctx Context) ldebug(msg &string) { +pub fn (mut ctx Context) ldebug(msg string) { ctx.log(msg, log.Level.debug) } diff --git a/src/web/parse.v b/src/web/parse.v index ee7a72cc..7af635fb 100644 --- a/src/web/parse.v +++ b/src/web/parse.v @@ -8,7 +8,7 @@ import net.http const attrs_to_ignore = ['auth'] // Parsing function attributes for methods and path. -fn parse_attrs(name string, attrs []string) ?([]http.Method, string) { +fn parse_attrs(name string, attrs []string) !([]http.Method, string) { if attrs.len == 0 { return [http.Method.get], '/$name' } @@ -61,7 +61,7 @@ fn parse_query_from_url(url urllib.URL) map[string]string { } // Extract form data from an HTTP request. -fn parse_form_from_request(request http.Request) ?(map[string]string, map[string][]http.FileData) { +fn parse_form_from_request(request http.Request) !(map[string]string, map[string][]http.FileData) { mut form := map[string]string{} mut files := map[string][]http.FileData{} if request.method in methods_with_form { diff --git a/src/web/web.v b/src/web/web.v index 1d1480fb..565baff7 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -24,7 +24,7 @@ pub: pub mut: // TCP connection to client. // But beware, do not store it for further use, after request processing web will close connection. - conn &net.TcpConn + conn &net.TcpConn = unsafe { nil } // Gives access to a shared logger object logger shared log.Log // time.ticks() from start of web connection handle. @@ -67,20 +67,20 @@ struct Route { pub fn (ctx Context) before_request() {} // send_string writes the given string to the TCP connection socket. -fn (mut ctx Context) send_string(s string) ? { - ctx.conn.write(s.bytes())? +fn (mut ctx Context) send_string(s string) ! { + ctx.conn.write(s.bytes())! } // send_reader reads at most `size` bytes from the given reader & writes them // to the TCP connection socket. Internally, a 10KB buffer is used, to avoid // having to store all bytes in memory at once. -fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ? { +fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ! { mut buf := []u8{len: 10_000} mut bytes_left := size // Repeat as long as the stream still has data for bytes_left > 0 { - bytes_read := reader.read(mut buf)? + bytes_read := reader.read(mut buf)! bytes_left -= u64(bytes_read) mut to_write := bytes_read @@ -96,20 +96,20 @@ fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ? { // send_custom_response sends the given http.Response to the client. It can be // used to overwrite the Context object & send a completely custom // http.Response instead. -fn (mut ctx Context) send_custom_response(resp &http.Response) ? { - ctx.send_string(resp.bytestr())? +fn (mut ctx Context) send_custom_response(resp &http.Response) ! { + ctx.send_string(resp.bytestr())! } // send_response_header constructs a valid HTTP response with an empty body & // sends it to the client. -pub fn (mut ctx Context) send_response_header() ? { +pub fn (mut ctx Context) send_response_header() ! { mut resp := http.new_response( header: ctx.header.join(headers_close) ) resp.header.add(.content_type, ctx.content_type) resp.set_status(ctx.status) - ctx.send_custom_response(resp)? + ctx.send_custom_response(resp)! } // send is a convenience function for sending the HTTP response with an empty @@ -260,13 +260,6 @@ pub fn (mut ctx Context) redirect(url string) Result { return Result{} } -// not_found Send an not_found response -pub fn (mut ctx Context) not_found() Result { - ctx.send_custom_response(http_404) or {} - - return Result{} -} - interface DbInterface { db voidptr } diff --git a/vieter.toml b/vieter.toml index d3922a43..74a73973 100644 --- a/vieter.toml +++ b/vieter.toml @@ -4,11 +4,11 @@ data_dir = "data" pkg_dir = "data/pkgs" log_level = "DEBUG" default_arch = "x86_64" +arch = "x86_64" address = "http://localhost:8000" -global_schedule = '* *' +# global_schedule = '* *' api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 -