diff --git a/CHANGELOG.md b/CHANGELOG.md index d2dd760..c55e16b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Migrated codebase to V 0.3.2 * Cron expression parser now uses bitfields instead of bool arrays +* Added option to deploy using agent-server architecture instead of cron daemon +* Allow force-building packages, meaning the build won't check if the + repository is already up to date +* Allow scheduling builds on the server from the CLI tool instead of building + them locally ### Fixed @@ -19,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * CLI no longer exits with non-zero status code when removing/patching target * Allow NULL values for branch in database +* Endpoint for adding targets now returns the correct id ## [0.4.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.4.0) diff --git a/README.md b/README.md index b9fff69..637d4c1 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ quicker. I chose [V](https://vlang.io/) as I've been very intrigued by this language for a while now. I wanted a fast language that I could code while relaxing, without having to exert too much mental effort & V seemed like the right choice for -that. +that. Sadly, this didn't quite turn out the way I expected, but I'm sticking +with it anyways ;p ## Features @@ -49,7 +50,7 @@ update`. I used to maintain a mirror that tracked the latest master, but nowadays, I maintain a Docker image containing the specific compiler version that Vieter -builds with. Currently, this is V 0.3. +builds with. Currently, this is V 0.3.2. ## Contributing diff --git a/docs/content/configuration.md b/docs/content/configuration.md index af941a2..95bf713 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -17,7 +17,7 @@ If a variable is both present in the config file & as an environment variable, the value in the environment variable is used. {{< hint info >}} -**Note** +**Note** All environment variables can also be provided from a file by appending them with `_FILE`. This for example allows you to provide the API key from a Docker secrets file. @@ -97,3 +97,25 @@ configuration variable required for each command. build`. * Default: `archlinux:base-devel` +### `vieter agent` + +* `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`, + `WARN`, `INFO` or `DEBUG`. + * Default: `WARN` +* `address`: *public* URL of the Vieter repository server to build for. From + this server jobs are retrieved. All built packages are published to this + server. +* `api_key`: API key of the above server. +* `data_dir`: directory to store log file in. +* `max_concurrent_builds`: how many builds to run at the same time. + * Default: `1` +* `polling_frequency`: how often (in seconds) to poll the server for new + builds. Note that the agent might poll more frequently when it's actively + processing builds. +* `image_rebuild_frequency`: Vieter periodically builds images that are then + used as a basis for running build containers. This is to prevent each build + from downloading an entire repository worth of dependencies. This setting + defines how frequently (in minutes) to rebuild these images. + * Default: `1440` (every 24 hours) +* `arch`: architecture for which this agent should pull down builds (e.g. + `x86_64`) diff --git a/docs/content/installation.md b/docs/content/installation.md index 87b9cba..21eda64 100644 --- a/docs/content/installation.md +++ b/docs/content/installation.md @@ -21,7 +21,7 @@ branch. This branch will be the most up to date, but does not give any guarantees about stability, so beware! Thanks to the single-binary design of Vieter, this image can be used both for -the repository server & the cron daemon. +the repository server, the cron daemon and the agent. Below is an example compose file to set up both the repository server & the cron daemon: @@ -76,7 +76,7 @@ architectures will build on both. ## Binary On the -[releases](https://git.rustybever.be/vieter/vieter/releases) +[releases](https://git.rustybever.be/vieter-v/vieter/releases) page, you can find statically compiled binaries for all released versions. This is the same binary as used inside the Docker images. @@ -106,5 +106,5 @@ guarantee that a compiler update won't temporarily break them. ## Building from source -The project [README](https://git.rustybever.be/vieter/vieter#building) contains -instructions for building Vieter from source. +The project [README](https://git.rustybever.be/vieter-v/vieter#building) +contains instructions for building Vieter from source. diff --git a/docs/content/usage/builds/schedule.md b/docs/content/usage/builds/schedule.md index 38f76a4..de59e25 100644 --- a/docs/content/usage/builds/schedule.md +++ b/docs/content/usage/builds/schedule.md @@ -37,6 +37,6 @@ Each section can consist of as many of these parts as necessary. ## CLI tool The Vieter binary contains a command that shows you the next matching times for -a given expression. This can be useful to understand the syntax. For more +a given expression. This can be useful for understanding the syntax. For more information, see [vieter-schedule(1)](https://rustybever.be/man/vieter/vieter-schedule.1.html). diff --git a/src/agent/agent.v b/src/agent/agent.v new file mode 100644 index 0000000..69b9947 --- /dev/null +++ b/src/agent/agent.v @@ -0,0 +1,27 @@ +module agent + +import log +import os +import util + +const log_file_name = 'vieter.agent.log' + +// agent starts an agent service +pub fn agent(conf Config) ! { + log_level := log.level_from_tag(conf.log_level) or { + return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + } + + mut logger := log.Log{ + level: log_level + } + + os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') } + + log_file := os.join_path_single(conf.data_dir, agent.log_file_name) + logger.set_full_logpath(log_file) + logger.log_to_console_too() + + mut d := agent_init(logger, conf) + d.run() +} diff --git a/src/agent/cli.v b/src/agent/cli.v new file mode 100644 index 0000000..1535e17 --- /dev/null +++ b/src/agent/cli.v @@ -0,0 +1,31 @@ +module agent + +import cli +import conf as vconf + +struct Config { +pub: + log_level string = 'WARN' + // Architecture that the agent represents + arch string + api_key string + address string + data_dir string + max_concurrent_builds int = 1 + polling_frequency int = 30 + image_rebuild_frequency int = 1440 +} + +// cmd returns the cli module that handles the cron daemon. +pub fn cmd() cli.Command { + return cli.Command{ + name: 'agent' + description: 'Start an agent daemon.' + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + + agent(conf)! + } + } +} diff --git a/src/agent/daemon.v b/src/agent/daemon.v new file mode 100644 index 0000000..0647733 --- /dev/null +++ b/src/agent/daemon.v @@ -0,0 +1,178 @@ +module agent + +import log +import sync.stdatomic +import build { BuildConfig } +import client +import time +import os + +const ( + build_empty = 0 + build_running = 1 + build_done = 2 +) + +struct AgentDaemon { + logger shared log.Log + conf Config + client client.Client +mut: + images ImageManager + // Which builds are currently running; length is conf.max_concurrent_builds + builds []BuildConfig + // Atomic variables used to detect when a build has finished; length is + // conf.max_concurrent_builds + atomics []u64 +} + +// agent_init initializes a new agent +fn agent_init(logger log.Log, conf Config) AgentDaemon { + mut d := AgentDaemon{ + logger: logger + client: client.new(conf.address, conf.api_key) + conf: conf + images: new_image_manager(conf.image_rebuild_frequency * 60) + builds: []BuildConfig{len: conf.max_concurrent_builds} + atomics: []u64{len: conf.max_concurrent_builds} + } + + return d +} + +// run starts the actual agent daemon. This function will run forever. +pub fn (mut d AgentDaemon) run() { + // This is just so that the very first time the loop is ran, the jobs are + // always polled + mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency) + mut sleep_time := 1 * time.second + mut finished, mut empty := 0, 0 + + for { + finished, empty = d.update_atomics() + + // No new finished builds and no free slots, so there's nothing to be + // done + if finished + empty == 0 { + time.sleep(1 * time.second) + continue + } + + // Builds have finished, so old builder images might have freed up. + // TODO this might query the docker daemon too frequently. + if finished > 0 { + d.images.clean_old_images() + } + + // The agent will always poll for new jobs after at most + // `polling_frequency` seconds. However, when jobs have finished, the + // agent will also poll for new jobs. This is because jobs are often + // clustered together (especially when mostly using the global cron + // schedule), so there's a much higher chance jobs are available. + if finished > 0 || time.now() >= last_poll_time.add_seconds(d.conf.polling_frequency) { + new_configs := d.client.poll_jobs(d.conf.arch, finished + empty) or { + d.lerror('Failed to poll jobs: $err.msg()') + + // TODO pick a better delay here + time.sleep(5 * time.second) + continue + } + last_poll_time = time.now() + + for config in new_configs { + // TODO handle this better than to just skip the config + // Make sure a recent build base image is available for + // building the config + d.images.refresh_image(config.base_image) or { + d.lerror(err.msg()) + continue + } + d.start_build(config) + } + + // No new jobs were scheduled and the agent isn't doing anything, + // so we just wait until the next polling period. + if new_configs.len == 0 && finished + empty == d.conf.max_concurrent_builds { + sleep_time = time.now() - last_poll_time + } + } + // The agent is not doing anything, so we just wait until the next poll + // time + else if finished + empty == d.conf.max_concurrent_builds { + sleep_time = time.now() - last_poll_time + } + + time.sleep(sleep_time) + } +} + +// update_atomics checks for each build whether it's completed, and sets it to +// empty again if so. The return value is a tuple `(finished, empty)` where +// `finished` is how many builds were just finished and thus set to empty, and +// `empty` is how many build slots were already empty. The amount of running +// builds can then be calculated by substracting these two values from the +// total allowed concurrent builds. +fn (mut d AgentDaemon) update_atomics() (int, int) { + mut finished := 0 + mut empty := 0 + + for i in 0 .. d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == agent.build_done { + stdatomic.store_u64(&d.atomics[i], agent.build_empty) + finished++ + } else if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { + empty++ + } + } + + return finished, empty +} + +// start_build starts a build for the given BuildConfig. +fn (mut d AgentDaemon) start_build(config BuildConfig) bool { + for i in 0 .. d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { + stdatomic.store_u64(&d.atomics[i], agent.build_running) + d.builds[i] = config + + go d.run_build(i, config) + + return true + } + } + + return false +} + +// run_build actually starts the build process for a given target. +fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) { + d.linfo('started build: $config') + + // 0 means success, 1 means failure + mut status := 0 + + new_config := BuildConfig{ + ...config + base_image: d.images.get(config.base_image) + } + + res := build.build_config(d.client.address, d.client.api_key, new_config) or { + d.ldebug('build_config error: $err.msg()') + status = 1 + + build.BuildResult{} + } + + if status == 0 { + d.linfo('Uploading build logs for $config') + + // TODO use the arch value here + build_arch := os.uname().machine + d.client.add_build_log(config.target_id, res.start_time, res.end_time, build_arch, + res.exit_code, res.logs) or { d.lerror('Failed to upload logs for $config') } + } else { + d.lwarn('an error occurred during build: $config') + } + + stdatomic.store_u64(&d.atomics[build_index], agent.build_done) +} diff --git a/src/agent/images.v b/src/agent/images.v new file mode 100644 index 0000000..185192e --- /dev/null +++ b/src/agent/images.v @@ -0,0 +1,79 @@ +module agent + +import time +import docker +import build + +// An ImageManager is a utility that creates builder images from given base +// images, updating these builder images if they've become too old. This +// structure can manage images from any number of base images, paving the way +// for configurable base images per target/repository. +struct ImageManager { + max_image_age int [required] +mut: + // For each base image, one or more builder images can exist at the same + // time + images map[string][]string [required] + // For each base image, we track when its newest image was built + timestamps map[string]time.Time [required] +} + +// new_image_manager initializes a new image manager. +fn new_image_manager(max_image_age int) ImageManager { + return ImageManager{ + max_image_age: max_image_age + images: map[string][]string{} + timestamps: map[string]time.Time{} + } +} + +// get returns the name of the newest image for the given base image. Note that +// this function should only be called *after* a first call to `refresh_image`. +pub fn (m &ImageManager) get(base_image string) string { + return m.images[base_image].last() +} + +// refresh_image builds a new builder image from the given base image if the +// previous builder image is too old or non-existent. This function will do +// nothing if these conditions aren't met, so it's safe to call it every time +// you want to ensure an image is up to date. +fn (mut m ImageManager) refresh_image(base_image string) ! { + if base_image in m.timestamps + && m.timestamps[base_image].add_seconds(m.max_image_age) > time.now() { + return + } + + // TODO use better image tags for built images + new_image := build.create_build_image(base_image) or { + return error('Failed to build builder image from base image $base_image') + } + + m.images[base_image] << new_image + m.timestamps[base_image] = time.now() +} + +// clean_old_images removes all older builder images that are no longer in use. +// The function will always leave at least one builder image, namely the newest +// one. +fn (mut m ImageManager) clean_old_images() { + mut dd := docker.new_conn() or { return } + + defer { + dd.close() or {} + } + + mut i := 0 + + for image in m.images.keys() { + i = 0 + + for i < m.images[image].len - 1 { + // For each builder image, we try to remove it by calling the Docker + // API. If the function returns an error or false, that means the image + // wasn't deleted. Therefore, we move the index over. If the function + // returns true, the array's length has decreased by one so we don't + // move the index. + dd.remove_image(m.images[image][i]) or { i += 1 } + } + } +} diff --git a/src/agent/log.v b/src/agent/log.v new file mode 100644 index 0000000..cd59207 --- /dev/null +++ b/src/agent/log.v @@ -0,0 +1,35 @@ +module agent + +import log + +// log a message with the given level +pub fn (mut d AgentDaemon) log(msg string, level log.Level) { + lock d.logger { + d.logger.send_output(msg, level) + } +} + +// lfatal create a log message with the fatal level +pub fn (mut d AgentDaemon) lfatal(msg string) { + d.log(msg, log.Level.fatal) +} + +// lerror create a log message with the error level +pub fn (mut d AgentDaemon) lerror(msg string) { + d.log(msg, log.Level.error) +} + +// lwarn create a log message with the warn level +pub fn (mut d AgentDaemon) lwarn(msg string) { + d.log(msg, log.Level.warn) +} + +// linfo create a log message with the info level +pub fn (mut d AgentDaemon) linfo(msg string) { + d.log(msg, log.Level.info) +} + +// ldebug create a log message with the debug level +pub fn (mut d AgentDaemon) ldebug(msg string) { + d.log(msg, log.Level.debug) +} diff --git a/src/build/build.v b/src/build/build.v index 247df6e..3d916bf 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -16,6 +16,22 @@ const ( '/usr/local/bin', '/usr/bin/site_perl', '/usr/bin/vendor_perl', '/usr/bin/core_perl'] ) +pub struct BuildConfig { +pub: + target_id int + kind string + url string + branch string + repo string + base_image string + force bool +} + +// str return a single-line string representation of a build log +pub fn (c BuildConfig) str() string { + return '{ target: $c.target_id, kind: $c.kind, url: $c.url, branch: $c.branch, repo: $c.repo, base_image: $c.base_image, force: $c.force }' +} + // create_build_image creates a builder image given some base image which can // then be used to build & package Arch images. It mostly just updates the // system, install some necessary packages & creates a non-root user to run @@ -93,10 +109,25 @@ pub: logs string } -// build_target builds, packages & publishes a given Arch package based on the +// build_target builds the given target. Internally it calls `build_config`. +pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool) !BuildResult { + config := BuildConfig{ + target_id: target.id + kind: target.kind + url: target.url + branch: target.branch + repo: target.repo + base_image: base_image_id + force: force + } + + return build_config(address, api_key, config) +} + +// build_config builds, packages & publishes a given Arch package based on the // provided target. The base image ID should be of an image previously created // by create_build_image. It returns the logs of the container. -pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult { +pub fn build_config(address string, api_key string, config BuildConfig) !BuildResult { mut dd := docker.new_conn()! defer { @@ -104,14 +135,14 @@ pub fn build_target(address string, api_key string, base_image_id string, target } build_arch := os.uname().machine - build_script := create_build_script(address, target, build_arch) + build_script := create_build_script(address, config, build_arch) // We convert the build script into a base64 string, which then gets passed // to the container as an env var base64_script := base64.encode_str(build_script) c := docker.NewContainer{ - image: '$base_image_id' + image: '$config.base_image' env: [ 'BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key', diff --git a/src/build/queue.v b/src/build/queue.v new file mode 100644 index 0000000..7902173 --- /dev/null +++ b/src/build/queue.v @@ -0,0 +1,229 @@ +module build + +import models { Target } +import cron.expression { CronExpression, parse_expression } +import time +import datatypes { MinHeap } +import util + +struct BuildJob { +pub mut: + // Time at which this build job was created/queued + created time.Time + // Next timestamp from which point this job is allowed to be executed + timestamp time.Time + // Required for calculating next timestamp after having pop'ed a job + ce CronExpression + // Actual build config sent to the agent + config BuildConfig + // Whether this is a one-time job + single bool +} + +// Allows BuildJob structs to be sorted according to their timestamp in +// MinHeaps +fn (r1 BuildJob) < (r2 BuildJob) bool { + return r1.timestamp < r2.timestamp +} + +// The build job queue is responsible for managing the list of scheduled builds +// for each architecture. Agents receive jobs from this queue. +pub struct BuildJobQueue { + // Schedule to use for targets without explicitely defined cron expression + default_schedule CronExpression + // Base image to use for targets without defined base image + default_base_image string +mut: + mutex shared util.Dummy + // For each architecture, a priority queue is tracked + queues map[string]MinHeap + // When a target is removed from the server or edited, its previous build + // configs will be invalid. This map allows for those to be simply skipped + // by ignoring any build configs created before this timestamp. + invalidated map[int]time.Time +} + +// new_job_queue initializes a new job queue +pub fn new_job_queue(default_schedule CronExpression, default_base_image string) BuildJobQueue { + return BuildJobQueue{ + default_schedule: default_schedule + default_base_image: default_base_image + invalidated: map[int]time.Time{} + } +} + +// insert_all executes insert for each architecture of the given Target. +pub fn (mut q BuildJobQueue) insert_all(target Target) ! { + for arch in target.arch { + q.insert(target: target, arch: arch.value)! + } +} + +[params] +pub struct InsertConfig { + target Target [required] + arch string [required] + single bool + force bool + now bool +} + +// insert a new target's job into the queue for the given architecture. This +// job will then be endlessly rescheduled after being pop'ed, unless removed +// explicitely. +pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! { + lock q.mutex { + if input.arch !in q.queues { + q.queues[input.arch] = MinHeap{} + } + + mut job := BuildJob{ + created: time.now() + single: input.single + config: BuildConfig{ + target_id: input.target.id + kind: input.target.kind + url: input.target.url + branch: input.target.branch + repo: input.target.repo + // TODO make this configurable + base_image: q.default_base_image + force: input.force + } + } + + if !input.now { + ce := if input.target.schedule != '' { + parse_expression(input.target.schedule) or { + return error("Error while parsing cron expression '$input.target.schedule' (id $input.target.id): $err.msg()") + } + } else { + q.default_schedule + } + + job.timestamp = ce.next_from_now()! + job.ce = ce + } else { + job.timestamp = time.now() + } + + q.queues[input.arch].insert(job) + } +} + +// reschedule the given job by calculating the next timestamp and re-adding it +// to its respective queue. This function is called by the pop functions +// *after* having pop'ed the job. +fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) ! { + new_timestamp := job.ce.next_from_now()! + + new_job := BuildJob{ + ...job + created: time.now() + timestamp: new_timestamp + } + + q.queues[arch].insert(new_job) +} + +// pop_invalid pops all invalid jobs. +fn (mut q BuildJobQueue) pop_invalid(arch string) { + for { + job := q.queues[arch].peek() or { return } + + if job.config.target_id in q.invalidated + && job.created < q.invalidated[job.config.target_id] { + // This pop *should* never fail according to the source code + q.queues[arch].pop() or {} + } else { + break + } + } +} + +// peek shows the first job for the given architecture that's ready to be +// executed, if present. +pub fn (mut q BuildJobQueue) peek(arch string) ?BuildJob { + // Even peek requires a write lock, because pop_invalid can modify the data + // structure + lock q.mutex { + if arch !in q.queues { + return none + } + + q.pop_invalid(arch) + job := q.queues[arch].peek()? + + if job.timestamp < time.now() { + return job + } + } + + return none +} + +// pop removes the first job for the given architecture that's ready to be +// executed from the queue and returns it, if present. +pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob { + lock q.mutex { + if arch !in q.queues { + return none + } + + q.pop_invalid(arch) + mut job := q.queues[arch].peek()? + + if job.timestamp < time.now() { + job = q.queues[arch].pop()? + + if !job.single { + // TODO how do we handle this properly? Is it even possible for a + // cron expression to not return a next time if it's already been + // used before? + q.reschedule(job, arch) or {} + } + + return job + } + } + + return none +} + +// pop_n tries to pop at most n available jobs for the given architecture. +pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { + lock q.mutex { + if arch !in q.queues { + return [] + } + + mut out := []BuildJob{} + + for out.len < n { + q.pop_invalid(arch) + mut job := q.queues[arch].peek() or { break } + + if job.timestamp < time.now() { + job = q.queues[arch].pop() or { break } + + if !job.single { + // TODO idem + q.reschedule(job, arch) or {} + } + + out << job + } else { + break + } + } + + return out + } + + return [] +} + +// invalidate a target's old build jobs. +pub fn (mut q BuildJobQueue) invalidate(target_id int) { + q.invalidated[target_id] = time.now() +} diff --git a/src/build/shell.v b/src/build/shell.v index e573d53..ac61e07 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -1,7 +1,5 @@ module build -import models { Target } - // escape_shell_string escapes any characters that could be interpreted // incorrectly by a shell. The resulting value should be safe to use inside an // echo statement. @@ -23,13 +21,13 @@ pub fn echo_commands(cmds []string) []string { } // create_build_script generates a shell script that builds a given Target. -fn create_build_script(address string, target &Target, build_arch string) string { - repo_url := '$address/$target.repo' +fn create_build_script(address string, config BuildConfig, build_arch string) string { + repo_url := '$address/$config.repo' mut commands := [ // This will later be replaced by a proper setting for changing the // mirrorlist - "echo -e '[$target.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" + "echo -e '[$config.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" // We need to update the package list of the repo we just added above. // This should however not pull in a lot of packages as long as the // builder image is rebuilt frequently. @@ -38,22 +36,22 @@ fn create_build_script(address string, target &Target, build_arch string) string 'su builder', ] - commands << match target.kind { + commands << match config.kind { 'git' { - if target.branch == '' { + if config.branch == '' { [ - "git clone --single-branch --depth 1 '$target.url' repo", + "git clone --single-branch --depth 1 '$config.url' repo", ] } else { [ - "git clone --single-branch --depth 1 --branch $target.branch '$target.url' repo", + "git clone --single-branch --depth 1 --branch $config.branch '$config.url' repo", ] } } 'url' { [ 'mkdir repo', - "curl -o repo/PKGBUILD -L '$target.url'", + "curl -o repo/PKGBUILD -L '$config.url'", ] } else { @@ -65,14 +63,22 @@ fn create_build_script(address string, target &Target, build_arch string) string 'cd repo', 'makepkg --nobuild --syncdeps --needed --noconfirm', 'source PKGBUILD', + ] + + if !config.force { // The build container checks whether the package is already present on // the server. - 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', - // If the above curl command succeeds, we don't need to rebuild the - // package. However, because we're in a su shell, the exit command will - // drop us back into the root shell. Therefore, we must check whether - // we're in root so we don't proceed. - '[ "\$(id -u)" == 0 ] && exit 0', + commands << [ + 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + // If the above curl command succeeds, we don't need to rebuild the + // package. However, because we're in a su shell, the exit command will + // drop us back into the root shell. Therefore, we must check whether + // we're in root so we don't proceed. + '[ "\$(id -u)" == 0 ] && exit 0', + ] + } + + commands << [ 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', ] diff --git a/src/build/shell_test.v b/src/build/shell_test.v index 341df88..8bb22d9 100644 --- a/src/build/shell_test.v +++ b/src/build/shell_test.v @@ -1,42 +1,46 @@ module build -import models { Target } - fn test_create_build_script_git_branch() { - target := Target{ - id: 1 + config := BuildConfig{ + target_id: 1 kind: 'git' url: 'https://examplerepo.com' branch: 'main' repo: 'vieter' + base_image: 'not-used:latest' } - build_script := create_build_script('https://example.com', target, 'x86_64') + + build_script := create_build_script('https://example.com', config, 'x86_64') expected := $embed_file('build_script_git_branch.sh') assert build_script == expected.to_string().trim_space() } fn test_create_build_script_git() { - target := Target{ - id: 1 + config := BuildConfig{ + target_id: 1 kind: 'git' url: 'https://examplerepo.com' repo: 'vieter' + base_image: 'not-used:latest' } - build_script := create_build_script('https://example.com', target, 'x86_64') + + build_script := create_build_script('https://example.com', config, 'x86_64') expected := $embed_file('build_script_git.sh') assert build_script == expected.to_string().trim_space() } fn test_create_build_script_url() { - target := Target{ - id: 1 + config := BuildConfig{ + target_id: 1 kind: 'url' url: 'https://examplerepo.com' repo: 'vieter' + base_image: 'not-used:latest' } - build_script := create_build_script('https://example.com', target, 'x86_64') + + build_script := create_build_script('https://example.com', config, 'x86_64') expected := $embed_file('build_script_url.sh') assert build_script == expected.to_string().trim_space() diff --git a/src/client/jobs.v b/src/client/jobs.v new file mode 100644 index 0000000..440affa --- /dev/null +++ b/src/client/jobs.v @@ -0,0 +1,26 @@ +module client + +import build { BuildConfig } +import web.response { Response } + +// poll_jobs requests a list of new build jobs from the server. +pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig { + data := c.send_request<[]BuildConfig>(.get, '/api/v1/jobs/poll', { + 'arch': arch + 'max': max.str() + })! + + return data.data +} + +// queue_job adds a new one-time build job for the given target to the job +// queue. +pub fn (c &Client) queue_job(target_id int, arch string, force bool) !Response { + data := c.send_request(.post, '/api/v1/jobs/queue', { + 'target': target_id.str() + 'arch': arch + 'force': force.str() + })! + + return data +} diff --git a/src/console/targets/build.v b/src/console/targets/build.v index 9368558..e18077d 100644 --- a/src/console/targets/build.v +++ b/src/console/targets/build.v @@ -6,7 +6,7 @@ import os import build // build locally builds the target with the given id. -fn build(conf Config, target_id int) ! { +fn build(conf Config, target_id int, force bool) ! { c := client.new(conf.address, conf.api_key) target := c.get_target(target_id)! @@ -16,7 +16,7 @@ fn build(conf Config, target_id int) ! { image_id := build.create_build_image(conf.base_image)! println('Running build...') - res := build.build_target(conf.address, conf.api_key, image_id, target)! + res := build.build_target(conf.address, conf.api_key, image_id, target, force)! println('Removing build image...') diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 4179363..b527896 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -182,11 +182,44 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Build the target with the given id & publish it.' + flags: [ + cli.Flag{ + name: 'force' + description: 'Build the target without checking whether it needs to be renewed.' + flag: cli.FlagType.bool + }, + cli.Flag{ + name: 'remote' + description: 'Schedule the build on the server instead of running it locally.' + flag: cli.FlagType.bool + }, + cli.Flag{ + name: 'arch' + description: 'Architecture to schedule build for. Required when using -remote.' + flag: cli.FlagType.string + }, + ] execute: fn (cmd cli.Command) ! { config_file := cmd.flags.get_string('config-file')! conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - build(conf, cmd.args[0].int())! + remote := cmd.flags.get_bool('remote')! + force := cmd.flags.get_bool('force')! + target_id := cmd.args[0].int() + + if remote { + arch := cmd.flags.get_string('arch')! + + if arch == '' { + return error('When scheduling the build remotely, you have to specify an architecture.') + } + + c := client.new(conf.address, conf.api_key) + res := c.queue_job(target_id, arch, force)! + println(res.message) + } else { + build(conf, target_id, force)! + } } }, ] diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index beed9fc..42edc92 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -79,7 +79,7 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { mut status := 0 res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(), - &sb.target) or { + &sb.target, false) or { d.ldebug('build_target error: $err.msg()') status = 1 diff --git a/src/db/logs.v b/src/db/logs.v index 923dde2..2745467 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -84,6 +84,8 @@ pub fn (db &VieterDb) add_build_log(log BuildLog) int { insert log into BuildLog } + // Here, this does work because a log doesn't contain any foreign keys, + // meaning the ORM only has to do a single add inserted_id := db.conn.last_id() as int return inserted_id diff --git a/src/db/targets.v b/src/db/targets.v index a705ebb..41e56df 100644 --- a/src/db/targets.v +++ b/src/db/targets.v @@ -38,14 +38,17 @@ pub fn (db &VieterDb) get_target(target_id int) ?Target { } // add_target inserts the given target into the database. -pub fn (db &VieterDb) add_target(repo Target) int { +pub fn (db &VieterDb) add_target(target Target) int { sql db.conn { - insert repo into Target + insert target into Target } - inserted_id := db.conn.last_id() as int + // ID of inserted target is the largest id + inserted_target := sql db.conn { + select from Target order by id desc limit 1 + } - return inserted_id + return inserted_target.id } // delete_target deletes the target with the given id from the database. diff --git a/src/main.v b/src/main.v index fc09f7e..34387bf 100644 --- a/src/main.v +++ b/src/main.v @@ -9,6 +9,7 @@ import console.schedule import console.man import console.aur import cron +import agent fn main() { mut app := cli.Command{ @@ -40,6 +41,7 @@ fn main() { schedule.cmd(), man.cmd(), aur.cmd(), + agent.cmd(), ] } app.setup() diff --git a/src/server/api_jobs.v b/src/server/api_jobs.v new file mode 100644 index 0000000..7795351 --- /dev/null +++ b/src/server/api_jobs.v @@ -0,0 +1,49 @@ +module server + +import web +import web.response { new_data_response, new_response } + +// v1_poll_job_queue allows agents to poll for new build jobs. +['/api/v1/jobs/poll'; auth; get] +fn (mut app App) v1_poll_job_queue() web.Result { + arch := app.query['arch'] or { + return app.json(.bad_request, new_response('Missing arch query arg.')) + } + + max_str := app.query['max'] or { + return app.json(.bad_request, new_response('Missing max query arg.')) + } + max := max_str.int() + + mut out := app.job_queue.pop_n(arch, max).map(it.config) + + return app.json(.ok, new_data_response(out)) +} + +// v1_queue_job allows queueing a new one-time build job for the given target. +['/api/v1/jobs/queue'; auth; post] +fn (mut app App) v1_queue_job() web.Result { + target_id := app.query['target'] or { + return app.json(.bad_request, new_response('Missing target query arg.')) + }.int() + + arch := app.query['arch'] or { + return app.json(.bad_request, new_response('Missing arch query arg.')) + } + + if arch == '' { + app.json(.bad_request, new_response('Empty arch query arg.')) + } + + force := 'force' in app.query + + target := app.db.get_target(target_id) or { + return app.json(.bad_request, new_response('Unknown target id.')) + } + + app.job_queue.insert(target: target, arch: arch, single: true, now: true, force: force) or { + return app.status(.internal_server_error) + } + + return app.status(.ok) +} diff --git a/src/server/api_targets.v b/src/server/api_targets.v index 16db7e9..dc39d37 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -12,17 +12,17 @@ fn (mut app App) v1_get_targets() web.Result { filter := models.from_params(app.query) or { return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } - repos := app.db.get_targets(filter) + targets := app.db.get_targets(filter) - return app.json(.ok, new_data_response(repos)) + return app.json(.ok, new_data_response(targets)) } // v1_get_single_target returns the information for a single target. ['/api/v1/targets/:id'; auth; get] fn (mut app App) v1_get_single_target(id int) web.Result { - repo := app.db.get_target(id) or { return app.not_found() } + target := app.db.get_target(id) or { return app.not_found() } - return app.json(.ok, new_data_response(repo)) + return app.json(.ok, new_data_response(target)) } // v1_post_target creates a new target from the provided query string. @@ -30,22 +30,27 @@ fn (mut app App) v1_get_single_target(id int) web.Result { fn (mut app App) v1_post_target() web.Result { mut params := app.query.clone() - // If a repo is created without specifying the arch, we assume it's meant + // If a target is created without specifying the arch, we assume it's meant // for the default architecture. if 'arch' !in params || params['arch'] == '' { params['arch'] = app.conf.default_arch } - new_repo := models.from_params(params) or { + mut new_target := models.from_params(params) or { return app.json(http.Status.bad_request, new_response(err.msg())) } // Ensure someone doesn't submit an invalid kind - if new_repo.kind !in models.valid_kinds { + if new_target.kind !in models.valid_kinds { return app.json(http.Status.bad_request, new_response('Invalid kind.')) } - id := app.db.add_target(new_repo) + id := app.db.add_target(new_target) + new_target.id = id + + // Add the target to the job queue + // TODO return better error here if it's the cron schedule that's incorrect + app.job_queue.insert_all(new_target) or { return app.status(.internal_server_error) } return app.json(.ok, new_data_response(id)) } @@ -54,6 +59,7 @@ fn (mut app App) v1_post_target() web.Result { ['/api/v1/targets/:id'; auth; delete] fn (mut app App) v1_delete_target(id int) web.Result { app.db.delete_target(id) + app.job_queue.invalidate(id) return app.json(.ok, new_response('')) } @@ -69,7 +75,10 @@ fn (mut app App) v1_patch_target(id int) web.Result { app.db.update_target_archs(id, arch_objs) } - repo := app.db.get_target(id) or { return app.status(.internal_server_error) } + target := app.db.get_target(id) or { return app.status(.internal_server_error) } - return app.json(.ok, new_data_response(repo)) + app.job_queue.invalidate(id) + app.job_queue.insert_all(target) or { return app.status(.internal_server_error) } + + return app.json(.ok, new_data_response(target)) } diff --git a/src/server/cli.v b/src/server/cli.v index a9644f3..2fede6c 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -5,12 +5,14 @@ import conf as vconf struct Config { pub: - log_level string = 'WARN' - pkg_dir string - data_dir string - api_key string - default_arch string - port int = 8000 + log_level string = 'WARN' + pkg_dir string + data_dir string + api_key string + default_arch string + global_schedule string = '0 3' + port int = 8000 + base_image string = 'archlinux:base-devel' } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/server.v b/src/server/server.v index d5f6135..74b1f37 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -6,6 +6,8 @@ import log import repo import util import db +import build { BuildJobQueue } +import cron.expression const ( log_file_name = 'vieter.log' @@ -20,7 +22,26 @@ pub: conf Config [required; web_global] pub mut: repo repo.RepoGroupManager [required; web_global] - db db.VieterDb + // Keys are the various architectures for packages + job_queue BuildJobQueue [required; web_global] + db db.VieterDb +} + +// init_job_queue populates a fresh job queue with all the targets currently +// stored in the database. +fn (mut app App) init_job_queue() ! { + // Initialize build queues + mut targets := app.db.get_targets(limit: 25) + mut i := u64(0) + + for targets.len > 0 { + for target in targets { + app.job_queue.insert_all(target)! + } + + i += 25 + targets = app.db.get_targets(limit: 25, offset: i) + } } // server starts the web server & starts listening for requests @@ -30,6 +51,10 @@ pub fn server(conf Config) ! { util.exit_with_message(1, "'any' is not allowed as the value for default_arch.") } + global_ce := expression.parse_expression(conf.global_schedule) or { + util.exit_with_message(1, 'Invalid global cron expression: $err.msg()') + } + // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -71,11 +96,17 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Failed to initialize database: $err.msg()') } - web.run(&App{ + mut app := &App{ logger: logger api_key: conf.api_key conf: conf repo: repo db: db - }, conf.port) + job_queue: build.new_job_queue(global_ce, conf.base_image) + } + app.init_job_queue() or { + util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()') + } + + web.run(app, conf.port) } diff --git a/vieter.toml b/vieter.toml index d3922a4..74a7397 100644 --- a/vieter.toml +++ b/vieter.toml @@ -4,11 +4,11 @@ data_dir = "data" pkg_dir = "data/pkgs" log_level = "DEBUG" default_arch = "x86_64" +arch = "x86_64" address = "http://localhost:8000" -global_schedule = '* *' +# global_schedule = '* *' api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 -