From 6f23d690a7a0a78d7d9203850c96204832149df0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Dec 2022 22:09:57 +0100 Subject: [PATCH] feat(agent): partially wrote daemon code --- src/agent/cli.v | 2 ++ src/agent/daemon.v | 77 ++++++++++++++++++++++++++++++++++++++++++++-- src/agent/images.v | 4 +++ src/build/build.v | 19 ++++++++++-- src/build/shell.v | 16 +++++----- src/client/jobs.v | 11 +++++++ 6 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 src/client/jobs.v diff --git a/src/agent/cli.v b/src/agent/cli.v index 063d960..1badbab 100644 --- a/src/agent/cli.v +++ b/src/agent/cli.v @@ -6,6 +6,8 @@ import conf as vconf struct Config { pub: log_level string = 'WARN' + // Architecture that the agent represents + arch string api_key string address string data_dir string diff --git a/src/agent/daemon.v b/src/agent/daemon.v index 0508790..aabcb44 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -4,6 +4,8 @@ import log import sync.stdatomic import build { BuildConfig } import client +import time +import os const ( build_empty = 0 @@ -14,6 +16,7 @@ const ( struct AgentDaemon { logger shared log.Log conf Config +mut: images ImageManager // Which builds are currently running; length is same as // conf.max_concurrent_builds @@ -41,13 +44,33 @@ pub fn (mut d AgentDaemon) run() { for { free_builds := d.update_atomics() - if free_builds > 0 { + // All build slots are taken, so there's nothing to be done + if free_builds == 0 { + time.sleep(1 * time.second) + continue + } + + // Builds have finished, so old builder images might have freed up. + d.images.clean_old_images() + + // Poll for new jobs + new_configs := d.client.poll_jobs(free_builds) or { + d.lerror('Failed to poll jobs: $err.msg()') + + time.sleep(1 * time.second) + continue + } + + // Schedule new jobs + for config in new_configs { + d.start_build(config) } } } // update_atomics checks for each build whether it's completed, and sets it to -// free again if so. The return value is how many fields are now set to free. +// free again if so. The return value is how many build slots are currently +// free. fn (mut d AgentDaemon) update_atomics() int { mut count := 0 @@ -62,3 +85,53 @@ fn (mut d AgentDaemon) update_atomics() int { return count } + +// start_build starts a build for the given BuildConfig object. +fn (mut d AgentDaemon) start_build(config BuildConfig) bool { + for i in 0 .. d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { + stdatomic.store_u64(&d.atomics[i], agent.build_running) + d.builds[i] = config + + go d.run_build(i, config) + + return true + } + } + + return false +} + +// run_build actually starts the build process for a given target. +fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) { + d.linfo('started build: $config.url -> $config.repo') + + // 0 means success, 1 means failure + mut status := 0 + + new_config := BuildConfig{ + ...config + base_image: d.images.get(config.base_image) + } + + res := build.build_config(d.client.address, d.client.api_key, new_config) or { + d.ldebug('build_config error: $err.msg()') + status = 1 + + build.BuildResult{} + } + + if status == 0 { + d.linfo('finished build: $config.url -> $config.repo; uploading logs...') + + build_arch := os.uname().machine + d.client.add_build_log(config.target_id, res.start_time, res.end_time, build_arch, + res.exit_code, res.logs) or { + d.lerror('Failed to upload logs for build: $config.url -> $config.repo') + } + } else { + d.linfo('an error occured during build: $config.url -> $config.repo') + } + + stdatomic.store_u64(&d.atomics[build_index], agent.build_done) +} diff --git a/src/agent/images.v b/src/agent/images.v index aee2be0..78bf2d0 100644 --- a/src/agent/images.v +++ b/src/agent/images.v @@ -19,6 +19,10 @@ fn new_image_manager(refresh_frequency int) ImageManager { } } +pub fn (m &ImageManager) get(base_image string) string { + return m.images[base_image].last() +} + fn (mut m ImageManager) refresh_image(base_image string) ! { // No need to refresh the image if the previous one is still new enough if base_image in m.timestamps diff --git a/src/build/build.v b/src/build/build.v index 13d3e45..744ce9c 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -103,10 +103,23 @@ pub: logs string } +pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult { +config := BuildConfig{ + target_id: target.id + kind: target.kind + url: target.url + branch: target.branch + repo: target.repo + base_image: base_image_id + } + + return build_config(address, api_key, config) +} + // build_target builds, packages & publishes a given Arch package based on the // provided target. The base image ID should be of an image previously created // by create_build_image. It returns the logs of the container. -pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult { +pub fn build_config(address string, api_key string, config BuildConfig) !BuildResult { mut dd := docker.new_conn()! defer { @@ -114,14 +127,14 @@ pub fn build_target(address string, api_key string, base_image_id string, target } build_arch := os.uname().machine - build_script := create_build_script(address, target, build_arch) + build_script := create_build_script(address, config, build_arch) // We convert the build script into a base64 string, which then gets passed // to the container as an env var base64_script := base64.encode_str(build_script) c := docker.NewContainer{ - image: '$base_image_id' + image: '$config.base_image' env: [ 'BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key', diff --git a/src/build/shell.v b/src/build/shell.v index e573d53..42ec3c0 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -23,13 +23,13 @@ pub fn echo_commands(cmds []string) []string { } // create_build_script generates a shell script that builds a given Target. -fn create_build_script(address string, target &Target, build_arch string) string { - repo_url := '$address/$target.repo' +fn create_build_script(address string, config BuildConfig, build_arch string) string { + repo_url := '$address/$config.repo' mut commands := [ // This will later be replaced by a proper setting for changing the // mirrorlist - "echo -e '[$target.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" + "echo -e '[$config.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" // We need to update the package list of the repo we just added above. // This should however not pull in a lot of packages as long as the // builder image is rebuilt frequently. @@ -38,22 +38,22 @@ fn create_build_script(address string, target &Target, build_arch string) string 'su builder', ] - commands << match target.kind { + commands << match config.kind { 'git' { - if target.branch == '' { + if config.branch == '' { [ - "git clone --single-branch --depth 1 '$target.url' repo", + "git clone --single-branch --depth 1 '$config.url' repo", ] } else { [ - "git clone --single-branch --depth 1 --branch $target.branch '$target.url' repo", + "git clone --single-branch --depth 1 --branch $config.branch '$config.url' repo", ] } } 'url' { [ 'mkdir repo', - "curl -o repo/PKGBUILD -L '$target.url'", + "curl -o repo/PKGBUILD -L '$config.url'", ] } else { diff --git a/src/client/jobs.v b/src/client/jobs.v new file mode 100644 index 0000000..281d6ce --- /dev/null +++ b/src/client/jobs.v @@ -0,0 +1,11 @@ +module client + +import build { BuildConfig } + +pub fn (c &Client) poll_jobs(max int) ![]BuildConfig { + data := c.send_request<[]BuildConfig>(.get, '/api/v1/jobs/poll', { + 'max': max.str() + })! + + return data.data +}