From f6c5e7c2469f474193270cd09264ee1fb022499c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 19:59:18 +0100 Subject: [PATCH 1/3] feat: add option to force-build package --- src/build/build.v | 4 +++- src/build/shell.v | 20 ++++++++++++++------ src/console/targets/build.v | 4 ++-- src/console/targets/targets.v | 9 ++++++++- src/cron/daemon/build.v | 2 +- vieter.toml | 2 +- 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 84d288c..6da851a 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -24,6 +24,7 @@ pub: branch string repo string base_image string + force bool } // create_build_image creates a builder image given some base image which can @@ -104,7 +105,7 @@ pub: } // build_target builds the given target. Internally it calls `build_config`. -pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult { +pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool) !BuildResult { config := BuildConfig{ target_id: target.id kind: target.kind @@ -112,6 +113,7 @@ pub fn build_target(address string, api_key string, base_image_id string, target branch: target.branch repo: target.repo base_image: base_image_id + force: force } return build_config(address, api_key, config) diff --git a/src/build/shell.v b/src/build/shell.v index c2d0c9b..ac61e07 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -63,14 +63,22 @@ fn create_build_script(address string, config BuildConfig, build_arch string) st 'cd repo', 'makepkg --nobuild --syncdeps --needed --noconfirm', 'source PKGBUILD', + ] + + if !config.force { // The build container checks whether the package is already present on // the server. - 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', - // If the above curl command succeeds, we don't need to rebuild the - // package. However, because we're in a su shell, the exit command will - // drop us back into the root shell. Therefore, we must check whether - // we're in root so we don't proceed. - '[ "\$(id -u)" == 0 ] && exit 0', + commands << [ + 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + // If the above curl command succeeds, we don't need to rebuild the + // package. However, because we're in a su shell, the exit command will + // drop us back into the root shell. Therefore, we must check whether + // we're in root so we don't proceed. + '[ "\$(id -u)" == 0 ] && exit 0', + ] + } + + commands << [ 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', ] diff --git a/src/console/targets/build.v b/src/console/targets/build.v index 9368558..e18077d 100644 --- a/src/console/targets/build.v +++ b/src/console/targets/build.v @@ -6,7 +6,7 @@ import os import build // build locally builds the target with the given id. -fn build(conf Config, target_id int) ! { +fn build(conf Config, target_id int, force bool) ! { c := client.new(conf.address, conf.api_key) target := c.get_target(target_id)! @@ -16,7 +16,7 @@ fn build(conf Config, target_id int) ! { image_id := build.create_build_image(conf.base_image)! println('Running build...') - res := build.build_target(conf.address, conf.api_key, image_id, target)! + res := build.build_target(conf.address, conf.api_key, image_id, target, force)! println('Removing build image...') diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 4179363..ffcd36c 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -182,11 +182,18 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Build the target with the given id & publish it.' + flags: [ + cli.Flag{ + name: 'force' + description: 'Build the target without checking whether it needs to be renewed.' + flag: cli.FlagType.bool + }, + ] execute: fn (cmd cli.Command) ! { config_file := cmd.flags.get_string('config-file')! conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - build(conf, cmd.args[0].int())! + build(conf, cmd.args[0].int(), cmd.flags.get_bool('force')!)! } }, ] diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index beed9fc..42edc92 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -79,7 +79,7 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { mut status := 0 res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(), - &sb.target) or { + &sb.target, false) or { d.ldebug('build_target error: $err.msg()') status = 1 diff --git a/vieter.toml b/vieter.toml index 9a68ae3..74a7397 100644 --- a/vieter.toml +++ b/vieter.toml @@ -8,7 +8,7 @@ arch = "x86_64" address = "http://localhost:8000" -global_schedule = '* *' +# global_schedule = '* *' api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 From 6a208dbe6ca73b25e1e0e30f3b3b266620061ebc Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 21:22:22 +0100 Subject: [PATCH 2/3] feat: allow queueing one-time builds --- src/build/queue.v | 46 ++++++++++++++++++++++++++--------------- src/server/api_builds.v | 21 +++++++++++++++++++ src/server/server.v | 2 +- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/src/build/queue.v b/src/build/queue.v index dd2bb87..1395a0b 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -16,6 +16,8 @@ pub: ce CronExpression // Actual build config sent to the agent config BuildConfig + // Whether this is a one-time job + single bool } // Allows BuildJob structs to be sorted according to their timestamp in @@ -53,22 +55,29 @@ pub fn new_job_queue(default_schedule CronExpression, default_base_image string) // insert_all executes insert for each architecture of the given Target. pub fn (mut q BuildJobQueue) insert_all(target Target) ! { for arch in target.arch { - q.insert(target, arch.value)! + q.insert(target: target, arch: arch.value)! } } +[params] +pub struct InsertConfig { + target Target [required] + arch string [required] + single bool +} + // insert a new target's job into the queue for the given architecture. This // job will then be endlessly rescheduled after being pop'ed, unless removed // explicitely. -pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { +pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! { lock q.mutex { - if arch !in q.queues { - q.queues[arch] = MinHeap{} + if input.arch !in q.queues { + q.queues[input.arch] = MinHeap{} } - ce := if target.schedule != '' { - parse_expression(target.schedule) or { - return error("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()") + ce := if input.target.schedule != '' { + parse_expression(input.target.schedule) or { + return error("Error while parsing cron expression '$input.target.schedule' (id $input.target.id): $err.msg()") } } else { q.default_schedule @@ -80,18 +89,19 @@ pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { created: time.now() timestamp: timestamp ce: ce + single: input.single config: BuildConfig{ - target_id: target.id - kind: target.kind - url: target.url - branch: target.branch - repo: target.repo + target_id: input.target.id + kind: input.target.kind + url: input.target.url + branch: input.target.branch + repo: input.target.repo // TODO make this configurable base_image: q.default_base_image } } - q.queues[arch].insert(job) + q.queues[input.arch].insert(job) } } @@ -158,10 +168,12 @@ pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob { if job.timestamp < time.now() { job = q.queues[arch].pop()? - // TODO how do we handle this properly? Is it even possible for a - // cron expression to not return a next time if it's already been - // used before? - q.reschedule(job, arch) or {} + if !job.single { + // TODO how do we handle this properly? Is it even possible for a + // cron expression to not return a next time if it's already been + // used before? + q.reschedule(job, arch) or {} + } return job } diff --git a/src/server/api_builds.v b/src/server/api_builds.v index 922b252..bc841ce 100644 --- a/src/server/api_builds.v +++ b/src/server/api_builds.v @@ -19,3 +19,24 @@ fn (mut app App) v1_poll_job_queue() web.Result { return app.json(.ok, new_data_response(out)) } + +['/api/v1/jobs/queue'; auth; post] +fn (mut app App) v1_queue_job() web.Result { + target_id := app.query['target'] or { + return app.json(.bad_request, new_response('Missing target query arg.')) + }.int() + + arch := app.query['arch'] or { + return app.json(.bad_request, new_response('Missing arch query arg.')) + } + + target := app.db.get_target(target_id) or { + return app.json(.bad_request, new_response('Unknown target id.')) + } + + app.job_queue.insert(target: target, arch: arch, single: true) or { + return app.status(.internal_server_error) + } + + return app.status(.ok) +} diff --git a/src/server/server.v b/src/server/server.v index 1e86906..6d18f09 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -37,7 +37,7 @@ fn (mut app App) init_job_queue() ! { for targets.len > 0 { for target in targets { for arch in target.arch { - app.job_queue.insert(target, arch.value)! + app.job_queue.insert(target: target, arch: arch.value)! } } From b3e26e5ffe726144c8853f381dd076c7c0ba68e3 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 22:03:04 +0100 Subject: [PATCH 3/3] feat: queue one-time builds from CLI --- src/build/queue.v | 40 +++++++++++++++---------- src/client/jobs.v | 11 +++++++ src/console/targets/targets.v | 29 +++++++++++++++++- src/server/{api_builds.v => api_jobs.v} | 8 ++++- 4 files changed, 70 insertions(+), 18 deletions(-) rename src/server/{api_builds.v => api_jobs.v} (83%) diff --git a/src/build/queue.v b/src/build/queue.v index 1395a0b..5d50f34 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -7,7 +7,7 @@ import datatypes { MinHeap } import util struct BuildJob { -pub: +pub mut: // Time at which this build job was created/queued created time.Time // Next timestamp from which point this job is allowed to be executed @@ -64,6 +64,8 @@ pub struct InsertConfig { target Target [required] arch string [required] single bool + force bool + now bool } // insert a new target's job into the queue for the given architecture. This @@ -75,20 +77,8 @@ pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! { q.queues[input.arch] = MinHeap{} } - ce := if input.target.schedule != '' { - parse_expression(input.target.schedule) or { - return error("Error while parsing cron expression '$input.target.schedule' (id $input.target.id): $err.msg()") - } - } else { - q.default_schedule - } - - timestamp := ce.next_from_now()! - - job := BuildJob{ + mut job := BuildJob{ created: time.now() - timestamp: timestamp - ce: ce single: input.single config: BuildConfig{ target_id: input.target.id @@ -98,9 +88,25 @@ pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! { repo: input.target.repo // TODO make this configurable base_image: q.default_base_image + force: input.force } } + if !input.now { + ce := if input.target.schedule != '' { + parse_expression(input.target.schedule) or { + return error("Error while parsing cron expression '$input.target.schedule' (id $input.target.id): $err.msg()") + } + } else { + q.default_schedule + } + + job.timestamp = ce.next_from_now()! + job.ce = ce + } else { + job.timestamp = time.now() + } + q.queues[input.arch].insert(job) } } @@ -198,8 +204,10 @@ pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { if job.timestamp < time.now() { job = q.queues[arch].pop() or { break } - // TODO idem - q.reschedule(job, arch) or {} + if !job.single { + // TODO idem + q.reschedule(job, arch) or {} + } out << job } else { diff --git a/src/client/jobs.v b/src/client/jobs.v index 7fee94f..2d8e99b 100644 --- a/src/client/jobs.v +++ b/src/client/jobs.v @@ -1,6 +1,7 @@ module client import build { BuildConfig } +import web.response { Response } // poll_jobs requests a list of new build jobs from the server. pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig { @@ -11,3 +12,13 @@ pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig { return data.data } + +pub fn (c &Client) queue_job(target_id int, arch string, force bool) !Response { + data := c.send_request(.post, '/api/v1/jobs/queue', { + 'target': target_id.str() + 'arch': arch + 'force': force.str() + })! + + return data +} diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index ffcd36c..bf2051f 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -188,12 +188,39 @@ pub fn cmd() cli.Command { description: 'Build the target without checking whether it needs to be renewed.' flag: cli.FlagType.bool }, + cli.Flag{ + name: 'remote' + description: 'Schedule the build on the server instead of running it locally.' + flag: cli.FlagType.bool + }, + cli.Flag{ + name: 'arch' + description: 'Architecture to schedule build for. Required when using -remote.' + flag: cli.FlagType.string + }, ] execute: fn (cmd cli.Command) ! { config_file := cmd.flags.get_string('config-file')! conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - build(conf, cmd.args[0].int(), cmd.flags.get_bool('force')!)! + remote := cmd.flags.get_bool('remote')! + force := cmd.flags.get_bool('force')! + target_id := cmd.args[0].int() + + if remote { + arch := cmd.flags.get_string('arch')! + + if arch == '' { + println('When scheduling the build remotely, you have to specify an architecture.') + exit(1) + } + + c := client.new(conf.address, conf.api_key) + res := c.queue_job(target_id, arch, force)! + println(res.message) + } else { + build(conf, target_id, force)! + } } }, ] diff --git a/src/server/api_builds.v b/src/server/api_jobs.v similarity index 83% rename from src/server/api_builds.v rename to src/server/api_jobs.v index bc841ce..b75e70e 100644 --- a/src/server/api_builds.v +++ b/src/server/api_jobs.v @@ -30,11 +30,17 @@ fn (mut app App) v1_queue_job() web.Result { return app.json(.bad_request, new_response('Missing arch query arg.')) } + if arch == '' { + app.json(.bad_request, new_response('Empty arch query arg.')) + } + + force := 'force' in app.query + target := app.db.get_target(target_id) or { return app.json(.bad_request, new_response('Unknown target id.')) } - app.job_queue.insert(target: target, arch: arch, single: true) or { + app.job_queue.insert(target: target, arch: arch, single: true, now: true, force: force) or { return app.status(.internal_server_error) }