From f7ca005ad530fd0861f63d10636521ef0b434990 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 6 Dec 2022 14:11:17 +0100 Subject: [PATCH 1/9] WIP server queeu stuff --- src/agent/agent.v | 2 +- src/agent/cli.v | 16 ++++++++-------- src/agent/daemon.v | 10 ++++------ src/build/build.v | 10 +++++----- src/main.v | 2 +- src/server/api_builds.v | 31 +++++++++++++++++++++++++++++++ src/server/server.v | 21 +++++++++++++++++++-- 7 files changed, 69 insertions(+), 23 deletions(-) create mode 100644 src/server/api_builds.v diff --git a/src/agent/agent.v b/src/agent/agent.v index 3affd21..1758c85 100644 --- a/src/agent/agent.v +++ b/src/agent/agent.v @@ -20,6 +20,6 @@ pub fn agent(conf Config) ! { logger.set_full_logpath(log_file) logger.log_to_console_too() - mut d := agent.agent_init(logger, conf) + mut d := agent_init(logger, conf) d.run() } diff --git a/src/agent/cli.v b/src/agent/cli.v index 46942ec..a0a249c 100644 --- a/src/agent/cli.v +++ b/src/agent/cli.v @@ -5,15 +5,15 @@ import conf as vconf struct Config { pub: - log_level string = 'WARN' - api_key string - address string - data_dir string - max_concurrent_builds int = 1 - polling_frequency int = 30 + log_level string = 'WARN' + api_key string + address string + data_dir string + max_concurrent_builds int = 1 + polling_frequency int = 30 // Architecture of agent - /* arch string */ - /* image_rebuild_frequency int = 1440 */ + // arch string + // image_rebuild_frequency int = 1440 } // cmd returns the cli module that handles the cron daemon. diff --git a/src/agent/daemon.v b/src/agent/daemon.v index 389a148..fd5fe04 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -13,13 +13,13 @@ const ( struct AgentDaemon { logger shared log.Log - conf Config + conf Config // Which builds are currently running; length is same as // conf.max_concurrent_builds builds []BuildConfig // Atomic variables used to detect when a build has finished; length is the // same as conf.max_concurrent_builds - client client.Client + client client.Client atomics []u64 } @@ -39,10 +39,8 @@ pub fn (mut d AgentDaemon) run() { for { free_builds := d.update_atomics() - if free_builds > 0 { - - } - + if free_builds > 0 { + } } } diff --git a/src/build/build.v b/src/build/build.v index b7c5cb6..01f9a53 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -18,11 +18,11 @@ const ( pub struct BuildConfig { pub: - id int - kind string - url string - branch string - repo string + id int + kind string + url string + branch string + repo string base_image string } diff --git a/src/main.v b/src/main.v index 424e328..34387bf 100644 --- a/src/main.v +++ b/src/main.v @@ -41,7 +41,7 @@ fn main() { schedule.cmd(), man.cmd(), aur.cmd(), - agent.cmd() + agent.cmd(), ] } app.setup() diff --git a/src/server/api_builds.v b/src/server/api_builds.v new file mode 100644 index 0000000..1870025 --- /dev/null +++ b/src/server/api_builds.v @@ -0,0 +1,31 @@ +module server + +import web +import web.response { new_response } +// import db +import time +// import os +// import util +// import models { BuildLog, BuildLogFilter } + +['/api/v1/builds/poll'; auth; get] +fn (mut app App) v1_poll_build_queue() web.Result { + arch := app.query['arch'] or { + return app.json(.bad_request, new_response('Missing arch query arg.')) + } + + max_str := app.query['max'] or { + return app.json(.bad_request, new_response('Missing max query arg.')) + } + max := max_str.int() + + + mut out := [] + + now := time.now() + lock app.build_queue { + for app.build_queue.len() > 0 && out.len() < max { + + } + } +} diff --git a/src/server/server.v b/src/server/server.v index d5f6135..c1691f2 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -6,6 +6,9 @@ import log import repo import util import db +import datatypes { MinHeap } +import build { BuildConfig } +import time const ( log_file_name = 'vieter.log' @@ -14,13 +17,26 @@ const ( logs_dir_name = 'logs' ) +struct ScheduledBuild { +pub: + timestamp time.Time + target_id int + build_config BuildConfig +} + +// Overloaded operator for comparing ScheduledBuild objects +fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool { + return r1.timestamp < r2.timestamp +} + struct App { web.Context pub: conf Config [required; web_global] pub mut: - repo repo.RepoGroupManager [required; web_global] - db db.VieterDb + repo repo.RepoGroupManager [required; web_global] + build_queue shared MinHeap + db db.VieterDb } // server starts the web server & starts listening for requests @@ -77,5 +93,6 @@ pub fn server(conf Config) ! { conf: conf repo: repo db: db + build_queue: MinHeap{} }, conf.port) } From 722acb8ab3a5b8ff0395fcf97a96f56878827dfc Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 6 Dec 2022 21:15:57 +0100 Subject: [PATCH 2/9] more wip --- src/build/build.v | 1 + src/server/api_builds.v | 26 +++++++++++++++--------- src/server/cli.v | 1 + src/server/server.v | 45 ++++++++++++++++++++++++++++++++++------- 4 files changed, 57 insertions(+), 16 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 01f9a53..648eeb0 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -19,6 +19,7 @@ const ( pub struct BuildConfig { pub: id int + target_id int kind string url string branch string diff --git a/src/server/api_builds.v b/src/server/api_builds.v index 1870025..34b34dc 100644 --- a/src/server/api_builds.v +++ b/src/server/api_builds.v @@ -1,9 +1,9 @@ module server import web -import web.response { new_response } -// import db +import web.response { new_data_response, new_response } import time +import build { BuildConfig } // import os // import util // import models { BuildLog, BuildLogFilter } @@ -19,13 +19,21 @@ fn (mut app App) v1_poll_build_queue() web.Result { } max := max_str.int() - - mut out := [] + mut out := []BuildConfig{} now := time.now() - lock app.build_queue { - for app.build_queue.len() > 0 && out.len() < max { - - } - } + + lock app.build_queues { + mut queue := app.build_queues[arch] or { return app.json(.ok, new_data_response(out)) } + + for queue.len() > 0 && out.len < max { + next := queue.peek() or { return app.status(.internal_server_error) } + + if next.timestamp < now { + out << queue.pop() or { return app.status(.internal_server_error) }.config + } + } + } + + return app.json(.ok, new_data_response(out)) } diff --git a/src/server/cli.v b/src/server/cli.v index a9644f3..067725c 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -10,6 +10,7 @@ pub: data_dir string api_key string default_arch string + global_schedule string = '0 3' port int = 8000 } diff --git a/src/server/server.v b/src/server/server.v index c1691f2..8849c87 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -9,6 +9,7 @@ import db import datatypes { MinHeap } import build { BuildConfig } import time +import cron.expression const ( log_file_name = 'vieter.log' @@ -19,9 +20,8 @@ const ( struct ScheduledBuild { pub: - timestamp time.Time - target_id int - build_config BuildConfig + timestamp time.Time + config BuildConfig } // Overloaded operator for comparing ScheduledBuild objects @@ -34,9 +34,40 @@ struct App { pub: conf Config [required; web_global] pub mut: - repo repo.RepoGroupManager [required; web_global] - build_queue shared MinHeap - db db.VieterDb + repo repo.RepoGroupManager [required; web_global] + // Keys are the various architectures for packages + build_queues shared map[string]MinHeap + db db.VieterDb +} + +fn (mut app App) init_build_queues() { + // Initialize build queues + mut i := 0 + mut targets := app.db.get_targets(limit: 25) + + default_ce := expression.parse_expression(conf.global_schedule) or { + return + } + + for targets.len > 0 { + for t in targets { + ce := parse_expression(t.schedule) or { default_ce } + + for arch in t.arch { + if arch !in app.build_queues { + app.build_queues[arch] = Minheap{} + } + +build_config := BuildConfig{ + + } + app.build_queues[arch].push(ScheduledBuild{ + timestamp: ce.next() + config: build_config + }) + } + } + } } // server starts the web server & starts listening for requests @@ -93,6 +124,6 @@ pub fn server(conf Config) ! { conf: conf repo: repo db: db - build_queue: MinHeap{} + build_queues: map[string]MinHeap{} }, conf.port) } From 05980cc09c6e4275187d424ba641dc8385cf37b6 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 12 Dec 2022 19:03:31 +0100 Subject: [PATCH 3/9] feat(build): start of server-side job queue --- src/build/build.v | 1 - src/build/queue.v | 70 +++++++++++++++++++++++++++++++++++++++ src/server/api_builds.v | 58 ++++++++++++++++---------------- src/server/cli.v | 13 ++++---- src/server/server.v | 73 +++++++++++++++++------------------------ 5 files changed, 137 insertions(+), 78 deletions(-) create mode 100644 src/build/queue.v diff --git a/src/build/build.v b/src/build/build.v index 648eeb0..13d3e45 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -18,7 +18,6 @@ const ( pub struct BuildConfig { pub: - id int target_id int kind string url string diff --git a/src/build/queue.v b/src/build/queue.v new file mode 100644 index 0000000..81d3fa9 --- /dev/null +++ b/src/build/queue.v @@ -0,0 +1,70 @@ +module build + +import models { Target } +import cron.expression { CronExpression, parse_expression } +import time +import datatypes { MinHeap } + +struct BuildJob { +pub: + // Earliest point this + timestamp time.Time + config BuildConfig +} + +// Overloaded operator for comparing ScheduledBuild objects +fn (r1 BuildJob) < (r2 BuildJob) bool { + return r1.timestamp < r2.timestamp +} + +pub struct BuildJobQueue { + // Schedule to use for targets without explicitely defined cron expression + default_schedule CronExpression + // Base image to use for targets without defined base image + default_base_image string +mut: + // For each architecture, a priority queue is tracked + queues map[string]MinHeap + // Each queued build job is also stored in a map, with the keys being the + // target IDs. This is used when removing or editing targets. + // jobs map[int]BuildJob +} + +pub fn new_job_queue(default_schedule CronExpression, default_base_image string) BuildJobQueue { + return BuildJobQueue{ + default_schedule: default_schedule + default_base_image: default_base_image + } +} + +// insert a new job into the queue for a given target on an architecture. +pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { + if arch !in q.queues { + q.queues[arch] = MinHeap{} + } + + ce := if target.schedule != '' { + parse_expression(target.schedule) or { + return error("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()") + } + } else { + q.default_schedule + } + + timestamp := ce.next_from_now()! + + job := BuildJob{ + timestamp: timestamp + config: BuildConfig{ + target_id: target.id + kind: target.kind + url: target.url + branch: target.branch + repo: target.repo + // TODO make this configurable + base_image: q.default_base_image + } + } + + q.queues[arch].insert(job) +} diff --git a/src/server/api_builds.v b/src/server/api_builds.v index 34b34dc..888fe9d 100644 --- a/src/server/api_builds.v +++ b/src/server/api_builds.v @@ -1,39 +1,39 @@ module server -import web -import web.response { new_data_response, new_response } -import time -import build { BuildConfig } -// import os -// import util -// import models { BuildLog, BuildLogFilter } +/* import web */ +/* import web.response { new_data_response, new_response } */ +/* import time */ +/* import build { BuildConfig } */ +/* // import os */ +/* // import util */ +/* // import models { BuildLog, BuildLogFilter } */ -['/api/v1/builds/poll'; auth; get] -fn (mut app App) v1_poll_build_queue() web.Result { - arch := app.query['arch'] or { - return app.json(.bad_request, new_response('Missing arch query arg.')) - } +/* ['/api/v1/builds/poll'; auth; get] */ +/* fn (mut app App) v1_poll_build_queue() web.Result { */ +/* arch := app.query['arch'] or { */ +/* return app.json(.bad_request, new_response('Missing arch query arg.')) */ +/* } */ - max_str := app.query['max'] or { - return app.json(.bad_request, new_response('Missing max query arg.')) - } - max := max_str.int() +/* max_str := app.query['max'] or { */ +/* return app.json(.bad_request, new_response('Missing max query arg.')) */ +/* } */ +/* max := max_str.int() */ - mut out := []BuildConfig{} +/* mut out := []BuildConfig{} */ - now := time.now() +/* now := time.now() */ - lock app.build_queues { - mut queue := app.build_queues[arch] or { return app.json(.ok, new_data_response(out)) } +/* lock app.build_queues { */ +/* mut queue := app.build_queues[arch] or { return app.json(.ok, new_data_response(out)) } */ - for queue.len() > 0 && out.len < max { - next := queue.peek() or { return app.status(.internal_server_error) } +/* for queue.len() > 0 && out.len < max { */ +/* next := queue.peek() or { return app.status(.internal_server_error) } */ - if next.timestamp < now { - out << queue.pop() or { return app.status(.internal_server_error) }.config - } - } - } +/* if next.timestamp < now { */ +/* out << queue.pop() or { return app.status(.internal_server_error) }.config */ +/* } */ +/* } */ +/* } */ - return app.json(.ok, new_data_response(out)) -} +/* return app.json(.ok, new_data_response(out)) */ +/* } */ diff --git a/src/server/cli.v b/src/server/cli.v index 067725c..2fede6c 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -5,13 +5,14 @@ import conf as vconf struct Config { pub: - log_level string = 'WARN' - pkg_dir string - data_dir string - api_key string - default_arch string + log_level string = 'WARN' + pkg_dir string + data_dir string + api_key string + default_arch string global_schedule string = '0 3' - port int = 8000 + port int = 8000 + base_image string = 'archlinux:base-devel' } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/server.v b/src/server/server.v index 8849c87..fb45e6d 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -6,9 +6,7 @@ import log import repo import util import db -import datatypes { MinHeap } -import build { BuildConfig } -import time +import build { BuildJobQueue } import cron.expression const ( @@ -18,17 +16,6 @@ const ( logs_dir_name = 'logs' ) -struct ScheduledBuild { -pub: - timestamp time.Time - config BuildConfig -} - -// Overloaded operator for comparing ScheduledBuild objects -fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool { - return r1.timestamp < r2.timestamp -} - struct App { web.Context pub: @@ -36,39 +23,35 @@ pub: pub mut: repo repo.RepoGroupManager [required; web_global] // Keys are the various architectures for packages - build_queues shared map[string]MinHeap + job_queue BuildJobQueue [required; web_global] db db.VieterDb } -fn (mut app App) init_build_queues() { - // Initialize build queues - mut i := 0 - mut targets := app.db.get_targets(limit: 25) +// fn (mut app App) init_build_queues() { +// // Initialize build queues +// mut i := 0 +// mut targets := app.db.get_targets(limit: 25) - default_ce := expression.parse_expression(conf.global_schedule) or { - return - } +// default_ce := expression.parse_expression(conf.global_schedule) or { return } - for targets.len > 0 { - for t in targets { - ce := parse_expression(t.schedule) or { default_ce } +// for targets.len > 0 { +// for t in targets { +// ce := parse_expression(t.schedule) or { default_ce } - for arch in t.arch { - if arch !in app.build_queues { - app.build_queues[arch] = Minheap{} - } - -build_config := BuildConfig{ - - } - app.build_queues[arch].push(ScheduledBuild{ - timestamp: ce.next() - config: build_config - }) - } - } - } -} +// for arch in t.arch { +// if arch !in app.build_queues { +// app.build_queues[arch] = Minheap{} +// } + +// build_config := BuildConfig{} +// app.build_queues[arch].push(ScheduledBuild{ +// timestamp: ce.next() +// config: build_config +// }) +// } +// } +// } +//} // server starts the web server & starts listening for requests pub fn server(conf Config) ! { @@ -77,6 +60,10 @@ pub fn server(conf Config) ! { util.exit_with_message(1, "'any' is not allowed as the value for default_arch.") } + global_ce := expression.parse_expression(conf.global_schedule) or { + util.exit_with_message(1, 'Invalid global cron expression: $err.msg()') + } + // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -118,12 +105,14 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Failed to initialize database: $err.msg()') } + mut queue := build.new_job_queue(global_ce, conf.base_image) + web.run(&App{ logger: logger api_key: conf.api_key conf: conf repo: repo db: db - build_queues: map[string]MinHeap{} + job_queue: queue }, conf.port) } From 27775074345ed3b642fedf78d2ffe1c82ac22cd4 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Dec 2022 20:33:51 +0100 Subject: [PATCH 4/9] feat(server): initialize job queue on start; api endpoint for polling jobs --- src/build/queue.v | 113 +++++++++++++++++++++++++++++++--------- src/server/api_builds.v | 50 ++++++------------ src/server/server.v | 52 ++++++++---------- 3 files changed, 129 insertions(+), 86 deletions(-) diff --git a/src/build/queue.v b/src/build/queue.v index 81d3fa9..65b279e 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -4,6 +4,7 @@ import models { Target } import cron.expression { CronExpression, parse_expression } import time import datatypes { MinHeap } +import util struct BuildJob { pub: @@ -23,6 +24,7 @@ pub struct BuildJobQueue { // Base image to use for targets without defined base image default_base_image string mut: + mutex shared util.Dummy // For each architecture, a priority queue is tracked queues map[string]MinHeap // Each queued build job is also stored in a map, with the keys being the @@ -39,32 +41,95 @@ pub fn new_job_queue(default_schedule CronExpression, default_base_image string) // insert a new job into the queue for a given target on an architecture. pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { - if arch !in q.queues { - q.queues[arch] = MinHeap{} - } - - ce := if target.schedule != '' { - parse_expression(target.schedule) or { - return error("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()") + lock q.mutex { + if arch !in q.queues { + q.queues[arch] = MinHeap{} } - } else { - q.default_schedule - } - timestamp := ce.next_from_now()! - - job := BuildJob{ - timestamp: timestamp - config: BuildConfig{ - target_id: target.id - kind: target.kind - url: target.url - branch: target.branch - repo: target.repo - // TODO make this configurable - base_image: q.default_base_image + ce := if target.schedule != '' { + parse_expression(target.schedule) or { + return error("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()") + } + } else { + q.default_schedule } - } - q.queues[arch].insert(job) + timestamp := ce.next_from_now()! + + job := BuildJob{ + timestamp: timestamp + config: BuildConfig{ + target_id: target.id + kind: target.kind + url: target.url + branch: target.branch + repo: target.repo + // TODO make this configurable + base_image: q.default_base_image + } + } + + q.queues[arch].insert(job) + } +} + +// peek shows the first job for the given architecture that's ready to be +// executed, if present. +pub fn (q &BuildJobQueue) peek(arch string) ?BuildJob { + rlock q.mutex { + if arch !in q.queues { + return none + } + + job := q.queues[arch].peek() or { return none } + + if job.timestamp < time.now() { + return job + } + } + + return none +} + +// pop removes the first job for the given architecture that's ready to be +// executed from the queue and returns it, if present. +pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob { + lock q.mutex { + if arch !in q.queues { + return none + } + + job := q.queues[arch].peek() or { return none } + + if job.timestamp < time.now() { + return q.queues[arch].pop() + } + } + + return none +} + +// pop_n tries to pop at most n available jobs for the given architecture. +pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { + lock q.mutex { + if arch !in q.queues { + return [] + } + + mut out := []BuildJob{} + + for out.len < n { + job := q.queues[arch].peek() or { break } + + if job.timestamp < time.now() { + out << q.queues[arch].pop() or { break } + } else { + break + } + } + + return out + } + + return [] } diff --git a/src/server/api_builds.v b/src/server/api_builds.v index 888fe9d..62948cd 100644 --- a/src/server/api_builds.v +++ b/src/server/api_builds.v @@ -1,39 +1,23 @@ module server -/* import web */ -/* import web.response { new_data_response, new_response } */ -/* import time */ -/* import build { BuildConfig } */ -/* // import os */ -/* // import util */ -/* // import models { BuildLog, BuildLogFilter } */ +import web +import web.response { new_data_response, new_response } +// import os +// import util +// import models { BuildLog, BuildLogFilter } -/* ['/api/v1/builds/poll'; auth; get] */ -/* fn (mut app App) v1_poll_build_queue() web.Result { */ -/* arch := app.query['arch'] or { */ -/* return app.json(.bad_request, new_response('Missing arch query arg.')) */ -/* } */ +['/api/v1/builds/poll'; auth; get] +fn (mut app App) v1_poll_build_queue() web.Result { + arch := app.query['arch'] or { + return app.json(.bad_request, new_response('Missing arch query arg.')) + } -/* max_str := app.query['max'] or { */ -/* return app.json(.bad_request, new_response('Missing max query arg.')) */ -/* } */ -/* max := max_str.int() */ + max_str := app.query['max'] or { + return app.json(.bad_request, new_response('Missing max query arg.')) + } + max := max_str.int() -/* mut out := []BuildConfig{} */ + mut out := app.job_queue.pop_n(arch, max) -/* now := time.now() */ - -/* lock app.build_queues { */ -/* mut queue := app.build_queues[arch] or { return app.json(.ok, new_data_response(out)) } */ - -/* for queue.len() > 0 && out.len < max { */ -/* next := queue.peek() or { return app.status(.internal_server_error) } */ - -/* if next.timestamp < now { */ -/* out << queue.pop() or { return app.status(.internal_server_error) }.config */ -/* } */ -/* } */ -/* } */ - -/* return app.json(.ok, new_data_response(out)) */ -/* } */ + return app.json(.ok, new_data_response(out)) +} diff --git a/src/server/server.v b/src/server/server.v index fb45e6d..e2c19c2 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -24,34 +24,25 @@ pub mut: repo repo.RepoGroupManager [required; web_global] // Keys are the various architectures for packages job_queue BuildJobQueue [required; web_global] - db db.VieterDb + db db.VieterDb } -// fn (mut app App) init_build_queues() { -// // Initialize build queues -// mut i := 0 -// mut targets := app.db.get_targets(limit: 25) +fn (mut app App) init_job_queue() ! { + // Initialize build queues + mut targets := app.db.get_targets(limit: 25) + mut i := u64(0) -// default_ce := expression.parse_expression(conf.global_schedule) or { return } + for targets.len > 0 { + for target in targets { + for arch in target.arch { + app.job_queue.insert(target, arch.value)! + } + } -// for targets.len > 0 { -// for t in targets { -// ce := parse_expression(t.schedule) or { default_ce } - -// for arch in t.arch { -// if arch !in app.build_queues { -// app.build_queues[arch] = Minheap{} -// } - -// build_config := BuildConfig{} -// app.build_queues[arch].push(ScheduledBuild{ -// timestamp: ce.next() -// config: build_config -// }) -// } -// } -// } -//} + i += 25 + targets = app.db.get_targets(limit: 25, offset: i) + } +} // server starts the web server & starts listening for requests pub fn server(conf Config) ! { @@ -105,14 +96,17 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Failed to initialize database: $err.msg()') } - mut queue := build.new_job_queue(global_ce, conf.base_image) - - web.run(&App{ + mut app := &App{ logger: logger api_key: conf.api_key conf: conf repo: repo db: db - job_queue: queue - }, conf.port) + job_queue: build.new_job_queue(global_ce, conf.base_image) + } + app.init_job_queue() or { + util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()') + } + + web.run(app, conf.port) } From 4f1be62b1b1b2cb64004554cba9998878ca77078 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Dec 2022 20:59:43 +0100 Subject: [PATCH 5/9] feat(server): properly reschedule jobs after polling --- src/build/queue.v | 50 ++++++++++++++++++++++++++++++++++------- src/server/api_builds.v | 6 ++--- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/build/queue.v b/src/build/queue.v index 65b279e..b704926 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -8,12 +8,16 @@ import util struct BuildJob { pub: - // Earliest point this + // Next timestamp from which point this job is allowed to be executed timestamp time.Time - config BuildConfig + // Required for calculating next timestamp after having pop'ed a job + ce CronExpression + // Actual build config sent to the agent + config BuildConfig } -// Overloaded operator for comparing ScheduledBuild objects +// Allows BuildJob structs to be sorted according to their timestamp in +// MinHeaps fn (r1 BuildJob) < (r2 BuildJob) bool { return r1.timestamp < r2.timestamp } @@ -39,7 +43,9 @@ pub fn new_job_queue(default_schedule CronExpression, default_base_image string) } } -// insert a new job into the queue for a given target on an architecture. +// insert a new target's job into the queue for the given architecture. This +// job will then be endlessly rescheduled after being pop'ed, unless removed +// explicitely. pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { lock q.mutex { if arch !in q.queues { @@ -58,6 +64,7 @@ pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { job := BuildJob{ timestamp: timestamp + ce: ce config: BuildConfig{ target_id: target.id kind: target.kind @@ -69,10 +76,25 @@ pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { } } + dump(job) q.queues[arch].insert(job) } } +// reschedule the given job by calculating the next timestamp and re-adding it +// to its respective queue. This function is called by the pop functions +// *after* having pop'ed the job. +fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) ! { + new_timestamp := job.ce.next_from_now()! + + new_job := BuildJob{ + ...job + timestamp: new_timestamp + } + + q.queues[arch].insert(new_job) +} + // peek shows the first job for the given architecture that's ready to be // executed, if present. pub fn (q &BuildJobQueue) peek(arch string) ?BuildJob { @@ -99,10 +121,17 @@ pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob { return none } - job := q.queues[arch].peek() or { return none } + mut job := q.queues[arch].peek() or { return none } if job.timestamp < time.now() { - return q.queues[arch].pop() + job = q.queues[arch].pop()? + + // TODO how do we handle this properly? Is it even possible for a + // cron expression to not return a next time if it's already been + // used before? + q.reschedule(job, arch) or {} + + return job } } @@ -119,10 +148,15 @@ pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { mut out := []BuildJob{} for out.len < n { - job := q.queues[arch].peek() or { break } + mut job := q.queues[arch].peek() or { break } if job.timestamp < time.now() { - out << q.queues[arch].pop() or { break } + job = q.queues[arch].pop() or { break } + + // TODO idem + q.reschedule(job, arch) or {} + + out << job } else { break } diff --git a/src/server/api_builds.v b/src/server/api_builds.v index 62948cd..ec3c8ec 100644 --- a/src/server/api_builds.v +++ b/src/server/api_builds.v @@ -6,8 +6,8 @@ import web.response { new_data_response, new_response } // import util // import models { BuildLog, BuildLogFilter } -['/api/v1/builds/poll'; auth; get] -fn (mut app App) v1_poll_build_queue() web.Result { +['/api/v1/jobs/poll'; auth; get] +fn (mut app App) v1_poll_job_queue() web.Result { arch := app.query['arch'] or { return app.json(.bad_request, new_response('Missing arch query arg.')) } @@ -17,7 +17,7 @@ fn (mut app App) v1_poll_build_queue() web.Result { } max := max_str.int() - mut out := app.job_queue.pop_n(arch, max) + mut out := app.job_queue.pop_n(arch, max).map(it.config) return app.json(.ok, new_data_response(out)) } From ca92ae2a0ee5eaafdefa2610b6f95a3e519a0173 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Dec 2022 21:21:58 +0100 Subject: [PATCH 6/9] feat(agent): begin reforming for new api --- src/agent/daemon.v | 7 ++++--- src/agent/images.v | 49 ++++++++++++++++++++++++++++++++++++++++++++++ src/agent/log.v | 35 +++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 src/agent/images.v create mode 100644 src/agent/log.v diff --git a/src/agent/daemon.v b/src/agent/daemon.v index fd5fe04..71f4780 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -14,6 +14,8 @@ const ( struct AgentDaemon { logger shared log.Log conf Config + // List of last built builder images + builder_images []string // Which builds are currently running; length is same as // conf.max_concurrent_builds builds []BuildConfig @@ -44,9 +46,8 @@ pub fn (mut d AgentDaemon) run() { } } -// clean_finished_builds checks for each build whether it's completed, and sets -// it to free again if so. The return value is how many fields are now set to -// free. +// update_atomics checks for each build whether it's completed, and sets it to +// free again if so. The return value is how many fields are now set to free. fn (mut d AgentDaemon) update_atomics() int { mut count := 0 diff --git a/src/agent/images.v b/src/agent/images.v new file mode 100644 index 0000000..454f85f --- /dev/null +++ b/src/agent/images.v @@ -0,0 +1,49 @@ +module agent + +import time +import docker + +struct ImageManager { + images map[string]string + timestamps map[string]time.Time +} + +// clean_old_base_images tries to remove any old but still present builder +// images. +fn (mut d AgentDaemon) clean_old_base_images() { + mut i := 0 + + mut dd := docker.new_conn() or { + d.lerror('Failed to connect to Docker socket.') + return + } + + defer { + dd.close() or {} + } + + for i < d.builder_images.len - 1 { + // For each builder image, we try to remove it by calling the Docker + // API. If the function returns an error or false, that means the image + // wasn't deleted. Therefore, we move the index over. If the function + // returns true, the array's length has decreased by one so we don't + // move the index. + dd.remove_image(d.builder_images[i]) or { i += 1 } + } +} + +// rebuild_base_image builds a builder image from the given base image. +/* fn (mut d AgentDaemon) build_base_image(base_image string) bool { */ +/* d.linfo('Rebuilding builder image....') */ + +/* d.builder_images << build.create_build_image(d.base_image) or { */ +/* d.lerror('Failed to rebuild base image. Retrying in ${daemon.rebuild_base_image_retry_timout}s...') */ +/* d.image_build_timestamp = time.now().add_seconds(daemon.rebuild_base_image_retry_timout) */ + +/* return false */ +/* } */ + +/* d.image_build_timestamp = time.now().add_seconds(60 * d.image_rebuild_frequency) */ + +/* return true */ +/* } */ diff --git a/src/agent/log.v b/src/agent/log.v new file mode 100644 index 0000000..d47df0f --- /dev/null +++ b/src/agent/log.v @@ -0,0 +1,35 @@ +module agent + +import log + +// log reate a log message with the given level +pub fn (mut d AgentDaemon) log(msg string, level log.Level) { + lock d.logger { + d.logger.send_output(msg, level) + } +} + +// lfatal create a log message with the fatal level +pub fn (mut d AgentDaemon) lfatal(msg string) { + d.log(msg, log.Level.fatal) +} + +// lerror create a log message with the error level +pub fn (mut d AgentDaemon) lerror(msg string) { + d.log(msg, log.Level.error) +} + +// lwarn create a log message with the warn level +pub fn (mut d AgentDaemon) lwarn(msg string) { + d.log(msg, log.Level.warn) +} + +// linfo create a log message with the info level +pub fn (mut d AgentDaemon) linfo(msg string) { + d.log(msg, log.Level.info) +} + +// ldebug create a log message with the debug level +pub fn (mut d AgentDaemon) ldebug(msg string) { + d.log(msg, log.Level.debug) +} From 18c94c8d0b0d1b6ad035522b36694bbeedbae770 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Dec 2022 21:50:34 +0100 Subject: [PATCH 7/9] feat(agent): wrote ImageManager --- src/agent/cli.v | 2 +- src/agent/daemon.v | 4 +-- src/agent/images.v | 73 ++++++++++++++++++++++++++-------------------- 3 files changed, 45 insertions(+), 34 deletions(-) diff --git a/src/agent/cli.v b/src/agent/cli.v index a0a249c..063d960 100644 --- a/src/agent/cli.v +++ b/src/agent/cli.v @@ -13,7 +13,7 @@ pub: polling_frequency int = 30 // Architecture of agent // arch string - // image_rebuild_frequency int = 1440 + image_rebuild_frequency int = 1440 } // cmd returns the cli module that handles the cron daemon. diff --git a/src/agent/daemon.v b/src/agent/daemon.v index 71f4780..0508790 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -14,8 +14,7 @@ const ( struct AgentDaemon { logger shared log.Log conf Config - // List of last built builder images - builder_images []string + images ImageManager // Which builds are currently running; length is same as // conf.max_concurrent_builds builds []BuildConfig @@ -30,6 +29,7 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon { logger: logger client: client.new(conf.address, conf.api_key) conf: conf + images: new_image_manager(conf.image_rebuild_frequency) builds: []BuildConfig{len: conf.max_concurrent_builds} atomics: []u64{len: conf.max_concurrent_builds} } diff --git a/src/agent/images.v b/src/agent/images.v index 454f85f..aee2be0 100644 --- a/src/agent/images.v +++ b/src/agent/images.v @@ -2,48 +2,59 @@ module agent import time import docker +import build struct ImageManager { - images map[string]string - timestamps map[string]time.Time +mut: + refresh_frequency int + images map[string][]string [required] + timestamps map[string]time.Time [required] } -// clean_old_base_images tries to remove any old but still present builder -// images. -fn (mut d AgentDaemon) clean_old_base_images() { - mut i := 0 +fn new_image_manager(refresh_frequency int) ImageManager { + return ImageManager{ + refresh_frequency: refresh_frequency + images: map[string][]string{} + timestamps: map[string]time.Time{} + } +} - mut dd := docker.new_conn() or { - d.lerror('Failed to connect to Docker socket.') +fn (mut m ImageManager) refresh_image(base_image string) ! { + // No need to refresh the image if the previous one is still new enough + if base_image in m.timestamps + && m.timestamps[base_image].add_seconds(m.refresh_frequency) > time.now() { return } + // TODO use better image tags for built images + new_image := build.create_build_image(base_image) or { + return error('Failed to build builder image from base image $base_image') + } + + m.images[base_image] << new_image + m.timestamps[base_image] = time.now() +} + +// clean_old_images tries to remove any old but still present builder images. +fn (mut m ImageManager) clean_old_images() { + mut dd := docker.new_conn() or { return } + defer { dd.close() or {} } - for i < d.builder_images.len - 1 { - // For each builder image, we try to remove it by calling the Docker - // API. If the function returns an error or false, that means the image - // wasn't deleted. Therefore, we move the index over. If the function - // returns true, the array's length has decreased by one so we don't - // move the index. - dd.remove_image(d.builder_images[i]) or { i += 1 } + mut i := 0 + + for image in m.images.keys() { + i = 0 + + for i < m.images[image].len - 1 { + // For each builder image, we try to remove it by calling the Docker + // API. If the function returns an error or false, that means the image + // wasn't deleted. Therefore, we move the index over. If the function + // returns true, the array's length has decreased by one so we don't + // move the index. + dd.remove_image(m.images[image][i]) or { i += 1 } + } } } - -// rebuild_base_image builds a builder image from the given base image. -/* fn (mut d AgentDaemon) build_base_image(base_image string) bool { */ -/* d.linfo('Rebuilding builder image....') */ - -/* d.builder_images << build.create_build_image(d.base_image) or { */ -/* d.lerror('Failed to rebuild base image. Retrying in ${daemon.rebuild_base_image_retry_timout}s...') */ -/* d.image_build_timestamp = time.now().add_seconds(daemon.rebuild_base_image_retry_timout) */ - -/* return false */ -/* } */ - -/* d.image_build_timestamp = time.now().add_seconds(60 * d.image_rebuild_frequency) */ - -/* return true */ -/* } */ From ddddff28f2f64a5aa4b545cc5ebc7cbebaf212af Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Dec 2022 22:09:57 +0100 Subject: [PATCH 8/9] feat(agent): partially wrote daemon code --- src/agent/cli.v | 2 ++ src/agent/daemon.v | 77 ++++++++++++++++++++++++++++++++++++++++++++-- src/agent/images.v | 4 +++ src/build/build.v | 19 ++++++++++-- src/build/shell.v | 16 +++++----- src/client/jobs.v | 11 +++++++ 6 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 src/client/jobs.v diff --git a/src/agent/cli.v b/src/agent/cli.v index 063d960..1badbab 100644 --- a/src/agent/cli.v +++ b/src/agent/cli.v @@ -6,6 +6,8 @@ import conf as vconf struct Config { pub: log_level string = 'WARN' + // Architecture that the agent represents + arch string api_key string address string data_dir string diff --git a/src/agent/daemon.v b/src/agent/daemon.v index 0508790..aabcb44 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -4,6 +4,8 @@ import log import sync.stdatomic import build { BuildConfig } import client +import time +import os const ( build_empty = 0 @@ -14,6 +16,7 @@ const ( struct AgentDaemon { logger shared log.Log conf Config +mut: images ImageManager // Which builds are currently running; length is same as // conf.max_concurrent_builds @@ -41,13 +44,33 @@ pub fn (mut d AgentDaemon) run() { for { free_builds := d.update_atomics() - if free_builds > 0 { + // All build slots are taken, so there's nothing to be done + if free_builds == 0 { + time.sleep(1 * time.second) + continue + } + + // Builds have finished, so old builder images might have freed up. + d.images.clean_old_images() + + // Poll for new jobs + new_configs := d.client.poll_jobs(free_builds) or { + d.lerror('Failed to poll jobs: $err.msg()') + + time.sleep(1 * time.second) + continue + } + + // Schedule new jobs + for config in new_configs { + d.start_build(config) } } } // update_atomics checks for each build whether it's completed, and sets it to -// free again if so. The return value is how many fields are now set to free. +// free again if so. The return value is how many build slots are currently +// free. fn (mut d AgentDaemon) update_atomics() int { mut count := 0 @@ -62,3 +85,53 @@ fn (mut d AgentDaemon) update_atomics() int { return count } + +// start_build starts a build for the given BuildConfig object. +fn (mut d AgentDaemon) start_build(config BuildConfig) bool { + for i in 0 .. d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { + stdatomic.store_u64(&d.atomics[i], agent.build_running) + d.builds[i] = config + + go d.run_build(i, config) + + return true + } + } + + return false +} + +// run_build actually starts the build process for a given target. +fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) { + d.linfo('started build: $config.url -> $config.repo') + + // 0 means success, 1 means failure + mut status := 0 + + new_config := BuildConfig{ + ...config + base_image: d.images.get(config.base_image) + } + + res := build.build_config(d.client.address, d.client.api_key, new_config) or { + d.ldebug('build_config error: $err.msg()') + status = 1 + + build.BuildResult{} + } + + if status == 0 { + d.linfo('finished build: $config.url -> $config.repo; uploading logs...') + + build_arch := os.uname().machine + d.client.add_build_log(config.target_id, res.start_time, res.end_time, build_arch, + res.exit_code, res.logs) or { + d.lerror('Failed to upload logs for build: $config.url -> $config.repo') + } + } else { + d.linfo('an error occured during build: $config.url -> $config.repo') + } + + stdatomic.store_u64(&d.atomics[build_index], agent.build_done) +} diff --git a/src/agent/images.v b/src/agent/images.v index aee2be0..78bf2d0 100644 --- a/src/agent/images.v +++ b/src/agent/images.v @@ -19,6 +19,10 @@ fn new_image_manager(refresh_frequency int) ImageManager { } } +pub fn (m &ImageManager) get(base_image string) string { + return m.images[base_image].last() +} + fn (mut m ImageManager) refresh_image(base_image string) ! { // No need to refresh the image if the previous one is still new enough if base_image in m.timestamps diff --git a/src/build/build.v b/src/build/build.v index 13d3e45..744ce9c 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -103,10 +103,23 @@ pub: logs string } +pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult { +config := BuildConfig{ + target_id: target.id + kind: target.kind + url: target.url + branch: target.branch + repo: target.repo + base_image: base_image_id + } + + return build_config(address, api_key, config) +} + // build_target builds, packages & publishes a given Arch package based on the // provided target. The base image ID should be of an image previously created // by create_build_image. It returns the logs of the container. -pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult { +pub fn build_config(address string, api_key string, config BuildConfig) !BuildResult { mut dd := docker.new_conn()! defer { @@ -114,14 +127,14 @@ pub fn build_target(address string, api_key string, base_image_id string, target } build_arch := os.uname().machine - build_script := create_build_script(address, target, build_arch) + build_script := create_build_script(address, config, build_arch) // We convert the build script into a base64 string, which then gets passed // to the container as an env var base64_script := base64.encode_str(build_script) c := docker.NewContainer{ - image: '$base_image_id' + image: '$config.base_image' env: [ 'BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key', diff --git a/src/build/shell.v b/src/build/shell.v index e573d53..42ec3c0 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -23,13 +23,13 @@ pub fn echo_commands(cmds []string) []string { } // create_build_script generates a shell script that builds a given Target. -fn create_build_script(address string, target &Target, build_arch string) string { - repo_url := '$address/$target.repo' +fn create_build_script(address string, config BuildConfig, build_arch string) string { + repo_url := '$address/$config.repo' mut commands := [ // This will later be replaced by a proper setting for changing the // mirrorlist - "echo -e '[$target.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" + "echo -e '[$config.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" // We need to update the package list of the repo we just added above. // This should however not pull in a lot of packages as long as the // builder image is rebuilt frequently. @@ -38,22 +38,22 @@ fn create_build_script(address string, target &Target, build_arch string) string 'su builder', ] - commands << match target.kind { + commands << match config.kind { 'git' { - if target.branch == '' { + if config.branch == '' { [ - "git clone --single-branch --depth 1 '$target.url' repo", + "git clone --single-branch --depth 1 '$config.url' repo", ] } else { [ - "git clone --single-branch --depth 1 --branch $target.branch '$target.url' repo", + "git clone --single-branch --depth 1 --branch $config.branch '$config.url' repo", ] } } 'url' { [ 'mkdir repo', - "curl -o repo/PKGBUILD -L '$target.url'", + "curl -o repo/PKGBUILD -L '$config.url'", ] } else { diff --git a/src/client/jobs.v b/src/client/jobs.v new file mode 100644 index 0000000..281d6ce --- /dev/null +++ b/src/client/jobs.v @@ -0,0 +1,11 @@ +module client + +import build { BuildConfig } + +pub fn (c &Client) poll_jobs(max int) ![]BuildConfig { + data := c.send_request<[]BuildConfig>(.get, '/api/v1/jobs/poll', { + 'max': max.str() + })! + + return data.data +} From d411fdc64b488be83efce72639a575511403c59f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Dec 2022 22:58:43 +0100 Subject: [PATCH 9/9] feat(agent): initial working version --- src/agent/cli.v | 4 ++-- src/agent/daemon.v | 39 ++++++++++++++++++++++++++++++++------- src/build/build.v | 18 +++++++++--------- src/build/queue.v | 1 - src/build/shell.v | 2 -- src/client/jobs.v | 5 +++-- vieter.toml | 2 +- 7 files changed, 47 insertions(+), 24 deletions(-) diff --git a/src/agent/cli.v b/src/agent/cli.v index 1badbab..a375f08 100644 --- a/src/agent/cli.v +++ b/src/agent/cli.v @@ -5,9 +5,9 @@ import conf as vconf struct Config { pub: - log_level string = 'WARN' + log_level string = 'WARN' // Architecture that the agent represents - arch string + arch string api_key string address string data_dir string diff --git a/src/agent/daemon.v b/src/agent/daemon.v index aabcb44..f060863 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -41,6 +41,10 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon { } pub fn (mut d AgentDaemon) run() { + // This is just so that the very first time the loop is ran, the jobs are + // always polled + mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency) + for { free_builds := d.update_atomics() @@ -54,16 +58,37 @@ pub fn (mut d AgentDaemon) run() { d.images.clean_old_images() // Poll for new jobs - new_configs := d.client.poll_jobs(free_builds) or { - d.lerror('Failed to poll jobs: $err.msg()') + if time.now() >= last_poll_time.add_seconds(d.conf.polling_frequency) { + new_configs := d.client.poll_jobs(d.conf.arch, free_builds) or { + d.lerror('Failed to poll jobs: $err.msg()') + + time.sleep(5 * time.second) + continue + } + last_poll_time = time.now() + + // Schedule new jobs + for config in new_configs { + // TODO handle this better than to just skip the config + // Make sure a recent build base image is available for building the config + d.images.refresh_image(config.base_image) or { + d.lerror(err.msg()) + continue + } + d.start_build(config) + } time.sleep(1 * time.second) - continue } - - // Schedule new jobs - for config in new_configs { - d.start_build(config) + // Builds are running, so check again after one second + else if free_builds < d.conf.max_concurrent_builds { + time.sleep(1 * time.second) + } + // The agent is not doing anything, so we just wait until the next poll + // time + else { + time_until_next_poll := time.now() - last_poll_time + time.sleep(time_until_next_poll) } } } diff --git a/src/build/build.v b/src/build/build.v index 744ce9c..2d51156 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -104,16 +104,16 @@ pub: } pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult { -config := BuildConfig{ - target_id: target.id - kind: target.kind - url: target.url - branch: target.branch - repo: target.repo - base_image: base_image_id - } + config := BuildConfig{ + target_id: target.id + kind: target.kind + url: target.url + branch: target.branch + repo: target.repo + base_image: base_image_id + } - return build_config(address, api_key, config) + return build_config(address, api_key, config) } // build_target builds, packages & publishes a given Arch package based on the diff --git a/src/build/queue.v b/src/build/queue.v index b704926..29036e4 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -76,7 +76,6 @@ pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { } } - dump(job) q.queues[arch].insert(job) } } diff --git a/src/build/shell.v b/src/build/shell.v index 42ec3c0..c2d0c9b 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -1,7 +1,5 @@ module build -import models { Target } - // escape_shell_string escapes any characters that could be interpreted // incorrectly by a shell. The resulting value should be safe to use inside an // echo statement. diff --git a/src/client/jobs.v b/src/client/jobs.v index 281d6ce..30f2531 100644 --- a/src/client/jobs.v +++ b/src/client/jobs.v @@ -2,9 +2,10 @@ module client import build { BuildConfig } -pub fn (c &Client) poll_jobs(max int) ![]BuildConfig { +pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig { data := c.send_request<[]BuildConfig>(.get, '/api/v1/jobs/poll', { - 'max': max.str() + 'arch': arch + 'max': max.str() })! return data.data diff --git a/vieter.toml b/vieter.toml index d3922a4..9a68ae3 100644 --- a/vieter.toml +++ b/vieter.toml @@ -4,6 +4,7 @@ data_dir = "data" pkg_dir = "data/pkgs" log_level = "DEBUG" default_arch = "x86_64" +arch = "x86_64" address = "http://localhost:8000" @@ -11,4 +12,3 @@ global_schedule = '* *' api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 -