From 6281ef76070f1ccf7b2bed05e06f9ca7cdf5322c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 6 Dec 2022 13:50:25 +0100 Subject: [PATCH 01/22] feat: start of agent code --- src/agent/agent.v | 25 ++++++++++++++++++ src/agent/cli.v | 31 ++++++++++++++++++++++ src/agent/daemon.v | 65 ++++++++++++++++++++++++++++++++++++++++++++++ src/build/build.v | 10 +++++++ src/main.v | 2 ++ 5 files changed, 133 insertions(+) create mode 100644 src/agent/agent.v create mode 100644 src/agent/cli.v create mode 100644 src/agent/daemon.v diff --git a/src/agent/agent.v b/src/agent/agent.v new file mode 100644 index 0000000..3affd21 --- /dev/null +++ b/src/agent/agent.v @@ -0,0 +1,25 @@ +module agent + +import log +import os + +const log_file_name = 'vieter.agent.log' + +// agent start an agent service +pub fn agent(conf Config) ! { + // Configure logger + log_level := log.level_from_tag(conf.log_level) or { + return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + } + + mut logger := log.Log{ + level: log_level + } + + log_file := os.join_path_single(conf.data_dir, agent.log_file_name) + logger.set_full_logpath(log_file) + logger.log_to_console_too() + + mut d := agent.agent_init(logger, conf) + d.run() +} diff --git a/src/agent/cli.v b/src/agent/cli.v new file mode 100644 index 0000000..46942ec --- /dev/null +++ b/src/agent/cli.v @@ -0,0 +1,31 @@ +module agent + +import cli +import conf as vconf + +struct Config { +pub: + log_level string = 'WARN' + api_key string + address string + data_dir string + max_concurrent_builds int = 1 + polling_frequency int = 30 + // Architecture of agent + /* arch string */ + /* image_rebuild_frequency int = 1440 */ +} + +// cmd returns the cli module that handles the cron daemon. +pub fn cmd() cli.Command { + return cli.Command{ + name: 'agent' + description: 'Start an agent service & start polling for new builds.' + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + + agent(conf)! + } + } +} diff --git a/src/agent/daemon.v b/src/agent/daemon.v new file mode 100644 index 0000000..389a148 --- /dev/null +++ b/src/agent/daemon.v @@ -0,0 +1,65 @@ +module agent + +import log +import sync.stdatomic +import build { BuildConfig } +import client + +const ( + build_empty = 0 + build_running = 1 + build_done = 2 +) + +struct AgentDaemon { + logger shared log.Log + conf Config + // Which builds are currently running; length is same as + // conf.max_concurrent_builds + builds []BuildConfig + // Atomic variables used to detect when a build has finished; length is the + // same as conf.max_concurrent_builds + client client.Client + atomics []u64 +} + +fn agent_init(logger log.Log, conf Config) AgentDaemon { + mut d := AgentDaemon{ + logger: logger + client: client.new(conf.address, conf.api_key) + conf: conf + builds: []BuildConfig{len: conf.max_concurrent_builds} + atomics: []u64{len: conf.max_concurrent_builds} + } + + return d +} + +pub fn (mut d AgentDaemon) run() { + for { + free_builds := d.update_atomics() + + if free_builds > 0 { + + } + + } +} + +// clean_finished_builds checks for each build whether it's completed, and sets +// it to free again if so. The return value is how many fields are now set to +// free. +fn (mut d AgentDaemon) update_atomics() int { + mut count := 0 + + for i in 0 .. d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == agent.build_done { + stdatomic.store_u64(&d.atomics[i], agent.build_empty) + count++ + } else if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { + count++ + } + } + + return count +} diff --git a/src/build/build.v b/src/build/build.v index 247df6e..b7c5cb6 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -16,6 +16,16 @@ const ( '/usr/local/bin', '/usr/bin/site_perl', '/usr/bin/vendor_perl', '/usr/bin/core_perl'] ) +pub struct BuildConfig { +pub: + id int + kind string + url string + branch string + repo string + base_image string +} + // create_build_image creates a builder image given some base image which can // then be used to build & package Arch images. It mostly just updates the // system, install some necessary packages & creates a non-root user to run diff --git a/src/main.v b/src/main.v index fc09f7e..424e328 100644 --- a/src/main.v +++ b/src/main.v @@ -9,6 +9,7 @@ import console.schedule import console.man import console.aur import cron +import agent fn main() { mut app := cli.Command{ @@ -40,6 +41,7 @@ fn main() { schedule.cmd(), man.cmd(), aur.cmd(), + agent.cmd() ] } app.setup() From 9a49d96e202208453169524585eb28882628f10f Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 6 Dec 2022 14:11:17 +0100 Subject: [PATCH 02/22] feat(build): start of server-side job queue --- src/agent/agent.v | 2 +- src/agent/cli.v | 16 +++++----- src/agent/daemon.v | 10 +++--- src/build/build.v | 10 +++--- src/build/queue.v | 70 +++++++++++++++++++++++++++++++++++++++++ src/main.v | 2 +- src/server/api_builds.v | 39 +++++++++++++++++++++++ src/server/cli.v | 14 +++++---- src/server/server.v | 39 ++++++++++++++++++++++- 9 files changed, 174 insertions(+), 28 deletions(-) create mode 100644 src/build/queue.v create mode 100644 src/server/api_builds.v diff --git a/src/agent/agent.v b/src/agent/agent.v index 3affd21..1758c85 100644 --- a/src/agent/agent.v +++ b/src/agent/agent.v @@ -20,6 +20,6 @@ pub fn agent(conf Config) ! { logger.set_full_logpath(log_file) logger.log_to_console_too() - mut d := agent.agent_init(logger, conf) + mut d := agent_init(logger, conf) d.run() } diff --git a/src/agent/cli.v b/src/agent/cli.v index 46942ec..a0a249c 100644 --- a/src/agent/cli.v +++ b/src/agent/cli.v @@ -5,15 +5,15 @@ import conf as vconf struct Config { pub: - log_level string = 'WARN' - api_key string - address string - data_dir string - max_concurrent_builds int = 1 - polling_frequency int = 30 + log_level string = 'WARN' + api_key string + address string + data_dir string + max_concurrent_builds int = 1 + polling_frequency int = 30 // Architecture of agent - /* arch string */ - /* image_rebuild_frequency int = 1440 */ + // arch string + // image_rebuild_frequency int = 1440 } // cmd returns the cli module that handles the cron daemon. diff --git a/src/agent/daemon.v b/src/agent/daemon.v index 389a148..fd5fe04 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -13,13 +13,13 @@ const ( struct AgentDaemon { logger shared log.Log - conf Config + conf Config // Which builds are currently running; length is same as // conf.max_concurrent_builds builds []BuildConfig // Atomic variables used to detect when a build has finished; length is the // same as conf.max_concurrent_builds - client client.Client + client client.Client atomics []u64 } @@ -39,10 +39,8 @@ pub fn (mut d AgentDaemon) run() { for { free_builds := d.update_atomics() - if free_builds > 0 { - - } - + if free_builds > 0 { + } } } diff --git a/src/build/build.v b/src/build/build.v index b7c5cb6..13d3e45 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -18,11 +18,11 @@ const ( pub struct BuildConfig { pub: - id int - kind string - url string - branch string - repo string + target_id int + kind string + url string + branch string + repo string base_image string } diff --git a/src/build/queue.v b/src/build/queue.v new file mode 100644 index 0000000..81d3fa9 --- /dev/null +++ b/src/build/queue.v @@ -0,0 +1,70 @@ +module build + +import models { Target } +import cron.expression { CronExpression, parse_expression } +import time +import datatypes { MinHeap } + +struct BuildJob { +pub: + // Earliest point this + timestamp time.Time + config BuildConfig +} + +// Overloaded operator for comparing ScheduledBuild objects +fn (r1 BuildJob) < (r2 BuildJob) bool { + return r1.timestamp < r2.timestamp +} + +pub struct BuildJobQueue { + // Schedule to use for targets without explicitely defined cron expression + default_schedule CronExpression + // Base image to use for targets without defined base image + default_base_image string +mut: + // For each architecture, a priority queue is tracked + queues map[string]MinHeap + // Each queued build job is also stored in a map, with the keys being the + // target IDs. This is used when removing or editing targets. + // jobs map[int]BuildJob +} + +pub fn new_job_queue(default_schedule CronExpression, default_base_image string) BuildJobQueue { + return BuildJobQueue{ + default_schedule: default_schedule + default_base_image: default_base_image + } +} + +// insert a new job into the queue for a given target on an architecture. +pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { + if arch !in q.queues { + q.queues[arch] = MinHeap{} + } + + ce := if target.schedule != '' { + parse_expression(target.schedule) or { + return error("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()") + } + } else { + q.default_schedule + } + + timestamp := ce.next_from_now()! + + job := BuildJob{ + timestamp: timestamp + config: BuildConfig{ + target_id: target.id + kind: target.kind + url: target.url + branch: target.branch + repo: target.repo + // TODO make this configurable + base_image: q.default_base_image + } + } + + q.queues[arch].insert(job) +} diff --git a/src/main.v b/src/main.v index 424e328..34387bf 100644 --- a/src/main.v +++ b/src/main.v @@ -41,7 +41,7 @@ fn main() { schedule.cmd(), man.cmd(), aur.cmd(), - agent.cmd() + agent.cmd(), ] } app.setup() diff --git a/src/server/api_builds.v b/src/server/api_builds.v new file mode 100644 index 0000000..888fe9d --- /dev/null +++ b/src/server/api_builds.v @@ -0,0 +1,39 @@ +module server + +/* import web */ +/* import web.response { new_data_response, new_response } */ +/* import time */ +/* import build { BuildConfig } */ +/* // import os */ +/* // import util */ +/* // import models { BuildLog, BuildLogFilter } */ + +/* ['/api/v1/builds/poll'; auth; get] */ +/* fn (mut app App) v1_poll_build_queue() web.Result { */ +/* arch := app.query['arch'] or { */ +/* return app.json(.bad_request, new_response('Missing arch query arg.')) */ +/* } */ + +/* max_str := app.query['max'] or { */ +/* return app.json(.bad_request, new_response('Missing max query arg.')) */ +/* } */ +/* max := max_str.int() */ + +/* mut out := []BuildConfig{} */ + +/* now := time.now() */ + +/* lock app.build_queues { */ +/* mut queue := app.build_queues[arch] or { return app.json(.ok, new_data_response(out)) } */ + +/* for queue.len() > 0 && out.len < max { */ +/* next := queue.peek() or { return app.status(.internal_server_error) } */ + +/* if next.timestamp < now { */ +/* out << queue.pop() or { return app.status(.internal_server_error) }.config */ +/* } */ +/* } */ +/* } */ + +/* return app.json(.ok, new_data_response(out)) */ +/* } */ diff --git a/src/server/cli.v b/src/server/cli.v index a9644f3..2fede6c 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -5,12 +5,14 @@ import conf as vconf struct Config { pub: - log_level string = 'WARN' - pkg_dir string - data_dir string - api_key string - default_arch string - port int = 8000 + log_level string = 'WARN' + pkg_dir string + data_dir string + api_key string + default_arch string + global_schedule string = '0 3' + port int = 8000 + base_image string = 'archlinux:base-devel' } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/server.v b/src/server/server.v index d5f6135..fb45e6d 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -6,6 +6,8 @@ import log import repo import util import db +import build { BuildJobQueue } +import cron.expression const ( log_file_name = 'vieter.log' @@ -20,9 +22,37 @@ pub: conf Config [required; web_global] pub mut: repo repo.RepoGroupManager [required; web_global] - db db.VieterDb + // Keys are the various architectures for packages + job_queue BuildJobQueue [required; web_global] + db db.VieterDb } +// fn (mut app App) init_build_queues() { +// // Initialize build queues +// mut i := 0 +// mut targets := app.db.get_targets(limit: 25) + +// default_ce := expression.parse_expression(conf.global_schedule) or { return } + +// for targets.len > 0 { +// for t in targets { +// ce := parse_expression(t.schedule) or { default_ce } + +// for arch in t.arch { +// if arch !in app.build_queues { +// app.build_queues[arch] = Minheap{} +// } + +// build_config := BuildConfig{} +// app.build_queues[arch].push(ScheduledBuild{ +// timestamp: ce.next() +// config: build_config +// }) +// } +// } +// } +//} + // server starts the web server & starts listening for requests pub fn server(conf Config) ! { // Prevent using 'any' as the default arch @@ -30,6 +60,10 @@ pub fn server(conf Config) ! { util.exit_with_message(1, "'any' is not allowed as the value for default_arch.") } + global_ce := expression.parse_expression(conf.global_schedule) or { + util.exit_with_message(1, 'Invalid global cron expression: $err.msg()') + } + // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -71,11 +105,14 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Failed to initialize database: $err.msg()') } + mut queue := build.new_job_queue(global_ce, conf.base_image) + web.run(&App{ logger: logger api_key: conf.api_key conf: conf repo: repo db: db + job_queue: queue }, conf.port) } From c57de4d8ee4994b6efcc3370690835e86bd893e4 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Dec 2022 20:33:51 +0100 Subject: [PATCH 03/22] feat(server): initialize job queue on start; api endpoint for polling jobs --- src/build/queue.v | 113 +++++++++++++++++++++++++++++++--------- src/server/api_builds.v | 50 ++++++------------ src/server/server.v | 52 ++++++++---------- 3 files changed, 129 insertions(+), 86 deletions(-) diff --git a/src/build/queue.v b/src/build/queue.v index 81d3fa9..65b279e 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -4,6 +4,7 @@ import models { Target } import cron.expression { CronExpression, parse_expression } import time import datatypes { MinHeap } +import util struct BuildJob { pub: @@ -23,6 +24,7 @@ pub struct BuildJobQueue { // Base image to use for targets without defined base image default_base_image string mut: + mutex shared util.Dummy // For each architecture, a priority queue is tracked queues map[string]MinHeap // Each queued build job is also stored in a map, with the keys being the @@ -39,32 +41,95 @@ pub fn new_job_queue(default_schedule CronExpression, default_base_image string) // insert a new job into the queue for a given target on an architecture. pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { - if arch !in q.queues { - q.queues[arch] = MinHeap{} - } - - ce := if target.schedule != '' { - parse_expression(target.schedule) or { - return error("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()") + lock q.mutex { + if arch !in q.queues { + q.queues[arch] = MinHeap{} } - } else { - q.default_schedule - } - timestamp := ce.next_from_now()! - - job := BuildJob{ - timestamp: timestamp - config: BuildConfig{ - target_id: target.id - kind: target.kind - url: target.url - branch: target.branch - repo: target.repo - // TODO make this configurable - base_image: q.default_base_image + ce := if target.schedule != '' { + parse_expression(target.schedule) or { + return error("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()") + } + } else { + q.default_schedule } - } - q.queues[arch].insert(job) + timestamp := ce.next_from_now()! + + job := BuildJob{ + timestamp: timestamp + config: BuildConfig{ + target_id: target.id + kind: target.kind + url: target.url + branch: target.branch + repo: target.repo + // TODO make this configurable + base_image: q.default_base_image + } + } + + q.queues[arch].insert(job) + } +} + +// peek shows the first job for the given architecture that's ready to be +// executed, if present. +pub fn (q &BuildJobQueue) peek(arch string) ?BuildJob { + rlock q.mutex { + if arch !in q.queues { + return none + } + + job := q.queues[arch].peek() or { return none } + + if job.timestamp < time.now() { + return job + } + } + + return none +} + +// pop removes the first job for the given architecture that's ready to be +// executed from the queue and returns it, if present. +pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob { + lock q.mutex { + if arch !in q.queues { + return none + } + + job := q.queues[arch].peek() or { return none } + + if job.timestamp < time.now() { + return q.queues[arch].pop() + } + } + + return none +} + +// pop_n tries to pop at most n available jobs for the given architecture. +pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { + lock q.mutex { + if arch !in q.queues { + return [] + } + + mut out := []BuildJob{} + + for out.len < n { + job := q.queues[arch].peek() or { break } + + if job.timestamp < time.now() { + out << q.queues[arch].pop() or { break } + } else { + break + } + } + + return out + } + + return [] } diff --git a/src/server/api_builds.v b/src/server/api_builds.v index 888fe9d..62948cd 100644 --- a/src/server/api_builds.v +++ b/src/server/api_builds.v @@ -1,39 +1,23 @@ module server -/* import web */ -/* import web.response { new_data_response, new_response } */ -/* import time */ -/* import build { BuildConfig } */ -/* // import os */ -/* // import util */ -/* // import models { BuildLog, BuildLogFilter } */ +import web +import web.response { new_data_response, new_response } +// import os +// import util +// import models { BuildLog, BuildLogFilter } -/* ['/api/v1/builds/poll'; auth; get] */ -/* fn (mut app App) v1_poll_build_queue() web.Result { */ -/* arch := app.query['arch'] or { */ -/* return app.json(.bad_request, new_response('Missing arch query arg.')) */ -/* } */ +['/api/v1/builds/poll'; auth; get] +fn (mut app App) v1_poll_build_queue() web.Result { + arch := app.query['arch'] or { + return app.json(.bad_request, new_response('Missing arch query arg.')) + } -/* max_str := app.query['max'] or { */ -/* return app.json(.bad_request, new_response('Missing max query arg.')) */ -/* } */ -/* max := max_str.int() */ + max_str := app.query['max'] or { + return app.json(.bad_request, new_response('Missing max query arg.')) + } + max := max_str.int() -/* mut out := []BuildConfig{} */ + mut out := app.job_queue.pop_n(arch, max) -/* now := time.now() */ - -/* lock app.build_queues { */ -/* mut queue := app.build_queues[arch] or { return app.json(.ok, new_data_response(out)) } */ - -/* for queue.len() > 0 && out.len < max { */ -/* next := queue.peek() or { return app.status(.internal_server_error) } */ - -/* if next.timestamp < now { */ -/* out << queue.pop() or { return app.status(.internal_server_error) }.config */ -/* } */ -/* } */ -/* } */ - -/* return app.json(.ok, new_data_response(out)) */ -/* } */ + return app.json(.ok, new_data_response(out)) +} diff --git a/src/server/server.v b/src/server/server.v index fb45e6d..e2c19c2 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -24,34 +24,25 @@ pub mut: repo repo.RepoGroupManager [required; web_global] // Keys are the various architectures for packages job_queue BuildJobQueue [required; web_global] - db db.VieterDb + db db.VieterDb } -// fn (mut app App) init_build_queues() { -// // Initialize build queues -// mut i := 0 -// mut targets := app.db.get_targets(limit: 25) +fn (mut app App) init_job_queue() ! { + // Initialize build queues + mut targets := app.db.get_targets(limit: 25) + mut i := u64(0) -// default_ce := expression.parse_expression(conf.global_schedule) or { return } + for targets.len > 0 { + for target in targets { + for arch in target.arch { + app.job_queue.insert(target, arch.value)! + } + } -// for targets.len > 0 { -// for t in targets { -// ce := parse_expression(t.schedule) or { default_ce } - -// for arch in t.arch { -// if arch !in app.build_queues { -// app.build_queues[arch] = Minheap{} -// } - -// build_config := BuildConfig{} -// app.build_queues[arch].push(ScheduledBuild{ -// timestamp: ce.next() -// config: build_config -// }) -// } -// } -// } -//} + i += 25 + targets = app.db.get_targets(limit: 25, offset: i) + } +} // server starts the web server & starts listening for requests pub fn server(conf Config) ! { @@ -105,14 +96,17 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Failed to initialize database: $err.msg()') } - mut queue := build.new_job_queue(global_ce, conf.base_image) - - web.run(&App{ + mut app := &App{ logger: logger api_key: conf.api_key conf: conf repo: repo db: db - job_queue: queue - }, conf.port) + job_queue: build.new_job_queue(global_ce, conf.base_image) + } + app.init_job_queue() or { + util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()') + } + + web.run(app, conf.port) } From 0a5c4295e008b3687d160957328a934c85489f9b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Dec 2022 20:59:43 +0100 Subject: [PATCH 04/22] feat(server): properly reschedule jobs after polling --- src/build/queue.v | 50 ++++++++++++++++++++++++++++++++++------- src/server/api_builds.v | 6 ++--- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/build/queue.v b/src/build/queue.v index 65b279e..b704926 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -8,12 +8,16 @@ import util struct BuildJob { pub: - // Earliest point this + // Next timestamp from which point this job is allowed to be executed timestamp time.Time - config BuildConfig + // Required for calculating next timestamp after having pop'ed a job + ce CronExpression + // Actual build config sent to the agent + config BuildConfig } -// Overloaded operator for comparing ScheduledBuild objects +// Allows BuildJob structs to be sorted according to their timestamp in +// MinHeaps fn (r1 BuildJob) < (r2 BuildJob) bool { return r1.timestamp < r2.timestamp } @@ -39,7 +43,9 @@ pub fn new_job_queue(default_schedule CronExpression, default_base_image string) } } -// insert a new job into the queue for a given target on an architecture. +// insert a new target's job into the queue for the given architecture. This +// job will then be endlessly rescheduled after being pop'ed, unless removed +// explicitely. pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { lock q.mutex { if arch !in q.queues { @@ -58,6 +64,7 @@ pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { job := BuildJob{ timestamp: timestamp + ce: ce config: BuildConfig{ target_id: target.id kind: target.kind @@ -69,10 +76,25 @@ pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { } } + dump(job) q.queues[arch].insert(job) } } +// reschedule the given job by calculating the next timestamp and re-adding it +// to its respective queue. This function is called by the pop functions +// *after* having pop'ed the job. +fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) ! { + new_timestamp := job.ce.next_from_now()! + + new_job := BuildJob{ + ...job + timestamp: new_timestamp + } + + q.queues[arch].insert(new_job) +} + // peek shows the first job for the given architecture that's ready to be // executed, if present. pub fn (q &BuildJobQueue) peek(arch string) ?BuildJob { @@ -99,10 +121,17 @@ pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob { return none } - job := q.queues[arch].peek() or { return none } + mut job := q.queues[arch].peek() or { return none } if job.timestamp < time.now() { - return q.queues[arch].pop() + job = q.queues[arch].pop()? + + // TODO how do we handle this properly? Is it even possible for a + // cron expression to not return a next time if it's already been + // used before? + q.reschedule(job, arch) or {} + + return job } } @@ -119,10 +148,15 @@ pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { mut out := []BuildJob{} for out.len < n { - job := q.queues[arch].peek() or { break } + mut job := q.queues[arch].peek() or { break } if job.timestamp < time.now() { - out << q.queues[arch].pop() or { break } + job = q.queues[arch].pop() or { break } + + // TODO idem + q.reschedule(job, arch) or {} + + out << job } else { break } diff --git a/src/server/api_builds.v b/src/server/api_builds.v index 62948cd..ec3c8ec 100644 --- a/src/server/api_builds.v +++ b/src/server/api_builds.v @@ -6,8 +6,8 @@ import web.response { new_data_response, new_response } // import util // import models { BuildLog, BuildLogFilter } -['/api/v1/builds/poll'; auth; get] -fn (mut app App) v1_poll_build_queue() web.Result { +['/api/v1/jobs/poll'; auth; get] +fn (mut app App) v1_poll_job_queue() web.Result { arch := app.query['arch'] or { return app.json(.bad_request, new_response('Missing arch query arg.')) } @@ -17,7 +17,7 @@ fn (mut app App) v1_poll_build_queue() web.Result { } max := max_str.int() - mut out := app.job_queue.pop_n(arch, max) + mut out := app.job_queue.pop_n(arch, max).map(it.config) return app.json(.ok, new_data_response(out)) } From 5bab1f77f0686a3db9eedb2ecab36d7592299655 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Dec 2022 21:21:58 +0100 Subject: [PATCH 05/22] feat(agent): begin reforming for new api --- src/agent/daemon.v | 7 ++++--- src/agent/images.v | 49 ++++++++++++++++++++++++++++++++++++++++++++++ src/agent/log.v | 35 +++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 src/agent/images.v create mode 100644 src/agent/log.v diff --git a/src/agent/daemon.v b/src/agent/daemon.v index fd5fe04..71f4780 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -14,6 +14,8 @@ const ( struct AgentDaemon { logger shared log.Log conf Config + // List of last built builder images + builder_images []string // Which builds are currently running; length is same as // conf.max_concurrent_builds builds []BuildConfig @@ -44,9 +46,8 @@ pub fn (mut d AgentDaemon) run() { } } -// clean_finished_builds checks for each build whether it's completed, and sets -// it to free again if so. The return value is how many fields are now set to -// free. +// update_atomics checks for each build whether it's completed, and sets it to +// free again if so. The return value is how many fields are now set to free. fn (mut d AgentDaemon) update_atomics() int { mut count := 0 diff --git a/src/agent/images.v b/src/agent/images.v new file mode 100644 index 0000000..454f85f --- /dev/null +++ b/src/agent/images.v @@ -0,0 +1,49 @@ +module agent + +import time +import docker + +struct ImageManager { + images map[string]string + timestamps map[string]time.Time +} + +// clean_old_base_images tries to remove any old but still present builder +// images. +fn (mut d AgentDaemon) clean_old_base_images() { + mut i := 0 + + mut dd := docker.new_conn() or { + d.lerror('Failed to connect to Docker socket.') + return + } + + defer { + dd.close() or {} + } + + for i < d.builder_images.len - 1 { + // For each builder image, we try to remove it by calling the Docker + // API. If the function returns an error or false, that means the image + // wasn't deleted. Therefore, we move the index over. If the function + // returns true, the array's length has decreased by one so we don't + // move the index. + dd.remove_image(d.builder_images[i]) or { i += 1 } + } +} + +// rebuild_base_image builds a builder image from the given base image. +/* fn (mut d AgentDaemon) build_base_image(base_image string) bool { */ +/* d.linfo('Rebuilding builder image....') */ + +/* d.builder_images << build.create_build_image(d.base_image) or { */ +/* d.lerror('Failed to rebuild base image. Retrying in ${daemon.rebuild_base_image_retry_timout}s...') */ +/* d.image_build_timestamp = time.now().add_seconds(daemon.rebuild_base_image_retry_timout) */ + +/* return false */ +/* } */ + +/* d.image_build_timestamp = time.now().add_seconds(60 * d.image_rebuild_frequency) */ + +/* return true */ +/* } */ diff --git a/src/agent/log.v b/src/agent/log.v new file mode 100644 index 0000000..d47df0f --- /dev/null +++ b/src/agent/log.v @@ -0,0 +1,35 @@ +module agent + +import log + +// log reate a log message with the given level +pub fn (mut d AgentDaemon) log(msg string, level log.Level) { + lock d.logger { + d.logger.send_output(msg, level) + } +} + +// lfatal create a log message with the fatal level +pub fn (mut d AgentDaemon) lfatal(msg string) { + d.log(msg, log.Level.fatal) +} + +// lerror create a log message with the error level +pub fn (mut d AgentDaemon) lerror(msg string) { + d.log(msg, log.Level.error) +} + +// lwarn create a log message with the warn level +pub fn (mut d AgentDaemon) lwarn(msg string) { + d.log(msg, log.Level.warn) +} + +// linfo create a log message with the info level +pub fn (mut d AgentDaemon) linfo(msg string) { + d.log(msg, log.Level.info) +} + +// ldebug create a log message with the debug level +pub fn (mut d AgentDaemon) ldebug(msg string) { + d.log(msg, log.Level.debug) +} From 7ef8d4b846a6245258a23503862e8f95a2985d81 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Dec 2022 21:50:34 +0100 Subject: [PATCH 06/22] feat(agent): wrote ImageManager --- src/agent/cli.v | 2 +- src/agent/daemon.v | 4 +-- src/agent/images.v | 73 ++++++++++++++++++++++++++-------------------- 3 files changed, 45 insertions(+), 34 deletions(-) diff --git a/src/agent/cli.v b/src/agent/cli.v index a0a249c..063d960 100644 --- a/src/agent/cli.v +++ b/src/agent/cli.v @@ -13,7 +13,7 @@ pub: polling_frequency int = 30 // Architecture of agent // arch string - // image_rebuild_frequency int = 1440 + image_rebuild_frequency int = 1440 } // cmd returns the cli module that handles the cron daemon. diff --git a/src/agent/daemon.v b/src/agent/daemon.v index 71f4780..0508790 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -14,8 +14,7 @@ const ( struct AgentDaemon { logger shared log.Log conf Config - // List of last built builder images - builder_images []string + images ImageManager // Which builds are currently running; length is same as // conf.max_concurrent_builds builds []BuildConfig @@ -30,6 +29,7 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon { logger: logger client: client.new(conf.address, conf.api_key) conf: conf + images: new_image_manager(conf.image_rebuild_frequency) builds: []BuildConfig{len: conf.max_concurrent_builds} atomics: []u64{len: conf.max_concurrent_builds} } diff --git a/src/agent/images.v b/src/agent/images.v index 454f85f..aee2be0 100644 --- a/src/agent/images.v +++ b/src/agent/images.v @@ -2,48 +2,59 @@ module agent import time import docker +import build struct ImageManager { - images map[string]string - timestamps map[string]time.Time +mut: + refresh_frequency int + images map[string][]string [required] + timestamps map[string]time.Time [required] } -// clean_old_base_images tries to remove any old but still present builder -// images. -fn (mut d AgentDaemon) clean_old_base_images() { - mut i := 0 +fn new_image_manager(refresh_frequency int) ImageManager { + return ImageManager{ + refresh_frequency: refresh_frequency + images: map[string][]string{} + timestamps: map[string]time.Time{} + } +} - mut dd := docker.new_conn() or { - d.lerror('Failed to connect to Docker socket.') +fn (mut m ImageManager) refresh_image(base_image string) ! { + // No need to refresh the image if the previous one is still new enough + if base_image in m.timestamps + && m.timestamps[base_image].add_seconds(m.refresh_frequency) > time.now() { return } + // TODO use better image tags for built images + new_image := build.create_build_image(base_image) or { + return error('Failed to build builder image from base image $base_image') + } + + m.images[base_image] << new_image + m.timestamps[base_image] = time.now() +} + +// clean_old_images tries to remove any old but still present builder images. +fn (mut m ImageManager) clean_old_images() { + mut dd := docker.new_conn() or { return } + defer { dd.close() or {} } - for i < d.builder_images.len - 1 { - // For each builder image, we try to remove it by calling the Docker - // API. If the function returns an error or false, that means the image - // wasn't deleted. Therefore, we move the index over. If the function - // returns true, the array's length has decreased by one so we don't - // move the index. - dd.remove_image(d.builder_images[i]) or { i += 1 } + mut i := 0 + + for image in m.images.keys() { + i = 0 + + for i < m.images[image].len - 1 { + // For each builder image, we try to remove it by calling the Docker + // API. If the function returns an error or false, that means the image + // wasn't deleted. Therefore, we move the index over. If the function + // returns true, the array's length has decreased by one so we don't + // move the index. + dd.remove_image(m.images[image][i]) or { i += 1 } + } } } - -// rebuild_base_image builds a builder image from the given base image. -/* fn (mut d AgentDaemon) build_base_image(base_image string) bool { */ -/* d.linfo('Rebuilding builder image....') */ - -/* d.builder_images << build.create_build_image(d.base_image) or { */ -/* d.lerror('Failed to rebuild base image. Retrying in ${daemon.rebuild_base_image_retry_timout}s...') */ -/* d.image_build_timestamp = time.now().add_seconds(daemon.rebuild_base_image_retry_timout) */ - -/* return false */ -/* } */ - -/* d.image_build_timestamp = time.now().add_seconds(60 * d.image_rebuild_frequency) */ - -/* return true */ -/* } */ From 6f23d690a7a0a78d7d9203850c96204832149df0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Dec 2022 22:09:57 +0100 Subject: [PATCH 07/22] feat(agent): partially wrote daemon code --- src/agent/cli.v | 2 ++ src/agent/daemon.v | 77 ++++++++++++++++++++++++++++++++++++++++++++-- src/agent/images.v | 4 +++ src/build/build.v | 19 ++++++++++-- src/build/shell.v | 16 +++++----- src/client/jobs.v | 11 +++++++ 6 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 src/client/jobs.v diff --git a/src/agent/cli.v b/src/agent/cli.v index 063d960..1badbab 100644 --- a/src/agent/cli.v +++ b/src/agent/cli.v @@ -6,6 +6,8 @@ import conf as vconf struct Config { pub: log_level string = 'WARN' + // Architecture that the agent represents + arch string api_key string address string data_dir string diff --git a/src/agent/daemon.v b/src/agent/daemon.v index 0508790..aabcb44 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -4,6 +4,8 @@ import log import sync.stdatomic import build { BuildConfig } import client +import time +import os const ( build_empty = 0 @@ -14,6 +16,7 @@ const ( struct AgentDaemon { logger shared log.Log conf Config +mut: images ImageManager // Which builds are currently running; length is same as // conf.max_concurrent_builds @@ -41,13 +44,33 @@ pub fn (mut d AgentDaemon) run() { for { free_builds := d.update_atomics() - if free_builds > 0 { + // All build slots are taken, so there's nothing to be done + if free_builds == 0 { + time.sleep(1 * time.second) + continue + } + + // Builds have finished, so old builder images might have freed up. + d.images.clean_old_images() + + // Poll for new jobs + new_configs := d.client.poll_jobs(free_builds) or { + d.lerror('Failed to poll jobs: $err.msg()') + + time.sleep(1 * time.second) + continue + } + + // Schedule new jobs + for config in new_configs { + d.start_build(config) } } } // update_atomics checks for each build whether it's completed, and sets it to -// free again if so. The return value is how many fields are now set to free. +// free again if so. The return value is how many build slots are currently +// free. fn (mut d AgentDaemon) update_atomics() int { mut count := 0 @@ -62,3 +85,53 @@ fn (mut d AgentDaemon) update_atomics() int { return count } + +// start_build starts a build for the given BuildConfig object. +fn (mut d AgentDaemon) start_build(config BuildConfig) bool { + for i in 0 .. d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { + stdatomic.store_u64(&d.atomics[i], agent.build_running) + d.builds[i] = config + + go d.run_build(i, config) + + return true + } + } + + return false +} + +// run_build actually starts the build process for a given target. +fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) { + d.linfo('started build: $config.url -> $config.repo') + + // 0 means success, 1 means failure + mut status := 0 + + new_config := BuildConfig{ + ...config + base_image: d.images.get(config.base_image) + } + + res := build.build_config(d.client.address, d.client.api_key, new_config) or { + d.ldebug('build_config error: $err.msg()') + status = 1 + + build.BuildResult{} + } + + if status == 0 { + d.linfo('finished build: $config.url -> $config.repo; uploading logs...') + + build_arch := os.uname().machine + d.client.add_build_log(config.target_id, res.start_time, res.end_time, build_arch, + res.exit_code, res.logs) or { + d.lerror('Failed to upload logs for build: $config.url -> $config.repo') + } + } else { + d.linfo('an error occured during build: $config.url -> $config.repo') + } + + stdatomic.store_u64(&d.atomics[build_index], agent.build_done) +} diff --git a/src/agent/images.v b/src/agent/images.v index aee2be0..78bf2d0 100644 --- a/src/agent/images.v +++ b/src/agent/images.v @@ -19,6 +19,10 @@ fn new_image_manager(refresh_frequency int) ImageManager { } } +pub fn (m &ImageManager) get(base_image string) string { + return m.images[base_image].last() +} + fn (mut m ImageManager) refresh_image(base_image string) ! { // No need to refresh the image if the previous one is still new enough if base_image in m.timestamps diff --git a/src/build/build.v b/src/build/build.v index 13d3e45..744ce9c 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -103,10 +103,23 @@ pub: logs string } +pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult { +config := BuildConfig{ + target_id: target.id + kind: target.kind + url: target.url + branch: target.branch + repo: target.repo + base_image: base_image_id + } + + return build_config(address, api_key, config) +} + // build_target builds, packages & publishes a given Arch package based on the // provided target. The base image ID should be of an image previously created // by create_build_image. It returns the logs of the container. -pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult { +pub fn build_config(address string, api_key string, config BuildConfig) !BuildResult { mut dd := docker.new_conn()! defer { @@ -114,14 +127,14 @@ pub fn build_target(address string, api_key string, base_image_id string, target } build_arch := os.uname().machine - build_script := create_build_script(address, target, build_arch) + build_script := create_build_script(address, config, build_arch) // We convert the build script into a base64 string, which then gets passed // to the container as an env var base64_script := base64.encode_str(build_script) c := docker.NewContainer{ - image: '$base_image_id' + image: '$config.base_image' env: [ 'BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key', diff --git a/src/build/shell.v b/src/build/shell.v index e573d53..42ec3c0 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -23,13 +23,13 @@ pub fn echo_commands(cmds []string) []string { } // create_build_script generates a shell script that builds a given Target. -fn create_build_script(address string, target &Target, build_arch string) string { - repo_url := '$address/$target.repo' +fn create_build_script(address string, config BuildConfig, build_arch string) string { + repo_url := '$address/$config.repo' mut commands := [ // This will later be replaced by a proper setting for changing the // mirrorlist - "echo -e '[$target.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" + "echo -e '[$config.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" // We need to update the package list of the repo we just added above. // This should however not pull in a lot of packages as long as the // builder image is rebuilt frequently. @@ -38,22 +38,22 @@ fn create_build_script(address string, target &Target, build_arch string) string 'su builder', ] - commands << match target.kind { + commands << match config.kind { 'git' { - if target.branch == '' { + if config.branch == '' { [ - "git clone --single-branch --depth 1 '$target.url' repo", + "git clone --single-branch --depth 1 '$config.url' repo", ] } else { [ - "git clone --single-branch --depth 1 --branch $target.branch '$target.url' repo", + "git clone --single-branch --depth 1 --branch $config.branch '$config.url' repo", ] } } 'url' { [ 'mkdir repo', - "curl -o repo/PKGBUILD -L '$target.url'", + "curl -o repo/PKGBUILD -L '$config.url'", ] } else { diff --git a/src/client/jobs.v b/src/client/jobs.v new file mode 100644 index 0000000..281d6ce --- /dev/null +++ b/src/client/jobs.v @@ -0,0 +1,11 @@ +module client + +import build { BuildConfig } + +pub fn (c &Client) poll_jobs(max int) ![]BuildConfig { + data := c.send_request<[]BuildConfig>(.get, '/api/v1/jobs/poll', { + 'max': max.str() + })! + + return data.data +} From 3611123f4549523f2420ac8a1157d146c9064c8d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Dec 2022 22:58:43 +0100 Subject: [PATCH 08/22] feat(agent): initial working version --- src/agent/cli.v | 4 ++-- src/agent/daemon.v | 39 ++++++++++++++++++++++++++++++++------- src/build/build.v | 18 +++++++++--------- src/build/queue.v | 1 - src/build/shell.v | 2 -- src/client/jobs.v | 5 +++-- vieter.toml | 2 +- 7 files changed, 47 insertions(+), 24 deletions(-) diff --git a/src/agent/cli.v b/src/agent/cli.v index 1badbab..a375f08 100644 --- a/src/agent/cli.v +++ b/src/agent/cli.v @@ -5,9 +5,9 @@ import conf as vconf struct Config { pub: - log_level string = 'WARN' + log_level string = 'WARN' // Architecture that the agent represents - arch string + arch string api_key string address string data_dir string diff --git a/src/agent/daemon.v b/src/agent/daemon.v index aabcb44..f060863 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -41,6 +41,10 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon { } pub fn (mut d AgentDaemon) run() { + // This is just so that the very first time the loop is ran, the jobs are + // always polled + mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency) + for { free_builds := d.update_atomics() @@ -54,16 +58,37 @@ pub fn (mut d AgentDaemon) run() { d.images.clean_old_images() // Poll for new jobs - new_configs := d.client.poll_jobs(free_builds) or { - d.lerror('Failed to poll jobs: $err.msg()') + if time.now() >= last_poll_time.add_seconds(d.conf.polling_frequency) { + new_configs := d.client.poll_jobs(d.conf.arch, free_builds) or { + d.lerror('Failed to poll jobs: $err.msg()') + + time.sleep(5 * time.second) + continue + } + last_poll_time = time.now() + + // Schedule new jobs + for config in new_configs { + // TODO handle this better than to just skip the config + // Make sure a recent build base image is available for building the config + d.images.refresh_image(config.base_image) or { + d.lerror(err.msg()) + continue + } + d.start_build(config) + } time.sleep(1 * time.second) - continue } - - // Schedule new jobs - for config in new_configs { - d.start_build(config) + // Builds are running, so check again after one second + else if free_builds < d.conf.max_concurrent_builds { + time.sleep(1 * time.second) + } + // The agent is not doing anything, so we just wait until the next poll + // time + else { + time_until_next_poll := time.now() - last_poll_time + time.sleep(time_until_next_poll) } } } diff --git a/src/build/build.v b/src/build/build.v index 744ce9c..2d51156 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -104,16 +104,16 @@ pub: } pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult { -config := BuildConfig{ - target_id: target.id - kind: target.kind - url: target.url - branch: target.branch - repo: target.repo - base_image: base_image_id - } + config := BuildConfig{ + target_id: target.id + kind: target.kind + url: target.url + branch: target.branch + repo: target.repo + base_image: base_image_id + } - return build_config(address, api_key, config) + return build_config(address, api_key, config) } // build_target builds, packages & publishes a given Arch package based on the diff --git a/src/build/queue.v b/src/build/queue.v index b704926..29036e4 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -76,7 +76,6 @@ pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { } } - dump(job) q.queues[arch].insert(job) } } diff --git a/src/build/shell.v b/src/build/shell.v index 42ec3c0..c2d0c9b 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -1,7 +1,5 @@ module build -import models { Target } - // escape_shell_string escapes any characters that could be interpreted // incorrectly by a shell. The resulting value should be safe to use inside an // echo statement. diff --git a/src/client/jobs.v b/src/client/jobs.v index 281d6ce..30f2531 100644 --- a/src/client/jobs.v +++ b/src/client/jobs.v @@ -2,9 +2,10 @@ module client import build { BuildConfig } -pub fn (c &Client) poll_jobs(max int) ![]BuildConfig { +pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig { data := c.send_request<[]BuildConfig>(.get, '/api/v1/jobs/poll', { - 'max': max.str() + 'arch': arch + 'max': max.str() })! return data.data diff --git a/vieter.toml b/vieter.toml index d3922a4..9a68ae3 100644 --- a/vieter.toml +++ b/vieter.toml @@ -4,6 +4,7 @@ data_dir = "data" pkg_dir = "data/pkgs" log_level = "DEBUG" default_arch = "x86_64" +arch = "x86_64" address = "http://localhost:8000" @@ -11,4 +12,3 @@ global_schedule = '* *' api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 - From 882a9a60a973427b7d0a181dc5f2c1117cd6188f Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 08:58:27 +0100 Subject: [PATCH 09/22] feat(build): allowed invalidating entries in build queue --- src/build/queue.v | 91 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 67 insertions(+), 24 deletions(-) diff --git a/src/build/queue.v b/src/build/queue.v index 29036e4..b559552 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -8,6 +8,8 @@ import util struct BuildJob { pub: + // Time at which this build job was created/queued + created time.Time // Next timestamp from which point this job is allowed to be executed timestamp time.Time // Required for calculating next timestamp after having pop'ed a job @@ -22,6 +24,8 @@ fn (r1 BuildJob) < (r2 BuildJob) bool { return r1.timestamp < r2.timestamp } +// The build job queue is responsible for managing the list of scheduled builds +// for each architecture. Agents receive jobs from this queue. pub struct BuildJobQueue { // Schedule to use for targets without explicitely defined cron expression default_schedule CronExpression @@ -31,15 +35,17 @@ mut: mutex shared util.Dummy // For each architecture, a priority queue is tracked queues map[string]MinHeap - // Each queued build job is also stored in a map, with the keys being the - // target IDs. This is used when removing or editing targets. - // jobs map[int]BuildJob + // When a target is removed from the server or edited, its previous build + // configs will be invalid. This map allows for those to be simply skipped + // by ignoring any build configs created before this timestamp. + invalidated map[int]time.Time } pub fn new_job_queue(default_schedule CronExpression, default_base_image string) BuildJobQueue { return BuildJobQueue{ default_schedule: default_schedule default_base_image: default_base_image + invalidated: map[int]time.Time{} } } @@ -63,6 +69,7 @@ pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { timestamp := ce.next_from_now()! job := BuildJob{ + created: time.now() timestamp: timestamp ce: ce config: BuildConfig{ @@ -88,6 +95,7 @@ fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) ! { new_job := BuildJob{ ...job + created: time.now() timestamp: new_timestamp } @@ -96,16 +104,26 @@ fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) ! { // peek shows the first job for the given architecture that's ready to be // executed, if present. -pub fn (q &BuildJobQueue) peek(arch string) ?BuildJob { +pub fn (mut q BuildJobQueue) peek(arch string) ?BuildJob { rlock q.mutex { if arch !in q.queues { return none } - job := q.queues[arch].peek() or { return none } + for { + job := q.queues[arch].peek() or { return none } - if job.timestamp < time.now() { - return job + // Skip any invalidated jobs + if job.config.target_id in q.invalidated + && job.created < q.invalidated[job.config.target_id] { + // This pop *should* never fail according to the source code + q.queues[arch].pop() or { return none } + continue + } + + if job.timestamp < time.now() { + return job + } } } @@ -120,17 +138,27 @@ pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob { return none } - mut job := q.queues[arch].peek() or { return none } + for { + mut job := q.queues[arch].peek() or { return none } - if job.timestamp < time.now() { - job = q.queues[arch].pop()? + // Skip any invalidated jobs + if job.config.target_id in q.invalidated + && job.created < q.invalidated[job.config.target_id] { + // This pop *should* never fail according to the source code + q.queues[arch].pop() or { return none } + continue + } - // TODO how do we handle this properly? Is it even possible for a - // cron expression to not return a next time if it's already been - // used before? - q.reschedule(job, arch) or {} + if job.timestamp < time.now() { + job = q.queues[arch].pop()? - return job + // TODO how do we handle this properly? Is it even possible for a + // cron expression to not return a next time if it's already been + // used before? + q.reschedule(job, arch) or {} + + return job + } } } @@ -146,18 +174,28 @@ pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { mut out := []BuildJob{} - for out.len < n { - mut job := q.queues[arch].peek() or { break } + outer: for out.len < n { + for { + mut job := q.queues[arch].peek() or { break outer } - if job.timestamp < time.now() { - job = q.queues[arch].pop() or { break } + // Skip any invalidated jobs + if job.config.target_id in q.invalidated + && job.created < q.invalidated[job.config.target_id] { + // This pop *should* never fail according to the source code + q.queues[arch].pop() or { break outer } + continue + } - // TODO idem - q.reschedule(job, arch) or {} + if job.timestamp < time.now() { + job = q.queues[arch].pop() or { break outer } - out << job - } else { - break + // TODO idem + q.reschedule(job, arch) or {} + + out << job + } else { + break outer + } } } @@ -166,3 +204,8 @@ pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { return [] } + +// invalidate a target's old build jobs. +pub fn (mut q BuildJobQueue) invalidate(target_id int) { + q.invalidated[target_id] = time.now() +} From b6168a3060752474bb1ba4bd961ac119eddce16f Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 12:38:39 +0100 Subject: [PATCH 10/22] fix(build): change tests to use BuildConfig instead --- src/build/shell_test.v | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/build/shell_test.v b/src/build/shell_test.v index 341df88..d228faf 100644 --- a/src/build/shell_test.v +++ b/src/build/shell_test.v @@ -1,42 +1,48 @@ module build -import models { Target } +import models fn test_create_build_script_git_branch() { - target := Target{ - id: 1 + config := BuildConfig{ + target_id: 1 kind: 'git' url: 'https://examplerepo.com' branch: 'main' repo: 'vieter' + base_image: 'not-used:latest' } - build_script := create_build_script('https://example.com', target, 'x86_64') + + build_script := create_build_script('https://example.com', config, 'x86_64') expected := $embed_file('build_script_git_branch.sh') assert build_script == expected.to_string().trim_space() } fn test_create_build_script_git() { - target := Target{ - id: 1 + config := BuildConfig{ + target_id: 1 kind: 'git' url: 'https://examplerepo.com' repo: 'vieter' + base_image: 'not-used:latest' } - build_script := create_build_script('https://example.com', target, 'x86_64') + + build_script := create_build_script('https://example.com', config, 'x86_64') expected := $embed_file('build_script_git.sh') assert build_script == expected.to_string().trim_space() } fn test_create_build_script_url() { - target := Target{ - id: 1 + config := BuildConfig{ + target_id: 1 kind: 'url' url: 'https://examplerepo.com' repo: 'vieter' + base_image: 'not-used:latest' } - build_script := create_build_script('https://example.com', target, 'x86_64') + + build_script := create_build_script('https://example.com', config, 'x86_64') expected := $embed_file('build_script_url.sh') assert build_script == expected.to_string().trim_space() From e742d3de6da36298ba4f34c2f12820a85e08fb47 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 13:46:07 +0100 Subject: [PATCH 11/22] fix(db): return correct id when adding targets --- src/db/logs.v | 2 ++ src/db/targets.v | 11 +++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/db/logs.v b/src/db/logs.v index 923dde2..2745467 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -84,6 +84,8 @@ pub fn (db &VieterDb) add_build_log(log BuildLog) int { insert log into BuildLog } + // Here, this does work because a log doesn't contain any foreign keys, + // meaning the ORM only has to do a single add inserted_id := db.conn.last_id() as int return inserted_id diff --git a/src/db/targets.v b/src/db/targets.v index a705ebb..41e56df 100644 --- a/src/db/targets.v +++ b/src/db/targets.v @@ -38,14 +38,17 @@ pub fn (db &VieterDb) get_target(target_id int) ?Target { } // add_target inserts the given target into the database. -pub fn (db &VieterDb) add_target(repo Target) int { +pub fn (db &VieterDb) add_target(target Target) int { sql db.conn { - insert repo into Target + insert target into Target } - inserted_id := db.conn.last_id() as int + // ID of inserted target is the largest id + inserted_target := sql db.conn { + select from Target order by id desc limit 1 + } - return inserted_id + return inserted_target.id } // delete_target deletes the target with the given id from the database. From 63427899217aae4390e01b72238758ac1457856d Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 13:58:51 +0100 Subject: [PATCH 12/22] feat(server): update job queue when adding, removing or updating targets --- src/build/queue.v | 7 +++++++ src/server/api_targets.v | 29 +++++++++++++++++++---------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/build/queue.v b/src/build/queue.v index b559552..a78e56a 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -49,6 +49,13 @@ pub fn new_job_queue(default_schedule CronExpression, default_base_image string) } } +// insert_all executes insert for each architecture of the given Target. +pub fn (mut q BuildJobQueue) insert_all(target Target) ! { + for arch in target.arch { + q.insert(target, arch.value)! + } +} + // insert a new target's job into the queue for the given architecture. This // job will then be endlessly rescheduled after being pop'ed, unless removed // explicitely. diff --git a/src/server/api_targets.v b/src/server/api_targets.v index 16db7e9..dc39d37 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -12,17 +12,17 @@ fn (mut app App) v1_get_targets() web.Result { filter := models.from_params(app.query) or { return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } - repos := app.db.get_targets(filter) + targets := app.db.get_targets(filter) - return app.json(.ok, new_data_response(repos)) + return app.json(.ok, new_data_response(targets)) } // v1_get_single_target returns the information for a single target. ['/api/v1/targets/:id'; auth; get] fn (mut app App) v1_get_single_target(id int) web.Result { - repo := app.db.get_target(id) or { return app.not_found() } + target := app.db.get_target(id) or { return app.not_found() } - return app.json(.ok, new_data_response(repo)) + return app.json(.ok, new_data_response(target)) } // v1_post_target creates a new target from the provided query string. @@ -30,22 +30,27 @@ fn (mut app App) v1_get_single_target(id int) web.Result { fn (mut app App) v1_post_target() web.Result { mut params := app.query.clone() - // If a repo is created without specifying the arch, we assume it's meant + // If a target is created without specifying the arch, we assume it's meant // for the default architecture. if 'arch' !in params || params['arch'] == '' { params['arch'] = app.conf.default_arch } - new_repo := models.from_params(params) or { + mut new_target := models.from_params(params) or { return app.json(http.Status.bad_request, new_response(err.msg())) } // Ensure someone doesn't submit an invalid kind - if new_repo.kind !in models.valid_kinds { + if new_target.kind !in models.valid_kinds { return app.json(http.Status.bad_request, new_response('Invalid kind.')) } - id := app.db.add_target(new_repo) + id := app.db.add_target(new_target) + new_target.id = id + + // Add the target to the job queue + // TODO return better error here if it's the cron schedule that's incorrect + app.job_queue.insert_all(new_target) or { return app.status(.internal_server_error) } return app.json(.ok, new_data_response(id)) } @@ -54,6 +59,7 @@ fn (mut app App) v1_post_target() web.Result { ['/api/v1/targets/:id'; auth; delete] fn (mut app App) v1_delete_target(id int) web.Result { app.db.delete_target(id) + app.job_queue.invalidate(id) return app.json(.ok, new_response('')) } @@ -69,7 +75,10 @@ fn (mut app App) v1_patch_target(id int) web.Result { app.db.update_target_archs(id, arch_objs) } - repo := app.db.get_target(id) or { return app.status(.internal_server_error) } + target := app.db.get_target(id) or { return app.status(.internal_server_error) } - return app.json(.ok, new_data_response(repo)) + app.job_queue.invalidate(id) + app.job_queue.insert_all(target) or { return app.status(.internal_server_error) } + + return app.json(.ok, new_data_response(target)) } From 5cbfc0ebcb45e08b5d445d6e3997f7a92628a797 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 17:42:49 +0100 Subject: [PATCH 13/22] feat(agent): clean up code a bit; add frequent polling when active --- src/agent/agent.v | 6 ++-- src/agent/cli.v | 16 +++++----- src/agent/daemon.v | 78 ++++++++++++++++++++++++++++------------------ src/agent/images.v | 31 +++++++++++++----- src/agent/log.v | 2 +- 5 files changed, 83 insertions(+), 50 deletions(-) diff --git a/src/agent/agent.v b/src/agent/agent.v index 1758c85..69b9947 100644 --- a/src/agent/agent.v +++ b/src/agent/agent.v @@ -2,12 +2,12 @@ module agent import log import os +import util const log_file_name = 'vieter.agent.log' -// agent start an agent service +// agent starts an agent service pub fn agent(conf Config) ! { - // Configure logger log_level := log.level_from_tag(conf.log_level) or { return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') } @@ -16,6 +16,8 @@ pub fn agent(conf Config) ! { level: log_level } + os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') } + log_file := os.join_path_single(conf.data_dir, agent.log_file_name) logger.set_full_logpath(log_file) logger.log_to_console_too() diff --git a/src/agent/cli.v b/src/agent/cli.v index a375f08..1535e17 100644 --- a/src/agent/cli.v +++ b/src/agent/cli.v @@ -7,14 +7,12 @@ struct Config { pub: log_level string = 'WARN' // Architecture that the agent represents - arch string - api_key string - address string - data_dir string - max_concurrent_builds int = 1 - polling_frequency int = 30 - // Architecture of agent - // arch string + arch string + api_key string + address string + data_dir string + max_concurrent_builds int = 1 + polling_frequency int = 30 image_rebuild_frequency int = 1440 } @@ -22,7 +20,7 @@ pub: pub fn cmd() cli.Command { return cli.Command{ name: 'agent' - description: 'Start an agent service & start polling for new builds.' + description: 'Start an agent daemon.' execute: fn (cmd cli.Command) ! { config_file := cmd.flags.get_string('config-file')! conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! diff --git a/src/agent/daemon.v b/src/agent/daemon.v index f060863..f753e25 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -16,17 +16,17 @@ const ( struct AgentDaemon { logger shared log.Log conf Config + client client.Client mut: images ImageManager - // Which builds are currently running; length is same as - // conf.max_concurrent_builds + // Which builds are currently running; length is conf.max_concurrent_builds builds []BuildConfig - // Atomic variables used to detect when a build has finished; length is the - // same as conf.max_concurrent_builds - client client.Client + // Atomic variables used to detect when a build has finished; length is + // conf.max_concurrent_builds atomics []u64 } +// agent_init initializes a new agent fn agent_init(logger log.Log, conf Config) AgentDaemon { mut d := AgentDaemon{ logger: logger @@ -40,37 +40,49 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon { return d } +// run starts the actual agent daemon. This function will run forever. pub fn (mut d AgentDaemon) run() { // This is just so that the very first time the loop is ran, the jobs are // always polled mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency) + mut sleep_time := 1 * time.second + mut finished, mut empty := 0, 0 for { - free_builds := d.update_atomics() + finished, empty = d.update_atomics() - // All build slots are taken, so there's nothing to be done - if free_builds == 0 { + // No new finished builds and no free slots, so there's nothing to be + // done + if finished + empty == 0 { time.sleep(1 * time.second) continue } // Builds have finished, so old builder images might have freed up. - d.images.clean_old_images() + // TODO this might query the docker daemon too frequently. + if finished > 0 { + d.images.clean_old_images() + } - // Poll for new jobs - if time.now() >= last_poll_time.add_seconds(d.conf.polling_frequency) { - new_configs := d.client.poll_jobs(d.conf.arch, free_builds) or { + // The agent will always poll for new jobs after at most + // `polling_frequency` seconds. However, when jobs have finished, the + // agent will also poll for new jobs. This is because jobs are often + // clustered together (especially when mostly using the global cron + // schedule), so there's a much higher chance jobs are available. + if finished > 0 || time.now() >= last_poll_time.add_seconds(d.conf.polling_frequency) { + new_configs := d.client.poll_jobs(d.conf.arch, finished + empty) or { d.lerror('Failed to poll jobs: $err.msg()') + // TODO pick a better delay here time.sleep(5 * time.second) continue } last_poll_time = time.now() - // Schedule new jobs for config in new_configs { // TODO handle this better than to just skip the config - // Make sure a recent build base image is available for building the config + // Make sure a recent build base image is available for + // building the config d.images.refresh_image(config.base_image) or { d.lerror(err.msg()) continue @@ -78,40 +90,45 @@ pub fn (mut d AgentDaemon) run() { d.start_build(config) } - time.sleep(1 * time.second) - } - // Builds are running, so check again after one second - else if free_builds < d.conf.max_concurrent_builds { - time.sleep(1 * time.second) + // No new jobs were scheduled and the agent isn't doing anything, + // so we just wait until the next polling period. + if new_configs.len == 0 && finished + empty == d.conf.max_concurrent_builds { + sleep_time = time.now() - last_poll_time + } } // The agent is not doing anything, so we just wait until the next poll // time - else { - time_until_next_poll := time.now() - last_poll_time - time.sleep(time_until_next_poll) + else if finished + empty == d.conf.max_concurrent_builds { + sleep_time = time.now() - last_poll_time } + + time.sleep(sleep_time) } } // update_atomics checks for each build whether it's completed, and sets it to -// free again if so. The return value is how many build slots are currently -// free. -fn (mut d AgentDaemon) update_atomics() int { - mut count := 0 +// empty again if so. The return value is a tuple `(finished, empty)` where +// `finished` is how many builds were just finished and thus set to empty, and +// `empty` is how many build slots were already empty. The amount of running +// builds can then be calculated by substracting these two values from the +// total allowed concurrent builds. +fn (mut d AgentDaemon) update_atomics() (int, int) { + mut finished := 0 + mut empty := 0 for i in 0 .. d.atomics.len { if stdatomic.load_u64(&d.atomics[i]) == agent.build_done { stdatomic.store_u64(&d.atomics[i], agent.build_empty) - count++ + finished++ } else if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { - count++ + empty++ } } - return count + return finished, empty } -// start_build starts a build for the given BuildConfig object. +// start_build starts a build for the given BuildConfig. fn (mut d AgentDaemon) start_build(config BuildConfig) bool { for i in 0 .. d.atomics.len { if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { @@ -149,6 +166,7 @@ fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) { if status == 0 { d.linfo('finished build: $config.url -> $config.repo; uploading logs...') + // TODO use the arch value here build_arch := os.uname().machine d.client.add_build_log(config.target_id, res.start_time, res.end_time, build_arch, res.exit_code, res.logs) or { diff --git a/src/agent/images.v b/src/agent/images.v index 78bf2d0..64a8f74 100644 --- a/src/agent/images.v +++ b/src/agent/images.v @@ -4,29 +4,42 @@ import time import docker import build +// An ImageManager is a utility that creates builder images from given base +// images, updating these builder images if they've become too old. This +// structure can manage images from any number of base images, paving the way +// for configurable base images per target/repository. struct ImageManager { mut: - refresh_frequency int - images map[string][]string [required] - timestamps map[string]time.Time [required] + max_image_age int [required] + // For each base images, one or more builder images can exist at the same + // time + images map[string][]string [required] + // For each base image, we track when its newest image was built + timestamps map[string]time.Time [required] } -fn new_image_manager(refresh_frequency int) ImageManager { +// new_image_manager initializes a new image manager. +fn new_image_manager(max_image_age int) ImageManager { return ImageManager{ - refresh_frequency: refresh_frequency + max_image_age: max_image_age images: map[string][]string{} timestamps: map[string]time.Time{} } } +// get returns the name of the newest image for the given base image. Note that +// this function should only be called *after* a first call to `refresh_image`. pub fn (m &ImageManager) get(base_image string) string { return m.images[base_image].last() } +// refresh_image builds a new builder image from the given base image if the +// previous builder image is too old or non-existent. This function will do +// nothing if these conditions aren't met, so it's safe to call it every time +// you want to ensure an image is up to date. fn (mut m ImageManager) refresh_image(base_image string) ! { - // No need to refresh the image if the previous one is still new enough if base_image in m.timestamps - && m.timestamps[base_image].add_seconds(m.refresh_frequency) > time.now() { + && m.timestamps[base_image].add_seconds(m.max_image_age) > time.now() { return } @@ -39,7 +52,9 @@ fn (mut m ImageManager) refresh_image(base_image string) ! { m.timestamps[base_image] = time.now() } -// clean_old_images tries to remove any old but still present builder images. +// clean_old_images removes all older builder images that are no longer in use. +// The function will always leave at least one builder image, namely the newest +// one. fn (mut m ImageManager) clean_old_images() { mut dd := docker.new_conn() or { return } diff --git a/src/agent/log.v b/src/agent/log.v index d47df0f..cd59207 100644 --- a/src/agent/log.v +++ b/src/agent/log.v @@ -2,7 +2,7 @@ module agent import log -// log reate a log message with the given level +// log a message with the given level pub fn (mut d AgentDaemon) log(msg string, level log.Level) { lock d.logger { d.logger.send_output(msg, level) From 03f2240ff63e2f115e348626941d8f7919bd3e0f Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 17:51:42 +0100 Subject: [PATCH 14/22] chore: please the linter --- src/build/build.v | 3 ++- src/build/queue.v | 1 + src/build/shell_test.v | 2 -- src/client/jobs.v | 1 + src/server/api_builds.v | 4 +--- src/server/server.v | 2 ++ 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 2d51156..84d288c 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -103,6 +103,7 @@ pub: logs string } +// build_target builds the given target. Internally it calls `build_config`. pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult { config := BuildConfig{ target_id: target.id @@ -116,7 +117,7 @@ pub fn build_target(address string, api_key string, base_image_id string, target return build_config(address, api_key, config) } -// build_target builds, packages & publishes a given Arch package based on the +// build_config builds, packages & publishes a given Arch package based on the // provided target. The base image ID should be of an image previously created // by create_build_image. It returns the logs of the container. pub fn build_config(address string, api_key string, config BuildConfig) !BuildResult { diff --git a/src/build/queue.v b/src/build/queue.v index a78e56a..2a28e62 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -41,6 +41,7 @@ mut: invalidated map[int]time.Time } +// new_job_queue initializes a new job queue pub fn new_job_queue(default_schedule CronExpression, default_base_image string) BuildJobQueue { return BuildJobQueue{ default_schedule: default_schedule diff --git a/src/build/shell_test.v b/src/build/shell_test.v index d228faf..8bb22d9 100644 --- a/src/build/shell_test.v +++ b/src/build/shell_test.v @@ -1,7 +1,5 @@ module build -import models - fn test_create_build_script_git_branch() { config := BuildConfig{ target_id: 1 diff --git a/src/client/jobs.v b/src/client/jobs.v index 30f2531..7fee94f 100644 --- a/src/client/jobs.v +++ b/src/client/jobs.v @@ -2,6 +2,7 @@ module client import build { BuildConfig } +// poll_jobs requests a list of new build jobs from the server. pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig { data := c.send_request<[]BuildConfig>(.get, '/api/v1/jobs/poll', { 'arch': arch diff --git a/src/server/api_builds.v b/src/server/api_builds.v index ec3c8ec..922b252 100644 --- a/src/server/api_builds.v +++ b/src/server/api_builds.v @@ -2,10 +2,8 @@ module server import web import web.response { new_data_response, new_response } -// import os -// import util -// import models { BuildLog, BuildLogFilter } +// v1_poll_job_queue allows agents to poll for new build jobs. ['/api/v1/jobs/poll'; auth; get] fn (mut app App) v1_poll_job_queue() web.Result { arch := app.query['arch'] or { diff --git a/src/server/server.v b/src/server/server.v index e2c19c2..1e86906 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -27,6 +27,8 @@ pub mut: db db.VieterDb } +// init_job_queue populates a fresh job queue with all the targets currently +// stored in the database. fn (mut app App) init_job_queue() ! { // Initialize build queues mut targets := app.db.get_targets(limit: 25) From d3151863ee88c7fdf75d6a569be25e76511c38c1 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 18:24:21 +0100 Subject: [PATCH 15/22] refactor(build): remove some code duplication from queue --- CHANGELOG.md | 2 ++ README.md | 5 +-- src/build/queue.v | 88 ++++++++++++++++++++--------------------------- 3 files changed, 43 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2dd760..aed7571 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Migrated codebase to V 0.3.2 * Cron expression parser now uses bitfields instead of bool arrays +* Added option to deploy using agent-server architecture instead of cron daemon ### Fixed @@ -19,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * CLI no longer exits with non-zero status code when removing/patching target * Allow NULL values for branch in database +* Endpoint for adding targets now returns the correct id ## [0.4.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.4.0) diff --git a/README.md b/README.md index b9fff69..637d4c1 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ quicker. I chose [V](https://vlang.io/) as I've been very intrigued by this language for a while now. I wanted a fast language that I could code while relaxing, without having to exert too much mental effort & V seemed like the right choice for -that. +that. Sadly, this didn't quite turn out the way I expected, but I'm sticking +with it anyways ;p ## Features @@ -49,7 +50,7 @@ update`. I used to maintain a mirror that tracked the latest master, but nowadays, I maintain a Docker image containing the specific compiler version that Vieter -builds with. Currently, this is V 0.3. +builds with. Currently, this is V 0.3.2. ## Contributing diff --git a/src/build/queue.v b/src/build/queue.v index 2a28e62..dd2bb87 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -110,6 +110,21 @@ fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) ! { q.queues[arch].insert(new_job) } +// pop_invalid pops all invalid jobs. +fn (mut q BuildJobQueue) pop_invalid(arch string) { + for { + job := q.queues[arch].peek() or { return } + + if job.config.target_id in q.invalidated + && job.created < q.invalidated[job.config.target_id] { + // This pop *should* never fail according to the source code + q.queues[arch].pop() or {} + } else { + break + } + } +} + // peek shows the first job for the given architecture that's ready to be // executed, if present. pub fn (mut q BuildJobQueue) peek(arch string) ?BuildJob { @@ -118,20 +133,11 @@ pub fn (mut q BuildJobQueue) peek(arch string) ?BuildJob { return none } - for { - job := q.queues[arch].peek() or { return none } + q.pop_invalid(arch) + job := q.queues[arch].peek()? - // Skip any invalidated jobs - if job.config.target_id in q.invalidated - && job.created < q.invalidated[job.config.target_id] { - // This pop *should* never fail according to the source code - q.queues[arch].pop() or { return none } - continue - } - - if job.timestamp < time.now() { - return job - } + if job.timestamp < time.now() { + return job } } @@ -146,27 +152,18 @@ pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob { return none } - for { - mut job := q.queues[arch].peek() or { return none } + q.pop_invalid(arch) + mut job := q.queues[arch].peek()? - // Skip any invalidated jobs - if job.config.target_id in q.invalidated - && job.created < q.invalidated[job.config.target_id] { - // This pop *should* never fail according to the source code - q.queues[arch].pop() or { return none } - continue - } + if job.timestamp < time.now() { + job = q.queues[arch].pop()? - if job.timestamp < time.now() { - job = q.queues[arch].pop()? + // TODO how do we handle this properly? Is it even possible for a + // cron expression to not return a next time if it's already been + // used before? + q.reschedule(job, arch) or {} - // TODO how do we handle this properly? Is it even possible for a - // cron expression to not return a next time if it's already been - // used before? - q.reschedule(job, arch) or {} - - return job - } + return job } } @@ -182,28 +179,19 @@ pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { mut out := []BuildJob{} - outer: for out.len < n { - for { - mut job := q.queues[arch].peek() or { break outer } + for out.len < n { + q.pop_invalid(arch) + mut job := q.queues[arch].peek() or { break } - // Skip any invalidated jobs - if job.config.target_id in q.invalidated - && job.created < q.invalidated[job.config.target_id] { - // This pop *should* never fail according to the source code - q.queues[arch].pop() or { break outer } - continue - } + if job.timestamp < time.now() { + job = q.queues[arch].pop() or { break } - if job.timestamp < time.now() { - job = q.queues[arch].pop() or { break outer } + // TODO idem + q.reschedule(job, arch) or {} - // TODO idem - q.reschedule(job, arch) or {} - - out << job - } else { - break outer - } + out << job + } else { + break } } From 8a2f720bdf1702c7ffaadd27230372fb94519ceb Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 19:33:21 +0100 Subject: [PATCH 16/22] docs(agent): added agent configuration docs --- docs/content/configuration.md | 24 +++++++++++++++++++++++- docs/content/installation.md | 8 ++++---- docs/content/usage/builds/schedule.md | 2 +- src/agent/daemon.v | 2 +- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/docs/content/configuration.md b/docs/content/configuration.md index af941a2..95bf713 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -17,7 +17,7 @@ If a variable is both present in the config file & as an environment variable, the value in the environment variable is used. {{< hint info >}} -**Note** +**Note** All environment variables can also be provided from a file by appending them with `_FILE`. This for example allows you to provide the API key from a Docker secrets file. @@ -97,3 +97,25 @@ configuration variable required for each command. build`. * Default: `archlinux:base-devel` +### `vieter agent` + +* `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`, + `WARN`, `INFO` or `DEBUG`. + * Default: `WARN` +* `address`: *public* URL of the Vieter repository server to build for. From + this server jobs are retrieved. All built packages are published to this + server. +* `api_key`: API key of the above server. +* `data_dir`: directory to store log file in. +* `max_concurrent_builds`: how many builds to run at the same time. + * Default: `1` +* `polling_frequency`: how often (in seconds) to poll the server for new + builds. Note that the agent might poll more frequently when it's actively + processing builds. +* `image_rebuild_frequency`: Vieter periodically builds images that are then + used as a basis for running build containers. This is to prevent each build + from downloading an entire repository worth of dependencies. This setting + defines how frequently (in minutes) to rebuild these images. + * Default: `1440` (every 24 hours) +* `arch`: architecture for which this agent should pull down builds (e.g. + `x86_64`) diff --git a/docs/content/installation.md b/docs/content/installation.md index 87b9cba..21eda64 100644 --- a/docs/content/installation.md +++ b/docs/content/installation.md @@ -21,7 +21,7 @@ branch. This branch will be the most up to date, but does not give any guarantees about stability, so beware! Thanks to the single-binary design of Vieter, this image can be used both for -the repository server & the cron daemon. +the repository server, the cron daemon and the agent. Below is an example compose file to set up both the repository server & the cron daemon: @@ -76,7 +76,7 @@ architectures will build on both. ## Binary On the -[releases](https://git.rustybever.be/vieter/vieter/releases) +[releases](https://git.rustybever.be/vieter-v/vieter/releases) page, you can find statically compiled binaries for all released versions. This is the same binary as used inside the Docker images. @@ -106,5 +106,5 @@ guarantee that a compiler update won't temporarily break them. ## Building from source -The project [README](https://git.rustybever.be/vieter/vieter#building) contains -instructions for building Vieter from source. +The project [README](https://git.rustybever.be/vieter-v/vieter#building) +contains instructions for building Vieter from source. diff --git a/docs/content/usage/builds/schedule.md b/docs/content/usage/builds/schedule.md index 38f76a4..de59e25 100644 --- a/docs/content/usage/builds/schedule.md +++ b/docs/content/usage/builds/schedule.md @@ -37,6 +37,6 @@ Each section can consist of as many of these parts as necessary. ## CLI tool The Vieter binary contains a command that shows you the next matching times for -a given expression. This can be useful to understand the syntax. For more +a given expression. This can be useful for understanding the syntax. For more information, see [vieter-schedule(1)](https://rustybever.be/man/vieter/vieter-schedule.1.html). diff --git a/src/agent/daemon.v b/src/agent/daemon.v index f753e25..ff29d5e 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -32,7 +32,7 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon { logger: logger client: client.new(conf.address, conf.api_key) conf: conf - images: new_image_manager(conf.image_rebuild_frequency) + images: new_image_manager(conf.image_rebuild_frequency * 60) builds: []BuildConfig{len: conf.max_concurrent_builds} atomics: []u64{len: conf.max_concurrent_builds} } From f6c5e7c2469f474193270cd09264ee1fb022499c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 19:59:18 +0100 Subject: [PATCH 17/22] feat: add option to force-build package --- src/build/build.v | 4 +++- src/build/shell.v | 20 ++++++++++++++------ src/console/targets/build.v | 4 ++-- src/console/targets/targets.v | 9 ++++++++- src/cron/daemon/build.v | 2 +- vieter.toml | 2 +- 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 84d288c..6da851a 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -24,6 +24,7 @@ pub: branch string repo string base_image string + force bool } // create_build_image creates a builder image given some base image which can @@ -104,7 +105,7 @@ pub: } // build_target builds the given target. Internally it calls `build_config`. -pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult { +pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool) !BuildResult { config := BuildConfig{ target_id: target.id kind: target.kind @@ -112,6 +113,7 @@ pub fn build_target(address string, api_key string, base_image_id string, target branch: target.branch repo: target.repo base_image: base_image_id + force: force } return build_config(address, api_key, config) diff --git a/src/build/shell.v b/src/build/shell.v index c2d0c9b..ac61e07 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -63,14 +63,22 @@ fn create_build_script(address string, config BuildConfig, build_arch string) st 'cd repo', 'makepkg --nobuild --syncdeps --needed --noconfirm', 'source PKGBUILD', + ] + + if !config.force { // The build container checks whether the package is already present on // the server. - 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', - // If the above curl command succeeds, we don't need to rebuild the - // package. However, because we're in a su shell, the exit command will - // drop us back into the root shell. Therefore, we must check whether - // we're in root so we don't proceed. - '[ "\$(id -u)" == 0 ] && exit 0', + commands << [ + 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + // If the above curl command succeeds, we don't need to rebuild the + // package. However, because we're in a su shell, the exit command will + // drop us back into the root shell. Therefore, we must check whether + // we're in root so we don't proceed. + '[ "\$(id -u)" == 0 ] && exit 0', + ] + } + + commands << [ 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', ] diff --git a/src/console/targets/build.v b/src/console/targets/build.v index 9368558..e18077d 100644 --- a/src/console/targets/build.v +++ b/src/console/targets/build.v @@ -6,7 +6,7 @@ import os import build // build locally builds the target with the given id. -fn build(conf Config, target_id int) ! { +fn build(conf Config, target_id int, force bool) ! { c := client.new(conf.address, conf.api_key) target := c.get_target(target_id)! @@ -16,7 +16,7 @@ fn build(conf Config, target_id int) ! { image_id := build.create_build_image(conf.base_image)! println('Running build...') - res := build.build_target(conf.address, conf.api_key, image_id, target)! + res := build.build_target(conf.address, conf.api_key, image_id, target, force)! println('Removing build image...') diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 4179363..ffcd36c 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -182,11 +182,18 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Build the target with the given id & publish it.' + flags: [ + cli.Flag{ + name: 'force' + description: 'Build the target without checking whether it needs to be renewed.' + flag: cli.FlagType.bool + }, + ] execute: fn (cmd cli.Command) ! { config_file := cmd.flags.get_string('config-file')! conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - build(conf, cmd.args[0].int())! + build(conf, cmd.args[0].int(), cmd.flags.get_bool('force')!)! } }, ] diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index beed9fc..42edc92 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -79,7 +79,7 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { mut status := 0 res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(), - &sb.target) or { + &sb.target, false) or { d.ldebug('build_target error: $err.msg()') status = 1 diff --git a/vieter.toml b/vieter.toml index 9a68ae3..74a7397 100644 --- a/vieter.toml +++ b/vieter.toml @@ -8,7 +8,7 @@ arch = "x86_64" address = "http://localhost:8000" -global_schedule = '* *' +# global_schedule = '* *' api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 From 6a208dbe6ca73b25e1e0e30f3b3b266620061ebc Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 21:22:22 +0100 Subject: [PATCH 18/22] feat: allow queueing one-time builds --- src/build/queue.v | 46 ++++++++++++++++++++++++++--------------- src/server/api_builds.v | 21 +++++++++++++++++++ src/server/server.v | 2 +- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/src/build/queue.v b/src/build/queue.v index dd2bb87..1395a0b 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -16,6 +16,8 @@ pub: ce CronExpression // Actual build config sent to the agent config BuildConfig + // Whether this is a one-time job + single bool } // Allows BuildJob structs to be sorted according to their timestamp in @@ -53,22 +55,29 @@ pub fn new_job_queue(default_schedule CronExpression, default_base_image string) // insert_all executes insert for each architecture of the given Target. pub fn (mut q BuildJobQueue) insert_all(target Target) ! { for arch in target.arch { - q.insert(target, arch.value)! + q.insert(target: target, arch: arch.value)! } } +[params] +pub struct InsertConfig { + target Target [required] + arch string [required] + single bool +} + // insert a new target's job into the queue for the given architecture. This // job will then be endlessly rescheduled after being pop'ed, unless removed // explicitely. -pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { +pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! { lock q.mutex { - if arch !in q.queues { - q.queues[arch] = MinHeap{} + if input.arch !in q.queues { + q.queues[input.arch] = MinHeap{} } - ce := if target.schedule != '' { - parse_expression(target.schedule) or { - return error("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()") + ce := if input.target.schedule != '' { + parse_expression(input.target.schedule) or { + return error("Error while parsing cron expression '$input.target.schedule' (id $input.target.id): $err.msg()") } } else { q.default_schedule @@ -80,18 +89,19 @@ pub fn (mut q BuildJobQueue) insert(target Target, arch string) ! { created: time.now() timestamp: timestamp ce: ce + single: input.single config: BuildConfig{ - target_id: target.id - kind: target.kind - url: target.url - branch: target.branch - repo: target.repo + target_id: input.target.id + kind: input.target.kind + url: input.target.url + branch: input.target.branch + repo: input.target.repo // TODO make this configurable base_image: q.default_base_image } } - q.queues[arch].insert(job) + q.queues[input.arch].insert(job) } } @@ -158,10 +168,12 @@ pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob { if job.timestamp < time.now() { job = q.queues[arch].pop()? - // TODO how do we handle this properly? Is it even possible for a - // cron expression to not return a next time if it's already been - // used before? - q.reschedule(job, arch) or {} + if !job.single { + // TODO how do we handle this properly? Is it even possible for a + // cron expression to not return a next time if it's already been + // used before? + q.reschedule(job, arch) or {} + } return job } diff --git a/src/server/api_builds.v b/src/server/api_builds.v index 922b252..bc841ce 100644 --- a/src/server/api_builds.v +++ b/src/server/api_builds.v @@ -19,3 +19,24 @@ fn (mut app App) v1_poll_job_queue() web.Result { return app.json(.ok, new_data_response(out)) } + +['/api/v1/jobs/queue'; auth; post] +fn (mut app App) v1_queue_job() web.Result { + target_id := app.query['target'] or { + return app.json(.bad_request, new_response('Missing target query arg.')) + }.int() + + arch := app.query['arch'] or { + return app.json(.bad_request, new_response('Missing arch query arg.')) + } + + target := app.db.get_target(target_id) or { + return app.json(.bad_request, new_response('Unknown target id.')) + } + + app.job_queue.insert(target: target, arch: arch, single: true) or { + return app.status(.internal_server_error) + } + + return app.status(.ok) +} diff --git a/src/server/server.v b/src/server/server.v index 1e86906..6d18f09 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -37,7 +37,7 @@ fn (mut app App) init_job_queue() ! { for targets.len > 0 { for target in targets { for arch in target.arch { - app.job_queue.insert(target, arch.value)! + app.job_queue.insert(target: target, arch: arch.value)! } } From 2cc3e8404e98329bb8b0dfd5dda39393b0abc2da Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Dec 2022 22:03:04 +0100 Subject: [PATCH 19/22] feat: queue one-time builds from CLI --- src/build/queue.v | 40 +++++++++++++++---------- src/client/jobs.v | 11 +++++++ src/console/targets/targets.v | 28 ++++++++++++++++- src/server/{api_builds.v => api_jobs.v} | 8 ++++- 4 files changed, 69 insertions(+), 18 deletions(-) rename src/server/{api_builds.v => api_jobs.v} (83%) diff --git a/src/build/queue.v b/src/build/queue.v index 1395a0b..5d50f34 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -7,7 +7,7 @@ import datatypes { MinHeap } import util struct BuildJob { -pub: +pub mut: // Time at which this build job was created/queued created time.Time // Next timestamp from which point this job is allowed to be executed @@ -64,6 +64,8 @@ pub struct InsertConfig { target Target [required] arch string [required] single bool + force bool + now bool } // insert a new target's job into the queue for the given architecture. This @@ -75,20 +77,8 @@ pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! { q.queues[input.arch] = MinHeap{} } - ce := if input.target.schedule != '' { - parse_expression(input.target.schedule) or { - return error("Error while parsing cron expression '$input.target.schedule' (id $input.target.id): $err.msg()") - } - } else { - q.default_schedule - } - - timestamp := ce.next_from_now()! - - job := BuildJob{ + mut job := BuildJob{ created: time.now() - timestamp: timestamp - ce: ce single: input.single config: BuildConfig{ target_id: input.target.id @@ -98,9 +88,25 @@ pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! { repo: input.target.repo // TODO make this configurable base_image: q.default_base_image + force: input.force } } + if !input.now { + ce := if input.target.schedule != '' { + parse_expression(input.target.schedule) or { + return error("Error while parsing cron expression '$input.target.schedule' (id $input.target.id): $err.msg()") + } + } else { + q.default_schedule + } + + job.timestamp = ce.next_from_now()! + job.ce = ce + } else { + job.timestamp = time.now() + } + q.queues[input.arch].insert(job) } } @@ -198,8 +204,10 @@ pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { if job.timestamp < time.now() { job = q.queues[arch].pop() or { break } - // TODO idem - q.reschedule(job, arch) or {} + if !job.single { + // TODO idem + q.reschedule(job, arch) or {} + } out << job } else { diff --git a/src/client/jobs.v b/src/client/jobs.v index 7fee94f..2d8e99b 100644 --- a/src/client/jobs.v +++ b/src/client/jobs.v @@ -1,6 +1,7 @@ module client import build { BuildConfig } +import web.response { Response } // poll_jobs requests a list of new build jobs from the server. pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig { @@ -11,3 +12,13 @@ pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig { return data.data } + +pub fn (c &Client) queue_job(target_id int, arch string, force bool) !Response { + data := c.send_request(.post, '/api/v1/jobs/queue', { + 'target': target_id.str() + 'arch': arch + 'force': force.str() + })! + + return data +} diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index ffcd36c..b527896 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -188,12 +188,38 @@ pub fn cmd() cli.Command { description: 'Build the target without checking whether it needs to be renewed.' flag: cli.FlagType.bool }, + cli.Flag{ + name: 'remote' + description: 'Schedule the build on the server instead of running it locally.' + flag: cli.FlagType.bool + }, + cli.Flag{ + name: 'arch' + description: 'Architecture to schedule build for. Required when using -remote.' + flag: cli.FlagType.string + }, ] execute: fn (cmd cli.Command) ! { config_file := cmd.flags.get_string('config-file')! conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - build(conf, cmd.args[0].int(), cmd.flags.get_bool('force')!)! + remote := cmd.flags.get_bool('remote')! + force := cmd.flags.get_bool('force')! + target_id := cmd.args[0].int() + + if remote { + arch := cmd.flags.get_string('arch')! + + if arch == '' { + return error('When scheduling the build remotely, you have to specify an architecture.') + } + + c := client.new(conf.address, conf.api_key) + res := c.queue_job(target_id, arch, force)! + println(res.message) + } else { + build(conf, target_id, force)! + } } }, ] diff --git a/src/server/api_builds.v b/src/server/api_jobs.v similarity index 83% rename from src/server/api_builds.v rename to src/server/api_jobs.v index bc841ce..b75e70e 100644 --- a/src/server/api_builds.v +++ b/src/server/api_jobs.v @@ -30,11 +30,17 @@ fn (mut app App) v1_queue_job() web.Result { return app.json(.bad_request, new_response('Missing arch query arg.')) } + if arch == '' { + app.json(.bad_request, new_response('Empty arch query arg.')) + } + + force := 'force' in app.query + target := app.db.get_target(target_id) or { return app.json(.bad_request, new_response('Unknown target id.')) } - app.job_queue.insert(target: target, arch: arch, single: true) or { + app.job_queue.insert(target: target, arch: arch, single: true, now: true, force: force) or { return app.status(.internal_server_error) } From d7a04c6ebff26a95d8680d3fb233bf370943a646 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 14 Dec 2022 16:03:57 +0100 Subject: [PATCH 20/22] chore: please the great lint --- CHANGELOG.md | 4 ++++ src/agent/images.v | 4 ++-- src/client/jobs.v | 2 ++ src/server/api_jobs.v | 1 + 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aed7571..c55e16b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Migrated codebase to V 0.3.2 * Cron expression parser now uses bitfields instead of bool arrays * Added option to deploy using agent-server architecture instead of cron daemon +* Allow force-building packages, meaning the build won't check if the + repository is already up to date +* Allow scheduling builds on the server from the CLI tool instead of building + them locally ### Fixed diff --git a/src/agent/images.v b/src/agent/images.v index 64a8f74..185192e 100644 --- a/src/agent/images.v +++ b/src/agent/images.v @@ -9,9 +9,9 @@ import build // structure can manage images from any number of base images, paving the way // for configurable base images per target/repository. struct ImageManager { -mut: max_image_age int [required] - // For each base images, one or more builder images can exist at the same +mut: + // For each base image, one or more builder images can exist at the same // time images map[string][]string [required] // For each base image, we track when its newest image was built diff --git a/src/client/jobs.v b/src/client/jobs.v index 2d8e99b..440affa 100644 --- a/src/client/jobs.v +++ b/src/client/jobs.v @@ -13,6 +13,8 @@ pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig { return data.data } +// queue_job adds a new one-time build job for the given target to the job +// queue. pub fn (c &Client) queue_job(target_id int, arch string, force bool) !Response { data := c.send_request(.post, '/api/v1/jobs/queue', { 'target': target_id.str() diff --git a/src/server/api_jobs.v b/src/server/api_jobs.v index b75e70e..7795351 100644 --- a/src/server/api_jobs.v +++ b/src/server/api_jobs.v @@ -20,6 +20,7 @@ fn (mut app App) v1_poll_job_queue() web.Result { return app.json(.ok, new_data_response(out)) } +// v1_queue_job allows queueing a new one-time build job for the given target. ['/api/v1/jobs/queue'; auth; post] fn (mut app App) v1_queue_job() web.Result { target_id := app.query['target'] or { From 51df1874f5b2d31b88ed68a02a910ac091b53af3 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 14 Dec 2022 16:33:50 +0100 Subject: [PATCH 21/22] agent: some better logging --- src/agent/daemon.v | 10 ++++------ src/build/build.v | 5 +++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/agent/daemon.v b/src/agent/daemon.v index ff29d5e..0647733 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -146,7 +146,7 @@ fn (mut d AgentDaemon) start_build(config BuildConfig) bool { // run_build actually starts the build process for a given target. fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) { - d.linfo('started build: $config.url -> $config.repo') + d.linfo('started build: $config') // 0 means success, 1 means failure mut status := 0 @@ -164,16 +164,14 @@ fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) { } if status == 0 { - d.linfo('finished build: $config.url -> $config.repo; uploading logs...') + d.linfo('Uploading build logs for $config') // TODO use the arch value here build_arch := os.uname().machine d.client.add_build_log(config.target_id, res.start_time, res.end_time, build_arch, - res.exit_code, res.logs) or { - d.lerror('Failed to upload logs for build: $config.url -> $config.repo') - } + res.exit_code, res.logs) or { d.lerror('Failed to upload logs for $config') } } else { - d.linfo('an error occured during build: $config.url -> $config.repo') + d.lwarn('an error occurred during build: $config') } stdatomic.store_u64(&d.atomics[build_index], agent.build_done) diff --git a/src/build/build.v b/src/build/build.v index 6da851a..3d916bf 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -27,6 +27,11 @@ pub: force bool } +// str return a single-line string representation of a build log +pub fn (c BuildConfig) str() string { + return '{ target: $c.target_id, kind: $c.kind, url: $c.url, branch: $c.branch, repo: $c.repo, base_image: $c.base_image, force: $c.force }' +} + // create_build_image creates a builder image given some base image which can // then be used to build & package Arch images. It mostly just updates the // system, install some necessary packages & creates a non-root user to run From 60cb91c18cf1cf7ea66835d1b8e73cb66ff05ab3 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 14 Dec 2022 17:23:51 +0100 Subject: [PATCH 22/22] chore: final read before merging --- src/build/queue.v | 4 +++- src/server/server.v | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/build/queue.v b/src/build/queue.v index 5d50f34..7902173 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -144,7 +144,9 @@ fn (mut q BuildJobQueue) pop_invalid(arch string) { // peek shows the first job for the given architecture that's ready to be // executed, if present. pub fn (mut q BuildJobQueue) peek(arch string) ?BuildJob { - rlock q.mutex { + // Even peek requires a write lock, because pop_invalid can modify the data + // structure + lock q.mutex { if arch !in q.queues { return none } diff --git a/src/server/server.v b/src/server/server.v index 6d18f09..74b1f37 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -36,9 +36,7 @@ fn (mut app App) init_job_queue() ! { for targets.len > 0 { for target in targets { - for arch in target.arch { - app.job_queue.insert(target: target, arch: arch.value)! - } + app.job_queue.insert_all(target)! } i += 25