From f7e1aba30bc8d95c28632ace658998804f60763f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 13 Apr 2022 16:12:22 +0200 Subject: [PATCH] Attempt at writing renew_queue function; seems to just stop in the middle --- src/cron/cli.v | 16 ++--- src/cron/cron.v | 12 +++- src/cron/daemon/daemon.v | 72 +++++++++++++++---- src/cron/{ => expression}/expression.v | 6 +- .../{ => expression}/expression_parse_test.v | 2 +- src/cron/{ => expression}/expression_test.v | 2 +- src/env/env.v | 2 +- src/git/git.v | 4 +- src/v.mod | 0 vieter.toml | 3 + 10 files changed, 89 insertions(+), 30 deletions(-) rename src/cron/{ => expression}/expression.v (98%) rename src/cron/{ => expression}/expression_parse_test.v (99%) rename src/cron/{ => expression}/expression_test.v (97%) create mode 100644 src/v.mod diff --git a/src/cron/cli.v b/src/cron/cli.v index 4d2b133..f4b20ec 100644 --- a/src/cron/cli.v +++ b/src/cron/cli.v @@ -5,14 +5,14 @@ import env struct Config { pub: - log_level string = 'WARN' - log_file string = 'vieter.log' - api_key string - address string - base_image string = 'archlinux:base-devel' - max_concurrent_builds int = 1 - api_update_frequency int = 60 - global_schedule string + log_level string = 'WARN' + log_file string = 'vieter.log' + api_key string + address string + base_image string = 'archlinux:base-devel' + max_concurrent_builds int = 1 + api_update_frequency int = 60 + global_schedule string } // cmd returns the cli module that handles the cron daemon. diff --git a/src/cron/cron.v b/src/cron/cron.v index 3d3ea9a..d8b4d95 100644 --- a/src/cron/cron.v +++ b/src/cron/cron.v @@ -5,12 +5,13 @@ import time import log import util import cron.daemon +import cron.expression // cron starts a cron daemon & starts periodically scheduling builds. pub fn cron(conf Config) ? { // Configure logger log_level := log.level_from_tag(conf.log_level) or { - util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') } mut logger := log.Log{ @@ -20,5 +21,12 @@ pub fn cron(conf Config) ? { logger.set_full_logpath(conf.log_file) logger.log_to_console_too() - d := daemon.init(conf) + ce := expression.parse_expression(conf.global_schedule) or { + return error('Error while parsing global cron expression: $err.msg()') + } + + mut d := daemon.init_daemon(logger, conf.address, conf.api_key, conf.base_image, ce, + conf.max_concurrent_builds, conf.api_update_frequency) ? + + d.run() ? } diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index a887717..ede9320 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -3,9 +3,12 @@ module daemon import git import time import log -import datatypes +import datatypes { MinHeap } +import cron.expression { CronExpression, parse_expression } struct ScheduledBuild { +pub: + repo_id string repo git.GitRepo timestamp time.Time } @@ -16,39 +19,84 @@ fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool { pub struct Daemon { mut: - conf Config + address string + api_key string + base_image string + global_schedule CronExpression + api_update_frequency int // Repos currently loaded from API. repos_map map[string]git.GitRepo // At what point to update the list of repositories. api_update_timestamp time.Time - queue datatypes.MinHeap + queue MinHeap // Which builds are currently running builds []git.GitRepo // Atomic variables used to detect when a build has finished; length is the // same as builds atomics []u64 - logger shared log.Log + logger shared log.Log } -// init -pub fn init(conf Config) Daemon { - return Daemon{ - conf: conf - atomics: [conf.max_concurrent_builds]u64{} +pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int) ?Daemon { + mut d := Daemon{ + address: address + api_key: api_key + base_image: base_image + global_schedule: global_schedule + api_update_frequency: api_update_frequency + atomics: []u64{len: max_concurrent_builds} + builds: []git.GitRepo{len: max_concurrent_builds} + logger: logger } -} -fn (mut d Daemon) run() ? { + // Initialize the repos & queue d.renew_repos() ? d.renew_queue() ? + + return d +} + +pub fn (mut d Daemon) run() ? { + println(d.queue) } fn (mut d Daemon) renew_repos() ? { - mut new_repos := git.get_repos(d.conf.address, d.conf.api_key) ? + mut new_repos := git.get_repos(d.address, d.api_key) ? d.repos_map = new_repos.move() + + d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency) } +// renew_queue replaces the old queue with a new one that reflects the newest +// values in repos_map. fn (mut d Daemon) renew_queue() ? { + mut new_queue := MinHeap{} + // Move any jobs that should have already started from the old queue onto + // the new one + now := time.now() + + for d.queue.len() > 0 && d.queue.peek() ?.timestamp < now { + new_queue.insert(d.queue.pop() ?) + } + + println('hey') + println(d.repos_map) + // For each repository in repos_map, parse their cron expression (or use + // the default one if not present) & add them to the queue + for id, repo in d.repos_map { + println('hey') + ce := parse_expression(repo.schedule) or { d.global_schedule } + // A repo that can't be scheduled will just be skipped for now + timestamp := ce.next(now) or { continue } + + new_queue.insert(ScheduledBuild{ + repo_id: id + repo: repo + timestamp: timestamp + }) + } + + d.queue = new_queue } diff --git a/src/cron/expression.v b/src/cron/expression/expression.v similarity index 98% rename from src/cron/expression.v rename to src/cron/expression/expression.v index b35c568..c122585 100644 --- a/src/cron/expression.v +++ b/src/cron/expression/expression.v @@ -1,8 +1,8 @@ -module cron +module expression import time -struct CronExpression { +pub struct CronExpression { minutes []int hours []int days []int @@ -219,7 +219,7 @@ fn parse_part(s string, min int, max int) ?[]int { // parse_expression parses an entire cron expression string into a // CronExpression object, if possible. -fn parse_expression(exp string) ?CronExpression { +pub fn parse_expression(exp string) ?CronExpression { // The filter allows for multiple spaces between parts mut parts := exp.split(' ').filter(it != '') diff --git a/src/cron/expression_parse_test.v b/src/cron/expression/expression_parse_test.v similarity index 99% rename from src/cron/expression_parse_test.v rename to src/cron/expression/expression_parse_test.v index 8f3ac38..18531c0 100644 --- a/src/cron/expression_parse_test.v +++ b/src/cron/expression/expression_parse_test.v @@ -1,4 +1,4 @@ -module cron +module expression // parse_range_error returns the returned error message. If the result is '', // that means the function didn't error. diff --git a/src/cron/expression_test.v b/src/cron/expression/expression_test.v similarity index 97% rename from src/cron/expression_test.v rename to src/cron/expression/expression_test.v index 0be9a64..ef0283a 100644 --- a/src/cron/expression_test.v +++ b/src/cron/expression/expression_test.v @@ -1,4 +1,4 @@ -module cron +module expression import time { parse } diff --git a/src/env/env.v b/src/env/env.v index 0124850..88f1650 100644 --- a/src/env/env.v +++ b/src/env/env.v @@ -58,7 +58,7 @@ pub fn load(path string) ?T { if s !is toml.Null { $if field.typ is string { res.$(field.name) = s.string() - }$else $if field.typ is int { + } $else $if field.typ is int { res.$(field.name) = s.int() } } diff --git a/src/git/git.v b/src/git/git.v index 45aed60..2023f34 100644 --- a/src/git/git.v +++ b/src/git/git.v @@ -15,7 +15,7 @@ pub mut: // Which repo the builder should publish packages to repo string // Cron schedule describing how frequently to build the repo. - schedule string + schedule string [optional] } // patch_from_params patches a GitRepo from a map[string]string, usually @@ -74,7 +74,7 @@ pub fn repo_from_params(params map[string]string) ?GitRepo { // If we're creating a new GitRepo, we want all fields to be present before // "patching". $for field in GitRepo.fields { - if field.name !in params { + if field.name !in params && !field.attrs.contains('optional') { return error('Missing parameter: ${field.name}.') } } diff --git a/src/v.mod b/src/v.mod new file mode 100644 index 0000000..e69de29 diff --git a/vieter.toml b/vieter.toml index 8e0447b..e646739 100644 --- a/vieter.toml +++ b/vieter.toml @@ -8,3 +8,6 @@ repos_file = "data/repos.json" default_arch = "x86_64" address = "http://localhost:8000" + +global_schedule = '0 3' +