From 3a20d6d47004576270b9f72b52dc4a66020535a2 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Fri, 10 Mar 2023 12:04:51 +0000 Subject: [PATCH 1/7] chore(deps): update dependency middleman to v4.4.3 --- docs/api/Gemfile.lock | 52 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/api/Gemfile.lock b/docs/api/Gemfile.lock index bd60f7f..f45b4a8 100644 --- a/docs/api/Gemfile.lock +++ b/docs/api/Gemfile.lock @@ -1,14 +1,14 @@ GEM remote: https://rubygems.org/ specs: - activesupport (6.1.4.1) + activesupport (6.1.7.2) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) zeitwerk (~> 2.3) - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) + addressable (2.8.1) + public_suffix (>= 2.0.2, < 6.0) autoprefixer-rails (10.2.5.0) execjs (< 2.8.0) backports (3.21.0) @@ -16,14 +16,14 @@ GEM coffee-script-source execjs coffee-script-source (1.12.2) - concurrent-ruby (1.1.9) + concurrent-ruby (1.1.10) contracts (0.13.0) dotenv (2.7.6) erubis (2.7.0) execjs (2.7.0) fast_blank (1.0.1) - fastimage (2.2.5) - ffi (1.15.4) + fastimage (2.2.6) + ffi (1.15.5) haml (5.2.2) temple (>= 0.8.0) tilt @@ -32,29 +32,29 @@ GEM hashie (3.6.0) i18n (1.6.0) concurrent-ruby (~> 1.0) - kramdown (2.3.1) + kramdown (2.3.2) rexml listen (3.0.8) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) memoist (0.16.2) - middleman (4.4.2) + middleman (4.4.3) coffee-script (~> 2.2) - haml (>= 4.0.5) + haml (>= 4.0.5, < 6.0) kramdown (>= 2.3.0) - middleman-cli (= 4.4.2) - middleman-core (= 4.4.2) + middleman-cli (= 4.4.3) + middleman-core (= 4.4.3) middleman-autoprefixer (3.0.0) autoprefixer-rails (~> 10.0) middleman-core (>= 4.0.0) - middleman-cli (4.4.2) + middleman-cli (4.4.3) thor (>= 0.17.0, < 2.0) - middleman-core (4.4.2) - activesupport (>= 6.1, < 7.0) + middleman-core (4.4.3) + activesupport (>= 6.1, < 7.1) addressable (~> 2.4) backports (~> 3.6) bundler (~> 2.0) - contracts (~> 0.13.0) + contracts (~> 0.13) dotenv erubis execjs (~> 2.0) @@ -63,7 +63,7 @@ GEM hamster (~> 3.0) hashie (~> 3.4) i18n (~> 1.6.0) - listen (~> 3.0.0) + listen (~> 3.0) memoist (~> 0.14) padrino-helpers (~> 0.15.0) parallel @@ -85,17 +85,17 @@ GEM nokogiri (1.13.4) mini_portile2 (~> 2.8.0) racc (~> 1.4) - padrino-helpers (0.15.1) + padrino-helpers (0.15.3) i18n (>= 0.6.7, < 2) - padrino-support (= 0.15.1) + padrino-support (= 0.15.3) tilt (>= 1.4.1, < 3) - padrino-support (0.15.1) + padrino-support (0.15.3) parallel (1.21.0) parslet (2.0.0) - public_suffix (4.0.6) + public_suffix (4.0.7) racc (1.6.0) - rack (2.2.3) - rb-fsevent (0.11.0) + rack (2.2.6.3) + rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) redcarpet (3.5.1) @@ -114,15 +114,15 @@ GEM rack (> 1, < 3) temple (0.8.2) thor (1.1.0) - tilt (2.0.10) + tilt (2.0.11) toml (0.3.0) parslet (>= 1.8.0, < 3.0.0) - tzinfo (2.0.4) + tzinfo (2.0.6) concurrent-ruby (~> 1.0) uglifier (3.2.0) execjs (>= 0.3.0, < 3) webrick (1.7.0) - zeitwerk (2.5.1) + zeitwerk (2.5.4) PLATFORMS ruby @@ -142,4 +142,4 @@ RUBY VERSION ruby 2.7.2p137 BUNDLED WITH - 2.2.22 + 2.4.8 From 094634084b741cce218364c8a5d5a28b5aaa9434 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 5 Apr 2023 10:50:30 +0200 Subject: [PATCH 2/7] feat(agent): use worker thread approach --- src/agent/daemon.v | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/agent/daemon.v b/src/agent/daemon.v index 4364a92..d49b45e 100644 --- a/src/agent/daemon.v +++ b/src/agent/daemon.v @@ -20,11 +20,13 @@ struct AgentDaemon { client client.Client mut: images ImageManager - // Which builds are currently running; length is conf.max_concurrent_builds - builds []BuildConfig // Atomic variables used to detect when a build has finished; length is - // conf.max_concurrent_builds + // conf.max_concurrent_builds. This approach is used as the difference + // between a recently finished build and an empty build slot is important + // for knowing whether the agent is currently "active". atomics []u64 + // Channel used to send builds to worker threads + build_channel chan BuildConfig } // agent_init initializes a new agent @@ -34,8 +36,8 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon { client: client.new(conf.address, conf.api_key) conf: conf images: new_image_manager(conf.image_rebuild_frequency * 60) - builds: []BuildConfig{len: conf.max_concurrent_builds} atomics: []u64{len: conf.max_concurrent_builds} + build_channel: chan BuildConfig{cap: conf.max_concurrent_builds} } return d @@ -43,6 +45,11 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon { // run starts the actual agent daemon. This function will run forever. pub fn (mut d AgentDaemon) run() { + // Spawn worker threads + for builder_index in 0 .. d.conf.max_concurrent_builds { + spawn d.builder_thread(d.build_channel, builder_index) + } + // This is just so that the very first time the loop is ran, the jobs are // always polled mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency) @@ -107,10 +114,10 @@ pub fn (mut d AgentDaemon) run() { // It's technically still possible that the build image is // removed in the very short period between building the // builder image and starting a build container with it. If - // this happens, faith really just didn't want you to do this + // this happens, fate really just didn't want you to do this // build. - d.start_build(config) + d.build_channel <- config running++ } } @@ -147,22 +154,6 @@ fn (mut d AgentDaemon) update_atomics() (int, int) { return finished, empty } -// start_build starts a build for the given BuildConfig. -fn (mut d AgentDaemon) start_build(config BuildConfig) bool { - for i in 0 .. d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { - stdatomic.store_u64(&d.atomics[i], agent.build_running) - d.builds[i] = config - - spawn d.run_build(i, config) - - return true - } - } - - return false -} - // run_build actually starts the build process for a given target. fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) { d.linfo('started build: ${config}') @@ -195,3 +186,12 @@ fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) { stdatomic.store_u64(&d.atomics[build_index], agent.build_done) } + +// builder_thread is a thread that constantly listens for builds to process +fn (mut d AgentDaemon) builder_thread(ch chan BuildConfig, builder_index int) { + for { + build_config := <-ch or { break } + + d.run_build(builder_index, build_config) + } +} From 7595eb7bbea77479d59fdbdf5782e3ca34b5b7c9 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 27 Apr 2023 13:23:58 +0200 Subject: [PATCH 3/7] fix: error when upload failed before all bytes received --- src/server/repo.v | 2 +- src/util/stream.v | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/server/repo.v b/src/server/repo.v index 051a7c2..8f8270d 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -68,7 +68,7 @@ fn (mut app App) put_package(repo_ string) web.Result { mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) util.reader_to_file(mut app.reader, length.int(), pkg_path) or { - app.lwarn("Failed to upload '${pkg_path}'") + app.lwarn("Failed to upload '${pkg_path}': ${err.msg()}") return app.status(.internal_server_error) } diff --git a/src/util/stream.v b/src/util/stream.v index 4b362fc..ef6e872 100644 --- a/src/util/stream.v +++ b/src/util/stream.v @@ -46,6 +46,10 @@ pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ! { to_write = to_write - bytes_written } } + + if bytes_left > 0 { + return error('Not all bytes were received.') + } } // match_array_in_array[T] returns how many elements of a2 overlap with a1. For From ac3a89500bbbbf9a4b95278913b61e4490ff5558 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 2 May 2023 11:46:19 +0200 Subject: [PATCH 4/7] feat: add non-functional build timeout setting --- src/build/build.v | 4 ++-- src/build/queue.v | 7 +++++-- src/console/targets/build.v | 4 ++-- src/console/targets/targets.v | 9 ++++++++- src/models/builds.v | 3 ++- src/models/targets.v | 3 ++- src/server/cli.v | 23 ++++++++++++----------- src/server/server.v | 2 +- 8 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index c69a613..756e8f6 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -94,8 +94,8 @@ pub: } // build_target builds the given target. Internally it calls `build_config`. -pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool) !BuildResult { - config := target.as_build_config(base_image_id, force) +pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool, timeout int) !BuildResult { + config := target.as_build_config(base_image_id, force, timeout) return build_config(address, api_key, config) } diff --git a/src/build/queue.v b/src/build/queue.v index 73068ac..bc4db9d 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -33,6 +33,8 @@ pub struct BuildJobQueue { default_schedule &cron.Expression // Base image to use for targets without defined base image default_base_image string + // After how many minutes a build should be forcefully cancelled + default_build_timeout int mut: mutex shared util.Dummy // For each architecture, a priority queue is tracked @@ -44,10 +46,11 @@ mut: } // new_job_queue initializes a new job queue -pub fn new_job_queue(default_schedule &cron.Expression, default_base_image string) BuildJobQueue { +pub fn new_job_queue(default_schedule &cron.Expression, default_base_image string, default_build_timeout int) BuildJobQueue { return BuildJobQueue{ default_schedule: unsafe { default_schedule } default_base_image: default_base_image + default_build_timeout: default_build_timeout invalidated: map[int]time.Time{} } } @@ -80,7 +83,7 @@ pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! { mut job := BuildJob{ created: time.now() single: input.single - config: input.target.as_build_config(q.default_base_image, input.force) + config: input.target.as_build_config(q.default_base_image, input.force, q.default_build_timeout) } if !input.now { diff --git a/src/console/targets/build.v b/src/console/targets/build.v index a59e6a1..93464af 100644 --- a/src/console/targets/build.v +++ b/src/console/targets/build.v @@ -6,7 +6,7 @@ import os import build // build locally builds the target with the given id. -fn build_target(conf Config, target_id int, force bool) ! { +fn build_target(conf Config, target_id int, force bool, timeout int) ! { c := client.new(conf.address, conf.api_key) target := c.get_target(target_id)! @@ -16,7 +16,7 @@ fn build_target(conf Config, target_id int, force bool) ! { image_id := build.create_build_image(conf.base_image)! println('Running build...') - res := build.build_target(conf.address, conf.api_key, image_id, target, force)! + res := build.build_target(conf.address, conf.api_key, image_id, target, force, timeout)! println('Removing build image...') diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 676fa0a..80fc36b 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -232,6 +232,12 @@ pub fn cmd() cli.Command { description: 'Architecture to schedule build for. Required when using -remote.' flag: cli.FlagType.string }, + cli.Flag{ + name: 'timeout' + description: 'After how many minutes to cancel the build. Only applies to local builds.' + flag: cli.FlagType.int + default_value: ['60'] + }, ] execute: fn (cmd cli.Command) ! { config_file := cmd.flags.get_string('config-file')! @@ -239,6 +245,7 @@ pub fn cmd() cli.Command { remote := cmd.flags.get_bool('remote')! force := cmd.flags.get_bool('force')! + timeout := cmd.flags.get_int('timeout')! target_id := cmd.args[0].int() if remote { @@ -251,7 +258,7 @@ pub fn cmd() cli.Command { c := client.new(conf_.address, conf_.api_key) c.queue_job(target_id, arch, force)! } else { - build_target(conf_, target_id, force)! + build_target(conf_, target_id, force, timeout)! } } }, diff --git a/src/models/builds.v b/src/models/builds.v index be2910c..6923115 100644 --- a/src/models/builds.v +++ b/src/models/builds.v @@ -10,9 +10,10 @@ pub: repo string base_image string force bool + timeout int } // str return a single-line string representation of a build log pub fn (c BuildConfig) str() string { - return '{ target: ${c.target_id}, kind: ${c.kind}, url: ${c.url}, branch: ${c.branch}, path: ${c.path}, repo: ${c.repo}, base_image: ${c.base_image}, force: ${c.force} }' + return '{ target: ${c.target_id}, kind: ${c.kind}, url: ${c.url}, branch: ${c.branch}, path: ${c.path}, repo: ${c.repo}, base_image: ${c.base_image}, force: ${c.force}, timeout: ${c.timeout} }' } diff --git a/src/models/targets.v b/src/models/targets.v index 3c0c9cf..14cc8a6 100644 --- a/src/models/targets.v +++ b/src/models/targets.v @@ -54,7 +54,7 @@ pub fn (t &Target) str() string { // as_build_config converts a Target into a BuildConfig, given some extra // needed information. -pub fn (t &Target) as_build_config(base_image string, force bool) BuildConfig { +pub fn (t &Target) as_build_config(base_image string, force bool, timeout int) BuildConfig { return BuildConfig{ target_id: t.id kind: t.kind @@ -64,6 +64,7 @@ pub fn (t &Target) as_build_config(base_image string, force bool) BuildConfig { repo: t.repo base_image: base_image force: force + timeout: timeout } } diff --git a/src/server/cli.v b/src/server/cli.v index 08ad5f8..c24812d 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -5,17 +5,18 @@ import conf as vconf struct Config { pub: - port int = 8000 - log_level string = 'WARN' - pkg_dir string - data_dir string - api_key string - default_arch string - global_schedule string = '0 3' - base_image string = 'archlinux:base-devel' - max_log_age int [empty_default] - log_removal_schedule string = '0 0' - collect_metrics bool [empty_default] + port int = 8000 + log_level string = 'WARN' + pkg_dir string + data_dir string + api_key string + default_arch string + global_schedule string = '0 3' + base_image string = 'archlinux:base-devel' + max_log_age int [empty_default] + log_removal_schedule string = '0 0' + collect_metrics bool [empty_default] + default_build_timeout int = 60 } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/server.v b/src/server/server.v index 4cccb27..e1516fa 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -108,7 +108,7 @@ pub fn server(conf Config) ! { repo: repo_ db: db collector: collector - job_queue: build.new_job_queue(global_ce, conf.base_image) + job_queue: build.new_job_queue(global_ce, conf.base_image, conf.default_build_timeout) } app.init_job_queue() or { util.exit_with_message(1, 'Failed to inialize job queue: ${err.msg()}') From afb38256ac7ab48900582b13becefcc2df99d79d Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 2 May 2023 14:49:32 +0200 Subject: [PATCH 5/7] feat: implement build timeout --- src/build/build.v | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/build/build.v b/src/build/build.v index 756e8f6..b864792 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -136,9 +136,17 @@ pub fn build_config(address string, api_key string, config BuildConfig) !BuildRe dd.container_start(id)! mut data := dd.container_inspect(id)! + start_time := time.now() // This loop waits until the container has stopped, so we can remove it after for data.state.running { + if time.now() - start_time > config.timeout * time.second { + dd.container_kill(id)! + dd.container_remove(id)! + + return error('Build killed due to timeout (${config.timeout}s)') + } + time.sleep(1 * time.second) data = dd.container_inspect(id)! From b278ebd73fe2a5096acd5e2a164b325a4b981035 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 2 May 2023 14:51:49 +0200 Subject: [PATCH 6/7] fix: set default timeout to 60 minutes --- src/console/targets/targets.v | 2 +- src/server/cli.v | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 80fc36b..f85c4c0 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -236,7 +236,7 @@ pub fn cmd() cli.Command { name: 'timeout' description: 'After how many minutes to cancel the build. Only applies to local builds.' flag: cli.FlagType.int - default_value: ['60'] + default_value: ['3600'] }, ] execute: fn (cmd cli.Command) ! { diff --git a/src/server/cli.v b/src/server/cli.v index c24812d..abb5fe3 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -16,7 +16,7 @@ pub: max_log_age int [empty_default] log_removal_schedule string = '0 0' collect_metrics bool [empty_default] - default_build_timeout int = 60 + default_build_timeout int = 3600 } // cmd returns the cli submodule that handles starting the server From 1fff407dfe81b7f3eb5a8c4be58bdf765b10eac5 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Wed, 3 May 2023 19:03:39 +0000 Subject: [PATCH 7/7] chore(deps): update dependency middleman to v4.5.0 --- docs/api/Gemfile.lock | 72 +++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/docs/api/Gemfile.lock b/docs/api/Gemfile.lock index bd60f7f..145183d 100644 --- a/docs/api/Gemfile.lock +++ b/docs/api/Gemfile.lock @@ -1,29 +1,29 @@ GEM remote: https://rubygems.org/ specs: - activesupport (6.1.4.1) + activesupport (6.1.7.3) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) zeitwerk (~> 2.3) - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) + addressable (2.8.4) + public_suffix (>= 2.0.2, < 6.0) autoprefixer-rails (10.2.5.0) execjs (< 2.8.0) - backports (3.21.0) + backports (3.24.1) coffee-script (2.4.1) coffee-script-source execjs coffee-script-source (1.12.2) - concurrent-ruby (1.1.9) - contracts (0.13.0) - dotenv (2.7.6) + concurrent-ruby (1.2.2) + contracts (0.17) + dotenv (2.8.1) erubis (2.7.0) execjs (2.7.0) fast_blank (1.0.1) - fastimage (2.2.5) - ffi (1.15.4) + fastimage (2.2.6) + ffi (1.15.5) haml (5.2.2) temple (>= 0.8.0) tilt @@ -32,29 +32,29 @@ GEM hashie (3.6.0) i18n (1.6.0) concurrent-ruby (~> 1.0) - kramdown (2.3.1) + kramdown (2.4.0) rexml - listen (3.0.8) - rb-fsevent (~> 0.9, >= 0.9.4) - rb-inotify (~> 0.9, >= 0.9.7) + listen (3.8.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) memoist (0.16.2) - middleman (4.4.2) + middleman (4.5.0) coffee-script (~> 2.2) haml (>= 4.0.5) kramdown (>= 2.3.0) - middleman-cli (= 4.4.2) - middleman-core (= 4.4.2) + middleman-cli (= 4.5.0) + middleman-core (= 4.5.0) middleman-autoprefixer (3.0.0) autoprefixer-rails (~> 10.0) middleman-core (>= 4.0.0) - middleman-cli (4.4.2) + middleman-cli (4.5.0) thor (>= 0.17.0, < 2.0) - middleman-core (4.4.2) - activesupport (>= 6.1, < 7.0) + middleman-core (4.5.0) + activesupport (>= 6.1, < 7.1) addressable (~> 2.4) backports (~> 3.6) bundler (~> 2.0) - contracts (~> 0.13.0) + contracts (~> 0.13) dotenv erubis execjs (~> 2.0) @@ -63,7 +63,7 @@ GEM hamster (~> 3.0) hashie (~> 3.4) i18n (~> 1.6.0) - listen (~> 3.0.0) + listen (~> 3.0) memoist (~> 0.14) padrino-helpers (~> 0.15.0) parallel @@ -81,21 +81,21 @@ GEM middleman-core (>= 3.2) rouge (~> 3.2) mini_portile2 (2.8.0) - minitest (5.14.4) + minitest (5.18.0) nokogiri (1.13.4) mini_portile2 (~> 2.8.0) racc (~> 1.4) - padrino-helpers (0.15.1) + padrino-helpers (0.15.3) i18n (>= 0.6.7, < 2) - padrino-support (= 0.15.1) + padrino-support (= 0.15.3) tilt (>= 1.4.1, < 3) - padrino-support (0.15.1) - parallel (1.21.0) + padrino-support (0.15.3) + parallel (1.23.0) parslet (2.0.0) - public_suffix (4.0.6) + public_suffix (4.0.7) racc (1.6.0) - rack (2.2.3) - rb-fsevent (0.11.0) + rack (2.2.7) + rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) redcarpet (3.5.1) @@ -112,17 +112,17 @@ GEM sprockets (3.7.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) - temple (0.8.2) - thor (1.1.0) - tilt (2.0.10) + temple (0.10.0) + thor (1.2.1) + tilt (2.0.11) toml (0.3.0) parslet (>= 1.8.0, < 3.0.0) - tzinfo (2.0.4) + tzinfo (2.0.6) concurrent-ruby (~> 1.0) uglifier (3.2.0) execjs (>= 0.3.0, < 3) - webrick (1.7.0) - zeitwerk (2.5.1) + webrick (1.8.1) + zeitwerk (2.6.8) PLATFORMS ruby @@ -142,4 +142,4 @@ RUBY VERSION ruby 2.7.2p137 BUNDLED WITH - 2.2.22 + 2.4.12