Compare commits
9 Commits
3a20d6d470
...
1fff407dfe
Author | SHA1 | Date |
---|---|---|
Renovate Bot | 1fff407dfe | |
Jef Roosens | de8764b281 | |
Jef Roosens | b278ebd73f | |
Jef Roosens | afb38256ac | |
Jef Roosens | ac3a89500b | |
Jef Roosens | 8a76860363 | |
Jef Roosens | 7595eb7bbe | |
Jef Roosens | 37f368b769 | |
Jef Roosens | 094634084b |
|
@ -1,24 +1,24 @@
|
|||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
activesupport (6.1.7.2)
|
||||
activesupport (6.1.7.3)
|
||||
concurrent-ruby (~> 1.0, >= 1.0.2)
|
||||
i18n (>= 1.6, < 2)
|
||||
minitest (>= 5.1)
|
||||
tzinfo (~> 2.0)
|
||||
zeitwerk (~> 2.3)
|
||||
addressable (2.8.1)
|
||||
addressable (2.8.4)
|
||||
public_suffix (>= 2.0.2, < 6.0)
|
||||
autoprefixer-rails (10.2.5.0)
|
||||
execjs (< 2.8.0)
|
||||
backports (3.21.0)
|
||||
backports (3.24.1)
|
||||
coffee-script (2.4.1)
|
||||
coffee-script-source
|
||||
execjs
|
||||
coffee-script-source (1.12.2)
|
||||
concurrent-ruby (1.1.10)
|
||||
contracts (0.13.0)
|
||||
dotenv (2.7.6)
|
||||
concurrent-ruby (1.2.2)
|
||||
contracts (0.17)
|
||||
dotenv (2.8.1)
|
||||
erubis (2.7.0)
|
||||
execjs (2.7.0)
|
||||
fast_blank (1.0.1)
|
||||
|
@ -32,24 +32,24 @@ GEM
|
|||
hashie (3.6.0)
|
||||
i18n (1.6.0)
|
||||
concurrent-ruby (~> 1.0)
|
||||
kramdown (2.3.2)
|
||||
kramdown (2.4.0)
|
||||
rexml
|
||||
listen (3.0.8)
|
||||
rb-fsevent (~> 0.9, >= 0.9.4)
|
||||
rb-inotify (~> 0.9, >= 0.9.7)
|
||||
listen (3.8.0)
|
||||
rb-fsevent (~> 0.10, >= 0.10.3)
|
||||
rb-inotify (~> 0.9, >= 0.9.10)
|
||||
memoist (0.16.2)
|
||||
middleman (4.4.3)
|
||||
middleman (4.5.0)
|
||||
coffee-script (~> 2.2)
|
||||
haml (>= 4.0.5, < 6.0)
|
||||
haml (>= 4.0.5)
|
||||
kramdown (>= 2.3.0)
|
||||
middleman-cli (= 4.4.3)
|
||||
middleman-core (= 4.4.3)
|
||||
middleman-cli (= 4.5.0)
|
||||
middleman-core (= 4.5.0)
|
||||
middleman-autoprefixer (3.0.0)
|
||||
autoprefixer-rails (~> 10.0)
|
||||
middleman-core (>= 4.0.0)
|
||||
middleman-cli (4.4.3)
|
||||
middleman-cli (4.5.0)
|
||||
thor (>= 0.17.0, < 2.0)
|
||||
middleman-core (4.4.3)
|
||||
middleman-core (4.5.0)
|
||||
activesupport (>= 6.1, < 7.1)
|
||||
addressable (~> 2.4)
|
||||
backports (~> 3.6)
|
||||
|
@ -81,7 +81,7 @@ GEM
|
|||
middleman-core (>= 3.2)
|
||||
rouge (~> 3.2)
|
||||
mini_portile2 (2.8.0)
|
||||
minitest (5.14.4)
|
||||
minitest (5.18.0)
|
||||
nokogiri (1.13.4)
|
||||
mini_portile2 (~> 2.8.0)
|
||||
racc (~> 1.4)
|
||||
|
@ -90,11 +90,11 @@ GEM
|
|||
padrino-support (= 0.15.3)
|
||||
tilt (>= 1.4.1, < 3)
|
||||
padrino-support (0.15.3)
|
||||
parallel (1.21.0)
|
||||
parallel (1.23.0)
|
||||
parslet (2.0.0)
|
||||
public_suffix (4.0.7)
|
||||
racc (1.6.0)
|
||||
rack (2.2.6.3)
|
||||
rack (2.2.7)
|
||||
rb-fsevent (0.11.2)
|
||||
rb-inotify (0.10.1)
|
||||
ffi (~> 1.0)
|
||||
|
@ -112,8 +112,8 @@ GEM
|
|||
sprockets (3.7.2)
|
||||
concurrent-ruby (~> 1.0)
|
||||
rack (> 1, < 3)
|
||||
temple (0.8.2)
|
||||
thor (1.1.0)
|
||||
temple (0.10.0)
|
||||
thor (1.2.1)
|
||||
tilt (2.0.11)
|
||||
toml (0.3.0)
|
||||
parslet (>= 1.8.0, < 3.0.0)
|
||||
|
@ -121,8 +121,8 @@ GEM
|
|||
concurrent-ruby (~> 1.0)
|
||||
uglifier (3.2.0)
|
||||
execjs (>= 0.3.0, < 3)
|
||||
webrick (1.7.0)
|
||||
zeitwerk (2.5.4)
|
||||
webrick (1.8.1)
|
||||
zeitwerk (2.6.8)
|
||||
|
||||
PLATFORMS
|
||||
ruby
|
||||
|
@ -142,4 +142,4 @@ RUBY VERSION
|
|||
ruby 2.7.2p137
|
||||
|
||||
BUNDLED WITH
|
||||
2.4.8
|
||||
2.4.12
|
||||
|
|
|
@ -20,11 +20,13 @@ struct AgentDaemon {
|
|||
client client.Client
|
||||
mut:
|
||||
images ImageManager
|
||||
// Which builds are currently running; length is conf.max_concurrent_builds
|
||||
builds []BuildConfig
|
||||
// Atomic variables used to detect when a build has finished; length is
|
||||
// conf.max_concurrent_builds
|
||||
// conf.max_concurrent_builds. This approach is used as the difference
|
||||
// between a recently finished build and an empty build slot is important
|
||||
// for knowing whether the agent is currently "active".
|
||||
atomics []u64
|
||||
// Channel used to send builds to worker threads
|
||||
build_channel chan BuildConfig
|
||||
}
|
||||
|
||||
// agent_init initializes a new agent
|
||||
|
@ -34,8 +36,8 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon {
|
|||
client: client.new(conf.address, conf.api_key)
|
||||
conf: conf
|
||||
images: new_image_manager(conf.image_rebuild_frequency * 60)
|
||||
builds: []BuildConfig{len: conf.max_concurrent_builds}
|
||||
atomics: []u64{len: conf.max_concurrent_builds}
|
||||
build_channel: chan BuildConfig{cap: conf.max_concurrent_builds}
|
||||
}
|
||||
|
||||
return d
|
||||
|
@ -43,6 +45,11 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon {
|
|||
|
||||
// run starts the actual agent daemon. This function will run forever.
|
||||
pub fn (mut d AgentDaemon) run() {
|
||||
// Spawn worker threads
|
||||
for builder_index in 0 .. d.conf.max_concurrent_builds {
|
||||
spawn d.builder_thread(d.build_channel, builder_index)
|
||||
}
|
||||
|
||||
// This is just so that the very first time the loop is ran, the jobs are
|
||||
// always polled
|
||||
mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency)
|
||||
|
@ -107,10 +114,10 @@ pub fn (mut d AgentDaemon) run() {
|
|||
// It's technically still possible that the build image is
|
||||
// removed in the very short period between building the
|
||||
// builder image and starting a build container with it. If
|
||||
// this happens, faith really just didn't want you to do this
|
||||
// this happens, fate really just didn't want you to do this
|
||||
// build.
|
||||
|
||||
d.start_build(config)
|
||||
d.build_channel <- config
|
||||
running++
|
||||
}
|
||||
}
|
||||
|
@ -147,22 +154,6 @@ fn (mut d AgentDaemon) update_atomics() (int, int) {
|
|||
return finished, empty
|
||||
}
|
||||
|
||||
// start_build starts a build for the given BuildConfig.
|
||||
fn (mut d AgentDaemon) start_build(config BuildConfig) bool {
|
||||
for i in 0 .. d.atomics.len {
|
||||
if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty {
|
||||
stdatomic.store_u64(&d.atomics[i], agent.build_running)
|
||||
d.builds[i] = config
|
||||
|
||||
spawn d.run_build(i, config)
|
||||
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// run_build actually starts the build process for a given target.
|
||||
fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) {
|
||||
d.linfo('started build: ${config}')
|
||||
|
@ -195,3 +186,12 @@ fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) {
|
|||
|
||||
stdatomic.store_u64(&d.atomics[build_index], agent.build_done)
|
||||
}
|
||||
|
||||
// builder_thread is a thread that constantly listens for builds to process
|
||||
fn (mut d AgentDaemon) builder_thread(ch chan BuildConfig, builder_index int) {
|
||||
for {
|
||||
build_config := <-ch or { break }
|
||||
|
||||
d.run_build(builder_index, build_config)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -94,8 +94,8 @@ pub:
|
|||
}
|
||||
|
||||
// build_target builds the given target. Internally it calls `build_config`.
|
||||
pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool) !BuildResult {
|
||||
config := target.as_build_config(base_image_id, force)
|
||||
pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool, timeout int) !BuildResult {
|
||||
config := target.as_build_config(base_image_id, force, timeout)
|
||||
|
||||
return build_config(address, api_key, config)
|
||||
}
|
||||
|
@ -136,9 +136,17 @@ pub fn build_config(address string, api_key string, config BuildConfig) !BuildRe
|
|||
dd.container_start(id)!
|
||||
|
||||
mut data := dd.container_inspect(id)!
|
||||
start_time := time.now()
|
||||
|
||||
// This loop waits until the container has stopped, so we can remove it after
|
||||
for data.state.running {
|
||||
if time.now() - start_time > config.timeout * time.second {
|
||||
dd.container_kill(id)!
|
||||
dd.container_remove(id)!
|
||||
|
||||
return error('Build killed due to timeout (${config.timeout}s)')
|
||||
}
|
||||
|
||||
time.sleep(1 * time.second)
|
||||
|
||||
data = dd.container_inspect(id)!
|
||||
|
|
|
@ -33,6 +33,8 @@ pub struct BuildJobQueue {
|
|||
default_schedule &cron.Expression
|
||||
// Base image to use for targets without defined base image
|
||||
default_base_image string
|
||||
// After how many minutes a build should be forcefully cancelled
|
||||
default_build_timeout int
|
||||
mut:
|
||||
mutex shared util.Dummy
|
||||
// For each architecture, a priority queue is tracked
|
||||
|
@ -44,10 +46,11 @@ mut:
|
|||
}
|
||||
|
||||
// new_job_queue initializes a new job queue
|
||||
pub fn new_job_queue(default_schedule &cron.Expression, default_base_image string) BuildJobQueue {
|
||||
pub fn new_job_queue(default_schedule &cron.Expression, default_base_image string, default_build_timeout int) BuildJobQueue {
|
||||
return BuildJobQueue{
|
||||
default_schedule: unsafe { default_schedule }
|
||||
default_base_image: default_base_image
|
||||
default_build_timeout: default_build_timeout
|
||||
invalidated: map[int]time.Time{}
|
||||
}
|
||||
}
|
||||
|
@ -80,7 +83,7 @@ pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! {
|
|||
mut job := BuildJob{
|
||||
created: time.now()
|
||||
single: input.single
|
||||
config: input.target.as_build_config(q.default_base_image, input.force)
|
||||
config: input.target.as_build_config(q.default_base_image, input.force, q.default_build_timeout)
|
||||
}
|
||||
|
||||
if !input.now {
|
||||
|
|
|
@ -6,7 +6,7 @@ import os
|
|||
import build
|
||||
|
||||
// build locally builds the target with the given id.
|
||||
fn build_target(conf Config, target_id int, force bool) ! {
|
||||
fn build_target(conf Config, target_id int, force bool, timeout int) ! {
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
target := c.get_target(target_id)!
|
||||
|
||||
|
@ -16,7 +16,7 @@ fn build_target(conf Config, target_id int, force bool) ! {
|
|||
image_id := build.create_build_image(conf.base_image)!
|
||||
|
||||
println('Running build...')
|
||||
res := build.build_target(conf.address, conf.api_key, image_id, target, force)!
|
||||
res := build.build_target(conf.address, conf.api_key, image_id, target, force, timeout)!
|
||||
|
||||
println('Removing build image...')
|
||||
|
||||
|
|
|
@ -232,6 +232,12 @@ pub fn cmd() cli.Command {
|
|||
description: 'Architecture to schedule build for. Required when using -remote.'
|
||||
flag: cli.FlagType.string
|
||||
},
|
||||
cli.Flag{
|
||||
name: 'timeout'
|
||||
description: 'After how many minutes to cancel the build. Only applies to local builds.'
|
||||
flag: cli.FlagType.int
|
||||
default_value: ['3600']
|
||||
},
|
||||
]
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
|
@ -239,6 +245,7 @@ pub fn cmd() cli.Command {
|
|||
|
||||
remote := cmd.flags.get_bool('remote')!
|
||||
force := cmd.flags.get_bool('force')!
|
||||
timeout := cmd.flags.get_int('timeout')!
|
||||
target_id := cmd.args[0].int()
|
||||
|
||||
if remote {
|
||||
|
@ -251,7 +258,7 @@ pub fn cmd() cli.Command {
|
|||
c := client.new(conf_.address, conf_.api_key)
|
||||
c.queue_job(target_id, arch, force)!
|
||||
} else {
|
||||
build_target(conf_, target_id, force)!
|
||||
build_target(conf_, target_id, force, timeout)!
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
@ -10,9 +10,10 @@ pub:
|
|||
repo string
|
||||
base_image string
|
||||
force bool
|
||||
timeout int
|
||||
}
|
||||
|
||||
// str return a single-line string representation of a build log
|
||||
pub fn (c BuildConfig) str() string {
|
||||
return '{ target: ${c.target_id}, kind: ${c.kind}, url: ${c.url}, branch: ${c.branch}, path: ${c.path}, repo: ${c.repo}, base_image: ${c.base_image}, force: ${c.force} }'
|
||||
return '{ target: ${c.target_id}, kind: ${c.kind}, url: ${c.url}, branch: ${c.branch}, path: ${c.path}, repo: ${c.repo}, base_image: ${c.base_image}, force: ${c.force}, timeout: ${c.timeout} }'
|
||||
}
|
||||
|
|
|
@ -54,7 +54,7 @@ pub fn (t &Target) str() string {
|
|||
|
||||
// as_build_config converts a Target into a BuildConfig, given some extra
|
||||
// needed information.
|
||||
pub fn (t &Target) as_build_config(base_image string, force bool) BuildConfig {
|
||||
pub fn (t &Target) as_build_config(base_image string, force bool, timeout int) BuildConfig {
|
||||
return BuildConfig{
|
||||
target_id: t.id
|
||||
kind: t.kind
|
||||
|
@ -64,6 +64,7 @@ pub fn (t &Target) as_build_config(base_image string, force bool) BuildConfig {
|
|||
repo: t.repo
|
||||
base_image: base_image
|
||||
force: force
|
||||
timeout: timeout
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -5,17 +5,18 @@ import conf as vconf
|
|||
|
||||
struct Config {
|
||||
pub:
|
||||
port int = 8000
|
||||
log_level string = 'WARN'
|
||||
pkg_dir string
|
||||
data_dir string
|
||||
api_key string
|
||||
default_arch string
|
||||
global_schedule string = '0 3'
|
||||
base_image string = 'archlinux:base-devel'
|
||||
max_log_age int [empty_default]
|
||||
log_removal_schedule string = '0 0'
|
||||
collect_metrics bool [empty_default]
|
||||
port int = 8000
|
||||
log_level string = 'WARN'
|
||||
pkg_dir string
|
||||
data_dir string
|
||||
api_key string
|
||||
default_arch string
|
||||
global_schedule string = '0 3'
|
||||
base_image string = 'archlinux:base-devel'
|
||||
max_log_age int [empty_default]
|
||||
log_removal_schedule string = '0 0'
|
||||
collect_metrics bool [empty_default]
|
||||
default_build_timeout int = 3600
|
||||
}
|
||||
|
||||
// cmd returns the cli submodule that handles starting the server
|
||||
|
|
|
@ -68,7 +68,7 @@ fn (mut app App) put_package(repo_ string) web.Result {
|
|||
mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true })
|
||||
|
||||
util.reader_to_file(mut app.reader, length.int(), pkg_path) or {
|
||||
app.lwarn("Failed to upload '${pkg_path}'")
|
||||
app.lwarn("Failed to upload '${pkg_path}': ${err.msg()}")
|
||||
|
||||
return app.status(.internal_server_error)
|
||||
}
|
||||
|
|
|
@ -108,7 +108,7 @@ pub fn server(conf Config) ! {
|
|||
repo: repo_
|
||||
db: db
|
||||
collector: collector
|
||||
job_queue: build.new_job_queue(global_ce, conf.base_image)
|
||||
job_queue: build.new_job_queue(global_ce, conf.base_image, conf.default_build_timeout)
|
||||
}
|
||||
app.init_job_queue() or {
|
||||
util.exit_with_message(1, 'Failed to inialize job queue: ${err.msg()}')
|
||||
|
|
|
@ -46,6 +46,10 @@ pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ! {
|
|||
to_write = to_write - bytes_written
|
||||
}
|
||||
}
|
||||
|
||||
if bytes_left > 0 {
|
||||
return error('Not all bytes were received.')
|
||||
}
|
||||
}
|
||||
|
||||
// match_array_in_array[T] returns how many elements of a2 overlap with a1. For
|
||||
|
|
Loading…
Reference in New Issue