Compare commits

...

12 Commits
dev ... dev

Author SHA1 Message Date
Jef Roosens 1a992806fa
fix: update PKGBUILD to use libvieter
ci/woodpecker/tag/lint Pipeline was successful Details
ci/woodpecker/tag/docs Pipeline was successful Details
ci/woodpecker/tag/arch-rel Pipeline was successful Details
ci/woodpecker/tag/build Pipeline was successful Details
ci/woodpecker/tag/gitea Pipeline was successful Details
ci/woodpecker/tag/test Pipeline was successful Details
ci/woodpecker/tag/man Pipeline was successful Details
ci/woodpecker/tag/docker Pipeline was successful Details
ci/woodpecker/push/lint Pipeline was successful Details
ci/woodpecker/push/arch Pipeline was successful Details
ci/woodpecker/push/build Pipeline was successful Details
ci/woodpecker/push/test Pipeline was successful Details
ci/woodpecker/push/man Pipeline was successful Details
ci/woodpecker/push/docker Pipeline was successful Details
ci/woodpecker/push/deploy Pipeline was successful Details
ci/woodpecker/push/docs Pipeline failed Details
2023-07-17 13:53:20 +02:00
Jef Roosens 22577d3411 Merge pull request 'Release 0.6.0' (#360) from release-0.6.0 into main
ci/woodpecker/push/build Pipeline was successful Details
ci/woodpecker/push/docker Pipeline was successful Details
ci/woodpecker/tag/build Pipeline was successful Details
ci/woodpecker/tag/lint Pipeline failed Details
ci/woodpecker/tag/test Pipeline failed Details
ci/woodpecker/tag/docs Pipeline failed Details
ci/woodpecker/tag/arch-rel Pipeline failed Details
ci/woodpecker/tag/docker unknown status Details
ci/woodpecker/tag/gitea unknown status Details
ci/woodpecker/tag/man unknown status Details
Reviewed-on: vieter-v/vieter#360
2023-07-17 13:35:03 +02:00
Jef Roosens 47c0f0405b
chore: update versions to 0.6.0
ci/woodpecker/push/lint Pipeline was successful Details
ci/woodpecker/push/docs Pipeline was successful Details
ci/woodpecker/pr/build Pipeline failed Details
ci/woodpecker/pr/docker unknown status Details
ci/woodpecker/push/build Pipeline was successful Details
ci/woodpecker/push/test Pipeline was successful Details
ci/woodpecker/push/man Pipeline was successful Details
2023-07-17 13:26:38 +02:00
Jef Roosens 076ee24b1b
chore: update changelog
ci/woodpecker/push/arch Pipeline is pending Details
ci/woodpecker/push/build Pipeline is pending Details
ci/woodpecker/push/deploy Pipeline is pending Details
ci/woodpecker/push/docker Pipeline is pending Details
ci/woodpecker/push/docs Pipeline is pending Details
ci/woodpecker/push/lint Pipeline is pending Details
ci/woodpecker/push/man Pipeline is pending Details
ci/woodpecker/push/test Pipeline is pending Details
2023-05-04 09:36:03 +02:00
Jef Roosens de8764b281 Merge pull request 'Implement global build timeout' (#358) from Chewing_Bever/vieter:build-timeout into dev
ci/woodpecker/push/lint Pipeline was successful Details
ci/woodpecker/push/arch Pipeline was successful Details
ci/woodpecker/push/build Pipeline was successful Details
ci/woodpecker/push/test Pipeline was successful Details
ci/woodpecker/push/man Pipeline was successful Details
ci/woodpecker/push/docker Pipeline was successful Details
ci/woodpecker/push/deploy Pipeline was successful Details
ci/woodpecker/push/docs Pipeline failed Details
Reviewed-on: vieter-v/vieter#358
2023-05-02 21:41:26 +02:00
Jef Roosens b278ebd73f
fix: set default timeout to 60 minutes
ci/woodpecker/pr/lint Pipeline was successful Details
ci/woodpecker/pr/docs Pipeline was successful Details
ci/woodpecker/pr/build Pipeline was successful Details
ci/woodpecker/pr/docker Pipeline was successful Details
ci/woodpecker/pr/man Pipeline was successful Details
ci/woodpecker/pr/test Pipeline was successful Details
2023-05-02 21:31:24 +02:00
Jef Roosens afb38256ac
feat: implement build timeout 2023-05-02 14:52:41 +02:00
Jef Roosens ac3a89500b
feat: add non-functional build timeout setting 2023-05-02 14:52:40 +02:00
Jef Roosens 8a76860363 Merge pull request 'Error when upload fails before all bytes are received' (#357) from Chewing_Bever/vieter:dev into dev
ci/woodpecker/push/docs Pipeline was successful Details
ci/woodpecker/push/lint Pipeline was successful Details
ci/woodpecker/push/arch Pipeline was successful Details
ci/woodpecker/push/build Pipeline was successful Details
ci/woodpecker/push/test Pipeline was successful Details
ci/woodpecker/push/man Pipeline was successful Details
ci/woodpecker/push/docker Pipeline was successful Details
ci/woodpecker/push/deploy Pipeline was successful Details
Reviewed-on: vieter-v/vieter#357
2023-04-28 12:44:22 +02:00
Jef Roosens 7595eb7bbe fix: error when upload failed before all bytes received
ci/woodpecker/pr/docs Pipeline was successful Details
ci/woodpecker/pr/lint Pipeline was successful Details
ci/woodpecker/pr/build Pipeline was successful Details
ci/woodpecker/pr/docker Pipeline was successful Details
ci/woodpecker/pr/man Pipeline was successful Details
ci/woodpecker/pr/test Pipeline was successful Details
2023-04-28 12:05:12 +02:00
Jef Roosens 37f368b769 Merge pull request 'Agent worker threads' (#355) from Chewing_Bever/vieter:agent-threads into dev
ci/woodpecker/push/lint Pipeline was successful Details
ci/woodpecker/push/arch Pipeline was successful Details
ci/woodpecker/push/build Pipeline was successful Details
ci/woodpecker/push/test Pipeline was successful Details
ci/woodpecker/push/man Pipeline was successful Details
ci/woodpecker/push/docker Pipeline was successful Details
ci/woodpecker/push/deploy Pipeline was successful Details
ci/woodpecker/push/docs Pipeline failed Details
Reviewed-on: vieter-v/vieter#355
2023-04-05 11:20:43 +02:00
Jef Roosens 094634084b
feat(agent): use worker thread approach
ci/woodpecker/pr/lint Pipeline was successful Details
ci/woodpecker/pr/docs Pipeline was successful Details
ci/woodpecker/pr/build Pipeline was successful Details
ci/woodpecker/pr/docker Pipeline was successful Details
ci/woodpecker/pr/man Pipeline was successful Details
ci/woodpecker/pr/test Pipeline was successful Details
2023-04-05 10:50:30 +02:00
14 changed files with 95 additions and 50 deletions

View File

@ -7,16 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev) ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
## [0.6.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.6.0)
### Added ### Added
* Metrics endpoint for Prometheus integration * Metrics endpoint for Prometheus integration
* Search in list of targets using API & CLI * Search in list of targets using API & CLI
* Allow filtering targets by arch value * Allow filtering targets by arch value
* Configurable global timeout for builds
### Changed ### Changed
* Rewrote cron expression logic in C * Rewrote cron expression logic in C
* Updated codebase to V commit after 0.3.3 * Updated codebase to V commit after 0.3.3
* Agents now use worker threads and no longer spawn a new thread for every
build
### Fixed
* Package upload now fails if TCP connection is closed before all bytes have
been received
### Removed ### Removed

View File

@ -3,7 +3,7 @@
pkgbase='vieter' pkgbase='vieter'
pkgname='vieter' pkgname='vieter'
pkgver='0.5.0' pkgver='0.6.0'
pkgrel=1 pkgrel=1
pkgdesc="Lightweight Arch repository server & package build system" pkgdesc="Lightweight Arch repository server & package build system"
depends=('glibc' 'openssl' 'libarchive' 'sqlite') depends=('glibc' 'openssl' 'libarchive' 'sqlite')
@ -11,13 +11,23 @@ makedepends=('git' 'vieter-vlang')
arch=('x86_64' 'aarch64') arch=('x86_64' 'aarch64')
url='https://git.rustybever.be/vieter-v/vieter' url='https://git.rustybever.be/vieter-v/vieter'
license=('AGPL3') license=('AGPL3')
source=("$pkgname::git+https://git.rustybever.be/vieter-v/vieter#tag=${pkgver//_/-}") source=(
md5sums=('SKIP') "$pkgname::git+https://git.rustybever.be/vieter-v/vieter#tag=${pkgver//_/-}"
"libvieter::git+https://git.rustybever.be/vieter-v/libvieter"
)
md5sums=('SKIP' 'SKIP')
prepare() { prepare() {
export VMODULES="$srcdir/.vmodules" cd "${pkgname}"
cd "$pkgname/src" && v install # Add the libvieter submodule
git submodule init
git config submodules.src/libvieter.url "${srcdir}/libvieter"
git -c protocol.file.allow=always submodule update
export VMODULES="${srcdir}/.vmodules"
cd src && v install
} }
build() { build() {

View File

@ -20,11 +20,13 @@ struct AgentDaemon {
client client.Client client client.Client
mut: mut:
images ImageManager images ImageManager
// Which builds are currently running; length is conf.max_concurrent_builds
builds []BuildConfig
// Atomic variables used to detect when a build has finished; length is // Atomic variables used to detect when a build has finished; length is
// conf.max_concurrent_builds // conf.max_concurrent_builds. This approach is used as the difference
// between a recently finished build and an empty build slot is important
// for knowing whether the agent is currently "active".
atomics []u64 atomics []u64
// Channel used to send builds to worker threads
build_channel chan BuildConfig
} }
// agent_init initializes a new agent // agent_init initializes a new agent
@ -34,8 +36,8 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon {
client: client.new(conf.address, conf.api_key) client: client.new(conf.address, conf.api_key)
conf: conf conf: conf
images: new_image_manager(conf.image_rebuild_frequency * 60) images: new_image_manager(conf.image_rebuild_frequency * 60)
builds: []BuildConfig{len: conf.max_concurrent_builds}
atomics: []u64{len: conf.max_concurrent_builds} atomics: []u64{len: conf.max_concurrent_builds}
build_channel: chan BuildConfig{cap: conf.max_concurrent_builds}
} }
return d return d
@ -43,6 +45,11 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon {
// run starts the actual agent daemon. This function will run forever. // run starts the actual agent daemon. This function will run forever.
pub fn (mut d AgentDaemon) run() { pub fn (mut d AgentDaemon) run() {
// Spawn worker threads
for builder_index in 0 .. d.conf.max_concurrent_builds {
spawn d.builder_thread(d.build_channel, builder_index)
}
// This is just so that the very first time the loop is ran, the jobs are // This is just so that the very first time the loop is ran, the jobs are
// always polled // always polled
mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency) mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency)
@ -107,10 +114,10 @@ pub fn (mut d AgentDaemon) run() {
// It's technically still possible that the build image is // It's technically still possible that the build image is
// removed in the very short period between building the // removed in the very short period between building the
// builder image and starting a build container with it. If // builder image and starting a build container with it. If
// this happens, faith really just didn't want you to do this // this happens, fate really just didn't want you to do this
// build. // build.
d.start_build(config) d.build_channel <- config
running++ running++
} }
} }
@ -147,22 +154,6 @@ fn (mut d AgentDaemon) update_atomics() (int, int) {
return finished, empty return finished, empty
} }
// start_build starts a build for the given BuildConfig.
fn (mut d AgentDaemon) start_build(config BuildConfig) bool {
for i in 0 .. d.atomics.len {
if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty {
stdatomic.store_u64(&d.atomics[i], agent.build_running)
d.builds[i] = config
spawn d.run_build(i, config)
return true
}
}
return false
}
// run_build actually starts the build process for a given target. // run_build actually starts the build process for a given target.
fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) { fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) {
d.linfo('started build: ${config}') d.linfo('started build: ${config}')
@ -195,3 +186,12 @@ fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) {
stdatomic.store_u64(&d.atomics[build_index], agent.build_done) stdatomic.store_u64(&d.atomics[build_index], agent.build_done)
} }
// builder_thread is a thread that constantly listens for builds to process
fn (mut d AgentDaemon) builder_thread(ch chan BuildConfig, builder_index int) {
for {
build_config := <-ch or { break }
d.run_build(builder_index, build_config)
}
}

View File

@ -94,8 +94,8 @@ pub:
} }
// build_target builds the given target. Internally it calls `build_config`. // build_target builds the given target. Internally it calls `build_config`.
pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool) !BuildResult { pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool, timeout int) !BuildResult {
config := target.as_build_config(base_image_id, force) config := target.as_build_config(base_image_id, force, timeout)
return build_config(address, api_key, config) return build_config(address, api_key, config)
} }
@ -136,9 +136,17 @@ pub fn build_config(address string, api_key string, config BuildConfig) !BuildRe
dd.container_start(id)! dd.container_start(id)!
mut data := dd.container_inspect(id)! mut data := dd.container_inspect(id)!
start_time := time.now()
// This loop waits until the container has stopped, so we can remove it after // This loop waits until the container has stopped, so we can remove it after
for data.state.running { for data.state.running {
if time.now() - start_time > config.timeout * time.second {
dd.container_kill(id)!
dd.container_remove(id)!
return error('Build killed due to timeout (${config.timeout}s)')
}
time.sleep(1 * time.second) time.sleep(1 * time.second)
data = dd.container_inspect(id)! data = dd.container_inspect(id)!

View File

@ -33,6 +33,8 @@ pub struct BuildJobQueue {
default_schedule &cron.Expression default_schedule &cron.Expression
// Base image to use for targets without defined base image // Base image to use for targets without defined base image
default_base_image string default_base_image string
// After how many minutes a build should be forcefully cancelled
default_build_timeout int
mut: mut:
mutex shared util.Dummy mutex shared util.Dummy
// For each architecture, a priority queue is tracked // For each architecture, a priority queue is tracked
@ -44,10 +46,11 @@ mut:
} }
// new_job_queue initializes a new job queue // new_job_queue initializes a new job queue
pub fn new_job_queue(default_schedule &cron.Expression, default_base_image string) BuildJobQueue { pub fn new_job_queue(default_schedule &cron.Expression, default_base_image string, default_build_timeout int) BuildJobQueue {
return BuildJobQueue{ return BuildJobQueue{
default_schedule: unsafe { default_schedule } default_schedule: unsafe { default_schedule }
default_base_image: default_base_image default_base_image: default_base_image
default_build_timeout: default_build_timeout
invalidated: map[int]time.Time{} invalidated: map[int]time.Time{}
} }
} }
@ -80,7 +83,7 @@ pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! {
mut job := BuildJob{ mut job := BuildJob{
created: time.now() created: time.now()
single: input.single single: input.single
config: input.target.as_build_config(q.default_base_image, input.force) config: input.target.as_build_config(q.default_base_image, input.force, q.default_build_timeout)
} }
if !input.now { if !input.now {

View File

@ -6,7 +6,7 @@ import os
import build import build
// build locally builds the target with the given id. // build locally builds the target with the given id.
fn build_target(conf Config, target_id int, force bool) ! { fn build_target(conf Config, target_id int, force bool, timeout int) ! {
c := client.new(conf.address, conf.api_key) c := client.new(conf.address, conf.api_key)
target := c.get_target(target_id)! target := c.get_target(target_id)!
@ -16,7 +16,7 @@ fn build_target(conf Config, target_id int, force bool) ! {
image_id := build.create_build_image(conf.base_image)! image_id := build.create_build_image(conf.base_image)!
println('Running build...') println('Running build...')
res := build.build_target(conf.address, conf.api_key, image_id, target, force)! res := build.build_target(conf.address, conf.api_key, image_id, target, force, timeout)!
println('Removing build image...') println('Removing build image...')

View File

@ -232,6 +232,12 @@ pub fn cmd() cli.Command {
description: 'Architecture to schedule build for. Required when using -remote.' description: 'Architecture to schedule build for. Required when using -remote.'
flag: cli.FlagType.string flag: cli.FlagType.string
}, },
cli.Flag{
name: 'timeout'
description: 'After how many minutes to cancel the build. Only applies to local builds.'
flag: cli.FlagType.int
default_value: ['3600']
},
] ]
execute: fn (cmd cli.Command) ! { execute: fn (cmd cli.Command) ! {
config_file := cmd.flags.get_string('config-file')! config_file := cmd.flags.get_string('config-file')!
@ -239,6 +245,7 @@ pub fn cmd() cli.Command {
remote := cmd.flags.get_bool('remote')! remote := cmd.flags.get_bool('remote')!
force := cmd.flags.get_bool('force')! force := cmd.flags.get_bool('force')!
timeout := cmd.flags.get_int('timeout')!
target_id := cmd.args[0].int() target_id := cmd.args[0].int()
if remote { if remote {
@ -251,7 +258,7 @@ pub fn cmd() cli.Command {
c := client.new(conf_.address, conf_.api_key) c := client.new(conf_.address, conf_.api_key)
c.queue_job(target_id, arch, force)! c.queue_job(target_id, arch, force)!
} else { } else {
build_target(conf_, target_id, force)! build_target(conf_, target_id, force, timeout)!
} }
} }
}, },

View File

@ -20,7 +20,7 @@ fn main() {
mut app := cli.Command{ mut app := cli.Command{
name: 'vieter' name: 'vieter'
description: 'Vieter is a lightweight implementation of an Arch repository server.' description: 'Vieter is a lightweight implementation of an Arch repository server.'
version: '0.5.0' version: '0.6.0'
posix_mode: true posix_mode: true
flags: [ flags: [
cli.Flag{ cli.Flag{

View File

@ -10,9 +10,10 @@ pub:
repo string repo string
base_image string base_image string
force bool force bool
timeout int
} }
// str return a single-line string representation of a build log // str return a single-line string representation of a build log
pub fn (c BuildConfig) str() string { pub fn (c BuildConfig) str() string {
return '{ target: ${c.target_id}, kind: ${c.kind}, url: ${c.url}, branch: ${c.branch}, path: ${c.path}, repo: ${c.repo}, base_image: ${c.base_image}, force: ${c.force} }' return '{ target: ${c.target_id}, kind: ${c.kind}, url: ${c.url}, branch: ${c.branch}, path: ${c.path}, repo: ${c.repo}, base_image: ${c.base_image}, force: ${c.force}, timeout: ${c.timeout} }'
} }

View File

@ -54,7 +54,7 @@ pub fn (t &Target) str() string {
// as_build_config converts a Target into a BuildConfig, given some extra // as_build_config converts a Target into a BuildConfig, given some extra
// needed information. // needed information.
pub fn (t &Target) as_build_config(base_image string, force bool) BuildConfig { pub fn (t &Target) as_build_config(base_image string, force bool, timeout int) BuildConfig {
return BuildConfig{ return BuildConfig{
target_id: t.id target_id: t.id
kind: t.kind kind: t.kind
@ -64,6 +64,7 @@ pub fn (t &Target) as_build_config(base_image string, force bool) BuildConfig {
repo: t.repo repo: t.repo
base_image: base_image base_image: base_image
force: force force: force
timeout: timeout
} }
} }

View File

@ -5,17 +5,18 @@ import conf as vconf
struct Config { struct Config {
pub: pub:
port int = 8000 port int = 8000
log_level string = 'WARN' log_level string = 'WARN'
pkg_dir string pkg_dir string
data_dir string data_dir string
api_key string api_key string
default_arch string default_arch string
global_schedule string = '0 3' global_schedule string = '0 3'
base_image string = 'archlinux:base-devel' base_image string = 'archlinux:base-devel'
max_log_age int [empty_default] max_log_age int [empty_default]
log_removal_schedule string = '0 0' log_removal_schedule string = '0 0'
collect_metrics bool [empty_default] collect_metrics bool [empty_default]
default_build_timeout int = 3600
} }
// cmd returns the cli submodule that handles starting the server // cmd returns the cli submodule that handles starting the server

View File

@ -68,7 +68,7 @@ fn (mut app App) put_package(repo_ string) web.Result {
mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true })
util.reader_to_file(mut app.reader, length.int(), pkg_path) or { util.reader_to_file(mut app.reader, length.int(), pkg_path) or {
app.lwarn("Failed to upload '${pkg_path}'") app.lwarn("Failed to upload '${pkg_path}': ${err.msg()}")
return app.status(.internal_server_error) return app.status(.internal_server_error)
} }

View File

@ -108,7 +108,7 @@ pub fn server(conf Config) ! {
repo: repo_ repo: repo_
db: db db: db
collector: collector collector: collector
job_queue: build.new_job_queue(global_ce, conf.base_image) job_queue: build.new_job_queue(global_ce, conf.base_image, conf.default_build_timeout)
} }
app.init_job_queue() or { app.init_job_queue() or {
util.exit_with_message(1, 'Failed to inialize job queue: ${err.msg()}') util.exit_with_message(1, 'Failed to inialize job queue: ${err.msg()}')

View File

@ -46,6 +46,10 @@ pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ! {
to_write = to_write - bytes_written to_write = to_write - bytes_written
} }
} }
if bytes_left > 0 {
return error('Not all bytes were received.')
}
} }
// match_array_in_array[T] returns how many elements of a2 overlap with a1. For // match_array_in_array[T] returns how many elements of a2 overlap with a1. For