test
/
vieter
forked from vieter-v/vieter
1
0
Fork 0
Code Issues Pull Requests Releases Activity

feat(cron): added automatic rebuilding of image; implemented builds

cron
Jef Roosens 2022-04-30 17:56:35 +02:00
parent 98c0e52b08
commit 369b4458c5
Signed by untrusted user: Jef Roosens
GPG Key ID: B75D4F293C7052DB
5 changed files with 94 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
src/build/build.v
View File

@ -10,7 +10,7 @@ const container_build_dir = '/build'
const build_image_repo = 'vieter-build'
fn create_build_image(base_image string) ?string {
pub fn create_build_image(base_image string) ?string {
commands := [
// Update repos & install required packages
'pacman -Syu --needed --noconfirm base-devel git'
@ -53,12 +53,13 @@ fn create_build_image(base_image string) ?string {
break
}
// Wait for 5 seconds
time.sleep(5000000000)
time.sleep(1 * time.second)
}
// Finally, we create the image from the container
// As the tag, we use the epoch value
// TODO also add the base image's name into the image name to prevent
// conflicts.
tag := time.sys_mono_now().str()
image := docker.create_image_from_container(id, 'vieter-build', tag) ?
docker.remove_container(id) ?
@ -66,6 +67,52 @@ fn create_build_image(base_image string) ?string {
return image.id
}
pub fn build_repo(address string, api_key string, base_image_id string, repo &git.GitRepo) ? {
build_arch := os.uname().machine
// TODO what to do with PKGBUILDs that build multiple packages?
commands := [
'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
'cd repo',
'makepkg --nobuild --nodeps',
'source PKGBUILD',
// The build container checks whether the package is already
// present on the server
'curl --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done',
]
// We convert the list of commands into a base64 string, which then gets
// passed to the container as an env var
cmds_str := base64.encode_str(commands.join('\n'))
c := docker.NewContainer{
image: '$base_image_id'
env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$api_key']
entrypoint: ['/bin/sh', '-c']
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e']
work_dir: '/build'
user: 'builder:builder'
}
id := docker.create_container(c) ?
docker.start_container(id) ?
// This loop waits until the container has stopped, so we can remove it after
for {
data := docker.inspect_container(id) ?
if !data.state.running {
break
}
// Wait for 5 seconds
time.sleep(1 * time.second)
}
docker.remove_container(id) ?
}
fn build(conf Config) ? {
build_arch := os.uname().machine
@ -85,47 +132,7 @@ fn build(conf Config) ? {
image_id := create_build_image(conf.base_image) ?
for repo in filtered_repos {
// TODO what to do with PKGBUILDs that build multiple packages?
commands := [
'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
'cd repo',
'makepkg --nobuild --nodeps',
'source PKGBUILD',
// The build container checks whether the package is already
// present on the server
'curl --head --fail $conf.address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/$repo.repo/publish; done',
]
// We convert the list of commands into a base64 string, which then gets
// passed to the container as an env var
cmds_str := base64.encode_str(commands.join('\n'))
c := docker.NewContainer{
image: '$image_id'
env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key']
entrypoint: ['/bin/sh', '-c']
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e']
work_dir: '/build'
user: 'builder:builder'
}
id := docker.create_container(c) ?
docker.start_container(id) ?
// This loop waits until the container has stopped, so we can remove it after
for {
data := docker.inspect_container(id) ?
if !data.state.running {
break
}
// Wait for 5 seconds
time.sleep(5000000000)
}
docker.remove_container(id) ?
build_repo(conf.address, conf.api_key, image_id, repo) ?
}
// Finally, we remove the builder image

15
src/cron/cli.v
View File

@ -5,13 +5,14 @@ import env
struct Config {
pub:
log_level string = 'WARN'
log_file string = 'vieter.log'
api_key string
address string
base_image string = 'archlinux:base-devel'
max_concurrent_builds int = 1
api_update_frequency int = 15
log_level string = 'WARN'
log_file string = 'vieter.log'
api_key string
address string
base_image string = 'archlinux:base-devel'
max_concurrent_builds int = 1
api_update_frequency int = 15
image_rebuild_frequency int = 1440
// Replicates the behavior of the original cron system
global_schedule string = '0 3'
}

2
src/cron/cron.v
View File

@ -23,7 +23,7 @@ pub fn cron(conf Config) ? {
}
mut d := daemon.init_daemon(logger, conf.address, conf.api_key, conf.base_image, ce,
conf.max_concurrent_builds, conf.api_update_frequency) ?
conf.max_concurrent_builds, conf.api_update_frequency, conf.image_rebuild_frequency) ?
d.run() ?
}

6
src/cron/daemon/build.v
View File

@ -3,6 +3,7 @@ module daemon
import time
import sync.stdatomic
import rand
import build
const build_empty = 0
@ -62,8 +63,9 @@ fn (mut d Daemon) start_build(sb ScheduledBuild) bool {
// run_build actually starts the build process for a given repo.
fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) ? {
d.linfo('build $sb.repo.url')
time.sleep(rand.int_in_range(1, 6) ? * time.second)
d.linfo('started build: ${sb.repo.url} ${sb.repo.branch}')
build.build_repo(d.address, d.api_key, d.builder_image, &sb.repo) ?
stdatomic.store_u64(&d.atomics[build_index], daemon.build_done)
}

39
src/cron/daemon/daemon.v
View File

@ -6,6 +6,7 @@ import log
import datatypes { MinHeap }
import cron.expression { CronExpression, parse_expression }
import math
import build
struct ScheduledBuild {
pub:
@ -20,16 +21,19 @@ fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool {
pub struct Daemon {
mut:
address string
api_key string
base_image string
global_schedule CronExpression
api_update_frequency int
address string
api_key string
base_image string
builder_image string
global_schedule CronExpression
api_update_frequency int
image_rebuild_frequency int
// Repos currently loaded from API.
repos_map map[string]git.GitRepo
// At what point to update the list of repositories.
api_update_timestamp time.Time
queue MinHeap<ScheduledBuild>
api_update_timestamp time.Time
image_build_timestamp time.Time
queue MinHeap<ScheduledBuild>
// Which builds are currently running
builds []ScheduledBuild
// Atomic variables used to detect when a build has finished; length is the
@ -40,13 +44,14 @@ mut:
// init_daemon initializes a new Daemon object. It renews the repositories &
// populates the build queue for the first time.
pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int) ?Daemon {
pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon {
mut d := Daemon{
address: address
api_key: api_key
base_image: base_image
global_schedule: global_schedule
api_update_frequency: api_update_frequency
image_rebuild_frequency: image_rebuild_frequency
atomics: []u64{len: max_concurrent_builds}
builds: []ScheduledBuild{len: max_concurrent_builds}
logger: logger
@ -55,6 +60,7 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st
// Initialize the repos & queue
d.renew_repos() ?
d.renew_queue() ?
d.rebuild_base_image() ?
return d
}
@ -78,7 +84,15 @@ pub fn (mut d Daemon) run() ? {
}
}
// TODO rebuild builder image when needed
// TODO remove old builder images.
// This issue is less trivial than it sounds, because a build could
// still be running when the image has to be rebuilt. That would
// prevent the image from being removed. Therefore, we will need to
// keep track of a list or something & remove an image once we have
// made sure it isn't being used anymore.
if time.now() >= d.image_build_timestamp {
d.rebuild_base_image() ?
}
// Schedules new builds when possible
d.start_new_builds() ?
@ -170,3 +184,10 @@ fn (mut d Daemon) renew_queue() ? {
d.schedule_build(id, repo) ?
}
}
fn (mut d Daemon) rebuild_base_image() ? {
d.linfo("Rebuilding builder image....")
d.builder_image = build.create_build_image(d.base_image) ?
d.image_build_timestamp = time.now().add_seconds(60 * d.image_rebuild_frequency)
}