forked from vieter-v/vieter
Compare commits
No commits in common. "6edd1b475d4bace8c7d3a747cb5f59d9dcbba671" and "ca92ae2a0ee5eaafdefa2610b6f95a3e519a0173" have entirely different histories.
6edd1b475d
...
ca92ae2a0e
|
|
@ -13,7 +13,7 @@ pub:
|
||||||
polling_frequency int = 30
|
polling_frequency int = 30
|
||||||
// Architecture of agent
|
// Architecture of agent
|
||||||
// arch string
|
// arch string
|
||||||
image_rebuild_frequency int = 1440
|
// image_rebuild_frequency int = 1440
|
||||||
}
|
}
|
||||||
|
|
||||||
// cmd returns the cli module that handles the cron daemon.
|
// cmd returns the cli module that handles the cron daemon.
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,6 @@ import log
|
||||||
import sync.stdatomic
|
import sync.stdatomic
|
||||||
import build { BuildConfig }
|
import build { BuildConfig }
|
||||||
import client
|
import client
|
||||||
import time
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
build_empty = 0
|
build_empty = 0
|
||||||
|
|
@ -15,8 +14,8 @@ const (
|
||||||
struct AgentDaemon {
|
struct AgentDaemon {
|
||||||
logger shared log.Log
|
logger shared log.Log
|
||||||
conf Config
|
conf Config
|
||||||
mut:
|
// List of last built builder images
|
||||||
images ImageManager
|
builder_images []string
|
||||||
// Which builds are currently running; length is same as
|
// Which builds are currently running; length is same as
|
||||||
// conf.max_concurrent_builds
|
// conf.max_concurrent_builds
|
||||||
builds []BuildConfig
|
builds []BuildConfig
|
||||||
|
|
@ -31,7 +30,6 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon {
|
||||||
logger: logger
|
logger: logger
|
||||||
client: client.new(conf.address, conf.api_key)
|
client: client.new(conf.address, conf.api_key)
|
||||||
conf: conf
|
conf: conf
|
||||||
images: new_image_manager(conf.image_rebuild_frequency)
|
|
||||||
builds: []BuildConfig{len: conf.max_concurrent_builds}
|
builds: []BuildConfig{len: conf.max_concurrent_builds}
|
||||||
atomics: []u64{len: conf.max_concurrent_builds}
|
atomics: []u64{len: conf.max_concurrent_builds}
|
||||||
}
|
}
|
||||||
|
|
@ -43,33 +41,13 @@ pub fn (mut d AgentDaemon) run() {
|
||||||
for {
|
for {
|
||||||
free_builds := d.update_atomics()
|
free_builds := d.update_atomics()
|
||||||
|
|
||||||
// All build slots are taken, so there's nothing to be done
|
if free_builds > 0 {
|
||||||
if free_builds == 0 {
|
|
||||||
time.sleep(1 * time.second)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Builds have finished, so old builder images might have freed up.
|
|
||||||
d.images.clean_old_images()
|
|
||||||
|
|
||||||
// Poll for new jobs
|
|
||||||
new_configs := d.client.poll_jobs(free_builds) or {
|
|
||||||
d.lerror('Failed to poll jobs: $err.msg()')
|
|
||||||
|
|
||||||
time.sleep(1 * time.second)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Schedule new jobs
|
|
||||||
for config in new_configs {
|
|
||||||
d.start_build(config)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// update_atomics checks for each build whether it's completed, and sets it to
|
// update_atomics checks for each build whether it's completed, and sets it to
|
||||||
// free again if so. The return value is how many build slots are currently
|
// free again if so. The return value is how many fields are now set to free.
|
||||||
// free.
|
|
||||||
fn (mut d AgentDaemon) update_atomics() int {
|
fn (mut d AgentDaemon) update_atomics() int {
|
||||||
mut count := 0
|
mut count := 0
|
||||||
|
|
||||||
|
|
@ -84,49 +62,3 @@ fn (mut d AgentDaemon) update_atomics() int {
|
||||||
|
|
||||||
return count
|
return count
|
||||||
}
|
}
|
||||||
|
|
||||||
// start_build starts a build for the given BuildConfig object.
|
|
||||||
fn (mut d AgentDaemon) start_build(config BuildConfig) bool {
|
|
||||||
for i in 0 .. d.atomics.len {
|
|
||||||
if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty {
|
|
||||||
stdatomic.store_u64(&d.atomics[i], agent.build_running)
|
|
||||||
d.builds[i] = config
|
|
||||||
|
|
||||||
go d.run_build(i, config)
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// run_build actually starts the build process for a given target.
|
|
||||||
fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) {
|
|
||||||
d.linfo('started build: $config.url -> $config.repo')
|
|
||||||
|
|
||||||
// 0 means success, 1 means failure
|
|
||||||
mut status := 0
|
|
||||||
|
|
||||||
res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(),
|
|
||||||
&sb.target) or {
|
|
||||||
d.ldebug('build_target error: $err.msg()')
|
|
||||||
status = 1
|
|
||||||
|
|
||||||
build.BuildResult{}
|
|
||||||
}
|
|
||||||
|
|
||||||
if status == 0 {
|
|
||||||
d.linfo('finished build: $sb.target.url -> $sb.target.repo; uploading logs...')
|
|
||||||
|
|
||||||
build_arch := os.uname().machine
|
|
||||||
d.client.add_build_log(sb.target.id, res.start_time, res.end_time, build_arch,
|
|
||||||
res.exit_code, res.logs) or {
|
|
||||||
d.lerror('Failed to upload logs for build: $sb.target.url -> $sb.target.repo')
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
d.linfo('an error occured during build: $sb.target.url -> $sb.target.repo')
|
|
||||||
}
|
|
||||||
|
|
||||||
stdatomic.store_u64(&d.atomics[build_index], agent.build_done)
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -2,59 +2,48 @@ module agent
|
||||||
|
|
||||||
import time
|
import time
|
||||||
import docker
|
import docker
|
||||||
import build
|
|
||||||
|
|
||||||
struct ImageManager {
|
struct ImageManager {
|
||||||
mut:
|
images map[string]string
|
||||||
refresh_frequency int
|
timestamps map[string]time.Time
|
||||||
images map[string][]string [required]
|
|
||||||
timestamps map[string]time.Time [required]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn new_image_manager(refresh_frequency int) ImageManager {
|
// clean_old_base_images tries to remove any old but still present builder
|
||||||
return ImageManager{
|
// images.
|
||||||
refresh_frequency: refresh_frequency
|
fn (mut d AgentDaemon) clean_old_base_images() {
|
||||||
images: map[string][]string{}
|
mut i := 0
|
||||||
timestamps: map[string]time.Time{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn (mut m ImageManager) refresh_image(base_image string) ! {
|
mut dd := docker.new_conn() or {
|
||||||
// No need to refresh the image if the previous one is still new enough
|
d.lerror('Failed to connect to Docker socket.')
|
||||||
if base_image in m.timestamps
|
|
||||||
&& m.timestamps[base_image].add_seconds(m.refresh_frequency) > time.now() {
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO use better image tags for built images
|
|
||||||
new_image := build.create_build_image(base_image) or {
|
|
||||||
return error('Failed to build builder image from base image $base_image')
|
|
||||||
}
|
|
||||||
|
|
||||||
m.images[base_image] << new_image
|
|
||||||
m.timestamps[base_image] = time.now()
|
|
||||||
}
|
|
||||||
|
|
||||||
// clean_old_images tries to remove any old but still present builder images.
|
|
||||||
fn (mut m ImageManager) clean_old_images() {
|
|
||||||
mut dd := docker.new_conn() or { return }
|
|
||||||
|
|
||||||
defer {
|
defer {
|
||||||
dd.close() or {}
|
dd.close() or {}
|
||||||
}
|
}
|
||||||
|
|
||||||
mut i := 0
|
for i < d.builder_images.len - 1 {
|
||||||
|
|
||||||
for image in m.images.keys() {
|
|
||||||
i = 0
|
|
||||||
|
|
||||||
for i < m.images[image].len - 1 {
|
|
||||||
// For each builder image, we try to remove it by calling the Docker
|
// For each builder image, we try to remove it by calling the Docker
|
||||||
// API. If the function returns an error or false, that means the image
|
// API. If the function returns an error or false, that means the image
|
||||||
// wasn't deleted. Therefore, we move the index over. If the function
|
// wasn't deleted. Therefore, we move the index over. If the function
|
||||||
// returns true, the array's length has decreased by one so we don't
|
// returns true, the array's length has decreased by one so we don't
|
||||||
// move the index.
|
// move the index.
|
||||||
dd.remove_image(m.images[image][i]) or { i += 1 }
|
dd.remove_image(d.builder_images[i]) or { i += 1 }
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// rebuild_base_image builds a builder image from the given base image.
|
||||||
|
/* fn (mut d AgentDaemon) build_base_image(base_image string) bool { */
|
||||||
|
/* d.linfo('Rebuilding builder image....') */
|
||||||
|
|
||||||
|
/* d.builder_images << build.create_build_image(d.base_image) or { */
|
||||||
|
/* d.lerror('Failed to rebuild base image. Retrying in ${daemon.rebuild_base_image_retry_timout}s...') */
|
||||||
|
/* d.image_build_timestamp = time.now().add_seconds(daemon.rebuild_base_image_retry_timout) */
|
||||||
|
|
||||||
|
/* return false */
|
||||||
|
/* } */
|
||||||
|
|
||||||
|
/* d.image_build_timestamp = time.now().add_seconds(60 * d.image_rebuild_frequency) */
|
||||||
|
|
||||||
|
/* return true */
|
||||||
|
/* } */
|
||||||
|
|
|
||||||
|
|
@ -103,23 +103,10 @@ pub:
|
||||||
logs string
|
logs string
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult {
|
|
||||||
config := BuildConfig{
|
|
||||||
target_id: target.id
|
|
||||||
kind: target.kind
|
|
||||||
url: target.url
|
|
||||||
branch: target.branch
|
|
||||||
repo: target.repo
|
|
||||||
base_image: base_image_id
|
|
||||||
}
|
|
||||||
|
|
||||||
return build_config(address, api_key, config)
|
|
||||||
}
|
|
||||||
|
|
||||||
// build_target builds, packages & publishes a given Arch package based on the
|
// build_target builds, packages & publishes a given Arch package based on the
|
||||||
// provided target. The base image ID should be of an image previously created
|
// provided target. The base image ID should be of an image previously created
|
||||||
// by create_build_image. It returns the logs of the container.
|
// by create_build_image. It returns the logs of the container.
|
||||||
pub fn build_config(address string, api_key string, config BuildConfig) !BuildResult {
|
pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult {
|
||||||
mut dd := docker.new_conn()!
|
mut dd := docker.new_conn()!
|
||||||
|
|
||||||
defer {
|
defer {
|
||||||
|
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
module client
|
|
||||||
|
|
||||||
import build { BuildConfig }
|
|
||||||
|
|
||||||
pub fn (c &Client) poll_jobs(max int) ![]BuildConfig {
|
|
||||||
data := c.send_request<[]BuildConfig>(.get, '/api/v1/jobs/poll', {
|
|
||||||
'max': max.str()
|
|
||||||
})!
|
|
||||||
|
|
||||||
return data.data
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue