From 27aa215effb206e2256ac67308935bc2340a0bdb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 8 May 2022 10:29:06 +0200 Subject: [PATCH 1/6] feat(docker): added function to retrieve container logs --- src/docker/containers.v | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/docker/containers.v b/src/docker/containers.v index d0f5a4d..8134370 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -76,3 +76,23 @@ pub fn remove_container(id string) ?bool { return res.status_code == 204 } + +pub fn get_container_logs(id string) ?string { + res := request('GET', urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true') ?) ? + mut res_bytes := res.text.bytes() + + // Docker uses a special "stream" format for their logs, so we have to + // clean up the data. + mut index := 0 + + for index < res_bytes.len { + // The reverse is required because V reads in the bytes differently + t := res_bytes[index + 4..index + 8].reverse() + len_length := unsafe { *(&u32(&t[0])) } + + res_bytes.delete_many(index, 8) + index += int(len_length) + } + + return res_bytes.bytestr() +} From 4b172cb5d8e9dc8d96f7c73fc84e0d7aa1b8b5ab Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 8 May 2022 13:17:54 +0200 Subject: [PATCH 2/6] feat(cli): `vieter build` now builds a single repo & uploads build logs --- src/build/build.v | 56 +++++++++++++++++++++++++---------------- src/build/cli.v | 8 ++++-- src/docker/containers.v | 29 +++++++++++++++++++-- 3 files changed, 68 insertions(+), 25 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 6f033e6..9505171 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -73,10 +73,17 @@ pub fn create_build_image(base_image string) ?string { return image.id } +struct BuildResult { + start_time time.Time + end_time time.Time + exit_code int + logs string +} + // build_repo builds, packages & publishes a given Arch package based on the // provided GitRepo. The base image ID should be of an image previously created -// by create_build_image. -pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ? { +// by create_build_image. It returns the logs of the container. +pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ?BuildResult { build_arch := os.uname().machine // TODO what to do with PKGBUILDs that build multiple packages? @@ -107,43 +114,50 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db id := docker.create_container(c) ? docker.start_container(id) ? + mut data := docker.inspect_container(id) ? + // This loop waits until the container has stopped, so we can remove it after for { - data := docker.inspect_container(id) ? - if !data.state.running { break } time.sleep(1 * time.second) + + data = docker.inspect_container(id) ? } + logs := docker.get_container_logs(id) ? + docker.remove_container(id) ? + + return BuildResult{ + start_time: data.state.start_time + end_time: data.state.end_time + exit_code: data.state.exit_code + logs: logs + } } // build builds every Git repo in the server's list. -fn build(conf Config) ? { +fn build(conf Config, repo_id int) ? { + c := client.new(conf.address, conf.api_key) + repo := c.get_git_repo(repo_id) ? + build_arch := os.uname().machine - // We get the repos map from the Vieter instance - repos := client.new(conf.address, conf.api_key).get_git_repos() ? - - // We filter out any repos that aren't allowed to be built on this - // architecture - filtered_repos := repos.filter(it.arch.map(it.value).contains(build_arch)) - - // No point in doing work if there's no repos present - if filtered_repos.len == 0 { - return - } - // First, we create a base image which has updated repos n stuff + println('Creating base image...') image_id := create_build_image(conf.base_image) ? - for repo in filtered_repos { - build_repo(conf.address, conf.api_key, image_id, repo) ? - } + println('Running build...') + res := build_repo(conf.address, conf.api_key, image_id, repo) ? - // Finally, we remove the builder image + // Remove the builder image + println('Removing build image...') docker.remove_image(image_id) ? + + // Upload the build log to the Vieter instance + println('Uploading logs to Vieter...') + c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code, res.logs) ? } diff --git a/src/build/cli.v b/src/build/cli.v index 0131396..5247e87 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -14,12 +14,16 @@ pub: pub fn cmd() cli.Command { return cli.Command{ name: 'build' - description: 'Run the build process.' + required_args: 1 + usage: 'id' + description: 'Build the repository with the given ID.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file') ? conf := env.load(config_file) ? - build(conf) ? + id := cmd.args[0].int() + + build(conf, id) ? } } } diff --git a/src/docker/containers.v b/src/docker/containers.v index 8134370..63095a6 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -2,6 +2,8 @@ module docker import json import net.urllib +import regex +import time struct Container { id string [json: Id] @@ -49,13 +51,28 @@ pub fn start_container(id string) ?bool { } struct ContainerInspect { -pub: +pub mut: state ContainerState [json: State] } struct ContainerState { pub: running bool [json: Running] + status string [json: Status] + exit_code int [json: ExitCode] + // These use a rather specific format so they have to be parsed later + start_time_str string [json: StartedAt] + end_time_str string [json: FinishedAt] +pub mut: + start_time time.Time [skip] + end_time time.Time [skip] +} + +fn docker_timestamp_to_time(s string) ?time.Time { + parts := s.split('.') + clipped := parts[0] + '.' + parts[1][..3] + + return time.parse_rfc3339(clipped) } // inspect_container returns the result of inspecting a container with a given @@ -67,7 +84,15 @@ pub fn inspect_container(id string) ?ContainerInspect { return error('Failed to inspect container.') } - return json.decode(ContainerInspect, res.text) or {} + mut data := json.decode(ContainerInspect, res.text) ? + + data.state.start_time = docker_timestamp_to_time(data.state.start_time_str) ? + + if data.state.status == "exited" { + data.state.end_time = docker_timestamp_to_time(data.state.end_time_str) ? + } + + return data } // remove_container removes a container with a given ID. From e79d18100f38599429006e3c07406e0526cd0b74 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 8 May 2022 14:53:35 +0200 Subject: [PATCH 3/6] chore: ran `make fmt` --- src/build/build.v | 14 ++++++-------- src/docker/containers.v | 12 ++++++------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 9505171..c97a5f7 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -75,9 +75,9 @@ pub fn create_build_image(base_image string) ?string { struct BuildResult { start_time time.Time - end_time time.Time - exit_code int - logs string + end_time time.Time + exit_code int + logs string } // build_repo builds, packages & publishes a given Arch package based on the @@ -94,7 +94,7 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db 'source PKGBUILD', // The build container checks whether the package is already // present on the server - 'curl --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + 'curl -s --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done', ] @@ -146,18 +146,16 @@ fn build(conf Config, repo_id int) ? { build_arch := os.uname().machine - // First, we create a base image which has updated repos n stuff println('Creating base image...') image_id := create_build_image(conf.base_image) ? println('Running build...') res := build_repo(conf.address, conf.api_key, image_id, repo) ? - // Remove the builder image println('Removing build image...') docker.remove_image(image_id) ? - // Upload the build log to the Vieter instance println('Uploading logs to Vieter...') - c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code, res.logs) ? + c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code, + res.logs) ? } diff --git a/src/docker/containers.v b/src/docker/containers.v index 63095a6..6d2eb3e 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -57,15 +57,15 @@ pub mut: struct ContainerState { pub: - running bool [json: Running] - status string [json: Status] - exit_code int [json: ExitCode] + running bool [json: Running] + status string [json: Status] + exit_code int [json: ExitCode] // These use a rather specific format so they have to be parsed later start_time_str string [json: StartedAt] - end_time_str string [json: FinishedAt] + end_time_str string [json: FinishedAt] pub mut: start_time time.Time [skip] - end_time time.Time [skip] + end_time time.Time [skip] } fn docker_timestamp_to_time(s string) ?time.Time { @@ -88,7 +88,7 @@ pub fn inspect_container(id string) ?ContainerInspect { data.state.start_time = docker_timestamp_to_time(data.state.start_time_str) ? - if data.state.status == "exited" { + if data.state.status == 'exited' { data.state.end_time = docker_timestamp_to_time(data.state.end_time_str) ? } From ea4c4fce1650543bcbf22e20d6ce01015120500f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 8 May 2022 15:07:54 +0200 Subject: [PATCH 4/6] feat(cron): upload logs after build --- src/build/build.v | 3 ++- src/cron/daemon/build.v | 12 ++++++++++-- src/docker/containers.v | 1 - 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index c97a5f7..774591d 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -73,7 +73,8 @@ pub fn create_build_image(base_image string) ?string { return image.id } -struct BuildResult { +pub struct BuildResult { +pub: start_time time.Time end_time time.Time exit_code int diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index d107fd3..aa08f9f 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -3,6 +3,7 @@ module daemon import time import sync.stdatomic import build +import os const ( build_empty = 0 @@ -77,13 +78,20 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { // 0 means success, 1 means failure mut status := 0 - build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(), &sb.repo) or { + res := build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(), + &sb.repo) or { d.ldebug('build_repo error: $err.msg()') status = 1 + + build.BuildResult{} } if status == 0 { - d.linfo('finished build: $sb.repo.url $sb.repo.branch') + d.linfo('finished build: $sb.repo.url $sb.repo.branch; uploading logs...') + + build_arch := os.uname().machine + d.client.add_build_log(sb.repo.id, res.start_time, res.end_time, build_arch, res.exit_code, + res.logs) or { d.lerror('Failed to upload logs for $sb.repo.url $sb.repo.arch') } } else { d.linfo('failed build: $sb.repo.url $sb.repo.branch') } diff --git a/src/docker/containers.v b/src/docker/containers.v index 6d2eb3e..fe0bb7b 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -2,7 +2,6 @@ module docker import json import net.urllib -import regex import time struct Container { From 5a5f7f83461f20785e18b006e8033316a9f24d9c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 9 May 2022 14:58:20 +0200 Subject: [PATCH 5/6] refactor(docker): use builtin parse_rfc3339 function --- src/docker/containers.v | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/docker/containers.v b/src/docker/containers.v index fe0bb7b..2258f3b 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -67,13 +67,6 @@ pub mut: end_time time.Time [skip] } -fn docker_timestamp_to_time(s string) ?time.Time { - parts := s.split('.') - clipped := parts[0] + '.' + parts[1][..3] - - return time.parse_rfc3339(clipped) -} - // inspect_container returns the result of inspecting a container with a given // ID. pub fn inspect_container(id string) ?ContainerInspect { @@ -85,10 +78,10 @@ pub fn inspect_container(id string) ?ContainerInspect { mut data := json.decode(ContainerInspect, res.text) ? - data.state.start_time = docker_timestamp_to_time(data.state.start_time_str) ? + data.state.start_time = time.parse_rfc3339(data.state.start_time_str) ? if data.state.status == 'exited' { - data.state.end_time = docker_timestamp_to_time(data.state.end_time_str) ? + data.state.end_time = time.parse_rfc3339(data.state.end_time_str) ? } return data @@ -101,6 +94,8 @@ pub fn remove_container(id string) ?bool { return res.status_code == 204 } +// get_container_logs retrieves the logs for a Docker container, both stdout & +// stderr. pub fn get_container_logs(id string) ?string { res := request('GET', urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true') ?) ? mut res_bytes := res.text.bytes() From 3821ed29fd4d0dd9fda4447dab444baf40d5cd6b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 9 May 2022 15:05:53 +0200 Subject: [PATCH 6/6] refactor(docker): simplified loop expression --- src/build/build.v | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 774591d..0a978aa 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -118,11 +118,7 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db mut data := docker.inspect_container(id) ? // This loop waits until the container has stopped, so we can remove it after - for { - if !data.state.running { - break - } - + for data.state.running { time.sleep(1 * time.second) data = docker.inspect_container(id) ?