Merge pull request 'Isolate builds, build when necessary' (#107) from build-system-upgrade into dev

Reviewed-on: Chewing_Bever/vieter#107
main
Jef Roosens 2022-02-25 22:08:02 +01:00
commit 09d0a40aae
6 changed files with 126 additions and 40 deletions

View File

@ -10,6 +10,8 @@ pipeline:
webhook:
image: chewingbever/vlang:latest
secrets:
- webhook
- webhook_1
- webhook_2
commands:
- curl -XPOST -s "$WEBHOOK"
- curl -XPOST -s "$WEBHOOK_1"
- curl -XPOST -s "$WEBHOOK_2"

View File

@ -2,7 +2,6 @@ module main
import docker
import encoding.base64
import rand
import time
import json
import server
@ -11,17 +10,10 @@ import net.http
const container_build_dir = '/build'
fn build() ? {
conf := env.load<env.BuildConfig>() ?
const build_image_repo = 'vieter-build'
// We get the repos list from the Vieter instance
mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ?
req.add_custom_header('X-Api-Key', conf.api_key) ?
res := req.do() ?
repos := json.decode([]server.GitRepo, res.text) ?
mut commands := [
fn create_build_image() ?string {
commands := [
// Update repos & install required packages
'pacman -Syu --needed --noconfirm base-devel git'
// Add a non-root user to run makepkg
@ -34,31 +26,11 @@ fn build() ? {
'mkdir /build',
'chown -R builder:builder /build',
]
// Each repo gets a unique UUID to avoid naming conflicts when cloning
mut uuids := []string{}
for repo in repos {
mut uuid := rand.uuid_v4()
// Just to be sure we don't have any collisions
for uuids.contains(uuid) {
uuid = rand.uuid_v4()
}
uuids << uuid
commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'"
commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\''
}
// We convert the list of commands into a base64 string, which then gets
// passed to the container as an env var
cmds_str := base64.encode_str(commands.join('\n'))
c := docker.NewContainer{
image: 'archlinux:latest'
env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key']
env: ['BUILD_SCRIPT=$cmds_str']
entrypoint: ['/bin/sh', '-c']
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e']
}
@ -81,5 +53,76 @@ fn build() ? {
time.sleep(5000000000)
}
// Finally, we create the image from the container
// As the tag, we use the epoch value
tag := time.sys_mono_now().str()
image := docker.create_image_from_container(id, 'vieter-build', tag) ?
docker.remove_container(id) ?
return image.id
}
fn build() ? {
conf := env.load<env.BuildConfig>() ?
// We get the repos list from the Vieter instance
mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ?
req.add_custom_header('X-Api-Key', conf.api_key) ?
res := req.do() ?
repos := json.decode([]server.GitRepo, res.text) ?
// No point in doing work if there's no repos present
if repos.len == 0 {
return
}
// First, we create a base image which has updated repos n stuff
image_id := create_build_image() ?
for repo in repos {
// TODO what to do with PKGBUILDs that build multiple packages?
commands := [
'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
'cd repo',
'makepkg --nobuild --nodeps',
'source PKGBUILD',
// The build container checks whether the package is already present on the server
'curl --head --fail $conf.address/\$pkgname-\$pkgver-\$pkgrel-\$(uname -m).pkg.tar.zst && exit 0',
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done',
]
// We convert the list of commands into a base64 string, which then gets
// passed to the container as an env var
cmds_str := base64.encode_str(commands.join('\n'))
c := docker.NewContainer{
image: '$image_id'
env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key']
entrypoint: ['/bin/sh', '-c']
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e']
work_dir: '/build'
user: 'builder:builder'
}
id := docker.create_container(c) ?
docker.start_container(id) ?
// This loop waits until the container has stopped, so we can remove it after
for {
data := docker.inspect_container(id) ?
if !data.state.running {
break
}
// Wait for 5 seconds
time.sleep(5000000000)
}
docker.remove_container(id) ?
}
// Finally, we remove the builder image
docker.remove_image(image_id) ?
}

View File

@ -20,6 +20,8 @@ pub struct NewContainer {
entrypoint []string [json: Entrypoint]
cmd []string [json: Cmd]
env []string [json: Env]
work_dir string [json: WorkingDir]
user string [json: User]
}
struct CreatedContainer {

View File

@ -91,8 +91,3 @@ pub fn request_with_json<T>(method string, url urllib.URL, data &T) ?http.Respon
return request_with_body(method, url, 'application/json', body)
}
// pull_image pulls tries to pull the image for the given image & tag
pub fn pull_image(image string, tag string) ?http.Response {
return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?)
}

View File

@ -0,0 +1,34 @@
module docker
import net.http
import net.urllib
import json
struct Image {
pub:
id string [json: Id]
}
// pull_image pulls tries to pull the image for the given image & tag
pub fn pull_image(image string, tag string) ?http.Response {
return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?)
}
// create_image_from_container creates a new image from a container with the
// given repo & tag, given the container's ID.
pub fn create_image_from_container(id string, repo string, tag string) ?Image {
res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag') ?) ?
if res.status_code != 201 {
return error('Failed to create image from container.')
}
return json.decode(Image, res.text) or {}
}
// remove_image removes the image with the given ID.
pub fn remove_image(id string) ?bool {
res := request('DELETE', urllib.parse('/v1.41/images/$id') ?) ?
return res.status_code == 200
}

View File

@ -6,6 +6,7 @@ import repo
import time
import rand
import util
import net.http
// healthcheck just returns a string, but can be used to quickly check if the
// server is still responsive.
@ -15,7 +16,7 @@ pub fn (mut app App) healthcheck() web.Result {
}
// get_root handles a GET request for a file on the root
['/:filename'; get]
['/:filename'; get; head]
fn (mut app App) get_root(filename string) web.Result {
mut full_path := ''
@ -27,6 +28,15 @@ fn (mut app App) get_root(filename string) web.Result {
full_path = os.join_path_single(app.repo.pkg_dir, filename)
}
// Scuffed way to respond to HEAD requests
if app.req.method == http.Method.head {
if os.exists(full_path) {
return app.ok('')
}
return app.not_found()
}
return app.file(full_path)
}