forked from vieter-v/vieter
Merge pull request 'Isolate builds, build when necessary' (#107) from build-system-upgrade into dev
Reviewed-on: Chewing_Bever/vieter#107main
commit
09d0a40aae
|
@ -10,6 +10,8 @@ pipeline:
|
||||||
webhook:
|
webhook:
|
||||||
image: chewingbever/vlang:latest
|
image: chewingbever/vlang:latest
|
||||||
secrets:
|
secrets:
|
||||||
- webhook
|
- webhook_1
|
||||||
|
- webhook_2
|
||||||
commands:
|
commands:
|
||||||
- curl -XPOST -s "$WEBHOOK"
|
- curl -XPOST -s "$WEBHOOK_1"
|
||||||
|
- curl -XPOST -s "$WEBHOOK_2"
|
||||||
|
|
107
src/build.v
107
src/build.v
|
@ -2,7 +2,6 @@ module main
|
||||||
|
|
||||||
import docker
|
import docker
|
||||||
import encoding.base64
|
import encoding.base64
|
||||||
import rand
|
|
||||||
import time
|
import time
|
||||||
import json
|
import json
|
||||||
import server
|
import server
|
||||||
|
@ -11,17 +10,10 @@ import net.http
|
||||||
|
|
||||||
const container_build_dir = '/build'
|
const container_build_dir = '/build'
|
||||||
|
|
||||||
fn build() ? {
|
const build_image_repo = 'vieter-build'
|
||||||
conf := env.load<env.BuildConfig>() ?
|
|
||||||
|
|
||||||
// We get the repos list from the Vieter instance
|
fn create_build_image() ?string {
|
||||||
mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ?
|
commands := [
|
||||||
req.add_custom_header('X-Api-Key', conf.api_key) ?
|
|
||||||
|
|
||||||
res := req.do() ?
|
|
||||||
repos := json.decode([]server.GitRepo, res.text) ?
|
|
||||||
|
|
||||||
mut commands := [
|
|
||||||
// Update repos & install required packages
|
// Update repos & install required packages
|
||||||
'pacman -Syu --needed --noconfirm base-devel git'
|
'pacman -Syu --needed --noconfirm base-devel git'
|
||||||
// Add a non-root user to run makepkg
|
// Add a non-root user to run makepkg
|
||||||
|
@ -34,31 +26,11 @@ fn build() ? {
|
||||||
'mkdir /build',
|
'mkdir /build',
|
||||||
'chown -R builder:builder /build',
|
'chown -R builder:builder /build',
|
||||||
]
|
]
|
||||||
|
|
||||||
// Each repo gets a unique UUID to avoid naming conflicts when cloning
|
|
||||||
mut uuids := []string{}
|
|
||||||
|
|
||||||
for repo in repos {
|
|
||||||
mut uuid := rand.uuid_v4()
|
|
||||||
|
|
||||||
// Just to be sure we don't have any collisions
|
|
||||||
for uuids.contains(uuid) {
|
|
||||||
uuid = rand.uuid_v4()
|
|
||||||
}
|
|
||||||
|
|
||||||
uuids << uuid
|
|
||||||
|
|
||||||
commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'"
|
|
||||||
commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\''
|
|
||||||
}
|
|
||||||
|
|
||||||
// We convert the list of commands into a base64 string, which then gets
|
|
||||||
// passed to the container as an env var
|
|
||||||
cmds_str := base64.encode_str(commands.join('\n'))
|
cmds_str := base64.encode_str(commands.join('\n'))
|
||||||
|
|
||||||
c := docker.NewContainer{
|
c := docker.NewContainer{
|
||||||
image: 'archlinux:latest'
|
image: 'archlinux:latest'
|
||||||
env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key']
|
env: ['BUILD_SCRIPT=$cmds_str']
|
||||||
entrypoint: ['/bin/sh', '-c']
|
entrypoint: ['/bin/sh', '-c']
|
||||||
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e']
|
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e']
|
||||||
}
|
}
|
||||||
|
@ -81,5 +53,76 @@ fn build() ? {
|
||||||
time.sleep(5000000000)
|
time.sleep(5000000000)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Finally, we create the image from the container
|
||||||
|
// As the tag, we use the epoch value
|
||||||
|
tag := time.sys_mono_now().str()
|
||||||
|
image := docker.create_image_from_container(id, 'vieter-build', tag) ?
|
||||||
docker.remove_container(id) ?
|
docker.remove_container(id) ?
|
||||||
|
|
||||||
|
return image.id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build() ? {
|
||||||
|
conf := env.load<env.BuildConfig>() ?
|
||||||
|
|
||||||
|
// We get the repos list from the Vieter instance
|
||||||
|
mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ?
|
||||||
|
req.add_custom_header('X-Api-Key', conf.api_key) ?
|
||||||
|
|
||||||
|
res := req.do() ?
|
||||||
|
repos := json.decode([]server.GitRepo, res.text) ?
|
||||||
|
|
||||||
|
// No point in doing work if there's no repos present
|
||||||
|
if repos.len == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// First, we create a base image which has updated repos n stuff
|
||||||
|
image_id := create_build_image() ?
|
||||||
|
|
||||||
|
for repo in repos {
|
||||||
|
// TODO what to do with PKGBUILDs that build multiple packages?
|
||||||
|
commands := [
|
||||||
|
'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
|
||||||
|
'cd repo',
|
||||||
|
'makepkg --nobuild --nodeps',
|
||||||
|
'source PKGBUILD',
|
||||||
|
// The build container checks whether the package is already present on the server
|
||||||
|
'curl --head --fail $conf.address/\$pkgname-\$pkgver-\$pkgrel-\$(uname -m).pkg.tar.zst && exit 0',
|
||||||
|
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done',
|
||||||
|
]
|
||||||
|
|
||||||
|
// We convert the list of commands into a base64 string, which then gets
|
||||||
|
// passed to the container as an env var
|
||||||
|
cmds_str := base64.encode_str(commands.join('\n'))
|
||||||
|
|
||||||
|
c := docker.NewContainer{
|
||||||
|
image: '$image_id'
|
||||||
|
env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key']
|
||||||
|
entrypoint: ['/bin/sh', '-c']
|
||||||
|
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e']
|
||||||
|
work_dir: '/build'
|
||||||
|
user: 'builder:builder'
|
||||||
|
}
|
||||||
|
|
||||||
|
id := docker.create_container(c) ?
|
||||||
|
docker.start_container(id) ?
|
||||||
|
|
||||||
|
// This loop waits until the container has stopped, so we can remove it after
|
||||||
|
for {
|
||||||
|
data := docker.inspect_container(id) ?
|
||||||
|
|
||||||
|
if !data.state.running {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for 5 seconds
|
||||||
|
time.sleep(5000000000)
|
||||||
|
}
|
||||||
|
|
||||||
|
docker.remove_container(id) ?
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finally, we remove the builder image
|
||||||
|
docker.remove_image(image_id) ?
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,6 +20,8 @@ pub struct NewContainer {
|
||||||
entrypoint []string [json: Entrypoint]
|
entrypoint []string [json: Entrypoint]
|
||||||
cmd []string [json: Cmd]
|
cmd []string [json: Cmd]
|
||||||
env []string [json: Env]
|
env []string [json: Env]
|
||||||
|
work_dir string [json: WorkingDir]
|
||||||
|
user string [json: User]
|
||||||
}
|
}
|
||||||
|
|
||||||
struct CreatedContainer {
|
struct CreatedContainer {
|
||||||
|
|
|
@ -91,8 +91,3 @@ pub fn request_with_json<T>(method string, url urllib.URL, data &T) ?http.Respon
|
||||||
|
|
||||||
return request_with_body(method, url, 'application/json', body)
|
return request_with_body(method, url, 'application/json', body)
|
||||||
}
|
}
|
||||||
|
|
||||||
// pull_image pulls tries to pull the image for the given image & tag
|
|
||||||
pub fn pull_image(image string, tag string) ?http.Response {
|
|
||||||
return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?)
|
|
||||||
}
|
|
||||||
|
|
|
@ -0,0 +1,34 @@
|
||||||
|
module docker
|
||||||
|
|
||||||
|
import net.http
|
||||||
|
import net.urllib
|
||||||
|
import json
|
||||||
|
|
||||||
|
struct Image {
|
||||||
|
pub:
|
||||||
|
id string [json: Id]
|
||||||
|
}
|
||||||
|
|
||||||
|
// pull_image pulls tries to pull the image for the given image & tag
|
||||||
|
pub fn pull_image(image string, tag string) ?http.Response {
|
||||||
|
return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?)
|
||||||
|
}
|
||||||
|
|
||||||
|
// create_image_from_container creates a new image from a container with the
|
||||||
|
// given repo & tag, given the container's ID.
|
||||||
|
pub fn create_image_from_container(id string, repo string, tag string) ?Image {
|
||||||
|
res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag') ?) ?
|
||||||
|
|
||||||
|
if res.status_code != 201 {
|
||||||
|
return error('Failed to create image from container.')
|
||||||
|
}
|
||||||
|
|
||||||
|
return json.decode(Image, res.text) or {}
|
||||||
|
}
|
||||||
|
|
||||||
|
// remove_image removes the image with the given ID.
|
||||||
|
pub fn remove_image(id string) ?bool {
|
||||||
|
res := request('DELETE', urllib.parse('/v1.41/images/$id') ?) ?
|
||||||
|
|
||||||
|
return res.status_code == 200
|
||||||
|
}
|
|
@ -6,6 +6,7 @@ import repo
|
||||||
import time
|
import time
|
||||||
import rand
|
import rand
|
||||||
import util
|
import util
|
||||||
|
import net.http
|
||||||
|
|
||||||
// healthcheck just returns a string, but can be used to quickly check if the
|
// healthcheck just returns a string, but can be used to quickly check if the
|
||||||
// server is still responsive.
|
// server is still responsive.
|
||||||
|
@ -15,7 +16,7 @@ pub fn (mut app App) healthcheck() web.Result {
|
||||||
}
|
}
|
||||||
|
|
||||||
// get_root handles a GET request for a file on the root
|
// get_root handles a GET request for a file on the root
|
||||||
['/:filename'; get]
|
['/:filename'; get; head]
|
||||||
fn (mut app App) get_root(filename string) web.Result {
|
fn (mut app App) get_root(filename string) web.Result {
|
||||||
mut full_path := ''
|
mut full_path := ''
|
||||||
|
|
||||||
|
@ -27,6 +28,15 @@ fn (mut app App) get_root(filename string) web.Result {
|
||||||
full_path = os.join_path_single(app.repo.pkg_dir, filename)
|
full_path = os.join_path_single(app.repo.pkg_dir, filename)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Scuffed way to respond to HEAD requests
|
||||||
|
if app.req.method == http.Method.head {
|
||||||
|
if os.exists(full_path) {
|
||||||
|
return app.ok('')
|
||||||
|
}
|
||||||
|
|
||||||
|
return app.not_found()
|
||||||
|
}
|
||||||
|
|
||||||
return app.file(full_path)
|
return app.file(full_path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue