From 540574b3c3c4d71df804e2ecd42028273f95e28d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 25 Feb 2022 20:52:30 +0100 Subject: [PATCH] Split builds into separate containers; made makepkg parallel --- src/build.v | 99 ++++++++++++++++++++++++++++++--------------- src/docker/docker.v | 5 --- src/docker/images.v | 31 ++++++++++++++ 3 files changed, 98 insertions(+), 37 deletions(-) create mode 100644 src/docker/images.v diff --git a/src/build.v b/src/build.v index fc1fe6f..d299d39 100644 --- a/src/build.v +++ b/src/build.v @@ -10,18 +10,10 @@ import env import net.http const container_build_dir = '/build' +const build_image_repo = 'vieter-build' -fn build() ? { - conf := env.load() ? - - // We get the repos list from the Vieter instance - mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? - req.add_custom_header('X-Api-Key', conf.api_key) ? - - res := req.do() ? - repos := json.decode([]server.GitRepo, res.text) ? - - mut commands := [ +fn create_build_image() ?string { + commands := [ // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' // Add a non-root user to run makepkg @@ -34,31 +26,11 @@ fn build() ? { 'mkdir /build', 'chown -R builder:builder /build', ] - - // Each repo gets a unique UUID to avoid naming conflicts when cloning - mut uuids := []string{} - - for repo in repos { - mut uuid := rand.uuid_v4() - - // Just to be sure we don't have any collisions - for uuids.contains(uuid) { - uuid = rand.uuid_v4() - } - - uuids << uuid - - commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'" - commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\'' - } - - // We convert the list of commands into a base64 string, which then gets - // passed to the container as an env var cmds_str := base64.encode_str(commands.join('\n')) c := docker.NewContainer{ image: 'archlinux:latest' - env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key'] + env: ['BUILD_SCRIPT=$cmds_str'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] } @@ -81,5 +53,68 @@ fn build() ? { time.sleep(5000000000) } + // Finally, we create the image from the container + // As the tag, we use the epoch value + tag := time.sys_mono_now().str() + image := docker.create_image_from_container(id, 'vieter-build', tag) ? docker.remove_container(id) ? + + return image.id +} + +fn build() ? { + conf := env.load() ? + + // We get the repos list from the Vieter instance + mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? + req.add_custom_header('X-Api-Key', conf.api_key) ? + + res := req.do() ? + repos := json.decode([]server.GitRepo, res.text) ? + + // No point in doing work if there's no repos present + if repos.len == 0 { + return + } + + // First, we create a base image which has updated repos n stuff + image_id := create_build_image() ? + + for repo in repos { + commands := [ + "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/repo'" + 'su builder -c \'cd /build/repo && MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\'' + ] + + // We convert the list of commands into a base64 string, which then gets + // passed to the container as an env var + cmds_str := base64.encode_str(commands.join('\n')) + + c := docker.NewContainer{ + image: '$image_id' + env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key'] + entrypoint: ['/bin/sh', '-c'] + cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] + } + + id := docker.create_container(c) ? + docker.start_container(id) ? + + // This loop waits until the container has stopped, so we can remove it after + for { + data := docker.inspect_container(id) ? + + if !data.state.running { + break + } + + // Wait for 5 seconds + time.sleep(5000000000) + } + + docker.remove_container(id) ? + } + + // Finally, we remove the builder image + docker.remove_image(image_id) ? } diff --git a/src/docker/docker.v b/src/docker/docker.v index e0dbf7d..a6f7640 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -91,8 +91,3 @@ pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Respon return request_with_body(method, url, 'application/json', body) } - -// pull_image pulls tries to pull the image for the given image & tag -pub fn pull_image(image string, tag string) ?http.Response { - return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) -} diff --git a/src/docker/images.v b/src/docker/images.v new file mode 100644 index 0000000..51a315d --- /dev/null +++ b/src/docker/images.v @@ -0,0 +1,31 @@ +module docker + +import net.http +import net.urllib +import json + +struct Image { +pub: + id string [json: Id] +} + +// pull_image pulls tries to pull the image for the given image & tag +pub fn pull_image(image string, tag string) ?http.Response { + return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) +} + +pub fn create_image_from_container(id string, repo string, tag string) ?Image { + res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag') ?) ? + + if res.status_code != 201 { + return error('Failed to create image from container.') + } + + return json.decode(Image, res.text) or {} +} + +pub fn remove_image(id string) ?bool { + res := request('DELETE', urllib.parse('/v1.41/images/$id') ?) ? + + return res.status_code == 200 +}