From 540574b3c3c4d71df804e2ecd42028273f95e28d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 25 Feb 2022 20:52:30 +0100 Subject: [PATCH 1/5] Split builds into separate containers; made makepkg parallel --- src/build.v | 99 ++++++++++++++++++++++++++++++--------------- src/docker/docker.v | 5 --- src/docker/images.v | 31 ++++++++++++++ 3 files changed, 98 insertions(+), 37 deletions(-) create mode 100644 src/docker/images.v diff --git a/src/build.v b/src/build.v index fc1fe6f..d299d39 100644 --- a/src/build.v +++ b/src/build.v @@ -10,18 +10,10 @@ import env import net.http const container_build_dir = '/build' +const build_image_repo = 'vieter-build' -fn build() ? { - conf := env.load() ? - - // We get the repos list from the Vieter instance - mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? - req.add_custom_header('X-Api-Key', conf.api_key) ? - - res := req.do() ? - repos := json.decode([]server.GitRepo, res.text) ? - - mut commands := [ +fn create_build_image() ?string { + commands := [ // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' // Add a non-root user to run makepkg @@ -34,31 +26,11 @@ fn build() ? { 'mkdir /build', 'chown -R builder:builder /build', ] - - // Each repo gets a unique UUID to avoid naming conflicts when cloning - mut uuids := []string{} - - for repo in repos { - mut uuid := rand.uuid_v4() - - // Just to be sure we don't have any collisions - for uuids.contains(uuid) { - uuid = rand.uuid_v4() - } - - uuids << uuid - - commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'" - commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\'' - } - - // We convert the list of commands into a base64 string, which then gets - // passed to the container as an env var cmds_str := base64.encode_str(commands.join('\n')) c := docker.NewContainer{ image: 'archlinux:latest' - env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key'] + env: ['BUILD_SCRIPT=$cmds_str'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] } @@ -81,5 +53,68 @@ fn build() ? { time.sleep(5000000000) } + // Finally, we create the image from the container + // As the tag, we use the epoch value + tag := time.sys_mono_now().str() + image := docker.create_image_from_container(id, 'vieter-build', tag) ? docker.remove_container(id) ? + + return image.id +} + +fn build() ? { + conf := env.load() ? + + // We get the repos list from the Vieter instance + mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ? + req.add_custom_header('X-Api-Key', conf.api_key) ? + + res := req.do() ? + repos := json.decode([]server.GitRepo, res.text) ? + + // No point in doing work if there's no repos present + if repos.len == 0 { + return + } + + // First, we create a base image which has updated repos n stuff + image_id := create_build_image() ? + + for repo in repos { + commands := [ + "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/repo'" + 'su builder -c \'cd /build/repo && MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\'' + ] + + // We convert the list of commands into a base64 string, which then gets + // passed to the container as an env var + cmds_str := base64.encode_str(commands.join('\n')) + + c := docker.NewContainer{ + image: '$image_id' + env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key'] + entrypoint: ['/bin/sh', '-c'] + cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] + } + + id := docker.create_container(c) ? + docker.start_container(id) ? + + // This loop waits until the container has stopped, so we can remove it after + for { + data := docker.inspect_container(id) ? + + if !data.state.running { + break + } + + // Wait for 5 seconds + time.sleep(5000000000) + } + + docker.remove_container(id) ? + } + + // Finally, we remove the builder image + docker.remove_image(image_id) ? } diff --git a/src/docker/docker.v b/src/docker/docker.v index e0dbf7d..a6f7640 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -91,8 +91,3 @@ pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Respon return request_with_body(method, url, 'application/json', body) } - -// pull_image pulls tries to pull the image for the given image & tag -pub fn pull_image(image string, tag string) ?http.Response { - return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) -} diff --git a/src/docker/images.v b/src/docker/images.v new file mode 100644 index 0000000..51a315d --- /dev/null +++ b/src/docker/images.v @@ -0,0 +1,31 @@ +module docker + +import net.http +import net.urllib +import json + +struct Image { +pub: + id string [json: Id] +} + +// pull_image pulls tries to pull the image for the given image & tag +pub fn pull_image(image string, tag string) ?http.Response { + return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) +} + +pub fn create_image_from_container(id string, repo string, tag string) ?Image { + res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag') ?) ? + + if res.status_code != 201 { + return error('Failed to create image from container.') + } + + return json.decode(Image, res.text) or {} +} + +pub fn remove_image(id string) ?bool { + res := request('DELETE', urllib.parse('/v1.41/images/$id') ?) ? + + return res.status_code == 200 +} -- 2.40.1 From a7cb08f93d96dcc7cfe055f4d5dea1fa5c74f3b2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 25 Feb 2022 21:47:28 +0100 Subject: [PATCH 2/5] Containers now check whether package should be rebuilt --- src/build.v | 16 ++++++++++++---- src/docker/containers.v | 2 ++ src/server/routes.v | 12 +++++++++++- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/build.v b/src/build.v index d299d39..167b492 100644 --- a/src/build.v +++ b/src/build.v @@ -81,9 +81,15 @@ fn build() ? { image_id := create_build_image() ? for repo in repos { + // TODO what to do with PKGBUILDs that build multiple packages? commands := [ - "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/repo'" - 'su builder -c \'cd /build/repo && MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\'' + "git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo" + 'cd repo' + "makepkg --nobuild --nodeps" + 'source PKGBUILD' + // The build container checks whether the package is already present on the server + "curl --head --fail $conf.address/\$pkgname-\$pkgver-\$pkgrel-\$(uname -m).pkg.tar.zst && exit 0" + 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done' ] // We convert the list of commands into a base64 string, which then gets @@ -94,7 +100,9 @@ fn build() ? { image: '$image_id' env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key'] entrypoint: ['/bin/sh', '-c'] - cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] + cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e'] + work_dir: '/build' + user: 'builder:builder' } id := docker.create_container(c) ? @@ -112,7 +120,7 @@ fn build() ? { time.sleep(5000000000) } - docker.remove_container(id) ? + // docker.remove_container(id) ? } // Finally, we remove the builder image diff --git a/src/docker/containers.v b/src/docker/containers.v index a6df345..37ff516 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -20,6 +20,8 @@ pub struct NewContainer { entrypoint []string [json: Entrypoint] cmd []string [json: Cmd] env []string [json: Env] + work_dir string [json: WorkingDir] + user string [json: User] } struct CreatedContainer { diff --git a/src/server/routes.v b/src/server/routes.v index 7a0ee38..0090666 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -6,6 +6,7 @@ import repo import time import rand import util +import net.http // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. @@ -15,7 +16,7 @@ pub fn (mut app App) healthcheck() web.Result { } // get_root handles a GET request for a file on the root -['/:filename'; get] +['/:filename'; get; head] fn (mut app App) get_root(filename string) web.Result { mut full_path := '' @@ -27,6 +28,15 @@ fn (mut app App) get_root(filename string) web.Result { full_path = os.join_path_single(app.repo.pkg_dir, filename) } + // Scuffed way to respond to HEAD requests + if app.req.method == http.Method.head { + if os.exists(full_path) { + return app.ok('') + } + + return app.not_found() + } + return app.file(full_path) } -- 2.40.1 From 732b53b6cec8039b018e157bb30b672c602aa786 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 25 Feb 2022 21:50:07 +0100 Subject: [PATCH 3/5] Forgot to uncomment line --- src/build.v | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build.v b/src/build.v index 167b492..d1fb21d 100644 --- a/src/build.v +++ b/src/build.v @@ -120,7 +120,7 @@ fn build() ? { time.sleep(5000000000) } - // docker.remove_container(id) ? + docker.remove_container(id) ? } // Finally, we remove the builder image -- 2.40.1 From 4a4362c1385934a3cdc5404199bfcc6e4e705a73 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 25 Feb 2022 21:54:16 +0100 Subject: [PATCH 4/5] Fixed linting errors --- src/build.v | 14 +++++++------- src/docker/containers.v | 4 ++-- src/docker/images.v | 3 +++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/build.v b/src/build.v index d1fb21d..51de64c 100644 --- a/src/build.v +++ b/src/build.v @@ -2,7 +2,6 @@ module main import docker import encoding.base64 -import rand import time import json import server @@ -10,6 +9,7 @@ import env import net.http const container_build_dir = '/build' + const build_image_repo = 'vieter-build' fn create_build_image() ?string { @@ -83,13 +83,13 @@ fn build() ? { for repo in repos { // TODO what to do with PKGBUILDs that build multiple packages? commands := [ - "git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo" - 'cd repo' - "makepkg --nobuild --nodeps" - 'source PKGBUILD' + 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', + 'cd repo', + 'makepkg --nobuild --nodeps', + 'source PKGBUILD', // The build container checks whether the package is already present on the server - "curl --head --fail $conf.address/\$pkgname-\$pkgver-\$pkgrel-\$(uname -m).pkg.tar.zst && exit 0" - 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done' + 'curl --head --fail $conf.address/\$pkgname-\$pkgver-\$pkgrel-\$(uname -m).pkg.tar.zst && exit 0', + 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done', ] // We convert the list of commands into a base64 string, which then gets diff --git a/src/docker/containers.v b/src/docker/containers.v index 37ff516..d0f5a4d 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -20,8 +20,8 @@ pub struct NewContainer { entrypoint []string [json: Entrypoint] cmd []string [json: Cmd] env []string [json: Env] - work_dir string [json: WorkingDir] - user string [json: User] + work_dir string [json: WorkingDir] + user string [json: User] } struct CreatedContainer { diff --git a/src/docker/images.v b/src/docker/images.v index 51a315d..e94ceca 100644 --- a/src/docker/images.v +++ b/src/docker/images.v @@ -14,6 +14,8 @@ pub fn pull_image(image string, tag string) ?http.Response { return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) } +// create_image_from_container creates a new image from a container with the +// given repo & tag, given the container's ID. pub fn create_image_from_container(id string, repo string, tag string) ?Image { res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag') ?) ? @@ -24,6 +26,7 @@ pub fn create_image_from_container(id string, repo string, tag string) ?Image { return json.decode(Image, res.text) or {} } +// remove_image removes the image with the given ID. pub fn remove_image(id string) ?bool { res := request('DELETE', urllib.parse('/v1.41/images/$id') ?) ? -- 2.40.1 From 88e048fbe2ef908123b482b8082709769d92ee67 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 25 Feb 2022 22:07:03 +0100 Subject: [PATCH 5/5] Updated deploy webhooks [CI SKIP] --- .woodpecker/.deploy.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.woodpecker/.deploy.yml b/.woodpecker/.deploy.yml index 56e6c2f..8cf3907 100644 --- a/.woodpecker/.deploy.yml +++ b/.woodpecker/.deploy.yml @@ -10,6 +10,8 @@ pipeline: webhook: image: chewingbever/vlang:latest secrets: - - webhook + - webhook_1 + - webhook_2 commands: - - curl -XPOST -s "$WEBHOOK" + - curl -XPOST -s "$WEBHOOK_1" + - curl -XPOST -s "$WEBHOOK_2" -- 2.40.1