From 9f753f9c93f53b0251e14233f5439e3fc9d6e39e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 12 May 2022 09:11:18 +0200 Subject: [PATCH 1/2] feat(build): add target repo to builds; update system for every build --- .woodpecker/.build.yml | 2 +- src/build/build.v | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 1698129..b0fd267 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -45,7 +45,7 @@ pipeline: - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')" - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH" - - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64` + - export SIGNATURE="$(echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64)" - > curl --silent diff --git a/src/build/build.v b/src/build/build.v index 16942bd..3c9cae5 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -102,16 +102,26 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Gi build_arch := os.uname().machine + repo_url := '$address/$repo.repo' + // TODO what to do with PKGBUILDs that build multiple packages? commands := [ + // This will later be replaced by a proper setting for changing the + // mirrorlist + "echo -e '[$repo.repo]\nServer = $address/\$repo/\$arch\nSigLevel = Optional' >> /etc/pacman.conf" + // We need to update the package list of the repo we just added above. + // This should however not pull in a lot of packages as long as the + // builder image is rebuilt frequently. + 'pacman -Syu --needed --noconfirm', + 'su builder', 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', 'cd repo', 'makepkg --nobuild --syncdeps --needed --noconfirm', 'source PKGBUILD', // The build container checks whether the package is already // present on the server - 'curl -s --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', - 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done', + 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', ] // We convert the list of commands into a base64 string, which then gets @@ -124,7 +134,7 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Gi entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e'] work_dir: '/build' - user: 'builder:builder' + // user: 'builder:builder' } id := dd.create_container(c)?.id -- 2.40.1 From 48e2ae7645f2de788801185bb096aee11cfbf983 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 1 Jun 2022 20:34:36 +0200 Subject: [PATCH 2/2] feat(build): show shell commands in build logs --- CHANGELOG.md | 9 +++++++ src/build/build.v | 33 +++++------------------ src/build/build_script.sh | 20 ++++++++++++++ src/build/shell.v | 55 +++++++++++++++++++++++++++++++++++++++ src/build/shell_test.v | 16 ++++++++++++ 5 files changed, 106 insertions(+), 27 deletions(-) create mode 100644 src/build/build_script.sh create mode 100644 src/build/shell.v create mode 100644 src/build/shell_test.v diff --git a/CHANGELOG.md b/CHANGELOG.md index f9fee6b..affd5c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * BuildLog: filter by start & end date, repo, exit code & arch * CLI flags to take advantage of above API improvements +### Changed + +* Packages from target repo are available during builds + * This can be used as a basic way to support AUR dependencies, by adding + the dependencies to the same repository +* Every build now updates its packages first instead of solely relying on the + updated builder image +* Build logs now show commands being executed + ## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) ### Added diff --git a/src/build/build.v b/src/build/build.v index 3c9cae5..2e86471 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -101,40 +101,19 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Gi } build_arch := os.uname().machine + build_script := create_build_script(address, repo, build_arch) - repo_url := '$address/$repo.repo' - - // TODO what to do with PKGBUILDs that build multiple packages? - commands := [ - // This will later be replaced by a proper setting for changing the - // mirrorlist - "echo -e '[$repo.repo]\nServer = $address/\$repo/\$arch\nSigLevel = Optional' >> /etc/pacman.conf" - // We need to update the package list of the repo we just added above. - // This should however not pull in a lot of packages as long as the - // builder image is rebuilt frequently. - 'pacman -Syu --needed --noconfirm', - 'su builder', - 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', - 'cd repo', - 'makepkg --nobuild --syncdeps --needed --noconfirm', - 'source PKGBUILD', - // The build container checks whether the package is already - // present on the server - 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', - 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', - ] - - // We convert the list of commands into a base64 string, which then gets - // passed to the container as an env var - cmds_str := base64.encode_str(commands.join('\n')) + // We convert the build script into a base64 string, which then gets passed + // to the container as an env var + base64_script := base64.encode_str(build_script) c := docker.NewContainer{ image: '$base_image_id' - env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$api_key'] + env: ['BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e'] work_dir: '/build' - // user: 'builder:builder' + user: '0:0' } id := dd.create_container(c)?.id diff --git a/src/build/build_script.sh b/src/build/build_script.sh new file mode 100644 index 0000000..29f163e --- /dev/null +++ b/src/build/build_script.sh @@ -0,0 +1,20 @@ +echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' +echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf +echo -e '+ pacman -Syu --needed --noconfirm' +pacman -Syu --needed --noconfirm +echo -e '+ su builder' +su builder +echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo' +git clone --single-branch --depth 1 --branch main https://examplerepo.com repo +echo -e '+ cd repo' +cd repo +echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' +makepkg --nobuild --syncdeps --needed --noconfirm +echo -e '+ source PKGBUILD' +source PKGBUILD +echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' +curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 +echo -e '+ [ "$(id -u)" == 0 ] && exit 0' +[ "$(id -u)" == 0 ] && exit 0 +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/shell.v b/src/build/shell.v new file mode 100644 index 0000000..a3121fe --- /dev/null +++ b/src/build/shell.v @@ -0,0 +1,55 @@ +module build + +import models { GitRepo } + +// escape_shell_string escapes any characters that could be interpreted +// incorrectly by a shell. The resulting value should be safe to use inside an +// echo statement. +fn escape_shell_string(s string) string { + return s.replace(r'\', r'\\').replace("'", r"'\''") +} + +// echo_commands takes a list of shell commands & prepends each one with +// an echo call displaying said command. +pub fn echo_commands(cmds []string) []string { + mut out := []string{cap: 2 * cmds.len} + + for cmd in cmds { + out << "echo -e '+ ${escape_shell_string(cmd)}'" + out << cmd + } + + return out +} + +// create_build_script generates a shell script that builds a given GitRepo. +fn create_build_script(address string, repo &GitRepo, build_arch string) string { + repo_url := '$address/$repo.repo' + + commands := echo_commands([ + // This will later be replaced by a proper setting for changing the + // mirrorlist + "echo -e '[$repo.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" + // We need to update the package list of the repo we just added above. + // This should however not pull in a lot of packages as long as the + // builder image is rebuilt frequently. + 'pacman -Syu --needed --noconfirm', + // makepkg can't run as root + 'su builder', + 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', + 'cd repo', + 'makepkg --nobuild --syncdeps --needed --noconfirm', + 'source PKGBUILD', + // The build container checks whether the package is already present on + // the server. + 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + // If the above curl command succeeds, we don't need to rebuild the + // package. However, because we're in a su shell, the exit command will + // drop us back into the root shell. Therefore, we must check whether + // we're in root so we don't proceed. + '[ "\$(id -u)" == 0 ] && exit 0', + 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', + ]) + + return commands.join('\n') +} diff --git a/src/build/shell_test.v b/src/build/shell_test.v new file mode 100644 index 0000000..46ab350 --- /dev/null +++ b/src/build/shell_test.v @@ -0,0 +1,16 @@ +module build + +import models { GitRepo } + +fn test_create_build_script() { + repo := GitRepo{ + id: 1 + url: 'https://examplerepo.com' + branch: 'main' + repo: 'vieter' + } + build_script := create_build_script('https://example.com', repo, 'x86_64') + expected := $embed_file('build_script.sh') + + assert build_script == expected.to_string().trim_space() +} -- 2.40.1