Merge pull request 'Repo in build & other build improvements' (#186) from Chewing_Bever/vieter:repo-in-builds into dev

Reviewed-on: vieter/vieter#186
Jef Roosens 2022-06-01 20:49:47 +02:00
commit 06df2c21f0
6 changed files with 107 additions and 18 deletions

View File

@ -45,7 +45,7 @@ pipeline:
- export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')"
- export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH"
- export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64`
- export SIGNATURE="$(echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64)"
- >
curl
--silent

View File

@ -16,6 +16,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* BuildLog: filter by start & end date, repo, exit code & arch
* CLI flags to take advantage of above API improvements
### Changed
* Packages from target repo are available during builds
* This can be used as a basic way to support AUR dependencies, by adding
the dependencies to the same repository
* Every build now updates its packages first instead of solely relying on the
updated builder image
* Build logs now show commands being executed
## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2)
### Added

View File

@ -101,30 +101,19 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Gi
}
build_arch := os.uname().machine
build_script := create_build_script(address, repo, build_arch)
// TODO what to do with PKGBUILDs that build multiple packages?
commands := [
'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
'cd repo',
'makepkg --nobuild --syncdeps --needed --noconfirm',
'source PKGBUILD',
// The build container checks whether the package is already
// present on the server
'curl -s --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done',
]
// We convert the list of commands into a base64 string, which then gets
// passed to the container as an env var
cmds_str := base64.encode_str(commands.join('\n'))
// We convert the build script into a base64 string, which then gets passed
// to the container as an env var
base64_script := base64.encode_str(build_script)
c := docker.NewContainer{
image: '$base_image_id'
env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$api_key']
env: ['BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key']
entrypoint: ['/bin/sh', '-c']
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e']
work_dir: '/build'
user: 'builder:builder'
user: '0:0'
}
id := dd.create_container(c)?.id

View File

@ -0,0 +1,20 @@
echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf'
echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
echo -e '+ pacman -Syu --needed --noconfirm'
pacman -Syu --needed --noconfirm
echo -e '+ su builder'
su builder
echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo'
git clone --single-branch --depth 1 --branch main https://examplerepo.com repo
echo -e '+ cd repo'
cd repo
echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'
makepkg --nobuild --syncdeps --needed --noconfirm
echo -e '+ source PKGBUILD'
source PKGBUILD
echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0'
curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
[ "$(id -u)" == 0 ] && exit 0
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done

55
src/build/shell.v 100644
View File

@ -0,0 +1,55 @@
module build
import models { GitRepo }
// escape_shell_string escapes any characters that could be interpreted
// incorrectly by a shell. The resulting value should be safe to use inside an
// echo statement.
fn escape_shell_string(s string) string {
return s.replace(r'\', r'\\').replace("'", r"'\''")
}
// echo_commands takes a list of shell commands & prepends each one with
// an echo call displaying said command.
pub fn echo_commands(cmds []string) []string {
mut out := []string{cap: 2 * cmds.len}
for cmd in cmds {
out << "echo -e '+ ${escape_shell_string(cmd)}'"
out << cmd
}
return out
}
// create_build_script generates a shell script that builds a given GitRepo.
fn create_build_script(address string, repo &GitRepo, build_arch string) string {
repo_url := '$address/$repo.repo'
commands := echo_commands([
// This will later be replaced by a proper setting for changing the
// mirrorlist
"echo -e '[$repo.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf"
// We need to update the package list of the repo we just added above.
// This should however not pull in a lot of packages as long as the
// builder image is rebuilt frequently.
'pacman -Syu --needed --noconfirm',
// makepkg can't run as root
'su builder',
'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
'cd repo',
'makepkg --nobuild --syncdeps --needed --noconfirm',
'source PKGBUILD',
// The build container checks whether the package is already present on
// the server.
'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
// If the above curl command succeeds, we don't need to rebuild the
// package. However, because we're in a su shell, the exit command will
// drop us back into the root shell. Therefore, we must check whether
// we're in root so we don't proceed.
'[ "\$(id -u)" == 0 ] && exit 0',
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done',
])
return commands.join('\n')
}

View File

@ -0,0 +1,16 @@
module build
import models { GitRepo }
fn test_create_build_script() {
repo := GitRepo{
id: 1
url: 'https://examplerepo.com'
branch: 'main'
repo: 'vieter'
}
build_script := create_build_script('https://example.com', repo, 'x86_64')
expected := $embed_file('build_script.sh')
assert build_script == expected.to_string().trim_space()
}