Merge pull request 'Repo in build & other build improvements' (#186) from Chewing_Bever/vieter:repo-in-builds into dev

Reviewed-on: vieter/vieter#186

.woodpecker/.build.yml

@@ -45,7 +45,7 @@ pipeline:
 
       - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')"
       - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH"
-      - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64`
+      - export SIGNATURE="$(echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64)"
       - >
         curl 
         --silent

CHANGELOG.md

@@ -16,6 +16,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
     * BuildLog: filter by start & end date, repo, exit code & arch
 * CLI flags to take advantage of above API improvements
 
+### Changed
+
+* Packages from target repo are available during builds
+    * This can be used as a basic way to support AUR dependencies, by adding
+      the dependencies to the same repository
+* Every build now updates its packages first instead of solely relying on the
+  updated builder image
+* Build logs now show commands being executed
+
 ## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2)
 
 ### Added

src/build/build.v

@@ -101,30 +101,19 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Gi
 	}
 
 	build_arch := os.uname().machine
+	build_script := create_build_script(address, repo, build_arch)
 
-	// TODO what to do with PKGBUILDs that build multiple packages?
+	// We convert the build script into a base64 string, which then gets passed
-	commands := [
+	// to the container as an env var
-		'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
+	base64_script := base64.encode_str(build_script)
-		'cd repo',
-		'makepkg --nobuild --syncdeps --needed --noconfirm',
-		'source PKGBUILD',
-		// The build container checks whether the package is already
-		// present on the server
-		'curl -s --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
-		'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done',
-	]
-
-	// We convert the list of commands into a base64 string, which then gets
-	// passed to the container as an env var
-	cmds_str := base64.encode_str(commands.join('\n'))
 
 	c := docker.NewContainer{
 		image: '$base_image_id'
-		env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$api_key']
+		env: ['BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key']
 		entrypoint: ['/bin/sh', '-c']
 		cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e']
 		work_dir: '/build'
-		user: 'builder:builder'
+		user: '0:0'
 	}
 
 	id := dd.create_container(c)?.id

src/build/build_script.sh

@@ -0,0 +1,20 @@
+echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf'
+echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
+echo -e '+ pacman -Syu --needed --noconfirm'
+pacman -Syu --needed --noconfirm
+echo -e '+ su builder'
+su builder
+echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo'
+git clone --single-branch --depth 1 --branch main https://examplerepo.com repo
+echo -e '+ cd repo'
+cd repo
+echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'
+makepkg --nobuild --syncdeps --needed --noconfirm
+echo -e '+ source PKGBUILD'
+source PKGBUILD
+echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0'
+curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
+echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
+[ "$(id -u)" == 0 ] && exit 0
+echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
+MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done

src/build/shell.v

@@ -0,0 +1,55 @@
+module build
+
+import models { GitRepo }
+
+// escape_shell_string escapes any characters that could be interpreted
+// incorrectly by a shell. The resulting value should be safe to use inside an
+// echo statement.
+fn escape_shell_string(s string) string {
+	return s.replace(r'\', r'\\').replace("'", r"'\''")
+}
+
+// echo_commands takes a list of shell commands & prepends each one with
+// an echo call displaying said command.
+pub fn echo_commands(cmds []string) []string {
+	mut out := []string{cap: 2 * cmds.len}
+
+	for cmd in cmds {
+		out << "echo -e '+ ${escape_shell_string(cmd)}'"
+		out << cmd
+	}
+
+	return out
+}
+
+// create_build_script generates a shell script that builds a given GitRepo.
+fn create_build_script(address string, repo &GitRepo, build_arch string) string {
+	repo_url := '$address/$repo.repo'
+
+	commands := echo_commands([
+		// This will later be replaced by a proper setting for changing the
+		// mirrorlist
+		"echo -e '[$repo.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf"
+		// We need to update the package list of the repo we just added above.
+		// This should however not pull in a lot of packages as long as the
+		// builder image is rebuilt frequently.
+		'pacman -Syu --needed --noconfirm',
+		// makepkg can't run as root
+		'su builder',
+		'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
+		'cd repo',
+		'makepkg --nobuild --syncdeps --needed --noconfirm',
+		'source PKGBUILD',
+		// The build container checks whether the package is already present on
+		// the server.
+		'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
+		// If the above curl command succeeds, we don't need to rebuild the
+		// package. However, because we're in a su shell, the exit command will
+		// drop us back into the root shell. Therefore, we must check whether
+		// we're in root so we don't proceed.
+		'[ "\$(id -u)" == 0 ] && exit 0',
+		'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done',
+	])
+
+	return commands.join('\n')
+}

src/build/shell_test.v

@@ -0,0 +1,16 @@
+module build
+
+import models { GitRepo }
+
+fn test_create_build_script() {
+	repo := GitRepo{
+		id: 1
+		url: 'https://examplerepo.com'
+		branch: 'main'
+		repo: 'vieter'
+	}
+	build_script := create_build_script('https://example.com', repo, 'x86_64')
+	expected := $embed_file('build_script.sh')
+
+	assert build_script == expected.to_string().trim_space()
+}