Merge pull request 'Isolate builds, build when necessary' (#107) from build-system-upgrade into dev

Reviewed-on: Chewing_Bever/vieter#107

.woodpecker/.deploy.yml

@@ -10,6 +10,8 @@ pipeline:
   webhook:
     image: chewingbever/vlang:latest
     secrets:
-      - webhook
+      - webhook_1
+      - webhook_2
     commands:
-      - curl -XPOST -s "$WEBHOOK"
+      - curl -XPOST -s "$WEBHOOK_1"
+      - curl -XPOST -s "$WEBHOOK_2"

src/build.v

@@ -2,7 +2,6 @@ module main
 
 import docker
 import encoding.base64
-import rand
 import time
 import json
 import server
@@ -11,17 +10,10 @@ import net.http
 
 const container_build_dir = '/build'
 
-fn build() ? {
+const build_image_repo = 'vieter-build'
-	conf := env.load<env.BuildConfig>() ?
 
-	// We get the repos list from the Vieter instance
+fn create_build_image() ?string {
-	mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ?
+	commands := [
-	req.add_custom_header('X-Api-Key', conf.api_key) ?
-
-	res := req.do() ?
-	repos := json.decode([]server.GitRepo, res.text) ?
-
-	mut commands := [
 		// Update repos & install required packages
 		'pacman -Syu --needed --noconfirm base-devel git'
 		// Add a non-root user to run makepkg
@@ -34,31 +26,11 @@ fn build() ? {
 		'mkdir /build',
 		'chown -R builder:builder /build',
 	]
-
-	// Each repo gets a unique UUID to avoid naming conflicts when cloning
-	mut uuids := []string{}
-
-	for repo in repos {
-		mut uuid := rand.uuid_v4()
-
-		// Just to be sure we don't have any collisions
-		for uuids.contains(uuid) {
-			uuid = rand.uuid_v4()
-		}
-
-		uuids << uuid
-
-		commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'"
-		commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done\''
-	}
-
-	// We convert the list of commands into a base64 string, which then gets
-	// passed to the container as an env var
 	cmds_str := base64.encode_str(commands.join('\n'))
 
 	c := docker.NewContainer{
 		image: 'archlinux:latest'
-		env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key']
+		env: ['BUILD_SCRIPT=$cmds_str']
 		entrypoint: ['/bin/sh', '-c']
 		cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e']
 	}
@@ -81,5 +53,76 @@ fn build() ? {
 		time.sleep(5000000000)
 	}
 
+	// Finally, we create the image from the container
+	// As the tag, we use the epoch value
+	tag := time.sys_mono_now().str()
+	image := docker.create_image_from_container(id, 'vieter-build', tag) ?
+	docker.remove_container(id) ?
+
+	return image.id
+}
+
+fn build() ? {
+	conf := env.load<env.BuildConfig>() ?
+
+	// We get the repos list from the Vieter instance
+	mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ?
+	req.add_custom_header('X-Api-Key', conf.api_key) ?
+
+	res := req.do() ?
+	repos := json.decode([]server.GitRepo, res.text) ?
+
+	// No point in doing work if there's no repos present
+	if repos.len == 0 {
+		return
+	}
+
+	// First, we create a base image which has updated repos n stuff
+	image_id := create_build_image() ?
+
+	for repo in repos {
+		// TODO what to do with PKGBUILDs that build multiple packages?
+		commands := [
+			'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
+			'cd repo',
+			'makepkg --nobuild --nodeps',
+			'source PKGBUILD',
+			// The build container checks whether the package is already present on the server
+			'curl --head --fail $conf.address/\$pkgname-\$pkgver-\$pkgrel-\$(uname -m).pkg.tar.zst && exit 0',
+			'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/publish; done',
+		]
+
+		// We convert the list of commands into a base64 string, which then gets
+		// passed to the container as an env var
+		cmds_str := base64.encode_str(commands.join('\n'))
+
+		c := docker.NewContainer{
+			image: '$image_id'
+			env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key']
+			entrypoint: ['/bin/sh', '-c']
+			cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e']
+			work_dir: '/build'
+			user: 'builder:builder'
+		}
+
+		id := docker.create_container(c) ?
+		docker.start_container(id) ?
+
+		// This loop waits until the container has stopped, so we can remove it after
+		for {
+			data := docker.inspect_container(id) ?
+
+			if !data.state.running {
+				break
+			}
+
+			// Wait for 5 seconds
+			time.sleep(5000000000)
+		}
+
 		docker.remove_container(id) ?
 	}
+
+	// Finally, we remove the builder image
+	docker.remove_image(image_id) ?
+}

src/docker/containers.v

@@ -20,6 +20,8 @@ pub struct NewContainer {
 	entrypoint []string [json: Entrypoint]
 	cmd        []string [json: Cmd]
 	env        []string [json: Env]
+	work_dir   string   [json: WorkingDir]
+	user       string   [json: User]
 }
 
 struct CreatedContainer {

src/docker/docker.v

@@ -91,8 +91,3 @@ pub fn request_with_json<T>(method string, url urllib.URL, data &T) ?http.Respon
 
 	return request_with_body(method, url, 'application/json', body)
 }
-
-// pull_image pulls tries to pull the image for the given image & tag
-pub fn pull_image(image string, tag string) ?http.Response {
-	return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?)
-}

src/docker/images.v

@@ -0,0 +1,34 @@
+module docker
+
+import net.http
+import net.urllib
+import json
+
+struct Image {
+pub:
+	id string [json: Id]
+}
+
+// pull_image pulls tries to pull the image for the given image & tag
+pub fn pull_image(image string, tag string) ?http.Response {
+	return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?)
+}
+
+// create_image_from_container creates a new image from a container with the
+// given repo & tag, given the container's ID.
+pub fn create_image_from_container(id string, repo string, tag string) ?Image {
+	res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag') ?) ?
+
+	if res.status_code != 201 {
+		return error('Failed to create image from container.')
+	}
+
+	return json.decode(Image, res.text) or {}
+}
+
+// remove_image removes the image with the given ID.
+pub fn remove_image(id string) ?bool {
+	res := request('DELETE', urllib.parse('/v1.41/images/$id') ?) ?
+
+	return res.status_code == 200
+}

src/server/routes.v

@@ -6,6 +6,7 @@ import repo
 import time
 import rand
 import util
+import net.http
 
 // healthcheck just returns a string, but can be used to quickly check if the
 // server is still responsive.
@@ -15,7 +16,7 @@ pub fn (mut app App) healthcheck() web.Result {
 }
 
 // get_root handles a GET request for a file on the root
-['/:filename'; get]
+['/:filename'; get; head]
 fn (mut app App) get_root(filename string) web.Result {
 	mut full_path := ''
 
@@ -27,6 +28,15 @@ fn (mut app App) get_root(filename string) web.Result {
 		full_path = os.join_path_single(app.repo.pkg_dir, filename)
 	}
 
+	// Scuffed way to respond to HEAD requests
+	if app.req.method == http.Method.head {
+		if os.exists(full_path) {
+			return app.ok('')
+		}
+
+		return app.not_found()
+	}
+
 	return app.file(full_path)
 }