ci: deploy man pages to website

Reviewed-on: vieter/vieter#186

.gitignore

@@ -26,3 +26,4 @@ gdb.txt
 
 # Generated docs
 _docs/
+/man/

.woodpecker/.build.yml

@@ -45,7 +45,7 @@ pipeline:
 
       - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')"
       - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH"
-      - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64`
+      - export SIGNATURE="$(echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64)"
       - >
         curl 
         --silent

.woodpecker/.man.yml

@@ -0,0 +1,36 @@
+platform: 'linux/amd64'
+branches:
+  exclude: [ main ]
+
+depends_on:
+  - build
+
+skip_clone: true
+
+pipeline:
+  generate:
+    image: 'chewingbever/vlang:latest'
+    pull: true
+    commands:
+      - curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64"
+      - chmod +x vieter
+      - ./vieter man man
+      - cd man
+
+      # Generate an HTML page from each man page
+      - for f in $(ls -1 *.1); do mandoc -Thtml -O style=mandoc.css,man=%N.%S.html $f > "$f.html"; done
+
+      # Download the mandoc.css file from the official site
+      - curl -o mandoc.css -L https://mandoc.bsd.lv/mandoc.css
+
+      - tar czvf ../man.tar.gz *.html mandoc.css
+
+  deploy:
+    image: 'curlimages/curl'
+    secrets:
+      - 'site_api_key'
+    commands:
+      - 'curl -XPOST --fail -s -H "Authorization: Bearer $SITE_API_KEY" -T man.tar.gz https://rustybever.be/api/deploy?dir=man-vieter'
+    when:
+      event: push
+      branch: dev

CHANGELOG.md

@@ -15,6 +15,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
     * GitRepo: filter by repo
     * BuildLog: filter by start & end date, repo, exit code & arch
 * CLI flags to take advantage of above API improvements
+* Added CLI command to generate all man pages
+* PKGBUILDs now install man pages
+* CLI man pages are now hosted on https://rustybever.be
+
+### Changed
+
+* Packages from target repo are available during builds
+    * This can be used as a basic way to support AUR dependencies, by adding
+      the dependencies to the same repository
+* Every build now updates its packages first instead of solely relying on the
+  updated builder image
+* Build logs now show commands being executed
 
 ## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2)
 

Makefile

@@ -60,6 +60,11 @@ api-docs:
 	rm -rf '$(SRC_DIR)/_docs'
 	cd '$(SRC_DIR)' && v doc -all -f html -m -readme .
 
+.PHONY: man
+man: vieter
+	rm -rf man
+	./vieter man man
+
 
 # =====OTHER=====
 .PHONY: lint

PKGBUILD

@@ -24,4 +24,7 @@ package() {
 
     install -dm755 "$pkgdir/usr/bin"
     install -Dm755 "$pkgname/pvieter" "$pkgdir/usr/bin/vieter"
+
+    install -dm755 "$pkgdir/usr/share/man/man1"
+    ./vieter man "$pkgdir/usr/share/man/man1"
 }

PKGBUILD.dev

@@ -32,4 +32,7 @@ package() {
 
     install -dm755 "$pkgdir/usr/bin"
     install -Dm755 "$pkgname/pvieter" "$pkgdir/usr/bin/vieter"
+
+    install -dm755 "$pkgdir/usr/share/man/man1"
+    ./vieter man "$pkgdir/usr/share/man/man1"
 }

src/build/build.v

@@ -101,30 +101,19 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Gi
 	}
 
 	build_arch := os.uname().machine
+	build_script := create_build_script(address, repo, build_arch)
 
-	// TODO what to do with PKGBUILDs that build multiple packages?
-	commands := [
-		'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
-		'cd repo',
-		'makepkg --nobuild --syncdeps --needed --noconfirm',
-		'source PKGBUILD',
-		// The build container checks whether the package is already
-		// present on the server
-		'curl -s --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
-		'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done',
-	]
-
-	// We convert the list of commands into a base64 string, which then gets
-	// passed to the container as an env var
-	cmds_str := base64.encode_str(commands.join('\n'))
+	// We convert the build script into a base64 string, which then gets passed
+	// to the container as an env var
+	base64_script := base64.encode_str(build_script)
 
 	c := docker.NewContainer{
 		image: '$base_image_id'
-		env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$api_key']
+		env: ['BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key']
 		entrypoint: ['/bin/sh', '-c']
 		cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e']
 		work_dir: '/build'
-		user: 'builder:builder'
+		user: '0:0'
 	}
 
 	id := dd.create_container(c)?.id

src/build/build_script.sh

@@ -0,0 +1,20 @@
+echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf'
+echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
+echo -e '+ pacman -Syu --needed --noconfirm'
+pacman -Syu --needed --noconfirm
+echo -e '+ su builder'
+su builder
+echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo'
+git clone --single-branch --depth 1 --branch main https://examplerepo.com repo
+echo -e '+ cd repo'
+cd repo
+echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'
+makepkg --nobuild --syncdeps --needed --noconfirm
+echo -e '+ source PKGBUILD'
+source PKGBUILD
+echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0'
+curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
+echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
+[ "$(id -u)" == 0 ] && exit 0
+echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
+MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done

src/build/shell.v

@@ -0,0 +1,55 @@
+module build
+
+import models { GitRepo }
+
+// escape_shell_string escapes any characters that could be interpreted
+// incorrectly by a shell. The resulting value should be safe to use inside an
+// echo statement.
+fn escape_shell_string(s string) string {
+	return s.replace(r'\', r'\\').replace("'", r"'\''")
+}
+
+// echo_commands takes a list of shell commands & prepends each one with
+// an echo call displaying said command.
+pub fn echo_commands(cmds []string) []string {
+	mut out := []string{cap: 2 * cmds.len}
+
+	for cmd in cmds {
+		out << "echo -e '+ ${escape_shell_string(cmd)}'"
+		out << cmd
+	}
+
+	return out
+}
+
+// create_build_script generates a shell script that builds a given GitRepo.
+fn create_build_script(address string, repo &GitRepo, build_arch string) string {
+	repo_url := '$address/$repo.repo'
+
+	commands := echo_commands([
+		// This will later be replaced by a proper setting for changing the
+		// mirrorlist
+		"echo -e '[$repo.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf"
+		// We need to update the package list of the repo we just added above.
+		// This should however not pull in a lot of packages as long as the
+		// builder image is rebuilt frequently.
+		'pacman -Syu --needed --noconfirm',
+		// makepkg can't run as root
+		'su builder',
+		'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
+		'cd repo',
+		'makepkg --nobuild --syncdeps --needed --noconfirm',
+		'source PKGBUILD',
+		// The build container checks whether the package is already present on
+		// the server.
+		'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
+		// If the above curl command succeeds, we don't need to rebuild the
+		// package. However, because we're in a su shell, the exit command will
+		// drop us back into the root shell. Therefore, we must check whether
+		// we're in root so we don't proceed.
+		'[ "\$(id -u)" == 0 ] && exit 0',
+		'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done',
+	])
+
+	return commands.join('\n')
+}

src/build/shell_test.v

@@ -0,0 +1,16 @@
+module build
+
+import models { GitRepo }
+
+fn test_create_build_script() {
+	repo := GitRepo{
+		id: 1
+		url: 'https://examplerepo.com'
+		branch: 'main'
+		repo: 'vieter'
+	}
+	build_script := create_build_script('https://example.com', repo, 'x86_64')
+	expected := $embed_file('build_script.sh')
+
+	assert build_script == expected.to_string().trim_space()
+}

src/console/console.v

@@ -2,6 +2,8 @@ module console
 
 import arrays
 import strings
+import cli
+import os
 
 // pretty_table converts a list of string data into a pretty table. Many thanks
 // to @hungrybluedev in the Vlang Discord for providing this code!
@@ -54,3 +56,15 @@ pub fn pretty_table(header []string, data [][]string) ?string {
 
 	return buffer.str()
 }
+
+// export_man_pages recursively generates all man pages for the given
+// cli.Command & writes them to the given directory.
+pub fn export_man_pages(cmd cli.Command, path string) ? {
+	man := cmd.manpage()
+	os.write_file(os.join_path_single(path, cmd.full_name().replace(' ', '-') + '.1'),
+		man)?
+
+	for sub_cmd in cmd.commands {
+		export_man_pages(sub_cmd, path)?
+	}
+}

src/console/man/man.v

@@ -0,0 +1,21 @@
+module man
+
+import cli
+import console
+import os
+
+// cmd returns the cli submodule that handles generating man pages.
+pub fn cmd() cli.Command {
+	return cli.Command{
+		name: 'man'
+		description: 'Generate all man pages & save them in the given directory.'
+		usage: 'dir'
+		required_args: 1
+		execute: fn (cmd cli.Command) ? {
+			root := cmd.root()
+			os.mkdir_all(cmd.args[0])?
+
+			console.export_man_pages(root, cmd.args[0])?
+		}
+	}
+}

src/main.v

@@ -6,6 +6,7 @@ import cli
 import console.git
 import console.logs
 import console.schedule
+import console.man
 import cron
 
 fn main() {
@@ -29,6 +30,7 @@ fn main() {
 			cron.cmd(),
 			logs.cmd(),
 			schedule.cmd(),
+			man.cmd(),
 		]
 	}
 	app.setup()