From 9c3af3411865f8ca1ea898a90635a1046a77e027 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 5 May 2022 09:35:19 +0200 Subject: [PATCH 01/24] ci: removed a comment --- .woodpecker/.build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index f9cab001..c89348b1 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -2,10 +2,7 @@ matrix: PLATFORM: - linux/amd64 - linux/arm64 - # I just don't have a performant enough runner for this platform - # - linux/arm/v7 -# These checks already get performed on the feature branches platform: ${PLATFORM} pipeline: From 8c5652c2301a55e0c44c07653f628ae7349e6bdf Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 5 May 2022 09:35:19 +0200 Subject: [PATCH 02/24] ci: made build upload failable; updated ci for use with PRs --- .woodpecker/.arch.yml | 4 ++++ .woodpecker/.build.yml | 16 +++++++--------- .woodpecker/.build_experimental.yml | 29 ----------------------------- .woodpecker/.deploy.yml | 4 +++- .woodpecker/.docker.yml | 28 ++++++++++++++++------------ .woodpecker/.gitea.yml | 5 ++--- .woodpecker/.lint.yml | 6 ++++-- .woodpecker/.test.yml | 8 ++++---- src/db/git.v | 4 +--- src/server/git.v | 1 - 10 files changed, 41 insertions(+), 64 deletions(-) delete mode 100644 .woodpecker/.build_experimental.yml diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index 742095c0..93c98407 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -25,6 +25,8 @@ pipeline: # inside the repo - curl -OL https://git.rustybever.be/Chewing_Bever/vieter/raw/branch/dev/PKGBUILD - makepkg -s --noconfirm --needed + when: + event: push publish: image: 'curlimages/curl' @@ -33,3 +35,5 @@ pipeline: - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done' secrets: - vieter_api_key + when: + event: push diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index f9cab001..a3b66ea6 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -1,11 +1,8 @@ matrix: PLATFORM: - - linux/amd64 - - linux/arm64 - # I just don't have a performant enough runner for this platform - # - linux/arm/v7 + - 'linux/amd64' + - 'linux/arm64' -# These checks already get performed on the feature branches platform: ${PLATFORM} pipeline: @@ -15,9 +12,9 @@ pipeline: commands: - make when: - event: push + event: [push, pull_request] branch: - exclude: [main, dev] + exclude: [main] prod: image: 'chewingbever/vlang:latest' @@ -35,7 +32,7 @@ pipeline: - strip -s pvieter - du -h pvieter when: - event: push + event: [push, pull_request] upload: image: 'chewingbever/vlang:latest' @@ -52,6 +49,7 @@ pipeline: - > curl --silent + --fail -XPUT -T pvieter -H "Host: $URL" @@ -60,4 +58,4 @@ pipeline: -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" https://$URL$OBJ_PATH when: - event: push + event: [push, pull_request] diff --git a/.woodpecker/.build_experimental.yml b/.woodpecker/.build_experimental.yml deleted file mode 100644 index 0129d2b4..00000000 --- a/.woodpecker/.build_experimental.yml +++ /dev/null @@ -1,29 +0,0 @@ -# These builds are not important for the project, but might be valuable for -# fixing bugs in the V compiler. - -platform: linux/amd64 -branches: - exclude: [master, dev] - -pipeline: - autofree: - image: 'chewingbever/vlang:latest' - pull: true - group: 'build' - commands: - - make autofree - - readelf -d afvieter - - du -h afvieter - when: - event: push - - skip-unused: - image: 'chewingbever/vlang:latest' - pull: true - group: 'build' - commands: - - make skip-unused - - readelf -d suvieter - - du -h suvieter - when: - event: push diff --git a/.woodpecker/.deploy.yml b/.woodpecker/.deploy.yml index dd77fb98..8e68641e 100644 --- a/.woodpecker/.deploy.yml +++ b/.woodpecker/.deploy.yml @@ -1,4 +1,4 @@ -branches: 'dev' +branches: [ 'dev' ] platform: 'linux/amd64' depends_on: - 'docker' @@ -14,3 +14,5 @@ pipeline: commands: - 'curl -XPOST -s --fail $WEBHOOK_APP' - 'curl -XPOST -s --fail $WEBHOOK_CRON' + when: + event: push diff --git a/.woodpecker/.docker.yml b/.woodpecker/.docker.yml index 9b605f3f..bab869b4 100644 --- a/.woodpecker/.docker.yml +++ b/.woodpecker/.docker.yml @@ -1,30 +1,34 @@ branches: [main, dev] -platform: linux/amd64 +platform: 'linux/amd64' depends_on: - build pipeline: dev: - image: woodpeckerci/plugin-docker-buildx - secrets: [ docker_username, docker_password ] + image: 'woodpeckerci/plugin-docker-buildx' + secrets: + - 'docker_username' + - 'docker_password' settings: - repo: chewingbever/vieter - tag: dev - platforms: [ linux/arm64/v8, linux/amd64 ] + repo: 'chewingbever/vieter' + tag: 'dev' + platforms: [ 'linux/arm64/v8', 'linux/amd64' ] build_args_from_env: - - CI_COMMIT_SHA + - 'CI_COMMIT_SHA' when: event: push branch: dev release: - image: woodpeckerci/plugin-docker-buildx - secrets: [ docker_username, docker_password ] + image: 'woodpeckerci/plugin-docker-buildx' + secrets: + - 'docker_username' + - 'docker_password' settings: - repo: chewingbever/vieter + repo: 'chewingbever/vieter' auto_tag: true - platforms: [ linux/arm64/v8, linux/amd64 ] + platforms: [ 'linux/arm64/v8', 'linux/amd64' ] build_args_from_env: - - CI_COMMIT_SHA + - 'CI_COMMIT_SHA' when: event: tag diff --git a/.woodpecker/.gitea.yml b/.woodpecker/.gitea.yml index c492d345..18770502 100644 --- a/.woodpecker/.gitea.yml +++ b/.woodpecker/.gitea.yml @@ -1,6 +1,5 @@ -# Yeah so this only works on tags so we'll worry about this later -platform: linux/amd64 -branches: main +platform: 'linux/amd64' +branches: [ 'main' ] depends_on: - build diff --git a/.woodpecker/.lint.yml b/.woodpecker/.lint.yml index b1c16fd8..b13aea4a 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/.lint.yml @@ -1,7 +1,7 @@ # These checks already get performed on the feature branches branches: - exclude: [ main, dev ] -platform: linux/amd64 + exclude: [ main ] +platform: 'linux/amd64' pipeline: lint: @@ -9,3 +9,5 @@ pipeline: pull: true commands: - make lint + when: + event: [ push, pull_request ] diff --git a/.woodpecker/.test.yml b/.woodpecker/.test.yml index 3800cc1d..03375705 100644 --- a/.woodpecker/.test.yml +++ b/.woodpecker/.test.yml @@ -1,10 +1,10 @@ matrix: PLATFORM: - - linux/amd64 - - linux/arm64 + - 'linux/amd64' + - 'linux/arm64' branches: - exclude: [main, dev] + exclude: [ main ] platform: ${PLATFORM} pipeline: @@ -14,4 +14,4 @@ pipeline: commands: - make test when: - event: push + event: [push, pull_request] diff --git a/src/db/git.v b/src/db/git.v index 2fb2a92e..c40086b2 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -118,11 +118,9 @@ pub fn (db &VieterDb) delete_git_repo(repo_id int) { // update_git_repo updates any non-array values for a given GitRepo. pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) { - // sql db.conn { - // update GitRepo set repo - //} mut values := []string{} + // TODO does this allow for SQL injection? $for field in GitRepo.fields { if field.name in params { // Any fields that are array types require their own update method diff --git a/src/server/git.v b/src/server/git.v index 6c852b84..c5cbc0a6 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -60,7 +60,6 @@ fn (mut app App) delete_repo(id int) web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - // repos.delete(id) app.db.delete_git_repo(id) return app.json(http.Status.ok, new_response('Repo removed successfully.')) From e00813398183bc1d38d936695739f45b6344c160 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 5 May 2022 23:29:08 +0200 Subject: [PATCH 03/24] ci(arch): changed PKGBUILD to new URL --- PKGBUILD | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/PKGBUILD b/PKGBUILD index eb866834..87c575ff 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -4,12 +4,12 @@ pkgbase='vieter' pkgname='vieter' pkgver=0.2.0.r25.g20112b8 pkgrel=1 -depends=('glibc' 'openssl' 'libarchive' 'gc') +depends=('glibc' 'openssl' 'libarchive' 'gc' 'sqlite') makedepends=('git' 'gcc' 'vieter-v') -arch=('x86_64' 'aarch64' 'armv7') -url='https://git.rustybever.be/Chewing_Bever/vieter' +arch=('x86_64' 'aarch64') +url='https://git.rustybever.be/vieter/vieter' license=('AGPL3') -source=($pkgname::git+https://git.rustybever.be/Chewing_Bever/vieter#branch=dev) +source=($pkgname::git+https://git.rustybever.be/vieter/vieter#branch=dev) md5sums=('SKIP') pkgver() { From 1990ade089856559c21793f1692c929685a361e6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 5 May 2022 23:30:54 +0200 Subject: [PATCH 04/24] ci: fixed some steps running when not required --- .woodpecker/.build.yml | 2 +- .woodpecker/.lint.yml | 2 +- .woodpecker/.test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index a3b66ea6..16981293 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -12,7 +12,7 @@ pipeline: commands: - make when: - event: [push, pull_request] + event: [pull_request] branch: exclude: [main] diff --git a/.woodpecker/.lint.yml b/.woodpecker/.lint.yml index b13aea4a..e70648d8 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/.lint.yml @@ -10,4 +10,4 @@ pipeline: commands: - make lint when: - event: [ push, pull_request ] + event: [ pull_request ] diff --git a/.woodpecker/.test.yml b/.woodpecker/.test.yml index 03375705..6b7b646d 100644 --- a/.woodpecker/.test.yml +++ b/.woodpecker/.test.yml @@ -14,4 +14,4 @@ pipeline: commands: - make test when: - event: [push, pull_request] + event: [pull_request] From 7fdbcdf3e7388ef6139afb5f55d07483609b9c98 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 5 May 2022 23:38:12 +0200 Subject: [PATCH 05/24] ci(arch): also change URL of downloaded PKGBUILD --- .woodpecker/.arch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index 93c98407..6b8f8f2f 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -23,7 +23,7 @@ pipeline: - su builder # Due to a bug with the V compiler, we can't just use the PKGBUILD from # inside the repo - - curl -OL https://git.rustybever.be/Chewing_Bever/vieter/raw/branch/dev/PKGBUILD + - curl -OL https://git.rustybever.be/vieter/vieter/raw/branch/dev/PKGBUILD - makepkg -s --noconfirm --needed when: event: push From a3b66801535166cb9bc7b5ee6173a0ba8c121e08 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 6 May 2022 08:31:59 +0200 Subject: [PATCH 06/24] cron: filter out repos with wrong architecture --- src/cron/daemon/daemon.v | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 35cca5f1..ffa2f6e2 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -9,6 +9,7 @@ import math import build import docker import db +import os const ( // How many seconds to wait before retrying to update API if failed @@ -19,7 +20,6 @@ const ( struct ScheduledBuild { pub: - repo_id string repo db.GitRepo timestamp time.Time } @@ -187,6 +187,10 @@ fn (mut d Daemon) renew_repos() { return } + // Filter out any repos that shouldn't run on this architecture + cur_arch := os.uname().machine + new_repos = new_repos.filter(it.arch.any(it.value == cur_arch)) + d.repos = new_repos d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency) From 356a34ab01f7a9ffbb9c36b6fd8c7de6bf8b1cb2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 6 May 2022 20:04:48 +0200 Subject: [PATCH 07/24] chore: bumped versions --- CHANGELOG.md | 8 ++++---- src/main.v | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f40ce526..2bbe4f06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter) +## [0.3.0-alpha.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.1) ### Changed @@ -24,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Binary no longer panics when an env var is missing -## [0.2.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.2.0) +## [0.2.0](https://git.rustybever.be/vieter/vieter/src/tag/0.2.0) ### Changed @@ -58,13 +58,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Packages with unknown fields in .PKGINFO are now allowed * Old packages are now properly removed -## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0) +## [0.1.0](https://git.rustybever.be/vieter/vieter/src/tag/0.1.0) ### Changed * Improved logging -## [0.1.0-rc.1](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0-rc.1) +## [0.1.0-rc.1](https://git.rustybever.be/vieter/vieter/src/tag/0.1.0-rc.1) ### Added diff --git a/src/main.v b/src/main.v index 37cabc3a..4ba6d30f 100644 --- a/src/main.v +++ b/src/main.v @@ -11,7 +11,7 @@ fn main() { mut app := cli.Command{ name: 'vieter' description: 'Vieter is a lightweight implementation of an Arch repository server.' - version: '0.2.0' + version: '0.3.0-alpha.1' flags: [ cli.Flag{ flag: cli.FlagType.string From 58c1ecd25e82f4466093151525454cf420ecfc6e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 14:16:30 +0200 Subject: [PATCH 08/24] db: added BuildLog & required methods --- src/db/db.v | 1 + src/db/git.v | 2 +- src/db/logs.v | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 src/db/logs.v diff --git a/src/db/db.v b/src/db/db.v index a75c34c4..5ec240de 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -12,6 +12,7 @@ pub fn init(db_path string) ?VieterDb { sql conn { create table GitRepo + create table BuildLog } return VieterDb{ diff --git a/src/db/git.v b/src/db/git.v index c40086b2..b7791401 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -94,7 +94,7 @@ pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo { // If a select statement fails, it returns a zeroed object. By // checking one of the required fields, we can see whether the query // returned a result or not. - if res.url == '' { + if res.id == 0 { return none } diff --git a/src/db/logs.v b/src/db/logs.v new file mode 100644 index 00000000..3e5b6008 --- /dev/null +++ b/src/db/logs.v @@ -0,0 +1,47 @@ +module db + +import time + +pub struct BuildLog { + id int [primary; sql: serial] + repo GitRepo [nonull] + start_time time.Time [nonull] + end_time time.Time [nonull] + exit_code int [nonull] +} + +// get_build_logs returns all BuildLog's in the database. +pub fn (db &VieterDb) get_build_logs() []BuildLog { + res := sql db.conn { + select from BuildLog order by id + } + + return res +} + +// get_build_log tries to return a specific BuildLog. +pub fn (db &VieterDb) get_build_log(id int) ?BuildLog { + res := sql db.conn { + select from BuildLog where id == id + } + + if res.id == 0 { + return none + } + + return res +} + +// add_build_log inserts the given BuildLog into the database. +pub fn (db &VieterDb) add_build_log(log BuildLog) { + sql db.conn { + insert log into BuildLog + } +} + +// delete_build_log delete the BuildLog with the given ID from the database. +pub fn (db &VieterDb) delete_build_log(id int) { + sql db.conn { + delete from BuildLog where id == id + } +} From 7e01dbafec2ecb6dd92debc79a2b9f0f107d7198 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 15:10:07 +0200 Subject: [PATCH 09/24] feat(server): added endpoints for listing & uploading build logs --- src/server/logs.v | 99 +++++++++++++++++++++++++++++++++++++++++++++ src/server/server.v | 9 +++++ 2 files changed, 108 insertions(+) create mode 100644 src/server/logs.v diff --git a/src/server/logs.v b/src/server/logs.v new file mode 100644 index 00000000..01116b4b --- /dev/null +++ b/src/server/logs.v @@ -0,0 +1,99 @@ +module server + +import web +import net.http +import net.urllib +import response { new_data_response, new_response } +import db +import time +import os +import util + +['/api/logs'; get] +fn (mut app App) get_logs() web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + logs := app.db.get_build_logs() + + return app.json(http.Status.ok, new_data_response(logs)) +} + +// parse_query_time unescapes an HTTP query parameter & tries to parse it as a +// time.Time struct. +fn parse_query_time(query string) ?time.Time { + unescaped := urllib.query_unescape(query) ? + t := time.parse(unescaped) ? + + return t +} + +['/api/logs'; post] +fn (mut app App) post_log() web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + // Parse query params + start_time := parse_query_time(app.query['startTime']) or { + return app.json(http.Status.bad_request, new_response('Invalid or missing start time.')) + } + + end_time := time.parse(app.query['endTime'].replace('_', ' ')) or { + return app.json(http.Status.bad_request, new_response('Invalid or missing end time.')) + } + + if 'exitCode' !in app.query { + return app.json(http.Status.bad_request, new_response('Missing exit code.')) + } + + exit_code := app.query['exitCode'].int() + + if 'arch' !in app.query { + return app.json(http.Status.bad_request, new_response("Missing parameter 'arch'.")) + } + + arch := app.query['arch'] + + repo := app.db.get_git_repo(app.query['repo'].int()) or { + return app.json(http.Status.bad_request, new_response('Unknown repo.')) + } + + // Store log in db + log := db.BuildLog{ + repo: repo + start_time: start_time + end_time: end_time + exit_code: exit_code + } + + app.db.add_build_log(log) + + repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo.id.str(), arch) + + // Create the logs directory of it doesn't exist + if !os.exists(repo_logs_dir) { + os.mkdir_all(repo_logs_dir) or { + app.lerror("Couldn't create dir '$repo_logs_dir'.") + + return app.json(http.Status.internal_server_error, new_response('An error occured while processing the request.')) + } + } + + // Stream log contents to correct file + file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + full_path := os.join_path_single(repo_logs_dir, file_name) + + if length := app.req.header.get(.content_length) { + util.reader_to_file(mut app.reader, length.int(), full_path) or { + app.lerror('An error occured while receiving logs: $err.msg()') + + return app.json(http.Status.internal_server_error, new_response('Failed to upload logs.')) + } + } else { + return app.status(http.Status.length_required) + } + + return app.json(http.Status.ok, new_response('Logs added successfully.')) +} diff --git a/src/server/server.v b/src/server/server.v index b2a2ad29..090aa76e 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -12,6 +12,7 @@ const ( log_file_name = 'vieter.log' repo_dir_name = 'repos' db_file_name = 'vieter.sqlite' + logs_dir_name = 'logs' ) struct App { @@ -37,6 +38,14 @@ pub fn server(conf Config) ? { os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') } + logs_dir := os.join_path_single(conf.data_dir, server.logs_dir_name) + + if !os.exists(logs_dir) { + os.mkdir(os.join_path_single(conf.data_dir, server.logs_dir_name)) or { + util.exit_with_message(1, 'Failed to create logs directory.') + } + } + mut logger := log.Log{ level: log_level } From 393e641a7666682c1e4ff38b83affbf596c461d6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 15:31:01 +0200 Subject: [PATCH 10/24] feat(server): allow filtering of builds per repo --- src/db/logs.v | 12 +++++++++++- src/server/logs.v | 16 +++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/db/logs.v b/src/db/logs.v index 3e5b6008..9a2405b4 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -4,7 +4,7 @@ import time pub struct BuildLog { id int [primary; sql: serial] - repo GitRepo [nonull] + repo_id int [nonull] start_time time.Time [nonull] end_time time.Time [nonull] exit_code int [nonull] @@ -19,6 +19,16 @@ pub fn (db &VieterDb) get_build_logs() []BuildLog { return res } +// get_build_logs_for_repo returns all BuildLog's in the database for a given +// repo. +pub fn (db &VieterDb) get_build_logs_for_repo(repo_id int) []BuildLog { + res := sql db.conn { + select from BuildLog where repo_id == repo_id order by id + } + + return res +} + // get_build_log tries to return a specific BuildLog. pub fn (db &VieterDb) get_build_log(id int) ?BuildLog { res := sql db.conn { diff --git a/src/server/logs.v b/src/server/logs.v index 01116b4b..8b0f297f 100644 --- a/src/server/logs.v +++ b/src/server/logs.v @@ -15,7 +15,11 @@ fn (mut app App) get_logs() web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - logs := app.db.get_build_logs() + logs := if 'repo' in app.query { + app.db.get_build_logs_for_repo(app.query['repo'].int()) + } else { + app.db.get_build_logs() + } return app.json(http.Status.ok, new_data_response(logs)) } @@ -56,13 +60,15 @@ fn (mut app App) post_log() web.Result { arch := app.query['arch'] - repo := app.db.get_git_repo(app.query['repo'].int()) or { - return app.json(http.Status.bad_request, new_response('Unknown repo.')) + repo := app.query['repo'].int() + + if repo == 0 { + return app.json(http.Status.bad_request, new_response('Invalid Git repo.')) } // Store log in db log := db.BuildLog{ - repo: repo + repo_id: repo start_time: start_time end_time: end_time exit_code: exit_code @@ -70,7 +76,7 @@ fn (mut app App) post_log() web.Result { app.db.add_build_log(log) - repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo.id.str(), arch) + repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo.str(), arch) // Create the logs directory of it doesn't exist if !os.exists(repo_logs_dir) { From 139142fcec4f66ad08e95ed4dcf6c3c30acc423e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 15:41:49 +0200 Subject: [PATCH 11/24] feat(server): added endpoint for content of build log --- src/db/logs.v | 2 ++ src/server/logs.v | 27 ++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/db/logs.v b/src/db/logs.v index 9a2405b4..589304e7 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -3,10 +3,12 @@ module db import time pub struct BuildLog { +pub: id int [primary; sql: serial] repo_id int [nonull] start_time time.Time [nonull] end_time time.Time [nonull] + arch string [nonull] exit_code int [nonull] } diff --git a/src/server/logs.v b/src/server/logs.v index 8b0f297f..9bddc210 100644 --- a/src/server/logs.v +++ b/src/server/logs.v @@ -24,6 +24,30 @@ fn (mut app App) get_logs() web.Result { return app.json(http.Status.ok, new_data_response(logs)) } +['/api/logs/:id'; get] +fn (mut app App) get_single_log(id int) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + log := app.db.get_build_log(id) or { return app.not_found() } + + return app.json(http.Status.ok, new_data_response(log)) +} + +['/api/logs/:id/content'; get] +fn (mut app App) get_log_contents(id int) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + log := app.db.get_build_log(id) or { return app.not_found() } + file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + full_path := os.join_path(app.conf.data_dir, server.logs_dir_name, log.repo_id.str(), log.arch, file_name) + + return app.file(full_path) +} + // parse_query_time unescapes an HTTP query parameter & tries to parse it as a // time.Time struct. fn parse_query_time(query string) ?time.Time { @@ -44,7 +68,7 @@ fn (mut app App) post_log() web.Result { return app.json(http.Status.bad_request, new_response('Invalid or missing start time.')) } - end_time := time.parse(app.query['endTime'].replace('_', ' ')) or { + end_time := parse_query_time(app.query['endTime']) or { return app.json(http.Status.bad_request, new_response('Invalid or missing end time.')) } @@ -71,6 +95,7 @@ fn (mut app App) post_log() web.Result { repo_id: repo start_time: start_time end_time: end_time + arch: arch exit_code: exit_code } From f42d3fd8b0f2005f96b94e39c66e94f300b2649e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 15:44:59 +0200 Subject: [PATCH 12/24] fix(server): prevent adding logs to non-existent repo --- src/db/git.v | 8 ++++++++ src/server/logs.v | 10 +++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/db/git.v b/src/db/git.v index b7791401..6eeecc8c 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -152,3 +152,11 @@ pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { } } } + +pub fn (db &VieterDb) git_repo_exists(repo_id int) bool { + db.get_git_repo(repo_id) or { + return false + } + + return true +} diff --git a/src/server/logs.v b/src/server/logs.v index 9bddc210..10b734da 100644 --- a/src/server/logs.v +++ b/src/server/logs.v @@ -84,15 +84,15 @@ fn (mut app App) post_log() web.Result { arch := app.query['arch'] - repo := app.query['repo'].int() + repo_id := app.query['repo'].int() - if repo == 0 { - return app.json(http.Status.bad_request, new_response('Invalid Git repo.')) + if !app.db.git_repo_exists(repo_id) { + return app.json(http.Status.bad_request, new_response('Unknown Git repo.')) } // Store log in db log := db.BuildLog{ - repo_id: repo + repo_id: repo_id start_time: start_time end_time: end_time arch: arch @@ -101,7 +101,7 @@ fn (mut app App) post_log() web.Result { app.db.add_build_log(log) - repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo.str(), arch) + repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo_id.str(), arch) // Create the logs directory of it doesn't exist if !os.exists(repo_logs_dir) { From 407b2269556e23d561ecd878c82c898502d28fc5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 16:10:27 +0200 Subject: [PATCH 13/24] refactor: moved client code into own module --- src/build/build.v | 4 +-- src/client/client.v | 40 +++++++++++++++++++++ src/client/git.v | 51 ++++++++++++++++++++++++++ src/cron/daemon/build.v | 2 +- src/cron/daemon/daemon.v | 10 +++--- src/git/cli.v | 17 ++++++--- src/git/client.v | 77 ---------------------------------------- 7 files changed, 110 insertions(+), 91 deletions(-) create mode 100644 src/client/client.v create mode 100644 src/client/git.v delete mode 100644 src/git/client.v diff --git a/src/build/build.v b/src/build/build.v index 15a5eb81..6f033e67 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -3,9 +3,9 @@ module build import docker import encoding.base64 import time -import git import os import db +import client const container_build_dir = '/build' @@ -126,7 +126,7 @@ fn build(conf Config) ? { build_arch := os.uname().machine // We get the repos map from the Vieter instance - repos := git.get_repos(conf.address, conf.api_key) ? + repos := client.new(conf.address, conf.api_key).get_git_repos() ? // We filter out any repos that aren't allowed to be built on this // architecture diff --git a/src/client/client.v b/src/client/client.v new file mode 100644 index 00000000..614b3dba --- /dev/null +++ b/src/client/client.v @@ -0,0 +1,40 @@ +module client + +import net.http +import response { Response } +import json + +pub struct Client { +pub: + address string + api_key string +} + +pub fn new(address string, api_key string) Client { + return Client{ + address: address + api_key: api_key + } +} + +// send_request is a convenience method for sending requests to the repos +// API. It mostly does string manipulation to create a query string containing +// the provided params. +fn (c &Client) send_request(method http.Method, url string, params map[string]string) ?Response { + mut full_url := '${c.address}$url' + + if params.len > 0 { + params_str := params.keys().map('$it=${params[it]}').join('&') + + full_url = '$full_url?$params_str' + } + + mut req := http.new_request(method, full_url, '') ? + req.add_custom_header('X-API-Key', c.api_key) ? + + res := req.do() ? + data := json.decode(Response, res.text) ? + + return data +} + diff --git a/src/client/git.v b/src/client/git.v new file mode 100644 index 00000000..5ed06200 --- /dev/null +++ b/src/client/git.v @@ -0,0 +1,51 @@ +module client + +import db +import net.http +import response { Response } + +// get_repos returns the current list of repos. +pub fn (c &Client) get_git_repos() ?[]db.GitRepo { + data := c.send_request<[]db.GitRepo>(http.Method.get, '/api/repos', {}) ? + + return data.data +} + +// get_repo returns the repo for a specific ID. +pub fn (c &Client) get_git_repo(id int) ?db.GitRepo { + data := c.send_request(http.Method.get, '/api/repos/$id', {}) ? + + return data.data +} + +// add_repo adds a new repo to the server. +pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []string) ?Response { + mut params := { + 'url': url + 'branch': branch + 'repo': repo + } + + if arch.len > 0 { + params['arch'] = arch.join(',') + } + + data := c.send_request(http.Method.post, '/api/repos', params) ? + + return data +} + +// remove_repo removes the repo with the given ID from the server. +pub fn (c &Client) remove_git_repo(id int) ?Response { + data := c.send_request(http.Method.delete, '/api/repos/$id', {}) ? + + return data +} + +// patch_repo sends a PATCH request to the given repo with the params as +// payload. +pub fn (c &Client) patch_git_repo(id int, params map[string]string) ?Response { + data := c.send_request(http.Method.patch, '/api/repos/$id', params) ? + + return data +} diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index e54a39e0..d107fd35 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -77,7 +77,7 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { // 0 means success, 1 means failure mut status := 0 - build.build_repo(d.address, d.api_key, d.builder_images.last(), &sb.repo) or { + build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(), &sb.repo) or { d.ldebug('build_repo error: $err.msg()') status = 1 } diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index ffa2f6e2..71fc5754 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -1,6 +1,5 @@ module daemon -import git import time import log import datatypes { MinHeap } @@ -10,6 +9,7 @@ import build import docker import db import os +import client const ( // How many seconds to wait before retrying to update API if failed @@ -31,8 +31,7 @@ fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool { pub struct Daemon { mut: - address string - api_key string + client client.Client base_image string builder_images []string global_schedule CronExpression @@ -56,8 +55,7 @@ mut: // populates the build queue for the first time. pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon { mut d := Daemon{ - address: address - api_key: api_key + client: client.new(address, api_key) base_image: base_image global_schedule: global_schedule api_update_frequency: api_update_frequency @@ -180,7 +178,7 @@ fn (mut d Daemon) schedule_build(repo db.GitRepo) { fn (mut d Daemon) renew_repos() { d.linfo('Renewing repos...') - mut new_repos := git.get_repos(d.address, d.api_key) or { + mut new_repos := d.client.get_git_repos() or { d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...') d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) diff --git a/src/git/cli.v b/src/git/cli.v index 634b7782..bdc0479d 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -3,6 +3,7 @@ module git import cli import env import cron.expression { parse_expression } +import client struct Config { address string [required] @@ -119,7 +120,8 @@ pub fn cmd() cli.Command { // list prints out a list of all repositories. fn list(conf Config) ? { - repos := get_repos(conf.address, conf.api_key) ? + c := client.new(conf.address, conf.api_key) + repos := c.get_git_repos() ? for repo in repos { println('$repo.id\t$repo.url\t$repo.branch\t$repo.repo') @@ -128,7 +130,8 @@ fn list(conf Config) ? { // add adds a new repository to the server's list. fn add(conf Config, url string, branch string, repo string) ? { - res := add_repo(conf.address, conf.api_key, url, branch, repo, []) ? + c := client.new(conf.address, conf.api_key) + res := c.add_git_repo(url, branch, repo, []) ? println(res.message) } @@ -139,7 +142,8 @@ fn remove(conf Config, id string) ? { id_int := id.int() if id_int != 0 { - res := remove_repo(conf.address, conf.api_key, id_int) ? + c := client.new(conf.address, conf.api_key) + res := c.remove_git_repo(id_int) ? println(res.message) } } @@ -156,7 +160,9 @@ fn patch(conf Config, id string, params map[string]string) ? { id_int := id.int() if id_int != 0 { - res := patch_repo(conf.address, conf.api_key, id_int, params) ? + + c := client.new(conf.address, conf.api_key) + res := c.patch_git_repo(id_int, params) ? println(res.message) } @@ -170,6 +176,7 @@ fn info(conf Config, id string) ? { return } - repo := get_repo(conf.address, conf.api_key, id_int) ? + c := client.new(conf.address, conf.api_key) + repo := c.get_git_repo(id_int) ? println(repo) } diff --git a/src/git/client.v b/src/git/client.v deleted file mode 100644 index b5f8e9fd..00000000 --- a/src/git/client.v +++ /dev/null @@ -1,77 +0,0 @@ -module git - -import json -import response { Response } -import net.http -import db - -// send_request is a convenience method for sending requests to the repos -// API. It mostly does string manipulation to create a query string containing -// the provided params. -fn send_request(method http.Method, address string, url string, api_key string, params map[string]string) ?Response { - mut full_url := '$address$url' - - if params.len > 0 { - params_str := params.keys().map('$it=${params[it]}').join('&') - - full_url = '$full_url?$params_str' - } - - mut req := http.new_request(method, full_url, '') ? - req.add_custom_header('X-API-Key', api_key) ? - - res := req.do() ? - data := json.decode(Response, res.text) ? - - return data -} - -// get_repos returns the current list of repos. -pub fn get_repos(address string, api_key string) ?[]db.GitRepo { - data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', api_key, - {}) ? - - return data.data -} - -// get_repo returns the repo for a specific ID. -pub fn get_repo(address string, api_key string, id int) ?db.GitRepo { - data := send_request(http.Method.get, address, '/api/repos/$id', api_key, - {}) ? - - return data.data -} - -// add_repo adds a new repo to the server. -pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response { - mut params := { - 'url': url - 'branch': branch - 'repo': repo - } - - if arch.len > 0 { - params['arch'] = arch.join(',') - } - - data := send_request(http.Method.post, address, '/api/repos', api_key, params) ? - - return data -} - -// remove_repo removes the repo with the given ID from the server. -pub fn remove_repo(address string, api_key string, id int) ?Response { - data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, - {}) ? - - return data -} - -// patch_repo sends a PATCH request to the given repo with the params as -// payload. -pub fn patch_repo(address string, api_key string, id int, params map[string]string) ?Response { - data := send_request(http.Method.patch, address, '/api/repos/$id', api_key, - params) ? - - return data -} From fa6603bd459c1f3c240d5f389e33441488cebcbf Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 19:38:28 +0200 Subject: [PATCH 14/24] feat(client): added client code for logs API --- src/client/client.v | 32 ++++++++++++++++++++---------- src/client/git.v | 18 ++++++++--------- src/client/logs.v | 42 ++++++++++++++++++++++++++++++++++++++++ src/cron/daemon/daemon.v | 2 +- src/db/git.v | 6 ++---- src/db/logs.v | 2 +- src/git/cli.v | 3 +-- src/server/logs.v | 5 +++-- 8 files changed, 81 insertions(+), 29 deletions(-) create mode 100644 src/client/logs.v diff --git a/src/client/client.v b/src/client/client.v index 614b3dba..7446e91b 100644 --- a/src/client/client.v +++ b/src/client/client.v @@ -1,6 +1,7 @@ module client -import net.http +import net.http { Method } +import net.urllib import response { Response } import json @@ -17,24 +18,35 @@ pub fn new(address string, api_key string) Client { } } -// send_request is a convenience method for sending requests to the repos -// API. It mostly does string manipulation to create a query string containing -// the provided params. -fn (c &Client) send_request(method http.Method, url string, params map[string]string) ?Response { - mut full_url := '${c.address}$url' +// send_request just calls send_request_with_body with an empty body. +fn (c &Client) send_request(method Method, url string, params map[string]string) ?Response { + return c.send_request_with_body(method, url, params, '') +} + +// send_request_with_body is a convenience method for sending requests to +// the repos API. It mostly does string manipulation to create a query string +// containing the provided params. +fn (c &Client) send_request_with_body(method Method, url string, params map[string]string, body string) ?Response { + mut full_url := '$c.address$url' if params.len > 0 { - params_str := params.keys().map('$it=${params[it]}').join('&') + mut params_escaped := map[string]string{} + + // Escape each query param + for k, v in params { + params_escaped[k] = urllib.query_escape(v) + } + + params_str := params_escaped.keys().map('$it=${params[it]}').join('&') full_url = '$full_url?$params_str' } - mut req := http.new_request(method, full_url, '') ? - req.add_custom_header('X-API-Key', c.api_key) ? + mut req := http.new_request(method, full_url, body) ? + req.add_custom_header('X-Api-Key', c.api_key) ? res := req.do() ? data := json.decode(Response, res.text) ? return data } - diff --git a/src/client/git.v b/src/client/git.v index 5ed06200..7f4c27ad 100644 --- a/src/client/git.v +++ b/src/client/git.v @@ -1,19 +1,19 @@ module client -import db -import net.http +import db { GitRepo } +import net.http { Method } import response { Response } // get_repos returns the current list of repos. -pub fn (c &Client) get_git_repos() ?[]db.GitRepo { - data := c.send_request<[]db.GitRepo>(http.Method.get, '/api/repos', {}) ? +pub fn (c &Client) get_git_repos() ?[]GitRepo { + data := c.send_request<[]GitRepo>(Method.get, '/api/repos', {}) ? return data.data } // get_repo returns the repo for a specific ID. -pub fn (c &Client) get_git_repo(id int) ?db.GitRepo { - data := c.send_request(http.Method.get, '/api/repos/$id', {}) ? +pub fn (c &Client) get_git_repo(id int) ?GitRepo { + data := c.send_request(Method.get, '/api/repos/$id', {}) ? return data.data } @@ -30,14 +30,14 @@ pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []s params['arch'] = arch.join(',') } - data := c.send_request(http.Method.post, '/api/repos', params) ? + data := c.send_request(Method.post, '/api/repos', params) ? return data } // remove_repo removes the repo with the given ID from the server. pub fn (c &Client) remove_git_repo(id int) ?Response { - data := c.send_request(http.Method.delete, '/api/repos/$id', {}) ? + data := c.send_request(Method.delete, '/api/repos/$id', {}) ? return data } @@ -45,7 +45,7 @@ pub fn (c &Client) remove_git_repo(id int) ?Response { // patch_repo sends a PATCH request to the given repo with the params as // payload. pub fn (c &Client) patch_git_repo(id int, params map[string]string) ?Response { - data := c.send_request(http.Method.patch, '/api/repos/$id', params) ? + data := c.send_request(Method.patch, '/api/repos/$id', params) ? return data } diff --git a/src/client/logs.v b/src/client/logs.v new file mode 100644 index 00000000..9182f691 --- /dev/null +++ b/src/client/logs.v @@ -0,0 +1,42 @@ +module client + +import db { BuildLog } +import net.http { Method } +import response { Response } +import time + +pub fn (c &Client) get_build_logs() ?Response<[]BuildLog> { + data := c.send_request<[]BuildLog>(Method.get, '/api/logs', {}) ? + + return data +} + +pub fn (c &Client) get_build_logs_for_repo(repo_id int) ?Response<[]BuildLog> { + params := { + 'repo': repo_id.str() + } + + data := c.send_request<[]BuildLog>(Method.get, '/api/logs', params) ? + + return data +} + +pub fn (c &Client) get_build_log(id int) ?Response { + data := c.send_request(Method.get, '/api/logs/$id', {}) ? + + return data +} + +pub fn (c &Client) add_build_log(repo_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { + params := { + 'repo': repo_id.str() + 'startTime': start_time.str() + 'endTime': end_time.str() + 'arch': arch + 'exitCode': exit_code.str() + } + + data := c.send_request_with_body(Method.post, '/api/logs', params, content) ? + + return data +} diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 71fc5754..ade8fcbf 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -31,7 +31,7 @@ fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool { pub struct Daemon { mut: - client client.Client + client client.Client base_image string builder_images []string global_schedule CronExpression diff --git a/src/db/git.v b/src/db/git.v index 6eeecc8c..7aba2845 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -154,9 +154,7 @@ pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { } pub fn (db &VieterDb) git_repo_exists(repo_id int) bool { - db.get_git_repo(repo_id) or { - return false - } - + db.get_git_repo(repo_id) or { return false } + return true } diff --git a/src/db/logs.v b/src/db/logs.v index 589304e7..9ce28652 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -8,7 +8,7 @@ pub: repo_id int [nonull] start_time time.Time [nonull] end_time time.Time [nonull] - arch string [nonull] + arch string [nonull] exit_code int [nonull] } diff --git a/src/git/cli.v b/src/git/cli.v index bdc0479d..3bf78d10 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -160,7 +160,6 @@ fn patch(conf Config, id string, params map[string]string) ? { id_int := id.int() if id_int != 0 { - c := client.new(conf.address, conf.api_key) res := c.patch_git_repo(id_int, params) ? @@ -176,7 +175,7 @@ fn info(conf Config, id string) ? { return } - c := client.new(conf.address, conf.api_key) + c := client.new(conf.address, conf.api_key) repo := c.get_git_repo(id_int) ? println(repo) } diff --git a/src/server/logs.v b/src/server/logs.v index 10b734da..f53464dc 100644 --- a/src/server/logs.v +++ b/src/server/logs.v @@ -29,7 +29,7 @@ fn (mut app App) get_single_log(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - + log := app.db.get_build_log(id) or { return app.not_found() } return app.json(http.Status.ok, new_data_response(log)) @@ -43,7 +43,8 @@ fn (mut app App) get_log_contents(id int) web.Result { log := app.db.get_build_log(id) or { return app.not_found() } file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') - full_path := os.join_path(app.conf.data_dir, server.logs_dir_name, log.repo_id.str(), log.arch, file_name) + full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.repo_id.str(), log.arch, + file_name) return app.file(full_path) } From 5b016df85ddcef017648039601ff08668d329ba8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 21:50:20 +0200 Subject: [PATCH 15/24] feat(cli): added commands for interacting with build logs --- src/client/client.v | 32 ++++++--- src/client/logs.v | 6 ++ src/console/console.v | 1 + src/{git/cli.v => console/git/git.v} | 0 src/console/logs/logs.v | 99 ++++++++++++++++++++++++++++ src/db/logs.v | 14 ++++ src/main.v | 4 +- 7 files changed, 145 insertions(+), 11 deletions(-) create mode 100644 src/console/console.v rename src/{git/cli.v => console/git/git.v} (100%) create mode 100644 src/console/logs/logs.v diff --git a/src/client/client.v b/src/client/client.v index 7446e91b..12c92d38 100644 --- a/src/client/client.v +++ b/src/client/client.v @@ -18,15 +18,7 @@ pub fn new(address string, api_key string) Client { } } -// send_request just calls send_request_with_body with an empty body. -fn (c &Client) send_request(method Method, url string, params map[string]string) ?Response { - return c.send_request_with_body(method, url, params, '') -} - -// send_request_with_body is a convenience method for sending requests to -// the repos API. It mostly does string manipulation to create a query string -// containing the provided params. -fn (c &Client) send_request_with_body(method Method, url string, params map[string]string, body string) ?Response { +fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) ?http.Response { mut full_url := '$c.address$url' if params.len > 0 { @@ -46,7 +38,27 @@ fn (c &Client) send_request_with_body(method Method, url string, params map[s req.add_custom_header('X-Api-Key', c.api_key) ? res := req.do() ? - data := json.decode(Response, res.text) ? + + return res +} + +// send_request just calls send_request_with_body with an empty body. +fn (c &Client) send_request(method Method, url string, params map[string]string) ?Response { + return c.send_request_with_body(method, url, params, '') +} + +// send_request_with_body is a convenience method for sending requests to +// the repos API. It mostly does string manipulation to create a query string +// containing the provided params. +fn (c &Client) send_request_with_body(method Method, url string, params map[string]string, body string) ?Response { + res_text := c.send_request_raw_response(method, url, params, body) ? + data := json.decode(Response, res_text) ? return data } + +fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) ?string { + res := c.send_request_raw(method, url, params, body) ? + + return res.text +} diff --git a/src/client/logs.v b/src/client/logs.v index 9182f691..19575a6e 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -27,6 +27,12 @@ pub fn (c &Client) get_build_log(id int) ?Response { return data } +pub fn (c &Client) get_build_log_content(id int) ?string { + data := c.send_request_raw_response(Method.get, '/api/logs/$id/content', {}, '') ? + + return data +} + pub fn (c &Client) add_build_log(repo_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { params := { 'repo': repo_id.str() diff --git a/src/console/console.v b/src/console/console.v new file mode 100644 index 00000000..6f296bdd --- /dev/null +++ b/src/console/console.v @@ -0,0 +1 @@ +module console diff --git a/src/git/cli.v b/src/console/git/git.v similarity index 100% rename from src/git/cli.v rename to src/console/git/git.v diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v new file mode 100644 index 00000000..14aeddf3 --- /dev/null +++ b/src/console/logs/logs.v @@ -0,0 +1,99 @@ +module logs + +import cli +import env +import client +import db + +struct Config { + address string [required] + api_key string [required] +} + +pub fn cmd() cli.Command { + return cli.Command{ + name: 'logs' + description: 'Interact with the build logs API.' + commands: [ + cli.Command{ + name: 'list' + description: 'List the build logs. If a repo ID is provided, only list the build logs for that repo.' + flags: [ + cli.Flag{ + name: 'repo' + description: 'ID of the Git repo to restrict list to.' + flag: cli.FlagType.int + }, + ] + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? + + repo_id := cmd.flags.get_int('repo') ? + + if repo_id == 0 { list(conf) ? } else { list_for_repo(conf, repo_id) ? } + } + }, + cli.Command{ + name: 'info' + required_args: 1 + usage: 'id' + description: 'Show all info for a specific build log.' + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? + + id := cmd.args[0].int() + info(conf, id) ? + } + }, + cli.Command{ + name: 'content' + required_args: 1 + usage: 'id' + description: 'Output the content of a build log to stdout.' + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? + + id := cmd.args[0].int() + content(conf, id) ? + } + }, + ] + } +} + +fn print_log_list(logs []db.BuildLog) { + for log in logs { + println('$log.id\t$log.start_time\t$log.exit_code') + } +} + +fn list(conf Config) ? { + c := client.new(conf.address, conf.api_key) + logs := c.get_build_logs() ?.data + + print_log_list(logs) +} + +fn list_for_repo(conf Config, repo_id int) ? { + c := client.new(conf.address, conf.api_key) + logs := c.get_build_logs_for_repo(repo_id) ?.data + + print_log_list(logs) +} + +fn info(conf Config, id int) ? { + c := client.new(conf.address, conf.api_key) + log := c.get_build_log(id) ?.data + + print(log) +} + +fn content(conf Config, id int) ? { + c := client.new(conf.address, conf.api_key) + content := c.get_build_log_content(id) ? + + println(content) +} diff --git a/src/db/logs.v b/src/db/logs.v index 9ce28652..05973af1 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -12,6 +12,20 @@ pub: exit_code int [nonull] } +pub fn (bl &BuildLog) str() string { + mut parts := [ + 'id: $bl.id', + 'repo id: $bl.repo_id', + 'start time: $bl.start_time', + 'end time: $bl.end_time', + 'arch: $bl.arch', + 'exit code: $bl.exit_code', + ] + str := parts.join('\n') + + return str +} + // get_build_logs returns all BuildLog's in the database. pub fn (db &VieterDb) get_build_logs() []BuildLog { res := sql db.conn { diff --git a/src/main.v b/src/main.v index 4ba6d30f..41d0d331 100644 --- a/src/main.v +++ b/src/main.v @@ -4,7 +4,8 @@ import os import server import cli import build -import git +import console.git +import console.logs import cron fn main() { @@ -27,6 +28,7 @@ fn main() { build.cmd(), git.cmd(), cron.cmd(), + logs.cmd(), ] } From 5f7d7c47801b3b6145e945c9a69c3d891085f593 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 22:06:17 +0200 Subject: [PATCH 16/24] doc: added documentation to all functions --- src/client/client.v | 9 ++++++--- src/client/git.v | 10 +++++----- src/client/logs.v | 5 +++++ src/console/logs/logs.v | 7 +++++++ src/db/git.v | 2 ++ src/db/logs.v | 1 + src/server/logs.v | 7 ++++++- 7 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/client/client.v b/src/client/client.v index 12c92d38..25224a51 100644 --- a/src/client/client.v +++ b/src/client/client.v @@ -11,6 +11,7 @@ pub: api_key string } +// new creates a new Client instance. pub fn new(address string, api_key string) Client { return Client{ address: address @@ -18,6 +19,8 @@ pub fn new(address string, api_key string) Client { } } +// send_request_raw sends an HTTP request, returning the http.Response object. +// It encodes the params so that they're safe to pass as HTTP query parameters. fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) ?http.Response { mut full_url := '$c.address$url' @@ -47,9 +50,8 @@ fn (c &Client) send_request(method Method, url string, params map[string]stri return c.send_request_with_body(method, url, params, '') } -// send_request_with_body is a convenience method for sending requests to -// the repos API. It mostly does string manipulation to create a query string -// containing the provided params. +// send_request_with_body calls send_request_raw_response & parses its +// output as a Response object. fn (c &Client) send_request_with_body(method Method, url string, params map[string]string, body string) ?Response { res_text := c.send_request_raw_response(method, url, params, body) ? data := json.decode(Response, res_text) ? @@ -57,6 +59,7 @@ fn (c &Client) send_request_with_body(method Method, url string, params map[s return data } +// send_request_raw_response returns the raw text response for an HTTP request. fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) ?string { res := c.send_request_raw(method, url, params, body) ? diff --git a/src/client/git.v b/src/client/git.v index 7f4c27ad..b09d4c22 100644 --- a/src/client/git.v +++ b/src/client/git.v @@ -4,21 +4,21 @@ import db { GitRepo } import net.http { Method } import response { Response } -// get_repos returns the current list of repos. +// get_git_repos returns the current list of repos. pub fn (c &Client) get_git_repos() ?[]GitRepo { data := c.send_request<[]GitRepo>(Method.get, '/api/repos', {}) ? return data.data } -// get_repo returns the repo for a specific ID. +// get_git_repo returns the repo for a specific ID. pub fn (c &Client) get_git_repo(id int) ?GitRepo { data := c.send_request(Method.get, '/api/repos/$id', {}) ? return data.data } -// add_repo adds a new repo to the server. +// add_git_repo adds a new repo to the server. pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []string) ?Response { mut params := { 'url': url @@ -35,14 +35,14 @@ pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []s return data } -// remove_repo removes the repo with the given ID from the server. +// remove_git_repo removes the repo with the given ID from the server. pub fn (c &Client) remove_git_repo(id int) ?Response { data := c.send_request(Method.delete, '/api/repos/$id', {}) ? return data } -// patch_repo sends a PATCH request to the given repo with the params as +// patch_git_repo sends a PATCH request to the given repo with the params as // payload. pub fn (c &Client) patch_git_repo(id int, params map[string]string) ?Response { data := c.send_request(Method.patch, '/api/repos/$id', params) ? diff --git a/src/client/logs.v b/src/client/logs.v index 19575a6e..8c532138 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -5,12 +5,14 @@ import net.http { Method } import response { Response } import time +// get_build_logs returns all build logs. pub fn (c &Client) get_build_logs() ?Response<[]BuildLog> { data := c.send_request<[]BuildLog>(Method.get, '/api/logs', {}) ? return data } +// get_build_logs_for_repo returns all build logs for a given repo. pub fn (c &Client) get_build_logs_for_repo(repo_id int) ?Response<[]BuildLog> { params := { 'repo': repo_id.str() @@ -21,18 +23,21 @@ pub fn (c &Client) get_build_logs_for_repo(repo_id int) ?Response<[]BuildLog> { return data } +// get_build_log returns a specific build log. pub fn (c &Client) get_build_log(id int) ?Response { data := c.send_request(Method.get, '/api/logs/$id', {}) ? return data } +// get_build_log_content returns the contents of the build log file. pub fn (c &Client) get_build_log_content(id int) ?string { data := c.send_request_raw_response(Method.get, '/api/logs/$id/content', {}, '') ? return data } +// add_build_log adds a new build log to the server. pub fn (c &Client) add_build_log(repo_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { params := { 'repo': repo_id.str() diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 14aeddf3..e3c7d14a 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -10,6 +10,7 @@ struct Config { api_key string [required] } +// cmd returns the cli module that handles the build repos API. pub fn cmd() cli.Command { return cli.Command{ name: 'logs' @@ -64,12 +65,14 @@ pub fn cmd() cli.Command { } } +// print_log_list prints a list of logs. fn print_log_list(logs []db.BuildLog) { for log in logs { println('$log.id\t$log.start_time\t$log.exit_code') } } +// list prints a list of all build logs. fn list(conf Config) ? { c := client.new(conf.address, conf.api_key) logs := c.get_build_logs() ?.data @@ -77,6 +80,7 @@ fn list(conf Config) ? { print_log_list(logs) } +// list prints a list of all build logs for a given repo. fn list_for_repo(conf Config, repo_id int) ? { c := client.new(conf.address, conf.api_key) logs := c.get_build_logs_for_repo(repo_id) ?.data @@ -84,6 +88,7 @@ fn list_for_repo(conf Config, repo_id int) ? { print_log_list(logs) } +// info print the detailed info for a given build log. fn info(conf Config, id int) ? { c := client.new(conf.address, conf.api_key) log := c.get_build_log(id) ?.data @@ -91,6 +96,8 @@ fn info(conf Config, id int) ? { print(log) } +// content outputs the contents of the log file for a given build log to +// stdout. fn content(conf Config, id int) ? { c := client.new(conf.address, conf.api_key) content := c.get_build_log_content(id) ? diff --git a/src/db/git.v b/src/db/git.v index 7aba2845..9a475a54 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -153,6 +153,8 @@ pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { } } +// git_repo_exists is a utility function that checks whether a repo with the +// given id exists. pub fn (db &VieterDb) git_repo_exists(repo_id int) bool { db.get_git_repo(repo_id) or { return false } diff --git a/src/db/logs.v b/src/db/logs.v index 05973af1..817db78b 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -12,6 +12,7 @@ pub: exit_code int [nonull] } +// str returns a string representation. pub fn (bl &BuildLog) str() string { mut parts := [ 'id: $bl.id', diff --git a/src/server/logs.v b/src/server/logs.v index f53464dc..b048dc4f 100644 --- a/src/server/logs.v +++ b/src/server/logs.v @@ -9,6 +9,8 @@ import time import os import util +// get_logs returns all build logs in the database. A 'repo' query param can +// optionally be added to limit the list of build logs to that repository. ['/api/logs'; get] fn (mut app App) get_logs() web.Result { if !app.is_authorized() { @@ -24,6 +26,7 @@ fn (mut app App) get_logs() web.Result { return app.json(http.Status.ok, new_data_response(logs)) } +// get_single_log returns the build log with the given id. ['/api/logs/:id'; get] fn (mut app App) get_single_log(id int) web.Result { if !app.is_authorized() { @@ -35,8 +38,9 @@ fn (mut app App) get_single_log(id int) web.Result { return app.json(http.Status.ok, new_data_response(log)) } +// get_log_content returns the actual build log file for the given id. ['/api/logs/:id/content'; get] -fn (mut app App) get_log_contents(id int) web.Result { +fn (mut app App) get_log_content(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } @@ -58,6 +62,7 @@ fn parse_query_time(query string) ?time.Time { return t } +// post_log adds a new log to the database. ['/api/logs'; post] fn (mut app App) post_log() web.Result { if !app.is_authorized() { From 30cce4fa72c4eaf632dc7a329392f1dd05edd326 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 22:13:35 +0200 Subject: [PATCH 17/24] chore: updated changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bbe4f06..7d9eb4d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev) + +### Added + +* Web API for adding & querying build logs +* CLI commands to access build logs API + ## [0.3.0-alpha.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.1) ### Changed From 27aa215effb206e2256ac67308935bc2340a0bdb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 8 May 2022 10:29:06 +0200 Subject: [PATCH 18/24] feat(docker): added function to retrieve container logs --- src/docker/containers.v | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/docker/containers.v b/src/docker/containers.v index d0f5a4d7..81343702 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -76,3 +76,23 @@ pub fn remove_container(id string) ?bool { return res.status_code == 204 } + +pub fn get_container_logs(id string) ?string { + res := request('GET', urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true') ?) ? + mut res_bytes := res.text.bytes() + + // Docker uses a special "stream" format for their logs, so we have to + // clean up the data. + mut index := 0 + + for index < res_bytes.len { + // The reverse is required because V reads in the bytes differently + t := res_bytes[index + 4..index + 8].reverse() + len_length := unsafe { *(&u32(&t[0])) } + + res_bytes.delete_many(index, 8) + index += int(len_length) + } + + return res_bytes.bytestr() +} From 4b172cb5d8e9dc8d96f7c73fc84e0d7aa1b8b5ab Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 8 May 2022 13:17:54 +0200 Subject: [PATCH 19/24] feat(cli): `vieter build` now builds a single repo & uploads build logs --- src/build/build.v | 56 +++++++++++++++++++++++++---------------- src/build/cli.v | 8 ++++-- src/docker/containers.v | 29 +++++++++++++++++++-- 3 files changed, 68 insertions(+), 25 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 6f033e67..9505171c 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -73,10 +73,17 @@ pub fn create_build_image(base_image string) ?string { return image.id } +struct BuildResult { + start_time time.Time + end_time time.Time + exit_code int + logs string +} + // build_repo builds, packages & publishes a given Arch package based on the // provided GitRepo. The base image ID should be of an image previously created -// by create_build_image. -pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ? { +// by create_build_image. It returns the logs of the container. +pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ?BuildResult { build_arch := os.uname().machine // TODO what to do with PKGBUILDs that build multiple packages? @@ -107,43 +114,50 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db id := docker.create_container(c) ? docker.start_container(id) ? + mut data := docker.inspect_container(id) ? + // This loop waits until the container has stopped, so we can remove it after for { - data := docker.inspect_container(id) ? - if !data.state.running { break } time.sleep(1 * time.second) + + data = docker.inspect_container(id) ? } + logs := docker.get_container_logs(id) ? + docker.remove_container(id) ? + + return BuildResult{ + start_time: data.state.start_time + end_time: data.state.end_time + exit_code: data.state.exit_code + logs: logs + } } // build builds every Git repo in the server's list. -fn build(conf Config) ? { +fn build(conf Config, repo_id int) ? { + c := client.new(conf.address, conf.api_key) + repo := c.get_git_repo(repo_id) ? + build_arch := os.uname().machine - // We get the repos map from the Vieter instance - repos := client.new(conf.address, conf.api_key).get_git_repos() ? - - // We filter out any repos that aren't allowed to be built on this - // architecture - filtered_repos := repos.filter(it.arch.map(it.value).contains(build_arch)) - - // No point in doing work if there's no repos present - if filtered_repos.len == 0 { - return - } - // First, we create a base image which has updated repos n stuff + println('Creating base image...') image_id := create_build_image(conf.base_image) ? - for repo in filtered_repos { - build_repo(conf.address, conf.api_key, image_id, repo) ? - } + println('Running build...') + res := build_repo(conf.address, conf.api_key, image_id, repo) ? - // Finally, we remove the builder image + // Remove the builder image + println('Removing build image...') docker.remove_image(image_id) ? + + // Upload the build log to the Vieter instance + println('Uploading logs to Vieter...') + c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code, res.logs) ? } diff --git a/src/build/cli.v b/src/build/cli.v index 01313960..5247e871 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -14,12 +14,16 @@ pub: pub fn cmd() cli.Command { return cli.Command{ name: 'build' - description: 'Run the build process.' + required_args: 1 + usage: 'id' + description: 'Build the repository with the given ID.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file') ? conf := env.load(config_file) ? - build(conf) ? + id := cmd.args[0].int() + + build(conf, id) ? } } } diff --git a/src/docker/containers.v b/src/docker/containers.v index 81343702..63095a6b 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -2,6 +2,8 @@ module docker import json import net.urllib +import regex +import time struct Container { id string [json: Id] @@ -49,13 +51,28 @@ pub fn start_container(id string) ?bool { } struct ContainerInspect { -pub: +pub mut: state ContainerState [json: State] } struct ContainerState { pub: running bool [json: Running] + status string [json: Status] + exit_code int [json: ExitCode] + // These use a rather specific format so they have to be parsed later + start_time_str string [json: StartedAt] + end_time_str string [json: FinishedAt] +pub mut: + start_time time.Time [skip] + end_time time.Time [skip] +} + +fn docker_timestamp_to_time(s string) ?time.Time { + parts := s.split('.') + clipped := parts[0] + '.' + parts[1][..3] + + return time.parse_rfc3339(clipped) } // inspect_container returns the result of inspecting a container with a given @@ -67,7 +84,15 @@ pub fn inspect_container(id string) ?ContainerInspect { return error('Failed to inspect container.') } - return json.decode(ContainerInspect, res.text) or {} + mut data := json.decode(ContainerInspect, res.text) ? + + data.state.start_time = docker_timestamp_to_time(data.state.start_time_str) ? + + if data.state.status == "exited" { + data.state.end_time = docker_timestamp_to_time(data.state.end_time_str) ? + } + + return data } // remove_container removes a container with a given ID. From e79d18100f38599429006e3c07406e0526cd0b74 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 8 May 2022 14:53:35 +0200 Subject: [PATCH 20/24] chore: ran `make fmt` --- src/build/build.v | 14 ++++++-------- src/docker/containers.v | 12 ++++++------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 9505171c..c97a5f7e 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -75,9 +75,9 @@ pub fn create_build_image(base_image string) ?string { struct BuildResult { start_time time.Time - end_time time.Time - exit_code int - logs string + end_time time.Time + exit_code int + logs string } // build_repo builds, packages & publishes a given Arch package based on the @@ -94,7 +94,7 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db 'source PKGBUILD', // The build container checks whether the package is already // present on the server - 'curl --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + 'curl -s --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done', ] @@ -146,18 +146,16 @@ fn build(conf Config, repo_id int) ? { build_arch := os.uname().machine - // First, we create a base image which has updated repos n stuff println('Creating base image...') image_id := create_build_image(conf.base_image) ? println('Running build...') res := build_repo(conf.address, conf.api_key, image_id, repo) ? - // Remove the builder image println('Removing build image...') docker.remove_image(image_id) ? - // Upload the build log to the Vieter instance println('Uploading logs to Vieter...') - c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code, res.logs) ? + c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code, + res.logs) ? } diff --git a/src/docker/containers.v b/src/docker/containers.v index 63095a6b..6d2eb3e5 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -57,15 +57,15 @@ pub mut: struct ContainerState { pub: - running bool [json: Running] - status string [json: Status] - exit_code int [json: ExitCode] + running bool [json: Running] + status string [json: Status] + exit_code int [json: ExitCode] // These use a rather specific format so they have to be parsed later start_time_str string [json: StartedAt] - end_time_str string [json: FinishedAt] + end_time_str string [json: FinishedAt] pub mut: start_time time.Time [skip] - end_time time.Time [skip] + end_time time.Time [skip] } fn docker_timestamp_to_time(s string) ?time.Time { @@ -88,7 +88,7 @@ pub fn inspect_container(id string) ?ContainerInspect { data.state.start_time = docker_timestamp_to_time(data.state.start_time_str) ? - if data.state.status == "exited" { + if data.state.status == 'exited' { data.state.end_time = docker_timestamp_to_time(data.state.end_time_str) ? } From ea4c4fce1650543bcbf22e20d6ce01015120500f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 8 May 2022 15:07:54 +0200 Subject: [PATCH 21/24] feat(cron): upload logs after build --- src/build/build.v | 3 ++- src/cron/daemon/build.v | 12 ++++++++++-- src/docker/containers.v | 1 - 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index c97a5f7e..774591d6 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -73,7 +73,8 @@ pub fn create_build_image(base_image string) ?string { return image.id } -struct BuildResult { +pub struct BuildResult { +pub: start_time time.Time end_time time.Time exit_code int diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index d107fd35..aa08f9f3 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -3,6 +3,7 @@ module daemon import time import sync.stdatomic import build +import os const ( build_empty = 0 @@ -77,13 +78,20 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { // 0 means success, 1 means failure mut status := 0 - build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(), &sb.repo) or { + res := build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(), + &sb.repo) or { d.ldebug('build_repo error: $err.msg()') status = 1 + + build.BuildResult{} } if status == 0 { - d.linfo('finished build: $sb.repo.url $sb.repo.branch') + d.linfo('finished build: $sb.repo.url $sb.repo.branch; uploading logs...') + + build_arch := os.uname().machine + d.client.add_build_log(sb.repo.id, res.start_time, res.end_time, build_arch, res.exit_code, + res.logs) or { d.lerror('Failed to upload logs for $sb.repo.url $sb.repo.arch') } } else { d.linfo('failed build: $sb.repo.url $sb.repo.branch') } diff --git a/src/docker/containers.v b/src/docker/containers.v index 6d2eb3e5..fe0bb7b1 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -2,7 +2,6 @@ module docker import json import net.urllib -import regex import time struct Container { From 5a5f7f83461f20785e18b006e8033316a9f24d9c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 9 May 2022 14:58:20 +0200 Subject: [PATCH 22/24] refactor(docker): use builtin parse_rfc3339 function --- src/docker/containers.v | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/docker/containers.v b/src/docker/containers.v index fe0bb7b1..2258f3bd 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -67,13 +67,6 @@ pub mut: end_time time.Time [skip] } -fn docker_timestamp_to_time(s string) ?time.Time { - parts := s.split('.') - clipped := parts[0] + '.' + parts[1][..3] - - return time.parse_rfc3339(clipped) -} - // inspect_container returns the result of inspecting a container with a given // ID. pub fn inspect_container(id string) ?ContainerInspect { @@ -85,10 +78,10 @@ pub fn inspect_container(id string) ?ContainerInspect { mut data := json.decode(ContainerInspect, res.text) ? - data.state.start_time = docker_timestamp_to_time(data.state.start_time_str) ? + data.state.start_time = time.parse_rfc3339(data.state.start_time_str) ? if data.state.status == 'exited' { - data.state.end_time = docker_timestamp_to_time(data.state.end_time_str) ? + data.state.end_time = time.parse_rfc3339(data.state.end_time_str) ? } return data @@ -101,6 +94,8 @@ pub fn remove_container(id string) ?bool { return res.status_code == 204 } +// get_container_logs retrieves the logs for a Docker container, both stdout & +// stderr. pub fn get_container_logs(id string) ?string { res := request('GET', urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true') ?) ? mut res_bytes := res.text.bytes() From 3821ed29fd4d0dd9fda4447dab444baf40d5cd6b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 9 May 2022 15:05:53 +0200 Subject: [PATCH 23/24] refactor(docker): simplified loop expression --- src/build/build.v | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 774591d6..0a978aa0 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -118,11 +118,7 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db mut data := docker.inspect_container(id) ? // This loop waits until the container has stopped, so we can remove it after - for { - if !data.state.running { - break - } - + for data.state.running { time.sleep(1 * time.second) data = docker.inspect_container(id) ? From 78fc3afcd3b53860c21e227fc452a876567976cd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 9 May 2022 15:16:30 +0200 Subject: [PATCH 24/24] feat(ci): also publish dev images as specific commit hash --- .woodpecker/.docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.woodpecker/.docker.yml b/.woodpecker/.docker.yml index bab869b4..f31490a1 100644 --- a/.woodpecker/.docker.yml +++ b/.woodpecker/.docker.yml @@ -11,7 +11,9 @@ pipeline: - 'docker_password' settings: repo: 'chewingbever/vieter' - tag: 'dev' + tags: + - 'dev' + - ${CI_COMMIT_SHA} platforms: [ 'linux/arm64/v8', 'linux/amd64' ] build_args_from_env: - 'CI_COMMIT_SHA'