From 0de5ffb45d2734b95c3c11dd25d2bfdf6dd2bf0e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 16 May 2022 17:34:51 +0200 Subject: [PATCH 01/19] chore: bumped versions --- CHANGELOG.md | 2 ++ PKGBUILD | 2 +- src/main.v | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c20393..7dc0a00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev) +## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) + ### Added * Web API for adding & querying build logs diff --git a/PKGBUILD b/PKGBUILD index 83ab896..49fcf54 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -3,7 +3,7 @@ pkgbase='vieter' pkgname='vieter' -pkgver='0.3.0_alpha.1' +pkgver='0.3.0_alpha.2' pkgrel=1 depends=('glibc' 'openssl' 'libarchive' 'sqlite') makedepends=('git' 'vieter-v') diff --git a/src/main.v b/src/main.v index dbfac09..6df45dc 100644 --- a/src/main.v +++ b/src/main.v @@ -11,7 +11,7 @@ fn main() { mut app := cli.Command{ name: 'vieter' description: 'Vieter is a lightweight implementation of an Arch repository server.' - version: '0.3.0-alpha.1' + version: '0.3.0-alpha.2' flags: [ cli.Flag{ flag: cli.FlagType.string From 1e079143cd29673ff2a20e8513dfc741493f0e9b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 17 May 2022 13:50:46 +0200 Subject: [PATCH 02/19] feat(server): added better query params to GitRepo API --- src/db/filter.v | 25 +++++++++++++++++++++++++ src/db/git.v | 14 ++++++++++++-- src/server/git.v | 5 ++++- 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 src/db/filter.v diff --git a/src/db/filter.v b/src/db/filter.v new file mode 100644 index 0000000..79e281b --- /dev/null +++ b/src/db/filter.v @@ -0,0 +1,25 @@ +module db + +pub struct GitRepoFilter { +pub mut: + limit u64 = 25 + offset u64 + repo string +} + +pub fn filter_from_params(params map[string]string) ?T { + mut o := GitRepoFilter{} + + $for field in T.fields { + if field.name in params { + val := params[field.name] + + $if field.typ is string { + o.$(field.name) = val + } $else $if field.typ is u64 { + o.$(field.name) = val.u64() + } + } + } + return o +} diff --git a/src/db/git.v b/src/db/git.v index 9a475a5..cb31bf0 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -77,9 +77,19 @@ pub fn git_repo_from_params(params map[string]string) ?GitRepo { } // get_git_repos returns all GitRepo's in the database. -pub fn (db &VieterDb) get_git_repos() []GitRepo { +pub fn (db &VieterDb) get_git_repos(filter GitRepoFilter) []GitRepo { + // This seems to currently be blocked by a bug in the ORM, I'll have to ask + // around. + if filter.repo != '' { + res := sql db.conn { + select from GitRepo where repo == filter.repo order by id limit filter.limit offset filter.offset + } + + return res + } + res := sql db.conn { - select from GitRepo order by id + select from GitRepo order by id limit filter.limit offset filter.offset } return res diff --git a/src/server/git.v b/src/server/git.v index c5cbc0a..921251f 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -12,7 +12,10 @@ fn (mut app App) get_repos() web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - repos := app.db.get_git_repos() + filter := db.filter_from_params(app.query) or { + return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) + } + repos := app.db.get_git_repos(filter) return app.json(http.Status.ok, new_data_response(repos)) } From 5e81dadce3bfd9a4a43c979507f0bd9be553a55c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 18 May 2022 08:22:13 +0200 Subject: [PATCH 03/19] feat: partially added filters to GitRepo CLI --- src/client/git.v | 8 +++++--- src/console/git/git.v | 46 ++++++++++++++++++++++++++++++++++++++++--- src/db/filter.v | 1 + src/db/git.v | 1 + src/util/util.v | 10 ++++++++++ 5 files changed, 60 insertions(+), 6 deletions(-) diff --git a/src/client/git.v b/src/client/git.v index 280caab..c21565a 100644 --- a/src/client/git.v +++ b/src/client/git.v @@ -1,12 +1,14 @@ module client -import db { GitRepo } +import db { GitRepo, GitRepoFilter } import net.http { Method } import response { Response } +import util // get_git_repos returns the current list of repos. -pub fn (c &Client) get_git_repos() ?[]GitRepo { - data := c.send_request<[]GitRepo>(Method.get, '/api/repos', {})? +pub fn (c &Client) get_git_repos(filter GitRepoFilter) ?[]GitRepo { + params := util.struct_to_map(filter) + data := c.send_request<[]GitRepo>(Method.get, '/api/repos', params)? return data.data } diff --git a/src/console/git/git.v b/src/console/git/git.v index 06d5f80..861bfb3 100644 --- a/src/console/git/git.v +++ b/src/console/git/git.v @@ -5,6 +5,7 @@ import env import cron.expression { parse_expression } import client import console +import db { GitRepoFilter } struct Config { address string [required] @@ -21,11 +22,50 @@ pub fn cmd() cli.Command { cli.Command{ name: 'list' description: 'List the current repos.' + flags: [ + cli.Flag{ + name: 'limit' + description: 'How many results to return.' + flag: cli.FlagType.int + }, + cli.Flag{ + name: 'offset' + description: 'Minimum index to return.' + flag: cli.FlagType.int + }, + cli.Flag{ + name: 'repo' + description: 'Only return Git repos that publish to this repo.' + flag: cli.FlagType.string + }, + cli.Flag{ + name: 'arch' + description: 'Only return repos enabled for this architecture.' + flag: cli.FlagType.string + }, + ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? conf := env.load(config_file)? - list(conf)? + mut filter := GitRepoFilter{} + + if limit := cmd.flags.get_int('limit') { + println('limit = $limit') + filter.limit = u64(limit) + } + + if offset := cmd.flags.get_int('offset') { + filter.limit = u64(offset) + } + + if repo := cmd.flags.get_string('repo') { + filter.repo = repo + } + + dump(filter) + + list(conf, filter)? } }, cli.Command{ @@ -133,9 +173,9 @@ pub fn cmd() cli.Command { // ID. If multiple or none are found, an error is raised. // list prints out a list of all repositories. -fn list(conf Config) ? { +fn list(conf Config, filter GitRepoFilter) ? { c := client.new(conf.address, conf.api_key) - repos := c.get_git_repos()? + repos := c.get_git_repos(filter)? data := repos.map([it.id.str(), it.url, it.branch, it.repo]) println(console.pretty_table(['id', 'url', 'branch', 'repo'], data)?) diff --git a/src/db/filter.v b/src/db/filter.v index 79e281b..07c890a 100644 --- a/src/db/filter.v +++ b/src/db/filter.v @@ -1,5 +1,6 @@ module db +[params] pub struct GitRepoFilter { pub mut: limit u64 = 25 diff --git a/src/db/git.v b/src/db/git.v index cb31bf0..7308346 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -78,6 +78,7 @@ pub fn git_repo_from_params(params map[string]string) ?GitRepo { // get_git_repos returns all GitRepo's in the database. pub fn (db &VieterDb) get_git_repos(filter GitRepoFilter) []GitRepo { + println(filter) // This seems to currently be blocked by a bug in the ORM, I'll have to ask // around. if filter.repo != '' { diff --git a/src/util/util.v b/src/util/util.v index 266bcb5..cbb4072 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -64,3 +64,13 @@ pub fn pretty_bytes(bytes int) string { return '${n:.2}${util.prefixes[i]}' } + +pub fn struct_to_map(o T) map[string]string { + mut m := map[string]string{} + + $for field in T.fields { + m[field.name] = o.$(field.name).str() + } + + return m +} From 6bd5b7cb48eb38ebccda393a886badd36e4c4346 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 19 May 2022 07:54:33 +0200 Subject: [PATCH 04/19] refactor: separated GitRepo types into own module feat: added more query params for GitRepo API --- src/build/build.v | 3 +- src/client/client.v | 5 ++- src/client/git.v | 5 ++- src/console/git/git.v | 16 ++++----- src/cron/daemon/daemon.v | 7 ++-- src/db/db.v | 1 + src/db/filter.v | 26 -------------- src/db/git.v | 77 +--------------------------------------- src/models/git.v | 52 +++++++++++++++++++++++++++ src/models/models.v | 36 +++++++++++++++++++ src/server/git.v | 7 ++-- src/util/util.v | 1 - 12 files changed, 114 insertions(+), 122 deletions(-) delete mode 100644 src/db/filter.v create mode 100644 src/models/git.v create mode 100644 src/models/models.v diff --git a/src/build/build.v b/src/build/build.v index fab6c35..1f26d03 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -7,6 +7,7 @@ import os import db import strings import util +import models { GitRepo } const ( container_build_dir = '/build' @@ -93,7 +94,7 @@ pub: // build_repo builds, packages & publishes a given Arch package based on the // provided GitRepo. The base image ID should be of an image previously created // by create_build_image. It returns the logs of the container. -pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ?BuildResult { +pub fn build_repo(address string, api_key string, base_image_id string, repo &GitRepo) ?BuildResult { mut dd := docker.new_conn()? defer { diff --git a/src/client/client.v b/src/client/client.v index 3b28073..7cb6be5 100644 --- a/src/client/client.v +++ b/src/client/client.v @@ -29,7 +29,10 @@ fn (c &Client) send_request_raw(method Method, url string, params map[string]str // Escape each query param for k, v in params { - params_escaped[k] = urllib.query_escape(v) + // An empty parameter should be the same as not providing it at all + if v != '' { + params_escaped[k] = urllib.query_escape(v) + } } params_str := params_escaped.keys().map('$it=${params[it]}').join('&') diff --git a/src/client/git.v b/src/client/git.v index c21565a..fd14718 100644 --- a/src/client/git.v +++ b/src/client/git.v @@ -1,13 +1,12 @@ module client -import db { GitRepo, GitRepoFilter } +import models { GitRepo, GitRepoFilter } import net.http { Method } import response { Response } -import util // get_git_repos returns the current list of repos. pub fn (c &Client) get_git_repos(filter GitRepoFilter) ?[]GitRepo { - params := util.struct_to_map(filter) + params := models.params_from(filter) data := c.send_request<[]GitRepo>(Method.get, '/api/repos', params)? return data.data diff --git a/src/console/git/git.v b/src/console/git/git.v index 861bfb3..b01f3dd 100644 --- a/src/console/git/git.v +++ b/src/console/git/git.v @@ -5,7 +5,7 @@ import env import cron.expression { parse_expression } import client import console -import db { GitRepoFilter } +import models { GitRepoFilter } struct Config { address string [required] @@ -50,21 +50,21 @@ pub fn cmd() cli.Command { mut filter := GitRepoFilter{} - if limit := cmd.flags.get_int('limit') { - println('limit = $limit') + limit := cmd.flags.get_int('limit')? + if limit != 0 { filter.limit = u64(limit) } - if offset := cmd.flags.get_int('offset') { - filter.limit = u64(offset) + offset := cmd.flags.get_int('offset')? + if offset != 0 { + filter.offset = u64(offset) } - if repo := cmd.flags.get_string('repo') { + repo := cmd.flags.get_string('repo')? + if repo != '' { filter.repo = repo } - dump(filter) - list(conf, filter)? } }, diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index da3b46e..e92dd68 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -10,6 +10,7 @@ import docker import db import os import client +import models { GitRepo } const ( // How many seconds to wait before retrying to update API if failed @@ -20,7 +21,7 @@ const ( struct ScheduledBuild { pub: - repo db.GitRepo + repo GitRepo timestamp time.Time } @@ -38,7 +39,7 @@ mut: api_update_frequency int image_rebuild_frequency int // Repos currently loaded from API. - repos []db.GitRepo + repos []GitRepo // At what point to update the list of repositories. api_update_timestamp time.Time image_build_timestamp time.Time @@ -149,7 +150,7 @@ pub fn (mut d Daemon) run() { } // schedule_build adds the next occurence of the given repo build to the queue. -fn (mut d Daemon) schedule_build(repo db.GitRepo) { +fn (mut d Daemon) schedule_build(repo GitRepo) { ce := if repo.schedule != '' { parse_expression(repo.schedule) or { // TODO This shouldn't return an error if the expression is empty. diff --git a/src/db/db.v b/src/db/db.v index 7c1acf1..a5756e6 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -1,6 +1,7 @@ module db import sqlite +import models struct VieterDb { conn sqlite.DB diff --git a/src/db/filter.v b/src/db/filter.v deleted file mode 100644 index 07c890a..0000000 --- a/src/db/filter.v +++ /dev/null @@ -1,26 +0,0 @@ -module db - -[params] -pub struct GitRepoFilter { -pub mut: - limit u64 = 25 - offset u64 - repo string -} - -pub fn filter_from_params(params map[string]string) ?T { - mut o := GitRepoFilter{} - - $for field in T.fields { - if field.name in params { - val := params[field.name] - - $if field.typ is string { - o.$(field.name) = val - } $else $if field.typ is u64 { - o.$(field.name) = val.u64() - } - } - } - return o -} diff --git a/src/db/git.v b/src/db/git.v index 7308346..d8887ff 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -1,84 +1,9 @@ module db -pub struct GitRepoArch { -pub: - id int [primary; sql: serial] - repo_id int [nonull] - value string [nonull] -} - -// str returns a string representation. -pub fn (gra &GitRepoArch) str() string { - return gra.value -} - -pub struct GitRepo { -pub mut: - id int [optional; primary; sql: serial] - // URL of the Git repository - url string [nonull] - // Branch of the Git repository to use - branch string [nonull] - // Which repo the builder should publish packages to - repo string [nonull] - // Cron schedule describing how frequently to build the repo. - schedule string [optional] - // On which architectures the package is allowed to be built. In reality, - // this controls which builders will periodically build the image. - arch []GitRepoArch [fkey: 'repo_id'] -} - -// str returns a string representation. -pub fn (gr &GitRepo) str() string { - mut parts := [ - 'id: $gr.id', - 'url: $gr.url', - 'branch: $gr.branch', - 'repo: $gr.repo', - 'schedule: $gr.schedule', - 'arch: ${gr.arch.map(it.value).join(', ')}', - ] - str := parts.join('\n') - - return str -} - -// patch_from_params patches a GitRepo from a map[string]string, usually -// provided from a web.App's params -pub fn (mut r GitRepo) patch_from_params(params map[string]string) { - $for field in GitRepo.fields { - if field.name in params { - $if field.typ is string { - r.$(field.name) = params[field.name] - // This specific type check is needed for the compiler to ensure - // our types are correct - } $else $if field.typ is []GitRepoArch { - r.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) - } - } - } -} - -// git_repo_from_params creates a GitRepo from a map[string]string, usually -// provided from a web.App's params -pub fn git_repo_from_params(params map[string]string) ?GitRepo { - mut repo := GitRepo{} - - // If we're creating a new GitRepo, we want all fields to be present before - // "patching". - $for field in GitRepo.fields { - if field.name !in params && !field.attrs.contains('optional') { - return error('Missing parameter: ${field.name}.') - } - } - repo.patch_from_params(params) - - return repo -} +import models { GitRepo, GitRepoArch, GitRepoFilter } // get_git_repos returns all GitRepo's in the database. pub fn (db &VieterDb) get_git_repos(filter GitRepoFilter) []GitRepo { - println(filter) // This seems to currently be blocked by a bug in the ORM, I'll have to ask // around. if filter.repo != '' { diff --git a/src/models/git.v b/src/models/git.v new file mode 100644 index 0000000..5dcc13a --- /dev/null +++ b/src/models/git.v @@ -0,0 +1,52 @@ +module models + +pub struct GitRepoArch { +pub: + id int [primary; sql: serial] + repo_id int [nonull] + value string [nonull] +} + +// str returns a string representation. +pub fn (gra &GitRepoArch) str() string { + return gra.value +} + +pub struct GitRepo { +pub mut: + id int [primary; sql: serial] + // URL of the Git repository + url string [nonull] + // Branch of the Git repository to use + branch string [nonull] + // Which repo the builder should publish packages to + repo string [nonull] + // Cron schedule describing how frequently to build the repo. + schedule string + // On which architectures the package is allowed to be built. In reality, + // this controls which builders will periodically build the image. + arch []GitRepoArch [fkey: 'repo_id'] +} + +// str returns a string representation. +pub fn (gr &GitRepo) str() string { + mut parts := [ + 'id: $gr.id', + 'url: $gr.url', + 'branch: $gr.branch', + 'repo: $gr.repo', + 'schedule: $gr.schedule', + 'arch: ${gr.arch.map(it.value).join(', ')}', + ] + str := parts.join('\n') + + return str +} + +[params] +pub struct GitRepoFilter { +pub mut: + limit u64 = 25 + offset u64 + repo string +} diff --git a/src/models/models.v b/src/models/models.v new file mode 100644 index 0000000..a32ae39 --- /dev/null +++ b/src/models/models.v @@ -0,0 +1,36 @@ +module models + +pub fn from_params(params map[string]string) ?T { + mut o := T{} + + patch_from_params(mut o, params)? + + return o +} + +pub fn patch_from_params(mut o T, params map[string]string) ? { + $for field in T.fields { + if field.name in params && params[field.name] != '' { + $if field.typ is string { + o.$(field.name) = params[field.name] + } $else $if field.typ is int { + o.$(field.name) = params[field.name].int() + } $else $if field.typ is u64 { + o.$(field.name) = params[field.name].u64() + } $else $if field.typ is []GitRepoArch { + o.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) + } + } else if field.attrs.contains('nonull') { + return error('Missing parameter: ${field.name}.') + } + } +} + +pub fn params_from(o &T) map[string]string { + mut out := map[string]string{} + + $for field in T.fields { + out[field.name] = o.$(field.name).str() + } + return out +} diff --git a/src/server/git.v b/src/server/git.v index 921251f..ec433c7 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -4,6 +4,7 @@ import web import net.http import response { new_data_response, new_response } import db +import models { GitRepoArch } // get_repos returns the current list of repos. ['/api/repos'; get] @@ -12,7 +13,7 @@ fn (mut app App) get_repos() web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - filter := db.filter_from_params(app.query) or { + filter := models.from_params(app.query) or { return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } repos := app.db.get_git_repos(filter) @@ -47,7 +48,7 @@ fn (mut app App) post_repo() web.Result { params['arch'] = app.conf.default_arch } - new_repo := db.git_repo_from_params(params) or { + new_repo := models.from_params(params) or { return app.json(http.Status.bad_request, new_response(err.msg())) } @@ -78,7 +79,7 @@ fn (mut app App) patch_repo(id int) web.Result { app.db.update_git_repo(id, app.query) if 'arch' in app.query { - arch_objs := app.query['arch'].split(',').map(db.GitRepoArch{ value: it }) + arch_objs := app.query['arch'].split(',').map(GitRepoArch{ value: it }) app.db.update_git_repo_archs(id, arch_objs) } diff --git a/src/util/util.v b/src/util/util.v index cbb4072..f9d47a8 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -71,6 +71,5 @@ pub fn struct_to_map(o T) map[string]string { $for field in T.fields { m[field.name] = o.$(field.name).str() } - return m } From 2fc25f1afefe34d4e196aab7b84f2a122045d154 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 19 May 2022 08:20:11 +0200 Subject: [PATCH 05/19] refactor: moved BuildLog to models --- src/client/logs.v | 2 +- src/console/logs/logs.v | 4 ++-- src/db/db.v | 2 +- src/db/git.v | 1 + src/db/logs.v | 27 +-------------------------- src/models/logs.v | 28 ++++++++++++++++++++++++++++ src/server/git.v | 2 +- src/server/logs.v | 3 ++- 8 files changed, 37 insertions(+), 32 deletions(-) create mode 100644 src/models/logs.v diff --git a/src/client/logs.v b/src/client/logs.v index cdacab9..3484f03 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -1,6 +1,6 @@ module client -import db { BuildLog } +import models { BuildLog } import net.http { Method } import response { Response } import time diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 6400e80..1d0bb55 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -3,8 +3,8 @@ module logs import cli import env import client -import db import console +import models { BuildLog } struct Config { address string [required] @@ -67,7 +67,7 @@ pub fn cmd() cli.Command { } // print_log_list prints a list of logs. -fn print_log_list(logs []db.BuildLog) ? { +fn print_log_list(logs []BuildLog) ? { data := logs.map([it.id.str(), it.repo_id.str(), it.start_time.str(), it.exit_code.str()]) diff --git a/src/db/db.v b/src/db/db.v index a5756e6..57b437e 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -1,7 +1,7 @@ module db import sqlite -import models +import models { BuildLog, GitRepo } struct VieterDb { conn sqlite.DB diff --git a/src/db/git.v b/src/db/git.v index d8887ff..8cc493f 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -66,6 +66,7 @@ pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) { } } values_str := values.join(', ') + // I think this is actual SQL & not the ORM language query := 'update GitRepo set $values_str where id == $repo_id' db.conn.exec_none(query) diff --git a/src/db/logs.v b/src/db/logs.v index 817db78..129ec4e 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -1,31 +1,6 @@ module db -import time - -pub struct BuildLog { -pub: - id int [primary; sql: serial] - repo_id int [nonull] - start_time time.Time [nonull] - end_time time.Time [nonull] - arch string [nonull] - exit_code int [nonull] -} - -// str returns a string representation. -pub fn (bl &BuildLog) str() string { - mut parts := [ - 'id: $bl.id', - 'repo id: $bl.repo_id', - 'start time: $bl.start_time', - 'end time: $bl.end_time', - 'arch: $bl.arch', - 'exit code: $bl.exit_code', - ] - str := parts.join('\n') - - return str -} +import models { BuildLog } // get_build_logs returns all BuildLog's in the database. pub fn (db &VieterDb) get_build_logs() []BuildLog { diff --git a/src/models/logs.v b/src/models/logs.v new file mode 100644 index 0000000..173336f --- /dev/null +++ b/src/models/logs.v @@ -0,0 +1,28 @@ +module models + +import time + +pub struct BuildLog { +pub: + id int [primary; sql: serial] + repo_id int [nonull] + start_time time.Time [nonull] + end_time time.Time [nonull] + arch string [nonull] + exit_code int [nonull] +} + +// str returns a string representation. +pub fn (bl &BuildLog) str() string { + mut parts := [ + 'id: $bl.id', + 'repo id: $bl.repo_id', + 'start time: $bl.start_time', + 'end time: $bl.end_time', + 'arch: $bl.arch', + 'exit code: $bl.exit_code', + ] + str := parts.join('\n') + + return str +} diff --git a/src/server/git.v b/src/server/git.v index ec433c7..c1bc6f3 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -4,7 +4,7 @@ import web import net.http import response { new_data_response, new_response } import db -import models { GitRepoArch } +import models { GitRepo, GitRepoArch, GitRepoFilter } // get_repos returns the current list of repos. ['/api/repos'; get] diff --git a/src/server/logs.v b/src/server/logs.v index 21331e5..af0b081 100644 --- a/src/server/logs.v +++ b/src/server/logs.v @@ -8,6 +8,7 @@ import db import time import os import util +import models { BuildLog } // get_logs returns all build logs in the database. A 'repo' query param can // optionally be added to limit the list of build logs to that repository. @@ -97,7 +98,7 @@ fn (mut app App) post_log() web.Result { } // Store log in db - log := db.BuildLog{ + log := BuildLog{ repo_id: repo_id start_time: start_time end_time: end_time From 0233b8559de41513153101a34311aaaa1147a211 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 19 May 2022 22:14:41 +0200 Subject: [PATCH 06/19] doc: added some missing docstrings --- src/build/build.v | 1 - src/cron/daemon/daemon.v | 1 - src/models/models.v | 5 +++++ src/util/util.v | 9 --------- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 1f26d03..16942bd 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -4,7 +4,6 @@ import docker import encoding.base64 import time import os -import db import strings import util import models { GitRepo } diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index e92dd68..82c219d 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -7,7 +7,6 @@ import cron.expression { CronExpression, parse_expression } import math import build import docker -import db import os import client import models { GitRepo } diff --git a/src/models/models.v b/src/models/models.v index a32ae39..924f45f 100644 --- a/src/models/models.v +++ b/src/models/models.v @@ -1,5 +1,7 @@ module models +// from_params creates a new instance of T from the given map by parsing all +// of its fields from the map. pub fn from_params(params map[string]string) ?T { mut o := T{} @@ -8,6 +10,8 @@ pub fn from_params(params map[string]string) ?T { return o } +// patch_from_params updates the given T object with the params defined in +// the map. pub fn patch_from_params(mut o T, params map[string]string) ? { $for field in T.fields { if field.name in params && params[field.name] != '' { @@ -26,6 +30,7 @@ pub fn patch_from_params(mut o T, params map[string]string) ? { } } +// params_from converts a given T struct into a map of strings. pub fn params_from(o &T) map[string]string { mut out := map[string]string{} diff --git a/src/util/util.v b/src/util/util.v index f9d47a8..266bcb5 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -64,12 +64,3 @@ pub fn pretty_bytes(bytes int) string { return '${n:.2}${util.prefixes[i]}' } - -pub fn struct_to_map(o T) map[string]string { - mut m := map[string]string{} - - $for field in T.fields { - m[field.name] = o.$(field.name).str() - } - return m -} From 7f6e9e636c378978082200e2aa5342c43bba8a51 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 25 May 2022 09:24:01 +0200 Subject: [PATCH 07/19] fix(cron): retrieve all GitRepo's instead of first 25 --- src/client/git.v | 23 ++++++++++++++++++++++- src/cron/daemon/daemon.v | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/client/git.v b/src/client/git.v index fd14718..4496c08 100644 --- a/src/client/git.v +++ b/src/client/git.v @@ -4,7 +4,7 @@ import models { GitRepo, GitRepoFilter } import net.http { Method } import response { Response } -// get_git_repos returns the current list of repos. +// get_git_repos returns a list of GitRepo's, given a filter object. pub fn (c &Client) get_git_repos(filter GitRepoFilter) ?[]GitRepo { params := models.params_from(filter) data := c.send_request<[]GitRepo>(Method.get, '/api/repos', params)? @@ -12,6 +12,27 @@ pub fn (c &Client) get_git_repos(filter GitRepoFilter) ?[]GitRepo { return data.data } +// get_all_git_repos retrieves *all* GitRepo's from the API using the default +// limit. +pub fn (c &Client) get_all_git_repos() ?[]GitRepo { + mut repos := []GitRepo{} + mut offset := u64(0) + + for { + sub_repos := c.get_git_repos(offset: offset)? + + if sub_repos.len == 0 { + break + } + + repos << sub_repos + + offset += u64(sub_repos.len) + } + + return repos +} + // get_git_repo returns the repo for a specific ID. pub fn (c &Client) get_git_repo(id int) ?GitRepo { data := c.send_request(Method.get, '/api/repos/$id', {})? diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 82c219d..f1206d6 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -178,7 +178,7 @@ fn (mut d Daemon) schedule_build(repo GitRepo) { fn (mut d Daemon) renew_repos() { d.linfo('Renewing repos...') - mut new_repos := d.client.get_git_repos() or { + mut new_repos := d.client.get_all_git_repos() or { d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...') d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) From bd4bb9a9fbe51451cbfb6f5abd11169809f5d275 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 26 May 2022 09:15:49 +0200 Subject: [PATCH 08/19] feat: added cli command for previewing cron schedules --- src/console/schedule/schedule.v | 34 +++++++++++++++++++++++++++++++++ src/main.v | 2 ++ 2 files changed, 36 insertions(+) create mode 100644 src/console/schedule/schedule.v diff --git a/src/console/schedule/schedule.v b/src/console/schedule/schedule.v new file mode 100644 index 0000000..b086b53 --- /dev/null +++ b/src/console/schedule/schedule.v @@ -0,0 +1,34 @@ +module schedule + +import cli +import cron.expression { parse_expression } + +// cmd returns the cli submodule for previewing a cron schedule. +pub fn cmd() cli.Command { + return cli.Command{ + name: 'schedule' + description: 'Preview the behavior of a cron schedule.' + flags: [ + cli.Flag{ + name: 'count' + description: 'How many scheduled times to show.' + flag: cli.FlagType.int + default_value: ['5'] + }, + ] + execute: fn (cmd cli.Command) ? { + exp := parse_expression(cmd.args.join(' '))? + + mut t := exp.next_from_now()? + println(t) + + count := cmd.flags.get_int('count')? + + for _ in 1 .. count { + t = exp.next(t)? + + println(t) + } + } + } +} diff --git a/src/main.v b/src/main.v index 6df45dc..885e0f3 100644 --- a/src/main.v +++ b/src/main.v @@ -5,6 +5,7 @@ import server import cli import console.git import console.logs +import console.schedule import cron fn main() { @@ -27,6 +28,7 @@ fn main() { git.cmd(), cron.cmd(), logs.cmd(), + schedule.cmd(), ] } app.setup() From 768da5b7907e27282744aa25946325d7df2d79f8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 26 May 2022 13:34:57 +0200 Subject: [PATCH 09/19] refactor: added CronExpression.next_n function --- src/console/schedule/schedule.v | 12 ++++-------- src/cron/expression/expression.v | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/console/schedule/schedule.v b/src/console/schedule/schedule.v index b086b53..8fceddd 100644 --- a/src/console/schedule/schedule.v +++ b/src/console/schedule/schedule.v @@ -2,11 +2,13 @@ module schedule import cli import cron.expression { parse_expression } +import time // cmd returns the cli submodule for previewing a cron schedule. pub fn cmd() cli.Command { return cli.Command{ name: 'schedule' + usage: 'schedule' description: 'Preview the behavior of a cron schedule.' flags: [ cli.Flag{ @@ -17,16 +19,10 @@ pub fn cmd() cli.Command { }, ] execute: fn (cmd cli.Command) ? { - exp := parse_expression(cmd.args.join(' '))? - - mut t := exp.next_from_now()? - println(t) - + ce := parse_expression(cmd.args.join(' '))? count := cmd.flags.get_int('count')? - for _ in 1 .. count { - t = exp.next(t)? - + for t in ce.next_n(time.now(), count)? { println(t) } } diff --git a/src/cron/expression/expression.v b/src/cron/expression/expression.v index 5eae332..17d2dde 100644 --- a/src/cron/expression/expression.v +++ b/src/cron/expression/expression.v @@ -121,6 +121,20 @@ pub fn (ce &CronExpression) next_from_now() ?time.Time { return ce.next(time.now()) } +// next_n returns the n next occurences of the expression, given a starting +// time. +pub fn (ce &CronExpression) next_n(ref time.Time, n int) ?[]time.Time { + mut times := []time.Time{cap: n} + + times << ce.next(ref)? + + for i in 1 .. n { + times << ce.next(times[i - 1])? + } + + return times +} + // parse_range parses a given string into a range of sorted integers, if // possible. fn parse_range(s string, min int, max int, mut bitv []bool) ? { From 0d5704ba157ecf2f5bce70a7e51e1880fa77f7b5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 24 May 2022 22:48:17 +0200 Subject: [PATCH 10/19] feat(server): initial implementation of migrations --- CHANGELOG.md | 4 ++ src/db/db.v | 52 ++++++++++++++++++++++++-- src/db/migrations/001-initial/down.sql | 3 ++ src/db/migrations/001-initial/up.sql | 22 +++++++++++ src/server/server.v | 4 +- 5 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 src/db/migrations/001-initial/down.sql create mode 100644 src/db/migrations/001-initial/up.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dc0a00..8c336fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev) +### Added + +* Database migrations + ## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) ### Added diff --git a/src/db/db.v b/src/db/db.v index 57b437e..9a06a03 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -1,19 +1,65 @@ module db import sqlite -import models { BuildLog, GitRepo } struct VieterDb { conn sqlite.DB } +struct MigrationVersion { + id int [primary] + version int +} + +const ( + migrations_up = [$embed_file('migrations/001-initial/up.sql')] + migrations_down = [$embed_file('migrations/001-initial/down.sql')] +) + // init initializes a database & adds the correct tables. pub fn init(db_path string) ?VieterDb { conn := sqlite.connect(db_path)? sql conn { - create table GitRepo - create table BuildLog + create table MigrationVersion + } + + cur_version := sql conn { + select from MigrationVersion limit 1 + } + + // If there's no row yet, we add it here + if cur_version == MigrationVersion{} { + sql conn { + insert cur_version into MigrationVersion + } + } + + // Apply each migration in order + for i in cur_version.version .. db.migrations_up.len { + migration := db.migrations_up[i].to_string() + + version_num := i + 1 + + // vfmt does not like these dots + println('Applying migration $version_num' + '...') + + // The sqlite library seems to not like it when multiple statements are + // passed in a single exec. Therefore, we split them & run them all + // separately. + for part in migration.split(';').map(it.trim_space()).filter(it != '') { + res := conn.exec_none(part) + + if res != sqlite.sqlite_done { + return error('An error occurred while applying migration $version_num') + } + } + + // The where clause doesn't really matter, as there will always only be + // one entry anyways. + sql conn { + update MigrationVersion set version = version_num where id > 0 + } } return VieterDb{ diff --git a/src/db/migrations/001-initial/down.sql b/src/db/migrations/001-initial/down.sql new file mode 100644 index 0000000..43ad40b --- /dev/null +++ b/src/db/migrations/001-initial/down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS BuildLog; +DROP TABLE IF EXISTS GitRepoArch; +DROP TABLE IF EXISTS GitRepo; diff --git a/src/db/migrations/001-initial/up.sql b/src/db/migrations/001-initial/up.sql new file mode 100644 index 0000000..ca0aace --- /dev/null +++ b/src/db/migrations/001-initial/up.sql @@ -0,0 +1,22 @@ +CREATE TABLE IF NOT EXISTS GitRepo ( + id INTEGER PRIMARY KEY, + url TEXT NOT NULL, + branch TEXT NOT NULL, + repo TEXT NOT NULL, + schedule TEXT +); + +CREATE TABLE IF NOT EXISTS GitRepoArch ( + id INTEGER PRIMARY KEY, + repo_id INTEGER NOT NULL, + value TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS BuildLog ( + id INTEGER PRIMARY KEY, + repo_id INTEGER NOT NULL, + start_time INTEGER NOT NULL, + end_time iNTEGER NOT NULL, + arch TEXT NOT NULL, + exit_code INTEGER NOT NULL +); diff --git a/src/server/server.v b/src/server/server.v index 090aa76..2309ee7 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -68,7 +68,9 @@ pub fn server(conf Config) ? { } db_file := os.join_path_single(conf.data_dir, server.db_file_name) - db := db.init(db_file) or { util.exit_with_message(1, 'Failed to initialize database.') } + db := db.init(db_file) or { + util.exit_with_message(1, 'Failed to initialize database: $err.msg()') + } web.run(&App{ logger: logger From 31e903ebeb167a47fe15ca829f450138ce69483b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 19 May 2022 22:43:38 +0200 Subject: [PATCH 11/19] feat(server): partial implementation of BuildLog API filter --- CHANGELOG.md | 6 ++++++ src/db/logs.v | 32 ++++++++++++++++++++++++++++---- src/models/logs.v | 13 +++++++++++++ 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c336fe..170c97c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Database migrations +* Query parameters for GitRepo API to filter responses +* Respective CLI flags for new GitRepo API parameters + +### Changed + +* Refactor of main types into `models` module ## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) diff --git a/src/db/logs.v b/src/db/logs.v index 129ec4e..49f905b 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -1,13 +1,37 @@ module db -import models { BuildLog } +import models { BuildLog, BuildLogFilter } +import time // get_build_logs returns all BuildLog's in the database. -pub fn (db &VieterDb) get_build_logs() []BuildLog { - res := sql db.conn { - select from BuildLog order by id +pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { + mut where_parts := []string{} + + if filter.repo != 0 { + where_parts << 'repo_id == $filter.repo' } + if filter.before != time.Time{} { + where_parts << 'start_time < $filter.before.unix_time()' + } + + if filter.after != time.Time{} { + where_parts << 'start_time < $filter.after.unix_time()' + } + + where_str := if where_parts.len > 0 { + ' where ' + where_parts.map('($it)').join(' and ') + } else { + '' + } + + query := 'select from BuildLog' + where_str + res := db.conn.exec(query) + +/* res := sql db.conn { */ +/* select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id */ +/* } */ + return res } diff --git a/src/models/logs.v b/src/models/logs.v index 173336f..1bfc7a6 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -26,3 +26,16 @@ pub fn (bl &BuildLog) str() string { return str } + +[params] +pub struct BuildLogFilter { +pub mut: + limit u64 = 25 + offset u64 + repo int + before time.Time + after time.Time + exit_codes_whitelist []u8 + exit_codes_blacklist []u8 + arch string +} From 4f32dec5b5e689afe78ed3df1fcde15525027e17 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 28 May 2022 22:22:55 +0200 Subject: [PATCH 12/19] feat(db): added function to convert sqlite output to struct --- src/db/db.v | 19 +++++++++++++++++++ src/db/logs.v | 17 +++++++++-------- src/models/logs.v | 2 +- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/db/db.v b/src/db/db.v index 9a06a03..d6cc057 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -1,6 +1,7 @@ module db import sqlite +import time struct VieterDb { conn sqlite.DB @@ -66,3 +67,21 @@ pub fn init(db_path string) ?VieterDb { conn: conn } } + +pub fn row_into(row sqlite.Row) T { + mut i := 0 + mut out := T{} + + $for field in T.fields { + $if field.typ is string { + out.$(field.name) = row.vals[i] + } $else $if field.typ is int { + out.$(field.name) = row.vals[i].int() + } $else $if field.typ is time.Time { + out.$(field.name) = time.unix(row.vals[i].int()) + } + + i += 1 + } + return out +} diff --git a/src/db/logs.v b/src/db/logs.v index 49f905b..4ec48c7 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -19,18 +19,19 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { where_parts << 'start_time < $filter.after.unix_time()' } - where_str := if where_parts.len > 0 { - ' where ' + where_parts.map('($it)').join(' and ') - } else { - '' + mut where_str := '' + + if where_parts.len > 0 { + where_str = ' where ' + where_parts.map('($it)').join(' and ') } query := 'select from BuildLog' + where_str - res := db.conn.exec(query) + rows, _ := db.conn.exec(query) + res := rows.map(row_into(it)) -/* res := sql db.conn { */ -/* select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id */ -/* } */ + // res := sql db.conn { + // select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id + // } return res } diff --git a/src/models/logs.v b/src/models/logs.v index 1bfc7a6..82fc52f 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -3,7 +3,7 @@ module models import time pub struct BuildLog { -pub: +pub mut: id int [primary; sql: serial] repo_id int [nonull] start_time time.Time [nonull] From a39c1aa5eb4031d3302ab1cf817f3c3903aa9412 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 29 May 2022 20:08:21 +0200 Subject: [PATCH 13/19] feat(server): added proper filtering the BuildLog API --- src/db/logs.v | 35 ++++++++++++++++++++++++++++------- src/models/logs.v | 3 +-- src/models/models.v | 7 +++++++ src/server/logs.v | 9 ++++----- 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/db/logs.v b/src/db/logs.v index 4ec48c7..806f3d8 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -16,23 +16,44 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { } if filter.after != time.Time{} { - where_parts << 'start_time < $filter.after.unix_time()' + where_parts << 'start_time > $filter.after.unix_time()' + } + + // NOTE: possible SQL injection + if filter.arch != '' { + where_parts << "arch == '$filter.arch'" + } + + println(filter.exit_codes) + + mut parts := []string{} + + for exp in filter.exit_codes { + if exp[0] == `!` { + code := exp[1..].int() + + parts << 'exit_code != $code' + }else { + code := exp.int() + + parts << 'exit_code == $code' + } + } + + if parts.len > 0 { + where_parts << parts.map('($it)').join(' or ') } mut where_str := '' if where_parts.len > 0 { - where_str = ' where ' + where_parts.map('($it)').join(' and ') + where_str = 'where ' + where_parts.map('($it)').join(' and ') } - query := 'select from BuildLog' + where_str + query := 'select * from BuildLog $where_str limit $filter.limit offset $filter.offset' rows, _ := db.conn.exec(query) res := rows.map(row_into(it)) - // res := sql db.conn { - // select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id - // } - return res } diff --git a/src/models/logs.v b/src/models/logs.v index 82fc52f..5ea9dd5 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -35,7 +35,6 @@ pub mut: repo int before time.Time after time.Time - exit_codes_whitelist []u8 - exit_codes_blacklist []u8 arch string + exit_codes []string } diff --git a/src/models/models.v b/src/models/models.v index 924f45f..0d0395a 100644 --- a/src/models/models.v +++ b/src/models/models.v @@ -1,5 +1,7 @@ module models +import time + // from_params creates a new instance of T from the given map by parsing all // of its fields from the map. pub fn from_params(params map[string]string) ?T { @@ -23,7 +25,12 @@ pub fn patch_from_params(mut o T, params map[string]string) ? { o.$(field.name) = params[field.name].u64() } $else $if field.typ is []GitRepoArch { o.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) + } $else $if field.typ is time.Time { + o.$(field.name) = time.unix(params[field.name].int()) + } $else $if field.typ is []string { + o.$(field.name) = params[field.name].split(',') } + } else if field.attrs.contains('nonull') { return error('Missing parameter: ${field.name}.') } diff --git a/src/server/logs.v b/src/server/logs.v index af0b081..51b364f 100644 --- a/src/server/logs.v +++ b/src/server/logs.v @@ -8,7 +8,7 @@ import db import time import os import util -import models { BuildLog } +import models { BuildLog, BuildLogFilter } // get_logs returns all build logs in the database. A 'repo' query param can // optionally be added to limit the list of build logs to that repository. @@ -18,11 +18,10 @@ fn (mut app App) get_logs() web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - logs := if 'repo' in app.query { - app.db.get_build_logs_for_repo(app.query['repo'].int()) - } else { - app.db.get_build_logs() + filter := models.from_params(app.query) or { + return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } + logs := app.db.get_build_logs(filter) return app.json(http.Status.ok, new_data_response(logs)) } From 401e0291e39dd0e3f92c98db993ec63e0ab7cc10 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 29 May 2022 21:07:46 +0200 Subject: [PATCH 14/19] feat(cli): added some filter flags to GitRepo CLI --- src/client/logs.v | 7 +++-- src/console/logs/logs.v | 67 ++++++++++++++++++++++++++++++++++++----- src/db/logs.v | 4 +-- src/models/logs.v | 12 ++++---- src/models/models.v | 9 ++++-- 5 files changed, 77 insertions(+), 22 deletions(-) diff --git a/src/client/logs.v b/src/client/logs.v index 3484f03..d4d373f 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -1,13 +1,14 @@ module client -import models { BuildLog } +import models { BuildLog, BuildLogFilter } import net.http { Method } import response { Response } import time // get_build_logs returns all build logs. -pub fn (c &Client) get_build_logs() ?Response<[]BuildLog> { - data := c.send_request<[]BuildLog>(Method.get, '/api/logs', {})? +pub fn (c &Client) get_build_logs(filter BuildLogFilter) ?Response<[]BuildLog> { + params := models.params_from(filter) + data := c.send_request<[]BuildLog>(Method.get, '/api/logs', params)? return data } diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 1d0bb55..48b9df9 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -4,7 +4,8 @@ import cli import env import client import console -import models { BuildLog } +import time +import models { BuildLog, BuildLogFilter } struct Config { address string [required] @@ -19,21 +20,71 @@ pub fn cmd() cli.Command { commands: [ cli.Command{ name: 'list' - description: 'List the build logs. If a repo ID is provided, only list the build logs for that repo.' + description: 'List build logs.' flags: [ cli.Flag{ - name: 'repo' - description: 'ID of the Git repo to restrict list to.' + name: 'limit' + description: 'How many results to return.' flag: cli.FlagType.int }, + cli.Flag{ + name: 'offset' + description: 'Minimum index to return.' + flag: cli.FlagType.int + }, + cli.Flag{ + name: 'repo' + description: 'Only return logs for this repo id.' + flag: cli.FlagType.int + }, + cli.Flag{ + name: 'today' + description: 'Only list logs started today (UTC time).' + flag: cli.FlagType.bool + }, + cli.Flag{ + name: 'failed' + description: 'Only list logs with non-zero exit codes.' + flag: cli.FlagType.bool + }, ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? conf := env.load(config_file)? - repo_id := cmd.flags.get_int('repo')? + mut filter := BuildLogFilter{} - if repo_id == 0 { list(conf)? } else { list_for_repo(conf, repo_id)? } + limit := cmd.flags.get_int('limit')? + if limit != 0 { + filter.limit = u64(limit) + } + + offset := cmd.flags.get_int('offset')? + if offset != 0 { + filter.offset = u64(offset) + } + + repo_id := cmd.flags.get_int('repo')? + if repo_id != 0 { + filter.repo = repo_id + } + + if cmd.flags.get_bool('today')? { + today := time.now() + + filter.after = time.new_time(time.Time{ + year: today.year + month: today.month + day: today.day + }) + filter.before = filter.after.add_days(1) + } + + if cmd.flags.get_bool('failed')? { + filter.exit_codes = ['!0'] + } + + list(conf, filter)? } }, cli.Command{ @@ -75,9 +126,9 @@ fn print_log_list(logs []BuildLog) ? { } // list prints a list of all build logs. -fn list(conf Config) ? { +fn list(conf Config, filter BuildLogFilter) ? { c := client.new(conf.address, conf.api_key) - logs := c.get_build_logs()?.data + logs := c.get_build_logs(filter)?.data print_log_list(logs)? } diff --git a/src/db/logs.v b/src/db/logs.v index 806f3d8..cac08e7 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -24,8 +24,6 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { where_parts << "arch == '$filter.arch'" } - println(filter.exit_codes) - mut parts := []string{} for exp in filter.exit_codes { @@ -33,7 +31,7 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { code := exp[1..].int() parts << 'exit_code != $code' - }else { + } else { code := exp.int() parts << 'exit_code == $code' diff --git a/src/models/logs.v b/src/models/logs.v index 5ea9dd5..c92dc07 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -30,11 +30,11 @@ pub fn (bl &BuildLog) str() string { [params] pub struct BuildLogFilter { pub mut: - limit u64 = 25 - offset u64 - repo int - before time.Time - after time.Time - arch string + limit u64 = 25 + offset u64 + repo int + before time.Time + after time.Time + arch string exit_codes []string } diff --git a/src/models/models.v b/src/models/models.v index 0d0395a..3a127bc 100644 --- a/src/models/models.v +++ b/src/models/models.v @@ -30,7 +30,6 @@ pub fn patch_from_params(mut o T, params map[string]string) ? { } $else $if field.typ is []string { o.$(field.name) = params[field.name].split(',') } - } else if field.attrs.contains('nonull') { return error('Missing parameter: ${field.name}.') } @@ -42,7 +41,13 @@ pub fn params_from(o &T) map[string]string { mut out := map[string]string{} $for field in T.fields { - out[field.name] = o.$(field.name).str() + $if field.typ is time.Time { + out[field.name] = o.$(field.name).unix_time().str() + } $else $if field.typ is []string { + out[field.name] = o.$(field.name).join(',') + } $else { + out[field.name] = o.$(field.name).str() + } } return out } From a4ffc2c0e393464eb3c09d89d2873f018e3a76b4 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 29 May 2022 22:16:39 +0200 Subject: [PATCH 15/19] feat(cli): added more advanced date flags for BuildLog CLI --- CHANGELOG.md | 11 +++++----- src/console/logs/logs.v | 45 ++++++++++++++++++++++++++++++++++++++++- src/db/db.v | 2 ++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 170c97c..f9fee6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Database migrations -* Query parameters for GitRepo API to filter responses -* Respective CLI flags for new GitRepo API parameters - -### Changed - -* Refactor of main types into `models` module +* Improved GitRepo & BuildLog API + * Pagination using `limit` & `offset` query params + * GitRepo: filter by repo + * BuildLog: filter by start & end date, repo, exit code & arch +* CLI flags to take advantage of above API improvements ## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 48b9df9..3b1c756 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -47,6 +47,21 @@ pub fn cmd() cli.Command { description: 'Only list logs with non-zero exit codes.' flag: cli.FlagType.bool }, + cli.Flag{ + name: 'day' + description: 'Only list logs started on this day. Format is YYYY-MM-DD.' + flag: cli.FlagType.string + }, + cli.Flag{ + name: 'before' + description: 'Only list logs started before this timestamp. Accepts any RFC 3339 date.' + flag: cli.FlagType.string + }, + cli.Flag{ + name: 'after' + description: 'Only list logs started after this timestamp. Accepts any RFC 3339 date.' + flag: cli.FlagType.string + }, ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? @@ -79,9 +94,37 @@ pub fn cmd() cli.Command { }) filter.before = filter.after.add_days(1) } + // The -today flag overwrites any of the other date flags. + else { + day_str := cmd.flags.get_string('day')? + before_str := cmd.flags.get_string('before')? + after_str := cmd.flags.get_string('after')? + + if day_str != '' { + day := time.parse_rfc3339(day_str)? + + filter.after = time.new_time(time.Time{ + year: day.year + month: day.month + day: day.day + }) + + filter.before = filter.after.add_days(1) + } else { + if before_str != '' { + filter.before = time.parse_rfc3339(before_str)? + } + + if after_str != '' { + filter.after = time.parse_rfc3339(after_str)? + } + } + } if cmd.flags.get_bool('failed')? { - filter.exit_codes = ['!0'] + filter.exit_codes = [ + '!0', + ] } list(conf, filter)? diff --git a/src/db/db.v b/src/db/db.v index d6cc057..fac1458 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -68,6 +68,8 @@ pub fn init(db_path string) ?VieterDb { } } +// row_into converts an sqlite.Row into a given type T by parsing each field +// from a string according to its type. pub fn row_into(row sqlite.Row) T { mut i := 0 mut out := T{} From edd71b41c2910ac27805c97150d80a9e26184763 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 31 May 2022 12:29:35 +0200 Subject: [PATCH 16/19] feat(cli): interpet input dates & print dates as local timezone --- src/console/logs/logs.v | 32 +++++++++++++++++++------------- src/docker/containers.v | 1 + 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 3b1c756..cb6997f 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -20,7 +20,7 @@ pub fn cmd() cli.Command { commands: [ cli.Command{ name: 'list' - description: 'List build logs.' + description: 'List build logs. All date strings in the output are converted to the local timezone. Any time strings provided as input should be in the local timezone as well.' flags: [ cli.Flag{ name: 'limit' @@ -39,7 +39,7 @@ pub fn cmd() cli.Command { }, cli.Flag{ name: 'today' - description: 'Only list logs started today (UTC time).' + description: 'Only list logs started today.' flag: cli.FlagType.bool }, cli.Flag{ @@ -49,17 +49,17 @@ pub fn cmd() cli.Command { }, cli.Flag{ name: 'day' - description: 'Only list logs started on this day. Format is YYYY-MM-DD.' + description: 'Only list logs started on this day. (format: YYYY-MM-DD)' flag: cli.FlagType.string }, cli.Flag{ name: 'before' - description: 'Only list logs started before this timestamp. Accepts any RFC 3339 date.' + description: 'Only list logs started before this timestamp. (format: YYYY-MM-DD HH:mm:ss)' flag: cli.FlagType.string }, cli.Flag{ name: 'after' - description: 'Only list logs started after this timestamp. Accepts any RFC 3339 date.' + description: 'Only list logs started after this timestamp. (format: YYYY-MM-DD HH:mm:ss)' flag: cli.FlagType.string }, ] @@ -84,6 +84,8 @@ pub fn cmd() cli.Command { filter.repo = repo_id } + tz_offset := time.offset() + if cmd.flags.get_bool('today')? { today := time.now() @@ -91,7 +93,7 @@ pub fn cmd() cli.Command { year: today.year month: today.month day: today.day - }) + }).add_seconds(-tz_offset) filter.before = filter.after.add_days(1) } // The -today flag overwrites any of the other date flags. @@ -102,21 +104,25 @@ pub fn cmd() cli.Command { if day_str != '' { day := time.parse_rfc3339(day_str)? - - filter.after = time.new_time(time.Time{ + day_utc := time.new_time(time.Time{ year: day.year month: day.month day: day.day - }) + }).add_seconds(-tz_offset) - filter.before = filter.after.add_days(1) + // The extra -1 is so we also return logs that + // started at exactly midnight (filter bounds are + // exclusive). therefore, we have to request logs + // started after 23:59:59 the previous day. + filter.after = day_utc.add_seconds(-1) + filter.before = day_utc.add_days(1) } else { if before_str != '' { - filter.before = time.parse_rfc3339(before_str)? + filter.before = time.parse(before_str)?.add_seconds(-tz_offset) } if after_str != '' { - filter.after = time.parse_rfc3339(after_str)? + filter.after = time.parse(after_str)?.add_seconds(-tz_offset) } } } @@ -162,7 +168,7 @@ pub fn cmd() cli.Command { // print_log_list prints a list of logs. fn print_log_list(logs []BuildLog) ? { - data := logs.map([it.id.str(), it.repo_id.str(), it.start_time.str(), + data := logs.map([it.id.str(), it.repo_id.str(), it.start_time.local().str(), it.exit_code.str()]) println(console.pretty_table(['id', 'repo', 'start time', 'exit code'], data)?) diff --git a/src/docker/containers.v b/src/docker/containers.v index 0bc59bb..8fbf027 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -83,6 +83,7 @@ pub fn (mut d DockerConn) inspect_container(id string) ?ContainerInspect { mut data := json.decode(ContainerInspect, body)? + // The Docker engine API *should* always return UTC time. data.state.start_time = time.parse_rfc3339(data.state.start_time_str)? if data.state.status == 'exited' { From aded6d438a2f2daa1beb36e6866f5601c8538b52 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 31 May 2022 12:41:50 +0200 Subject: [PATCH 17/19] feat(cli): use correct timezones strings for log info; show build duration --- src/models/logs.v | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/models/logs.v b/src/models/logs.v index c92dc07..7f5a5fe 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -17,8 +17,9 @@ pub fn (bl &BuildLog) str() string { mut parts := [ 'id: $bl.id', 'repo id: $bl.repo_id', - 'start time: $bl.start_time', - 'end time: $bl.end_time', + 'start time: $bl.start_time.local()', + 'end time: $bl.end_time.local()', + 'duration: ${bl.end_time - bl.start_time}', 'arch: $bl.arch', 'exit code: $bl.exit_code', ] From 9f753f9c93f53b0251e14233f5439e3fc9d6e39e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 12 May 2022 09:11:18 +0200 Subject: [PATCH 18/19] feat(build): add target repo to builds; update system for every build --- .woodpecker/.build.yml | 2 +- src/build/build.v | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 1698129..b0fd267 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -45,7 +45,7 @@ pipeline: - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')" - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH" - - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64` + - export SIGNATURE="$(echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64)" - > curl --silent diff --git a/src/build/build.v b/src/build/build.v index 16942bd..3c9cae5 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -102,16 +102,26 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Gi build_arch := os.uname().machine + repo_url := '$address/$repo.repo' + // TODO what to do with PKGBUILDs that build multiple packages? commands := [ + // This will later be replaced by a proper setting for changing the + // mirrorlist + "echo -e '[$repo.repo]\nServer = $address/\$repo/\$arch\nSigLevel = Optional' >> /etc/pacman.conf" + // We need to update the package list of the repo we just added above. + // This should however not pull in a lot of packages as long as the + // builder image is rebuilt frequently. + 'pacman -Syu --needed --noconfirm', + 'su builder', 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', 'cd repo', 'makepkg --nobuild --syncdeps --needed --noconfirm', 'source PKGBUILD', // The build container checks whether the package is already // present on the server - 'curl -s --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', - 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done', + 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', ] // We convert the list of commands into a base64 string, which then gets @@ -124,7 +134,7 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Gi entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e'] work_dir: '/build' - user: 'builder:builder' + // user: 'builder:builder' } id := dd.create_container(c)?.id From 48e2ae7645f2de788801185bb096aee11cfbf983 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 1 Jun 2022 20:34:36 +0200 Subject: [PATCH 19/19] feat(build): show shell commands in build logs --- CHANGELOG.md | 9 +++++++ src/build/build.v | 33 +++++------------------ src/build/build_script.sh | 20 ++++++++++++++ src/build/shell.v | 55 +++++++++++++++++++++++++++++++++++++++ src/build/shell_test.v | 16 ++++++++++++ 5 files changed, 106 insertions(+), 27 deletions(-) create mode 100644 src/build/build_script.sh create mode 100644 src/build/shell.v create mode 100644 src/build/shell_test.v diff --git a/CHANGELOG.md b/CHANGELOG.md index f9fee6b..affd5c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * BuildLog: filter by start & end date, repo, exit code & arch * CLI flags to take advantage of above API improvements +### Changed + +* Packages from target repo are available during builds + * This can be used as a basic way to support AUR dependencies, by adding + the dependencies to the same repository +* Every build now updates its packages first instead of solely relying on the + updated builder image +* Build logs now show commands being executed + ## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) ### Added diff --git a/src/build/build.v b/src/build/build.v index 3c9cae5..2e86471 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -101,40 +101,19 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Gi } build_arch := os.uname().machine + build_script := create_build_script(address, repo, build_arch) - repo_url := '$address/$repo.repo' - - // TODO what to do with PKGBUILDs that build multiple packages? - commands := [ - // This will later be replaced by a proper setting for changing the - // mirrorlist - "echo -e '[$repo.repo]\nServer = $address/\$repo/\$arch\nSigLevel = Optional' >> /etc/pacman.conf" - // We need to update the package list of the repo we just added above. - // This should however not pull in a lot of packages as long as the - // builder image is rebuilt frequently. - 'pacman -Syu --needed --noconfirm', - 'su builder', - 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', - 'cd repo', - 'makepkg --nobuild --syncdeps --needed --noconfirm', - 'source PKGBUILD', - // The build container checks whether the package is already - // present on the server - 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', - 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', - ] - - // We convert the list of commands into a base64 string, which then gets - // passed to the container as an env var - cmds_str := base64.encode_str(commands.join('\n')) + // We convert the build script into a base64 string, which then gets passed + // to the container as an env var + base64_script := base64.encode_str(build_script) c := docker.NewContainer{ image: '$base_image_id' - env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$api_key'] + env: ['BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key'] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e'] work_dir: '/build' - // user: 'builder:builder' + user: '0:0' } id := dd.create_container(c)?.id diff --git a/src/build/build_script.sh b/src/build/build_script.sh new file mode 100644 index 0000000..29f163e --- /dev/null +++ b/src/build/build_script.sh @@ -0,0 +1,20 @@ +echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' +echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf +echo -e '+ pacman -Syu --needed --noconfirm' +pacman -Syu --needed --noconfirm +echo -e '+ su builder' +su builder +echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo' +git clone --single-branch --depth 1 --branch main https://examplerepo.com repo +echo -e '+ cd repo' +cd repo +echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' +makepkg --nobuild --syncdeps --needed --noconfirm +echo -e '+ source PKGBUILD' +source PKGBUILD +echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' +curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 +echo -e '+ [ "$(id -u)" == 0 ] && exit 0' +[ "$(id -u)" == 0 ] && exit 0 +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/shell.v b/src/build/shell.v new file mode 100644 index 0000000..a3121fe --- /dev/null +++ b/src/build/shell.v @@ -0,0 +1,55 @@ +module build + +import models { GitRepo } + +// escape_shell_string escapes any characters that could be interpreted +// incorrectly by a shell. The resulting value should be safe to use inside an +// echo statement. +fn escape_shell_string(s string) string { + return s.replace(r'\', r'\\').replace("'", r"'\''") +} + +// echo_commands takes a list of shell commands & prepends each one with +// an echo call displaying said command. +pub fn echo_commands(cmds []string) []string { + mut out := []string{cap: 2 * cmds.len} + + for cmd in cmds { + out << "echo -e '+ ${escape_shell_string(cmd)}'" + out << cmd + } + + return out +} + +// create_build_script generates a shell script that builds a given GitRepo. +fn create_build_script(address string, repo &GitRepo, build_arch string) string { + repo_url := '$address/$repo.repo' + + commands := echo_commands([ + // This will later be replaced by a proper setting for changing the + // mirrorlist + "echo -e '[$repo.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" + // We need to update the package list of the repo we just added above. + // This should however not pull in a lot of packages as long as the + // builder image is rebuilt frequently. + 'pacman -Syu --needed --noconfirm', + // makepkg can't run as root + 'su builder', + 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', + 'cd repo', + 'makepkg --nobuild --syncdeps --needed --noconfirm', + 'source PKGBUILD', + // The build container checks whether the package is already present on + // the server. + 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + // If the above curl command succeeds, we don't need to rebuild the + // package. However, because we're in a su shell, the exit command will + // drop us back into the root shell. Therefore, we must check whether + // we're in root so we don't proceed. + '[ "\$(id -u)" == 0 ] && exit 0', + 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', + ]) + + return commands.join('\n') +} diff --git a/src/build/shell_test.v b/src/build/shell_test.v new file mode 100644 index 0000000..46ab350 --- /dev/null +++ b/src/build/shell_test.v @@ -0,0 +1,16 @@ +module build + +import models { GitRepo } + +fn test_create_build_script() { + repo := GitRepo{ + id: 1 + url: 'https://examplerepo.com' + branch: 'main' + repo: 'vieter' + } + build_script := create_build_script('https://example.com', repo, 'x86_64') + expected := $embed_file('build_script.sh') + + assert build_script == expected.to_string().trim_space() +}