From 7f6e9e636c378978082200e2aa5342c43bba8a51 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 25 May 2022 09:24:01 +0200 Subject: [PATCH 1/7] fix(cron): retrieve all GitRepo's instead of first 25 --- src/client/git.v | 23 ++++++++++++++++++++++- src/cron/daemon/daemon.v | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/client/git.v b/src/client/git.v index fd14718..4496c08 100644 --- a/src/client/git.v +++ b/src/client/git.v @@ -4,7 +4,7 @@ import models { GitRepo, GitRepoFilter } import net.http { Method } import response { Response } -// get_git_repos returns the current list of repos. +// get_git_repos returns a list of GitRepo's, given a filter object. pub fn (c &Client) get_git_repos(filter GitRepoFilter) ?[]GitRepo { params := models.params_from(filter) data := c.send_request<[]GitRepo>(Method.get, '/api/repos', params)? @@ -12,6 +12,27 @@ pub fn (c &Client) get_git_repos(filter GitRepoFilter) ?[]GitRepo { return data.data } +// get_all_git_repos retrieves *all* GitRepo's from the API using the default +// limit. +pub fn (c &Client) get_all_git_repos() ?[]GitRepo { + mut repos := []GitRepo{} + mut offset := u64(0) + + for { + sub_repos := c.get_git_repos(offset: offset)? + + if sub_repos.len == 0 { + break + } + + repos << sub_repos + + offset += u64(sub_repos.len) + } + + return repos +} + // get_git_repo returns the repo for a specific ID. pub fn (c &Client) get_git_repo(id int) ?GitRepo { data := c.send_request(Method.get, '/api/repos/$id', {})? diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 82c219d..f1206d6 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -178,7 +178,7 @@ fn (mut d Daemon) schedule_build(repo GitRepo) { fn (mut d Daemon) renew_repos() { d.linfo('Renewing repos...') - mut new_repos := d.client.get_git_repos() or { + mut new_repos := d.client.get_all_git_repos() or { d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...') d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) From bd4bb9a9fbe51451cbfb6f5abd11169809f5d275 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 26 May 2022 09:15:49 +0200 Subject: [PATCH 2/7] feat: added cli command for previewing cron schedules --- src/console/schedule/schedule.v | 34 +++++++++++++++++++++++++++++++++ src/main.v | 2 ++ 2 files changed, 36 insertions(+) create mode 100644 src/console/schedule/schedule.v diff --git a/src/console/schedule/schedule.v b/src/console/schedule/schedule.v new file mode 100644 index 0000000..b086b53 --- /dev/null +++ b/src/console/schedule/schedule.v @@ -0,0 +1,34 @@ +module schedule + +import cli +import cron.expression { parse_expression } + +// cmd returns the cli submodule for previewing a cron schedule. +pub fn cmd() cli.Command { + return cli.Command{ + name: 'schedule' + description: 'Preview the behavior of a cron schedule.' + flags: [ + cli.Flag{ + name: 'count' + description: 'How many scheduled times to show.' + flag: cli.FlagType.int + default_value: ['5'] + }, + ] + execute: fn (cmd cli.Command) ? { + exp := parse_expression(cmd.args.join(' '))? + + mut t := exp.next_from_now()? + println(t) + + count := cmd.flags.get_int('count')? + + for _ in 1 .. count { + t = exp.next(t)? + + println(t) + } + } + } +} diff --git a/src/main.v b/src/main.v index 6df45dc..885e0f3 100644 --- a/src/main.v +++ b/src/main.v @@ -5,6 +5,7 @@ import server import cli import console.git import console.logs +import console.schedule import cron fn main() { @@ -27,6 +28,7 @@ fn main() { git.cmd(), cron.cmd(), logs.cmd(), + schedule.cmd(), ] } app.setup() From 768da5b7907e27282744aa25946325d7df2d79f8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 26 May 2022 13:34:57 +0200 Subject: [PATCH 3/7] refactor: added CronExpression.next_n function --- src/console/schedule/schedule.v | 12 ++++-------- src/cron/expression/expression.v | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/console/schedule/schedule.v b/src/console/schedule/schedule.v index b086b53..8fceddd 100644 --- a/src/console/schedule/schedule.v +++ b/src/console/schedule/schedule.v @@ -2,11 +2,13 @@ module schedule import cli import cron.expression { parse_expression } +import time // cmd returns the cli submodule for previewing a cron schedule. pub fn cmd() cli.Command { return cli.Command{ name: 'schedule' + usage: 'schedule' description: 'Preview the behavior of a cron schedule.' flags: [ cli.Flag{ @@ -17,16 +19,10 @@ pub fn cmd() cli.Command { }, ] execute: fn (cmd cli.Command) ? { - exp := parse_expression(cmd.args.join(' '))? - - mut t := exp.next_from_now()? - println(t) - + ce := parse_expression(cmd.args.join(' '))? count := cmd.flags.get_int('count')? - for _ in 1 .. count { - t = exp.next(t)? - + for t in ce.next_n(time.now(), count)? { println(t) } } diff --git a/src/cron/expression/expression.v b/src/cron/expression/expression.v index 5eae332..17d2dde 100644 --- a/src/cron/expression/expression.v +++ b/src/cron/expression/expression.v @@ -121,6 +121,20 @@ pub fn (ce &CronExpression) next_from_now() ?time.Time { return ce.next(time.now()) } +// next_n returns the n next occurences of the expression, given a starting +// time. +pub fn (ce &CronExpression) next_n(ref time.Time, n int) ?[]time.Time { + mut times := []time.Time{cap: n} + + times << ce.next(ref)? + + for i in 1 .. n { + times << ce.next(times[i - 1])? + } + + return times +} + // parse_range parses a given string into a range of sorted integers, if // possible. fn parse_range(s string, min int, max int, mut bitv []bool) ? { From 0d5704ba157ecf2f5bce70a7e51e1880fa77f7b5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 24 May 2022 22:48:17 +0200 Subject: [PATCH 4/7] feat(server): initial implementation of migrations --- CHANGELOG.md | 4 ++ src/db/db.v | 52 ++++++++++++++++++++++++-- src/db/migrations/001-initial/down.sql | 3 ++ src/db/migrations/001-initial/up.sql | 22 +++++++++++ src/server/server.v | 4 +- 5 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 src/db/migrations/001-initial/down.sql create mode 100644 src/db/migrations/001-initial/up.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dc0a00..8c336fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev) +### Added + +* Database migrations + ## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) ### Added diff --git a/src/db/db.v b/src/db/db.v index 57b437e..9a06a03 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -1,19 +1,65 @@ module db import sqlite -import models { BuildLog, GitRepo } struct VieterDb { conn sqlite.DB } +struct MigrationVersion { + id int [primary] + version int +} + +const ( + migrations_up = [$embed_file('migrations/001-initial/up.sql')] + migrations_down = [$embed_file('migrations/001-initial/down.sql')] +) + // init initializes a database & adds the correct tables. pub fn init(db_path string) ?VieterDb { conn := sqlite.connect(db_path)? sql conn { - create table GitRepo - create table BuildLog + create table MigrationVersion + } + + cur_version := sql conn { + select from MigrationVersion limit 1 + } + + // If there's no row yet, we add it here + if cur_version == MigrationVersion{} { + sql conn { + insert cur_version into MigrationVersion + } + } + + // Apply each migration in order + for i in cur_version.version .. db.migrations_up.len { + migration := db.migrations_up[i].to_string() + + version_num := i + 1 + + // vfmt does not like these dots + println('Applying migration $version_num' + '...') + + // The sqlite library seems to not like it when multiple statements are + // passed in a single exec. Therefore, we split them & run them all + // separately. + for part in migration.split(';').map(it.trim_space()).filter(it != '') { + res := conn.exec_none(part) + + if res != sqlite.sqlite_done { + return error('An error occurred while applying migration $version_num') + } + } + + // The where clause doesn't really matter, as there will always only be + // one entry anyways. + sql conn { + update MigrationVersion set version = version_num where id > 0 + } } return VieterDb{ diff --git a/src/db/migrations/001-initial/down.sql b/src/db/migrations/001-initial/down.sql new file mode 100644 index 0000000..43ad40b --- /dev/null +++ b/src/db/migrations/001-initial/down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS BuildLog; +DROP TABLE IF EXISTS GitRepoArch; +DROP TABLE IF EXISTS GitRepo; diff --git a/src/db/migrations/001-initial/up.sql b/src/db/migrations/001-initial/up.sql new file mode 100644 index 0000000..ca0aace --- /dev/null +++ b/src/db/migrations/001-initial/up.sql @@ -0,0 +1,22 @@ +CREATE TABLE IF NOT EXISTS GitRepo ( + id INTEGER PRIMARY KEY, + url TEXT NOT NULL, + branch TEXT NOT NULL, + repo TEXT NOT NULL, + schedule TEXT +); + +CREATE TABLE IF NOT EXISTS GitRepoArch ( + id INTEGER PRIMARY KEY, + repo_id INTEGER NOT NULL, + value TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS BuildLog ( + id INTEGER PRIMARY KEY, + repo_id INTEGER NOT NULL, + start_time INTEGER NOT NULL, + end_time iNTEGER NOT NULL, + arch TEXT NOT NULL, + exit_code INTEGER NOT NULL +); diff --git a/src/server/server.v b/src/server/server.v index 090aa76..2309ee7 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -68,7 +68,9 @@ pub fn server(conf Config) ? { } db_file := os.join_path_single(conf.data_dir, server.db_file_name) - db := db.init(db_file) or { util.exit_with_message(1, 'Failed to initialize database.') } + db := db.init(db_file) or { + util.exit_with_message(1, 'Failed to initialize database: $err.msg()') + } web.run(&App{ logger: logger From 74baccb507523a10ce3514807a9c4e8433ab36a0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 19 May 2022 22:43:38 +0200 Subject: [PATCH 5/7] WIP: BuildLogFilter --- src/db/logs.v | 6 +++--- src/models/logs.v | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/db/logs.v b/src/db/logs.v index 129ec4e..5c64e1f 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -1,11 +1,11 @@ module db -import models { BuildLog } +import models { BuildLog, BuildLogFilter } // get_build_logs returns all BuildLog's in the database. -pub fn (db &VieterDb) get_build_logs() []BuildLog { +pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { res := sql db.conn { - select from BuildLog order by id + select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id } return res diff --git a/src/models/logs.v b/src/models/logs.v index 173336f..0124d4e 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -26,3 +26,13 @@ pub fn (bl &BuildLog) str() string { return str } + +[params] +pub struct BuildLogFilter { +pub mut: + repo int + before time.Time + after time.Time + failed_only bool + arch string +} From a374924ea1e2bad90bcb4b3c801e2977d3ea4be4 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 22 May 2022 20:29:16 +0200 Subject: [PATCH 6/7] WIP: BuildLog filters --- CHANGELOG.md | 6 ++++++ src/db/logs.v | 28 ++++++++++++++++++++++++++-- src/models/logs.v | 13 ++++++++----- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c336fe..170c97c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Database migrations +* Query parameters for GitRepo API to filter responses +* Respective CLI flags for new GitRepo API parameters + +### Changed + +* Refactor of main types into `models` module ## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) diff --git a/src/db/logs.v b/src/db/logs.v index 5c64e1f..49f905b 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -1,13 +1,37 @@ module db import models { BuildLog, BuildLogFilter } +import time // get_build_logs returns all BuildLog's in the database. pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { - res := sql db.conn { - select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id + mut where_parts := []string{} + + if filter.repo != 0 { + where_parts << 'repo_id == $filter.repo' } + if filter.before != time.Time{} { + where_parts << 'start_time < $filter.before.unix_time()' + } + + if filter.after != time.Time{} { + where_parts << 'start_time < $filter.after.unix_time()' + } + + where_str := if where_parts.len > 0 { + ' where ' + where_parts.map('($it)').join(' and ') + } else { + '' + } + + query := 'select from BuildLog' + where_str + res := db.conn.exec(query) + +/* res := sql db.conn { */ +/* select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id */ +/* } */ + return res } diff --git a/src/models/logs.v b/src/models/logs.v index 0124d4e..1bfc7a6 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -30,9 +30,12 @@ pub fn (bl &BuildLog) str() string { [params] pub struct BuildLogFilter { pub mut: - repo int - before time.Time - after time.Time - failed_only bool - arch string + limit u64 = 25 + offset u64 + repo int + before time.Time + after time.Time + exit_codes_whitelist []u8 + exit_codes_blacklist []u8 + arch string } From 546d79ed2ef34edb5f563c4b9d7628f8078da09d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 28 May 2022 22:22:55 +0200 Subject: [PATCH 7/7] feat(db): added function to convert sqlite output to struct --- src/db/db.v | 19 +++++++++++++++++++ src/db/logs.v | 17 +++++++++-------- src/models/logs.v | 2 +- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/db/db.v b/src/db/db.v index 9a06a03..d6cc057 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -1,6 +1,7 @@ module db import sqlite +import time struct VieterDb { conn sqlite.DB @@ -66,3 +67,21 @@ pub fn init(db_path string) ?VieterDb { conn: conn } } + +pub fn row_into(row sqlite.Row) T { + mut i := 0 + mut out := T{} + + $for field in T.fields { + $if field.typ is string { + out.$(field.name) = row.vals[i] + } $else $if field.typ is int { + out.$(field.name) = row.vals[i].int() + } $else $if field.typ is time.Time { + out.$(field.name) = time.unix(row.vals[i].int()) + } + + i += 1 + } + return out +} diff --git a/src/db/logs.v b/src/db/logs.v index 49f905b..4ec48c7 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -19,18 +19,19 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { where_parts << 'start_time < $filter.after.unix_time()' } - where_str := if where_parts.len > 0 { - ' where ' + where_parts.map('($it)').join(' and ') - } else { - '' + mut where_str := '' + + if where_parts.len > 0 { + where_str = ' where ' + where_parts.map('($it)').join(' and ') } query := 'select from BuildLog' + where_str - res := db.conn.exec(query) + rows, _ := db.conn.exec(query) + res := rows.map(row_into(it)) -/* res := sql db.conn { */ -/* select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id */ -/* } */ + // res := sql db.conn { + // select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id + // } return res } diff --git a/src/models/logs.v b/src/models/logs.v index 1bfc7a6..82fc52f 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -3,7 +3,7 @@ module models import time pub struct BuildLog { -pub: +pub mut: id int [primary; sql: serial] repo_id int [nonull] start_time time.Time [nonull]