From 31e903ebeb167a47fe15ca829f450138ce69483b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 19 May 2022 22:43:38 +0200 Subject: [PATCH 1/5] feat(server): partial implementation of BuildLog API filter --- CHANGELOG.md | 6 ++++++ src/db/logs.v | 32 ++++++++++++++++++++++++++++---- src/models/logs.v | 13 +++++++++++++ 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c336fe..170c97c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Database migrations +* Query parameters for GitRepo API to filter responses +* Respective CLI flags for new GitRepo API parameters + +### Changed + +* Refactor of main types into `models` module ## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) diff --git a/src/db/logs.v b/src/db/logs.v index 129ec4e..49f905b 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -1,13 +1,37 @@ module db -import models { BuildLog } +import models { BuildLog, BuildLogFilter } +import time // get_build_logs returns all BuildLog's in the database. -pub fn (db &VieterDb) get_build_logs() []BuildLog { - res := sql db.conn { - select from BuildLog order by id +pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { + mut where_parts := []string{} + + if filter.repo != 0 { + where_parts << 'repo_id == $filter.repo' } + if filter.before != time.Time{} { + where_parts << 'start_time < $filter.before.unix_time()' + } + + if filter.after != time.Time{} { + where_parts << 'start_time < $filter.after.unix_time()' + } + + where_str := if where_parts.len > 0 { + ' where ' + where_parts.map('($it)').join(' and ') + } else { + '' + } + + query := 'select from BuildLog' + where_str + res := db.conn.exec(query) + +/* res := sql db.conn { */ +/* select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id */ +/* } */ + return res } diff --git a/src/models/logs.v b/src/models/logs.v index 173336f..1bfc7a6 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -26,3 +26,16 @@ pub fn (bl &BuildLog) str() string { return str } + +[params] +pub struct BuildLogFilter { +pub mut: + limit u64 = 25 + offset u64 + repo int + before time.Time + after time.Time + exit_codes_whitelist []u8 + exit_codes_blacklist []u8 + arch string +} From 4f32dec5b5e689afe78ed3df1fcde15525027e17 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 28 May 2022 22:22:55 +0200 Subject: [PATCH 2/5] feat(db): added function to convert sqlite output to struct --- src/db/db.v | 19 +++++++++++++++++++ src/db/logs.v | 17 +++++++++-------- src/models/logs.v | 2 +- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/db/db.v b/src/db/db.v index 9a06a03..d6cc057 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -1,6 +1,7 @@ module db import sqlite +import time struct VieterDb { conn sqlite.DB @@ -66,3 +67,21 @@ pub fn init(db_path string) ?VieterDb { conn: conn } } + +pub fn row_into(row sqlite.Row) T { + mut i := 0 + mut out := T{} + + $for field in T.fields { + $if field.typ is string { + out.$(field.name) = row.vals[i] + } $else $if field.typ is int { + out.$(field.name) = row.vals[i].int() + } $else $if field.typ is time.Time { + out.$(field.name) = time.unix(row.vals[i].int()) + } + + i += 1 + } + return out +} diff --git a/src/db/logs.v b/src/db/logs.v index 49f905b..4ec48c7 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -19,18 +19,19 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { where_parts << 'start_time < $filter.after.unix_time()' } - where_str := if where_parts.len > 0 { - ' where ' + where_parts.map('($it)').join(' and ') - } else { - '' + mut where_str := '' + + if where_parts.len > 0 { + where_str = ' where ' + where_parts.map('($it)').join(' and ') } query := 'select from BuildLog' + where_str - res := db.conn.exec(query) + rows, _ := db.conn.exec(query) + res := rows.map(row_into(it)) -/* res := sql db.conn { */ -/* select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id */ -/* } */ + // res := sql db.conn { + // select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id + // } return res } diff --git a/src/models/logs.v b/src/models/logs.v index 1bfc7a6..82fc52f 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -3,7 +3,7 @@ module models import time pub struct BuildLog { -pub: +pub mut: id int [primary; sql: serial] repo_id int [nonull] start_time time.Time [nonull] From a39c1aa5eb4031d3302ab1cf817f3c3903aa9412 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 29 May 2022 20:08:21 +0200 Subject: [PATCH 3/5] feat(server): added proper filtering the BuildLog API --- src/db/logs.v | 35 ++++++++++++++++++++++++++++------- src/models/logs.v | 3 +-- src/models/models.v | 7 +++++++ src/server/logs.v | 9 ++++----- 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/db/logs.v b/src/db/logs.v index 4ec48c7..806f3d8 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -16,23 +16,44 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { } if filter.after != time.Time{} { - where_parts << 'start_time < $filter.after.unix_time()' + where_parts << 'start_time > $filter.after.unix_time()' + } + + // NOTE: possible SQL injection + if filter.arch != '' { + where_parts << "arch == '$filter.arch'" + } + + println(filter.exit_codes) + + mut parts := []string{} + + for exp in filter.exit_codes { + if exp[0] == `!` { + code := exp[1..].int() + + parts << 'exit_code != $code' + }else { + code := exp.int() + + parts << 'exit_code == $code' + } + } + + if parts.len > 0 { + where_parts << parts.map('($it)').join(' or ') } mut where_str := '' if where_parts.len > 0 { - where_str = ' where ' + where_parts.map('($it)').join(' and ') + where_str = 'where ' + where_parts.map('($it)').join(' and ') } - query := 'select from BuildLog' + where_str + query := 'select * from BuildLog $where_str limit $filter.limit offset $filter.offset' rows, _ := db.conn.exec(query) res := rows.map(row_into(it)) - // res := sql db.conn { - // select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id - // } - return res } diff --git a/src/models/logs.v b/src/models/logs.v index 82fc52f..5ea9dd5 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -35,7 +35,6 @@ pub mut: repo int before time.Time after time.Time - exit_codes_whitelist []u8 - exit_codes_blacklist []u8 arch string + exit_codes []string } diff --git a/src/models/models.v b/src/models/models.v index 924f45f..0d0395a 100644 --- a/src/models/models.v +++ b/src/models/models.v @@ -1,5 +1,7 @@ module models +import time + // from_params creates a new instance of T from the given map by parsing all // of its fields from the map. pub fn from_params(params map[string]string) ?T { @@ -23,7 +25,12 @@ pub fn patch_from_params(mut o T, params map[string]string) ? { o.$(field.name) = params[field.name].u64() } $else $if field.typ is []GitRepoArch { o.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) + } $else $if field.typ is time.Time { + o.$(field.name) = time.unix(params[field.name].int()) + } $else $if field.typ is []string { + o.$(field.name) = params[field.name].split(',') } + } else if field.attrs.contains('nonull') { return error('Missing parameter: ${field.name}.') } diff --git a/src/server/logs.v b/src/server/logs.v index af0b081..51b364f 100644 --- a/src/server/logs.v +++ b/src/server/logs.v @@ -8,7 +8,7 @@ import db import time import os import util -import models { BuildLog } +import models { BuildLog, BuildLogFilter } // get_logs returns all build logs in the database. A 'repo' query param can // optionally be added to limit the list of build logs to that repository. @@ -18,11 +18,10 @@ fn (mut app App) get_logs() web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - logs := if 'repo' in app.query { - app.db.get_build_logs_for_repo(app.query['repo'].int()) - } else { - app.db.get_build_logs() + filter := models.from_params(app.query) or { + return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } + logs := app.db.get_build_logs(filter) return app.json(http.Status.ok, new_data_response(logs)) } From 401e0291e39dd0e3f92c98db993ec63e0ab7cc10 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 29 May 2022 21:07:46 +0200 Subject: [PATCH 4/5] feat(cli): added some filter flags to GitRepo CLI --- src/client/logs.v | 7 +++-- src/console/logs/logs.v | 67 ++++++++++++++++++++++++++++++++++++----- src/db/logs.v | 4 +-- src/models/logs.v | 12 ++++---- src/models/models.v | 9 ++++-- 5 files changed, 77 insertions(+), 22 deletions(-) diff --git a/src/client/logs.v b/src/client/logs.v index 3484f03..d4d373f 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -1,13 +1,14 @@ module client -import models { BuildLog } +import models { BuildLog, BuildLogFilter } import net.http { Method } import response { Response } import time // get_build_logs returns all build logs. -pub fn (c &Client) get_build_logs() ?Response<[]BuildLog> { - data := c.send_request<[]BuildLog>(Method.get, '/api/logs', {})? +pub fn (c &Client) get_build_logs(filter BuildLogFilter) ?Response<[]BuildLog> { + params := models.params_from(filter) + data := c.send_request<[]BuildLog>(Method.get, '/api/logs', params)? return data } diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 1d0bb55..48b9df9 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -4,7 +4,8 @@ import cli import env import client import console -import models { BuildLog } +import time +import models { BuildLog, BuildLogFilter } struct Config { address string [required] @@ -19,21 +20,71 @@ pub fn cmd() cli.Command { commands: [ cli.Command{ name: 'list' - description: 'List the build logs. If a repo ID is provided, only list the build logs for that repo.' + description: 'List build logs.' flags: [ cli.Flag{ - name: 'repo' - description: 'ID of the Git repo to restrict list to.' + name: 'limit' + description: 'How many results to return.' flag: cli.FlagType.int }, + cli.Flag{ + name: 'offset' + description: 'Minimum index to return.' + flag: cli.FlagType.int + }, + cli.Flag{ + name: 'repo' + description: 'Only return logs for this repo id.' + flag: cli.FlagType.int + }, + cli.Flag{ + name: 'today' + description: 'Only list logs started today (UTC time).' + flag: cli.FlagType.bool + }, + cli.Flag{ + name: 'failed' + description: 'Only list logs with non-zero exit codes.' + flag: cli.FlagType.bool + }, ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? conf := env.load(config_file)? - repo_id := cmd.flags.get_int('repo')? + mut filter := BuildLogFilter{} - if repo_id == 0 { list(conf)? } else { list_for_repo(conf, repo_id)? } + limit := cmd.flags.get_int('limit')? + if limit != 0 { + filter.limit = u64(limit) + } + + offset := cmd.flags.get_int('offset')? + if offset != 0 { + filter.offset = u64(offset) + } + + repo_id := cmd.flags.get_int('repo')? + if repo_id != 0 { + filter.repo = repo_id + } + + if cmd.flags.get_bool('today')? { + today := time.now() + + filter.after = time.new_time(time.Time{ + year: today.year + month: today.month + day: today.day + }) + filter.before = filter.after.add_days(1) + } + + if cmd.flags.get_bool('failed')? { + filter.exit_codes = ['!0'] + } + + list(conf, filter)? } }, cli.Command{ @@ -75,9 +126,9 @@ fn print_log_list(logs []BuildLog) ? { } // list prints a list of all build logs. -fn list(conf Config) ? { +fn list(conf Config, filter BuildLogFilter) ? { c := client.new(conf.address, conf.api_key) - logs := c.get_build_logs()?.data + logs := c.get_build_logs(filter)?.data print_log_list(logs)? } diff --git a/src/db/logs.v b/src/db/logs.v index 806f3d8..cac08e7 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -24,8 +24,6 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { where_parts << "arch == '$filter.arch'" } - println(filter.exit_codes) - mut parts := []string{} for exp in filter.exit_codes { @@ -33,7 +31,7 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { code := exp[1..].int() parts << 'exit_code != $code' - }else { + } else { code := exp.int() parts << 'exit_code == $code' diff --git a/src/models/logs.v b/src/models/logs.v index 5ea9dd5..c92dc07 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -30,11 +30,11 @@ pub fn (bl &BuildLog) str() string { [params] pub struct BuildLogFilter { pub mut: - limit u64 = 25 - offset u64 - repo int - before time.Time - after time.Time - arch string + limit u64 = 25 + offset u64 + repo int + before time.Time + after time.Time + arch string exit_codes []string } diff --git a/src/models/models.v b/src/models/models.v index 0d0395a..3a127bc 100644 --- a/src/models/models.v +++ b/src/models/models.v @@ -30,7 +30,6 @@ pub fn patch_from_params(mut o T, params map[string]string) ? { } $else $if field.typ is []string { o.$(field.name) = params[field.name].split(',') } - } else if field.attrs.contains('nonull') { return error('Missing parameter: ${field.name}.') } @@ -42,7 +41,13 @@ pub fn params_from(o &T) map[string]string { mut out := map[string]string{} $for field in T.fields { - out[field.name] = o.$(field.name).str() + $if field.typ is time.Time { + out[field.name] = o.$(field.name).unix_time().str() + } $else $if field.typ is []string { + out[field.name] = o.$(field.name).join(',') + } $else { + out[field.name] = o.$(field.name).str() + } } return out } From a4ffc2c0e393464eb3c09d89d2873f018e3a76b4 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 29 May 2022 22:16:39 +0200 Subject: [PATCH 5/5] feat(cli): added more advanced date flags for BuildLog CLI --- CHANGELOG.md | 11 +++++----- src/console/logs/logs.v | 45 ++++++++++++++++++++++++++++++++++++++++- src/db/db.v | 2 ++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 170c97c..f9fee6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Database migrations -* Query parameters for GitRepo API to filter responses -* Respective CLI flags for new GitRepo API parameters - -### Changed - -* Refactor of main types into `models` module +* Improved GitRepo & BuildLog API + * Pagination using `limit` & `offset` query params + * GitRepo: filter by repo + * BuildLog: filter by start & end date, repo, exit code & arch +* CLI flags to take advantage of above API improvements ## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 48b9df9..3b1c756 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -47,6 +47,21 @@ pub fn cmd() cli.Command { description: 'Only list logs with non-zero exit codes.' flag: cli.FlagType.bool }, + cli.Flag{ + name: 'day' + description: 'Only list logs started on this day. Format is YYYY-MM-DD.' + flag: cli.FlagType.string + }, + cli.Flag{ + name: 'before' + description: 'Only list logs started before this timestamp. Accepts any RFC 3339 date.' + flag: cli.FlagType.string + }, + cli.Flag{ + name: 'after' + description: 'Only list logs started after this timestamp. Accepts any RFC 3339 date.' + flag: cli.FlagType.string + }, ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? @@ -79,9 +94,37 @@ pub fn cmd() cli.Command { }) filter.before = filter.after.add_days(1) } + // The -today flag overwrites any of the other date flags. + else { + day_str := cmd.flags.get_string('day')? + before_str := cmd.flags.get_string('before')? + after_str := cmd.flags.get_string('after')? + + if day_str != '' { + day := time.parse_rfc3339(day_str)? + + filter.after = time.new_time(time.Time{ + year: day.year + month: day.month + day: day.day + }) + + filter.before = filter.after.add_days(1) + } else { + if before_str != '' { + filter.before = time.parse_rfc3339(before_str)? + } + + if after_str != '' { + filter.after = time.parse_rfc3339(after_str)? + } + } + } if cmd.flags.get_bool('failed')? { - filter.exit_codes = ['!0'] + filter.exit_codes = [ + '!0', + ] } list(conf, filter)? diff --git a/src/db/db.v b/src/db/db.v index d6cc057..fac1458 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -68,6 +68,8 @@ pub fn init(db_path string) ?VieterDb { } } +// row_into converts an sqlite.Row into a given type T by parsing each field +// from a string according to its type. pub fn row_into(row sqlite.Row) T { mut i := 0 mut out := T{}