From 74baccb507523a10ce3514807a9c4e8433ab36a0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 19 May 2022 22:43:38 +0200 Subject: [PATCH 1/6] WIP: BuildLogFilter --- src/db/logs.v | 6 +++--- src/models/logs.v | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/db/logs.v b/src/db/logs.v index 129ec4e..5c64e1f 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -1,11 +1,11 @@ module db -import models { BuildLog } +import models { BuildLog, BuildLogFilter } // get_build_logs returns all BuildLog's in the database. -pub fn (db &VieterDb) get_build_logs() []BuildLog { +pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { res := sql db.conn { - select from BuildLog order by id + select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id } return res diff --git a/src/models/logs.v b/src/models/logs.v index 173336f..0124d4e 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -26,3 +26,13 @@ pub fn (bl &BuildLog) str() string { return str } + +[params] +pub struct BuildLogFilter { +pub mut: + repo int + before time.Time + after time.Time + failed_only bool + arch string +} From a374924ea1e2bad90bcb4b3c801e2977d3ea4be4 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 22 May 2022 20:29:16 +0200 Subject: [PATCH 2/6] WIP: BuildLog filters --- CHANGELOG.md | 6 ++++++ src/db/logs.v | 28 ++++++++++++++++++++++++++-- src/models/logs.v | 13 ++++++++----- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c336fe..170c97c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Database migrations +* Query parameters for GitRepo API to filter responses +* Respective CLI flags for new GitRepo API parameters + +### Changed + +* Refactor of main types into `models` module ## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) diff --git a/src/db/logs.v b/src/db/logs.v index 5c64e1f..49f905b 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -1,13 +1,37 @@ module db import models { BuildLog, BuildLogFilter } +import time // get_build_logs returns all BuildLog's in the database. pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { - res := sql db.conn { - select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id + mut where_parts := []string{} + + if filter.repo != 0 { + where_parts << 'repo_id == $filter.repo' } + if filter.before != time.Time{} { + where_parts << 'start_time < $filter.before.unix_time()' + } + + if filter.after != time.Time{} { + where_parts << 'start_time < $filter.after.unix_time()' + } + + where_str := if where_parts.len > 0 { + ' where ' + where_parts.map('($it)').join(' and ') + } else { + '' + } + + query := 'select from BuildLog' + where_str + res := db.conn.exec(query) + +/* res := sql db.conn { */ +/* select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id */ +/* } */ + return res } diff --git a/src/models/logs.v b/src/models/logs.v index 0124d4e..1bfc7a6 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -30,9 +30,12 @@ pub fn (bl &BuildLog) str() string { [params] pub struct BuildLogFilter { pub mut: - repo int - before time.Time - after time.Time - failed_only bool - arch string + limit u64 = 25 + offset u64 + repo int + before time.Time + after time.Time + exit_codes_whitelist []u8 + exit_codes_blacklist []u8 + arch string } From 546d79ed2ef34edb5f563c4b9d7628f8078da09d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 28 May 2022 22:22:55 +0200 Subject: [PATCH 3/6] feat(db): added function to convert sqlite output to struct --- src/db/db.v | 19 +++++++++++++++++++ src/db/logs.v | 17 +++++++++-------- src/models/logs.v | 2 +- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/db/db.v b/src/db/db.v index 9a06a03..d6cc057 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -1,6 +1,7 @@ module db import sqlite +import time struct VieterDb { conn sqlite.DB @@ -66,3 +67,21 @@ pub fn init(db_path string) ?VieterDb { conn: conn } } + +pub fn row_into(row sqlite.Row) T { + mut i := 0 + mut out := T{} + + $for field in T.fields { + $if field.typ is string { + out.$(field.name) = row.vals[i] + } $else $if field.typ is int { + out.$(field.name) = row.vals[i].int() + } $else $if field.typ is time.Time { + out.$(field.name) = time.unix(row.vals[i].int()) + } + + i += 1 + } + return out +} diff --git a/src/db/logs.v b/src/db/logs.v index 49f905b..4ec48c7 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -19,18 +19,19 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { where_parts << 'start_time < $filter.after.unix_time()' } - where_str := if where_parts.len > 0 { - ' where ' + where_parts.map('($it)').join(' and ') - } else { - '' + mut where_str := '' + + if where_parts.len > 0 { + where_str = ' where ' + where_parts.map('($it)').join(' and ') } query := 'select from BuildLog' + where_str - res := db.conn.exec(query) + rows, _ := db.conn.exec(query) + res := rows.map(row_into(it)) -/* res := sql db.conn { */ -/* select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id */ -/* } */ + // res := sql db.conn { + // select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id + // } return res } diff --git a/src/models/logs.v b/src/models/logs.v index 1bfc7a6..82fc52f 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -3,7 +3,7 @@ module models import time pub struct BuildLog { -pub: +pub mut: id int [primary; sql: serial] repo_id int [nonull] start_time time.Time [nonull] From 31e903ebeb167a47fe15ca829f450138ce69483b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 19 May 2022 22:43:38 +0200 Subject: [PATCH 4/6] feat(server): partial implementation of BuildLog API filter --- CHANGELOG.md | 6 ++++++ src/db/logs.v | 32 ++++++++++++++++++++++++++++---- src/models/logs.v | 13 +++++++++++++ 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c336fe..170c97c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Database migrations +* Query parameters for GitRepo API to filter responses +* Respective CLI flags for new GitRepo API parameters + +### Changed + +* Refactor of main types into `models` module ## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) diff --git a/src/db/logs.v b/src/db/logs.v index 129ec4e..49f905b 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -1,13 +1,37 @@ module db -import models { BuildLog } +import models { BuildLog, BuildLogFilter } +import time // get_build_logs returns all BuildLog's in the database. -pub fn (db &VieterDb) get_build_logs() []BuildLog { - res := sql db.conn { - select from BuildLog order by id +pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { + mut where_parts := []string{} + + if filter.repo != 0 { + where_parts << 'repo_id == $filter.repo' } + if filter.before != time.Time{} { + where_parts << 'start_time < $filter.before.unix_time()' + } + + if filter.after != time.Time{} { + where_parts << 'start_time < $filter.after.unix_time()' + } + + where_str := if where_parts.len > 0 { + ' where ' + where_parts.map('($it)').join(' and ') + } else { + '' + } + + query := 'select from BuildLog' + where_str + res := db.conn.exec(query) + +/* res := sql db.conn { */ +/* select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id */ +/* } */ + return res } diff --git a/src/models/logs.v b/src/models/logs.v index 173336f..1bfc7a6 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -26,3 +26,16 @@ pub fn (bl &BuildLog) str() string { return str } + +[params] +pub struct BuildLogFilter { +pub mut: + limit u64 = 25 + offset u64 + repo int + before time.Time + after time.Time + exit_codes_whitelist []u8 + exit_codes_blacklist []u8 + arch string +} From 4f32dec5b5e689afe78ed3df1fcde15525027e17 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 28 May 2022 22:22:55 +0200 Subject: [PATCH 5/6] feat(db): added function to convert sqlite output to struct --- src/db/db.v | 19 +++++++++++++++++++ src/db/logs.v | 17 +++++++++-------- src/models/logs.v | 2 +- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/db/db.v b/src/db/db.v index 9a06a03..d6cc057 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -1,6 +1,7 @@ module db import sqlite +import time struct VieterDb { conn sqlite.DB @@ -66,3 +67,21 @@ pub fn init(db_path string) ?VieterDb { conn: conn } } + +pub fn row_into(row sqlite.Row) T { + mut i := 0 + mut out := T{} + + $for field in T.fields { + $if field.typ is string { + out.$(field.name) = row.vals[i] + } $else $if field.typ is int { + out.$(field.name) = row.vals[i].int() + } $else $if field.typ is time.Time { + out.$(field.name) = time.unix(row.vals[i].int()) + } + + i += 1 + } + return out +} diff --git a/src/db/logs.v b/src/db/logs.v index 49f905b..4ec48c7 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -19,18 +19,19 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { where_parts << 'start_time < $filter.after.unix_time()' } - where_str := if where_parts.len > 0 { - ' where ' + where_parts.map('($it)').join(' and ') - } else { - '' + mut where_str := '' + + if where_parts.len > 0 { + where_str = ' where ' + where_parts.map('($it)').join(' and ') } query := 'select from BuildLog' + where_str - res := db.conn.exec(query) + rows, _ := db.conn.exec(query) + res := rows.map(row_into(it)) -/* res := sql db.conn { */ -/* select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id */ -/* } */ + // res := sql db.conn { + // select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id + // } return res } diff --git a/src/models/logs.v b/src/models/logs.v index 1bfc7a6..82fc52f 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -3,7 +3,7 @@ module models import time pub struct BuildLog { -pub: +pub mut: id int [primary; sql: serial] repo_id int [nonull] start_time time.Time [nonull] From a39c1aa5eb4031d3302ab1cf817f3c3903aa9412 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 29 May 2022 20:08:21 +0200 Subject: [PATCH 6/6] feat(server): added proper filtering the BuildLog API --- src/db/logs.v | 35 ++++++++++++++++++++++++++++------- src/models/logs.v | 3 +-- src/models/models.v | 7 +++++++ src/server/logs.v | 9 ++++----- 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/db/logs.v b/src/db/logs.v index 4ec48c7..806f3d8 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -16,23 +16,44 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { } if filter.after != time.Time{} { - where_parts << 'start_time < $filter.after.unix_time()' + where_parts << 'start_time > $filter.after.unix_time()' + } + + // NOTE: possible SQL injection + if filter.arch != '' { + where_parts << "arch == '$filter.arch'" + } + + println(filter.exit_codes) + + mut parts := []string{} + + for exp in filter.exit_codes { + if exp[0] == `!` { + code := exp[1..].int() + + parts << 'exit_code != $code' + }else { + code := exp.int() + + parts << 'exit_code == $code' + } + } + + if parts.len > 0 { + where_parts << parts.map('($it)').join(' or ') } mut where_str := '' if where_parts.len > 0 { - where_str = ' where ' + where_parts.map('($it)').join(' and ') + where_str = 'where ' + where_parts.map('($it)').join(' and ') } - query := 'select from BuildLog' + where_str + query := 'select * from BuildLog $where_str limit $filter.limit offset $filter.offset' rows, _ := db.conn.exec(query) res := rows.map(row_into(it)) - // res := sql db.conn { - // select from BuildLog where filter.repo == 0 || repo_id == filter.repo order by id - // } - return res } diff --git a/src/models/logs.v b/src/models/logs.v index 82fc52f..5ea9dd5 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -35,7 +35,6 @@ pub mut: repo int before time.Time after time.Time - exit_codes_whitelist []u8 - exit_codes_blacklist []u8 arch string + exit_codes []string } diff --git a/src/models/models.v b/src/models/models.v index 924f45f..0d0395a 100644 --- a/src/models/models.v +++ b/src/models/models.v @@ -1,5 +1,7 @@ module models +import time + // from_params creates a new instance of T from the given map by parsing all // of its fields from the map. pub fn from_params(params map[string]string) ?T { @@ -23,7 +25,12 @@ pub fn patch_from_params(mut o T, params map[string]string) ? { o.$(field.name) = params[field.name].u64() } $else $if field.typ is []GitRepoArch { o.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) + } $else $if field.typ is time.Time { + o.$(field.name) = time.unix(params[field.name].int()) + } $else $if field.typ is []string { + o.$(field.name) = params[field.name].split(',') } + } else if field.attrs.contains('nonull') { return error('Missing parameter: ${field.name}.') } diff --git a/src/server/logs.v b/src/server/logs.v index af0b081..51b364f 100644 --- a/src/server/logs.v +++ b/src/server/logs.v @@ -8,7 +8,7 @@ import db import time import os import util -import models { BuildLog } +import models { BuildLog, BuildLogFilter } // get_logs returns all build logs in the database. A 'repo' query param can // optionally be added to limit the list of build logs to that repository. @@ -18,11 +18,10 @@ fn (mut app App) get_logs() web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - logs := if 'repo' in app.query { - app.db.get_build_logs_for_repo(app.query['repo'].int()) - } else { - app.db.get_build_logs() + filter := models.from_params(app.query) or { + return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } + logs := app.db.get_build_logs(filter) return app.json(http.Status.ok, new_data_response(logs)) }