feat(db): implemented iterator over targets

dev
Jef Roosens 2022-12-31 16:10:47 +01:00
parent f8f611f5c5
commit c9edb55abc
Signed by untrusted user: Jef Roosens
GPG Key ID: B75D4F293C7052DB
5 changed files with 130 additions and 66 deletions

View File

@ -7,7 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev) ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
### Added
* Metrics endpoint for Prometheus integration * Metrics endpoint for Prometheus integration
* Search in list of targets using API & CLI
## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0) ## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0)

View File

@ -1,56 +1,6 @@
module db module db
import models { Target, TargetArch, TargetFilter } import models { Target, TargetArch, TargetFilter }
import math
// get_targets returns all targets in the database.
pub fn (db &VieterDb) get_targets(filter TargetFilter) []Target {
window_size := 32
mut out := []Target{}
mut targets := []Target{cap: window_size}
mut offset := 0
mut filtered_offset := u64(0)
for out.len < filter.limit {
targets = sql db.conn {
select from Target order by id limit window_size offset offset
}
offset += targets.len
if targets.len == 0 {
break
}
if filter.repo != '' {
targets = targets.filter(it.repo == filter.repo)
}
if filter.query != '' {
targets = targets.filter(it.url.contains(filter.query) || it.path.contains(filter.query)
|| it.branch.contains(filter.query))
}
if filtered_offset > filter.offset {
end_index := math.min(filter.limit - u64(out.len), u64(targets.len))
out << targets[0..end_index]
}
// We start counting targets in the middle of the current window
else if filtered_offset + u64(targets.len) > filter.offset {
start_index := filter.offset - filtered_offset
end_index := start_index +
math.min(filter.limit - u64(out.len), u64(targets.len) - start_index)
out << targets[start_index..end_index]
}
filtered_offset += u64(targets.len)
}
return out
}
// get_target tries to return a specific target. // get_target tries to return a specific target.
pub fn (db &VieterDb) get_target(target_id int) ?Target { pub fn (db &VieterDb) get_target(target_id int) ?Target {

View File

@ -0,0 +1,123 @@
module db
import models { Target, TargetFilter }
import sqlite
// Iterator providing a filtered view into the list of targets currently stored
// in the database. It replaces functionality usually performed in the database
// using SQL queries that can't currently be used due to missing stuff in V's
// ORM.
pub struct TargetsIterator {
conn sqlite.DB
filter TargetFilter
window_size int = 32
mut:
window []Target
window_index u64
// Offset in entire list of unfiltered targets
offset int
// Offset in filtered list of targets
filtered_offset u64
started bool
done bool
}
// targets returns an iterator allowing filtered access to the list of targets.
pub fn (db &VieterDb) targets(filter TargetFilter) TargetsIterator {
window_size := 32
return TargetsIterator{
conn: db.conn
filter: filter
window: []Target{cap: window_size}
window_size: window_size
}
}
// advance_window moves the sliding window over the filtered list of targets
// until it either reaches the end of the list of targets, or has encountered a
// non-empty window.
fn (mut ti TargetsIterator) advance_window() {
for {
ti.window = sql ti.conn {
select from Target order by id limit ti.window_size offset ti.offset
}
ti.offset += ti.window.len
if ti.window.len == 0 {
ti.done = true
return
}
if ti.filter.repo != '' {
ti.window = ti.window.filter(it.repo == ti.filter.repo)
}
if ti.filter.query != '' {
ti.window = ti.window.filter(it.url.contains(ti.filter.query)
|| it.path.contains(ti.filter.query) || it.branch.contains(ti.filter.query))
}
if ti.window.len > 0 {
break
}
}
}
// next returns the next target, if possible.
pub fn (mut ti TargetsIterator) next() ?Target {
if ti.done {
return none
}
// The first call to `next` will cause the sliding window to move to where the requested offset starts
if !ti.started {
ti.advance_window()
// Skip all matched targets until the requested offset
for !ti.done && ti.filtered_offset + u64(ti.window.len) <= ti.filter.offset {
ti.filtered_offset += u64(ti.window.len)
ti.advance_window()
}
if ti.done {
return none
}
left_inside_window := ti.filter.offset - ti.filtered_offset
ti.window_index = left_inside_window
ti.filtered_offset += left_inside_window
ti.started = true
}
return_value := ti.window[ti.window_index]
ti.window_index++
ti.filtered_offset++
// Next call will be past the requested offset
if ti.filter.limit > 0 && ti.filtered_offset == ti.filter.offset + ti.filter.limit {
ti.done = true
}
// Ensure the next call has a new valid window
if ti.window_index == u64(ti.window.len) {
ti.advance_window()
ti.window_index = 0
}
return return_value
}
// collect consumes the entire iterator & returns the result as an array.
pub fn (mut ti TargetsIterator) collect() []Target {
mut out := []Target{}
for t in ti {
out << t
}
return out
}

View File

@ -11,9 +11,9 @@ fn (mut app App) v1_get_targets() web.Result {
filter := models.from_params<TargetFilter>(app.query) or { filter := models.from_params<TargetFilter>(app.query) or {
return app.json(.bad_request, new_response('Invalid query parameters.')) return app.json(.bad_request, new_response('Invalid query parameters.'))
} }
targets := app.db.get_targets(filter) mut iter := app.db.targets(filter)
return app.json(.ok, new_data_response(targets)) return app.json(.ok, new_data_response(iter.collect()))
} }
// v1_get_single_target returns the information for a single target. // v1_get_single_target returns the information for a single target.
@ -81,6 +81,3 @@ fn (mut app App) v1_patch_target(id int) web.Result {
return app.json(.ok, new_data_response(target)) return app.json(.ok, new_data_response(target))
} }
['/api/v1/targets/search'; auth; get; markused]
fn (mut app App) v1_search_targets()

View File

@ -31,17 +31,8 @@ pub mut:
// init_job_queue populates a fresh job queue with all the targets currently // init_job_queue populates a fresh job queue with all the targets currently
// stored in the database. // stored in the database.
fn (mut app App) init_job_queue() ! { fn (mut app App) init_job_queue() ! {
// Initialize build queues for target in app.db.targets(limit: 0) {
mut targets := app.db.get_targets(limit: 25) app.job_queue.insert_all(target)!
mut i := u64(0)
for targets.len > 0 {
for target in targets {
app.job_queue.insert_all(target)!
}
i += 25
targets = app.db.get_targets(limit: 25, offset: i)
} }
} }