Search targets using API #332

Merged
Jef Roosens merged 5 commits from Chewing_Bever/vieter:search-targets into dev 2023-01-04 14:50:50 +01:00
9 changed files with 164 additions and 34 deletions

View File

@ -7,7 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
### Added
* Metrics endpoint for Prometheus integration
* Search in list of targets using API & CLI
* Allow filtering targets by arch value
## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0)

View File

@ -55,6 +55,8 @@ Parameter | Description
limit | Maximum amount of results to return.
offset | Offset of results.
repo | Limit results to targets that publish to the given repo.
query | Only return targets that have this substring in their URL, path or branch.
arch | Only return targets that publish to this arch.
## Get specific target

View File

@ -40,6 +40,17 @@ pub fn cmd() cli.Command {
description: 'Only return targets that publish to this repo.'
flag: cli.FlagType.string
},
cli.Flag{
name: 'query'
abbrev: 'q'
description: 'Search string to filter targets by.'
flag: cli.FlagType.string
},
cli.Flag{
name: 'arch'
description: 'Only list targets that build for this arch.'
flag: cli.FlagType.string
},
]
execute: fn (cmd cli.Command) ! {
config_file := cmd.flags.get_string('config-file')!
@ -62,6 +73,16 @@ pub fn cmd() cli.Command {
filter.repo = repo
}
query := cmd.flags.get_string('query')!
if query != '' {
filter.query = query
}
arch := cmd.flags.get_string('arch')!
if arch != '' {
filter.arch = arch
}
raw := cmd.flags.get_bool('raw')!
list(conf, filter, raw)!

View File

@ -1,25 +1,6 @@
module db
import models { Target, TargetArch, TargetFilter }
// get_targets returns all targets in the database.
pub fn (db &VieterDb) get_targets(filter TargetFilter) []Target {
// This seems to currently be blocked by a bug in the ORM, I'll have to ask
// around.
if filter.repo != '' {
res := sql db.conn {
select from Target where repo == filter.repo order by id limit filter.limit offset filter.offset
}
return res
}
res := sql db.conn {
select from Target order by id limit filter.limit offset filter.offset
}
return res
}
import models { Target, TargetArch }
// get_target tries to return a specific target.
pub fn (db &VieterDb) get_target(target_id int) ?Target {

View File

@ -0,0 +1,129 @@
module db
import models { Target, TargetFilter }
import sqlite
// Iterator providing a filtered view into the list of targets currently stored
// in the database. It replaces functionality usually performed in the database
// using SQL queries that can't currently be used due to missing stuff in V's
// ORM.
pub struct TargetsIterator {
conn sqlite.DB
filter TargetFilter
window_size int = 32
mut:
window []Target
window_index u64
// Offset in entire list of unfiltered targets
offset int
// Offset in filtered list of targets
filtered_offset u64
started bool
done bool
}
// targets returns an iterator allowing filtered access to the list of targets.
pub fn (db &VieterDb) targets(filter TargetFilter) TargetsIterator {
window_size := 32
return TargetsIterator{
conn: db.conn
filter: filter
window: []Target{cap: window_size}
window_size: window_size
}
}
// advance_window moves the sliding window over the filtered list of targets
// until it either reaches the end of the list of targets, or has encountered a
// non-empty window.
fn (mut ti TargetsIterator) advance_window() {
for {
ti.window = sql ti.conn {
select from Target order by id limit ti.window_size offset ti.offset
}
ti.offset += ti.window.len
if ti.window.len == 0 {
ti.done = true
return
}
if ti.filter.repo != '' {
ti.window = ti.window.filter(it.repo == ti.filter.repo)
}
if ti.filter.arch != '' {
ti.window = ti.window.filter(it.arch.any(it.value == ti.filter.arch))
}
if ti.filter.query != '' {
ti.window = ti.window.filter(it.url.contains(ti.filter.query)
|| it.path.contains(ti.filter.query) || it.branch.contains(ti.filter.query))
}
// We break out of the loop once we found a non-empty window
if ti.window.len > 0 {
break
}
}
}
// next returns the next target, if possible.
pub fn (mut ti TargetsIterator) next() ?Target {
if ti.done {
return none
}
// The first call to `next` will cause the sliding window to move to where
// the requested offset starts
if !ti.started {
ti.advance_window()
// Skip all matched targets until the requested offset
for !ti.done && ti.filtered_offset + u64(ti.window.len) <= ti.filter.offset {
ti.filtered_offset += u64(ti.window.len)
ti.advance_window()
}
if ti.done {
return none
}
left_inside_window := ti.filter.offset - ti.filtered_offset
ti.window_index = left_inside_window
ti.filtered_offset += left_inside_window
ti.started = true
}
return_value := ti.window[ti.window_index]
ti.window_index++
ti.filtered_offset++
// Next call will be past the requested offset
if ti.filter.limit > 0 && ti.filtered_offset == ti.filter.offset + ti.filter.limit {
ti.done = true
}
// Ensure the next call has a new valid window
if ti.window_index == u64(ti.window.len) {
ti.advance_window()
ti.window_index = 0
}
return return_value
}
// collect consumes the entire iterator & returns the result as an array.
pub fn (mut ti TargetsIterator) collect() []Target {
mut out := []Target{}
for t in ti {
out << t
}
return out
}

View File

@ -73,4 +73,6 @@ pub mut:
limit u64 = 25
offset u64
repo string
query string
arch string
}

View File

@ -11,9 +11,9 @@ fn (mut app App) v1_get_targets() web.Result {
filter := models.from_params<TargetFilter>(app.query) or {
return app.json(.bad_request, new_response('Invalid query parameters.'))
}
targets := app.db.get_targets(filter)
mut iter := app.db.targets(filter)
return app.json(.ok, new_data_response(targets))
return app.json(.ok, new_data_response(iter.collect()))
}
// v1_get_single_target returns the information for a single target.

View File

@ -31,17 +31,8 @@ pub mut:
// init_job_queue populates a fresh job queue with all the targets currently
// stored in the database.
fn (mut app App) init_job_queue() ! {
// Initialize build queues
mut targets := app.db.get_targets(limit: 25)
mut i := u64(0)
for targets.len > 0 {
for target in targets {
app.job_queue.insert_all(target)!
}
i += 25
targets = app.db.get_targets(limit: 25, offset: i)
for target in app.db.targets(limit: 0) {
app.job_queue.insert_all(target)!
}
}

View File

@ -12,5 +12,5 @@ address = "http://localhost:8000"
api_update_frequency = 2
image_rebuild_frequency = 1
max_concurrent_builds = 3
max_log_age = 64
# max_log_age = 64
collect_metrics = true