From c9edb55abcf2bbee5848a5c27dc2cfd662325671 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 31 Dec 2022 16:10:47 +0100 Subject: [PATCH] feat(db): implemented iterator over targets --- CHANGELOG.md | 3 + src/db/targets.v | 50 ---------------- src/db/targets_iter.v | 123 +++++++++++++++++++++++++++++++++++++++ src/server/api_targets.v | 7 +-- src/server/server.v | 13 +---- 5 files changed, 130 insertions(+), 66 deletions(-) create mode 100644 src/db/targets_iter.v diff --git a/CHANGELOG.md b/CHANGELOG.md index 72c54406..2ab14e8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev) +### Added + * Metrics endpoint for Prometheus integration +* Search in list of targets using API & CLI ## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0) diff --git a/src/db/targets.v b/src/db/targets.v index fba227e2..e022a70f 100644 --- a/src/db/targets.v +++ b/src/db/targets.v @@ -1,56 +1,6 @@ module db import models { Target, TargetArch, TargetFilter } -import math - -// get_targets returns all targets in the database. -pub fn (db &VieterDb) get_targets(filter TargetFilter) []Target { - window_size := 32 - - mut out := []Target{} - mut targets := []Target{cap: window_size} - - mut offset := 0 - mut filtered_offset := u64(0) - - for out.len < filter.limit { - targets = sql db.conn { - select from Target order by id limit window_size offset offset - } - offset += targets.len - - if targets.len == 0 { - break - } - - if filter.repo != '' { - targets = targets.filter(it.repo == filter.repo) - } - - if filter.query != '' { - targets = targets.filter(it.url.contains(filter.query) || it.path.contains(filter.query) - || it.branch.contains(filter.query)) - } - - if filtered_offset > filter.offset { - end_index := math.min(filter.limit - u64(out.len), u64(targets.len)) - - out << targets[0..end_index] - } - // We start counting targets in the middle of the current window - else if filtered_offset + u64(targets.len) > filter.offset { - start_index := filter.offset - filtered_offset - end_index := start_index + - math.min(filter.limit - u64(out.len), u64(targets.len) - start_index) - - out << targets[start_index..end_index] - } - - filtered_offset += u64(targets.len) - } - - return out -} // get_target tries to return a specific target. pub fn (db &VieterDb) get_target(target_id int) ?Target { diff --git a/src/db/targets_iter.v b/src/db/targets_iter.v new file mode 100644 index 00000000..16b10809 --- /dev/null +++ b/src/db/targets_iter.v @@ -0,0 +1,123 @@ +module db + +import models { Target, TargetFilter } +import sqlite + +// Iterator providing a filtered view into the list of targets currently stored +// in the database. It replaces functionality usually performed in the database +// using SQL queries that can't currently be used due to missing stuff in V's +// ORM. +pub struct TargetsIterator { + conn sqlite.DB + filter TargetFilter + window_size int = 32 +mut: + window []Target + window_index u64 + // Offset in entire list of unfiltered targets + offset int + // Offset in filtered list of targets + filtered_offset u64 + started bool + done bool +} + +// targets returns an iterator allowing filtered access to the list of targets. +pub fn (db &VieterDb) targets(filter TargetFilter) TargetsIterator { + window_size := 32 + + return TargetsIterator{ + conn: db.conn + filter: filter + window: []Target{cap: window_size} + window_size: window_size + } +} + +// advance_window moves the sliding window over the filtered list of targets +// until it either reaches the end of the list of targets, or has encountered a +// non-empty window. +fn (mut ti TargetsIterator) advance_window() { + for { + ti.window = sql ti.conn { + select from Target order by id limit ti.window_size offset ti.offset + } + ti.offset += ti.window.len + + if ti.window.len == 0 { + ti.done = true + + return + } + + if ti.filter.repo != '' { + ti.window = ti.window.filter(it.repo == ti.filter.repo) + } + + if ti.filter.query != '' { + ti.window = ti.window.filter(it.url.contains(ti.filter.query) + || it.path.contains(ti.filter.query) || it.branch.contains(ti.filter.query)) + } + + if ti.window.len > 0 { + break + } + } +} + +// next returns the next target, if possible. +pub fn (mut ti TargetsIterator) next() ?Target { + if ti.done { + return none + } + + // The first call to `next` will cause the sliding window to move to where the requested offset starts + if !ti.started { + ti.advance_window() + + // Skip all matched targets until the requested offset + for !ti.done && ti.filtered_offset + u64(ti.window.len) <= ti.filter.offset { + ti.filtered_offset += u64(ti.window.len) + ti.advance_window() + } + + if ti.done { + return none + } + + left_inside_window := ti.filter.offset - ti.filtered_offset + ti.window_index = left_inside_window + ti.filtered_offset += left_inside_window + + ti.started = true + } + + return_value := ti.window[ti.window_index] + + ti.window_index++ + ti.filtered_offset++ + + // Next call will be past the requested offset + if ti.filter.limit > 0 && ti.filtered_offset == ti.filter.offset + ti.filter.limit { + ti.done = true + } + + // Ensure the next call has a new valid window + if ti.window_index == u64(ti.window.len) { + ti.advance_window() + ti.window_index = 0 + } + + return return_value +} + +// collect consumes the entire iterator & returns the result as an array. +pub fn (mut ti TargetsIterator) collect() []Target { + mut out := []Target{} + + for t in ti { + out << t + } + + return out +} diff --git a/src/server/api_targets.v b/src/server/api_targets.v index f04fdaeb..f47467aa 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -11,9 +11,9 @@ fn (mut app App) v1_get_targets() web.Result { filter := models.from_params(app.query) or { return app.json(.bad_request, new_response('Invalid query parameters.')) } - targets := app.db.get_targets(filter) + mut iter := app.db.targets(filter) - return app.json(.ok, new_data_response(targets)) + return app.json(.ok, new_data_response(iter.collect())) } // v1_get_single_target returns the information for a single target. @@ -81,6 +81,3 @@ fn (mut app App) v1_patch_target(id int) web.Result { return app.json(.ok, new_data_response(target)) } - -['/api/v1/targets/search'; auth; get; markused] -fn (mut app App) v1_search_targets() diff --git a/src/server/server.v b/src/server/server.v index 76e7ad60..5dd1a202 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -31,17 +31,8 @@ pub mut: // init_job_queue populates a fresh job queue with all the targets currently // stored in the database. fn (mut app App) init_job_queue() ! { - // Initialize build queues - mut targets := app.db.get_targets(limit: 25) - mut i := u64(0) - - for targets.len > 0 { - for target in targets { - app.job_queue.insert_all(target)! - } - - i += 25 - targets = app.db.get_targets(limit: 25, offset: i) + for target in app.db.targets(limit: 0) { + app.job_queue.insert_all(target)! } }