diff --git a/CHANGELOG.md b/CHANGELOG.md index be5f445..72c5440 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev) -### Added - * Metrics endpoint for Prometheus integration -* Search in list of targets using API & CLI -* Allow filtering targets by arch value ## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0) diff --git a/docs/api/source/includes/_targets.md b/docs/api/source/includes/_targets.md index 1a5f3e0..b71da84 100644 --- a/docs/api/source/includes/_targets.md +++ b/docs/api/source/includes/_targets.md @@ -55,8 +55,6 @@ Parameter | Description limit | Maximum amount of results to return. offset | Offset of results. repo | Limit results to targets that publish to the given repo. -query | Only return targets that have this substring in their URL, path or branch. -arch | Only return targets that publish to this arch. ## Get specific target diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 6152a53..3c0d755 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -40,17 +40,6 @@ pub fn cmd() cli.Command { description: 'Only return targets that publish to this repo.' flag: cli.FlagType.string }, - cli.Flag{ - name: 'query' - abbrev: 'q' - description: 'Search string to filter targets by.' - flag: cli.FlagType.string - }, - cli.Flag{ - name: 'arch' - description: 'Only list targets that build for this arch.' - flag: cli.FlagType.string - }, ] execute: fn (cmd cli.Command) ! { config_file := cmd.flags.get_string('config-file')! @@ -73,16 +62,6 @@ pub fn cmd() cli.Command { filter.repo = repo } - query := cmd.flags.get_string('query')! - if query != '' { - filter.query = query - } - - arch := cmd.flags.get_string('arch')! - if arch != '' { - filter.arch = arch - } - raw := cmd.flags.get_bool('raw')! list(conf, filter, raw)! diff --git a/src/db/targets.v b/src/db/targets.v index 2644f49..41e56df 100644 --- a/src/db/targets.v +++ b/src/db/targets.v @@ -1,6 +1,25 @@ module db -import models { Target, TargetArch } +import models { Target, TargetArch, TargetFilter } + +// get_targets returns all targets in the database. +pub fn (db &VieterDb) get_targets(filter TargetFilter) []Target { + // This seems to currently be blocked by a bug in the ORM, I'll have to ask + // around. + if filter.repo != '' { + res := sql db.conn { + select from Target where repo == filter.repo order by id limit filter.limit offset filter.offset + } + + return res + } + + res := sql db.conn { + select from Target order by id limit filter.limit offset filter.offset + } + + return res +} // get_target tries to return a specific target. pub fn (db &VieterDb) get_target(target_id int) ?Target { diff --git a/src/db/targets_iter.v b/src/db/targets_iter.v deleted file mode 100644 index 081de1f..0000000 --- a/src/db/targets_iter.v +++ /dev/null @@ -1,129 +0,0 @@ -module db - -import models { Target, TargetFilter } -import sqlite - -// Iterator providing a filtered view into the list of targets currently stored -// in the database. It replaces functionality usually performed in the database -// using SQL queries that can't currently be used due to missing stuff in V's -// ORM. -pub struct TargetsIterator { - conn sqlite.DB - filter TargetFilter - window_size int = 32 -mut: - window []Target - window_index u64 - // Offset in entire list of unfiltered targets - offset int - // Offset in filtered list of targets - filtered_offset u64 - started bool - done bool -} - -// targets returns an iterator allowing filtered access to the list of targets. -pub fn (db &VieterDb) targets(filter TargetFilter) TargetsIterator { - window_size := 32 - - return TargetsIterator{ - conn: db.conn - filter: filter - window: []Target{cap: window_size} - window_size: window_size - } -} - -// advance_window moves the sliding window over the filtered list of targets -// until it either reaches the end of the list of targets, or has encountered a -// non-empty window. -fn (mut ti TargetsIterator) advance_window() { - for { - ti.window = sql ti.conn { - select from Target order by id limit ti.window_size offset ti.offset - } - ti.offset += ti.window.len - - if ti.window.len == 0 { - ti.done = true - - return - } - - if ti.filter.repo != '' { - ti.window = ti.window.filter(it.repo == ti.filter.repo) - } - - if ti.filter.arch != '' { - ti.window = ti.window.filter(it.arch.any(it.value == ti.filter.arch)) - } - - if ti.filter.query != '' { - ti.window = ti.window.filter(it.url.contains(ti.filter.query) - || it.path.contains(ti.filter.query) || it.branch.contains(ti.filter.query)) - } - - // We break out of the loop once we found a non-empty window - if ti.window.len > 0 { - break - } - } -} - -// next returns the next target, if possible. -pub fn (mut ti TargetsIterator) next() ?Target { - if ti.done { - return none - } - - // The first call to `next` will cause the sliding window to move to where - // the requested offset starts - if !ti.started { - ti.advance_window() - - // Skip all matched targets until the requested offset - for !ti.done && ti.filtered_offset + u64(ti.window.len) <= ti.filter.offset { - ti.filtered_offset += u64(ti.window.len) - ti.advance_window() - } - - if ti.done { - return none - } - - left_inside_window := ti.filter.offset - ti.filtered_offset - ti.window_index = left_inside_window - ti.filtered_offset += left_inside_window - - ti.started = true - } - - return_value := ti.window[ti.window_index] - - ti.window_index++ - ti.filtered_offset++ - - // Next call will be past the requested offset - if ti.filter.limit > 0 && ti.filtered_offset == ti.filter.offset + ti.filter.limit { - ti.done = true - } - - // Ensure the next call has a new valid window - if ti.window_index == u64(ti.window.len) { - ti.advance_window() - ti.window_index = 0 - } - - return return_value -} - -// collect consumes the entire iterator & returns the result as an array. -pub fn (mut ti TargetsIterator) collect() []Target { - mut out := []Target{} - - for t in ti { - out << t - } - - return out -} diff --git a/src/models/targets.v b/src/models/targets.v index a0c88d0..af3cb0d 100644 --- a/src/models/targets.v +++ b/src/models/targets.v @@ -73,6 +73,4 @@ pub mut: limit u64 = 25 offset u64 repo string - query string - arch string } diff --git a/src/server/api_targets.v b/src/server/api_targets.v index f47467a..4bb7d12 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -11,9 +11,9 @@ fn (mut app App) v1_get_targets() web.Result { filter := models.from_params(app.query) or { return app.json(.bad_request, new_response('Invalid query parameters.')) } - mut iter := app.db.targets(filter) + targets := app.db.get_targets(filter) - return app.json(.ok, new_data_response(iter.collect())) + return app.json(.ok, new_data_response(targets)) } // v1_get_single_target returns the information for a single target. diff --git a/src/server/server.v b/src/server/server.v index 5dd1a20..76e7ad6 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -31,8 +31,17 @@ pub mut: // init_job_queue populates a fresh job queue with all the targets currently // stored in the database. fn (mut app App) init_job_queue() ! { - for target in app.db.targets(limit: 0) { - app.job_queue.insert_all(target)! + // Initialize build queues + mut targets := app.db.get_targets(limit: 25) + mut i := u64(0) + + for targets.len > 0 { + for target in targets { + app.job_queue.insert_all(target)! + } + + i += 25 + targets = app.db.get_targets(limit: 25, offset: i) } } diff --git a/vieter.toml b/vieter.toml index 7744a56..31eadc0 100644 --- a/vieter.toml +++ b/vieter.toml @@ -12,5 +12,5 @@ address = "http://localhost:8000" api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 -# max_log_age = 64 +max_log_age = 64 collect_metrics = true