diff --git a/src/db/targets.v b/src/db/targets.v index fba227e..7a162df 100644 --- a/src/db/targets.v +++ b/src/db/targets.v @@ -2,6 +2,107 @@ module db import models { Target, TargetArch, TargetFilter } import math +import sqlite + +pub struct TargetsIterator { + conn sqlite.DB + filter TargetFilter + window_size int = 32 +mut: + window []Target + window_index u64 + // Offset in entire list of unfiltered targets + offset int + // Offset in filtered list of targets + filtered_offset u64 + done bool +} + +pub fn (db &VieterDb) targets(filter TargetFilter) TargetsIterator { + window_size := 32 + + return TargetsIterator{ + conn: db.conn + filter: filter + window: []Target{cap: window_size} + window_size: window_size + } +} + +/* [direct_array_access] */ +pub fn (mut ti TargetsIterator) next() ?Target { + if ti.done { + return none + } + + ti.window_index++ + + // limit 0 means no limit + if ti.filter.limit > 0 && ti.filtered_offset == ti.filter.offset + ti.filter.limit { + ti.done = true + + return none + } + + ti.filtered_offset++ + + // The very first time, targets contains no elements so this will be true + // as well. + if ti.window_index >= ti.window.len { + // Loop until we encounter a non-empty window or have gone through the + // entire database + for { + ti.window = sql ti.conn { + select from Target order by id limit ti.window_size offset ti.offset + } + ti.offset += ti.window.len + + if ti.window.len == 0 { + ti.done = true + + return none + } + + if ti.filter.repo != '' { + ti.window = ti.window.filter(it.repo == ti.filter.repo) + } + + if ti.filter.query != '' { + ti.window = ti.window.filter(it.url.contains(ti.filter.query) + || it.path.contains(ti.filter.query) || it.branch.contains(ti.filter.query)) + } + + // Skip window if we're not past the requested offset yet + if ti.filtered_offset + u64(ti.window.len) < ti.filter.offset { + ti.filtered_offset += u64(ti.window.len) + continue + } + else if ti.window.len > 0 { + if ti.filtered_offset >= ti.filter.offset { + ti.window_index = 0 + } else { + ti.window_index = ti.filter.offset - ti.filtered_offset + ti.filtered_offset += ti.window_index + } + + break + } + } + } + + + return ti.window[ti.window_index] +} + +pub fn (mut ti TargetsIterator) collect() []Target { + mut out := []Target{} + + for t in ti { + out << t + } + + return out +} // get_targets returns all targets in the database. pub fn (db &VieterDb) get_targets(filter TargetFilter) []Target { diff --git a/src/server/api_targets.v b/src/server/api_targets.v index f04fdae..1d6854f 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -11,9 +11,9 @@ fn (mut app App) v1_get_targets() web.Result { filter := models.from_params(app.query) or { return app.json(.bad_request, new_response('Invalid query parameters.')) } - targets := app.db.get_targets(filter) + mut iter := app.db.targets(filter) - return app.json(.ok, new_data_response(targets)) + return app.json(.ok, new_data_response(iter.collect())) } // v1_get_single_target returns the information for a single target. diff --git a/src/server/server.v b/src/server/server.v index 178f657..fd44f43 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -30,17 +30,8 @@ pub mut: // init_job_queue populates a fresh job queue with all the targets currently // stored in the database. fn (mut app App) init_job_queue() ! { - // Initialize build queues - mut targets := app.db.get_targets(limit: 25) - mut i := u64(0) - - for targets.len > 0 { - for target in targets { - app.job_queue.insert_all(target)! - } - - i += 25 - targets = app.db.get_targets(limit: 25, offset: i) + for target in app.db.targets(limit: 0) { + app.job_queue.insert_all(target)! } }