feat(db): implemented iterator over targets

CHANGELOG.md

@@ -7,7 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
 
+### Added
+
 * Metrics endpoint for Prometheus integration
+* Search in list of targets using API & CLI
 
 ## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0)
 

src/db/targets.v

@@ -1,56 +1,6 @@
 module db
 
 import models { Target, TargetArch, TargetFilter }
-import math
-
-// get_targets returns all targets in the database.
-pub fn (db &VieterDb) get_targets(filter TargetFilter) []Target {
-	window_size := 32
-
-	mut out := []Target{}
-	mut targets := []Target{cap: window_size}
-
-	mut offset := 0
-	mut filtered_offset := u64(0)
-
-	for out.len < filter.limit {
-		targets = sql db.conn {
-			select from Target order by id limit window_size offset offset
-		}
-		offset += targets.len
-
-		if targets.len == 0 {
-			break
-		}
-
-		if filter.repo != '' {
-			targets = targets.filter(it.repo == filter.repo)
-		}
-
-		if filter.query != '' {
-			targets = targets.filter(it.url.contains(filter.query) || it.path.contains(filter.query)
-				|| it.branch.contains(filter.query))
-		}
-
-		if filtered_offset > filter.offset {
-			end_index := math.min(filter.limit - u64(out.len), u64(targets.len))
-
-			out << targets[0..end_index]
-		}
-		// We start counting targets in the middle of the current window
-		else if filtered_offset + u64(targets.len) > filter.offset {
-			start_index := filter.offset - filtered_offset
-			end_index := start_index +
-				math.min(filter.limit - u64(out.len), u64(targets.len) - start_index)
-
-			out << targets[start_index..end_index]
-		}
-
-		filtered_offset += u64(targets.len)
-	}
-
-	return out
-}
 
 // get_target tries to return a specific target.
 pub fn (db &VieterDb) get_target(target_id int) ?Target {

src/db/targets_iter.v

@@ -0,0 +1,123 @@
+module db
+
+import models { Target, TargetFilter }
+import sqlite
+
+// Iterator providing a filtered view into the list of targets currently stored
+// in the database. It replaces functionality usually performed in the database
+// using SQL queries that can't currently be used due to missing stuff in V's
+// ORM.
+pub struct TargetsIterator {
+	conn        sqlite.DB
+	filter      TargetFilter
+	window_size int = 32
+mut:
+	window       []Target
+	window_index u64
+	// Offset in entire list of unfiltered targets
+	offset int
+	// Offset in filtered list of targets
+	filtered_offset u64
+	started         bool
+	done            bool
+}
+
+// targets returns an iterator allowing filtered access to the list of targets.
+pub fn (db &VieterDb) targets(filter TargetFilter) TargetsIterator {
+	window_size := 32
+
+	return TargetsIterator{
+		conn: db.conn
+		filter: filter
+		window: []Target{cap: window_size}
+		window_size: window_size
+	}
+}
+
+// advance_window moves the sliding window over the filtered list of targets
+// until it either reaches the end of the list of targets, or has encountered a
+// non-empty window.
+fn (mut ti TargetsIterator) advance_window() {
+	for {
+		ti.window = sql ti.conn {
+			select from Target order by id limit ti.window_size offset ti.offset
+		}
+		ti.offset += ti.window.len
+
+		if ti.window.len == 0 {
+			ti.done = true
+
+			return
+		}
+
+		if ti.filter.repo != '' {
+			ti.window = ti.window.filter(it.repo == ti.filter.repo)
+		}
+
+		if ti.filter.query != '' {
+			ti.window = ti.window.filter(it.url.contains(ti.filter.query)
+				|| it.path.contains(ti.filter.query) || it.branch.contains(ti.filter.query))
+		}
+
+		if ti.window.len > 0 {
+			break
+		}
+	}
+}
+
+// next returns the next target, if possible.
+pub fn (mut ti TargetsIterator) next() ?Target {
+	if ti.done {
+		return none
+	}
+
+	// The first call to `next` will cause the sliding window to move to where the requested offset starts
+	if !ti.started {
+		ti.advance_window()
+
+		// Skip all matched targets until the requested offset
+		for !ti.done && ti.filtered_offset + u64(ti.window.len) <= ti.filter.offset {
+			ti.filtered_offset += u64(ti.window.len)
+			ti.advance_window()
+		}
+
+		if ti.done {
+			return none
+		}
+
+		left_inside_window := ti.filter.offset - ti.filtered_offset
+		ti.window_index = left_inside_window
+		ti.filtered_offset += left_inside_window
+
+		ti.started = true
+	}
+
+	return_value := ti.window[ti.window_index]
+
+	ti.window_index++
+	ti.filtered_offset++
+
+	// Next call will be past the requested offset
+	if ti.filter.limit > 0 && ti.filtered_offset == ti.filter.offset + ti.filter.limit {
+		ti.done = true
+	}
+
+	// Ensure the next call has a new valid window
+	if ti.window_index == u64(ti.window.len) {
+		ti.advance_window()
+		ti.window_index = 0
+	}
+
+	return return_value
+}
+
+// collect consumes the entire iterator & returns the result as an array.
+pub fn (mut ti TargetsIterator) collect() []Target {
+	mut out := []Target{}
+
+	for t in ti {
+		out << t
+	}
+
+	return out
+}

src/server/api_targets.v

@@ -11,9 +11,9 @@ fn (mut app App) v1_get_targets() web.Result {
 	filter := models.from_params<TargetFilter>(app.query) or {
 		return app.json(.bad_request, new_response('Invalid query parameters.'))
 	}
-	targets := app.db.get_targets(filter)
+	mut iter := app.db.targets(filter)
 
-	return app.json(.ok, new_data_response(targets))
+	return app.json(.ok, new_data_response(iter.collect()))
 }
 
 // v1_get_single_target returns the information for a single target.
@@ -81,6 +81,3 @@ fn (mut app App) v1_patch_target(id int) web.Result {
 
 	return app.json(.ok, new_data_response(target))
 }
-
-['/api/v1/targets/search'; auth; get; markused]
-fn (mut app App) v1_search_targets()

src/server/server.v

@@ -31,18 +31,9 @@ pub mut:
 // init_job_queue populates a fresh job queue with all the targets currently
 // stored in the database.
 fn (mut app App) init_job_queue() ! {
-	// Initialize build queues
+	for target in app.db.targets(limit: 0) {
-	mut targets := app.db.get_targets(limit: 25)
-	mut i := u64(0)
-
-	for targets.len > 0 {
-		for target in targets {
 		app.job_queue.insert_all(target)!
 	}
-
-		i += 25
-		targets = app.db.get_targets(limit: 25, offset: i)
-	}
 }
 
 // server starts the web server & starts listening for requests