wip fancy iterator

src/db/targets.v

@@ -2,6 +2,107 @@ module db
 
 import models { Target, TargetArch, TargetFilter }
 import math
+import sqlite
+
+pub struct TargetsIterator {
+	conn        sqlite.DB
+	filter      TargetFilter
+	window_size int = 32
+mut:
+	window       []Target
+	window_index u64
+	// Offset in entire list of unfiltered targets
+	offset int
+	// Offset in filtered list of targets
+	filtered_offset u64
+	done            bool
+}
+
+pub fn (db &VieterDb) targets(filter TargetFilter) TargetsIterator {
+	window_size := 32
+
+	return TargetsIterator{
+		conn: db.conn
+		filter: filter
+		window: []Target{cap: window_size}
+		window_size: window_size
+	}
+}
+
+/* [direct_array_access] */
+pub fn (mut ti TargetsIterator) next() ?Target {
+	if ti.done {
+		return none
+	}
+
+	ti.window_index++
+
+	// limit 0 means no limit
+	if ti.filter.limit > 0 && ti.filtered_offset == ti.filter.offset + ti.filter.limit {
+		ti.done = true
+
+		return none
+	}
+
+	ti.filtered_offset++
+
+	// The very first time, targets contains no elements so this will be true
+	// as well.
+	if ti.window_index >= ti.window.len {
+		// Loop until we encounter a non-empty window or have gone through the
+		// entire database
+		for {
+			ti.window = sql ti.conn {
+				select from Target order by id limit ti.window_size offset ti.offset
+			}
+			ti.offset += ti.window.len
+
+			if ti.window.len == 0 {
+				ti.done = true
+
+				return none
+			}
+
+			if ti.filter.repo != '' {
+				ti.window = ti.window.filter(it.repo == ti.filter.repo)
+			}
+
+			if ti.filter.query != '' {
+				ti.window = ti.window.filter(it.url.contains(ti.filter.query)
+					|| it.path.contains(ti.filter.query) || it.branch.contains(ti.filter.query))
+			}
+
+			// Skip window if we're not past the requested offset yet
+			if ti.filtered_offset + u64(ti.window.len) < ti.filter.offset {
+				ti.filtered_offset += u64(ti.window.len)
+				continue
+			}
+			else if ti.window.len > 0 {
+				if ti.filtered_offset >= ti.filter.offset {
+					ti.window_index = 0
+				} else {
+					ti.window_index = ti.filter.offset - ti.filtered_offset
+					ti.filtered_offset += ti.window_index
+				}
+
+				break
+			}
+		}
+	}
+
+
+	return ti.window[ti.window_index]
+}
+
+pub fn (mut ti TargetsIterator) collect() []Target {
+	mut out := []Target{}
+
+	for t in ti {
+		out << t
+	}
+
+	return out
+}
 
 // get_targets returns all targets in the database.
 pub fn (db &VieterDb) get_targets(filter TargetFilter) []Target {

src/server/api_targets.v

@@ -11,9 +11,9 @@ fn (mut app App) v1_get_targets() web.Result {
 	filter := models.from_params<TargetFilter>(app.query) or {
 		return app.json(.bad_request, new_response('Invalid query parameters.'))
 	}
-	targets := app.db.get_targets(filter)
+	mut iter := app.db.targets(filter)
 
-	return app.json(.ok, new_data_response(targets))
+	return app.json(.ok, new_data_response(iter.collect()))
 }
 
 // v1_get_single_target returns the information for a single target.

src/server/server.v

@@ -30,17 +30,8 @@ pub mut:
 // init_job_queue populates a fresh job queue with all the targets currently
 // stored in the database.
 fn (mut app App) init_job_queue() ! {
-	// Initialize build queues
+	for target in app.db.targets(limit: 0) {
-	mut targets := app.db.get_targets(limit: 25)
+		app.job_queue.insert_all(target)!
-	mut i := u64(0)
-
-	for targets.len > 0 {
-		for target in targets {
-			app.job_queue.insert_all(target)!
-		}
-
-		i += 25
-		targets = app.db.get_targets(limit: 25, offset: i)
 	}
 }