Merge pull request 'Search targets using API' (#332) from Chewing_Bever/vieter:search-targets into dev

Reviewed-on: vieter-v/vieter#332

CHANGELOG.md

@@ -7,7 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
 
+### Added
+
 * Metrics endpoint for Prometheus integration
+* Search in list of targets using API & CLI
+* Allow filtering targets by arch value
 
 ## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0)
 

docs/api/source/includes/_targets.md

@@ -55,6 +55,8 @@ Parameter | Description
 limit | Maximum amount of results to return.
 offset | Offset of results.
 repo | Limit results to targets that publish to the given repo.
+query | Only return targets that have this substring in their URL, path or branch.
+arch | Only return targets that publish to this arch.
 
 ## Get specific target
 

src/console/targets/targets.v

@@ -40,6 +40,17 @@ pub fn cmd() cli.Command {
 						description: 'Only return targets that publish to this repo.'
 						flag: cli.FlagType.string
 					},
+					cli.Flag{
+						name: 'query'
+						abbrev: 'q'
+						description: 'Search string to filter targets by.'
+						flag: cli.FlagType.string
+					},
+					cli.Flag{
+						name: 'arch'
+						description: 'Only list targets that build for this arch.'
+						flag: cli.FlagType.string
+					},
 				]
 				execute: fn (cmd cli.Command) ! {
 					config_file := cmd.flags.get_string('config-file')!
@@ -62,6 +73,16 @@ pub fn cmd() cli.Command {
 						filter.repo = repo
 					}
 
+					query := cmd.flags.get_string('query')!
+					if query != '' {
+						filter.query = query
+					}
+
+					arch := cmd.flags.get_string('arch')!
+					if arch != '' {
+						filter.arch = arch
+					}
+
 					raw := cmd.flags.get_bool('raw')!
 
 					list(conf, filter, raw)!

src/db/targets.v

@@ -1,25 +1,6 @@
 module db
 
-import models { Target, TargetArch, TargetFilter }
+import models { Target, TargetArch }
-
-// get_targets returns all targets in the database.
-pub fn (db &VieterDb) get_targets(filter TargetFilter) []Target {
-	// This seems to currently be blocked by a bug in the ORM, I'll have to ask
-	// around.
-	if filter.repo != '' {
-		res := sql db.conn {
-			select from Target where repo == filter.repo order by id limit filter.limit offset filter.offset
-		}
-
-		return res
-	}
-
-	res := sql db.conn {
-		select from Target order by id limit filter.limit offset filter.offset
-	}
-
-	return res
-}
 
 // get_target tries to return a specific target.
 pub fn (db &VieterDb) get_target(target_id int) ?Target {

src/db/targets_iter.v

@@ -0,0 +1,129 @@
+module db
+
+import models { Target, TargetFilter }
+import sqlite
+
+// Iterator providing a filtered view into the list of targets currently stored
+// in the database. It replaces functionality usually performed in the database
+// using SQL queries that can't currently be used due to missing stuff in V's
+// ORM.
+pub struct TargetsIterator {
+	conn        sqlite.DB
+	filter      TargetFilter
+	window_size int = 32
+mut:
+	window       []Target
+	window_index u64
+	// Offset in entire list of unfiltered targets
+	offset int
+	// Offset in filtered list of targets
+	filtered_offset u64
+	started         bool
+	done            bool
+}
+
+// targets returns an iterator allowing filtered access to the list of targets.
+pub fn (db &VieterDb) targets(filter TargetFilter) TargetsIterator {
+	window_size := 32
+
+	return TargetsIterator{
+		conn: db.conn
+		filter: filter
+		window: []Target{cap: window_size}
+		window_size: window_size
+	}
+}
+
+// advance_window moves the sliding window over the filtered list of targets
+// until it either reaches the end of the list of targets, or has encountered a
+// non-empty window.
+fn (mut ti TargetsIterator) advance_window() {
+	for {
+		ti.window = sql ti.conn {
+			select from Target order by id limit ti.window_size offset ti.offset
+		}
+		ti.offset += ti.window.len
+
+		if ti.window.len == 0 {
+			ti.done = true
+
+			return
+		}
+
+		if ti.filter.repo != '' {
+			ti.window = ti.window.filter(it.repo == ti.filter.repo)
+		}
+
+		if ti.filter.arch != '' {
+			ti.window = ti.window.filter(it.arch.any(it.value == ti.filter.arch))
+		}
+
+		if ti.filter.query != '' {
+			ti.window = ti.window.filter(it.url.contains(ti.filter.query)
+				|| it.path.contains(ti.filter.query) || it.branch.contains(ti.filter.query))
+		}
+
+		// We break out of the loop once we found a non-empty window
+		if ti.window.len > 0 {
+			break
+		}
+	}
+}
+
+// next returns the next target, if possible.
+pub fn (mut ti TargetsIterator) next() ?Target {
+	if ti.done {
+		return none
+	}
+
+	// The first call to `next` will cause the sliding window to move to where
+	// the requested offset starts
+	if !ti.started {
+		ti.advance_window()
+
+		// Skip all matched targets until the requested offset
+		for !ti.done && ti.filtered_offset + u64(ti.window.len) <= ti.filter.offset {
+			ti.filtered_offset += u64(ti.window.len)
+			ti.advance_window()
+		}
+
+		if ti.done {
+			return none
+		}
+
+		left_inside_window := ti.filter.offset - ti.filtered_offset
+		ti.window_index = left_inside_window
+		ti.filtered_offset += left_inside_window
+
+		ti.started = true
+	}
+
+	return_value := ti.window[ti.window_index]
+
+	ti.window_index++
+	ti.filtered_offset++
+
+	// Next call will be past the requested offset
+	if ti.filter.limit > 0 && ti.filtered_offset == ti.filter.offset + ti.filter.limit {
+		ti.done = true
+	}
+
+	// Ensure the next call has a new valid window
+	if ti.window_index == u64(ti.window.len) {
+		ti.advance_window()
+		ti.window_index = 0
+	}
+
+	return return_value
+}
+
+// collect consumes the entire iterator & returns the result as an array.
+pub fn (mut ti TargetsIterator) collect() []Target {
+	mut out := []Target{}
+
+	for t in ti {
+		out << t
+	}
+
+	return out
+}

src/models/targets.v

@@ -73,4 +73,6 @@ pub mut:
 	limit  u64 = 25
 	offset u64
 	repo   string
+	query  string
+	arch   string
 }

src/server/api_targets.v

@@ -11,9 +11,9 @@ fn (mut app App) v1_get_targets() web.Result {
 	filter := models.from_params<TargetFilter>(app.query) or {
 		return app.json(.bad_request, new_response('Invalid query parameters.'))
 	}
-	targets := app.db.get_targets(filter)
+	mut iter := app.db.targets(filter)
 
-	return app.json(.ok, new_data_response(targets))
+	return app.json(.ok, new_data_response(iter.collect()))
 }
 
 // v1_get_single_target returns the information for a single target.

src/server/server.v

@@ -31,17 +31,8 @@ pub mut:
 // init_job_queue populates a fresh job queue with all the targets currently
 // stored in the database.
 fn (mut app App) init_job_queue() ! {
-	// Initialize build queues
+	for target in app.db.targets(limit: 0) {
-	mut targets := app.db.get_targets(limit: 25)
+		app.job_queue.insert_all(target)!
-	mut i := u64(0)
-
-	for targets.len > 0 {
-		for target in targets {
-			app.job_queue.insert_all(target)!
-		}
-
-		i += 25
-		targets = app.db.get_targets(limit: 25, offset: i)
 	}
 }
 

vieter.toml

@@ -12,5 +12,5 @@ address = "http://localhost:8000"
 api_update_frequency = 2
 image_rebuild_frequency = 1
 max_concurrent_builds = 3
-max_log_age = 64
+# max_log_age = 64
 collect_metrics = true