chore: update docs; final read

Reviewed-on: #325

.woodpecker/man.yml

@@ -8,15 +8,21 @@ branches:
 depends_on:
   - build
 
-skip_clone: true
-
 pipeline:
-  generate:
+  install-modules:
     image: *vlang_image
     pull: true
     commands:
-      - curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64"
+      - export VMODULES=$PWD/.vmodules
-      - chmod +x vieter
+      - 'cd src && v install'
+
+  generate:
+    image: *vlang_image
+    commands:
+      # - curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64"
+      # - chmod +x vieter
+      - export VMODULES=$PWD/.vmodules
+      - make
       - ./vieter man man
       - cd man
 

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
 
+### Added
+
+* Metrics endpoint for Prometheus integration
+* Search in list of targets using API & CLI
+* Allow filtering targets by arch value
+
 ## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0)
 
 ### Added

docs/api/source/includes/_targets.md

@@ -55,6 +55,8 @@ Parameter | Description
 limit | Maximum amount of results to return.
 offset | Offset of results.
 repo | Limit results to targets that publish to the given repo.
+query | Only return targets that have this substring in their URL, path or branch.
+arch | Only return targets that publish to this arch.
 
 ## Get specific target
 

src/console/targets/targets.v

@@ -40,6 +40,17 @@ pub fn cmd() cli.Command {
 						description: 'Only return targets that publish to this repo.'
 						flag: cli.FlagType.string
 					},
+					cli.Flag{
+						name: 'query'
+						abbrev: 'q'
+						description: 'Search string to filter targets by.'
+						flag: cli.FlagType.string
+					},
+					cli.Flag{
+						name: 'arch'
+						description: 'Only list targets that build for this arch.'
+						flag: cli.FlagType.string
+					},
 				]
 				execute: fn (cmd cli.Command) ! {
 					config_file := cmd.flags.get_string('config-file')!
@@ -62,6 +73,16 @@ pub fn cmd() cli.Command {
 						filter.repo = repo
 					}
 
+					query := cmd.flags.get_string('query')!
+					if query != '' {
+						filter.query = query
+					}
+
+					arch := cmd.flags.get_string('arch')!
+					if arch != '' {
+						filter.arch = arch
+					}
+
 					raw := cmd.flags.get_bool('raw')!
 
 					list(conf, filter, raw)!

src/db/targets.v

@@ -1,25 +1,6 @@
 module db
 
-import models { Target, TargetArch, TargetFilter }
+import models { Target, TargetArch }
-
-// get_targets returns all targets in the database.
-pub fn (db &VieterDb) get_targets(filter TargetFilter) []Target {
-	// This seems to currently be blocked by a bug in the ORM, I'll have to ask
-	// around.
-	if filter.repo != '' {
-		res := sql db.conn {
-			select from Target where repo == filter.repo order by id limit filter.limit offset filter.offset
-		}
-
-		return res
-	}
-
-	res := sql db.conn {
-		select from Target order by id limit filter.limit offset filter.offset
-	}
-
-	return res
-}
 
 // get_target tries to return a specific target.
 pub fn (db &VieterDb) get_target(target_id int) ?Target {

src/db/targets_iter.v

@@ -0,0 +1,129 @@
+module db
+
+import models { Target, TargetFilter }
+import sqlite
+
+// Iterator providing a filtered view into the list of targets currently stored
+// in the database. It replaces functionality usually performed in the database
+// using SQL queries that can't currently be used due to missing stuff in V's
+// ORM.
+pub struct TargetsIterator {
+	conn        sqlite.DB
+	filter      TargetFilter
+	window_size int = 32
+mut:
+	window       []Target
+	window_index u64
+	// Offset in entire list of unfiltered targets
+	offset int
+	// Offset in filtered list of targets
+	filtered_offset u64
+	started         bool
+	done            bool
+}
+
+// targets returns an iterator allowing filtered access to the list of targets.
+pub fn (db &VieterDb) targets(filter TargetFilter) TargetsIterator {
+	window_size := 32
+
+	return TargetsIterator{
+		conn: db.conn
+		filter: filter
+		window: []Target{cap: window_size}
+		window_size: window_size
+	}
+}
+
+// advance_window moves the sliding window over the filtered list of targets
+// until it either reaches the end of the list of targets, or has encountered a
+// non-empty window.
+fn (mut ti TargetsIterator) advance_window() {
+	for {
+		ti.window = sql ti.conn {
+			select from Target order by id limit ti.window_size offset ti.offset
+		}
+		ti.offset += ti.window.len
+
+		if ti.window.len == 0 {
+			ti.done = true
+
+			return
+		}
+
+		if ti.filter.repo != '' {
+			ti.window = ti.window.filter(it.repo == ti.filter.repo)
+		}
+
+		if ti.filter.arch != '' {
+			ti.window = ti.window.filter(it.arch.any(it.value == ti.filter.arch))
+		}
+
+		if ti.filter.query != '' {
+			ti.window = ti.window.filter(it.url.contains(ti.filter.query)
+				|| it.path.contains(ti.filter.query) || it.branch.contains(ti.filter.query))
+		}
+
+		// We break out of the loop once we found a non-empty window
+		if ti.window.len > 0 {
+			break
+		}
+	}
+}
+
+// next returns the next target, if possible.
+pub fn (mut ti TargetsIterator) next() ?Target {
+	if ti.done {
+		return none
+	}
+
+	// The first call to `next` will cause the sliding window to move to where
+	// the requested offset starts
+	if !ti.started {
+		ti.advance_window()
+
+		// Skip all matched targets until the requested offset
+		for !ti.done && ti.filtered_offset + u64(ti.window.len) <= ti.filter.offset {
+			ti.filtered_offset += u64(ti.window.len)
+			ti.advance_window()
+		}
+
+		if ti.done {
+			return none
+		}
+
+		left_inside_window := ti.filter.offset - ti.filtered_offset
+		ti.window_index = left_inside_window
+		ti.filtered_offset += left_inside_window
+
+		ti.started = true
+	}
+
+	return_value := ti.window[ti.window_index]
+
+	ti.window_index++
+	ti.filtered_offset++
+
+	// Next call will be past the requested offset
+	if ti.filter.limit > 0 && ti.filtered_offset == ti.filter.offset + ti.filter.limit {
+		ti.done = true
+	}
+
+	// Ensure the next call has a new valid window
+	if ti.window_index == u64(ti.window.len) {
+		ti.advance_window()
+		ti.window_index = 0
+	}
+
+	return return_value
+}
+
+// collect consumes the entire iterator & returns the result as an array.
+pub fn (mut ti TargetsIterator) collect() []Target {
+	mut out := []Target{}
+
+	for t in ti {
+		out << t
+	}
+
+	return out
+}

src/models/targets.v

@@ -73,4 +73,6 @@ pub mut:
 	limit  u64 = 25
 	offset u64
 	repo   string
+	query  string
+	arch   string
 }

src/server/api_metrics.v

@@ -0,0 +1,20 @@
+module server
+
+import metrics
+import web
+
+// v1_metrics serves a Prometheus-compatible metrics endpoint.
+['/api/v1/metrics'; get; markused]
+fn (mut app App) v1_metrics() web.Result {
+	if !app.conf.collect_metrics {
+		return app.status(.not_found)
+	}
+
+	mut exporter := metrics.new_prometheus_exporter([0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5,
+		10])
+	exporter.load('vieter_', app.collector)
+
+	// TODO stream to connection instead
+	body := exporter.export_to_string() or { return app.status(.internal_server_error) }
+	return app.body(.ok, 'text/plain', body)
+}

src/server/api_targets.v

@@ -11,9 +11,9 @@ fn (mut app App) v1_get_targets() web.Result {
 	filter := models.from_params<TargetFilter>(app.query) or {
 		return app.json(.bad_request, new_response('Invalid query parameters.'))
 	}
-	targets := app.db.get_targets(filter)
+	mut iter := app.db.targets(filter)
 
-	return app.json(.ok, new_data_response(targets))
+	return app.json(.ok, new_data_response(iter.collect()))
 }
 
 // v1_get_single_target returns the information for a single target.

src/server/cli.v

@@ -15,6 +15,7 @@ pub:
 	base_image           string = 'archlinux:base-devel'
 	max_log_age          int    [empty_default]
 	log_removal_schedule string = '0 0'
+	collect_metrics      bool   [empty_default]
 }
 
 // cmd returns the cli submodule that handles starting the server

src/server/server.v

@@ -8,6 +8,7 @@ import util
 import db
 import build { BuildJobQueue }
 import cron.expression
+import metrics
 
 const (
 	log_file_name = 'vieter.log'
@@ -30,17 +31,8 @@ pub mut:
 // init_job_queue populates a fresh job queue with all the targets currently
 // stored in the database.
 fn (mut app App) init_job_queue() ! {
-	// Initialize build queues
+	for target in app.db.targets(limit: 0) {
-	mut targets := app.db.get_targets(limit: 25)
+		app.job_queue.insert_all(target)!
-	mut i := u64(0)
-
-	for targets.len > 0 {
-		for target in targets {
-			app.job_queue.insert_all(target)!
-		}
-
-		i += 25
-		targets = app.db.get_targets(limit: 25, offset: i)
 	}
 }
 
@@ -100,12 +92,19 @@ pub fn server(conf Config) ! {
 		util.exit_with_message(1, 'Failed to initialize database: $err.msg()')
 	}
 
+	collector := if conf.collect_metrics {
+		&metrics.MetricsCollector(metrics.new_default_collector())
+	} else {
+		&metrics.MetricsCollector(metrics.new_null_collector())
+	}
+
 	mut app := &App{
 		logger: logger
 		api_key: conf.api_key
 		conf: conf
 		repo: repo
 		db: db
+		collector: collector
 		job_queue: build.new_job_queue(global_ce, conf.base_image)
 	}
 	app.init_job_queue() or {

src/v.mod

@@ -2,6 +2,7 @@ Module {
     dependencies: [
         'https://git.rustybever.be/vieter-v/conf',
         'https://git.rustybever.be/vieter-v/docker',
-        'https://git.rustybever.be/vieter-v/aur'
+        'https://git.rustybever.be/vieter-v/aur',
+        'https://git.rustybever.be/vieter-v/metrics'
     ]
 }

src/web/web.v

@@ -11,6 +11,7 @@ import net.urllib
 import time
 import json
 import log
+import metrics
 
 // The Context struct represents the Context which hold the HTTP request and response.
 // It has fields for the query, form, files.
@@ -27,6 +28,8 @@ pub mut:
 	conn &net.TcpConn = unsafe { nil }
 	// Gives access to a shared logger object
 	logger shared log.Log
+	// Used to collect metrics on the web server
+	collector &metrics.MetricsCollector
 	// time.ticks() from start of web connection handle.
 	// You can use it to determine how much time is spent on your request.
 	page_gen_start i64
@@ -145,6 +148,15 @@ pub fn (ctx &Context) is_authenticated() bool {
 	return false
 }
 
+// body sends the given body as an HTTP response.
+pub fn (mut ctx Context) body(status http.Status, content_type string, body string) Result {
+	ctx.status = status
+	ctx.content_type = content_type
+	ctx.send_response(body)
+
+	return Result{}
+}
+
 // json<T> HTTP_OK with json_s as payload with content-type `application/json`
 pub fn (mut ctx Context) json<T>(status http.Status, j T) Result {
 	ctx.status = status
@@ -319,6 +331,22 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T, routes map[string]Route) {
 			app.logger.flush()
 		}
 
+		// Record how long request took to process
+		labels := [
+			['method', app.req.method.str()]!,
+			['path', app.req.url]!,
+			// Not all methods properly set this value yet I think
+			['status', app.status.int().str()]!,
+		]
+		app.collector.counter_increment(name: 'http_requests_total', labels: labels)
+		// Prometheus prefers metrics containing base units, as defined here
+		// https://prometheus.io/docs/practices/naming/
+		app.collector.histogram_record(f64(time.ticks() - app.page_gen_start) / 1000,
+			
+			name: 'http_requests_duration_seconds'
+			labels: labels
+		)
+
 		unsafe {
 			free(app)
 		}
@@ -384,6 +412,7 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T, routes map[string]Route) {
 		static_mime_types: app.static_mime_types
 		reader: reader
 		logger: app.logger
+		collector: app.collector
 		api_key: app.api_key
 	}
 

vieter.toml

@@ -12,4 +12,5 @@ address = "http://localhost:8000"
 api_update_frequency = 2
 image_rebuild_frequency = 1
 max_concurrent_builds = 3
-max_log_age = 64
+# max_log_age = 64
+collect_metrics = true