Merge pull request 'Better API & CLI: GitRepo' (#188) from Chewing_Bever/vieter:better-api into dev

Reviewed-on: vieter/vieter#188

src/build/build.v

@@ -4,9 +4,9 @@ import docker
 import encoding.base64
 import time
 import os
-import db
 import strings
 import util
+import models { GitRepo }
 
 const (
 	container_build_dir = '/build'
@@ -93,7 +93,7 @@ pub:
 // build_repo builds, packages & publishes a given Arch package based on the
 // provided GitRepo. The base image ID should be of an image previously created
 // by create_build_image. It returns the logs of the container.
-pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ?BuildResult {
+pub fn build_repo(address string, api_key string, base_image_id string, repo &GitRepo) ?BuildResult {
 	mut dd := docker.new_conn()?
 
 	defer {

src/client/client.v

@@ -29,8 +29,11 @@ fn (c &Client) send_request_raw(method Method, url string, params map[string]str
 
 		// Escape each query param
 		for k, v in params {
+			// An empty parameter should be the same as not providing it at all
+			if v != '' {
 				params_escaped[k] = urllib.query_escape(v)
 			}
+		}
 
 		params_str := params_escaped.keys().map('$it=${params[it]}').join('&')
 

src/client/git.v

@@ -1,12 +1,13 @@
 module client
 
-import db { GitRepo }
+import models { GitRepo, GitRepoFilter }
 import net.http { Method }
 import response { Response }
 
 // get_git_repos returns the current list of repos.
-pub fn (c &Client) get_git_repos() ?[]GitRepo {
-	data := c.send_request<[]GitRepo>(Method.get, '/api/repos', {})?
+pub fn (c &Client) get_git_repos(filter GitRepoFilter) ?[]GitRepo {
+	params := models.params_from(filter)
+	data := c.send_request<[]GitRepo>(Method.get, '/api/repos', params)?
 
 	return data.data
 }

src/client/logs.v

@@ -1,6 +1,6 @@
 module client
 
-import db { BuildLog }
+import models { BuildLog }
 import net.http { Method }
 import response { Response }
 import time

src/console/git/git.v

@@ -5,6 +5,7 @@ import env
 import cron.expression { parse_expression }
 import client
 import console
+import models { GitRepoFilter }
 
 struct Config {
 	address    string [required]
@@ -21,11 +22,50 @@ pub fn cmd() cli.Command {
 			cli.Command{
 				name: 'list'
 				description: 'List the current repos.'
+				flags: [
+					cli.Flag{
+						name: 'limit'
+						description: 'How many results to return.'
+						flag: cli.FlagType.int
+					},
+					cli.Flag{
+						name: 'offset'
+						description: 'Minimum index to return.'
+						flag: cli.FlagType.int
+					},
+					cli.Flag{
+						name: 'repo'
+						description: 'Only return Git repos that publish to this repo.'
+						flag: cli.FlagType.string
+					},
+					cli.Flag{
+						name: 'arch'
+						description: 'Only return repos enabled for this architecture.'
+						flag: cli.FlagType.string
+					},
+				]
 				execute: fn (cmd cli.Command) ? {
 					config_file := cmd.flags.get_string('config-file')?
 					conf := env.load<Config>(config_file)?
 
-					list(conf)?
+					mut filter := GitRepoFilter{}
+
+					limit := cmd.flags.get_int('limit')?
+					if limit != 0 {
+						filter.limit = u64(limit)
+					}
+
+					offset := cmd.flags.get_int('offset')?
+					if offset != 0 {
+						filter.offset = u64(offset)
+					}
+
+					repo := cmd.flags.get_string('repo')?
+					if repo != '' {
+						filter.repo = repo
+					}
+
+					list(conf, filter)?
 				}
 			},
 			cli.Command{
@@ -133,9 +173,9 @@ pub fn cmd() cli.Command {
 // ID. If multiple or none are found, an error is raised.
 
 // list prints out a list of all repositories.
-fn list(conf Config) ? {
+fn list(conf Config, filter GitRepoFilter) ? {
 	c := client.new(conf.address, conf.api_key)
-	repos := c.get_git_repos()?
+	repos := c.get_git_repos(filter)?
 	data := repos.map([it.id.str(), it.url, it.branch, it.repo])
 
 	println(console.pretty_table(['id', 'url', 'branch', 'repo'], data)?)

src/console/logs/logs.v

@@ -3,8 +3,8 @@ module logs
 import cli
 import env
 import client
-import db
 import console
+import models { BuildLog }
 
 struct Config {
 	address string [required]
@@ -67,7 +67,7 @@ pub fn cmd() cli.Command {
 }
 
 // print_log_list prints a list of logs.
-fn print_log_list(logs []db.BuildLog) ? {
+fn print_log_list(logs []BuildLog) ? {
 	data := logs.map([it.id.str(), it.repo_id.str(), it.start_time.str(),
 		it.exit_code.str()])
 

src/cron/daemon/daemon.v

@@ -7,9 +7,9 @@ import cron.expression { CronExpression, parse_expression }
 import math
 import build
 import docker
-import db
 import os
 import client
+import models { GitRepo }
 
 const (
 	// How many seconds to wait before retrying to update API if failed
@@ -20,7 +20,7 @@ const (
 
 struct ScheduledBuild {
 pub:
-	repo      db.GitRepo
+	repo      GitRepo
 	timestamp time.Time
 }
 
@@ -38,7 +38,7 @@ mut:
 	api_update_frequency    int
 	image_rebuild_frequency int
 	// Repos currently loaded from API.
-	repos []db.GitRepo
+	repos []GitRepo
 	// At what point to update the list of repositories.
 	api_update_timestamp  time.Time
 	image_build_timestamp time.Time
@@ -149,7 +149,7 @@ pub fn (mut d Daemon) run() {
 }
 
 // schedule_build adds the next occurence of the given repo build to the queue.
-fn (mut d Daemon) schedule_build(repo db.GitRepo) {
+fn (mut d Daemon) schedule_build(repo GitRepo) {
 	ce := if repo.schedule != '' {
 		parse_expression(repo.schedule) or {
 			// TODO This shouldn't return an error if the expression is empty.

src/db/db.v

@@ -1,6 +1,7 @@
 module db
 
 import sqlite
+import models { BuildLog, GitRepo }
 
 struct VieterDb {
 	conn sqlite.DB

src/db/git.v

@@ -1,85 +1,21 @@
 module db
 
-pub struct GitRepoArch {
-pub:
-	id      int    [primary; sql: serial]
-	repo_id int    [nonull]
-	value   string [nonull]
-}
-
-// str returns a string representation.
-pub fn (gra &GitRepoArch) str() string {
-	return gra.value
-}
-
-pub struct GitRepo {
-pub mut:
-	id int [optional; primary; sql: serial]
-	// URL of the Git repository
-	url string [nonull]
-	// Branch of the Git repository to use
-	branch string [nonull]
-	// Which repo the builder should publish packages to
-	repo string [nonull]
-	// Cron schedule describing how frequently to build the repo.
-	schedule string [optional]
-	// On which architectures the package is allowed to be built. In reality,
-	// this controls which builders will periodically build the image.
-	arch []GitRepoArch [fkey: 'repo_id']
-}
-
-// str returns a string representation.
-pub fn (gr &GitRepo) str() string {
-	mut parts := [
-		'id: $gr.id',
-		'url: $gr.url',
-		'branch: $gr.branch',
-		'repo: $gr.repo',
-		'schedule: $gr.schedule',
-		'arch: ${gr.arch.map(it.value).join(', ')}',
-	]
-	str := parts.join('\n')
-
-	return str
-}
-
-// patch_from_params patches a GitRepo from a map[string]string, usually
-// provided from a web.App's params
-pub fn (mut r GitRepo) patch_from_params(params map[string]string) {
-	$for field in GitRepo.fields {
-		if field.name in params {
-			$if field.typ is string {
-				r.$(field.name) = params[field.name]
-				// This specific type check is needed for the compiler to ensure
-				// our types are correct
-			} $else $if field.typ is []GitRepoArch {
-				r.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it })
-			}
-		}
-	}
-}
-
-// git_repo_from_params creates a GitRepo from a map[string]string, usually
-// provided from a web.App's params
-pub fn git_repo_from_params(params map[string]string) ?GitRepo {
-	mut repo := GitRepo{}
-
-	// If we're creating a new GitRepo, we want all fields to be present before
-	// "patching".
-	$for field in GitRepo.fields {
-		if field.name !in params && !field.attrs.contains('optional') {
-			return error('Missing parameter: ${field.name}.')
-		}
-	}
-	repo.patch_from_params(params)
-
-	return repo
-}
+import models { GitRepo, GitRepoArch, GitRepoFilter }
 
 // get_git_repos returns all GitRepo's in the database.
-pub fn (db &VieterDb) get_git_repos() []GitRepo {
+pub fn (db &VieterDb) get_git_repos(filter GitRepoFilter) []GitRepo {
+	// This seems to currently be blocked by a bug in the ORM, I'll have to ask
+	// around.
+	if filter.repo != '' {
 		res := sql db.conn {
-		select from GitRepo order by id
+			select from GitRepo where repo == filter.repo order by id limit filter.limit offset filter.offset
+		}
+
+		return res
+	}
+
+	res := sql db.conn {
+		select from GitRepo order by id limit filter.limit offset filter.offset
 	}
 
 	return res
@@ -130,6 +66,7 @@ pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) {
 		}
 	}
 	values_str := values.join(', ')
+	// I think this is actual SQL & not the ORM language
 	query := 'update GitRepo set $values_str where id == $repo_id'
 
 	db.conn.exec_none(query)

src/db/logs.v

@@ -1,31 +1,6 @@
 module db
 
-import time
-
-pub struct BuildLog {
-pub:
-	id         int       [primary; sql: serial]
-	repo_id    int       [nonull]
-	start_time time.Time [nonull]
-	end_time   time.Time [nonull]
-	arch       string    [nonull]
-	exit_code  int       [nonull]
-}
-
-// str returns a string representation.
-pub fn (bl &BuildLog) str() string {
-	mut parts := [
-		'id: $bl.id',
-		'repo id: $bl.repo_id',
-		'start time: $bl.start_time',
-		'end time: $bl.end_time',
-		'arch: $bl.arch',
-		'exit code: $bl.exit_code',
-	]
-	str := parts.join('\n')
-
-	return str
-}
+import models { BuildLog }
 
 // get_build_logs returns all BuildLog's in the database.
 pub fn (db &VieterDb) get_build_logs() []BuildLog {

src/models/git.v

@@ -0,0 +1,52 @@
+module models
+
+pub struct GitRepoArch {
+pub:
+	id      int    [primary; sql: serial]
+	repo_id int    [nonull]
+	value   string [nonull]
+}
+
+// str returns a string representation.
+pub fn (gra &GitRepoArch) str() string {
+	return gra.value
+}
+
+pub struct GitRepo {
+pub mut:
+	id int [primary; sql: serial]
+	// URL of the Git repository
+	url string [nonull]
+	// Branch of the Git repository to use
+	branch string [nonull]
+	// Which repo the builder should publish packages to
+	repo string [nonull]
+	// Cron schedule describing how frequently to build the repo.
+	schedule string
+	// On which architectures the package is allowed to be built. In reality,
+	// this controls which builders will periodically build the image.
+	arch []GitRepoArch [fkey: 'repo_id']
+}
+
+// str returns a string representation.
+pub fn (gr &GitRepo) str() string {
+	mut parts := [
+		'id: $gr.id',
+		'url: $gr.url',
+		'branch: $gr.branch',
+		'repo: $gr.repo',
+		'schedule: $gr.schedule',
+		'arch: ${gr.arch.map(it.value).join(', ')}',
+	]
+	str := parts.join('\n')
+
+	return str
+}
+
+[params]
+pub struct GitRepoFilter {
+pub mut:
+	limit  u64 = 25
+	offset u64
+	repo   string
+}

src/models/logs.v

@@ -0,0 +1,28 @@
+module models
+
+import time
+
+pub struct BuildLog {
+pub:
+	id         int       [primary; sql: serial]
+	repo_id    int       [nonull]
+	start_time time.Time [nonull]
+	end_time   time.Time [nonull]
+	arch       string    [nonull]
+	exit_code  int       [nonull]
+}
+
+// str returns a string representation.
+pub fn (bl &BuildLog) str() string {
+	mut parts := [
+		'id: $bl.id',
+		'repo id: $bl.repo_id',
+		'start time: $bl.start_time',
+		'end time: $bl.end_time',
+		'arch: $bl.arch',
+		'exit code: $bl.exit_code',
+	]
+	str := parts.join('\n')
+
+	return str
+}

src/models/models.v

@@ -0,0 +1,41 @@
+module models
+
+// from_params<T> creates a new instance of T from the given map by parsing all
+// of its fields from the map.
+pub fn from_params<T>(params map[string]string) ?T {
+	mut o := T{}
+
+	patch_from_params<T>(mut o, params)?
+
+	return o
+}
+
+// patch_from_params<T> updates the given T object with the params defined in
+// the map.
+pub fn patch_from_params<T>(mut o T, params map[string]string) ? {
+	$for field in T.fields {
+		if field.name in params && params[field.name] != '' {
+			$if field.typ is string {
+				o.$(field.name) = params[field.name]
+			} $else $if field.typ is int {
+				o.$(field.name) = params[field.name].int()
+			} $else $if field.typ is u64 {
+				o.$(field.name) = params[field.name].u64()
+			} $else $if field.typ is []GitRepoArch {
+				o.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it })
+			}
+		} else if field.attrs.contains('nonull') {
+			return error('Missing parameter: ${field.name}.')
+		}
+	}
+}
+
+// params_from<T> converts a given T struct into a map of strings.
+pub fn params_from<T>(o &T) map[string]string {
+	mut out := map[string]string{}
+
+	$for field in T.fields {
+		out[field.name] = o.$(field.name).str()
+	}
+	return out
+}

src/server/git.v

@@ -4,6 +4,7 @@ import web
 import net.http
 import response { new_data_response, new_response }
 import db
+import models { GitRepo, GitRepoArch, GitRepoFilter }
 
 // get_repos returns the current list of repos.
 ['/api/repos'; get]
@@ -12,7 +13,10 @@ fn (mut app App) get_repos() web.Result {
 		return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
 	}
 
-	repos := app.db.get_git_repos()
+	filter := models.from_params<GitRepoFilter>(app.query) or {
+		return app.json(http.Status.bad_request, new_response('Invalid query parameters.'))
+	}
+	repos := app.db.get_git_repos(filter)
 
 	return app.json(http.Status.ok, new_data_response(repos))
 }
@@ -44,7 +48,7 @@ fn (mut app App) post_repo() web.Result {
 		params['arch'] = app.conf.default_arch
 	}
 
-	new_repo := db.git_repo_from_params(params) or {
+	new_repo := models.from_params<GitRepo>(params) or {
 		return app.json(http.Status.bad_request, new_response(err.msg()))
 	}
 
@@ -75,7 +79,7 @@ fn (mut app App) patch_repo(id int) web.Result {
 	app.db.update_git_repo(id, app.query)
 
 	if 'arch' in app.query {
-		arch_objs := app.query['arch'].split(',').map(db.GitRepoArch{ value: it })
+		arch_objs := app.query['arch'].split(',').map(GitRepoArch{ value: it })
 
 		app.db.update_git_repo_archs(id, arch_objs)
 	}

src/server/logs.v

@@ -8,6 +8,7 @@ import db
 import time
 import os
 import util
+import models { BuildLog }
 
 // get_logs returns all build logs in the database. A 'repo' query param can
 // optionally be added to limit the list of build logs to that repository.
@@ -97,7 +98,7 @@ fn (mut app App) post_log() web.Result {
 	}
 
 	// Store log in db
-	log := db.BuildLog{
+	log := BuildLog{
 		repo_id: repo_id
 		start_time: start_time
 		end_time: end_time