From 393e641a7666682c1e4ff38b83affbf596c461d6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 15:31:01 +0200 Subject: [PATCH 1/4] feat(server): allow filtering of builds per repo --- src/db/logs.v | 12 +++++++++++- src/server/logs.v | 16 +++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/db/logs.v b/src/db/logs.v index 3e5b600..9a2405b 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -4,7 +4,7 @@ import time pub struct BuildLog { id int [primary; sql: serial] - repo GitRepo [nonull] + repo_id int [nonull] start_time time.Time [nonull] end_time time.Time [nonull] exit_code int [nonull] @@ -19,6 +19,16 @@ pub fn (db &VieterDb) get_build_logs() []BuildLog { return res } +// get_build_logs_for_repo returns all BuildLog's in the database for a given +// repo. +pub fn (db &VieterDb) get_build_logs_for_repo(repo_id int) []BuildLog { + res := sql db.conn { + select from BuildLog where repo_id == repo_id order by id + } + + return res +} + // get_build_log tries to return a specific BuildLog. pub fn (db &VieterDb) get_build_log(id int) ?BuildLog { res := sql db.conn { diff --git a/src/server/logs.v b/src/server/logs.v index 01116b4..8b0f297 100644 --- a/src/server/logs.v +++ b/src/server/logs.v @@ -15,7 +15,11 @@ fn (mut app App) get_logs() web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - logs := app.db.get_build_logs() + logs := if 'repo' in app.query { + app.db.get_build_logs_for_repo(app.query['repo'].int()) + } else { + app.db.get_build_logs() + } return app.json(http.Status.ok, new_data_response(logs)) } @@ -56,13 +60,15 @@ fn (mut app App) post_log() web.Result { arch := app.query['arch'] - repo := app.db.get_git_repo(app.query['repo'].int()) or { - return app.json(http.Status.bad_request, new_response('Unknown repo.')) + repo := app.query['repo'].int() + + if repo == 0 { + return app.json(http.Status.bad_request, new_response('Invalid Git repo.')) } // Store log in db log := db.BuildLog{ - repo: repo + repo_id: repo start_time: start_time end_time: end_time exit_code: exit_code @@ -70,7 +76,7 @@ fn (mut app App) post_log() web.Result { app.db.add_build_log(log) - repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo.id.str(), arch) + repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo.str(), arch) // Create the logs directory of it doesn't exist if !os.exists(repo_logs_dir) { From 139142fcec4f66ad08e95ed4dcf6c3c30acc423e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 15:41:49 +0200 Subject: [PATCH 2/4] feat(server): added endpoint for content of build log --- src/db/logs.v | 2 ++ src/server/logs.v | 27 ++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/db/logs.v b/src/db/logs.v index 9a2405b..589304e 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -3,10 +3,12 @@ module db import time pub struct BuildLog { +pub: id int [primary; sql: serial] repo_id int [nonull] start_time time.Time [nonull] end_time time.Time [nonull] + arch string [nonull] exit_code int [nonull] } diff --git a/src/server/logs.v b/src/server/logs.v index 8b0f297..9bddc21 100644 --- a/src/server/logs.v +++ b/src/server/logs.v @@ -24,6 +24,30 @@ fn (mut app App) get_logs() web.Result { return app.json(http.Status.ok, new_data_response(logs)) } +['/api/logs/:id'; get] +fn (mut app App) get_single_log(id int) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + log := app.db.get_build_log(id) or { return app.not_found() } + + return app.json(http.Status.ok, new_data_response(log)) +} + +['/api/logs/:id/content'; get] +fn (mut app App) get_log_contents(id int) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + log := app.db.get_build_log(id) or { return app.not_found() } + file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + full_path := os.join_path(app.conf.data_dir, server.logs_dir_name, log.repo_id.str(), log.arch, file_name) + + return app.file(full_path) +} + // parse_query_time unescapes an HTTP query parameter & tries to parse it as a // time.Time struct. fn parse_query_time(query string) ?time.Time { @@ -44,7 +68,7 @@ fn (mut app App) post_log() web.Result { return app.json(http.Status.bad_request, new_response('Invalid or missing start time.')) } - end_time := time.parse(app.query['endTime'].replace('_', ' ')) or { + end_time := parse_query_time(app.query['endTime']) or { return app.json(http.Status.bad_request, new_response('Invalid or missing end time.')) } @@ -71,6 +95,7 @@ fn (mut app App) post_log() web.Result { repo_id: repo start_time: start_time end_time: end_time + arch: arch exit_code: exit_code } From f42d3fd8b0f2005f96b94e39c66e94f300b2649e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 15:44:59 +0200 Subject: [PATCH 3/4] fix(server): prevent adding logs to non-existent repo --- src/db/git.v | 8 ++++++++ src/server/logs.v | 10 +++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/db/git.v b/src/db/git.v index b779140..6eeecc8 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -152,3 +152,11 @@ pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { } } } + +pub fn (db &VieterDb) git_repo_exists(repo_id int) bool { + db.get_git_repo(repo_id) or { + return false + } + + return true +} diff --git a/src/server/logs.v b/src/server/logs.v index 9bddc21..10b734d 100644 --- a/src/server/logs.v +++ b/src/server/logs.v @@ -84,15 +84,15 @@ fn (mut app App) post_log() web.Result { arch := app.query['arch'] - repo := app.query['repo'].int() + repo_id := app.query['repo'].int() - if repo == 0 { - return app.json(http.Status.bad_request, new_response('Invalid Git repo.')) + if !app.db.git_repo_exists(repo_id) { + return app.json(http.Status.bad_request, new_response('Unknown Git repo.')) } // Store log in db log := db.BuildLog{ - repo_id: repo + repo_id: repo_id start_time: start_time end_time: end_time arch: arch @@ -101,7 +101,7 @@ fn (mut app App) post_log() web.Result { app.db.add_build_log(log) - repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo.str(), arch) + repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo_id.str(), arch) // Create the logs directory of it doesn't exist if !os.exists(repo_logs_dir) { From 407b2269556e23d561ecd878c82c898502d28fc5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 7 May 2022 16:10:27 +0200 Subject: [PATCH 4/4] refactor: moved client code into own module --- src/build/build.v | 4 +-- src/client/client.v | 40 +++++++++++++++++++++ src/client/git.v | 51 ++++++++++++++++++++++++++ src/cron/daemon/build.v | 2 +- src/cron/daemon/daemon.v | 10 +++--- src/git/cli.v | 17 ++++++--- src/git/client.v | 77 ---------------------------------------- 7 files changed, 110 insertions(+), 91 deletions(-) create mode 100644 src/client/client.v create mode 100644 src/client/git.v delete mode 100644 src/git/client.v diff --git a/src/build/build.v b/src/build/build.v index 15a5eb8..6f033e6 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -3,9 +3,9 @@ module build import docker import encoding.base64 import time -import git import os import db +import client const container_build_dir = '/build' @@ -126,7 +126,7 @@ fn build(conf Config) ? { build_arch := os.uname().machine // We get the repos map from the Vieter instance - repos := git.get_repos(conf.address, conf.api_key) ? + repos := client.new(conf.address, conf.api_key).get_git_repos() ? // We filter out any repos that aren't allowed to be built on this // architecture diff --git a/src/client/client.v b/src/client/client.v new file mode 100644 index 0000000..614b3db --- /dev/null +++ b/src/client/client.v @@ -0,0 +1,40 @@ +module client + +import net.http +import response { Response } +import json + +pub struct Client { +pub: + address string + api_key string +} + +pub fn new(address string, api_key string) Client { + return Client{ + address: address + api_key: api_key + } +} + +// send_request is a convenience method for sending requests to the repos +// API. It mostly does string manipulation to create a query string containing +// the provided params. +fn (c &Client) send_request(method http.Method, url string, params map[string]string) ?Response { + mut full_url := '${c.address}$url' + + if params.len > 0 { + params_str := params.keys().map('$it=${params[it]}').join('&') + + full_url = '$full_url?$params_str' + } + + mut req := http.new_request(method, full_url, '') ? + req.add_custom_header('X-API-Key', c.api_key) ? + + res := req.do() ? + data := json.decode(Response, res.text) ? + + return data +} + diff --git a/src/client/git.v b/src/client/git.v new file mode 100644 index 0000000..5ed0620 --- /dev/null +++ b/src/client/git.v @@ -0,0 +1,51 @@ +module client + +import db +import net.http +import response { Response } + +// get_repos returns the current list of repos. +pub fn (c &Client) get_git_repos() ?[]db.GitRepo { + data := c.send_request<[]db.GitRepo>(http.Method.get, '/api/repos', {}) ? + + return data.data +} + +// get_repo returns the repo for a specific ID. +pub fn (c &Client) get_git_repo(id int) ?db.GitRepo { + data := c.send_request(http.Method.get, '/api/repos/$id', {}) ? + + return data.data +} + +// add_repo adds a new repo to the server. +pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []string) ?Response { + mut params := { + 'url': url + 'branch': branch + 'repo': repo + } + + if arch.len > 0 { + params['arch'] = arch.join(',') + } + + data := c.send_request(http.Method.post, '/api/repos', params) ? + + return data +} + +// remove_repo removes the repo with the given ID from the server. +pub fn (c &Client) remove_git_repo(id int) ?Response { + data := c.send_request(http.Method.delete, '/api/repos/$id', {}) ? + + return data +} + +// patch_repo sends a PATCH request to the given repo with the params as +// payload. +pub fn (c &Client) patch_git_repo(id int, params map[string]string) ?Response { + data := c.send_request(http.Method.patch, '/api/repos/$id', params) ? + + return data +} diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index e54a39e..d107fd3 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -77,7 +77,7 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { // 0 means success, 1 means failure mut status := 0 - build.build_repo(d.address, d.api_key, d.builder_images.last(), &sb.repo) or { + build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(), &sb.repo) or { d.ldebug('build_repo error: $err.msg()') status = 1 } diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index ffa2f6e..71fc575 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -1,6 +1,5 @@ module daemon -import git import time import log import datatypes { MinHeap } @@ -10,6 +9,7 @@ import build import docker import db import os +import client const ( // How many seconds to wait before retrying to update API if failed @@ -31,8 +31,7 @@ fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool { pub struct Daemon { mut: - address string - api_key string + client client.Client base_image string builder_images []string global_schedule CronExpression @@ -56,8 +55,7 @@ mut: // populates the build queue for the first time. pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon { mut d := Daemon{ - address: address - api_key: api_key + client: client.new(address, api_key) base_image: base_image global_schedule: global_schedule api_update_frequency: api_update_frequency @@ -180,7 +178,7 @@ fn (mut d Daemon) schedule_build(repo db.GitRepo) { fn (mut d Daemon) renew_repos() { d.linfo('Renewing repos...') - mut new_repos := git.get_repos(d.address, d.api_key) or { + mut new_repos := d.client.get_git_repos() or { d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...') d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) diff --git a/src/git/cli.v b/src/git/cli.v index 634b778..bdc0479 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -3,6 +3,7 @@ module git import cli import env import cron.expression { parse_expression } +import client struct Config { address string [required] @@ -119,7 +120,8 @@ pub fn cmd() cli.Command { // list prints out a list of all repositories. fn list(conf Config) ? { - repos := get_repos(conf.address, conf.api_key) ? + c := client.new(conf.address, conf.api_key) + repos := c.get_git_repos() ? for repo in repos { println('$repo.id\t$repo.url\t$repo.branch\t$repo.repo') @@ -128,7 +130,8 @@ fn list(conf Config) ? { // add adds a new repository to the server's list. fn add(conf Config, url string, branch string, repo string) ? { - res := add_repo(conf.address, conf.api_key, url, branch, repo, []) ? + c := client.new(conf.address, conf.api_key) + res := c.add_git_repo(url, branch, repo, []) ? println(res.message) } @@ -139,7 +142,8 @@ fn remove(conf Config, id string) ? { id_int := id.int() if id_int != 0 { - res := remove_repo(conf.address, conf.api_key, id_int) ? + c := client.new(conf.address, conf.api_key) + res := c.remove_git_repo(id_int) ? println(res.message) } } @@ -156,7 +160,9 @@ fn patch(conf Config, id string, params map[string]string) ? { id_int := id.int() if id_int != 0 { - res := patch_repo(conf.address, conf.api_key, id_int, params) ? + + c := client.new(conf.address, conf.api_key) + res := c.patch_git_repo(id_int, params) ? println(res.message) } @@ -170,6 +176,7 @@ fn info(conf Config, id string) ? { return } - repo := get_repo(conf.address, conf.api_key, id_int) ? + c := client.new(conf.address, conf.api_key) + repo := c.get_git_repo(id_int) ? println(repo) } diff --git a/src/git/client.v b/src/git/client.v deleted file mode 100644 index b5f8e9f..0000000 --- a/src/git/client.v +++ /dev/null @@ -1,77 +0,0 @@ -module git - -import json -import response { Response } -import net.http -import db - -// send_request is a convenience method for sending requests to the repos -// API. It mostly does string manipulation to create a query string containing -// the provided params. -fn send_request(method http.Method, address string, url string, api_key string, params map[string]string) ?Response { - mut full_url := '$address$url' - - if params.len > 0 { - params_str := params.keys().map('$it=${params[it]}').join('&') - - full_url = '$full_url?$params_str' - } - - mut req := http.new_request(method, full_url, '') ? - req.add_custom_header('X-API-Key', api_key) ? - - res := req.do() ? - data := json.decode(Response, res.text) ? - - return data -} - -// get_repos returns the current list of repos. -pub fn get_repos(address string, api_key string) ?[]db.GitRepo { - data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', api_key, - {}) ? - - return data.data -} - -// get_repo returns the repo for a specific ID. -pub fn get_repo(address string, api_key string, id int) ?db.GitRepo { - data := send_request(http.Method.get, address, '/api/repos/$id', api_key, - {}) ? - - return data.data -} - -// add_repo adds a new repo to the server. -pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response { - mut params := { - 'url': url - 'branch': branch - 'repo': repo - } - - if arch.len > 0 { - params['arch'] = arch.join(',') - } - - data := send_request(http.Method.post, address, '/api/repos', api_key, params) ? - - return data -} - -// remove_repo removes the repo with the given ID from the server. -pub fn remove_repo(address string, api_key string, id int) ?Response { - data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, - {}) ? - - return data -} - -// patch_repo sends a PATCH request to the given repo with the params as -// payload. -pub fn patch_repo(address string, api_key string, id int, params map[string]string) ?Response { - data := send_request(http.Method.patch, address, '/api/repos/$id', api_key, - params) ? - - return data -}