diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bbe4f06..7d9eb4d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev) + +### Added + +* Web API for adding & querying build logs +* CLI commands to access build logs API + ## [0.3.0-alpha.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.1) ### Changed diff --git a/src/build/build.v b/src/build/build.v index 15a5eb81..6f033e67 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -3,9 +3,9 @@ module build import docker import encoding.base64 import time -import git import os import db +import client const container_build_dir = '/build' @@ -126,7 +126,7 @@ fn build(conf Config) ? { build_arch := os.uname().machine // We get the repos map from the Vieter instance - repos := git.get_repos(conf.address, conf.api_key) ? + repos := client.new(conf.address, conf.api_key).get_git_repos() ? // We filter out any repos that aren't allowed to be built on this // architecture diff --git a/src/client/client.v b/src/client/client.v new file mode 100644 index 00000000..25224a51 --- /dev/null +++ b/src/client/client.v @@ -0,0 +1,67 @@ +module client + +import net.http { Method } +import net.urllib +import response { Response } +import json + +pub struct Client { +pub: + address string + api_key string +} + +// new creates a new Client instance. +pub fn new(address string, api_key string) Client { + return Client{ + address: address + api_key: api_key + } +} + +// send_request_raw sends an HTTP request, returning the http.Response object. +// It encodes the params so that they're safe to pass as HTTP query parameters. +fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) ?http.Response { + mut full_url := '$c.address$url' + + if params.len > 0 { + mut params_escaped := map[string]string{} + + // Escape each query param + for k, v in params { + params_escaped[k] = urllib.query_escape(v) + } + + params_str := params_escaped.keys().map('$it=${params[it]}').join('&') + + full_url = '$full_url?$params_str' + } + + mut req := http.new_request(method, full_url, body) ? + req.add_custom_header('X-Api-Key', c.api_key) ? + + res := req.do() ? + + return res +} + +// send_request just calls send_request_with_body with an empty body. +fn (c &Client) send_request(method Method, url string, params map[string]string) ?Response { + return c.send_request_with_body(method, url, params, '') +} + +// send_request_with_body calls send_request_raw_response & parses its +// output as a Response object. +fn (c &Client) send_request_with_body(method Method, url string, params map[string]string, body string) ?Response { + res_text := c.send_request_raw_response(method, url, params, body) ? + data := json.decode(Response, res_text) ? + + return data +} + +// send_request_raw_response returns the raw text response for an HTTP request. +fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) ?string { + res := c.send_request_raw(method, url, params, body) ? + + return res.text +} diff --git a/src/client/git.v b/src/client/git.v new file mode 100644 index 00000000..b09d4c22 --- /dev/null +++ b/src/client/git.v @@ -0,0 +1,51 @@ +module client + +import db { GitRepo } +import net.http { Method } +import response { Response } + +// get_git_repos returns the current list of repos. +pub fn (c &Client) get_git_repos() ?[]GitRepo { + data := c.send_request<[]GitRepo>(Method.get, '/api/repos', {}) ? + + return data.data +} + +// get_git_repo returns the repo for a specific ID. +pub fn (c &Client) get_git_repo(id int) ?GitRepo { + data := c.send_request(Method.get, '/api/repos/$id', {}) ? + + return data.data +} + +// add_git_repo adds a new repo to the server. +pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []string) ?Response { + mut params := { + 'url': url + 'branch': branch + 'repo': repo + } + + if arch.len > 0 { + params['arch'] = arch.join(',') + } + + data := c.send_request(Method.post, '/api/repos', params) ? + + return data +} + +// remove_git_repo removes the repo with the given ID from the server. +pub fn (c &Client) remove_git_repo(id int) ?Response { + data := c.send_request(Method.delete, '/api/repos/$id', {}) ? + + return data +} + +// patch_git_repo sends a PATCH request to the given repo with the params as +// payload. +pub fn (c &Client) patch_git_repo(id int, params map[string]string) ?Response { + data := c.send_request(Method.patch, '/api/repos/$id', params) ? + + return data +} diff --git a/src/client/logs.v b/src/client/logs.v new file mode 100644 index 00000000..8c532138 --- /dev/null +++ b/src/client/logs.v @@ -0,0 +1,53 @@ +module client + +import db { BuildLog } +import net.http { Method } +import response { Response } +import time + +// get_build_logs returns all build logs. +pub fn (c &Client) get_build_logs() ?Response<[]BuildLog> { + data := c.send_request<[]BuildLog>(Method.get, '/api/logs', {}) ? + + return data +} + +// get_build_logs_for_repo returns all build logs for a given repo. +pub fn (c &Client) get_build_logs_for_repo(repo_id int) ?Response<[]BuildLog> { + params := { + 'repo': repo_id.str() + } + + data := c.send_request<[]BuildLog>(Method.get, '/api/logs', params) ? + + return data +} + +// get_build_log returns a specific build log. +pub fn (c &Client) get_build_log(id int) ?Response { + data := c.send_request(Method.get, '/api/logs/$id', {}) ? + + return data +} + +// get_build_log_content returns the contents of the build log file. +pub fn (c &Client) get_build_log_content(id int) ?string { + data := c.send_request_raw_response(Method.get, '/api/logs/$id/content', {}, '') ? + + return data +} + +// add_build_log adds a new build log to the server. +pub fn (c &Client) add_build_log(repo_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { + params := { + 'repo': repo_id.str() + 'startTime': start_time.str() + 'endTime': end_time.str() + 'arch': arch + 'exitCode': exit_code.str() + } + + data := c.send_request_with_body(Method.post, '/api/logs', params, content) ? + + return data +} diff --git a/src/console/console.v b/src/console/console.v new file mode 100644 index 00000000..6f296bdd --- /dev/null +++ b/src/console/console.v @@ -0,0 +1 @@ +module console diff --git a/src/git/cli.v b/src/console/git/git.v similarity index 90% rename from src/git/cli.v rename to src/console/git/git.v index 634b7782..3bf78d10 100644 --- a/src/git/cli.v +++ b/src/console/git/git.v @@ -3,6 +3,7 @@ module git import cli import env import cron.expression { parse_expression } +import client struct Config { address string [required] @@ -119,7 +120,8 @@ pub fn cmd() cli.Command { // list prints out a list of all repositories. fn list(conf Config) ? { - repos := get_repos(conf.address, conf.api_key) ? + c := client.new(conf.address, conf.api_key) + repos := c.get_git_repos() ? for repo in repos { println('$repo.id\t$repo.url\t$repo.branch\t$repo.repo') @@ -128,7 +130,8 @@ fn list(conf Config) ? { // add adds a new repository to the server's list. fn add(conf Config, url string, branch string, repo string) ? { - res := add_repo(conf.address, conf.api_key, url, branch, repo, []) ? + c := client.new(conf.address, conf.api_key) + res := c.add_git_repo(url, branch, repo, []) ? println(res.message) } @@ -139,7 +142,8 @@ fn remove(conf Config, id string) ? { id_int := id.int() if id_int != 0 { - res := remove_repo(conf.address, conf.api_key, id_int) ? + c := client.new(conf.address, conf.api_key) + res := c.remove_git_repo(id_int) ? println(res.message) } } @@ -156,7 +160,8 @@ fn patch(conf Config, id string, params map[string]string) ? { id_int := id.int() if id_int != 0 { - res := patch_repo(conf.address, conf.api_key, id_int, params) ? + c := client.new(conf.address, conf.api_key) + res := c.patch_git_repo(id_int, params) ? println(res.message) } @@ -170,6 +175,7 @@ fn info(conf Config, id string) ? { return } - repo := get_repo(conf.address, conf.api_key, id_int) ? + c := client.new(conf.address, conf.api_key) + repo := c.get_git_repo(id_int) ? println(repo) } diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v new file mode 100644 index 00000000..e3c7d14a --- /dev/null +++ b/src/console/logs/logs.v @@ -0,0 +1,106 @@ +module logs + +import cli +import env +import client +import db + +struct Config { + address string [required] + api_key string [required] +} + +// cmd returns the cli module that handles the build repos API. +pub fn cmd() cli.Command { + return cli.Command{ + name: 'logs' + description: 'Interact with the build logs API.' + commands: [ + cli.Command{ + name: 'list' + description: 'List the build logs. If a repo ID is provided, only list the build logs for that repo.' + flags: [ + cli.Flag{ + name: 'repo' + description: 'ID of the Git repo to restrict list to.' + flag: cli.FlagType.int + }, + ] + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? + + repo_id := cmd.flags.get_int('repo') ? + + if repo_id == 0 { list(conf) ? } else { list_for_repo(conf, repo_id) ? } + } + }, + cli.Command{ + name: 'info' + required_args: 1 + usage: 'id' + description: 'Show all info for a specific build log.' + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? + + id := cmd.args[0].int() + info(conf, id) ? + } + }, + cli.Command{ + name: 'content' + required_args: 1 + usage: 'id' + description: 'Output the content of a build log to stdout.' + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file') ? + conf := env.load(config_file) ? + + id := cmd.args[0].int() + content(conf, id) ? + } + }, + ] + } +} + +// print_log_list prints a list of logs. +fn print_log_list(logs []db.BuildLog) { + for log in logs { + println('$log.id\t$log.start_time\t$log.exit_code') + } +} + +// list prints a list of all build logs. +fn list(conf Config) ? { + c := client.new(conf.address, conf.api_key) + logs := c.get_build_logs() ?.data + + print_log_list(logs) +} + +// list prints a list of all build logs for a given repo. +fn list_for_repo(conf Config, repo_id int) ? { + c := client.new(conf.address, conf.api_key) + logs := c.get_build_logs_for_repo(repo_id) ?.data + + print_log_list(logs) +} + +// info print the detailed info for a given build log. +fn info(conf Config, id int) ? { + c := client.new(conf.address, conf.api_key) + log := c.get_build_log(id) ?.data + + print(log) +} + +// content outputs the contents of the log file for a given build log to +// stdout. +fn content(conf Config, id int) ? { + c := client.new(conf.address, conf.api_key) + content := c.get_build_log_content(id) ? + + println(content) +} diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index e54a39e0..d107fd35 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -77,7 +77,7 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { // 0 means success, 1 means failure mut status := 0 - build.build_repo(d.address, d.api_key, d.builder_images.last(), &sb.repo) or { + build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(), &sb.repo) or { d.ldebug('build_repo error: $err.msg()') status = 1 } diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index ffa2f6e2..ade8fcbf 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -1,6 +1,5 @@ module daemon -import git import time import log import datatypes { MinHeap } @@ -10,6 +9,7 @@ import build import docker import db import os +import client const ( // How many seconds to wait before retrying to update API if failed @@ -31,8 +31,7 @@ fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool { pub struct Daemon { mut: - address string - api_key string + client client.Client base_image string builder_images []string global_schedule CronExpression @@ -56,8 +55,7 @@ mut: // populates the build queue for the first time. pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon { mut d := Daemon{ - address: address - api_key: api_key + client: client.new(address, api_key) base_image: base_image global_schedule: global_schedule api_update_frequency: api_update_frequency @@ -180,7 +178,7 @@ fn (mut d Daemon) schedule_build(repo db.GitRepo) { fn (mut d Daemon) renew_repos() { d.linfo('Renewing repos...') - mut new_repos := git.get_repos(d.address, d.api_key) or { + mut new_repos := d.client.get_git_repos() or { d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...') d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) diff --git a/src/db/db.v b/src/db/db.v index a75c34c4..5ec240de 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -12,6 +12,7 @@ pub fn init(db_path string) ?VieterDb { sql conn { create table GitRepo + create table BuildLog } return VieterDb{ diff --git a/src/db/git.v b/src/db/git.v index c40086b2..9a475a54 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -94,7 +94,7 @@ pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo { // If a select statement fails, it returns a zeroed object. By // checking one of the required fields, we can see whether the query // returned a result or not. - if res.url == '' { + if res.id == 0 { return none } @@ -152,3 +152,11 @@ pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { } } } + +// git_repo_exists is a utility function that checks whether a repo with the +// given id exists. +pub fn (db &VieterDb) git_repo_exists(repo_id int) bool { + db.get_git_repo(repo_id) or { return false } + + return true +} diff --git a/src/db/logs.v b/src/db/logs.v new file mode 100644 index 00000000..817db78b --- /dev/null +++ b/src/db/logs.v @@ -0,0 +1,74 @@ +module db + +import time + +pub struct BuildLog { +pub: + id int [primary; sql: serial] + repo_id int [nonull] + start_time time.Time [nonull] + end_time time.Time [nonull] + arch string [nonull] + exit_code int [nonull] +} + +// str returns a string representation. +pub fn (bl &BuildLog) str() string { + mut parts := [ + 'id: $bl.id', + 'repo id: $bl.repo_id', + 'start time: $bl.start_time', + 'end time: $bl.end_time', + 'arch: $bl.arch', + 'exit code: $bl.exit_code', + ] + str := parts.join('\n') + + return str +} + +// get_build_logs returns all BuildLog's in the database. +pub fn (db &VieterDb) get_build_logs() []BuildLog { + res := sql db.conn { + select from BuildLog order by id + } + + return res +} + +// get_build_logs_for_repo returns all BuildLog's in the database for a given +// repo. +pub fn (db &VieterDb) get_build_logs_for_repo(repo_id int) []BuildLog { + res := sql db.conn { + select from BuildLog where repo_id == repo_id order by id + } + + return res +} + +// get_build_log tries to return a specific BuildLog. +pub fn (db &VieterDb) get_build_log(id int) ?BuildLog { + res := sql db.conn { + select from BuildLog where id == id + } + + if res.id == 0 { + return none + } + + return res +} + +// add_build_log inserts the given BuildLog into the database. +pub fn (db &VieterDb) add_build_log(log BuildLog) { + sql db.conn { + insert log into BuildLog + } +} + +// delete_build_log delete the BuildLog with the given ID from the database. +pub fn (db &VieterDb) delete_build_log(id int) { + sql db.conn { + delete from BuildLog where id == id + } +} diff --git a/src/git/client.v b/src/git/client.v deleted file mode 100644 index b5f8e9fd..00000000 --- a/src/git/client.v +++ /dev/null @@ -1,77 +0,0 @@ -module git - -import json -import response { Response } -import net.http -import db - -// send_request is a convenience method for sending requests to the repos -// API. It mostly does string manipulation to create a query string containing -// the provided params. -fn send_request(method http.Method, address string, url string, api_key string, params map[string]string) ?Response { - mut full_url := '$address$url' - - if params.len > 0 { - params_str := params.keys().map('$it=${params[it]}').join('&') - - full_url = '$full_url?$params_str' - } - - mut req := http.new_request(method, full_url, '') ? - req.add_custom_header('X-API-Key', api_key) ? - - res := req.do() ? - data := json.decode(Response, res.text) ? - - return data -} - -// get_repos returns the current list of repos. -pub fn get_repos(address string, api_key string) ?[]db.GitRepo { - data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', api_key, - {}) ? - - return data.data -} - -// get_repo returns the repo for a specific ID. -pub fn get_repo(address string, api_key string, id int) ?db.GitRepo { - data := send_request(http.Method.get, address, '/api/repos/$id', api_key, - {}) ? - - return data.data -} - -// add_repo adds a new repo to the server. -pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response { - mut params := { - 'url': url - 'branch': branch - 'repo': repo - } - - if arch.len > 0 { - params['arch'] = arch.join(',') - } - - data := send_request(http.Method.post, address, '/api/repos', api_key, params) ? - - return data -} - -// remove_repo removes the repo with the given ID from the server. -pub fn remove_repo(address string, api_key string, id int) ?Response { - data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, - {}) ? - - return data -} - -// patch_repo sends a PATCH request to the given repo with the params as -// payload. -pub fn patch_repo(address string, api_key string, id int, params map[string]string) ?Response { - data := send_request(http.Method.patch, address, '/api/repos/$id', api_key, - params) ? - - return data -} diff --git a/src/main.v b/src/main.v index 4ba6d30f..41d0d331 100644 --- a/src/main.v +++ b/src/main.v @@ -4,7 +4,8 @@ import os import server import cli import build -import git +import console.git +import console.logs import cron fn main() { @@ -27,6 +28,7 @@ fn main() { build.cmd(), git.cmd(), cron.cmd(), + logs.cmd(), ] } diff --git a/src/server/logs.v b/src/server/logs.v new file mode 100644 index 00000000..b048dc4f --- /dev/null +++ b/src/server/logs.v @@ -0,0 +1,136 @@ +module server + +import web +import net.http +import net.urllib +import response { new_data_response, new_response } +import db +import time +import os +import util + +// get_logs returns all build logs in the database. A 'repo' query param can +// optionally be added to limit the list of build logs to that repository. +['/api/logs'; get] +fn (mut app App) get_logs() web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + logs := if 'repo' in app.query { + app.db.get_build_logs_for_repo(app.query['repo'].int()) + } else { + app.db.get_build_logs() + } + + return app.json(http.Status.ok, new_data_response(logs)) +} + +// get_single_log returns the build log with the given id. +['/api/logs/:id'; get] +fn (mut app App) get_single_log(id int) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + log := app.db.get_build_log(id) or { return app.not_found() } + + return app.json(http.Status.ok, new_data_response(log)) +} + +// get_log_content returns the actual build log file for the given id. +['/api/logs/:id/content'; get] +fn (mut app App) get_log_content(id int) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + log := app.db.get_build_log(id) or { return app.not_found() } + file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.repo_id.str(), log.arch, + file_name) + + return app.file(full_path) +} + +// parse_query_time unescapes an HTTP query parameter & tries to parse it as a +// time.Time struct. +fn parse_query_time(query string) ?time.Time { + unescaped := urllib.query_unescape(query) ? + t := time.parse(unescaped) ? + + return t +} + +// post_log adds a new log to the database. +['/api/logs'; post] +fn (mut app App) post_log() web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + // Parse query params + start_time := parse_query_time(app.query['startTime']) or { + return app.json(http.Status.bad_request, new_response('Invalid or missing start time.')) + } + + end_time := parse_query_time(app.query['endTime']) or { + return app.json(http.Status.bad_request, new_response('Invalid or missing end time.')) + } + + if 'exitCode' !in app.query { + return app.json(http.Status.bad_request, new_response('Missing exit code.')) + } + + exit_code := app.query['exitCode'].int() + + if 'arch' !in app.query { + return app.json(http.Status.bad_request, new_response("Missing parameter 'arch'.")) + } + + arch := app.query['arch'] + + repo_id := app.query['repo'].int() + + if !app.db.git_repo_exists(repo_id) { + return app.json(http.Status.bad_request, new_response('Unknown Git repo.')) + } + + // Store log in db + log := db.BuildLog{ + repo_id: repo_id + start_time: start_time + end_time: end_time + arch: arch + exit_code: exit_code + } + + app.db.add_build_log(log) + + repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo_id.str(), arch) + + // Create the logs directory of it doesn't exist + if !os.exists(repo_logs_dir) { + os.mkdir_all(repo_logs_dir) or { + app.lerror("Couldn't create dir '$repo_logs_dir'.") + + return app.json(http.Status.internal_server_error, new_response('An error occured while processing the request.')) + } + } + + // Stream log contents to correct file + file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + full_path := os.join_path_single(repo_logs_dir, file_name) + + if length := app.req.header.get(.content_length) { + util.reader_to_file(mut app.reader, length.int(), full_path) or { + app.lerror('An error occured while receiving logs: $err.msg()') + + return app.json(http.Status.internal_server_error, new_response('Failed to upload logs.')) + } + } else { + return app.status(http.Status.length_required) + } + + return app.json(http.Status.ok, new_response('Logs added successfully.')) +} diff --git a/src/server/server.v b/src/server/server.v index b2a2ad29..090aa76e 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -12,6 +12,7 @@ const ( log_file_name = 'vieter.log' repo_dir_name = 'repos' db_file_name = 'vieter.sqlite' + logs_dir_name = 'logs' ) struct App { @@ -37,6 +38,14 @@ pub fn server(conf Config) ? { os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') } + logs_dir := os.join_path_single(conf.data_dir, server.logs_dir_name) + + if !os.exists(logs_dir) { + os.mkdir(os.join_path_single(conf.data_dir, server.logs_dir_name)) or { + util.exit_with_message(1, 'Failed to create logs directory.') + } + } + mut logger := log.Log{ level: log_level }