forked from vieter-v/vieter
				
			Merge pull request 'Build logs API & CLI + refactoring' (#169) from Chewing_Bever/vieter:build-logs into dev
Reviewed-on: vieter/vieter#169main
						commit
						7e5f0c5a53
					
				|  | @ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. | ||||||
| The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), | ||||||
| and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). | ||||||
| 
 | 
 | ||||||
|  | ## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev) | ||||||
|  | 
 | ||||||
|  | ### Added | ||||||
|  | 
 | ||||||
|  | * Web API for adding & querying build logs | ||||||
|  | * CLI commands to access build logs API | ||||||
|  | 
 | ||||||
| ## [0.3.0-alpha.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.1) | ## [0.3.0-alpha.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.1) | ||||||
| 
 | 
 | ||||||
| ### Changed | ### Changed | ||||||
|  |  | ||||||
|  | @ -3,9 +3,9 @@ module build | ||||||
| import docker | import docker | ||||||
| import encoding.base64 | import encoding.base64 | ||||||
| import time | import time | ||||||
| import git |  | ||||||
| import os | import os | ||||||
| import db | import db | ||||||
|  | import client | ||||||
| 
 | 
 | ||||||
| const container_build_dir = '/build' | const container_build_dir = '/build' | ||||||
| 
 | 
 | ||||||
|  | @ -126,7 +126,7 @@ fn build(conf Config) ? { | ||||||
| 	build_arch := os.uname().machine | 	build_arch := os.uname().machine | ||||||
| 
 | 
 | ||||||
| 	// We get the repos map from the Vieter instance | 	// We get the repos map from the Vieter instance | ||||||
| 	repos := git.get_repos(conf.address, conf.api_key) ? | 	repos := client.new(conf.address, conf.api_key).get_git_repos() ? | ||||||
| 
 | 
 | ||||||
| 	// We filter out any repos that aren't allowed to be built on this | 	// We filter out any repos that aren't allowed to be built on this | ||||||
| 	// architecture | 	// architecture | ||||||
|  |  | ||||||
|  | @ -0,0 +1,67 @@ | ||||||
|  | module client | ||||||
|  | 
 | ||||||
|  | import net.http { Method } | ||||||
|  | import net.urllib | ||||||
|  | import response { Response } | ||||||
|  | import json | ||||||
|  | 
 | ||||||
|  | pub struct Client { | ||||||
|  | pub: | ||||||
|  | 	address string | ||||||
|  | 	api_key string | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // new creates a new Client instance. | ||||||
|  | pub fn new(address string, api_key string) Client { | ||||||
|  | 	return Client{ | ||||||
|  | 		address: address | ||||||
|  | 		api_key: api_key | ||||||
|  | 	} | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // send_request_raw sends an HTTP request, returning the http.Response object. | ||||||
|  | // It encodes the params so that they're safe to pass as HTTP query parameters. | ||||||
|  | fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) ?http.Response { | ||||||
|  | 	mut full_url := '$c.address$url' | ||||||
|  | 
 | ||||||
|  | 	if params.len > 0 { | ||||||
|  | 		mut params_escaped := map[string]string{} | ||||||
|  | 
 | ||||||
|  | 		// Escape each query param | ||||||
|  | 		for k, v in params { | ||||||
|  | 			params_escaped[k] = urllib.query_escape(v) | ||||||
|  | 		} | ||||||
|  | 
 | ||||||
|  | 		params_str := params_escaped.keys().map('$it=${params[it]}').join('&') | ||||||
|  | 
 | ||||||
|  | 		full_url = '$full_url?$params_str' | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	mut req := http.new_request(method, full_url, body) ? | ||||||
|  | 	req.add_custom_header('X-Api-Key', c.api_key) ? | ||||||
|  | 
 | ||||||
|  | 	res := req.do() ? | ||||||
|  | 
 | ||||||
|  | 	return res | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // send_request<T> just calls send_request_with_body<T> with an empty body. | ||||||
|  | fn (c &Client) send_request<T>(method Method, url string, params map[string]string) ?Response<T> { | ||||||
|  | 	return c.send_request_with_body<T>(method, url, params, '') | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // send_request_with_body<T> calls send_request_raw_response & parses its | ||||||
|  | // output as a Response<T> object. | ||||||
|  | fn (c &Client) send_request_with_body<T>(method Method, url string, params map[string]string, body string) ?Response<T> { | ||||||
|  | 	res_text := c.send_request_raw_response(method, url, params, body) ? | ||||||
|  | 	data := json.decode(Response<T>, res_text) ? | ||||||
|  | 
 | ||||||
|  | 	return data | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // send_request_raw_response returns the raw text response for an HTTP request. | ||||||
|  | fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) ?string { | ||||||
|  | 	res := c.send_request_raw(method, url, params, body) ? | ||||||
|  | 
 | ||||||
|  | 	return res.text | ||||||
|  | } | ||||||
|  | @ -0,0 +1,51 @@ | ||||||
|  | module client | ||||||
|  | 
 | ||||||
|  | import db { GitRepo } | ||||||
|  | import net.http { Method } | ||||||
|  | import response { Response } | ||||||
|  | 
 | ||||||
|  | // get_git_repos returns the current list of repos. | ||||||
|  | pub fn (c &Client) get_git_repos() ?[]GitRepo { | ||||||
|  | 	data := c.send_request<[]GitRepo>(Method.get, '/api/repos', {}) ? | ||||||
|  | 
 | ||||||
|  | 	return data.data | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // get_git_repo returns the repo for a specific ID. | ||||||
|  | pub fn (c &Client) get_git_repo(id int) ?GitRepo { | ||||||
|  | 	data := c.send_request<GitRepo>(Method.get, '/api/repos/$id', {}) ? | ||||||
|  | 
 | ||||||
|  | 	return data.data | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // add_git_repo adds a new repo to the server. | ||||||
|  | pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []string) ?Response<string> { | ||||||
|  | 	mut params := { | ||||||
|  | 		'url':    url | ||||||
|  | 		'branch': branch | ||||||
|  | 		'repo':   repo | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	if arch.len > 0 { | ||||||
|  | 		params['arch'] = arch.join(',') | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	data := c.send_request<string>(Method.post, '/api/repos', params) ? | ||||||
|  | 
 | ||||||
|  | 	return data | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // remove_git_repo removes the repo with the given ID from the server. | ||||||
|  | pub fn (c &Client) remove_git_repo(id int) ?Response<string> { | ||||||
|  | 	data := c.send_request<string>(Method.delete, '/api/repos/$id', {}) ? | ||||||
|  | 
 | ||||||
|  | 	return data | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // patch_git_repo sends a PATCH request to the given repo with the params as | ||||||
|  | // payload. | ||||||
|  | pub fn (c &Client) patch_git_repo(id int, params map[string]string) ?Response<string> { | ||||||
|  | 	data := c.send_request<string>(Method.patch, '/api/repos/$id', params) ? | ||||||
|  | 
 | ||||||
|  | 	return data | ||||||
|  | } | ||||||
|  | @ -0,0 +1,53 @@ | ||||||
|  | module client | ||||||
|  | 
 | ||||||
|  | import db { BuildLog } | ||||||
|  | import net.http { Method } | ||||||
|  | import response { Response } | ||||||
|  | import time | ||||||
|  | 
 | ||||||
|  | // get_build_logs returns all build logs. | ||||||
|  | pub fn (c &Client) get_build_logs() ?Response<[]BuildLog> { | ||||||
|  | 	data := c.send_request<[]BuildLog>(Method.get, '/api/logs', {}) ? | ||||||
|  | 
 | ||||||
|  | 	return data | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // get_build_logs_for_repo returns all build logs for a given repo. | ||||||
|  | pub fn (c &Client) get_build_logs_for_repo(repo_id int) ?Response<[]BuildLog> { | ||||||
|  | 	params := { | ||||||
|  | 		'repo': repo_id.str() | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	data := c.send_request<[]BuildLog>(Method.get, '/api/logs', params) ? | ||||||
|  | 
 | ||||||
|  | 	return data | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // get_build_log returns a specific build log. | ||||||
|  | pub fn (c &Client) get_build_log(id int) ?Response<BuildLog> { | ||||||
|  | 	data := c.send_request<BuildLog>(Method.get, '/api/logs/$id', {}) ? | ||||||
|  | 
 | ||||||
|  | 	return data | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // get_build_log_content returns the contents of the build log file. | ||||||
|  | pub fn (c &Client) get_build_log_content(id int) ?string { | ||||||
|  | 	data := c.send_request_raw_response(Method.get, '/api/logs/$id/content', {}, '') ? | ||||||
|  | 
 | ||||||
|  | 	return data | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // add_build_log adds a new build log to the server. | ||||||
|  | pub fn (c &Client) add_build_log(repo_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response<string> { | ||||||
|  | 	params := { | ||||||
|  | 		'repo':      repo_id.str() | ||||||
|  | 		'startTime': start_time.str() | ||||||
|  | 		'endTime':   end_time.str() | ||||||
|  | 		'arch':      arch | ||||||
|  | 		'exitCode':  exit_code.str() | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	data := c.send_request_with_body<string>(Method.post, '/api/logs', params, content) ? | ||||||
|  | 
 | ||||||
|  | 	return data | ||||||
|  | } | ||||||
|  | @ -0,0 +1 @@ | ||||||
|  | module console | ||||||
|  | @ -3,6 +3,7 @@ module git | ||||||
| import cli | import cli | ||||||
| import env | import env | ||||||
| import cron.expression { parse_expression } | import cron.expression { parse_expression } | ||||||
|  | import client | ||||||
| 
 | 
 | ||||||
| struct Config { | struct Config { | ||||||
| 	address string [required] | 	address string [required] | ||||||
|  | @ -119,7 +120,8 @@ pub fn cmd() cli.Command { | ||||||
| 
 | 
 | ||||||
| // list prints out a list of all repositories. | // list prints out a list of all repositories. | ||||||
| fn list(conf Config) ? { | fn list(conf Config) ? { | ||||||
| 	repos := get_repos(conf.address, conf.api_key) ? | 	c := client.new(conf.address, conf.api_key) | ||||||
|  | 	repos := c.get_git_repos() ? | ||||||
| 
 | 
 | ||||||
| 	for repo in repos { | 	for repo in repos { | ||||||
| 		println('$repo.id\t$repo.url\t$repo.branch\t$repo.repo') | 		println('$repo.id\t$repo.url\t$repo.branch\t$repo.repo') | ||||||
|  | @ -128,7 +130,8 @@ fn list(conf Config) ? { | ||||||
| 
 | 
 | ||||||
| // add adds a new repository to the server's list. | // add adds a new repository to the server's list. | ||||||
| fn add(conf Config, url string, branch string, repo string) ? { | fn add(conf Config, url string, branch string, repo string) ? { | ||||||
| 	res := add_repo(conf.address, conf.api_key, url, branch, repo, []) ? | 	c := client.new(conf.address, conf.api_key) | ||||||
|  | 	res := c.add_git_repo(url, branch, repo, []) ? | ||||||
| 
 | 
 | ||||||
| 	println(res.message) | 	println(res.message) | ||||||
| } | } | ||||||
|  | @ -139,7 +142,8 @@ fn remove(conf Config, id string) ? { | ||||||
| 	id_int := id.int() | 	id_int := id.int() | ||||||
| 
 | 
 | ||||||
| 	if id_int != 0 { | 	if id_int != 0 { | ||||||
| 		res := remove_repo(conf.address, conf.api_key, id_int) ? | 		c := client.new(conf.address, conf.api_key) | ||||||
|  | 		res := c.remove_git_repo(id_int) ? | ||||||
| 		println(res.message) | 		println(res.message) | ||||||
| 	} | 	} | ||||||
| } | } | ||||||
|  | @ -156,7 +160,8 @@ fn patch(conf Config, id string, params map[string]string) ? { | ||||||
| 
 | 
 | ||||||
| 	id_int := id.int() | 	id_int := id.int() | ||||||
| 	if id_int != 0 { | 	if id_int != 0 { | ||||||
| 		res := patch_repo(conf.address, conf.api_key, id_int, params) ? | 		c := client.new(conf.address, conf.api_key) | ||||||
|  | 		res := c.patch_git_repo(id_int, params) ? | ||||||
| 
 | 
 | ||||||
| 		println(res.message) | 		println(res.message) | ||||||
| 	} | 	} | ||||||
|  | @ -170,6 +175,7 @@ fn info(conf Config, id string) ? { | ||||||
| 		return | 		return | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	repo := get_repo(conf.address, conf.api_key, id_int) ? | 	c := client.new(conf.address, conf.api_key) | ||||||
|  | 	repo := c.get_git_repo(id_int) ? | ||||||
| 	println(repo) | 	println(repo) | ||||||
| } | } | ||||||
|  | @ -0,0 +1,106 @@ | ||||||
|  | module logs | ||||||
|  | 
 | ||||||
|  | import cli | ||||||
|  | import env | ||||||
|  | import client | ||||||
|  | import db | ||||||
|  | 
 | ||||||
|  | struct Config { | ||||||
|  | 	address string [required] | ||||||
|  | 	api_key string [required] | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // cmd returns the cli module that handles the build repos API. | ||||||
|  | pub fn cmd() cli.Command { | ||||||
|  | 	return cli.Command{ | ||||||
|  | 		name: 'logs' | ||||||
|  | 		description: 'Interact with the build logs API.' | ||||||
|  | 		commands: [ | ||||||
|  | 			cli.Command{ | ||||||
|  | 				name: 'list' | ||||||
|  | 				description: 'List the build logs. If a repo ID is provided, only list the build logs for that repo.' | ||||||
|  | 				flags: [ | ||||||
|  | 					cli.Flag{ | ||||||
|  | 						name: 'repo' | ||||||
|  | 						description: 'ID of the Git repo to restrict list to.' | ||||||
|  | 						flag: cli.FlagType.int | ||||||
|  | 					}, | ||||||
|  | 				] | ||||||
|  | 				execute: fn (cmd cli.Command) ? { | ||||||
|  | 					config_file := cmd.flags.get_string('config-file') ? | ||||||
|  | 					conf := env.load<Config>(config_file) ? | ||||||
|  | 
 | ||||||
|  | 					repo_id := cmd.flags.get_int('repo') ? | ||||||
|  | 
 | ||||||
|  | 					if repo_id == 0 { list(conf) ? } else { list_for_repo(conf, repo_id) ? } | ||||||
|  | 				} | ||||||
|  | 			}, | ||||||
|  | 			cli.Command{ | ||||||
|  | 				name: 'info' | ||||||
|  | 				required_args: 1 | ||||||
|  | 				usage: 'id' | ||||||
|  | 				description: 'Show all info for a specific build log.' | ||||||
|  | 				execute: fn (cmd cli.Command) ? { | ||||||
|  | 					config_file := cmd.flags.get_string('config-file') ? | ||||||
|  | 					conf := env.load<Config>(config_file) ? | ||||||
|  | 
 | ||||||
|  | 					id := cmd.args[0].int() | ||||||
|  | 					info(conf, id) ? | ||||||
|  | 				} | ||||||
|  | 			}, | ||||||
|  | 			cli.Command{ | ||||||
|  | 				name: 'content' | ||||||
|  | 				required_args: 1 | ||||||
|  | 				usage: 'id' | ||||||
|  | 				description: 'Output the content of a build log to stdout.' | ||||||
|  | 				execute: fn (cmd cli.Command) ? { | ||||||
|  | 					config_file := cmd.flags.get_string('config-file') ? | ||||||
|  | 					conf := env.load<Config>(config_file) ? | ||||||
|  | 
 | ||||||
|  | 					id := cmd.args[0].int() | ||||||
|  | 					content(conf, id) ? | ||||||
|  | 				} | ||||||
|  | 			}, | ||||||
|  | 		] | ||||||
|  | 	} | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // print_log_list prints a list of logs. | ||||||
|  | fn print_log_list(logs []db.BuildLog) { | ||||||
|  | 	for log in logs { | ||||||
|  | 		println('$log.id\t$log.start_time\t$log.exit_code') | ||||||
|  | 	} | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // list prints a list of all build logs. | ||||||
|  | fn list(conf Config) ? { | ||||||
|  | 	c := client.new(conf.address, conf.api_key) | ||||||
|  | 	logs := c.get_build_logs() ?.data | ||||||
|  | 
 | ||||||
|  | 	print_log_list(logs) | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // list prints a list of all build logs for a given repo. | ||||||
|  | fn list_for_repo(conf Config, repo_id int) ? { | ||||||
|  | 	c := client.new(conf.address, conf.api_key) | ||||||
|  | 	logs := c.get_build_logs_for_repo(repo_id) ?.data | ||||||
|  | 
 | ||||||
|  | 	print_log_list(logs) | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // info print the detailed info for a given build log. | ||||||
|  | fn info(conf Config, id int) ? { | ||||||
|  | 	c := client.new(conf.address, conf.api_key) | ||||||
|  | 	log := c.get_build_log(id) ?.data | ||||||
|  | 
 | ||||||
|  | 	print(log) | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // content outputs the contents of the log file for a given build log to | ||||||
|  | // stdout. | ||||||
|  | fn content(conf Config, id int) ? { | ||||||
|  | 	c := client.new(conf.address, conf.api_key) | ||||||
|  | 	content := c.get_build_log_content(id) ? | ||||||
|  | 
 | ||||||
|  | 	println(content) | ||||||
|  | } | ||||||
|  | @ -77,7 +77,7 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { | ||||||
| 	// 0 means success, 1 means failure | 	// 0 means success, 1 means failure | ||||||
| 	mut status := 0 | 	mut status := 0 | ||||||
| 
 | 
 | ||||||
| 	build.build_repo(d.address, d.api_key, d.builder_images.last(), &sb.repo) or { | 	build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(), &sb.repo) or { | ||||||
| 		d.ldebug('build_repo error: $err.msg()') | 		d.ldebug('build_repo error: $err.msg()') | ||||||
| 		status = 1 | 		status = 1 | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
|  | @ -1,6 +1,5 @@ | ||||||
| module daemon | module daemon | ||||||
| 
 | 
 | ||||||
| import git |  | ||||||
| import time | import time | ||||||
| import log | import log | ||||||
| import datatypes { MinHeap } | import datatypes { MinHeap } | ||||||
|  | @ -10,6 +9,7 @@ import build | ||||||
| import docker | import docker | ||||||
| import db | import db | ||||||
| import os | import os | ||||||
|  | import client | ||||||
| 
 | 
 | ||||||
| const ( | const ( | ||||||
| 	// How many seconds to wait before retrying to update API if failed | 	// How many seconds to wait before retrying to update API if failed | ||||||
|  | @ -31,8 +31,7 @@ fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool { | ||||||
| 
 | 
 | ||||||
| pub struct Daemon { | pub struct Daemon { | ||||||
| mut: | mut: | ||||||
| 	address                 string | 	client                  client.Client | ||||||
| 	api_key                 string |  | ||||||
| 	base_image              string | 	base_image              string | ||||||
| 	builder_images          []string | 	builder_images          []string | ||||||
| 	global_schedule         CronExpression | 	global_schedule         CronExpression | ||||||
|  | @ -56,8 +55,7 @@ mut: | ||||||
| // populates the build queue for the first time. | // populates the build queue for the first time. | ||||||
| pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon { | pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon { | ||||||
| 	mut d := Daemon{ | 	mut d := Daemon{ | ||||||
| 		address: address | 		client: client.new(address, api_key) | ||||||
| 		api_key: api_key |  | ||||||
| 		base_image: base_image | 		base_image: base_image | ||||||
| 		global_schedule: global_schedule | 		global_schedule: global_schedule | ||||||
| 		api_update_frequency: api_update_frequency | 		api_update_frequency: api_update_frequency | ||||||
|  | @ -180,7 +178,7 @@ fn (mut d Daemon) schedule_build(repo db.GitRepo) { | ||||||
| fn (mut d Daemon) renew_repos() { | fn (mut d Daemon) renew_repos() { | ||||||
| 	d.linfo('Renewing repos...') | 	d.linfo('Renewing repos...') | ||||||
| 
 | 
 | ||||||
| 	mut new_repos := git.get_repos(d.address, d.api_key) or { | 	mut new_repos := d.client.get_git_repos() or { | ||||||
| 		d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...') | 		d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...') | ||||||
| 		d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) | 		d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
|  | @ -12,6 +12,7 @@ pub fn init(db_path string) ?VieterDb { | ||||||
| 
 | 
 | ||||||
| 	sql conn { | 	sql conn { | ||||||
| 		create table GitRepo | 		create table GitRepo | ||||||
|  | 		create table BuildLog | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	return VieterDb{ | 	return VieterDb{ | ||||||
|  |  | ||||||
							
								
								
									
										10
									
								
								src/db/git.v
								
								
								
								
							
							
						
						
									
										10
									
								
								src/db/git.v
								
								
								
								
							|  | @ -94,7 +94,7 @@ pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo { | ||||||
| 	// If a select statement fails, it returns a zeroed object. By | 	// If a select statement fails, it returns a zeroed object. By | ||||||
| 	// checking one of the required fields, we can see whether the query | 	// checking one of the required fields, we can see whether the query | ||||||
| 	// returned a result or not. | 	// returned a result or not. | ||||||
| 	if res.url == '' { | 	if res.id == 0 { | ||||||
| 		return none | 		return none | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
|  | @ -152,3 +152,11 @@ pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { | ||||||
| 		} | 		} | ||||||
| 	} | 	} | ||||||
| } | } | ||||||
|  | 
 | ||||||
|  | // git_repo_exists is a utility function that checks whether a repo with the | ||||||
|  | // given id exists. | ||||||
|  | pub fn (db &VieterDb) git_repo_exists(repo_id int) bool { | ||||||
|  | 	db.get_git_repo(repo_id) or { return false } | ||||||
|  | 
 | ||||||
|  | 	return true | ||||||
|  | } | ||||||
|  |  | ||||||
|  | @ -0,0 +1,74 @@ | ||||||
|  | module db | ||||||
|  | 
 | ||||||
|  | import time | ||||||
|  | 
 | ||||||
|  | pub struct BuildLog { | ||||||
|  | pub: | ||||||
|  | 	id         int       [primary; sql: serial] | ||||||
|  | 	repo_id    int       [nonull] | ||||||
|  | 	start_time time.Time [nonull] | ||||||
|  | 	end_time   time.Time [nonull] | ||||||
|  | 	arch       string    [nonull] | ||||||
|  | 	exit_code  int       [nonull] | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // str returns a string representation. | ||||||
|  | pub fn (bl &BuildLog) str() string { | ||||||
|  | 	mut parts := [ | ||||||
|  | 		'id: $bl.id', | ||||||
|  | 		'repo id: $bl.repo_id', | ||||||
|  | 		'start time: $bl.start_time', | ||||||
|  | 		'end time: $bl.end_time', | ||||||
|  | 		'arch: $bl.arch', | ||||||
|  | 		'exit code: $bl.exit_code', | ||||||
|  | 	] | ||||||
|  | 	str := parts.join('\n') | ||||||
|  | 
 | ||||||
|  | 	return str | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // get_build_logs returns all BuildLog's in the database. | ||||||
|  | pub fn (db &VieterDb) get_build_logs() []BuildLog { | ||||||
|  | 	res := sql db.conn { | ||||||
|  | 		select from BuildLog order by id | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	return res | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // get_build_logs_for_repo returns all BuildLog's in the database for a given | ||||||
|  | // repo. | ||||||
|  | pub fn (db &VieterDb) get_build_logs_for_repo(repo_id int) []BuildLog { | ||||||
|  | 	res := sql db.conn { | ||||||
|  | 		select from BuildLog where repo_id == repo_id order by id | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	return res | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // get_build_log tries to return a specific BuildLog. | ||||||
|  | pub fn (db &VieterDb) get_build_log(id int) ?BuildLog { | ||||||
|  | 	res := sql db.conn { | ||||||
|  | 		select from BuildLog where id == id | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	if res.id == 0 { | ||||||
|  | 		return none | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	return res | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // add_build_log inserts the given BuildLog into the database. | ||||||
|  | pub fn (db &VieterDb) add_build_log(log BuildLog) { | ||||||
|  | 	sql db.conn { | ||||||
|  | 		insert log into BuildLog | ||||||
|  | 	} | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // delete_build_log delete the BuildLog with the given ID from the database. | ||||||
|  | pub fn (db &VieterDb) delete_build_log(id int) { | ||||||
|  | 	sql db.conn { | ||||||
|  | 		delete from BuildLog where id == id | ||||||
|  | 	} | ||||||
|  | } | ||||||
|  | @ -1,77 +0,0 @@ | ||||||
| module git |  | ||||||
| 
 |  | ||||||
| import json |  | ||||||
| import response { Response } |  | ||||||
| import net.http |  | ||||||
| import db |  | ||||||
| 
 |  | ||||||
| // send_request<T> is a convenience method for sending requests to the repos |  | ||||||
| // API. It mostly does string manipulation to create a query string containing |  | ||||||
| // the provided params. |  | ||||||
| fn send_request<T>(method http.Method, address string, url string, api_key string, params map[string]string) ?Response<T> { |  | ||||||
| 	mut full_url := '$address$url' |  | ||||||
| 
 |  | ||||||
| 	if params.len > 0 { |  | ||||||
| 		params_str := params.keys().map('$it=${params[it]}').join('&') |  | ||||||
| 
 |  | ||||||
| 		full_url = '$full_url?$params_str' |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	mut req := http.new_request(method, full_url, '') ? |  | ||||||
| 	req.add_custom_header('X-API-Key', api_key) ? |  | ||||||
| 
 |  | ||||||
| 	res := req.do() ? |  | ||||||
| 	data := json.decode(Response<T>, res.text) ? |  | ||||||
| 
 |  | ||||||
| 	return data |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| // get_repos returns the current list of repos. |  | ||||||
| pub fn get_repos(address string, api_key string) ?[]db.GitRepo { |  | ||||||
| 	data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', api_key, |  | ||||||
| 		{}) ? |  | ||||||
| 
 |  | ||||||
| 	return data.data |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| // get_repo returns the repo for a specific ID. |  | ||||||
| pub fn get_repo(address string, api_key string, id int) ?db.GitRepo { |  | ||||||
| 	data := send_request<db.GitRepo>(http.Method.get, address, '/api/repos/$id', api_key, |  | ||||||
| 		{}) ? |  | ||||||
| 
 |  | ||||||
| 	return data.data |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| // add_repo adds a new repo to the server. |  | ||||||
| pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response<string> { |  | ||||||
| 	mut params := { |  | ||||||
| 		'url':    url |  | ||||||
| 		'branch': branch |  | ||||||
| 		'repo':   repo |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	if arch.len > 0 { |  | ||||||
| 		params['arch'] = arch.join(',') |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	data := send_request<string>(http.Method.post, address, '/api/repos', api_key, params) ? |  | ||||||
| 
 |  | ||||||
| 	return data |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| // remove_repo removes the repo with the given ID from the server. |  | ||||||
| pub fn remove_repo(address string, api_key string, id int) ?Response<string> { |  | ||||||
| 	data := send_request<string>(http.Method.delete, address, '/api/repos/$id', api_key, |  | ||||||
| 		{}) ? |  | ||||||
| 
 |  | ||||||
| 	return data |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| // patch_repo sends a PATCH request to the given repo with the params as |  | ||||||
| // payload. |  | ||||||
| pub fn patch_repo(address string, api_key string, id int, params map[string]string) ?Response<string> { |  | ||||||
| 	data := send_request<string>(http.Method.patch, address, '/api/repos/$id', api_key, |  | ||||||
| 		params) ? |  | ||||||
| 
 |  | ||||||
| 	return data |  | ||||||
| } |  | ||||||
|  | @ -4,7 +4,8 @@ import os | ||||||
| import server | import server | ||||||
| import cli | import cli | ||||||
| import build | import build | ||||||
| import git | import console.git | ||||||
|  | import console.logs | ||||||
| import cron | import cron | ||||||
| 
 | 
 | ||||||
| fn main() { | fn main() { | ||||||
|  | @ -27,6 +28,7 @@ fn main() { | ||||||
| 			build.cmd(), | 			build.cmd(), | ||||||
| 			git.cmd(), | 			git.cmd(), | ||||||
| 			cron.cmd(), | 			cron.cmd(), | ||||||
|  | 			logs.cmd(), | ||||||
| 		] | 		] | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
|  | @ -0,0 +1,136 @@ | ||||||
|  | module server | ||||||
|  | 
 | ||||||
|  | import web | ||||||
|  | import net.http | ||||||
|  | import net.urllib | ||||||
|  | import response { new_data_response, new_response } | ||||||
|  | import db | ||||||
|  | import time | ||||||
|  | import os | ||||||
|  | import util | ||||||
|  | 
 | ||||||
|  | // get_logs returns all build logs in the database. A 'repo' query param can | ||||||
|  | // optionally be added to limit the list of build logs to that repository. | ||||||
|  | ['/api/logs'; get] | ||||||
|  | fn (mut app App) get_logs() web.Result { | ||||||
|  | 	if !app.is_authorized() { | ||||||
|  | 		return app.json(http.Status.unauthorized, new_response('Unauthorized.')) | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	logs := if 'repo' in app.query { | ||||||
|  | 		app.db.get_build_logs_for_repo(app.query['repo'].int()) | ||||||
|  | 	} else { | ||||||
|  | 		app.db.get_build_logs() | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	return app.json(http.Status.ok, new_data_response(logs)) | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // get_single_log returns the build log with the given id. | ||||||
|  | ['/api/logs/:id'; get] | ||||||
|  | fn (mut app App) get_single_log(id int) web.Result { | ||||||
|  | 	if !app.is_authorized() { | ||||||
|  | 		return app.json(http.Status.unauthorized, new_response('Unauthorized.')) | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	log := app.db.get_build_log(id) or { return app.not_found() } | ||||||
|  | 
 | ||||||
|  | 	return app.json(http.Status.ok, new_data_response(log)) | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // get_log_content returns the actual build log file for the given id. | ||||||
|  | ['/api/logs/:id/content'; get] | ||||||
|  | fn (mut app App) get_log_content(id int) web.Result { | ||||||
|  | 	if !app.is_authorized() { | ||||||
|  | 		return app.json(http.Status.unauthorized, new_response('Unauthorized.')) | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	log := app.db.get_build_log(id) or { return app.not_found() } | ||||||
|  | 	file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') | ||||||
|  | 	full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.repo_id.str(), log.arch, | ||||||
|  | 		file_name) | ||||||
|  | 
 | ||||||
|  | 	return app.file(full_path) | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // parse_query_time unescapes an HTTP query parameter & tries to parse it as a | ||||||
|  | // time.Time struct. | ||||||
|  | fn parse_query_time(query string) ?time.Time { | ||||||
|  | 	unescaped := urllib.query_unescape(query) ? | ||||||
|  | 	t := time.parse(unescaped) ? | ||||||
|  | 
 | ||||||
|  | 	return t | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // post_log adds a new log to the database. | ||||||
|  | ['/api/logs'; post] | ||||||
|  | fn (mut app App) post_log() web.Result { | ||||||
|  | 	if !app.is_authorized() { | ||||||
|  | 		return app.json(http.Status.unauthorized, new_response('Unauthorized.')) | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	// Parse query params | ||||||
|  | 	start_time := parse_query_time(app.query['startTime']) or { | ||||||
|  | 		return app.json(http.Status.bad_request, new_response('Invalid or missing start time.')) | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	end_time := parse_query_time(app.query['endTime']) or { | ||||||
|  | 		return app.json(http.Status.bad_request, new_response('Invalid or missing end time.')) | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	if 'exitCode' !in app.query { | ||||||
|  | 		return app.json(http.Status.bad_request, new_response('Missing exit code.')) | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	exit_code := app.query['exitCode'].int() | ||||||
|  | 
 | ||||||
|  | 	if 'arch' !in app.query { | ||||||
|  | 		return app.json(http.Status.bad_request, new_response("Missing parameter 'arch'.")) | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	arch := app.query['arch'] | ||||||
|  | 
 | ||||||
|  | 	repo_id := app.query['repo'].int() | ||||||
|  | 
 | ||||||
|  | 	if !app.db.git_repo_exists(repo_id) { | ||||||
|  | 		return app.json(http.Status.bad_request, new_response('Unknown Git repo.')) | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	// Store log in db | ||||||
|  | 	log := db.BuildLog{ | ||||||
|  | 		repo_id: repo_id | ||||||
|  | 		start_time: start_time | ||||||
|  | 		end_time: end_time | ||||||
|  | 		arch: arch | ||||||
|  | 		exit_code: exit_code | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	app.db.add_build_log(log) | ||||||
|  | 
 | ||||||
|  | 	repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo_id.str(), arch) | ||||||
|  | 
 | ||||||
|  | 	// Create the logs directory of it doesn't exist | ||||||
|  | 	if !os.exists(repo_logs_dir) { | ||||||
|  | 		os.mkdir_all(repo_logs_dir) or { | ||||||
|  | 			app.lerror("Couldn't create dir '$repo_logs_dir'.") | ||||||
|  | 
 | ||||||
|  | 			return app.json(http.Status.internal_server_error, new_response('An error occured while processing the request.')) | ||||||
|  | 		} | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	// Stream log contents to correct file | ||||||
|  | 	file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss') | ||||||
|  | 	full_path := os.join_path_single(repo_logs_dir, file_name) | ||||||
|  | 
 | ||||||
|  | 	if length := app.req.header.get(.content_length) { | ||||||
|  | 		util.reader_to_file(mut app.reader, length.int(), full_path) or { | ||||||
|  | 			app.lerror('An error occured while receiving logs: $err.msg()') | ||||||
|  | 
 | ||||||
|  | 			return app.json(http.Status.internal_server_error, new_response('Failed to upload logs.')) | ||||||
|  | 		} | ||||||
|  | 	} else { | ||||||
|  | 		return app.status(http.Status.length_required) | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	return app.json(http.Status.ok, new_response('Logs added successfully.')) | ||||||
|  | } | ||||||
|  | @ -12,6 +12,7 @@ const ( | ||||||
| 	log_file_name = 'vieter.log' | 	log_file_name = 'vieter.log' | ||||||
| 	repo_dir_name = 'repos' | 	repo_dir_name = 'repos' | ||||||
| 	db_file_name  = 'vieter.sqlite' | 	db_file_name  = 'vieter.sqlite' | ||||||
|  | 	logs_dir_name = 'logs' | ||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
| struct App { | struct App { | ||||||
|  | @ -37,6 +38,14 @@ pub fn server(conf Config) ? { | ||||||
| 
 | 
 | ||||||
| 	os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') } | 	os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') } | ||||||
| 
 | 
 | ||||||
|  | 	logs_dir := os.join_path_single(conf.data_dir, server.logs_dir_name) | ||||||
|  | 
 | ||||||
|  | 	if !os.exists(logs_dir) { | ||||||
|  | 		os.mkdir(os.join_path_single(conf.data_dir, server.logs_dir_name)) or { | ||||||
|  | 			util.exit_with_message(1, 'Failed to create logs directory.') | ||||||
|  | 		} | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
| 	mut logger := log.Log{ | 	mut logger := log.Log{ | ||||||
| 		level: log_level | 		level: log_level | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
		Loading…
	
		Reference in New Issue