From 44696fc11b200aa903641f852282f32a96de117b Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 15 Jun 2022 13:20:29 +0200 Subject: [PATCH 01/62] refactor: migrated env code to own external module --- README.md | 4 ++ src/console/git/git.v | 14 +++--- src/console/logs/logs.v | 8 ++-- src/cron/cli.v | 4 +- src/env/README.md | 7 --- src/env/env.v | 102 ---------------------------------------- src/server/cli.v | 4 +- src/v.mod | 5 ++ 8 files changed, 24 insertions(+), 124 deletions(-) delete mode 100644 src/env/README.md delete mode 100644 src/env/env.v diff --git a/README.md b/README.md index 892396a2..31f262e1 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,10 @@ Besides a V installer, Vieter also requires the following libraries to work: * openssl * sqlite3 +Vieter also depends on some external V modules which you can install using `cd +src && v install`. Make sure to keep these dependencies up to date using `v +update`. + ### Compiler Vieter compiles with the standard Vlang compiler. However, I do maintain a diff --git a/src/console/git/git.v b/src/console/git/git.v index e27c1c4e..303909e4 100644 --- a/src/console/git/git.v +++ b/src/console/git/git.v @@ -1,7 +1,7 @@ module git import cli -import env +import vieter.vconf import cron.expression { parse_expression } import client import console @@ -41,7 +41,7 @@ pub fn cmd() cli.Command { ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := env.load(config_file)? + conf := vconf.load(default_path: config_file)? mut filter := GitRepoFilter{} @@ -70,7 +70,7 @@ pub fn cmd() cli.Command { description: 'Add a new repository.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := env.load(config_file)? + conf := vconf.load(default_path: config_file)? add(conf, cmd.args[0], cmd.args[1], cmd.args[2])? } @@ -82,7 +82,7 @@ pub fn cmd() cli.Command { description: 'Remove a repository that matches the given ID prefix.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := env.load(config_file)? + conf := vconf.load(default_path: config_file)? remove(conf, cmd.args[0])? } @@ -94,7 +94,7 @@ pub fn cmd() cli.Command { description: 'Show detailed information for the repo matching the ID prefix.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := env.load(config_file)? + conf := vconf.load(default_path: config_file)? info(conf, cmd.args[0])? } @@ -133,7 +133,7 @@ pub fn cmd() cli.Command { ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := env.load(config_file)? + conf := vconf.load(default_path: config_file)? found := cmd.flags.get_all_found() @@ -155,7 +155,7 @@ pub fn cmd() cli.Command { description: 'Build the repo with the given id & publish it.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := env.load(config_file)? + conf := vconf.load(default_path: config_file)? build(conf, cmd.args[0].int())? } diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index cb6997fc..d63e284a 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -1,7 +1,7 @@ module logs import cli -import env +import vieter.vconf import client import console import time @@ -65,7 +65,7 @@ pub fn cmd() cli.Command { ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := env.load(config_file)? + conf := vconf.load(default_path: config_file)? mut filter := BuildLogFilter{} @@ -143,7 +143,7 @@ pub fn cmd() cli.Command { description: 'Show all info for a specific build log.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := env.load(config_file)? + conf := vconf.load(default_path: config_file)? id := cmd.args[0].int() info(conf, id)? @@ -156,7 +156,7 @@ pub fn cmd() cli.Command { description: 'Output the content of a build log to stdout.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := env.load(config_file)? + conf := vconf.load(default_path: config_file)? id := cmd.args[0].int() content(conf, id)? diff --git a/src/cron/cli.v b/src/cron/cli.v index 9703c663..10a4bdd9 100644 --- a/src/cron/cli.v +++ b/src/cron/cli.v @@ -1,7 +1,7 @@ module cron import cli -import env +import vieter.vconf struct Config { pub: @@ -24,7 +24,7 @@ pub fn cmd() cli.Command { description: 'Start the cron service that periodically runs builds.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := env.load(config_file)? + conf := vconf.load(default_path: config_file)? cron(conf)? } diff --git a/src/env/README.md b/src/env/README.md deleted file mode 100644 index 135e8fa9..00000000 --- a/src/env/README.md +++ /dev/null @@ -1,7 +0,0 @@ -This module provides a framework for parsing a configuration, defined as a -struct, from both a TOML configuration file & environment variables. Some -notable features are: - -* Overwrite values in config file using environment variables -* Allow default values in config struct -* Read environment variable value from file diff --git a/src/env/env.v b/src/env/env.v deleted file mode 100644 index 5ed19552..00000000 --- a/src/env/env.v +++ /dev/null @@ -1,102 +0,0 @@ -module env - -import os -import toml - -const ( - // The prefix that every environment variable should have - prefix = 'VIETER_' - // The suffix an environment variable in order for it to be loaded from a file - // instead - file_suffix = '_FILE' -) - -// get_env_var tries to read the contents of the given environment variable. It -// looks for either `${env.prefix}${field_name.to_upper()}` or -// `${env.prefix}${field_name.to_upper()}${env.file_suffix}`, returning the -// contents of the file instead if the latter. If both or neither exist, the -// function returns an error. -fn get_env_var(field_name string) ?string { - env_var_name := '$env.prefix$field_name.to_upper()' - env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix' - env_var := os.getenv(env_var_name) - env_file := os.getenv(env_file_name) - - // If both are missing, we return an empty string - if env_var == '' && env_file == '' { - return '' - } - - // If they're both set, we report a conflict - if env_var != '' && env_file != '' { - return error('Only one of $env_var_name or $env_file_name can be defined.') - } - - // If it's the env var itself, we return it. - // I'm pretty sure this also prevents variable ending in _FILE (e.g. - // VIETER_LOG_FILE) from being mistakingely read as an _FILE suffixed env - // var. - if env_var != '' { - return env_var - } - - // Otherwise, we process the file - return os.read_file(env_file) or { - error('Failed to read file defined in $env_file_name: ${err.msg()}.') - } -} - -// load attempts to create an object of type T from the given path to a toml -// file & environment variables. For each field, it will select either a value -// given from an environment variable, a value defined in the config file or a -// configured default if present, in that order. -pub fn load(path string) ?T { - mut res := T{} - - if os.exists(path) { - // We don't use reflect here because reflect also sets any fields not - // in the toml back to their zero value, which we don't want - doc := toml.parse_file(path)? - - $for field in T.fields { - s := doc.value(field.name) - - if s !is toml.Null { - $if field.typ is string { - res.$(field.name) = s.string() - } $else $if field.typ is int { - res.$(field.name) = s.int() - } - } - } - } - - $for field in T.fields { - env_value := get_env_var(field.name)? - - // The value of an env var will always take precedence over the toml - // file. - if env_value != '' { - $if field.typ is string { - res.$(field.name) = env_value - } $else $if field.typ is int { - res.$(field.name) = env_value.int() - } - } - - // Now, we check whether a value is present. If there isn't, that means - // it isn't in the config file, nor is there a default or an env var. - mut has_value := false - - $if field.typ is string { - has_value = res.$(field.name) != '' - } $else $if field.typ is int { - has_value = res.$(field.name) != 0 - } - - if !has_value { - return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.") - } - } - return res -} diff --git a/src/server/cli.v b/src/server/cli.v index 556efcfe..cf0306cb 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -1,7 +1,7 @@ module server import cli -import env +import vieter.vconf struct Config { pub: @@ -19,7 +19,7 @@ pub fn cmd() cli.Command { description: 'Start the Vieter server.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := env.load(config_file)? + conf := vconf.load(default_path: config_file)? server(conf)? } diff --git a/src/v.mod b/src/v.mod index e69de29b..7f459174 100644 --- a/src/v.mod +++ b/src/v.mod @@ -0,0 +1,5 @@ +Module { + dependencies: [ + 'https://git.rustybever.be/vieter/vconf' + ] +} From 592241c743d4f7f7c9ec065f422a3ccf5e2068d4 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 15 Jun 2022 17:47:12 +0200 Subject: [PATCH 02/62] chore: updated CI config & PKGBUILDs to new module split --- .woodpecker/.build.yml | 13 +++++++++++-- PKGBUILD | 8 ++++++++ PKGBUILD.dev | 8 ++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index b0fd267e..d073b6c9 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -6,10 +6,19 @@ matrix: platform: ${PLATFORM} pipeline: - debug: + install-modules: image: 'chewingbever/vlang:latest' pull: true commands: + - export VMODULES=$PWD/.vmodules + - 'cd src && v install' + when: + event: [pull_request] + + debug: + image: 'chewingbever/vlang:latest' + commands: + - export VMODULES=$PWD/.vmodules - make when: event: [pull_request] @@ -18,10 +27,10 @@ pipeline: prod: image: 'chewingbever/vlang:latest' - pull: true environment: - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static commands: + - export VMODULES=$PWD/.vmodules # Apparently this -D is *very* important - CFLAGS='-DGC_THREADS=1' make prod # Make sure the binary is actually statically built diff --git a/PKGBUILD b/PKGBUILD index 639ce95c..33faec55 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -14,7 +14,15 @@ license=('AGPL3') source=("$pkgname::git+https://git.rustybever.be/vieter/vieter#tag=${pkgver//_/-}") md5sums=('SKIP') +prepare() { + export VMODULES="$srcdir/.vmodules" + + cd "$pkgname/src" && v install +} + build() { + export VMODULES="$srcdir/.vmodules" + cd "$pkgname" make prod diff --git a/PKGBUILD.dev b/PKGBUILD.dev index d25abb34..bff1459e 100644 --- a/PKGBUILD.dev +++ b/PKGBUILD.dev @@ -22,7 +22,15 @@ pkgver() { git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g' } +prepare() { + export VMODULES="$srcdir/.vmodules" + + cd "$pkgname/src" && v install +} + build() { + export VMODULES="$srcdir/.vmodules" + cd "$pkgname" make prod From 233dd20345c005401dcd28ebd2ebc29a79608531 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 15 Jun 2022 19:59:48 +0200 Subject: [PATCH 03/62] fix(ci): also install modules when building on dev --- .woodpecker/.build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index d073b6c9..7cb7d535 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -13,7 +13,7 @@ pipeline: - export VMODULES=$PWD/.vmodules - 'cd src && v install' when: - event: [pull_request] + event: [push, pull_request] debug: image: 'chewingbever/vlang:latest' From 339267e6b2a58bd25adf8599d667085cb48850f6 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 15 Jun 2022 22:54:27 +0200 Subject: [PATCH 04/62] fix: added VIETER_ prefix to vconf.load calls --- src/console/git/git.v | 12 ++++++------ src/console/logs/logs.v | 6 +++--- src/cron/cli.v | 2 +- src/server/cli.v | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/console/git/git.v b/src/console/git/git.v index 303909e4..54cfaaa8 100644 --- a/src/console/git/git.v +++ b/src/console/git/git.v @@ -41,7 +41,7 @@ pub fn cmd() cli.Command { ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(default_path: config_file)? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? mut filter := GitRepoFilter{} @@ -70,7 +70,7 @@ pub fn cmd() cli.Command { description: 'Add a new repository.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(default_path: config_file)? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? add(conf, cmd.args[0], cmd.args[1], cmd.args[2])? } @@ -82,7 +82,7 @@ pub fn cmd() cli.Command { description: 'Remove a repository that matches the given ID prefix.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(default_path: config_file)? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? remove(conf, cmd.args[0])? } @@ -94,7 +94,7 @@ pub fn cmd() cli.Command { description: 'Show detailed information for the repo matching the ID prefix.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(default_path: config_file)? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? info(conf, cmd.args[0])? } @@ -133,7 +133,7 @@ pub fn cmd() cli.Command { ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(default_path: config_file)? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? found := cmd.flags.get_all_found() @@ -155,7 +155,7 @@ pub fn cmd() cli.Command { description: 'Build the repo with the given id & publish it.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(default_path: config_file)? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? build(conf, cmd.args[0].int())? } diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index d63e284a..f9dd3636 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -65,7 +65,7 @@ pub fn cmd() cli.Command { ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(default_path: config_file)? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? mut filter := BuildLogFilter{} @@ -143,7 +143,7 @@ pub fn cmd() cli.Command { description: 'Show all info for a specific build log.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(default_path: config_file)? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? id := cmd.args[0].int() info(conf, id)? @@ -156,7 +156,7 @@ pub fn cmd() cli.Command { description: 'Output the content of a build log to stdout.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(default_path: config_file)? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? id := cmd.args[0].int() content(conf, id)? diff --git a/src/cron/cli.v b/src/cron/cli.v index 10a4bdd9..c2e3d39a 100644 --- a/src/cron/cli.v +++ b/src/cron/cli.v @@ -24,7 +24,7 @@ pub fn cmd() cli.Command { description: 'Start the cron service that periodically runs builds.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(default_path: config_file)? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? cron(conf)? } diff --git a/src/server/cli.v b/src/server/cli.v index cf0306cb..53393f70 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -19,7 +19,7 @@ pub fn cmd() cli.Command { description: 'Start the Vieter server.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? - conf := vconf.load(default_path: config_file)? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? server(conf)? } From 6b79f7b5edf3f7d6ebd7b913d75034529fdd83d2 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 14 Jun 2022 12:54:32 +0200 Subject: [PATCH 05/62] feat(server): moved api routes under /v1 namespace --- CHANGELOG.md | 4 ++++ src/client/git.v | 10 +++++----- src/client/logs.v | 10 +++++----- src/server/git.v | 30 +++++++++++++++--------------- src/server/logs.v | 24 ++++++++++++------------ 5 files changed, 41 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99a795a4..9adee075 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev) +### Changed + +* Moved all API routes under `/v1` namespace + ## [0.3.0](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0) Nothing besides bumping the versions. diff --git a/src/client/git.v b/src/client/git.v index 4496c08a..7b5e52e1 100644 --- a/src/client/git.v +++ b/src/client/git.v @@ -7,7 +7,7 @@ import response { Response } // get_git_repos returns a list of GitRepo's, given a filter object. pub fn (c &Client) get_git_repos(filter GitRepoFilter) ?[]GitRepo { params := models.params_from(filter) - data := c.send_request<[]GitRepo>(Method.get, '/api/repos', params)? + data := c.send_request<[]GitRepo>(Method.get, '/api/v1/repos', params)? return data.data } @@ -35,7 +35,7 @@ pub fn (c &Client) get_all_git_repos() ?[]GitRepo { // get_git_repo returns the repo for a specific ID. pub fn (c &Client) get_git_repo(id int) ?GitRepo { - data := c.send_request(Method.get, '/api/repos/$id', {})? + data := c.send_request(Method.get, '/api/v1/repos/$id', {})? return data.data } @@ -52,14 +52,14 @@ pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []s params['arch'] = arch.join(',') } - data := c.send_request(Method.post, '/api/repos', params)? + data := c.send_request(Method.post, '/api/v1/repos', params)? return data } // remove_git_repo removes the repo with the given ID from the server. pub fn (c &Client) remove_git_repo(id int) ?Response { - data := c.send_request(Method.delete, '/api/repos/$id', {})? + data := c.send_request(Method.delete, '/api/v1/repos/$id', {})? return data } @@ -67,7 +67,7 @@ pub fn (c &Client) remove_git_repo(id int) ?Response { // patch_git_repo sends a PATCH request to the given repo with the params as // payload. pub fn (c &Client) patch_git_repo(id int, params map[string]string) ?Response { - data := c.send_request(Method.patch, '/api/repos/$id', params)? + data := c.send_request(Method.patch, '/api/v1/repos/$id', params)? return data } diff --git a/src/client/logs.v b/src/client/logs.v index 739de238..2d78fc15 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -8,7 +8,7 @@ import time // get_build_logs returns all build logs. pub fn (c &Client) get_build_logs(filter BuildLogFilter) ?Response<[]BuildLog> { params := models.params_from(filter) - data := c.send_request<[]BuildLog>(Method.get, '/api/logs', params)? + data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)? return data } @@ -19,21 +19,21 @@ pub fn (c &Client) get_build_logs_for_repo(repo_id int) ?Response<[]BuildLog> { 'repo': repo_id.str() } - data := c.send_request<[]BuildLog>(Method.get, '/api/logs', params)? + data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)? return data } // get_build_log returns a specific build log. pub fn (c &Client) get_build_log(id int) ?Response { - data := c.send_request(Method.get, '/api/logs/$id', {})? + data := c.send_request(Method.get, '/api/v1/logs/$id', {})? return data } // get_build_log_content returns the contents of the build log file. pub fn (c &Client) get_build_log_content(id int) ?string { - data := c.send_request_raw_response(Method.get, '/api/logs/$id/content', {}, '')? + data := c.send_request_raw_response(Method.get, '/api/v1/logs/$id/content', {}, '')? return data } @@ -48,7 +48,7 @@ pub fn (c &Client) add_build_log(repo_id int, start_time time.Time, end_time tim 'exitCode': exit_code.str() } - data := c.send_request_with_body(Method.post, '/api/logs', params, content)? + data := c.send_request_with_body(Method.post, '/api/v1/logs', params, content)? return data } diff --git a/src/server/git.v b/src/server/git.v index c1bc6f34..f3fe010d 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -6,9 +6,9 @@ import response { new_data_response, new_response } import db import models { GitRepo, GitRepoArch, GitRepoFilter } -// get_repos returns the current list of repos. -['/api/repos'; get] -fn (mut app App) get_repos() web.Result { +// v1_get_repos returns the current list of repos. +['/api/v1/repos'; get] +fn (mut app App) v1_get_repos() web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } @@ -21,9 +21,9 @@ fn (mut app App) get_repos() web.Result { return app.json(http.Status.ok, new_data_response(repos)) } -// get_single_repo returns the information for a single repo. -['/api/repos/:id'; get] -fn (mut app App) get_single_repo(id int) web.Result { +// v1_get_single_repo returns the information for a single repo. +['/api/v1/repos/:id'; get] +fn (mut app App) v1_get_single_repo(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } @@ -33,9 +33,9 @@ fn (mut app App) get_single_repo(id int) web.Result { return app.json(http.Status.ok, new_data_response(repo)) } -// post_repo creates a new repo from the provided query string. -['/api/repos'; post] -fn (mut app App) post_repo() web.Result { +// v1_post_repo creates a new repo from the provided query string. +['/api/v1/repos'; post] +fn (mut app App) v1_post_repo() web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } @@ -57,9 +57,9 @@ fn (mut app App) post_repo() web.Result { return app.json(http.Status.ok, new_response('Repo added successfully.')) } -// delete_repo removes a given repo from the server's list. -['/api/repos/:id'; delete] -fn (mut app App) delete_repo(id int) web.Result { +// v1_delete_repo removes a given repo from the server's list. +['/api/v1/repos/:id'; delete] +fn (mut app App) v1_delete_repo(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } @@ -69,9 +69,9 @@ fn (mut app App) delete_repo(id int) web.Result { return app.json(http.Status.ok, new_response('Repo removed successfully.')) } -// patch_repo updates a repo's data with the given query params. -['/api/repos/:id'; patch] -fn (mut app App) patch_repo(id int) web.Result { +// v1_patch_repo updates a repo's data with the given query params. +['/api/v1/repos/:id'; patch] +fn (mut app App) v1_patch_repo(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } diff --git a/src/server/logs.v b/src/server/logs.v index 314e3228..748c3a66 100644 --- a/src/server/logs.v +++ b/src/server/logs.v @@ -10,10 +10,10 @@ import os import util import models { BuildLog, BuildLogFilter } -// get_logs returns all build logs in the database. A 'repo' query param can +// v1_get_logs returns all build logs in the database. A 'repo' query param can // optionally be added to limit the list of build logs to that repository. -['/api/logs'; get] -fn (mut app App) get_logs() web.Result { +['/api/v1/logs'; get] +fn (mut app App) v1_get_logs() web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } @@ -26,9 +26,9 @@ fn (mut app App) get_logs() web.Result { return app.json(http.Status.ok, new_data_response(logs)) } -// get_single_log returns the build log with the given id. -['/api/logs/:id'; get] -fn (mut app App) get_single_log(id int) web.Result { +// v1_get_single_log returns the build log with the given id. +['/api/v1/logs/:id'; get] +fn (mut app App) v1_get_single_log(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } @@ -38,9 +38,9 @@ fn (mut app App) get_single_log(id int) web.Result { return app.json(http.Status.ok, new_data_response(log)) } -// get_log_content returns the actual build log file for the given id. -['/api/logs/:id/content'; get] -fn (mut app App) get_log_content(id int) web.Result { +// v1_get_log_content returns the actual build log file for the given id. +['/api/v1/logs/:id/content'; get] +fn (mut app App) v1_get_log_content(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } @@ -62,9 +62,9 @@ fn parse_query_time(query string) ?time.Time { return t } -// post_log adds a new log to the database. -['/api/logs'; post] -fn (mut app App) post_log() web.Result { +// v1_post_log adds a new log to the database. +['/api/v1/logs'; post] +fn (mut app App) v1_post_log() web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } From 4d581da7bf08fa73510c29aff9e11b1741dcce1d Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 14 Jun 2022 20:31:00 +0200 Subject: [PATCH 06/62] refactor: renamed api routes & client code to 'targets' --- src/client/git.v | 73 ----------------------------- src/client/targets.v | 73 +++++++++++++++++++++++++++++ src/console/git/build.v | 2 +- src/console/git/git.v | 10 ++-- src/cron/daemon/daemon.v | 2 +- src/server/README.md | 6 +++ src/server/{logs.v => api_logs.v} | 0 src/server/{git.v => api_targets.v} | 30 ++++++------ 8 files changed, 101 insertions(+), 95 deletions(-) delete mode 100644 src/client/git.v create mode 100644 src/client/targets.v create mode 100644 src/server/README.md rename src/server/{logs.v => api_logs.v} (100%) rename src/server/{git.v => api_targets.v} (72%) diff --git a/src/client/git.v b/src/client/git.v deleted file mode 100644 index 7b5e52e1..00000000 --- a/src/client/git.v +++ /dev/null @@ -1,73 +0,0 @@ -module client - -import models { GitRepo, GitRepoFilter } -import net.http { Method } -import response { Response } - -// get_git_repos returns a list of GitRepo's, given a filter object. -pub fn (c &Client) get_git_repos(filter GitRepoFilter) ?[]GitRepo { - params := models.params_from(filter) - data := c.send_request<[]GitRepo>(Method.get, '/api/v1/repos', params)? - - return data.data -} - -// get_all_git_repos retrieves *all* GitRepo's from the API using the default -// limit. -pub fn (c &Client) get_all_git_repos() ?[]GitRepo { - mut repos := []GitRepo{} - mut offset := u64(0) - - for { - sub_repos := c.get_git_repos(offset: offset)? - - if sub_repos.len == 0 { - break - } - - repos << sub_repos - - offset += u64(sub_repos.len) - } - - return repos -} - -// get_git_repo returns the repo for a specific ID. -pub fn (c &Client) get_git_repo(id int) ?GitRepo { - data := c.send_request(Method.get, '/api/v1/repos/$id', {})? - - return data.data -} - -// add_git_repo adds a new repo to the server. -pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []string) ?Response { - mut params := { - 'url': url - 'branch': branch - 'repo': repo - } - - if arch.len > 0 { - params['arch'] = arch.join(',') - } - - data := c.send_request(Method.post, '/api/v1/repos', params)? - - return data -} - -// remove_git_repo removes the repo with the given ID from the server. -pub fn (c &Client) remove_git_repo(id int) ?Response { - data := c.send_request(Method.delete, '/api/v1/repos/$id', {})? - - return data -} - -// patch_git_repo sends a PATCH request to the given repo with the params as -// payload. -pub fn (c &Client) patch_git_repo(id int, params map[string]string) ?Response { - data := c.send_request(Method.patch, '/api/v1/repos/$id', params)? - - return data -} diff --git a/src/client/targets.v b/src/client/targets.v new file mode 100644 index 00000000..90ff1a2d --- /dev/null +++ b/src/client/targets.v @@ -0,0 +1,73 @@ +module client + +import models { GitRepo, GitRepoFilter } +import net.http { Method } +import response { Response } + +// get_targets returns a list of GitRepo's, given a filter object. +pub fn (c &Client) get_targets(filter GitRepoFilter) ?[]GitRepo { + params := models.params_from(filter) + data := c.send_request<[]GitRepo>(Method.get, '/api/v1/targets', params)? + + return data.data +} + +// get_all_targets retrieves *all* GitRepo's from the API using the default +// limit. +pub fn (c &Client) get_all_targets() ?[]GitRepo { + mut repos := []GitRepo{} + mut offset := u64(0) + + for { + sub_repos := c.get_targets(offset: offset)? + + if sub_repos.len == 0 { + break + } + + repos << sub_repos + + offset += u64(sub_repos.len) + } + + return repos +} + +// get_target returns the repo for a specific ID. +pub fn (c &Client) get_target(id int) ?GitRepo { + data := c.send_request(Method.get, '/api/v1/targets/$id', {})? + + return data.data +} + +// add_target adds a new repo to the server. +pub fn (c &Client) add_target(url string, branch string, repo string, arch []string) ?Response { + mut params := { + 'url': url + 'branch': branch + 'repo': repo + } + + if arch.len > 0 { + params['arch'] = arch.join(',') + } + + data := c.send_request(Method.post, '/api/v1/targets', params)? + + return data +} + +// remove_target removes the repo with the given ID from the server. +pub fn (c &Client) remove_target(id int) ?Response { + data := c.send_request(Method.delete, '/api/v1/targets/$id', {})? + + return data +} + +// patch_target sends a PATCH request to the given repo with the params as +// payload. +pub fn (c &Client) patch_target(id int, params map[string]string) ?Response { + data := c.send_request(Method.patch, '/api/v1/targets/$id', params)? + + return data +} diff --git a/src/console/git/build.v b/src/console/git/build.v index fac760df..b3c46a18 100644 --- a/src/console/git/build.v +++ b/src/console/git/build.v @@ -8,7 +8,7 @@ import build // build builds every Git repo in the server's list. fn build(conf Config, repo_id int) ? { c := client.new(conf.address, conf.api_key) - repo := c.get_git_repo(repo_id)? + repo := c.get_target(repo_id)? build_arch := os.uname().machine diff --git a/src/console/git/git.v b/src/console/git/git.v index 54cfaaa8..6a4b9a87 100644 --- a/src/console/git/git.v +++ b/src/console/git/git.v @@ -170,7 +170,7 @@ pub fn cmd() cli.Command { // list prints out a list of all repositories. fn list(conf Config, filter GitRepoFilter) ? { c := client.new(conf.address, conf.api_key) - repos := c.get_git_repos(filter)? + repos := c.get_targets(filter)? data := repos.map([it.id.str(), it.url, it.branch, it.repo]) println(console.pretty_table(['id', 'url', 'branch', 'repo'], data)?) @@ -179,7 +179,7 @@ fn list(conf Config, filter GitRepoFilter) ? { // add adds a new repository to the server's list. fn add(conf Config, url string, branch string, repo string) ? { c := client.new(conf.address, conf.api_key) - res := c.add_git_repo(url, branch, repo, [])? + res := c.add_target(url, branch, repo, [])? println(res.message) } @@ -191,7 +191,7 @@ fn remove(conf Config, id string) ? { if id_int != 0 { c := client.new(conf.address, conf.api_key) - res := c.remove_git_repo(id_int)? + res := c.remove_target(id_int)? println(res.message) } } @@ -209,7 +209,7 @@ fn patch(conf Config, id string, params map[string]string) ? { id_int := id.int() if id_int != 0 { c := client.new(conf.address, conf.api_key) - res := c.patch_git_repo(id_int, params)? + res := c.patch_target(id_int, params)? println(res.message) } @@ -224,6 +224,6 @@ fn info(conf Config, id string) ? { } c := client.new(conf.address, conf.api_key) - repo := c.get_git_repo(id_int)? + repo := c.get_target(id_int)? println(repo) } diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index f1206d6d..1f36e740 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -178,7 +178,7 @@ fn (mut d Daemon) schedule_build(repo GitRepo) { fn (mut d Daemon) renew_repos() { d.linfo('Renewing repos...') - mut new_repos := d.client.get_all_git_repos() or { + mut new_repos := d.client.get_all_targets() or { d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...') d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) diff --git a/src/server/README.md b/src/server/README.md new file mode 100644 index 00000000..ded99857 --- /dev/null +++ b/src/server/README.md @@ -0,0 +1,6 @@ +This module contains the Vieter HTTP server, consisting of the repository +implementation & the REST API. + +**NOTE**: vweb defines the priority order of routes by the file names in this +module. Therefore, it's very important that all API routes are defined in files +prefixed with `api_`, as this is before the word `routes` alphabetically. diff --git a/src/server/logs.v b/src/server/api_logs.v similarity index 100% rename from src/server/logs.v rename to src/server/api_logs.v diff --git a/src/server/git.v b/src/server/api_targets.v similarity index 72% rename from src/server/git.v rename to src/server/api_targets.v index f3fe010d..67117b92 100644 --- a/src/server/git.v +++ b/src/server/api_targets.v @@ -6,9 +6,9 @@ import response { new_data_response, new_response } import db import models { GitRepo, GitRepoArch, GitRepoFilter } -// v1_get_repos returns the current list of repos. -['/api/v1/repos'; get] -fn (mut app App) v1_get_repos() web.Result { +// v1_get_targets returns the current list of repos. +['/api/v1/targets'; get] +fn (mut app App) v1_get_targets() web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } @@ -21,9 +21,9 @@ fn (mut app App) v1_get_repos() web.Result { return app.json(http.Status.ok, new_data_response(repos)) } -// v1_get_single_repo returns the information for a single repo. -['/api/v1/repos/:id'; get] -fn (mut app App) v1_get_single_repo(id int) web.Result { +// v1_get_single_target returns the information for a single repo. +['/api/v1/targets/:id'; get] +fn (mut app App) v1_get_single_target(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } @@ -33,9 +33,9 @@ fn (mut app App) v1_get_single_repo(id int) web.Result { return app.json(http.Status.ok, new_data_response(repo)) } -// v1_post_repo creates a new repo from the provided query string. -['/api/v1/repos'; post] -fn (mut app App) v1_post_repo() web.Result { +// v1_post_target creates a new repo from the provided query string. +['/api/v1/targets'; post] +fn (mut app App) v1_post_target() web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } @@ -57,9 +57,9 @@ fn (mut app App) v1_post_repo() web.Result { return app.json(http.Status.ok, new_response('Repo added successfully.')) } -// v1_delete_repo removes a given repo from the server's list. -['/api/v1/repos/:id'; delete] -fn (mut app App) v1_delete_repo(id int) web.Result { +// v1_delete_target removes a given repo from the server's list. +['/api/v1/targets/:id'; delete] +fn (mut app App) v1_delete_target(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } @@ -69,9 +69,9 @@ fn (mut app App) v1_delete_repo(id int) web.Result { return app.json(http.Status.ok, new_response('Repo removed successfully.')) } -// v1_patch_repo updates a repo's data with the given query params. -['/api/v1/repos/:id'; patch] -fn (mut app App) v1_patch_repo(id int) web.Result { +// v1_patch_target updates a repo's data with the given query params. +['/api/v1/targets/:id'; patch] +fn (mut app App) v1_patch_target(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } From faec08f846492718a5fefdb5d0d234bc5928e70c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 14 Jun 2022 20:38:20 +0200 Subject: [PATCH 07/62] refactor(console): renamed stuff to 'targets' --- src/console/{git => targets}/build.v | 4 ++-- src/console/{git/git.v => targets/targets.v} | 20 ++++++++++---------- src/main.v | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) rename src/console/{git => targets}/build.v (90%) rename src/console/{git/git.v => targets/targets.v} (91%) diff --git a/src/console/git/build.v b/src/console/targets/build.v similarity index 90% rename from src/console/git/build.v rename to src/console/targets/build.v index b3c46a18..d50dfd2c 100644 --- a/src/console/git/build.v +++ b/src/console/targets/build.v @@ -1,11 +1,11 @@ -module git +module targets import client import docker import os import build -// build builds every Git repo in the server's list. +// build locally builds the target with the given id. fn build(conf Config, repo_id int) ? { c := client.new(conf.address, conf.api_key) repo := c.get_target(repo_id)? diff --git a/src/console/git/git.v b/src/console/targets/targets.v similarity index 91% rename from src/console/git/git.v rename to src/console/targets/targets.v index 6a4b9a87..9917913c 100644 --- a/src/console/git/git.v +++ b/src/console/targets/targets.v @@ -1,4 +1,4 @@ -module git +module targets import cli import vieter.vconf @@ -16,12 +16,12 @@ struct Config { // cmd returns the cli submodule that handles the repos API interaction pub fn cmd() cli.Command { return cli.Command{ - name: 'repos' - description: 'Interact with the repos API.' + name: 'targets' + description: 'Interact with the targets API.' commands: [ cli.Command{ name: 'list' - description: 'List the current repos.' + description: 'List the current targets.' flags: [ cli.Flag{ name: 'limit' @@ -35,7 +35,7 @@ pub fn cmd() cli.Command { }, cli.Flag{ name: 'repo' - description: 'Only return Git repos that publish to this repo.' + description: 'Only return targets that publish to this repo.' flag: cli.FlagType.string }, ] @@ -67,7 +67,7 @@ pub fn cmd() cli.Command { name: 'add' required_args: 3 usage: 'url branch repo' - description: 'Add a new repository.' + description: 'Add a new Git repository target.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? @@ -79,7 +79,7 @@ pub fn cmd() cli.Command { name: 'remove' required_args: 1 usage: 'id' - description: 'Remove a repository that matches the given ID prefix.' + description: 'Remove a target that matches the given id.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? @@ -91,7 +91,7 @@ pub fn cmd() cli.Command { name: 'info' required_args: 1 usage: 'id' - description: 'Show detailed information for the repo matching the ID prefix.' + description: 'Show detailed information for the target matching the id.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? @@ -103,7 +103,7 @@ pub fn cmd() cli.Command { name: 'edit' required_args: 1 usage: 'id' - description: 'Edit the repository that matches the given ID prefix.' + description: 'Edit the Git repository target that matches the given id.' flags: [ cli.Flag{ name: 'url' @@ -152,7 +152,7 @@ pub fn cmd() cli.Command { name: 'build' required_args: 1 usage: 'id' - description: 'Build the repo with the given id & publish it.' + description: 'Build the target with the given id & publish it.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? diff --git a/src/main.v b/src/main.v index e1979de9..cba410ce 100644 --- a/src/main.v +++ b/src/main.v @@ -3,7 +3,7 @@ module main import os import server import cli -import console.git +import console.targets import console.logs import console.schedule import console.man @@ -26,7 +26,7 @@ fn main() { ] commands: [ server.cmd(), - git.cmd(), + targets.cmd(), cron.cmd(), logs.cmd(), schedule.cmd(), From 102a7f88999102a795646d3136475e7ac1238dfe Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 14 Jun 2022 22:25:40 +0200 Subject: [PATCH 08/62] refactor: renamed codebase to "targets" --- src/build/build.v | 6 +- src/build/shell.v | 6 +- src/build/shell_test.v | 6 +- src/client/logs.v | 6 +- src/client/targets.v | 36 +++---- src/console/logs/logs.v | 22 ++--- src/console/targets/targets.v | 6 +- src/cron/daemon/daemon.v | 8 +- src/db/db.v | 8 +- src/db/git.v | 99 ------------------- src/db/logs.v | 12 +-- .../migrations/002-rename-to-targets/down.sql | 5 + .../migrations/002-rename-to-targets/up.sql | 5 + src/db/targets.v | 99 +++++++++++++++++++ src/models/logs.v | 6 +- src/models/models.v | 4 +- src/models/{git.v => targets.v} | 18 ++-- src/server/api_logs.v | 14 +-- src/server/api_targets.v | 30 +++--- 19 files changed, 205 insertions(+), 191 deletions(-) delete mode 100644 src/db/git.v create mode 100644 src/db/migrations/002-rename-to-targets/down.sql create mode 100644 src/db/migrations/002-rename-to-targets/up.sql create mode 100644 src/db/targets.v rename src/models/{git.v => targets.v} (75%) diff --git a/src/build/build.v b/src/build/build.v index 2e86471f..3c6b439a 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -6,7 +6,7 @@ import time import os import strings import util -import models { GitRepo } +import models { Target } const ( container_build_dir = '/build' @@ -91,9 +91,9 @@ pub: } // build_repo builds, packages & publishes a given Arch package based on the -// provided GitRepo. The base image ID should be of an image previously created +// provided target. The base image ID should be of an image previously created // by create_build_image. It returns the logs of the container. -pub fn build_repo(address string, api_key string, base_image_id string, repo &GitRepo) ?BuildResult { +pub fn build_repo(address string, api_key string, base_image_id string, repo &Target) ?BuildResult { mut dd := docker.new_conn()? defer { diff --git a/src/build/shell.v b/src/build/shell.v index a3121fe8..c5944bf5 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -1,6 +1,6 @@ module build -import models { GitRepo } +import models { Target } // escape_shell_string escapes any characters that could be interpreted // incorrectly by a shell. The resulting value should be safe to use inside an @@ -22,8 +22,8 @@ pub fn echo_commands(cmds []string) []string { return out } -// create_build_script generates a shell script that builds a given GitRepo. -fn create_build_script(address string, repo &GitRepo, build_arch string) string { +// create_build_script generates a shell script that builds a given Target. +fn create_build_script(address string, repo &Target, build_arch string) string { repo_url := '$address/$repo.repo' commands := echo_commands([ diff --git a/src/build/shell_test.v b/src/build/shell_test.v index 46ab3501..85340549 100644 --- a/src/build/shell_test.v +++ b/src/build/shell_test.v @@ -1,15 +1,15 @@ module build -import models { GitRepo } +import models { Target } fn test_create_build_script() { - repo := GitRepo{ + target := Target{ id: 1 url: 'https://examplerepo.com' branch: 'main' repo: 'vieter' } - build_script := create_build_script('https://example.com', repo, 'x86_64') + build_script := create_build_script('https://example.com', target, 'x86_64') expected := $embed_file('build_script.sh') assert build_script == expected.to_string().trim_space() diff --git a/src/client/logs.v b/src/client/logs.v index 2d78fc15..49ef679b 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -13,10 +13,10 @@ pub fn (c &Client) get_build_logs(filter BuildLogFilter) ?Response<[]BuildLog> { return data } -// get_build_logs_for_repo returns all build logs for a given repo. -pub fn (c &Client) get_build_logs_for_repo(repo_id int) ?Response<[]BuildLog> { +// get_build_logs_for_target returns all build logs for a given target. +pub fn (c &Client) get_build_logs_for_target(target_id int) ?Response<[]BuildLog> { params := { - 'repo': repo_id.str() + 'repo': target_id.str() } data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)? diff --git a/src/client/targets.v b/src/client/targets.v index 90ff1a2d..86d8c2a9 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -1,46 +1,46 @@ module client -import models { GitRepo, GitRepoFilter } +import models { Target, TargetFilter } import net.http { Method } import response { Response } -// get_targets returns a list of GitRepo's, given a filter object. -pub fn (c &Client) get_targets(filter GitRepoFilter) ?[]GitRepo { +// get_targets returns a list of targets, given a filter object. +pub fn (c &Client) get_targets(filter TargetFilter) ?[]Target { params := models.params_from(filter) - data := c.send_request<[]GitRepo>(Method.get, '/api/v1/targets', params)? + data := c.send_request<[]Target>(Method.get, '/api/v1/targets', params)? return data.data } -// get_all_targets retrieves *all* GitRepo's from the API using the default +// get_all_targets retrieves *all* targs from the API using the default // limit. -pub fn (c &Client) get_all_targets() ?[]GitRepo { - mut repos := []GitRepo{} +pub fn (c &Client) get_all_targets() ?[]Target { + mut targets := []Target{} mut offset := u64(0) for { - sub_repos := c.get_targets(offset: offset)? + sub_targets := c.get_targets(offset: offset)? - if sub_repos.len == 0 { + if sub_targets.len == 0 { break } - repos << sub_repos + targets << sub_targets - offset += u64(sub_repos.len) + offset += u64(sub_targets.len) } - return repos + return targets } -// get_target returns the repo for a specific ID. -pub fn (c &Client) get_target(id int) ?GitRepo { - data := c.send_request(Method.get, '/api/v1/targets/$id', {})? +// get_target returns the target for a specific id. +pub fn (c &Client) get_target(id int) ?Target { + data := c.send_request(Method.get, '/api/v1/targets/$id', {})? return data.data } -// add_target adds a new repo to the server. +// add_target adds a new target to the server. pub fn (c &Client) add_target(url string, branch string, repo string, arch []string) ?Response { mut params := { 'url': url @@ -57,14 +57,14 @@ pub fn (c &Client) add_target(url string, branch string, repo string, arch []str return data } -// remove_target removes the repo with the given ID from the server. +// remove_target removes the target with the given id from the server. pub fn (c &Client) remove_target(id int) ?Response { data := c.send_request(Method.delete, '/api/v1/targets/$id', {})? return data } -// patch_target sends a PATCH request to the given repo with the params as +// patch_target sends a PATCH request to the given target with the params as // payload. pub fn (c &Client) patch_target(id int, params map[string]string) ?Response { data := c.send_request(Method.patch, '/api/v1/targets/$id', params)? diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index f9dd3636..5155ae88 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -12,7 +12,7 @@ struct Config { api_key string [required] } -// cmd returns the cli module that handles the build repos API. +// cmd returns the cli module that handles the build logs API. pub fn cmd() cli.Command { return cli.Command{ name: 'logs' @@ -33,8 +33,8 @@ pub fn cmd() cli.Command { flag: cli.FlagType.int }, cli.Flag{ - name: 'repo' - description: 'Only return logs for this repo id.' + name: 'target' + description: 'Only return logs for this target id.' flag: cli.FlagType.int }, cli.Flag{ @@ -79,9 +79,9 @@ pub fn cmd() cli.Command { filter.offset = u64(offset) } - repo_id := cmd.flags.get_int('repo')? - if repo_id != 0 { - filter.repo = repo_id + target_id := cmd.flags.get_int('target')? + if target_id != 0 { + filter.target = target_id } tz_offset := time.offset() @@ -168,10 +168,10 @@ pub fn cmd() cli.Command { // print_log_list prints a list of logs. fn print_log_list(logs []BuildLog) ? { - data := logs.map([it.id.str(), it.repo_id.str(), it.start_time.local().str(), + data := logs.map([it.id.str(), it.target_id.str(), it.start_time.local().str(), it.exit_code.str()]) - println(console.pretty_table(['id', 'repo', 'start time', 'exit code'], data)?) + println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)?) } // list prints a list of all build logs. @@ -182,10 +182,10 @@ fn list(conf Config, filter BuildLogFilter) ? { print_log_list(logs)? } -// list prints a list of all build logs for a given repo. -fn list_for_repo(conf Config, repo_id int) ? { +// list prints a list of all build logs for a given target. +fn list_for_target(conf Config, target_id int) ? { c := client.new(conf.address, conf.api_key) - logs := c.get_build_logs_for_repo(repo_id)?.data + logs := c.get_build_logs_for_target(target_id)?.data print_log_list(logs)? } diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 9917913c..530184d4 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -5,7 +5,7 @@ import vieter.vconf import cron.expression { parse_expression } import client import console -import models { GitRepoFilter } +import models { TargetFilter } struct Config { address string [required] @@ -43,7 +43,7 @@ pub fn cmd() cli.Command { config_file := cmd.flags.get_string('config-file')? conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? - mut filter := GitRepoFilter{} + mut filter := TargetFilter{} limit := cmd.flags.get_int('limit')? if limit != 0 { @@ -168,7 +168,7 @@ pub fn cmd() cli.Command { // ID. If multiple or none are found, an error is raised. // list prints out a list of all repositories. -fn list(conf Config, filter GitRepoFilter) ? { +fn list(conf Config, filter TargetFilter) ? { c := client.new(conf.address, conf.api_key) repos := c.get_targets(filter)? data := repos.map([it.id.str(), it.url, it.branch, it.repo]) diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 1f36e740..7b514aa4 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -9,7 +9,7 @@ import build import docker import os import client -import models { GitRepo } +import models { Target } const ( // How many seconds to wait before retrying to update API if failed @@ -20,7 +20,7 @@ const ( struct ScheduledBuild { pub: - repo GitRepo + repo Target timestamp time.Time } @@ -38,7 +38,7 @@ mut: api_update_frequency int image_rebuild_frequency int // Repos currently loaded from API. - repos []GitRepo + repos []Target // At what point to update the list of repositories. api_update_timestamp time.Time image_build_timestamp time.Time @@ -149,7 +149,7 @@ pub fn (mut d Daemon) run() { } // schedule_build adds the next occurence of the given repo build to the queue. -fn (mut d Daemon) schedule_build(repo GitRepo) { +fn (mut d Daemon) schedule_build(repo Target) { ce := if repo.schedule != '' { parse_expression(repo.schedule) or { // TODO This shouldn't return an error if the expression is empty. diff --git a/src/db/db.v b/src/db/db.v index fac14581..64a57d22 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -13,8 +13,12 @@ struct MigrationVersion { } const ( - migrations_up = [$embed_file('migrations/001-initial/up.sql')] - migrations_down = [$embed_file('migrations/001-initial/down.sql')] + migrations_up = [ + $embed_file('migrations/001-initial/up.sql'), + $embed_file('migrations/002-rename-to-targets/up.sql'), + ] + migrations_down = [$embed_file('migrations/001-initial/down.sql'), + $embed_file('migrations/002-rename-to-targets/down.sql')] ) // init initializes a database & adds the correct tables. diff --git a/src/db/git.v b/src/db/git.v deleted file mode 100644 index 8cc493f3..00000000 --- a/src/db/git.v +++ /dev/null @@ -1,99 +0,0 @@ -module db - -import models { GitRepo, GitRepoArch, GitRepoFilter } - -// get_git_repos returns all GitRepo's in the database. -pub fn (db &VieterDb) get_git_repos(filter GitRepoFilter) []GitRepo { - // This seems to currently be blocked by a bug in the ORM, I'll have to ask - // around. - if filter.repo != '' { - res := sql db.conn { - select from GitRepo where repo == filter.repo order by id limit filter.limit offset filter.offset - } - - return res - } - - res := sql db.conn { - select from GitRepo order by id limit filter.limit offset filter.offset - } - - return res -} - -// get_git_repo tries to return a specific GitRepo. -pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo { - res := sql db.conn { - select from GitRepo where id == repo_id - } - - // If a select statement fails, it returns a zeroed object. By - // checking one of the required fields, we can see whether the query - // returned a result or not. - if res.id == 0 { - return none - } - - return res -} - -// add_git_repo inserts the given GitRepo into the database. -pub fn (db &VieterDb) add_git_repo(repo GitRepo) { - sql db.conn { - insert repo into GitRepo - } -} - -// delete_git_repo deletes the repo with the given ID from the database. -pub fn (db &VieterDb) delete_git_repo(repo_id int) { - sql db.conn { - delete from GitRepo where id == repo_id - delete from GitRepoArch where repo_id == repo_id - } -} - -// update_git_repo updates any non-array values for a given GitRepo. -pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) { - mut values := []string{} - - // TODO does this allow for SQL injection? - $for field in GitRepo.fields { - if field.name in params { - // Any fields that are array types require their own update method - $if field.typ is string { - values << "$field.name = '${params[field.name]}'" - } - } - } - values_str := values.join(', ') - // I think this is actual SQL & not the ORM language - query := 'update GitRepo set $values_str where id == $repo_id' - - db.conn.exec_none(query) -} - -// update_git_repo_archs updates a given GitRepo's arch value. -pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { - archs_with_id := archs.map(GitRepoArch{ - ...it - repo_id: repo_id - }) - - sql db.conn { - delete from GitRepoArch where repo_id == repo_id - } - - for arch in archs_with_id { - sql db.conn { - insert arch into GitRepoArch - } - } -} - -// git_repo_exists is a utility function that checks whether a repo with the -// given id exists. -pub fn (db &VieterDb) git_repo_exists(repo_id int) bool { - db.get_git_repo(repo_id) or { return false } - - return true -} diff --git a/src/db/logs.v b/src/db/logs.v index cac08e7f..af5f53c3 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -7,8 +7,8 @@ import time pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { mut where_parts := []string{} - if filter.repo != 0 { - where_parts << 'repo_id == $filter.repo' + if filter.target != 0 { + where_parts << 'target_id == $filter.target' } if filter.before != time.Time{} { @@ -55,11 +55,11 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog { return res } -// get_build_logs_for_repo returns all BuildLog's in the database for a given -// repo. -pub fn (db &VieterDb) get_build_logs_for_repo(repo_id int) []BuildLog { +// get_build_logs_for_target returns all BuildLog's in the database for a given +// target. +pub fn (db &VieterDb) get_build_logs_for_target(target_id int) []BuildLog { res := sql db.conn { - select from BuildLog where repo_id == repo_id order by id + select from BuildLog where target_id == target_id order by id } return res diff --git a/src/db/migrations/002-rename-to-targets/down.sql b/src/db/migrations/002-rename-to-targets/down.sql new file mode 100644 index 00000000..861bfa9f --- /dev/null +++ b/src/db/migrations/002-rename-to-targets/down.sql @@ -0,0 +1,5 @@ +ALTER TABLE Target RENAME TO GitRepo; +ALTER TABLE TargetArch RENAME TO GitRepoArch; + +ALTER TABLE GitRepoArch RENAME COLUMN target_id TO repo_id; +ALTER TABLE BuildLog RENAME COLUMN target_id TO repo_id; diff --git a/src/db/migrations/002-rename-to-targets/up.sql b/src/db/migrations/002-rename-to-targets/up.sql new file mode 100644 index 00000000..081e3eee --- /dev/null +++ b/src/db/migrations/002-rename-to-targets/up.sql @@ -0,0 +1,5 @@ +ALTER TABLE GitRepo RENAME TO Target; +ALTER TABLE GitRepoArch RENAME TO TargetArch; + +ALTER TABLE TargetArch RENAME COLUMN repo_id TO target_id; +ALTER TABLE BuildLog RENAME COLUMN repo_id TO target_id; diff --git a/src/db/targets.v b/src/db/targets.v new file mode 100644 index 00000000..91020337 --- /dev/null +++ b/src/db/targets.v @@ -0,0 +1,99 @@ +module db + +import models { Target, TargetArch, TargetFilter } + +// get_targets returns all targets in the database. +pub fn (db &VieterDb) get_targets(filter TargetFilter) []Target { + // This seems to currently be blocked by a bug in the ORM, I'll have to ask + // around. + if filter.repo != '' { + res := sql db.conn { + select from Target where repo == filter.repo order by id limit filter.limit offset filter.offset + } + + return res + } + + res := sql db.conn { + select from Target order by id limit filter.limit offset filter.offset + } + + return res +} + +// get_target tries to return a specific target. +pub fn (db &VieterDb) get_target(target_id int) ?Target { + res := sql db.conn { + select from Target where id == target_id + } + + // If a select statement fails, it returns a zeroed object. By + // checking one of the required fields, we can see whether the query + // returned a result or not. + if res.id == 0 { + return none + } + + return res +} + +// add_target inserts the given target into the database. +pub fn (db &VieterDb) add_target(repo Target) { + sql db.conn { + insert repo into Target + } +} + +// delete_target deletes the target with the given id from the database. +pub fn (db &VieterDb) delete_target(target_id int) { + sql db.conn { + delete from Target where id == target_id + delete from TargetArch where target_id == target_id + } +} + +// update_target updates any non-array values for a given target. +pub fn (db &VieterDb) update_target(target_id int, params map[string]string) { + mut values := []string{} + + // TODO does this allow for SQL injection? + $for field in Target.fields { + if field.name in params { + // Any fields that are array types require their own update method + $if field.typ is string { + values << "$field.name = '${params[field.name]}'" + } + } + } + values_str := values.join(', ') + // I think this is actual SQL & not the ORM language + query := 'update Target set $values_str where id == $target_id' + + db.conn.exec_none(query) +} + +// update_target_archs updates a given target's arch value. +pub fn (db &VieterDb) update_target_archs(target_id int, archs []TargetArch) { + archs_with_id := archs.map(TargetArch{ + ...it + target_id: target_id + }) + + sql db.conn { + delete from TargetArch where target_id == target_id + } + + for arch in archs_with_id { + sql db.conn { + insert arch into TargetArch + } + } +} + +// target_exists is a utility function that checks whether a target with the +// given id exists. +pub fn (db &VieterDb) target_exists(target_id int) bool { + db.get_target(target_id) or { return false } + + return true +} diff --git a/src/models/logs.v b/src/models/logs.v index 7f5a5fe5..12907d84 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -5,7 +5,7 @@ import time pub struct BuildLog { pub mut: id int [primary; sql: serial] - repo_id int [nonull] + target_id int [nonull] start_time time.Time [nonull] end_time time.Time [nonull] arch string [nonull] @@ -16,7 +16,7 @@ pub mut: pub fn (bl &BuildLog) str() string { mut parts := [ 'id: $bl.id', - 'repo id: $bl.repo_id', + 'target id: $bl.target_id', 'start time: $bl.start_time.local()', 'end time: $bl.end_time.local()', 'duration: ${bl.end_time - bl.start_time}', @@ -33,7 +33,7 @@ pub struct BuildLogFilter { pub mut: limit u64 = 25 offset u64 - repo int + target int before time.Time after time.Time arch string diff --git a/src/models/models.v b/src/models/models.v index 3a127bcc..b6103d31 100644 --- a/src/models/models.v +++ b/src/models/models.v @@ -23,8 +23,8 @@ pub fn patch_from_params(mut o T, params map[string]string) ? { o.$(field.name) = params[field.name].int() } $else $if field.typ is u64 { o.$(field.name) = params[field.name].u64() - } $else $if field.typ is []GitRepoArch { - o.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) + } $else $if field.typ is []TargetArch { + o.$(field.name) = params[field.name].split(',').map(TargetArch{ value: it }) } $else $if field.typ is time.Time { o.$(field.name) = time.unix(params[field.name].int()) } $else $if field.typ is []string { diff --git a/src/models/git.v b/src/models/targets.v similarity index 75% rename from src/models/git.v rename to src/models/targets.v index 5dcc13a7..537bbc4d 100644 --- a/src/models/git.v +++ b/src/models/targets.v @@ -1,18 +1,18 @@ module models -pub struct GitRepoArch { +pub struct TargetArch { pub: - id int [primary; sql: serial] - repo_id int [nonull] - value string [nonull] + id int [primary; sql: serial] + target_id int [nonull] + value string [nonull] } // str returns a string representation. -pub fn (gra &GitRepoArch) str() string { +pub fn (gra &TargetArch) str() string { return gra.value } -pub struct GitRepo { +pub struct Target { pub mut: id int [primary; sql: serial] // URL of the Git repository @@ -25,11 +25,11 @@ pub mut: schedule string // On which architectures the package is allowed to be built. In reality, // this controls which builders will periodically build the image. - arch []GitRepoArch [fkey: 'repo_id'] + arch []TargetArch [fkey: 'target_id'] } // str returns a string representation. -pub fn (gr &GitRepo) str() string { +pub fn (gr &Target) str() string { mut parts := [ 'id: $gr.id', 'url: $gr.url', @@ -44,7 +44,7 @@ pub fn (gr &GitRepo) str() string { } [params] -pub struct GitRepoFilter { +pub struct TargetFilter { pub mut: limit u64 = 25 offset u64 diff --git a/src/server/api_logs.v b/src/server/api_logs.v index 748c3a66..fa3338ea 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -10,7 +10,7 @@ import os import util import models { BuildLog, BuildLogFilter } -// v1_get_logs returns all build logs in the database. A 'repo' query param can +// v1_get_logs returns all build logs in the database. A 'target' query param can // optionally be added to limit the list of build logs to that repository. ['/api/v1/logs'; get] fn (mut app App) v1_get_logs() web.Result { @@ -47,7 +47,7 @@ fn (mut app App) v1_get_log_content(id int) web.Result { log := app.db.get_build_log(id) or { return app.not_found() } file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') - full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.repo_id.str(), log.arch, + full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), log.arch, file_name) return app.file(full_path) @@ -96,15 +96,15 @@ fn (mut app App) v1_post_log() web.Result { arch := app.query['arch'] - repo_id := app.query['repo'].int() + target_id := app.query['target'].int() - if !app.db.git_repo_exists(repo_id) { - return app.json(http.Status.bad_request, new_response('Unknown Git repo.')) + if !app.db.target_exists(target_id) { + return app.json(http.Status.bad_request, new_response('Unknown target.')) } // Store log in db log := BuildLog{ - repo_id: repo_id + target_id: target_id start_time: start_time end_time: end_time arch: arch @@ -113,7 +113,7 @@ fn (mut app App) v1_post_log() web.Result { app.db.add_build_log(log) - repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo_id.str(), arch) + repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, target_id.str(), arch) // Create the logs directory of it doesn't exist if !os.exists(repo_logs_dir) { diff --git a/src/server/api_targets.v b/src/server/api_targets.v index 67117b92..f807fe94 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -4,36 +4,36 @@ import web import net.http import response { new_data_response, new_response } import db -import models { GitRepo, GitRepoArch, GitRepoFilter } +import models { Target, TargetArch, TargetFilter } -// v1_get_targets returns the current list of repos. +// v1_get_targets returns the current list of targets. ['/api/v1/targets'; get] fn (mut app App) v1_get_targets() web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - filter := models.from_params(app.query) or { + filter := models.from_params(app.query) or { return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } - repos := app.db.get_git_repos(filter) + repos := app.db.get_targets(filter) return app.json(http.Status.ok, new_data_response(repos)) } -// v1_get_single_target returns the information for a single repo. +// v1_get_single_target returns the information for a single target. ['/api/v1/targets/:id'; get] fn (mut app App) v1_get_single_target(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - repo := app.db.get_git_repo(id) or { return app.not_found() } + repo := app.db.get_target(id) or { return app.not_found() } return app.json(http.Status.ok, new_data_response(repo)) } -// v1_post_target creates a new repo from the provided query string. +// v1_post_target creates a new target from the provided query string. ['/api/v1/targets'; post] fn (mut app App) v1_post_target() web.Result { if !app.is_authorized() { @@ -48,40 +48,40 @@ fn (mut app App) v1_post_target() web.Result { params['arch'] = app.conf.default_arch } - new_repo := models.from_params(params) or { + new_repo := models.from_params(params) or { return app.json(http.Status.bad_request, new_response(err.msg())) } - app.db.add_git_repo(new_repo) + app.db.add_target(new_repo) return app.json(http.Status.ok, new_response('Repo added successfully.')) } -// v1_delete_target removes a given repo from the server's list. +// v1_delete_target removes a given target from the server's list. ['/api/v1/targets/:id'; delete] fn (mut app App) v1_delete_target(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - app.db.delete_git_repo(id) + app.db.delete_target(id) return app.json(http.Status.ok, new_response('Repo removed successfully.')) } -// v1_patch_target updates a repo's data with the given query params. +// v1_patch_target updates a target's data with the given query params. ['/api/v1/targets/:id'; patch] fn (mut app App) v1_patch_target(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - app.db.update_git_repo(id, app.query) + app.db.update_target(id, app.query) if 'arch' in app.query { - arch_objs := app.query['arch'].split(',').map(GitRepoArch{ value: it }) + arch_objs := app.query['arch'].split(',').map(TargetArch{ value: it }) - app.db.update_git_repo_archs(id, arch_objs) + app.db.update_target_archs(id, arch_objs) } return app.json(http.Status.ok, new_response('Repo updated successfully.')) From 9727b8620302276ed9f65b8d24c12273371bb99e Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 14 Jun 2022 22:41:28 +0200 Subject: [PATCH 09/62] docs(api): updated to new "targets" naming --- docs/api/source/includes/_logs.md | 10 ++-- .../source/includes/{_git.md => _targets.md} | 51 +++++++++---------- docs/api/source/index.html.md | 2 +- 3 files changed, 31 insertions(+), 32 deletions(-) rename docs/api/source/includes/{_git.md => _targets.md} (71%) diff --git a/docs/api/source/includes/_logs.md b/docs/api/source/includes/_logs.md index d4f76320..a38bf484 100644 --- a/docs/api/source/includes/_logs.md +++ b/docs/api/source/includes/_logs.md @@ -24,7 +24,7 @@ curl \ "data": [ { "id": 1, - "repo_id": 3, + "target_id": 3, "start_time": 1652008554, "end_time": 1652008559, "arch": "x86_64", @@ -46,7 +46,7 @@ Parameter | Description --------- | ----------- limit | Maximum amount of results to return. offset | Offset of results. -repo | Only return builds published to this repository. +target | Only return builds for this target id. before | Only return logs started before this time (UTC epoch) after | Only return logs started after this time (UTC epoch) arch | Only return logs built on this architecture @@ -58,7 +58,7 @@ exit_codes | Comma-separated list of exit codes to limit result to; using `!` as ```shell curl \ -H 'X-Api-Key: secret' \ - https://example.com/api/logs/15 + https://example.com/api/logs/1 ``` > JSON output format @@ -68,7 +68,7 @@ curl \ "message": "", "data": { "id": 1, - "repo_id": 3, + "target_id": 3, "start_time": 1652008554, "end_time": 1652008559, "arch": "x86_64", @@ -129,11 +129,11 @@ Publish a new build log to the server. Parameter | Description --------- | ----------- -id | ID of requested log startTime | Start time of the build (UTC epoch) endTime | End time of the build (UTC epoch) arch | Architecture on which the build was done exitCode | Exit code of the build container +target | id of target this build is for ### Request body diff --git a/docs/api/source/includes/_git.md b/docs/api/source/includes/_targets.md similarity index 71% rename from docs/api/source/includes/_git.md rename to docs/api/source/includes/_targets.md index 8458834e..05cf0015 100644 --- a/docs/api/source/includes/_git.md +++ b/docs/api/source/includes/_targets.md @@ -1,4 +1,4 @@ -# Git Repositories +# Targets -Endpoints for interacting with the list of Git repositories stored on the -server. +Endpoints for interacting with the list of targets stored on the server. -## List repos +## List targets ```shell curl \ -H 'X-Api-Key: secret' \ - https://example.com/api/repos?offset=10&limit=20 + https://example.com/api/targets?offset=10&limit=20 ``` > JSON output format @@ -32,7 +31,7 @@ curl \ "arch": [ { "id": 1, - "repo_id": 1, + "target_id": 1, "value": "x86_64" } ] @@ -41,11 +40,11 @@ curl \ } ``` -Retrieve a list of Git repositories. +Retrieve a list of targets. ### HTTP Request -`GET /api/repos` +`GET /api/targets` ### Query Parameters @@ -53,14 +52,14 @@ Parameter | Description --------- | ----------- limit | Maximum amount of results to return. offset | Offset of results. -repo | Limit results to repositories that publish to the given repo. +repo | Limit results to targets that publish to the given repo. -## Get a repo +## Get specific target ```shell curl \ -H 'X-Api-Key: secret' \ - https://example.com/api/repos/15 + https://example.com/api/targets/1 ``` > JSON output format @@ -77,7 +76,7 @@ curl \ "arch": [ { "id": 1, - "repo_id": 1, + "target_id": 1, "value": "x86_64" } ] @@ -85,25 +84,25 @@ curl \ } ``` -Get info about a specific Git repository. +Get info about a specific target. ### HTTP Request -`GET /api/repos/:id` +`GET /api/targets/:id` ### URL Parameters Parameter | Description --------- | ----------- -id | ID of requested repo +id | id of requested target -## Create a new repo +## Create a new target -Create a new Git repository with the given data. +Create a new target with the given data. ### HTTP Request -`POST /api/repos` +`POST /api/targets` ### Query Parameters @@ -115,19 +114,19 @@ repo | Vieter repository to publish built packages to. schedule | Cron build schedule (syntax explained [here](https://rustybever.be/docs/vieter/usage/builds/schedule/)) arch | Comma-separated list of architectures to build package on. -## Modify a repo +## Modify a target -Modify the data of an existing Git repository. +Modify the data of an existing target. ### HTTP Request -`PATCH /api/repos/:id` +`PATCH /api/targets/:id` ### URL Parameters Parameter | Description --------- | ----------- -id | ID of requested repo +id | id of target to modify ### Query Parameters @@ -139,16 +138,16 @@ repo | Vieter repository to publish built packages to. schedule | Cron build schedule arch | Comma-separated list of architectures to build package on. -## Remove a repo +## Remove a target -Remove a Git repository from the server. +Remove a target from the server. ### HTTP Request -`DELETE /api/repos/:id` +`DELETE /api/targets/:id` ### URL Parameters Parameter | Description --------- | ----------- -id | ID of repo to remove +id | id of target to remove diff --git a/docs/api/source/index.html.md b/docs/api/source/index.html.md index 74774982..4bfddb8e 100644 --- a/docs/api/source/index.html.md +++ b/docs/api/source/index.html.md @@ -9,7 +9,7 @@ toc_footers: includes: - repository - - git + - targets - logs search: true From fcdcf9c5ca0985c9fbaded2e9580e3f5c90b79dc Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 15 Jun 2022 12:10:58 +0200 Subject: [PATCH 10/62] feat(server): add config option for server port --- CHANGELOG.md | 6 ++++++ docs/content/configuration.md | 3 ++- src/server/cli.v | 1 + src/server/server.v | 3 +-- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9adee075..5fe1de7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev) +### Added + +* Server port can now be configured + ### Changed * Moved all API routes under `/v1` namespace +* Renamed `vieter repos` to `vieter targets` +* Renamed `/v1/api/repos` namespace to `/v1/api/targets` ## [0.3.0](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0) diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 600c6f35..f4bf9649 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -45,7 +45,8 @@ configuration variable required for each command. This prevents the server from being confused when an `any` package is published as the very first package for a repository. * Git repositories added without an `arch` value use this value instead. - +* `port`: HTTP port to run on + * Default: `8000` ### `vieter cron` diff --git a/src/server/cli.v b/src/server/cli.v index 53393f70..a62f56ec 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -10,6 +10,7 @@ pub: data_dir string api_key string default_arch string + port int = 8000 } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/server.v b/src/server/server.v index 2309ee7b..1a9df3f8 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -8,7 +8,6 @@ import util import db const ( - port = 8000 log_file_name = 'vieter.log' repo_dir_name = 'repos' db_file_name = 'vieter.sqlite' @@ -77,5 +76,5 @@ pub fn server(conf Config) ? { conf: conf repo: repo db: db - }, server.port) + }, conf.port) } From cf94b644005d4d11f45b99a053ad665204c9603c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 16 Jun 2022 17:38:41 +0200 Subject: [PATCH 11/62] docs: forgot some renames --- CHANGELOG.md | 2 +- docs/content/configuration.md | 6 +++--- docs/content/usage/builds/_index.md | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fe1de7b..109e65c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Moved all API routes under `/v1` namespace * Renamed `vieter repos` to `vieter targets` -* Renamed `/v1/api/repos` namespace to `/v1/api/targets` +* Renamed `/api/v1/repos` namespace to `/api/v1/targets` ## [0.3.0](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0) diff --git a/docs/content/configuration.md b/docs/content/configuration.md index f4bf9649..af941a22 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -26,7 +26,7 @@ secrets file. ## Commands The first argument passed to Vieter determines which command you wish to use. -Each of these can contain subcommands (e.g. `vieter repos list`), but all +Each of these can contain subcommands (e.g. `vieter targets list`), but all subcommands will use the same configuration. Below you can find the configuration variable required for each command. @@ -89,11 +89,11 @@ configuration variable required for each command. * `api_key`: the API key to use when authenticating requests. * `address`: Base URL of your Vieter instance, e.g. https://example.com -### `vieter repos` +### `vieter targets` * `api_key`: the API key to use when authenticating requests. * `address`: Base URL of your Vieter instance, e.g. https://example.com -* `base_image`: image to use when building a package using `vieter repos +* `base_image`: image to use when building a package using `vieter targets build`. * Default: `archlinux:base-devel` diff --git a/docs/content/usage/builds/_index.md b/docs/content/usage/builds/_index.md index cd463a4b..7babb062 100644 --- a/docs/content/usage/builds/_index.md +++ b/docs/content/usage/builds/_index.md @@ -16,11 +16,11 @@ info to the system. The Vieter repository server exposes an HTTP API for this info). For ease of use, the Vieter binary contains a CLI interface for interacting with this API (see [Configuration](/configuration) for configuration details). The [man -pages](https://rustybever.be/man/vieter/vieter-repos.1.html) describe this in +pages](https://rustybever.be/man/vieter/vieter-targets.1.html) describe this in greater detail, but the basic usage is as follows: ``` -vieter repos add some-url some-branch some-repository +vieter targets add some-url some-branch some-repository ``` Here, `some-url` is the URL of the Git repository containing the PKGBUILD. This @@ -37,7 +37,7 @@ should be published. The above command intentionally leaves out a few parameters to make the CLI more useable. For information on how to modify all parameters using the CLI, see -[vieter-repos-edit(1)](https://rustybever.be/man/vieter/vieter-repos-edit.1.html). +[vieter-targets-edit(1)](https://rustybever.be/man/vieter/vieter-targets-edit.1.html). ## Reading logs From 3d38df6d038a2e75053461ad0c2829d10fb8a4e2 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 16 Jun 2022 18:10:05 +0200 Subject: [PATCH 12/62] fix(client): use new "target" name for param --- src/client/logs.v | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/logs.v b/src/client/logs.v index 49ef679b..f242f6e8 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -39,9 +39,9 @@ pub fn (c &Client) get_build_log_content(id int) ?string { } // add_build_log adds a new build log to the server. -pub fn (c &Client) add_build_log(repo_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { +pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { params := { - 'repo': repo_id.str() + 'target': target_id.str() 'startTime': start_time.unix_time().str() 'endTime': end_time.unix_time().str() 'arch': arch From 1b7c14e7dcd8850d622687e27eb454c6f8d5b05c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 15 Jun 2022 22:15:11 +0200 Subject: [PATCH 13/62] feat(server): no longer calculate md5 hashes for packages --- CHANGELOG.md | 4 ++++ src/package/package.v | 10 ++++------ src/repo/repo.v | 2 +- src/util/util.v | 19 ++++++------------- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 109e65c6..d01409a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Renamed `vieter repos` to `vieter targets` * Renamed `/api/v1/repos` namespace to `/api/v1/targets` +### Removed + +* md5 hashes are no longer calculated for packages + ## [0.3.0](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0) Nothing besides bumping the versions. diff --git a/src/package/package.v b/src/package/package.v index 273322fa..86bf40ad 100644 --- a/src/package/package.v +++ b/src/package/package.v @@ -42,8 +42,8 @@ pub mut: checkdepends []string } -// checksum calculates the md5 & sha256 hash of the package -pub fn (p &Pkg) checksum() ?(string, string) { +// checksum calculates the sha256 hash of the package +pub fn (p &Pkg) checksum() ?string { return util.hash_file(p.path) } @@ -201,8 +201,7 @@ pub fn (pkg &Pkg) filename() string { } // to_desc returns a desc file valid string representation -// TODO calculate md5 & sha256 instead of believing the file -pub fn (pkg &Pkg) to_desc() string { +pub fn (pkg &Pkg) to_desc() ?string { p := pkg.info // filename @@ -223,9 +222,8 @@ pub fn (pkg &Pkg) to_desc() string { desc += format_entry('CSIZE', p.csize.str()) desc += format_entry('ISIZE', p.size.str()) - md5sum, sha256sum := pkg.checksum() or { '', '' } + sha256sum := pkg.checksum()? - desc += format_entry('MD5SUM', md5sum) desc += format_entry('SHA256SUM', sha256sum) // TODO add pgpsig stuff diff --git a/src/repo/repo.v b/src/repo/repo.v index 817ec304..c4b85c0b 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -139,7 +139,7 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') } - os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or { + os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()?) or { os.rmdir_all(pkg_dir)? return error('Failed to write desc file.') diff --git a/src/util/util.v b/src/util/util.v index 266bcb5b..4cd374e7 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -1,7 +1,6 @@ module util import os -import crypto.md5 import crypto.sha256 const ( @@ -23,12 +22,10 @@ pub fn exit_with_message(code int, msg string) { exit(code) } -// hash_file returns the md5 & sha256 hash of a given file -// TODO actually implement sha256 -pub fn hash_file(path &string) ?(string, string) { +// hash_file returns the sha256 hash of a given file +pub fn hash_file(path &string) ?string { file := os.open(path) or { return error('Failed to open file.') } - mut md5sum := md5.new() mut sha256sum := sha256.new() buf_size := int(1_000_000) @@ -40,16 +37,12 @@ pub fn hash_file(path &string) ?(string, string) { bytes_read := file.read(mut buf) or { return error('Failed to read from file.') } bytes_left -= u64(bytes_read) - // For now we'll assume that this always works - md5sum.write(buf[..bytes_read]) or { - return error('Failed to update md5 checksum. This should never happen.') - } - sha256sum.write(buf[..bytes_read]) or { - return error('Failed to update sha256 checksum. This should never happen.') - } + // This function never actually fails, but returns an option to follow + // the Writer interface. + sha256sum.write(buf[..bytes_read])? } - return md5sum.checksum().hex(), sha256sum.checksum().hex() + return sha256sum.checksum().hex() } // pretty_bytes converts a byte count to human-readable version From a8d647cca365dec1c036c055783d9cd3ef632973 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 16 Jun 2022 22:38:42 +0200 Subject: [PATCH 14/62] fix(docs): use versioned endpoints in HTTP API docs --- docs/api/source/includes/_logs.md | 14 +++++++------- docs/api/source/includes/_targets.md | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/api/source/includes/_logs.md b/docs/api/source/includes/_logs.md index a38bf484..2797e60d 100644 --- a/docs/api/source/includes/_logs.md +++ b/docs/api/source/includes/_logs.md @@ -13,7 +13,7 @@ Endpoints for interacting with stored build logs. ```shell curl \ -H 'X-Api-Key: secret' \ - https://example.com/api/logs?offset=10&limit=20 + https://example.com/api/v1/logs?offset=10&limit=20 ``` > JSON output format @@ -38,7 +38,7 @@ Retrieve a list of build logs. ### HTTP Request -`GET /api/logs` +`GET /api/v1/logs` ### Query Parameters @@ -58,7 +58,7 @@ exit_codes | Comma-separated list of exit codes to limit result to; using `!` as ```shell curl \ -H 'X-Api-Key: secret' \ - https://example.com/api/logs/1 + https://example.com/api/v1/logs/1 ``` > JSON output format @@ -81,7 +81,7 @@ Retrieve info about a specific build log. ### HTTP Request -`GET /api/logs/:id` +`GET /api/v1/logs/:id` ### URL Parameters @@ -94,7 +94,7 @@ id | ID of requested log ```shell curl \ -H 'X-Api-Key: secret' \ - https://example.com/api/logs/15/content + https://example.com/api/v1/logs/15/content ``` Retrieve the contents of a build log. The response is the build log in @@ -102,7 +102,7 @@ plaintext. ### HTTP Request -`GET /api/logs/:id/content` +`GET /api/v1/logs/:id/content` ### URL Parameters @@ -123,7 +123,7 @@ Publish a new build log to the server. ### HTTP Request -`POST /api/logs` +`POST /api/v1/logs` ### Query parameters diff --git a/docs/api/source/includes/_targets.md b/docs/api/source/includes/_targets.md index 05cf0015..f7ea21e5 100644 --- a/docs/api/source/includes/_targets.md +++ b/docs/api/source/includes/_targets.md @@ -13,7 +13,7 @@ Endpoints for interacting with the list of targets stored on the server. ```shell curl \ -H 'X-Api-Key: secret' \ - https://example.com/api/targets?offset=10&limit=20 + https://example.com/api/v1/targets?offset=10&limit=20 ``` > JSON output format @@ -44,7 +44,7 @@ Retrieve a list of targets. ### HTTP Request -`GET /api/targets` +`GET /api/v1/targets` ### Query Parameters @@ -59,7 +59,7 @@ repo | Limit results to targets that publish to the given repo. ```shell curl \ -H 'X-Api-Key: secret' \ - https://example.com/api/targets/1 + https://example.com/api/v1/targets/1 ``` > JSON output format @@ -88,7 +88,7 @@ Get info about a specific target. ### HTTP Request -`GET /api/targets/:id` +`GET /api/v1/targets/:id` ### URL Parameters @@ -102,7 +102,7 @@ Create a new target with the given data. ### HTTP Request -`POST /api/targets` +`POST /api/v1/targets` ### Query Parameters @@ -120,7 +120,7 @@ Modify the data of an existing target. ### HTTP Request -`PATCH /api/targets/:id` +`PATCH /api/v1/targets/:id` ### URL Parameters @@ -144,7 +144,7 @@ Remove a target from the server. ### HTTP Request -`DELETE /api/targets/:id` +`DELETE /api/v1/targets/:id` ### URL Parameters From bb5643bb036eff0d8ece2a94ad05d14b42dcf901 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 17 Jun 2022 13:45:21 +0200 Subject: [PATCH 15/62] feat: added ability to specify kind of target --- src/client/targets.v | 21 +++++----- src/console/targets/targets.v | 40 ++++++++++++++----- src/db/db.v | 8 +++- .../migrations/003-target-url-type/down.sql | 4 ++ src/db/migrations/003-target-url-type/up.sql | 1 + src/models/targets.v | 9 +++-- 6 files changed, 58 insertions(+), 25 deletions(-) create mode 100644 src/db/migrations/003-target-url-type/down.sql create mode 100644 src/db/migrations/003-target-url-type/up.sql diff --git a/src/client/targets.v b/src/client/targets.v index 86d8c2a9..d7cc4161 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -40,18 +40,17 @@ pub fn (c &Client) get_target(id int) ?Target { return data.data } +pub struct NewTarget { + kind string + url string + branch string + repo string + arch []string +} + // add_target adds a new target to the server. -pub fn (c &Client) add_target(url string, branch string, repo string, arch []string) ?Response { - mut params := { - 'url': url - 'branch': branch - 'repo': repo - } - - if arch.len > 0 { - params['arch'] = arch.join(',') - } - +pub fn (c &Client) add_target(t NewTarget) ?Response { + params := models.params_from(t) data := c.send_request(Method.post, '/api/v1/targets', params)? return data diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 530184d4..71a5c92c 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -3,7 +3,7 @@ module targets import cli import vieter.vconf import cron.expression { parse_expression } -import client +import client { NewTarget } import console import models { TargetFilter } @@ -65,14 +65,36 @@ pub fn cmd() cli.Command { }, cli.Command{ name: 'add' - required_args: 3 - usage: 'url branch repo' - description: 'Add a new Git repository target.' + required_args: 2 + usage: 'url repo' + description: "Add a new target with the given URL & target repo." + flags: [ + cli.Flag{ + name: 'kind' + description: "Kind of target to add. Defaults to 'git' if not specified. One of 'git', 'url'." + flag: cli.FlagType.string + default_value: ['git'] + }, + cli.Flag{ + name: 'branch' + description: "Which branch to clone; only applies to kind 'git'." + flag: cli.FlagType.string + } + ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? - add(conf, cmd.args[0], cmd.args[1], cmd.args[2])? + kind := cmd.flags.get_string('kind')? + + t := NewTarget{ + kind: cmd.flags.get_string('kind')? + url: cmd.args[0] + repo: cmd.args[1] + branch: cmd.flags.get_string('branch') or { '' } + } + + add(conf, t)? } }, cli.Command{ @@ -171,15 +193,15 @@ pub fn cmd() cli.Command { fn list(conf Config, filter TargetFilter) ? { c := client.new(conf.address, conf.api_key) repos := c.get_targets(filter)? - data := repos.map([it.id.str(), it.url, it.branch, it.repo]) + data := repos.map([it.id.str(), it.kind, it.url, it.repo]) - println(console.pretty_table(['id', 'url', 'branch', 'repo'], data)?) + println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?) } // add adds a new repository to the server's list. -fn add(conf Config, url string, branch string, repo string) ? { +fn add(conf Config, t &NewTarget) ? { c := client.new(conf.address, conf.api_key) - res := c.add_target(url, branch, repo, [])? + res := c.add_target(t)? println(res.message) } diff --git a/src/db/db.v b/src/db/db.v index 64a57d22..eaf71abb 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -16,9 +16,13 @@ const ( migrations_up = [ $embed_file('migrations/001-initial/up.sql'), $embed_file('migrations/002-rename-to-targets/up.sql'), + $embed_file('migrations/003-target-url-type/up.sql'), + ] + migrations_down = [ + $embed_file('migrations/001-initial/down.sql'), + $embed_file('migrations/002-rename-to-targets/down.sql'), + $embed_file('migrations/003-target-url-type/down.sql'), ] - migrations_down = [$embed_file('migrations/001-initial/down.sql'), - $embed_file('migrations/002-rename-to-targets/down.sql')] ) // init initializes a database & adds the correct tables. diff --git a/src/db/migrations/003-target-url-type/down.sql b/src/db/migrations/003-target-url-type/down.sql new file mode 100644 index 00000000..9d9b45c6 --- /dev/null +++ b/src/db/migrations/003-target-url-type/down.sql @@ -0,0 +1,4 @@ +-- I'm not sure whether I should remove any non-git targets here. Keeping them +-- will result in invalid targets, but removing them means losing data. +ALTER TABLE Target DROP COLUMN kind; + diff --git a/src/db/migrations/003-target-url-type/up.sql b/src/db/migrations/003-target-url-type/up.sql new file mode 100644 index 00000000..f6be4f47 --- /dev/null +++ b/src/db/migrations/003-target-url-type/up.sql @@ -0,0 +1 @@ +ALTER TABLE Target ADD COLUMN kind TEXT NOT NULL DEFAULT 'git'; diff --git a/src/models/targets.v b/src/models/targets.v index 537bbc4d..b8fc88a8 100644 --- a/src/models/targets.v +++ b/src/models/targets.v @@ -15,10 +15,13 @@ pub fn (gra &TargetArch) str() string { pub struct Target { pub mut: id int [primary; sql: serial] - // URL of the Git repository + kind string [nonull] + // If kind is git: URL of the Git repository + // If kind is url: URL to PKGBUILD file url string [nonull] - // Branch of the Git repository to use - branch string [nonull] + // Branch of the Git repository to use; only applicable when kind is git. + // If not provided, the repository is cloned with the default branch. + branch string // Which repo the builder should publish packages to repo string [nonull] // Cron schedule describing how frequently to build the repo. From bd079645090280eecd5e827ddd03220c8f9f53dd Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 17 Jun 2022 13:56:38 +0200 Subject: [PATCH 16/62] feat(api): prevent invalid kind values --- src/client/targets.v | 8 ++++---- src/console/targets/targets.v | 13 ++++++++----- src/models/targets.v | 5 ++++- src/server/api_targets.v | 5 +++++ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/client/targets.v b/src/client/targets.v index d7cc4161..82c78780 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -41,11 +41,11 @@ pub fn (c &Client) get_target(id int) ?Target { } pub struct NewTarget { - kind string - url string + kind string + url string branch string - repo string - arch []string + repo string + arch []string } // add_target adds a new target to the server. diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 71a5c92c..a9934f1f 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -67,7 +67,7 @@ pub fn cmd() cli.Command { name: 'add' required_args: 2 usage: 'url repo' - description: "Add a new target with the given URL & target repo." + description: 'Add a new target with the given URL & target repo.' flags: [ cli.Flag{ name: 'kind' @@ -79,14 +79,12 @@ pub fn cmd() cli.Command { name: 'branch' description: "Which branch to clone; only applies to kind 'git'." flag: cli.FlagType.string - } + }, ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? - kind := cmd.flags.get_string('kind')? - t := NewTarget{ kind: cmd.flags.get_string('kind')? url: cmd.args[0] @@ -152,6 +150,11 @@ pub fn cmd() cli.Command { description: 'Cron schedule for repository.' flag: cli.FlagType.string }, + cli.Flag{ + name: 'kind' + description: 'Kind of target.' + flag: cli.FlagType.string + }, ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? @@ -195,7 +198,7 @@ fn list(conf Config, filter TargetFilter) ? { repos := c.get_targets(filter)? data := repos.map([it.id.str(), it.kind, it.url, it.repo]) - println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?) + println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?) } // add adds a new repository to the server's list. diff --git a/src/models/targets.v b/src/models/targets.v index b8fc88a8..c8aa5357 100644 --- a/src/models/targets.v +++ b/src/models/targets.v @@ -1,5 +1,7 @@ module models +pub const valid_kinds = ['git', 'url'] + pub struct TargetArch { pub: id int [primary; sql: serial] @@ -14,7 +16,7 @@ pub fn (gra &TargetArch) str() string { pub struct Target { pub mut: - id int [primary; sql: serial] + id int [primary; sql: serial] kind string [nonull] // If kind is git: URL of the Git repository // If kind is url: URL to PKGBUILD file @@ -35,6 +37,7 @@ pub mut: pub fn (gr &Target) str() string { mut parts := [ 'id: $gr.id', + 'kind: $gr.kind', 'url: $gr.url', 'branch: $gr.branch', 'repo: $gr.repo', diff --git a/src/server/api_targets.v b/src/server/api_targets.v index f807fe94..3867c94b 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -52,6 +52,11 @@ fn (mut app App) v1_post_target() web.Result { return app.json(http.Status.bad_request, new_response(err.msg())) } + // Ensure someone doesn't submit an invalid kind + if new_repo.kind !in models.valid_kinds { + return app.json(http.Status.bad_request, new_response('Invalid kind.')) + } + app.db.add_target(new_repo) return app.json(http.Status.ok, new_response('Repo added successfully.')) From 8f91c1fde52fd7d3a6a11e4c3fa54e0cc414afd5 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 17 Jun 2022 14:31:34 +0200 Subject: [PATCH 17/62] feat(build): added support for 'url' kind --- .../{build_script.sh => build_script_git.sh} | 4 +- src/build/build_script_git_branch.sh | 20 ++++++++++ src/build/build_script_url.sh | 22 +++++++++++ src/build/shell.v | 39 +++++++++++++++---- src/build/shell_test.v | 31 ++++++++++++++- src/console/targets/targets.v | 5 +-- 6 files changed, 107 insertions(+), 14 deletions(-) rename src/build/{build_script.sh => build_script_git.sh} (87%) create mode 100644 src/build/build_script_git_branch.sh create mode 100644 src/build/build_script_url.sh diff --git a/src/build/build_script.sh b/src/build/build_script_git.sh similarity index 87% rename from src/build/build_script.sh rename to src/build/build_script_git.sh index 29f163ed..73e0965d 100644 --- a/src/build/build_script.sh +++ b/src/build/build_script_git.sh @@ -4,8 +4,8 @@ echo -e '+ pacman -Syu --needed --noconfirm' pacman -Syu --needed --noconfirm echo -e '+ su builder' su builder -echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo' -git clone --single-branch --depth 1 --branch main https://examplerepo.com repo +echo -e '+ git clone --single-branch --depth 1 '\''https://examplerepo.com'\'' repo' +git clone --single-branch --depth 1 'https://examplerepo.com' repo echo -e '+ cd repo' cd repo echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' diff --git a/src/build/build_script_git_branch.sh b/src/build/build_script_git_branch.sh new file mode 100644 index 00000000..be1ff4fb --- /dev/null +++ b/src/build/build_script_git_branch.sh @@ -0,0 +1,20 @@ +echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' +echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf +echo -e '+ pacman -Syu --needed --noconfirm' +pacman -Syu --needed --noconfirm +echo -e '+ su builder' +su builder +echo -e '+ git clone --single-branch --depth 1 --branch main '\''https://examplerepo.com'\'' repo' +git clone --single-branch --depth 1 --branch main 'https://examplerepo.com' repo +echo -e '+ cd repo' +cd repo +echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' +makepkg --nobuild --syncdeps --needed --noconfirm +echo -e '+ source PKGBUILD' +source PKGBUILD +echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' +curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 +echo -e '+ [ "$(id -u)" == 0 ] && exit 0' +[ "$(id -u)" == 0 ] && exit 0 +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/build_script_url.sh b/src/build/build_script_url.sh new file mode 100644 index 00000000..3bc97e12 --- /dev/null +++ b/src/build/build_script_url.sh @@ -0,0 +1,22 @@ +echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' +echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf +echo -e '+ pacman -Syu --needed --noconfirm' +pacman -Syu --needed --noconfirm +echo -e '+ su builder' +su builder +echo -e '+ mkdir repo' +mkdir repo +echo -e '+ curl -o repo/PKGBUILD -L '\''https://examplerepo.com'\''' +curl -o repo/PKGBUILD -L 'https://examplerepo.com' +echo -e '+ cd repo' +cd repo +echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' +makepkg --nobuild --syncdeps --needed --noconfirm +echo -e '+ source PKGBUILD' +source PKGBUILD +echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' +curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 +echo -e '+ [ "$(id -u)" == 0 ] && exit 0' +[ "$(id -u)" == 0 ] && exit 0 +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/shell.v b/src/build/shell.v index c5944bf5..e573d53c 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -23,20 +23,45 @@ pub fn echo_commands(cmds []string) []string { } // create_build_script generates a shell script that builds a given Target. -fn create_build_script(address string, repo &Target, build_arch string) string { - repo_url := '$address/$repo.repo' +fn create_build_script(address string, target &Target, build_arch string) string { + repo_url := '$address/$target.repo' - commands := echo_commands([ + mut commands := [ // This will later be replaced by a proper setting for changing the // mirrorlist - "echo -e '[$repo.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" + "echo -e '[$target.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" // We need to update the package list of the repo we just added above. // This should however not pull in a lot of packages as long as the // builder image is rebuilt frequently. 'pacman -Syu --needed --noconfirm', // makepkg can't run as root 'su builder', - 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', + ] + + commands << match target.kind { + 'git' { + if target.branch == '' { + [ + "git clone --single-branch --depth 1 '$target.url' repo", + ] + } else { + [ + "git clone --single-branch --depth 1 --branch $target.branch '$target.url' repo", + ] + } + } + 'url' { + [ + 'mkdir repo', + "curl -o repo/PKGBUILD -L '$target.url'", + ] + } + else { + panic("Invalid kind. This shouldn't be possible.") + } + } + + commands << [ 'cd repo', 'makepkg --nobuild --syncdeps --needed --noconfirm', 'source PKGBUILD', @@ -49,7 +74,7 @@ fn create_build_script(address string, repo &Target, build_arch string) string { // we're in root so we don't proceed. '[ "\$(id -u)" == 0 ] && exit 0', 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', - ]) + ] - return commands.join('\n') + return echo_commands(commands).join('\n') } diff --git a/src/build/shell_test.v b/src/build/shell_test.v index 85340549..341df888 100644 --- a/src/build/shell_test.v +++ b/src/build/shell_test.v @@ -2,15 +2,42 @@ module build import models { Target } -fn test_create_build_script() { +fn test_create_build_script_git_branch() { target := Target{ id: 1 + kind: 'git' url: 'https://examplerepo.com' branch: 'main' repo: 'vieter' } build_script := create_build_script('https://example.com', target, 'x86_64') - expected := $embed_file('build_script.sh') + expected := $embed_file('build_script_git_branch.sh') + + assert build_script == expected.to_string().trim_space() +} + +fn test_create_build_script_git() { + target := Target{ + id: 1 + kind: 'git' + url: 'https://examplerepo.com' + repo: 'vieter' + } + build_script := create_build_script('https://example.com', target, 'x86_64') + expected := $embed_file('build_script_git.sh') + + assert build_script == expected.to_string().trim_space() +} + +fn test_create_build_script_url() { + target := Target{ + id: 1 + kind: 'url' + url: 'https://examplerepo.com' + repo: 'vieter' + } + build_script := create_build_script('https://example.com', target, 'x86_64') + expected := $embed_file('build_script_url.sh') assert build_script == expected.to_string().trim_space() } diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index a9934f1f..ce1cd149 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -123,11 +123,11 @@ pub fn cmd() cli.Command { name: 'edit' required_args: 1 usage: 'id' - description: 'Edit the Git repository target that matches the given id.' + description: 'Edit the target that matches the given id.' flags: [ cli.Flag{ name: 'url' - description: 'URL of the Git repository.' + description: 'URL value. Meaning depends on kind of target.' flag: cli.FlagType.string }, cli.Flag{ @@ -211,7 +211,6 @@ fn add(conf Config, t &NewTarget) ? { // remove removes a repository from the server's list. fn remove(conf Config, id string) ? { - // id, _ := get_repo_by_prefix(conf, id_prefix) ? id_int := id.int() if id_int != 0 { From 449656eb97a016fdf5fc115414fc58d3c4445012 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 17 Jun 2022 14:52:59 +0200 Subject: [PATCH 18/62] docs: updated to new 'kind' field --- CHANGELOG.md | 6 ++++++ docs/api/source/includes/_targets.md | 4 ++++ docs/content/usage/builds/_index.md | 16 ++++++++-------- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d01409a0..eee1f9ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Server port can now be configured +* Targets now have a 'kind' field describing whether it's a Git repository or a + URL to a PKGBUILD +* Targets with kind 'url' can provide a direct URL to a PKGBUILD instead of + providing a Git repository ### Changed * Moved all API routes under `/v1` namespace * Renamed `vieter repos` to `vieter targets` * Renamed `/api/v1/repos` namespace to `/api/v1/targets` +* Branch name for 'git' targets is now optional; if not provided, the + repository will be cloned with the default branch ### Removed diff --git a/docs/api/source/includes/_targets.md b/docs/api/source/includes/_targets.md index f7ea21e5..c7061c86 100644 --- a/docs/api/source/includes/_targets.md +++ b/docs/api/source/includes/_targets.md @@ -24,6 +24,7 @@ curl \ "data": [ { "id": 1, + "kind": "git", "url": "https://aur.archlinux.org/discord-ptb.git", "branch": "master", "repo": "bur", @@ -69,6 +70,7 @@ curl \ "message": "", "data": { "id": 1, + "kind": "git", "url": "https://aur.archlinux.org/discord-ptb.git", "branch": "master", "repo": "bur", @@ -108,6 +110,7 @@ Create a new target with the given data. Parameter | Description --------- | ----------- +kind | Kind of target to add; one of 'git', 'url'. url | URL of the Git repository. branch | Branch of the Git repository. repo | Vieter repository to publish built packages to. @@ -132,6 +135,7 @@ id | id of target to modify Parameter | Description --------- | ----------- +kind | Kind of target; one of 'git', 'url'. url | URL of the Git repository. branch | Branch of the Git repository. repo | Vieter repository to publish built packages to. diff --git a/docs/content/usage/builds/_index.md b/docs/content/usage/builds/_index.md index 7babb062..e6c0b1c7 100644 --- a/docs/content/usage/builds/_index.md +++ b/docs/content/usage/builds/_index.md @@ -20,24 +20,24 @@ pages](https://rustybever.be/man/vieter/vieter-targets.1.html) describe this in greater detail, but the basic usage is as follows: ``` -vieter targets add some-url some-branch some-repository +vieter targets add some-url some-repository ``` Here, `some-url` is the URL of the Git repository containing the PKGBUILD. This URL is passed to `git clone`, meaning the repository should be public. Vieter expects the same format as an AUR Git repository, so you can directly use AUR -URLs here. +URLs here. Alternatively, you can also provide the URL to a PKGBUILD file +instead. See +[vieter-targets-add(1)](https://rustybever.be/man/vieter/vieter-targets-add.1.html) +for more information. -`some-branch` is the branch of the Git repository the build should check out. -If you're using an AUR package, this should be `master`. - -Finally, `some-repo` is the repository to which the built package archives -should be published. +`some-repo` is the repository to which the built package archives should be +published. The above command intentionally leaves out a few parameters to make the CLI more useable. For information on how to modify all parameters using the CLI, see -[vieter-targets-edit(1)](https://rustybever.be/man/vieter/vieter-targets-edit.1.html). +[vieter-targets(1)](https://rustybever.be/man/vieter/vieter-targets.1.html). ## Reading logs From 5e11a91f3d1a7d0932b6d74598f36dff94cf008d Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 17 Jun 2022 16:23:47 +0200 Subject: [PATCH 19/62] refactor: renamed cron & build code to use "target" naming --- src/build/build.v | 6 ++-- src/console/targets/build.v | 8 ++--- src/cron/daemon/build.v | 20 +++++++----- src/cron/daemon/daemon.v | 65 +++++++++++++++++++------------------ 4 files changed, 51 insertions(+), 48 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 3c6b439a..cd0007fb 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -90,10 +90,10 @@ pub: logs string } -// build_repo builds, packages & publishes a given Arch package based on the +// build_target builds, packages & publishes a given Arch package based on the // provided target. The base image ID should be of an image previously created // by create_build_image. It returns the logs of the container. -pub fn build_repo(address string, api_key string, base_image_id string, repo &Target) ?BuildResult { +pub fn build_target(address string, api_key string, base_image_id string, target &Target) ?BuildResult { mut dd := docker.new_conn()? defer { @@ -101,7 +101,7 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Ta } build_arch := os.uname().machine - build_script := create_build_script(address, repo, build_arch) + build_script := create_build_script(address, target, build_arch) // We convert the build script into a base64 string, which then gets passed // to the container as an env var diff --git a/src/console/targets/build.v b/src/console/targets/build.v index d50dfd2c..1f522a6c 100644 --- a/src/console/targets/build.v +++ b/src/console/targets/build.v @@ -6,9 +6,9 @@ import os import build // build locally builds the target with the given id. -fn build(conf Config, repo_id int) ? { +fn build(conf Config, target_id int) ? { c := client.new(conf.address, conf.api_key) - repo := c.get_target(repo_id)? + target := c.get_target(target_id)? build_arch := os.uname().machine @@ -16,7 +16,7 @@ fn build(conf Config, repo_id int) ? { image_id := build.create_build_image(conf.base_image)? println('Running build...') - res := build.build_repo(conf.address, conf.api_key, image_id, repo)? + res := build.build_target(conf.address, conf.api_key, image_id, target)? println('Removing build image...') @@ -29,6 +29,6 @@ fn build(conf Config, repo_id int) ? { dd.remove_image(image_id)? println('Uploading logs to Vieter...') - c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code, + c.add_build_log(target.id, res.start_time, res.end_time, build_arch, res.exit_code, res.logs)? } diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index aa08f9f3..beed9fc8 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -71,29 +71,31 @@ fn (mut d Daemon) start_build(sb ScheduledBuild) bool { return false } -// run_build actually starts the build process for a given repo. +// run_build actually starts the build process for a given target. fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { - d.linfo('started build: $sb.repo.url $sb.repo.branch') + d.linfo('started build: $sb.target.url -> $sb.target.repo') // 0 means success, 1 means failure mut status := 0 - res := build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(), - &sb.repo) or { - d.ldebug('build_repo error: $err.msg()') + res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(), + &sb.target) or { + d.ldebug('build_target error: $err.msg()') status = 1 build.BuildResult{} } if status == 0 { - d.linfo('finished build: $sb.repo.url $sb.repo.branch; uploading logs...') + d.linfo('finished build: $sb.target.url -> $sb.target.repo; uploading logs...') build_arch := os.uname().machine - d.client.add_build_log(sb.repo.id, res.start_time, res.end_time, build_arch, res.exit_code, - res.logs) or { d.lerror('Failed to upload logs for $sb.repo.url $sb.repo.arch') } + d.client.add_build_log(sb.target.id, res.start_time, res.end_time, build_arch, + res.exit_code, res.logs) or { + d.lerror('Failed to upload logs for build: $sb.target.url -> $sb.target.repo') + } } else { - d.linfo('failed build: $sb.repo.url $sb.repo.branch') + d.linfo('an error occured during build: $sb.target.url -> $sb.target.repo') } stdatomic.store_u64(&d.atomics[build_index], daemon.build_done) diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 7b514aa4..9a06dcd8 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -20,7 +20,7 @@ const ( struct ScheduledBuild { pub: - repo Target + target Target timestamp time.Time } @@ -37,9 +37,9 @@ mut: global_schedule CronExpression api_update_frequency int image_rebuild_frequency int - // Repos currently loaded from API. - repos []Target - // At what point to update the list of repositories. + // Targets currently loaded from API. + targets []Target + // At what point to update the list of targets. api_update_timestamp time.Time image_build_timestamp time.Time queue MinHeap @@ -51,7 +51,7 @@ mut: logger shared log.Log } -// init_daemon initializes a new Daemon object. It renews the repositories & +// init_daemon initializes a new Daemon object. It renews the targets & // populates the build queue for the first time. pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon { mut d := Daemon{ @@ -65,8 +65,8 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st logger: logger } - // Initialize the repos & queue - d.renew_repos() + // Initialize the targets & queue + d.renew_targets() d.renew_queue() if !d.rebuild_base_image() { return error('The base image failed to build. The Vieter cron daemon cannot run without an initial builder image.') @@ -76,21 +76,21 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st } // run starts the actual daemon process. It runs builds when possible & -// periodically refreshes the list of repositories to ensure we stay in sync. +// periodically refreshes the list of targets to ensure we stay in sync. pub fn (mut d Daemon) run() { for { finished_builds := d.clean_finished_builds() // Update the API's contents if needed & renew the queue if time.now() >= d.api_update_timestamp { - d.renew_repos() + d.renew_targets() d.renew_queue() } // The finished builds should only be rescheduled if the API contents // haven't been renewed. else { for sb in finished_builds { - d.schedule_build(sb.repo) + d.schedule_build(sb.target) } } @@ -114,7 +114,7 @@ pub fn (mut d Daemon) run() { // every second to clean up any finished builds & start new ones. mut delay := time.Duration(1 * time.second) - // Sleep either until we have to refresh the repos or when the next + // Sleep either until we have to refresh the targets or when the next // build has to start, with a minimum of 1 second. if d.current_build_count() == 0 { now := time.now() @@ -148,12 +148,13 @@ pub fn (mut d Daemon) run() { } } -// schedule_build adds the next occurence of the given repo build to the queue. -fn (mut d Daemon) schedule_build(repo Target) { - ce := if repo.schedule != '' { - parse_expression(repo.schedule) or { +// schedule_build adds the next occurence of the given targets build to the +// queue. +fn (mut d Daemon) schedule_build(target Target) { + ce := if target.schedule != '' { + parse_expression(target.schedule) or { // TODO This shouldn't return an error if the expression is empty. - d.lerror("Error while parsing cron expression '$repo.schedule' (id $repo.id): $err.msg()") + d.lerror("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()") d.global_schedule } @@ -161,41 +162,41 @@ fn (mut d Daemon) schedule_build(repo Target) { d.global_schedule } - // A repo that can't be scheduled will just be skipped for now + // A target that can't be scheduled will just be skipped for now timestamp := ce.next_from_now() or { - d.lerror("Couldn't calculate next timestamp from '$repo.schedule'; skipping") + d.lerror("Couldn't calculate next timestamp from '$target.schedule'; skipping") return } d.queue.insert(ScheduledBuild{ - repo: repo + target: target timestamp: timestamp }) } -// renew_repos requests the newest list of Git repos from the server & replaces +// renew_targets requests the newest list of targets from the server & replaces // the old one. -fn (mut d Daemon) renew_repos() { - d.linfo('Renewing repos...') +fn (mut d Daemon) renew_targets() { + d.linfo('Renewing targets...') - mut new_repos := d.client.get_all_targets() or { - d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...') + mut new_targets := d.client.get_all_targets() or { + d.lerror('Failed to renew targets. Retrying in ${daemon.api_update_retry_timeout}s...') d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) return } - // Filter out any repos that shouldn't run on this architecture + // Filter out any targets that shouldn't run on this architecture cur_arch := os.uname().machine - new_repos = new_repos.filter(it.arch.any(it.value == cur_arch)) + new_targets = new_targets.filter(it.arch.any(it.value == cur_arch)) - d.repos = new_repos + d.targets = new_targets d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency) } // renew_queue replaces the old queue with a new one that reflects the newest -// values in repos_map. +// values in targets. fn (mut d Daemon) renew_queue() { d.linfo('Renewing queue...') mut new_queue := MinHeap{} @@ -225,10 +226,10 @@ fn (mut d Daemon) renew_queue() { d.queue = new_queue - // For each repository in repos_map, parse their cron expression (or use - // the default one if not present) & add them to the queue - for repo in d.repos { - d.schedule_build(repo) + // For each target in targets, parse their cron expression (or use the + // default one if not present) & add them to the queue + for target in d.targets { + d.schedule_build(target) } } From 4200f5c8de675b60fc70e9edc852ea5bff9b8e0b Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 17 Jun 2022 20:19:15 +0200 Subject: [PATCH 20/62] fix(build): explicitely set PATH variable in build containers --- CHANGELOG.md | 1 + src/build/build.v | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eee1f9ff..e5bd6c58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Renamed `/api/v1/repos` namespace to `/api/v1/targets` * Branch name for 'git' targets is now optional; if not provided, the repository will be cloned with the default branch +* Build containers now explicitely set the PATH variable ### Removed diff --git a/src/build/build.v b/src/build/build.v index cd0007fb..8048ec41 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -11,6 +11,9 @@ import models { Target } const ( container_build_dir = '/build' build_image_repo = 'vieter-build' + // Contents of PATH variable in build containers + path_dirs = ['/sbin', '/bin', '/usr/sbin', '/usr/bin', '/usr/local/sbin', + '/usr/local/bin', '/usr/bin/site_perl', '/usr/bin/vendor_perl', '/usr/bin/core_perl'] ) // create_build_image creates a builder image given some base image which can @@ -109,7 +112,13 @@ pub fn build_target(address string, api_key string, base_image_id string, target c := docker.NewContainer{ image: '$base_image_id' - env: ['BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key'] + env: [ + 'BUILD_SCRIPT=$base64_script', + 'API_KEY=$api_key', + // `archlinux:base-devel` does not correctly set the path variable, + // causing certain builds to fail. This fixes it. + 'PATH=${build.path_dirs.join(':')}', + ] entrypoint: ['/bin/sh', '-c'] cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e'] work_dir: '/build' From 39eb03077e1437aa2a25a807972e425ba71524b6 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 20 Jun 2022 17:31:05 +0200 Subject: [PATCH 21/62] docs: mention matrix room in README [CI SKIP] --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 31f262e1..70906f13 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ # Vieter -## Documentation - I host documentation for Vieter over at https://rustybever.be/docs/vieter/. API documentation for the current codebase can be found at https://rustybever.be/api-docs/vieter/. +For more information, questions or just a chat, there's +[#vieter:rustybever.be](https://matrix.to/#/#vieter:rustybever.be) on Matrix! + ## Overview Vieter is a restart of the Pieter project. The goal is to create a simple, From d060366dcb42ef1ee32fb6d8459b29b5e6a9bd92 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 18 Jun 2022 18:28:59 +0200 Subject: [PATCH 22/62] refactor: move docker code to external vdocker module --- src/build/build.v | 2 +- src/console/targets/build.v | 2 +- src/cron/daemon/daemon.v | 2 +- src/docker/README.md | 3 - src/docker/containers.v | 123 -------------------------------- src/docker/docker.v | 137 ------------------------------------ src/docker/images.v | 61 ---------------- src/docker/stream.v | 135 ----------------------------------- src/v.mod | 3 +- 9 files changed, 5 insertions(+), 463 deletions(-) delete mode 100644 src/docker/README.md delete mode 100644 src/docker/containers.v delete mode 100644 src/docker/docker.v delete mode 100644 src/docker/images.v delete mode 100644 src/docker/stream.v diff --git a/src/build/build.v b/src/build/build.v index 8048ec41..6688296e 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -1,6 +1,6 @@ module build -import docker +import vieter.vdocker as docker import encoding.base64 import time import os diff --git a/src/console/targets/build.v b/src/console/targets/build.v index 1f522a6c..7403e2ea 100644 --- a/src/console/targets/build.v +++ b/src/console/targets/build.v @@ -1,7 +1,7 @@ module targets import client -import docker +import vieter.vdocker as docker import os import build diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 9a06dcd8..0ab792aa 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -6,7 +6,7 @@ import datatypes { MinHeap } import cron.expression { CronExpression, parse_expression } import math import build -import docker +import vieter.vdocker as docker import os import client import models { Target } diff --git a/src/docker/README.md b/src/docker/README.md deleted file mode 100644 index 4cc8971a..00000000 --- a/src/docker/README.md +++ /dev/null @@ -1,3 +0,0 @@ -This module implements part of the Docker Engine API v1.41 -([documentation](https://docs.docker.com/engine/api/v1.41/)) using socket-based -HTTP communication. diff --git a/src/docker/containers.v b/src/docker/containers.v deleted file mode 100644 index 8fbf0274..00000000 --- a/src/docker/containers.v +++ /dev/null @@ -1,123 +0,0 @@ -module docker - -import json -import net.urllib -import time -import net.http { Method } - -struct DockerError { - message string -} - -pub struct NewContainer { - image string [json: Image] - entrypoint []string [json: Entrypoint] - cmd []string [json: Cmd] - env []string [json: Env] - work_dir string [json: WorkingDir] - user string [json: User] -} - -struct CreatedContainer { -pub: - id string [json: Id] - warnings []string [json: Warnings] -} - -// create_container creates a new container with the given config. -pub fn (mut d DockerConn) create_container(c NewContainer) ?CreatedContainer { - d.send_request_with_json(Method.post, urllib.parse('/v1.41/containers/create')?, c)? - head, res := d.read_response()? - - if head.status_code != 201 { - data := json.decode(DockerError, res)? - - return error(data.message) - } - - data := json.decode(CreatedContainer, res)? - - return data -} - -// start_container starts the container with the given id. -pub fn (mut d DockerConn) start_container(id string) ? { - d.send_request(Method.post, urllib.parse('/v1.41/containers/$id/start')?)? - head, body := d.read_response()? - - if head.status_code != 204 { - data := json.decode(DockerError, body)? - - return error(data.message) - } -} - -struct ContainerInspect { -pub mut: - state ContainerState [json: State] -} - -struct ContainerState { -pub: - running bool [json: Running] - status string [json: Status] - exit_code int [json: ExitCode] - // These use a rather specific format so they have to be parsed later - start_time_str string [json: StartedAt] - end_time_str string [json: FinishedAt] -pub mut: - start_time time.Time [skip] - end_time time.Time [skip] -} - -// inspect_container returns detailed information for a given container. -pub fn (mut d DockerConn) inspect_container(id string) ?ContainerInspect { - d.send_request(Method.get, urllib.parse('/v1.41/containers/$id/json')?)? - head, body := d.read_response()? - - if head.status_code != 200 { - data := json.decode(DockerError, body)? - - return error(data.message) - } - - mut data := json.decode(ContainerInspect, body)? - - // The Docker engine API *should* always return UTC time. - data.state.start_time = time.parse_rfc3339(data.state.start_time_str)? - - if data.state.status == 'exited' { - data.state.end_time = time.parse_rfc3339(data.state.end_time_str)? - } - - return data -} - -// remove_container removes the container with the given id. -pub fn (mut d DockerConn) remove_container(id string) ? { - d.send_request(Method.delete, urllib.parse('/v1.41/containers/$id')?)? - head, body := d.read_response()? - - if head.status_code != 204 { - data := json.decode(DockerError, body)? - - return error(data.message) - } -} - -// get_container_logs returns a reader object allowing access to the -// container's logs. -pub fn (mut d DockerConn) get_container_logs(id string) ?&StreamFormatReader { - d.send_request(Method.get, urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true')?)? - head := d.read_response_head()? - - if head.status_code != 200 { - content_length := head.header.get(http.CommonHeader.content_length)?.int() - body := d.read_response_body(content_length)? - data := json.decode(DockerError, body)? - - return error(data.message) - } - - return d.get_stream_format_reader() -} diff --git a/src/docker/docker.v b/src/docker/docker.v deleted file mode 100644 index ccc6bedd..00000000 --- a/src/docker/docker.v +++ /dev/null @@ -1,137 +0,0 @@ -module docker - -import net.unix -import io -import net.http -import strings -import net.urllib -import json -import util - -const ( - socket = '/var/run/docker.sock' - buf_len = 10 * 1024 - http_separator = [u8(`\r`), `\n`, `\r`, `\n`] - http_chunk_separator = [u8(`\r`), `\n`] -) - -pub struct DockerConn { -mut: - socket &unix.StreamConn - reader &io.BufferedReader -} - -// new_conn creates a new connection to the Docker daemon. -pub fn new_conn() ?&DockerConn { - s := unix.connect_stream(docker.socket)? - - d := &DockerConn{ - socket: s - reader: io.new_buffered_reader(reader: s) - } - - return d -} - -// close closes the underlying socket connection. -pub fn (mut d DockerConn) close() ? { - d.socket.close()? -} - -// send_request sends an HTTP request without body. -pub fn (mut d DockerConn) send_request(method http.Method, url urllib.URL) ? { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' - - d.socket.write_string(req)? - - // When starting a new request, the reader needs to be reset. - d.reader = io.new_buffered_reader(reader: d.socket) -} - -// send_request_with_body sends an HTTP request with the given body. -pub fn (mut d DockerConn) send_request_with_body(method http.Method, url urllib.URL, content_type string, body string) ? { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' - - d.socket.write_string(req)? - - // When starting a new request, the reader needs to be reset. - d.reader = io.new_buffered_reader(reader: d.socket) -} - -// send_request_with_json is a convenience wrapper around -// send_request_with_body that encodes the input as JSON. -pub fn (mut d DockerConn) send_request_with_json(method http.Method, url urllib.URL, data &T) ? { - body := json.encode(data) - - return d.send_request_with_body(method, url, 'application/json', body) -} - -// read_response_head consumes the socket's contents until it encounters -// '\r\n\r\n', after which it parses the response as an HTTP response. -// Importantly, this function never consumes the reader past the HTTP -// separator, so the body can be read fully later on. -pub fn (mut d DockerConn) read_response_head() ?http.Response { - mut res := []u8{} - - util.read_until_separator(mut d.reader, mut res, docker.http_separator)? - - return http.parse_response(res.bytestr()) -} - -// read_response_body reads `length` bytes from the stream. It can be used when -// the response encoding isn't chunked to fully read it. -pub fn (mut d DockerConn) read_response_body(length int) ?string { - if length == 0 { - return '' - } - - mut buf := []u8{len: docker.buf_len} - mut c := 0 - mut builder := strings.new_builder(docker.buf_len) - - for builder.len < length { - c = d.reader.read(mut buf) or { break } - - builder.write(buf[..c])? - } - - return builder.str() -} - -// read_response is a convenience function which always consumes the entire -// response & returns it. It should only be used when we're certain that the -// result isn't too large. -pub fn (mut d DockerConn) read_response() ?(http.Response, string) { - head := d.read_response_head()? - - if head.header.get(http.CommonHeader.transfer_encoding) or { '' } == 'chunked' { - mut builder := strings.new_builder(1024) - mut body := d.get_chunked_response_reader() - - util.reader_to_writer(mut body, mut builder)? - - return head, builder.str() - } - - content_length := head.header.get(http.CommonHeader.content_length)?.int() - res := d.read_response_body(content_length)? - - return head, res -} - -// get_chunked_response_reader returns a ChunkedResponseReader using the socket -// as reader. -pub fn (mut d DockerConn) get_chunked_response_reader() &ChunkedResponseReader { - r := new_chunked_response_reader(d.reader) - - return r -} - -// get_stream_format_reader returns a StreamFormatReader using the socket as -// reader. -pub fn (mut d DockerConn) get_stream_format_reader() &StreamFormatReader { - r := new_chunked_response_reader(d.reader) - r2 := new_stream_format_reader(r) - - return r2 -} diff --git a/src/docker/images.v b/src/docker/images.v deleted file mode 100644 index 6161565a..00000000 --- a/src/docker/images.v +++ /dev/null @@ -1,61 +0,0 @@ -module docker - -import net.http { Method } -import net.urllib -import json - -struct Image { -pub: - id string [json: Id] -} - -// pull_image pulls the given image:tag. -pub fn (mut d DockerConn) pull_image(image string, tag string) ? { - d.send_request(Method.post, urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?)? - head := d.read_response_head()? - - if head.status_code != 200 { - content_length := head.header.get(http.CommonHeader.content_length)?.int() - body := d.read_response_body(content_length)? - data := json.decode(DockerError, body)? - - return error(data.message) - } - - // Keep reading the body until the pull has completed - mut body := d.get_chunked_response_reader() - - mut buf := []u8{len: 1024} - - for { - body.read(mut buf) or { break } - } -} - -// create_image_from_container creates a new image from a container. -pub fn (mut d DockerConn) create_image_from_container(id string, repo string, tag string) ?Image { - d.send_request(Method.post, urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag')?)? - head, body := d.read_response()? - - if head.status_code != 201 { - data := json.decode(DockerError, body)? - - return error(data.message) - } - - data := json.decode(Image, body)? - - return data -} - -// remove_image removes the image with the given id. -pub fn (mut d DockerConn) remove_image(id string) ? { - d.send_request(Method.delete, urllib.parse('/v1.41/images/$id')?)? - head, body := d.read_response()? - - if head.status_code != 200 { - data := json.decode(DockerError, body)? - - return error(data.message) - } -} diff --git a/src/docker/stream.v b/src/docker/stream.v deleted file mode 100644 index 001f4b3e..00000000 --- a/src/docker/stream.v +++ /dev/null @@ -1,135 +0,0 @@ -module docker - -import io -import util -import encoding.binary -import encoding.hex - -// ChunkedResponseReader parses an underlying HTTP chunked response, exposing -// it as if it was a continuous stream of data. -struct ChunkedResponseReader { -mut: - reader io.BufferedReader - bytes_left_in_chunk u64 - started bool -} - -// new_chunked_response_reader creates a new ChunkedResponseReader on the heap -// with the provided reader. -pub fn new_chunked_response_reader(reader io.BufferedReader) &ChunkedResponseReader { - r := &ChunkedResponseReader{ - reader: reader - } - - return r -} - -// read satisfies the io.Reader interface. -pub fn (mut r ChunkedResponseReader) read(mut buf []u8) ?int { - if r.bytes_left_in_chunk == 0 { - // An io.BufferedReader always returns none if its stream has - // ended. - r.bytes_left_in_chunk = r.read_chunk_size()? - } - - mut c := 0 - - // Make sure we don't read more than we can safely read. This is to avoid - // the underlying reader from becoming out of sync with our parsing: - if buf.len > r.bytes_left_in_chunk { - c = r.reader.read(mut buf[..r.bytes_left_in_chunk])? - } else { - c = r.reader.read(mut buf)? - } - - r.bytes_left_in_chunk -= u64(c) - - return c -} - -// read_chunk_size advances the reader & reads the size of the next HTTP chunk. -// This function should only be called if the previous chunk has been -// completely consumed. -fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { - if r.started { - mut buf := []u8{len: 2} - - // Each chunk ends with a `\r\n` which we want to skip first - r.reader.read(mut buf)? - } - - r.started = true - - mut res := []u8{} - util.read_until_separator(mut r.reader, mut res, http_chunk_separator)? - - // The length of the next chunk is provided as a hexadecimal - mut num_data := hex.decode(res#[..-2].bytestr())? - - for num_data.len < 8 { - num_data.insert(0, 0) - } - - num := binary.big_endian_u64(num_data) - - // This only occurs for the very last chunk, which always reports a size of - // 0. - if num == 0 { - return none - } - - return num -} - -// StreamFormatReader parses an underlying stream of Docker logs, removing the -// header bytes. -struct StreamFormatReader { -mut: - reader ChunkedResponseReader - bytes_left_in_chunk u32 -} - -// new_stream_format_reader creates a new StreamFormatReader using the given -// reader. -pub fn new_stream_format_reader(reader ChunkedResponseReader) &StreamFormatReader { - r := &StreamFormatReader{ - reader: reader - } - - return r -} - -// read satisfies the io.Reader interface. -pub fn (mut r StreamFormatReader) read(mut buf []u8) ?int { - if r.bytes_left_in_chunk == 0 { - r.bytes_left_in_chunk = r.read_chunk_size()? - } - - mut c := 0 - - if buf.len > r.bytes_left_in_chunk { - c = r.reader.read(mut buf[..r.bytes_left_in_chunk])? - } else { - c = r.reader.read(mut buf)? - } - - r.bytes_left_in_chunk -= u32(c) - - return c -} - -// read_chunk_size advances the reader & reads the header bytes for the length -// of the next chunk. -fn (mut r StreamFormatReader) read_chunk_size() ?u32 { - mut buf := []u8{len: 8} - - r.reader.read(mut buf)? - - num := binary.big_endian_u32(buf[4..]) - - if num == 0 { - return none - } - - return num -} diff --git a/src/v.mod b/src/v.mod index 7f459174..15e3518a 100644 --- a/src/v.mod +++ b/src/v.mod @@ -1,5 +1,6 @@ Module { dependencies: [ - 'https://git.rustybever.be/vieter/vconf' + 'https://git.rustybever.be/vieter/vconf', + 'https://git.rustybever.be/vieter/vdocker' ] } From 461f22716995e51db40519312f49f08fd18d9e21 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 22 Jun 2022 09:18:47 +0200 Subject: [PATCH 23/62] refactor: use new module names --- src/build/build.v | 22 +++++++++++----------- src/console/logs/logs.v | 2 +- src/console/targets/build.v | 2 +- src/console/targets/targets.v | 2 +- src/cron/cli.v | 2 +- src/cron/daemon/daemon.v | 2 +- src/server/cli.v | 2 +- src/v.mod | 4 ++-- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 6688296e..2ad70a6f 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -1,6 +1,6 @@ module build -import vieter.vdocker as docker +import vieter_v.docker import encoding.base64 import time import os @@ -59,13 +59,13 @@ pub fn create_build_image(base_image string) ?string { // We pull the provided image dd.pull_image(image_name, image_tag)? - id := dd.create_container(c)?.id + id := dd.container_create(c)?.id // id := docker.create_container(c)? - dd.start_container(id)? + dd.container_start(id)? // This loop waits until the container has stopped, so we can remove it after for { - data := dd.inspect_container(id)? + data := dd.container_inspect(id)? if !data.state.running { break @@ -80,7 +80,7 @@ pub fn create_build_image(base_image string) ?string { // conflicts. tag := time.sys_mono_now().str() image := dd.create_image_from_container(id, 'vieter-build', tag)? - dd.remove_container(id)? + dd.container_remove(id)? return image.id } @@ -125,25 +125,25 @@ pub fn build_target(address string, api_key string, base_image_id string, target user: '0:0' } - id := dd.create_container(c)?.id - dd.start_container(id)? + id := dd.container_create(c)?.id + dd.container_start(id)? - mut data := dd.inspect_container(id)? + mut data := dd.container_inspect(id)? // This loop waits until the container has stopped, so we can remove it after for data.state.running { time.sleep(1 * time.second) - data = dd.inspect_container(id)? + data = dd.container_inspect(id)? } - mut logs_stream := dd.get_container_logs(id)? + mut logs_stream := dd.container_get_logs(id)? // Read in the entire stream mut logs_builder := strings.new_builder(10 * 1024) util.reader_to_writer(mut logs_stream, mut logs_builder)? - dd.remove_container(id)? + dd.container_remove(id)? return BuildResult{ start_time: data.state.start_time diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 5155ae88..0f023bc7 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -1,7 +1,7 @@ module logs import cli -import vieter.vconf +import vieter_v.conf as vconf import client import console import time diff --git a/src/console/targets/build.v b/src/console/targets/build.v index 7403e2ea..6337aa3e 100644 --- a/src/console/targets/build.v +++ b/src/console/targets/build.v @@ -1,7 +1,7 @@ module targets import client -import vieter.vdocker as docker +import vieter_v.docker import os import build diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index ce1cd149..66d48fb3 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -1,7 +1,7 @@ module targets import cli -import vieter.vconf +import vieter_v.conf as vconf import cron.expression { parse_expression } import client { NewTarget } import console diff --git a/src/cron/cli.v b/src/cron/cli.v index c2e3d39a..4d958339 100644 --- a/src/cron/cli.v +++ b/src/cron/cli.v @@ -1,7 +1,7 @@ module cron import cli -import vieter.vconf +import vieter_v.conf as vconf struct Config { pub: diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 0ab792aa..934d35a0 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -6,7 +6,7 @@ import datatypes { MinHeap } import cron.expression { CronExpression, parse_expression } import math import build -import vieter.vdocker as docker +import vieter_v.docker import os import client import models { Target } diff --git a/src/server/cli.v b/src/server/cli.v index a62f56ec..6fd09c54 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -1,7 +1,7 @@ module server import cli -import vieter.vconf +import vieter_v.conf as vconf struct Config { pub: diff --git a/src/v.mod b/src/v.mod index 15e3518a..5b890629 100644 --- a/src/v.mod +++ b/src/v.mod @@ -1,6 +1,6 @@ Module { dependencies: [ - 'https://git.rustybever.be/vieter/vconf', - 'https://git.rustybever.be/vieter/vdocker' + 'https://git.rustybever.be/vieter-v/conf', + 'https://git.rustybever.be/vieter-v/docker' ] } From e58ac496807501ba91c7fcbb7c6a2a51c25b53ca Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 22 Jun 2022 09:27:22 +0200 Subject: [PATCH 24/62] chore: hopefully changed all URLs to new org --- .woodpecker/.arch-rel.yml | 2 +- .woodpecker/.arch.yml | 2 +- CHANGELOG.md | 16 ++++++++-------- Makefile | 2 +- PKGBUILD | 4 ++-- PKGBUILD.dev | 4 ++-- README.md | 6 +++--- docs/config.toml | 4 ++-- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.woodpecker/.arch-rel.yml b/.woodpecker/.arch-rel.yml index b8f4c7ae..f5f228ef 100644 --- a/.woodpecker/.arch-rel.yml +++ b/.woodpecker/.arch-rel.yml @@ -23,7 +23,7 @@ pipeline: - su builder # Due to a bug with the V compiler, we can't just use the PKGBUILD from # inside the repo - - curl -OL "https://git.rustybever.be/vieter/vieter/raw/tag/$CI_COMMIT_TAG/PKGBUILD" + - curl -OL "https://git.rustybever.be/vieter-v/vieter/raw/tag/$CI_COMMIT_TAG/PKGBUILD" - makepkg -s --noconfirm --needed when: event: tag diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index b2a59ba5..8f1a6ff0 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -23,7 +23,7 @@ pipeline: - su builder # Due to a bug with the V compiler, we can't just use the PKGBUILD from # inside the repo - - curl -o PKGBUILD -L https://git.rustybever.be/vieter/vieter/raw/branch/dev/PKGBUILD.dev + - curl -o PKGBUILD -L https://git.rustybever.be/vieter-v/vieter/raw/branch/dev/PKGBUILD.dev - makepkg -s --noconfirm --needed when: event: push diff --git a/CHANGELOG.md b/CHANGELOG.md index e5bd6c58..cf2b8293 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev) +## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev) ### Added @@ -28,11 +28,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * md5 hashes are no longer calculated for packages -## [0.3.0](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0) +## [0.3.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.3.0) Nothing besides bumping the versions. -## [0.3.0-rc.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-rc.1) +## [0.3.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.3.0-rc.1) ### Added @@ -60,7 +60,7 @@ Nothing besides bumping the versions. * `POST /api/logs` now correctly uses epoch timestamps instead of strings -## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) +## [0.3.0-alpha.2](https://git.rustybever.be/vieter-v/vieter/src/tag/0.3.0-alpha.2) ### Added @@ -85,7 +85,7 @@ Nothing besides bumping the versions. * `vieter-git` is the latest commit on the dev branch * Full refactor of Docker socket code -## [0.3.0-alpha.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.1) +## [0.3.0-alpha.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.3.0-alpha.1) ### Changed @@ -104,7 +104,7 @@ Nothing besides bumping the versions. * Binary no longer panics when an env var is missing -## [0.2.0](https://git.rustybever.be/vieter/vieter/src/tag/0.2.0) +## [0.2.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.2.0) ### Changed @@ -138,13 +138,13 @@ Nothing besides bumping the versions. * Packages with unknown fields in .PKGINFO are now allowed * Old packages are now properly removed -## [0.1.0](https://git.rustybever.be/vieter/vieter/src/tag/0.1.0) +## [0.1.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.1.0) ### Changed * Improved logging -## [0.1.0-rc.1](https://git.rustybever.be/vieter/vieter/src/tag/0.1.0-rc.1) +## [0.1.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.1.0-rc.1) ### Added diff --git a/Makefile b/Makefile index 7eb35472..96410d79 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ test: .PHONY: v v: v/v v/v: - git clone --single-branch https://git.rustybever.be/Chewing_Bever/v v + git clone --single-branch https://git.rustybever.be/vieter-v/v v make -C v .PHONY: clean diff --git a/PKGBUILD b/PKGBUILD index 33faec55..6b664d1f 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -9,9 +9,9 @@ pkgdesc="Vieter is a lightweight implementation of an Arch repository server." depends=('glibc' 'openssl' 'libarchive' 'sqlite') makedepends=('git' 'vieter-v') arch=('x86_64' 'aarch64') -url='https://git.rustybever.be/vieter/vieter' +url='https://git.rustybever.be/vieter-v/vieter' license=('AGPL3') -source=("$pkgname::git+https://git.rustybever.be/vieter/vieter#tag=${pkgver//_/-}") +source=("$pkgname::git+https://git.rustybever.be/vieter-v/vieter#tag=${pkgver//_/-}") md5sums=('SKIP') prepare() { diff --git a/PKGBUILD.dev b/PKGBUILD.dev index bff1459e..045e5765 100644 --- a/PKGBUILD.dev +++ b/PKGBUILD.dev @@ -9,9 +9,9 @@ pkgdesc="Vieter is a lightweight implementation of an Arch repository server." depends=('glibc' 'openssl' 'libarchive' 'sqlite') makedepends=('git' 'vieter-v') arch=('x86_64' 'aarch64') -url='https://git.rustybever.be/vieter/vieter' +url='https://git.rustybever.be/vieter-v/vieter' license=('AGPL3') -source=("$pkgname::git+https://git.rustybever.be/vieter/vieter#branch=dev") +source=("$pkgname::git+https://git.rustybever.be/vieter-v/vieter#branch=dev") md5sums=('SKIP') provides=('vieter') conflicts=('vieter') diff --git a/README.md b/README.md index 70906f13..5911ea21 100644 --- a/README.md +++ b/README.md @@ -49,9 +49,9 @@ update`. ### Compiler Vieter compiles with the standard Vlang compiler. However, I do maintain a -[mirror](https://git.rustybever.be/vieter/v). This is to ensure my CI does not -break without reason, as I control when & how frequently the mirror is updated -to reflect the official repository. +[mirror](https://git.rustybever.be/vieter-v/v). This is to ensure my CI does +not break without reason, as I control when & how frequently the mirror is +updated to reflect the official repository. If you encounter issues using the latest V compiler, try using my mirror instead. `make v` will clone the repository & build the mirror. Afterwards, diff --git a/docs/config.toml b/docs/config.toml index f8e23cdc..7d23d06a 100644 --- a/docs/config.toml +++ b/docs/config.toml @@ -38,7 +38,7 @@ enableGitInfo = true weight = 20 [[menu.after]] name = "Vieter" - url = "https://git.rustybever.be/vieter/vieter" + url = "https://git.rustybever.be/vieter-v/vieter" weight = 30 [[menu.after]] name = "Hugo Theme" @@ -70,7 +70,7 @@ enableGitInfo = true # Set source repository location. # Used for 'Last Modified' and 'Edit this page' links. - BookRepo = 'https://git.rustybever.be/vieter/vieter' + BookRepo = 'https://git.rustybever.be/vieter-v/vieter' # (Optional, default 'commit') Specifies commit portion of the link to the page's last modified # commit hash for 'doc' page type. From a4c2508fe7f42b0fe92271fb47d6ff97b82df730 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 22 Jun 2022 09:33:48 +0200 Subject: [PATCH 25/62] chore: removed unnecessary things from Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 96410d79..ed44df92 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ v/v: .PHONY: clean clean: - rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' 'docs/public' + rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' 'docs/public' # =====EXPERIMENTAL===== From 9dd9222a69c52aa3424878fca166b3212f00b91c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 22 Jun 2022 09:44:25 +0200 Subject: [PATCH 26/62] fix(ci): use extended hugo; install modules for tests --- .woodpecker/.docs.yml | 2 +- .woodpecker/.test.yml | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.woodpecker/.docs.yml b/.woodpecker/.docs.yml index c342e388..051d852e 100644 --- a/.woodpecker/.docs.yml +++ b/.woodpecker/.docs.yml @@ -4,7 +4,7 @@ branches: pipeline: docs: - image: 'klakegg/hugo:alpine' + image: 'klakegg/hugo:ext-alpine' group: 'generate' commands: - apk add git diff --git a/.woodpecker/.test.yml b/.woodpecker/.test.yml index 6b7b646d..4cd6eaf0 100644 --- a/.woodpecker/.test.yml +++ b/.woodpecker/.test.yml @@ -8,6 +8,15 @@ branches: platform: ${PLATFORM} pipeline: + install-modules: + image: 'chewingbever/vlang:latest' + pull: true + commands: + - export VMODULES=$PWD/.vmodules + - 'cd src && v install' + when: + event: [pull_request] + test: image: 'chewingbever/vlang:latest' pull: true From c8fc4c6a96d75082dba977f63e230eca9d632102 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 22 Jun 2022 09:59:44 +0200 Subject: [PATCH 27/62] fix(ci): set VMODULES for tests --- .woodpecker/.test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker/.test.yml b/.woodpecker/.test.yml index 4cd6eaf0..a19dbd42 100644 --- a/.woodpecker/.test.yml +++ b/.woodpecker/.test.yml @@ -21,6 +21,7 @@ pipeline: image: 'chewingbever/vlang:latest' pull: true commands: + - export VMODULES=$PWD/.vmodules - make test when: event: [pull_request] From 25d87fb5e6cef2e9db90dc3b2c51eff1c49c8099 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 22 Jun 2022 20:17:11 +0200 Subject: [PATCH 28/62] fix: make code compile with updated V version --- src/archive.c.v | 4 ++-- src/db/db.v | 2 +- src/package/package.v | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/archive.c.v b/src/archive.c.v index 1f0d1dd5..a40cdef8 100644 --- a/src/archive.c.v +++ b/src/archive.c.v @@ -4,7 +4,7 @@ #include "archive.h" -struct C.archive {} +pub struct C.archive {} // Create a new archive struct for reading fn C.archive_read_new() &C.archive @@ -71,7 +71,7 @@ fn C.archive_filter_code(&C.archive, int) int #include "archive_entry.h" -struct C.archive_entry {} +pub struct C.archive_entry {} // Create a new archive_entry struct fn C.archive_entry_new() &C.archive_entry diff --git a/src/db/db.v b/src/db/db.v index eaf71abb..9459c052 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -3,7 +3,7 @@ module db import sqlite import time -struct VieterDb { +pub struct VieterDb { conn sqlite.DB } diff --git a/src/package/package.v b/src/package/package.v index 86bf40ad..9eaf5a23 100644 --- a/src/package/package.v +++ b/src/package/package.v @@ -4,7 +4,7 @@ import os import util // Represents a read archive -struct Pkg { +pub struct Pkg { pub: path string [required] info PkgInfo [required] From 6336d801d3631c205b0c29ce528878a99a581fba Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 22 Jun 2022 20:43:14 +0200 Subject: [PATCH 29/62] docs: mention AUR packages --- docs/content/installation.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/content/installation.md b/docs/content/installation.md index b5bdbaf0..87b9cba9 100644 --- a/docs/content/installation.md +++ b/docs/content/installation.md @@ -96,6 +96,14 @@ SigLevel = Optional Afterwards, you can update your system & install the `vieter` package for the latest official release or `vieter-git` for the latest development release. +### AUR + +If you prefer building the packages locally (or on your own Vieter instance), +there's the `[vieter](https://aur.archlinux.org/packages/vieter)` & +`[vieter-git](https://aur.archlinux.org/packages/vieter-git)` packages on the +AUR. These packages build using the `vlang-git` compiler package, so I can't +guarantee that a compiler update won't temporarily break them. + ## Building from source The project [README](https://git.rustybever.be/vieter/vieter#building) contains From 487b2357272050d1fcabf9cead73872fa5ea524e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 22 Jun 2022 16:19:07 +0200 Subject: [PATCH 30/62] feat(cli): add aur search command --- .gitignore | 3 +++ src/console/aur/aur.v | 26 ++++++++++++++++++++++++++ src/main.v | 2 ++ src/v.mod | 3 ++- 4 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 src/console/aur/aur.v diff --git a/.gitignore b/.gitignore index 4d9f94f2..a2804fe6 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ gdb.txt # Generated docs _docs/ /man/ + +# VLS logs +vls.log diff --git a/src/console/aur/aur.v b/src/console/aur/aur.v new file mode 100644 index 00000000..3f20dfca --- /dev/null +++ b/src/console/aur/aur.v @@ -0,0 +1,26 @@ +module aur + +import cli +import console +import vieter_v.aur + +pub fn cmd() cli.Command { + return cli.Command{ + name: 'aur' + description: 'Interact with the AUR.' + commands: [ + cli.Command{ + name: 'search' + description: 'Search for packages.' + required_args: 1 + execute: fn (cmd cli.Command) ? { + c := aur.new() + pkgs := c.search(cmd.args[0])? + data := pkgs.map([it.name, it.description]) + + println(console.pretty_table(['name', 'description'], data)?) + } + }, + ] + } +} diff --git a/src/main.v b/src/main.v index cba410ce..4ade9305 100644 --- a/src/main.v +++ b/src/main.v @@ -7,6 +7,7 @@ import console.targets import console.logs import console.schedule import console.man +import console.aur import cron fn main() { @@ -31,6 +32,7 @@ fn main() { logs.cmd(), schedule.cmd(), man.cmd(), + aur.cmd(), ] } app.setup() diff --git a/src/v.mod b/src/v.mod index 5b890629..710c976c 100644 --- a/src/v.mod +++ b/src/v.mod @@ -1,6 +1,7 @@ Module { dependencies: [ 'https://git.rustybever.be/vieter-v/conf', - 'https://git.rustybever.be/vieter-v/docker' + 'https://git.rustybever.be/vieter-v/docker', + 'https://git.rustybever.be/vieter-v/aur' ] } From 1a940f2f98e6c2c7794e4f06b2913a32316cfbfd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 22 Jun 2022 16:38:53 +0200 Subject: [PATCH 31/62] feat(cli): added "aur add" command --- CHANGELOG.md | 1 + src/console/aur/aur.v | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf2b8293..a03c18cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 URL to a PKGBUILD * Targets with kind 'url' can provide a direct URL to a PKGBUILD instead of providing a Git repository +* CLI commands for searching the AUR & directly adding packages ### Changed diff --git a/src/console/aur/aur.v b/src/console/aur/aur.v index 3f20dfca..c98f8e68 100644 --- a/src/console/aur/aur.v +++ b/src/console/aur/aur.v @@ -2,8 +2,16 @@ module aur import cli import console +import client import vieter_v.aur +import vieter_v.conf as vconf +struct Config { + address string [required] + api_key string [required] +} + +// cmd returns the cli module for interacting with the AUR API. pub fn cmd() cli.Command { return cli.Command{ name: 'aur' @@ -21,6 +29,34 @@ pub fn cmd() cli.Command { println(console.pretty_table(['name', 'description'], data)?) } }, + cli.Command{ + name: 'add' + usage: 'repo pkg-name [pkg-name...]' + description: 'Add the given AUR package(s) to Vieter. Non-existent packages will be silently ignored.' + required_args: 2 + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + + c := aur.new() + pkgs := c.info(cmd.args[1..])? + + vc := client.new(conf.address, conf.api_key) + + for pkg in pkgs { + vc.add_target( + kind: 'git' + url: 'https://aur.archlinux.org/$pkg.package_base' + '.git' + repo: cmd.args[0] + ) or { + println('Failed to add $pkg.name: $err.msg()') + continue + } + + println('Added $pkg.name' + '.') + } + } + }, ] } } From 0d0fb323f235c0381d8737ccd777bfa6167a7e4b Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 3 Jul 2022 14:11:29 +0200 Subject: [PATCH 32/62] chore: switched to vlang 0.3 Docker image --- .woodpecker/.build.yml | 8 ++++---- .woodpecker/.docs.yml | 2 +- .woodpecker/.gitea.yml | 2 +- .woodpecker/.lint.yml | 2 +- .woodpecker/.man.yml | 2 +- .woodpecker/.test.yml | 4 ++-- Dockerfile | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 7cb7d535..580fa69e 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -7,7 +7,7 @@ platform: ${PLATFORM} pipeline: install-modules: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true commands: - export VMODULES=$PWD/.vmodules @@ -16,7 +16,7 @@ pipeline: event: [push, pull_request] debug: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' commands: - export VMODULES=$PWD/.vmodules - make @@ -26,7 +26,7 @@ pipeline: exclude: [main] prod: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' environment: - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static commands: @@ -44,7 +44,7 @@ pipeline: event: [push, pull_request] upload: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' secrets: [ s3_username, s3_password ] commands: # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f diff --git a/.woodpecker/.docs.yml b/.woodpecker/.docs.yml index 051d852e..da495fcb 100644 --- a/.woodpecker/.docs.yml +++ b/.woodpecker/.docs.yml @@ -11,7 +11,7 @@ pipeline: - make docs api-docs: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true group: 'generate' commands: diff --git a/.woodpecker/.gitea.yml b/.woodpecker/.gitea.yml index d0825c2c..55f991e2 100644 --- a/.woodpecker/.gitea.yml +++ b/.woodpecker/.gitea.yml @@ -8,7 +8,7 @@ skip_clone: true pipeline: prepare: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true secrets: [ s3_username, s3_password ] commands: diff --git a/.woodpecker/.lint.yml b/.woodpecker/.lint.yml index e70648d8..75a81059 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/.lint.yml @@ -5,7 +5,7 @@ platform: 'linux/amd64' pipeline: lint: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true commands: - make lint diff --git a/.woodpecker/.man.yml b/.woodpecker/.man.yml index 0b808866..1a30b03d 100644 --- a/.woodpecker/.man.yml +++ b/.woodpecker/.man.yml @@ -9,7 +9,7 @@ skip_clone: true pipeline: generate: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true commands: - curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64" diff --git a/.woodpecker/.test.yml b/.woodpecker/.test.yml index a19dbd42..6c267fab 100644 --- a/.woodpecker/.test.yml +++ b/.woodpecker/.test.yml @@ -9,7 +9,7 @@ platform: ${PLATFORM} pipeline: install-modules: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true commands: - export VMODULES=$PWD/.vmodules @@ -18,7 +18,7 @@ pipeline: event: [pull_request] test: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true commands: - export VMODULES=$PWD/.vmodules diff --git a/Dockerfile b/Dockerfile index 5997adca..7aed9179 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM chewingbever/vlang:latest AS builder +FROM chewingbever/vlang:0.3 AS builder ARG TARGETPLATFORM ARG CI_COMMIT_SHA From 0f6630b9404acb2b2217f52091c38d636ffc81f0 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 17 Jul 2022 13:38:46 +0200 Subject: [PATCH 33/62] chore: updated PKGBUILDs to use vlang package --- PKGBUILD | 4 ++-- PKGBUILD.dev | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/PKGBUILD b/PKGBUILD index 6b664d1f..14ea71b2 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -5,9 +5,9 @@ pkgbase='vieter' pkgname='vieter' pkgver='0.3.0' pkgrel=1 -pkgdesc="Vieter is a lightweight implementation of an Arch repository server." +pkgdesc="Lightweight Arch repository server & package build system" depends=('glibc' 'openssl' 'libarchive' 'sqlite') -makedepends=('git' 'vieter-v') +makedepends=('git' 'vlang') arch=('x86_64' 'aarch64') url='https://git.rustybever.be/vieter-v/vieter' license=('AGPL3') diff --git a/PKGBUILD.dev b/PKGBUILD.dev index 045e5765..79c7f37f 100644 --- a/PKGBUILD.dev +++ b/PKGBUILD.dev @@ -5,9 +5,9 @@ pkgbase='vieter-git' pkgname='vieter-git' pkgver=0.2.0.r25.g20112b8 pkgrel=1 -pkgdesc="Vieter is a lightweight implementation of an Arch repository server." +pkgdesc="Lightweight Arch repository server & package build system (development version)" depends=('glibc' 'openssl' 'libarchive' 'sqlite') -makedepends=('git' 'vieter-v') +makedepends=('git' 'vlang') arch=('x86_64' 'aarch64') url='https://git.rustybever.be/vieter-v/vieter' license=('AGPL3') From 49ddb312dea5d298952ad460922ed3f518677214 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 11 Aug 2022 19:07:54 +0200 Subject: [PATCH 34/62] feat(server): added endpoint to remove package from arch-repo --- docs/api/source/includes/_repository.md | 29 +++++++++++++++++++++++++ src/repo/repo.v | 2 +- src/server/routes.v | 23 ++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/docs/api/source/includes/_repository.md b/docs/api/source/includes/_repository.md index fbbc329a..9764e014 100644 --- a/docs/api/source/includes/_repository.md +++ b/docs/api/source/includes/_repository.md @@ -93,3 +93,32 @@ other already present arch-repos. Parameter | Description --------- | ----------- repo | Repository to publish package to + +## Remove package from arch-repo + + + +```shell +curl \ + -H 'X-Api-Key: secret' \ + -XDELETE \ + https://example.com/vieter/x86_64/mike +``` + +This endpoint allows you to remove a package from a given arch-repo. + +### HTTP Request + +`DELETE /:repo/:arch/:pkg` + +### URL Parameters + +Parameter | Description +--------- | ----------- +repo | Repository to delete package from +arch | Specific arch-repo to remove package from +pkg | Name of package to remove (without any version information) diff --git a/src/repo/repo.v b/src/repo/repo.v index c4b85c0b..7de12cdb 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -158,7 +158,7 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac // remove_pkg_from_arch_repo removes a package from an arch-repo's database. It // returns false if the package wasn't present in the database. It also // optionally re-syncs the repo archives. -fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool { +pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool { repo_dir := os.join_path(r.repos_dir, repo, arch) // If the repository doesn't exist yet, the result is automatically false diff --git a/src/server/routes.v b/src/server/routes.v index fbf37dfb..3b86e20b 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -57,6 +57,29 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re return app.file(full_path) } +['/:repo/:arch/:pkg'; delete] +fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { + app.lerror('Error while deleting package: $err.msg()') + + return app.json(http.Status.internal_server_error, new_response('Failed to delete package.')) + } + + if res { + app.linfo("Removed package '$pkg' from '$repo/$arch'") + + return app.json(http.Status.ok, new_response('Package removed.')) + } else { + app.linfo("Tried removing package '$pkg' from '$repo/$arch', but it doesn't exist.") + + return app.json(http.Status.not_found, new_response('Package not found.')) + } +} + // put_package handles publishing a package to a repository. ['/:repo/publish'; post] fn (mut app App) put_package(repo string) web.Result { From 6283cbea9cf0c655ea43a146bada31aef5ee9d7f Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 11 Aug 2022 19:16:38 +0200 Subject: [PATCH 35/62] feat(repo): added function to remove arch-repo --- src/repo/{repo.v => add.v} | 48 --------------------------- src/repo/remove.v | 68 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 48 deletions(-) rename src/repo/{repo.v => add.v} (77%) create mode 100644 src/repo/remove.v diff --git a/src/repo/repo.v b/src/repo/add.v similarity index 77% rename from src/repo/repo.v rename to src/repo/add.v index 7de12cdb..0985c7a6 100644 --- a/src/repo/repo.v +++ b/src/repo/add.v @@ -154,51 +154,3 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac return true } - -// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It -// returns false if the package wasn't present in the database. It also -// optionally re-syncs the repo archives. -pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool { - repo_dir := os.join_path(r.repos_dir, repo, arch) - - // If the repository doesn't exist yet, the result is automatically false - if !os.exists(repo_dir) { - return false - } - - // We iterate over every directory in the repo dir - // TODO filter so we only check directories - for d in os.ls(repo_dir)? { - // Because a repository only allows a single version of each package, - // we need only compare whether the name of the package is the same, - // not the version. - name := d.split('-')#[..-2].join('-') - - if name == pkg_name { - // We lock the mutex here to prevent other routines from creating a - // new archive while we remove an entry - lock r.mutex { - os.rmdir_all(os.join_path_single(repo_dir, d))? - } - - // Also remove the package archive - repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch) - - archives := os.ls(repo_pkg_dir)?.filter(it.split('-')#[..-3].join('-') == name) - - for archive_name in archives { - full_path := os.join_path_single(repo_pkg_dir, archive_name) - os.rm(full_path)? - } - - // Sync the db archives if requested - if sync { - r.sync(repo, arch)? - } - - return true - } - } - - return false -} diff --git a/src/repo/remove.v b/src/repo/remove.v new file mode 100644 index 00000000..dd4f4004 --- /dev/null +++ b/src/repo/remove.v @@ -0,0 +1,68 @@ +module repo + +import os + +// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It +// returns false if the package wasn't present in the database. It also +// optionally re-syncs the repo archives. +pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool { + repo_dir := os.join_path(r.repos_dir, repo, arch) + + // If the repository doesn't exist yet, the result is automatically false + if !os.exists(repo_dir) { + return false + } + + // We iterate over every directory in the repo dir + // TODO filter so we only check directories + for d in os.ls(repo_dir)? { + // Because a repository only allows a single version of each package, + // we need only compare whether the name of the package is the same, + // not the version. + name := d.split('-')#[..-2].join('-') + + if name == pkg_name { + // We lock the mutex here to prevent other routines from creating a + // new archive while we remove an entry + lock r.mutex { + os.rmdir_all(os.join_path_single(repo_dir, d))? + } + + // Also remove the package archive + repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch) + + archives := os.ls(repo_pkg_dir)?.filter(it.split('-')#[..-3].join('-') == name) + + for archive_name in archives { + full_path := os.join_path_single(repo_pkg_dir, archive_name) + os.rm(full_path)? + } + + // Sync the db archives if requested + if sync { + r.sync(repo, arch)? + } + + return true + } + } + + return false +} + +// remove_arch_repo removes an arch-repo & its packages. +pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) ?bool { + repo_dir := os.join_path(r.repos_dir, repo, arch) + + // If the repository doesn't exist yet, the result is automatically false + if !os.exists(repo_dir) { + return false + } + + os.rmdir_all(repo_dir)? + + pkg_dir := os.join_path(r.pkg_dir, repo, arch) + os.rmdir_all(pkg_dir)? + + return true +} From 68b7e5e71ec636fe1fdcb7338f02dd3f70e922ed Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 11 Aug 2022 19:28:17 +0200 Subject: [PATCH 36/62] feat(server): added routes for removing arch-repo & repo --- docs/api/source/includes/_repository.md | 55 ++++++++++++++++++ src/repo/remove.v | 17 ++++++ src/server/{routes.v => repo.v} | 23 -------- src/server/repo_remove.v | 77 +++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 23 deletions(-) rename src/server/{routes.v => repo.v} (82%) create mode 100644 src/server/repo_remove.v diff --git a/docs/api/source/includes/_repository.md b/docs/api/source/includes/_repository.md index 9764e014..a8469043 100644 --- a/docs/api/source/includes/_repository.md +++ b/docs/api/source/includes/_repository.md @@ -122,3 +122,58 @@ Parameter | Description repo | Repository to delete package from arch | Specific arch-repo to remove package from pkg | Name of package to remove (without any version information) + +## Remove arch-repo + + + +```shell +curl \ + -H 'X-Api-Key: secret' \ + -XDELETE \ + https://example.com/vieter/x86_64 +``` + +This endpoint allows remove an entire arch-repo. + +### HTTP Request + +`DELETE /:repo/:arch` + +### URL Parameters + +Parameter | Description +--------- | ----------- +repo | Repository to delete arch-repo from +arch | Specific architecture to remove package + +## Remove repo + + + +```shell +curl \ + -H 'X-Api-Key: secret' \ + -XDELETE \ + https://example.com/vieter +``` + +This endpoint allows remove an entire repo. + +### HTTP Request + +`DELETE /:repo` + +### URL Parameters + +Parameter | Description +--------- | ----------- +repo | Repository to delete diff --git a/src/repo/remove.v b/src/repo/remove.v index dd4f4004..add921cb 100644 --- a/src/repo/remove.v +++ b/src/repo/remove.v @@ -66,3 +66,20 @@ pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) ?bool { return true } + +// remove_repo removes a repo & its packages. +pub fn (r &RepoGroupManager) remove_repo(repo string) ?bool { + repo_dir := os.join_path_single(r.repos_dir, repo) + + // If the repository doesn't exist yet, the result is automatically false + if !os.exists(repo_dir) { + return false + } + + os.rmdir_all(repo_dir)? + + pkg_dir := os.join_path_single(r.pkg_dir, repo) + os.rmdir_all(pkg_dir)? + + return true +} diff --git a/src/server/routes.v b/src/server/repo.v similarity index 82% rename from src/server/routes.v rename to src/server/repo.v index 3b86e20b..fbf37dfb 100644 --- a/src/server/routes.v +++ b/src/server/repo.v @@ -57,29 +57,6 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re return app.file(full_path) } -['/:repo/:arch/:pkg'; delete] -fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - - res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { - app.lerror('Error while deleting package: $err.msg()') - - return app.json(http.Status.internal_server_error, new_response('Failed to delete package.')) - } - - if res { - app.linfo("Removed package '$pkg' from '$repo/$arch'") - - return app.json(http.Status.ok, new_response('Package removed.')) - } else { - app.linfo("Tried removing package '$pkg' from '$repo/$arch', but it doesn't exist.") - - return app.json(http.Status.not_found, new_response('Package not found.')) - } -} - // put_package handles publishing a package to a repository. ['/:repo/publish'; post] fn (mut app App) put_package(repo string) web.Result { diff --git a/src/server/repo_remove.v b/src/server/repo_remove.v new file mode 100644 index 00000000..642f26f2 --- /dev/null +++ b/src/server/repo_remove.v @@ -0,0 +1,77 @@ +module server + +import web +import net.http +import response { new_response } + +// delete_package tries to remove the given package. +['/:repo/:arch/:pkg'; delete] +fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { + app.lerror('Error while deleting package: $err.msg()') + + return app.json(http.Status.internal_server_error, new_response('Failed to delete package.')) + } + + if res { + app.linfo("Removed package '$pkg' from '$repo/$arch'") + + return app.json(http.Status.ok, new_response('Package removed.')) + } else { + app.linfo("Tried removing package '$pkg' from '$repo/$arch', but it doesn't exist.") + + return app.json(http.Status.not_found, new_response('Package not found.')) + } +} + +// delete_arch_repo tries to remove the given arch-repo. +['/:repo/:arch'; delete] +fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + res := app.repo.remove_arch_repo(repo, arch) or { + app.lerror('Error while deleting arch-repo: $err.msg()') + + return app.json(http.Status.internal_server_error, new_response('Failed to delete arch-repo.')) + } + + if res { + app.linfo("Removed '$repo/$arch'") + + return app.json(http.Status.ok, new_response('Arch-repo removed.')) + } else { + app.linfo("Tried removing '$repo/$arch', but it doesn't exist.") + + return app.json(http.Status.not_found, new_response('Arch-repo not found.')) + } +} + +// delete_repo tries to remove the given repo. +['/:repo'; delete] +fn (mut app App) delete_repo(repo string) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + res := app.repo.remove_repo(repo) or { + app.lerror('Error while deleting repo: $err.msg()') + + return app.json(http.Status.internal_server_error, new_response('Failed to delete repo.')) + } + + if res { + app.linfo("Removed '$repo'") + + return app.json(http.Status.ok, new_response('Repo removed.')) + } else { + app.linfo("Tried removing '$repo', but it doesn't exist.") + + return app.json(http.Status.not_found, new_response('Repo not found.')) + } +} From ba3b00572b3ddf31490f054b5bbf2288913da5af Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 11 Aug 2022 19:44:22 +0200 Subject: [PATCH 37/62] chore: updated changelog [CI SKIP] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a03c18cc..b7138a84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Targets with kind 'url' can provide a direct URL to a PKGBUILD instead of providing a Git repository * CLI commands for searching the AUR & directly adding packages +* HTTP routes for removing packages, arch-repos & repos ### Changed From 50918da67214ff69a29343f04a68f52e28f602dc Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 11 Aug 2022 19:47:26 +0200 Subject: [PATCH 38/62] docs: fixed some typos --- docs/api/source/includes/_repository.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api/source/includes/_repository.md b/docs/api/source/includes/_repository.md index a8469043..ff17f719 100644 --- a/docs/api/source/includes/_repository.md +++ b/docs/api/source/includes/_repository.md @@ -138,7 +138,7 @@ curl \ https://example.com/vieter/x86_64 ``` -This endpoint allows remove an entire arch-repo. +This endpoint allows removing an entire arch-repo. ### HTTP Request @@ -149,7 +149,7 @@ This endpoint allows remove an entire arch-repo. Parameter | Description --------- | ----------- repo | Repository to delete arch-repo from -arch | Specific architecture to remove package +arch | Specific architecture to remove ## Remove repo @@ -166,7 +166,7 @@ curl \ https://example.com/vieter ``` -This endpoint allows remove an entire repo. +This endpoint allows removing an entire repo. ### HTTP Request From 3a73ea0632df8c804d3a6d7a3ed7ae59f001ff65 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 12 Aug 2022 14:39:42 +0200 Subject: [PATCH 39/62] refactor(web): simplified web framework in general --- src/web/consts.v | 125 +++++++++++++++++++++ src/web/web.v | 275 +++++++++++++++-------------------------------- 2 files changed, 209 insertions(+), 191 deletions(-) create mode 100644 src/web/consts.v diff --git a/src/web/consts.v b/src/web/consts.v new file mode 100644 index 00000000..7b1d2b47 --- /dev/null +++ b/src/web/consts.v @@ -0,0 +1,125 @@ +module web + +import net.http + +// A dummy structure that returns from routes to indicate that you actually sent something to a user +[noinit] +pub struct Result {} + +pub const ( + methods_with_form = [http.Method.post, .put, .patch] + headers_close = http.new_custom_header_from_map({ + 'Server': 'VWeb' + http.CommonHeader.connection.str(): 'close' + }) or { panic('should never fail') } + + http_302 = http.new_response( + status: .found + body: '302 Found' + header: headers_close + ) + http_400 = http.new_response( + status: .bad_request + body: '400 Bad Request' + header: http.new_header( + key: .content_type + value: 'text/plain' + ).join(headers_close) + ) + http_404 = http.new_response( + status: .not_found + body: '404 Not Found' + header: http.new_header( + key: .content_type + value: 'text/plain' + ).join(headers_close) + ) + http_500 = http.new_response( + status: .internal_server_error + body: '500 Internal Server Error' + header: http.new_header( + key: .content_type + value: 'text/plain' + ).join(headers_close) + ) + mime_types = { + '.aac': 'audio/aac' + '.abw': 'application/x-abiword' + '.arc': 'application/x-freearc' + '.avi': 'video/x-msvideo' + '.azw': 'application/vnd.amazon.ebook' + '.bin': 'application/octet-stream' + '.bmp': 'image/bmp' + '.bz': 'application/x-bzip' + '.bz2': 'application/x-bzip2' + '.cda': 'application/x-cdf' + '.csh': 'application/x-csh' + '.css': 'text/css' + '.csv': 'text/csv' + '.doc': 'application/msword' + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + '.eot': 'application/vnd.ms-fontobject' + '.epub': 'application/epub+zip' + '.gz': 'application/gzip' + '.gif': 'image/gif' + '.htm': 'text/html' + '.html': 'text/html' + '.ico': 'image/vnd.microsoft.icon' + '.ics': 'text/calendar' + '.jar': 'application/java-archive' + '.jpeg': 'image/jpeg' + '.jpg': 'image/jpeg' + '.js': 'text/javascript' + '.json': 'application/json' + '.jsonld': 'application/ld+json' + '.mid': 'audio/midi audio/x-midi' + '.midi': 'audio/midi audio/x-midi' + '.mjs': 'text/javascript' + '.mp3': 'audio/mpeg' + '.mp4': 'video/mp4' + '.mpeg': 'video/mpeg' + '.mpkg': 'application/vnd.apple.installer+xml' + '.odp': 'application/vnd.oasis.opendocument.presentation' + '.ods': 'application/vnd.oasis.opendocument.spreadsheet' + '.odt': 'application/vnd.oasis.opendocument.text' + '.oga': 'audio/ogg' + '.ogv': 'video/ogg' + '.ogx': 'application/ogg' + '.opus': 'audio/opus' + '.otf': 'font/otf' + '.png': 'image/png' + '.pdf': 'application/pdf' + '.php': 'application/x-httpd-php' + '.ppt': 'application/vnd.ms-powerpoint' + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation' + '.rar': 'application/vnd.rar' + '.rtf': 'application/rtf' + '.sh': 'application/x-sh' + '.svg': 'image/svg+xml' + '.swf': 'application/x-shockwave-flash' + '.tar': 'application/x-tar' + '.tif': 'image/tiff' + '.tiff': 'image/tiff' + '.ts': 'video/mp2t' + '.ttf': 'font/ttf' + '.txt': 'text/plain' + '.vsd': 'application/vnd.visio' + '.wav': 'audio/wav' + '.weba': 'audio/webm' + '.webm': 'video/webm' + '.webp': 'image/webp' + '.woff': 'font/woff' + '.woff2': 'font/woff2' + '.xhtml': 'application/xhtml+xml' + '.xls': 'application/vnd.ms-excel' + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + '.xml': 'application/xml' + '.xul': 'application/vnd.mozilla.xul+xml' + '.zip': 'application/zip' + '.3gp': 'video/3gpp' + '.3g2': 'video/3gpp2' + '.7z': 'application/x-7z-compressed' + } + max_http_post_size = 1024 * 1024 + default_port = 8080 +) diff --git a/src/web/web.v b/src/web/web.v index b053904d..9fe0ddcb 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -12,146 +12,23 @@ import time import json import log -// A dummy structure that returns from routes to indicate that you actually sent something to a user -[noinit] -pub struct Result {} - -pub const ( - methods_with_form = [http.Method.post, .put, .patch] - headers_close = http.new_custom_header_from_map({ - 'Server': 'VWeb' - http.CommonHeader.connection.str(): 'close' - }) or { panic('should never fail') } - - http_302 = http.new_response( - status: .found - body: '302 Found' - header: headers_close - ) - http_400 = http.new_response( - status: .bad_request - body: '400 Bad Request' - header: http.new_header( - key: .content_type - value: 'text/plain' - ).join(headers_close) - ) - http_404 = http.new_response( - status: .not_found - body: '404 Not Found' - header: http.new_header( - key: .content_type - value: 'text/plain' - ).join(headers_close) - ) - http_500 = http.new_response( - status: .internal_server_error - body: '500 Internal Server Error' - header: http.new_header( - key: .content_type - value: 'text/plain' - ).join(headers_close) - ) - mime_types = { - '.aac': 'audio/aac' - '.abw': 'application/x-abiword' - '.arc': 'application/x-freearc' - '.avi': 'video/x-msvideo' - '.azw': 'application/vnd.amazon.ebook' - '.bin': 'application/octet-stream' - '.bmp': 'image/bmp' - '.bz': 'application/x-bzip' - '.bz2': 'application/x-bzip2' - '.cda': 'application/x-cdf' - '.csh': 'application/x-csh' - '.css': 'text/css' - '.csv': 'text/csv' - '.doc': 'application/msword' - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' - '.eot': 'application/vnd.ms-fontobject' - '.epub': 'application/epub+zip' - '.gz': 'application/gzip' - '.gif': 'image/gif' - '.htm': 'text/html' - '.html': 'text/html' - '.ico': 'image/vnd.microsoft.icon' - '.ics': 'text/calendar' - '.jar': 'application/java-archive' - '.jpeg': 'image/jpeg' - '.jpg': 'image/jpeg' - '.js': 'text/javascript' - '.json': 'application/json' - '.jsonld': 'application/ld+json' - '.mid': 'audio/midi audio/x-midi' - '.midi': 'audio/midi audio/x-midi' - '.mjs': 'text/javascript' - '.mp3': 'audio/mpeg' - '.mp4': 'video/mp4' - '.mpeg': 'video/mpeg' - '.mpkg': 'application/vnd.apple.installer+xml' - '.odp': 'application/vnd.oasis.opendocument.presentation' - '.ods': 'application/vnd.oasis.opendocument.spreadsheet' - '.odt': 'application/vnd.oasis.opendocument.text' - '.oga': 'audio/ogg' - '.ogv': 'video/ogg' - '.ogx': 'application/ogg' - '.opus': 'audio/opus' - '.otf': 'font/otf' - '.png': 'image/png' - '.pdf': 'application/pdf' - '.php': 'application/x-httpd-php' - '.ppt': 'application/vnd.ms-powerpoint' - '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation' - '.rar': 'application/vnd.rar' - '.rtf': 'application/rtf' - '.sh': 'application/x-sh' - '.svg': 'image/svg+xml' - '.swf': 'application/x-shockwave-flash' - '.tar': 'application/x-tar' - '.tif': 'image/tiff' - '.tiff': 'image/tiff' - '.ts': 'video/mp2t' - '.ttf': 'font/ttf' - '.txt': 'text/plain' - '.vsd': 'application/vnd.visio' - '.wav': 'audio/wav' - '.weba': 'audio/webm' - '.webm': 'video/webm' - '.webp': 'image/webp' - '.woff': 'font/woff' - '.woff2': 'font/woff2' - '.xhtml': 'application/xhtml+xml' - '.xls': 'application/vnd.ms-excel' - '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' - '.xml': 'application/xml' - '.xul': 'application/vnd.mozilla.xul+xml' - '.zip': 'application/zip' - '.3gp': 'video/3gpp' - '.3g2': 'video/3gpp2' - '.7z': 'application/x-7z-compressed' - } - max_http_post_size = 1024 * 1024 - default_port = 8080 -) - // The Context struct represents the Context which hold the HTTP request and response. // It has fields for the query, form, files. pub struct Context { -mut: - content_type string = 'text/plain' - status http.Status = http.Status.ok pub: // HTTP Request req http.Request // TODO Response pub mut: - done bool - // time.ticks() from start of web connection handle. - // You can use it to determine how much time is spent on your request. - page_gen_start i64 // TCP connection to client. // But beware, do not store it for further use, after request processing web will close connection. - conn &net.TcpConn + conn &net.TcpConn + // Gives access to a shared logger object + logger shared log.Log + // REQUEST + // time.ticks() from start of web connection handle. + // You can use it to determine how much time is spent on your request. + page_gen_start i64 static_files map[string]string static_mime_types map[string]string // Map containing query params for the route. @@ -161,14 +38,13 @@ pub mut: form map[string]string // Files from multipart-form. files map[string][]http.FileData - - header http.Header // response headers - // ? It doesn't seem to be used anywhere - form_error string // Allows reading the request body reader io.BufferedReader - // Gives access to a shared logger object - logger shared log.Log + // RESPONSE + status http.Status = http.Status.ok + content_type string = 'text/plain' + // response headers + header http.Header } struct FileData { @@ -188,40 +64,68 @@ struct Route { // Probably you can use it for check user session cookie or add header. pub fn (ctx Context) before_request() {} -// send_string -fn send_string(mut conn net.TcpConn, s string) ? { - conn.write(s.bytes())? +// send_string writes the given string to the TCP connection socket. +fn (mut ctx Context) send_string(s string) ? { + ctx.conn.write(s.bytes())? } -// send_response_to_client sends a response to the client -[manualfree] -pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bool { - if ctx.done { - return false +// send_reader reads at most `size` bytes from the given reader & writes them +// to the TCP connection socket. Internally, a 10KB buffer is used, to avoid +// having to store all bytes in memory at once. +fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ? { + mut buf := []u8{len: 10_000} + mut bytes_left := size + + // Repeat as long as the stream still has data + for bytes_left > 0 { + bytes_read := reader.read(mut buf)? + bytes_left -= u64(bytes_read) + + mut to_write := bytes_read + + for to_write > 0 { + // TODO don't just loop infinitely here + bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { continue } + + to_write = to_write - bytes_written + } } - ctx.done = true - - // build header - header := http.new_header_from_map({ - http.CommonHeader.content_type: mimetype - http.CommonHeader.content_length: res.len.str() - }).join(ctx.header) +} +// send_response_header constructs a valid HTTP response with an empty body & +// sends it to the client. +pub fn (mut ctx Context) send_response_header() ? { mut resp := http.Response{ - header: header.join(web.headers_close) - body: res + header: ctx.header.join(headers_close) } resp.set_version(.v1_1) resp.set_status(ctx.status) - send_string(mut ctx.conn, resp.bytestr()) or { return false } + ctx.send_string(resp.bytestr())? +} + +// send_response constructs the resulting HTTP response with the given body +// string & sends it to the client. +pub fn (mut ctx Context) send_response(res string) bool { + ctx.send_response_header() or { return false } + ctx.send_string(res) or { return false } + + return true +} + +// send_reader_response constructs the resulting HTTP response with the given +// body & streams the reader's contents to the client. +pub fn (mut ctx Context) send_reader_response(mut reader io.Reader, size u64) bool { + ctx.send_response_header() or { return false } + ctx.send_reader(mut reader, size) or { return false } + return true } // text responds to a request with some plaintext. pub fn (mut ctx Context) text(status http.Status, s string) Result { ctx.status = status - - ctx.send_response_to_client('text/plain', s) + ctx.content_type = 'text/plain' + ctx.send_response(s) return Result{} } @@ -229,9 +133,10 @@ pub fn (mut ctx Context) text(status http.Status, s string) Result { // json HTTP_OK with json_s as payload with content-type `application/json` pub fn (mut ctx Context) json(status http.Status, j T) Result { ctx.status = status + ctx.content_type = 'application/json' json_s := json.encode(j) - ctx.send_response_to_client('application/json', json_s) + ctx.send_response(json_s) return Result{} } @@ -239,10 +144,6 @@ pub fn (mut ctx Context) json(status http.Status, j T) Result { // file Response HTTP_OK with file as payload // This function manually implements responses because it needs to stream the file contents pub fn (mut ctx Context) file(f_path string) Result { - if ctx.done { - return Result{} - } - if !os.is_file(f_path) { return ctx.not_found() } @@ -266,7 +167,7 @@ pub fn (mut ctx Context) file(f_path string) Result { // We open the file before sending the headers in case reading fails file_size := os.file_size(f_path) - file := os.open(f_path) or { + mut file := os.open(f_path) or { eprintln(err.msg()) ctx.server_error(500) return Result{} @@ -279,32 +180,32 @@ pub fn (mut ctx Context) file(f_path string) Result { }).join(ctx.header) mut resp := http.Response{ - header: header.join(web.headers_close) + header: header.join(headers_close) } resp.set_version(.v1_1) resp.set_status(ctx.status) - send_string(mut ctx.conn, resp.bytestr()) or { return Result{} } + ctx.send_string(resp.bytestr()) or { return Result{} } + ctx.send_reader(mut file, file_size) or { return Result{} } - mut buf := []u8{len: 1_000_000} - mut bytes_left := file_size + // mut buf := []u8{len: 1_000_000} + // mut bytes_left := file_size - // Repeat as long as the stream still has data - for bytes_left > 0 { - // TODO check if just breaking here is safe - bytes_read := file.read(mut buf) or { break } - bytes_left -= u64(bytes_read) + // // Repeat as long as the stream still has data + // for bytes_left > 0 { + // // TODO check if just breaking here is safe + // bytes_read := file.read(mut buf) or { break } + // bytes_left -= u64(bytes_read) - mut to_write := bytes_read + // mut to_write := bytes_read - for to_write > 0 { - // TODO don't just loop infinitely here - bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { continue } + // for to_write > 0 { + // // TODO don't just loop infinitely here + // bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { continue } - to_write = to_write - bytes_written - } - } + // to_write = to_write - bytes_written + // } + // } - ctx.done = true return Result{} } @@ -319,23 +220,16 @@ pub fn (mut ctx Context) server_error(ecode int) Result { $if debug { eprintln('> ctx.server_error ecode: $ecode') } - if ctx.done { - return Result{} - } - send_string(mut ctx.conn, web.http_500.bytestr()) or {} + ctx.send_string(http_500.bytestr()) or {} return Result{} } // redirect Redirect to an url pub fn (mut ctx Context) redirect(url string) Result { - if ctx.done { - return Result{} - } - ctx.done = true - mut resp := web.http_302 + mut resp := http_302 resp.header = resp.header.join(ctx.header) resp.header.add(.location, url) - send_string(mut ctx.conn, resp.bytestr()) or { return Result{} } + ctx.send_string(resp.bytestr()) or { return Result{} } return Result{} } @@ -532,7 +426,7 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { } } // Route not found - conn.write(web.http_404.bytes()) or {} + conn.write(http_404.bytes()) or {} } // route_matches returns wether a route matches @@ -597,7 +491,6 @@ pub fn (ctx &Context) ip() string { // error Set s to the form error pub fn (mut ctx Context) error(s string) { println('web error: $s') - ctx.form_error = s } // filter Do not delete. From e7b45bf251736cbd60fcb49aa45bd2b7012125fe Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 12 Aug 2022 15:08:05 +0200 Subject: [PATCH 40/62] feat(web): file() now handles HEAD requests --- src/server/repo.v | 9 ----- src/web/web.v | 90 +++++++++++------------------------------------ 2 files changed, 20 insertions(+), 79 deletions(-) diff --git a/src/server/repo.v b/src/server/repo.v index fbf37dfb..4a417fb0 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -45,15 +45,6 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re full_path = os.join_path(app.repo.repos_dir, repo, arch, filename, 'desc') } - // Scuffed way to respond to HEAD requests - if app.req.method == http.Method.head { - if os.exists(full_path) { - return app.status(http.Status.ok) - } - - return app.not_found() - } - return app.file(full_path) } diff --git a/src/web/web.v b/src/web/web.v index 9fe0ddcb..c4cfee76 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -25,10 +25,10 @@ pub mut: conn &net.TcpConn // Gives access to a shared logger object logger shared log.Log - // REQUEST // time.ticks() from start of web connection handle. // You can use it to determine how much time is spent on your request. page_gen_start i64 + // REQUEST static_files map[string]string static_mime_types map[string]string // Map containing query params for the route. @@ -103,6 +103,12 @@ pub fn (mut ctx Context) send_response_header() ? { ctx.send_string(resp.bytestr())? } +// send is a convenience function for sending the HTTP response with an empty +// body. +pub fn (mut ctx Context) send() bool { + return ctx.send_response('') +} + // send_response constructs the resulting HTTP response with the given body // string & sends it to the client. pub fn (mut ctx Context) send_response(res string) bool { @@ -144,67 +150,32 @@ pub fn (mut ctx Context) json(status http.Status, j T) Result { // file Response HTTP_OK with file as payload // This function manually implements responses because it needs to stream the file contents pub fn (mut ctx Context) file(f_path string) Result { + // If the file doesn't exist, just respond with a 404 if !os.is_file(f_path) { - return ctx.not_found() + ctx.status = .not_found + ctx.send() + + return Result{} } - // ext := os.file_ext(f_path) - // data := os.read_file(f_path) or { - // eprint(err.msg()) - // ctx.server_error(500) - // return Result{} - // } - // content_type := web.mime_types[ext] - // if content_type == '' { - // eprintln('no MIME type found for extension $ext') - // ctx.server_error(500) + file_size := os.file_size(f_path) + ctx.header.add(http.CommonHeader.content_length, file_size.str()) - // return Result{} - // } + // A HEAD request only returns the size of the file. + if ctx.req.method == .head { + ctx.send() - // First, we return the headers for the request + return Result{} + } // We open the file before sending the headers in case reading fails - file_size := os.file_size(f_path) - mut file := os.open(f_path) or { eprintln(err.msg()) ctx.server_error(500) return Result{} } - // build header - header := http.new_header_from_map({ - // http.CommonHeader.content_type: content_type - http.CommonHeader.content_length: file_size.str() - }).join(ctx.header) - - mut resp := http.Response{ - header: header.join(headers_close) - } - resp.set_version(.v1_1) - resp.set_status(ctx.status) - ctx.send_string(resp.bytestr()) or { return Result{} } - ctx.send_reader(mut file, file_size) or { return Result{} } - - // mut buf := []u8{len: 1_000_000} - // mut bytes_left := file_size - - // // Repeat as long as the stream still has data - // for bytes_left > 0 { - // // TODO check if just breaking here is safe - // bytes_read := file.read(mut buf) or { break } - // bytes_left -= u64(bytes_read) - - // mut to_write := bytes_read - - // for to_write > 0 { - // // TODO don't just loop infinitely here - // bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { continue } - - // to_write = to_write - bytes_written - // } - // } + ctx.send_reader_response(mut file, file_size) return Result{} } @@ -472,27 +443,6 @@ fn route_matches(url_words []string, route_words []string) ?[]string { return params } -// ip Returns the ip address from the current user -pub fn (ctx &Context) ip() string { - mut ip := ctx.req.header.get(.x_forwarded_for) or { '' } - if ip == '' { - ip = ctx.req.header.get_custom('X-Real-Ip') or { '' } - } - - if ip.contains(',') { - ip = ip.all_before(',') - } - if ip == '' { - ip = ctx.conn.peer_ip() or { '' } - } - return ip -} - -// error Set s to the form error -pub fn (mut ctx Context) error(s string) { - println('web error: $s') -} - // filter Do not delete. // It used by `vlib/v/gen/c/str_intp.v:130` for string interpolation inside web templates // TODO: move it to template render From cc5df95a1a9b742ca3945bfda16c1df53b294a0c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 12 Aug 2022 17:11:44 +0200 Subject: [PATCH 41/62] feat(web): file() now supports HTTP byte range --- src/server/repo.v | 2 +- src/web/web.v | 64 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/src/server/repo.v b/src/server/repo.v index 4a417fb0..2253a441 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -13,7 +13,7 @@ import response { new_response } // server is still responsive. ['/health'; get] pub fn (mut app App) healthcheck() web.Result { - return app.json(http.Status.ok, new_response('Healthy.')) + return app.json(.ok, new_response('Healthy.')) } // get_repo_file handles all Pacman-related routes. It returns both the diff --git a/src/web/web.v b/src/web/web.v index c4cfee76..51acd2f2 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -27,7 +27,7 @@ pub mut: logger shared log.Log // time.ticks() from start of web connection handle. // You can use it to determine how much time is spent on your request. - page_gen_start i64 + page_gen_start i64 // REQUEST static_files map[string]string static_mime_types map[string]string @@ -84,8 +84,7 @@ fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ? { mut to_write := bytes_read for to_write > 0 { - // TODO don't just loop infinitely here - bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { continue } + bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { break } to_write = to_write - bytes_written } @@ -127,15 +126,6 @@ pub fn (mut ctx Context) send_reader_response(mut reader io.Reader, size u64) bo return true } -// text responds to a request with some plaintext. -pub fn (mut ctx Context) text(status http.Status, s string) Result { - ctx.status = status - ctx.content_type = 'text/plain' - ctx.send_response(s) - - return Result{} -} - // json HTTP_OK with json_s as payload with content-type `application/json` pub fn (mut ctx Context) json(status http.Status, j T) Result { ctx.status = status @@ -158,6 +148,8 @@ pub fn (mut ctx Context) file(f_path string) Result { return Result{} } + ctx.header.add(.accept_ranges, 'bytes') + file_size := os.file_size(f_path) ctx.header.add(http.CommonHeader.content_length, file_size.str()) @@ -168,14 +160,53 @@ pub fn (mut ctx Context) file(f_path string) Result { return Result{} } - // We open the file before sending the headers in case reading fails mut file := os.open(f_path) or { eprintln(err.msg()) ctx.server_error(500) return Result{} } - ctx.send_reader_response(mut file, file_size) + defer { + file.close() + } + + if range_str := ctx.req.header.get(.range) { + mut parts := range_str.split_nth('=', 2) + + if parts[0] != 'bytes' { + ctx.status = .requested_range_not_satisfiable + ctx.header.delete(.content_length) + ctx.send() + return Result{} + } + + parts = parts[1].split_nth('-', 2) + + start := parts[0].i64() + end := if parts[1] == '' { file_size - 1 } else { parts[1].u64() } + + // Either the actual number 0 or the result of an invalid integer + if end == 0 { + ctx.status = .requested_range_not_satisfiable + ctx.header.delete(.content_length) + ctx.send() + return Result{} + } + + // Move cursor to start of data to read + file.seek(start, .start) or { + ctx.server_error(500) + return Result{} + } + + length := end - u64(start) + 1 + + ctx.status = .partial_content + ctx.header.set(.content_length, length.str()) + ctx.send_reader_response(mut file, length) + } else { + ctx.send_reader_response(mut file, file_size) + } return Result{} } @@ -183,7 +214,10 @@ pub fn (mut ctx Context) file(f_path string) Result { // status responds with an empty textual response, essentially only returning // the given status code. pub fn (mut ctx Context) status(status http.Status) Result { - return ctx.text(status, '') + ctx.status = status + ctx.send() + + return Result{} } // server_error Response a server error From e23635a1d394f135c60b67a6517d78db0ed0b579 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 13 Aug 2022 13:16:31 +0200 Subject: [PATCH 42/62] refactor: moved response module to web.response --- README.md | 11 ++--------- src/client/client.v | 2 +- src/client/logs.v | 2 +- src/client/targets.v | 2 +- src/server/api_logs.v | 2 +- src/server/api_targets.v | 2 +- src/server/repo.v | 2 +- src/server/repo_remove.v | 2 +- src/{ => web}/response/response.v | 0 9 files changed, 9 insertions(+), 16 deletions(-) rename src/{ => web}/response/response.v (100%) diff --git a/README.md b/README.md index 5911ea21..29ec0f01 100644 --- a/README.md +++ b/README.md @@ -48,15 +48,8 @@ update`. ### Compiler -Vieter compiles with the standard Vlang compiler. However, I do maintain a -[mirror](https://git.rustybever.be/vieter-v/v). This is to ensure my CI does -not break without reason, as I control when & how frequently the mirror is -updated to reflect the official repository. - -If you encounter issues using the latest V compiler, try using my mirror -instead. `make v` will clone the repository & build the mirror. Afterwards, -prepending any make command with `V_PATH=v/v` tells make to use the locally -compiled mirror instead. +I used to maintain a mirror that tracked the latest master, but nowadays, I +solely target V 0.3 as a compiler. ## Contributing diff --git a/src/client/client.v b/src/client/client.v index 2bb1ac20..24e44448 100644 --- a/src/client/client.v +++ b/src/client/client.v @@ -2,7 +2,7 @@ module client import net.http { Method } import net.urllib -import response { Response } +import web.response { Response } import json pub struct Client { diff --git a/src/client/logs.v b/src/client/logs.v index f242f6e8..b52c3d09 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -2,7 +2,7 @@ module client import models { BuildLog, BuildLogFilter } import net.http { Method } -import response { Response } +import web.response { Response } import time // get_build_logs returns all build logs. diff --git a/src/client/targets.v b/src/client/targets.v index 82c78780..f5258a43 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -2,7 +2,7 @@ module client import models { Target, TargetFilter } import net.http { Method } -import response { Response } +import web.response { Response } // get_targets returns a list of targets, given a filter object. pub fn (c &Client) get_targets(filter TargetFilter) ?[]Target { diff --git a/src/server/api_logs.v b/src/server/api_logs.v index fa3338ea..6728392c 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -3,7 +3,7 @@ module server import web import net.http import net.urllib -import response { new_data_response, new_response } +import web.response { new_data_response, new_response } import db import time import os diff --git a/src/server/api_targets.v b/src/server/api_targets.v index 3867c94b..4cc3a584 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -2,7 +2,7 @@ module server import web import net.http -import response { new_data_response, new_response } +import web.response { new_data_response, new_response } import db import models { Target, TargetArch, TargetFilter } diff --git a/src/server/repo.v b/src/server/repo.v index 2253a441..242fd2df 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -7,7 +7,7 @@ import time import rand import util import net.http -import response { new_response } +import web.response { new_response } // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. diff --git a/src/server/repo_remove.v b/src/server/repo_remove.v index 642f26f2..5d5ef156 100644 --- a/src/server/repo_remove.v +++ b/src/server/repo_remove.v @@ -2,7 +2,7 @@ module server import web import net.http -import response { new_response } +import web.response { new_response } // delete_package tries to remove the given package. ['/:repo/:arch/:pkg'; delete] diff --git a/src/response/response.v b/src/web/response/response.v similarity index 100% rename from src/response/response.v rename to src/web/response/response.v From 9268ef0302d9ec3d73dc1fa7f38809655f2adbeb Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 13 Aug 2022 17:49:05 +0200 Subject: [PATCH 43/62] refactor(web): some small cleanup --- CHANGELOG.md | 2 ++ src/server/repo_remove.v | 2 +- src/web/consts.v | 2 +- src/web/web.v | 33 ++++++++++++++++----------------- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7138a84..5aa0e434 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 providing a Git repository * CLI commands for searching the AUR & directly adding packages * HTTP routes for removing packages, arch-repos & repos +* All endpoints serving files now support HTTP byte range requests ### Changed @@ -25,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Branch name for 'git' targets is now optional; if not provided, the repository will be cloned with the default branch * Build containers now explicitely set the PATH variable +* Refactor of web framework ### Removed diff --git a/src/server/repo_remove.v b/src/server/repo_remove.v index 5d5ef156..316b3872 100644 --- a/src/server/repo_remove.v +++ b/src/server/repo_remove.v @@ -8,7 +8,7 @@ import web.response { new_response } ['/:repo/:arch/:pkg'; delete] fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result { if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + return app.json(.unauthorized, new_response('Unauthorized.')) } res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { diff --git a/src/web/consts.v b/src/web/consts.v index 7b1d2b47..1b5bf08e 100644 --- a/src/web/consts.v +++ b/src/web/consts.v @@ -9,7 +9,7 @@ pub struct Result {} pub const ( methods_with_form = [http.Method.post, .put, .patch] headers_close = http.new_custom_header_from_map({ - 'Server': 'VWeb' + 'Server': 'Vieter' http.CommonHeader.connection.str(): 'close' }) or { panic('should never fail') } diff --git a/src/web/web.v b/src/web/web.v index 51acd2f2..8434a803 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -91,15 +91,22 @@ fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ? { } } +// send_custom_response sends the given http.Response to the client. It can be +// used to overwrite the Context object & send a completely custom +// http.Response instead. +fn (mut ctx Context) send_custom_response(resp &http.Response) ? { + ctx.send_string(resp.bytestr())? +} + // send_response_header constructs a valid HTTP response with an empty body & // sends it to the client. pub fn (mut ctx Context) send_response_header() ? { mut resp := http.Response{ header: ctx.header.join(headers_close) } - resp.set_version(.v1_1) resp.set_status(ctx.status) - ctx.send_string(resp.bytestr())? + + ctx.send_custom_response(resp)? } // send is a convenience function for sending the HTTP response with an empty @@ -222,10 +229,8 @@ pub fn (mut ctx Context) status(status http.Status) Result { // server_error Response a server error pub fn (mut ctx Context) server_error(ecode int) Result { - $if debug { - eprintln('> ctx.server_error ecode: $ecode') - } - ctx.send_string(http_500.bytestr()) or {} + ctx.send_custom_response(http_500) or {} + return Result{} } @@ -234,23 +239,17 @@ pub fn (mut ctx Context) redirect(url string) Result { mut resp := http_302 resp.header = resp.header.join(ctx.header) resp.header.add(.location, url) - ctx.send_string(resp.bytestr()) or { return Result{} } + + ctx.send_custom_response(resp) or {} + return Result{} } // not_found Send an not_found response pub fn (mut ctx Context) not_found() Result { - return ctx.status(http.Status.not_found) -} + ctx.send_custom_response(http_404) or {} -// add_header Adds an header to the response with key and val -pub fn (mut ctx Context) add_header(key string, val string) { - ctx.header.add_custom(key, val) or {} -} - -// get_header Returns the header data from the key -pub fn (ctx &Context) get_header(key string) string { - return ctx.req.header.get_custom(key) or { '' } + return Result{} } interface DbInterface { From 4887af26d3e7733ce3be5db570a4861590aade33 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 4 Sep 2022 19:32:22 +0200 Subject: [PATCH 44/62] feat(web): added authentication as function attribute --- src/server/server.v | 1 + src/web/consts.v | 8 +++++++ src/web/parse.v | 6 ++++- src/web/web.v | 53 ++++++++++++++++++++++++++++----------------- 4 files changed, 47 insertions(+), 21 deletions(-) diff --git a/src/server/server.v b/src/server/server.v index 1a9df3f8..9903cea9 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -73,6 +73,7 @@ pub fn server(conf Config) ? { web.run(&App{ logger: logger + api_key: conf.api_key conf: conf repo: repo db: db diff --git a/src/web/consts.v b/src/web/consts.v index 1b5bf08e..df8cdb25 100644 --- a/src/web/consts.v +++ b/src/web/consts.v @@ -26,6 +26,14 @@ pub const ( value: 'text/plain' ).join(headers_close) ) + http_401 = http.new_response( + status: .unauthorized + body: '401 Unauthorized' + header: http.new_header( + key: .content_type + value: 'text/plain' + ).join(headers_close) + ) http_404 = http.new_response( status: .not_found body: '404 Not Found' diff --git a/src/web/parse.v b/src/web/parse.v index a095f0c9..ee7a72cc 100644 --- a/src/web/parse.v +++ b/src/web/parse.v @@ -3,6 +3,10 @@ module web import net.urllib import net.http +// Method attributes that should be ignored when parsing, as they're used +// elsewhere. +const attrs_to_ignore = ['auth'] + // Parsing function attributes for methods and path. fn parse_attrs(name string, attrs []string) ?([]http.Method, string) { if attrs.len == 0 { @@ -32,7 +36,7 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) { } i++ } - if x.len > 0 { + if x.len > 0 && x.any(!web.attrs_to_ignore.contains(it)) { return IError(http.UnexpectedExtraAttributeError{ attributes: x }) diff --git a/src/web/web.v b/src/web/web.v index 8434a803..0847c7e6 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -18,6 +18,8 @@ pub struct Context { pub: // HTTP Request req http.Request + // API key used when authenticating requests + api_key string // TODO Response pub mut: // TCP connection to client. @@ -101,9 +103,10 @@ fn (mut ctx Context) send_custom_response(resp &http.Response) ? { // send_response_header constructs a valid HTTP response with an empty body & // sends it to the client. pub fn (mut ctx Context) send_response_header() ? { - mut resp := http.Response{ + mut resp := http.new_response( header: ctx.header.join(headers_close) - } + ) + resp.header.add(.content_type, ctx.content_type) resp.set_status(ctx.status) ctx.send_custom_response(resp)? @@ -133,6 +136,15 @@ pub fn (mut ctx Context) send_reader_response(mut reader io.Reader, size u64) bo return true } +// is_authenticated checks whether the request passes a correct API key. +pub fn (ctx &Context) is_authenticated() bool { + if provided_key := ctx.req.header.get_custom('X-Api-Key') { + return provided_key == ctx.api_key + } + + return false +} + // json HTTP_OK with json_s as payload with content-type `application/json` pub fn (mut ctx Context) json(status http.Status, j T) Result { ctx.status = status @@ -177,9 +189,12 @@ pub fn (mut ctx Context) file(f_path string) Result { file.close() } + // Currently, this only supports a single provided range, e.g. + // bytes=0-1023, and not multiple ranges, e.g. bytes=0-50, 100-150 if range_str := ctx.req.header.get(.range) { mut parts := range_str.split_nth('=', 2) + // We only support the 'bytes' range type if parts[0] != 'bytes' { ctx.status = .requested_range_not_satisfiable ctx.header.delete(.content_length) @@ -376,8 +391,10 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { static_mime_types: app.static_mime_types reader: reader logger: app.logger + api_key: app.api_key } + // Calling middleware... app.before_request() @@ -394,31 +411,27 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { // Used for route matching route_words := route.path.split('/').filter(it != '') - // Route immediate matches first + // Route immediate matches & index files first // For example URL `/register` matches route `/:user`, but `fn register()` // should be called first. - if !route.path.contains('/:') && url_words == route_words { - // We found a match - if head.method == .post && method.args.len > 0 { - // TODO implement POST requests - // Populate method args with form values - // mut args := []string{cap: method.args.len} - // for param in method.args { - // args << form[param.name] - // } - // app.$method(args) - } else { - app.$method() + if (!route.path.contains('/:') && url_words == route_words) + || (url_words.len == 0 && route_words == ['index'] && method.name == 'index') { + // Check whether the request is authorised + if 'auth' in method.attrs && !app.is_authenticated() { + conn.write(http_401.bytes()) or {} + return } - return - } - if url_words.len == 0 && route_words == ['index'] && method.name == 'index' { + // We found a match app.$method() return - } + } else if params := route_matches(url_words, route_words) { + // Check whether the request is authorised + if 'auth' in method.attrs && !app.is_authenticated() { + conn.write(http_401.bytes()) or {} + return + } - if params := route_matches(url_words, route_words) { method_args := params.clone() if method_args.len != method.args.len { eprintln('warning: uneven parameters count ($method.args.len) in `$method.name`, compared to the web route `$method.attrs` ($method_args.len)') From 272f14b2645a56aedd608fd0880a768d43bfa75e Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 4 Sep 2022 19:36:08 +0200 Subject: [PATCH 45/62] refactor(server): migrated all routes to new auth system --- src/server/api_logs.v | 24 ++++-------------------- src/server/api_targets.v | 30 +++++------------------------- src/server/auth.v | 12 ------------ src/server/repo.v | 6 +----- src/server/repo_remove.v | 18 +++--------------- src/web/web.v | 1 - 6 files changed, 13 insertions(+), 78 deletions(-) delete mode 100644 src/server/auth.v diff --git a/src/server/api_logs.v b/src/server/api_logs.v index 6728392c..021c1aca 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -12,12 +12,8 @@ import models { BuildLog, BuildLogFilter } // v1_get_logs returns all build logs in the database. A 'target' query param can // optionally be added to limit the list of build logs to that repository. -['/api/v1/logs'; get] +['/api/v1/logs'; auth; get] fn (mut app App) v1_get_logs() web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - filter := models.from_params(app.query) or { return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } @@ -27,24 +23,16 @@ fn (mut app App) v1_get_logs() web.Result { } // v1_get_single_log returns the build log with the given id. -['/api/v1/logs/:id'; get] +['/api/v1/logs/:id'; auth; get] fn (mut app App) v1_get_single_log(id int) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - log := app.db.get_build_log(id) or { return app.not_found() } return app.json(http.Status.ok, new_data_response(log)) } // v1_get_log_content returns the actual build log file for the given id. -['/api/v1/logs/:id/content'; get] +['/api/v1/logs/:id/content'; auth; get] fn (mut app App) v1_get_log_content(id int) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - log := app.db.get_build_log(id) or { return app.not_found() } file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), log.arch, @@ -63,12 +51,8 @@ fn parse_query_time(query string) ?time.Time { } // v1_post_log adds a new log to the database. -['/api/v1/logs'; post] +['/api/v1/logs'; auth; post] fn (mut app App) v1_post_log() web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - // Parse query params start_time_int := app.query['startTime'].int() diff --git a/src/server/api_targets.v b/src/server/api_targets.v index 4cc3a584..c9e7963d 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -7,12 +7,8 @@ import db import models { Target, TargetArch, TargetFilter } // v1_get_targets returns the current list of targets. -['/api/v1/targets'; get] +['/api/v1/targets'; auth; get] fn (mut app App) v1_get_targets() web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - filter := models.from_params(app.query) or { return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } @@ -22,24 +18,16 @@ fn (mut app App) v1_get_targets() web.Result { } // v1_get_single_target returns the information for a single target. -['/api/v1/targets/:id'; get] +['/api/v1/targets/:id'; auth; get] fn (mut app App) v1_get_single_target(id int) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - repo := app.db.get_target(id) or { return app.not_found() } return app.json(http.Status.ok, new_data_response(repo)) } // v1_post_target creates a new target from the provided query string. -['/api/v1/targets'; post] +['/api/v1/targets'; auth; post] fn (mut app App) v1_post_target() web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - mut params := app.query.clone() // If a repo is created without specifying the arch, we assume it's meant @@ -63,24 +51,16 @@ fn (mut app App) v1_post_target() web.Result { } // v1_delete_target removes a given target from the server's list. -['/api/v1/targets/:id'; delete] +['/api/v1/targets/:id'; auth; delete] fn (mut app App) v1_delete_target(id int) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - app.db.delete_target(id) return app.json(http.Status.ok, new_response('Repo removed successfully.')) } // v1_patch_target updates a target's data with the given query params. -['/api/v1/targets/:id'; patch] +['/api/v1/targets/:id'; auth; patch] fn (mut app App) v1_patch_target(id int) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - app.db.update_target(id, app.query) if 'arch' in app.query { diff --git a/src/server/auth.v b/src/server/auth.v deleted file mode 100644 index 7c8a676f..00000000 --- a/src/server/auth.v +++ /dev/null @@ -1,12 +0,0 @@ -module server - -import net.http - -// is_authorized checks whether the provided API key is correct. -fn (mut app App) is_authorized() bool { - x_header := app.req.header.get_custom('X-Api-Key', http.HeaderQueryConfig{ exact: true }) or { - return false - } - - return x_header.trim_space() == app.conf.api_key -} diff --git a/src/server/repo.v b/src/server/repo.v index 242fd2df..5ed5d15c 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -49,12 +49,8 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re } // put_package handles publishing a package to a repository. -['/:repo/publish'; post] +['/:repo/publish'; auth; post] fn (mut app App) put_package(repo string) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - mut pkg_path := '' if length := app.req.header.get(.content_length) { diff --git a/src/server/repo_remove.v b/src/server/repo_remove.v index 316b3872..fdc40e87 100644 --- a/src/server/repo_remove.v +++ b/src/server/repo_remove.v @@ -5,12 +5,8 @@ import net.http import web.response { new_response } // delete_package tries to remove the given package. -['/:repo/:arch/:pkg'; delete] +['/:repo/:arch/:pkg'; auth; delete] fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result { - if !app.is_authorized() { - return app.json(.unauthorized, new_response('Unauthorized.')) - } - res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { app.lerror('Error while deleting package: $err.msg()') @@ -29,12 +25,8 @@ fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result } // delete_arch_repo tries to remove the given arch-repo. -['/:repo/:arch'; delete] +['/:repo/:arch'; auth; delete] fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - res := app.repo.remove_arch_repo(repo, arch) or { app.lerror('Error while deleting arch-repo: $err.msg()') @@ -53,12 +45,8 @@ fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { } // delete_repo tries to remove the given repo. -['/:repo'; delete] +['/:repo'; auth; delete] fn (mut app App) delete_repo(repo string) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - res := app.repo.remove_repo(repo) or { app.lerror('Error while deleting repo: $err.msg()') diff --git a/src/web/web.v b/src/web/web.v index 0847c7e6..1d1480fb 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -394,7 +394,6 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { api_key: app.api_key } - // Calling middleware... app.before_request() From 9dfdfbf72446f04b38eb6962bfd98600d380e7ca Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 4 Sep 2022 20:16:58 +0200 Subject: [PATCH 46/62] chore: update README [CI SKIP] --- Makefile | 7 ------- README.md | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index ed44df92..69bd7957 100644 --- a/Makefile +++ b/Makefile @@ -83,13 +83,6 @@ fmt: test: $(V) test $(SRC_DIR) -# Build & patch the V compiler -.PHONY: v -v: v/v -v/v: - git clone --single-branch https://git.rustybever.be/vieter-v/v v - make -C v - .PHONY: clean clean: rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' 'docs/public' diff --git a/README.md b/README.md index 29ec0f01..b9fff692 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,6 @@ that. Besides a V installer, Vieter also requires the following libraries to work: -* gc * libarchive * openssl * sqlite3 @@ -49,7 +48,8 @@ update`. ### Compiler I used to maintain a mirror that tracked the latest master, but nowadays, I -solely target V 0.3 as a compiler. +maintain a Docker image containing the specific compiler version that Vieter +builds with. Currently, this is V 0.3. ## Contributing From 7b59277931172997759572ac5a00930d677a4d67 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 5 Sep 2022 10:10:02 +0200 Subject: [PATCH 47/62] feat: adding target returns id of added entry --- src/client/targets.v | 4 ++-- src/console/targets/targets.v | 2 +- src/db/targets.v | 6 +++++- src/server/api_targets.v | 6 +++--- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/client/targets.v b/src/client/targets.v index f5258a43..c5e44fea 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -49,9 +49,9 @@ pub struct NewTarget { } // add_target adds a new target to the server. -pub fn (c &Client) add_target(t NewTarget) ?Response { +pub fn (c &Client) add_target(t NewTarget) ?Response { params := models.params_from(t) - data := c.send_request(Method.post, '/api/v1/targets', params)? + data := c.send_request(Method.post, '/api/v1/targets', params)? return data } diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 66d48fb3..a5559360 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -206,7 +206,7 @@ fn add(conf Config, t &NewTarget) ? { c := client.new(conf.address, conf.api_key) res := c.add_target(t)? - println(res.message) + println("Target added with id $res.data") } // remove removes a repository from the server's list. diff --git a/src/db/targets.v b/src/db/targets.v index 91020337..a705ebb5 100644 --- a/src/db/targets.v +++ b/src/db/targets.v @@ -38,10 +38,14 @@ pub fn (db &VieterDb) get_target(target_id int) ?Target { } // add_target inserts the given target into the database. -pub fn (db &VieterDb) add_target(repo Target) { +pub fn (db &VieterDb) add_target(repo Target) int { sql db.conn { insert repo into Target } + + inserted_id := db.conn.last_id() as int + + return inserted_id } // delete_target deletes the target with the given id from the database. diff --git a/src/server/api_targets.v b/src/server/api_targets.v index c9e7963d..5757d76b 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -2,7 +2,7 @@ module server import web import net.http -import web.response { new_data_response, new_response } +import web.response { new_data_response, new_response, new_full_response } import db import models { Target, TargetArch, TargetFilter } @@ -45,9 +45,9 @@ fn (mut app App) v1_post_target() web.Result { return app.json(http.Status.bad_request, new_response('Invalid kind.')) } - app.db.add_target(new_repo) + id := app.db.add_target(new_repo) - return app.json(http.Status.ok, new_response('Repo added successfully.')) + return app.json(http.Status.ok, new_data_response(id)) } // v1_delete_target removes a given target from the server's list. From 210508f1ee251b3cb61ab5bf146805f10b600c1c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 20:34:02 +0200 Subject: [PATCH 48/62] feat: logs api now also returns id --- CHANGELOG.md | 1 + src/client/logs.v | 4 ++-- src/console/targets/targets.v | 2 +- src/db/logs.v | 6 +++++- src/server/api_logs.v | 9 +++++---- src/server/api_targets.v | 10 +++++----- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5aa0e434..00a3539c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 repository will be cloned with the default branch * Build containers now explicitely set the PATH variable * Refactor of web framework +* API endpoints now return id of newly created entries ### Removed diff --git a/src/client/logs.v b/src/client/logs.v index b52c3d09..b4142450 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -39,7 +39,7 @@ pub fn (c &Client) get_build_log_content(id int) ?string { } // add_build_log adds a new build log to the server. -pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { +pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { params := { 'target': target_id.str() 'startTime': start_time.unix_time().str() @@ -48,7 +48,7 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t 'exitCode': exit_code.str() } - data := c.send_request_with_body(Method.post, '/api/v1/logs', params, content)? + data := c.send_request_with_body(Method.post, '/api/v1/logs', params, content)? return data } diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index a5559360..2784dbc3 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -206,7 +206,7 @@ fn add(conf Config, t &NewTarget) ? { c := client.new(conf.address, conf.api_key) res := c.add_target(t)? - println("Target added with id $res.data") + println('Target added with id $res.data') } // remove removes a repository from the server's list. diff --git a/src/db/logs.v b/src/db/logs.v index af5f53c3..923dde2d 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -79,10 +79,14 @@ pub fn (db &VieterDb) get_build_log(id int) ?BuildLog { } // add_build_log inserts the given BuildLog into the database. -pub fn (db &VieterDb) add_build_log(log BuildLog) { +pub fn (db &VieterDb) add_build_log(log BuildLog) int { sql db.conn { insert log into BuildLog } + + inserted_id := db.conn.last_id() as int + + return inserted_id } // delete_build_log delete the BuildLog with the given ID from the database. diff --git a/src/server/api_logs.v b/src/server/api_logs.v index 021c1aca..287755a9 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -19,7 +19,7 @@ fn (mut app App) v1_get_logs() web.Result { } logs := app.db.get_build_logs(filter) - return app.json(http.Status.ok, new_data_response(logs)) + return app.json(.ok, new_data_response(logs)) } // v1_get_single_log returns the build log with the given id. @@ -27,7 +27,7 @@ fn (mut app App) v1_get_logs() web.Result { fn (mut app App) v1_get_single_log(id int) web.Result { log := app.db.get_build_log(id) or { return app.not_found() } - return app.json(http.Status.ok, new_data_response(log)) + return app.json(.ok, new_data_response(log)) } // v1_get_log_content returns the actual build log file for the given id. @@ -95,7 +95,8 @@ fn (mut app App) v1_post_log() web.Result { exit_code: exit_code } - app.db.add_build_log(log) + // id of newly created log + log_id := app.db.add_build_log(log) repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, target_id.str(), arch) @@ -122,5 +123,5 @@ fn (mut app App) v1_post_log() web.Result { return app.status(http.Status.length_required) } - return app.json(http.Status.ok, new_response('Logs added successfully.')) + return app.json(.ok, new_data_response(log_id)) } diff --git a/src/server/api_targets.v b/src/server/api_targets.v index 5757d76b..6f284af6 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -2,7 +2,7 @@ module server import web import net.http -import web.response { new_data_response, new_response, new_full_response } +import web.response { new_data_response, new_response } import db import models { Target, TargetArch, TargetFilter } @@ -14,7 +14,7 @@ fn (mut app App) v1_get_targets() web.Result { } repos := app.db.get_targets(filter) - return app.json(http.Status.ok, new_data_response(repos)) + return app.json(.ok, new_data_response(repos)) } // v1_get_single_target returns the information for a single target. @@ -22,7 +22,7 @@ fn (mut app App) v1_get_targets() web.Result { fn (mut app App) v1_get_single_target(id int) web.Result { repo := app.db.get_target(id) or { return app.not_found() } - return app.json(http.Status.ok, new_data_response(repo)) + return app.json(.ok, new_data_response(repo)) } // v1_post_target creates a new target from the provided query string. @@ -55,7 +55,7 @@ fn (mut app App) v1_post_target() web.Result { fn (mut app App) v1_delete_target(id int) web.Result { app.db.delete_target(id) - return app.json(http.Status.ok, new_response('Repo removed successfully.')) + return app.status(.ok) } // v1_patch_target updates a target's data with the given query params. @@ -69,5 +69,5 @@ fn (mut app App) v1_patch_target(id int) web.Result { app.db.update_target_archs(id, arch_objs) } - return app.json(http.Status.ok, new_response('Repo updated successfully.')) + return app.status(.ok) } From b6cd2f0bc280dfddfa85a91188ebe05433eabbf8 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 21:24:29 +0200 Subject: [PATCH 49/62] feat(server): repo POST requests now return information --- CHANGELOG.md | 1 + src/repo/add.v | 43 +++++++++++++++++++------------------------ src/server/repo.v | 21 ++++++--------------- 3 files changed, 26 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00a3539c..c875f2e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Build containers now explicitely set the PATH variable * Refactor of web framework * API endpoints now return id of newly created entries +* Repo POST requests now return information on published package ### Removed diff --git a/src/repo/add.v b/src/repo/add.v index 0985c7a6..608ca501 100644 --- a/src/repo/add.v +++ b/src/repo/add.v @@ -23,8 +23,9 @@ pub: pub struct RepoAddResult { pub: - added bool [required] - pkg &package.Pkg [required] + name string + version string + archs []string } // new creates a new RepoGroupManager & creates the directories as needed @@ -53,10 +54,10 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re return error('Failed to read package file: $err.msg()') } - added := r.add_pkg_in_repo(repo, pkg)? + archs := r.add_pkg_in_repo(repo, pkg)? // If the add was successful, we move the file to the packages directory - for arch in added { + for arch in archs { repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch)) dest_path := os.join_path_single(repo_pkg_path, pkg.filename()) @@ -71,8 +72,9 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re os.rm(pkg_path)? return RepoAddResult{ - added: added.len > 0 - pkg: &pkg + name: pkg.info.name + version: pkg.info.version + archs: archs } } @@ -87,11 +89,9 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin // A package not of arch 'any' can be handled easily by adding it to the // respective repo if pkg.info.arch != 'any' { - if r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)? { - return [pkg.info.arch] - } else { - return [] - } + r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)? + + return [pkg.info.arch] } mut arch_repos := []string{} @@ -113,25 +113,22 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin arch_repos << r.default_arch } - mut added := []string{} - - // We add the package to each repository. If any of the repositories - // return true, the result of the function is also true. + // Add the package to each found architecture + // NOTE: if any of these fail, the function fails. This means the user does + // not know which arch-repositories did succeed in adding the package, if + // any. for arch in arch_repos { - if r.add_pkg_in_arch_repo(repo, arch, pkg)? { - added << arch - } + r.add_pkg_in_arch_repo(repo, arch, pkg)? } - return added + return arch_repos } // add_pkg_in_arch_repo is the function that actually adds a package to a given // arch-repo. It records the package's data in the arch-repo's desc & files // files, and afterwards updates the db & files archives to reflect these -// changes. The function returns false if the package was already present in -// the repo, and true otherwise. -fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool { +// changes. +fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ? { pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version') // Remove the previous version of the package, if present @@ -151,6 +148,4 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac } r.sync(repo, arch)? - - return true } diff --git a/src/server/repo.v b/src/server/repo.v index 5ed5d15c..526d4e76 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -6,8 +6,7 @@ import repo import time import rand import util -import net.http -import web.response { new_response } +import web.response { new_data_response, new_response } // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. @@ -65,7 +64,7 @@ fn (mut app App) put_package(repo string) web.Result { util.reader_to_file(mut app.reader, length.int(), pkg_path) or { app.lwarn("Failed to upload '$pkg_path'") - return app.json(http.Status.internal_server_error, new_response('Failed to upload file.')) + return app.status(.internal_server_error) } sw.stop() @@ -74,7 +73,7 @@ fn (mut app App) put_package(repo string) web.Result { app.lwarn('Tried to upload package without specifying a Content-Length.') // length required - return app.status(http.Status.length_required) + return app.status(.length_required) } res := app.repo.add_pkg_from_path(repo, pkg_path) or { @@ -82,18 +81,10 @@ fn (mut app App) put_package(repo string) web.Result { os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg()") } - return app.json(http.Status.internal_server_error, new_response('Failed to add package.')) + return app.status(.internal_server_error) } - if !res.added { - os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg()") } + app.linfo("Added '$res.name-$res.version' to '$repo (${res.archs.join(',')})'.") - app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo'.") - - return app.json(http.Status.bad_request, new_response('File already exists.')) - } - - app.linfo("Added '$res.pkg.full_name()' to repo '$repo ($res.pkg.info.arch)'.") - - return app.json(http.Status.ok, new_response('Package added successfully.')) + return app.json(.ok, new_data_response(res)) } From cf67b46df0cbd69444f2e6046937c74583021d84 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 21:28:37 +0200 Subject: [PATCH 50/62] feat(server): less verbose repo DELETE responses --- src/server/repo_remove.v | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/server/repo_remove.v b/src/server/repo_remove.v index fdc40e87..694f0854 100644 --- a/src/server/repo_remove.v +++ b/src/server/repo_remove.v @@ -1,8 +1,6 @@ module server import web -import net.http -import web.response { new_response } // delete_package tries to remove the given package. ['/:repo/:arch/:pkg'; auth; delete] @@ -10,17 +8,17 @@ fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { app.lerror('Error while deleting package: $err.msg()') - return app.json(http.Status.internal_server_error, new_response('Failed to delete package.')) + return app.status(.internal_server_error) } if res { app.linfo("Removed package '$pkg' from '$repo/$arch'") - return app.json(http.Status.ok, new_response('Package removed.')) + return app.status(.ok) } else { app.linfo("Tried removing package '$pkg' from '$repo/$arch', but it doesn't exist.") - return app.json(http.Status.not_found, new_response('Package not found.')) + return app.status(.not_found) } } @@ -30,17 +28,17 @@ fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { res := app.repo.remove_arch_repo(repo, arch) or { app.lerror('Error while deleting arch-repo: $err.msg()') - return app.json(http.Status.internal_server_error, new_response('Failed to delete arch-repo.')) + return app.status(.internal_server_error) } if res { - app.linfo("Removed '$repo/$arch'") + app.linfo("Removed arch-repo '$repo/$arch'") - return app.json(http.Status.ok, new_response('Arch-repo removed.')) + return app.status(.ok) } else { app.linfo("Tried removing '$repo/$arch', but it doesn't exist.") - return app.json(http.Status.not_found, new_response('Arch-repo not found.')) + return app.status(.not_found) } } @@ -50,16 +48,16 @@ fn (mut app App) delete_repo(repo string) web.Result { res := app.repo.remove_repo(repo) or { app.lerror('Error while deleting repo: $err.msg()') - return app.json(http.Status.internal_server_error, new_response('Failed to delete repo.')) + return app.status(.internal_server_error) } if res { - app.linfo("Removed '$repo'") + app.linfo("Removed repo '$repo'") - return app.json(http.Status.ok, new_response('Repo removed.')) + return app.status(.ok) } else { app.linfo("Tried removing '$repo', but it doesn't exist.") - return app.json(http.Status.not_found, new_response('Repo not found.')) + return app.status(.not_found) } } From 8a08788935bb9514007f41d17d18cb2d1b879ecf Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 21:50:29 +0200 Subject: [PATCH 51/62] feat(console): tabled outputs now optionally return without decorations --- src/console/console.v | 5 +++++ src/console/logs/logs.v | 20 +++++++++++++------- src/console/targets/targets.v | 12 +++++++++--- src/main.v | 7 +++++++ 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/console/console.v b/src/console/console.v index 7d782bac..caf4cca7 100644 --- a/src/console/console.v +++ b/src/console/console.v @@ -5,6 +5,11 @@ import strings import cli import os +// tabbed_table returns a simple textual table, with tabs as separators. +pub fn tabbed_table(data [][]string) string { + return data.map(it.join('\t')).join('\n') +} + // pretty_table converts a list of string data into a pretty table. Many thanks // to @hungrybluedev in the Vlang Discord for providing this code! // https://ptb.discord.com/channels/592103645835821068/592106336838352923/970278787143045192 diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 0f023bc7..41830c25 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -133,7 +133,9 @@ pub fn cmd() cli.Command { ] } - list(conf, filter)? + raw := cmd.flags.get_bool('raw')? + + list(conf, filter, raw)? } }, cli.Command{ @@ -167,27 +169,31 @@ pub fn cmd() cli.Command { } // print_log_list prints a list of logs. -fn print_log_list(logs []BuildLog) ? { +fn print_log_list(logs []BuildLog, raw bool) ? { data := logs.map([it.id.str(), it.target_id.str(), it.start_time.local().str(), it.exit_code.str()]) - println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)?) + if raw { + println(console.tabbed_table(data)) + } else { + println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)?) + } } // list prints a list of all build logs. -fn list(conf Config, filter BuildLogFilter) ? { +fn list(conf Config, filter BuildLogFilter, raw bool) ? { c := client.new(conf.address, conf.api_key) logs := c.get_build_logs(filter)?.data - print_log_list(logs)? + print_log_list(logs, raw)? } // list prints a list of all build logs for a given target. -fn list_for_target(conf Config, target_id int) ? { +fn list_for_target(conf Config, target_id int, raw bool) ? { c := client.new(conf.address, conf.api_key) logs := c.get_build_logs_for_target(target_id)?.data - print_log_list(logs)? + print_log_list(logs, raw)? } // info print the detailed info for a given build log. diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 2784dbc3..198e0626 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -60,7 +60,9 @@ pub fn cmd() cli.Command { filter.repo = repo } - list(conf, filter)? + raw := cmd.flags.get_bool('raw')? + + list(conf, filter, raw)? } }, cli.Command{ @@ -193,12 +195,16 @@ pub fn cmd() cli.Command { // ID. If multiple or none are found, an error is raised. // list prints out a list of all repositories. -fn list(conf Config, filter TargetFilter) ? { +fn list(conf Config, filter TargetFilter, raw bool) ? { c := client.new(conf.address, conf.api_key) repos := c.get_targets(filter)? data := repos.map([it.id.str(), it.kind, it.url, it.repo]) - println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?) + if raw { + println(console.tabbed_table(data)) + } else { + println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?) + } } // add adds a new repository to the server's list. diff --git a/src/main.v b/src/main.v index 4ade9305..0e98bd26 100644 --- a/src/main.v +++ b/src/main.v @@ -24,6 +24,13 @@ fn main() { global: true default_value: [os.expand_tilde_to_home('~/.vieterrc')] }, + cli.Flag{ + flag: cli.FlagType.bool + name: 'raw' + abbrev: 'r' + description: 'Only output minimal information (no formatted tables, etc.)' + global: true + }, ] commands: [ server.cmd(), From fab8ca20b82e7f088d95efdc945a600050fefd6b Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 1 Oct 2022 16:02:43 +0200 Subject: [PATCH 52/62] cli: targets add now supports raw flag --- src/console/targets/targets.v | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 198e0626..56400117 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -94,7 +94,9 @@ pub fn cmd() cli.Command { branch: cmd.flags.get_string('branch') or { '' } } - add(conf, t)? + raw := cmd.flags.get_bool('raw')? + + add(conf, t, raw)? } }, cli.Command{ @@ -208,11 +210,15 @@ fn list(conf Config, filter TargetFilter, raw bool) ? { } // add adds a new repository to the server's list. -fn add(conf Config, t &NewTarget) ? { +fn add(conf Config, t &NewTarget, raw bool) ? { c := client.new(conf.address, conf.api_key) res := c.add_target(t)? - println('Target added with id $res.data') + if raw { + println(res.data) + } else { + println('Target added with id $res.data') + } } // remove removes a repository from the server's list. From 851a446a954e619e474251533960b00a4b64bef3 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 1 Oct 2022 16:21:49 +0200 Subject: [PATCH 53/62] chore: updated changelog [CI SKIP] --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c875f2e7..60b70b60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * CLI commands for searching the AUR & directly adding packages * HTTP routes for removing packages, arch-repos & repos * All endpoints serving files now support HTTP byte range requests +* Better CLI UX + * When adding targets, the ID of the created target is returned + * The `-r` flag only shows raw data of action + * When adding a target, only ID is shown and not surrounding text + * Tabled output returns a tab-separated list (easy to script using + `cut`) ### Changed From 575c04189da2088261b6c3d5921ded948312e568 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 22:20:02 +0200 Subject: [PATCH 54/62] fix(client): allow empty values as params --- src/client/client.v | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/client/client.v b/src/client/client.v index 24e44448..d68ff18f 100644 --- a/src/client/client.v +++ b/src/client/client.v @@ -30,12 +30,10 @@ fn (c &Client) send_request_raw(method Method, url string, params map[string]str // Escape each query param for k, v in params { // An empty parameter should be the same as not providing it at all - if v != '' { - params_escaped[k] = urllib.query_escape(v) - } + params_escaped[k] = urllib.query_escape(v) } - params_str := params_escaped.keys().map('$it=${params[it]}').join('&') + params_str := params_escaped.keys().map('$it=${params_escaped[it]}').join('&') full_url = '$full_url?$params_str' } From 95d32e2d515edf27a919e847fef5f5459f3f7234 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 22:24:29 +0200 Subject: [PATCH 55/62] fix(server): prevent `api` as a repository name --- CHANGELOG.md | 2 ++ src/server/repo.v | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60b70b60..79ceed49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Refactor of web framework * API endpoints now return id of newly created entries * Repo POST requests now return information on published package +* `api` can no longer be used as a repository name +* CLI client now allows setting values to an empty value ### Removed diff --git a/src/server/repo.v b/src/server/repo.v index 526d4e76..06ab72e6 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -50,6 +50,12 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re // put_package handles publishing a package to a repository. ['/:repo/publish'; auth; post] fn (mut app App) put_package(repo string) web.Result { + // api is a reserved keyword for api routes & should never be allowed to be + // a repository. + if repo.to_lower() == 'api' { + return app.json(.bad_request, new_response("'api' is a reserved keyword & cannot be used as a repository name.")) + } + mut pkg_path := '' if length := app.req.header.get(.content_length) { From 847d77b2bc6e1547da489048b196dd2cb641c4f5 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 15 Sep 2022 13:42:49 +0200 Subject: [PATCH 56/62] chore(ci): refactor ci configs a bit --- .woodpecker/{.arch-rel.yml => arch-rel.yml} | 0 .woodpecker/{.arch.yml => arch.yml} | 0 .woodpecker/{.build.yml => build.yml} | 11 +++++++---- .woodpecker/{.deploy.yml => deploy.yml} | 0 .woodpecker/{.docker.yml => docker.yml} | 0 .woodpecker/{.docs.yml => docs.yml} | 5 ++++- .woodpecker/{.gitea.yml => gitea.yml} | 5 ++++- .woodpecker/{.lint.yml => lint.yml} | 5 ++++- .woodpecker/{.man.yml => man.yml} | 5 ++++- .woodpecker/{.test.yml => test.yml} | 7 +++++-- 10 files changed, 28 insertions(+), 10 deletions(-) rename .woodpecker/{.arch-rel.yml => arch-rel.yml} (100%) rename .woodpecker/{.arch.yml => arch.yml} (100%) rename .woodpecker/{.build.yml => build.yml} (92%) rename .woodpecker/{.deploy.yml => deploy.yml} (100%) rename .woodpecker/{.docker.yml => docker.yml} (100%) rename .woodpecker/{.docs.yml => docs.yml} (93%) rename .woodpecker/{.gitea.yml => gitea.yml} (89%) rename .woodpecker/{.lint.yml => lint.yml} (73%) rename .woodpecker/{.man.yml => man.yml} (92%) rename .woodpecker/{.test.yml => test.yml} (80%) diff --git a/.woodpecker/.arch-rel.yml b/.woodpecker/arch-rel.yml similarity index 100% rename from .woodpecker/.arch-rel.yml rename to .woodpecker/arch-rel.yml diff --git a/.woodpecker/.arch.yml b/.woodpecker/arch.yml similarity index 100% rename from .woodpecker/.arch.yml rename to .woodpecker/arch.yml diff --git a/.woodpecker/.build.yml b/.woodpecker/build.yml similarity index 92% rename from .woodpecker/.build.yml rename to .woodpecker/build.yml index 580fa69e..9ee80853 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/build.yml @@ -1,3 +1,6 @@ +variables: + - &vlang_image 'chewingbever/vlang:0.3' + matrix: PLATFORM: - 'linux/amd64' @@ -7,7 +10,7 @@ platform: ${PLATFORM} pipeline: install-modules: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true commands: - export VMODULES=$PWD/.vmodules @@ -16,7 +19,7 @@ pipeline: event: [push, pull_request] debug: - image: 'chewingbever/vlang:0.3' + image: *vlang_image commands: - export VMODULES=$PWD/.vmodules - make @@ -26,7 +29,7 @@ pipeline: exclude: [main] prod: - image: 'chewingbever/vlang:0.3' + image: *vlang_image environment: - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static commands: @@ -44,7 +47,7 @@ pipeline: event: [push, pull_request] upload: - image: 'chewingbever/vlang:0.3' + image: *vlang_image secrets: [ s3_username, s3_password ] commands: # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f diff --git a/.woodpecker/.deploy.yml b/.woodpecker/deploy.yml similarity index 100% rename from .woodpecker/.deploy.yml rename to .woodpecker/deploy.yml diff --git a/.woodpecker/.docker.yml b/.woodpecker/docker.yml similarity index 100% rename from .woodpecker/.docker.yml rename to .woodpecker/docker.yml diff --git a/.woodpecker/.docs.yml b/.woodpecker/docs.yml similarity index 93% rename from .woodpecker/.docs.yml rename to .woodpecker/docs.yml index da495fcb..048b1ad9 100644 --- a/.woodpecker/.docs.yml +++ b/.woodpecker/docs.yml @@ -1,3 +1,6 @@ +variables: + - &vlang_image 'chewingbever/vlang:0.3' + platform: 'linux/amd64' branches: exclude: [ main ] @@ -11,7 +14,7 @@ pipeline: - make docs api-docs: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true group: 'generate' commands: diff --git a/.woodpecker/.gitea.yml b/.woodpecker/gitea.yml similarity index 89% rename from .woodpecker/.gitea.yml rename to .woodpecker/gitea.yml index 55f991e2..8e3b9d44 100644 --- a/.woodpecker/.gitea.yml +++ b/.woodpecker/gitea.yml @@ -1,3 +1,6 @@ +variables: + - &vlang_image 'chewingbever/vlang:0.3' + platform: 'linux/amd64' branches: [ 'main' ] depends_on: @@ -8,7 +11,7 @@ skip_clone: true pipeline: prepare: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true secrets: [ s3_username, s3_password ] commands: diff --git a/.woodpecker/.lint.yml b/.woodpecker/lint.yml similarity index 73% rename from .woodpecker/.lint.yml rename to .woodpecker/lint.yml index 75a81059..c80ce338 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/lint.yml @@ -1,3 +1,6 @@ +variables: + - &vlang_image 'chewingbever/vlang:0.3' + # These checks already get performed on the feature branches branches: exclude: [ main ] @@ -5,7 +8,7 @@ platform: 'linux/amd64' pipeline: lint: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true commands: - make lint diff --git a/.woodpecker/.man.yml b/.woodpecker/man.yml similarity index 92% rename from .woodpecker/.man.yml rename to .woodpecker/man.yml index 1a30b03d..86a1bd8c 100644 --- a/.woodpecker/.man.yml +++ b/.woodpecker/man.yml @@ -1,3 +1,6 @@ +variables: + - &vlang_image 'chewingbever/vlang:0.3' + platform: 'linux/amd64' branches: exclude: [ main ] @@ -9,7 +12,7 @@ skip_clone: true pipeline: generate: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true commands: - curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64" diff --git a/.woodpecker/.test.yml b/.woodpecker/test.yml similarity index 80% rename from .woodpecker/.test.yml rename to .woodpecker/test.yml index 6c267fab..08b7534f 100644 --- a/.woodpecker/.test.yml +++ b/.woodpecker/test.yml @@ -1,3 +1,6 @@ +variables: + - &vlang_image 'chewingbever/vlang:0.3' + matrix: PLATFORM: - 'linux/amd64' @@ -9,7 +12,7 @@ platform: ${PLATFORM} pipeline: install-modules: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true commands: - export VMODULES=$PWD/.vmodules @@ -18,7 +21,7 @@ pipeline: event: [pull_request] test: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true commands: - export VMODULES=$PWD/.vmodules From 559ef3e505495c9b87b1f6dfb781c7230a7caa1a Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 20:34:02 +0200 Subject: [PATCH 57/62] feat: logs api now also returns id --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79ceed49..b4efd58b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Repo POST requests now return information on published package * `api` can no longer be used as a repository name * CLI client now allows setting values to an empty value +* API endpoints now return id of newly created entries ### Removed From ae98c3e717431952eb248cac3fdc99b85e946d7a Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 21:24:29 +0200 Subject: [PATCH 58/62] feat(server): repo POST requests now return information --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4efd58b..16ed2a7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `api` can no longer be used as a repository name * CLI client now allows setting values to an empty value * API endpoints now return id of newly created entries +* Repo POST requests now return information on published package ### Removed From f34eefd59bfcfdc996f12779cd3101d740037d24 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 1 Oct 2022 16:35:43 +0200 Subject: [PATCH 59/62] fix(server): prevent `api` as a repository name --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16ed2a7f..113dc21d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * CLI client now allows setting values to an empty value * API endpoints now return id of newly created entries * Repo POST requests now return information on published package +* `api` can no longer be used as a repository name +* CLI client now allows setting values to an empty value ### Removed From 15c2d7274310de6fa77c99933ba6891493ec3e01 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 1 Oct 2022 16:53:16 +0200 Subject: [PATCH 60/62] docs: updated json response types for create routes [CI SKIP] --- docs/api/source/includes/_logs.md | 11 +++++++++++ docs/api/source/includes/_targets.md | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/api/source/includes/_logs.md b/docs/api/source/includes/_logs.md index 2797e60d..1c14e71a 100644 --- a/docs/api/source/includes/_logs.md +++ b/docs/api/source/includes/_logs.md @@ -112,6 +112,17 @@ id | ID of requested log ## Publish build log +> JSON output format + +```json +{ + "message": "", + "data": { + "id": 15 + } +} +``` +