Compare commits

...

14 Commits

25 changed files with 154 additions and 95 deletions

View File

@ -1,3 +1,6 @@
variables:
- &vlang_image 'chewingbever/vlang:0.3'
matrix:
PLATFORM:
- 'linux/amd64'
@ -7,7 +10,7 @@ platform: ${PLATFORM}
pipeline:
install-modules:
image: 'chewingbever/vlang:0.3'
image: *vlang_image
pull: true
commands:
- export VMODULES=$PWD/.vmodules
@ -16,7 +19,7 @@ pipeline:
event: [push, pull_request]
debug:
image: 'chewingbever/vlang:0.3'
image: *vlang_image
commands:
- export VMODULES=$PWD/.vmodules
- make
@ -26,7 +29,7 @@ pipeline:
exclude: [main]
prod:
image: 'chewingbever/vlang:0.3'
image: *vlang_image
environment:
- LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static
commands:
@ -44,7 +47,7 @@ pipeline:
event: [push, pull_request]
upload:
image: 'chewingbever/vlang:0.3'
image: *vlang_image
secrets: [ s3_username, s3_password ]
commands:
# https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f

View File

@ -1,3 +1,6 @@
variables:
- &vlang_image 'chewingbever/vlang:0.3'
platform: 'linux/amd64'
branches:
exclude: [ main ]
@ -11,7 +14,7 @@ pipeline:
- make docs
api-docs:
image: 'chewingbever/vlang:0.3'
image: *vlang_image
pull: true
group: 'generate'
commands:

View File

@ -1,3 +1,6 @@
variables:
- &vlang_image 'chewingbever/vlang:0.3'
platform: 'linux/amd64'
branches: [ 'main' ]
depends_on:
@ -8,7 +11,7 @@ skip_clone: true
pipeline:
prepare:
image: 'chewingbever/vlang:0.3'
image: *vlang_image
pull: true
secrets: [ s3_username, s3_password ]
commands:

View File

@ -1,3 +1,6 @@
variables:
- &vlang_image 'chewingbever/vlang:0.3'
# These checks already get performed on the feature branches
branches:
exclude: [ main ]
@ -5,7 +8,7 @@ platform: 'linux/amd64'
pipeline:
lint:
image: 'chewingbever/vlang:0.3'
image: *vlang_image
pull: true
commands:
- make lint

View File

@ -1,3 +1,6 @@
variables:
- &vlang_image 'chewingbever/vlang:0.3'
platform: 'linux/amd64'
branches:
exclude: [ main ]
@ -9,7 +12,7 @@ skip_clone: true
pipeline:
generate:
image: 'chewingbever/vlang:0.3'
image: *vlang_image
pull: true
commands:
- curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64"

View File

@ -1,3 +1,6 @@
variables:
- &vlang_image 'chewingbever/vlang:0.3'
matrix:
PLATFORM:
- 'linux/amd64'
@ -9,7 +12,7 @@ platform: ${PLATFORM}
pipeline:
install-modules:
image: 'chewingbever/vlang:0.3'
image: *vlang_image
pull: true
commands:
- export VMODULES=$PWD/.vmodules
@ -18,7 +21,7 @@ pipeline:
event: [pull_request]
test:
image: 'chewingbever/vlang:0.3'
image: *vlang_image
pull: true
commands:
- export VMODULES=$PWD/.vmodules

View File

@ -17,6 +17,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* CLI commands for searching the AUR & directly adding packages
* HTTP routes for removing packages, arch-repos & repos
* All endpoints serving files now support HTTP byte range requests
* Better CLI UX
* When adding targets, the ID of the created target is returned
* The `-r` flag only shows raw data of action
* When adding a target, only ID is shown and not surrounding text
* Tabled output returns a tab-separated list (easy to script using
`cut`)
### Changed
@ -27,6 +33,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
repository will be cloned with the default branch
* Build containers now explicitely set the PATH variable
* Refactor of web framework
* API endpoints now return id of newly created entries
* Repo POST requests now return information on published package
* `api` can no longer be used as a repository name
* CLI client now allows setting values to an empty value
* API endpoints now return id of newly created entries
* Repo POST requests now return information on published package
* `api` can no longer be used as a repository name
* CLI client now allows setting values to an empty value
### Removed

View File

@ -30,12 +30,10 @@ fn (c &Client) send_request_raw(method Method, url string, params map[string]str
// Escape each query param
for k, v in params {
// An empty parameter should be the same as not providing it at all
if v != '' {
params_escaped[k] = urllib.query_escape(v)
}
params_escaped[k] = urllib.query_escape(v)
}
params_str := params_escaped.keys().map('$it=${params[it]}').join('&')
params_str := params_escaped.keys().map('$it=${params_escaped[it]}').join('&')
full_url = '$full_url?$params_str'
}

View File

@ -39,7 +39,7 @@ pub fn (c &Client) get_build_log_content(id int) ?string {
}
// add_build_log adds a new build log to the server.
pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response<string> {
pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response<int> {
params := {
'target': target_id.str()
'startTime': start_time.unix_time().str()
@ -48,7 +48,7 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t
'exitCode': exit_code.str()
}
data := c.send_request_with_body<string>(Method.post, '/api/v1/logs', params, content)?
data := c.send_request_with_body<int>(Method.post, '/api/v1/logs', params, content)?
return data
}

View File

@ -49,9 +49,9 @@ pub struct NewTarget {
}
// add_target adds a new target to the server.
pub fn (c &Client) add_target(t NewTarget) ?Response<string> {
pub fn (c &Client) add_target(t NewTarget) ?Response<int> {
params := models.params_from<NewTarget>(t)
data := c.send_request<string>(Method.post, '/api/v1/targets', params)?
data := c.send_request<int>(Method.post, '/api/v1/targets', params)?
return data
}

View File

@ -5,6 +5,11 @@ import strings
import cli
import os
// tabbed_table returns a simple textual table, with tabs as separators.
pub fn tabbed_table(data [][]string) string {
return data.map(it.join('\t')).join('\n')
}
// pretty_table converts a list of string data into a pretty table. Many thanks
// to @hungrybluedev in the Vlang Discord for providing this code!
// https://ptb.discord.com/channels/592103645835821068/592106336838352923/970278787143045192

View File

@ -133,7 +133,9 @@ pub fn cmd() cli.Command {
]
}
list(conf, filter)?
raw := cmd.flags.get_bool('raw')?
list(conf, filter, raw)?
}
},
cli.Command{
@ -167,27 +169,31 @@ pub fn cmd() cli.Command {
}
// print_log_list prints a list of logs.
fn print_log_list(logs []BuildLog) ? {
fn print_log_list(logs []BuildLog, raw bool) ? {
data := logs.map([it.id.str(), it.target_id.str(), it.start_time.local().str(),
it.exit_code.str()])
println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)?)
if raw {
println(console.tabbed_table(data))
} else {
println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)?)
}
}
// list prints a list of all build logs.
fn list(conf Config, filter BuildLogFilter) ? {
fn list(conf Config, filter BuildLogFilter, raw bool) ? {
c := client.new(conf.address, conf.api_key)
logs := c.get_build_logs(filter)?.data
print_log_list(logs)?
print_log_list(logs, raw)?
}
// list prints a list of all build logs for a given target.
fn list_for_target(conf Config, target_id int) ? {
fn list_for_target(conf Config, target_id int, raw bool) ? {
c := client.new(conf.address, conf.api_key)
logs := c.get_build_logs_for_target(target_id)?.data
print_log_list(logs)?
print_log_list(logs, raw)?
}
// info print the detailed info for a given build log.

View File

@ -60,7 +60,9 @@ pub fn cmd() cli.Command {
filter.repo = repo
}
list(conf, filter)?
raw := cmd.flags.get_bool('raw')?
list(conf, filter, raw)?
}
},
cli.Command{
@ -92,7 +94,9 @@ pub fn cmd() cli.Command {
branch: cmd.flags.get_string('branch') or { '' }
}
add(conf, t)?
raw := cmd.flags.get_bool('raw')?
add(conf, t, raw)?
}
},
cli.Command{
@ -193,20 +197,28 @@ pub fn cmd() cli.Command {
// ID. If multiple or none are found, an error is raised.
// list prints out a list of all repositories.
fn list(conf Config, filter TargetFilter) ? {
fn list(conf Config, filter TargetFilter, raw bool) ? {
c := client.new(conf.address, conf.api_key)
repos := c.get_targets(filter)?
data := repos.map([it.id.str(), it.kind, it.url, it.repo])
println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?)
if raw {
println(console.tabbed_table(data))
} else {
println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?)
}
}
// add adds a new repository to the server's list.
fn add(conf Config, t &NewTarget) ? {
fn add(conf Config, t &NewTarget, raw bool) ? {
c := client.new(conf.address, conf.api_key)
res := c.add_target(t)?
println(res.message)
if raw {
println(res.data)
} else {
println('Target added with id $res.data')
}
}
// remove removes a repository from the server's list.

View File

@ -79,10 +79,14 @@ pub fn (db &VieterDb) get_build_log(id int) ?BuildLog {
}
// add_build_log inserts the given BuildLog into the database.
pub fn (db &VieterDb) add_build_log(log BuildLog) {
pub fn (db &VieterDb) add_build_log(log BuildLog) int {
sql db.conn {
insert log into BuildLog
}
inserted_id := db.conn.last_id() as int
return inserted_id
}
// delete_build_log delete the BuildLog with the given ID from the database.

View File

@ -38,10 +38,14 @@ pub fn (db &VieterDb) get_target(target_id int) ?Target {
}
// add_target inserts the given target into the database.
pub fn (db &VieterDb) add_target(repo Target) {
pub fn (db &VieterDb) add_target(repo Target) int {
sql db.conn {
insert repo into Target
}
inserted_id := db.conn.last_id() as int
return inserted_id
}
// delete_target deletes the target with the given id from the database.

View File

@ -24,6 +24,13 @@ fn main() {
global: true
default_value: [os.expand_tilde_to_home('~/.vieterrc')]
},
cli.Flag{
flag: cli.FlagType.bool
name: 'raw'
abbrev: 'r'
description: 'Only output minimal information (no formatted tables, etc.)'
global: true
},
]
commands: [
server.cmd(),

View File

@ -23,8 +23,9 @@ pub:
pub struct RepoAddResult {
pub:
added bool [required]
pkg &package.Pkg [required]
name string
version string
archs []string
}
// new creates a new RepoGroupManager & creates the directories as needed
@ -53,10 +54,10 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re
return error('Failed to read package file: $err.msg()')
}
added := r.add_pkg_in_repo(repo, pkg)?
archs := r.add_pkg_in_repo(repo, pkg)?
// If the add was successful, we move the file to the packages directory
for arch in added {
for arch in archs {
repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch))
dest_path := os.join_path_single(repo_pkg_path, pkg.filename())
@ -71,8 +72,9 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re
os.rm(pkg_path)?
return RepoAddResult{
added: added.len > 0
pkg: &pkg
name: pkg.info.name
version: pkg.info.version
archs: archs
}
}
@ -87,11 +89,9 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin
// A package not of arch 'any' can be handled easily by adding it to the
// respective repo
if pkg.info.arch != 'any' {
if r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)? {
return [pkg.info.arch]
} else {
return []
}
r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)?
return [pkg.info.arch]
}
mut arch_repos := []string{}
@ -113,25 +113,22 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin
arch_repos << r.default_arch
}
mut added := []string{}
// We add the package to each repository. If any of the repositories
// return true, the result of the function is also true.
// Add the package to each found architecture
// NOTE: if any of these fail, the function fails. This means the user does
// not know which arch-repositories did succeed in adding the package, if
// any.
for arch in arch_repos {
if r.add_pkg_in_arch_repo(repo, arch, pkg)? {
added << arch
}
r.add_pkg_in_arch_repo(repo, arch, pkg)?
}
return added
return arch_repos
}
// add_pkg_in_arch_repo is the function that actually adds a package to a given
// arch-repo. It records the package's data in the arch-repo's desc & files
// files, and afterwards updates the db & files archives to reflect these
// changes. The function returns false if the package was already present in
// the repo, and true otherwise.
fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool {
// changes.
fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ? {
pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version')
// Remove the previous version of the package, if present
@ -151,6 +148,4 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac
}
r.sync(repo, arch)?
return true
}

View File

@ -19,7 +19,7 @@ fn (mut app App) v1_get_logs() web.Result {
}
logs := app.db.get_build_logs(filter)
return app.json(http.Status.ok, new_data_response(logs))
return app.json(.ok, new_data_response(logs))
}
// v1_get_single_log returns the build log with the given id.
@ -27,7 +27,7 @@ fn (mut app App) v1_get_logs() web.Result {
fn (mut app App) v1_get_single_log(id int) web.Result {
log := app.db.get_build_log(id) or { return app.not_found() }
return app.json(http.Status.ok, new_data_response(log))
return app.json(.ok, new_data_response(log))
}
// v1_get_log_content returns the actual build log file for the given id.
@ -95,7 +95,8 @@ fn (mut app App) v1_post_log() web.Result {
exit_code: exit_code
}
app.db.add_build_log(log)
// id of newly created log
log_id := app.db.add_build_log(log)
repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, target_id.str(), arch)
@ -122,5 +123,5 @@ fn (mut app App) v1_post_log() web.Result {
return app.status(http.Status.length_required)
}
return app.json(http.Status.ok, new_response('Logs added successfully.'))
return app.json(.ok, new_data_response(log_id))
}

View File

@ -14,7 +14,7 @@ fn (mut app App) v1_get_targets() web.Result {
}
repos := app.db.get_targets(filter)
return app.json(http.Status.ok, new_data_response(repos))
return app.json(.ok, new_data_response(repos))
}
// v1_get_single_target returns the information for a single target.
@ -22,7 +22,7 @@ fn (mut app App) v1_get_targets() web.Result {
fn (mut app App) v1_get_single_target(id int) web.Result {
repo := app.db.get_target(id) or { return app.not_found() }
return app.json(http.Status.ok, new_data_response(repo))
return app.json(.ok, new_data_response(repo))
}
// v1_post_target creates a new target from the provided query string.
@ -45,9 +45,9 @@ fn (mut app App) v1_post_target() web.Result {
return app.json(http.Status.bad_request, new_response('Invalid kind.'))
}
app.db.add_target(new_repo)
id := app.db.add_target(new_repo)
return app.json(http.Status.ok, new_response('Repo added successfully.'))
return app.json(http.Status.ok, new_data_response(id))
}
// v1_delete_target removes a given target from the server's list.
@ -55,7 +55,7 @@ fn (mut app App) v1_post_target() web.Result {
fn (mut app App) v1_delete_target(id int) web.Result {
app.db.delete_target(id)
return app.json(http.Status.ok, new_response('Repo removed successfully.'))
return app.status(.ok)
}
// v1_patch_target updates a target's data with the given query params.
@ -69,5 +69,5 @@ fn (mut app App) v1_patch_target(id int) web.Result {
app.db.update_target_archs(id, arch_objs)
}
return app.json(http.Status.ok, new_response('Repo updated successfully.'))
return app.status(.ok)
}

View File

@ -6,8 +6,7 @@ import repo
import time
import rand
import util
import net.http
import web.response { new_response }
import web.response { new_data_response, new_response }
// healthcheck just returns a string, but can be used to quickly check if the
// server is still responsive.
@ -51,6 +50,12 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re
// put_package handles publishing a package to a repository.
['/:repo/publish'; auth; post]
fn (mut app App) put_package(repo string) web.Result {
// api is a reserved keyword for api routes & should never be allowed to be
// a repository.
if repo.to_lower() == 'api' {
return app.json(.bad_request, new_response("'api' is a reserved keyword & cannot be used as a repository name."))
}
mut pkg_path := ''
if length := app.req.header.get(.content_length) {
@ -65,7 +70,7 @@ fn (mut app App) put_package(repo string) web.Result {
util.reader_to_file(mut app.reader, length.int(), pkg_path) or {
app.lwarn("Failed to upload '$pkg_path'")
return app.json(http.Status.internal_server_error, new_response('Failed to upload file.'))
return app.status(.internal_server_error)
}
sw.stop()
@ -74,7 +79,7 @@ fn (mut app App) put_package(repo string) web.Result {
app.lwarn('Tried to upload package without specifying a Content-Length.')
// length required
return app.status(http.Status.length_required)
return app.status(.length_required)
}
res := app.repo.add_pkg_from_path(repo, pkg_path) or {
@ -82,18 +87,10 @@ fn (mut app App) put_package(repo string) web.Result {
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg()") }
return app.json(http.Status.internal_server_error, new_response('Failed to add package.'))
return app.status(.internal_server_error)
}
if !res.added {
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg()") }
app.linfo("Added '$res.name-$res.version' to '$repo (${res.archs.join(',')})'.")
app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo'.")
return app.json(http.Status.bad_request, new_response('File already exists.'))
}
app.linfo("Added '$res.pkg.full_name()' to repo '$repo ($res.pkg.info.arch)'.")
return app.json(http.Status.ok, new_response('Package added successfully.'))
return app.json(.ok, new_data_response(res))
}

View File

@ -1,8 +1,6 @@
module server
import web
import net.http
import web.response { new_response }
// delete_package tries to remove the given package.
['/:repo/:arch/:pkg'; auth; delete]
@ -10,17 +8,17 @@ fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result
res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or {
app.lerror('Error while deleting package: $err.msg()')
return app.json(http.Status.internal_server_error, new_response('Failed to delete package.'))
return app.status(.internal_server_error)
}
if res {
app.linfo("Removed package '$pkg' from '$repo/$arch'")
return app.json(http.Status.ok, new_response('Package removed.'))
return app.status(.ok)
} else {
app.linfo("Tried removing package '$pkg' from '$repo/$arch', but it doesn't exist.")
return app.json(http.Status.not_found, new_response('Package not found.'))
return app.status(.not_found)
}
}
@ -30,17 +28,17 @@ fn (mut app App) delete_arch_repo(repo string, arch string) web.Result {
res := app.repo.remove_arch_repo(repo, arch) or {
app.lerror('Error while deleting arch-repo: $err.msg()')
return app.json(http.Status.internal_server_error, new_response('Failed to delete arch-repo.'))
return app.status(.internal_server_error)
}
if res {
app.linfo("Removed '$repo/$arch'")
app.linfo("Removed arch-repo '$repo/$arch'")
return app.json(http.Status.ok, new_response('Arch-repo removed.'))
return app.status(.ok)
} else {
app.linfo("Tried removing '$repo/$arch', but it doesn't exist.")
return app.json(http.Status.not_found, new_response('Arch-repo not found.'))
return app.status(.not_found)
}
}
@ -50,16 +48,16 @@ fn (mut app App) delete_repo(repo string) web.Result {
res := app.repo.remove_repo(repo) or {
app.lerror('Error while deleting repo: $err.msg()')
return app.json(http.Status.internal_server_error, new_response('Failed to delete repo.'))
return app.status(.internal_server_error)
}
if res {
app.linfo("Removed '$repo'")
app.linfo("Removed repo '$repo'")
return app.json(http.Status.ok, new_response('Repo removed.'))
return app.status(.ok)
} else {
app.linfo("Tried removing '$repo', but it doesn't exist.")
return app.json(http.Status.not_found, new_response('Repo not found.'))
return app.status(.not_found)
}
}