From 487b2357272050d1fcabf9cead73872fa5ea524e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 22 Jun 2022 16:19:07 +0200 Subject: [PATCH 001/140] feat(cli): add aur search command --- .gitignore | 3 +++ src/console/aur/aur.v | 26 ++++++++++++++++++++++++++ src/main.v | 2 ++ src/v.mod | 3 ++- 4 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 src/console/aur/aur.v diff --git a/.gitignore b/.gitignore index 4d9f94f..a2804fe 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ gdb.txt # Generated docs _docs/ /man/ + +# VLS logs +vls.log diff --git a/src/console/aur/aur.v b/src/console/aur/aur.v new file mode 100644 index 0000000..3f20dfc --- /dev/null +++ b/src/console/aur/aur.v @@ -0,0 +1,26 @@ +module aur + +import cli +import console +import vieter_v.aur + +pub fn cmd() cli.Command { + return cli.Command{ + name: 'aur' + description: 'Interact with the AUR.' + commands: [ + cli.Command{ + name: 'search' + description: 'Search for packages.' + required_args: 1 + execute: fn (cmd cli.Command) ? { + c := aur.new() + pkgs := c.search(cmd.args[0])? + data := pkgs.map([it.name, it.description]) + + println(console.pretty_table(['name', 'description'], data)?) + } + }, + ] + } +} diff --git a/src/main.v b/src/main.v index cba410c..4ade930 100644 --- a/src/main.v +++ b/src/main.v @@ -7,6 +7,7 @@ import console.targets import console.logs import console.schedule import console.man +import console.aur import cron fn main() { @@ -31,6 +32,7 @@ fn main() { logs.cmd(), schedule.cmd(), man.cmd(), + aur.cmd(), ] } app.setup() diff --git a/src/v.mod b/src/v.mod index 5b89062..710c976 100644 --- a/src/v.mod +++ b/src/v.mod @@ -1,6 +1,7 @@ Module { dependencies: [ 'https://git.rustybever.be/vieter-v/conf', - 'https://git.rustybever.be/vieter-v/docker' + 'https://git.rustybever.be/vieter-v/docker', + 'https://git.rustybever.be/vieter-v/aur' ] } From 1a940f2f98e6c2c7794e4f06b2913a32316cfbfd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 22 Jun 2022 16:38:53 +0200 Subject: [PATCH 002/140] feat(cli): added "aur add" command --- CHANGELOG.md | 1 + src/console/aur/aur.v | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf2b829..a03c18c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 URL to a PKGBUILD * Targets with kind 'url' can provide a direct URL to a PKGBUILD instead of providing a Git repository +* CLI commands for searching the AUR & directly adding packages ### Changed diff --git a/src/console/aur/aur.v b/src/console/aur/aur.v index 3f20dfc..c98f8e6 100644 --- a/src/console/aur/aur.v +++ b/src/console/aur/aur.v @@ -2,8 +2,16 @@ module aur import cli import console +import client import vieter_v.aur +import vieter_v.conf as vconf +struct Config { + address string [required] + api_key string [required] +} + +// cmd returns the cli module for interacting with the AUR API. pub fn cmd() cli.Command { return cli.Command{ name: 'aur' @@ -21,6 +29,34 @@ pub fn cmd() cli.Command { println(console.pretty_table(['name', 'description'], data)?) } }, + cli.Command{ + name: 'add' + usage: 'repo pkg-name [pkg-name...]' + description: 'Add the given AUR package(s) to Vieter. Non-existent packages will be silently ignored.' + required_args: 2 + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? + + c := aur.new() + pkgs := c.info(cmd.args[1..])? + + vc := client.new(conf.address, conf.api_key) + + for pkg in pkgs { + vc.add_target( + kind: 'git' + url: 'https://aur.archlinux.org/$pkg.package_base' + '.git' + repo: cmd.args[0] + ) or { + println('Failed to add $pkg.name: $err.msg()') + continue + } + + println('Added $pkg.name' + '.') + } + } + }, ] } } From 0d0fb323f235c0381d8737ccd777bfa6167a7e4b Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 3 Jul 2022 14:11:29 +0200 Subject: [PATCH 003/140] chore: switched to vlang 0.3 Docker image --- .woodpecker/.build.yml | 8 ++++---- .woodpecker/.docs.yml | 2 +- .woodpecker/.gitea.yml | 2 +- .woodpecker/.lint.yml | 2 +- .woodpecker/.man.yml | 2 +- .woodpecker/.test.yml | 4 ++-- Dockerfile | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 7cb7d53..580fa69 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -7,7 +7,7 @@ platform: ${PLATFORM} pipeline: install-modules: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true commands: - export VMODULES=$PWD/.vmodules @@ -16,7 +16,7 @@ pipeline: event: [push, pull_request] debug: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' commands: - export VMODULES=$PWD/.vmodules - make @@ -26,7 +26,7 @@ pipeline: exclude: [main] prod: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' environment: - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static commands: @@ -44,7 +44,7 @@ pipeline: event: [push, pull_request] upload: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' secrets: [ s3_username, s3_password ] commands: # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f diff --git a/.woodpecker/.docs.yml b/.woodpecker/.docs.yml index 051d852..da495fc 100644 --- a/.woodpecker/.docs.yml +++ b/.woodpecker/.docs.yml @@ -11,7 +11,7 @@ pipeline: - make docs api-docs: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true group: 'generate' commands: diff --git a/.woodpecker/.gitea.yml b/.woodpecker/.gitea.yml index d0825c2..55f991e 100644 --- a/.woodpecker/.gitea.yml +++ b/.woodpecker/.gitea.yml @@ -8,7 +8,7 @@ skip_clone: true pipeline: prepare: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true secrets: [ s3_username, s3_password ] commands: diff --git a/.woodpecker/.lint.yml b/.woodpecker/.lint.yml index e70648d..75a8105 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/.lint.yml @@ -5,7 +5,7 @@ platform: 'linux/amd64' pipeline: lint: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true commands: - make lint diff --git a/.woodpecker/.man.yml b/.woodpecker/.man.yml index 0b80886..1a30b03 100644 --- a/.woodpecker/.man.yml +++ b/.woodpecker/.man.yml @@ -9,7 +9,7 @@ skip_clone: true pipeline: generate: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true commands: - curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64" diff --git a/.woodpecker/.test.yml b/.woodpecker/.test.yml index a19dbd4..6c267fa 100644 --- a/.woodpecker/.test.yml +++ b/.woodpecker/.test.yml @@ -9,7 +9,7 @@ platform: ${PLATFORM} pipeline: install-modules: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true commands: - export VMODULES=$PWD/.vmodules @@ -18,7 +18,7 @@ pipeline: event: [pull_request] test: - image: 'chewingbever/vlang:latest' + image: 'chewingbever/vlang:0.3' pull: true commands: - export VMODULES=$PWD/.vmodules diff --git a/Dockerfile b/Dockerfile index 5997adc..7aed917 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM chewingbever/vlang:latest AS builder +FROM chewingbever/vlang:0.3 AS builder ARG TARGETPLATFORM ARG CI_COMMIT_SHA From 0f6630b9404acb2b2217f52091c38d636ffc81f0 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 17 Jul 2022 13:38:46 +0200 Subject: [PATCH 004/140] chore: updated PKGBUILDs to use vlang package --- PKGBUILD | 4 ++-- PKGBUILD.dev | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/PKGBUILD b/PKGBUILD index 6b664d1..14ea71b 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -5,9 +5,9 @@ pkgbase='vieter' pkgname='vieter' pkgver='0.3.0' pkgrel=1 -pkgdesc="Vieter is a lightweight implementation of an Arch repository server." +pkgdesc="Lightweight Arch repository server & package build system" depends=('glibc' 'openssl' 'libarchive' 'sqlite') -makedepends=('git' 'vieter-v') +makedepends=('git' 'vlang') arch=('x86_64' 'aarch64') url='https://git.rustybever.be/vieter-v/vieter' license=('AGPL3') diff --git a/PKGBUILD.dev b/PKGBUILD.dev index 045e576..79c7f37 100644 --- a/PKGBUILD.dev +++ b/PKGBUILD.dev @@ -5,9 +5,9 @@ pkgbase='vieter-git' pkgname='vieter-git' pkgver=0.2.0.r25.g20112b8 pkgrel=1 -pkgdesc="Vieter is a lightweight implementation of an Arch repository server." +pkgdesc="Lightweight Arch repository server & package build system (development version)" depends=('glibc' 'openssl' 'libarchive' 'sqlite') -makedepends=('git' 'vieter-v') +makedepends=('git' 'vlang') arch=('x86_64' 'aarch64') url='https://git.rustybever.be/vieter-v/vieter' license=('AGPL3') From 49ddb312dea5d298952ad460922ed3f518677214 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 11 Aug 2022 19:07:54 +0200 Subject: [PATCH 005/140] feat(server): added endpoint to remove package from arch-repo --- docs/api/source/includes/_repository.md | 29 +++++++++++++++++++++++++ src/repo/repo.v | 2 +- src/server/routes.v | 23 ++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/docs/api/source/includes/_repository.md b/docs/api/source/includes/_repository.md index fbbc329..9764e01 100644 --- a/docs/api/source/includes/_repository.md +++ b/docs/api/source/includes/_repository.md @@ -93,3 +93,32 @@ other already present arch-repos. Parameter | Description --------- | ----------- repo | Repository to publish package to + +## Remove package from arch-repo + + + +```shell +curl \ + -H 'X-Api-Key: secret' \ + -XDELETE \ + https://example.com/vieter/x86_64/mike +``` + +This endpoint allows you to remove a package from a given arch-repo. + +### HTTP Request + +`DELETE /:repo/:arch/:pkg` + +### URL Parameters + +Parameter | Description +--------- | ----------- +repo | Repository to delete package from +arch | Specific arch-repo to remove package from +pkg | Name of package to remove (without any version information) diff --git a/src/repo/repo.v b/src/repo/repo.v index c4b85c0..7de12cd 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -158,7 +158,7 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac // remove_pkg_from_arch_repo removes a package from an arch-repo's database. It // returns false if the package wasn't present in the database. It also // optionally re-syncs the repo archives. -fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool { +pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool { repo_dir := os.join_path(r.repos_dir, repo, arch) // If the repository doesn't exist yet, the result is automatically false diff --git a/src/server/routes.v b/src/server/routes.v index fbf37df..3b86e20 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -57,6 +57,29 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re return app.file(full_path) } +['/:repo/:arch/:pkg'; delete] +fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { + app.lerror('Error while deleting package: $err.msg()') + + return app.json(http.Status.internal_server_error, new_response('Failed to delete package.')) + } + + if res { + app.linfo("Removed package '$pkg' from '$repo/$arch'") + + return app.json(http.Status.ok, new_response('Package removed.')) + } else { + app.linfo("Tried removing package '$pkg' from '$repo/$arch', but it doesn't exist.") + + return app.json(http.Status.not_found, new_response('Package not found.')) + } +} + // put_package handles publishing a package to a repository. ['/:repo/publish'; post] fn (mut app App) put_package(repo string) web.Result { From 6283cbea9cf0c655ea43a146bada31aef5ee9d7f Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 11 Aug 2022 19:16:38 +0200 Subject: [PATCH 006/140] feat(repo): added function to remove arch-repo --- src/repo/{repo.v => add.v} | 48 --------------------------- src/repo/remove.v | 68 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 48 deletions(-) rename src/repo/{repo.v => add.v} (77%) create mode 100644 src/repo/remove.v diff --git a/src/repo/repo.v b/src/repo/add.v similarity index 77% rename from src/repo/repo.v rename to src/repo/add.v index 7de12cd..0985c7a 100644 --- a/src/repo/repo.v +++ b/src/repo/add.v @@ -154,51 +154,3 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac return true } - -// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It -// returns false if the package wasn't present in the database. It also -// optionally re-syncs the repo archives. -pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool { - repo_dir := os.join_path(r.repos_dir, repo, arch) - - // If the repository doesn't exist yet, the result is automatically false - if !os.exists(repo_dir) { - return false - } - - // We iterate over every directory in the repo dir - // TODO filter so we only check directories - for d in os.ls(repo_dir)? { - // Because a repository only allows a single version of each package, - // we need only compare whether the name of the package is the same, - // not the version. - name := d.split('-')#[..-2].join('-') - - if name == pkg_name { - // We lock the mutex here to prevent other routines from creating a - // new archive while we remove an entry - lock r.mutex { - os.rmdir_all(os.join_path_single(repo_dir, d))? - } - - // Also remove the package archive - repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch) - - archives := os.ls(repo_pkg_dir)?.filter(it.split('-')#[..-3].join('-') == name) - - for archive_name in archives { - full_path := os.join_path_single(repo_pkg_dir, archive_name) - os.rm(full_path)? - } - - // Sync the db archives if requested - if sync { - r.sync(repo, arch)? - } - - return true - } - } - - return false -} diff --git a/src/repo/remove.v b/src/repo/remove.v new file mode 100644 index 0000000..dd4f400 --- /dev/null +++ b/src/repo/remove.v @@ -0,0 +1,68 @@ +module repo + +import os + +// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It +// returns false if the package wasn't present in the database. It also +// optionally re-syncs the repo archives. +pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool { + repo_dir := os.join_path(r.repos_dir, repo, arch) + + // If the repository doesn't exist yet, the result is automatically false + if !os.exists(repo_dir) { + return false + } + + // We iterate over every directory in the repo dir + // TODO filter so we only check directories + for d in os.ls(repo_dir)? { + // Because a repository only allows a single version of each package, + // we need only compare whether the name of the package is the same, + // not the version. + name := d.split('-')#[..-2].join('-') + + if name == pkg_name { + // We lock the mutex here to prevent other routines from creating a + // new archive while we remove an entry + lock r.mutex { + os.rmdir_all(os.join_path_single(repo_dir, d))? + } + + // Also remove the package archive + repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch) + + archives := os.ls(repo_pkg_dir)?.filter(it.split('-')#[..-3].join('-') == name) + + for archive_name in archives { + full_path := os.join_path_single(repo_pkg_dir, archive_name) + os.rm(full_path)? + } + + // Sync the db archives if requested + if sync { + r.sync(repo, arch)? + } + + return true + } + } + + return false +} + +// remove_arch_repo removes an arch-repo & its packages. +pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) ?bool { + repo_dir := os.join_path(r.repos_dir, repo, arch) + + // If the repository doesn't exist yet, the result is automatically false + if !os.exists(repo_dir) { + return false + } + + os.rmdir_all(repo_dir)? + + pkg_dir := os.join_path(r.pkg_dir, repo, arch) + os.rmdir_all(pkg_dir)? + + return true +} From 68b7e5e71ec636fe1fdcb7338f02dd3f70e922ed Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 11 Aug 2022 19:28:17 +0200 Subject: [PATCH 007/140] feat(server): added routes for removing arch-repo & repo --- docs/api/source/includes/_repository.md | 55 ++++++++++++++++++ src/repo/remove.v | 17 ++++++ src/server/{routes.v => repo.v} | 23 -------- src/server/repo_remove.v | 77 +++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 23 deletions(-) rename src/server/{routes.v => repo.v} (82%) create mode 100644 src/server/repo_remove.v diff --git a/docs/api/source/includes/_repository.md b/docs/api/source/includes/_repository.md index 9764e01..a846904 100644 --- a/docs/api/source/includes/_repository.md +++ b/docs/api/source/includes/_repository.md @@ -122,3 +122,58 @@ Parameter | Description repo | Repository to delete package from arch | Specific arch-repo to remove package from pkg | Name of package to remove (without any version information) + +## Remove arch-repo + + + +```shell +curl \ + -H 'X-Api-Key: secret' \ + -XDELETE \ + https://example.com/vieter/x86_64 +``` + +This endpoint allows remove an entire arch-repo. + +### HTTP Request + +`DELETE /:repo/:arch` + +### URL Parameters + +Parameter | Description +--------- | ----------- +repo | Repository to delete arch-repo from +arch | Specific architecture to remove package + +## Remove repo + + + +```shell +curl \ + -H 'X-Api-Key: secret' \ + -XDELETE \ + https://example.com/vieter +``` + +This endpoint allows remove an entire repo. + +### HTTP Request + +`DELETE /:repo` + +### URL Parameters + +Parameter | Description +--------- | ----------- +repo | Repository to delete diff --git a/src/repo/remove.v b/src/repo/remove.v index dd4f400..add921c 100644 --- a/src/repo/remove.v +++ b/src/repo/remove.v @@ -66,3 +66,20 @@ pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) ?bool { return true } + +// remove_repo removes a repo & its packages. +pub fn (r &RepoGroupManager) remove_repo(repo string) ?bool { + repo_dir := os.join_path_single(r.repos_dir, repo) + + // If the repository doesn't exist yet, the result is automatically false + if !os.exists(repo_dir) { + return false + } + + os.rmdir_all(repo_dir)? + + pkg_dir := os.join_path_single(r.pkg_dir, repo) + os.rmdir_all(pkg_dir)? + + return true +} diff --git a/src/server/routes.v b/src/server/repo.v similarity index 82% rename from src/server/routes.v rename to src/server/repo.v index 3b86e20..fbf37df 100644 --- a/src/server/routes.v +++ b/src/server/repo.v @@ -57,29 +57,6 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re return app.file(full_path) } -['/:repo/:arch/:pkg'; delete] -fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - - res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { - app.lerror('Error while deleting package: $err.msg()') - - return app.json(http.Status.internal_server_error, new_response('Failed to delete package.')) - } - - if res { - app.linfo("Removed package '$pkg' from '$repo/$arch'") - - return app.json(http.Status.ok, new_response('Package removed.')) - } else { - app.linfo("Tried removing package '$pkg' from '$repo/$arch', but it doesn't exist.") - - return app.json(http.Status.not_found, new_response('Package not found.')) - } -} - // put_package handles publishing a package to a repository. ['/:repo/publish'; post] fn (mut app App) put_package(repo string) web.Result { diff --git a/src/server/repo_remove.v b/src/server/repo_remove.v new file mode 100644 index 0000000..642f26f --- /dev/null +++ b/src/server/repo_remove.v @@ -0,0 +1,77 @@ +module server + +import web +import net.http +import response { new_response } + +// delete_package tries to remove the given package. +['/:repo/:arch/:pkg'; delete] +fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { + app.lerror('Error while deleting package: $err.msg()') + + return app.json(http.Status.internal_server_error, new_response('Failed to delete package.')) + } + + if res { + app.linfo("Removed package '$pkg' from '$repo/$arch'") + + return app.json(http.Status.ok, new_response('Package removed.')) + } else { + app.linfo("Tried removing package '$pkg' from '$repo/$arch', but it doesn't exist.") + + return app.json(http.Status.not_found, new_response('Package not found.')) + } +} + +// delete_arch_repo tries to remove the given arch-repo. +['/:repo/:arch'; delete] +fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + res := app.repo.remove_arch_repo(repo, arch) or { + app.lerror('Error while deleting arch-repo: $err.msg()') + + return app.json(http.Status.internal_server_error, new_response('Failed to delete arch-repo.')) + } + + if res { + app.linfo("Removed '$repo/$arch'") + + return app.json(http.Status.ok, new_response('Arch-repo removed.')) + } else { + app.linfo("Tried removing '$repo/$arch', but it doesn't exist.") + + return app.json(http.Status.not_found, new_response('Arch-repo not found.')) + } +} + +// delete_repo tries to remove the given repo. +['/:repo'; delete] +fn (mut app App) delete_repo(repo string) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + res := app.repo.remove_repo(repo) or { + app.lerror('Error while deleting repo: $err.msg()') + + return app.json(http.Status.internal_server_error, new_response('Failed to delete repo.')) + } + + if res { + app.linfo("Removed '$repo'") + + return app.json(http.Status.ok, new_response('Repo removed.')) + } else { + app.linfo("Tried removing '$repo', but it doesn't exist.") + + return app.json(http.Status.not_found, new_response('Repo not found.')) + } +} From ba3b00572b3ddf31490f054b5bbf2288913da5af Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 11 Aug 2022 19:44:22 +0200 Subject: [PATCH 008/140] chore: updated changelog [CI SKIP] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a03c18c..b7138a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Targets with kind 'url' can provide a direct URL to a PKGBUILD instead of providing a Git repository * CLI commands for searching the AUR & directly adding packages +* HTTP routes for removing packages, arch-repos & repos ### Changed From 50918da67214ff69a29343f04a68f52e28f602dc Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 11 Aug 2022 19:47:26 +0200 Subject: [PATCH 009/140] docs: fixed some typos --- docs/api/source/includes/_repository.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api/source/includes/_repository.md b/docs/api/source/includes/_repository.md index a846904..ff17f71 100644 --- a/docs/api/source/includes/_repository.md +++ b/docs/api/source/includes/_repository.md @@ -138,7 +138,7 @@ curl \ https://example.com/vieter/x86_64 ``` -This endpoint allows remove an entire arch-repo. +This endpoint allows removing an entire arch-repo. ### HTTP Request @@ -149,7 +149,7 @@ This endpoint allows remove an entire arch-repo. Parameter | Description --------- | ----------- repo | Repository to delete arch-repo from -arch | Specific architecture to remove package +arch | Specific architecture to remove ## Remove repo @@ -166,7 +166,7 @@ curl \ https://example.com/vieter ``` -This endpoint allows remove an entire repo. +This endpoint allows removing an entire repo. ### HTTP Request From 3a73ea0632df8c804d3a6d7a3ed7ae59f001ff65 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 12 Aug 2022 14:39:42 +0200 Subject: [PATCH 010/140] refactor(web): simplified web framework in general --- src/web/consts.v | 125 +++++++++++++++++++++ src/web/web.v | 275 +++++++++++++++-------------------------------- 2 files changed, 209 insertions(+), 191 deletions(-) create mode 100644 src/web/consts.v diff --git a/src/web/consts.v b/src/web/consts.v new file mode 100644 index 0000000..7b1d2b4 --- /dev/null +++ b/src/web/consts.v @@ -0,0 +1,125 @@ +module web + +import net.http + +// A dummy structure that returns from routes to indicate that you actually sent something to a user +[noinit] +pub struct Result {} + +pub const ( + methods_with_form = [http.Method.post, .put, .patch] + headers_close = http.new_custom_header_from_map({ + 'Server': 'VWeb' + http.CommonHeader.connection.str(): 'close' + }) or { panic('should never fail') } + + http_302 = http.new_response( + status: .found + body: '302 Found' + header: headers_close + ) + http_400 = http.new_response( + status: .bad_request + body: '400 Bad Request' + header: http.new_header( + key: .content_type + value: 'text/plain' + ).join(headers_close) + ) + http_404 = http.new_response( + status: .not_found + body: '404 Not Found' + header: http.new_header( + key: .content_type + value: 'text/plain' + ).join(headers_close) + ) + http_500 = http.new_response( + status: .internal_server_error + body: '500 Internal Server Error' + header: http.new_header( + key: .content_type + value: 'text/plain' + ).join(headers_close) + ) + mime_types = { + '.aac': 'audio/aac' + '.abw': 'application/x-abiword' + '.arc': 'application/x-freearc' + '.avi': 'video/x-msvideo' + '.azw': 'application/vnd.amazon.ebook' + '.bin': 'application/octet-stream' + '.bmp': 'image/bmp' + '.bz': 'application/x-bzip' + '.bz2': 'application/x-bzip2' + '.cda': 'application/x-cdf' + '.csh': 'application/x-csh' + '.css': 'text/css' + '.csv': 'text/csv' + '.doc': 'application/msword' + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + '.eot': 'application/vnd.ms-fontobject' + '.epub': 'application/epub+zip' + '.gz': 'application/gzip' + '.gif': 'image/gif' + '.htm': 'text/html' + '.html': 'text/html' + '.ico': 'image/vnd.microsoft.icon' + '.ics': 'text/calendar' + '.jar': 'application/java-archive' + '.jpeg': 'image/jpeg' + '.jpg': 'image/jpeg' + '.js': 'text/javascript' + '.json': 'application/json' + '.jsonld': 'application/ld+json' + '.mid': 'audio/midi audio/x-midi' + '.midi': 'audio/midi audio/x-midi' + '.mjs': 'text/javascript' + '.mp3': 'audio/mpeg' + '.mp4': 'video/mp4' + '.mpeg': 'video/mpeg' + '.mpkg': 'application/vnd.apple.installer+xml' + '.odp': 'application/vnd.oasis.opendocument.presentation' + '.ods': 'application/vnd.oasis.opendocument.spreadsheet' + '.odt': 'application/vnd.oasis.opendocument.text' + '.oga': 'audio/ogg' + '.ogv': 'video/ogg' + '.ogx': 'application/ogg' + '.opus': 'audio/opus' + '.otf': 'font/otf' + '.png': 'image/png' + '.pdf': 'application/pdf' + '.php': 'application/x-httpd-php' + '.ppt': 'application/vnd.ms-powerpoint' + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation' + '.rar': 'application/vnd.rar' + '.rtf': 'application/rtf' + '.sh': 'application/x-sh' + '.svg': 'image/svg+xml' + '.swf': 'application/x-shockwave-flash' + '.tar': 'application/x-tar' + '.tif': 'image/tiff' + '.tiff': 'image/tiff' + '.ts': 'video/mp2t' + '.ttf': 'font/ttf' + '.txt': 'text/plain' + '.vsd': 'application/vnd.visio' + '.wav': 'audio/wav' + '.weba': 'audio/webm' + '.webm': 'video/webm' + '.webp': 'image/webp' + '.woff': 'font/woff' + '.woff2': 'font/woff2' + '.xhtml': 'application/xhtml+xml' + '.xls': 'application/vnd.ms-excel' + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + '.xml': 'application/xml' + '.xul': 'application/vnd.mozilla.xul+xml' + '.zip': 'application/zip' + '.3gp': 'video/3gpp' + '.3g2': 'video/3gpp2' + '.7z': 'application/x-7z-compressed' + } + max_http_post_size = 1024 * 1024 + default_port = 8080 +) diff --git a/src/web/web.v b/src/web/web.v index b053904..9fe0ddc 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -12,146 +12,23 @@ import time import json import log -// A dummy structure that returns from routes to indicate that you actually sent something to a user -[noinit] -pub struct Result {} - -pub const ( - methods_with_form = [http.Method.post, .put, .patch] - headers_close = http.new_custom_header_from_map({ - 'Server': 'VWeb' - http.CommonHeader.connection.str(): 'close' - }) or { panic('should never fail') } - - http_302 = http.new_response( - status: .found - body: '302 Found' - header: headers_close - ) - http_400 = http.new_response( - status: .bad_request - body: '400 Bad Request' - header: http.new_header( - key: .content_type - value: 'text/plain' - ).join(headers_close) - ) - http_404 = http.new_response( - status: .not_found - body: '404 Not Found' - header: http.new_header( - key: .content_type - value: 'text/plain' - ).join(headers_close) - ) - http_500 = http.new_response( - status: .internal_server_error - body: '500 Internal Server Error' - header: http.new_header( - key: .content_type - value: 'text/plain' - ).join(headers_close) - ) - mime_types = { - '.aac': 'audio/aac' - '.abw': 'application/x-abiword' - '.arc': 'application/x-freearc' - '.avi': 'video/x-msvideo' - '.azw': 'application/vnd.amazon.ebook' - '.bin': 'application/octet-stream' - '.bmp': 'image/bmp' - '.bz': 'application/x-bzip' - '.bz2': 'application/x-bzip2' - '.cda': 'application/x-cdf' - '.csh': 'application/x-csh' - '.css': 'text/css' - '.csv': 'text/csv' - '.doc': 'application/msword' - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' - '.eot': 'application/vnd.ms-fontobject' - '.epub': 'application/epub+zip' - '.gz': 'application/gzip' - '.gif': 'image/gif' - '.htm': 'text/html' - '.html': 'text/html' - '.ico': 'image/vnd.microsoft.icon' - '.ics': 'text/calendar' - '.jar': 'application/java-archive' - '.jpeg': 'image/jpeg' - '.jpg': 'image/jpeg' - '.js': 'text/javascript' - '.json': 'application/json' - '.jsonld': 'application/ld+json' - '.mid': 'audio/midi audio/x-midi' - '.midi': 'audio/midi audio/x-midi' - '.mjs': 'text/javascript' - '.mp3': 'audio/mpeg' - '.mp4': 'video/mp4' - '.mpeg': 'video/mpeg' - '.mpkg': 'application/vnd.apple.installer+xml' - '.odp': 'application/vnd.oasis.opendocument.presentation' - '.ods': 'application/vnd.oasis.opendocument.spreadsheet' - '.odt': 'application/vnd.oasis.opendocument.text' - '.oga': 'audio/ogg' - '.ogv': 'video/ogg' - '.ogx': 'application/ogg' - '.opus': 'audio/opus' - '.otf': 'font/otf' - '.png': 'image/png' - '.pdf': 'application/pdf' - '.php': 'application/x-httpd-php' - '.ppt': 'application/vnd.ms-powerpoint' - '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation' - '.rar': 'application/vnd.rar' - '.rtf': 'application/rtf' - '.sh': 'application/x-sh' - '.svg': 'image/svg+xml' - '.swf': 'application/x-shockwave-flash' - '.tar': 'application/x-tar' - '.tif': 'image/tiff' - '.tiff': 'image/tiff' - '.ts': 'video/mp2t' - '.ttf': 'font/ttf' - '.txt': 'text/plain' - '.vsd': 'application/vnd.visio' - '.wav': 'audio/wav' - '.weba': 'audio/webm' - '.webm': 'video/webm' - '.webp': 'image/webp' - '.woff': 'font/woff' - '.woff2': 'font/woff2' - '.xhtml': 'application/xhtml+xml' - '.xls': 'application/vnd.ms-excel' - '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' - '.xml': 'application/xml' - '.xul': 'application/vnd.mozilla.xul+xml' - '.zip': 'application/zip' - '.3gp': 'video/3gpp' - '.3g2': 'video/3gpp2' - '.7z': 'application/x-7z-compressed' - } - max_http_post_size = 1024 * 1024 - default_port = 8080 -) - // The Context struct represents the Context which hold the HTTP request and response. // It has fields for the query, form, files. pub struct Context { -mut: - content_type string = 'text/plain' - status http.Status = http.Status.ok pub: // HTTP Request req http.Request // TODO Response pub mut: - done bool - // time.ticks() from start of web connection handle. - // You can use it to determine how much time is spent on your request. - page_gen_start i64 // TCP connection to client. // But beware, do not store it for further use, after request processing web will close connection. - conn &net.TcpConn + conn &net.TcpConn + // Gives access to a shared logger object + logger shared log.Log + // REQUEST + // time.ticks() from start of web connection handle. + // You can use it to determine how much time is spent on your request. + page_gen_start i64 static_files map[string]string static_mime_types map[string]string // Map containing query params for the route. @@ -161,14 +38,13 @@ pub mut: form map[string]string // Files from multipart-form. files map[string][]http.FileData - - header http.Header // response headers - // ? It doesn't seem to be used anywhere - form_error string // Allows reading the request body reader io.BufferedReader - // Gives access to a shared logger object - logger shared log.Log + // RESPONSE + status http.Status = http.Status.ok + content_type string = 'text/plain' + // response headers + header http.Header } struct FileData { @@ -188,40 +64,68 @@ struct Route { // Probably you can use it for check user session cookie or add header. pub fn (ctx Context) before_request() {} -// send_string -fn send_string(mut conn net.TcpConn, s string) ? { - conn.write(s.bytes())? +// send_string writes the given string to the TCP connection socket. +fn (mut ctx Context) send_string(s string) ? { + ctx.conn.write(s.bytes())? } -// send_response_to_client sends a response to the client -[manualfree] -pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bool { - if ctx.done { - return false +// send_reader reads at most `size` bytes from the given reader & writes them +// to the TCP connection socket. Internally, a 10KB buffer is used, to avoid +// having to store all bytes in memory at once. +fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ? { + mut buf := []u8{len: 10_000} + mut bytes_left := size + + // Repeat as long as the stream still has data + for bytes_left > 0 { + bytes_read := reader.read(mut buf)? + bytes_left -= u64(bytes_read) + + mut to_write := bytes_read + + for to_write > 0 { + // TODO don't just loop infinitely here + bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { continue } + + to_write = to_write - bytes_written + } } - ctx.done = true - - // build header - header := http.new_header_from_map({ - http.CommonHeader.content_type: mimetype - http.CommonHeader.content_length: res.len.str() - }).join(ctx.header) +} +// send_response_header constructs a valid HTTP response with an empty body & +// sends it to the client. +pub fn (mut ctx Context) send_response_header() ? { mut resp := http.Response{ - header: header.join(web.headers_close) - body: res + header: ctx.header.join(headers_close) } resp.set_version(.v1_1) resp.set_status(ctx.status) - send_string(mut ctx.conn, resp.bytestr()) or { return false } + ctx.send_string(resp.bytestr())? +} + +// send_response constructs the resulting HTTP response with the given body +// string & sends it to the client. +pub fn (mut ctx Context) send_response(res string) bool { + ctx.send_response_header() or { return false } + ctx.send_string(res) or { return false } + + return true +} + +// send_reader_response constructs the resulting HTTP response with the given +// body & streams the reader's contents to the client. +pub fn (mut ctx Context) send_reader_response(mut reader io.Reader, size u64) bool { + ctx.send_response_header() or { return false } + ctx.send_reader(mut reader, size) or { return false } + return true } // text responds to a request with some plaintext. pub fn (mut ctx Context) text(status http.Status, s string) Result { ctx.status = status - - ctx.send_response_to_client('text/plain', s) + ctx.content_type = 'text/plain' + ctx.send_response(s) return Result{} } @@ -229,9 +133,10 @@ pub fn (mut ctx Context) text(status http.Status, s string) Result { // json HTTP_OK with json_s as payload with content-type `application/json` pub fn (mut ctx Context) json(status http.Status, j T) Result { ctx.status = status + ctx.content_type = 'application/json' json_s := json.encode(j) - ctx.send_response_to_client('application/json', json_s) + ctx.send_response(json_s) return Result{} } @@ -239,10 +144,6 @@ pub fn (mut ctx Context) json(status http.Status, j T) Result { // file Response HTTP_OK with file as payload // This function manually implements responses because it needs to stream the file contents pub fn (mut ctx Context) file(f_path string) Result { - if ctx.done { - return Result{} - } - if !os.is_file(f_path) { return ctx.not_found() } @@ -266,7 +167,7 @@ pub fn (mut ctx Context) file(f_path string) Result { // We open the file before sending the headers in case reading fails file_size := os.file_size(f_path) - file := os.open(f_path) or { + mut file := os.open(f_path) or { eprintln(err.msg()) ctx.server_error(500) return Result{} @@ -279,32 +180,32 @@ pub fn (mut ctx Context) file(f_path string) Result { }).join(ctx.header) mut resp := http.Response{ - header: header.join(web.headers_close) + header: header.join(headers_close) } resp.set_version(.v1_1) resp.set_status(ctx.status) - send_string(mut ctx.conn, resp.bytestr()) or { return Result{} } + ctx.send_string(resp.bytestr()) or { return Result{} } + ctx.send_reader(mut file, file_size) or { return Result{} } - mut buf := []u8{len: 1_000_000} - mut bytes_left := file_size + // mut buf := []u8{len: 1_000_000} + // mut bytes_left := file_size - // Repeat as long as the stream still has data - for bytes_left > 0 { - // TODO check if just breaking here is safe - bytes_read := file.read(mut buf) or { break } - bytes_left -= u64(bytes_read) + // // Repeat as long as the stream still has data + // for bytes_left > 0 { + // // TODO check if just breaking here is safe + // bytes_read := file.read(mut buf) or { break } + // bytes_left -= u64(bytes_read) - mut to_write := bytes_read + // mut to_write := bytes_read - for to_write > 0 { - // TODO don't just loop infinitely here - bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { continue } + // for to_write > 0 { + // // TODO don't just loop infinitely here + // bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { continue } - to_write = to_write - bytes_written - } - } + // to_write = to_write - bytes_written + // } + // } - ctx.done = true return Result{} } @@ -319,23 +220,16 @@ pub fn (mut ctx Context) server_error(ecode int) Result { $if debug { eprintln('> ctx.server_error ecode: $ecode') } - if ctx.done { - return Result{} - } - send_string(mut ctx.conn, web.http_500.bytestr()) or {} + ctx.send_string(http_500.bytestr()) or {} return Result{} } // redirect Redirect to an url pub fn (mut ctx Context) redirect(url string) Result { - if ctx.done { - return Result{} - } - ctx.done = true - mut resp := web.http_302 + mut resp := http_302 resp.header = resp.header.join(ctx.header) resp.header.add(.location, url) - send_string(mut ctx.conn, resp.bytestr()) or { return Result{} } + ctx.send_string(resp.bytestr()) or { return Result{} } return Result{} } @@ -532,7 +426,7 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { } } // Route not found - conn.write(web.http_404.bytes()) or {} + conn.write(http_404.bytes()) or {} } // route_matches returns wether a route matches @@ -597,7 +491,6 @@ pub fn (ctx &Context) ip() string { // error Set s to the form error pub fn (mut ctx Context) error(s string) { println('web error: $s') - ctx.form_error = s } // filter Do not delete. From e7b45bf251736cbd60fcb49aa45bd2b7012125fe Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 12 Aug 2022 15:08:05 +0200 Subject: [PATCH 011/140] feat(web): file() now handles HEAD requests --- src/server/repo.v | 9 ----- src/web/web.v | 90 +++++++++++------------------------------------ 2 files changed, 20 insertions(+), 79 deletions(-) diff --git a/src/server/repo.v b/src/server/repo.v index fbf37df..4a417fb 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -45,15 +45,6 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re full_path = os.join_path(app.repo.repos_dir, repo, arch, filename, 'desc') } - // Scuffed way to respond to HEAD requests - if app.req.method == http.Method.head { - if os.exists(full_path) { - return app.status(http.Status.ok) - } - - return app.not_found() - } - return app.file(full_path) } diff --git a/src/web/web.v b/src/web/web.v index 9fe0ddc..c4cfee7 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -25,10 +25,10 @@ pub mut: conn &net.TcpConn // Gives access to a shared logger object logger shared log.Log - // REQUEST // time.ticks() from start of web connection handle. // You can use it to determine how much time is spent on your request. page_gen_start i64 + // REQUEST static_files map[string]string static_mime_types map[string]string // Map containing query params for the route. @@ -103,6 +103,12 @@ pub fn (mut ctx Context) send_response_header() ? { ctx.send_string(resp.bytestr())? } +// send is a convenience function for sending the HTTP response with an empty +// body. +pub fn (mut ctx Context) send() bool { + return ctx.send_response('') +} + // send_response constructs the resulting HTTP response with the given body // string & sends it to the client. pub fn (mut ctx Context) send_response(res string) bool { @@ -144,67 +150,32 @@ pub fn (mut ctx Context) json(status http.Status, j T) Result { // file Response HTTP_OK with file as payload // This function manually implements responses because it needs to stream the file contents pub fn (mut ctx Context) file(f_path string) Result { + // If the file doesn't exist, just respond with a 404 if !os.is_file(f_path) { - return ctx.not_found() + ctx.status = .not_found + ctx.send() + + return Result{} } - // ext := os.file_ext(f_path) - // data := os.read_file(f_path) or { - // eprint(err.msg()) - // ctx.server_error(500) - // return Result{} - // } - // content_type := web.mime_types[ext] - // if content_type == '' { - // eprintln('no MIME type found for extension $ext') - // ctx.server_error(500) + file_size := os.file_size(f_path) + ctx.header.add(http.CommonHeader.content_length, file_size.str()) - // return Result{} - // } + // A HEAD request only returns the size of the file. + if ctx.req.method == .head { + ctx.send() - // First, we return the headers for the request + return Result{} + } // We open the file before sending the headers in case reading fails - file_size := os.file_size(f_path) - mut file := os.open(f_path) or { eprintln(err.msg()) ctx.server_error(500) return Result{} } - // build header - header := http.new_header_from_map({ - // http.CommonHeader.content_type: content_type - http.CommonHeader.content_length: file_size.str() - }).join(ctx.header) - - mut resp := http.Response{ - header: header.join(headers_close) - } - resp.set_version(.v1_1) - resp.set_status(ctx.status) - ctx.send_string(resp.bytestr()) or { return Result{} } - ctx.send_reader(mut file, file_size) or { return Result{} } - - // mut buf := []u8{len: 1_000_000} - // mut bytes_left := file_size - - // // Repeat as long as the stream still has data - // for bytes_left > 0 { - // // TODO check if just breaking here is safe - // bytes_read := file.read(mut buf) or { break } - // bytes_left -= u64(bytes_read) - - // mut to_write := bytes_read - - // for to_write > 0 { - // // TODO don't just loop infinitely here - // bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { continue } - - // to_write = to_write - bytes_written - // } - // } + ctx.send_reader_response(mut file, file_size) return Result{} } @@ -472,27 +443,6 @@ fn route_matches(url_words []string, route_words []string) ?[]string { return params } -// ip Returns the ip address from the current user -pub fn (ctx &Context) ip() string { - mut ip := ctx.req.header.get(.x_forwarded_for) or { '' } - if ip == '' { - ip = ctx.req.header.get_custom('X-Real-Ip') or { '' } - } - - if ip.contains(',') { - ip = ip.all_before(',') - } - if ip == '' { - ip = ctx.conn.peer_ip() or { '' } - } - return ip -} - -// error Set s to the form error -pub fn (mut ctx Context) error(s string) { - println('web error: $s') -} - // filter Do not delete. // It used by `vlib/v/gen/c/str_intp.v:130` for string interpolation inside web templates // TODO: move it to template render From cc5df95a1a9b742ca3945bfda16c1df53b294a0c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 12 Aug 2022 17:11:44 +0200 Subject: [PATCH 012/140] feat(web): file() now supports HTTP byte range --- src/server/repo.v | 2 +- src/web/web.v | 64 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/src/server/repo.v b/src/server/repo.v index 4a417fb..2253a44 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -13,7 +13,7 @@ import response { new_response } // server is still responsive. ['/health'; get] pub fn (mut app App) healthcheck() web.Result { - return app.json(http.Status.ok, new_response('Healthy.')) + return app.json(.ok, new_response('Healthy.')) } // get_repo_file handles all Pacman-related routes. It returns both the diff --git a/src/web/web.v b/src/web/web.v index c4cfee7..51acd2f 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -27,7 +27,7 @@ pub mut: logger shared log.Log // time.ticks() from start of web connection handle. // You can use it to determine how much time is spent on your request. - page_gen_start i64 + page_gen_start i64 // REQUEST static_files map[string]string static_mime_types map[string]string @@ -84,8 +84,7 @@ fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ? { mut to_write := bytes_read for to_write > 0 { - // TODO don't just loop infinitely here - bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { continue } + bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { break } to_write = to_write - bytes_written } @@ -127,15 +126,6 @@ pub fn (mut ctx Context) send_reader_response(mut reader io.Reader, size u64) bo return true } -// text responds to a request with some plaintext. -pub fn (mut ctx Context) text(status http.Status, s string) Result { - ctx.status = status - ctx.content_type = 'text/plain' - ctx.send_response(s) - - return Result{} -} - // json HTTP_OK with json_s as payload with content-type `application/json` pub fn (mut ctx Context) json(status http.Status, j T) Result { ctx.status = status @@ -158,6 +148,8 @@ pub fn (mut ctx Context) file(f_path string) Result { return Result{} } + ctx.header.add(.accept_ranges, 'bytes') + file_size := os.file_size(f_path) ctx.header.add(http.CommonHeader.content_length, file_size.str()) @@ -168,14 +160,53 @@ pub fn (mut ctx Context) file(f_path string) Result { return Result{} } - // We open the file before sending the headers in case reading fails mut file := os.open(f_path) or { eprintln(err.msg()) ctx.server_error(500) return Result{} } - ctx.send_reader_response(mut file, file_size) + defer { + file.close() + } + + if range_str := ctx.req.header.get(.range) { + mut parts := range_str.split_nth('=', 2) + + if parts[0] != 'bytes' { + ctx.status = .requested_range_not_satisfiable + ctx.header.delete(.content_length) + ctx.send() + return Result{} + } + + parts = parts[1].split_nth('-', 2) + + start := parts[0].i64() + end := if parts[1] == '' { file_size - 1 } else { parts[1].u64() } + + // Either the actual number 0 or the result of an invalid integer + if end == 0 { + ctx.status = .requested_range_not_satisfiable + ctx.header.delete(.content_length) + ctx.send() + return Result{} + } + + // Move cursor to start of data to read + file.seek(start, .start) or { + ctx.server_error(500) + return Result{} + } + + length := end - u64(start) + 1 + + ctx.status = .partial_content + ctx.header.set(.content_length, length.str()) + ctx.send_reader_response(mut file, length) + } else { + ctx.send_reader_response(mut file, file_size) + } return Result{} } @@ -183,7 +214,10 @@ pub fn (mut ctx Context) file(f_path string) Result { // status responds with an empty textual response, essentially only returning // the given status code. pub fn (mut ctx Context) status(status http.Status) Result { - return ctx.text(status, '') + ctx.status = status + ctx.send() + + return Result{} } // server_error Response a server error From e23635a1d394f135c60b67a6517d78db0ed0b579 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 13 Aug 2022 13:16:31 +0200 Subject: [PATCH 013/140] refactor: moved response module to web.response --- README.md | 11 ++--------- src/client/client.v | 2 +- src/client/logs.v | 2 +- src/client/targets.v | 2 +- src/server/api_logs.v | 2 +- src/server/api_targets.v | 2 +- src/server/repo.v | 2 +- src/server/repo_remove.v | 2 +- src/{ => web}/response/response.v | 0 9 files changed, 9 insertions(+), 16 deletions(-) rename src/{ => web}/response/response.v (100%) diff --git a/README.md b/README.md index 5911ea2..29ec0f0 100644 --- a/README.md +++ b/README.md @@ -48,15 +48,8 @@ update`. ### Compiler -Vieter compiles with the standard Vlang compiler. However, I do maintain a -[mirror](https://git.rustybever.be/vieter-v/v). This is to ensure my CI does -not break without reason, as I control when & how frequently the mirror is -updated to reflect the official repository. - -If you encounter issues using the latest V compiler, try using my mirror -instead. `make v` will clone the repository & build the mirror. Afterwards, -prepending any make command with `V_PATH=v/v` tells make to use the locally -compiled mirror instead. +I used to maintain a mirror that tracked the latest master, but nowadays, I +solely target V 0.3 as a compiler. ## Contributing diff --git a/src/client/client.v b/src/client/client.v index 2bb1ac2..24e4444 100644 --- a/src/client/client.v +++ b/src/client/client.v @@ -2,7 +2,7 @@ module client import net.http { Method } import net.urllib -import response { Response } +import web.response { Response } import json pub struct Client { diff --git a/src/client/logs.v b/src/client/logs.v index f242f6e..b52c3d0 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -2,7 +2,7 @@ module client import models { BuildLog, BuildLogFilter } import net.http { Method } -import response { Response } +import web.response { Response } import time // get_build_logs returns all build logs. diff --git a/src/client/targets.v b/src/client/targets.v index 82c7878..f5258a4 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -2,7 +2,7 @@ module client import models { Target, TargetFilter } import net.http { Method } -import response { Response } +import web.response { Response } // get_targets returns a list of targets, given a filter object. pub fn (c &Client) get_targets(filter TargetFilter) ?[]Target { diff --git a/src/server/api_logs.v b/src/server/api_logs.v index fa3338e..6728392 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -3,7 +3,7 @@ module server import web import net.http import net.urllib -import response { new_data_response, new_response } +import web.response { new_data_response, new_response } import db import time import os diff --git a/src/server/api_targets.v b/src/server/api_targets.v index 3867c94..4cc3a58 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -2,7 +2,7 @@ module server import web import net.http -import response { new_data_response, new_response } +import web.response { new_data_response, new_response } import db import models { Target, TargetArch, TargetFilter } diff --git a/src/server/repo.v b/src/server/repo.v index 2253a44..242fd2d 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -7,7 +7,7 @@ import time import rand import util import net.http -import response { new_response } +import web.response { new_response } // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. diff --git a/src/server/repo_remove.v b/src/server/repo_remove.v index 642f26f..5d5ef15 100644 --- a/src/server/repo_remove.v +++ b/src/server/repo_remove.v @@ -2,7 +2,7 @@ module server import web import net.http -import response { new_response } +import web.response { new_response } // delete_package tries to remove the given package. ['/:repo/:arch/:pkg'; delete] diff --git a/src/response/response.v b/src/web/response/response.v similarity index 100% rename from src/response/response.v rename to src/web/response/response.v From 9268ef0302d9ec3d73dc1fa7f38809655f2adbeb Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 13 Aug 2022 17:49:05 +0200 Subject: [PATCH 014/140] refactor(web): some small cleanup --- CHANGELOG.md | 2 ++ src/server/repo_remove.v | 2 +- src/web/consts.v | 2 +- src/web/web.v | 33 ++++++++++++++++----------------- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7138a8..5aa0e43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 providing a Git repository * CLI commands for searching the AUR & directly adding packages * HTTP routes for removing packages, arch-repos & repos +* All endpoints serving files now support HTTP byte range requests ### Changed @@ -25,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Branch name for 'git' targets is now optional; if not provided, the repository will be cloned with the default branch * Build containers now explicitely set the PATH variable +* Refactor of web framework ### Removed diff --git a/src/server/repo_remove.v b/src/server/repo_remove.v index 5d5ef15..316b387 100644 --- a/src/server/repo_remove.v +++ b/src/server/repo_remove.v @@ -8,7 +8,7 @@ import web.response { new_response } ['/:repo/:arch/:pkg'; delete] fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result { if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + return app.json(.unauthorized, new_response('Unauthorized.')) } res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { diff --git a/src/web/consts.v b/src/web/consts.v index 7b1d2b4..1b5bf08 100644 --- a/src/web/consts.v +++ b/src/web/consts.v @@ -9,7 +9,7 @@ pub struct Result {} pub const ( methods_with_form = [http.Method.post, .put, .patch] headers_close = http.new_custom_header_from_map({ - 'Server': 'VWeb' + 'Server': 'Vieter' http.CommonHeader.connection.str(): 'close' }) or { panic('should never fail') } diff --git a/src/web/web.v b/src/web/web.v index 51acd2f..8434a80 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -91,15 +91,22 @@ fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ? { } } +// send_custom_response sends the given http.Response to the client. It can be +// used to overwrite the Context object & send a completely custom +// http.Response instead. +fn (mut ctx Context) send_custom_response(resp &http.Response) ? { + ctx.send_string(resp.bytestr())? +} + // send_response_header constructs a valid HTTP response with an empty body & // sends it to the client. pub fn (mut ctx Context) send_response_header() ? { mut resp := http.Response{ header: ctx.header.join(headers_close) } - resp.set_version(.v1_1) resp.set_status(ctx.status) - ctx.send_string(resp.bytestr())? + + ctx.send_custom_response(resp)? } // send is a convenience function for sending the HTTP response with an empty @@ -222,10 +229,8 @@ pub fn (mut ctx Context) status(status http.Status) Result { // server_error Response a server error pub fn (mut ctx Context) server_error(ecode int) Result { - $if debug { - eprintln('> ctx.server_error ecode: $ecode') - } - ctx.send_string(http_500.bytestr()) or {} + ctx.send_custom_response(http_500) or {} + return Result{} } @@ -234,23 +239,17 @@ pub fn (mut ctx Context) redirect(url string) Result { mut resp := http_302 resp.header = resp.header.join(ctx.header) resp.header.add(.location, url) - ctx.send_string(resp.bytestr()) or { return Result{} } + + ctx.send_custom_response(resp) or {} + return Result{} } // not_found Send an not_found response pub fn (mut ctx Context) not_found() Result { - return ctx.status(http.Status.not_found) -} + ctx.send_custom_response(http_404) or {} -// add_header Adds an header to the response with key and val -pub fn (mut ctx Context) add_header(key string, val string) { - ctx.header.add_custom(key, val) or {} -} - -// get_header Returns the header data from the key -pub fn (ctx &Context) get_header(key string) string { - return ctx.req.header.get_custom(key) or { '' } + return Result{} } interface DbInterface { From 4887af26d3e7733ce3be5db570a4861590aade33 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 4 Sep 2022 19:32:22 +0200 Subject: [PATCH 015/140] feat(web): added authentication as function attribute --- src/server/server.v | 1 + src/web/consts.v | 8 +++++++ src/web/parse.v | 6 ++++- src/web/web.v | 53 ++++++++++++++++++++++++++++----------------- 4 files changed, 47 insertions(+), 21 deletions(-) diff --git a/src/server/server.v b/src/server/server.v index 1a9df3f..9903cea 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -73,6 +73,7 @@ pub fn server(conf Config) ? { web.run(&App{ logger: logger + api_key: conf.api_key conf: conf repo: repo db: db diff --git a/src/web/consts.v b/src/web/consts.v index 1b5bf08..df8cdb2 100644 --- a/src/web/consts.v +++ b/src/web/consts.v @@ -26,6 +26,14 @@ pub const ( value: 'text/plain' ).join(headers_close) ) + http_401 = http.new_response( + status: .unauthorized + body: '401 Unauthorized' + header: http.new_header( + key: .content_type + value: 'text/plain' + ).join(headers_close) + ) http_404 = http.new_response( status: .not_found body: '404 Not Found' diff --git a/src/web/parse.v b/src/web/parse.v index a095f0c..ee7a72c 100644 --- a/src/web/parse.v +++ b/src/web/parse.v @@ -3,6 +3,10 @@ module web import net.urllib import net.http +// Method attributes that should be ignored when parsing, as they're used +// elsewhere. +const attrs_to_ignore = ['auth'] + // Parsing function attributes for methods and path. fn parse_attrs(name string, attrs []string) ?([]http.Method, string) { if attrs.len == 0 { @@ -32,7 +36,7 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) { } i++ } - if x.len > 0 { + if x.len > 0 && x.any(!web.attrs_to_ignore.contains(it)) { return IError(http.UnexpectedExtraAttributeError{ attributes: x }) diff --git a/src/web/web.v b/src/web/web.v index 8434a80..0847c7e 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -18,6 +18,8 @@ pub struct Context { pub: // HTTP Request req http.Request + // API key used when authenticating requests + api_key string // TODO Response pub mut: // TCP connection to client. @@ -101,9 +103,10 @@ fn (mut ctx Context) send_custom_response(resp &http.Response) ? { // send_response_header constructs a valid HTTP response with an empty body & // sends it to the client. pub fn (mut ctx Context) send_response_header() ? { - mut resp := http.Response{ + mut resp := http.new_response( header: ctx.header.join(headers_close) - } + ) + resp.header.add(.content_type, ctx.content_type) resp.set_status(ctx.status) ctx.send_custom_response(resp)? @@ -133,6 +136,15 @@ pub fn (mut ctx Context) send_reader_response(mut reader io.Reader, size u64) bo return true } +// is_authenticated checks whether the request passes a correct API key. +pub fn (ctx &Context) is_authenticated() bool { + if provided_key := ctx.req.header.get_custom('X-Api-Key') { + return provided_key == ctx.api_key + } + + return false +} + // json HTTP_OK with json_s as payload with content-type `application/json` pub fn (mut ctx Context) json(status http.Status, j T) Result { ctx.status = status @@ -177,9 +189,12 @@ pub fn (mut ctx Context) file(f_path string) Result { file.close() } + // Currently, this only supports a single provided range, e.g. + // bytes=0-1023, and not multiple ranges, e.g. bytes=0-50, 100-150 if range_str := ctx.req.header.get(.range) { mut parts := range_str.split_nth('=', 2) + // We only support the 'bytes' range type if parts[0] != 'bytes' { ctx.status = .requested_range_not_satisfiable ctx.header.delete(.content_length) @@ -376,8 +391,10 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { static_mime_types: app.static_mime_types reader: reader logger: app.logger + api_key: app.api_key } + // Calling middleware... app.before_request() @@ -394,31 +411,27 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { // Used for route matching route_words := route.path.split('/').filter(it != '') - // Route immediate matches first + // Route immediate matches & index files first // For example URL `/register` matches route `/:user`, but `fn register()` // should be called first. - if !route.path.contains('/:') && url_words == route_words { - // We found a match - if head.method == .post && method.args.len > 0 { - // TODO implement POST requests - // Populate method args with form values - // mut args := []string{cap: method.args.len} - // for param in method.args { - // args << form[param.name] - // } - // app.$method(args) - } else { - app.$method() + if (!route.path.contains('/:') && url_words == route_words) + || (url_words.len == 0 && route_words == ['index'] && method.name == 'index') { + // Check whether the request is authorised + if 'auth' in method.attrs && !app.is_authenticated() { + conn.write(http_401.bytes()) or {} + return } - return - } - if url_words.len == 0 && route_words == ['index'] && method.name == 'index' { + // We found a match app.$method() return - } + } else if params := route_matches(url_words, route_words) { + // Check whether the request is authorised + if 'auth' in method.attrs && !app.is_authenticated() { + conn.write(http_401.bytes()) or {} + return + } - if params := route_matches(url_words, route_words) { method_args := params.clone() if method_args.len != method.args.len { eprintln('warning: uneven parameters count ($method.args.len) in `$method.name`, compared to the web route `$method.attrs` ($method_args.len)') From 272f14b2645a56aedd608fd0880a768d43bfa75e Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 4 Sep 2022 19:36:08 +0200 Subject: [PATCH 016/140] refactor(server): migrated all routes to new auth system --- src/server/api_logs.v | 24 ++++-------------------- src/server/api_targets.v | 30 +++++------------------------- src/server/auth.v | 12 ------------ src/server/repo.v | 6 +----- src/server/repo_remove.v | 18 +++--------------- src/web/web.v | 1 - 6 files changed, 13 insertions(+), 78 deletions(-) delete mode 100644 src/server/auth.v diff --git a/src/server/api_logs.v b/src/server/api_logs.v index 6728392..021c1ac 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -12,12 +12,8 @@ import models { BuildLog, BuildLogFilter } // v1_get_logs returns all build logs in the database. A 'target' query param can // optionally be added to limit the list of build logs to that repository. -['/api/v1/logs'; get] +['/api/v1/logs'; auth; get] fn (mut app App) v1_get_logs() web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - filter := models.from_params(app.query) or { return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } @@ -27,24 +23,16 @@ fn (mut app App) v1_get_logs() web.Result { } // v1_get_single_log returns the build log with the given id. -['/api/v1/logs/:id'; get] +['/api/v1/logs/:id'; auth; get] fn (mut app App) v1_get_single_log(id int) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - log := app.db.get_build_log(id) or { return app.not_found() } return app.json(http.Status.ok, new_data_response(log)) } // v1_get_log_content returns the actual build log file for the given id. -['/api/v1/logs/:id/content'; get] +['/api/v1/logs/:id/content'; auth; get] fn (mut app App) v1_get_log_content(id int) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - log := app.db.get_build_log(id) or { return app.not_found() } file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), log.arch, @@ -63,12 +51,8 @@ fn parse_query_time(query string) ?time.Time { } // v1_post_log adds a new log to the database. -['/api/v1/logs'; post] +['/api/v1/logs'; auth; post] fn (mut app App) v1_post_log() web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - // Parse query params start_time_int := app.query['startTime'].int() diff --git a/src/server/api_targets.v b/src/server/api_targets.v index 4cc3a58..c9e7963 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -7,12 +7,8 @@ import db import models { Target, TargetArch, TargetFilter } // v1_get_targets returns the current list of targets. -['/api/v1/targets'; get] +['/api/v1/targets'; auth; get] fn (mut app App) v1_get_targets() web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - filter := models.from_params(app.query) or { return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } @@ -22,24 +18,16 @@ fn (mut app App) v1_get_targets() web.Result { } // v1_get_single_target returns the information for a single target. -['/api/v1/targets/:id'; get] +['/api/v1/targets/:id'; auth; get] fn (mut app App) v1_get_single_target(id int) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - repo := app.db.get_target(id) or { return app.not_found() } return app.json(http.Status.ok, new_data_response(repo)) } // v1_post_target creates a new target from the provided query string. -['/api/v1/targets'; post] +['/api/v1/targets'; auth; post] fn (mut app App) v1_post_target() web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - mut params := app.query.clone() // If a repo is created without specifying the arch, we assume it's meant @@ -63,24 +51,16 @@ fn (mut app App) v1_post_target() web.Result { } // v1_delete_target removes a given target from the server's list. -['/api/v1/targets/:id'; delete] +['/api/v1/targets/:id'; auth; delete] fn (mut app App) v1_delete_target(id int) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - app.db.delete_target(id) return app.json(http.Status.ok, new_response('Repo removed successfully.')) } // v1_patch_target updates a target's data with the given query params. -['/api/v1/targets/:id'; patch] +['/api/v1/targets/:id'; auth; patch] fn (mut app App) v1_patch_target(id int) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - app.db.update_target(id, app.query) if 'arch' in app.query { diff --git a/src/server/auth.v b/src/server/auth.v deleted file mode 100644 index 7c8a676..0000000 --- a/src/server/auth.v +++ /dev/null @@ -1,12 +0,0 @@ -module server - -import net.http - -// is_authorized checks whether the provided API key is correct. -fn (mut app App) is_authorized() bool { - x_header := app.req.header.get_custom('X-Api-Key', http.HeaderQueryConfig{ exact: true }) or { - return false - } - - return x_header.trim_space() == app.conf.api_key -} diff --git a/src/server/repo.v b/src/server/repo.v index 242fd2d..5ed5d15 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -49,12 +49,8 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re } // put_package handles publishing a package to a repository. -['/:repo/publish'; post] +['/:repo/publish'; auth; post] fn (mut app App) put_package(repo string) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - mut pkg_path := '' if length := app.req.header.get(.content_length) { diff --git a/src/server/repo_remove.v b/src/server/repo_remove.v index 316b387..fdc40e8 100644 --- a/src/server/repo_remove.v +++ b/src/server/repo_remove.v @@ -5,12 +5,8 @@ import net.http import web.response { new_response } // delete_package tries to remove the given package. -['/:repo/:arch/:pkg'; delete] +['/:repo/:arch/:pkg'; auth; delete] fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result { - if !app.is_authorized() { - return app.json(.unauthorized, new_response('Unauthorized.')) - } - res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { app.lerror('Error while deleting package: $err.msg()') @@ -29,12 +25,8 @@ fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result } // delete_arch_repo tries to remove the given arch-repo. -['/:repo/:arch'; delete] +['/:repo/:arch'; auth; delete] fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - res := app.repo.remove_arch_repo(repo, arch) or { app.lerror('Error while deleting arch-repo: $err.msg()') @@ -53,12 +45,8 @@ fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { } // delete_repo tries to remove the given repo. -['/:repo'; delete] +['/:repo'; auth; delete] fn (mut app App) delete_repo(repo string) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - res := app.repo.remove_repo(repo) or { app.lerror('Error while deleting repo: $err.msg()') diff --git a/src/web/web.v b/src/web/web.v index 0847c7e..1d1480f 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -394,7 +394,6 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { api_key: app.api_key } - // Calling middleware... app.before_request() From 9dfdfbf72446f04b38eb6962bfd98600d380e7ca Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 4 Sep 2022 20:16:58 +0200 Subject: [PATCH 017/140] chore: update README [CI SKIP] --- Makefile | 7 ------- README.md | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index ed44df9..69bd795 100644 --- a/Makefile +++ b/Makefile @@ -83,13 +83,6 @@ fmt: test: $(V) test $(SRC_DIR) -# Build & patch the V compiler -.PHONY: v -v: v/v -v/v: - git clone --single-branch https://git.rustybever.be/vieter-v/v v - make -C v - .PHONY: clean clean: rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' 'docs/public' diff --git a/README.md b/README.md index 29ec0f0..b9fff69 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,6 @@ that. Besides a V installer, Vieter also requires the following libraries to work: -* gc * libarchive * openssl * sqlite3 @@ -49,7 +48,8 @@ update`. ### Compiler I used to maintain a mirror that tracked the latest master, but nowadays, I -solely target V 0.3 as a compiler. +maintain a Docker image containing the specific compiler version that Vieter +builds with. Currently, this is V 0.3. ## Contributing From 7b59277931172997759572ac5a00930d677a4d67 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 5 Sep 2022 10:10:02 +0200 Subject: [PATCH 018/140] feat: adding target returns id of added entry --- src/client/targets.v | 4 ++-- src/console/targets/targets.v | 2 +- src/db/targets.v | 6 +++++- src/server/api_targets.v | 6 +++--- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/client/targets.v b/src/client/targets.v index f5258a4..c5e44fe 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -49,9 +49,9 @@ pub struct NewTarget { } // add_target adds a new target to the server. -pub fn (c &Client) add_target(t NewTarget) ?Response { +pub fn (c &Client) add_target(t NewTarget) ?Response { params := models.params_from(t) - data := c.send_request(Method.post, '/api/v1/targets', params)? + data := c.send_request(Method.post, '/api/v1/targets', params)? return data } diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 66d48fb..a555936 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -206,7 +206,7 @@ fn add(conf Config, t &NewTarget) ? { c := client.new(conf.address, conf.api_key) res := c.add_target(t)? - println(res.message) + println("Target added with id $res.data") } // remove removes a repository from the server's list. diff --git a/src/db/targets.v b/src/db/targets.v index 9102033..a705ebb 100644 --- a/src/db/targets.v +++ b/src/db/targets.v @@ -38,10 +38,14 @@ pub fn (db &VieterDb) get_target(target_id int) ?Target { } // add_target inserts the given target into the database. -pub fn (db &VieterDb) add_target(repo Target) { +pub fn (db &VieterDb) add_target(repo Target) int { sql db.conn { insert repo into Target } + + inserted_id := db.conn.last_id() as int + + return inserted_id } // delete_target deletes the target with the given id from the database. diff --git a/src/server/api_targets.v b/src/server/api_targets.v index c9e7963..5757d76 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -2,7 +2,7 @@ module server import web import net.http -import web.response { new_data_response, new_response } +import web.response { new_data_response, new_response, new_full_response } import db import models { Target, TargetArch, TargetFilter } @@ -45,9 +45,9 @@ fn (mut app App) v1_post_target() web.Result { return app.json(http.Status.bad_request, new_response('Invalid kind.')) } - app.db.add_target(new_repo) + id := app.db.add_target(new_repo) - return app.json(http.Status.ok, new_response('Repo added successfully.')) + return app.json(http.Status.ok, new_data_response(id)) } // v1_delete_target removes a given target from the server's list. From 210508f1ee251b3cb61ab5bf146805f10b600c1c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 20:34:02 +0200 Subject: [PATCH 019/140] feat: logs api now also returns id --- CHANGELOG.md | 1 + src/client/logs.v | 4 ++-- src/console/targets/targets.v | 2 +- src/db/logs.v | 6 +++++- src/server/api_logs.v | 9 +++++---- src/server/api_targets.v | 10 +++++----- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5aa0e43..00a3539 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 repository will be cloned with the default branch * Build containers now explicitely set the PATH variable * Refactor of web framework +* API endpoints now return id of newly created entries ### Removed diff --git a/src/client/logs.v b/src/client/logs.v index b52c3d0..b414245 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -39,7 +39,7 @@ pub fn (c &Client) get_build_log_content(id int) ?string { } // add_build_log adds a new build log to the server. -pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { +pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { params := { 'target': target_id.str() 'startTime': start_time.unix_time().str() @@ -48,7 +48,7 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t 'exitCode': exit_code.str() } - data := c.send_request_with_body(Method.post, '/api/v1/logs', params, content)? + data := c.send_request_with_body(Method.post, '/api/v1/logs', params, content)? return data } diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index a555936..2784dbc 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -206,7 +206,7 @@ fn add(conf Config, t &NewTarget) ? { c := client.new(conf.address, conf.api_key) res := c.add_target(t)? - println("Target added with id $res.data") + println('Target added with id $res.data') } // remove removes a repository from the server's list. diff --git a/src/db/logs.v b/src/db/logs.v index af5f53c..923dde2 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -79,10 +79,14 @@ pub fn (db &VieterDb) get_build_log(id int) ?BuildLog { } // add_build_log inserts the given BuildLog into the database. -pub fn (db &VieterDb) add_build_log(log BuildLog) { +pub fn (db &VieterDb) add_build_log(log BuildLog) int { sql db.conn { insert log into BuildLog } + + inserted_id := db.conn.last_id() as int + + return inserted_id } // delete_build_log delete the BuildLog with the given ID from the database. diff --git a/src/server/api_logs.v b/src/server/api_logs.v index 021c1ac..287755a 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -19,7 +19,7 @@ fn (mut app App) v1_get_logs() web.Result { } logs := app.db.get_build_logs(filter) - return app.json(http.Status.ok, new_data_response(logs)) + return app.json(.ok, new_data_response(logs)) } // v1_get_single_log returns the build log with the given id. @@ -27,7 +27,7 @@ fn (mut app App) v1_get_logs() web.Result { fn (mut app App) v1_get_single_log(id int) web.Result { log := app.db.get_build_log(id) or { return app.not_found() } - return app.json(http.Status.ok, new_data_response(log)) + return app.json(.ok, new_data_response(log)) } // v1_get_log_content returns the actual build log file for the given id. @@ -95,7 +95,8 @@ fn (mut app App) v1_post_log() web.Result { exit_code: exit_code } - app.db.add_build_log(log) + // id of newly created log + log_id := app.db.add_build_log(log) repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, target_id.str(), arch) @@ -122,5 +123,5 @@ fn (mut app App) v1_post_log() web.Result { return app.status(http.Status.length_required) } - return app.json(http.Status.ok, new_response('Logs added successfully.')) + return app.json(.ok, new_data_response(log_id)) } diff --git a/src/server/api_targets.v b/src/server/api_targets.v index 5757d76..6f284af 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -2,7 +2,7 @@ module server import web import net.http -import web.response { new_data_response, new_response, new_full_response } +import web.response { new_data_response, new_response } import db import models { Target, TargetArch, TargetFilter } @@ -14,7 +14,7 @@ fn (mut app App) v1_get_targets() web.Result { } repos := app.db.get_targets(filter) - return app.json(http.Status.ok, new_data_response(repos)) + return app.json(.ok, new_data_response(repos)) } // v1_get_single_target returns the information for a single target. @@ -22,7 +22,7 @@ fn (mut app App) v1_get_targets() web.Result { fn (mut app App) v1_get_single_target(id int) web.Result { repo := app.db.get_target(id) or { return app.not_found() } - return app.json(http.Status.ok, new_data_response(repo)) + return app.json(.ok, new_data_response(repo)) } // v1_post_target creates a new target from the provided query string. @@ -55,7 +55,7 @@ fn (mut app App) v1_post_target() web.Result { fn (mut app App) v1_delete_target(id int) web.Result { app.db.delete_target(id) - return app.json(http.Status.ok, new_response('Repo removed successfully.')) + return app.status(.ok) } // v1_patch_target updates a target's data with the given query params. @@ -69,5 +69,5 @@ fn (mut app App) v1_patch_target(id int) web.Result { app.db.update_target_archs(id, arch_objs) } - return app.json(http.Status.ok, new_response('Repo updated successfully.')) + return app.status(.ok) } From b6cd2f0bc280dfddfa85a91188ebe05433eabbf8 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 21:24:29 +0200 Subject: [PATCH 020/140] feat(server): repo POST requests now return information --- CHANGELOG.md | 1 + src/repo/add.v | 43 +++++++++++++++++++------------------------ src/server/repo.v | 21 ++++++--------------- 3 files changed, 26 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00a3539..c875f2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Build containers now explicitely set the PATH variable * Refactor of web framework * API endpoints now return id of newly created entries +* Repo POST requests now return information on published package ### Removed diff --git a/src/repo/add.v b/src/repo/add.v index 0985c7a..608ca50 100644 --- a/src/repo/add.v +++ b/src/repo/add.v @@ -23,8 +23,9 @@ pub: pub struct RepoAddResult { pub: - added bool [required] - pkg &package.Pkg [required] + name string + version string + archs []string } // new creates a new RepoGroupManager & creates the directories as needed @@ -53,10 +54,10 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re return error('Failed to read package file: $err.msg()') } - added := r.add_pkg_in_repo(repo, pkg)? + archs := r.add_pkg_in_repo(repo, pkg)? // If the add was successful, we move the file to the packages directory - for arch in added { + for arch in archs { repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch)) dest_path := os.join_path_single(repo_pkg_path, pkg.filename()) @@ -71,8 +72,9 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re os.rm(pkg_path)? return RepoAddResult{ - added: added.len > 0 - pkg: &pkg + name: pkg.info.name + version: pkg.info.version + archs: archs } } @@ -87,11 +89,9 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin // A package not of arch 'any' can be handled easily by adding it to the // respective repo if pkg.info.arch != 'any' { - if r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)? { - return [pkg.info.arch] - } else { - return [] - } + r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)? + + return [pkg.info.arch] } mut arch_repos := []string{} @@ -113,25 +113,22 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin arch_repos << r.default_arch } - mut added := []string{} - - // We add the package to each repository. If any of the repositories - // return true, the result of the function is also true. + // Add the package to each found architecture + // NOTE: if any of these fail, the function fails. This means the user does + // not know which arch-repositories did succeed in adding the package, if + // any. for arch in arch_repos { - if r.add_pkg_in_arch_repo(repo, arch, pkg)? { - added << arch - } + r.add_pkg_in_arch_repo(repo, arch, pkg)? } - return added + return arch_repos } // add_pkg_in_arch_repo is the function that actually adds a package to a given // arch-repo. It records the package's data in the arch-repo's desc & files // files, and afterwards updates the db & files archives to reflect these -// changes. The function returns false if the package was already present in -// the repo, and true otherwise. -fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool { +// changes. +fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ? { pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version') // Remove the previous version of the package, if present @@ -151,6 +148,4 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac } r.sync(repo, arch)? - - return true } diff --git a/src/server/repo.v b/src/server/repo.v index 5ed5d15..526d4e7 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -6,8 +6,7 @@ import repo import time import rand import util -import net.http -import web.response { new_response } +import web.response { new_data_response, new_response } // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. @@ -65,7 +64,7 @@ fn (mut app App) put_package(repo string) web.Result { util.reader_to_file(mut app.reader, length.int(), pkg_path) or { app.lwarn("Failed to upload '$pkg_path'") - return app.json(http.Status.internal_server_error, new_response('Failed to upload file.')) + return app.status(.internal_server_error) } sw.stop() @@ -74,7 +73,7 @@ fn (mut app App) put_package(repo string) web.Result { app.lwarn('Tried to upload package without specifying a Content-Length.') // length required - return app.status(http.Status.length_required) + return app.status(.length_required) } res := app.repo.add_pkg_from_path(repo, pkg_path) or { @@ -82,18 +81,10 @@ fn (mut app App) put_package(repo string) web.Result { os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg()") } - return app.json(http.Status.internal_server_error, new_response('Failed to add package.')) + return app.status(.internal_server_error) } - if !res.added { - os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg()") } + app.linfo("Added '$res.name-$res.version' to '$repo (${res.archs.join(',')})'.") - app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo'.") - - return app.json(http.Status.bad_request, new_response('File already exists.')) - } - - app.linfo("Added '$res.pkg.full_name()' to repo '$repo ($res.pkg.info.arch)'.") - - return app.json(http.Status.ok, new_response('Package added successfully.')) + return app.json(.ok, new_data_response(res)) } From cf67b46df0cbd69444f2e6046937c74583021d84 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 21:28:37 +0200 Subject: [PATCH 021/140] feat(server): less verbose repo DELETE responses --- src/server/repo_remove.v | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/server/repo_remove.v b/src/server/repo_remove.v index fdc40e8..694f085 100644 --- a/src/server/repo_remove.v +++ b/src/server/repo_remove.v @@ -1,8 +1,6 @@ module server import web -import net.http -import web.response { new_response } // delete_package tries to remove the given package. ['/:repo/:arch/:pkg'; auth; delete] @@ -10,17 +8,17 @@ fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { app.lerror('Error while deleting package: $err.msg()') - return app.json(http.Status.internal_server_error, new_response('Failed to delete package.')) + return app.status(.internal_server_error) } if res { app.linfo("Removed package '$pkg' from '$repo/$arch'") - return app.json(http.Status.ok, new_response('Package removed.')) + return app.status(.ok) } else { app.linfo("Tried removing package '$pkg' from '$repo/$arch', but it doesn't exist.") - return app.json(http.Status.not_found, new_response('Package not found.')) + return app.status(.not_found) } } @@ -30,17 +28,17 @@ fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { res := app.repo.remove_arch_repo(repo, arch) or { app.lerror('Error while deleting arch-repo: $err.msg()') - return app.json(http.Status.internal_server_error, new_response('Failed to delete arch-repo.')) + return app.status(.internal_server_error) } if res { - app.linfo("Removed '$repo/$arch'") + app.linfo("Removed arch-repo '$repo/$arch'") - return app.json(http.Status.ok, new_response('Arch-repo removed.')) + return app.status(.ok) } else { app.linfo("Tried removing '$repo/$arch', but it doesn't exist.") - return app.json(http.Status.not_found, new_response('Arch-repo not found.')) + return app.status(.not_found) } } @@ -50,16 +48,16 @@ fn (mut app App) delete_repo(repo string) web.Result { res := app.repo.remove_repo(repo) or { app.lerror('Error while deleting repo: $err.msg()') - return app.json(http.Status.internal_server_error, new_response('Failed to delete repo.')) + return app.status(.internal_server_error) } if res { - app.linfo("Removed '$repo'") + app.linfo("Removed repo '$repo'") - return app.json(http.Status.ok, new_response('Repo removed.')) + return app.status(.ok) } else { app.linfo("Tried removing '$repo', but it doesn't exist.") - return app.json(http.Status.not_found, new_response('Repo not found.')) + return app.status(.not_found) } } From 8a08788935bb9514007f41d17d18cb2d1b879ecf Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 21:50:29 +0200 Subject: [PATCH 022/140] feat(console): tabled outputs now optionally return without decorations --- src/console/console.v | 5 +++++ src/console/logs/logs.v | 20 +++++++++++++------- src/console/targets/targets.v | 12 +++++++++--- src/main.v | 7 +++++++ 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/console/console.v b/src/console/console.v index 7d782ba..caf4cca 100644 --- a/src/console/console.v +++ b/src/console/console.v @@ -5,6 +5,11 @@ import strings import cli import os +// tabbed_table returns a simple textual table, with tabs as separators. +pub fn tabbed_table(data [][]string) string { + return data.map(it.join('\t')).join('\n') +} + // pretty_table converts a list of string data into a pretty table. Many thanks // to @hungrybluedev in the Vlang Discord for providing this code! // https://ptb.discord.com/channels/592103645835821068/592106336838352923/970278787143045192 diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 0f023bc..41830c2 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -133,7 +133,9 @@ pub fn cmd() cli.Command { ] } - list(conf, filter)? + raw := cmd.flags.get_bool('raw')? + + list(conf, filter, raw)? } }, cli.Command{ @@ -167,27 +169,31 @@ pub fn cmd() cli.Command { } // print_log_list prints a list of logs. -fn print_log_list(logs []BuildLog) ? { +fn print_log_list(logs []BuildLog, raw bool) ? { data := logs.map([it.id.str(), it.target_id.str(), it.start_time.local().str(), it.exit_code.str()]) - println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)?) + if raw { + println(console.tabbed_table(data)) + } else { + println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)?) + } } // list prints a list of all build logs. -fn list(conf Config, filter BuildLogFilter) ? { +fn list(conf Config, filter BuildLogFilter, raw bool) ? { c := client.new(conf.address, conf.api_key) logs := c.get_build_logs(filter)?.data - print_log_list(logs)? + print_log_list(logs, raw)? } // list prints a list of all build logs for a given target. -fn list_for_target(conf Config, target_id int) ? { +fn list_for_target(conf Config, target_id int, raw bool) ? { c := client.new(conf.address, conf.api_key) logs := c.get_build_logs_for_target(target_id)?.data - print_log_list(logs)? + print_log_list(logs, raw)? } // info print the detailed info for a given build log. diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 2784dbc..198e062 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -60,7 +60,9 @@ pub fn cmd() cli.Command { filter.repo = repo } - list(conf, filter)? + raw := cmd.flags.get_bool('raw')? + + list(conf, filter, raw)? } }, cli.Command{ @@ -193,12 +195,16 @@ pub fn cmd() cli.Command { // ID. If multiple or none are found, an error is raised. // list prints out a list of all repositories. -fn list(conf Config, filter TargetFilter) ? { +fn list(conf Config, filter TargetFilter, raw bool) ? { c := client.new(conf.address, conf.api_key) repos := c.get_targets(filter)? data := repos.map([it.id.str(), it.kind, it.url, it.repo]) - println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?) + if raw { + println(console.tabbed_table(data)) + } else { + println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?) + } } // add adds a new repository to the server's list. diff --git a/src/main.v b/src/main.v index 4ade930..0e98bd2 100644 --- a/src/main.v +++ b/src/main.v @@ -24,6 +24,13 @@ fn main() { global: true default_value: [os.expand_tilde_to_home('~/.vieterrc')] }, + cli.Flag{ + flag: cli.FlagType.bool + name: 'raw' + abbrev: 'r' + description: 'Only output minimal information (no formatted tables, etc.)' + global: true + }, ] commands: [ server.cmd(), From fab8ca20b82e7f088d95efdc945a600050fefd6b Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 1 Oct 2022 16:02:43 +0200 Subject: [PATCH 023/140] cli: targets add now supports raw flag --- src/console/targets/targets.v | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 198e062..5640011 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -94,7 +94,9 @@ pub fn cmd() cli.Command { branch: cmd.flags.get_string('branch') or { '' } } - add(conf, t)? + raw := cmd.flags.get_bool('raw')? + + add(conf, t, raw)? } }, cli.Command{ @@ -208,11 +210,15 @@ fn list(conf Config, filter TargetFilter, raw bool) ? { } // add adds a new repository to the server's list. -fn add(conf Config, t &NewTarget) ? { +fn add(conf Config, t &NewTarget, raw bool) ? { c := client.new(conf.address, conf.api_key) res := c.add_target(t)? - println('Target added with id $res.data') + if raw { + println(res.data) + } else { + println('Target added with id $res.data') + } } // remove removes a repository from the server's list. From 851a446a954e619e474251533960b00a4b64bef3 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 1 Oct 2022 16:21:49 +0200 Subject: [PATCH 024/140] chore: updated changelog [CI SKIP] --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c875f2e..60b70b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * CLI commands for searching the AUR & directly adding packages * HTTP routes for removing packages, arch-repos & repos * All endpoints serving files now support HTTP byte range requests +* Better CLI UX + * When adding targets, the ID of the created target is returned + * The `-r` flag only shows raw data of action + * When adding a target, only ID is shown and not surrounding text + * Tabled output returns a tab-separated list (easy to script using + `cut`) ### Changed From 575c04189da2088261b6c3d5921ded948312e568 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 22:20:02 +0200 Subject: [PATCH 025/140] fix(client): allow empty values as params --- src/client/client.v | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/client/client.v b/src/client/client.v index 24e4444..d68ff18 100644 --- a/src/client/client.v +++ b/src/client/client.v @@ -30,12 +30,10 @@ fn (c &Client) send_request_raw(method Method, url string, params map[string]str // Escape each query param for k, v in params { // An empty parameter should be the same as not providing it at all - if v != '' { - params_escaped[k] = urllib.query_escape(v) - } + params_escaped[k] = urllib.query_escape(v) } - params_str := params_escaped.keys().map('$it=${params[it]}').join('&') + params_str := params_escaped.keys().map('$it=${params_escaped[it]}').join('&') full_url = '$full_url?$params_str' } From 95d32e2d515edf27a919e847fef5f5459f3f7234 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 22:24:29 +0200 Subject: [PATCH 026/140] fix(server): prevent `api` as a repository name --- CHANGELOG.md | 2 ++ src/server/repo.v | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60b70b6..79ceed4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Refactor of web framework * API endpoints now return id of newly created entries * Repo POST requests now return information on published package +* `api` can no longer be used as a repository name +* CLI client now allows setting values to an empty value ### Removed diff --git a/src/server/repo.v b/src/server/repo.v index 526d4e7..06ab72e 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -50,6 +50,12 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re // put_package handles publishing a package to a repository. ['/:repo/publish'; auth; post] fn (mut app App) put_package(repo string) web.Result { + // api is a reserved keyword for api routes & should never be allowed to be + // a repository. + if repo.to_lower() == 'api' { + return app.json(.bad_request, new_response("'api' is a reserved keyword & cannot be used as a repository name.")) + } + mut pkg_path := '' if length := app.req.header.get(.content_length) { From 847d77b2bc6e1547da489048b196dd2cb641c4f5 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 15 Sep 2022 13:42:49 +0200 Subject: [PATCH 027/140] chore(ci): refactor ci configs a bit --- .woodpecker/{.arch-rel.yml => arch-rel.yml} | 0 .woodpecker/{.arch.yml => arch.yml} | 0 .woodpecker/{.build.yml => build.yml} | 11 +++++++---- .woodpecker/{.deploy.yml => deploy.yml} | 0 .woodpecker/{.docker.yml => docker.yml} | 0 .woodpecker/{.docs.yml => docs.yml} | 5 ++++- .woodpecker/{.gitea.yml => gitea.yml} | 5 ++++- .woodpecker/{.lint.yml => lint.yml} | 5 ++++- .woodpecker/{.man.yml => man.yml} | 5 ++++- .woodpecker/{.test.yml => test.yml} | 7 +++++-- 10 files changed, 28 insertions(+), 10 deletions(-) rename .woodpecker/{.arch-rel.yml => arch-rel.yml} (100%) rename .woodpecker/{.arch.yml => arch.yml} (100%) rename .woodpecker/{.build.yml => build.yml} (92%) rename .woodpecker/{.deploy.yml => deploy.yml} (100%) rename .woodpecker/{.docker.yml => docker.yml} (100%) rename .woodpecker/{.docs.yml => docs.yml} (93%) rename .woodpecker/{.gitea.yml => gitea.yml} (89%) rename .woodpecker/{.lint.yml => lint.yml} (73%) rename .woodpecker/{.man.yml => man.yml} (92%) rename .woodpecker/{.test.yml => test.yml} (80%) diff --git a/.woodpecker/.arch-rel.yml b/.woodpecker/arch-rel.yml similarity index 100% rename from .woodpecker/.arch-rel.yml rename to .woodpecker/arch-rel.yml diff --git a/.woodpecker/.arch.yml b/.woodpecker/arch.yml similarity index 100% rename from .woodpecker/.arch.yml rename to .woodpecker/arch.yml diff --git a/.woodpecker/.build.yml b/.woodpecker/build.yml similarity index 92% rename from .woodpecker/.build.yml rename to .woodpecker/build.yml index 580fa69..9ee8085 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/build.yml @@ -1,3 +1,6 @@ +variables: + - &vlang_image 'chewingbever/vlang:0.3' + matrix: PLATFORM: - 'linux/amd64' @@ -7,7 +10,7 @@ platform: ${PLATFORM} pipeline: install-modules: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true commands: - export VMODULES=$PWD/.vmodules @@ -16,7 +19,7 @@ pipeline: event: [push, pull_request] debug: - image: 'chewingbever/vlang:0.3' + image: *vlang_image commands: - export VMODULES=$PWD/.vmodules - make @@ -26,7 +29,7 @@ pipeline: exclude: [main] prod: - image: 'chewingbever/vlang:0.3' + image: *vlang_image environment: - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static commands: @@ -44,7 +47,7 @@ pipeline: event: [push, pull_request] upload: - image: 'chewingbever/vlang:0.3' + image: *vlang_image secrets: [ s3_username, s3_password ] commands: # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f diff --git a/.woodpecker/.deploy.yml b/.woodpecker/deploy.yml similarity index 100% rename from .woodpecker/.deploy.yml rename to .woodpecker/deploy.yml diff --git a/.woodpecker/.docker.yml b/.woodpecker/docker.yml similarity index 100% rename from .woodpecker/.docker.yml rename to .woodpecker/docker.yml diff --git a/.woodpecker/.docs.yml b/.woodpecker/docs.yml similarity index 93% rename from .woodpecker/.docs.yml rename to .woodpecker/docs.yml index da495fc..048b1ad 100644 --- a/.woodpecker/.docs.yml +++ b/.woodpecker/docs.yml @@ -1,3 +1,6 @@ +variables: + - &vlang_image 'chewingbever/vlang:0.3' + platform: 'linux/amd64' branches: exclude: [ main ] @@ -11,7 +14,7 @@ pipeline: - make docs api-docs: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true group: 'generate' commands: diff --git a/.woodpecker/.gitea.yml b/.woodpecker/gitea.yml similarity index 89% rename from .woodpecker/.gitea.yml rename to .woodpecker/gitea.yml index 55f991e..8e3b9d4 100644 --- a/.woodpecker/.gitea.yml +++ b/.woodpecker/gitea.yml @@ -1,3 +1,6 @@ +variables: + - &vlang_image 'chewingbever/vlang:0.3' + platform: 'linux/amd64' branches: [ 'main' ] depends_on: @@ -8,7 +11,7 @@ skip_clone: true pipeline: prepare: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true secrets: [ s3_username, s3_password ] commands: diff --git a/.woodpecker/.lint.yml b/.woodpecker/lint.yml similarity index 73% rename from .woodpecker/.lint.yml rename to .woodpecker/lint.yml index 75a8105..c80ce33 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/lint.yml @@ -1,3 +1,6 @@ +variables: + - &vlang_image 'chewingbever/vlang:0.3' + # These checks already get performed on the feature branches branches: exclude: [ main ] @@ -5,7 +8,7 @@ platform: 'linux/amd64' pipeline: lint: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true commands: - make lint diff --git a/.woodpecker/.man.yml b/.woodpecker/man.yml similarity index 92% rename from .woodpecker/.man.yml rename to .woodpecker/man.yml index 1a30b03..86a1bd8 100644 --- a/.woodpecker/.man.yml +++ b/.woodpecker/man.yml @@ -1,3 +1,6 @@ +variables: + - &vlang_image 'chewingbever/vlang:0.3' + platform: 'linux/amd64' branches: exclude: [ main ] @@ -9,7 +12,7 @@ skip_clone: true pipeline: generate: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true commands: - curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64" diff --git a/.woodpecker/.test.yml b/.woodpecker/test.yml similarity index 80% rename from .woodpecker/.test.yml rename to .woodpecker/test.yml index 6c267fa..08b7534 100644 --- a/.woodpecker/.test.yml +++ b/.woodpecker/test.yml @@ -1,3 +1,6 @@ +variables: + - &vlang_image 'chewingbever/vlang:0.3' + matrix: PLATFORM: - 'linux/amd64' @@ -9,7 +12,7 @@ platform: ${PLATFORM} pipeline: install-modules: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true commands: - export VMODULES=$PWD/.vmodules @@ -18,7 +21,7 @@ pipeline: event: [pull_request] test: - image: 'chewingbever/vlang:0.3' + image: *vlang_image pull: true commands: - export VMODULES=$PWD/.vmodules From 559ef3e505495c9b87b1f6dfb781c7230a7caa1a Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 20:34:02 +0200 Subject: [PATCH 028/140] feat: logs api now also returns id --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79ceed4..b4efd58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Repo POST requests now return information on published package * `api` can no longer be used as a repository name * CLI client now allows setting values to an empty value +* API endpoints now return id of newly created entries ### Removed From ae98c3e717431952eb248cac3fdc99b85e946d7a Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 11 Sep 2022 21:24:29 +0200 Subject: [PATCH 029/140] feat(server): repo POST requests now return information --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4efd58..16ed2a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `api` can no longer be used as a repository name * CLI client now allows setting values to an empty value * API endpoints now return id of newly created entries +* Repo POST requests now return information on published package ### Removed From f34eefd59bfcfdc996f12779cd3101d740037d24 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 1 Oct 2022 16:35:43 +0200 Subject: [PATCH 030/140] fix(server): prevent `api` as a repository name --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16ed2a7..113dc21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * CLI client now allows setting values to an empty value * API endpoints now return id of newly created entries * Repo POST requests now return information on published package +* `api` can no longer be used as a repository name +* CLI client now allows setting values to an empty value ### Removed From 15c2d7274310de6fa77c99933ba6891493ec3e01 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 1 Oct 2022 16:53:16 +0200 Subject: [PATCH 031/140] docs: updated json response types for create routes [CI SKIP] --- docs/api/source/includes/_logs.md | 11 +++++++++++ docs/api/source/includes/_targets.md | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/api/source/includes/_logs.md b/docs/api/source/includes/_logs.md index 2797e60..1c14e71 100644 --- a/docs/api/source/includes/_logs.md +++ b/docs/api/source/includes/_logs.md @@ -112,6 +112,17 @@ id | ID of requested log ## Publish build log +> JSON output format + +```json +{ + "message": "", + "data": { + "id": 15 + } +} +``` +