GreekStapler
/
vieter
forked from vieter-v/vieter
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Merge branch 'dev' into restful-api

main
Jef Roosens 2022-04-06 21:03:52 +02:00
parent 56517c0ff0 4c3f322e6f
commit 6b495b4e6e
Signed by untrusted user: Jef Roosens
GPG Key ID: B75D4F293C7052DB
15 changed files with 303 additions and 299 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
.woodpecker/.build.yml
View File

@ -37,22 +37,6 @@ pipeline:
when:
event: push
cli:
image: 'chewingbever/vlang:latest'
environment:
- LDFLAGS=-static
commands:
- make cli-prod
# Make sure the binary is actually statically built
- readelf -d vieterctl
- du -h vieterctl
- '[ "$(readelf -d vieterctl | grep NEEDED | wc -l)" = 0 ]'
# This removes so much, it's amazing
- strip -s vieterctl
- du -h vieterctl
when:
event: push
upload:
image: 'chewingbever/vlang:latest'
secrets: [ s3_username, s3_password ]
@ -75,20 +59,5 @@ pipeline:
-H "Content-Type: $CONTENT_TYPE"
-H "Authorization: AWS $S3_USERNAME:$SIGNATURE"
https://$URL$OBJ_PATH
# Also update the CLI tool
- export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieterctl-$(echo '${PLATFORM}' | sed 's:/:-:g')"
- export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH"
- export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64`
- >
curl
--silent
-XPUT
-T vieterctl
-H "Host: $URL"
-H "Date: $DATE"
-H "Content-Type: $CONTENT_TYPE"
-H "Authorization: AWS $S3_USERNAME:$SIGNATURE"
https://$URL$OBJ_PATH
when:
event: push

4
.woodpecker/.docker.yml
View File

@ -10,7 +10,7 @@ pipeline:
settings:
repo: chewingbever/vieter
tag: dev
platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
platforms: [ linux/arm64/v8, linux/amd64 ]
build_args_from_env:
- CI_COMMIT_SHA
when:
@ -23,7 +23,7 @@ pipeline:
settings:
repo: chewingbever/vieter
auto_tag: true
platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
platforms: [ linux/arm64/v8, linux/amd64 ]
build_args_from_env:
- CI_COMMIT_SHA
when:

35
Makefile
View File

@ -23,13 +23,7 @@ dvieter: $(SOURCES)
# Run the debug build inside gdb
.PHONY: gdb
gdb: dvieter
VIETER_API_KEY=test \
VIETER_DOWNLOAD_DIR=data/downloads \
VIETER_REPO_DIR=data/repo \
VIETER_PKG_DIR=data/pkgs \
VIETER_LOG_LEVEL=DEBUG \
VIETER_REPOS_FILE=data/repos.json \
gdb --args ./dvieter
gdb --args './dvieter -f vieter.toml server'
# Optimised production build
.PHONY: prod
@ -42,38 +36,15 @@ pvieter: $(SOURCES)
c:
$(V) -o vieter.c $(SRC_DIR)
# Build the CLI tool
.PHONY: cli
cli: dvieterctl
dvieterctl: cli.v
$(V_PATH) -showcc -g -o dvieterctl cli.v
.PHONY: cli-prod
cli-prod: vieterctl
vieterctl: cli.v
cli-prod:
$(V_PATH) -showcc -o vieterctl -prod cli.v
# =====EXECUTION=====
# Run the server in the default 'data' directory
.PHONY: run
run: vieter
VIETER_API_KEY=test \
VIETER_DOWNLOAD_DIR=data/downloads \
VIETER_REPO_DIR=data/repo \
VIETER_PKG_DIR=data/pkgs \
VIETER_LOG_LEVEL=DEBUG \
VIETER_REPOS_FILE=data/repos.json \
./vieter server
./vieter -f vieter.toml server
.PHONY: run-prod
run-prod: prod
VIETER_API_KEY=test \
VIETER_DOWNLOAD_DIR=data/downloads \
VIETER_REPO_DIR=data/repo \
VIETER_PKG_DIR=data/pkgs \
VIETER_LOG_LEVEL=DEBUG \
./pvieter server
./pvieter -f vieter.toml server
# =====OTHER=====
.PHONY: lint

13
PKGBUILD
View File

@ -1,7 +1,7 @@
# Maintainer: Jef Roosens
pkgbase='vieter'
pkgname=('vieter' 'vieterctl')
pkgname='vieter'
pkgver=0.1.0.rc1.r45.g6d3ff8a
pkgrel=1
depends=('glibc' 'openssl' 'libarchive' 'gc')
@ -23,21 +23,12 @@ build() {
# Build the compiler
CFLAGS= make v
# Build the server & the CLI tool
make prod
make cli-prod
}
package_vieter() {
package() {
pkgdesc="Vieter is a lightweight implementation of an Arch repository server."
install -dm755 "$pkgdir/usr/bin"
install -Dm755 "$pkgbase/pvieter" "$pkgdir/usr/bin/vieter"
}
package_vieterctl() {
pkgdesc="Allows you to configure a Vieter server's list of Git repositories."
install -dm755 "$pkgdir/usr/bin"
install -Dm755 "$pkgbase/vieterctl" "$pkgdir/usr/bin/vieterctl"
}

84
cli.v
View File

@ -1,84 +0,0 @@
import os
import toml
import net.http
struct Config {
address string [required]
api_key string [required]
}
fn list(conf Config) ? {
mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ?
req.add_custom_header('X-API-Key', conf.api_key) ?
res := req.do() ?
println(res.text)
}
fn add(conf Config, args []string) ? {
if args.len < 2 {
eprintln('Not enough arguments.')
exit(1)
}
if args.len > 2 {
eprintln('Too many arguments.')
exit(1)
}
mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=${args[0]}&branch=${args[1]}', '') ?
req.add_custom_header('X-API-Key', conf.api_key) ?
res := req.do() ?
println(res.text)
}
fn remove(conf Config, args []string) ? {
if args.len < 2 {
eprintln('Not enough arguments.')
exit(1)
}
if args.len > 2 {
eprintln('Too many arguments.')
exit(1)
}
mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=${args[0]}&branch=${args[1]}', '') ?
req.add_custom_header('X-API-Key', conf.api_key) ?
res := req.do() ?
println(res.text)
}
fn main() {
conf_path := os.expand_tilde_to_home('~/.vieterrc')
if !os.is_file(conf_path) {
exit(1)
}
conf := toml.parse_file(conf_path) ?.reflect<Config>()
args := os.args[1..]
if args.len == 0 {
eprintln('No action provided.')
exit(1)
}
action := args[0]
match action {
'list' { list(conf) ? }
'add' { add(conf, args[1..]) ? }
'remove' { remove(conf, args[1..]) ? }
else {
eprintln("Invalid action '$action'.")
exit(1)
}
}
}

13
src/build.v → src/build/build.v
View File

@ -1,12 +1,11 @@
module main
module build
import docker
import encoding.base64
import time
import json
import server
import env
import net.http
import git
import json
const container_build_dir = '/build'
@ -62,15 +61,13 @@ fn create_build_image() ?string {
return image.id
}
fn build() ? {
conf := env.load<env.BuildConfig>() ?
fn build(conf Config) ? {
// We get the repos list from the Vieter instance
mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ?
req.add_custom_header('X-Api-Key', conf.api_key) ?
res := req.do() ?
repos := json.decode([]server.GitRepo, res.text) ?
repos := json.decode([]git.GitRepo, res.text) ?
// No point in doing work if there's no repos present
if repos.len == 0 {

24
src/build/cli.v 100644
View File

@ -0,0 +1,24 @@
module build
import cli
import env
pub struct Config {
pub:
api_key string
address string
}
// cmd returns the cli submodule that handles the build process
pub fn cmd() cli.Command {
return cli.Command{
name: 'build'
description: 'Run the build process.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
build(conf) ?
}
}
}

68
src/env.v
View File

@ -1,6 +1,7 @@
module env
import os
import toml
// The prefix that every environment variable should have
const prefix = 'VIETER_'
@ -9,32 +10,15 @@ const prefix = 'VIETER_'
// instead
const file_suffix = '_FILE'
pub struct ServerConfig {
pub:
log_level string [default: WARN]
log_file string [default: 'vieter.log']
pkg_dir string
download_dir string
api_key string
repo_dir string
repos_file string
}
pub struct BuildConfig {
pub:
api_key string
address string
}
fn get_env_var(field_name string) ?string {
env_var_name := '$env.prefix$field_name.to_upper()'
env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix'
env_var := os.getenv(env_var_name)
env_file := os.getenv(env_file_name)
// If both aren't set, we report them missing
// If both are missing, we return an empty string
if env_var == '' && env_file == '' {
return error('Either $env_var_name or $env_file_name is required.')
return ''
}
// If they're both set, we report a conflict
@ -56,30 +40,42 @@ fn get_env_var(field_name string) ?string {
}
}
// load<T> attempts to create the given type from environment variables. For
// each field, the corresponding env var is its name in uppercase prepended
// with the hardcoded prefix. If this one isn't present, it looks for the env
// var with the file_suffix suffix.
pub fn load<T>() ?T {
res := T{}
// load<T> attempts to create an object of type T from the given path to a toml
// file & environment variables. For each field, it will select either a value
// given from an environment variable, a value defined in the config file or a
// configured default if present, in that order.
pub fn load<T>(path string) ?T {
mut res := T{}
if os.exists(path) {
// We don't use reflect here because reflect also sets any fields not
// in the toml back to their zero value, which we don't want
doc := toml.parse_file(path) ?
$for field in T.fields {
res.$(field.name) = get_env_var(field.name) or {
// We use the default instead, if it's present
mut default := ''
s := doc.value(field.name)
for attr in field.attrs {
if attr.starts_with('default: ') {
default = attr[9..]
break
// We currently only support strings
if s.type_name() == 'string' {
res.$(field.name) = s.string()
}
}
}
if default == '' {
return err
}
$for field in T.fields {
$if field.typ is string {
env_value := get_env_var(field.name) ?
default
// The value of the env var will always be chosen over the config
// file
if env_value != '' {
res.$(field.name) = env_value
}
// If there's no value from the toml file either, we try to find a
// default value
else if res.$(field.name) == '' {
return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.")
}
}
}
return res

83
src/git/cli.v 100644
View File

@ -0,0 +1,83 @@
module git
import cli
import env
import net.http
struct Config {
address string [required]
api_key string [required]
}
// cmd returns the cli submodule that handles the repos API interaction
pub fn cmd() cli.Command {
return cli.Command{
name: 'repos'
description: 'Interact with the repos API.'
commands: [
cli.Command{
name: 'list'
description: 'List the current repos.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
list(conf) ?
}
},
cli.Command{
name: 'add'
required_args: 2
usage: 'url branch'
description: 'Add a new repository.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
add(conf, cmd.args[0], cmd.args[1]) ?
}
},
cli.Command{
name: 'remove'
required_args: 2
usage: 'url branch'
description: 'Remove a repository.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
remove(conf, cmd.args[0], cmd.args[1]) ?
}
},
]
}
}
fn list(conf Config) ? {
mut req := http.new_request(http.Method.get, '$conf.address/api/repos', '') ?
req.add_custom_header('X-API-Key', conf.api_key) ?
res := req.do() ?
println(res.text)
}
fn add(conf Config, url string, branch string) ? {
mut req := http.new_request(http.Method.post, '$conf.address/api/repos?url=$url&branch=$branch',
'') ?
req.add_custom_header('X-API-Key', conf.api_key) ?
res := req.do() ?
println(res.text)
}
fn remove(conf Config, url string, branch string) ? {
mut req := http.new_request(http.Method.delete, '$conf.address/api/repos?url=$url&branch=$branch',
'') ?
req.add_custom_header('X-API-Key', conf.api_key) ?
res := req.do() ?
println(res.text)
}

74
src/git/git.v 100644
View File

@ -0,0 +1,74 @@
module git
import os
import json
pub struct GitRepo {
pub mut:
// URL of the Git repository
url string
// Branch of the Git repository to use
branch string
// On which architectures the package is allowed to be built. In reality,
// this controls which builders will periodically build the image.
arch []string
}
pub fn (mut r GitRepo) patch_from_params(params map[string]string) {
$for field in GitRepo.fields {
if field.name in params {
$if field.typ is string {
r.$(field.name) = params[field.name]
// This specific type check is needed for the compiler to ensure
// our types are correct
} $else $if field.typ is []string {
r.$(field.name) = params[field.name].split(',')
}
}
}
}
pub fn read_repos(path string) ?map[string]GitRepo {
if !os.exists(path) {
mut f := os.create(path) ?
defer {
f.close()
}
f.write_string('{}') ?
return {}
}
content := os.read_file(path) ?
res := json.decode(map[string]GitRepo, content) ?
return res
}
pub fn write_repos(path string, repos &map[string]GitRepo) ? {
mut f := os.create(path) ?
defer {
f.close()
}
value := json.encode(repos)
f.write_string(value) ?
}
pub fn repo_from_params(params map[string]string) ?GitRepo {
mut repo := GitRepo{}
// If we're creating a new GitRepo, we want all fields to be present before
// "patching".
$for field in GitRepo.fields {
if field.name !in params {
return error('Missing parameter: ${field.name}.')
}
}
repo.patch_from_params(params)
return repo
}

32
src/main.v
View File

@ -2,16 +2,32 @@ module main
import os
import server
import util
import cli
import build
import git
fn main() {
if os.args.len == 1 {
util.exit_with_message(1, 'No action provided.')
mut app := cli.Command{
name: 'vieter'
description: 'Vieter is a lightweight implementation of an Arch repository server.'
version: '0.1.0'
flags: [
cli.Flag{
flag: cli.FlagType.string
name: 'config-file'
abbrev: 'f'
description: 'Location of Vieter config file; defaults to ~/.vieterrc.'
global: true
default_value: [os.expand_tilde_to_home('~/.vieterrc')]
},
]
commands: [
server.cmd(),
build.cmd(),
git.cmd(),
]
}
match os.args[1] {
'server' { server.server() ? }
'build' { build() ? }
else { util.exit_with_message(1, 'Unknown action: ${os.args[1]}') }
}
app.setup()
app.parse(os.args)
}

29
src/server/cli.v 100644
View File

@ -0,0 +1,29 @@
module server
import cli
import env
struct Config {
pub:
log_level string = 'WARN'
log_file string = 'vieter.log'
pkg_dir string
download_dir string
api_key string
repo_dir string
repos_file string
}
// cmd returns the cli submodule that handles starting the server
pub fn cmd() cli.Command {
return cli.Command{
name: 'server'
description: 'Start the Vieter server.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
server(conf) ?
}
}
}

92
src/server/git.v
View File

@ -1,79 +1,11 @@
module server
import web
import os
import json
import rand
import git
import net.http
import rand
pub struct GitRepo {
pub mut:
// URL of the Git repository
url string
// Branch of the Git repository to use
branch string
// On which architectures the package is allowed to be built. In reality,
// this controls which builders will periodically build the image.
arch []string
}
fn (mut r GitRepo) patch_from_params(params map[string]string) {
$for field in GitRepo.fields {
if field.name in params {
$if field.typ is string {
r.$(field.name) = params[field.name]
// This specific type check is needed for the compiler to ensure
// our types are correct
} $else $if field.typ is []string {
r.$(field.name) = params[field.name].split(',')
}
}
}
}
fn repo_from_params(params map[string]string) ?GitRepo {
mut repo := GitRepo{}
// If we're creating a new GitRepo, we want all fields to be present before
// "patching".
$for field in GitRepo.fields {
if field.name !in params {
return error('Missing parameter: ${field.name}.')
}
}
repo.patch_from_params(params)
return repo
}
fn read_repos(path string) ?map[string]GitRepo {
if !os.exists(path) {
mut f := os.create(path) ?
defer {
f.close()
}
f.write_string('{}') ?
return {}
}
content := os.read_file(path) ?
res := json.decode(map[string]GitRepo, content) ?
return res
}
fn write_repos(path string, repos &map[string]GitRepo) ? {
mut f := os.create(path) ?
defer {
f.close()
}
value := json.encode(repos)
f.write_string(value) ?
}
const repos_file = 'repos.json'
['/api/repos'; get]
fn (mut app App) get_repos() web.Result {
@ -82,7 +14,7 @@ fn (mut app App) get_repos() web.Result {
}
repos := rlock app.git_mutex {
read_repos(app.conf.repos_file) or {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
return app.status(http.Status.internal_server_error)
@ -99,7 +31,7 @@ fn (mut app App) get_single_repo(id string) web.Result {
}
repos := rlock app.git_mutex {
read_repos(app.conf.repos_file) or {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
return app.status(http.Status.internal_server_error)
@ -121,14 +53,14 @@ fn (mut app App) post_repo() web.Result {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
new_repo := repo_from_params(app.query) or {
new_repo := git.repo_from_params(app.query) or {
return app.json(http.Status.bad_request, new_response(err.msg))
}
id := rand.uuid_v4()
mut repos := rlock app.git_mutex {
read_repos(app.conf.repos_file) or {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
return app.status(http.Status.internal_server_error)
@ -145,7 +77,7 @@ fn (mut app App) post_repo() web.Result {
repos[id] = new_repo
lock app.git_mutex {
write_repos(app.conf.repos_file, &repos) or {
git.write_repos(app.conf.repos_file, &repos) or {
return app.status(http.Status.internal_server_error)
}
}
@ -160,7 +92,7 @@ fn (mut app App) delete_repo(id string) web.Result {
}
mut repos := rlock app.git_mutex {
read_repos(app.conf.repos_file) or {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
return app.status(http.Status.internal_server_error)
@ -174,7 +106,7 @@ fn (mut app App) delete_repo(id string) web.Result {
repos.delete(id)
lock app.git_mutex {
write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) }
git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) }
}
return app.json(http.Status.ok, new_response('Repo removed successfully.'))
@ -187,7 +119,7 @@ fn (mut app App) patch_repo(id string) web.Result {
}
mut repos := rlock app.git_mutex {
read_repos(app.conf.repos_file) or {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
return app.status(http.Status.internal_server_error)
@ -201,7 +133,7 @@ fn (mut app App) patch_repo(id string) web.Result {
repos[id].patch_from_params(app.query)
lock app.git_mutex {
write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) }
git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) }
}
return app.json(http.Status.ok, new_response('Repo updated successfully.'))

7
src/server/server.v
View File

@ -4,7 +4,6 @@ import web
import os
import log
import repo
import env
import util
const port = 8000
@ -12,7 +11,7 @@ const port = 8000
struct App {
web.Context
pub:
conf env.ServerConfig [required; web_global]
conf Config [required; web_global]
pub mut:
repo repo.Repo [required; web_global]
// This is used to claim the file lock on the repos file
@ -20,9 +19,7 @@ pub mut:
}
// server starts the web server & starts listening for requests
pub fn server() ? {
conf := env.load<env.ServerConfig>() ?
pub fn server(conf Config) ? {
// Configure logger
log_level := log.level_from_tag(conf.log_level) or {
util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')

9
vieter.toml 100644
View File

@ -0,0 +1,9 @@
# This file contains settings used during development
api_key = "test"
download_dir = "data/downloads"
repo_dir = "data/repo"
pkg_dir = "data/pkgs"
# log_level = "DEBUG"
repos_file = "data/repos.json"
address = "http://localhost:8000"