LordMZTE
/
vieter
forked from vieter-v/vieter
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Merge pull request 'Sqlite backend & simplifying of config variables' (#158) from Chewing_Bever/vieter:sqlite-backend into dev 

Reviewed-on: vieter/vieter#158
dev
Jef Roosens 2022-05-05 23:18:43 +02:00
parent 03318586ed 8c5652c230
commit 1a076a7a8c
26 changed files with 335 additions and 324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.woodpecker/.arch.yml
View File

@ -25,6 +25,8 @@ pipeline:
# inside the repo
- curl -OL https://git.rustybever.be/Chewing_Bever/vieter/raw/branch/dev/PKGBUILD
- makepkg -s --noconfirm --needed
when:
event: push
publish:
image: 'curlimages/curl'
@ -33,3 +35,5 @@ pipeline:
- 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done'
secrets:
- vieter_api_key
when:
event: push

18
.woodpecker/.build.yml
View File

@ -1,11 +1,8 @@
matrix:
PLATFORM:
- linux/amd64
- linux/arm64
# I just don't have a performant enough runner for this platform
# - linux/arm/v7
- 'linux/amd64'
- 'linux/arm64'
# These checks already get performed on the feature branches
platform: ${PLATFORM}
pipeline:
@ -15,15 +12,15 @@ pipeline:
commands:
- make
when:
event: push
event: [push, pull_request]
branch:
exclude: [main, dev]
exclude: [main]
prod:
image: 'chewingbever/vlang:latest'
pull: true
environment:
- LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static
- LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static
commands:
# Apparently this -D is *very* important
- CFLAGS='-DGC_THREADS=1' make prod
@ -35,7 +32,7 @@ pipeline:
- strip -s pvieter
- du -h pvieter
when:
event: push
event: [push, pull_request]
upload:
image: 'chewingbever/vlang:latest'
@ -52,6 +49,7 @@ pipeline:
- >
curl
--silent
--fail
-XPUT
-T pvieter
-H "Host: $URL"
@ -60,4 +58,4 @@ pipeline:
-H "Authorization: AWS $S3_USERNAME:$SIGNATURE"
https://$URL$OBJ_PATH
when:
event: push
event: [push, pull_request]

29
.woodpecker/.build_experimental.yml
View File

@ -1,29 +0,0 @@
# These builds are not important for the project, but might be valuable for
# fixing bugs in the V compiler.
platform: linux/amd64
branches:
exclude: [master, dev]
pipeline:
autofree:
image: 'chewingbever/vlang:latest'
pull: true
group: 'build'
commands:
- make autofree
- readelf -d afvieter
- du -h afvieter
when:
event: push
skip-unused:
image: 'chewingbever/vlang:latest'
pull: true
group: 'build'
commands:
- make skip-unused
- readelf -d suvieter
- du -h suvieter
when:
event: push

4
.woodpecker/.deploy.yml
View File

@ -1,4 +1,4 @@
branches: 'dev'
branches: [ 'dev' ]
platform: 'linux/amd64'
depends_on:
- 'docker'
@ -14,3 +14,5 @@ pipeline:
commands:
- 'curl -XPOST -s --fail $WEBHOOK_APP'
- 'curl -XPOST -s --fail $WEBHOOK_CRON'
when:
event: push

28
.woodpecker/.docker.yml
View File

@ -1,30 +1,34 @@
branches: [main, dev]
platform: linux/amd64
platform: 'linux/amd64'
depends_on:
- build
pipeline:
dev:
image: woodpeckerci/plugin-docker-buildx
secrets: [ docker_username, docker_password ]
image: 'woodpeckerci/plugin-docker-buildx'
secrets:
- 'docker_username'
- 'docker_password'
settings:
repo: chewingbever/vieter
tag: dev
platforms: [ linux/arm64/v8, linux/amd64 ]
repo: 'chewingbever/vieter'
tag: 'dev'
platforms: [ 'linux/arm64/v8', 'linux/amd64' ]
build_args_from_env:
- CI_COMMIT_SHA
- 'CI_COMMIT_SHA'
when:
event: push
branch: dev
release:
image: woodpeckerci/plugin-docker-buildx
secrets: [ docker_username, docker_password ]
image: 'woodpeckerci/plugin-docker-buildx'
secrets:
- 'docker_username'
- 'docker_password'
settings:
repo: chewingbever/vieter
repo: 'chewingbever/vieter'
auto_tag: true
platforms: [ linux/arm64/v8, linux/amd64 ]
platforms: [ 'linux/arm64/v8', 'linux/amd64' ]
build_args_from_env:
- CI_COMMIT_SHA
- 'CI_COMMIT_SHA'
when:
event: tag

5
.woodpecker/.gitea.yml
View File

@ -1,6 +1,5 @@
# Yeah so this only works on tags so we'll worry about this later
platform: linux/amd64
branches: main
platform: 'linux/amd64'
branches: [ 'main' ]
depends_on:
- build

6
.woodpecker/.lint.yml
View File

@ -1,7 +1,7 @@
# These checks already get performed on the feature branches
branches:
exclude: [ main, dev ]
platform: linux/amd64
exclude: [ main ]
platform: 'linux/amd64'
pipeline:
lint:
@ -9,3 +9,5 @@ pipeline:
pull: true
commands:
- make lint
when:
event: [ push, pull_request ]

8
.woodpecker/.test.yml
View File

@ -1,10 +1,10 @@
matrix:
PLATFORM:
- linux/amd64
- linux/arm64
- 'linux/amd64'
- 'linux/arm64'
branches:
exclude: [main, dev]
exclude: [ main ]
platform: ${PLATFORM}
pipeline:
@ -14,4 +14,4 @@ pipeline:
commands:
- make test
when:
event: push
event: [push, pull_request]

3
CHANGELOG.md
View File

@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
* Switched from compiler fork to fully vanilla compiler mirror
* `download_dir`, `repos_file` & `repos_dir` config values have been replaced
with `data_dir`
* Storage of metadata (e.g. Git repositories) is now done using Sqlite
### Added

8
Dockerfile
View File

@ -23,7 +23,7 @@ RUN if [ -n "${CI_COMMIT_SHA}" ]; then \
"https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \
chmod +x vieter ; \
else \
LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' make prod && \
LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static' make prod && \
mv pvieter vieter ; \
fi
@ -31,10 +31,8 @@ RUN if [ -n "${CI_COMMIT_SHA}" ]; then \
FROM busybox:1.35.0
ENV PATH=/bin \
VIETER_REPOS_DIR=/data/repos \
VIETER_PKG_DIR=/data/pkgs \
VIETER_DOWNLOAD_DIR=/data/downloads \
VIETER_REPOS_FILE=/data/repos.json
VIETER_DATA_DIR=/data \
VIETER_PKG_DIR=/data/pkgs
COPY --from=builder /app/dumb-init /app/vieter /bin/

25
docs/content/configuration.md
View File

@ -35,18 +35,10 @@ passed to them. Each mode requires a different configuration.
* `log_file`: log file to write logs to. Defaults to `vieter.log` in the
current directory.
* `pkg_dir`: where Vieter should store the actual package archives.
* `download_dir`: where Vieter should initially download uploaded files.
* `data_dir`: where Vieter stores the repositories, log file & database.
* `api_key`: the API key to use when authenticating requests.
* `repo_dir`: where Vieter should store the contents of the repository.
* `repos_file`: JSON file where the list of Git repositories is saved
* `default_arch`: architecture to always add packages of arch `any` to.
{{< hint info >}}
**Note**
Because Vieter hard links files between `download_dir` & `pkg_dir`, they need
to be on the same file system.
{{< /hint >}}
### Builder
* `api_key`: the API key to use when authenticating requests.
@ -62,3 +54,18 @@ to be on the same file system.
* `api_key`: the API key to use when authenticating requests.
* `address`: Base your URL of your Vieter instance, e.g. https://example.com
### Cron
* `log_level`: defines how much logs to show. Valid values are one of `FATAL`,
`ERROR`, `WARN`, `INFO` or `DEBUG`. Defaults to `WARN`
* `api_key`: the API key to use when authenticating requests.
* `address`: Base your URL of your Vieter instance, e.g. https://example.com.
This *must* be the publicly facing URL of your Vieter instance.
* `data_dir`: where Vieter stores the log file.
* `base_image`: Docker image from which to create the builder images.
* `max_concurrent_builds`: amount of builds to run at once.
* `api_update_frequency`: how frequenty to check for changes in the repo list.
* `image_rebuild+frequency`: how frequently to rebuild the builder image
* `global_schedule`: cron schedule to use for any repo without an individual
schedule

7
src/build/build.v
View File

@ -5,6 +5,7 @@ import encoding.base64
import time
import git
import os
import db
const container_build_dir = '/build'
@ -75,7 +76,7 @@ pub fn create_build_image(base_image string) ?string {
// build_repo builds, packages & publishes a given Arch package based on the
// provided GitRepo. The base image ID should be of an image previously created
// by create_build_image.
pub fn build_repo(address string, api_key string, base_image_id string, repo &git.GitRepo) ? {
pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ? {
build_arch := os.uname().machine
// TODO what to do with PKGBUILDs that build multiple packages?
@ -125,11 +126,11 @@ fn build(conf Config) ? {
build_arch := os.uname().machine
// We get the repos map from the Vieter instance
repos_map := git.get_repos(conf.address, conf.api_key) ?
repos := git.get_repos(conf.address, conf.api_key) ?
// We filter out any repos that aren't allowed to be built on this
// architecture
filtered_repos := repos_map.keys().map(repos_map[it]).filter(it.arch.contains(build_arch))
filtered_repos := repos.filter(it.arch.map(it.value).contains(build_arch))
// No point in doing work if there's no repos present
if filtered_repos.len == 0 {

2
src/cron/cli.v
View File

@ -6,9 +6,9 @@ import env
struct Config {
pub:
log_level string = 'WARN'
log_file string = 'vieter.log'
api_key string
address string
data_dir string
base_image string = 'archlinux:base-devel'
max_concurrent_builds int = 1
api_update_frequency int = 15

6
src/cron/cron.v
View File

@ -3,6 +3,9 @@ module cron
import log
import cron.daemon
import cron.expression
import os
const log_file_name = 'vieter.cron.log'
// cron starts a cron daemon & starts periodically scheduling builds.
pub fn cron(conf Config) ? {
@ -15,7 +18,8 @@ pub fn cron(conf Config) ? {
level: log_level
}
logger.set_full_logpath(conf.log_file)
log_file := os.join_path_single(conf.data_dir, cron.log_file_name)
logger.set_full_logpath(log_file)
logger.log_to_console_too()
ce := expression.parse_expression(conf.global_schedule) or {

10
src/cron/daemon/build.v
View File

@ -4,11 +4,11 @@ import time
import sync.stdatomic
import build
const build_empty = 0
const build_running = 1
const build_done = 2
const (
build_empty = 0
build_running = 1
build_done = 2
)
// clean_finished_builds removes finished builds from the build slots & returns
// them.

25
src/cron/daemon/daemon.v
View File

@ -8,17 +8,19 @@ import cron.expression { CronExpression, parse_expression }
import math
import build
import docker
import db
const (
// How many seconds to wait before retrying to update API if failed
const api_update_retry_timeout = 5
api_update_retry_timeout = 5
// How many seconds to wait before retrying to rebuild image if failed
const rebuild_base_image_retry_timout = 30
rebuild_base_image_retry_timout = 30
)
struct ScheduledBuild {
pub:
repo_id string
repo git.GitRepo
repo db.GitRepo
timestamp time.Time
}
@ -37,7 +39,7 @@ mut:
api_update_frequency int
image_rebuild_frequency int
// Repos currently loaded from API.
repos_map map[string]git.GitRepo
repos []db.GitRepo
// At what point to update the list of repositories.
api_update_timestamp time.Time
image_build_timestamp time.Time
@ -90,7 +92,7 @@ pub fn (mut d Daemon) run() {
// haven't been renewed.
else {
for sb in finished_builds {
d.schedule_build(sb.repo_id, sb.repo)
d.schedule_build(sb.repo)
}
}
@ -149,11 +151,11 @@ pub fn (mut d Daemon) run() {
}
// schedule_build adds the next occurence of the given repo build to the queue.
fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) {
fn (mut d Daemon) schedule_build(repo db.GitRepo) {
ce := if repo.schedule != '' {
parse_expression(repo.schedule) or {
// TODO This shouldn't return an error if the expression is empty.
d.lerror("Error while parsing cron expression '$repo.schedule' ($repo_id): $err.msg()")
d.lerror("Error while parsing cron expression '$repo.schedule' (id $repo.id): $err.msg()")
d.global_schedule
}
@ -168,7 +170,6 @@ fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) {
}
d.queue.insert(ScheduledBuild{
repo_id: repo_id
repo: repo
timestamp: timestamp
})
@ -186,7 +187,7 @@ fn (mut d Daemon) renew_repos() {
return
}
d.repos_map = new_repos.move()
d.repos = new_repos
d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency)
}
@ -224,8 +225,8 @@ fn (mut d Daemon) renew_queue() {
// For each repository in repos_map, parse their cron expression (or use
// the default one if not present) & add them to the queue
for id, repo in d.repos_map {
d.schedule_build(id, repo)
for repo in d.repos {
d.schedule_build(repo)
}
}

20
src/db/db.v 100644
View File

@ -0,0 +1,20 @@
module db
import sqlite
struct VieterDb {
conn sqlite.DB
}
// init initializes a database & adds the correct tables.
pub fn init(db_path string) ?VieterDb {
conn := sqlite.connect(db_path) ?
sql conn {
create table GitRepo
}
return VieterDb{
conn: conn
}
}

154
src/db/git.v 100644
View File

@ -0,0 +1,154 @@
module db
pub struct GitRepoArch {
pub:
id int [primary; sql: serial]
repo_id int [nonull]
value string [nonull]
}
// str returns a string representation.
pub fn (gra &GitRepoArch) str() string {
return gra.value
}
pub struct GitRepo {
pub mut:
id int [optional; primary; sql: serial]
// URL of the Git repository
url string [nonull]
// Branch of the Git repository to use
branch string [nonull]
// Which repo the builder should publish packages to
repo string [nonull]
// Cron schedule describing how frequently to build the repo.
schedule string [optional]
// On which architectures the package is allowed to be built. In reality,
// this controls which builders will periodically build the image.
arch []GitRepoArch [fkey: 'repo_id']
}
// str returns a string representation.
pub fn (gr &GitRepo) str() string {
mut parts := [
'id: $gr.id',
'url: $gr.url',
'branch: $gr.branch',
'repo: $gr.repo',
'schedule: $gr.schedule',
'arch: ${gr.arch.map(it.value).join(', ')}',
]
str := parts.join('\n')
return str
}
// patch_from_params patches a GitRepo from a map[string]string, usually
// provided from a web.App's params
pub fn (mut r GitRepo) patch_from_params(params map[string]string) {
$for field in GitRepo.fields {
if field.name in params {
$if field.typ is string {
r.$(field.name) = params[field.name]
// This specific type check is needed for the compiler to ensure
// our types are correct
} $else $if field.typ is []GitRepoArch {
r.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it })
}
}
}
}
// git_repo_from_params creates a GitRepo from a map[string]string, usually
// provided from a web.App's params
pub fn git_repo_from_params(params map[string]string) ?GitRepo {
mut repo := GitRepo{}
// If we're creating a new GitRepo, we want all fields to be present before
// "patching".
$for field in GitRepo.fields {
if field.name !in params && !field.attrs.contains('optional') {
return error('Missing parameter: ${field.name}.')
}
}
repo.patch_from_params(params)
return repo
}
// get_git_repos returns all GitRepo's in the database.
pub fn (db &VieterDb) get_git_repos() []GitRepo {
res := sql db.conn {
select from GitRepo order by id
}
return res
}
// get_git_repo tries to return a specific GitRepo.
pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo {
res := sql db.conn {
select from GitRepo where id == repo_id
}
// If a select statement fails, it returns a zeroed object. By
// checking one of the required fields, we can see whether the query
// returned a result or not.
if res.url == '' {
return none
}
return res
}
// add_git_repo inserts the given GitRepo into the database.
pub fn (db &VieterDb) add_git_repo(repo GitRepo) {
sql db.conn {
insert repo into GitRepo
}
}
// delete_git_repo deletes the repo with the given ID from the database.
pub fn (db &VieterDb) delete_git_repo(repo_id int) {
sql db.conn {
delete from GitRepo where id == repo_id
delete from GitRepoArch where repo_id == repo_id
}
}
// update_git_repo updates any non-array values for a given GitRepo.
pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) {
mut values := []string{}
// TODO does this allow for SQL injection?
$for field in GitRepo.fields {
if field.name in params {
// Any fields that are array types require their own update method
$if field.typ is string {
values << "$field.name = '${params[field.name]}'"
}
}
}
values_str := values.join(', ')
query := 'update GitRepo set $values_str where id == $repo_id'
db.conn.exec_none(query)
}
// update_git_repo_archs updates a given GitRepo's arch value.
pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) {
archs_with_id := archs.map(GitRepoArch{
...it
repo_id: repo_id
})
sql db.conn {
delete from GitRepoArch where repo_id == repo_id
}
for arch in archs_with_id {
sql db.conn {
insert arch into GitRepoArch
}
}
}

56
src/git/cli.v
View File

@ -116,34 +116,13 @@ pub fn cmd() cli.Command {
// get_repo_by_prefix tries to find the repo with the given prefix in its
// ID. If multiple or none are found, an error is raised.
fn get_repo_by_prefix(conf Config, id_prefix string) ?(string, GitRepo) {
repos := get_repos(conf.address, conf.api_key) ?
mut res := map[string]GitRepo{}
for id, repo in repos {
if id.starts_with(id_prefix) {
res[id] = repo
}
}
if res.len == 0 {
return error('No repo found for given prefix.')
}
if res.len > 1 {
return error('Multiple repos found for given prefix.')
}
return res.keys()[0], res[res.keys()[0]]
}
// list prints out a list of all repositories.
fn list(conf Config) ? {
repos := get_repos(conf.address, conf.api_key) ?
for id, details in repos {
println('${id[..8]}\t$details.url\t$details.branch\t$details.repo')
for repo in repos {
println('$repo.id\t$repo.url\t$repo.branch\t$repo.repo')
}
}
@ -155,15 +134,18 @@ fn add(conf Config, url string, branch string, repo string) ? {
}
// remove removes a repository from the server's list.
fn remove(conf Config, id_prefix string) ? {
id, _ := get_repo_by_prefix(conf, id_prefix) ?
res := remove_repo(conf.address, conf.api_key, id) ?
fn remove(conf Config, id string) ? {
// id, _ := get_repo_by_prefix(conf, id_prefix) ?
id_int := id.int()
if id_int != 0 {
res := remove_repo(conf.address, conf.api_key, id_int) ?
println(res.message)
}
}
// patch patches a given repository with the provided params.
fn patch(conf Config, id_prefix string, params map[string]string) ? {
fn patch(conf Config, id string, params map[string]string) ? {
// We check the cron expression first because it's useless to send an
// invalid one to the server.
if 'schedule' in params && params['schedule'] != '' {
@ -172,20 +154,22 @@ fn patch(conf Config, id_prefix string, params map[string]string) ? {
}
}
id, _ := get_repo_by_prefix(conf, id_prefix) ?
res := patch_repo(conf.address, conf.api_key, id, params) ?
id_int := id.int()
if id_int != 0 {
res := patch_repo(conf.address, conf.api_key, id_int, params) ?
println(res.message)
}
}
// info shows detailed information for a given repo.
fn info(conf Config, id_prefix string) ? {
id, repo := get_repo_by_prefix(conf, id_prefix) ?
fn info(conf Config, id string) ? {
id_int := id.int()
println('id: $id')
if id_int == 0 {
return
}
$for field in GitRepo.fields {
val := repo.$(field.name)
println('$field.name: $val')
}
repo := get_repo(conf.address, conf.api_key, id_int) ?
println(repo)
}

17
src/git/client.v
View File

@ -3,6 +3,7 @@ module git
import json
import response { Response }
import net.http
import db
// send_request<T> is a convenience method for sending requests to the repos
// API. It mostly does string manipulation to create a query string containing
@ -26,8 +27,16 @@ fn send_request<T>(method http.Method, address string, url string, api_key strin
}
// get_repos returns the current list of repos.
pub fn get_repos(address string, api_key string) ?map[string]GitRepo {
data := send_request<map[string]GitRepo>(http.Method.get, address, '/api/repos', api_key,
pub fn get_repos(address string, api_key string) ?[]db.GitRepo {
data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', api_key,
{}) ?
return data.data
}
// get_repo returns the repo for a specific ID.
pub fn get_repo(address string, api_key string, id int) ?db.GitRepo {
data := send_request<db.GitRepo>(http.Method.get, address, '/api/repos/$id', api_key,
{}) ?
return data.data
@ -51,7 +60,7 @@ pub fn add_repo(address string, api_key string, url string, branch string, repo
}
// remove_repo removes the repo with the given ID from the server.
pub fn remove_repo(address string, api_key string, id string) ?Response<string> {
pub fn remove_repo(address string, api_key string, id int) ?Response<string> {
data := send_request<string>(http.Method.delete, address, '/api/repos/$id', api_key,
{}) ?
@ -60,7 +69,7 @@ pub fn remove_repo(address string, api_key string, id string) ?Response<string>
// patch_repo sends a PATCH request to the given repo with the params as
// payload.
pub fn patch_repo(address string, api_key string, id string, params map[string]string) ?Response<string> {
pub fn patch_repo(address string, api_key string, id int, params map[string]string) ?Response<string> {
data := send_request<string>(http.Method.patch, address, '/api/repos/$id', api_key,
params) ?

84
src/git/git.v
View File

@ -1,84 +0,0 @@
module git
import os
import json
pub struct GitRepo {
pub mut:
// URL of the Git repository
url string
// Branch of the Git repository to use
branch string
// On which architectures the package is allowed to be built. In reality,
// this controls which builders will periodically build the image.
arch []string
// Which repo the builder should publish packages to
repo string
// Cron schedule describing how frequently to build the repo.
schedule string [optional]
}
// patch_from_params patches a GitRepo from a map[string]string, usually
// provided from a web.App's params
pub fn (mut r GitRepo) patch_from_params(params map[string]string) {
$for field in GitRepo.fields {
if field.name in params {
$if field.typ is string {
r.$(field.name) = params[field.name]
// This specific type check is needed for the compiler to ensure
// our types are correct
} $else $if field.typ is []string {
r.$(field.name) = params[field.name].split(',')
}
}
}
}
// read_repos reads the provided path & parses it into a map of GitRepo's.
pub fn read_repos(path string) ?map[string]GitRepo {
if !os.exists(path) {
mut f := os.create(path) ?
defer {
f.close()
}
f.write_string('{}') ?
return {}
}
content := os.read_file(path) ?
res := json.decode(map[string]GitRepo, content) ?
return res
}
// write_repos writes a map of GitRepo's back to disk given the provided path.
pub fn write_repos(path string, repos &map[string]GitRepo) ? {
mut f := os.create(path) ?
defer {
f.close()
}
value := json.encode(repos)
f.write_string(value) ?
}
// repo_from_params creates a GitRepo from a map[string]string, usually
// provided from a web.App's params
pub fn repo_from_params(params map[string]string) ?GitRepo {
mut repo := GitRepo{}
// If we're creating a new GitRepo, we want all fields to be present before
// "patching".
$for field in GitRepo.fields {
if field.name !in params && !field.attrs.contains('optional') {
return error('Missing parameter: ${field.name}.')
}
}
repo.patch_from_params(params)
return repo
}

5
src/server/cli.v
View File

@ -6,12 +6,9 @@ import env
struct Config {
pub:
log_level string = 'WARN'
log_file string = 'vieter.log'
pkg_dir string
download_dir string
data_dir string
api_key string
repos_dir string
repos_file string
default_arch string
}

96
src/server/git.v
View File

@ -1,12 +1,9 @@
module server
import web
import git
import net.http
import rand
import response { new_data_response, new_response }
const repos_file = 'repos.json'
import db
// get_repos returns the current list of repos.
['/api/repos'; get]
@ -15,37 +12,19 @@ fn (mut app App) get_repos() web.Result {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
repos := rlock app.git_mutex {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file: $err.msg()')
return app.status(http.Status.internal_server_error)
}
}
repos := app.db.get_git_repos()
return app.json(http.Status.ok, new_data_response(repos))
}
// get_single_repo returns the information for a single repo.
['/api/repos/:id'; get]
fn (mut app App) get_single_repo(id string) web.Result {
fn (mut app App) get_single_repo(id int) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
repos := rlock app.git_mutex {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
return app.status(http.Status.internal_server_error)
}
}
if id !in repos {
return app.not_found()
}
repo := repos[id]
repo := app.db.get_git_repo(id) or { return app.not_found() }
return app.json(http.Status.ok, new_data_response(repo))
}
@ -65,89 +44,40 @@ fn (mut app App) post_repo() web.Result {
params['arch'] = app.conf.default_arch
}
new_repo := git.repo_from_params(params) or {
new_repo := db.git_repo_from_params(params) or {
return app.json(http.Status.bad_request, new_response(err.msg()))
}
id := rand.uuid_v4()
mut repos := rlock app.git_mutex {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
return app.status(http.Status.internal_server_error)
}
}
// We need to check for duplicates
for _, repo in repos {
if repo == new_repo {
return app.json(http.Status.bad_request, new_response('Duplicate repository.'))
}
}
repos[id] = new_repo
lock app.git_mutex {
git.write_repos(app.conf.repos_file, &repos) or {
return app.status(http.Status.internal_server_error)
}
}
app.db.add_git_repo(new_repo)
return app.json(http.Status.ok, new_response('Repo added successfully.'))
}
// delete_repo removes a given repo from the server's list.
['/api/repos/:id'; delete]
fn (mut app App) delete_repo(id string) web.Result {
fn (mut app App) delete_repo(id int) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
mut repos := rlock app.git_mutex {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
return app.status(http.Status.internal_server_error)
}
}
if id !in repos {
return app.not_found()
}
repos.delete(id)
lock app.git_mutex {
git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) }
}
app.db.delete_git_repo(id)
return app.json(http.Status.ok, new_response('Repo removed successfully.'))
}
// patch_repo updates a repo's data with the given query params.
['/api/repos/:id'; patch]
fn (mut app App) patch_repo(id string) web.Result {
fn (mut app App) patch_repo(id int) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
mut repos := rlock app.git_mutex {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
app.db.update_git_repo(id, app.query)
return app.status(http.Status.internal_server_error)
}
}
if 'arch' in app.query {
arch_objs := app.query['arch'].split(',').map(db.GitRepoArch{ value: it })
if id !in repos {
return app.not_found()
}
repos[id].patch_from_params(app.query)
lock app.git_mutex {
git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) }
app.db.update_git_repo_archs(id, arch_objs)
}
return app.json(http.Status.ok, new_response('Repo updated successfully.'))

2
src/server/routes.v
View File

@ -68,7 +68,7 @@ fn (mut app App) put_package(repo string) web.Result {
if length := app.req.header.get(.content_length) {
// Generate a random filename for the temp file
pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4())
pkg_path = os.join_path_single(app.repo.pkg_dir, rand.uuid_v4())
app.ldebug("Uploading $length bytes (${util.pretty_bytes(length.int())}) to '$pkg_path'.")

25
src/server/server.v
View File

@ -5,8 +5,14 @@ import os
import log
import repo
import util
import db
const port = 8000
const (
port = 8000
log_file_name = 'vieter.log'
repo_dir_name = 'repos'
db_file_name = 'vieter.sqlite'
)
struct App {
web.Context
@ -14,8 +20,7 @@ pub:
conf Config [required; web_global]
pub mut:
repo repo.RepoGroupManager [required; web_global]
// This is used to claim the file lock on the repos file
git_mutex shared util.Dummy
db db.VieterDb
}
// server starts the web server & starts listening for requests
@ -30,11 +35,14 @@ pub fn server(conf Config) ? {
util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
}
os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') }
mut logger := log.Log{
level: log_level
}
logger.set_full_logpath(conf.log_file)
log_file := os.join_path_single(conf.data_dir, server.log_file_name)
logger.set_full_logpath(log_file)
logger.log_to_console_too()
defer {
@ -43,19 +51,20 @@ pub fn server(conf Config) ? {
logger.close()
}
repo_dir := os.join_path_single(conf.data_dir, server.repo_dir_name)
// This also creates the directories if needed
repo := repo.new(conf.repos_dir, conf.pkg_dir, conf.default_arch) or {
repo := repo.new(repo_dir, conf.pkg_dir, conf.default_arch) or {
logger.error(err.msg())
exit(1)
}
os.mkdir_all(conf.download_dir) or {
util.exit_with_message(1, 'Failed to create download directory.')
}
db_file := os.join_path_single(conf.data_dir, server.db_file_name)
db := db.init(db_file) or { util.exit_with_message(1, 'Failed to initialize database.') }
web.run(&App{
logger: logger
conf: conf
repo: repo
db: db
}, server.port)
}

4
vieter.toml
View File

@ -1,10 +1,8 @@
# This file contains settings used during development
api_key = "test"
download_dir = "data/downloads"
repos_dir = "data/repos"
data_dir = "data"
pkg_dir = "data/pkgs"
log_level = "DEBUG"
repos_file = "data/repos.json"
default_arch = "x86_64"
address = "http://localhost:8000"