test
/
vieter
forked from vieter-v/vieter
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Merge pull request 'Release 0.4.0' (#274) from release-0.4.0 into main 

Reviewed-on: vieter-v/vieter#274
Jef Roosens 2022-10-01 17:16:27 +02:00
parent 4866cfa635 66928216e5
commit ae29fe5ef8
82 changed files with 1550 additions and 1591 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -27,3 +27,6 @@ gdb.txt
# Generated docs
_docs/
/man/
# VLS logs
vls.log

17
.woodpecker/.test.yml
View File

@ -1,17 +0,0 @@
matrix:
PLATFORM:
- 'linux/amd64'
- 'linux/arm64'
branches:
exclude: [ main ]
platform: ${PLATFORM}
pipeline:
test:
image: 'chewingbever/vlang:latest'
pull: true
commands:
- make test
when:
event: [pull_request]

2
.woodpecker/.arch-rel.yml → .woodpecker/arch-rel.yml
View File

@ -23,7 +23,7 @@ pipeline:
- su builder
# Due to a bug with the V compiler, we can't just use the PKGBUILD from
# inside the repo
- curl -OL "https://git.rustybever.be/vieter/vieter/raw/tag/$CI_COMMIT_TAG/PKGBUILD"
- curl -OL "https://git.rustybever.be/vieter-v/vieter/raw/tag/$CI_COMMIT_TAG/PKGBUILD"
- makepkg -s --noconfirm --needed
when:
event: tag

2
.woodpecker/.arch.yml → .woodpecker/arch.yml
View File

@ -23,7 +23,7 @@ pipeline:
- su builder
# Due to a bug with the V compiler, we can't just use the PKGBUILD from
# inside the repo
- curl -o PKGBUILD -L https://git.rustybever.be/vieter/vieter/raw/branch/dev/PKGBUILD.dev
- curl -o PKGBUILD -L https://git.rustybever.be/vieter-v/vieter/raw/branch/dev/PKGBUILD.dev
- makepkg -s --noconfirm --needed
when:
event: push

22
.woodpecker/.build.yml → .woodpecker/build.yml
View File

@ -1,3 +1,6 @@
variables:
- &vlang_image 'chewingbever/vlang:0.3'
matrix:
PLATFORM:
- 'linux/amd64'
@ -6,10 +9,19 @@ matrix:
platform: ${PLATFORM}
pipeline:
debug:
image: 'chewingbever/vlang:latest'
install-modules:
image: *vlang_image
pull: true
commands:
- export VMODULES=$PWD/.vmodules
- 'cd src && v install'
when:
event: [push, pull_request]
debug:
image: *vlang_image
commands:
- export VMODULES=$PWD/.vmodules
- make
when:
event: [pull_request]
@ -17,11 +29,11 @@ pipeline:
exclude: [main]
prod:
image: 'chewingbever/vlang:latest'
pull: true
image: *vlang_image
environment:
- LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static
commands:
- export VMODULES=$PWD/.vmodules
# Apparently this -D is *very* important
- CFLAGS='-DGC_THREADS=1' make prod
# Make sure the binary is actually statically built
@ -35,7 +47,7 @@ pipeline:
event: [push, pull_request]
upload:
image: 'chewingbever/vlang:latest'
image: *vlang_image
secrets: [ s3_username, s3_password ]
commands:
# https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f

0
.woodpecker/.deploy.yml → .woodpecker/deploy.yml
View File

0
.woodpecker/.docker.yml → .woodpecker/docker.yml
View File

7
.woodpecker/.docs.yml → .woodpecker/docs.yml
View File

@ -1,17 +1,20 @@
variables:
- &vlang_image 'chewingbever/vlang:0.3'
platform: 'linux/amd64'
branches:
exclude: [ main ]
pipeline:
docs:
image: 'klakegg/hugo:alpine'
image: 'klakegg/hugo:ext-alpine'
group: 'generate'
commands:
- apk add git
- make docs
api-docs:
image: 'chewingbever/vlang:latest'
image: *vlang_image
pull: true
group: 'generate'
commands:

5
.woodpecker/.gitea.yml → .woodpecker/gitea.yml
View File

@ -1,3 +1,6 @@
variables:
- &vlang_image 'chewingbever/vlang:0.3'
platform: 'linux/amd64'
branches: [ 'main' ]
depends_on:
@ -8,7 +11,7 @@ skip_clone: true
pipeline:
prepare:
image: 'chewingbever/vlang:latest'
image: *vlang_image
pull: true
secrets: [ s3_username, s3_password ]
commands:

5
.woodpecker/.lint.yml → .woodpecker/lint.yml
View File

@ -1,3 +1,6 @@
variables:
- &vlang_image 'chewingbever/vlang:0.3'
# These checks already get performed on the feature branches
branches:
exclude: [ main ]
@ -5,7 +8,7 @@ platform: 'linux/amd64'
pipeline:
lint:
image: 'chewingbever/vlang:latest'
image: *vlang_image
pull: true
commands:
- make lint

5
.woodpecker/.man.yml → .woodpecker/man.yml
View File

@ -1,3 +1,6 @@
variables:
- &vlang_image 'chewingbever/vlang:0.3'
platform: 'linux/amd64'
branches:
exclude: [ main ]
@ -9,7 +12,7 @@ skip_clone: true
pipeline:
generate:
image: 'chewingbever/vlang:latest'
image: *vlang_image
pull: true
commands:
- curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64"

30
.woodpecker/test.yml 100644
View File

@ -0,0 +1,30 @@
variables:
- &vlang_image 'chewingbever/vlang:0.3'
matrix:
PLATFORM:
- 'linux/amd64'
- 'linux/arm64'
branches:
exclude: [ main ]
platform: ${PLATFORM}
pipeline:
install-modules:
image: *vlang_image
pull: true
commands:
- export VMODULES=$PWD/.vmodules
- 'cd src && v install'
when:
event: [pull_request]
test:
image: *vlang_image
pull: true
commands:
- export VMODULES=$PWD/.vmodules
- make test
when:
event: [pull_request]

53
CHANGELOG.md
View File

@ -5,13 +5,50 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev)
## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
## [0.3.0](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0)
## [0.4.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.4.0)
### Added
* Server port can now be configured
* Targets now have a 'kind' field describing whether it's a Git repository or a
URL to a PKGBUILD
* Targets with kind 'url' can provide a direct URL to a PKGBUILD instead of
providing a Git repository
* CLI commands for searching the AUR & directly adding packages
* HTTP routes for removing packages, arch-repos & repos
* All endpoints serving files now support HTTP byte range requests
* Better CLI UX
* When adding targets, the ID of the created target is returned
* The `-r` flag only shows raw data of action
* When adding a target, only ID is shown and not surrounding text
* Tabled output returns a tab-separated list (easy to script using
`cut`)
### Changed
* Moved all API routes under `/v1` namespace
* Renamed `vieter repos` to `vieter targets`
* Renamed `/api/v1/repos` namespace to `/api/v1/targets`
* Branch name for 'git' targets is now optional; if not provided, the
repository will be cloned with the default branch
* Build containers now explicitely set the PATH variable
* Refactor of web framework
* API endpoints now return id of newly created entries
* Repo POST requests now return information on published package
* `api` can no longer be used as a repository name
* CLI client now allows setting values to an empty value
### Removed
* md5 hashes are no longer calculated for packages
## [0.3.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.3.0)
Nothing besides bumping the versions.
## [0.3.0-rc.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-rc.1)
## [0.3.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.3.0-rc.1)
### Added
@ -39,7 +76,7 @@ Nothing besides bumping the versions.
* `POST /api/logs` now correctly uses epoch timestamps instead of strings
## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2)
## [0.3.0-alpha.2](https://git.rustybever.be/vieter-v/vieter/src/tag/0.3.0-alpha.2)
### Added
@ -64,7 +101,7 @@ Nothing besides bumping the versions.
* `vieter-git` is the latest commit on the dev branch
* Full refactor of Docker socket code
## [0.3.0-alpha.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.1)
## [0.3.0-alpha.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.3.0-alpha.1)
### Changed
@ -83,7 +120,7 @@ Nothing besides bumping the versions.
* Binary no longer panics when an env var is missing
## [0.2.0](https://git.rustybever.be/vieter/vieter/src/tag/0.2.0)
## [0.2.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.2.0)
### Changed
@ -117,13 +154,13 @@ Nothing besides bumping the versions.
* Packages with unknown fields in .PKGINFO are now allowed
* Old packages are now properly removed
## [0.1.0](https://git.rustybever.be/vieter/vieter/src/tag/0.1.0)
## [0.1.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.1.0)
### Changed
* Improved logging
## [0.1.0-rc.1](https://git.rustybever.be/vieter/vieter/src/tag/0.1.0-rc.1)
## [0.1.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.1.0-rc.1)
### Added

2
Dockerfile
View File

@ -1,4 +1,4 @@
FROM chewingbever/vlang:latest AS builder
FROM chewingbever/vlang:0.3 AS builder
ARG TARGETPLATFORM
ARG CI_COMMIT_SHA

9
Makefile
View File

@ -83,16 +83,9 @@ fmt:
test:
$(V) test $(SRC_DIR)
# Build & patch the V compiler
.PHONY: v
v: v/v
v/v:
git clone --single-branch https://git.rustybever.be/Chewing_Bever/v v
make -C v
.PHONY: clean
clean:
rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' 'docs/public'
rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' 'docs/public'
# =====EXPERIMENTAL=====

18
PKGBUILD
View File

@ -3,18 +3,26 @@
pkgbase='vieter'
pkgname='vieter'
pkgver='0.3.0'
pkgver='0.4.0'
pkgrel=1
pkgdesc="Vieter is a lightweight implementation of an Arch repository server."
pkgdesc="Lightweight Arch repository server & package build system"
depends=('glibc' 'openssl' 'libarchive' 'sqlite')
makedepends=('git' 'vieter-v')
makedepends=('git' 'vlang')
arch=('x86_64' 'aarch64')
url='https://git.rustybever.be/vieter/vieter'
url='https://git.rustybever.be/vieter-v/vieter'
license=('AGPL3')
source=("$pkgname::git+https://git.rustybever.be/vieter/vieter#tag=${pkgver//_/-}")
source=("$pkgname::git+https://git.rustybever.be/vieter-v/vieter#tag=${pkgver//_/-}")
md5sums=('SKIP')
prepare() {
export VMODULES="$srcdir/.vmodules"
cd "$pkgname/src" && v install
}
build() {
export VMODULES="$srcdir/.vmodules"
cd "$pkgname"
make prod

16
PKGBUILD.dev
View File

@ -5,13 +5,13 @@ pkgbase='vieter-git'
pkgname='vieter-git'
pkgver=0.2.0.r25.g20112b8
pkgrel=1
pkgdesc="Vieter is a lightweight implementation of an Arch repository server."
pkgdesc="Lightweight Arch repository server & package build system (development version)"
depends=('glibc' 'openssl' 'libarchive' 'sqlite')
makedepends=('git' 'vieter-v')
makedepends=('git' 'vlang')
arch=('x86_64' 'aarch64')
url='https://git.rustybever.be/vieter/vieter'
url='https://git.rustybever.be/vieter-v/vieter'
license=('AGPL3')
source=("$pkgname::git+https://git.rustybever.be/vieter/vieter#branch=dev")
source=("$pkgname::git+https://git.rustybever.be/vieter-v/vieter#branch=dev")
md5sums=('SKIP')
provides=('vieter')
conflicts=('vieter')
@ -22,7 +22,15 @@ pkgver() {
git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g'
}
prepare() {
export VMODULES="$srcdir/.vmodules"
cd "$pkgname/src" && v install
}
build() {
export VMODULES="$srcdir/.vmodules"
cd "$pkgname"
make prod

22
README.md
View File

@ -1,11 +1,12 @@
# Vieter
## Documentation
I host documentation for Vieter over at https://rustybever.be/docs/vieter/. API
documentation for the current codebase can be found at
https://rustybever.be/api-docs/vieter/.
For more information, questions or just a chat, there's
[#vieter:rustybever.be](https://matrix.to/#/#vieter:rustybever.be) on Matrix!
## Overview
Vieter is a restart of the Pieter project. The goal is to create a simple,
@ -36,22 +37,19 @@ that.
Besides a V installer, Vieter also requires the following libraries to work:
* gc
* libarchive
* openssl
* sqlite3
Vieter also depends on some external V modules which you can install using `cd
src && v install`. Make sure to keep these dependencies up to date using `v
update`.
### Compiler
Vieter compiles with the standard Vlang compiler. However, I do maintain a
[mirror](https://git.rustybever.be/vieter/v). This is to ensure my CI does not
break without reason, as I control when & how frequently the mirror is updated
to reflect the official repository.
If you encounter issues using the latest V compiler, try using my mirror
instead. `make v` will clone the repository & build the mirror. Afterwards,
prepending any make command with `V_PATH=v/v` tells make to use the locally
compiled mirror instead.
I used to maintain a mirror that tracked the latest master, but nowadays, I
maintain a Docker image containing the specific compiler version that Vieter
builds with. Currently, this is V 0.3.
## Contributing

33
docs/api/source/includes/_logs.md
View File

@ -13,7 +13,7 @@ Endpoints for interacting with stored build logs.
```shell
curl \
-H 'X-Api-Key: secret' \
https://example.com/api/logs?offset=10&limit=20
https://example.com/api/v1/logs?offset=10&limit=20
```
> JSON output format
@ -24,7 +24,7 @@ curl \
"data": [
{
"id": 1,
"repo_id": 3,
"target_id": 3,
"start_time": 1652008554,
"end_time": 1652008559,
"arch": "x86_64",
@ -38,7 +38,7 @@ Retrieve a list of build logs.
### HTTP Request
`GET /api/logs`
`GET /api/v1/logs`
### Query Parameters
@ -46,7 +46,7 @@ Parameter | Description
--------- | -----------
limit | Maximum amount of results to return.
offset | Offset of results.
repo | Only return builds published to this repository.
target | Only return builds for this target id.
before | Only return logs started before this time (UTC epoch)
after | Only return logs started after this time (UTC epoch)
arch | Only return logs built on this architecture
@ -58,7 +58,7 @@ exit_codes | Comma-separated list of exit codes to limit result to; using `!` as
```shell
curl \
-H 'X-Api-Key: secret' \
https://example.com/api/logs/15
https://example.com/api/v1/logs/1
```
> JSON output format
@ -68,7 +68,7 @@ curl \
"message": "",
"data": {
"id": 1,
"repo_id": 3,
"target_id": 3,
"start_time": 1652008554,
"end_time": 1652008559,
"arch": "x86_64",
@ -81,7 +81,7 @@ Retrieve info about a specific build log.
### HTTP Request
`GET /api/logs/:id`
`GET /api/v1/logs/:id`
### URL Parameters
@ -94,7 +94,7 @@ id | ID of requested log
```shell
curl \
-H 'X-Api-Key: secret' \
https://example.com/api/logs/15/content
https://example.com/api/v1/logs/15/content
```
Retrieve the contents of a build log. The response is the build log in
@ -102,7 +102,7 @@ plaintext.
### HTTP Request
`GET /api/logs/:id/content`
`GET /api/v1/logs/:id/content`
### URL Parameters
@ -112,6 +112,17 @@ id | ID of requested log
## Publish build log
> JSON output format
```json
{
"message": "",
"data": {
"id": 15
}
}
```
<aside class="warning">
You should probably not use this endpoint, as it's used by the build system to
@ -123,17 +134,17 @@ Publish a new build log to the server.
### HTTP Request
`POST /api/logs`
`POST /api/v1/logs`
### Query parameters
Parameter | Description
--------- | -----------
id | ID of requested log
startTime | Start time of the build (UTC epoch)
endTime | End time of the build (UTC epoch)
arch | Architecture on which the build was done
exitCode | Exit code of the build container
target | id of target this build is for
### Request body

84
docs/api/source/includes/_repository.md
View File

@ -93,3 +93,87 @@ other already present arch-repos.
Parameter | Description
--------- | -----------
repo | Repository to publish package to
## Remove package from arch-repo
<aside class="notice">
This endpoint requests authentication.
</aside>
```shell
curl \
-H 'X-Api-Key: secret' \
-XDELETE \
https://example.com/vieter/x86_64/mike
```
This endpoint allows you to remove a package from a given arch-repo.
### HTTP Request
`DELETE /:repo/:arch/:pkg`
### URL Parameters
Parameter | Description
--------- | -----------
repo | Repository to delete package from
arch | Specific arch-repo to remove package from
pkg | Name of package to remove (without any version information)
## Remove arch-repo
<aside class="notice">
This endpoint requests authentication.
</aside>
```shell
curl \
-H 'X-Api-Key: secret' \
-XDELETE \
https://example.com/vieter/x86_64
```
This endpoint allows removing an entire arch-repo.
### HTTP Request
`DELETE /:repo/:arch`
### URL Parameters
Parameter | Description
--------- | -----------
repo | Repository to delete arch-repo from
arch | Specific architecture to remove
## Remove repo
<aside class="notice">
This endpoint requests authentication.
</aside>
```shell
curl \
-H 'X-Api-Key: secret' \
-XDELETE \
https://example.com/vieter
```
This endpoint allows removing an entire repo.
### HTTP Request
`DELETE /:repo`
### URL Parameters
Parameter | Description
--------- | -----------
repo | Repository to delete

66
docs/api/source/includes/_git.md → docs/api/source/includes/_targets.md
View File

@ -1,4 +1,4 @@
# Git Repositories
# Targets
<aside class="notice">
@ -6,15 +6,14 @@ All routes in this section require authentication.
</aside>
Endpoints for interacting with the list of Git repositories stored on the
server.
Endpoints for interacting with the list of targets stored on the server.
## List repos
## List targets
```shell
curl \
-H 'X-Api-Key: secret' \
https://example.com/api/repos?offset=10&limit=20
https://example.com/api/v1/targets?offset=10&limit=20
```
> JSON output format
@ -25,6 +24,7 @@ curl \
"data": [
{
"id": 1,
"kind": "git",
"url": "https://aur.archlinux.org/discord-ptb.git",
"branch": "master",
"repo": "bur",
@ -32,7 +32,7 @@ curl \
"arch": [
{
"id": 1,
"repo_id": 1,
"target_id": 1,
"value": "x86_64"
}
]
@ -41,11 +41,11 @@ curl \
}
```
Retrieve a list of Git repositories.
Retrieve a list of targets.
### HTTP Request
`GET /api/repos`
`GET /api/v1/targets`
### Query Parameters
@ -53,14 +53,14 @@ Parameter | Description
--------- | -----------
limit | Maximum amount of results to return.
offset | Offset of results.
repo | Limit results to repositories that publish to the given repo.
repo | Limit results to targets that publish to the given repo.
## Get a repo
## Get specific target
```shell
curl \
-H 'X-Api-Key: secret' \
https://example.com/api/repos/15
https://example.com/api/v1/targets/1
```
> JSON output format
@ -70,6 +70,7 @@ curl \
"message": "",
"data": {
"id": 1,
"kind": "git",
"url": "https://aur.archlinux.org/discord-ptb.git",
"branch": "master",
"repo": "bur",
@ -77,7 +78,7 @@ curl \
"arch": [
{
"id": 1,
"repo_id": 1,
"target_id": 1,
"value": "x86_64"
}
]
@ -85,70 +86,83 @@ curl \
}
```
Get info about a specific Git repository.
Get info about a specific target.
### HTTP Request
`GET /api/repos/:id`
`GET /api/v1/targets/:id`
### URL Parameters
Parameter | Description
--------- | -----------
id | ID of requested repo
id | id of requested target
## Create a new repo
## Create a new target
Create a new Git repository with the given data.
> JSON output format
```json
{
"message": "",
"data": {
"id": 15
}
}
```
Create a new target with the given data.
### HTTP Request
`POST /api/repos`
`POST /api/v1/targets`
### Query Parameters
Parameter | Description
--------- | -----------
kind | Kind of target to add; one of 'git', 'url'.
url | URL of the Git repository.
branch | Branch of the Git repository.
repo | Vieter repository to publish built packages to.
schedule | Cron build schedule (syntax explained [here](https://rustybever.be/docs/vieter/usage/builds/schedule/))
arch | Comma-separated list of architectures to build package on.
## Modify a repo
## Modify a target
Modify the data of an existing Git repository.
Modify the data of an existing target.
### HTTP Request
`PATCH /api/repos/:id`
`PATCH /api/v1/targets/:id`
### URL Parameters
Parameter | Description
--------- | -----------
id | ID of requested repo
id | id of target to modify
### Query Parameters
Parameter | Description
--------- | -----------
kind | Kind of target; one of 'git', 'url'.
url | URL of the Git repository.
branch | Branch of the Git repository.
repo | Vieter repository to publish built packages to.
schedule | Cron build schedule
arch | Comma-separated list of architectures to build package on.
## Remove a repo
## Remove a target
Remove a Git repository from the server.
Remove a target from the server.
### HTTP Request
`DELETE /api/repos/:id`
`DELETE /api/v1/targets/:id`
### URL Parameters
Parameter | Description
--------- | -----------
id | ID of repo to remove
id | id of target to remove

2
docs/api/source/index.html.md
View File

@ -9,7 +9,7 @@ toc_footers:
includes:
- repository
- git
- targets
- logs
search: true

4
docs/config.toml
View File

@ -38,7 +38,7 @@ enableGitInfo = true
weight = 20
[[menu.after]]
name = "Vieter"
url = "https://git.rustybever.be/vieter/vieter"
url = "https://git.rustybever.be/vieter-v/vieter"
weight = 30
[[menu.after]]
name = "Hugo Theme"
@ -70,7 +70,7 @@ enableGitInfo = true
# Set source repository location.
# Used for 'Last Modified' and 'Edit this page' links.
BookRepo = 'https://git.rustybever.be/vieter/vieter'
BookRepo = 'https://git.rustybever.be/vieter-v/vieter'
# (Optional, default 'commit') Specifies commit portion of the link to the page's last modified
# commit hash for 'doc' page type.

9
docs/content/configuration.md
View File

@ -26,7 +26,7 @@ secrets file.
## Commands
The first argument passed to Vieter determines which command you wish to use.
Each of these can contain subcommands (e.g. `vieter repos list`), but all
Each of these can contain subcommands (e.g. `vieter targets list`), but all
subcommands will use the same configuration. Below you can find the
configuration variable required for each command.
@ -45,7 +45,8 @@ configuration variable required for each command.
This prevents the server from being confused when an `any` package is
published as the very first package for a repository.
* Git repositories added without an `arch` value use this value instead.
* `port`: HTTP port to run on
* Default: `8000`
### `vieter cron`
@ -88,11 +89,11 @@ configuration variable required for each command.
* `api_key`: the API key to use when authenticating requests.
* `address`: Base URL of your Vieter instance, e.g. https://example.com
### `vieter repos`
### `vieter targets`
* `api_key`: the API key to use when authenticating requests.
* `address`: Base URL of your Vieter instance, e.g. https://example.com
* `base_image`: image to use when building a package using `vieter repos
* `base_image`: image to use when building a package using `vieter targets
build`.
* Default: `archlinux:base-devel`

8
docs/content/installation.md
View File

@ -96,6 +96,14 @@ SigLevel = Optional
Afterwards, you can update your system & install the `vieter` package for the
latest official release or `vieter-git` for the latest development release.
### AUR
If you prefer building the packages locally (or on your own Vieter instance),
there's the `[vieter](https://aur.archlinux.org/packages/vieter)` &
`[vieter-git](https://aur.archlinux.org/packages/vieter-git)` packages on the
AUR. These packages build using the `vlang-git` compiler package, so I can't
guarantee that a compiler update won't temporarily break them.
## Building from source
The project [README](https://git.rustybever.be/vieter/vieter#building) contains

18
docs/content/usage/builds/_index.md
View File

@ -16,28 +16,28 @@ info to the system. The Vieter repository server exposes an HTTP API for this
info). For ease of use, the Vieter binary contains a CLI interface for
interacting with this API (see [Configuration](/configuration) for
configuration details). The [man
pages](https://rustybever.be/man/vieter/vieter-repos.1.html) describe this in
pages](https://rustybever.be/man/vieter/vieter-targets.1.html) describe this in
greater detail, but the basic usage is as follows:
```
vieter repos add some-url some-branch some-repository
vieter targets add some-url some-repository
```
Here, `some-url` is the URL of the Git repository containing the PKGBUILD. This
URL is passed to `git clone`, meaning the repository should be public. Vieter
expects the same format as an AUR Git repository, so you can directly use AUR
URLs here.
URLs here. Alternatively, you can also provide the URL to a PKGBUILD file
instead. See
[vieter-targets-add(1)](https://rustybever.be/man/vieter/vieter-targets-add.1.html)
for more information.
`some-branch` is the branch of the Git repository the build should check out.
If you're using an AUR package, this should be `master`.
Finally, `some-repo` is the repository to which the built package archives
should be published.
`some-repo` is the repository to which the built package archives should be
published.
The above command intentionally leaves out a few parameters to make the CLI
more useable. For information on how to modify all parameters using the CLI,
see
[vieter-repos-edit(1)](https://rustybever.be/man/vieter/vieter-repos-edit.1.html).
[vieter-targets(1)](https://rustybever.be/man/vieter/vieter-targets.1.html).
## Reading logs

4
src/archive.c.v
View File

@ -4,7 +4,7 @@
#include "archive.h"
struct C.archive {}
pub struct C.archive {}
// Create a new archive struct for reading
fn C.archive_read_new() &C.archive
@ -71,7 +71,7 @@ fn C.archive_filter_code(&C.archive, int) int
#include "archive_entry.h"
struct C.archive_entry {}
pub struct C.archive_entry {}
// Create a new archive_entry struct
fn C.archive_entry_new() &C.archive_entry

43
src/build/build.v
View File

@ -1,16 +1,19 @@
module build
import docker
import vieter_v.docker
import encoding.base64
import time
import os
import strings
import util
import models { GitRepo }
import models { Target }
const (
container_build_dir = '/build'
build_image_repo = 'vieter-build'
// Contents of PATH variable in build containers
path_dirs = ['/sbin', '/bin', '/usr/sbin', '/usr/bin', '/usr/local/sbin',
'/usr/local/bin', '/usr/bin/site_perl', '/usr/bin/vendor_perl', '/usr/bin/core_perl']
)
// create_build_image creates a builder image given some base image which can
@ -56,13 +59,13 @@ pub fn create_build_image(base_image string) ?string {
// We pull the provided image
dd.pull_image(image_name, image_tag)?
id := dd.create_container(c)?.id
id := dd.container_create(c)?.id
// id := docker.create_container(c)?
dd.start_container(id)?
dd.container_start(id)?
// This loop waits until the container has stopped, so we can remove it after
for {
data := dd.inspect_container(id)?
data := dd.container_inspect(id)?
if !data.state.running {
break
@ -77,7 +80,7 @@ pub fn create_build_image(base_image string) ?string {
// conflicts.
tag := time.sys_mono_now().str()
image := dd.create_image_from_container(id, 'vieter-build', tag)?
dd.remove_container(id)?
dd.container_remove(id)?
return image.id
}
@ -90,10 +93,10 @@ pub:
logs string
}
// build_repo builds, packages & publishes a given Arch package based on the
// provided GitRepo. The base image ID should be of an image previously created
// build_target builds, packages & publishes a given Arch package based on the
// provided target. The base image ID should be of an image previously created
// by create_build_image. It returns the logs of the container.
pub fn build_repo(address string, api_key string, base_image_id string, repo &GitRepo) ?BuildResult {
pub fn build_target(address string, api_key string, base_image_id string, target &Target) ?BuildResult {
mut dd := docker.new_conn()?
defer {
@ -101,7 +104,7 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Gi
}
build_arch := os.uname().machine
build_script := create_build_script(address, repo, build_arch)
build_script := create_build_script(address, target, build_arch)
// We convert the build script into a base64 string, which then gets passed
// to the container as an env var
@ -109,32 +112,38 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Gi
c := docker.NewContainer{
image: '$base_image_id'
env: ['BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key']
env: [
'BUILD_SCRIPT=$base64_script',
'API_KEY=$api_key',
// `archlinux:base-devel` does not correctly set the path variable,
// causing certain builds to fail. This fixes it.
'PATH=${build.path_dirs.join(':')}',
]
entrypoint: ['/bin/sh', '-c']
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e']
work_dir: '/build'
user: '0:0'
}
id := dd.create_container(c)?.id
dd.start_container(id)?
id := dd.container_create(c)?.id
dd.container_start(id)?
mut data := dd.inspect_container(id)?
mut data := dd.container_inspect(id)?
// This loop waits until the container has stopped, so we can remove it after
for data.state.running {
time.sleep(1 * time.second)
data = dd.inspect_container(id)?
data = dd.container_inspect(id)?
}
mut logs_stream := dd.get_container_logs(id)?
mut logs_stream := dd.container_get_logs(id)?
// Read in the entire stream
mut logs_builder := strings.new_builder(10 * 1024)
util.reader_to_writer(mut logs_stream, mut logs_builder)?
dd.remove_container(id)?
dd.container_remove(id)?
return BuildResult{
start_time: data.state.start_time

4
src/build/build_script.sh → src/build/build_script_git.sh
View File

@ -4,8 +4,8 @@ echo -e '+ pacman -Syu --needed --noconfirm'
pacman -Syu --needed --noconfirm
echo -e '+ su builder'
su builder
echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo'
git clone --single-branch --depth 1 --branch main https://examplerepo.com repo
echo -e '+ git clone --single-branch --depth 1 '\''https://examplerepo.com'\'' repo'
git clone --single-branch --depth 1 'https://examplerepo.com' repo
echo -e '+ cd repo'
cd repo
echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'

20
src/build/build_script_git_branch.sh 100644
View File

@ -0,0 +1,20 @@
echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf'
echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
echo -e '+ pacman -Syu --needed --noconfirm'
pacman -Syu --needed --noconfirm
echo -e '+ su builder'
su builder
echo -e '+ git clone --single-branch --depth 1 --branch main '\''https://examplerepo.com'\'' repo'
git clone --single-branch --depth 1 --branch main 'https://examplerepo.com' repo
echo -e '+ cd repo'
cd repo
echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'
makepkg --nobuild --syncdeps --needed --noconfirm
echo -e '+ source PKGBUILD'
source PKGBUILD
echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0'
curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
[ "$(id -u)" == 0 ] && exit 0
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done

22
src/build/build_script_url.sh 100644
View File

@ -0,0 +1,22 @@
echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf'
echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
echo -e '+ pacman -Syu --needed --noconfirm'
pacman -Syu --needed --noconfirm
echo -e '+ su builder'
su builder
echo -e '+ mkdir repo'
mkdir repo
echo -e '+ curl -o repo/PKGBUILD -L '\''https://examplerepo.com'\'''
curl -o repo/PKGBUILD -L 'https://examplerepo.com'
echo -e '+ cd repo'
cd repo
echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'
makepkg --nobuild --syncdeps --needed --noconfirm
echo -e '+ source PKGBUILD'
source PKGBUILD
echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0'
curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
[ "$(id -u)" == 0 ] && exit 0
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done

43
src/build/shell.v
View File

@ -1,6 +1,6 @@
module build
import models { GitRepo }
import models { Target }
// escape_shell_string escapes any characters that could be interpreted
// incorrectly by a shell. The resulting value should be safe to use inside an
@ -22,21 +22,46 @@ pub fn echo_commands(cmds []string) []string {
return out
}
// create_build_script generates a shell script that builds a given GitRepo.
fn create_build_script(address string, repo &GitRepo, build_arch string) string {
repo_url := '$address/$repo.repo'
// create_build_script generates a shell script that builds a given Target.
fn create_build_script(address string, target &Target, build_arch string) string {
repo_url := '$address/$target.repo'
commands := echo_commands([
mut commands := [
// This will later be replaced by a proper setting for changing the
// mirrorlist
"echo -e '[$repo.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf"
"echo -e '[$target.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf"
// We need to update the package list of the repo we just added above.
// This should however not pull in a lot of packages as long as the
// builder image is rebuilt frequently.
'pacman -Syu --needed --noconfirm',
// makepkg can't run as root
'su builder',
'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
]
commands << match target.kind {
'git' {
if target.branch == '' {
[
"git clone --single-branch --depth 1 '$target.url' repo",
]
} else {
[
"git clone --single-branch --depth 1 --branch $target.branch '$target.url' repo",
]
}
}
'url' {
[
'mkdir repo',
"curl -o repo/PKGBUILD -L '$target.url'",
]
}
else {
panic("Invalid kind. This shouldn't be possible.")
}
}
commands << [
'cd repo',
'makepkg --nobuild --syncdeps --needed --noconfirm',
'source PKGBUILD',
@ -49,7 +74,7 @@ fn create_build_script(address string, repo &GitRepo, build_arch string) string
// we're in root so we don't proceed.
'[ "\$(id -u)" == 0 ] && exit 0',
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done',
])
]
return commands.join('\n')
return echo_commands(commands).join('\n')
}

37
src/build/shell_test.v
View File

@ -1,16 +1,43 @@
module build
import models { GitRepo }
import models { Target }
fn test_create_build_script() {
repo := GitRepo{
fn test_create_build_script_git_branch() {
target := Target{
id: 1
kind: 'git'
url: 'https://examplerepo.com'
branch: 'main'
repo: 'vieter'
}
build_script := create_build_script('https://example.com', repo, 'x86_64')
expected := $embed_file('build_script.sh')
build_script := create_build_script('https://example.com', target, 'x86_64')
expected := $embed_file('build_script_git_branch.sh')
assert build_script == expected.to_string().trim_space()
}
fn test_create_build_script_git() {
target := Target{
id: 1
kind: 'git'
url: 'https://examplerepo.com'
repo: 'vieter'
}
build_script := create_build_script('https://example.com', target, 'x86_64')
expected := $embed_file('build_script_git.sh')
assert build_script == expected.to_string().trim_space()
}
fn test_create_build_script_url() {
target := Target{
id: 1
kind: 'url'
url: 'https://examplerepo.com'
repo: 'vieter'
}
build_script := create_build_script('https://example.com', target, 'x86_64')
expected := $embed_file('build_script_url.sh')
assert build_script == expected.to_string().trim_space()
}

6
src/client/client.v
View File

@ -2,7 +2,7 @@ module client
import net.http { Method }
import net.urllib
import response { Response }
import web.response { Response }
import json
pub struct Client {
@ -30,12 +30,10 @@ fn (c &Client) send_request_raw(method Method, url string, params map[string]str
// Escape each query param
for k, v in params {
// An empty parameter should be the same as not providing it at all
if v != '' {
params_escaped[k] = urllib.query_escape(v)
}
}
params_str := params_escaped.keys().map('$it=${params[it]}').join('&')
params_str := params_escaped.keys().map('$it=${params_escaped[it]}').join('&')
full_url = '$full_url?$params_str'
}

73
src/client/git.v
View File

@ -1,73 +0,0 @@
module client
import models { GitRepo, GitRepoFilter }
import net.http { Method }
import response { Response }
// get_git_repos returns a list of GitRepo's, given a filter object.
pub fn (c &Client) get_git_repos(filter GitRepoFilter) ?[]GitRepo {
params := models.params_from(filter)
data := c.send_request<[]GitRepo>(Method.get, '/api/repos', params)?
return data.data
}
// get_all_git_repos retrieves *all* GitRepo's from the API using the default
// limit.
pub fn (c &Client) get_all_git_repos() ?[]GitRepo {
mut repos := []GitRepo{}
mut offset := u64(0)
for {
sub_repos := c.get_git_repos(offset: offset)?
if sub_repos.len == 0 {
break
}
repos << sub_repos
offset += u64(sub_repos.len)
}
return repos
}
// get_git_repo returns the repo for a specific ID.
pub fn (c &Client) get_git_repo(id int) ?GitRepo {
data := c.send_request<GitRepo>(Method.get, '/api/repos/$id', {})?
return data.data
}
// add_git_repo adds a new repo to the server.
pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []string) ?Response<string> {
mut params := {
'url': url
'branch': branch
'repo': repo
}
if arch.len > 0 {
params['arch'] = arch.join(',')
}
data := c.send_request<string>(Method.post, '/api/repos', params)?
return data
}
// remove_git_repo removes the repo with the given ID from the server.
pub fn (c &Client) remove_git_repo(id int) ?Response<string> {
data := c.send_request<string>(Method.delete, '/api/repos/$id', {})?
return data
}
// patch_git_repo sends a PATCH request to the given repo with the params as
// payload.
pub fn (c &Client) patch_git_repo(id int, params map[string]string) ?Response<string> {
data := c.send_request<string>(Method.patch, '/api/repos/$id', params)?
return data
}

22
src/client/logs.v
View File

@ -2,53 +2,53 @@ module client
import models { BuildLog, BuildLogFilter }
import net.http { Method }
import response { Response }
import web.response { Response }
import time
// get_build_logs returns all build logs.
pub fn (c &Client) get_build_logs(filter BuildLogFilter) ?Response<[]BuildLog> {
params := models.params_from(filter)
data := c.send_request<[]BuildLog>(Method.get, '/api/logs', params)?
data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)?
return data
}
// get_build_logs_for_repo returns all build logs for a given repo.
pub fn (c &Client) get_build_logs_for_repo(repo_id int) ?Response<[]BuildLog> {
// get_build_logs_for_target returns all build logs for a given target.
pub fn (c &Client) get_build_logs_for_target(target_id int) ?Response<[]BuildLog> {
params := {
'repo': repo_id.str()
'repo': target_id.str()
}
data := c.send_request<[]BuildLog>(Method.get, '/api/logs', params)?
data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)?
return data
}
// get_build_log returns a specific build log.
pub fn (c &Client) get_build_log(id int) ?Response<BuildLog> {
data := c.send_request<BuildLog>(Method.get, '/api/logs/$id', {})?
data := c.send_request<BuildLog>(Method.get, '/api/v1/logs/$id', {})?
return data
}
// get_build_log_content returns the contents of the build log file.
pub fn (c &Client) get_build_log_content(id int) ?string {
data := c.send_request_raw_response(Method.get, '/api/logs/$id/content', {}, '')?
data := c.send_request_raw_response(Method.get, '/api/v1/logs/$id/content', {}, '')?
return data
}
// add_build_log adds a new build log to the server.
pub fn (c &Client) add_build_log(repo_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response<string> {
pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response<int> {
params := {
'repo': repo_id.str()
'target': target_id.str()
'startTime': start_time.unix_time().str()
'endTime': end_time.unix_time().str()
'arch': arch
'exitCode': exit_code.str()
}
data := c.send_request_with_body<string>(Method.post, '/api/logs', params, content)?
data := c.send_request_with_body<int>(Method.post, '/api/v1/logs', params, content)?
return data
}

72
src/client/targets.v 100644
View File

@ -0,0 +1,72 @@
module client
import models { Target, TargetFilter }
import net.http { Method }
import web.response { Response }
// get_targets returns a list of targets, given a filter object.
pub fn (c &Client) get_targets(filter TargetFilter) ?[]Target {
params := models.params_from(filter)
data := c.send_request<[]Target>(Method.get, '/api/v1/targets', params)?
return data.data
}
// get_all_targets retrieves *all* targs from the API using the default
// limit.
pub fn (c &Client) get_all_targets() ?[]Target {
mut targets := []Target{}
mut offset := u64(0)
for {
sub_targets := c.get_targets(offset: offset)?
if sub_targets.len == 0 {
break
}
targets << sub_targets
offset += u64(sub_targets.len)
}
return targets
}
// get_target returns the target for a specific id.
pub fn (c &Client) get_target(id int) ?Target {
data := c.send_request<Target>(Method.get, '/api/v1/targets/$id', {})?
return data.data
}
pub struct NewTarget {
kind string
url string
branch string
repo string
arch []string
}
// add_target adds a new target to the server.
pub fn (c &Client) add_target(t NewTarget) ?Response<int> {
params := models.params_from<NewTarget>(t)
data := c.send_request<int>(Method.post, '/api/v1/targets', params)?
return data
}
// remove_target removes the target with the given id from the server.
pub fn (c &Client) remove_target(id int) ?Response<string> {
data := c.send_request<string>(Method.delete, '/api/v1/targets/$id', {})?
return data
}
// patch_target sends a PATCH request to the given target with the params as
// payload.
pub fn (c &Client) patch_target(id int, params map[string]string) ?Response<string> {
data := c.send_request<string>(Method.patch, '/api/v1/targets/$id', params)?
return data
}

62
src/console/aur/aur.v 100644
View File

@ -0,0 +1,62 @@
module aur
import cli
import console
import client
import vieter_v.aur
import vieter_v.conf as vconf
struct Config {
address string [required]
api_key string [required]
}
// cmd returns the cli module for interacting with the AUR API.
pub fn cmd() cli.Command {
return cli.Command{
name: 'aur'
description: 'Interact with the AUR.'
commands: [
cli.Command{
name: 'search'
description: 'Search for packages.'
required_args: 1
execute: fn (cmd cli.Command) ? {
c := aur.new()
pkgs := c.search(cmd.args[0])?
data := pkgs.map([it.name, it.description])
println(console.pretty_table(['name', 'description'], data)?)
}
},
cli.Command{
name: 'add'
usage: 'repo pkg-name [pkg-name...]'
description: 'Add the given AUR package(s) to Vieter. Non-existent packages will be silently ignored.'
required_args: 2
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file')?
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
c := aur.new()
pkgs := c.info(cmd.args[1..])?
vc := client.new(conf.address, conf.api_key)
for pkg in pkgs {
vc.add_target(
kind: 'git'
url: 'https://aur.archlinux.org/$pkg.package_base' + '.git'
repo: cmd.args[0]
) or {
println('Failed to add $pkg.name: $err.msg()')
continue
}
println('Added $pkg.name' + '.')
}
}
},
]
}
}

5
src/console/console.v
View File

@ -5,6 +5,11 @@ import strings
import cli
import os
// tabbed_table returns a simple textual table, with tabs as separators.
pub fn tabbed_table(data [][]string) string {
return data.map(it.join('\t')).join('\n')
}
// pretty_table converts a list of string data into a pretty table. Many thanks
// to @hungrybluedev in the Vlang Discord for providing this code!
// https://ptb.discord.com/channels/592103645835821068/592106336838352923/970278787143045192

46
src/console/logs/logs.v
View File

@ -1,7 +1,7 @@
module logs
import cli
import env
import vieter_v.conf as vconf
import client
import console
import time
@ -12,7 +12,7 @@ struct Config {
api_key string [required]
}
// cmd returns the cli module that handles the build repos API.
// cmd returns the cli module that handles the build logs API.
pub fn cmd() cli.Command {
return cli.Command{
name: 'logs'
@ -33,8 +33,8 @@ pub fn cmd() cli.Command {
flag: cli.FlagType.int
},
cli.Flag{
name: 'repo'
description: 'Only return logs for this repo id.'
name: 'target'
description: 'Only return logs for this target id.'
flag: cli.FlagType.int
},
cli.Flag{
@ -65,7 +65,7 @@ pub fn cmd() cli.Command {
]
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file')?
conf := env.load<Config>(config_file)?
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
mut filter := BuildLogFilter{}
@ -79,9 +79,9 @@ pub fn cmd() cli.Command {
filter.offset = u64(offset)
}
repo_id := cmd.flags.get_int('repo')?
if repo_id != 0 {
filter.repo = repo_id
target_id := cmd.flags.get_int('target')?
if target_id != 0 {
filter.target = target_id
}
tz_offset := time.offset()
@ -133,7 +133,9 @@ pub fn cmd() cli.Command {
]
}
list(conf, filter)?
raw := cmd.flags.get_bool('raw')?
list(conf, filter, raw)?
}
},
cli.Command{
@ -143,7 +145,7 @@ pub fn cmd() cli.Command {
description: 'Show all info for a specific build log.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file')?
conf := env.load<Config>(config_file)?
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
id := cmd.args[0].int()
info(conf, id)?
@ -156,7 +158,7 @@ pub fn cmd() cli.Command {
description: 'Output the content of a build log to stdout.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file')?
conf := env.load<Config>(config_file)?
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
id := cmd.args[0].int()
content(conf, id)?
@ -167,27 +169,31 @@ pub fn cmd() cli.Command {
}
// print_log_list prints a list of logs.
fn print_log_list(logs []BuildLog) ? {
data := logs.map([it.id.str(), it.repo_id.str(), it.start_time.local().str(),
fn print_log_list(logs []BuildLog, raw bool) ? {
data := logs.map([it.id.str(), it.target_id.str(), it.start_time.local().str(),
it.exit_code.str()])
println(console.pretty_table(['id', 'repo', 'start time', 'exit code'], data)?)
if raw {
println(console.tabbed_table(data))
} else {
println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)?)
}
}
// list prints a list of all build logs.
fn list(conf Config, filter BuildLogFilter) ? {
fn list(conf Config, filter BuildLogFilter, raw bool) ? {
c := client.new(conf.address, conf.api_key)
logs := c.get_build_logs(filter)?.data
print_log_list(logs)?
print_log_list(logs, raw)?
}
// list prints a list of all build logs for a given repo.
fn list_for_repo(conf Config, repo_id int) ? {
// list prints a list of all build logs for a given target.
fn list_for_target(conf Config, target_id int, raw bool) ? {
c := client.new(conf.address, conf.api_key)
logs := c.get_build_logs_for_repo(repo_id)?.data
logs := c.get_build_logs_for_target(target_id)?.data
print_log_list(logs)?
print_log_list(logs, raw)?
}
// info print the detailed info for a given build log.

14
src/console/git/build.v → src/console/targets/build.v
View File

@ -1,14 +1,14 @@
module git
module targets
import client
import docker
import vieter_v.docker
import os
import build
// build builds every Git repo in the server's list.
fn build(conf Config, repo_id int) ? {
// build locally builds the target with the given id.
fn build(conf Config, target_id int) ? {
c := client.new(conf.address, conf.api_key)
repo := c.get_git_repo(repo_id)?
target := c.get_target(target_id)?
build_arch := os.uname().machine
@ -16,7 +16,7 @@ fn build(conf Config, repo_id int) ? {
image_id := build.create_build_image(conf.base_image)?
println('Running build...')
res := build.build_repo(conf.address, conf.api_key, image_id, repo)?
res := build.build_target(conf.address, conf.api_key, image_id, target)?
println('Removing build image...')
@ -29,6 +29,6 @@ fn build(conf Config, repo_id int) ? {
dd.remove_image(image_id)?
println('Uploading logs to Vieter...')
c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code,
c.add_build_log(target.id, res.start_time, res.end_time, build_arch, res.exit_code,
res.logs)?
}

108
src/console/git/git.v → src/console/targets/targets.v
View File

@ -1,11 +1,11 @@
module git
module targets
import cli
import env
import vieter_v.conf as vconf
import cron.expression { parse_expression }
import client
import client { NewTarget }
import console
import models { GitRepoFilter }
import models { TargetFilter }
struct Config {
address string [required]
@ -16,12 +16,12 @@ struct Config {
// cmd returns the cli submodule that handles the repos API interaction
pub fn cmd() cli.Command {
return cli.Command{
name: 'repos'
description: 'Interact with the repos API.'
name: 'targets'
description: 'Interact with the targets API.'
commands: [
cli.Command{
name: 'list'
description: 'List the current repos.'
description: 'List the current targets.'
flags: [
cli.Flag{
name: 'limit'
@ -35,15 +35,15 @@ pub fn cmd() cli.Command {
},
cli.Flag{
name: 'repo'
description: 'Only return Git repos that publish to this repo.'
description: 'Only return targets that publish to this repo.'
flag: cli.FlagType.string
},
]
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file')?
conf := env.load<Config>(config_file)?
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
mut filter := GitRepoFilter{}
mut filter := TargetFilter{}
limit := cmd.flags.get_int('limit')?
if limit != 0 {
@ -60,29 +60,53 @@ pub fn cmd() cli.Command {
filter.repo = repo
}
list(conf, filter)?
raw := cmd.flags.get_bool('raw')?
list(conf, filter, raw)?
}
},
cli.Command{
name: 'add'
required_args: 3
usage: 'url branch repo'
description: 'Add a new repository.'
required_args: 2
usage: 'url repo'
description: 'Add a new target with the given URL & target repo.'
flags: [
cli.Flag{
name: 'kind'
description: "Kind of target to add. Defaults to 'git' if not specified. One of 'git', 'url'."
flag: cli.FlagType.string
default_value: ['git']
},
cli.Flag{
name: 'branch'
description: "Which branch to clone; only applies to kind 'git'."
flag: cli.FlagType.string
},
]
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file')?
conf := env.load<Config>(config_file)?
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
add(conf, cmd.args[0], cmd.args[1], cmd.args[2])?
t := NewTarget{
kind: cmd.flags.get_string('kind')?
url: cmd.args[0]
repo: cmd.args[1]
branch: cmd.flags.get_string('branch') or { '' }
}
raw := cmd.flags.get_bool('raw')?
add(conf, t, raw)?
}
},
cli.Command{
name: 'remove'
required_args: 1
usage: 'id'
description: 'Remove a repository that matches the given ID prefix.'
description: 'Remove a target that matches the given id.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file')?
conf := env.load<Config>(config_file)?
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
remove(conf, cmd.args[0])?
}
@ -91,10 +115,10 @@ pub fn cmd() cli.Command {
name: 'info'
required_args: 1
usage: 'id'
description: 'Show detailed information for the repo matching the ID prefix.'
description: 'Show detailed information for the target matching the id.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file')?
conf := env.load<Config>(config_file)?
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
info(conf, cmd.args[0])?
}
@ -103,11 +127,11 @@ pub fn cmd() cli.Command {
name: 'edit'
required_args: 1
usage: 'id'
description: 'Edit the repository that matches the given ID prefix.'
description: 'Edit the target that matches the given id.'
flags: [
cli.Flag{
name: 'url'
description: 'URL of the Git repository.'
description: 'URL value. Meaning depends on kind of target.'
flag: cli.FlagType.string
},
cli.Flag{
@ -130,10 +154,15 @@ pub fn cmd() cli.Command {
description: 'Cron schedule for repository.'
flag: cli.FlagType.string
},
cli.Flag{
name: 'kind'
description: 'Kind of target.'
flag: cli.FlagType.string
},
]
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file')?
conf := env.load<Config>(config_file)?
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
found := cmd.flags.get_all_found()
@ -152,10 +181,10 @@ pub fn cmd() cli.Command {
name: 'build'
required_args: 1
usage: 'id'
description: 'Build the repo with the given id & publish it.'
description: 'Build the target with the given id & publish it.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file')?
conf := env.load<Config>(config_file)?
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
build(conf, cmd.args[0].int())?
}
@ -168,30 +197,37 @@ pub fn cmd() cli.Command {
// ID. If multiple or none are found, an error is raised.
// list prints out a list of all repositories.
fn list(conf Config, filter GitRepoFilter) ? {
fn list(conf Config, filter TargetFilter, raw bool) ? {
c := client.new(conf.address, conf.api_key)
repos := c.get_git_repos(filter)?
data := repos.map([it.id.str(), it.url, it.branch, it.repo])
repos := c.get_targets(filter)?
data := repos.map([it.id.str(), it.kind, it.url, it.repo])
println(console.pretty_table(['id', 'url', 'branch', 'repo'], data)?)
if raw {
println(console.tabbed_table(data))
} else {
println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?)
}
}
// add adds a new repository to the server's list.
fn add(conf Config, url string, branch string, repo string) ? {
fn add(conf Config, t &NewTarget, raw bool) ? {
c := client.new(conf.address, conf.api_key)
res := c.add_git_repo(url, branch, repo, [])?
res := c.add_target(t)?
println(res.message)
if raw {
println(res.data)
} else {
println('Target added with id $res.data')
}
}
// remove removes a repository from the server's list.
fn remove(conf Config, id string) ? {
// id, _ := get_repo_by_prefix(conf, id_prefix) ?
id_int := id.int()
if id_int != 0 {
c := client.new(conf.address, conf.api_key)
res := c.remove_git_repo(id_int)?
res := c.remove_target(id_int)?
println(res.message)
}
}
@ -209,7 +245,7 @@ fn patch(conf Config, id string, params map[string]string) ? {
id_int := id.int()
if id_int != 0 {
c := client.new(conf.address, conf.api_key)
res := c.patch_git_repo(id_int, params)?
res := c.patch_target(id_int, params)?
println(res.message)
}
@ -224,6 +260,6 @@ fn info(conf Config, id string) ? {
}
c := client.new(conf.address, conf.api_key)
repo := c.get_git_repo(id_int)?
repo := c.get_target(id_int)?
println(repo)
}

4
src/cron/cli.v
View File

@ -1,7 +1,7 @@
module cron
import cli
import env
import vieter_v.conf as vconf
struct Config {
pub:
@ -24,7 +24,7 @@ pub fn cmd() cli.Command {
description: 'Start the cron service that periodically runs builds.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file')?
conf := env.load<Config>(config_file)?
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
cron(conf)?
}

20
src/cron/daemon/build.v
View File

@ -71,29 +71,31 @@ fn (mut d Daemon) start_build(sb ScheduledBuild) bool {
return false
}
// run_build actually starts the build process for a given repo.
// run_build actually starts the build process for a given target.
fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) {
d.linfo('started build: $sb.repo.url $sb.repo.branch')
d.linfo('started build: $sb.target.url -> $sb.target.repo')
// 0 means success, 1 means failure
mut status := 0
res := build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(),
&sb.repo) or {
d.ldebug('build_repo error: $err.msg()')
res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(),
&sb.target) or {
d.ldebug('build_target error: $err.msg()')
status = 1
build.BuildResult{}
}
if status == 0 {
d.linfo('finished build: $sb.repo.url $sb.repo.branch; uploading logs...')
d.linfo('finished build: $sb.target.url -> $sb.target.repo; uploading logs...')
build_arch := os.uname().machine
d.client.add_build_log(sb.repo.id, res.start_time, res.end_time, build_arch, res.exit_code,
res.logs) or { d.lerror('Failed to upload logs for $sb.repo.url $sb.repo.arch') }
d.client.add_build_log(sb.target.id, res.start_time, res.end_time, build_arch,
res.exit_code, res.logs) or {
d.lerror('Failed to upload logs for build: $sb.target.url -> $sb.target.repo')
}
} else {
d.linfo('failed build: $sb.repo.url $sb.repo.branch')
d.linfo('an error occured during build: $sb.target.url -> $sb.target.repo')
}
stdatomic.store_u64(&d.atomics[build_index], daemon.build_done)

69
src/cron/daemon/daemon.v
View File

@ -6,10 +6,10 @@ import datatypes { MinHeap }
import cron.expression { CronExpression, parse_expression }
import math
import build
import docker
import vieter_v.docker
import os
import client
import models { GitRepo }
import models { Target }
const (
// How many seconds to wait before retrying to update API if failed
@ -20,7 +20,7 @@ const (
struct ScheduledBuild {
pub:
repo GitRepo
target Target
timestamp time.Time
}
@ -37,9 +37,9 @@ mut:
global_schedule CronExpression
api_update_frequency int
image_rebuild_frequency int
// Repos currently loaded from API.
repos []GitRepo
// At what point to update the list of repositories.
// Targets currently loaded from API.
targets []Target
// At what point to update the list of targets.
api_update_timestamp time.Time
image_build_timestamp time.Time
queue MinHeap<ScheduledBuild>
@ -51,7 +51,7 @@ mut:
logger shared log.Log
}
// init_daemon initializes a new Daemon object. It renews the repositories &
// init_daemon initializes a new Daemon object. It renews the targets &
// populates the build queue for the first time.
pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon {
mut d := Daemon{
@ -65,8 +65,8 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st
logger: logger
}
// Initialize the repos & queue
d.renew_repos()
// Initialize the targets & queue
d.renew_targets()
d.renew_queue()
if !d.rebuild_base_image() {
return error('The base image failed to build. The Vieter cron daemon cannot run without an initial builder image.')
@ -76,21 +76,21 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st
}
// run starts the actual daemon process. It runs builds when possible &
// periodically refreshes the list of repositories to ensure we stay in sync.
// periodically refreshes the list of targets to ensure we stay in sync.
pub fn (mut d Daemon) run() {
for {
finished_builds := d.clean_finished_builds()
// Update the API's contents if needed & renew the queue
if time.now() >= d.api_update_timestamp {
d.renew_repos()
d.renew_targets()
d.renew_queue()
}
// The finished builds should only be rescheduled if the API contents
// haven't been renewed.
else {
for sb in finished_builds {
d.schedule_build(sb.repo)
d.schedule_build(sb.target)
}
}
@ -114,7 +114,7 @@ pub fn (mut d Daemon) run() {
// every second to clean up any finished builds & start new ones.
mut delay := time.Duration(1 * time.second)
// Sleep either until we have to refresh the repos or when the next
// Sleep either until we have to refresh the targets or when the next
// build has to start, with a minimum of 1 second.
if d.current_build_count() == 0 {
now := time.now()
@ -148,12 +148,13 @@ pub fn (mut d Daemon) run() {
}
}
// schedule_build adds the next occurence of the given repo build to the queue.
fn (mut d Daemon) schedule_build(repo GitRepo) {
ce := if repo.schedule != '' {
parse_expression(repo.schedule) or {
// schedule_build adds the next occurence of the given targets build to the
// queue.
fn (mut d Daemon) schedule_build(target Target) {
ce := if target.schedule != '' {
parse_expression(target.schedule) or {
// TODO This shouldn't return an error if the expression is empty.
d.lerror("Error while parsing cron expression '$repo.schedule' (id $repo.id): $err.msg()")
d.lerror("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()")
d.global_schedule
}
@ -161,41 +162,41 @@ fn (mut d Daemon) schedule_build(repo GitRepo) {
d.global_schedule
}
// A repo that can't be scheduled will just be skipped for now
// A target that can't be scheduled will just be skipped for now
timestamp := ce.next_from_now() or {
d.lerror("Couldn't calculate next timestamp from '$repo.schedule'; skipping")
d.lerror("Couldn't calculate next timestamp from '$target.schedule'; skipping")
return
}
d.queue.insert(ScheduledBuild{
repo: repo
target: target
timestamp: timestamp
})
}
// renew_repos requests the newest list of Git repos from the server & replaces
// renew_targets requests the newest list of targets from the server & replaces
// the old one.
fn (mut d Daemon) renew_repos() {
d.linfo('Renewing repos...')
fn (mut d Daemon) renew_targets() {
d.linfo('Renewing targets...')
mut new_repos := d.client.get_all_git_repos() or {
d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...')
mut new_targets := d.client.get_all_targets() or {
d.lerror('Failed to renew targets. Retrying in ${daemon.api_update_retry_timeout}s...')
d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout)
return
}
// Filter out any repos that shouldn't run on this architecture
// Filter out any targets that shouldn't run on this architecture
cur_arch := os.uname().machine
new_repos = new_repos.filter(it.arch.any(it.value == cur_arch))
new_targets = new_targets.filter(it.arch.any(it.value == cur_arch))
d.repos = new_repos
d.targets = new_targets
d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency)
}
// renew_queue replaces the old queue with a new one that reflects the newest
// values in repos_map.
// values in targets.
fn (mut d Daemon) renew_queue() {
d.linfo('Renewing queue...')
mut new_queue := MinHeap<ScheduledBuild>{}
@ -225,10 +226,10 @@ fn (mut d Daemon) renew_queue() {
d.queue = new_queue
// For each repository in repos_map, parse their cron expression (or use
// the default one if not present) & add them to the queue
for repo in d.repos {
d.schedule_build(repo)
// For each target in targets, parse their cron expression (or use the
// default one if not present) & add them to the queue
for target in d.targets {
d.schedule_build(target)
}
}

14
src/db/db.v
View File

@ -3,7 +3,7 @@ module db
import sqlite
import time
struct VieterDb {
pub struct VieterDb {
conn sqlite.DB
}
@ -13,8 +13,16 @@ struct MigrationVersion {
}
const (
migrations_up = [$embed_file('migrations/001-initial/up.sql')]
migrations_down = [$embed_file('migrations/001-initial/down.sql')]
migrations_up = [
$embed_file('migrations/001-initial/up.sql'),
$embed_file('migrations/002-rename-to-targets/up.sql'),
$embed_file('migrations/003-target-url-type/up.sql'),
]
migrations_down = [
$embed_file('migrations/001-initial/down.sql'),
$embed_file('migrations/002-rename-to-targets/down.sql'),
$embed_file('migrations/003-target-url-type/down.sql'),
]
)
// init initializes a database & adds the correct tables.

99
src/db/git.v
View File

@ -1,99 +0,0 @@
module db
import models { GitRepo, GitRepoArch, GitRepoFilter }
// get_git_repos returns all GitRepo's in the database.
pub fn (db &VieterDb) get_git_repos(filter GitRepoFilter) []GitRepo {
// This seems to currently be blocked by a bug in the ORM, I'll have to ask
// around.
if filter.repo != '' {
res := sql db.conn {
select from GitRepo where repo == filter.repo order by id limit filter.limit offset filter.offset
}
return res
}
res := sql db.conn {
select from GitRepo order by id limit filter.limit offset filter.offset
}
return res
}
// get_git_repo tries to return a specific GitRepo.
pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo {
res := sql db.conn {
select from GitRepo where id == repo_id
}
// If a select statement fails, it returns a zeroed object. By
// checking one of the required fields, we can see whether the query
// returned a result or not.
if res.id == 0 {
return none
}
return res
}
// add_git_repo inserts the given GitRepo into the database.
pub fn (db &VieterDb) add_git_repo(repo GitRepo) {
sql db.conn {
insert repo into GitRepo
}
}
// delete_git_repo deletes the repo with the given ID from the database.
pub fn (db &VieterDb) delete_git_repo(repo_id int) {
sql db.conn {
delete from GitRepo where id == repo_id
delete from GitRepoArch where repo_id == repo_id
}
}
// update_git_repo updates any non-array values for a given GitRepo.
pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) {
mut values := []string{}
// TODO does this allow for SQL injection?
$for field in GitRepo.fields {
if field.name in params {
// Any fields that are array types require their own update method
$if field.typ is string {
values << "$field.name = '${params[field.name]}'"
}
}
}
values_str := values.join(', ')
// I think this is actual SQL & not the ORM language
query := 'update GitRepo set $values_str where id == $repo_id'
db.conn.exec_none(query)
}
// update_git_repo_archs updates a given GitRepo's arch value.
pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) {
archs_with_id := archs.map(GitRepoArch{
...it
repo_id: repo_id
})
sql db.conn {
delete from GitRepoArch where repo_id == repo_id
}
for arch in archs_with_id {
sql db.conn {
insert arch into GitRepoArch
}
}
}
// git_repo_exists is a utility function that checks whether a repo with the
// given id exists.
pub fn (db &VieterDb) git_repo_exists(repo_id int) bool {
db.get_git_repo(repo_id) or { return false }
return true
}

18
src/db/logs.v
View File

@ -7,8 +7,8 @@ import time
pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog {
mut where_parts := []string{}
if filter.repo != 0 {
where_parts << 'repo_id == $filter.repo'
if filter.target != 0 {
where_parts << 'target_id == $filter.target'
}
if filter.before != time.Time{} {
@ -55,11 +55,11 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog {
return res
}
// get_build_logs_for_repo returns all BuildLog's in the database for a given
// repo.
pub fn (db &VieterDb) get_build_logs_for_repo(repo_id int) []BuildLog {
// get_build_logs_for_target returns all BuildLog's in the database for a given
// target.
pub fn (db &VieterDb) get_build_logs_for_target(target_id int) []BuildLog {
res := sql db.conn {
select from BuildLog where repo_id == repo_id order by id
select from BuildLog where target_id == target_id order by id
}
return res
@ -79,10 +79,14 @@ pub fn (db &VieterDb) get_build_log(id int) ?BuildLog {
}
// add_build_log inserts the given BuildLog into the database.
pub fn (db &VieterDb) add_build_log(log BuildLog) {
pub fn (db &VieterDb) add_build_log(log BuildLog) int {
sql db.conn {
insert log into BuildLog
}
inserted_id := db.conn.last_id() as int
return inserted_id
}
// delete_build_log delete the BuildLog with the given ID from the database.

5
src/db/migrations/002-rename-to-targets/down.sql 100644
View File

@ -0,0 +1,5 @@
ALTER TABLE Target RENAME TO GitRepo;
ALTER TABLE TargetArch RENAME TO GitRepoArch;
ALTER TABLE GitRepoArch RENAME COLUMN target_id TO repo_id;
ALTER TABLE BuildLog RENAME COLUMN target_id TO repo_id;

5
src/db/migrations/002-rename-to-targets/up.sql 100644
View File

@ -0,0 +1,5 @@
ALTER TABLE GitRepo RENAME TO Target;
ALTER TABLE GitRepoArch RENAME TO TargetArch;
ALTER TABLE TargetArch RENAME COLUMN repo_id TO target_id;
ALTER TABLE BuildLog RENAME COLUMN repo_id TO target_id;

4
src/db/migrations/003-target-url-type/down.sql 100644
View File

@ -0,0 +1,4 @@
-- I'm not sure whether I should remove any non-git targets here. Keeping them
-- will result in invalid targets, but removing them means losing data.
ALTER TABLE Target DROP COLUMN kind;

1
src/db/migrations/003-target-url-type/up.sql 100644
View File

@ -0,0 +1 @@
ALTER TABLE Target ADD COLUMN kind TEXT NOT NULL DEFAULT 'git';

103
src/db/targets.v 100644
View File

@ -0,0 +1,103 @@
module db
import models { Target, TargetArch, TargetFilter }
// get_targets returns all targets in the database.
pub fn (db &VieterDb) get_targets(filter TargetFilter) []Target {
// This seems to currently be blocked by a bug in the ORM, I'll have to ask
// around.
if filter.repo != '' {
res := sql db.conn {
select from Target where repo == filter.repo order by id limit filter.limit offset filter.offset
}
return res
}
res := sql db.conn {
select from Target order by id limit filter.limit offset filter.offset
}
return res
}
// get_target tries to return a specific target.
pub fn (db &VieterDb) get_target(target_id int) ?Target {
res := sql db.conn {
select from Target where id == target_id
}
// If a select statement fails, it returns a zeroed object. By
// checking one of the required fields, we can see whether the query
// returned a result or not.
if res.id == 0 {
return none
}
return res
}
// add_target inserts the given target into the database.
pub fn (db &VieterDb) add_target(repo Target) int {
sql db.conn {
insert repo into Target
}
inserted_id := db.conn.last_id() as int
return inserted_id
}
// delete_target deletes the target with the given id from the database.
pub fn (db &VieterDb) delete_target(target_id int) {
sql db.conn {
delete from Target where id == target_id
delete from TargetArch where target_id == target_id
}
}
// update_target updates any non-array values for a given target.
pub fn (db &VieterDb) update_target(target_id int, params map[string]string) {
mut values := []string{}
// TODO does this allow for SQL injection?
$for field in Target.fields {
if field.name in params {
// Any fields that are array types require their own update method
$if field.typ is string {
values << "$field.name = '${params[field.name]}'"
}
}
}
values_str := values.join(', ')
// I think this is actual SQL & not the ORM language
query := 'update Target set $values_str where id == $target_id'
db.conn.exec_none(query)
}
// update_target_archs updates a given target's arch value.
pub fn (db &VieterDb) update_target_archs(target_id int, archs []TargetArch) {
archs_with_id := archs.map(TargetArch{
...it
target_id: target_id
})
sql db.conn {
delete from TargetArch where target_id == target_id
}
for arch in archs_with_id {
sql db.conn {
insert arch into TargetArch
}
}
}
// target_exists is a utility function that checks whether a target with the
// given id exists.
pub fn (db &VieterDb) target_exists(target_id int) bool {
db.get_target(target_id) or { return false }
return true
}

3
src/docker/README.md
View File

@ -1,3 +0,0 @@
This module implements part of the Docker Engine API v1.41
([documentation](https://docs.docker.com/engine/api/v1.41/)) using socket-based
HTTP communication.

123
src/docker/containers.v
View File

@ -1,123 +0,0 @@
module docker
import json
import net.urllib
import time
import net.http { Method }
struct DockerError {
message string
}
pub struct NewContainer {
image string [json: Image]
entrypoint []string [json: Entrypoint]
cmd []string [json: Cmd]
env []string [json: Env]
work_dir string [json: WorkingDir]
user string [json: User]
}
struct CreatedContainer {
pub:
id string [json: Id]
warnings []string [json: Warnings]
}
// create_container creates a new container with the given config.
pub fn (mut d DockerConn) create_container(c NewContainer) ?CreatedContainer {
d.send_request_with_json(Method.post, urllib.parse('/v1.41/containers/create')?, c)?
head, res := d.read_response()?
if head.status_code != 201 {
data := json.decode(DockerError, res)?
return error(data.message)
}
data := json.decode(CreatedContainer, res)?
return data
}
// start_container starts the container with the given id.
pub fn (mut d DockerConn) start_container(id string) ? {
d.send_request(Method.post, urllib.parse('/v1.41/containers/$id/start')?)?
head, body := d.read_response()?
if head.status_code != 204 {
data := json.decode(DockerError, body)?
return error(data.message)
}
}
struct ContainerInspect {
pub mut:
state ContainerState [json: State]
}
struct ContainerState {
pub:
running bool [json: Running]
status string [json: Status]
exit_code int [json: ExitCode]
// These use a rather specific format so they have to be parsed later
start_time_str string [json: StartedAt]
end_time_str string [json: FinishedAt]
pub mut:
start_time time.Time [skip]
end_time time.Time [skip]
}
// inspect_container returns detailed information for a given container.
pub fn (mut d DockerConn) inspect_container(id string) ?ContainerInspect {
d.send_request(Method.get, urllib.parse('/v1.41/containers/$id/json')?)?
head, body := d.read_response()?
if head.status_code != 200 {
data := json.decode(DockerError, body)?
return error(data.message)
}
mut data := json.decode(ContainerInspect, body)?
// The Docker engine API *should* always return UTC time.
data.state.start_time = time.parse_rfc3339(data.state.start_time_str)?
if data.state.status == 'exited' {
data.state.end_time = time.parse_rfc3339(data.state.end_time_str)?
}
return data
}
// remove_container removes the container with the given id.
pub fn (mut d DockerConn) remove_container(id string) ? {
d.send_request(Method.delete, urllib.parse('/v1.41/containers/$id')?)?
head, body := d.read_response()?
if head.status_code != 204 {
data := json.decode(DockerError, body)?
return error(data.message)
}
}
// get_container_logs returns a reader object allowing access to the
// container's logs.
pub fn (mut d DockerConn) get_container_logs(id string) ?&StreamFormatReader {
d.send_request(Method.get, urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true')?)?
head := d.read_response_head()?
if head.status_code != 200 {
content_length := head.header.get(http.CommonHeader.content_length)?.int()
body := d.read_response_body(content_length)?
data := json.decode(DockerError, body)?
return error(data.message)
}
return d.get_stream_format_reader()
}

137
src/docker/docker.v
View File

@ -1,137 +0,0 @@
module docker
import net.unix
import io
import net.http
import strings
import net.urllib
import json
import util
const (
socket = '/var/run/docker.sock'
buf_len = 10 * 1024
http_separator = [u8(`\r`), `\n`, `\r`, `\n`]
http_chunk_separator = [u8(`\r`), `\n`]
)
pub struct DockerConn {
mut:
socket &unix.StreamConn
reader &io.BufferedReader
}
// new_conn creates a new connection to the Docker daemon.
pub fn new_conn() ?&DockerConn {
s := unix.connect_stream(docker.socket)?
d := &DockerConn{
socket: s
reader: io.new_buffered_reader(reader: s)
}
return d
}
// close closes the underlying socket connection.
pub fn (mut d DockerConn) close() ? {
d.socket.close()?
}
// send_request sends an HTTP request without body.
pub fn (mut d DockerConn) send_request(method http.Method, url urllib.URL) ? {
req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n'
d.socket.write_string(req)?
// When starting a new request, the reader needs to be reset.
d.reader = io.new_buffered_reader(reader: d.socket)
}
// send_request_with_body sends an HTTP request with the given body.
pub fn (mut d DockerConn) send_request_with_body(method http.Method, url urllib.URL, content_type string, body string) ? {
req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n'
d.socket.write_string(req)?
// When starting a new request, the reader needs to be reset.
d.reader = io.new_buffered_reader(reader: d.socket)
}
// send_request_with_json<T> is a convenience wrapper around
// send_request_with_body that encodes the input as JSON.
pub fn (mut d DockerConn) send_request_with_json<T>(method http.Method, url urllib.URL, data &T) ? {
body := json.encode(data)
return d.send_request_with_body(method, url, 'application/json', body)
}
// read_response_head consumes the socket's contents until it encounters
// '\r\n\r\n', after which it parses the response as an HTTP response.
// Importantly, this function never consumes the reader past the HTTP
// separator, so the body can be read fully later on.
pub fn (mut d DockerConn) read_response_head() ?http.Response {
mut res := []u8{}
util.read_until_separator(mut d.reader, mut res, docker.http_separator)?
return http.parse_response(res.bytestr())
}
// read_response_body reads `length` bytes from the stream. It can be used when
// the response encoding isn't chunked to fully read it.
pub fn (mut d DockerConn) read_response_body(length int) ?string {
if length == 0 {
return ''
}
mut buf := []u8{len: docker.buf_len}
mut c := 0
mut builder := strings.new_builder(docker.buf_len)
for builder.len < length {
c = d.reader.read(mut buf) or { break }
builder.write(buf[..c])?
}
return builder.str()
}
// read_response is a convenience function which always consumes the entire
// response & returns it. It should only be used when we're certain that the
// result isn't too large.
pub fn (mut d DockerConn) read_response() ?(http.Response, string) {
head := d.read_response_head()?
if head.header.get(http.CommonHeader.transfer_encoding) or { '' } == 'chunked' {
mut builder := strings.new_builder(1024)
mut body := d.get_chunked_response_reader()
util.reader_to_writer(mut body, mut builder)?
return head, builder.str()
}
content_length := head.header.get(http.CommonHeader.content_length)?.int()
res := d.read_response_body(content_length)?
return head, res
}
// get_chunked_response_reader returns a ChunkedResponseReader using the socket
// as reader.
pub fn (mut d DockerConn) get_chunked_response_reader() &ChunkedResponseReader {
r := new_chunked_response_reader(d.reader)
return r
}
// get_stream_format_reader returns a StreamFormatReader using the socket as
// reader.
pub fn (mut d DockerConn) get_stream_format_reader() &StreamFormatReader {
r := new_chunked_response_reader(d.reader)
r2 := new_stream_format_reader(r)
return r2
}

61
src/docker/images.v
View File

@ -1,61 +0,0 @@
module docker
import net.http { Method }
import net.urllib
import json
struct Image {
pub:
id string [json: Id]
}
// pull_image pulls the given image:tag.
pub fn (mut d DockerConn) pull_image(image string, tag string) ? {
d.send_request(Method.post, urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?)?
head := d.read_response_head()?
if head.status_code != 200 {
content_length := head.header.get(http.CommonHeader.content_length)?.int()
body := d.read_response_body(content_length)?
data := json.decode(DockerError, body)?
return error(data.message)
}
// Keep reading the body until the pull has completed
mut body := d.get_chunked_response_reader()
mut buf := []u8{len: 1024}
for {
body.read(mut buf) or { break }
}
}
// create_image_from_container creates a new image from a container.
pub fn (mut d DockerConn) create_image_from_container(id string, repo string, tag string) ?Image {
d.send_request(Method.post, urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag')?)?
head, body := d.read_response()?
if head.status_code != 201 {
data := json.decode(DockerError, body)?
return error(data.message)
}
data := json.decode(Image, body)?
return data
}
// remove_image removes the image with the given id.
pub fn (mut d DockerConn) remove_image(id string) ? {
d.send_request(Method.delete, urllib.parse('/v1.41/images/$id')?)?
head, body := d.read_response()?
if head.status_code != 200 {
data := json.decode(DockerError, body)?
return error(data.message)
}
}

135
src/docker/stream.v
View File

@ -1,135 +0,0 @@
module docker
import io
import util
import encoding.binary
import encoding.hex
// ChunkedResponseReader parses an underlying HTTP chunked response, exposing
// it as if it was a continuous stream of data.
struct ChunkedResponseReader {
mut:
reader io.BufferedReader
bytes_left_in_chunk u64
started bool
}
// new_chunked_response_reader creates a new ChunkedResponseReader on the heap
// with the provided reader.
pub fn new_chunked_response_reader(reader io.BufferedReader) &ChunkedResponseReader {
r := &ChunkedResponseReader{
reader: reader
}
return r
}
// read satisfies the io.Reader interface.
pub fn (mut r ChunkedResponseReader) read(mut buf []u8) ?int {
if r.bytes_left_in_chunk == 0 {
// An io.BufferedReader always returns none if its stream has
// ended.
r.bytes_left_in_chunk = r.read_chunk_size()?
}
mut c := 0
// Make sure we don't read more than we can safely read. This is to avoid
// the underlying reader from becoming out of sync with our parsing:
if buf.len > r.bytes_left_in_chunk {
c = r.reader.read(mut buf[..r.bytes_left_in_chunk])?
} else {
c = r.reader.read(mut buf)?
}
r.bytes_left_in_chunk -= u64(c)
return c
}
// read_chunk_size advances the reader & reads the size of the next HTTP chunk.
// This function should only be called if the previous chunk has been
// completely consumed.
fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 {
if r.started {
mut buf := []u8{len: 2}
// Each chunk ends with a `\r\n` which we want to skip first
r.reader.read(mut buf)?
}
r.started = true
mut res := []u8{}
util.read_until_separator(mut r.reader, mut res, http_chunk_separator)?
// The length of the next chunk is provided as a hexadecimal
mut num_data := hex.decode(res#[..-2].bytestr())?
for num_data.len < 8 {
num_data.insert(0, 0)
}
num := binary.big_endian_u64(num_data)
// This only occurs for the very last chunk, which always reports a size of
// 0.
if num == 0 {
return none
}
return num
}
// StreamFormatReader parses an underlying stream of Docker logs, removing the
// header bytes.
struct StreamFormatReader {
mut:
reader ChunkedResponseReader
bytes_left_in_chunk u32
}
// new_stream_format_reader creates a new StreamFormatReader using the given
// reader.
pub fn new_stream_format_reader(reader ChunkedResponseReader) &StreamFormatReader {
r := &StreamFormatReader{
reader: reader
}
return r
}
// read satisfies the io.Reader interface.
pub fn (mut r StreamFormatReader) read(mut buf []u8) ?int {
if r.bytes_left_in_chunk == 0 {
r.bytes_left_in_chunk = r.read_chunk_size()?
}
mut c := 0
if buf.len > r.bytes_left_in_chunk {
c = r.reader.read(mut buf[..r.bytes_left_in_chunk])?
} else {
c = r.reader.read(mut buf)?
}
r.bytes_left_in_chunk -= u32(c)
return c
}
// read_chunk_size advances the reader & reads the header bytes for the length
// of the next chunk.
fn (mut r StreamFormatReader) read_chunk_size() ?u32 {
mut buf := []u8{len: 8}
r.reader.read(mut buf)?
num := binary.big_endian_u32(buf[4..])
if num == 0 {
return none
}
return num
}

7
src/env/README.md vendored
View File

@ -1,7 +0,0 @@
This module provides a framework for parsing a configuration, defined as a
struct, from both a TOML configuration file & environment variables. Some
notable features are:
* Overwrite values in config file using environment variables
* Allow default values in config struct
* Read environment variable value from file

102
src/env/env.v vendored
View File

@ -1,102 +0,0 @@
module env
import os
import toml
const (
// The prefix that every environment variable should have
prefix = 'VIETER_'
// The suffix an environment variable in order for it to be loaded from a file
// instead
file_suffix = '_FILE'
)
// get_env_var tries to read the contents of the given environment variable. It
// looks for either `${env.prefix}${field_name.to_upper()}` or
// `${env.prefix}${field_name.to_upper()}${env.file_suffix}`, returning the
// contents of the file instead if the latter. If both or neither exist, the
// function returns an error.
fn get_env_var(field_name string) ?string {
env_var_name := '$env.prefix$field_name.to_upper()'
env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix'
env_var := os.getenv(env_var_name)
env_file := os.getenv(env_file_name)
// If both are missing, we return an empty string
if env_var == '' && env_file == '' {
return ''
}
// If they're both set, we report a conflict
if env_var != '' && env_file != '' {
return error('Only one of $env_var_name or $env_file_name can be defined.')
}
// If it's the env var itself, we return it.
// I'm pretty sure this also prevents variable ending in _FILE (e.g.
// VIETER_LOG_FILE) from being mistakingely read as an _FILE suffixed env
// var.
if env_var != '' {
return env_var
}
// Otherwise, we process the file
return os.read_file(env_file) or {
error('Failed to read file defined in $env_file_name: ${err.msg()}.')
}
}
// load<T> attempts to create an object of type T from the given path to a toml
// file & environment variables. For each field, it will select either a value
// given from an environment variable, a value defined in the config file or a
// configured default if present, in that order.
pub fn load<T>(path string) ?T {
mut res := T{}
if os.exists(path) {
// We don't use reflect here because reflect also sets any fields not
// in the toml back to their zero value, which we don't want
doc := toml.parse_file(path)?
$for field in T.fields {
s := doc.value(field.name)
if s !is toml.Null {
$if field.typ is string {
res.$(field.name) = s.string()
} $else $if field.typ is int {
res.$(field.name) = s.int()
}
}
}
}
$for field in T.fields {
env_value := get_env_var(field.name)?
// The value of an env var will always take precedence over the toml
// file.
if env_value != '' {
$if field.typ is string {
res.$(field.name) = env_value
} $else $if field.typ is int {
res.$(field.name) = env_value.int()
}
}
// Now, we check whether a value is present. If there isn't, that means
// it isn't in the config file, nor is there a default or an env var.
mut has_value := false
$if field.typ is string {
has_value = res.$(field.name) != ''
} $else $if field.typ is int {
has_value = res.$(field.name) != 0
}
if !has_value {
return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.")
}
}
return res
}

15
src/main.v
View File

@ -3,17 +3,18 @@ module main
import os
import server
import cli
import console.git
import console.targets
import console.logs
import console.schedule
import console.man
import console.aur
import cron
fn main() {
mut app := cli.Command{
name: 'vieter'
description: 'Vieter is a lightweight implementation of an Arch repository server.'
version: '0.3.0'
version: '0.4.0'
flags: [
cli.Flag{
flag: cli.FlagType.string
@ -23,14 +24,22 @@ fn main() {
global: true
default_value: [os.expand_tilde_to_home('~/.vieterrc')]
},
cli.Flag{
flag: cli.FlagType.bool
name: 'raw'
abbrev: 'r'
description: 'Only output minimal information (no formatted tables, etc.)'
global: true
},
]
commands: [
server.cmd(),
git.cmd(),
targets.cmd(),
cron.cmd(),
logs.cmd(),
schedule.cmd(),
man.cmd(),
aur.cmd(),
]
}
app.setup()

6
src/models/logs.v
View File

@ -5,7 +5,7 @@ import time
pub struct BuildLog {
pub mut:
id int [primary; sql: serial]
repo_id int [nonull]
target_id int [nonull]
start_time time.Time [nonull]
end_time time.Time [nonull]
arch string [nonull]
@ -16,7 +16,7 @@ pub mut:
pub fn (bl &BuildLog) str() string {
mut parts := [
'id: $bl.id',
'repo id: $bl.repo_id',
'target id: $bl.target_id',
'start time: $bl.start_time.local()',
'end time: $bl.end_time.local()',
'duration: ${bl.end_time - bl.start_time}',
@ -33,7 +33,7 @@ pub struct BuildLogFilter {
pub mut:
limit u64 = 25
offset u64
repo int
target int
before time.Time
after time.Time
arch string

4
src/models/models.v
View File

@ -23,8 +23,8 @@ pub fn patch_from_params<T>(mut o T, params map[string]string) ? {
o.$(field.name) = params[field.name].int()
} $else $if field.typ is u64 {
o.$(field.name) = params[field.name].u64()
} $else $if field.typ is []GitRepoArch {
o.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it })
} $else $if field.typ is []TargetArch {
o.$(field.name) = params[field.name].split(',').map(TargetArch{ value: it })
} $else $if field.typ is time.Time {
o.$(field.name) = time.unix(params[field.name].int())
} $else $if field.typ is []string {

26
src/models/git.v → src/models/targets.v
View File

@ -1,37 +1,43 @@
module models
pub struct GitRepoArch {
pub const valid_kinds = ['git', 'url']
pub struct TargetArch {
pub:
id int [primary; sql: serial]
repo_id int [nonull]
target_id int [nonull]
value string [nonull]
}
// str returns a string representation.
pub fn (gra &GitRepoArch) str() string {
pub fn (gra &TargetArch) str() string {
return gra.value
}
pub struct GitRepo {
pub struct Target {
pub mut:
id int [primary; sql: serial]
// URL of the Git repository
kind string [nonull]
// If kind is git: URL of the Git repository
// If kind is url: URL to PKGBUILD file
url string [nonull]
// Branch of the Git repository to use
branch string [nonull]
// Branch of the Git repository to use; only applicable when kind is git.
// If not provided, the repository is cloned with the default branch.
branch string
// Which repo the builder should publish packages to
repo string [nonull]
// Cron schedule describing how frequently to build the repo.
schedule string
// On which architectures the package is allowed to be built. In reality,
// this controls which builders will periodically build the image.
arch []GitRepoArch [fkey: 'repo_id']
arch []TargetArch [fkey: 'target_id']
}
// str returns a string representation.
pub fn (gr &GitRepo) str() string {
pub fn (gr &Target) str() string {
mut parts := [
'id: $gr.id',
'kind: $gr.kind',
'url: $gr.url',
'branch: $gr.branch',
'repo: $gr.repo',
@ -44,7 +50,7 @@ pub fn (gr &GitRepo) str() string {
}
[params]
pub struct GitRepoFilter {
pub struct TargetFilter {
pub mut:
limit u64 = 25
offset u64

12
src/package/package.v
View File

@ -4,7 +4,7 @@ import os
import util
// Represents a read archive
struct Pkg {
pub struct Pkg {
pub:
path string [required]
info PkgInfo [required]
@ -42,8 +42,8 @@ pub mut:
checkdepends []string
}
// checksum calculates the md5 & sha256 hash of the package
pub fn (p &Pkg) checksum() ?(string, string) {
// checksum calculates the sha256 hash of the package
pub fn (p &Pkg) checksum() ?string {
return util.hash_file(p.path)
}
@ -201,8 +201,7 @@ pub fn (pkg &Pkg) filename() string {
}
// to_desc returns a desc file valid string representation
// TODO calculate md5 & sha256 instead of believing the file
pub fn (pkg &Pkg) to_desc() string {
pub fn (pkg &Pkg) to_desc() ?string {
p := pkg.info
// filename
@ -223,9 +222,8 @@ pub fn (pkg &Pkg) to_desc() string {
desc += format_entry('CSIZE', p.csize.str())
desc += format_entry('ISIZE', p.size.str())
md5sum, sha256sum := pkg.checksum() or { '', '' }
sha256sum := pkg.checksum()?
desc += format_entry('MD5SUM', md5sum)
desc += format_entry('SHA256SUM', sha256sum)
// TODO add pgpsig stuff

91
src/repo/repo.v → src/repo/add.v
View File

@ -23,8 +23,9 @@ pub:
pub struct RepoAddResult {
pub:
added bool [required]
pkg &package.Pkg [required]
name string
version string
archs []string
}
// new creates a new RepoGroupManager & creates the directories as needed
@ -53,10 +54,10 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re
return error('Failed to read package file: $err.msg()')
}
added := r.add_pkg_in_repo(repo, pkg)?
archs := r.add_pkg_in_repo(repo, pkg)?
// If the add was successful, we move the file to the packages directory
for arch in added {
for arch in archs {
repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch))
dest_path := os.join_path_single(repo_pkg_path, pkg.filename())
@ -71,8 +72,9 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re
os.rm(pkg_path)?
return RepoAddResult{
added: added.len > 0
pkg: &pkg
name: pkg.info.name
version: pkg.info.version
archs: archs
}
}
@ -87,11 +89,9 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin
// A package not of arch 'any' can be handled easily by adding it to the
// respective repo
if pkg.info.arch != 'any' {
if r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)? {
r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)?
return [pkg.info.arch]
} else {
return []
}
}
mut arch_repos := []string{}
@ -113,25 +113,22 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin
arch_repos << r.default_arch
}
mut added := []string{}
// We add the package to each repository. If any of the repositories
// return true, the result of the function is also true.
// Add the package to each found architecture
// NOTE: if any of these fail, the function fails. This means the user does
// not know which arch-repositories did succeed in adding the package, if
// any.
for arch in arch_repos {
if r.add_pkg_in_arch_repo(repo, arch, pkg)? {
added << arch
}
r.add_pkg_in_arch_repo(repo, arch, pkg)?
}
return added
return arch_repos
}
// add_pkg_in_arch_repo is the function that actually adds a package to a given
// arch-repo. It records the package's data in the arch-repo's desc & files
// files, and afterwards updates the db & files archives to reflect these
// changes. The function returns false if the package was already present in
// the repo, and true otherwise.
fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool {
// changes.
fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ? {
pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version')
// Remove the previous version of the package, if present
@ -139,7 +136,7 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac
os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') }
os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or {
os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()?) or {
os.rmdir_all(pkg_dir)?
return error('Failed to write desc file.')
@ -151,54 +148,4 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac
}
r.sync(repo, arch)?
return true
}
// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It
// returns false if the package wasn't present in the database. It also
// optionally re-syncs the repo archives.
fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool {
repo_dir := os.join_path(r.repos_dir, repo, arch)
// If the repository doesn't exist yet, the result is automatically false
if !os.exists(repo_dir) {
return false
}
// We iterate over every directory in the repo dir
// TODO filter so we only check directories
for d in os.ls(repo_dir)? {
// Because a repository only allows a single version of each package,
// we need only compare whether the name of the package is the same,
// not the version.
name := d.split('-')#[..-2].join('-')
if name == pkg_name {
// We lock the mutex here to prevent other routines from creating a
// new archive while we remove an entry
lock r.mutex {
os.rmdir_all(os.join_path_single(repo_dir, d))?
}
// Also remove the package archive
repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch)
archives := os.ls(repo_pkg_dir)?.filter(it.split('-')#[..-3].join('-') == name)
for archive_name in archives {
full_path := os.join_path_single(repo_pkg_dir, archive_name)
os.rm(full_path)?
}
// Sync the db archives if requested
if sync {
r.sync(repo, arch)?
}
return true
}
}
return false
}

85
src/repo/remove.v 100644
View File

@ -0,0 +1,85 @@
module repo
import os
// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It
// returns false if the package wasn't present in the database. It also
// optionally re-syncs the repo archives.
pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool {
repo_dir := os.join_path(r.repos_dir, repo, arch)
// If the repository doesn't exist yet, the result is automatically false
if !os.exists(repo_dir) {
return false
}
// We iterate over every directory in the repo dir
// TODO filter so we only check directories
for d in os.ls(repo_dir)? {
// Because a repository only allows a single version of each package,
// we need only compare whether the name of the package is the same,
// not the version.
name := d.split('-')#[..-2].join('-')
if name == pkg_name {
// We lock the mutex here to prevent other routines from creating a
// new archive while we remove an entry
lock r.mutex {
os.rmdir_all(os.join_path_single(repo_dir, d))?
}
// Also remove the package archive
repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch)
archives := os.ls(repo_pkg_dir)?.filter(it.split('-')#[..-3].join('-') == name)
for archive_name in archives {
full_path := os.join_path_single(repo_pkg_dir, archive_name)
os.rm(full_path)?
}
// Sync the db archives if requested
if sync {
r.sync(repo, arch)?
}
return true
}
}
return false
}
// remove_arch_repo removes an arch-repo & its packages.
pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) ?bool {
repo_dir := os.join_path(r.repos_dir, repo, arch)
// If the repository doesn't exist yet, the result is automatically false
if !os.exists(repo_dir) {
return false
}
os.rmdir_all(repo_dir)?
pkg_dir := os.join_path(r.pkg_dir, repo, arch)
os.rmdir_all(pkg_dir)?
return true
}
// remove_repo removes a repo & its packages.
pub fn (r &RepoGroupManager) remove_repo(repo string) ?bool {
repo_dir := os.join_path_single(r.repos_dir, repo)
// If the repository doesn't exist yet, the result is automatically false
if !os.exists(repo_dir) {
return false
}
os.rmdir_all(repo_dir)?
pkg_dir := os.join_path_single(r.pkg_dir, repo)
os.rmdir_all(pkg_dir)?
return true
}

6
src/server/README.md 100644
View File

@ -0,0 +1,6 @@
This module contains the Vieter HTTP server, consisting of the repository
implementation & the REST API.
**NOTE**: vweb defines the priority order of routes by the file names in this
module. Therefore, it's very important that all API routes are defined in files
prefixed with `api_`, as this is before the word `routes` alphabetically.

63
src/server/logs.v → src/server/api_logs.v
View File

@ -3,51 +3,39 @@ module server
import web
import net.http
import net.urllib
import response { new_data_response, new_response }
import web.response { new_data_response, new_response }
import db
import time
import os
import util
import models { BuildLog, BuildLogFilter }
// get_logs returns all build logs in the database. A 'repo' query param can
// v1_get_logs returns all build logs in the database. A 'target' query param can
// optionally be added to limit the list of build logs to that repository.
['/api/logs'; get]
fn (mut app App) get_logs() web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
['/api/v1/logs'; auth; get]
fn (mut app App) v1_get_logs() web.Result {
filter := models.from_params<BuildLogFilter>(app.query) or {
return app.json(http.Status.bad_request, new_response('Invalid query parameters.'))
}
logs := app.db.get_build_logs(filter)
return app.json(http.Status.ok, new_data_response(logs))
}
// get_single_log returns the build log with the given id.
['/api/logs/:id'; get]
fn (mut app App) get_single_log(id int) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
return app.json(.ok, new_data_response(logs))
}
// v1_get_single_log returns the build log with the given id.
['/api/v1/logs/:id'; auth; get]
fn (mut app App) v1_get_single_log(id int) web.Result {
log := app.db.get_build_log(id) or { return app.not_found() }
return app.json(http.Status.ok, new_data_response(log))
}
// get_log_content returns the actual build log file for the given id.
['/api/logs/:id/content'; get]
fn (mut app App) get_log_content(id int) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
return app.json(.ok, new_data_response(log))
}
// v1_get_log_content returns the actual build log file for the given id.
['/api/v1/logs/:id/content'; auth; get]
fn (mut app App) v1_get_log_content(id int) web.Result {
log := app.db.get_build_log(id) or { return app.not_found() }
file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.repo_id.str(), log.arch,
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), log.arch,
file_name)
return app.file(full_path)
@ -62,13 +50,9 @@ fn parse_query_time(query string) ?time.Time {
return t
}
// post_log adds a new log to the database.
['/api/logs'; post]
fn (mut app App) post_log() web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
// v1_post_log adds a new log to the database.
['/api/v1/logs'; auth; post]
fn (mut app App) v1_post_log() web.Result {
// Parse query params
start_time_int := app.query['startTime'].int()
@ -96,24 +80,25 @@ fn (mut app App) post_log() web.Result {
arch := app.query['arch']
repo_id := app.query['repo'].int()
target_id := app.query['target'].int()
if !app.db.git_repo_exists(repo_id) {
return app.json(http.Status.bad_request, new_response('Unknown Git repo.'))
if !app.db.target_exists(target_id) {
return app.json(http.Status.bad_request, new_response('Unknown target.'))
}
// Store log in db
log := BuildLog{
repo_id: repo_id
target_id: target_id
start_time: start_time
end_time: end_time
arch: arch
exit_code: exit_code
}
app.db.add_build_log(log)
// id of newly created log
log_id := app.db.add_build_log(log)
repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo_id.str(), arch)
repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, target_id.str(), arch)
// Create the logs directory of it doesn't exist
if !os.exists(repo_logs_dir) {
@ -138,5 +123,5 @@ fn (mut app App) post_log() web.Result {
return app.status(http.Status.length_required)
}
return app.json(http.Status.ok, new_response('Logs added successfully.'))
return app.json(.ok, new_data_response(log_id))
}

73
src/server/api_targets.v 100644
View File

@ -0,0 +1,73 @@
module server
import web
import net.http
import web.response { new_data_response, new_response }
import db
import models { Target, TargetArch, TargetFilter }
// v1_get_targets returns the current list of targets.
['/api/v1/targets'; auth; get]
fn (mut app App) v1_get_targets() web.Result {
filter := models.from_params<TargetFilter>(app.query) or {
return app.json(http.Status.bad_request, new_response('Invalid query parameters.'))
}
repos := app.db.get_targets(filter)
return app.json(.ok, new_data_response(repos))
}
// v1_get_single_target returns the information for a single target.
['/api/v1/targets/:id'; auth; get]
fn (mut app App) v1_get_single_target(id int) web.Result {
repo := app.db.get_target(id) or { return app.not_found() }
return app.json(.ok, new_data_response(repo))
}
// v1_post_target creates a new target from the provided query string.
['/api/v1/targets'; auth; post]
fn (mut app App) v1_post_target() web.Result {
mut params := app.query.clone()
// If a repo is created without specifying the arch, we assume it's meant
// for the default architecture.
if 'arch' !in params {
params['arch'] = app.conf.default_arch
}
new_repo := models.from_params<Target>(params) or {
return app.json(http.Status.bad_request, new_response(err.msg()))
}
// Ensure someone doesn't submit an invalid kind
if new_repo.kind !in models.valid_kinds {
return app.json(http.Status.bad_request, new_response('Invalid kind.'))
}
id := app.db.add_target(new_repo)
return app.json(http.Status.ok, new_data_response(id))
}
// v1_delete_target removes a given target from the server's list.
['/api/v1/targets/:id'; auth; delete]
fn (mut app App) v1_delete_target(id int) web.Result {
app.db.delete_target(id)
return app.status(.ok)
}
// v1_patch_target updates a target's data with the given query params.
['/api/v1/targets/:id'; auth; patch]
fn (mut app App) v1_patch_target(id int) web.Result {
app.db.update_target(id, app.query)
if 'arch' in app.query {
arch_objs := app.query['arch'].split(',').map(TargetArch{ value: it })
app.db.update_target_archs(id, arch_objs)
}
return app.status(.ok)
}

12
src/server/auth.v
View File

@ -1,12 +0,0 @@
module server
import net.http
// is_authorized checks whether the provided API key is correct.
fn (mut app App) is_authorized() bool {
x_header := app.req.header.get_custom('X-Api-Key', http.HeaderQueryConfig{ exact: true }) or {
return false
}
return x_header.trim_space() == app.conf.api_key
}

5
src/server/cli.v
View File

@ -1,7 +1,7 @@
module server
import cli
import env
import vieter_v.conf as vconf
struct Config {
pub:
@ -10,6 +10,7 @@ pub:
data_dir string
api_key string
default_arch string
port int = 8000
}
// cmd returns the cli submodule that handles starting the server
@ -19,7 +20,7 @@ pub fn cmd() cli.Command {
description: 'Start the Vieter server.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file')?
conf := env.load<Config>(config_file)?
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
server(conf)?
}

88
src/server/git.v
View File

@ -1,88 +0,0 @@
module server
import web
import net.http
import response { new_data_response, new_response }
import db
import models { GitRepo, GitRepoArch, GitRepoFilter }
// get_repos returns the current list of repos.
['/api/repos'; get]
fn (mut app App) get_repos() web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
filter := models.from_params<GitRepoFilter>(app.query) or {
return app.json(http.Status.bad_request, new_response('Invalid query parameters.'))
}
repos := app.db.get_git_repos(filter)
return app.json(http.Status.ok, new_data_response(repos))
}
// get_single_repo returns the information for a single repo.
['/api/repos/:id'; get]
fn (mut app App) get_single_repo(id int) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
repo := app.db.get_git_repo(id) or { return app.not_found() }
return app.json(http.Status.ok, new_data_response(repo))
}
// post_repo creates a new repo from the provided query string.
['/api/repos'; post]
fn (mut app App) post_repo() web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
mut params := app.query.clone()
// If a repo is created without specifying the arch, we assume it's meant
// for the default architecture.
if 'arch' !in params {
params['arch'] = app.conf.default_arch
}
new_repo := models.from_params<GitRepo>(params) or {
return app.json(http.Status.bad_request, new_response(err.msg()))
}
app.db.add_git_repo(new_repo)
return app.json(http.Status.ok, new_response('Repo added successfully.'))
}
// delete_repo removes a given repo from the server's list.
['/api/repos/:id'; delete]
fn (mut app App) delete_repo(id int) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
app.db.delete_git_repo(id)
return app.json(http.Status.ok, new_response('Repo removed successfully.'))
}
// patch_repo updates a repo's data with the given query params.
['/api/repos/:id'; patch]
fn (mut app App) patch_repo(id int) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
app.db.update_git_repo(id, app.query)
if 'arch' in app.query {
arch_objs := app.query['arch'].split(',').map(GitRepoArch{ value: it })
app.db.update_git_repo_archs(id, arch_objs)
}
return app.json(http.Status.ok, new_response('Repo updated successfully.'))
}

40
src/server/routes.v → src/server/repo.v
View File

@ -6,14 +6,13 @@ import repo
import time
import rand
import util
import net.http
import response { new_response }
import web.response { new_data_response, new_response }
// healthcheck just returns a string, but can be used to quickly check if the
// server is still responsive.
['/health'; get]
pub fn (mut app App) healthcheck() web.Result {
return app.json(http.Status.ok, new_response('Healthy.'))
return app.json(.ok, new_response('Healthy.'))
}
// get_repo_file handles all Pacman-related routes. It returns both the
@ -45,23 +44,16 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re
full_path = os.join_path(app.repo.repos_dir, repo, arch, filename, 'desc')
}
// Scuffed way to respond to HEAD requests
if app.req.method == http.Method.head {
if os.exists(full_path) {
return app.status(http.Status.ok)
}
return app.not_found()
}
return app.file(full_path)
}
// put_package handles publishing a package to a repository.
['/:repo/publish'; post]
['/:repo/publish'; auth; post]
fn (mut app App) put_package(repo string) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
// api is a reserved keyword for api routes & should never be allowed to be
// a repository.
if repo.to_lower() == 'api' {
return app.json(.bad_request, new_response("'api' is a reserved keyword & cannot be used as a repository name."))
}
mut pkg_path := ''
@ -78,7 +70,7 @@ fn (mut app App) put_package(repo string) web.Result {
util.reader_to_file(mut app.reader, length.int(), pkg_path) or {
app.lwarn("Failed to upload '$pkg_path'")
return app.json(http.Status.internal_server_error, new_response('Failed to upload file.'))
return app.status(.internal_server_error)
}
sw.stop()
@ -87,7 +79,7 @@ fn (mut app App) put_package(repo string) web.Result {
app.lwarn('Tried to upload package without specifying a Content-Length.')
// length required
return app.status(http.Status.length_required)
return app.status(.length_required)
}
res := app.repo.add_pkg_from_path(repo, pkg_path) or {
@ -95,18 +87,10 @@ fn (mut app App) put_package(repo string) web.Result {
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg()") }
return app.json(http.Status.internal_server_error, new_response('Failed to add package.'))
return app.status(.internal_server_error)
}
if !res.added {
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg()") }
app.linfo("Added '$res.name-$res.version' to '$repo (${res.archs.join(',')})'.")
app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo'.")
return app.json(http.Status.bad_request, new_response('File already exists.'))
}
app.linfo("Added '$res.pkg.full_name()' to repo '$repo ($res.pkg.info.arch)'.")
return app.json(http.Status.ok, new_response('Package added successfully.'))
return app.json(.ok, new_data_response(res))
}

63
src/server/repo_remove.v 100644
View File

@ -0,0 +1,63 @@
module server
import web
// delete_package tries to remove the given package.
['/:repo/:arch/:pkg'; auth; delete]
fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result {
res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or {
app.lerror('Error while deleting package: $err.msg()')
return app.status(.internal_server_error)
}
if res {
app.linfo("Removed package '$pkg' from '$repo/$arch'")
return app.status(.ok)
} else {
app.linfo("Tried removing package '$pkg' from '$repo/$arch', but it doesn't exist.")
return app.status(.not_found)
}
}
// delete_arch_repo tries to remove the given arch-repo.
['/:repo/:arch'; auth; delete]
fn (mut app App) delete_arch_repo(repo string, arch string) web.Result {
res := app.repo.remove_arch_repo(repo, arch) or {
app.lerror('Error while deleting arch-repo: $err.msg()')
return app.status(.internal_server_error)
}
if res {
app.linfo("Removed arch-repo '$repo/$arch'")
return app.status(.ok)
} else {
app.linfo("Tried removing '$repo/$arch', but it doesn't exist.")
return app.status(.not_found)
}
}
// delete_repo tries to remove the given repo.
['/:repo'; auth; delete]
fn (mut app App) delete_repo(repo string) web.Result {
res := app.repo.remove_repo(repo) or {
app.lerror('Error while deleting repo: $err.msg()')
return app.status(.internal_server_error)
}
if res {
app.linfo("Removed repo '$repo'")
return app.status(.ok)
} else {
app.linfo("Tried removing '$repo', but it doesn't exist.")
return app.status(.not_found)
}
}

4
src/server/server.v
View File

@ -8,7 +8,6 @@ import util
import db
const (
port = 8000
log_file_name = 'vieter.log'
repo_dir_name = 'repos'
db_file_name = 'vieter.sqlite'
@ -74,8 +73,9 @@ pub fn server(conf Config) ? {
web.run(&App{
logger: logger
api_key: conf.api_key
conf: conf
repo: repo
db: db
}, server.port)
}, conf.port)
}

19
src/util/util.v
View File

@ -1,7 +1,6 @@
module util
import os
import crypto.md5
import crypto.sha256
const (
@ -23,12 +22,10 @@ pub fn exit_with_message(code int, msg string) {
exit(code)
}
// hash_file returns the md5 & sha256 hash of a given file
// TODO actually implement sha256
pub fn hash_file(path &string) ?(string, string) {
// hash_file returns the sha256 hash of a given file
pub fn hash_file(path &string) ?string {
file := os.open(path) or { return error('Failed to open file.') }
mut md5sum := md5.new()
mut sha256sum := sha256.new()
buf_size := int(1_000_000)
@ -40,16 +37,12 @@ pub fn hash_file(path &string) ?(string, string) {
bytes_read := file.read(mut buf) or { return error('Failed to read from file.') }
bytes_left -= u64(bytes_read)
// For now we'll assume that this always works
md5sum.write(buf[..bytes_read]) or {
return error('Failed to update md5 checksum. This should never happen.')
}
sha256sum.write(buf[..bytes_read]) or {
return error('Failed to update sha256 checksum. This should never happen.')
}
// This function never actually fails, but returns an option to follow
// the Writer interface.
sha256sum.write(buf[..bytes_read])?
}
return md5sum.checksum().hex(), sha256sum.checksum().hex()
return sha256sum.checksum().hex()
}
// pretty_bytes converts a byte count to human-readable version

7
src/v.mod
View File

@ -0,0 +1,7 @@
Module {
dependencies: [
'https://git.rustybever.be/vieter-v/conf',
'https://git.rustybever.be/vieter-v/docker',
'https://git.rustybever.be/vieter-v/aur'
]
}

133
src/web/consts.v 100644
View File

@ -0,0 +1,133 @@
module web
import net.http
// A dummy structure that returns from routes to indicate that you actually sent something to a user
[noinit]
pub struct Result {}
pub const (
methods_with_form = [http.Method.post, .put, .patch]
headers_close = http.new_custom_header_from_map({
'Server': 'Vieter'
http.CommonHeader.connection.str(): 'close'
}) or { panic('should never fail') }
http_302 = http.new_response(
status: .found
body: '302 Found'
header: headers_close
)
http_400 = http.new_response(
status: .bad_request
body: '400 Bad Request'
header: http.new_header(
key: .content_type
value: 'text/plain'
).join(headers_close)
)
http_401 = http.new_response(
status: .unauthorized
body: '401 Unauthorized'
header: http.new_header(
key: .content_type
value: 'text/plain'
).join(headers_close)
)
http_404 = http.new_response(
status: .not_found
body: '404 Not Found'
header: http.new_header(
key: .content_type
value: 'text/plain'
).join(headers_close)
)
http_500 = http.new_response(
status: .internal_server_error
body: '500 Internal Server Error'
header: http.new_header(
key: .content_type
value: 'text/plain'
).join(headers_close)
)
mime_types = {
'.aac': 'audio/aac'
'.abw': 'application/x-abiword'
'.arc': 'application/x-freearc'
'.avi': 'video/x-msvideo'
'.azw': 'application/vnd.amazon.ebook'
'.bin': 'application/octet-stream'
'.bmp': 'image/bmp'
'.bz': 'application/x-bzip'
'.bz2': 'application/x-bzip2'
'.cda': 'application/x-cdf'
'.csh': 'application/x-csh'
'.css': 'text/css'
'.csv': 'text/csv'
'.doc': 'application/msword'
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
'.eot': 'application/vnd.ms-fontobject'
'.epub': 'application/epub+zip'
'.gz': 'application/gzip'
'.gif': 'image/gif'
'.htm': 'text/html'
'.html': 'text/html'
'.ico': 'image/vnd.microsoft.icon'
'.ics': 'text/calendar'
'.jar': 'application/java-archive'
'.jpeg': 'image/jpeg'
'.jpg': 'image/jpeg'
'.js': 'text/javascript'
'.json': 'application/json'
'.jsonld': 'application/ld+json'
'.mid': 'audio/midi audio/x-midi'
'.midi': 'audio/midi audio/x-midi'
'.mjs': 'text/javascript'
'.mp3': 'audio/mpeg'
'.mp4': 'video/mp4'
'.mpeg': 'video/mpeg'
'.mpkg': 'application/vnd.apple.installer+xml'
'.odp': 'application/vnd.oasis.opendocument.presentation'
'.ods': 'application/vnd.oasis.opendocument.spreadsheet'
'.odt': 'application/vnd.oasis.opendocument.text'
'.oga': 'audio/ogg'
'.ogv': 'video/ogg'
'.ogx': 'application/ogg'
'.opus': 'audio/opus'
'.otf': 'font/otf'
'.png': 'image/png'
'.pdf': 'application/pdf'
'.php': 'application/x-httpd-php'
'.ppt': 'application/vnd.ms-powerpoint'
'.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
'.rar': 'application/vnd.rar'
'.rtf': 'application/rtf'
'.sh': 'application/x-sh'
'.svg': 'image/svg+xml'
'.swf': 'application/x-shockwave-flash'
'.tar': 'application/x-tar'
'.tif': 'image/tiff'
'.tiff': 'image/tiff'
'.ts': 'video/mp2t'
'.ttf': 'font/ttf'
'.txt': 'text/plain'
'.vsd': 'application/vnd.visio'
'.wav': 'audio/wav'
'.weba': 'audio/webm'
'.webm': 'video/webm'
'.webp': 'image/webp'
'.woff': 'font/woff'
'.woff2': 'font/woff2'
'.xhtml': 'application/xhtml+xml'
'.xls': 'application/vnd.ms-excel'
'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
'.xml': 'application/xml'
'.xul': 'application/vnd.mozilla.xul+xml'
'.zip': 'application/zip'
'.3gp': 'video/3gpp'
'.3g2': 'video/3gpp2'
'.7z': 'application/x-7z-compressed'
}
max_http_post_size = 1024 * 1024
default_port = 8080
)

6
src/web/parse.v
View File

@ -3,6 +3,10 @@ module web
import net.urllib
import net.http
// Method attributes that should be ignored when parsing, as they're used
// elsewhere.
const attrs_to_ignore = ['auth']
// Parsing function attributes for methods and path.
fn parse_attrs(name string, attrs []string) ?([]http.Method, string) {
if attrs.len == 0 {
@ -32,7 +36,7 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) {
}
i++
}
if x.len > 0 {
if x.len > 0 && x.any(!web.attrs_to_ignore.contains(it)) {
return IError(http.UnexpectedExtraAttributeError{
attributes: x
})

0
src/response/response.v → src/web/response/response.v
View File

448
src/web/web.v
View File

@ -12,146 +12,25 @@ import time
import json
import log
// A dummy structure that returns from routes to indicate that you actually sent something to a user
[noinit]
pub struct Result {}
pub const (
methods_with_form = [http.Method.post, .put, .patch]
headers_close = http.new_custom_header_from_map({
'Server': 'VWeb'
http.CommonHeader.connection.str(): 'close'
}) or { panic('should never fail') }
http_302 = http.new_response(
status: .found
body: '302 Found'
header: headers_close
)
http_400 = http.new_response(
status: .bad_request
body: '400 Bad Request'
header: http.new_header(
key: .content_type
value: 'text/plain'
).join(headers_close)
)
http_404 = http.new_response(
status: .not_found
body: '404 Not Found'
header: http.new_header(
key: .content_type
value: 'text/plain'
).join(headers_close)
)
http_500 = http.new_response(
status: .internal_server_error
body: '500 Internal Server Error'
header: http.new_header(
key: .content_type
value: 'text/plain'
).join(headers_close)
)
mime_types = {
'.aac': 'audio/aac'
'.abw': 'application/x-abiword'
'.arc': 'application/x-freearc'
'.avi': 'video/x-msvideo'
'.azw': 'application/vnd.amazon.ebook'
'.bin': 'application/octet-stream'
'.bmp': 'image/bmp'
'.bz': 'application/x-bzip'
'.bz2': 'application/x-bzip2'
'.cda': 'application/x-cdf'
'.csh': 'application/x-csh'
'.css': 'text/css'
'.csv': 'text/csv'
'.doc': 'application/msword'
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
'.eot': 'application/vnd.ms-fontobject'
'.epub': 'application/epub+zip'
'.gz': 'application/gzip'
'.gif': 'image/gif'
'.htm': 'text/html'
'.html': 'text/html'
'.ico': 'image/vnd.microsoft.icon'
'.ics': 'text/calendar'
'.jar': 'application/java-archive'
'.jpeg': 'image/jpeg'
'.jpg': 'image/jpeg'
'.js': 'text/javascript'
'.json': 'application/json'
'.jsonld': 'application/ld+json'
'.mid': 'audio/midi audio/x-midi'
'.midi': 'audio/midi audio/x-midi'
'.mjs': 'text/javascript'
'.mp3': 'audio/mpeg'
'.mp4': 'video/mp4'
'.mpeg': 'video/mpeg'
'.mpkg': 'application/vnd.apple.installer+xml'
'.odp': 'application/vnd.oasis.opendocument.presentation'
'.ods': 'application/vnd.oasis.opendocument.spreadsheet'
'.odt': 'application/vnd.oasis.opendocument.text'
'.oga': 'audio/ogg'
'.ogv': 'video/ogg'
'.ogx': 'application/ogg'
'.opus': 'audio/opus'
'.otf': 'font/otf'
'.png': 'image/png'
'.pdf': 'application/pdf'
'.php': 'application/x-httpd-php'
'.ppt': 'application/vnd.ms-powerpoint'
'.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
'.rar': 'application/vnd.rar'
'.rtf': 'application/rtf'
'.sh': 'application/x-sh'
'.svg': 'image/svg+xml'
'.swf': 'application/x-shockwave-flash'
'.tar': 'application/x-tar'
'.tif': 'image/tiff'
'.tiff': 'image/tiff'
'.ts': 'video/mp2t'
'.ttf': 'font/ttf'
'.txt': 'text/plain'
'.vsd': 'application/vnd.visio'
'.wav': 'audio/wav'
'.weba': 'audio/webm'
'.webm': 'video/webm'
'.webp': 'image/webp'
'.woff': 'font/woff'
'.woff2': 'font/woff2'
'.xhtml': 'application/xhtml+xml'
'.xls': 'application/vnd.ms-excel'
'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
'.xml': 'application/xml'
'.xul': 'application/vnd.mozilla.xul+xml'
'.zip': 'application/zip'
'.3gp': 'video/3gpp'
'.3g2': 'video/3gpp2'
'.7z': 'application/x-7z-compressed'
}
max_http_post_size = 1024 * 1024
default_port = 8080
)
// The Context struct represents the Context which hold the HTTP request and response.
// It has fields for the query, form, files.
pub struct Context {
mut:
content_type string = 'text/plain'
status http.Status = http.Status.ok
pub:
// HTTP Request
req http.Request
// API key used when authenticating requests
api_key string
// TODO Response
pub mut:
done bool
// time.ticks() from start of web connection handle.
// You can use it to determine how much time is spent on your request.
page_gen_start i64
// TCP connection to client.
// But beware, do not store it for further use, after request processing web will close connection.
conn &net.TcpConn
// Gives access to a shared logger object
logger shared log.Log
// time.ticks() from start of web connection handle.
// You can use it to determine how much time is spent on your request.
page_gen_start i64
// REQUEST
static_files map[string]string
static_mime_types map[string]string
// Map containing query params for the route.
@ -161,14 +40,13 @@ pub mut:
form map[string]string
// Files from multipart-form.
files map[string][]http.FileData
header http.Header // response headers
// ? It doesn't seem to be used anywhere
form_error string
// Allows reading the request body
reader io.BufferedReader
// Gives access to a shared logger object
logger shared log.Log
// RESPONSE
status http.Status = http.Status.ok
content_type string = 'text/plain'
// response headers
header http.Header
}
struct FileData {
@ -188,50 +66,92 @@ struct Route {
// Probably you can use it for check user session cookie or add header.
pub fn (ctx Context) before_request() {}
// send_string
fn send_string(mut conn net.TcpConn, s string) ? {
conn.write(s.bytes())?
// send_string writes the given string to the TCP connection socket.
fn (mut ctx Context) send_string(s string) ? {
ctx.conn.write(s.bytes())?
}
// send_response_to_client sends a response to the client
[manualfree]
pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bool {
if ctx.done {
return false
}
ctx.done = true
// send_reader reads at most `size` bytes from the given reader & writes them
// to the TCP connection socket. Internally, a 10KB buffer is used, to avoid
// having to store all bytes in memory at once.
fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ? {
mut buf := []u8{len: 10_000}
mut bytes_left := size
// build header
header := http.new_header_from_map({
http.CommonHeader.content_type: mimetype
http.CommonHeader.content_length: res.len.str()
}).join(ctx.header)
// Repeat as long as the stream still has data
for bytes_left > 0 {
bytes_read := reader.read(mut buf)?
bytes_left -= u64(bytes_read)
mut resp := http.Response{
header: header.join(web.headers_close)
body: res
mut to_write := bytes_read
for to_write > 0 {
bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { break }
to_write = to_write - bytes_written
}
resp.set_version(.v1_1)
}
}
// send_custom_response sends the given http.Response to the client. It can be
// used to overwrite the Context object & send a completely custom
// http.Response instead.
fn (mut ctx Context) send_custom_response(resp &http.Response) ? {
ctx.send_string(resp.bytestr())?
}
// send_response_header constructs a valid HTTP response with an empty body &
// sends it to the client.
pub fn (mut ctx Context) send_response_header() ? {
mut resp := http.new_response(
header: ctx.header.join(headers_close)
)
resp.header.add(.content_type, ctx.content_type)
resp.set_status(ctx.status)
send_string(mut ctx.conn, resp.bytestr()) or { return false }
ctx.send_custom_response(resp)?
}
// send is a convenience function for sending the HTTP response with an empty
// body.
pub fn (mut ctx Context) send() bool {
return ctx.send_response('')
}
// send_response constructs the resulting HTTP response with the given body
// string & sends it to the client.
pub fn (mut ctx Context) send_response(res string) bool {
ctx.send_response_header() or { return false }
ctx.send_string(res) or { return false }
return true
}
// text responds to a request with some plaintext.
pub fn (mut ctx Context) text(status http.Status, s string) Result {
ctx.status = status
// send_reader_response constructs the resulting HTTP response with the given
// body & streams the reader's contents to the client.
pub fn (mut ctx Context) send_reader_response(mut reader io.Reader, size u64) bool {
ctx.send_response_header() or { return false }
ctx.send_reader(mut reader, size) or { return false }
ctx.send_response_to_client('text/plain', s)
return true
}
return Result{}
// is_authenticated checks whether the request passes a correct API key.
pub fn (ctx &Context) is_authenticated() bool {
if provided_key := ctx.req.header.get_custom('X-Api-Key') {
return provided_key == ctx.api_key
}
return false
}
// json<T> HTTP_OK with json_s as payload with content-type `application/json`
pub fn (mut ctx Context) json<T>(status http.Status, j T) Result {
ctx.status = status
ctx.content_type = 'application/json'
json_s := json.encode(j)
ctx.send_response_to_client('application/json', json_s)
ctx.send_response(json_s)
return Result{}
}
@ -239,119 +159,112 @@ pub fn (mut ctx Context) json<T>(status http.Status, j T) Result {
// file Response HTTP_OK with file as payload
// This function manually implements responses because it needs to stream the file contents
pub fn (mut ctx Context) file(f_path string) Result {
if ctx.done {
// If the file doesn't exist, just respond with a 404
if !os.is_file(f_path) {
ctx.status = .not_found
ctx.send()
return Result{}
}
if !os.is_file(f_path) {
return ctx.not_found()
ctx.header.add(.accept_ranges, 'bytes')
file_size := os.file_size(f_path)
ctx.header.add(http.CommonHeader.content_length, file_size.str())
// A HEAD request only returns the size of the file.
if ctx.req.method == .head {
ctx.send()
return Result{}
}
// ext := os.file_ext(f_path)
// data := os.read_file(f_path) or {
// eprint(err.msg())
// ctx.server_error(500)
// return Result{}
// }
// content_type := web.mime_types[ext]
// if content_type == '' {
// eprintln('no MIME type found for extension $ext')
// ctx.server_error(500)
// return Result{}
// }
// First, we return the headers for the request
// We open the file before sending the headers in case reading fails
file_size := os.file_size(f_path)
file := os.open(f_path) or {
mut file := os.open(f_path) or {
eprintln(err.msg())
ctx.server_error(500)
return Result{}
}
// build header
header := http.new_header_from_map({
// http.CommonHeader.content_type: content_type
http.CommonHeader.content_length: file_size.str()
}).join(ctx.header)
mut resp := http.Response{
header: header.join(web.headers_close)
}
resp.set_version(.v1_1)
resp.set_status(ctx.status)
send_string(mut ctx.conn, resp.bytestr()) or { return Result{} }
mut buf := []u8{len: 1_000_000}
mut bytes_left := file_size
// Repeat as long as the stream still has data
for bytes_left > 0 {
// TODO check if just breaking here is safe
bytes_read := file.read(mut buf) or { break }
bytes_left -= u64(bytes_read)
mut to_write := bytes_read
for to_write > 0 {
// TODO don't just loop infinitely here
bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { continue }
to_write = to_write - bytes_written
}
defer {
file.close()
}
// Currently, this only supports a single provided range, e.g.
// bytes=0-1023, and not multiple ranges, e.g. bytes=0-50, 100-150
if range_str := ctx.req.header.get(.range) {
mut parts := range_str.split_nth('=', 2)
// We only support the 'bytes' range type
if parts[0] != 'bytes' {
ctx.status = .requested_range_not_satisfiable
ctx.header.delete(.content_length)
ctx.send()
return Result{}
}
parts = parts[1].split_nth('-', 2)
start := parts[0].i64()
end := if parts[1] == '' { file_size - 1 } else { parts[1].u64() }
// Either the actual number 0 or the result of an invalid integer
if end == 0 {
ctx.status = .requested_range_not_satisfiable
ctx.header.delete(.content_length)
ctx.send()
return Result{}
}
// Move cursor to start of data to read
file.seek(start, .start) or {
ctx.server_error(500)
return Result{}
}
length := end - u64(start) + 1
ctx.status = .partial_content
ctx.header.set(.content_length, length.str())
ctx.send_reader_response(mut file, length)
} else {
ctx.send_reader_response(mut file, file_size)
}
ctx.done = true
return Result{}
}
// status responds with an empty textual response, essentially only returning
// the given status code.
pub fn (mut ctx Context) status(status http.Status) Result {
return ctx.text(status, '')
ctx.status = status
ctx.send()
return Result{}
}
// server_error Response a server error
pub fn (mut ctx Context) server_error(ecode int) Result {
$if debug {
eprintln('> ctx.server_error ecode: $ecode')
}
if ctx.done {
return Result{}
}
send_string(mut ctx.conn, web.http_500.bytestr()) or {}
ctx.send_custom_response(http_500) or {}
return Result{}
}
// redirect Redirect to an url
pub fn (mut ctx Context) redirect(url string) Result {
if ctx.done {
return Result{}
}
ctx.done = true
mut resp := web.http_302
mut resp := http_302
resp.header = resp.header.join(ctx.header)
resp.header.add(.location, url)
send_string(mut ctx.conn, resp.bytestr()) or { return Result{} }
ctx.send_custom_response(resp) or {}
return Result{}
}
// not_found Send an not_found response
pub fn (mut ctx Context) not_found() Result {
return ctx.status(http.Status.not_found)
}
ctx.send_custom_response(http_404) or {}
// add_header Adds an header to the response with key and val
pub fn (mut ctx Context) add_header(key string, val string) {
ctx.header.add_custom(key, val) or {}
}
// get_header Returns the header data from the key
pub fn (ctx &Context) get_header(key string) string {
return ctx.req.header.get_custom(key) or { '' }
return Result{}
}
interface DbInterface {
@ -478,6 +391,7 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T, routes map[string]Route) {
static_mime_types: app.static_mime_types
reader: reader
logger: app.logger
api_key: app.api_key
}
// Calling middleware...
@ -496,31 +410,27 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T, routes map[string]Route) {
// Used for route matching
route_words := route.path.split('/').filter(it != '')
// Route immediate matches first
// Route immediate matches & index files first
// For example URL `/register` matches route `/:user`, but `fn register()`
// should be called first.
if !route.path.contains('/:') && url_words == route_words {
if (!route.path.contains('/:') && url_words == route_words)
|| (url_words.len == 0 && route_words == ['index'] && method.name == 'index') {
// Check whether the request is authorised
if 'auth' in method.attrs && !app.is_authenticated() {
conn.write(http_401.bytes()) or {}
return
}
// We found a match
if head.method == .post && method.args.len > 0 {
// TODO implement POST requests
// Populate method args with form values
// mut args := []string{cap: method.args.len}
// for param in method.args {
// args << form[param.name]
// }
// app.$method(args)
} else {
app.$method()
}
return
} else if params := route_matches(url_words, route_words) {
// Check whether the request is authorised
if 'auth' in method.attrs && !app.is_authenticated() {
conn.write(http_401.bytes()) or {}
return
}
if url_words.len == 0 && route_words == ['index'] && method.name == 'index' {
app.$method()
return
}
if params := route_matches(url_words, route_words) {
method_args := params.clone()
if method_args.len != method.args.len {
eprintln('warning: uneven parameters count ($method.args.len) in `$method.name`, compared to the web route `$method.attrs` ($method_args.len)')
@ -532,7 +442,7 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T, routes map[string]Route) {
}
}
// Route not found
conn.write(web.http_404.bytes()) or {}
conn.write(http_404.bytes()) or {}
}
// route_matches returns wether a route matches
@ -578,28 +488,6 @@ fn route_matches(url_words []string, route_words []string) ?[]string {
return params
}
// ip Returns the ip address from the current user
pub fn (ctx &Context) ip() string {
mut ip := ctx.req.header.get(.x_forwarded_for) or { '' }
if ip == '' {
ip = ctx.req.header.get_custom('X-Real-Ip') or { '' }
}
if ip.contains(',') {
ip = ip.all_before(',')
}
if ip == '' {
ip = ctx.conn.peer_ip() or { '' }
}
return ip
}
// error Set s to the form error
pub fn (mut ctx Context) error(s string) {
println('web error: $s')
ctx.form_error = s
}
// filter Do not delete.
// It used by `vlib/v/gen/c/str_intp.v:130` for string interpolation inside web templates
// TODO: move it to template render