Compare commits
1 Commits
Author | SHA1 | Date |
---|---|---|
Jef Roosens | 8099acff01 |
|
@ -1,4 +0,0 @@
|
||||||
# To stay consistent with the V formatting style, we use tabs
|
|
||||||
UseTab: Always
|
|
||||||
IndentWidth: 4
|
|
||||||
TabWidth: 4
|
|
|
@ -5,5 +5,6 @@ root = true
|
||||||
end_of_line = lf
|
end_of_line = lf
|
||||||
insert_final_newline = true
|
insert_final_newline = true
|
||||||
|
|
||||||
[*.{v,c,h}]
|
[*.v]
|
||||||
|
# vfmt wants it :(
|
||||||
indent_style = tab
|
indent_style = tab
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
vieter.c
|
*.c
|
||||||
/data/
|
/data/
|
||||||
|
|
||||||
# Build artifacts
|
# Build artifacts
|
||||||
|
|
|
@ -1,6 +1,3 @@
|
||||||
[submodule "docs/themes/hugo-book"]
|
[submodule "docs/themes/hugo-book"]
|
||||||
path = docs/themes/hugo-book
|
path = docs/themes/hugo-book
|
||||||
url = https://github.com/alex-shpak/hugo-book
|
url = https://github.com/alex-shpak/hugo-book
|
||||||
[submodule "src/libvieter"]
|
|
||||||
path = src/libvieter
|
|
||||||
url = https://git.rustybever.be/vieter-v/libvieter
|
|
||||||
|
|
|
@ -10,7 +10,6 @@ skip_clone: true
|
||||||
pipeline:
|
pipeline:
|
||||||
build:
|
build:
|
||||||
image: 'git.rustybever.be/vieter-v/vieter-builder'
|
image: 'git.rustybever.be/vieter-v/vieter-builder'
|
||||||
pull: true
|
|
||||||
commands:
|
commands:
|
||||||
# Add the vieter repository so we can use the compiler
|
# Add the vieter repository so we can use the compiler
|
||||||
- echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
|
- echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
|
||||||
|
|
|
@ -10,7 +10,6 @@ skip_clone: true
|
||||||
pipeline:
|
pipeline:
|
||||||
build:
|
build:
|
||||||
image: 'git.rustybever.be/vieter-v/vieter-builder'
|
image: 'git.rustybever.be/vieter-v/vieter-builder'
|
||||||
pull: true
|
|
||||||
commands:
|
commands:
|
||||||
# Add the vieter repository so we can use the compiler
|
# Add the vieter repository so we can use the compiler
|
||||||
- echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
|
- echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
variables:
|
variables:
|
||||||
- &vlang_image 'git.rustybever.be/vieter/vlang:5d4c9dc9fc11bf8648541c934adb64f27cb94e37-alpine3.17'
|
- &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2'
|
||||||
|
|
||||||
matrix:
|
matrix:
|
||||||
PLATFORM:
|
PLATFORM:
|
||||||
|
@ -57,7 +57,7 @@ pipeline:
|
||||||
|
|
||||||
- export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')"
|
- export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')"
|
||||||
- export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH"
|
- export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH"
|
||||||
- export SIGNATURE="$(echo -en $SIG_STRING | openssl dgst -sha1 -hmac $S3_PASSWORD -binary | base64)"
|
- export SIGNATURE="$(echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64)"
|
||||||
- >
|
- >
|
||||||
curl
|
curl
|
||||||
--silent
|
--silent
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
variables:
|
variables:
|
||||||
- &vlang_image 'git.rustybever.be/vieter/vlang:5d4c9dc9fc11bf8648541c934adb64f27cb94e37-alpine3.17'
|
- &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2'
|
||||||
|
|
||||||
platform: 'linux/amd64'
|
platform: 'linux/amd64'
|
||||||
branches:
|
branches:
|
||||||
|
@ -21,9 +21,8 @@ pipeline:
|
||||||
- make api-docs
|
- make api-docs
|
||||||
|
|
||||||
slate-docs:
|
slate-docs:
|
||||||
image: 'slatedocs/slate:v2.13.0'
|
image: 'slatedocs/slate'
|
||||||
group: 'generate'
|
group: 'generate'
|
||||||
# Slate requires a specific directory to run in
|
|
||||||
commands:
|
commands:
|
||||||
- cd docs/api
|
- cd docs/api
|
||||||
- bundle exec middleman build --clean
|
- bundle exec middleman build --clean
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
variables:
|
variables:
|
||||||
- &vlang_image 'git.rustybever.be/vieter/vlang:5d4c9dc9fc11bf8648541c934adb64f27cb94e37-alpine3.17'
|
- &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2'
|
||||||
|
|
||||||
platform: 'linux/amd64'
|
platform: 'linux/amd64'
|
||||||
branches: [ 'main' ]
|
branches: [ 'main' ]
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
variables:
|
variables:
|
||||||
- &vlang_image 'git.rustybever.be/vieter/vlang:5d4c9dc9fc11bf8648541c934adb64f27cb94e37-alpine3.17'
|
- &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2'
|
||||||
|
|
||||||
# These checks already get performed on the feature branches
|
# These checks already get performed on the feature branches
|
||||||
branches:
|
branches:
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
variables:
|
variables:
|
||||||
- &vlang_image 'git.rustybever.be/vieter/vlang:5d4c9dc9fc11bf8648541c934adb64f27cb94e37-alpine3.17'
|
- &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2'
|
||||||
|
|
||||||
platform: 'linux/amd64'
|
platform: 'linux/amd64'
|
||||||
branches:
|
branches:
|
||||||
|
@ -8,21 +8,15 @@ branches:
|
||||||
depends_on:
|
depends_on:
|
||||||
- build
|
- build
|
||||||
|
|
||||||
|
skip_clone: true
|
||||||
|
|
||||||
pipeline:
|
pipeline:
|
||||||
install-modules:
|
generate:
|
||||||
image: *vlang_image
|
image: *vlang_image
|
||||||
pull: true
|
pull: true
|
||||||
commands:
|
commands:
|
||||||
- export VMODULES=$PWD/.vmodules
|
- curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64"
|
||||||
- 'cd src && v install'
|
- chmod +x vieter
|
||||||
|
|
||||||
generate:
|
|
||||||
image: *vlang_image
|
|
||||||
commands:
|
|
||||||
# - curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64"
|
|
||||||
# - chmod +x vieter
|
|
||||||
- export VMODULES=$PWD/.vmodules
|
|
||||||
- make
|
|
||||||
- ./vieter man man
|
- ./vieter man man
|
||||||
- cd man
|
- cd man
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
variables:
|
variables:
|
||||||
- &vlang_image 'git.rustybever.be/vieter/vlang:5d4c9dc9fc11bf8648541c934adb64f27cb94e37-alpine3.17'
|
- &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2'
|
||||||
|
|
||||||
matrix:
|
matrix:
|
||||||
PLATFORM:
|
PLATFORM:
|
||||||
|
|
25
CHANGELOG.md
25
CHANGELOG.md
|
@ -7,31 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||||
|
|
||||||
## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
|
## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
|
||||||
|
|
||||||
## [0.6.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.6.0)
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
* Metrics endpoint for Prometheus integration
|
|
||||||
* Search in list of targets using API & CLI
|
|
||||||
* Allow filtering targets by arch value
|
|
||||||
* Configurable global timeout for builds
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
|
|
||||||
* Rewrote cron expression logic in C
|
|
||||||
* Updated codebase to V commit after 0.3.3
|
|
||||||
* Agents now use worker threads and no longer spawn a new thread for every
|
|
||||||
build
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
|
|
||||||
* Package upload now fails if TCP connection is closed before all bytes have
|
|
||||||
been received
|
|
||||||
|
|
||||||
### Removed
|
|
||||||
|
|
||||||
* Deprecated cron daemon
|
|
||||||
|
|
||||||
## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0)
|
## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0)
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
29
Makefile
29
Makefile
|
@ -1,20 +1,16 @@
|
||||||
# =====CONFIG=====
|
# =====CONFIG=====
|
||||||
SRC_DIR := src
|
SRC_DIR := src
|
||||||
SRCS != find '$(SRC_DIR)' -iname '*.v'
|
SOURCES != find '$(SRC_DIR)' -iname '*.v'
|
||||||
|
|
||||||
V_PATH ?= v
|
V_PATH ?= v
|
||||||
V := $(V_PATH) -showcc -gc boehm -d use_openssl -skip-unused
|
V := $(V_PATH) -showcc -gc boehm -W -d use_openssl -skip-unused
|
||||||
|
|
||||||
all: vieter
|
all: vieter
|
||||||
|
|
||||||
|
|
||||||
# =====COMPILATION=====
|
# =====COMPILATION=====
|
||||||
.PHONY: libvieter
|
|
||||||
libvieter:
|
|
||||||
make -C '$(SRC_DIR)/libvieter' CFLAGS='-O3'
|
|
||||||
|
|
||||||
# Regular binary
|
# Regular binary
|
||||||
vieter: $(SOURCES) libvieter
|
vieter: $(SOURCES)
|
||||||
$(V) -g -o vieter $(SRC_DIR)
|
$(V) -g -o vieter $(SRC_DIR)
|
||||||
|
|
||||||
# Debug build using gcc
|
# Debug build using gcc
|
||||||
|
@ -22,7 +18,7 @@ vieter: $(SOURCES) libvieter
|
||||||
# multi-threaded and causes issues when running vieter inside gdb.
|
# multi-threaded and causes issues when running vieter inside gdb.
|
||||||
.PHONY: debug
|
.PHONY: debug
|
||||||
debug: dvieter
|
debug: dvieter
|
||||||
dvieter: $(SOURCES) libvieter
|
dvieter: $(SOURCES)
|
||||||
$(V_PATH) -showcc -keepc -cg -o dvieter $(SRC_DIR)
|
$(V_PATH) -showcc -keepc -cg -o dvieter $(SRC_DIR)
|
||||||
|
|
||||||
# Run the debug build inside gdb
|
# Run the debug build inside gdb
|
||||||
|
@ -33,12 +29,12 @@ gdb: dvieter
|
||||||
# Optimised production build
|
# Optimised production build
|
||||||
.PHONY: prod
|
.PHONY: prod
|
||||||
prod: pvieter
|
prod: pvieter
|
||||||
pvieter: $(SOURCES) libvieter
|
pvieter: $(SOURCES)
|
||||||
$(V) -o pvieter -prod $(SRC_DIR)
|
$(V) -o pvieter -prod $(SRC_DIR)
|
||||||
|
|
||||||
# Only generate C code
|
# Only generate C code
|
||||||
.PHONY: c
|
.PHONY: c
|
||||||
c: $(SOURCES) libvieter
|
c: $(SOURCES)
|
||||||
$(V) -o vieter.c $(SRC_DIR)
|
$(V) -o vieter.c $(SRC_DIR)
|
||||||
|
|
||||||
|
|
||||||
|
@ -71,7 +67,6 @@ man: vieter
|
||||||
|
|
||||||
|
|
||||||
# =====OTHER=====
|
# =====OTHER=====
|
||||||
# Linting
|
|
||||||
.PHONY: lint
|
.PHONY: lint
|
||||||
lint:
|
lint:
|
||||||
$(V) fmt -verify $(SRC_DIR)
|
$(V) fmt -verify $(SRC_DIR)
|
||||||
|
@ -79,24 +74,18 @@ lint:
|
||||||
$(V_PATH) missdoc -p $(SRC_DIR)
|
$(V_PATH) missdoc -p $(SRC_DIR)
|
||||||
@ [ $$($(V_PATH) missdoc -p $(SRC_DIR) | wc -l) = 0 ]
|
@ [ $$($(V_PATH) missdoc -p $(SRC_DIR) | wc -l) = 0 ]
|
||||||
|
|
||||||
|
# Format the V codebase
|
||||||
# Formatting
|
|
||||||
.PHONY: fmt
|
.PHONY: fmt
|
||||||
fmt:
|
fmt:
|
||||||
$(V) fmt -w $(SRC_DIR)
|
$(V) fmt -w $(SRC_DIR)
|
||||||
|
|
||||||
|
|
||||||
# Testing
|
|
||||||
.PHONY: test
|
.PHONY: test
|
||||||
test: libvieter
|
test:
|
||||||
$(V) -g test $(SRC_DIR)
|
$(V) test $(SRC_DIR)
|
||||||
|
|
||||||
|
|
||||||
# Cleaning
|
|
||||||
.PHONY: clean
|
.PHONY: clean
|
||||||
clean:
|
clean:
|
||||||
rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' 'docs/public'
|
rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' 'docs/public'
|
||||||
make -C '$(SRC_DIR)/libvieter' clean
|
|
||||||
|
|
||||||
|
|
||||||
# =====EXPERIMENTAL=====
|
# =====EXPERIMENTAL=====
|
||||||
|
|
22
PKGBUILD
22
PKGBUILD
|
@ -3,31 +3,21 @@
|
||||||
|
|
||||||
pkgbase='vieter'
|
pkgbase='vieter'
|
||||||
pkgname='vieter'
|
pkgname='vieter'
|
||||||
pkgver='0.6.0'
|
pkgver='0.5.0'
|
||||||
pkgrel=1
|
pkgrel=1
|
||||||
pkgdesc="Lightweight Arch repository server & package build system"
|
pkgdesc="Lightweight Arch repository server & package build system"
|
||||||
depends=('glibc' 'openssl' 'libarchive' 'sqlite')
|
depends=('glibc' 'openssl' 'libarchive' 'sqlite')
|
||||||
makedepends=('git' 'vieter-vlang')
|
makedepends=('git' 'vlang')
|
||||||
arch=('x86_64' 'aarch64')
|
arch=('x86_64' 'aarch64')
|
||||||
url='https://git.rustybever.be/vieter-v/vieter'
|
url='https://git.rustybever.be/vieter-v/vieter'
|
||||||
license=('AGPL3')
|
license=('AGPL3')
|
||||||
source=(
|
source=("$pkgname::git+https://git.rustybever.be/vieter-v/vieter#tag=${pkgver//_/-}")
|
||||||
"$pkgname::git+https://git.rustybever.be/vieter-v/vieter#tag=${pkgver//_/-}"
|
md5sums=('SKIP')
|
||||||
"libvieter::git+https://git.rustybever.be/vieter-v/libvieter"
|
|
||||||
)
|
|
||||||
md5sums=('SKIP' 'SKIP')
|
|
||||||
|
|
||||||
prepare() {
|
prepare() {
|
||||||
cd "${pkgname}"
|
export VMODULES="$srcdir/.vmodules"
|
||||||
|
|
||||||
# Add the libvieter submodule
|
cd "$pkgname/src" && v install
|
||||||
git submodule init
|
|
||||||
git config submodules.src/libvieter.url "${srcdir}/libvieter"
|
|
||||||
git -c protocol.file.allow=always submodule update
|
|
||||||
|
|
||||||
export VMODULES="${srcdir}/.vmodules"
|
|
||||||
|
|
||||||
cd src && v install
|
|
||||||
}
|
}
|
||||||
|
|
||||||
build() {
|
build() {
|
||||||
|
|
34
PKGBUILD.dev
34
PKGBUILD.dev
|
@ -7,41 +7,31 @@ pkgver=0.2.0.r25.g20112b8
|
||||||
pkgrel=1
|
pkgrel=1
|
||||||
pkgdesc="Lightweight Arch repository server & package build system (development version)"
|
pkgdesc="Lightweight Arch repository server & package build system (development version)"
|
||||||
depends=('glibc' 'openssl' 'libarchive' 'sqlite')
|
depends=('glibc' 'openssl' 'libarchive' 'sqlite')
|
||||||
makedepends=('git' 'vieter-vlang')
|
makedepends=('git' 'vlang')
|
||||||
arch=('x86_64' 'aarch64')
|
arch=('x86_64' 'aarch64')
|
||||||
url='https://git.rustybever.be/vieter-v/vieter'
|
url='https://git.rustybever.be/vieter-v/vieter'
|
||||||
license=('AGPL3')
|
license=('AGPL3')
|
||||||
source=(
|
source=("$pkgname::git+https://git.rustybever.be/vieter-v/vieter#branch=dev")
|
||||||
"${pkgname}::git+https://git.rustybever.be/vieter-v/vieter#branch=dev"
|
md5sums=('SKIP')
|
||||||
"libvieter::git+https://git.rustybever.be/vieter-v/libvieter"
|
|
||||||
)
|
|
||||||
md5sums=('SKIP' 'SKIP')
|
|
||||||
provides=('vieter')
|
provides=('vieter')
|
||||||
conflicts=('vieter')
|
conflicts=('vieter')
|
||||||
|
|
||||||
pkgver() {
|
pkgver() {
|
||||||
cd "${pkgname}"
|
cd "$pkgname"
|
||||||
|
|
||||||
git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g'
|
git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g'
|
||||||
}
|
}
|
||||||
|
|
||||||
prepare() {
|
prepare() {
|
||||||
cd "${pkgname}"
|
export VMODULES="$srcdir/.vmodules"
|
||||||
|
|
||||||
# Add the libvieter submodule
|
cd "$pkgname/src" && v install
|
||||||
git submodule init
|
|
||||||
git config submodules.src/libvieter.url "${srcdir}/libvieter"
|
|
||||||
git -c protocol.file.allow=always submodule update
|
|
||||||
|
|
||||||
export VMODULES="${srcdir}/.vmodules"
|
|
||||||
|
|
||||||
cd src && v install
|
|
||||||
}
|
}
|
||||||
|
|
||||||
build() {
|
build() {
|
||||||
export VMODULES="${srcdir}/.vmodules"
|
export VMODULES="$srcdir/.vmodules"
|
||||||
|
|
||||||
cd "${pkgname}"
|
cd "$pkgname"
|
||||||
|
|
||||||
make prod
|
make prod
|
||||||
|
|
||||||
|
@ -52,9 +42,9 @@ build() {
|
||||||
}
|
}
|
||||||
|
|
||||||
package() {
|
package() {
|
||||||
install -dm755 "${pkgdir}/usr/bin"
|
install -dm755 "$pkgdir/usr/bin"
|
||||||
install -Dm755 "${pkgname}/pvieter" "${pkgdir}/usr/bin/vieter"
|
install -Dm755 "$pkgname/pvieter" "$pkgdir/usr/bin/vieter"
|
||||||
|
|
||||||
install -dm755 "${pkgdir}/usr/share/man/man1"
|
install -dm755 "$pkgdir/usr/share/man/man1"
|
||||||
install -Dm644 "${pkgname}/man"/*.1 "${pkgdir}/usr/share/man/man1"
|
install -Dm644 "$pkgname/man"/*.1 "$pkgdir/usr/share/man/man1"
|
||||||
}
|
}
|
||||||
|
|
|
@ -48,9 +48,9 @@ update`.
|
||||||
|
|
||||||
### Compiler
|
### Compiler
|
||||||
|
|
||||||
V is developed using a specific compiler commit that is usually updated
|
I used to maintain a mirror that tracked the latest master, but nowadays, I
|
||||||
whenever a new version is released. Information on this can be found in the
|
maintain a Docker image containing the specific compiler version that Vieter
|
||||||
[tools](https://git.rustybever.be/vieter-v/tools) repository.
|
builds with. Currently, this is V 0.3.2.
|
||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
|
|
|
@ -55,8 +55,6 @@ Parameter | Description
|
||||||
limit | Maximum amount of results to return.
|
limit | Maximum amount of results to return.
|
||||||
offset | Offset of results.
|
offset | Offset of results.
|
||||||
repo | Limit results to targets that publish to the given repo.
|
repo | Limit results to targets that publish to the given repo.
|
||||||
query | Only return targets that have this substring in their URL, path or branch.
|
|
||||||
arch | Only return targets that publish to this arch.
|
|
||||||
|
|
||||||
## Get specific target
|
## Get specific target
|
||||||
|
|
||||||
|
|
|
@ -23,9 +23,9 @@ pub fn cmd() cli.Command {
|
||||||
description: 'Start an agent daemon.'
|
description: 'Start an agent daemon.'
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
agent(conf_)!
|
agent(conf)!
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,13 +20,11 @@ struct AgentDaemon {
|
||||||
client client.Client
|
client client.Client
|
||||||
mut:
|
mut:
|
||||||
images ImageManager
|
images ImageManager
|
||||||
|
// Which builds are currently running; length is conf.max_concurrent_builds
|
||||||
|
builds []BuildConfig
|
||||||
// Atomic variables used to detect when a build has finished; length is
|
// Atomic variables used to detect when a build has finished; length is
|
||||||
// conf.max_concurrent_builds. This approach is used as the difference
|
// conf.max_concurrent_builds
|
||||||
// between a recently finished build and an empty build slot is important
|
|
||||||
// for knowing whether the agent is currently "active".
|
|
||||||
atomics []u64
|
atomics []u64
|
||||||
// Channel used to send builds to worker threads
|
|
||||||
build_channel chan BuildConfig
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// agent_init initializes a new agent
|
// agent_init initializes a new agent
|
||||||
|
@ -36,8 +34,8 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon {
|
||||||
client: client.new(conf.address, conf.api_key)
|
client: client.new(conf.address, conf.api_key)
|
||||||
conf: conf
|
conf: conf
|
||||||
images: new_image_manager(conf.image_rebuild_frequency * 60)
|
images: new_image_manager(conf.image_rebuild_frequency * 60)
|
||||||
|
builds: []BuildConfig{len: conf.max_concurrent_builds}
|
||||||
atomics: []u64{len: conf.max_concurrent_builds}
|
atomics: []u64{len: conf.max_concurrent_builds}
|
||||||
build_channel: chan BuildConfig{cap: conf.max_concurrent_builds}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return d
|
return d
|
||||||
|
@ -45,11 +43,6 @@ fn agent_init(logger log.Log, conf Config) AgentDaemon {
|
||||||
|
|
||||||
// run starts the actual agent daemon. This function will run forever.
|
// run starts the actual agent daemon. This function will run forever.
|
||||||
pub fn (mut d AgentDaemon) run() {
|
pub fn (mut d AgentDaemon) run() {
|
||||||
// Spawn worker threads
|
|
||||||
for builder_index in 0 .. d.conf.max_concurrent_builds {
|
|
||||||
spawn d.builder_thread(d.build_channel, builder_index)
|
|
||||||
}
|
|
||||||
|
|
||||||
// This is just so that the very first time the loop is ran, the jobs are
|
// This is just so that the very first time the loop is ran, the jobs are
|
||||||
// always polled
|
// always polled
|
||||||
mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency)
|
mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency)
|
||||||
|
@ -58,7 +51,7 @@ pub fn (mut d AgentDaemon) run() {
|
||||||
|
|
||||||
for {
|
for {
|
||||||
if sleep_time > 0 {
|
if sleep_time > 0 {
|
||||||
d.ldebug('Sleeping for ${sleep_time}')
|
d.ldebug('Sleeping for $sleep_time')
|
||||||
time.sleep(sleep_time)
|
time.sleep(sleep_time)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -87,14 +80,14 @@ pub fn (mut d AgentDaemon) run() {
|
||||||
d.ldebug('Polling for new jobs')
|
d.ldebug('Polling for new jobs')
|
||||||
|
|
||||||
new_configs := d.client.poll_jobs(d.conf.arch, finished + empty) or {
|
new_configs := d.client.poll_jobs(d.conf.arch, finished + empty) or {
|
||||||
d.lerror('Failed to poll jobs: ${err.msg()}')
|
d.lerror('Failed to poll jobs: $err.msg()')
|
||||||
|
|
||||||
// TODO pick a better delay here
|
// TODO pick a better delay here
|
||||||
sleep_time = 5 * time.second
|
sleep_time = 5 * time.second
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
d.ldebug('Received ${new_configs.len} jobs')
|
d.ldebug('Received $new_configs.len jobs')
|
||||||
|
|
||||||
last_poll_time = time.now()
|
last_poll_time = time.now()
|
||||||
|
|
||||||
|
@ -102,7 +95,7 @@ pub fn (mut d AgentDaemon) run() {
|
||||||
// Make sure a recent build base image is available for
|
// Make sure a recent build base image is available for
|
||||||
// building the config
|
// building the config
|
||||||
if !d.images.up_to_date(config.base_image) {
|
if !d.images.up_to_date(config.base_image) {
|
||||||
d.linfo('Building builder image from base image ${config.base_image}')
|
d.linfo('Building builder image from base image $config.base_image')
|
||||||
|
|
||||||
// TODO handle this better than to just skip the config
|
// TODO handle this better than to just skip the config
|
||||||
d.images.refresh_image(config.base_image) or {
|
d.images.refresh_image(config.base_image) or {
|
||||||
|
@ -114,10 +107,10 @@ pub fn (mut d AgentDaemon) run() {
|
||||||
// It's technically still possible that the build image is
|
// It's technically still possible that the build image is
|
||||||
// removed in the very short period between building the
|
// removed in the very short period between building the
|
||||||
// builder image and starting a build container with it. If
|
// builder image and starting a build container with it. If
|
||||||
// this happens, fate really just didn't want you to do this
|
// this happens, faith really just didn't want you to do this
|
||||||
// build.
|
// build.
|
||||||
|
|
||||||
d.build_channel <- config
|
d.start_build(config)
|
||||||
running++
|
running++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -154,9 +147,25 @@ fn (mut d AgentDaemon) update_atomics() (int, int) {
|
||||||
return finished, empty
|
return finished, empty
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// start_build starts a build for the given BuildConfig.
|
||||||
|
fn (mut d AgentDaemon) start_build(config BuildConfig) bool {
|
||||||
|
for i in 0 .. d.atomics.len {
|
||||||
|
if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty {
|
||||||
|
stdatomic.store_u64(&d.atomics[i], agent.build_running)
|
||||||
|
d.builds[i] = config
|
||||||
|
|
||||||
|
go d.run_build(i, config)
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// run_build actually starts the build process for a given target.
|
// run_build actually starts the build process for a given target.
|
||||||
fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) {
|
fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) {
|
||||||
d.linfo('started build: ${config}')
|
d.linfo('started build: $config')
|
||||||
|
|
||||||
// 0 means success, 1 means failure
|
// 0 means success, 1 means failure
|
||||||
mut status := 0
|
mut status := 0
|
||||||
|
@ -167,31 +176,22 @@ fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) {
|
||||||
}
|
}
|
||||||
|
|
||||||
res := build.build_config(d.client.address, d.client.api_key, new_config) or {
|
res := build.build_config(d.client.address, d.client.api_key, new_config) or {
|
||||||
d.ldebug('build_config error: ${err.msg()}')
|
d.ldebug('build_config error: $err.msg()')
|
||||||
status = 1
|
status = 1
|
||||||
|
|
||||||
build.BuildResult{}
|
build.BuildResult{}
|
||||||
}
|
}
|
||||||
|
|
||||||
if status == 0 {
|
if status == 0 {
|
||||||
d.linfo('Uploading build logs for ${config}')
|
d.linfo('Uploading build logs for $config')
|
||||||
|
|
||||||
// TODO use the arch value here
|
// TODO use the arch value here
|
||||||
build_arch := os.uname().machine
|
build_arch := os.uname().machine
|
||||||
d.client.add_build_log(config.target_id, res.start_time, res.end_time, build_arch,
|
d.client.add_build_log(config.target_id, res.start_time, res.end_time, build_arch,
|
||||||
res.exit_code, res.logs) or { d.lerror('Failed to upload logs for ${config}') }
|
res.exit_code, res.logs) or { d.lerror('Failed to upload logs for $config') }
|
||||||
} else {
|
} else {
|
||||||
d.lwarn('an error occurred during build: ${config}')
|
d.lwarn('an error occurred during build: $config')
|
||||||
}
|
}
|
||||||
|
|
||||||
stdatomic.store_u64(&d.atomics[build_index], agent.build_done)
|
stdatomic.store_u64(&d.atomics[build_index], agent.build_done)
|
||||||
}
|
}
|
||||||
|
|
||||||
// builder_thread is a thread that constantly listens for builds to process
|
|
||||||
fn (mut d AgentDaemon) builder_thread(ch chan BuildConfig, builder_index int) {
|
|
||||||
for {
|
|
||||||
build_config := <-ch or { break }
|
|
||||||
|
|
||||||
d.run_build(builder_index, build_config)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -71,7 +71,7 @@ pub fn (mut m ImageManager) up_to_date(base_image string) bool {
|
||||||
fn (mut m ImageManager) refresh_image(base_image string) ! {
|
fn (mut m ImageManager) refresh_image(base_image string) ! {
|
||||||
// TODO use better image tags for built images
|
// TODO use better image tags for built images
|
||||||
new_image := build.create_build_image(base_image) or {
|
new_image := build.create_build_image(base_image) or {
|
||||||
return error('Failed to build builder image from base image ${base_image}')
|
return error('Failed to build builder image from base image $base_image')
|
||||||
}
|
}
|
||||||
|
|
||||||
m.images[base_image] << new_image
|
m.images[base_image] << new_image
|
||||||
|
@ -99,7 +99,7 @@ fn (mut m ImageManager) clean_old_images() {
|
||||||
// wasn't deleted. Therefore, we move the index over. If the function
|
// wasn't deleted. Therefore, we move the index over. If the function
|
||||||
// returns true, the array's length has decreased by one so we don't
|
// returns true, the array's length has decreased by one so we don't
|
||||||
// move the index.
|
// move the index.
|
||||||
dd.image_remove(m.images[image][i]) or {
|
dd.remove_image(m.images[image][i]) or {
|
||||||
// The image was removed by an external event
|
// The image was removed by an external event
|
||||||
if err.code() == 404 {
|
if err.code() == 404 {
|
||||||
m.images[image].delete(i)
|
m.images[image].delete(i)
|
||||||
|
|
|
@ -1,36 +1,35 @@
|
||||||
module agent
|
module agent
|
||||||
|
|
||||||
|
import log
|
||||||
|
|
||||||
|
// log a message with the given level
|
||||||
|
pub fn (mut d AgentDaemon) log(msg string, level log.Level) {
|
||||||
|
lock d.logger {
|
||||||
|
d.logger.send_output(msg, level)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// lfatal create a log message with the fatal level
|
// lfatal create a log message with the fatal level
|
||||||
pub fn (mut d AgentDaemon) lfatal(msg string) {
|
pub fn (mut d AgentDaemon) lfatal(msg string) {
|
||||||
lock d.logger {
|
d.log(msg, log.Level.fatal)
|
||||||
d.logger.fatal(msg)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// lerror create a log message with the error level
|
// lerror create a log message with the error level
|
||||||
pub fn (mut d AgentDaemon) lerror(msg string) {
|
pub fn (mut d AgentDaemon) lerror(msg string) {
|
||||||
lock d.logger {
|
d.log(msg, log.Level.error)
|
||||||
d.logger.error(msg)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// lwarn create a log message with the warn level
|
// lwarn create a log message with the warn level
|
||||||
pub fn (mut d AgentDaemon) lwarn(msg string) {
|
pub fn (mut d AgentDaemon) lwarn(msg string) {
|
||||||
lock d.logger {
|
d.log(msg, log.Level.warn)
|
||||||
d.logger.warn(msg)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// linfo create a log message with the info level
|
// linfo create a log message with the info level
|
||||||
pub fn (mut d AgentDaemon) linfo(msg string) {
|
pub fn (mut d AgentDaemon) linfo(msg string) {
|
||||||
lock d.logger {
|
d.log(msg, log.Level.info)
|
||||||
d.logger.info(msg)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ldebug create a log message with the debug level
|
// ldebug create a log message with the debug level
|
||||||
pub fn (mut d AgentDaemon) ldebug(msg string) {
|
pub fn (mut d AgentDaemon) ldebug(msg string) {
|
||||||
lock d.logger {
|
d.log(msg, log.Level.debug)
|
||||||
d.logger.debug(msg)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -45,7 +45,7 @@ pub fn create_build_image(base_image string) !string {
|
||||||
|
|
||||||
c := docker.NewContainer{
|
c := docker.NewContainer{
|
||||||
image: base_image
|
image: base_image
|
||||||
env: ['BUILD_SCRIPT=${cmds_str}']
|
env: ['BUILD_SCRIPT=$cmds_str']
|
||||||
entrypoint: ['/bin/sh', '-c']
|
entrypoint: ['/bin/sh', '-c']
|
||||||
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e']
|
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e']
|
||||||
}
|
}
|
||||||
|
@ -57,7 +57,7 @@ pub fn create_build_image(base_image string) !string {
|
||||||
image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' }
|
image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' }
|
||||||
|
|
||||||
// We pull the provided image
|
// We pull the provided image
|
||||||
dd.image_pull(image_name, image_tag)!
|
dd.pull_image(image_name, image_tag)!
|
||||||
|
|
||||||
id := dd.container_create(c)!.id
|
id := dd.container_create(c)!.id
|
||||||
// id := docker.create_container(c)!
|
// id := docker.create_container(c)!
|
||||||
|
@ -79,7 +79,7 @@ pub fn create_build_image(base_image string) !string {
|
||||||
// TODO also add the base image's name into the image name to prevent
|
// TODO also add the base image's name into the image name to prevent
|
||||||
// conflicts.
|
// conflicts.
|
||||||
tag := time.sys_mono_now().str()
|
tag := time.sys_mono_now().str()
|
||||||
image := dd.image_from_container(id, 'vieter-build', tag)!
|
image := dd.create_image_from_container(id, 'vieter-build', tag)!
|
||||||
dd.container_remove(id)!
|
dd.container_remove(id)!
|
||||||
|
|
||||||
return image.id
|
return image.id
|
||||||
|
@ -94,8 +94,8 @@ pub:
|
||||||
}
|
}
|
||||||
|
|
||||||
// build_target builds the given target. Internally it calls `build_config`.
|
// build_target builds the given target. Internally it calls `build_config`.
|
||||||
pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool, timeout int) !BuildResult {
|
pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool) !BuildResult {
|
||||||
config := target.as_build_config(base_image_id, force, timeout)
|
config := target.as_build_config(base_image_id, force)
|
||||||
|
|
||||||
return build_config(address, api_key, config)
|
return build_config(address, api_key, config)
|
||||||
}
|
}
|
||||||
|
@ -118,10 +118,10 @@ pub fn build_config(address string, api_key string, config BuildConfig) !BuildRe
|
||||||
base64_script := base64.encode_str(build_script)
|
base64_script := base64.encode_str(build_script)
|
||||||
|
|
||||||
c := docker.NewContainer{
|
c := docker.NewContainer{
|
||||||
image: '${config.base_image}'
|
image: '$config.base_image'
|
||||||
env: [
|
env: [
|
||||||
'BUILD_SCRIPT=${base64_script}',
|
'BUILD_SCRIPT=$base64_script',
|
||||||
'API_KEY=${api_key}',
|
'API_KEY=$api_key',
|
||||||
// `archlinux:base-devel` does not correctly set the path variable,
|
// `archlinux:base-devel` does not correctly set the path variable,
|
||||||
// causing certain builds to fail. This fixes it.
|
// causing certain builds to fail. This fixes it.
|
||||||
'PATH=${build.path_dirs.join(':')}',
|
'PATH=${build.path_dirs.join(':')}',
|
||||||
|
@ -136,17 +136,9 @@ pub fn build_config(address string, api_key string, config BuildConfig) !BuildRe
|
||||||
dd.container_start(id)!
|
dd.container_start(id)!
|
||||||
|
|
||||||
mut data := dd.container_inspect(id)!
|
mut data := dd.container_inspect(id)!
|
||||||
start_time := time.now()
|
|
||||||
|
|
||||||
// This loop waits until the container has stopped, so we can remove it after
|
// This loop waits until the container has stopped, so we can remove it after
|
||||||
for data.state.running {
|
for data.state.running {
|
||||||
if time.now() - start_time > config.timeout * time.second {
|
|
||||||
dd.container_kill(id)!
|
|
||||||
dd.container_remove(id)!
|
|
||||||
|
|
||||||
return error('Build killed due to timeout (${config.timeout}s)')
|
|
||||||
}
|
|
||||||
|
|
||||||
time.sleep(1 * time.second)
|
time.sleep(1 * time.second)
|
||||||
|
|
||||||
data = dd.container_inspect(id)!
|
data = dd.container_inspect(id)!
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
module build
|
module build
|
||||||
|
|
||||||
import models { BuildConfig, Target }
|
import models { BuildConfig, Target }
|
||||||
import cron
|
import cron.expression { CronExpression, parse_expression }
|
||||||
import time
|
import time
|
||||||
import datatypes { MinHeap }
|
import datatypes { MinHeap }
|
||||||
import util
|
import util
|
||||||
|
@ -13,7 +13,7 @@ pub mut:
|
||||||
// Next timestamp from which point this job is allowed to be executed
|
// Next timestamp from which point this job is allowed to be executed
|
||||||
timestamp time.Time
|
timestamp time.Time
|
||||||
// Required for calculating next timestamp after having pop'ed a job
|
// Required for calculating next timestamp after having pop'ed a job
|
||||||
ce &cron.Expression = unsafe { nil }
|
ce CronExpression
|
||||||
// Actual build config sent to the agent
|
// Actual build config sent to the agent
|
||||||
config BuildConfig
|
config BuildConfig
|
||||||
// Whether this is a one-time job
|
// Whether this is a one-time job
|
||||||
|
@ -30,15 +30,13 @@ fn (r1 BuildJob) < (r2 BuildJob) bool {
|
||||||
// for each architecture. Agents receive jobs from this queue.
|
// for each architecture. Agents receive jobs from this queue.
|
||||||
pub struct BuildJobQueue {
|
pub struct BuildJobQueue {
|
||||||
// Schedule to use for targets without explicitely defined cron expression
|
// Schedule to use for targets without explicitely defined cron expression
|
||||||
default_schedule &cron.Expression
|
default_schedule CronExpression
|
||||||
// Base image to use for targets without defined base image
|
// Base image to use for targets without defined base image
|
||||||
default_base_image string
|
default_base_image string
|
||||||
// After how many minutes a build should be forcefully cancelled
|
|
||||||
default_build_timeout int
|
|
||||||
mut:
|
mut:
|
||||||
mutex shared util.Dummy
|
mutex shared util.Dummy
|
||||||
// For each architecture, a priority queue is tracked
|
// For each architecture, a priority queue is tracked
|
||||||
queues map[string]MinHeap[BuildJob]
|
queues map[string]MinHeap<BuildJob>
|
||||||
// When a target is removed from the server or edited, its previous build
|
// When a target is removed from the server or edited, its previous build
|
||||||
// configs will be invalid. This map allows for those to be simply skipped
|
// configs will be invalid. This map allows for those to be simply skipped
|
||||||
// by ignoring any build configs created before this timestamp.
|
// by ignoring any build configs created before this timestamp.
|
||||||
|
@ -46,11 +44,10 @@ mut:
|
||||||
}
|
}
|
||||||
|
|
||||||
// new_job_queue initializes a new job queue
|
// new_job_queue initializes a new job queue
|
||||||
pub fn new_job_queue(default_schedule &cron.Expression, default_base_image string, default_build_timeout int) BuildJobQueue {
|
pub fn new_job_queue(default_schedule CronExpression, default_base_image string) BuildJobQueue {
|
||||||
return BuildJobQueue{
|
return BuildJobQueue{
|
||||||
default_schedule: unsafe { default_schedule }
|
default_schedule: default_schedule
|
||||||
default_base_image: default_base_image
|
default_base_image: default_base_image
|
||||||
default_build_timeout: default_build_timeout
|
|
||||||
invalidated: map[int]time.Time{}
|
invalidated: map[int]time.Time{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -77,25 +74,25 @@ pub struct InsertConfig {
|
||||||
pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! {
|
pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! {
|
||||||
lock q.mutex {
|
lock q.mutex {
|
||||||
if input.arch !in q.queues {
|
if input.arch !in q.queues {
|
||||||
q.queues[input.arch] = MinHeap[BuildJob]{}
|
q.queues[input.arch] = MinHeap<BuildJob>{}
|
||||||
}
|
}
|
||||||
|
|
||||||
mut job := BuildJob{
|
mut job := BuildJob{
|
||||||
created: time.now()
|
created: time.now()
|
||||||
single: input.single
|
single: input.single
|
||||||
config: input.target.as_build_config(q.default_base_image, input.force, q.default_build_timeout)
|
config: input.target.as_build_config(q.default_base_image, input.force)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !input.now {
|
if !input.now {
|
||||||
ce := if input.target.schedule != '' {
|
ce := if input.target.schedule != '' {
|
||||||
cron.parse_expression(input.target.schedule) or {
|
parse_expression(input.target.schedule) or {
|
||||||
return error("Error while parsing cron expression '${input.target.schedule}' (id ${input.target.id}): ${err.msg()}")
|
return error("Error while parsing cron expression '$input.target.schedule' (id $input.target.id): $err.msg()")
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
q.default_schedule
|
q.default_schedule
|
||||||
}
|
}
|
||||||
|
|
||||||
job.timestamp = ce.next_from_now()
|
job.timestamp = ce.next_from_now()!
|
||||||
job.ce = ce
|
job.ce = ce
|
||||||
} else {
|
} else {
|
||||||
job.timestamp = time.now()
|
job.timestamp = time.now()
|
||||||
|
@ -108,8 +105,8 @@ pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! {
|
||||||
// reschedule the given job by calculating the next timestamp and re-adding it
|
// reschedule the given job by calculating the next timestamp and re-adding it
|
||||||
// to its respective queue. This function is called by the pop functions
|
// to its respective queue. This function is called by the pop functions
|
||||||
// *after* having pop'ed the job.
|
// *after* having pop'ed the job.
|
||||||
fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) {
|
fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) ! {
|
||||||
new_timestamp := job.ce.next_from_now()
|
new_timestamp := job.ce.next_from_now()!
|
||||||
|
|
||||||
new_job := BuildJob{
|
new_job := BuildJob{
|
||||||
...job
|
...job
|
||||||
|
@ -146,7 +143,7 @@ pub fn (mut q BuildJobQueue) peek(arch string) ?BuildJob {
|
||||||
}
|
}
|
||||||
|
|
||||||
q.pop_invalid(arch)
|
q.pop_invalid(arch)
|
||||||
job := q.queues[arch].peek() or { return none }
|
job := q.queues[arch].peek()?
|
||||||
|
|
||||||
if job.timestamp < time.now() {
|
if job.timestamp < time.now() {
|
||||||
return job
|
return job
|
||||||
|
@ -165,13 +162,16 @@ pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob {
|
||||||
}
|
}
|
||||||
|
|
||||||
q.pop_invalid(arch)
|
q.pop_invalid(arch)
|
||||||
mut job := q.queues[arch].peek() or { return none }
|
mut job := q.queues[arch].peek()?
|
||||||
|
|
||||||
if job.timestamp < time.now() {
|
if job.timestamp < time.now() {
|
||||||
job = q.queues[arch].pop() or { return none }
|
job = q.queues[arch].pop()?
|
||||||
|
|
||||||
if !job.single {
|
if !job.single {
|
||||||
q.reschedule(job, arch)
|
// TODO how do we handle this properly? Is it even possible for a
|
||||||
|
// cron expression to not return a next time if it's already been
|
||||||
|
// used before?
|
||||||
|
q.reschedule(job, arch) or {}
|
||||||
}
|
}
|
||||||
|
|
||||||
return job
|
return job
|
||||||
|
@ -198,7 +198,8 @@ pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob {
|
||||||
job = q.queues[arch].pop() or { break }
|
job = q.queues[arch].pop() or { break }
|
||||||
|
|
||||||
if !job.single {
|
if !job.single {
|
||||||
q.reschedule(job, arch)
|
// TODO idem
|
||||||
|
q.reschedule(job, arch) or {}
|
||||||
}
|
}
|
||||||
|
|
||||||
out << job
|
out << job
|
||||||
|
|
|
@ -24,12 +24,12 @@ pub fn echo_commands(cmds []string) []string {
|
||||||
|
|
||||||
// create_build_script generates a shell script that builds a given Target.
|
// create_build_script generates a shell script that builds a given Target.
|
||||||
fn create_build_script(address string, config BuildConfig, build_arch string) string {
|
fn create_build_script(address string, config BuildConfig, build_arch string) string {
|
||||||
repo_url := '${address}/${config.repo}'
|
repo_url := '$address/$config.repo'
|
||||||
|
|
||||||
mut commands := [
|
mut commands := [
|
||||||
// This will later be replaced by a proper setting for changing the
|
// This will later be replaced by a proper setting for changing the
|
||||||
// mirrorlist
|
// mirrorlist
|
||||||
"echo -e '[${config.repo}]\\nServer = ${address}/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf"
|
"echo -e '[$config.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf"
|
||||||
// We need to update the package list of the repo we just added above.
|
// We need to update the package list of the repo we just added above.
|
||||||
// This should however not pull in a lot of packages as long as the
|
// This should however not pull in a lot of packages as long as the
|
||||||
// builder image is rebuilt frequently.
|
// builder image is rebuilt frequently.
|
||||||
|
@ -42,18 +42,18 @@ fn create_build_script(address string, config BuildConfig, build_arch string) st
|
||||||
'git' {
|
'git' {
|
||||||
if config.branch == '' {
|
if config.branch == '' {
|
||||||
[
|
[
|
||||||
"git clone --single-branch --depth 1 '${config.url}' repo",
|
"git clone --single-branch --depth 1 '$config.url' repo",
|
||||||
]
|
]
|
||||||
} else {
|
} else {
|
||||||
[
|
[
|
||||||
"git clone --single-branch --depth 1 --branch ${config.branch} '${config.url}' repo",
|
"git clone --single-branch --depth 1 --branch $config.branch '$config.url' repo",
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
'url' {
|
'url' {
|
||||||
[
|
[
|
||||||
'mkdir repo',
|
'mkdir repo',
|
||||||
"curl -o repo/PKGBUILD -L '${config.url}'",
|
"curl -o repo/PKGBUILD -L '$config.url'",
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
|
@ -62,7 +62,7 @@ fn create_build_script(address string, config BuildConfig, build_arch string) st
|
||||||
}
|
}
|
||||||
|
|
||||||
commands << if config.path != '' {
|
commands << if config.path != '' {
|
||||||
"cd 'repo/${config.path}'"
|
"cd 'repo/$config.path'"
|
||||||
} else {
|
} else {
|
||||||
'cd repo'
|
'cd repo'
|
||||||
}
|
}
|
||||||
|
@ -76,7 +76,7 @@ fn create_build_script(address string, config BuildConfig, build_arch string) st
|
||||||
// The build container checks whether the package is already present on
|
// The build container checks whether the package is already present on
|
||||||
// the server.
|
// the server.
|
||||||
commands << [
|
commands << [
|
||||||
'curl -s --head --fail ${repo_url}/${build_arch}/\$pkgname-\$pkgver-\$pkgrel && exit 0',
|
'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
|
||||||
// If the above curl command succeeds, we don't need to rebuild the
|
// If the above curl command succeeds, we don't need to rebuild the
|
||||||
// package. However, because we're in a su shell, the exit command will
|
// package. However, because we're in a su shell, the exit command will
|
||||||
// drop us back into the root shell. Therefore, we must check whether
|
// drop us back into the root shell. Therefore, we must check whether
|
||||||
|
@ -86,7 +86,7 @@ fn create_build_script(address string, config BuildConfig, build_arch string) st
|
||||||
}
|
}
|
||||||
|
|
||||||
commands << [
|
commands << [
|
||||||
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" ${repo_url}/publish; done',
|
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done',
|
||||||
]
|
]
|
||||||
|
|
||||||
return echo_commands(commands).join('\n')
|
return echo_commands(commands).join('\n')
|
||||||
|
|
|
@ -22,7 +22,7 @@ pub fn new(address string, api_key string) Client {
|
||||||
// send_request_raw sends an HTTP request, returning the http.Response object.
|
// send_request_raw sends an HTTP request, returning the http.Response object.
|
||||||
// It encodes the params so that they're safe to pass as HTTP query parameters.
|
// It encodes the params so that they're safe to pass as HTTP query parameters.
|
||||||
fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) !http.Response {
|
fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) !http.Response {
|
||||||
mut full_url := '${c.address}${url}'
|
mut full_url := '$c.address$url'
|
||||||
|
|
||||||
if params.len > 0 {
|
if params.len > 0 {
|
||||||
mut params_escaped := map[string]string{}
|
mut params_escaped := map[string]string{}
|
||||||
|
@ -33,9 +33,9 @@ fn (c &Client) send_request_raw(method Method, url string, params map[string]str
|
||||||
params_escaped[k] = urllib.query_escape(v)
|
params_escaped[k] = urllib.query_escape(v)
|
||||||
}
|
}
|
||||||
|
|
||||||
params_str := params_escaped.keys().map('${it}=${params_escaped[it]}').join('&')
|
params_str := params_escaped.keys().map('$it=${params_escaped[it]}').join('&')
|
||||||
|
|
||||||
full_url = '${full_url}?${params_str}'
|
full_url = '$full_url?$params_str'
|
||||||
}
|
}
|
||||||
|
|
||||||
// Looking at the source code, this function doesn't actually fail, so I'm
|
// Looking at the source code, this function doesn't actually fail, so I'm
|
||||||
|
@ -49,13 +49,13 @@ fn (c &Client) send_request_raw(method Method, url string, params map[string]str
|
||||||
}
|
}
|
||||||
|
|
||||||
// send_request<T> just calls send_request_with_body<T> with an empty body.
|
// send_request<T> just calls send_request_with_body<T> with an empty body.
|
||||||
fn (c &Client) send_request[T](method Method, url string, params map[string]string) !Response[T] {
|
fn (c &Client) send_request<T>(method Method, url string, params map[string]string) !Response<T> {
|
||||||
return c.send_request_with_body[T](method, url, params, '')
|
return c.send_request_with_body<T>(method, url, params, '')
|
||||||
}
|
}
|
||||||
|
|
||||||
// send_request_with_body<T> calls send_request_raw_response & parses its
|
// send_request_with_body<T> calls send_request_raw_response & parses its
|
||||||
// output as a Response<T> object.
|
// output as a Response<T> object.
|
||||||
fn (c &Client) send_request_with_body[T](method Method, url string, params map[string]string, body string) !Response[T] {
|
fn (c &Client) send_request_with_body<T>(method Method, url string, params map[string]string, body string) !Response<T> {
|
||||||
res := c.send_request_raw(method, url, params, body)!
|
res := c.send_request_raw(method, url, params, body)!
|
||||||
status := res.status()
|
status := res.status()
|
||||||
|
|
||||||
|
@ -64,12 +64,12 @@ fn (c &Client) send_request_with_body[T](method Method, url string, params map[s
|
||||||
if status.is_error() {
|
if status.is_error() {
|
||||||
// A non-successful status call will have an empty body
|
// A non-successful status call will have an empty body
|
||||||
if res.body == '' {
|
if res.body == '' {
|
||||||
return error('Error ${res.status_code} (${status.str()}): (empty response)')
|
return error('Error $res.status_code ($status.str()): (empty response)')
|
||||||
}
|
}
|
||||||
|
|
||||||
data := json.decode(Response[string], res.body)!
|
data := json.decode(Response<string>, res.body)!
|
||||||
|
|
||||||
return error('Status ${res.status_code} (${status.str()}): ${data.message}')
|
return error('Status $res.status_code ($status.str()): $data.message')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Just return an empty successful response
|
// Just return an empty successful response
|
||||||
|
@ -77,7 +77,7 @@ fn (c &Client) send_request_with_body[T](method Method, url string, params map[s
|
||||||
return new_data_response(T{})
|
return new_data_response(T{})
|
||||||
}
|
}
|
||||||
|
|
||||||
data := json.decode(Response[T], res.body)!
|
data := json.decode(Response<T>, res.body)!
|
||||||
|
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,7 @@ import models { BuildConfig }
|
||||||
|
|
||||||
// poll_jobs requests a list of new build jobs from the server.
|
// poll_jobs requests a list of new build jobs from the server.
|
||||||
pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig {
|
pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig {
|
||||||
data := c.send_request[[]BuildConfig](.get, '/api/v1/jobs/poll', {
|
data := c.send_request<[]BuildConfig>(.get, '/api/v1/jobs/poll', {
|
||||||
'arch': arch
|
'arch': arch
|
||||||
'max': max.str()
|
'max': max.str()
|
||||||
})!
|
})!
|
||||||
|
@ -15,7 +15,7 @@ pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig {
|
||||||
// queue_job adds a new one-time build job for the given target to the job
|
// queue_job adds a new one-time build job for the given target to the job
|
||||||
// queue.
|
// queue.
|
||||||
pub fn (c &Client) queue_job(target_id int, arch string, force bool) ! {
|
pub fn (c &Client) queue_job(target_id int, arch string, force bool) ! {
|
||||||
c.send_request[string](.post, '/api/v1/jobs/queue', {
|
c.send_request<string>(.post, '/api/v1/jobs/queue', {
|
||||||
'target': target_id.str()
|
'target': target_id.str()
|
||||||
'arch': arch
|
'arch': arch
|
||||||
'force': force.str()
|
'force': force.str()
|
||||||
|
|
|
@ -7,27 +7,27 @@ import time
|
||||||
// get_build_logs returns all build logs.
|
// get_build_logs returns all build logs.
|
||||||
pub fn (c &Client) get_build_logs(filter BuildLogFilter) ![]BuildLog {
|
pub fn (c &Client) get_build_logs(filter BuildLogFilter) ![]BuildLog {
|
||||||
params := models.params_from(filter)
|
params := models.params_from(filter)
|
||||||
data := c.send_request[[]BuildLog](.get, '/api/v1/logs', params)!
|
data := c.send_request<[]BuildLog>(.get, '/api/v1/logs', params)!
|
||||||
|
|
||||||
return data.data
|
return data.data
|
||||||
}
|
}
|
||||||
|
|
||||||
// get_build_log returns a specific build log.
|
// get_build_log returns a specific build log.
|
||||||
pub fn (c &Client) get_build_log(id int) !BuildLog {
|
pub fn (c &Client) get_build_log(id int) !BuildLog {
|
||||||
data := c.send_request[BuildLog](.get, '/api/v1/logs/${id}', {})!
|
data := c.send_request<BuildLog>(.get, '/api/v1/logs/$id', {})!
|
||||||
|
|
||||||
return data.data
|
return data.data
|
||||||
}
|
}
|
||||||
|
|
||||||
// get_build_log_content returns the contents of the build log file.
|
// get_build_log_content returns the contents of the build log file.
|
||||||
pub fn (c &Client) get_build_log_content(id int) !string {
|
pub fn (c &Client) get_build_log_content(id int) !string {
|
||||||
data := c.send_request_raw_response(.get, '/api/v1/logs/${id}/content', {}, '')!
|
data := c.send_request_raw_response(.get, '/api/v1/logs/$id/content', {}, '')!
|
||||||
|
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
// add_build_log adds a new build log to the server.
|
// add_build_log adds a new build log to the server.
|
||||||
pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) !Response[int] {
|
pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) !Response<int> {
|
||||||
params := {
|
params := {
|
||||||
'target': target_id.str()
|
'target': target_id.str()
|
||||||
'startTime': start_time.unix_time().str()
|
'startTime': start_time.unix_time().str()
|
||||||
|
@ -36,12 +36,12 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t
|
||||||
'exitCode': exit_code.str()
|
'exitCode': exit_code.str()
|
||||||
}
|
}
|
||||||
|
|
||||||
data := c.send_request_with_body[int](.post, '/api/v1/logs', params, content)!
|
data := c.send_request_with_body<int>(.post, '/api/v1/logs', params, content)!
|
||||||
|
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
// remove_build_log removes the build log with the given id from the server.
|
// remove_build_log removes the build log with the given id from the server.
|
||||||
pub fn (c &Client) remove_build_log(id int) ! {
|
pub fn (c &Client) remove_build_log(id int) ! {
|
||||||
c.send_request[string](.delete, '/api/v1/logs/${id}', {})!
|
c.send_request<string>(.delete, '/api/v1/logs/$id', {})!
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,15 +2,15 @@ module client
|
||||||
|
|
||||||
// remove_repo removes an entire repository.
|
// remove_repo removes an entire repository.
|
||||||
pub fn (c &Client) remove_repo(repo string) ! {
|
pub fn (c &Client) remove_repo(repo string) ! {
|
||||||
c.send_request[string](.delete, '/${repo}', {})!
|
c.send_request<string>(.delete, '/$repo', {})!
|
||||||
}
|
}
|
||||||
|
|
||||||
// remove_arch_repo removes an entire arch-repo.
|
// remove_arch_repo removes an entire arch-repo.
|
||||||
pub fn (c &Client) remove_arch_repo(repo string, arch string) ! {
|
pub fn (c &Client) remove_arch_repo(repo string, arch string) ! {
|
||||||
c.send_request[string](.delete, '/${repo}/${arch}', {})!
|
c.send_request<string>(.delete, '/$repo/$arch', {})!
|
||||||
}
|
}
|
||||||
|
|
||||||
// remove_package removes a single package from the given arch-repo.
|
// remove_package removes a single package from the given arch-repo.
|
||||||
pub fn (c &Client) remove_package(repo string, arch string, pkgname string) ! {
|
pub fn (c &Client) remove_package(repo string, arch string, pkgname string) ! {
|
||||||
c.send_request[string](.delete, '/${repo}/${arch}/${pkgname}', {})!
|
c.send_request<string>(.delete, '/$repo/$arch/$pkgname', {})!
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,7 +5,7 @@ import models { Target, TargetFilter }
|
||||||
// get_targets returns a list of targets, given a filter object.
|
// get_targets returns a list of targets, given a filter object.
|
||||||
pub fn (c &Client) get_targets(filter TargetFilter) ![]Target {
|
pub fn (c &Client) get_targets(filter TargetFilter) ![]Target {
|
||||||
params := models.params_from(filter)
|
params := models.params_from(filter)
|
||||||
data := c.send_request[[]Target](.get, '/api/v1/targets', params)!
|
data := c.send_request<[]Target>(.get, '/api/v1/targets', params)!
|
||||||
|
|
||||||
return data.data
|
return data.data
|
||||||
}
|
}
|
||||||
|
@ -33,7 +33,7 @@ pub fn (c &Client) get_all_targets() ![]Target {
|
||||||
|
|
||||||
// get_target returns the target for a specific id.
|
// get_target returns the target for a specific id.
|
||||||
pub fn (c &Client) get_target(id int) !Target {
|
pub fn (c &Client) get_target(id int) !Target {
|
||||||
data := c.send_request[Target](.get, '/api/v1/targets/${id}', {})!
|
data := c.send_request<Target>(.get, '/api/v1/targets/$id', {})!
|
||||||
|
|
||||||
return data.data
|
return data.data
|
||||||
}
|
}
|
||||||
|
@ -49,15 +49,15 @@ pub struct NewTarget {
|
||||||
|
|
||||||
// add_target adds a new target to the server.
|
// add_target adds a new target to the server.
|
||||||
pub fn (c &Client) add_target(t NewTarget) !int {
|
pub fn (c &Client) add_target(t NewTarget) !int {
|
||||||
params := models.params_from[NewTarget](t)
|
params := models.params_from<NewTarget>(t)
|
||||||
data := c.send_request[int](.post, '/api/v1/targets', params)!
|
data := c.send_request<int>(.post, '/api/v1/targets', params)!
|
||||||
|
|
||||||
return data.data
|
return data.data
|
||||||
}
|
}
|
||||||
|
|
||||||
// remove_target removes the target with the given id from the server.
|
// remove_target removes the target with the given id from the server.
|
||||||
pub fn (c &Client) remove_target(id int) !string {
|
pub fn (c &Client) remove_target(id int) !string {
|
||||||
data := c.send_request[string](.delete, '/api/v1/targets/${id}', {})!
|
data := c.send_request<string>(.delete, '/api/v1/targets/$id', {})!
|
||||||
|
|
||||||
return data.data
|
return data.data
|
||||||
}
|
}
|
||||||
|
@ -65,7 +65,7 @@ pub fn (c &Client) remove_target(id int) !string {
|
||||||
// patch_target sends a PATCH request to the given target with the params as
|
// patch_target sends a PATCH request to the given target with the params as
|
||||||
// payload.
|
// payload.
|
||||||
pub fn (c &Client) patch_target(id int, params map[string]string) !string {
|
pub fn (c &Client) patch_target(id int, params map[string]string) !string {
|
||||||
data := c.send_request[string](.patch, '/api/v1/targets/${id}', params)!
|
data := c.send_request<string>(.patch, '/api/v1/targets/$id', params)!
|
||||||
|
|
||||||
return data.data
|
return data.data
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,24 +36,24 @@ pub fn cmd() cli.Command {
|
||||||
required_args: 2
|
required_args: 2
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
c := aur.new()
|
c := aur.new()
|
||||||
pkgs := c.info(cmd.args[1..])!
|
pkgs := c.info(cmd.args[1..])!
|
||||||
|
|
||||||
vc := client.new(conf_.address, conf_.api_key)
|
vc := client.new(conf.address, conf.api_key)
|
||||||
|
|
||||||
for pkg in pkgs {
|
for pkg in pkgs {
|
||||||
vc.add_target(
|
vc.add_target(
|
||||||
kind: 'git'
|
kind: 'git'
|
||||||
url: 'https://aur.archlinux.org/${pkg.package_base}' + '.git'
|
url: 'https://aur.archlinux.org/$pkg.package_base' + '.git'
|
||||||
repo: cmd.args[0]
|
repo: cmd.args[0]
|
||||||
) or {
|
) or {
|
||||||
println('Failed to add ${pkg.name}: ${err.msg()}')
|
println('Failed to add $pkg.name: $err.msg()')
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
println('Added ${pkg.name}' + '.')
|
println('Added $pkg.name' + '.')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
@ -74,7 +74,7 @@ pub fn cmd() cli.Command {
|
||||||
]
|
]
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
mut filter := BuildLogFilter{}
|
mut filter := BuildLogFilter{}
|
||||||
|
|
||||||
|
@ -146,7 +146,7 @@ pub fn cmd() cli.Command {
|
||||||
|
|
||||||
raw := cmd.flags.get_bool('raw')!
|
raw := cmd.flags.get_bool('raw')!
|
||||||
|
|
||||||
list(conf_, filter, raw)!
|
list(conf, filter, raw)!
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
cli.Command{
|
cli.Command{
|
||||||
|
@ -156,9 +156,9 @@ pub fn cmd() cli.Command {
|
||||||
description: 'Remove a build log that matches the given id.'
|
description: 'Remove a build log that matches the given id.'
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
remove(conf_, cmd.args[0])!
|
remove(conf, cmd.args[0])!
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
cli.Command{
|
cli.Command{
|
||||||
|
@ -168,10 +168,10 @@ pub fn cmd() cli.Command {
|
||||||
description: 'Show all info for a specific build log.'
|
description: 'Show all info for a specific build log.'
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
id := cmd.args[0].int()
|
id := cmd.args[0].int()
|
||||||
info(conf_, id)!
|
info(conf, id)!
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
cli.Command{
|
cli.Command{
|
||||||
|
@ -181,10 +181,10 @@ pub fn cmd() cli.Command {
|
||||||
description: 'Output the content of a build log to stdout.'
|
description: 'Output the content of a build log to stdout.'
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
id := cmd.args[0].int()
|
id := cmd.args[0].int()
|
||||||
content(conf_, id)!
|
content(conf, id)!
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
@ -204,16 +204,16 @@ fn print_log_list(logs []BuildLog, raw bool) ! {
|
||||||
}
|
}
|
||||||
|
|
||||||
// list prints a list of all build logs.
|
// list prints a list of all build logs.
|
||||||
fn list(conf_ Config, filter BuildLogFilter, raw bool) ! {
|
fn list(conf Config, filter BuildLogFilter, raw bool) ! {
|
||||||
c := client.new(conf_.address, conf_.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
logs := c.get_build_logs(filter)!
|
logs := c.get_build_logs(filter)!
|
||||||
|
|
||||||
print_log_list(logs, raw)!
|
print_log_list(logs, raw)!
|
||||||
}
|
}
|
||||||
|
|
||||||
// info print the detailed info for a given build log.
|
// info print the detailed info for a given build log.
|
||||||
fn info(conf_ Config, id int) ! {
|
fn info(conf Config, id int) ! {
|
||||||
c := client.new(conf_.address, conf_.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
log := c.get_build_log(id)!
|
log := c.get_build_log(id)!
|
||||||
|
|
||||||
print(log)
|
print(log)
|
||||||
|
@ -221,15 +221,15 @@ fn info(conf_ Config, id int) ! {
|
||||||
|
|
||||||
// content outputs the contents of the log file for a given build log to
|
// content outputs the contents of the log file for a given build log to
|
||||||
// stdout.
|
// stdout.
|
||||||
fn content(conf_ Config, id int) ! {
|
fn content(conf Config, id int) ! {
|
||||||
c := client.new(conf_.address, conf_.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
content := c.get_build_log_content(id)!
|
content := c.get_build_log_content(id)!
|
||||||
|
|
||||||
println(content)
|
println(content)
|
||||||
}
|
}
|
||||||
|
|
||||||
// remove removes a build log from the server's list.
|
// remove removes a build log from the server's list.
|
||||||
fn remove(conf_ Config, id string) ! {
|
fn remove(conf Config, id string) ! {
|
||||||
c := client.new(conf_.address, conf_.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
c.remove_build_log(id.int())!
|
c.remove_build_log(id.int())!
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,7 +28,7 @@ pub fn cmd() cli.Command {
|
||||||
]
|
]
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
if cmd.args.len < 3 {
|
if cmd.args.len < 3 {
|
||||||
if !cmd.flags.get_bool('force')! {
|
if !cmd.flags.get_bool('force')! {
|
||||||
|
@ -36,14 +36,14 @@ pub fn cmd() cli.Command {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
client_ := client.new(conf_.address, conf_.api_key)
|
client := client.new(conf.address, conf.api_key)
|
||||||
|
|
||||||
if cmd.args.len == 1 {
|
if cmd.args.len == 1 {
|
||||||
client_.remove_repo(cmd.args[0])!
|
client.remove_repo(cmd.args[0])!
|
||||||
} else if cmd.args.len == 2 {
|
} else if cmd.args.len == 2 {
|
||||||
client_.remove_arch_repo(cmd.args[0], cmd.args[1])!
|
client.remove_arch_repo(cmd.args[0], cmd.args[1])!
|
||||||
} else {
|
} else {
|
||||||
client_.remove_package(cmd.args[0], cmd.args[1], cmd.args[2])!
|
client.remove_package(cmd.args[0], cmd.args[1], cmd.args[2])!
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
module schedule
|
module schedule
|
||||||
|
|
||||||
import cli
|
import cli
|
||||||
import cron
|
import cron.expression { parse_expression }
|
||||||
import time
|
import time
|
||||||
|
|
||||||
// cmd returns the cli submodule for previewing a cron schedule.
|
// cmd returns the cli submodule for previewing a cron schedule.
|
||||||
|
@ -19,10 +19,10 @@ pub fn cmd() cli.Command {
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
ce := cron.parse_expression(cmd.args.join(' '))!
|
ce := parse_expression(cmd.args.join(' '))!
|
||||||
count := cmd.flags.get_int('count')!
|
count := cmd.flags.get_int('count')!
|
||||||
|
|
||||||
for t in ce.next_n(time.now(), count) {
|
for t in ce.next_n(time.now(), count)! {
|
||||||
println(t)
|
println(t)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,7 +6,7 @@ import os
|
||||||
import build
|
import build
|
||||||
|
|
||||||
// build locally builds the target with the given id.
|
// build locally builds the target with the given id.
|
||||||
fn build_target(conf Config, target_id int, force bool, timeout int) ! {
|
fn build(conf Config, target_id int, force bool) ! {
|
||||||
c := client.new(conf.address, conf.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
target := c.get_target(target_id)!
|
target := c.get_target(target_id)!
|
||||||
|
|
||||||
|
@ -16,7 +16,7 @@ fn build_target(conf Config, target_id int, force bool, timeout int) ! {
|
||||||
image_id := build.create_build_image(conf.base_image)!
|
image_id := build.create_build_image(conf.base_image)!
|
||||||
|
|
||||||
println('Running build...')
|
println('Running build...')
|
||||||
res := build.build_target(conf.address, conf.api_key, image_id, target, force, timeout)!
|
res := build.build_target(conf.address, conf.api_key, image_id, target, force)!
|
||||||
|
|
||||||
println('Removing build image...')
|
println('Removing build image...')
|
||||||
|
|
||||||
|
@ -26,7 +26,7 @@ fn build_target(conf Config, target_id int, force bool, timeout int) ! {
|
||||||
dd.close() or {}
|
dd.close() or {}
|
||||||
}
|
}
|
||||||
|
|
||||||
dd.image_remove(image_id)!
|
dd.remove_image(image_id)!
|
||||||
|
|
||||||
println('Uploading logs to Vieter...')
|
println('Uploading logs to Vieter...')
|
||||||
c.add_build_log(target.id, res.start_time, res.end_time, build_arch, res.exit_code,
|
c.add_build_log(target.id, res.start_time, res.end_time, build_arch, res.exit_code,
|
||||||
|
|
|
@ -2,7 +2,7 @@ module targets
|
||||||
|
|
||||||
import cli
|
import cli
|
||||||
import conf as vconf
|
import conf as vconf
|
||||||
import cron
|
import cron.expression { parse_expression }
|
||||||
import client { NewTarget }
|
import client { NewTarget }
|
||||||
import console
|
import console
|
||||||
import models { TargetFilter }
|
import models { TargetFilter }
|
||||||
|
@ -40,21 +40,10 @@ pub fn cmd() cli.Command {
|
||||||
description: 'Only return targets that publish to this repo.'
|
description: 'Only return targets that publish to this repo.'
|
||||||
flag: cli.FlagType.string
|
flag: cli.FlagType.string
|
||||||
},
|
},
|
||||||
cli.Flag{
|
|
||||||
name: 'query'
|
|
||||||
abbrev: 'q'
|
|
||||||
description: 'Search string to filter targets by.'
|
|
||||||
flag: cli.FlagType.string
|
|
||||||
},
|
|
||||||
cli.Flag{
|
|
||||||
name: 'arch'
|
|
||||||
description: 'Only list targets that build for this arch.'
|
|
||||||
flag: cli.FlagType.string
|
|
||||||
},
|
|
||||||
]
|
]
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
mut filter := TargetFilter{}
|
mut filter := TargetFilter{}
|
||||||
|
|
||||||
|
@ -73,19 +62,9 @@ pub fn cmd() cli.Command {
|
||||||
filter.repo = repo
|
filter.repo = repo
|
||||||
}
|
}
|
||||||
|
|
||||||
query := cmd.flags.get_string('query')!
|
|
||||||
if query != '' {
|
|
||||||
filter.query = query
|
|
||||||
}
|
|
||||||
|
|
||||||
arch := cmd.flags.get_string('arch')!
|
|
||||||
if arch != '' {
|
|
||||||
filter.arch = arch
|
|
||||||
}
|
|
||||||
|
|
||||||
raw := cmd.flags.get_bool('raw')!
|
raw := cmd.flags.get_bool('raw')!
|
||||||
|
|
||||||
list(conf_, filter, raw)!
|
list(conf, filter, raw)!
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
cli.Command{
|
cli.Command{
|
||||||
|
@ -113,7 +92,7 @@ pub fn cmd() cli.Command {
|
||||||
]
|
]
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
t := NewTarget{
|
t := NewTarget{
|
||||||
kind: cmd.flags.get_string('kind')!
|
kind: cmd.flags.get_string('kind')!
|
||||||
|
@ -125,7 +104,7 @@ pub fn cmd() cli.Command {
|
||||||
|
|
||||||
raw := cmd.flags.get_bool('raw')!
|
raw := cmd.flags.get_bool('raw')!
|
||||||
|
|
||||||
add(conf_, t, raw)!
|
add(conf, t, raw)!
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
cli.Command{
|
cli.Command{
|
||||||
|
@ -135,9 +114,9 @@ pub fn cmd() cli.Command {
|
||||||
description: 'Remove a target that matches the given id.'
|
description: 'Remove a target that matches the given id.'
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
remove(conf_, cmd.args[0])!
|
remove(conf, cmd.args[0])!
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
cli.Command{
|
cli.Command{
|
||||||
|
@ -147,9 +126,9 @@ pub fn cmd() cli.Command {
|
||||||
description: 'Show detailed information for the target matching the id.'
|
description: 'Show detailed information for the target matching the id.'
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
info(conf_, cmd.args[0])!
|
info(conf, cmd.args[0])!
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
cli.Command{
|
cli.Command{
|
||||||
|
@ -196,7 +175,7 @@ pub fn cmd() cli.Command {
|
||||||
]
|
]
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
found := cmd.flags.get_all_found()
|
found := cmd.flags.get_all_found()
|
||||||
|
|
||||||
|
@ -208,7 +187,7 @@ pub fn cmd() cli.Command {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
patch(conf_, cmd.args[0], params)!
|
patch(conf, cmd.args[0], params)!
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
cli.Command{
|
cli.Command{
|
||||||
|
@ -232,20 +211,13 @@ pub fn cmd() cli.Command {
|
||||||
description: 'Architecture to schedule build for. Required when using -remote.'
|
description: 'Architecture to schedule build for. Required when using -remote.'
|
||||||
flag: cli.FlagType.string
|
flag: cli.FlagType.string
|
||||||
},
|
},
|
||||||
cli.Flag{
|
|
||||||
name: 'timeout'
|
|
||||||
description: 'After how many minutes to cancel the build. Only applies to local builds.'
|
|
||||||
flag: cli.FlagType.int
|
|
||||||
default_value: ['3600']
|
|
||||||
},
|
|
||||||
]
|
]
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
remote := cmd.flags.get_bool('remote')!
|
remote := cmd.flags.get_bool('remote')!
|
||||||
force := cmd.flags.get_bool('force')!
|
force := cmd.flags.get_bool('force')!
|
||||||
timeout := cmd.flags.get_int('timeout')!
|
|
||||||
target_id := cmd.args[0].int()
|
target_id := cmd.args[0].int()
|
||||||
|
|
||||||
if remote {
|
if remote {
|
||||||
|
@ -255,10 +227,10 @@ pub fn cmd() cli.Command {
|
||||||
return error('When scheduling the build remotely, you have to specify an architecture.')
|
return error('When scheduling the build remotely, you have to specify an architecture.')
|
||||||
}
|
}
|
||||||
|
|
||||||
c := client.new(conf_.address, conf_.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
c.queue_job(target_id, arch, force)!
|
c.queue_job(target_id, arch, force)!
|
||||||
} else {
|
} else {
|
||||||
build_target(conf_, target_id, force, timeout)!
|
build(conf, target_id, force)!
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
@ -267,8 +239,8 @@ pub fn cmd() cli.Command {
|
||||||
}
|
}
|
||||||
|
|
||||||
// list prints out a list of all repositories.
|
// list prints out a list of all repositories.
|
||||||
fn list(conf_ Config, filter TargetFilter, raw bool) ! {
|
fn list(conf Config, filter TargetFilter, raw bool) ! {
|
||||||
c := client.new(conf_.address, conf_.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
targets := c.get_targets(filter)!
|
targets := c.get_targets(filter)!
|
||||||
data := targets.map([it.id.str(), it.kind, it.url, it.repo])
|
data := targets.map([it.id.str(), it.kind, it.url, it.repo])
|
||||||
|
|
||||||
|
@ -280,40 +252,40 @@ fn list(conf_ Config, filter TargetFilter, raw bool) ! {
|
||||||
}
|
}
|
||||||
|
|
||||||
// add adds a new target to the server's list.
|
// add adds a new target to the server's list.
|
||||||
fn add(conf_ Config, t &NewTarget, raw bool) ! {
|
fn add(conf Config, t &NewTarget, raw bool) ! {
|
||||||
c := client.new(conf_.address, conf_.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
target_id := c.add_target(t)!
|
target_id := c.add_target(t)!
|
||||||
|
|
||||||
if raw {
|
if raw {
|
||||||
println(target_id)
|
println(target_id)
|
||||||
} else {
|
} else {
|
||||||
println('Target added with id ${target_id}')
|
println('Target added with id $target_id')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// remove removes a target from the server's list.
|
// remove removes a target from the server's list.
|
||||||
fn remove(conf_ Config, id string) ! {
|
fn remove(conf Config, id string) ! {
|
||||||
c := client.new(conf_.address, conf_.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
c.remove_target(id.int())!
|
c.remove_target(id.int())!
|
||||||
}
|
}
|
||||||
|
|
||||||
// patch patches a given target with the provided params.
|
// patch patches a given target with the provided params.
|
||||||
fn patch(conf_ Config, id string, params map[string]string) ! {
|
fn patch(conf Config, id string, params map[string]string) ! {
|
||||||
// We check the cron expression first because it's useless to send an
|
// We check the cron expression first because it's useless to send an
|
||||||
// invalid one to the server.
|
// invalid one to the server.
|
||||||
if 'schedule' in params && params['schedule'] != '' {
|
if 'schedule' in params && params['schedule'] != '' {
|
||||||
cron.parse_expression(params['schedule']) or {
|
parse_expression(params['schedule']) or {
|
||||||
return error('Invalid cron expression: ${err.msg()}')
|
return error('Invalid cron expression: $err.msg()')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
c := client.new(conf_.address, conf_.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
c.patch_target(id.int(), params)!
|
c.patch_target(id.int(), params)!
|
||||||
}
|
}
|
||||||
|
|
||||||
// info shows detailed information for a given target.
|
// info shows detailed information for a given target.
|
||||||
fn info(conf_ Config, id string) ! {
|
fn info(conf Config, id string) ! {
|
||||||
c := client.new(conf_.address, conf_.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
target := c.get_target(id.int())!
|
target := c.get_target(id.int())!
|
||||||
println(target)
|
println(target)
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,32 @@
|
||||||
|
module cron
|
||||||
|
|
||||||
|
import cli
|
||||||
|
import conf as vconf
|
||||||
|
|
||||||
|
struct Config {
|
||||||
|
pub:
|
||||||
|
log_level string = 'WARN'
|
||||||
|
api_key string
|
||||||
|
address string
|
||||||
|
data_dir string
|
||||||
|
base_image string = 'archlinux:base-devel'
|
||||||
|
max_concurrent_builds int = 1
|
||||||
|
api_update_frequency int = 15
|
||||||
|
image_rebuild_frequency int = 1440
|
||||||
|
// Replicates the behavior of the original cron system
|
||||||
|
global_schedule string = '0 3'
|
||||||
|
}
|
||||||
|
|
||||||
|
// cmd returns the cli module that handles the cron daemon.
|
||||||
|
pub fn cmd() cli.Command {
|
||||||
|
return cli.Command{
|
||||||
|
name: 'cron'
|
||||||
|
description: 'Start the cron service that periodically runs builds.'
|
||||||
|
execute: fn (cmd cli.Command) ! {
|
||||||
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
|
cron(conf)!
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,33 @@
|
||||||
|
module cron
|
||||||
|
|
||||||
|
import log
|
||||||
|
import cron.daemon
|
||||||
|
import cron.expression
|
||||||
|
import os
|
||||||
|
|
||||||
|
const log_file_name = 'vieter.cron.log'
|
||||||
|
|
||||||
|
// cron starts a cron daemon & starts periodically scheduling builds.
|
||||||
|
pub fn cron(conf Config) ! {
|
||||||
|
// Configure logger
|
||||||
|
log_level := log.level_from_tag(conf.log_level) or {
|
||||||
|
return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
|
||||||
|
}
|
||||||
|
|
||||||
|
mut logger := log.Log{
|
||||||
|
level: log_level
|
||||||
|
}
|
||||||
|
|
||||||
|
log_file := os.join_path_single(conf.data_dir, cron.log_file_name)
|
||||||
|
logger.set_full_logpath(log_file)
|
||||||
|
logger.log_to_console_too()
|
||||||
|
|
||||||
|
ce := expression.parse_expression(conf.global_schedule) or {
|
||||||
|
return error('Error while parsing global cron expression: $err.msg()')
|
||||||
|
}
|
||||||
|
|
||||||
|
mut d := daemon.init_daemon(logger, conf.address, conf.api_key, conf.base_image, ce,
|
||||||
|
conf.max_concurrent_builds, conf.api_update_frequency, conf.image_rebuild_frequency)!
|
||||||
|
|
||||||
|
d.run()
|
||||||
|
}
|
|
@ -0,0 +1,115 @@
|
||||||
|
module daemon
|
||||||
|
|
||||||
|
import time
|
||||||
|
import sync.stdatomic
|
||||||
|
import build
|
||||||
|
import os
|
||||||
|
|
||||||
|
const (
|
||||||
|
build_empty = 0
|
||||||
|
build_running = 1
|
||||||
|
build_done = 2
|
||||||
|
)
|
||||||
|
|
||||||
|
// clean_finished_builds removes finished builds from the build slots & returns
|
||||||
|
// them.
|
||||||
|
fn (mut d Daemon) clean_finished_builds() []ScheduledBuild {
|
||||||
|
mut out := []ScheduledBuild{}
|
||||||
|
|
||||||
|
for i in 0 .. d.atomics.len {
|
||||||
|
if stdatomic.load_u64(&d.atomics[i]) == daemon.build_done {
|
||||||
|
stdatomic.store_u64(&d.atomics[i], daemon.build_empty)
|
||||||
|
out << d.builds[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// update_builds starts as many builds as possible.
|
||||||
|
fn (mut d Daemon) start_new_builds() {
|
||||||
|
now := time.now()
|
||||||
|
|
||||||
|
for d.queue.len() > 0 {
|
||||||
|
elem := d.queue.peek() or {
|
||||||
|
d.lerror("queue.peek() unexpectedly returned an error. This shouldn't happen.")
|
||||||
|
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
if elem.timestamp < now {
|
||||||
|
sb := d.queue.pop() or {
|
||||||
|
d.lerror("queue.pop() unexpectedly returned an error. This shouldn't happen.")
|
||||||
|
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// If this build couldn't be scheduled, no more will be possible.
|
||||||
|
if !d.start_build(sb) {
|
||||||
|
d.queue.insert(sb)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// start_build starts a build for the given ScheduledBuild object.
|
||||||
|
fn (mut d Daemon) start_build(sb ScheduledBuild) bool {
|
||||||
|
for i in 0 .. d.atomics.len {
|
||||||
|
if stdatomic.load_u64(&d.atomics[i]) == daemon.build_empty {
|
||||||
|
stdatomic.store_u64(&d.atomics[i], daemon.build_running)
|
||||||
|
d.builds[i] = sb
|
||||||
|
|
||||||
|
go d.run_build(i, sb)
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// run_build actually starts the build process for a given target.
|
||||||
|
fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) {
|
||||||
|
d.linfo('started build: $sb.target.url -> $sb.target.repo')
|
||||||
|
|
||||||
|
// 0 means success, 1 means failure
|
||||||
|
mut status := 0
|
||||||
|
|
||||||
|
res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(),
|
||||||
|
&sb.target, false) or {
|
||||||
|
d.ldebug('build_target error: $err.msg()')
|
||||||
|
status = 1
|
||||||
|
|
||||||
|
build.BuildResult{}
|
||||||
|
}
|
||||||
|
|
||||||
|
if status == 0 {
|
||||||
|
d.linfo('finished build: $sb.target.url -> $sb.target.repo; uploading logs...')
|
||||||
|
|
||||||
|
build_arch := os.uname().machine
|
||||||
|
d.client.add_build_log(sb.target.id, res.start_time, res.end_time, build_arch,
|
||||||
|
res.exit_code, res.logs) or {
|
||||||
|
d.lerror('Failed to upload logs for build: $sb.target.url -> $sb.target.repo')
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
d.linfo('an error occured during build: $sb.target.url -> $sb.target.repo')
|
||||||
|
}
|
||||||
|
|
||||||
|
stdatomic.store_u64(&d.atomics[build_index], daemon.build_done)
|
||||||
|
}
|
||||||
|
|
||||||
|
// current_build_count returns how many builds are currently running.
|
||||||
|
fn (mut d Daemon) current_build_count() int {
|
||||||
|
mut res := 0
|
||||||
|
|
||||||
|
for i in 0 .. d.atomics.len {
|
||||||
|
if stdatomic.load_u64(&d.atomics[i]) == daemon.build_running {
|
||||||
|
res += 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return res
|
||||||
|
}
|
|
@ -0,0 +1,274 @@
|
||||||
|
module daemon
|
||||||
|
|
||||||
|
import time
|
||||||
|
import log
|
||||||
|
import datatypes { MinHeap }
|
||||||
|
import cron.expression { CronExpression, parse_expression }
|
||||||
|
import math
|
||||||
|
import build
|
||||||
|
import docker
|
||||||
|
import os
|
||||||
|
import client
|
||||||
|
import models { Target }
|
||||||
|
|
||||||
|
const (
|
||||||
|
// How many seconds to wait before retrying to update API if failed
|
||||||
|
api_update_retry_timeout = 5
|
||||||
|
// How many seconds to wait before retrying to rebuild image if failed
|
||||||
|
rebuild_base_image_retry_timout = 30
|
||||||
|
)
|
||||||
|
|
||||||
|
struct ScheduledBuild {
|
||||||
|
pub:
|
||||||
|
target Target
|
||||||
|
timestamp time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// Overloaded operator for comparing ScheduledBuild objects
|
||||||
|
fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool {
|
||||||
|
return r1.timestamp < r2.timestamp
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct Daemon {
|
||||||
|
mut:
|
||||||
|
client client.Client
|
||||||
|
base_image string
|
||||||
|
builder_images []string
|
||||||
|
global_schedule CronExpression
|
||||||
|
api_update_frequency int
|
||||||
|
image_rebuild_frequency int
|
||||||
|
// Targets currently loaded from API.
|
||||||
|
targets []Target
|
||||||
|
// At what point to update the list of targets.
|
||||||
|
api_update_timestamp time.Time
|
||||||
|
image_build_timestamp time.Time
|
||||||
|
queue MinHeap<ScheduledBuild>
|
||||||
|
// Which builds are currently running
|
||||||
|
builds []ScheduledBuild
|
||||||
|
// Atomic variables used to detect when a build has finished; length is the
|
||||||
|
// same as builds
|
||||||
|
atomics []u64
|
||||||
|
logger shared log.Log
|
||||||
|
}
|
||||||
|
|
||||||
|
// init_daemon initializes a new Daemon object. It renews the targets &
|
||||||
|
// populates the build queue for the first time.
|
||||||
|
pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) !Daemon {
|
||||||
|
mut d := Daemon{
|
||||||
|
client: client.new(address, api_key)
|
||||||
|
base_image: base_image
|
||||||
|
global_schedule: global_schedule
|
||||||
|
api_update_frequency: api_update_frequency
|
||||||
|
image_rebuild_frequency: image_rebuild_frequency
|
||||||
|
atomics: []u64{len: max_concurrent_builds}
|
||||||
|
builds: []ScheduledBuild{len: max_concurrent_builds}
|
||||||
|
logger: logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize the targets & queue
|
||||||
|
d.renew_targets()
|
||||||
|
d.renew_queue()
|
||||||
|
if !d.rebuild_base_image() {
|
||||||
|
return error('The base image failed to build. The Vieter cron daemon cannot run without an initial builder image.')
|
||||||
|
}
|
||||||
|
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
// run starts the actual daemon process. It runs builds when possible &
|
||||||
|
// periodically refreshes the list of targets to ensure we stay in sync.
|
||||||
|
pub fn (mut d Daemon) run() {
|
||||||
|
for {
|
||||||
|
finished_builds := d.clean_finished_builds()
|
||||||
|
|
||||||
|
// Update the API's contents if needed & renew the queue
|
||||||
|
if time.now() >= d.api_update_timestamp {
|
||||||
|
d.renew_targets()
|
||||||
|
d.renew_queue()
|
||||||
|
}
|
||||||
|
// The finished builds should only be rescheduled if the API contents
|
||||||
|
// haven't been renewed.
|
||||||
|
else {
|
||||||
|
for sb in finished_builds {
|
||||||
|
d.schedule_build(sb.target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO remove old builder images.
|
||||||
|
// This issue is less trivial than it sounds, because a build could
|
||||||
|
// still be running when the image has to be rebuilt. That would
|
||||||
|
// prevent the image from being removed. Therefore, we will need to
|
||||||
|
// keep track of a list or something & remove an image once we have
|
||||||
|
// made sure it isn't being used anymore.
|
||||||
|
if time.now() >= d.image_build_timestamp {
|
||||||
|
d.rebuild_base_image()
|
||||||
|
// In theory, executing this function here allows an old builder
|
||||||
|
// image to exist for at most image_rebuild_frequency minutes.
|
||||||
|
d.clean_old_base_images()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Schedules new builds when possible
|
||||||
|
d.start_new_builds()
|
||||||
|
|
||||||
|
// If there are builds currently running, the daemon should refresh
|
||||||
|
// every second to clean up any finished builds & start new ones.
|
||||||
|
mut delay := time.Duration(1 * time.second)
|
||||||
|
|
||||||
|
// Sleep either until we have to refresh the targets or when the next
|
||||||
|
// build has to start, with a minimum of 1 second.
|
||||||
|
if d.current_build_count() == 0 {
|
||||||
|
now := time.now()
|
||||||
|
delay = d.api_update_timestamp - now
|
||||||
|
|
||||||
|
if d.queue.len() > 0 {
|
||||||
|
elem := d.queue.peek() or {
|
||||||
|
d.lerror("queue.peek() unexpectedly returned an error. This shouldn't happen.")
|
||||||
|
|
||||||
|
// This is just a fallback option. In theory, queue.peek()
|
||||||
|
// should *never* return an error or none, because we check
|
||||||
|
// its len beforehand.
|
||||||
|
time.sleep(1)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
time_until_next_job := elem.timestamp - now
|
||||||
|
|
||||||
|
delay = math.min(delay, time_until_next_job)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// We sleep for at least one second. This is to prevent the program
|
||||||
|
// from looping agressively when a cronjob can be scheduled, but
|
||||||
|
// there's no spots free for it to be started.
|
||||||
|
delay = math.max(delay, 1 * time.second)
|
||||||
|
|
||||||
|
d.ldebug('Sleeping for ${delay}...')
|
||||||
|
|
||||||
|
time.sleep(delay)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// schedule_build adds the next occurence of the given targets build to the
|
||||||
|
// queue.
|
||||||
|
fn (mut d Daemon) schedule_build(target Target) {
|
||||||
|
ce := if target.schedule != '' {
|
||||||
|
parse_expression(target.schedule) or {
|
||||||
|
// TODO This shouldn't return an error if the expression is empty.
|
||||||
|
d.lerror("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()")
|
||||||
|
|
||||||
|
d.global_schedule
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
d.global_schedule
|
||||||
|
}
|
||||||
|
|
||||||
|
// A target that can't be scheduled will just be skipped for now
|
||||||
|
timestamp := ce.next_from_now() or {
|
||||||
|
d.lerror("Couldn't calculate next timestamp from '$target.schedule'; skipping")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
d.queue.insert(ScheduledBuild{
|
||||||
|
target: target
|
||||||
|
timestamp: timestamp
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// renew_targets requests the newest list of targets from the server & replaces
|
||||||
|
// the old one.
|
||||||
|
fn (mut d Daemon) renew_targets() {
|
||||||
|
d.linfo('Renewing targets...')
|
||||||
|
|
||||||
|
mut new_targets := d.client.get_all_targets() or {
|
||||||
|
d.lerror('Failed to renew targets. Retrying in ${daemon.api_update_retry_timeout}s...')
|
||||||
|
d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter out any targets that shouldn't run on this architecture
|
||||||
|
cur_arch := os.uname().machine
|
||||||
|
new_targets = new_targets.filter(it.arch.any(it.value == cur_arch))
|
||||||
|
|
||||||
|
d.targets = new_targets
|
||||||
|
|
||||||
|
d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency)
|
||||||
|
}
|
||||||
|
|
||||||
|
// renew_queue replaces the old queue with a new one that reflects the newest
|
||||||
|
// values in targets.
|
||||||
|
fn (mut d Daemon) renew_queue() {
|
||||||
|
d.linfo('Renewing queue...')
|
||||||
|
mut new_queue := MinHeap<ScheduledBuild>{}
|
||||||
|
|
||||||
|
// Move any jobs that should have already started from the old queue onto
|
||||||
|
// the new one
|
||||||
|
now := time.now()
|
||||||
|
|
||||||
|
// For some reason, using
|
||||||
|
// ```v
|
||||||
|
// for d.queue.len() > 0 && d.queue.peek() !.timestamp < now {
|
||||||
|
//```
|
||||||
|
// here causes the function to prematurely just exit, without any errors or anything, very weird
|
||||||
|
// https://github.com/vlang/v/issues/14042
|
||||||
|
for d.queue.len() > 0 {
|
||||||
|
elem := d.queue.pop() or {
|
||||||
|
d.lerror("queue.pop() returned an error. This shouldn't happen.")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if elem.timestamp < now {
|
||||||
|
new_queue.insert(elem)
|
||||||
|
} else {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
d.queue = new_queue
|
||||||
|
|
||||||
|
// For each target in targets, parse their cron expression (or use the
|
||||||
|
// default one if not present) & add them to the queue
|
||||||
|
for target in d.targets {
|
||||||
|
d.schedule_build(target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rebuild_base_image recreates the builder image.
|
||||||
|
fn (mut d Daemon) rebuild_base_image() bool {
|
||||||
|
d.linfo('Rebuilding builder image....')
|
||||||
|
|
||||||
|
d.builder_images << build.create_build_image(d.base_image) or {
|
||||||
|
d.lerror('Failed to rebuild base image. Retrying in ${daemon.rebuild_base_image_retry_timout}s...')
|
||||||
|
d.image_build_timestamp = time.now().add_seconds(daemon.rebuild_base_image_retry_timout)
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
d.image_build_timestamp = time.now().add_seconds(60 * d.image_rebuild_frequency)
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// clean_old_base_images tries to remove any old but still present builder
|
||||||
|
// images.
|
||||||
|
fn (mut d Daemon) clean_old_base_images() {
|
||||||
|
mut i := 0
|
||||||
|
|
||||||
|
mut dd := docker.new_conn() or {
|
||||||
|
d.lerror('Failed to connect to Docker socket.')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
defer {
|
||||||
|
dd.close() or {}
|
||||||
|
}
|
||||||
|
|
||||||
|
for i < d.builder_images.len - 1 {
|
||||||
|
// For each builder image, we try to remove it by calling the Docker
|
||||||
|
// API. If the function returns an error or false, that means the image
|
||||||
|
// wasn't deleted. Therefore, we move the index over. If the function
|
||||||
|
// returns true, the array's length has decreased by one so we don't
|
||||||
|
// move the index.
|
||||||
|
dd.remove_image(d.builder_images[i]) or { i += 1 }
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
module daemon
|
||||||
|
|
||||||
|
import log
|
||||||
|
|
||||||
|
// log reate a log message with the given level
|
||||||
|
pub fn (mut d Daemon) log(msg string, level log.Level) {
|
||||||
|
lock d.logger {
|
||||||
|
d.logger.send_output(msg, level)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// lfatal create a log message with the fatal level
|
||||||
|
pub fn (mut d Daemon) lfatal(msg string) {
|
||||||
|
d.log(msg, log.Level.fatal)
|
||||||
|
}
|
||||||
|
|
||||||
|
// lerror create a log message with the error level
|
||||||
|
pub fn (mut d Daemon) lerror(msg string) {
|
||||||
|
d.log(msg, log.Level.error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// lwarn create a log message with the warn level
|
||||||
|
pub fn (mut d Daemon) lwarn(msg string) {
|
||||||
|
d.log(msg, log.Level.warn)
|
||||||
|
}
|
||||||
|
|
||||||
|
// linfo create a log message with the info level
|
||||||
|
pub fn (mut d Daemon) linfo(msg string) {
|
||||||
|
d.log(msg, log.Level.info)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ldebug create a log message with the debug level
|
||||||
|
pub fn (mut d Daemon) ldebug(msg string) {
|
||||||
|
d.log(msg, log.Level.debug)
|
||||||
|
}
|
|
@ -1,101 +0,0 @@
|
||||||
module cron
|
|
||||||
|
|
||||||
#flag -I @VMODROOT/libvieter/include
|
|
||||||
#flag -L @VMODROOT/libvieter/build
|
|
||||||
#flag -lvieter
|
|
||||||
#include "vieter_cron.h"
|
|
||||||
|
|
||||||
[typedef]
|
|
||||||
pub struct C.vieter_cron_expression {
|
|
||||||
minutes &u8
|
|
||||||
hours &u8
|
|
||||||
days &u8
|
|
||||||
months &u8
|
|
||||||
minute_count u8
|
|
||||||
hour_count u8
|
|
||||||
day_count u8
|
|
||||||
month_count u8
|
|
||||||
}
|
|
||||||
|
|
||||||
pub type Expression = C.vieter_cron_expression
|
|
||||||
|
|
||||||
// == returns whether the two expressions are equal by value.
|
|
||||||
fn (ce1 Expression) == (ce2 Expression) bool {
|
|
||||||
if ce1.month_count != ce2.month_count || ce1.day_count != ce2.day_count
|
|
||||||
|| ce1.hour_count != ce2.hour_count || ce1.minute_count != ce2.minute_count {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
for i in 0 .. ce1.month_count {
|
|
||||||
unsafe {
|
|
||||||
if ce1.months[i] != ce2.months[i] {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i in 0 .. ce1.day_count {
|
|
||||||
unsafe {
|
|
||||||
if ce1.days[i] != ce2.days[i] {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i in 0 .. ce1.hour_count {
|
|
||||||
unsafe {
|
|
||||||
if ce1.hours[i] != ce2.hours[i] {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i in 0 .. ce1.minute_count {
|
|
||||||
unsafe {
|
|
||||||
if ce1.minutes[i] != ce2.minutes[i] {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
[typedef]
|
|
||||||
struct C.vieter_cron_simple_time {
|
|
||||||
year int
|
|
||||||
month int
|
|
||||||
day int
|
|
||||||
hour int
|
|
||||||
minute int
|
|
||||||
}
|
|
||||||
|
|
||||||
type SimpleTime = C.vieter_cron_simple_time
|
|
||||||
|
|
||||||
enum ParseError as u8 {
|
|
||||||
ok = 0
|
|
||||||
invalid_expression = 1
|
|
||||||
invalid_number = 2
|
|
||||||
out_of_range = 3
|
|
||||||
too_many_parts = 4
|
|
||||||
not_enough_parts = 5
|
|
||||||
}
|
|
||||||
|
|
||||||
// str returns the string representation of a ParseError.
|
|
||||||
fn (e ParseError) str() string {
|
|
||||||
return match e {
|
|
||||||
.ok { '' }
|
|
||||||
.invalid_expression { 'Invalid expression' }
|
|
||||||
.invalid_number { 'Invalid number' }
|
|
||||||
.out_of_range { 'Out of range' }
|
|
||||||
.too_many_parts { 'Too many parts' }
|
|
||||||
.not_enough_parts { 'Not enough parts' }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn C.vieter_cron_expr_init() &C.vieter_cron_expression
|
|
||||||
|
|
||||||
fn C.vieter_cron_expr_free(ce &C.vieter_cron_expression)
|
|
||||||
|
|
||||||
fn C.vieter_cron_expr_next(out &C.vieter_cron_simple_time, ce &C.vieter_cron_expression, ref &C.vieter_cron_simple_time)
|
|
||||||
|
|
||||||
fn C.vieter_cron_expr_next_from_now(out &C.vieter_cron_simple_time, ce &C.vieter_cron_expression)
|
|
||||||
|
|
||||||
fn C.vieter_cron_expr_parse(out &C.vieter_cron_expression, s &char) ParseError
|
|
|
@ -1,73 +0,0 @@
|
||||||
module cron
|
|
||||||
|
|
||||||
import time
|
|
||||||
|
|
||||||
// free the memory associated with the Expression.
|
|
||||||
[unsafe]
|
|
||||||
pub fn (ce &Expression) free() {
|
|
||||||
C.vieter_cron_expr_free(ce)
|
|
||||||
}
|
|
||||||
|
|
||||||
// parse_expression parses a string into an Expression.
|
|
||||||
pub fn parse_expression(exp string) !&Expression {
|
|
||||||
out := C.vieter_cron_expr_init()
|
|
||||||
res := C.vieter_cron_expr_parse(out, exp.str)
|
|
||||||
|
|
||||||
if res != .ok {
|
|
||||||
return error(res.str())
|
|
||||||
}
|
|
||||||
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// next calculates the next occurence of the cron schedule, given a reference
|
|
||||||
// point.
|
|
||||||
pub fn (ce &Expression) next(ref time.Time) time.Time {
|
|
||||||
st := SimpleTime{
|
|
||||||
year: ref.year
|
|
||||||
month: ref.month
|
|
||||||
day: ref.day
|
|
||||||
hour: ref.hour
|
|
||||||
minute: ref.minute
|
|
||||||
}
|
|
||||||
|
|
||||||
out := SimpleTime{}
|
|
||||||
C.vieter_cron_expr_next(&out, ce, &st)
|
|
||||||
|
|
||||||
return time.new_time(time.Time{
|
|
||||||
year: out.year
|
|
||||||
month: out.month
|
|
||||||
day: out.day
|
|
||||||
hour: out.hour
|
|
||||||
minute: out.minute
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// next_from_now calculates the next occurence of the cron schedule with the
|
|
||||||
// current time as reference.
|
|
||||||
pub fn (ce &Expression) next_from_now() time.Time {
|
|
||||||
out := SimpleTime{}
|
|
||||||
C.vieter_cron_expr_next_from_now(&out, ce)
|
|
||||||
|
|
||||||
return time.new_time(time.Time{
|
|
||||||
year: out.year
|
|
||||||
month: out.month
|
|
||||||
day: out.day
|
|
||||||
hour: out.hour
|
|
||||||
minute: out.minute
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// next_n returns the n next occurences of the expression, given a starting
|
|
||||||
// time.
|
|
||||||
pub fn (ce &Expression) next_n(ref time.Time, n int) []time.Time {
|
|
||||||
mut times := []time.Time{cap: n}
|
|
||||||
|
|
||||||
times << ce.next(ref)
|
|
||||||
|
|
||||||
for i in 1 .. n {
|
|
||||||
times << ce.next(times[i - 1])
|
|
||||||
}
|
|
||||||
|
|
||||||
return times
|
|
||||||
}
|
|
|
@ -0,0 +1,136 @@
|
||||||
|
module expression
|
||||||
|
|
||||||
|
import time
|
||||||
|
|
||||||
|
pub struct CronExpression {
|
||||||
|
minutes []int
|
||||||
|
hours []int
|
||||||
|
days []int
|
||||||
|
months []int
|
||||||
|
}
|
||||||
|
|
||||||
|
// next calculates the earliest time this cron expression is valid. It will
|
||||||
|
// always pick a moment in the future, even if ref matches completely up to the
|
||||||
|
// minute. This function conciously does not take gap years into account.
|
||||||
|
pub fn (ce &CronExpression) next(ref time.Time) !time.Time {
|
||||||
|
// If the given ref matches the next cron occurence up to the minute, it
|
||||||
|
// will return that value. Because we always want to return a value in the
|
||||||
|
// future, we artifically shift the ref 60 seconds to make sure we always
|
||||||
|
// match in the future. A shift of 60 seconds is enough because the cron
|
||||||
|
// expression does not allow for accuracy smaller than one minute.
|
||||||
|
sref := ref
|
||||||
|
|
||||||
|
// For all of these values, the rule is the following: if their value is
|
||||||
|
// the length of their respective array in the CronExpression object, that
|
||||||
|
// means we've looped back around. This means that the "bigger" value has
|
||||||
|
// to be incremented by one. For example, if the minutes have looped
|
||||||
|
// around, that means that the hour has to be incremented as well.
|
||||||
|
mut minute_index := 0
|
||||||
|
mut hour_index := 0
|
||||||
|
mut day_index := 0
|
||||||
|
mut month_index := 0
|
||||||
|
|
||||||
|
// This chain is the same logic multiple times, namely that if a "bigger"
|
||||||
|
// value loops around, then the smaller value will always reset as well.
|
||||||
|
// For example, if we're going to a new day, the hour & minute will always
|
||||||
|
// be their smallest value again.
|
||||||
|
for month_index < ce.months.len && sref.month > ce.months[month_index] {
|
||||||
|
month_index++
|
||||||
|
}
|
||||||
|
|
||||||
|
if month_index < ce.months.len && sref.month == ce.months[month_index] {
|
||||||
|
for day_index < ce.days.len && sref.day > ce.days[day_index] {
|
||||||
|
day_index++
|
||||||
|
}
|
||||||
|
|
||||||
|
if day_index < ce.days.len && ce.days[day_index] == sref.day {
|
||||||
|
for hour_index < ce.hours.len && sref.hour > ce.hours[hour_index] {
|
||||||
|
hour_index++
|
||||||
|
}
|
||||||
|
|
||||||
|
if hour_index < ce.hours.len && ce.hours[hour_index] == sref.hour {
|
||||||
|
// Minute is the only value where we explicitely make sure we
|
||||||
|
// can't match sref's value exactly. This is to ensure we only
|
||||||
|
// return values in the future.
|
||||||
|
for minute_index < ce.minutes.len && sref.minute >= ce.minutes[minute_index] {
|
||||||
|
minute_index++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Here, we increment the "bigger" values by one if the smaller ones loop
|
||||||
|
// around. The order is important, as it allows a sort-of waterfall effect
|
||||||
|
// to occur which updates all values if required.
|
||||||
|
if minute_index == ce.minutes.len && hour_index < ce.hours.len {
|
||||||
|
hour_index += 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if hour_index == ce.hours.len && day_index < ce.days.len {
|
||||||
|
day_index += 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if day_index == ce.days.len && month_index < ce.months.len {
|
||||||
|
month_index += 1
|
||||||
|
}
|
||||||
|
|
||||||
|
mut minute := ce.minutes[minute_index % ce.minutes.len]
|
||||||
|
mut hour := ce.hours[hour_index % ce.hours.len]
|
||||||
|
mut day := ce.days[day_index % ce.days.len]
|
||||||
|
|
||||||
|
// Sometimes, we end up with a day that does not exist within the selected
|
||||||
|
// month, e.g. day 30 in February. When this occurs, we reset day back to
|
||||||
|
// the smallest value & loop over to the next month that does have this
|
||||||
|
// day.
|
||||||
|
if day > time.month_days[ce.months[month_index % ce.months.len] - 1] {
|
||||||
|
day = ce.days[0]
|
||||||
|
month_index += 1
|
||||||
|
|
||||||
|
for day > time.month_days[ce.months[month_index & ce.months.len] - 1] {
|
||||||
|
month_index += 1
|
||||||
|
|
||||||
|
// If for whatever reason the day value ends up being something
|
||||||
|
// that can't be scheduled in any month, we have to make sure we
|
||||||
|
// don't create an infinite loop.
|
||||||
|
if month_index == 2 * ce.months.len {
|
||||||
|
return error('No schedulable moment.')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
month := ce.months[month_index % ce.months.len]
|
||||||
|
mut year := sref.year
|
||||||
|
|
||||||
|
// If the month loops over, we need to increment the year.
|
||||||
|
if month_index >= ce.months.len {
|
||||||
|
year++
|
||||||
|
}
|
||||||
|
|
||||||
|
return time.new_time(time.Time{
|
||||||
|
year: year
|
||||||
|
month: month
|
||||||
|
day: day
|
||||||
|
minute: minute
|
||||||
|
hour: hour
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// next_from_now returns the result of ce.next(ref) where ref is the result of
|
||||||
|
// time.now().
|
||||||
|
pub fn (ce &CronExpression) next_from_now() !time.Time {
|
||||||
|
return ce.next(time.now())
|
||||||
|
}
|
||||||
|
|
||||||
|
// next_n returns the n next occurences of the expression, given a starting
|
||||||
|
// time.
|
||||||
|
pub fn (ce &CronExpression) next_n(ref time.Time, n int) ![]time.Time {
|
||||||
|
mut times := []time.Time{cap: n}
|
||||||
|
|
||||||
|
times << ce.next(ref)!
|
||||||
|
|
||||||
|
for i in 1 .. n {
|
||||||
|
times << ce.next(times[i - 1])!
|
||||||
|
}
|
||||||
|
|
||||||
|
return times
|
||||||
|
}
|
|
@ -0,0 +1,146 @@
|
||||||
|
module expression
|
||||||
|
|
||||||
|
import bitfield
|
||||||
|
|
||||||
|
// parse_range parses a given string into a range of sorted integers. Its
|
||||||
|
// result is a BitField with set bits for all numbers in the result.
|
||||||
|
fn parse_range(s string, min int, max int) !bitfield.BitField {
|
||||||
|
mut start := min
|
||||||
|
mut end := max
|
||||||
|
mut interval := 1
|
||||||
|
mut bf := bitfield.new(max - min + 1)
|
||||||
|
|
||||||
|
exps := s.split('/')
|
||||||
|
|
||||||
|
if exps.len > 2 {
|
||||||
|
return error('Invalid expression.')
|
||||||
|
}
|
||||||
|
|
||||||
|
if exps[0] != '*' {
|
||||||
|
dash_parts := exps[0].split('-')
|
||||||
|
|
||||||
|
if dash_parts.len > 2 {
|
||||||
|
return error('Invalid expression.')
|
||||||
|
}
|
||||||
|
|
||||||
|
start = dash_parts[0].int()
|
||||||
|
|
||||||
|
// The builtin parsing functions return zero if the string can't be
|
||||||
|
// parsed into a number, so we have to explicitely check whether they
|
||||||
|
// actually entered zero or if it's an invalid number.
|
||||||
|
if start == 0 && dash_parts[0] != '0' {
|
||||||
|
return error('Invalid number.')
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check whether the start value is out of range
|
||||||
|
if start < min || start > max {
|
||||||
|
return error('Out of range.')
|
||||||
|
}
|
||||||
|
|
||||||
|
if dash_parts.len == 2 {
|
||||||
|
end = dash_parts[1].int()
|
||||||
|
|
||||||
|
if end == 0 && dash_parts[1] != '0' {
|
||||||
|
return error('Invalid number.')
|
||||||
|
}
|
||||||
|
|
||||||
|
if end < start || end > max {
|
||||||
|
return error('Out of range.')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if exps.len > 1 {
|
||||||
|
interval = exps[1].int()
|
||||||
|
|
||||||
|
// interval being zero is always invalid, but we want to check why
|
||||||
|
// it's invalid for better error messages.
|
||||||
|
if interval == 0 {
|
||||||
|
if exps[1] != '0' {
|
||||||
|
return error('Invalid number.')
|
||||||
|
} else {
|
||||||
|
return error('Step size zero not allowed.')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if interval > max - min {
|
||||||
|
return error('Step size too large.')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Here, s solely consists of a number, so that's the only value we
|
||||||
|
// should return.
|
||||||
|
else if exps[0] != '*' && !exps[0].contains('-') {
|
||||||
|
bf.set_bit(start - min)
|
||||||
|
return bf
|
||||||
|
}
|
||||||
|
|
||||||
|
for start <= end {
|
||||||
|
bf.set_bit(start - min)
|
||||||
|
start += interval
|
||||||
|
}
|
||||||
|
|
||||||
|
return bf
|
||||||
|
}
|
||||||
|
|
||||||
|
// bf_to_ints takes a BitField and converts it into the expected list of actual
|
||||||
|
// integers.
|
||||||
|
fn bf_to_ints(bf bitfield.BitField, min int) []int {
|
||||||
|
mut out := []int{}
|
||||||
|
|
||||||
|
for i in 0 .. bf.get_size() {
|
||||||
|
if bf.get_bit(i) == 1 {
|
||||||
|
out << min + i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// parse_part parses a given part of a cron expression & returns the
|
||||||
|
// corresponding array of ints.
|
||||||
|
fn parse_part(s string, min int, max int) ![]int {
|
||||||
|
mut bf := bitfield.new(max - min + 1)
|
||||||
|
|
||||||
|
for range in s.split(',') {
|
||||||
|
bf2 := parse_range(range, min, max)!
|
||||||
|
bf = bitfield.bf_or(bf, bf2)
|
||||||
|
}
|
||||||
|
|
||||||
|
return bf_to_ints(bf, min)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parse_expression parses an entire cron expression string into a
|
||||||
|
// CronExpression object, if possible.
|
||||||
|
pub fn parse_expression(exp string) !CronExpression {
|
||||||
|
// The filter allows for multiple spaces between parts
|
||||||
|
mut parts := exp.split(' ').filter(it != '')
|
||||||
|
|
||||||
|
if parts.len < 2 || parts.len > 4 {
|
||||||
|
return error('Expression must contain between 2 and 4 space-separated parts.')
|
||||||
|
}
|
||||||
|
|
||||||
|
// For ease of use, we allow the user to only specify as many parts as they
|
||||||
|
// need.
|
||||||
|
for parts.len < 4 {
|
||||||
|
parts << '*'
|
||||||
|
}
|
||||||
|
|
||||||
|
mut part_results := [][]int{}
|
||||||
|
|
||||||
|
mins := [0, 0, 1, 1]
|
||||||
|
maxs := [59, 23, 31, 12]
|
||||||
|
|
||||||
|
// This for loop allows us to more clearly propagate the error to the user.
|
||||||
|
for i, min in mins {
|
||||||
|
part_results << parse_part(parts[i], min, maxs[i]) or {
|
||||||
|
return error('An error occurred with part $i: $err.msg()')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return CronExpression{
|
||||||
|
minutes: part_results[0]
|
||||||
|
hours: part_results[1]
|
||||||
|
days: part_results[2]
|
||||||
|
months: part_results[3]
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,89 @@
|
||||||
|
module expression
|
||||||
|
|
||||||
|
// parse_range_error returns the returned error message. If the result is '',
|
||||||
|
// that means the function didn't error.
|
||||||
|
fn parse_range_error(s string, min int, max int) string {
|
||||||
|
parse_range(s, min, max) or { return err.msg }
|
||||||
|
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
|
||||||
|
// =====parse_range=====
|
||||||
|
fn test_range_star_range() ! {
|
||||||
|
bf := parse_range('*', 0, 5)!
|
||||||
|
|
||||||
|
assert bf_to_ints(bf, 0) == [0, 1, 2, 3, 4, 5]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_range_number() ! {
|
||||||
|
bf := parse_range('4', 0, 5)!
|
||||||
|
|
||||||
|
assert bf_to_ints(bf, 0) == [4]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_range_number_too_large() ! {
|
||||||
|
assert parse_range_error('10', 0, 6) == 'Out of range.'
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_range_number_too_small() ! {
|
||||||
|
assert parse_range_error('0', 2, 6) == 'Out of range.'
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_range_number_invalid() ! {
|
||||||
|
assert parse_range_error('x', 0, 6) == 'Invalid number.'
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_range_step_star_1() ! {
|
||||||
|
bf := parse_range('*/4', 0, 20)!
|
||||||
|
|
||||||
|
assert bf_to_ints(bf, 0) == [0, 4, 8, 12, 16, 20]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_range_step_star_2() ! {
|
||||||
|
bf := parse_range('*/3', 1, 8)!
|
||||||
|
|
||||||
|
assert bf_to_ints(bf, 1) == [1, 4, 7]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_range_step_star_too_large() ! {
|
||||||
|
assert parse_range_error('*/21', 0, 20) == 'Step size too large.'
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_range_step_zero() ! {
|
||||||
|
assert parse_range_error('*/0', 0, 20) == 'Step size zero not allowed.'
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_range_step_number() ! {
|
||||||
|
bf := parse_range('5/4', 2, 22)!
|
||||||
|
|
||||||
|
assert bf_to_ints(bf, 2) == [5, 9, 13, 17, 21]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_range_step_number_too_large() ! {
|
||||||
|
assert parse_range_error('10/4', 0, 5) == 'Out of range.'
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_range_step_number_too_small() ! {
|
||||||
|
assert parse_range_error('2/4', 5, 10) == 'Out of range.'
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_range_dash() ! {
|
||||||
|
bf := parse_range('4-8', 0, 9)!
|
||||||
|
|
||||||
|
assert bf_to_ints(bf, 0) == [4, 5, 6, 7, 8]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_range_dash_step() ! {
|
||||||
|
bf := parse_range('4-8/2', 0, 9)!
|
||||||
|
|
||||||
|
assert bf_to_ints(bf, 0) == [4, 6, 8]
|
||||||
|
}
|
||||||
|
|
||||||
|
// =====parse_part=====
|
||||||
|
fn test_part_single() ! {
|
||||||
|
assert parse_part('*', 0, 5)! == [0, 1, 2, 3, 4, 5]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_part_multiple() ! {
|
||||||
|
assert parse_part('*/2,2/3', 1, 8)! == [1, 2, 3, 5, 7, 8]
|
||||||
|
}
|
|
@ -1,4 +1,4 @@
|
||||||
module cron
|
module expression
|
||||||
|
|
||||||
import time { parse }
|
import time { parse }
|
||||||
|
|
||||||
|
@ -7,7 +7,7 @@ fn util_test_time(exp string, t1_str string, t2_str string) ! {
|
||||||
t1 := parse(t1_str)!
|
t1 := parse(t1_str)!
|
||||||
t2 := parse(t2_str)!
|
t2 := parse(t2_str)!
|
||||||
|
|
||||||
t3 := ce.next(t1)
|
t3 := ce.next(t1)!
|
||||||
|
|
||||||
assert t2.year == t3.year
|
assert t2.year == t3.year
|
||||||
assert t2.month == t3.month
|
assert t2.month == t3.month
|
||||||
|
@ -18,18 +18,17 @@ fn util_test_time(exp string, t1_str string, t2_str string) ! {
|
||||||
|
|
||||||
fn test_next_simple() ! {
|
fn test_next_simple() ! {
|
||||||
// Very simple
|
// Very simple
|
||||||
// util_test_time('0 3', '2002-01-01 00:00:00', '2002-01-01 03:00:00')!
|
util_test_time('0 3', '2002-01-01 00:00:00', '2002-01-01 03:00:00')!
|
||||||
|
|
||||||
// Overlap to next day
|
// Overlap to next day
|
||||||
mut exp := '0 3 '
|
util_test_time('0 3', '2002-01-01 03:00:00', '2002-01-02 03:00:00')!
|
||||||
util_test_time(exp, '2002-01-01 03:00:00', '2002-01-02 03:00:00')!
|
util_test_time('0 3', '2002-01-01 04:00:00', '2002-01-02 03:00:00')!
|
||||||
util_test_time(exp, '2002-01-01 04:00:00', '2002-01-02 03:00:00')!
|
|
||||||
|
|
||||||
util_test_time('0 3-7/4,7-19', '2002-01-01 04:00:00', '2002-01-01 07:00:00')!
|
util_test_time('0 3/4', '2002-01-01 04:00:00', '2002-01-01 07:00:00')!
|
||||||
|
|
||||||
//// Overlap to next month
|
// Overlap to next month
|
||||||
util_test_time('0 3', '2002-11-31 04:00:00', '2002-12-01 03:00:00')!
|
util_test_time('0 3', '2002-11-31 04:00:00', '2002-12-01 03:00:00')!
|
||||||
|
|
||||||
//// Overlap to next year
|
// Overlap to next year
|
||||||
util_test_time('0 3', '2002-12-31 04:00:00', '2003-01-01 03:00:00')!
|
util_test_time('0 3', '2002-12-31 04:00:00', '2003-01-01 03:00:00')!
|
||||||
}
|
}
|
|
@ -1,42 +0,0 @@
|
||||||
module cron
|
|
||||||
|
|
||||||
fn test_not_allowed() {
|
|
||||||
illegal_expressions := [
|
|
||||||
'4 *-7',
|
|
||||||
'4 *-7/4',
|
|
||||||
'4 7/*',
|
|
||||||
'0 0 30 2',
|
|
||||||
'0 /5',
|
|
||||||
'0 ',
|
|
||||||
'0',
|
|
||||||
' 0',
|
|
||||||
' 0 ',
|
|
||||||
'1 2 3 4~9',
|
|
||||||
'1 1-3-5',
|
|
||||||
'0 5/2-5',
|
|
||||||
'',
|
|
||||||
'1 1/2/3',
|
|
||||||
'*5 8',
|
|
||||||
'x 8',
|
|
||||||
]
|
|
||||||
|
|
||||||
mut res := false
|
|
||||||
|
|
||||||
for exp in illegal_expressions {
|
|
||||||
res = false
|
|
||||||
parse_expression(exp) or { res = true }
|
|
||||||
assert res, "'${exp}' should produce an error"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn test_auto_extend() ! {
|
|
||||||
ce1 := parse_expression('5 5')!
|
|
||||||
ce2 := parse_expression('5 5 *')!
|
|
||||||
ce3 := parse_expression('5 5 * *')!
|
|
||||||
|
|
||||||
assert ce1 == ce2 && ce2 == ce3
|
|
||||||
}
|
|
||||||
|
|
||||||
fn test_four() {
|
|
||||||
parse_expression('0 1 2 3 ') or { assert false }
|
|
||||||
}
|
|
|
@ -1,6 +1,6 @@
|
||||||
module dbms
|
module db
|
||||||
|
|
||||||
import db.sqlite
|
import sqlite
|
||||||
import time
|
import time
|
||||||
|
|
||||||
pub struct VieterDb {
|
pub struct VieterDb {
|
||||||
|
@ -49,13 +49,13 @@ pub fn init(db_path string) !VieterDb {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Apply each migration in order
|
// Apply each migration in order
|
||||||
for i in cur_version.version .. dbms.migrations_up.len {
|
for i in cur_version.version .. db.migrations_up.len {
|
||||||
migration := dbms.migrations_up[i].to_string()
|
migration := db.migrations_up[i].to_string()
|
||||||
|
|
||||||
version_num := i + 1
|
version_num := i + 1
|
||||||
|
|
||||||
// vfmt does not like these dots
|
// vfmt does not like these dots
|
||||||
println('Applying migration ${version_num}' + '...')
|
println('Applying migration $version_num' + '...')
|
||||||
|
|
||||||
// The sqlite library seems to not like it when multiple statements are
|
// The sqlite library seems to not like it when multiple statements are
|
||||||
// passed in a single exec. Therefore, we split them & run them all
|
// passed in a single exec. Therefore, we split them & run them all
|
||||||
|
@ -64,7 +64,7 @@ pub fn init(db_path string) !VieterDb {
|
||||||
res := conn.exec_none(part)
|
res := conn.exec_none(part)
|
||||||
|
|
||||||
if res != sqlite.sqlite_done {
|
if res != sqlite.sqlite_done {
|
||||||
return error('An error occurred while applying migration ${version_num}: SQLite error code ${res}')
|
return error('An error occurred while applying migration $version_num: SQLite error code $res')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -80,9 +80,9 @@ pub fn init(db_path string) !VieterDb {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// row_into[T] converts an sqlite.Row into a given type T by parsing each field
|
// row_into<T> converts an sqlite.Row into a given type T by parsing each field
|
||||||
// from a string according to its type.
|
// from a string according to its type.
|
||||||
pub fn row_into[T](row sqlite.Row) T {
|
pub fn row_into<T>(row sqlite.Row) T {
|
||||||
mut i := 0
|
mut i := 0
|
||||||
mut out := T{}
|
mut out := T{}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
module dbms
|
module db
|
||||||
|
|
||||||
import models { BuildLog, BuildLogFilter }
|
import models { BuildLog, BuildLogFilter }
|
||||||
import time
|
import time
|
||||||
|
@ -8,20 +8,20 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog {
|
||||||
mut where_parts := []string{}
|
mut where_parts := []string{}
|
||||||
|
|
||||||
if filter.target != 0 {
|
if filter.target != 0 {
|
||||||
where_parts << 'target_id == ${filter.target}'
|
where_parts << 'target_id == $filter.target'
|
||||||
}
|
}
|
||||||
|
|
||||||
if filter.before != time.Time{} {
|
if filter.before != time.Time{} {
|
||||||
where_parts << 'start_time < ${filter.before.unix_time()}'
|
where_parts << 'start_time < $filter.before.unix_time()'
|
||||||
}
|
}
|
||||||
|
|
||||||
if filter.after != time.Time{} {
|
if filter.after != time.Time{} {
|
||||||
where_parts << 'start_time > ${filter.after.unix_time()}'
|
where_parts << 'start_time > $filter.after.unix_time()'
|
||||||
}
|
}
|
||||||
|
|
||||||
// NOTE: possible SQL injection
|
// NOTE: possible SQL injection
|
||||||
if filter.arch != '' {
|
if filter.arch != '' {
|
||||||
where_parts << "arch == '${filter.arch}'"
|
where_parts << "arch == '$filter.arch'"
|
||||||
}
|
}
|
||||||
|
|
||||||
mut parts := []string{}
|
mut parts := []string{}
|
||||||
|
@ -30,27 +30,27 @@ pub fn (db &VieterDb) get_build_logs(filter BuildLogFilter) []BuildLog {
|
||||||
if exp[0] == `!` {
|
if exp[0] == `!` {
|
||||||
code := exp[1..].int()
|
code := exp[1..].int()
|
||||||
|
|
||||||
parts << 'exit_code != ${code}'
|
parts << 'exit_code != $code'
|
||||||
} else {
|
} else {
|
||||||
code := exp.int()
|
code := exp.int()
|
||||||
|
|
||||||
parts << 'exit_code == ${code}'
|
parts << 'exit_code == $code'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if parts.len > 0 {
|
if parts.len > 0 {
|
||||||
where_parts << parts.map('(${it})').join(' or ')
|
where_parts << parts.map('($it)').join(' or ')
|
||||||
}
|
}
|
||||||
|
|
||||||
mut where_str := ''
|
mut where_str := ''
|
||||||
|
|
||||||
if where_parts.len > 0 {
|
if where_parts.len > 0 {
|
||||||
where_str = 'where ' + where_parts.map('(${it})').join(' and ')
|
where_str = 'where ' + where_parts.map('($it)').join(' and ')
|
||||||
}
|
}
|
||||||
|
|
||||||
query := 'select * from BuildLog ${where_str} limit ${filter.limit} offset ${filter.offset}'
|
query := 'select * from BuildLog $where_str limit $filter.limit offset $filter.offset'
|
||||||
rows, _ := db.conn.exec(query)
|
rows, _ := db.conn.exec(query)
|
||||||
res := rows.map(row_into[BuildLog](it))
|
res := rows.map(row_into<BuildLog>(it))
|
||||||
|
|
||||||
return res
|
return res
|
||||||
}
|
}
|
|
@ -1,6 +1,25 @@
|
||||||
module dbms
|
module db
|
||||||
|
|
||||||
import models { Target, TargetArch }
|
import models { Target, TargetArch, TargetFilter }
|
||||||
|
|
||||||
|
// get_targets returns all targets in the database.
|
||||||
|
pub fn (db &VieterDb) get_targets(filter TargetFilter) []Target {
|
||||||
|
// This seems to currently be blocked by a bug in the ORM, I'll have to ask
|
||||||
|
// around.
|
||||||
|
if filter.repo != '' {
|
||||||
|
res := sql db.conn {
|
||||||
|
select from Target where repo == filter.repo order by id limit filter.limit offset filter.offset
|
||||||
|
}
|
||||||
|
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
res := sql db.conn {
|
||||||
|
select from Target order by id limit filter.limit offset filter.offset
|
||||||
|
}
|
||||||
|
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
// get_target tries to return a specific target.
|
// get_target tries to return a specific target.
|
||||||
pub fn (db &VieterDb) get_target(target_id int) ?Target {
|
pub fn (db &VieterDb) get_target(target_id int) ?Target {
|
||||||
|
@ -49,13 +68,13 @@ pub fn (db &VieterDb) update_target(target_id int, params map[string]string) {
|
||||||
if field.name in params {
|
if field.name in params {
|
||||||
// Any fields that are array types require their own update method
|
// Any fields that are array types require their own update method
|
||||||
$if field.typ is string {
|
$if field.typ is string {
|
||||||
values << "${field.name} = '${params[field.name]}'"
|
values << "$field.name = '${params[field.name]}'"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
values_str := values.join(', ')
|
values_str := values.join(', ')
|
||||||
// I think this is actual SQL & not the ORM language
|
// I think this is actual SQL & not the ORM language
|
||||||
query := 'update Target set ${values_str} where id == ${target_id}'
|
query := 'update Target set $values_str where id == $target_id'
|
||||||
|
|
||||||
db.conn.exec_none(query)
|
db.conn.exec_none(query)
|
||||||
}
|
}
|
|
@ -1,129 +0,0 @@
|
||||||
module dbms
|
|
||||||
|
|
||||||
import models { Target, TargetFilter }
|
|
||||||
import db.sqlite
|
|
||||||
|
|
||||||
// Iterator providing a filtered view into the list of targets currently stored
|
|
||||||
// in the database. It replaces functionality usually performed in the database
|
|
||||||
// using SQL queries that can't currently be used due to missing stuff in V's
|
|
||||||
// ORM.
|
|
||||||
pub struct TargetsIterator {
|
|
||||||
conn sqlite.DB
|
|
||||||
filter TargetFilter
|
|
||||||
window_size int = 32
|
|
||||||
mut:
|
|
||||||
window []Target
|
|
||||||
window_index u64
|
|
||||||
// Offset in entire list of unfiltered targets
|
|
||||||
offset int
|
|
||||||
// Offset in filtered list of targets
|
|
||||||
filtered_offset u64
|
|
||||||
started bool
|
|
||||||
done bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// targets returns an iterator allowing filtered access to the list of targets.
|
|
||||||
pub fn (db &VieterDb) targets(filter TargetFilter) TargetsIterator {
|
|
||||||
window_size := 32
|
|
||||||
|
|
||||||
return TargetsIterator{
|
|
||||||
conn: db.conn
|
|
||||||
filter: filter
|
|
||||||
window: []Target{cap: window_size}
|
|
||||||
window_size: window_size
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// advance_window moves the sliding window over the filtered list of targets
|
|
||||||
// until it either reaches the end of the list of targets, or has encountered a
|
|
||||||
// non-empty window.
|
|
||||||
fn (mut ti TargetsIterator) advance_window() {
|
|
||||||
for {
|
|
||||||
ti.window = sql ti.conn {
|
|
||||||
select from Target order by id limit ti.window_size offset ti.offset
|
|
||||||
}
|
|
||||||
ti.offset += ti.window.len
|
|
||||||
|
|
||||||
if ti.window.len == 0 {
|
|
||||||
ti.done = true
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if ti.filter.repo != '' {
|
|
||||||
ti.window = ti.window.filter(it.repo == ti.filter.repo)
|
|
||||||
}
|
|
||||||
|
|
||||||
if ti.filter.arch != '' {
|
|
||||||
ti.window = ti.window.filter(it.arch.any(it.value == ti.filter.arch))
|
|
||||||
}
|
|
||||||
|
|
||||||
if ti.filter.query != '' {
|
|
||||||
ti.window = ti.window.filter(it.url.contains(ti.filter.query)
|
|
||||||
|| it.path.contains(ti.filter.query) || it.branch.contains(ti.filter.query))
|
|
||||||
}
|
|
||||||
|
|
||||||
// We break out of the loop once we found a non-empty window
|
|
||||||
if ti.window.len > 0 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// next returns the next target, if possible.
|
|
||||||
pub fn (mut ti TargetsIterator) next() ?Target {
|
|
||||||
if ti.done {
|
|
||||||
return none
|
|
||||||
}
|
|
||||||
|
|
||||||
// The first call to `next` will cause the sliding window to move to where
|
|
||||||
// the requested offset starts
|
|
||||||
if !ti.started {
|
|
||||||
ti.advance_window()
|
|
||||||
|
|
||||||
// Skip all matched targets until the requested offset
|
|
||||||
for !ti.done && ti.filtered_offset + u64(ti.window.len) <= ti.filter.offset {
|
|
||||||
ti.filtered_offset += u64(ti.window.len)
|
|
||||||
ti.advance_window()
|
|
||||||
}
|
|
||||||
|
|
||||||
if ti.done {
|
|
||||||
return none
|
|
||||||
}
|
|
||||||
|
|
||||||
left_inside_window := ti.filter.offset - ti.filtered_offset
|
|
||||||
ti.window_index = left_inside_window
|
|
||||||
ti.filtered_offset += left_inside_window
|
|
||||||
|
|
||||||
ti.started = true
|
|
||||||
}
|
|
||||||
|
|
||||||
return_value := ti.window[ti.window_index]
|
|
||||||
|
|
||||||
ti.window_index++
|
|
||||||
ti.filtered_offset++
|
|
||||||
|
|
||||||
// Next call will be past the requested offset
|
|
||||||
if ti.filter.limit > 0 && ti.filtered_offset == ti.filter.offset + ti.filter.limit {
|
|
||||||
ti.done = true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure the next call has a new valid window
|
|
||||||
if ti.window_index == u64(ti.window.len) {
|
|
||||||
ti.advance_window()
|
|
||||||
ti.window_index = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return return_value
|
|
||||||
}
|
|
||||||
|
|
||||||
// collect consumes the entire iterator & returns the result as an array.
|
|
||||||
pub fn (mut ti TargetsIterator) collect() []Target {
|
|
||||||
mut out := []Target{}
|
|
||||||
|
|
||||||
for t in ti {
|
|
||||||
out << t
|
|
||||||
}
|
|
||||||
|
|
||||||
return out
|
|
||||||
}
|
|
|
@ -1 +0,0 @@
|
||||||
Subproject commit 379a05a7b6b604c107360e0a679fb3ea5400e02c
|
|
|
@ -9,6 +9,7 @@ import console.schedule
|
||||||
import console.man
|
import console.man
|
||||||
import console.aur
|
import console.aur
|
||||||
import console.repos
|
import console.repos
|
||||||
|
import cron
|
||||||
import agent
|
import agent
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
|
@ -20,7 +21,7 @@ fn main() {
|
||||||
mut app := cli.Command{
|
mut app := cli.Command{
|
||||||
name: 'vieter'
|
name: 'vieter'
|
||||||
description: 'Vieter is a lightweight implementation of an Arch repository server.'
|
description: 'Vieter is a lightweight implementation of an Arch repository server.'
|
||||||
version: '0.6.0'
|
version: '0.5.0'
|
||||||
posix_mode: true
|
posix_mode: true
|
||||||
flags: [
|
flags: [
|
||||||
cli.Flag{
|
cli.Flag{
|
||||||
|
@ -42,6 +43,7 @@ fn main() {
|
||||||
commands: [
|
commands: [
|
||||||
server.cmd(),
|
server.cmd(),
|
||||||
targets.cmd(),
|
targets.cmd(),
|
||||||
|
cron.cmd(),
|
||||||
logs.cmd(),
|
logs.cmd(),
|
||||||
schedule.cmd(),
|
schedule.cmd(),
|
||||||
man.cmd(),
|
man.cmd(),
|
||||||
|
|
|
@ -10,10 +10,9 @@ pub:
|
||||||
repo string
|
repo string
|
||||||
base_image string
|
base_image string
|
||||||
force bool
|
force bool
|
||||||
timeout int
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// str return a single-line string representation of a build log
|
// str return a single-line string representation of a build log
|
||||||
pub fn (c BuildConfig) str() string {
|
pub fn (c BuildConfig) str() string {
|
||||||
return '{ target: ${c.target_id}, kind: ${c.kind}, url: ${c.url}, branch: ${c.branch}, path: ${c.path}, repo: ${c.repo}, base_image: ${c.base_image}, force: ${c.force}, timeout: ${c.timeout} }'
|
return '{ target: $c.target_id, kind: $c.kind, url: $c.url, branch: $c.branch, path: $c.path, repo: $c.repo, base_image: $c.base_image, force: $c.force }'
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,13 +16,13 @@ pub mut:
|
||||||
// str returns a string representation.
|
// str returns a string representation.
|
||||||
pub fn (bl &BuildLog) str() string {
|
pub fn (bl &BuildLog) str() string {
|
||||||
mut parts := [
|
mut parts := [
|
||||||
'id: ${bl.id}',
|
'id: $bl.id',
|
||||||
'target id: ${bl.target_id}',
|
'target id: $bl.target_id',
|
||||||
'start time: ${bl.start_time.local()}',
|
'start time: $bl.start_time.local()',
|
||||||
'end time: ${bl.end_time.local()}',
|
'end time: $bl.end_time.local()',
|
||||||
'duration: ${bl.end_time - bl.start_time}',
|
'duration: ${bl.end_time - bl.start_time}',
|
||||||
'arch: ${bl.arch}',
|
'arch: $bl.arch',
|
||||||
'exit code: ${bl.exit_code}',
|
'exit code: $bl.exit_code',
|
||||||
]
|
]
|
||||||
str := parts.join('\n')
|
str := parts.join('\n')
|
||||||
|
|
||||||
|
|
|
@ -2,19 +2,19 @@ module models
|
||||||
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
// from_params[T] creates a new instance of T from the given map by parsing all
|
// from_params<T> creates a new instance of T from the given map by parsing all
|
||||||
// of its fields from the map.
|
// of its fields from the map.
|
||||||
pub fn from_params[T](params map[string]string) ?T {
|
pub fn from_params<T>(params map[string]string) ?T {
|
||||||
mut o := T{}
|
mut o := T{}
|
||||||
|
|
||||||
patch_from_params[T](mut o, params)?
|
patch_from_params<T>(mut o, params)?
|
||||||
|
|
||||||
return o
|
return o
|
||||||
}
|
}
|
||||||
|
|
||||||
// patch_from_params[T] updates the given T object with the params defined in
|
// patch_from_params<T> updates the given T object with the params defined in
|
||||||
// the map.
|
// the map.
|
||||||
pub fn patch_from_params[T](mut o T, params map[string]string) ? {
|
pub fn patch_from_params<T>(mut o T, params map[string]string) ? {
|
||||||
$for field in T.fields {
|
$for field in T.fields {
|
||||||
if field.name in params && params[field.name] != '' {
|
if field.name in params && params[field.name] != '' {
|
||||||
$if field.typ is string {
|
$if field.typ is string {
|
||||||
|
@ -36,8 +36,8 @@ pub fn patch_from_params[T](mut o T, params map[string]string) ? {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// params_from[T] converts a given T struct into a map of strings.
|
// params_from<T> converts a given T struct into a map of strings.
|
||||||
pub fn params_from[T](o &T) map[string]string {
|
pub fn params_from<T>(o &T) map[string]string {
|
||||||
mut out := map[string]string{}
|
mut out := map[string]string{}
|
||||||
|
|
||||||
$for field in T.fields {
|
$for field in T.fields {
|
||||||
|
|
|
@ -38,13 +38,13 @@ pub mut:
|
||||||
// str returns a string representation.
|
// str returns a string representation.
|
||||||
pub fn (t &Target) str() string {
|
pub fn (t &Target) str() string {
|
||||||
mut parts := [
|
mut parts := [
|
||||||
'id: ${t.id}',
|
'id: $t.id',
|
||||||
'kind: ${t.kind}',
|
'kind: $t.kind',
|
||||||
'url: ${t.url}',
|
'url: $t.url',
|
||||||
'branch: ${t.branch}',
|
'branch: $t.branch',
|
||||||
'path: ${t.path}',
|
'path: $t.path',
|
||||||
'repo: ${t.repo}',
|
'repo: $t.repo',
|
||||||
'schedule: ${t.schedule}',
|
'schedule: $t.schedule',
|
||||||
'arch: ${t.arch.map(it.value).join(', ')}',
|
'arch: ${t.arch.map(it.value).join(', ')}',
|
||||||
]
|
]
|
||||||
str := parts.join('\n')
|
str := parts.join('\n')
|
||||||
|
@ -54,7 +54,7 @@ pub fn (t &Target) str() string {
|
||||||
|
|
||||||
// as_build_config converts a Target into a BuildConfig, given some extra
|
// as_build_config converts a Target into a BuildConfig, given some extra
|
||||||
// needed information.
|
// needed information.
|
||||||
pub fn (t &Target) as_build_config(base_image string, force bool, timeout int) BuildConfig {
|
pub fn (t &Target) as_build_config(base_image string, force bool) BuildConfig {
|
||||||
return BuildConfig{
|
return BuildConfig{
|
||||||
target_id: t.id
|
target_id: t.id
|
||||||
kind: t.kind
|
kind: t.kind
|
||||||
|
@ -64,7 +64,6 @@ pub fn (t &Target) as_build_config(base_image string, force bool, timeout int) B
|
||||||
repo: t.repo
|
repo: t.repo
|
||||||
base_image: base_image
|
base_image: base_image
|
||||||
force: force
|
force: force
|
||||||
timeout: timeout
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -74,6 +73,4 @@ pub mut:
|
||||||
limit u64 = 25
|
limit u64 = 25
|
||||||
offset u64
|
offset u64
|
||||||
repo string
|
repo string
|
||||||
query string
|
|
||||||
arch string
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,14 +3,14 @@ module package
|
||||||
// format_entry returns a string properly formatted to be added to a desc file.
|
// format_entry returns a string properly formatted to be added to a desc file.
|
||||||
[inline]
|
[inline]
|
||||||
fn format_entry(key string, value string) string {
|
fn format_entry(key string, value string) string {
|
||||||
return '\n%${key}%\n${value}\n'
|
return '\n%$key%\n$value\n'
|
||||||
}
|
}
|
||||||
|
|
||||||
// full_name returns the properly formatted name for the package, including
|
// full_name returns the properly formatted name for the package, including
|
||||||
// version & architecture
|
// version & architecture
|
||||||
pub fn (pkg &Pkg) full_name() string {
|
pub fn (pkg &Pkg) full_name() string {
|
||||||
p := pkg.info
|
p := pkg.info
|
||||||
return '${p.name}-${p.version}-${p.arch}'
|
return '$p.name-$p.version-$p.arch'
|
||||||
}
|
}
|
||||||
|
|
||||||
// filename returns the correct filename of the package file
|
// filename returns the correct filename of the package file
|
||||||
|
@ -20,10 +20,10 @@ pub fn (pkg &Pkg) filename() string {
|
||||||
1 { '.tar.gz' }
|
1 { '.tar.gz' }
|
||||||
6 { '.tar.xz' }
|
6 { '.tar.xz' }
|
||||||
14 { '.tar.zst' }
|
14 { '.tar.zst' }
|
||||||
else { panic("Another compression code shouldn't be possible. Faulty code: ${pkg.compression}") }
|
else { panic("Another compression code shouldn't be possible. Faulty code: $pkg.compression") }
|
||||||
}
|
}
|
||||||
|
|
||||||
return '${pkg.full_name()}.pkg${ext}'
|
return '${pkg.full_name()}.pkg$ext'
|
||||||
}
|
}
|
||||||
|
|
||||||
// to_desc returns a desc file valid string representation
|
// to_desc returns a desc file valid string representation
|
||||||
|
@ -31,7 +31,7 @@ pub fn (pkg &Pkg) to_desc() !string {
|
||||||
p := pkg.info
|
p := pkg.info
|
||||||
|
|
||||||
// filename
|
// filename
|
||||||
mut desc := '%FILENAME%\n${pkg.filename()}\n'
|
mut desc := '%FILENAME%\n$pkg.filename()\n'
|
||||||
|
|
||||||
desc += format_entry('NAME', p.name)
|
desc += format_entry('NAME', p.name)
|
||||||
desc += format_entry('BASE', p.base)
|
desc += format_entry('BASE', p.base)
|
||||||
|
@ -94,10 +94,10 @@ pub fn (pkg &Pkg) to_desc() !string {
|
||||||
desc += format_entry('CHECKDEPENDS', p.checkdepends.join_lines())
|
desc += format_entry('CHECKDEPENDS', p.checkdepends.join_lines())
|
||||||
}
|
}
|
||||||
|
|
||||||
return '${desc}\n'
|
return '$desc\n'
|
||||||
}
|
}
|
||||||
|
|
||||||
// to_files returns a files file valid string representation
|
// to_files returns a files file valid string representation
|
||||||
pub fn (pkg &Pkg) to_files() string {
|
pub fn (pkg &Pkg) to_files() string {
|
||||||
return '%FILES%\n${pkg.files.join_lines()}\n'
|
return '%FILES%\n$pkg.files.join_lines()\n'
|
||||||
}
|
}
|
||||||
|
|
|
@ -103,7 +103,7 @@ fn parse_pkg_info_string(pkg_info_str &string) !PkgInfo {
|
||||||
// NOTE: this command only supports zstd-, xz- & gzip-compressed tarballs.
|
// NOTE: this command only supports zstd-, xz- & gzip-compressed tarballs.
|
||||||
pub fn read_pkg_archive(pkg_path string) !Pkg {
|
pub fn read_pkg_archive(pkg_path string) !Pkg {
|
||||||
if !os.is_file(pkg_path) {
|
if !os.is_file(pkg_path) {
|
||||||
return error("'${pkg_path}' doesn't exist or isn't a file.")
|
return error("'$pkg_path' doesn't exist or isn't a file.")
|
||||||
}
|
}
|
||||||
|
|
||||||
a := C.archive_read_new()
|
a := C.archive_read_new()
|
||||||
|
|
|
@ -31,15 +31,11 @@ pub:
|
||||||
// new creates a new RepoGroupManager & creates the directories as needed
|
// new creates a new RepoGroupManager & creates the directories as needed
|
||||||
pub fn new(repos_dir string, pkg_dir string, default_arch string) !RepoGroupManager {
|
pub fn new(repos_dir string, pkg_dir string, default_arch string) !RepoGroupManager {
|
||||||
if !os.is_dir(repos_dir) {
|
if !os.is_dir(repos_dir) {
|
||||||
os.mkdir_all(repos_dir) or {
|
os.mkdir_all(repos_dir) or { return error('Failed to create repos directory: $err.msg()') }
|
||||||
return error('Failed to create repos directory: ${err.msg()}')
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if !os.is_dir(pkg_dir) {
|
if !os.is_dir(pkg_dir) {
|
||||||
os.mkdir_all(pkg_dir) or {
|
os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg()') }
|
||||||
return error('Failed to create package directory: ${err.msg()}')
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return RepoGroupManager{
|
return RepoGroupManager{
|
||||||
|
@ -55,7 +51,7 @@ pub fn new(repos_dir string, pkg_dir string, default_arch string) !RepoGroupMana
|
||||||
// the right subdirectories in r.pkg_dir if it was successfully added.
|
// the right subdirectories in r.pkg_dir if it was successfully added.
|
||||||
pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) !RepoAddResult {
|
pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) !RepoAddResult {
|
||||||
pkg := package.read_pkg_archive(pkg_path) or {
|
pkg := package.read_pkg_archive(pkg_path) or {
|
||||||
return error('Failed to read package file: ${err.msg()}')
|
return error('Failed to read package file: $err.msg()')
|
||||||
}
|
}
|
||||||
|
|
||||||
archs := r.add_pkg_in_repo(repo, pkg)!
|
archs := r.add_pkg_in_repo(repo, pkg)!
|
||||||
|
@ -133,7 +129,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ![]strin
|
||||||
// files, and afterwards updates the db & files archives to reflect these
|
// files, and afterwards updates the db & files archives to reflect these
|
||||||
// changes.
|
// changes.
|
||||||
fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ! {
|
fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ! {
|
||||||
pkg_dir := os.join_path(r.repos_dir, repo, arch, '${pkg.info.name}-${pkg.info.version}')
|
pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version')
|
||||||
|
|
||||||
// Remove the previous version of the package, if present
|
// Remove the previous version of the package, if present
|
||||||
r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false)!
|
r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false)!
|
||||||
|
|
|
@ -5,7 +5,7 @@ import os
|
||||||
// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It
|
// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It
|
||||||
// returns false if the package wasn't present in the database. It also
|
// returns false if the package wasn't present in the database. It also
|
||||||
// optionally re-syncs the repo archives.
|
// optionally re-syncs the repo archives.
|
||||||
pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, perform_sync bool) !bool {
|
pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) !bool {
|
||||||
repo_dir := os.join_path(r.repos_dir, repo, arch)
|
repo_dir := os.join_path(r.repos_dir, repo, arch)
|
||||||
|
|
||||||
// If the repository doesn't exist yet, the result is automatically false
|
// If the repository doesn't exist yet, the result is automatically false
|
||||||
|
@ -39,7 +39,7 @@ pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sync the db archives if requested
|
// Sync the db archives if requested
|
||||||
if perform_sync {
|
if sync {
|
||||||
r.sync(repo, arch)!
|
r.sync(repo, arch)!
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,7 @@ module server
|
||||||
import web
|
import web
|
||||||
import net.urllib
|
import net.urllib
|
||||||
import web.response { new_data_response, new_response }
|
import web.response { new_data_response, new_response }
|
||||||
|
import db
|
||||||
import time
|
import time
|
||||||
import os
|
import os
|
||||||
import util
|
import util
|
||||||
|
@ -12,7 +13,7 @@ import models { BuildLog, BuildLogFilter }
|
||||||
// optionally be added to limit the list of build logs to that repository.
|
// optionally be added to limit the list of build logs to that repository.
|
||||||
['/api/v1/logs'; auth; get; markused]
|
['/api/v1/logs'; auth; get; markused]
|
||||||
fn (mut app App) v1_get_logs() web.Result {
|
fn (mut app App) v1_get_logs() web.Result {
|
||||||
filter := models.from_params[BuildLogFilter](app.query) or {
|
filter := models.from_params<BuildLogFilter>(app.query) or {
|
||||||
return app.json(.bad_request, new_response('Invalid query parameters.'))
|
return app.json(.bad_request, new_response('Invalid query parameters.'))
|
||||||
}
|
}
|
||||||
logs := app.db.get_build_logs(filter)
|
logs := app.db.get_build_logs(filter)
|
||||||
|
@ -100,7 +101,7 @@ fn (mut app App) v1_post_log() web.Result {
|
||||||
// Create the logs directory of it doesn't exist
|
// Create the logs directory of it doesn't exist
|
||||||
if !os.exists(os.dir(log_file_path)) {
|
if !os.exists(os.dir(log_file_path)) {
|
||||||
os.mkdir_all(os.dir(log_file_path)) or {
|
os.mkdir_all(os.dir(log_file_path)) or {
|
||||||
app.lerror('Error while creating log file: ${err.msg()}')
|
app.lerror('Error while creating log file: $err.msg()')
|
||||||
|
|
||||||
return app.status(.internal_server_error)
|
return app.status(.internal_server_error)
|
||||||
}
|
}
|
||||||
|
@ -108,7 +109,7 @@ fn (mut app App) v1_post_log() web.Result {
|
||||||
|
|
||||||
if length := app.req.header.get(.content_length) {
|
if length := app.req.header.get(.content_length) {
|
||||||
util.reader_to_file(mut app.reader, length.int(), log_file_path) or {
|
util.reader_to_file(mut app.reader, length.int(), log_file_path) or {
|
||||||
app.lerror('An error occured while receiving logs: ${err.msg()}')
|
app.lerror('An error occured while receiving logs: $err.msg()')
|
||||||
|
|
||||||
return app.status(.internal_server_error)
|
return app.status(.internal_server_error)
|
||||||
}
|
}
|
||||||
|
@ -126,7 +127,7 @@ fn (mut app App) v1_delete_log(id int) web.Result {
|
||||||
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
|
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
|
||||||
|
|
||||||
os.rm(full_path) or {
|
os.rm(full_path) or {
|
||||||
app.lerror('Failed to remove log file ${full_path}: ${err.msg()}')
|
app.lerror('Failed to remove log file $full_path: $err.msg()')
|
||||||
|
|
||||||
return app.status(.internal_server_error)
|
return app.status(.internal_server_error)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,19 +0,0 @@
|
||||||
module server
|
|
||||||
|
|
||||||
import metrics
|
|
||||||
import web
|
|
||||||
|
|
||||||
// v1_metrics serves a Prometheus-compatible metrics endpoint.
|
|
||||||
['/api/v1/metrics'; get; markused]
|
|
||||||
fn (mut app App) v1_metrics() web.Result {
|
|
||||||
if !app.conf.collect_metrics {
|
|
||||||
return app.status(.not_found)
|
|
||||||
}
|
|
||||||
|
|
||||||
mut exporter := metrics.new_prometheus_exporter()
|
|
||||||
exporter.load('vieter_', app.collector)
|
|
||||||
|
|
||||||
// TODO stream to connection instead
|
|
||||||
body := exporter.export_to_string() or { return app.status(.internal_server_error) }
|
|
||||||
return app.body(.ok, 'text/plain', body)
|
|
||||||
}
|
|
|
@ -2,17 +2,18 @@ module server
|
||||||
|
|
||||||
import web
|
import web
|
||||||
import web.response { new_data_response, new_response }
|
import web.response { new_data_response, new_response }
|
||||||
|
import db
|
||||||
import models { Target, TargetArch, TargetFilter }
|
import models { Target, TargetArch, TargetFilter }
|
||||||
|
|
||||||
// v1_get_targets returns the current list of targets.
|
// v1_get_targets returns the current list of targets.
|
||||||
['/api/v1/targets'; auth; get; markused]
|
['/api/v1/targets'; auth; get; markused]
|
||||||
fn (mut app App) v1_get_targets() web.Result {
|
fn (mut app App) v1_get_targets() web.Result {
|
||||||
filter := models.from_params[TargetFilter](app.query) or {
|
filter := models.from_params<TargetFilter>(app.query) or {
|
||||||
return app.json(.bad_request, new_response('Invalid query parameters.'))
|
return app.json(.bad_request, new_response('Invalid query parameters.'))
|
||||||
}
|
}
|
||||||
mut iter := app.db.targets(filter)
|
targets := app.db.get_targets(filter)
|
||||||
|
|
||||||
return app.json(.ok, new_data_response(iter.collect()))
|
return app.json(.ok, new_data_response(targets))
|
||||||
}
|
}
|
||||||
|
|
||||||
// v1_get_single_target returns the information for a single target.
|
// v1_get_single_target returns the information for a single target.
|
||||||
|
@ -34,7 +35,7 @@ fn (mut app App) v1_post_target() web.Result {
|
||||||
params['arch'] = app.conf.default_arch
|
params['arch'] = app.conf.default_arch
|
||||||
}
|
}
|
||||||
|
|
||||||
mut new_target := models.from_params[Target](params) or {
|
mut new_target := models.from_params<Target>(params) or {
|
||||||
return app.json(.bad_request, new_response(err.msg()))
|
return app.json(.bad_request, new_response(err.msg()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -15,8 +15,6 @@ pub:
|
||||||
base_image string = 'archlinux:base-devel'
|
base_image string = 'archlinux:base-devel'
|
||||||
max_log_age int [empty_default]
|
max_log_age int [empty_default]
|
||||||
log_removal_schedule string = '0 0'
|
log_removal_schedule string = '0 0'
|
||||||
collect_metrics bool [empty_default]
|
|
||||||
default_build_timeout int = 3600
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// cmd returns the cli submodule that handles starting the server
|
// cmd returns the cli submodule that handles starting the server
|
||||||
|
@ -26,9 +24,9 @@ pub fn cmd() cli.Command {
|
||||||
description: 'Start the Vieter server.'
|
description: 'Start the Vieter server.'
|
||||||
execute: fn (cmd cli.Command) ! {
|
execute: fn (cmd cli.Command) ! {
|
||||||
config_file := cmd.flags.get_string('config-file')!
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
conf_ := vconf.load[Config](prefix: 'VIETER_', default_path: config_file)!
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
server(conf_)!
|
server(conf)!
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,16 +3,20 @@ module server
|
||||||
import time
|
import time
|
||||||
import models { BuildLog }
|
import models { BuildLog }
|
||||||
import os
|
import os
|
||||||
import cron
|
import cron.expression { CronExpression }
|
||||||
|
|
||||||
const fallback_log_removal_frequency = 24 * time.hour
|
const fallback_log_removal_frequency = 24 * time.hour
|
||||||
|
|
||||||
// log_removal_daemon removes old build logs every `log_removal_frequency`.
|
// log_removal_daemon removes old build logs every `log_removal_frequency`.
|
||||||
fn (mut app App) log_removal_daemon(schedule &cron.Expression) {
|
fn (mut app App) log_removal_daemon(schedule CronExpression) {
|
||||||
|
mut start_time := time.Time{}
|
||||||
|
|
||||||
for {
|
for {
|
||||||
|
start_time = time.now()
|
||||||
|
|
||||||
mut too_old_timestamp := time.now().add_days(-app.conf.max_log_age)
|
mut too_old_timestamp := time.now().add_days(-app.conf.max_log_age)
|
||||||
|
|
||||||
app.linfo('Cleaning logs before ${too_old_timestamp}')
|
app.linfo('Cleaning logs before $too_old_timestamp')
|
||||||
|
|
||||||
mut logs := []BuildLog{}
|
mut logs := []BuildLog{}
|
||||||
mut counter := 0
|
mut counter := 0
|
||||||
|
@ -29,7 +33,7 @@ fn (mut app App) log_removal_daemon(schedule &cron.Expression) {
|
||||||
log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
|
log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
|
||||||
|
|
||||||
os.rm(log_file_path) or {
|
os.rm(log_file_path) or {
|
||||||
app.lerror('Failed to remove log file ${log_file_path}: ${err.msg()}')
|
app.lerror('Failed to remove log file $log_file_path: $err.msg()')
|
||||||
failed += 1
|
failed += 1
|
||||||
|
|
||||||
continue
|
continue
|
||||||
|
@ -44,10 +48,15 @@ fn (mut app App) log_removal_daemon(schedule &cron.Expression) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
app.linfo('Cleaned ${counter} logs (${failed} failed)')
|
app.linfo('Cleaned $counter logs ($failed failed)')
|
||||||
|
|
||||||
// Sleep until the next cycle
|
// Sleep until the next cycle
|
||||||
next_time := schedule.next_from_now()
|
next_time := schedule.next_from_now() or {
|
||||||
|
app.lerror("Log removal daemon couldn't calculate next time: $err.msg(); fallback to $server.fallback_log_removal_frequency")
|
||||||
|
|
||||||
|
start_time.add(server.fallback_log_removal_frequency)
|
||||||
|
}
|
||||||
|
|
||||||
time.sleep(next_time - time.now())
|
time.sleep(next_time - time.now())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,15 +19,15 @@ pub fn (mut app App) healthcheck() web.Result {
|
||||||
// repository's archives, but also package archives or the contents of a
|
// repository's archives, but also package archives or the contents of a
|
||||||
// package's desc file.
|
// package's desc file.
|
||||||
['/:repo/:arch/:filename'; get; head; markused]
|
['/:repo/:arch/:filename'; get; head; markused]
|
||||||
fn (mut app App) get_repo_file(repo_ string, arch string, filename string) web.Result {
|
fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result {
|
||||||
mut full_path := ''
|
mut full_path := ''
|
||||||
|
|
||||||
db_exts := ['.db', '.files', '.db.tar.gz', '.files.tar.gz']
|
db_exts := ['.db', '.files', '.db.tar.gz', '.files.tar.gz']
|
||||||
|
|
||||||
// There's no point in having the ability to serve db archives with wrong
|
// There's no point in having the ability to serve db archives with wrong
|
||||||
// filenames
|
// filenames
|
||||||
if db_exts.any(filename == '${repo_}${it}') {
|
if db_exts.any(filename == '$repo$it') {
|
||||||
full_path = os.join_path(app.repo.repos_dir, repo_, arch, filename)
|
full_path = os.join_path(app.repo.repos_dir, repo, arch, filename)
|
||||||
|
|
||||||
// repo-add does this using symlinks, but we just change the requested
|
// repo-add does this using symlinks, but we just change the requested
|
||||||
// path
|
// path
|
||||||
|
@ -35,13 +35,13 @@ fn (mut app App) get_repo_file(repo_ string, arch string, filename string) web.R
|
||||||
full_path += '.tar.gz'
|
full_path += '.tar.gz'
|
||||||
}
|
}
|
||||||
} else if filename.contains('.pkg') {
|
} else if filename.contains('.pkg') {
|
||||||
full_path = os.join_path(app.repo.pkg_dir, repo_, arch, filename)
|
full_path = os.join_path(app.repo.pkg_dir, repo, arch, filename)
|
||||||
}
|
}
|
||||||
// Default behavior is to return the desc file for the package, if present.
|
// Default behavior is to return the desc file for the package, if present.
|
||||||
// This can then also be used by the build system to properly check whether
|
// This can then also be used by the build system to properly check whether
|
||||||
// a package is present in an arch-repo.
|
// a package is present in an arch-repo.
|
||||||
else {
|
else {
|
||||||
full_path = os.join_path(app.repo.repos_dir, repo_, arch, filename, 'desc')
|
full_path = os.join_path(app.repo.repos_dir, repo, arch, filename, 'desc')
|
||||||
}
|
}
|
||||||
|
|
||||||
return app.file(full_path)
|
return app.file(full_path)
|
||||||
|
@ -49,10 +49,10 @@ fn (mut app App) get_repo_file(repo_ string, arch string, filename string) web.R
|
||||||
|
|
||||||
// put_package handles publishing a package to a repository.
|
// put_package handles publishing a package to a repository.
|
||||||
['/:repo/publish'; auth; markused; post]
|
['/:repo/publish'; auth; markused; post]
|
||||||
fn (mut app App) put_package(repo_ string) web.Result {
|
fn (mut app App) put_package(repo string) web.Result {
|
||||||
// api is a reserved keyword for api routes & should never be allowed to be
|
// api is a reserved keyword for api routes & should never be allowed to be
|
||||||
// a repository.
|
// a repository.
|
||||||
if repo_.to_lower() == 'api' {
|
if repo.to_lower() == 'api' {
|
||||||
return app.json(.bad_request, new_response("'api' is a reserved keyword & cannot be used as a repository name."))
|
return app.json(.bad_request, new_response("'api' is a reserved keyword & cannot be used as a repository name."))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -62,19 +62,19 @@ fn (mut app App) put_package(repo_ string) web.Result {
|
||||||
// Generate a random filename for the temp file
|
// Generate a random filename for the temp file
|
||||||
pkg_path = os.join_path_single(app.repo.pkg_dir, rand.uuid_v4())
|
pkg_path = os.join_path_single(app.repo.pkg_dir, rand.uuid_v4())
|
||||||
|
|
||||||
app.ldebug("Uploading ${length} bytes (${util.pretty_bytes(length.int())}) to '${pkg_path}'.")
|
app.ldebug("Uploading $length bytes (${util.pretty_bytes(length.int())}) to '$pkg_path'.")
|
||||||
|
|
||||||
// This is used to time how long it takes to upload a file
|
// This is used to time how long it takes to upload a file
|
||||||
mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true })
|
mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true })
|
||||||
|
|
||||||
util.reader_to_file(mut app.reader, length.int(), pkg_path) or {
|
util.reader_to_file(mut app.reader, length.int(), pkg_path) or {
|
||||||
app.lwarn("Failed to upload '${pkg_path}': ${err.msg()}")
|
app.lwarn("Failed to upload '$pkg_path'")
|
||||||
|
|
||||||
return app.status(.internal_server_error)
|
return app.status(.internal_server_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
sw.stop()
|
sw.stop()
|
||||||
app.ldebug("Upload of '${pkg_path}' completed in ${sw.elapsed().seconds():.3}s.")
|
app.ldebug("Upload of '$pkg_path' completed in ${sw.elapsed().seconds():.3}s.")
|
||||||
} else {
|
} else {
|
||||||
app.lwarn('Tried to upload package without specifying a Content-Length.')
|
app.lwarn('Tried to upload package without specifying a Content-Length.')
|
||||||
|
|
||||||
|
@ -82,15 +82,15 @@ fn (mut app App) put_package(repo_ string) web.Result {
|
||||||
return app.status(.length_required)
|
return app.status(.length_required)
|
||||||
}
|
}
|
||||||
|
|
||||||
res := app.repo.add_pkg_from_path(repo_, pkg_path) or {
|
res := app.repo.add_pkg_from_path(repo, pkg_path) or {
|
||||||
app.lerror('Error while adding package: ${err.msg()}')
|
app.lerror('Error while adding package: $err.msg()')
|
||||||
|
|
||||||
os.rm(pkg_path) or { app.lerror("Failed to remove download '${pkg_path}': ${err.msg()}") }
|
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg()") }
|
||||||
|
|
||||||
return app.status(.internal_server_error)
|
return app.status(.internal_server_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
app.linfo("Added '${res.name}-${res.version}' to '${repo_} (${res.archs.join(',')})'.")
|
app.linfo("Added '$res.name-$res.version' to '$repo (${res.archs.join(',')})'.")
|
||||||
|
|
||||||
return app.json(.ok, new_data_response(res))
|
return app.json(.ok, new_data_response(res))
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,17 +6,17 @@ import web
|
||||||
['/:repo/:arch/:pkg'; auth; delete; markused]
|
['/:repo/:arch/:pkg'; auth; delete; markused]
|
||||||
fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result {
|
fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result {
|
||||||
res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or {
|
res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or {
|
||||||
app.lerror('Error while deleting package: ${err.msg()}')
|
app.lerror('Error while deleting package: $err.msg()')
|
||||||
|
|
||||||
return app.status(.internal_server_error)
|
return app.status(.internal_server_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
if res {
|
if res {
|
||||||
app.linfo("Removed package '${pkg}' from '${repo}/${arch}'")
|
app.linfo("Removed package '$pkg' from '$repo/$arch'")
|
||||||
|
|
||||||
return app.status(.ok)
|
return app.status(.ok)
|
||||||
} else {
|
} else {
|
||||||
app.linfo("Tried removing package '${pkg}' from '${repo}/${arch}', but it doesn't exist.")
|
app.linfo("Tried removing package '$pkg' from '$repo/$arch', but it doesn't exist.")
|
||||||
|
|
||||||
return app.status(.not_found)
|
return app.status(.not_found)
|
||||||
}
|
}
|
||||||
|
@ -26,17 +26,17 @@ fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result
|
||||||
['/:repo/:arch'; auth; delete; markused]
|
['/:repo/:arch'; auth; delete; markused]
|
||||||
fn (mut app App) delete_arch_repo(repo string, arch string) web.Result {
|
fn (mut app App) delete_arch_repo(repo string, arch string) web.Result {
|
||||||
res := app.repo.remove_arch_repo(repo, arch) or {
|
res := app.repo.remove_arch_repo(repo, arch) or {
|
||||||
app.lerror('Error while deleting arch-repo: ${err.msg()}')
|
app.lerror('Error while deleting arch-repo: $err.msg()')
|
||||||
|
|
||||||
return app.status(.internal_server_error)
|
return app.status(.internal_server_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
if res {
|
if res {
|
||||||
app.linfo("Removed arch-repo '${repo}/${arch}'")
|
app.linfo("Removed arch-repo '$repo/$arch'")
|
||||||
|
|
||||||
return app.status(.ok)
|
return app.status(.ok)
|
||||||
} else {
|
} else {
|
||||||
app.linfo("Tried removing '${repo}/${arch}', but it doesn't exist.")
|
app.linfo("Tried removing '$repo/$arch', but it doesn't exist.")
|
||||||
|
|
||||||
return app.status(.not_found)
|
return app.status(.not_found)
|
||||||
}
|
}
|
||||||
|
@ -46,17 +46,17 @@ fn (mut app App) delete_arch_repo(repo string, arch string) web.Result {
|
||||||
['/:repo'; auth; delete; markused]
|
['/:repo'; auth; delete; markused]
|
||||||
fn (mut app App) delete_repo(repo string) web.Result {
|
fn (mut app App) delete_repo(repo string) web.Result {
|
||||||
res := app.repo.remove_repo(repo) or {
|
res := app.repo.remove_repo(repo) or {
|
||||||
app.lerror('Error while deleting repo: ${err.msg()}')
|
app.lerror('Error while deleting repo: $err.msg()')
|
||||||
|
|
||||||
return app.status(.internal_server_error)
|
return app.status(.internal_server_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
if res {
|
if res {
|
||||||
app.linfo("Removed repo '${repo}'")
|
app.linfo("Removed repo '$repo'")
|
||||||
|
|
||||||
return app.status(.ok)
|
return app.status(.ok)
|
||||||
} else {
|
} else {
|
||||||
app.linfo("Tried removing '${repo}', but it doesn't exist.")
|
app.linfo("Tried removing '$repo', but it doesn't exist.")
|
||||||
|
|
||||||
return app.status(.not_found)
|
return app.status(.not_found)
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,10 +5,9 @@ import os
|
||||||
import log
|
import log
|
||||||
import repo
|
import repo
|
||||||
import util
|
import util
|
||||||
import dbms
|
import db
|
||||||
import build { BuildJobQueue }
|
import build { BuildJobQueue }
|
||||||
import cron
|
import cron.expression
|
||||||
import metrics
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
log_file_name = 'vieter.log'
|
log_file_name = 'vieter.log'
|
||||||
|
@ -25,15 +24,24 @@ pub mut:
|
||||||
repo repo.RepoGroupManager [required; web_global]
|
repo repo.RepoGroupManager [required; web_global]
|
||||||
// Keys are the various architectures for packages
|
// Keys are the various architectures for packages
|
||||||
job_queue BuildJobQueue [required; web_global]
|
job_queue BuildJobQueue [required; web_global]
|
||||||
db dbms.VieterDb
|
db db.VieterDb
|
||||||
}
|
}
|
||||||
|
|
||||||
// init_job_queue populates a fresh job queue with all the targets currently
|
// init_job_queue populates a fresh job queue with all the targets currently
|
||||||
// stored in the database.
|
// stored in the database.
|
||||||
fn (mut app App) init_job_queue() ! {
|
fn (mut app App) init_job_queue() ! {
|
||||||
for target in app.db.targets(limit: 0) {
|
// Initialize build queues
|
||||||
|
mut targets := app.db.get_targets(limit: 25)
|
||||||
|
mut i := u64(0)
|
||||||
|
|
||||||
|
for targets.len > 0 {
|
||||||
|
for target in targets {
|
||||||
app.job_queue.insert_all(target)!
|
app.job_queue.insert_all(target)!
|
||||||
}
|
}
|
||||||
|
|
||||||
|
i += 25
|
||||||
|
targets = app.db.get_targets(limit: 25, offset: i)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// server starts the web server & starts listening for requests
|
// server starts the web server & starts listening for requests
|
||||||
|
@ -43,12 +51,12 @@ pub fn server(conf Config) ! {
|
||||||
util.exit_with_message(1, "'any' is not allowed as the value for default_arch.")
|
util.exit_with_message(1, "'any' is not allowed as the value for default_arch.")
|
||||||
}
|
}
|
||||||
|
|
||||||
global_ce := cron.parse_expression(conf.global_schedule) or {
|
global_ce := expression.parse_expression(conf.global_schedule) or {
|
||||||
util.exit_with_message(1, 'Invalid global cron expression: ${err.msg()}')
|
util.exit_with_message(1, 'Invalid global cron expression: $err.msg()')
|
||||||
}
|
}
|
||||||
|
|
||||||
log_removal_ce := cron.parse_expression(conf.log_removal_schedule) or {
|
log_removal_ce := expression.parse_expression(conf.log_removal_schedule) or {
|
||||||
util.exit_with_message(1, 'Invalid log removal cron expression: ${err.msg()}')
|
util.exit_with_message(1, 'Invalid log removal cron expression: $err.msg()')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Configure logger
|
// Configure logger
|
||||||
|
@ -82,40 +90,30 @@ pub fn server(conf Config) ! {
|
||||||
|
|
||||||
repo_dir := os.join_path_single(conf.data_dir, server.repo_dir_name)
|
repo_dir := os.join_path_single(conf.data_dir, server.repo_dir_name)
|
||||||
// This also creates the directories if needed
|
// This also creates the directories if needed
|
||||||
repo_ := repo.new(repo_dir, conf.pkg_dir, conf.default_arch) or {
|
repo := repo.new(repo_dir, conf.pkg_dir, conf.default_arch) or {
|
||||||
logger.error(err.msg())
|
logger.error(err.msg())
|
||||||
exit(1)
|
exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
db_file := os.join_path_single(conf.data_dir, server.db_file_name)
|
db_file := os.join_path_single(conf.data_dir, server.db_file_name)
|
||||||
db := dbms.init(db_file) or {
|
db := db.init(db_file) or {
|
||||||
util.exit_with_message(1, 'Failed to initialize database: ${err.msg()}')
|
util.exit_with_message(1, 'Failed to initialize database: $err.msg()')
|
||||||
}
|
}
|
||||||
|
|
||||||
mut collector := if conf.collect_metrics {
|
|
||||||
&metrics.MetricsCollector(metrics.new_default_collector())
|
|
||||||
} else {
|
|
||||||
&metrics.MetricsCollector(metrics.new_null_collector())
|
|
||||||
}
|
|
||||||
|
|
||||||
collector.histogram_buckets_set('http_requests_duration_seconds', [0.001, 0.005, 0.01, 0.05,
|
|
||||||
0.1, 0.5, 1, 5, 10])
|
|
||||||
|
|
||||||
mut app := &App{
|
mut app := &App{
|
||||||
logger: logger
|
logger: logger
|
||||||
api_key: conf.api_key
|
api_key: conf.api_key
|
||||||
conf: conf
|
conf: conf
|
||||||
repo: repo_
|
repo: repo
|
||||||
db: db
|
db: db
|
||||||
collector: collector
|
job_queue: build.new_job_queue(global_ce, conf.base_image)
|
||||||
job_queue: build.new_job_queue(global_ce, conf.base_image, conf.default_build_timeout)
|
|
||||||
}
|
}
|
||||||
app.init_job_queue() or {
|
app.init_job_queue() or {
|
||||||
util.exit_with_message(1, 'Failed to inialize job queue: ${err.msg()}')
|
util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()')
|
||||||
}
|
}
|
||||||
|
|
||||||
if conf.max_log_age > 0 {
|
if conf.max_log_age > 0 {
|
||||||
spawn app.log_removal_daemon(log_removal_ce)
|
go app.log_removal_daemon(log_removal_ce)
|
||||||
}
|
}
|
||||||
|
|
||||||
web.run(app, conf.port)
|
web.run(app, conf.port)
|
||||||
|
|
|
@ -6,7 +6,7 @@ import os
|
||||||
|
|
||||||
// reader_to_writer tries to consume the entire reader & write it to the writer.
|
// reader_to_writer tries to consume the entire reader & write it to the writer.
|
||||||
pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ! {
|
pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ! {
|
||||||
mut buf := []u8{len: 10 * 1024}
|
mut buf := []u8{len: 8192}
|
||||||
|
|
||||||
for {
|
for {
|
||||||
bytes_read := reader.read(mut buf) or { break }
|
bytes_read := reader.read(mut buf) or { break }
|
||||||
|
@ -27,7 +27,7 @@ pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ! {
|
||||||
file.close()
|
file.close()
|
||||||
}
|
}
|
||||||
|
|
||||||
mut buf := []u8{len: reader_buf_size}
|
mut buf := []u8{len: 8192}
|
||||||
mut bytes_left := length
|
mut bytes_left := length
|
||||||
|
|
||||||
// Repeat as long as the stream still has data
|
// Repeat as long as the stream still has data
|
||||||
|
@ -46,54 +46,4 @@ pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ! {
|
||||||
to_write = to_write - bytes_written
|
to_write = to_write - bytes_written
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if bytes_left > 0 {
|
|
||||||
return error('Not all bytes were received.')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// match_array_in_array[T] returns how many elements of a2 overlap with a1. For
|
|
||||||
// example, if a1 = "abcd" & a2 = "cd", the result will be 2. If the match is
|
|
||||||
// not at the end of a1, the result is 0.
|
|
||||||
pub fn match_array_in_array[T](a1 []T, a2 []T) int {
|
|
||||||
mut i := 0
|
|
||||||
mut match_len := 0
|
|
||||||
|
|
||||||
for i + match_len < a1.len {
|
|
||||||
if a1[i + match_len] == a2[match_len] {
|
|
||||||
match_len += 1
|
|
||||||
} else {
|
|
||||||
i += match_len + 1
|
|
||||||
match_len = 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return match_len
|
|
||||||
}
|
|
||||||
|
|
||||||
// read_until_separator consumes an io.Reader until it encounters some
|
|
||||||
// separator array. The data read is stored inside the provided res array.
|
|
||||||
pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ! {
|
|
||||||
mut buf := []u8{len: sep.len}
|
|
||||||
|
|
||||||
for {
|
|
||||||
c := reader.read(mut buf)!
|
|
||||||
res << buf[..c]
|
|
||||||
|
|
||||||
match_len := match_array_in_array(buf[..c], sep)
|
|
||||||
|
|
||||||
if match_len == sep.len {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
if match_len > 0 {
|
|
||||||
match_left := sep.len - match_len
|
|
||||||
c2 := reader.read(mut buf[..match_left])!
|
|
||||||
res << buf[..c2]
|
|
||||||
|
|
||||||
if buf[..c2] == sep[match_len..] {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,10 +3,7 @@ module util
|
||||||
import os
|
import os
|
||||||
import crypto.sha256
|
import crypto.sha256
|
||||||
|
|
||||||
const (
|
const prefixes = ['B', 'KB', 'MB', 'GB']
|
||||||
reader_buf_size = 1_000_000
|
|
||||||
prefixes = ['B', 'KB', 'MB', 'GB']
|
|
||||||
)
|
|
||||||
|
|
||||||
// Dummy struct to work around the fact that you can only share structs, maps &
|
// Dummy struct to work around the fact that you can only share structs, maps &
|
||||||
// arrays
|
// arrays
|
||||||
|
@ -24,18 +21,13 @@ pub fn exit_with_message(code int, msg string) {
|
||||||
|
|
||||||
// hash_file returns the sha256 hash of a given file
|
// hash_file returns the sha256 hash of a given file
|
||||||
pub fn hash_file(path &string) !string {
|
pub fn hash_file(path &string) !string {
|
||||||
file := os.open(path) or { return error('Failed to open file.') }
|
file := os.open(path)!
|
||||||
|
|
||||||
mut sha256sum := sha256.new()
|
mut sha256sum := sha256.new()
|
||||||
|
mut buf := []u8{len: 8192}
|
||||||
|
|
||||||
buf_size := int(1_000_000)
|
for !file.eof() {
|
||||||
mut buf := []u8{len: buf_size}
|
bytes_read := file.read(mut buf)!
|
||||||
mut bytes_left := os.file_size(path)
|
|
||||||
|
|
||||||
for bytes_left > 0 {
|
|
||||||
// TODO check if just breaking here is safe
|
|
||||||
bytes_read := file.read(mut buf) or { return error('Failed to read from file.') }
|
|
||||||
bytes_left -= u64(bytes_read)
|
|
||||||
|
|
||||||
// This function never actually fails, but returns an option to follow
|
// This function never actually fails, but returns an option to follow
|
||||||
// the Writer interface.
|
// the Writer interface.
|
||||||
|
|
|
@ -2,7 +2,6 @@ Module {
|
||||||
dependencies: [
|
dependencies: [
|
||||||
'https://git.rustybever.be/vieter-v/conf',
|
'https://git.rustybever.be/vieter-v/conf',
|
||||||
'https://git.rustybever.be/vieter-v/docker',
|
'https://git.rustybever.be/vieter-v/docker',
|
||||||
'https://git.rustybever.be/vieter-v/aur',
|
'https://git.rustybever.be/vieter-v/aur'
|
||||||
'https://git.rustybever.be/vieter-v/metrics'
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,36 +1,35 @@
|
||||||
module web
|
module web
|
||||||
|
|
||||||
|
import log
|
||||||
|
|
||||||
|
// log reate a log message with the given level
|
||||||
|
pub fn (mut ctx Context) log(msg string, level log.Level) {
|
||||||
|
lock ctx.logger {
|
||||||
|
ctx.logger.send_output(msg, level)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// lfatal create a log message with the fatal level
|
// lfatal create a log message with the fatal level
|
||||||
pub fn (mut ctx Context) lfatal(msg string) {
|
pub fn (mut ctx Context) lfatal(msg string) {
|
||||||
lock ctx.logger {
|
ctx.log(msg, log.Level.fatal)
|
||||||
ctx.logger.fatal(msg)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// lerror create a log message with the error level
|
// lerror create a log message with the error level
|
||||||
pub fn (mut ctx Context) lerror(msg string) {
|
pub fn (mut ctx Context) lerror(msg string) {
|
||||||
lock ctx.logger {
|
ctx.log(msg, log.Level.error)
|
||||||
ctx.logger.error(msg)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// lwarn create a log message with the warn level
|
// lwarn create a log message with the warn level
|
||||||
pub fn (mut ctx Context) lwarn(msg string) {
|
pub fn (mut ctx Context) lwarn(msg string) {
|
||||||
lock ctx.logger {
|
ctx.log(msg, log.Level.warn)
|
||||||
ctx.logger.warn(msg)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// linfo create a log message with the info level
|
// linfo create a log message with the info level
|
||||||
pub fn (mut ctx Context) linfo(msg string) {
|
pub fn (mut ctx Context) linfo(msg string) {
|
||||||
lock ctx.logger {
|
ctx.log(msg, log.Level.info)
|
||||||
ctx.logger.info(msg)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ldebug create a log message with the debug level
|
// ldebug create a log message with the debug level
|
||||||
pub fn (mut ctx Context) ldebug(msg string) {
|
pub fn (mut ctx Context) ldebug(msg string) {
|
||||||
lock ctx.logger {
|
ctx.log(msg, log.Level.debug)
|
||||||
ctx.logger.debug(msg)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,7 +10,7 @@ const attrs_to_ignore = ['auth', 'markused']
|
||||||
// Parsing function attributes for methods and path.
|
// Parsing function attributes for methods and path.
|
||||||
fn parse_attrs(name string, attrs []string) !([]http.Method, string) {
|
fn parse_attrs(name string, attrs []string) !([]http.Method, string) {
|
||||||
if attrs.len == 0 {
|
if attrs.len == 0 {
|
||||||
return [http.Method.get], '/${name}'
|
return [http.Method.get], '/$name'
|
||||||
}
|
}
|
||||||
|
|
||||||
mut x := attrs.clone()
|
mut x := attrs.clone()
|
||||||
|
@ -45,7 +45,7 @@ fn parse_attrs(name string, attrs []string) !([]http.Method, string) {
|
||||||
methods = [http.Method.get]
|
methods = [http.Method.get]
|
||||||
}
|
}
|
||||||
if path == '' {
|
if path == '' {
|
||||||
path = '/${name}'
|
path = '/$name'
|
||||||
}
|
}
|
||||||
// Make path lowercase for case-insensitive comparisons
|
// Make path lowercase for case-insensitive comparisons
|
||||||
return methods, path.to_lower()
|
return methods, path.to_lower()
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
module response
|
module response
|
||||||
|
|
||||||
pub struct Response[T] {
|
pub struct Response<T> {
|
||||||
pub:
|
pub:
|
||||||
message string
|
message string
|
||||||
data T
|
data T
|
||||||
|
@ -8,26 +8,26 @@ pub:
|
||||||
|
|
||||||
// new_response constructs a new Response<String> object with the given message
|
// new_response constructs a new Response<String> object with the given message
|
||||||
// & an empty data field.
|
// & an empty data field.
|
||||||
pub fn new_response(message string) Response[string] {
|
pub fn new_response(message string) Response<string> {
|
||||||
return Response[string]{
|
return Response<string>{
|
||||||
message: message
|
message: message
|
||||||
data: ''
|
data: ''
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// new_data_response[T] constructs a new Response<T> object with the given data
|
// new_data_response<T> constructs a new Response<T> object with the given data
|
||||||
// & an empty message field.
|
// & an empty message field.
|
||||||
pub fn new_data_response[T](data T) Response[T] {
|
pub fn new_data_response<T>(data T) Response<T> {
|
||||||
return Response[T]{
|
return Response<T>{
|
||||||
message: ''
|
message: ''
|
||||||
data: data
|
data: data
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// new_full_response[T] constructs a new Response<T> object with the given
|
// new_full_response<T> constructs a new Response<T> object with the given
|
||||||
// message & data.
|
// message & data.
|
||||||
pub fn new_full_response[T](message string, data T) Response[T] {
|
pub fn new_full_response<T>(message string, data T) Response<T> {
|
||||||
return Response[T]{
|
return Response<T>{
|
||||||
message: message
|
message: message
|
||||||
data: data
|
data: data
|
||||||
}
|
}
|
||||||
|
|
100
src/web/web.v
100
src/web/web.v
|
@ -11,7 +11,6 @@ import net.urllib
|
||||||
import time
|
import time
|
||||||
import json
|
import json
|
||||||
import log
|
import log
|
||||||
import metrics
|
|
||||||
|
|
||||||
// The Context struct represents the Context which hold the HTTP request and response.
|
// The Context struct represents the Context which hold the HTTP request and response.
|
||||||
// It has fields for the query, form, files.
|
// It has fields for the query, form, files.
|
||||||
|
@ -28,8 +27,6 @@ pub mut:
|
||||||
conn &net.TcpConn = unsafe { nil }
|
conn &net.TcpConn = unsafe { nil }
|
||||||
// Gives access to a shared logger object
|
// Gives access to a shared logger object
|
||||||
logger shared log.Log
|
logger shared log.Log
|
||||||
// Used to collect metrics on the web server
|
|
||||||
collector &metrics.MetricsCollector
|
|
||||||
// time.ticks() from start of web connection handle.
|
// time.ticks() from start of web connection handle.
|
||||||
// You can use it to determine how much time is spent on your request.
|
// You can use it to determine how much time is spent on your request.
|
||||||
page_gen_start i64
|
page_gen_start i64
|
||||||
|
@ -44,7 +41,7 @@ pub mut:
|
||||||
// Files from multipart-form.
|
// Files from multipart-form.
|
||||||
files map[string][]http.FileData
|
files map[string][]http.FileData
|
||||||
// Allows reading the request body
|
// Allows reading the request body
|
||||||
reader &io.BufferedReader = unsafe { nil }
|
reader io.BufferedReader
|
||||||
// RESPONSE
|
// RESPONSE
|
||||||
status http.Status = http.Status.ok
|
status http.Status = http.Status.ok
|
||||||
content_type string = 'text/plain'
|
content_type string = 'text/plain'
|
||||||
|
@ -78,7 +75,7 @@ fn (mut ctx Context) send_string(s string) ! {
|
||||||
// to the TCP connection socket. Internally, a 10KB buffer is used, to avoid
|
// to the TCP connection socket. Internally, a 10KB buffer is used, to avoid
|
||||||
// having to store all bytes in memory at once.
|
// having to store all bytes in memory at once.
|
||||||
fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ! {
|
fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ! {
|
||||||
mut buf := []u8{len: 10_000}
|
mut buf := []u8{len: 8192}
|
||||||
mut bytes_left := size
|
mut bytes_left := size
|
||||||
|
|
||||||
// Repeat as long as the stream still has data
|
// Repeat as long as the stream still has data
|
||||||
|
@ -148,17 +145,8 @@ pub fn (ctx &Context) is_authenticated() bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// body sends the given body as an HTTP response.
|
// json<T> HTTP_OK with json_s as payload with content-type `application/json`
|
||||||
pub fn (mut ctx Context) body(status http.Status, content_type string, body string) Result {
|
pub fn (mut ctx Context) json<T>(status http.Status, j T) Result {
|
||||||
ctx.status = status
|
|
||||||
ctx.content_type = content_type
|
|
||||||
ctx.send_response(body)
|
|
||||||
|
|
||||||
return Result{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// json[T] HTTP_OK with json_s as payload with content-type `application/json`
|
|
||||||
pub fn (mut ctx Context) json[T](status http.Status, j T) Result {
|
|
||||||
ctx.status = status
|
ctx.status = status
|
||||||
ctx.content_type = 'application/json'
|
ctx.content_type = 'application/json'
|
||||||
|
|
||||||
|
@ -168,15 +156,11 @@ pub fn (mut ctx Context) json[T](status http.Status, j T) Result {
|
||||||
return Result{}
|
return Result{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// file Response HTTP_OK with file as payload
|
// File serves the given file path by streaming its content to the client.
|
||||||
// This function manually implements responses because it needs to stream the file contents
|
|
||||||
pub fn (mut ctx Context) file(f_path string) Result {
|
pub fn (mut ctx Context) file(f_path string) Result {
|
||||||
// If the file doesn't exist, just respond with a 404
|
// If the file doesn't exist, just respond with a 404
|
||||||
if !os.is_file(f_path) {
|
if !os.is_file(f_path) {
|
||||||
ctx.status = .not_found
|
return ctx.status(.not_found)
|
||||||
ctx.send()
|
|
||||||
|
|
||||||
return Result{}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.header.add(.accept_ranges, 'bytes')
|
ctx.header.add(.accept_ranges, 'bytes')
|
||||||
|
@ -192,9 +176,11 @@ pub fn (mut ctx Context) file(f_path string) Result {
|
||||||
}
|
}
|
||||||
|
|
||||||
mut file := os.open(f_path) or {
|
mut file := os.open(f_path) or {
|
||||||
eprintln(err.msg())
|
lock ctx.logger {
|
||||||
ctx.server_error(500)
|
ctx.logger.error('Error in file(): $err.msg()')
|
||||||
return Result{}
|
}
|
||||||
|
|
||||||
|
return ctx.status(.internal_server_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer {
|
defer {
|
||||||
|
@ -229,8 +215,11 @@ pub fn (mut ctx Context) file(f_path string) Result {
|
||||||
|
|
||||||
// Move cursor to start of data to read
|
// Move cursor to start of data to read
|
||||||
file.seek(start, .start) or {
|
file.seek(start, .start) or {
|
||||||
ctx.server_error(500)
|
lock ctx.logger {
|
||||||
return Result{}
|
ctx.logger.error('Error in file(): $err.msg()')
|
||||||
|
}
|
||||||
|
|
||||||
|
return ctx.status(.internal_server_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
length := end - u64(start) + 1
|
length := end - u64(start) + 1
|
||||||
|
@ -254,13 +243,6 @@ pub fn (mut ctx Context) status(status http.Status) Result {
|
||||||
return Result{}
|
return Result{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// server_error Response a server error
|
|
||||||
pub fn (mut ctx Context) server_error(ecode int) Result {
|
|
||||||
ctx.send_custom_response(http_500) or {}
|
|
||||||
|
|
||||||
return Result{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// redirect Redirect to an url
|
// redirect Redirect to an url
|
||||||
pub fn (mut ctx Context) redirect(url string) Result {
|
pub fn (mut ctx Context) redirect(url string) Result {
|
||||||
mut resp := http_302
|
mut resp := http_302
|
||||||
|
@ -278,14 +260,14 @@ interface DbInterface {
|
||||||
|
|
||||||
// run runs the app
|
// run runs the app
|
||||||
[manualfree]
|
[manualfree]
|
||||||
pub fn run[T](global_app &T, port int) {
|
pub fn run<T>(global_app &T, port int) {
|
||||||
mut l := net.listen_tcp(.ip6, ':${port}') or { panic('failed to listen ${err.code()} ${err}') }
|
mut l := net.listen_tcp(.ip6, ':$port') or { panic('failed to listen $err.code() $err') }
|
||||||
|
|
||||||
// Parsing methods attributes
|
// Parsing methods attributes
|
||||||
mut routes := map[string]Route{}
|
mut routes := map[string]Route{}
|
||||||
$for method in T.methods {
|
$for method in T.methods {
|
||||||
http_methods, route_path := parse_attrs(method.name, method.attrs) or {
|
http_methods, route_path := parse_attrs(method.name, method.attrs) or {
|
||||||
eprintln('error parsing method attributes: ${err}')
|
eprintln('error parsing method attributes: $err')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -294,7 +276,7 @@ pub fn run[T](global_app &T, port int) {
|
||||||
path: route_path
|
path: route_path
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
println('[Vweb] Running app on http://localhost:${port}')
|
println('[Vweb] Running app on http://localhost:$port')
|
||||||
for {
|
for {
|
||||||
// Create a new app object for each connection, copy global data like db connections
|
// Create a new app object for each connection, copy global data like db connections
|
||||||
mut request_app := &T{}
|
mut request_app := &T{}
|
||||||
|
@ -311,16 +293,16 @@ pub fn run[T](global_app &T, port int) {
|
||||||
request_app.Context = global_app.Context // copy the context ref that contains static files map etc
|
request_app.Context = global_app.Context // copy the context ref that contains static files map etc
|
||||||
mut conn := l.accept() or {
|
mut conn := l.accept() or {
|
||||||
// failures should not panic
|
// failures should not panic
|
||||||
eprintln('accept() failed with error: ${err.msg()}')
|
eprintln('accept() failed with error: $err.msg()')
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
spawn handle_conn[T](mut conn, mut request_app, routes)
|
go handle_conn<T>(mut conn, mut request_app, routes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// handle_conn handles a connection
|
// handle_conn handles a connection
|
||||||
[manualfree]
|
[manualfree]
|
||||||
fn handle_conn[T](mut conn net.TcpConn, mut app T, routes map[string]Route) {
|
fn handle_conn<T>(mut conn net.TcpConn, mut app T, routes map[string]Route) {
|
||||||
conn.set_read_timeout(30 * time.second)
|
conn.set_read_timeout(30 * time.second)
|
||||||
conn.set_write_timeout(30 * time.second)
|
conn.set_write_timeout(30 * time.second)
|
||||||
|
|
||||||
|
@ -331,23 +313,6 @@ fn handle_conn[T](mut conn net.TcpConn, mut app T, routes map[string]Route) {
|
||||||
app.logger.flush()
|
app.logger.flush()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Record how long request took to process
|
|
||||||
path := urllib.parse(app.req.url) or { urllib.URL{} }.path
|
|
||||||
labels := [
|
|
||||||
['method', app.req.method.str()]!,
|
|
||||||
['path', path]!,
|
|
||||||
// Not all methods properly set this value yet I think
|
|
||||||
['status', app.status.int().str()]!,
|
|
||||||
]
|
|
||||||
app.collector.counter_increment(name: 'http_requests_total', labels: labels)
|
|
||||||
// Prometheus prefers metrics containing base units, as defined here
|
|
||||||
// https://prometheus.io/docs/practices/naming/
|
|
||||||
app.collector.histogram_record(f64(time.ticks() - app.page_gen_start) / 1000,
|
|
||||||
|
|
||||||
name: 'http_requests_duration_seconds'
|
|
||||||
labels: labels
|
|
||||||
)
|
|
||||||
|
|
||||||
unsafe {
|
unsafe {
|
||||||
free(app)
|
free(app)
|
||||||
}
|
}
|
||||||
|
@ -363,16 +328,19 @@ fn handle_conn[T](mut conn net.TcpConn, mut app T, routes map[string]Route) {
|
||||||
// Request parse
|
// Request parse
|
||||||
head := http.parse_request_head(mut reader) or {
|
head := http.parse_request_head(mut reader) or {
|
||||||
// Prevents errors from being thrown when BufferedReader is empty
|
// Prevents errors from being thrown when BufferedReader is empty
|
||||||
if '${err}' != 'none' {
|
if '$err' != 'none' {
|
||||||
eprintln('error parsing request head: ${err}')
|
lock app.logger {
|
||||||
|
app.logger.error('error parsing request head: $err')
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// The healthcheck spams the logs, which isn't very useful
|
// The healthcheck spams the logs, which isn't very useful
|
||||||
if head.url != '/health' {
|
if head.url != '/health' {
|
||||||
lock app.logger {
|
lock app.logger {
|
||||||
app.logger.debug('${head.method} ${head.url} ${head.version}')
|
app.logger.debug('$head.method $head.url $head.version')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -386,7 +354,10 @@ fn handle_conn[T](mut conn net.TcpConn, mut app T, routes map[string]Route) {
|
||||||
|
|
||||||
// URL Parse
|
// URL Parse
|
||||||
url := urllib.parse(head.url) or {
|
url := urllib.parse(head.url) or {
|
||||||
eprintln('error parsing path: ${err}')
|
lock app.logger {
|
||||||
|
app.logger.error('error parsing path: $err')
|
||||||
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -413,7 +384,6 @@ fn handle_conn[T](mut conn net.TcpConn, mut app T, routes map[string]Route) {
|
||||||
static_mime_types: app.static_mime_types
|
static_mime_types: app.static_mime_types
|
||||||
reader: reader
|
reader: reader
|
||||||
logger: app.logger
|
logger: app.logger
|
||||||
collector: app.collector
|
|
||||||
api_key: app.api_key
|
api_key: app.api_key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -424,7 +394,7 @@ fn handle_conn[T](mut conn net.TcpConn, mut app T, routes map[string]Route) {
|
||||||
$for method in T.methods {
|
$for method in T.methods {
|
||||||
$if method.return_type is Result {
|
$if method.return_type is Result {
|
||||||
route := routes[method.name] or {
|
route := routes[method.name] or {
|
||||||
eprintln('parsed attributes for the `${method.name}` are not found, skipping...')
|
eprintln('parsed attributes for the `$method.name` are not found, skipping...')
|
||||||
Route{}
|
Route{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -456,7 +426,7 @@ fn handle_conn[T](mut conn net.TcpConn, mut app T, routes map[string]Route) {
|
||||||
|
|
||||||
method_args := params.clone()
|
method_args := params.clone()
|
||||||
if method_args.len != method.args.len {
|
if method_args.len != method.args.len {
|
||||||
eprintln('warning: uneven parameters count (${method.args.len}) in `${method.name}`, compared to the web route `${method.attrs}` (${method_args.len})')
|
eprintln('warning: uneven parameters count ($method.args.len) in `$method.name`, compared to the web route `$method.attrs` ($method_args.len)')
|
||||||
}
|
}
|
||||||
app.$method(method_args)
|
app.$method(method_args)
|
||||||
return
|
return
|
||||||
|
|
|
@ -12,6 +12,4 @@ address = "http://localhost:8000"
|
||||||
api_update_frequency = 2
|
api_update_frequency = 2
|
||||||
image_rebuild_frequency = 1
|
image_rebuild_frequency = 1
|
||||||
max_concurrent_builds = 3
|
max_concurrent_builds = 3
|
||||||
# max_log_age = 64
|
max_log_age = 64
|
||||||
log_removal_schedule = '* * *'
|
|
||||||
collect_metrics = true
|
|
||||||
|
|
Loading…
Reference in New Issue