diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..2e6afb4 --- /dev/null +++ b/.clang-format @@ -0,0 +1,4 @@ +# To stay consistent with the V formatting style, we use tabs +UseTab: Always +IndentWidth: 4 +TabWidth: 4 diff --git a/.editorconfig b/.editorconfig index e23a3c7..e9c1e63 100644 --- a/.editorconfig +++ b/.editorconfig @@ -5,6 +5,5 @@ root = true end_of_line = lf insert_final_newline = true -[*.v] -# vfmt wants it :( +[*.{v,c,h}] indent_style = tab diff --git a/.gitignore b/.gitignore index aaec9ef..daeb3d3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -*.c +vieter.c /data/ # Build artifacts diff --git a/.gitmodules b/.gitmodules index 47029a0..24af818 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "docs/themes/hugo-book"] path = docs/themes/hugo-book url = https://github.com/alex-shpak/hugo-book +[submodule "src/libvieter"] + path = src/libvieter + url = https://git.rustybever.be/vieter-v/libvieter diff --git a/.woodpecker/build.yml b/.woodpecker/build.yml index f10e2a5..0785392 100644 --- a/.woodpecker/build.yml +++ b/.woodpecker/build.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' + - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2-alpine3.17' matrix: PLATFORM: @@ -57,7 +57,7 @@ pipeline: - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')" - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH" - - export SIGNATURE="$(echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64)" + - export SIGNATURE="$(echo -en $SIG_STRING | openssl dgst -sha1 -hmac $S3_PASSWORD -binary | base64)" - > curl --silent diff --git a/.woodpecker/docs.yml b/.woodpecker/docs.yml index cf4874e..e51f3d7 100644 --- a/.woodpecker/docs.yml +++ b/.woodpecker/docs.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' + - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2-alpine3.17' platform: 'linux/amd64' branches: diff --git a/.woodpecker/gitea.yml b/.woodpecker/gitea.yml index 9034f33..cff0eb9 100644 --- a/.woodpecker/gitea.yml +++ b/.woodpecker/gitea.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' + - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2-alpine3.17' platform: 'linux/amd64' branches: [ 'main' ] diff --git a/.woodpecker/lint.yml b/.woodpecker/lint.yml index ec64d13..1babcbc 100644 --- a/.woodpecker/lint.yml +++ b/.woodpecker/lint.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' + - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2-alpine3.17' # These checks already get performed on the feature branches branches: diff --git a/.woodpecker/man.yml b/.woodpecker/man.yml index 8c6ca06..9ad8dcf 100644 --- a/.woodpecker/man.yml +++ b/.woodpecker/man.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' + - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2-alpine3.17' platform: 'linux/amd64' branches: @@ -8,15 +8,21 @@ branches: depends_on: - build -skip_clone: true - pipeline: - generate: + install-modules: image: *vlang_image pull: true commands: - - curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64" - - chmod +x vieter + - export VMODULES=$PWD/.vmodules + - 'cd src && v install' + + generate: + image: *vlang_image + commands: + # - curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64" + # - chmod +x vieter + - export VMODULES=$PWD/.vmodules + - make - ./vieter man man - cd man diff --git a/.woodpecker/test.yml b/.woodpecker/test.yml index 39cb9f9..91ef7c6 100644 --- a/.woodpecker/test.yml +++ b/.woodpecker/test.yml @@ -1,5 +1,5 @@ variables: - - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2' + - &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2-alpine3.17' matrix: PLATFORM: diff --git a/CHANGELOG.md b/CHANGELOG.md index 54d833a..4c572bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* Metrics endpoint for Prometheus integration +* Search in list of targets using API & CLI +* Allow filtering targets by arch value + +### Changed + +* Rewrote cron expression logic in C + +### Removed + +* Deprecated cron daemon + +## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0) + +### Added + +* CLI commands for removing packages, arch-repos & repositories + +## [0.5.0-rc.2](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.2) + +### Added + +* API route for removing logs & accompanying CLI command +* Daemon for periodically removing old logs +* CLI flag to filter logs by specific exit codes + +### Changed + +* Use `--long-option` instead of `-long-option` for CLI + +## [0.5.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.1) + +### Added + * Allow specifying subdirectory inside Git repository * Added option to deploy using agent-server architecture instead of cron daemon * Allow scheduling builds on the server from the CLI tool instead of building diff --git a/Makefile b/Makefile index e716807..1521676 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,20 @@ # =====CONFIG===== SRC_DIR := src -SOURCES != find '$(SRC_DIR)' -iname '*.v' +SRCS != find '$(SRC_DIR)' -iname '*.v' V_PATH ?= v -V := $(V_PATH) -showcc -gc boehm -W -d use_openssl +V := $(V_PATH) -showcc -gc boehm -W -d use_openssl -skip-unused all: vieter # =====COMPILATION===== +.PHONY: libvieter +libvieter: + make -C '$(SRC_DIR)/libvieter' CFLAGS='-O3' + # Regular binary -vieter: $(SOURCES) +vieter: $(SOURCES) libvieter $(V) -g -o vieter $(SRC_DIR) # Debug build using gcc @@ -18,7 +22,7 @@ vieter: $(SOURCES) # multi-threaded and causes issues when running vieter inside gdb. .PHONY: debug debug: dvieter -dvieter: $(SOURCES) +dvieter: $(SOURCES) libvieter $(V_PATH) -showcc -keepc -cg -o dvieter $(SRC_DIR) # Run the debug build inside gdb @@ -29,12 +33,12 @@ gdb: dvieter # Optimised production build .PHONY: prod prod: pvieter -pvieter: $(SOURCES) +pvieter: $(SOURCES) libvieter $(V) -o pvieter -prod $(SRC_DIR) # Only generate C code .PHONY: c -c: $(SOURCES) +c: $(SOURCES) libvieter $(V) -o vieter.c $(SRC_DIR) @@ -67,6 +71,7 @@ man: vieter # =====OTHER===== +# Linting .PHONY: lint lint: $(V) fmt -verify $(SRC_DIR) @@ -74,18 +79,24 @@ lint: $(V_PATH) missdoc -p $(SRC_DIR) @ [ $$($(V_PATH) missdoc -p $(SRC_DIR) | wc -l) = 0 ] -# Format the V codebase + +# Formatting .PHONY: fmt fmt: $(V) fmt -w $(SRC_DIR) -.PHONY: test -test: - $(V) test $(SRC_DIR) +# Testing +.PHONY: test +test: libvieter + $(V) -g test $(SRC_DIR) + + +# Cleaning .PHONY: clean clean: rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' 'docs/public' + make -C '$(SRC_DIR)/libvieter' clean # =====EXPERIMENTAL===== diff --git a/PKGBUILD b/PKGBUILD index b600ba0..bf9c621 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -3,7 +3,7 @@ pkgbase='vieter' pkgname='vieter' -pkgver='0.4.0' +pkgver='0.5.0' pkgrel=1 pkgdesc="Lightweight Arch repository server & package build system" depends=('glibc' 'openssl' 'libarchive' 'sqlite') diff --git a/PKGBUILD.dev b/PKGBUILD.dev index 79c7f37..b07585a 100644 --- a/PKGBUILD.dev +++ b/PKGBUILD.dev @@ -11,27 +11,37 @@ makedepends=('git' 'vlang') arch=('x86_64' 'aarch64') url='https://git.rustybever.be/vieter-v/vieter' license=('AGPL3') -source=("$pkgname::git+https://git.rustybever.be/vieter-v/vieter#branch=dev") -md5sums=('SKIP') +source=( + "${pkgname}::git+https://git.rustybever.be/vieter-v/vieter#branch=dev" + "libvieter::git+https://git.rustybever.be/vieter-v/libvieter" +) +md5sums=('SKIP' 'SKIP') provides=('vieter') conflicts=('vieter') pkgver() { - cd "$pkgname" + cd "${pkgname}" git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g' } prepare() { - export VMODULES="$srcdir/.vmodules" + cd "${pkgname}" - cd "$pkgname/src" && v install + # Add the libvieter submodule + git submodule init + git config submodules.src/libvieter.url "${srcdir}/libvieter" + git -c protocol.file.allow=always submodule update + + export VMODULES="${srcdir}/.vmodules" + + cd src && v install } build() { - export VMODULES="$srcdir/.vmodules" + export VMODULES="${srcdir}/.vmodules" - cd "$pkgname" + cd "${pkgname}" make prod @@ -42,9 +52,9 @@ build() { } package() { - install -dm755 "$pkgdir/usr/bin" - install -Dm755 "$pkgname/pvieter" "$pkgdir/usr/bin/vieter" + install -dm755 "${pkgdir}/usr/bin" + install -Dm755 "${pkgname}/pvieter" "${pkgdir}/usr/bin/vieter" - install -dm755 "$pkgdir/usr/share/man/man1" - install -Dm644 "$pkgname/man"/*.1 "$pkgdir/usr/share/man/man1" + install -dm755 "${pkgdir}/usr/share/man/man1" + install -Dm644 "${pkgname}/man"/*.1 "${pkgdir}/usr/share/man/man1" } diff --git a/docs/api/Gemfile b/docs/api/Gemfile index 7604fd4..3010875 100644 --- a/docs/api/Gemfile +++ b/docs/api/Gemfile @@ -6,7 +6,7 @@ gem 'middleman', '~> 4.4' gem 'middleman-syntax', '~> 3.2' gem 'middleman-autoprefixer', '~> 3.0' gem 'middleman-sprockets', '~> 4.1' -gem 'rouge', '~> 3.21' +gem 'rouge', '~> 4.0' gem 'redcarpet', '~> 3.5.0' gem 'nokogiri', '~> 1.13.3' gem 'sass' diff --git a/docs/api/source/includes/_jobs.md b/docs/api/source/includes/_jobs.md new file mode 100644 index 0000000..a25309d --- /dev/null +++ b/docs/api/source/includes/_jobs.md @@ -0,0 +1,78 @@ +# Jobs + + + +## Manually schedule a job + +```shell +curl \ + -H 'X-Api-Key: secret' \ + https://example.com/api/v1/jobs/queue?target=10&force&arch=x86_64 +``` + +Manually schedule a job on the server. + +### HTTP Request + +`POST /api/v1/jobs/queue` + +### Query Parameters + +Parameter | Description +--------- | ----------- +target | Id of target to schedule build for +arch | Architecture to build on +force | Whether it's a forced build (true if present) + +## Poll for new jobs + + + +```shell +curl \ + -H 'X-Api-Key: secret' \ + https://example.com/api/v1/jobs/poll?arch=x86_64&max=2 +``` + +> JSON output format + +```json +{ + "message": "", + "data": [ + { + "target_id": 1, + "kind": "git", + "url": "https://aur.archlinux.org/discord-ptb.git", + "branch": "master", + "path": "", + "repo": "bur", + "base_image": "archlinux:base-devel", + "force": true + } + ] +} +``` + +Poll the server for new builds. + +### HTTP Request + +`GET /api/v1/jobs/poll` + +### Query Parameters + +Parameter | Description +--------- | ----------- +arch | For which architecture to receive jobs +max | How many jobs to receive at most diff --git a/docs/api/source/includes/_logs.md b/docs/api/source/includes/_logs.md index 1c14e71..d6134b7 100644 --- a/docs/api/source/includes/_logs.md +++ b/docs/api/source/includes/_logs.md @@ -125,8 +125,8 @@ id | ID of requested log @@ -149,3 +149,24 @@ target | id of target this build is for ### Request body Plaintext contents of the build log. + +## Remove a build log + +```shell +curl \ + -XDELETE \ + -H 'X-Api-Key: secret' \ + https://example.com/api/v1/logs/1 +``` + +Remove a build log from the server. + +### HTTP Request + +`DELETE /api/v1/logs/:id` + +### URL Parameters + +Parameter | Description +--------- | ----------- +id | id of log to remove diff --git a/docs/api/source/includes/_targets.md b/docs/api/source/includes/_targets.md index 93a4e86..1a5f3e0 100644 --- a/docs/api/source/includes/_targets.md +++ b/docs/api/source/includes/_targets.md @@ -27,6 +27,7 @@ curl \ "kind": "git", "url": "https://aur.archlinux.org/discord-ptb.git", "branch": "master", + "path" : "", "repo": "bur", "schedule": "", "arch": [ @@ -54,6 +55,8 @@ Parameter | Description limit | Maximum amount of results to return. offset | Offset of results. repo | Limit results to targets that publish to the given repo. +query | Only return targets that have this substring in their URL, path or branch. +arch | Only return targets that publish to this arch. ## Get specific target @@ -73,8 +76,9 @@ curl \ "kind": "git", "url": "https://aur.archlinux.org/discord-ptb.git", "branch": "master", + "path": "", "repo": "bur", - "schedule": "0 3", + "schedule": "0 2", "arch": [ { "id": 1, @@ -124,6 +128,7 @@ Parameter | Description kind | Kind of target to add; one of 'git', 'url'. url | URL of the Git repository. branch | Branch of the Git repository. +path | Subdirectory inside Git repository to use. repo | Vieter repository to publish built packages to. schedule | Cron build schedule (syntax explained [here](https://rustybever.be/docs/vieter/usage/builds/schedule/)) arch | Comma-separated list of architectures to build package on. @@ -149,12 +154,20 @@ Parameter | Description kind | Kind of target; one of 'git', 'url'. url | URL of the Git repository. branch | Branch of the Git repository. +path | Subdirectory inside Git repository to use. repo | Vieter repository to publish built packages to. schedule | Cron build schedule arch | Comma-separated list of architectures to build package on. ## Remove a target +```shell +curl \ + -XDELETE \ + -H 'X-Api-Key: secret' \ + https://example.com/api/v1/targets/1 +``` + Remove a target from the server. ### HTTP Request diff --git a/docs/api/source/index.html.md b/docs/api/source/index.html.md index 4bfddb8..f61e44a 100644 --- a/docs/api/source/index.html.md +++ b/docs/api/source/index.html.md @@ -11,6 +11,7 @@ includes: - repository - targets - logs + - jobs search: true diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 95bf713..612c505 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -32,11 +32,11 @@ configuration variable required for each command. ### `vieter server` +* `port`: HTTP port to run on + * Default: `8000` * `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`, `WARN`, `INFO` or `DEBUG`. * Default: `WARN` -* `log_file`: log file to write logs to. - * Default: `vieter.log` (in the current directory) * `pkg_dir`: where Vieter should store the actual package archives. * `data_dir`: where Vieter stores the repositories, log file & database. * `api_key`: the API key to use when authenticating requests. @@ -44,9 +44,26 @@ configuration variable required for each command. * Packages with architecture `any` are always added to this architecture. This prevents the server from being confused when an `any` package is published as the very first package for a repository. - * Git repositories added without an `arch` value use this value instead. -* `port`: HTTP port to run on - * Default: `8000` + * Targets added without an `arch` value use this value instead. +* `global_schedule`: build schedule for any target that does not have a + schedule defined. For information about this syntax, see + [here](/usage/builds/schedule). + * Default: `0 3` (3AM every night) +* `base_image`: Docker image to use when building a package. Any Pacman-based + distro image should work, as long as `/etc/pacman.conf` is used & + `base-devel` exists in the repositories. Make sure that the image supports + the architecture of your cron daemon. + * Default: `archlinux:base-devel` (only works on `x86_64`). If you require + `aarch64` support, consider using + [`menci/archlinuxarm:base-devel`](https://hub.docker.com/r/menci/archlinuxarm) + ([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the + image used for the Vieter CI builds. +* `max_log_age`: maximum age of logs (in days). Logs older than this will get + cleaned by the log removal daemon. If set to zero, no logs are ever removed. + The age of logs is determined by the time the build was started. + * Default: `0` +* `log_removal_schedule`: cron schedule defining when to clean old logs. + * Default: `0 0` (every day at midnight) ### `vieter cron` diff --git a/docs/content/installation.md b/docs/content/installation.md index 21eda64..5b8e2d8 100644 --- a/docs/content/installation.md +++ b/docs/content/installation.md @@ -23,15 +23,15 @@ guarantees about stability, so beware! Thanks to the single-binary design of Vieter, this image can be used both for the repository server, the cron daemon and the agent. -Below is an example compose file to set up both the repository server & the -cron daemon: +Below is a minimal compose file to set up both the repository server & a build +agent: ```yaml version: '3' services: server: - image: 'chewingbever/vieter:dev' + image: 'chewingbever/vieter:0.5.0-rc.1' restart: 'always' environment: @@ -41,18 +41,19 @@ services: - 'data:/data' cron: - image: 'chewingbever/vieter:dev' + image: 'chewingbever/vieter:0.5.0-rc.1' restart: 'always' + # Required to connect to the Docker daemon user: root - command: 'vieter cron' + command: 'vieter agent' environment: - 'VIETER_API_KEY=secret' # MUST be public URL of Vieter repository - 'VIETER_ADDRESS=https://example.com' - - 'VIETER_DEFAULT_ARCH=x86_64' + # Architecture for which the agent builds + - 'VIETER_ARCH=x86_64' - 'VIETER_MAX_CONCURRENT_BUILDS=2' - - 'VIETER_GLOBAL_SCHEDULE=0 3' volumes: - '/var/run/docker.sock:/var/run/docker.sock' @@ -63,14 +64,17 @@ volumes: If you do not require the build system, the repository server can be used independently as well. +Of course, Vieter allows a lot more configuration than this. This compose file +is meant as a starting point for setting up your installation. + {{< hint info >}} **Note** -Builds are executed on the cron daemon's system using the host's Docker daemon. -A cron daemon on a specific architecture will only build packages for that -specific architecture. Therefore, if you wish to build packages for both -`x86_64` & `aarch64`, you'll have to deploy two cron daemons, one on each -architecture. Afterwards, any Git repositories enabled for those two -architectures will build on both. +Builds are executed on the agent's system using the host's Docker daemon. An +agent for a specific `arch` will only build packages for that specific +architecture. Therefore, if you wish to build packages for both `x86_64` & +`aarch64`, you'll have to deploy two agents, one on each architecture. +Afterwards, any Git repositories enabled for those two architectures will build +on both. {{< /hint >}} ## Binary @@ -99,9 +103,9 @@ latest official release or `vieter-git` for the latest development release. ### AUR If you prefer building the packages locally (or on your own Vieter instance), -there's the `[vieter](https://aur.archlinux.org/packages/vieter)` & -`[vieter-git](https://aur.archlinux.org/packages/vieter-git)` packages on the -AUR. These packages build using the `vlang-git` compiler package, so I can't +there's the [`vieter`](https://aur.archlinux.org/packages/vieter) & +[`vieter-git`](https://aur.archlinux.org/packages/vieter-git) packages on the +AUR. These packages build using the `vlang` compiler package, so I can't guarantee that a compiler update won't temporarily break them. ## Building from source diff --git a/docs/content/other/_index.md b/docs/content/other/_index.md deleted file mode 100644 index 394456b..0000000 --- a/docs/content/other/_index.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -weight: 100 ---- diff --git a/docs/content/other/builds-in-depth.md b/docs/content/other/builds-in-depth.md deleted file mode 100644 index d8df6ec..0000000 --- a/docs/content/other/builds-in-depth.md +++ /dev/null @@ -1,81 +0,0 @@ -# Builds In-depth - -For those interested, this page describes how the build system works -internally. - -## Builder image - -Every cron daemon perodically creates a builder image that is then used as a -base for all builds. This is done to prevent build containers having to pull -down a bunch of updates when they update their system. - -The build container is created by running the following commands inside a -container started from the image defined in `base_image`: - -```sh -# Update repos & install required packages -pacman -Syu --needed --noconfirm base-devel git -# Add a non-root user to run makepkg -groupadd -g 1000 builder -useradd -mg builder builder -# Make sure they can use sudo without a password -echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers -# Create the directory for the builds & make it writeable for the -# build user -mkdir /build -chown -R builder:builder /build -``` - -This script updates the packages to their latest versions & creates a non-root -user to use when running `makepkg`. - -This script is base64-encoded & passed to the container as an environment -variable. The container's entrypoint is set to `/bin/sh -c` & its command -argument to `echo $BUILD_SCRIPT | base64 -d | /bin/sh -e`, with the -`BUILD_SCRIPT` environment variable containing the base64-encoded script. - -Once the container exits, a new Docker image is created from it. This image is -then used as the base for any builds. - -## Running builds - -Each build has its own Docker container, using the builder image as its base. -The same base64-based technique as above is used, just with a different script. -To make the build logs more clear, each command is appended by an echo command -printing the next command to stdout. - -Given the Git repository URL is `https://examplerepo.com` with branch `main`, -the URL of the Vieter server is `https://example.com` and `vieter` is the -repository we wish to publish to, we get the following script: - -```sh -echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' -echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf -echo -e '+ pacman -Syu --needed --noconfirm' -pacman -Syu --needed --noconfirm -echo -e '+ su builder' -su builder -echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo' -git clone --single-branch --depth 1 --branch main https://examplerepo.com repo -echo -e '+ cd repo' -cd repo -echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' -makepkg --nobuild --syncdeps --needed --noconfirm -echo -e '+ source PKGBUILD' -source PKGBUILD -echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' -curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 -echo -e '+ [ "$(id -u)" == 0 ] && exit 0' -[ "$(id -u)" == 0 ] && exit 0 -echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' -MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done -``` - -This script: - -1. Adds the target repository as a repository in the build container -2. Updates mirrors & packages -3. Clones the Git repository -4. Runs `makepkg` without building to calculate `pkgver` -5. Checks whether the package version is already present on the server -6. If not, run `makepkg` & publish any generated package archives to the server diff --git a/docs/content/usage/builds/cleanup.md b/docs/content/usage/builds/cleanup.md new file mode 100644 index 0000000..724a75f --- /dev/null +++ b/docs/content/usage/builds/cleanup.md @@ -0,0 +1,23 @@ +--- +weight: 20 +--- + +# Cleanup + +Vieter stores the logs of every single package build. While this is great for +debugging why builds fail, it also causes an active or long-running Vieter +instance to accumulate thousands of logs. + +To combat this, a log removal daemon can be enabled that periodically removes +old build logs. By starting your server with the `max_log_age` variable (see +[Configuration](/configuration#vieter-server)), a daemon will get enabled that +periodically removes logs older than this setting. By default, this will happen +every day at midnight, but this behavior can be changed using the +`log_removal_schedule` variable. + +{{< hint info >}} +**Note** +The daemon will always run a removal of logs on startup. Therefore, it's +possible the daemon will be *very* active when first enabling this setting. +After the initial surge of logs to remove, it'll calm down again. +{{< /hint >}} diff --git a/docs/content/usage/builds/schedule.md b/docs/content/usage/builds/schedule.md index de59e25..d3802fd 100644 --- a/docs/content/usage/builds/schedule.md +++ b/docs/content/usage/builds/schedule.md @@ -1,3 +1,7 @@ +--- +weight: 10 +--- + # Cron schedule syntax The Vieter cron daemon uses a subset of the cron expression syntax to schedule diff --git a/src/agent/images.v b/src/agent/images.v index 1fec567..5fba0f7 100644 --- a/src/agent/images.v +++ b/src/agent/images.v @@ -99,7 +99,7 @@ fn (mut m ImageManager) clean_old_images() { // wasn't deleted. Therefore, we move the index over. If the function // returns true, the array's length has decreased by one so we don't // move the index. - dd.remove_image(m.images[image][i]) or { + dd.image_remove(m.images[image][i]) or { // The image was removed by an external event if err.code() == 404 { m.images[image].delete(i) diff --git a/src/agent/log.v b/src/agent/log.v index cd59207..fcd8373 100644 --- a/src/agent/log.v +++ b/src/agent/log.v @@ -1,35 +1,36 @@ module agent -import log - -// log a message with the given level -pub fn (mut d AgentDaemon) log(msg string, level log.Level) { - lock d.logger { - d.logger.send_output(msg, level) - } -} - // lfatal create a log message with the fatal level pub fn (mut d AgentDaemon) lfatal(msg string) { - d.log(msg, log.Level.fatal) + lock d.logger { + d.logger.fatal(msg) + } } // lerror create a log message with the error level pub fn (mut d AgentDaemon) lerror(msg string) { - d.log(msg, log.Level.error) + lock d.logger { + d.logger.error(msg) + } } // lwarn create a log message with the warn level pub fn (mut d AgentDaemon) lwarn(msg string) { - d.log(msg, log.Level.warn) + lock d.logger { + d.logger.warn(msg) + } } // linfo create a log message with the info level pub fn (mut d AgentDaemon) linfo(msg string) { - d.log(msg, log.Level.info) + lock d.logger { + d.logger.info(msg) + } } // ldebug create a log message with the debug level pub fn (mut d AgentDaemon) ldebug(msg string) { - d.log(msg, log.Level.debug) + lock d.logger { + d.logger.debug(msg) + } } diff --git a/src/build/build.v b/src/build/build.v index 712c93b..dfea8c0 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -57,7 +57,7 @@ pub fn create_build_image(base_image string) !string { image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' } // We pull the provided image - dd.pull_image(image_name, image_tag)! + dd.image_pull(image_name, image_tag)! id := dd.container_create(c)!.id // id := docker.create_container(c)! @@ -79,7 +79,7 @@ pub fn create_build_image(base_image string) !string { // TODO also add the base image's name into the image name to prevent // conflicts. tag := time.sys_mono_now().str() - image := dd.create_image_from_container(id, 'vieter-build', tag)! + image := dd.image_from_container(id, 'vieter-build', tag)! dd.container_remove(id)! return image.id diff --git a/src/build/queue.v b/src/build/queue.v index e74529c..abd4ec6 100644 --- a/src/build/queue.v +++ b/src/build/queue.v @@ -1,7 +1,7 @@ module build import models { BuildConfig, Target } -import cron.expression { CronExpression, parse_expression } +import cron import time import datatypes { MinHeap } import util @@ -13,7 +13,7 @@ pub mut: // Next timestamp from which point this job is allowed to be executed timestamp time.Time // Required for calculating next timestamp after having pop'ed a job - ce CronExpression + ce &cron.Expression = unsafe { nil } // Actual build config sent to the agent config BuildConfig // Whether this is a one-time job @@ -30,7 +30,7 @@ fn (r1 BuildJob) < (r2 BuildJob) bool { // for each architecture. Agents receive jobs from this queue. pub struct BuildJobQueue { // Schedule to use for targets without explicitely defined cron expression - default_schedule CronExpression + default_schedule &cron.Expression // Base image to use for targets without defined base image default_base_image string mut: @@ -44,9 +44,9 @@ mut: } // new_job_queue initializes a new job queue -pub fn new_job_queue(default_schedule CronExpression, default_base_image string) BuildJobQueue { +pub fn new_job_queue(default_schedule &cron.Expression, default_base_image string) BuildJobQueue { return BuildJobQueue{ - default_schedule: default_schedule + default_schedule: unsafe { default_schedule } default_base_image: default_base_image invalidated: map[int]time.Time{} } @@ -85,14 +85,14 @@ pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! { if !input.now { ce := if input.target.schedule != '' { - parse_expression(input.target.schedule) or { + cron.parse_expression(input.target.schedule) or { return error("Error while parsing cron expression '$input.target.schedule' (id $input.target.id): $err.msg()") } } else { q.default_schedule } - job.timestamp = ce.next_from_now()! + job.timestamp = ce.next_from_now() job.ce = ce } else { job.timestamp = time.now() @@ -105,8 +105,8 @@ pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! { // reschedule the given job by calculating the next timestamp and re-adding it // to its respective queue. This function is called by the pop functions // *after* having pop'ed the job. -fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) ! { - new_timestamp := job.ce.next_from_now()! +fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) { + new_timestamp := job.ce.next_from_now() new_job := BuildJob{ ...job @@ -168,10 +168,7 @@ pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob { job = q.queues[arch].pop()? if !job.single { - // TODO how do we handle this properly? Is it even possible for a - // cron expression to not return a next time if it's already been - // used before? - q.reschedule(job, arch) or {} + q.reschedule(job, arch) } return job @@ -198,8 +195,7 @@ pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { job = q.queues[arch].pop() or { break } if !job.single { - // TODO idem - q.reschedule(job, arch) or {} + q.reschedule(job, arch) } out << job diff --git a/src/client/logs.v b/src/client/logs.v index 85063bc..6553837 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -1,28 +1,27 @@ module client import models { BuildLog, BuildLogFilter } -import net.http { Method } import web.response { Response } import time // get_build_logs returns all build logs. pub fn (c &Client) get_build_logs(filter BuildLogFilter) ![]BuildLog { params := models.params_from(filter) - data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)! + data := c.send_request<[]BuildLog>(.get, '/api/v1/logs', params)! return data.data } // get_build_log returns a specific build log. pub fn (c &Client) get_build_log(id int) !BuildLog { - data := c.send_request(Method.get, '/api/v1/logs/$id', {})! + data := c.send_request(.get, '/api/v1/logs/$id', {})! return data.data } // get_build_log_content returns the contents of the build log file. pub fn (c &Client) get_build_log_content(id int) !string { - data := c.send_request_raw_response(Method.get, '/api/v1/logs/$id/content', {}, '')! + data := c.send_request_raw_response(.get, '/api/v1/logs/$id/content', {}, '')! return data } @@ -37,7 +36,12 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t 'exitCode': exit_code.str() } - data := c.send_request_with_body(Method.post, '/api/v1/logs', params, content)! + data := c.send_request_with_body(.post, '/api/v1/logs', params, content)! return data } + +// remove_build_log removes the build log with the given id from the server. +pub fn (c &Client) remove_build_log(id int) ! { + c.send_request(.delete, '/api/v1/logs/$id', {})! +} diff --git a/src/client/repos.v b/src/client/repos.v new file mode 100644 index 0000000..9644e9b --- /dev/null +++ b/src/client/repos.v @@ -0,0 +1,16 @@ +module client + +// remove_repo removes an entire repository. +pub fn (c &Client) remove_repo(repo string) ! { + c.send_request(.delete, '/$repo', {})! +} + +// remove_arch_repo removes an entire arch-repo. +pub fn (c &Client) remove_arch_repo(repo string, arch string) ! { + c.send_request(.delete, '/$repo/$arch', {})! +} + +// remove_package removes a single package from the given arch-repo. +pub fn (c &Client) remove_package(repo string, arch string, pkgname string) ! { + c.send_request(.delete, '/$repo/$arch/$pkgname', {})! +} diff --git a/src/client/targets.v b/src/client/targets.v index da6a9e4..565832e 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -1,12 +1,11 @@ module client import models { Target, TargetFilter } -import net.http { Method } // get_targets returns a list of targets, given a filter object. pub fn (c &Client) get_targets(filter TargetFilter) ![]Target { params := models.params_from(filter) - data := c.send_request<[]Target>(Method.get, '/api/v1/targets', params)! + data := c.send_request<[]Target>(.get, '/api/v1/targets', params)! return data.data } @@ -34,7 +33,7 @@ pub fn (c &Client) get_all_targets() ![]Target { // get_target returns the target for a specific id. pub fn (c &Client) get_target(id int) !Target { - data := c.send_request(Method.get, '/api/v1/targets/$id', {})! + data := c.send_request(.get, '/api/v1/targets/$id', {})! return data.data } @@ -51,14 +50,14 @@ pub struct NewTarget { // add_target adds a new target to the server. pub fn (c &Client) add_target(t NewTarget) !int { params := models.params_from(t) - data := c.send_request(Method.post, '/api/v1/targets', params)! + data := c.send_request(.post, '/api/v1/targets', params)! return data.data } // remove_target removes the target with the given id from the server. pub fn (c &Client) remove_target(id int) !string { - data := c.send_request(Method.delete, '/api/v1/targets/$id', {})! + data := c.send_request(.delete, '/api/v1/targets/$id', {})! return data.data } @@ -66,7 +65,7 @@ pub fn (c &Client) remove_target(id int) !string { // patch_target sends a PATCH request to the given target with the params as // payload. pub fn (c &Client) patch_target(id int, params map[string]string) !string { - data := c.send_request(Method.patch, '/api/v1/targets/$id', params)! + data := c.send_request(.patch, '/api/v1/targets/$id', params)! return data.data } diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 3064a58..35ce4d7 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -24,11 +24,13 @@ pub fn cmd() cli.Command { flags: [ cli.Flag{ name: 'limit' + abbrev: 'l' description: 'How many results to return.' flag: cli.FlagType.int }, cli.Flag{ name: 'offset' + abbrev: 'o' description: 'Minimum index to return.' flag: cli.FlagType.int }, @@ -39,16 +41,18 @@ pub fn cmd() cli.Command { }, cli.Flag{ name: 'today' - description: 'Only list logs started today.' + abbrev: 't' + description: 'Only list logs started today. This flag overwrites any other date-related flag.' flag: cli.FlagType.bool }, cli.Flag{ name: 'failed' - description: 'Only list logs with non-zero exit codes.' + description: 'Only list logs with non-zero exit codes. This flag overwrites the --code flag.' flag: cli.FlagType.bool }, cli.Flag{ name: 'day' + abbrev: 'd' description: 'Only list logs started on this day. (format: YYYY-MM-DD)' flag: cli.FlagType.string }, @@ -62,6 +66,11 @@ pub fn cmd() cli.Command { description: 'Only list logs started after this timestamp. (format: YYYY-MM-DD HH:mm:ss)' flag: cli.FlagType.string }, + cli.Flag{ + name: 'code' + description: 'Only return logs with the given exit code. Prepend with `!` to exclude instead of include. Can be specified multiple times.' + flag: cli.FlagType.string_array + }, ] execute: fn (cmd cli.Command) ! { config_file := cmd.flags.get_string('config-file')! @@ -131,6 +140,8 @@ pub fn cmd() cli.Command { filter.exit_codes = [ '!0', ] + } else { + filter.exit_codes = cmd.flags.get_strings('code')! } raw := cmd.flags.get_bool('raw')! @@ -138,6 +149,18 @@ pub fn cmd() cli.Command { list(conf, filter, raw)! } }, + cli.Command{ + name: 'remove' + required_args: 1 + usage: 'id' + description: 'Remove a build log that matches the given id.' + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + + remove(conf, cmd.args[0])! + } + }, cli.Command{ name: 'info' required_args: 1 @@ -204,3 +227,9 @@ fn content(conf Config, id int) ! { println(content) } + +// remove removes a build log from the server's list. +fn remove(conf Config, id string) ! { + c := client.new(conf.address, conf.api_key) + c.remove_build_log(id.int())! +} diff --git a/src/console/repos/repos.v b/src/console/repos/repos.v new file mode 100644 index 0000000..729208e --- /dev/null +++ b/src/console/repos/repos.v @@ -0,0 +1,52 @@ +module repos + +import cli +import conf as vconf +import client + +struct Config { + address string [required] + api_key string [required] +} + +// cmd returns the cli module that handles modifying the repository contents. +pub fn cmd() cli.Command { + return cli.Command{ + name: 'repos' + description: 'Interact with the repositories & packages stored on the server.' + commands: [ + cli.Command{ + name: 'remove' + required_args: 1 + usage: 'repo [arch [pkgname]]' + description: 'Remove a repo, arch-repo, or package from the server.' + flags: [ + cli.Flag{ + name: 'force' + flag: cli.FlagType.bool + }, + ] + execute: fn (cmd cli.Command) ! { + config_file := cmd.flags.get_string('config-file')! + conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! + + if cmd.args.len < 3 { + if !cmd.flags.get_bool('force')! { + return error('Removing an arch-repo or repository is a very destructive command. If you really do wish to perform this operation, explicitely add the --force flag.') + } + } + + client := client.new(conf.address, conf.api_key) + + if cmd.args.len == 1 { + client.remove_repo(cmd.args[0])! + } else if cmd.args.len == 2 { + client.remove_arch_repo(cmd.args[0], cmd.args[1])! + } else { + client.remove_package(cmd.args[0], cmd.args[1], cmd.args[2])! + } + } + }, + ] + } +} diff --git a/src/console/schedule/schedule.v b/src/console/schedule/schedule.v index 7ce0516..ceabf24 100644 --- a/src/console/schedule/schedule.v +++ b/src/console/schedule/schedule.v @@ -1,7 +1,7 @@ module schedule import cli -import cron.expression { parse_expression } +import cron import time // cmd returns the cli submodule for previewing a cron schedule. @@ -19,10 +19,10 @@ pub fn cmd() cli.Command { }, ] execute: fn (cmd cli.Command) ! { - ce := parse_expression(cmd.args.join(' '))! + ce := cron.parse_expression(cmd.args.join(' '))! count := cmd.flags.get_int('count')! - for t in ce.next_n(time.now(), count)! { + for t in ce.next_n(time.now(), count) { println(t) } } diff --git a/src/console/targets/build.v b/src/console/targets/build.v index e18077d..b8cbe7f 100644 --- a/src/console/targets/build.v +++ b/src/console/targets/build.v @@ -26,7 +26,7 @@ fn build(conf Config, target_id int, force bool) ! { dd.close() or {} } - dd.remove_image(image_id)! + dd.image_remove(image_id)! println('Uploading logs to Vieter...') c.add_build_log(target.id, res.start_time, res.end_time, build_arch, res.exit_code, diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 94deebd..709c196 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -2,7 +2,7 @@ module targets import cli import conf as vconf -import cron.expression { parse_expression } +import cron import client { NewTarget } import console import models { TargetFilter } @@ -25,11 +25,13 @@ pub fn cmd() cli.Command { flags: [ cli.Flag{ name: 'limit' + abbrev: 'l' description: 'How many results to return.' flag: cli.FlagType.int }, cli.Flag{ name: 'offset' + abbrev: 'o' description: 'Minimum index to return.' flag: cli.FlagType.int }, @@ -38,6 +40,17 @@ pub fn cmd() cli.Command { description: 'Only return targets that publish to this repo.' flag: cli.FlagType.string }, + cli.Flag{ + name: 'query' + abbrev: 'q' + description: 'Search string to filter targets by.' + flag: cli.FlagType.string + }, + cli.Flag{ + name: 'arch' + description: 'Only list targets that build for this arch.' + flag: cli.FlagType.string + }, ] execute: fn (cmd cli.Command) ! { config_file := cmd.flags.get_string('config-file')! @@ -60,6 +73,16 @@ pub fn cmd() cli.Command { filter.repo = repo } + query := cmd.flags.get_string('query')! + if query != '' { + filter.query = query + } + + arch := cmd.flags.get_string('arch')! + if arch != '' { + filter.arch = arch + } + raw := cmd.flags.get_bool('raw')! list(conf, filter, raw)! @@ -272,7 +295,7 @@ fn patch(conf Config, id string, params map[string]string) ! { // We check the cron expression first because it's useless to send an // invalid one to the server. if 'schedule' in params && params['schedule'] != '' { - parse_expression(params['schedule']) or { + cron.parse_expression(params['schedule']) or { return error('Invalid cron expression: $err.msg()') } } diff --git a/src/cron/cli.v b/src/cron/cli.v deleted file mode 100644 index 16a3537..0000000 --- a/src/cron/cli.v +++ /dev/null @@ -1,32 +0,0 @@ -module cron - -import cli -import conf as vconf - -struct Config { -pub: - log_level string = 'WARN' - api_key string - address string - data_dir string - base_image string = 'archlinux:base-devel' - max_concurrent_builds int = 1 - api_update_frequency int = 15 - image_rebuild_frequency int = 1440 - // Replicates the behavior of the original cron system - global_schedule string = '0 3' -} - -// cmd returns the cli module that handles the cron daemon. -pub fn cmd() cli.Command { - return cli.Command{ - name: 'cron' - description: 'Start the cron service that periodically runs builds.' - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - - cron(conf)! - } - } -} diff --git a/src/cron/cron.v b/src/cron/cron.v deleted file mode 100644 index f1d6b7b..0000000 --- a/src/cron/cron.v +++ /dev/null @@ -1,33 +0,0 @@ -module cron - -import log -import cron.daemon -import cron.expression -import os - -const log_file_name = 'vieter.cron.log' - -// cron starts a cron daemon & starts periodically scheduling builds. -pub fn cron(conf Config) ! { - // Configure logger - log_level := log.level_from_tag(conf.log_level) or { - return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') - } - - mut logger := log.Log{ - level: log_level - } - - log_file := os.join_path_single(conf.data_dir, cron.log_file_name) - logger.set_full_logpath(log_file) - logger.log_to_console_too() - - ce := expression.parse_expression(conf.global_schedule) or { - return error('Error while parsing global cron expression: $err.msg()') - } - - mut d := daemon.init_daemon(logger, conf.address, conf.api_key, conf.base_image, ce, - conf.max_concurrent_builds, conf.api_update_frequency, conf.image_rebuild_frequency)! - - d.run() -} diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v deleted file mode 100644 index 42edc92..0000000 --- a/src/cron/daemon/build.v +++ /dev/null @@ -1,115 +0,0 @@ -module daemon - -import time -import sync.stdatomic -import build -import os - -const ( - build_empty = 0 - build_running = 1 - build_done = 2 -) - -// clean_finished_builds removes finished builds from the build slots & returns -// them. -fn (mut d Daemon) clean_finished_builds() []ScheduledBuild { - mut out := []ScheduledBuild{} - - for i in 0 .. d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == daemon.build_done { - stdatomic.store_u64(&d.atomics[i], daemon.build_empty) - out << d.builds[i] - } - } - - return out -} - -// update_builds starts as many builds as possible. -fn (mut d Daemon) start_new_builds() { - now := time.now() - - for d.queue.len() > 0 { - elem := d.queue.peek() or { - d.lerror("queue.peek() unexpectedly returned an error. This shouldn't happen.") - - break - } - - if elem.timestamp < now { - sb := d.queue.pop() or { - d.lerror("queue.pop() unexpectedly returned an error. This shouldn't happen.") - - break - } - - // If this build couldn't be scheduled, no more will be possible. - if !d.start_build(sb) { - d.queue.insert(sb) - break - } - } else { - break - } - } -} - -// start_build starts a build for the given ScheduledBuild object. -fn (mut d Daemon) start_build(sb ScheduledBuild) bool { - for i in 0 .. d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == daemon.build_empty { - stdatomic.store_u64(&d.atomics[i], daemon.build_running) - d.builds[i] = sb - - go d.run_build(i, sb) - - return true - } - } - - return false -} - -// run_build actually starts the build process for a given target. -fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { - d.linfo('started build: $sb.target.url -> $sb.target.repo') - - // 0 means success, 1 means failure - mut status := 0 - - res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(), - &sb.target, false) or { - d.ldebug('build_target error: $err.msg()') - status = 1 - - build.BuildResult{} - } - - if status == 0 { - d.linfo('finished build: $sb.target.url -> $sb.target.repo; uploading logs...') - - build_arch := os.uname().machine - d.client.add_build_log(sb.target.id, res.start_time, res.end_time, build_arch, - res.exit_code, res.logs) or { - d.lerror('Failed to upload logs for build: $sb.target.url -> $sb.target.repo') - } - } else { - d.linfo('an error occured during build: $sb.target.url -> $sb.target.repo') - } - - stdatomic.store_u64(&d.atomics[build_index], daemon.build_done) -} - -// current_build_count returns how many builds are currently running. -fn (mut d Daemon) current_build_count() int { - mut res := 0 - - for i in 0 .. d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == daemon.build_running { - res += 1 - } - } - - return res -} diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v deleted file mode 100644 index 0d30a23..0000000 --- a/src/cron/daemon/daemon.v +++ /dev/null @@ -1,274 +0,0 @@ -module daemon - -import time -import log -import datatypes { MinHeap } -import cron.expression { CronExpression, parse_expression } -import math -import build -import docker -import os -import client -import models { Target } - -const ( - // How many seconds to wait before retrying to update API if failed - api_update_retry_timeout = 5 - // How many seconds to wait before retrying to rebuild image if failed - rebuild_base_image_retry_timout = 30 -) - -struct ScheduledBuild { -pub: - target Target - timestamp time.Time -} - -// Overloaded operator for comparing ScheduledBuild objects -fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool { - return r1.timestamp < r2.timestamp -} - -pub struct Daemon { -mut: - client client.Client - base_image string - builder_images []string - global_schedule CronExpression - api_update_frequency int - image_rebuild_frequency int - // Targets currently loaded from API. - targets []Target - // At what point to update the list of targets. - api_update_timestamp time.Time - image_build_timestamp time.Time - queue MinHeap - // Which builds are currently running - builds []ScheduledBuild - // Atomic variables used to detect when a build has finished; length is the - // same as builds - atomics []u64 - logger shared log.Log -} - -// init_daemon initializes a new Daemon object. It renews the targets & -// populates the build queue for the first time. -pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) !Daemon { - mut d := Daemon{ - client: client.new(address, api_key) - base_image: base_image - global_schedule: global_schedule - api_update_frequency: api_update_frequency - image_rebuild_frequency: image_rebuild_frequency - atomics: []u64{len: max_concurrent_builds} - builds: []ScheduledBuild{len: max_concurrent_builds} - logger: logger - } - - // Initialize the targets & queue - d.renew_targets() - d.renew_queue() - if !d.rebuild_base_image() { - return error('The base image failed to build. The Vieter cron daemon cannot run without an initial builder image.') - } - - return d -} - -// run starts the actual daemon process. It runs builds when possible & -// periodically refreshes the list of targets to ensure we stay in sync. -pub fn (mut d Daemon) run() { - for { - finished_builds := d.clean_finished_builds() - - // Update the API's contents if needed & renew the queue - if time.now() >= d.api_update_timestamp { - d.renew_targets() - d.renew_queue() - } - // The finished builds should only be rescheduled if the API contents - // haven't been renewed. - else { - for sb in finished_builds { - d.schedule_build(sb.target) - } - } - - // TODO remove old builder images. - // This issue is less trivial than it sounds, because a build could - // still be running when the image has to be rebuilt. That would - // prevent the image from being removed. Therefore, we will need to - // keep track of a list or something & remove an image once we have - // made sure it isn't being used anymore. - if time.now() >= d.image_build_timestamp { - d.rebuild_base_image() - // In theory, executing this function here allows an old builder - // image to exist for at most image_rebuild_frequency minutes. - d.clean_old_base_images() - } - - // Schedules new builds when possible - d.start_new_builds() - - // If there are builds currently running, the daemon should refresh - // every second to clean up any finished builds & start new ones. - mut delay := time.Duration(1 * time.second) - - // Sleep either until we have to refresh the targets or when the next - // build has to start, with a minimum of 1 second. - if d.current_build_count() == 0 { - now := time.now() - delay = d.api_update_timestamp - now - - if d.queue.len() > 0 { - elem := d.queue.peek() or { - d.lerror("queue.peek() unexpectedly returned an error. This shouldn't happen.") - - // This is just a fallback option. In theory, queue.peek() - // should *never* return an error or none, because we check - // its len beforehand. - time.sleep(1) - continue - } - - time_until_next_job := elem.timestamp - now - - delay = math.min(delay, time_until_next_job) - } - } - - // We sleep for at least one second. This is to prevent the program - // from looping agressively when a cronjob can be scheduled, but - // there's no spots free for it to be started. - delay = math.max(delay, 1 * time.second) - - d.ldebug('Sleeping for ${delay}...') - - time.sleep(delay) - } -} - -// schedule_build adds the next occurence of the given targets build to the -// queue. -fn (mut d Daemon) schedule_build(target Target) { - ce := if target.schedule != '' { - parse_expression(target.schedule) or { - // TODO This shouldn't return an error if the expression is empty. - d.lerror("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()") - - d.global_schedule - } - } else { - d.global_schedule - } - - // A target that can't be scheduled will just be skipped for now - timestamp := ce.next_from_now() or { - d.lerror("Couldn't calculate next timestamp from '$target.schedule'; skipping") - return - } - - d.queue.insert(ScheduledBuild{ - target: target - timestamp: timestamp - }) -} - -// renew_targets requests the newest list of targets from the server & replaces -// the old one. -fn (mut d Daemon) renew_targets() { - d.linfo('Renewing targets...') - - mut new_targets := d.client.get_all_targets() or { - d.lerror('Failed to renew targets. Retrying in ${daemon.api_update_retry_timeout}s...') - d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) - - return - } - - // Filter out any targets that shouldn't run on this architecture - cur_arch := os.uname().machine - new_targets = new_targets.filter(it.arch.any(it.value == cur_arch)) - - d.targets = new_targets - - d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency) -} - -// renew_queue replaces the old queue with a new one that reflects the newest -// values in targets. -fn (mut d Daemon) renew_queue() { - d.linfo('Renewing queue...') - mut new_queue := MinHeap{} - - // Move any jobs that should have already started from the old queue onto - // the new one - now := time.now() - - // For some reason, using - // ```v - // for d.queue.len() > 0 && d.queue.peek() !.timestamp < now { - //``` - // here causes the function to prematurely just exit, without any errors or anything, very weird - // https://github.com/vlang/v/issues/14042 - for d.queue.len() > 0 { - elem := d.queue.pop() or { - d.lerror("queue.pop() returned an error. This shouldn't happen.") - continue - } - - if elem.timestamp < now { - new_queue.insert(elem) - } else { - break - } - } - - d.queue = new_queue - - // For each target in targets, parse their cron expression (or use the - // default one if not present) & add them to the queue - for target in d.targets { - d.schedule_build(target) - } -} - -// rebuild_base_image recreates the builder image. -fn (mut d Daemon) rebuild_base_image() bool { - d.linfo('Rebuilding builder image....') - - d.builder_images << build.create_build_image(d.base_image) or { - d.lerror('Failed to rebuild base image. Retrying in ${daemon.rebuild_base_image_retry_timout}s...') - d.image_build_timestamp = time.now().add_seconds(daemon.rebuild_base_image_retry_timout) - - return false - } - - d.image_build_timestamp = time.now().add_seconds(60 * d.image_rebuild_frequency) - - return true -} - -// clean_old_base_images tries to remove any old but still present builder -// images. -fn (mut d Daemon) clean_old_base_images() { - mut i := 0 - - mut dd := docker.new_conn() or { - d.lerror('Failed to connect to Docker socket.') - return - } - - defer { - dd.close() or {} - } - - for i < d.builder_images.len - 1 { - // For each builder image, we try to remove it by calling the Docker - // API. If the function returns an error or false, that means the image - // wasn't deleted. Therefore, we move the index over. If the function - // returns true, the array's length has decreased by one so we don't - // move the index. - dd.remove_image(d.builder_images[i]) or { i += 1 } - } -} diff --git a/src/cron/daemon/log.v b/src/cron/daemon/log.v deleted file mode 100644 index 95a50e7..0000000 --- a/src/cron/daemon/log.v +++ /dev/null @@ -1,35 +0,0 @@ -module daemon - -import log - -// log reate a log message with the given level -pub fn (mut d Daemon) log(msg string, level log.Level) { - lock d.logger { - d.logger.send_output(msg, level) - } -} - -// lfatal create a log message with the fatal level -pub fn (mut d Daemon) lfatal(msg string) { - d.log(msg, log.Level.fatal) -} - -// lerror create a log message with the error level -pub fn (mut d Daemon) lerror(msg string) { - d.log(msg, log.Level.error) -} - -// lwarn create a log message with the warn level -pub fn (mut d Daemon) lwarn(msg string) { - d.log(msg, log.Level.warn) -} - -// linfo create a log message with the info level -pub fn (mut d Daemon) linfo(msg string) { - d.log(msg, log.Level.info) -} - -// ldebug create a log message with the debug level -pub fn (mut d Daemon) ldebug(msg string) { - d.log(msg, log.Level.debug) -} diff --git a/src/cron/expression.c.v b/src/cron/expression.c.v new file mode 100644 index 0000000..e9686d6 --- /dev/null +++ b/src/cron/expression.c.v @@ -0,0 +1,101 @@ +module cron + +#flag -I @VMODROOT/libvieter/include +#flag -L @VMODROOT/libvieter/build +#flag -lvieter +#include "vieter_cron.h" + +[typedef] +pub struct C.vieter_cron_expression { + minutes &u8 + hours &u8 + days &u8 + months &u8 + minute_count u8 + hour_count u8 + day_count u8 + month_count u8 +} + +pub type Expression = C.vieter_cron_expression + +// == returns whether the two expressions are equal by value. +fn (ce1 Expression) == (ce2 Expression) bool { + if ce1.month_count != ce2.month_count || ce1.day_count != ce2.day_count + || ce1.hour_count != ce2.hour_count || ce1.minute_count != ce2.minute_count { + return false + } + + for i in 0 .. ce1.month_count { + unsafe { + if ce1.months[i] != ce2.months[i] { + return false + } + } + } + for i in 0 .. ce1.day_count { + unsafe { + if ce1.days[i] != ce2.days[i] { + return false + } + } + } + for i in 0 .. ce1.hour_count { + unsafe { + if ce1.hours[i] != ce2.hours[i] { + return false + } + } + } + for i in 0 .. ce1.minute_count { + unsafe { + if ce1.minutes[i] != ce2.minutes[i] { + return false + } + } + } + + return true +} + +[typedef] +struct C.vieter_cron_simple_time { + year int + month int + day int + hour int + minute int +} + +type SimpleTime = C.vieter_cron_simple_time + +enum ParseError as u8 { + ok = 0 + invalid_expression = 1 + invalid_number = 2 + out_of_range = 3 + too_many_parts = 4 + not_enough_parts = 5 +} + +// str returns the string representation of a ParseError. +fn (e ParseError) str() string { + return match e { + .ok { '' } + .invalid_expression { 'Invalid expression' } + .invalid_number { 'Invalid number' } + .out_of_range { 'Out of range' } + .too_many_parts { 'Too many parts' } + .not_enough_parts { 'Not enough parts' } + } +} + +fn C.vieter_cron_expr_init() &C.vieter_cron_expression + +fn C.vieter_cron_expr_free(ce &C.vieter_cron_expression) + +fn C.vieter_cron_expr_next(out &C.vieter_cron_simple_time, ce &C.vieter_cron_expression, ref &C.vieter_cron_simple_time) + +fn C.vieter_cron_expr_next_from_now(out &C.vieter_cron_simple_time, ce &C.vieter_cron_expression) + +fn C.vieter_cron_expr_parse(out &C.vieter_cron_expression, s &char) ParseError diff --git a/src/cron/expression.v b/src/cron/expression.v new file mode 100644 index 0000000..62692fa --- /dev/null +++ b/src/cron/expression.v @@ -0,0 +1,73 @@ +module cron + +import time + +// free the memory associated with the Expression. +[unsafe] +pub fn (ce &Expression) free() { + C.vieter_cron_expr_free(ce) +} + +// parse_expression parses a string into an Expression. +pub fn parse_expression(exp string) !&Expression { + out := C.vieter_cron_expr_init() + res := C.vieter_cron_expr_parse(out, exp.str) + + if res != .ok { + return error(res.str()) + } + + return out +} + +// next calculates the next occurence of the cron schedule, given a reference +// point. +pub fn (ce &Expression) next(ref time.Time) time.Time { + st := SimpleTime{ + year: ref.year + month: ref.month + day: ref.day + hour: ref.hour + minute: ref.minute + } + + out := SimpleTime{} + C.vieter_cron_expr_next(&out, ce, &st) + + return time.new_time(time.Time{ + year: out.year + month: out.month + day: out.day + hour: out.hour + minute: out.minute + }) +} + +// next_from_now calculates the next occurence of the cron schedule with the +// current time as reference. +pub fn (ce &Expression) next_from_now() time.Time { + out := SimpleTime{} + C.vieter_cron_expr_next_from_now(&out, ce) + + return time.new_time(time.Time{ + year: out.year + month: out.month + day: out.day + hour: out.hour + minute: out.minute + }) +} + +// next_n returns the n next occurences of the expression, given a starting +// time. +pub fn (ce &Expression) next_n(ref time.Time, n int) []time.Time { + mut times := []time.Time{cap: n} + + times << ce.next(ref) + + for i in 1 .. n { + times << ce.next(times[i - 1]) + } + + return times +} diff --git a/src/cron/expression/expression.v b/src/cron/expression/expression.v deleted file mode 100644 index c3ff8c5..0000000 --- a/src/cron/expression/expression.v +++ /dev/null @@ -1,136 +0,0 @@ -module expression - -import time - -pub struct CronExpression { - minutes []int - hours []int - days []int - months []int -} - -// next calculates the earliest time this cron expression is valid. It will -// always pick a moment in the future, even if ref matches completely up to the -// minute. This function conciously does not take gap years into account. -pub fn (ce &CronExpression) next(ref time.Time) !time.Time { - // If the given ref matches the next cron occurence up to the minute, it - // will return that value. Because we always want to return a value in the - // future, we artifically shift the ref 60 seconds to make sure we always - // match in the future. A shift of 60 seconds is enough because the cron - // expression does not allow for accuracy smaller than one minute. - sref := ref - - // For all of these values, the rule is the following: if their value is - // the length of their respective array in the CronExpression object, that - // means we've looped back around. This means that the "bigger" value has - // to be incremented by one. For example, if the minutes have looped - // around, that means that the hour has to be incremented as well. - mut minute_index := 0 - mut hour_index := 0 - mut day_index := 0 - mut month_index := 0 - - // This chain is the same logic multiple times, namely that if a "bigger" - // value loops around, then the smaller value will always reset as well. - // For example, if we're going to a new day, the hour & minute will always - // be their smallest value again. - for month_index < ce.months.len && sref.month > ce.months[month_index] { - month_index++ - } - - if month_index < ce.months.len && sref.month == ce.months[month_index] { - for day_index < ce.days.len && sref.day > ce.days[day_index] { - day_index++ - } - - if day_index < ce.days.len && ce.days[day_index] == sref.day { - for hour_index < ce.hours.len && sref.hour > ce.hours[hour_index] { - hour_index++ - } - - if hour_index < ce.hours.len && ce.hours[hour_index] == sref.hour { - // Minute is the only value where we explicitely make sure we - // can't match sref's value exactly. This is to ensure we only - // return values in the future. - for minute_index < ce.minutes.len && sref.minute >= ce.minutes[minute_index] { - minute_index++ - } - } - } - } - - // Here, we increment the "bigger" values by one if the smaller ones loop - // around. The order is important, as it allows a sort-of waterfall effect - // to occur which updates all values if required. - if minute_index == ce.minutes.len && hour_index < ce.hours.len { - hour_index += 1 - } - - if hour_index == ce.hours.len && day_index < ce.days.len { - day_index += 1 - } - - if day_index == ce.days.len && month_index < ce.months.len { - month_index += 1 - } - - mut minute := ce.minutes[minute_index % ce.minutes.len] - mut hour := ce.hours[hour_index % ce.hours.len] - mut day := ce.days[day_index % ce.days.len] - - // Sometimes, we end up with a day that does not exist within the selected - // month, e.g. day 30 in February. When this occurs, we reset day back to - // the smallest value & loop over to the next month that does have this - // day. - if day > time.month_days[ce.months[month_index % ce.months.len] - 1] { - day = ce.days[0] - month_index += 1 - - for day > time.month_days[ce.months[month_index & ce.months.len] - 1] { - month_index += 1 - - // If for whatever reason the day value ends up being something - // that can't be scheduled in any month, we have to make sure we - // don't create an infinite loop. - if month_index == 2 * ce.months.len { - return error('No schedulable moment.') - } - } - } - - month := ce.months[month_index % ce.months.len] - mut year := sref.year - - // If the month loops over, we need to increment the year. - if month_index >= ce.months.len { - year++ - } - - return time.new_time(time.Time{ - year: year - month: month - day: day - minute: minute - hour: hour - }) -} - -// next_from_now returns the result of ce.next(ref) where ref is the result of -// time.now(). -pub fn (ce &CronExpression) next_from_now() !time.Time { - return ce.next(time.now()) -} - -// next_n returns the n next occurences of the expression, given a starting -// time. -pub fn (ce &CronExpression) next_n(ref time.Time, n int) ![]time.Time { - mut times := []time.Time{cap: n} - - times << ce.next(ref)! - - for i in 1 .. n { - times << ce.next(times[i - 1])! - } - - return times -} diff --git a/src/cron/expression/expression_parse.v b/src/cron/expression/expression_parse.v deleted file mode 100644 index 4aaec5b..0000000 --- a/src/cron/expression/expression_parse.v +++ /dev/null @@ -1,146 +0,0 @@ -module expression - -import bitfield - -// parse_range parses a given string into a range of sorted integers. Its -// result is a BitField with set bits for all numbers in the result. -fn parse_range(s string, min int, max int) !bitfield.BitField { - mut start := min - mut end := max - mut interval := 1 - mut bf := bitfield.new(max - min + 1) - - exps := s.split('/') - - if exps.len > 2 { - return error('Invalid expression.') - } - - if exps[0] != '*' { - dash_parts := exps[0].split('-') - - if dash_parts.len > 2 { - return error('Invalid expression.') - } - - start = dash_parts[0].int() - - // The builtin parsing functions return zero if the string can't be - // parsed into a number, so we have to explicitely check whether they - // actually entered zero or if it's an invalid number. - if start == 0 && dash_parts[0] != '0' { - return error('Invalid number.') - } - - // Check whether the start value is out of range - if start < min || start > max { - return error('Out of range.') - } - - if dash_parts.len == 2 { - end = dash_parts[1].int() - - if end == 0 && dash_parts[1] != '0' { - return error('Invalid number.') - } - - if end < start || end > max { - return error('Out of range.') - } - } - } - - if exps.len > 1 { - interval = exps[1].int() - - // interval being zero is always invalid, but we want to check why - // it's invalid for better error messages. - if interval == 0 { - if exps[1] != '0' { - return error('Invalid number.') - } else { - return error('Step size zero not allowed.') - } - } - - if interval > max - min { - return error('Step size too large.') - } - } - // Here, s solely consists of a number, so that's the only value we - // should return. - else if exps[0] != '*' && !exps[0].contains('-') { - bf.set_bit(start - min) - return bf - } - - for start <= end { - bf.set_bit(start - min) - start += interval - } - - return bf -} - -// bf_to_ints takes a BitField and converts it into the expected list of actual -// integers. -fn bf_to_ints(bf bitfield.BitField, min int) []int { - mut out := []int{} - - for i in 0 .. bf.get_size() { - if bf.get_bit(i) == 1 { - out << min + i - } - } - - return out -} - -// parse_part parses a given part of a cron expression & returns the -// corresponding array of ints. -fn parse_part(s string, min int, max int) ![]int { - mut bf := bitfield.new(max - min + 1) - - for range in s.split(',') { - bf2 := parse_range(range, min, max)! - bf = bitfield.bf_or(bf, bf2) - } - - return bf_to_ints(bf, min) -} - -// parse_expression parses an entire cron expression string into a -// CronExpression object, if possible. -pub fn parse_expression(exp string) !CronExpression { - // The filter allows for multiple spaces between parts - mut parts := exp.split(' ').filter(it != '') - - if parts.len < 2 || parts.len > 4 { - return error('Expression must contain between 2 and 4 space-separated parts.') - } - - // For ease of use, we allow the user to only specify as many parts as they - // need. - for parts.len < 4 { - parts << '*' - } - - mut part_results := [][]int{} - - mins := [0, 0, 1, 1] - maxs := [59, 23, 31, 12] - - // This for loop allows us to more clearly propagate the error to the user. - for i, min in mins { - part_results << parse_part(parts[i], min, maxs[i]) or { - return error('An error occurred with part $i: $err.msg()') - } - } - - return CronExpression{ - minutes: part_results[0] - hours: part_results[1] - days: part_results[2] - months: part_results[3] - } -} diff --git a/src/cron/expression/expression_parse_test.v b/src/cron/expression/expression_parse_test.v deleted file mode 100644 index 92e8291..0000000 --- a/src/cron/expression/expression_parse_test.v +++ /dev/null @@ -1,89 +0,0 @@ -module expression - -// parse_range_error returns the returned error message. If the result is '', -// that means the function didn't error. -fn parse_range_error(s string, min int, max int) string { - parse_range(s, min, max) or { return err.msg } - - return '' -} - -// =====parse_range===== -fn test_range_star_range() ! { - bf := parse_range('*', 0, 5)! - - assert bf_to_ints(bf, 0) == [0, 1, 2, 3, 4, 5] -} - -fn test_range_number() ! { - bf := parse_range('4', 0, 5)! - - assert bf_to_ints(bf, 0) == [4] -} - -fn test_range_number_too_large() ! { - assert parse_range_error('10', 0, 6) == 'Out of range.' -} - -fn test_range_number_too_small() ! { - assert parse_range_error('0', 2, 6) == 'Out of range.' -} - -fn test_range_number_invalid() ! { - assert parse_range_error('x', 0, 6) == 'Invalid number.' -} - -fn test_range_step_star_1() ! { - bf := parse_range('*/4', 0, 20)! - - assert bf_to_ints(bf, 0) == [0, 4, 8, 12, 16, 20] -} - -fn test_range_step_star_2() ! { - bf := parse_range('*/3', 1, 8)! - - assert bf_to_ints(bf, 1) == [1, 4, 7] -} - -fn test_range_step_star_too_large() ! { - assert parse_range_error('*/21', 0, 20) == 'Step size too large.' -} - -fn test_range_step_zero() ! { - assert parse_range_error('*/0', 0, 20) == 'Step size zero not allowed.' -} - -fn test_range_step_number() ! { - bf := parse_range('5/4', 2, 22)! - - assert bf_to_ints(bf, 2) == [5, 9, 13, 17, 21] -} - -fn test_range_step_number_too_large() ! { - assert parse_range_error('10/4', 0, 5) == 'Out of range.' -} - -fn test_range_step_number_too_small() ! { - assert parse_range_error('2/4', 5, 10) == 'Out of range.' -} - -fn test_range_dash() ! { - bf := parse_range('4-8', 0, 9)! - - assert bf_to_ints(bf, 0) == [4, 5, 6, 7, 8] -} - -fn test_range_dash_step() ! { - bf := parse_range('4-8/2', 0, 9)! - - assert bf_to_ints(bf, 0) == [4, 6, 8] -} - -// =====parse_part===== -fn test_part_single() ! { - assert parse_part('*', 0, 5)! == [0, 1, 2, 3, 4, 5] -} - -fn test_part_multiple() ! { - assert parse_part('*/2,2/3', 1, 8)! == [1, 2, 3, 5, 7, 8] -} diff --git a/src/cron/expression/expression_test.v b/src/cron/expression_test.v similarity index 56% rename from src/cron/expression/expression_test.v rename to src/cron/expression_test.v index 82bf959..c7065f8 100644 --- a/src/cron/expression/expression_test.v +++ b/src/cron/expression_test.v @@ -1,4 +1,4 @@ -module expression +module cron import time { parse } @@ -7,7 +7,7 @@ fn util_test_time(exp string, t1_str string, t2_str string) ! { t1 := parse(t1_str)! t2 := parse(t2_str)! - t3 := ce.next(t1)! + t3 := ce.next(t1) assert t2.year == t3.year assert t2.month == t3.month @@ -18,17 +18,18 @@ fn util_test_time(exp string, t1_str string, t2_str string) ! { fn test_next_simple() ! { // Very simple - util_test_time('0 3', '2002-01-01 00:00:00', '2002-01-01 03:00:00')! + // util_test_time('0 3', '2002-01-01 00:00:00', '2002-01-01 03:00:00')! // Overlap to next day - util_test_time('0 3', '2002-01-01 03:00:00', '2002-01-02 03:00:00')! - util_test_time('0 3', '2002-01-01 04:00:00', '2002-01-02 03:00:00')! + mut exp := '0 3 ' + util_test_time(exp, '2002-01-01 03:00:00', '2002-01-02 03:00:00')! + util_test_time(exp, '2002-01-01 04:00:00', '2002-01-02 03:00:00')! - util_test_time('0 3/4', '2002-01-01 04:00:00', '2002-01-01 07:00:00')! + util_test_time('0 3-7/4,7-19', '2002-01-01 04:00:00', '2002-01-01 07:00:00')! - // Overlap to next month + //// Overlap to next month util_test_time('0 3', '2002-11-31 04:00:00', '2002-12-01 03:00:00')! - // Overlap to next year + //// Overlap to next year util_test_time('0 3', '2002-12-31 04:00:00', '2003-01-01 03:00:00')! } diff --git a/src/cron/parse_test.v b/src/cron/parse_test.v new file mode 100644 index 0000000..0dce7c2 --- /dev/null +++ b/src/cron/parse_test.v @@ -0,0 +1,42 @@ +module cron + +fn test_not_allowed() { + illegal_expressions := [ + '4 *-7', + '4 *-7/4', + '4 7/*', + '0 0 30 2', + '0 /5', + '0 ', + '0', + ' 0', + ' 0 ', + '1 2 3 4~9', + '1 1-3-5', + '0 5/2-5', + '', + '1 1/2/3', + '*5 8', + 'x 8', + ] + + mut res := false + + for exp in illegal_expressions { + res = false + parse_expression(exp) or { res = true } + assert res, "'$exp' should produce an error" + } +} + +fn test_auto_extend() ! { + ce1 := parse_expression('5 5')! + ce2 := parse_expression('5 5 *')! + ce3 := parse_expression('5 5 * *')! + + assert ce1 == ce2 && ce2 == ce3 +} + +fn test_four() { + parse_expression('0 1 2 3 ') or { assert false } +} diff --git a/src/db/targets.v b/src/db/targets.v index 41e56df..2644f49 100644 --- a/src/db/targets.v +++ b/src/db/targets.v @@ -1,25 +1,6 @@ module db -import models { Target, TargetArch, TargetFilter } - -// get_targets returns all targets in the database. -pub fn (db &VieterDb) get_targets(filter TargetFilter) []Target { - // This seems to currently be blocked by a bug in the ORM, I'll have to ask - // around. - if filter.repo != '' { - res := sql db.conn { - select from Target where repo == filter.repo order by id limit filter.limit offset filter.offset - } - - return res - } - - res := sql db.conn { - select from Target order by id limit filter.limit offset filter.offset - } - - return res -} +import models { Target, TargetArch } // get_target tries to return a specific target. pub fn (db &VieterDb) get_target(target_id int) ?Target { diff --git a/src/db/targets_iter.v b/src/db/targets_iter.v new file mode 100644 index 0000000..081de1f --- /dev/null +++ b/src/db/targets_iter.v @@ -0,0 +1,129 @@ +module db + +import models { Target, TargetFilter } +import sqlite + +// Iterator providing a filtered view into the list of targets currently stored +// in the database. It replaces functionality usually performed in the database +// using SQL queries that can't currently be used due to missing stuff in V's +// ORM. +pub struct TargetsIterator { + conn sqlite.DB + filter TargetFilter + window_size int = 32 +mut: + window []Target + window_index u64 + // Offset in entire list of unfiltered targets + offset int + // Offset in filtered list of targets + filtered_offset u64 + started bool + done bool +} + +// targets returns an iterator allowing filtered access to the list of targets. +pub fn (db &VieterDb) targets(filter TargetFilter) TargetsIterator { + window_size := 32 + + return TargetsIterator{ + conn: db.conn + filter: filter + window: []Target{cap: window_size} + window_size: window_size + } +} + +// advance_window moves the sliding window over the filtered list of targets +// until it either reaches the end of the list of targets, or has encountered a +// non-empty window. +fn (mut ti TargetsIterator) advance_window() { + for { + ti.window = sql ti.conn { + select from Target order by id limit ti.window_size offset ti.offset + } + ti.offset += ti.window.len + + if ti.window.len == 0 { + ti.done = true + + return + } + + if ti.filter.repo != '' { + ti.window = ti.window.filter(it.repo == ti.filter.repo) + } + + if ti.filter.arch != '' { + ti.window = ti.window.filter(it.arch.any(it.value == ti.filter.arch)) + } + + if ti.filter.query != '' { + ti.window = ti.window.filter(it.url.contains(ti.filter.query) + || it.path.contains(ti.filter.query) || it.branch.contains(ti.filter.query)) + } + + // We break out of the loop once we found a non-empty window + if ti.window.len > 0 { + break + } + } +} + +// next returns the next target, if possible. +pub fn (mut ti TargetsIterator) next() ?Target { + if ti.done { + return none + } + + // The first call to `next` will cause the sliding window to move to where + // the requested offset starts + if !ti.started { + ti.advance_window() + + // Skip all matched targets until the requested offset + for !ti.done && ti.filtered_offset + u64(ti.window.len) <= ti.filter.offset { + ti.filtered_offset += u64(ti.window.len) + ti.advance_window() + } + + if ti.done { + return none + } + + left_inside_window := ti.filter.offset - ti.filtered_offset + ti.window_index = left_inside_window + ti.filtered_offset += left_inside_window + + ti.started = true + } + + return_value := ti.window[ti.window_index] + + ti.window_index++ + ti.filtered_offset++ + + // Next call will be past the requested offset + if ti.filter.limit > 0 && ti.filtered_offset == ti.filter.offset + ti.filter.limit { + ti.done = true + } + + // Ensure the next call has a new valid window + if ti.window_index == u64(ti.window.len) { + ti.advance_window() + ti.window_index = 0 + } + + return return_value +} + +// collect consumes the entire iterator & returns the result as an array. +pub fn (mut ti TargetsIterator) collect() []Target { + mut out := []Target{} + + for t in ti { + out << t + } + + return out +} diff --git a/src/libvieter b/src/libvieter new file mode 160000 index 0000000..379a05a --- /dev/null +++ b/src/libvieter @@ -0,0 +1 @@ +Subproject commit 379a05a7b6b604c107360e0a679fb3ea5400e02c diff --git a/src/main.v b/src/main.v index fe0364f..ce9ec81 100644 --- a/src/main.v +++ b/src/main.v @@ -8,7 +8,7 @@ import console.logs import console.schedule import console.man import console.aur -import cron +import console.repos import agent fn main() { @@ -20,7 +20,8 @@ fn main() { mut app := cli.Command{ name: 'vieter' description: 'Vieter is a lightweight implementation of an Arch repository server.' - version: '0.4.0' + version: '0.5.0' + posix_mode: true flags: [ cli.Flag{ flag: cli.FlagType.string @@ -41,12 +42,12 @@ fn main() { commands: [ server.cmd(), targets.cmd(), - cron.cmd(), logs.cmd(), schedule.cmd(), man.cmd(), aur.cmd(), agent.cmd(), + repos.cmd(), ] } app.setup() diff --git a/src/models/logs.v b/src/models/logs.v index 12907d8..66a3a0a 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -1,6 +1,7 @@ module models import time +import os pub struct BuildLog { pub mut: @@ -28,6 +29,13 @@ pub fn (bl &BuildLog) str() string { return str } +// path returns the path to the log file, relative to the logs directory +pub fn (bl &BuildLog) path() string { + filename := bl.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + + return os.join_path(bl.target_id.str(), bl.arch, filename) +} + [params] pub struct BuildLogFilter { pub mut: diff --git a/src/models/targets.v b/src/models/targets.v index af3cb0d..a0c88d0 100644 --- a/src/models/targets.v +++ b/src/models/targets.v @@ -73,4 +73,6 @@ pub mut: limit u64 = 25 offset u64 repo string + query string + arch string } diff --git a/src/server/api_jobs.v b/src/server/api_jobs.v index 7795351..62bcb27 100644 --- a/src/server/api_jobs.v +++ b/src/server/api_jobs.v @@ -4,7 +4,7 @@ import web import web.response { new_data_response, new_response } // v1_poll_job_queue allows agents to poll for new build jobs. -['/api/v1/jobs/poll'; auth; get] +['/api/v1/jobs/poll'; auth; get; markused] fn (mut app App) v1_poll_job_queue() web.Result { arch := app.query['arch'] or { return app.json(.bad_request, new_response('Missing arch query arg.')) @@ -21,7 +21,7 @@ fn (mut app App) v1_poll_job_queue() web.Result { } // v1_queue_job allows queueing a new one-time build job for the given target. -['/api/v1/jobs/queue'; auth; post] +['/api/v1/jobs/queue'; auth; markused; post] fn (mut app App) v1_queue_job() web.Result { target_id := app.query['target'] or { return app.json(.bad_request, new_response('Missing target query arg.')) diff --git a/src/server/api_logs.v b/src/server/api_logs.v index c7521dd..3db4204 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -11,7 +11,7 @@ import models { BuildLog, BuildLogFilter } // v1_get_logs returns all build logs in the database. A 'target' query param can // optionally be added to limit the list of build logs to that repository. -['/api/v1/logs'; auth; get] +['/api/v1/logs'; auth; get; markused] fn (mut app App) v1_get_logs() web.Result { filter := models.from_params(app.query) or { return app.json(.bad_request, new_response('Invalid query parameters.')) @@ -22,7 +22,7 @@ fn (mut app App) v1_get_logs() web.Result { } // v1_get_single_log returns the build log with the given id. -['/api/v1/logs/:id'; auth; get] +['/api/v1/logs/:id'; auth; get; markused] fn (mut app App) v1_get_single_log(id int) web.Result { log := app.db.get_build_log(id) or { return app.status(.not_found) } @@ -30,7 +30,7 @@ fn (mut app App) v1_get_single_log(id int) web.Result { } // v1_get_log_content returns the actual build log file for the given id. -['/api/v1/logs/:id/content'; auth; get] +['/api/v1/logs/:id/content'; auth; get; markused] fn (mut app App) v1_get_log_content(id int) web.Result { log := app.db.get_build_log(id) or { return app.status(.not_found) } file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') @@ -50,7 +50,7 @@ fn parse_query_time(query string) !time.Time { } // v1_post_log adds a new log to the database. -['/api/v1/logs'; auth; post] +['/api/v1/logs'; auth; markused; post] fn (mut app App) v1_post_log() web.Result { // Parse query params start_time_int := app.query['startTime'].int() @@ -86,7 +86,7 @@ fn (mut app App) v1_post_log() web.Result { } // Store log in db - log := BuildLog{ + mut log := BuildLog{ target_id: target_id start_time: start_time end_time: end_time @@ -95,25 +95,20 @@ fn (mut app App) v1_post_log() web.Result { } // id of newly created log - log_id := app.db.add_build_log(log) - - repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, target_id.str(), arch) + log.id = app.db.add_build_log(log) + log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) // Create the logs directory of it doesn't exist - if !os.exists(repo_logs_dir) { - os.mkdir_all(repo_logs_dir) or { - app.lerror("Couldn't create dir '$repo_logs_dir'.") + if !os.exists(os.dir(log_file_path)) { + os.mkdir_all(os.dir(log_file_path)) or { + app.lerror('Error while creating log file: $err.msg()') return app.status(.internal_server_error) } } - // Stream log contents to correct file - file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss') - full_path := os.join_path_single(repo_logs_dir, file_name) - if length := app.req.header.get(.content_length) { - util.reader_to_file(mut app.reader, length.int(), full_path) or { + util.reader_to_file(mut app.reader, length.int(), log_file_path) or { app.lerror('An error occured while receiving logs: $err.msg()') return app.status(.internal_server_error) @@ -122,5 +117,22 @@ fn (mut app App) v1_post_log() web.Result { return app.status(.length_required) } - return app.json(.ok, new_data_response(log_id)) + return app.json(.ok, new_data_response(log.id)) +} + +// v1_delete_log allows removing a build log from the system. +['/api/v1/logs/:id'; auth; delete; markused] +fn (mut app App) v1_delete_log(id int) web.Result { + log := app.db.get_build_log(id) or { return app.status(.not_found) } + full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) + + os.rm(full_path) or { + app.lerror('Failed to remove log file $full_path: $err.msg()') + + return app.status(.internal_server_error) + } + + app.db.delete_build_log(id) + + return app.status(.ok) } diff --git a/src/server/api_metrics.v b/src/server/api_metrics.v new file mode 100644 index 0000000..5ba0452 --- /dev/null +++ b/src/server/api_metrics.v @@ -0,0 +1,19 @@ +module server + +import metrics +import web + +// v1_metrics serves a Prometheus-compatible metrics endpoint. +['/api/v1/metrics'; get; markused] +fn (mut app App) v1_metrics() web.Result { + if !app.conf.collect_metrics { + return app.status(.not_found) + } + + mut exporter := metrics.new_prometheus_exporter() + exporter.load('vieter_', app.collector) + + // TODO stream to connection instead + body := exporter.export_to_string() or { return app.status(.internal_server_error) } + return app.body(.ok, 'text/plain', body) +} diff --git a/src/server/api_targets.v b/src/server/api_targets.v index cd5cb0a..f47467a 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -6,18 +6,18 @@ import db import models { Target, TargetArch, TargetFilter } // v1_get_targets returns the current list of targets. -['/api/v1/targets'; auth; get] +['/api/v1/targets'; auth; get; markused] fn (mut app App) v1_get_targets() web.Result { filter := models.from_params(app.query) or { return app.json(.bad_request, new_response('Invalid query parameters.')) } - targets := app.db.get_targets(filter) + mut iter := app.db.targets(filter) - return app.json(.ok, new_data_response(targets)) + return app.json(.ok, new_data_response(iter.collect())) } // v1_get_single_target returns the information for a single target. -['/api/v1/targets/:id'; auth; get] +['/api/v1/targets/:id'; auth; get; markused] fn (mut app App) v1_get_single_target(id int) web.Result { target := app.db.get_target(id) or { return app.status(.not_found) } @@ -25,7 +25,7 @@ fn (mut app App) v1_get_single_target(id int) web.Result { } // v1_post_target creates a new target from the provided query string. -['/api/v1/targets'; auth; post] +['/api/v1/targets'; auth; markused; post] fn (mut app App) v1_post_target() web.Result { mut params := app.query.clone() @@ -55,7 +55,7 @@ fn (mut app App) v1_post_target() web.Result { } // v1_delete_target removes a given target from the server's list. -['/api/v1/targets/:id'; auth; delete] +['/api/v1/targets/:id'; auth; delete; markused] fn (mut app App) v1_delete_target(id int) web.Result { app.db.delete_target(id) app.job_queue.invalidate(id) @@ -64,7 +64,7 @@ fn (mut app App) v1_delete_target(id int) web.Result { } // v1_patch_target updates a target's data with the given query params. -['/api/v1/targets/:id'; auth; patch] +['/api/v1/targets/:id'; auth; markused; patch] fn (mut app App) v1_patch_target(id int) web.Result { app.db.update_target(id, app.query) diff --git a/src/server/cli.v b/src/server/cli.v index 2fede6c..9a8b144 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -5,14 +5,17 @@ import conf as vconf struct Config { pub: - log_level string = 'WARN' - pkg_dir string - data_dir string - api_key string - default_arch string - global_schedule string = '0 3' - port int = 8000 - base_image string = 'archlinux:base-devel' + port int = 8000 + log_level string = 'WARN' + pkg_dir string + data_dir string + api_key string + default_arch string + global_schedule string = '0 3' + base_image string = 'archlinux:base-devel' + max_log_age int [empty_default] + log_removal_schedule string = '0 0' + collect_metrics bool [empty_default] } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/log_removal.v b/src/server/log_removal.v new file mode 100644 index 0000000..27dc0db --- /dev/null +++ b/src/server/log_removal.v @@ -0,0 +1,53 @@ +module server + +import time +import models { BuildLog } +import os +import cron + +const fallback_log_removal_frequency = 24 * time.hour + +// log_removal_daemon removes old build logs every `log_removal_frequency`. +fn (mut app App) log_removal_daemon(schedule &cron.Expression) { + for { + mut too_old_timestamp := time.now().add_days(-app.conf.max_log_age) + + app.linfo('Cleaning logs before $too_old_timestamp') + + mut logs := []BuildLog{} + mut counter := 0 + mut failed := u64(0) + + // Remove old logs + for { + // The offset is used to skip logs that failed to remove. Besides + // this, we don't need to move the offset, because all previously + // oldest logs will have been removed. + logs = app.db.get_build_logs(before: too_old_timestamp, offset: failed, limit: 50) + + for log in logs { + log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) + + os.rm(log_file_path) or { + app.lerror('Failed to remove log file $log_file_path: $err.msg()') + failed += 1 + + continue + } + app.db.delete_build_log(log.id) + + counter += 1 + } + + if logs.len < 50 { + break + } + } + + app.linfo('Cleaned $counter logs ($failed failed)') + + // Sleep until the next cycle + next_time := schedule.next_from_now() + time.sleep(next_time - time.now()) + } +} diff --git a/src/server/repo.v b/src/server/repo.v index 06ab72e..38d07fe 100644 --- a/src/server/repo.v +++ b/src/server/repo.v @@ -10,7 +10,7 @@ import web.response { new_data_response, new_response } // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. -['/health'; get] +['/health'; get; markused] pub fn (mut app App) healthcheck() web.Result { return app.json(.ok, new_response('Healthy.')) } @@ -18,7 +18,7 @@ pub fn (mut app App) healthcheck() web.Result { // get_repo_file handles all Pacman-related routes. It returns both the // repository's archives, but also package archives or the contents of a // package's desc file. -['/:repo/:arch/:filename'; get; head] +['/:repo/:arch/:filename'; get; head; markused] fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result { mut full_path := '' @@ -48,7 +48,7 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re } // put_package handles publishing a package to a repository. -['/:repo/publish'; auth; post] +['/:repo/publish'; auth; markused; post] fn (mut app App) put_package(repo string) web.Result { // api is a reserved keyword for api routes & should never be allowed to be // a repository. diff --git a/src/server/repo_remove.v b/src/server/repo_remove.v index 694f085..9e6d747 100644 --- a/src/server/repo_remove.v +++ b/src/server/repo_remove.v @@ -3,7 +3,7 @@ module server import web // delete_package tries to remove the given package. -['/:repo/:arch/:pkg'; auth; delete] +['/:repo/:arch/:pkg'; auth; delete; markused] fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result { res := app.repo.remove_pkg_from_arch_repo(repo, arch, pkg, true) or { app.lerror('Error while deleting package: $err.msg()') @@ -23,7 +23,7 @@ fn (mut app App) delete_package(repo string, arch string, pkg string) web.Result } // delete_arch_repo tries to remove the given arch-repo. -['/:repo/:arch'; auth; delete] +['/:repo/:arch'; auth; delete; markused] fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { res := app.repo.remove_arch_repo(repo, arch) or { app.lerror('Error while deleting arch-repo: $err.msg()') @@ -43,7 +43,7 @@ fn (mut app App) delete_arch_repo(repo string, arch string) web.Result { } // delete_repo tries to remove the given repo. -['/:repo'; auth; delete] +['/:repo'; auth; delete; markused] fn (mut app App) delete_repo(repo string) web.Result { res := app.repo.remove_repo(repo) or { app.lerror('Error while deleting repo: $err.msg()') diff --git a/src/server/server.v b/src/server/server.v index 74b1f37..c6bdd1a 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -7,7 +7,8 @@ import repo import util import db import build { BuildJobQueue } -import cron.expression +import cron +import metrics const ( log_file_name = 'vieter.log' @@ -30,17 +31,8 @@ pub mut: // init_job_queue populates a fresh job queue with all the targets currently // stored in the database. fn (mut app App) init_job_queue() ! { - // Initialize build queues - mut targets := app.db.get_targets(limit: 25) - mut i := u64(0) - - for targets.len > 0 { - for target in targets { - app.job_queue.insert_all(target)! - } - - i += 25 - targets = app.db.get_targets(limit: 25, offset: i) + for target in app.db.targets(limit: 0) { + app.job_queue.insert_all(target)! } } @@ -51,10 +43,14 @@ pub fn server(conf Config) ! { util.exit_with_message(1, "'any' is not allowed as the value for default_arch.") } - global_ce := expression.parse_expression(conf.global_schedule) or { + global_ce := cron.parse_expression(conf.global_schedule) or { util.exit_with_message(1, 'Invalid global cron expression: $err.msg()') } + log_removal_ce := cron.parse_expression(conf.log_removal_schedule) or { + util.exit_with_message(1, 'Invalid log removal cron expression: $err.msg()') + } + // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -96,17 +92,31 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Failed to initialize database: $err.msg()') } + mut collector := if conf.collect_metrics { + &metrics.MetricsCollector(metrics.new_default_collector()) + } else { + &metrics.MetricsCollector(metrics.new_null_collector()) + } + + collector.histogram_buckets_set('http_requests_duration_seconds', [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, + 10] ) + mut app := &App{ logger: logger api_key: conf.api_key conf: conf repo: repo db: db + collector: collector job_queue: build.new_job_queue(global_ce, conf.base_image) } app.init_job_queue() or { util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()') } + if conf.max_log_age > 0 { + go app.log_removal_daemon(log_removal_ce) + } + web.run(app, conf.port) } diff --git a/src/v.mod b/src/v.mod index 710c976..461af6a 100644 --- a/src/v.mod +++ b/src/v.mod @@ -2,6 +2,7 @@ Module { dependencies: [ 'https://git.rustybever.be/vieter-v/conf', 'https://git.rustybever.be/vieter-v/docker', - 'https://git.rustybever.be/vieter-v/aur' + 'https://git.rustybever.be/vieter-v/aur', + 'https://git.rustybever.be/vieter-v/metrics' ] } diff --git a/src/web/logging.v b/src/web/logging.v index 12b07d7..7ba649c 100644 --- a/src/web/logging.v +++ b/src/web/logging.v @@ -1,35 +1,36 @@ module web -import log - -// log reate a log message with the given level -pub fn (mut ctx Context) log(msg string, level log.Level) { - lock ctx.logger { - ctx.logger.send_output(msg, level) - } -} - // lfatal create a log message with the fatal level pub fn (mut ctx Context) lfatal(msg string) { - ctx.log(msg, log.Level.fatal) + lock ctx.logger { + ctx.logger.fatal(msg) + } } // lerror create a log message with the error level pub fn (mut ctx Context) lerror(msg string) { - ctx.log(msg, log.Level.error) + lock ctx.logger { + ctx.logger.error(msg) + } } // lwarn create a log message with the warn level pub fn (mut ctx Context) lwarn(msg string) { - ctx.log(msg, log.Level.warn) + lock ctx.logger { + ctx.logger.warn(msg) + } } // linfo create a log message with the info level pub fn (mut ctx Context) linfo(msg string) { - ctx.log(msg, log.Level.info) + lock ctx.logger { + ctx.logger.info(msg) + } } // ldebug create a log message with the debug level pub fn (mut ctx Context) ldebug(msg string) { - ctx.log(msg, log.Level.debug) + lock ctx.logger { + ctx.logger.debug(msg) + } } diff --git a/src/web/parse.v b/src/web/parse.v index 7af635f..889944b 100644 --- a/src/web/parse.v +++ b/src/web/parse.v @@ -5,7 +5,7 @@ import net.http // Method attributes that should be ignored when parsing, as they're used // elsewhere. -const attrs_to_ignore = ['auth'] +const attrs_to_ignore = ['auth', 'markused'] // Parsing function attributes for methods and path. fn parse_attrs(name string, attrs []string) !([]http.Method, string) { diff --git a/src/web/web.v b/src/web/web.v index 565baff..f0f3523 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -11,6 +11,7 @@ import net.urllib import time import json import log +import metrics // The Context struct represents the Context which hold the HTTP request and response. // It has fields for the query, form, files. @@ -27,6 +28,8 @@ pub mut: conn &net.TcpConn = unsafe { nil } // Gives access to a shared logger object logger shared log.Log + // Used to collect metrics on the web server + collector &metrics.MetricsCollector // time.ticks() from start of web connection handle. // You can use it to determine how much time is spent on your request. page_gen_start i64 @@ -145,6 +148,15 @@ pub fn (ctx &Context) is_authenticated() bool { return false } +// body sends the given body as an HTTP response. +pub fn (mut ctx Context) body(status http.Status, content_type string, body string) Result { + ctx.status = status + ctx.content_type = content_type + ctx.send_response(body) + + return Result{} +} + // json HTTP_OK with json_s as payload with content-type `application/json` pub fn (mut ctx Context) json(status http.Status, j T) Result { ctx.status = status @@ -319,6 +331,22 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { app.logger.flush() } + // Record how long request took to process + labels := [ + ['method', app.req.method.str()]!, + ['path', app.req.url]!, + // Not all methods properly set this value yet I think + ['status', app.status.int().str()]!, + ] + app.collector.counter_increment(name: 'http_requests_total', labels: labels) + // Prometheus prefers metrics containing base units, as defined here + // https://prometheus.io/docs/practices/naming/ + app.collector.histogram_record(f64(time.ticks() - app.page_gen_start) / 1000, + + name: 'http_requests_duration_seconds' + labels: labels + ) + unsafe { free(app) } @@ -384,6 +412,7 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { static_mime_types: app.static_mime_types reader: reader logger: app.logger + collector: app.collector api_key: app.api_key } diff --git a/vieter.toml b/vieter.toml index 74a7397..34b4f4e 100644 --- a/vieter.toml +++ b/vieter.toml @@ -12,3 +12,6 @@ address = "http://localhost:8000" api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 +# max_log_age = 64 +log_removal_schedule = '* * *' +collect_metrics = true