diff --git a/CHANGELOG.md b/CHANGELOG.md index e615698..27d9096 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,17 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev) -## [0.5.0-rc.2](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.2) - -### Added - -* API route for removing logs & accompanying CLI command -* Daemon for periodically removing old logs - -### Changed - -* Use `--long-option` instead of `-long-option` for CLI - ## [0.5.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.1) ### Added diff --git a/PKGBUILD b/PKGBUILD index 5e9530a..94db654 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -3,7 +3,7 @@ pkgbase='vieter' pkgname='vieter' -pkgver='0.5.0_rc.2' +pkgver='0.5.0-rc.1' pkgrel=1 pkgdesc="Lightweight Arch repository server & package build system" depends=('glibc' 'openssl' 'libarchive' 'sqlite') diff --git a/docs/api/source/includes/_jobs.md b/docs/api/source/includes/_jobs.md deleted file mode 100644 index a25309d..0000000 --- a/docs/api/source/includes/_jobs.md +++ /dev/null @@ -1,78 +0,0 @@ -# Jobs - - - -## Manually schedule a job - -```shell -curl \ - -H 'X-Api-Key: secret' \ - https://example.com/api/v1/jobs/queue?target=10&force&arch=x86_64 -``` - -Manually schedule a job on the server. - -### HTTP Request - -`POST /api/v1/jobs/queue` - -### Query Parameters - -Parameter | Description ---------- | ----------- -target | Id of target to schedule build for -arch | Architecture to build on -force | Whether it's a forced build (true if present) - -## Poll for new jobs - - - -```shell -curl \ - -H 'X-Api-Key: secret' \ - https://example.com/api/v1/jobs/poll?arch=x86_64&max=2 -``` - -> JSON output format - -```json -{ - "message": "", - "data": [ - { - "target_id": 1, - "kind": "git", - "url": "https://aur.archlinux.org/discord-ptb.git", - "branch": "master", - "path": "", - "repo": "bur", - "base_image": "archlinux:base-devel", - "force": true - } - ] -} -``` - -Poll the server for new builds. - -### HTTP Request - -`GET /api/v1/jobs/poll` - -### Query Parameters - -Parameter | Description ---------- | ----------- -arch | For which architecture to receive jobs -max | How many jobs to receive at most diff --git a/docs/api/source/includes/_logs.md b/docs/api/source/includes/_logs.md index d6134b7..1c14e71 100644 --- a/docs/api/source/includes/_logs.md +++ b/docs/api/source/includes/_logs.md @@ -125,8 +125,8 @@ id | ID of requested log @@ -149,24 +149,3 @@ target | id of target this build is for ### Request body Plaintext contents of the build log. - -## Remove a build log - -```shell -curl \ - -XDELETE \ - -H 'X-Api-Key: secret' \ - https://example.com/api/v1/logs/1 -``` - -Remove a build log from the server. - -### HTTP Request - -`DELETE /api/v1/logs/:id` - -### URL Parameters - -Parameter | Description ---------- | ----------- -id | id of log to remove diff --git a/docs/api/source/includes/_targets.md b/docs/api/source/includes/_targets.md index b71da84..93a4e86 100644 --- a/docs/api/source/includes/_targets.md +++ b/docs/api/source/includes/_targets.md @@ -27,7 +27,6 @@ curl \ "kind": "git", "url": "https://aur.archlinux.org/discord-ptb.git", "branch": "master", - "path" : "", "repo": "bur", "schedule": "", "arch": [ @@ -74,9 +73,8 @@ curl \ "kind": "git", "url": "https://aur.archlinux.org/discord-ptb.git", "branch": "master", - "path": "", "repo": "bur", - "schedule": "0 2", + "schedule": "0 3", "arch": [ { "id": 1, @@ -126,7 +124,6 @@ Parameter | Description kind | Kind of target to add; one of 'git', 'url'. url | URL of the Git repository. branch | Branch of the Git repository. -path | Subdirectory inside Git repository to use. repo | Vieter repository to publish built packages to. schedule | Cron build schedule (syntax explained [here](https://rustybever.be/docs/vieter/usage/builds/schedule/)) arch | Comma-separated list of architectures to build package on. @@ -152,20 +149,12 @@ Parameter | Description kind | Kind of target; one of 'git', 'url'. url | URL of the Git repository. branch | Branch of the Git repository. -path | Subdirectory inside Git repository to use. repo | Vieter repository to publish built packages to. schedule | Cron build schedule arch | Comma-separated list of architectures to build package on. ## Remove a target -```shell -curl \ - -XDELETE \ - -H 'X-Api-Key: secret' \ - https://example.com/api/v1/targets/1 -``` - Remove a target from the server. ### HTTP Request diff --git a/docs/api/source/index.html.md b/docs/api/source/index.html.md index f61e44a..4bfddb8 100644 --- a/docs/api/source/index.html.md +++ b/docs/api/source/index.html.md @@ -11,7 +11,6 @@ includes: - repository - targets - logs - - jobs search: true diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 45c5de6..95bf713 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -32,11 +32,11 @@ configuration variable required for each command. ### `vieter server` -* `port`: HTTP port to run on - * Default: `8000` * `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`, `WARN`, `INFO` or `DEBUG`. * Default: `WARN` +* `log_file`: log file to write logs to. + * Default: `vieter.log` (in the current directory) * `pkg_dir`: where Vieter should store the actual package archives. * `data_dir`: where Vieter stores the repositories, log file & database. * `api_key`: the API key to use when authenticating requests. @@ -44,27 +44,9 @@ configuration variable required for each command. * Packages with architecture `any` are always added to this architecture. This prevents the server from being confused when an `any` package is published as the very first package for a repository. - * Targets added without an `arch` value use this value instead. -* `global_schedule`: build schedule for any target that does not have a - schedule defined. For information about this syntax, see - [here](/usage/builds/schedule). - * Default: `0 3` (3AM every night) -* `base_image`: Docker image to use when building a package. Any Pacman-based - distro image should work, as long as `/etc/pacman.conf` is used & - `base-devel` exists in the repositories. Make sure that the image supports - the architecture of your cron daemon. - * Default: `archlinux:base-devel` (only works on `x86_64`). If you require - `aarch64` support, consider using - [`menci/archlinuxarm:base-devel`](https://hub.docker.com/r/menci/archlinuxarm) - ([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the - image used for the Vieter CI builds. -* `max_log_age`: maximum age of logs (in days). Logs older than this will get - cleaned by the log removal daemon. If set to a negative value, no logs are - ever removed. The age of logs is determined by the time the build was - started. - * Default: `-1` -* `log_removal_schedule`: cron schedule defining when to clean old logs. - * Default: `0 0` (every day at midnight) + * Git repositories added without an `arch` value use this value instead. +* `port`: HTTP port to run on + * Default: `8000` ### `vieter cron` diff --git a/docs/content/installation.md b/docs/content/installation.md index 5b8e2d8..21eda64 100644 --- a/docs/content/installation.md +++ b/docs/content/installation.md @@ -23,15 +23,15 @@ guarantees about stability, so beware! Thanks to the single-binary design of Vieter, this image can be used both for the repository server, the cron daemon and the agent. -Below is a minimal compose file to set up both the repository server & a build -agent: +Below is an example compose file to set up both the repository server & the +cron daemon: ```yaml version: '3' services: server: - image: 'chewingbever/vieter:0.5.0-rc.1' + image: 'chewingbever/vieter:dev' restart: 'always' environment: @@ -41,19 +41,18 @@ services: - 'data:/data' cron: - image: 'chewingbever/vieter:0.5.0-rc.1' + image: 'chewingbever/vieter:dev' restart: 'always' - # Required to connect to the Docker daemon user: root - command: 'vieter agent' + command: 'vieter cron' environment: - 'VIETER_API_KEY=secret' # MUST be public URL of Vieter repository - 'VIETER_ADDRESS=https://example.com' - # Architecture for which the agent builds - - 'VIETER_ARCH=x86_64' + - 'VIETER_DEFAULT_ARCH=x86_64' - 'VIETER_MAX_CONCURRENT_BUILDS=2' + - 'VIETER_GLOBAL_SCHEDULE=0 3' volumes: - '/var/run/docker.sock:/var/run/docker.sock' @@ -64,17 +63,14 @@ volumes: If you do not require the build system, the repository server can be used independently as well. -Of course, Vieter allows a lot more configuration than this. This compose file -is meant as a starting point for setting up your installation. - {{< hint info >}} **Note** -Builds are executed on the agent's system using the host's Docker daemon. An -agent for a specific `arch` will only build packages for that specific -architecture. Therefore, if you wish to build packages for both `x86_64` & -`aarch64`, you'll have to deploy two agents, one on each architecture. -Afterwards, any Git repositories enabled for those two architectures will build -on both. +Builds are executed on the cron daemon's system using the host's Docker daemon. +A cron daemon on a specific architecture will only build packages for that +specific architecture. Therefore, if you wish to build packages for both +`x86_64` & `aarch64`, you'll have to deploy two cron daemons, one on each +architecture. Afterwards, any Git repositories enabled for those two +architectures will build on both. {{< /hint >}} ## Binary @@ -103,9 +99,9 @@ latest official release or `vieter-git` for the latest development release. ### AUR If you prefer building the packages locally (or on your own Vieter instance), -there's the [`vieter`](https://aur.archlinux.org/packages/vieter) & -[`vieter-git`](https://aur.archlinux.org/packages/vieter-git) packages on the -AUR. These packages build using the `vlang` compiler package, so I can't +there's the `[vieter](https://aur.archlinux.org/packages/vieter)` & +`[vieter-git](https://aur.archlinux.org/packages/vieter-git)` packages on the +AUR. These packages build using the `vlang-git` compiler package, so I can't guarantee that a compiler update won't temporarily break them. ## Building from source diff --git a/docs/content/usage/builds/cleanup.md b/docs/content/usage/builds/cleanup.md deleted file mode 100644 index 724a75f..0000000 --- a/docs/content/usage/builds/cleanup.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -weight: 20 ---- - -# Cleanup - -Vieter stores the logs of every single package build. While this is great for -debugging why builds fail, it also causes an active or long-running Vieter -instance to accumulate thousands of logs. - -To combat this, a log removal daemon can be enabled that periodically removes -old build logs. By starting your server with the `max_log_age` variable (see -[Configuration](/configuration#vieter-server)), a daemon will get enabled that -periodically removes logs older than this setting. By default, this will happen -every day at midnight, but this behavior can be changed using the -`log_removal_schedule` variable. - -{{< hint info >}} -**Note** -The daemon will always run a removal of logs on startup. Therefore, it's -possible the daemon will be *very* active when first enabling this setting. -After the initial surge of logs to remove, it'll calm down again. -{{< /hint >}} diff --git a/docs/content/usage/builds/schedule.md b/docs/content/usage/builds/schedule.md index d3802fd..de59e25 100644 --- a/docs/content/usage/builds/schedule.md +++ b/docs/content/usage/builds/schedule.md @@ -1,7 +1,3 @@ ---- -weight: 10 ---- - # Cron schedule syntax The Vieter cron daemon uses a subset of the cron expression syntax to schedule diff --git a/src/client/logs.v b/src/client/logs.v index 2ddb2e2..85063bc 100644 --- a/src/client/logs.v +++ b/src/client/logs.v @@ -41,8 +41,3 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t return data } - -// remove_build_log removes the build log with the given id from the server. -pub fn (c &Client) remove_build_log(id int) ! { - c.send_request(.delete, '/api/v1/logs/$id', {})! -} diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v index 19c46f6..3064a58 100644 --- a/src/console/logs/logs.v +++ b/src/console/logs/logs.v @@ -138,18 +138,6 @@ pub fn cmd() cli.Command { list(conf, filter, raw)! } }, - cli.Command{ - name: 'remove' - required_args: 1 - usage: 'id' - description: 'Remove a build log that matches the given id.' - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - - remove(conf, cmd.args[0])! - } - }, cli.Command{ name: 'info' required_args: 1 @@ -216,9 +204,3 @@ fn content(conf Config, id int) ! { println(content) } - -// remove removes a build log from the server's list. -fn remove(conf Config, id string) ! { - c := client.new(conf.address, conf.api_key) - c.remove_build_log(id.int())! -} diff --git a/src/main.v b/src/main.v index eda38e7..1053c2f 100644 --- a/src/main.v +++ b/src/main.v @@ -20,8 +20,7 @@ fn main() { mut app := cli.Command{ name: 'vieter' description: 'Vieter is a lightweight implementation of an Arch repository server.' - version: '0.5.0-rc.2' - posix_mode: true + version: '0.5.0-rc.1' flags: [ cli.Flag{ flag: cli.FlagType.string diff --git a/src/models/logs.v b/src/models/logs.v index 66a3a0a..12907d8 100644 --- a/src/models/logs.v +++ b/src/models/logs.v @@ -1,7 +1,6 @@ module models import time -import os pub struct BuildLog { pub mut: @@ -29,13 +28,6 @@ pub fn (bl &BuildLog) str() string { return str } -// path returns the path to the log file, relative to the logs directory -pub fn (bl &BuildLog) path() string { - filename := bl.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') - - return os.join_path(bl.target_id.str(), bl.arch, filename) -} - [params] pub struct BuildLogFilter { pub mut: diff --git a/src/server/api_logs.v b/src/server/api_logs.v index 13b50b9..c7521dd 100644 --- a/src/server/api_logs.v +++ b/src/server/api_logs.v @@ -86,7 +86,7 @@ fn (mut app App) v1_post_log() web.Result { } // Store log in db - mut log := BuildLog{ + log := BuildLog{ target_id: target_id start_time: start_time end_time: end_time @@ -95,20 +95,25 @@ fn (mut app App) v1_post_log() web.Result { } // id of newly created log - log.id = app.db.add_build_log(log) - log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) + log_id := app.db.add_build_log(log) + + repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, target_id.str(), arch) // Create the logs directory of it doesn't exist - if !os.exists(os.dir(log_file_path)) { - os.mkdir_all(os.dir(log_file_path)) or { - app.lerror('Error while creating log file: $err.msg()') + if !os.exists(repo_logs_dir) { + os.mkdir_all(repo_logs_dir) or { + app.lerror("Couldn't create dir '$repo_logs_dir'.") return app.status(.internal_server_error) } } + // Stream log contents to correct file + file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + full_path := os.join_path_single(repo_logs_dir, file_name) + if length := app.req.header.get(.content_length) { - util.reader_to_file(mut app.reader, length.int(), log_file_path) or { + util.reader_to_file(mut app.reader, length.int(), full_path) or { app.lerror('An error occured while receiving logs: $err.msg()') return app.status(.internal_server_error) @@ -117,22 +122,5 @@ fn (mut app App) v1_post_log() web.Result { return app.status(.length_required) } - return app.json(.ok, new_data_response(log.id)) -} - -// v1_delete_log allows removing a build log from the system. -['/api/v1/logs/:id'; auth; delete] -fn (mut app App) v1_delete_log(id int) web.Result { - log := app.db.get_build_log(id) or { return app.status(.not_found) } - full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) - - os.rm(full_path) or { - app.lerror('Failed to remove log file $full_path: $err.msg()') - - return app.status(.internal_server_error) - } - - app.db.delete_build_log(id) - - return app.status(.ok) + return app.json(.ok, new_data_response(log_id)) } diff --git a/src/server/cli.v b/src/server/cli.v index 795f764..2fede6c 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -5,16 +5,14 @@ import conf as vconf struct Config { pub: - port int = 8000 - log_level string = 'WARN' - pkg_dir string - data_dir string - api_key string - default_arch string - global_schedule string = '0 3' - base_image string = 'archlinux:base-devel' - max_log_age int = -1 - log_removal_schedule string = '0 0' + log_level string = 'WARN' + pkg_dir string + data_dir string + api_key string + default_arch string + global_schedule string = '0 3' + port int = 8000 + base_image string = 'archlinux:base-devel' } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/log_removal.v b/src/server/log_removal.v deleted file mode 100644 index 8e1a8c2..0000000 --- a/src/server/log_removal.v +++ /dev/null @@ -1,62 +0,0 @@ -module server - -import time -import models { BuildLog } -import os -import cron.expression { CronExpression } - -const fallback_log_removal_frequency = 24 * time.hour - -// log_removal_daemon removes old build logs every `log_removal_frequency`. -fn (mut app App) log_removal_daemon(schedule CronExpression) { - mut start_time := time.Time{} - - for { - start_time = time.now() - - mut too_old_timestamp := time.now().add_days(-app.conf.max_log_age) - - app.linfo('Cleaning logs before $too_old_timestamp') - - mut logs := []BuildLog{} - mut counter := 0 - mut failed := u64(0) - - // Remove old logs - for { - // The offset is used to skip logs that failed to remove. Besides - // this, we don't need to move the offset, because all previously - // oldest logs will have been removed. - logs = app.db.get_build_logs(before: too_old_timestamp, offset: failed, limit: 50) - - for log in logs { - log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path()) - - os.rm(log_file_path) or { - app.lerror('Failed to remove log file $log_file_path: $err.msg()') - failed += 1 - - continue - } - app.db.delete_build_log(log.id) - - counter += 1 - } - - if logs.len < 50 { - break - } - } - - app.linfo('Cleaned $counter logs ($failed failed)') - - // Sleep until the next cycle - next_time := schedule.next_from_now() or { - app.lerror("Log removal daemon couldn't calculate next time: $err.msg(); fallback to $server.fallback_log_removal_frequency") - - start_time.add(server.fallback_log_removal_frequency) - } - - time.sleep(next_time - time.now()) - } -} diff --git a/src/server/server.v b/src/server/server.v index 178f657..74b1f37 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -55,10 +55,6 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Invalid global cron expression: $err.msg()') } - log_removal_ce := expression.parse_expression(conf.log_removal_schedule) or { - util.exit_with_message(1, 'Invalid log removal cron expression: $err.msg()') - } - // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -112,9 +108,5 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()') } - if conf.max_log_age > 0 { - go app.log_removal_daemon(log_removal_ce) - } - web.run(app, conf.port) } diff --git a/vieter.toml b/vieter.toml index 1f839f0..74a7397 100644 --- a/vieter.toml +++ b/vieter.toml @@ -12,4 +12,3 @@ address = "http://localhost:8000" api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 -max_log_age = 64