forked from vieter-v/vieter
Merge pull request 'Release 0.5.0: release candidate 2' (#320) from release-0.5.0-rc.2 into main
Reviewed-on: vieter-v/vieter#320web-stuff 0.5.0-rc.2
commit
dc517c23c5
11
CHANGELOG.md
11
CHANGELOG.md
|
@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||||
|
|
||||||
## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
|
## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
|
||||||
|
|
||||||
|
## [0.5.0-rc.2](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.2)
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
* API route for removing logs & accompanying CLI command
|
||||||
|
* Daemon for periodically removing old logs
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
* Use `--long-option` instead of `-long-option` for CLI
|
||||||
|
|
||||||
## [0.5.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.1)
|
## [0.5.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.1)
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
2
PKGBUILD
2
PKGBUILD
|
@ -3,7 +3,7 @@
|
||||||
|
|
||||||
pkgbase='vieter'
|
pkgbase='vieter'
|
||||||
pkgname='vieter'
|
pkgname='vieter'
|
||||||
pkgver='0.5.0-rc.1'
|
pkgver='0.5.0_rc.2'
|
||||||
pkgrel=1
|
pkgrel=1
|
||||||
pkgdesc="Lightweight Arch repository server & package build system"
|
pkgdesc="Lightweight Arch repository server & package build system"
|
||||||
depends=('glibc' 'openssl' 'libarchive' 'sqlite')
|
depends=('glibc' 'openssl' 'libarchive' 'sqlite')
|
||||||
|
|
|
@ -0,0 +1,78 @@
|
||||||
|
# Jobs
|
||||||
|
|
||||||
|
<aside class="notice">
|
||||||
|
|
||||||
|
All routes in this section require authentication.
|
||||||
|
|
||||||
|
</aside>
|
||||||
|
|
||||||
|
## Manually schedule a job
|
||||||
|
|
||||||
|
```shell
|
||||||
|
curl \
|
||||||
|
-H 'X-Api-Key: secret' \
|
||||||
|
https://example.com/api/v1/jobs/queue?target=10&force&arch=x86_64
|
||||||
|
```
|
||||||
|
|
||||||
|
Manually schedule a job on the server.
|
||||||
|
|
||||||
|
### HTTP Request
|
||||||
|
|
||||||
|
`POST /api/v1/jobs/queue`
|
||||||
|
|
||||||
|
### Query Parameters
|
||||||
|
|
||||||
|
Parameter | Description
|
||||||
|
--------- | -----------
|
||||||
|
target | Id of target to schedule build for
|
||||||
|
arch | Architecture to build on
|
||||||
|
force | Whether it's a forced build (true if present)
|
||||||
|
|
||||||
|
## Poll for new jobs
|
||||||
|
|
||||||
|
<aside class="warning">
|
||||||
|
|
||||||
|
This endpoint is used by the agents and should not be used manually. It's just
|
||||||
|
here for completeness. Requests to this endpoint modify the build queue,
|
||||||
|
meaning manual requests can cause builds to be skipped.
|
||||||
|
|
||||||
|
</aside>
|
||||||
|
|
||||||
|
```shell
|
||||||
|
curl \
|
||||||
|
-H 'X-Api-Key: secret' \
|
||||||
|
https://example.com/api/v1/jobs/poll?arch=x86_64&max=2
|
||||||
|
```
|
||||||
|
|
||||||
|
> JSON output format
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"message": "",
|
||||||
|
"data": [
|
||||||
|
{
|
||||||
|
"target_id": 1,
|
||||||
|
"kind": "git",
|
||||||
|
"url": "https://aur.archlinux.org/discord-ptb.git",
|
||||||
|
"branch": "master",
|
||||||
|
"path": "",
|
||||||
|
"repo": "bur",
|
||||||
|
"base_image": "archlinux:base-devel",
|
||||||
|
"force": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Poll the server for new builds.
|
||||||
|
|
||||||
|
### HTTP Request
|
||||||
|
|
||||||
|
`GET /api/v1/jobs/poll`
|
||||||
|
|
||||||
|
### Query Parameters
|
||||||
|
|
||||||
|
Parameter | Description
|
||||||
|
--------- | -----------
|
||||||
|
arch | For which architecture to receive jobs
|
||||||
|
max | How many jobs to receive at most
|
|
@ -125,8 +125,8 @@ id | ID of requested log
|
||||||
|
|
||||||
<aside class="warning">
|
<aside class="warning">
|
||||||
|
|
||||||
You should probably not use this endpoint, as it's used by the build system to
|
This endpoint is used by the agents and should not be used manually unless you
|
||||||
publish its logs.
|
know what you're doing. It's just here for completeness.
|
||||||
|
|
||||||
</aside>
|
</aside>
|
||||||
|
|
||||||
|
@ -149,3 +149,24 @@ target | id of target this build is for
|
||||||
### Request body
|
### Request body
|
||||||
|
|
||||||
Plaintext contents of the build log.
|
Plaintext contents of the build log.
|
||||||
|
|
||||||
|
## Remove a build log
|
||||||
|
|
||||||
|
```shell
|
||||||
|
curl \
|
||||||
|
-XDELETE \
|
||||||
|
-H 'X-Api-Key: secret' \
|
||||||
|
https://example.com/api/v1/logs/1
|
||||||
|
```
|
||||||
|
|
||||||
|
Remove a build log from the server.
|
||||||
|
|
||||||
|
### HTTP Request
|
||||||
|
|
||||||
|
`DELETE /api/v1/logs/:id`
|
||||||
|
|
||||||
|
### URL Parameters
|
||||||
|
|
||||||
|
Parameter | Description
|
||||||
|
--------- | -----------
|
||||||
|
id | id of log to remove
|
||||||
|
|
|
@ -27,6 +27,7 @@ curl \
|
||||||
"kind": "git",
|
"kind": "git",
|
||||||
"url": "https://aur.archlinux.org/discord-ptb.git",
|
"url": "https://aur.archlinux.org/discord-ptb.git",
|
||||||
"branch": "master",
|
"branch": "master",
|
||||||
|
"path" : "",
|
||||||
"repo": "bur",
|
"repo": "bur",
|
||||||
"schedule": "",
|
"schedule": "",
|
||||||
"arch": [
|
"arch": [
|
||||||
|
@ -73,8 +74,9 @@ curl \
|
||||||
"kind": "git",
|
"kind": "git",
|
||||||
"url": "https://aur.archlinux.org/discord-ptb.git",
|
"url": "https://aur.archlinux.org/discord-ptb.git",
|
||||||
"branch": "master",
|
"branch": "master",
|
||||||
|
"path": "",
|
||||||
"repo": "bur",
|
"repo": "bur",
|
||||||
"schedule": "0 3",
|
"schedule": "0 2",
|
||||||
"arch": [
|
"arch": [
|
||||||
{
|
{
|
||||||
"id": 1,
|
"id": 1,
|
||||||
|
@ -124,6 +126,7 @@ Parameter | Description
|
||||||
kind | Kind of target to add; one of 'git', 'url'.
|
kind | Kind of target to add; one of 'git', 'url'.
|
||||||
url | URL of the Git repository.
|
url | URL of the Git repository.
|
||||||
branch | Branch of the Git repository.
|
branch | Branch of the Git repository.
|
||||||
|
path | Subdirectory inside Git repository to use.
|
||||||
repo | Vieter repository to publish built packages to.
|
repo | Vieter repository to publish built packages to.
|
||||||
schedule | Cron build schedule (syntax explained [here](https://rustybever.be/docs/vieter/usage/builds/schedule/))
|
schedule | Cron build schedule (syntax explained [here](https://rustybever.be/docs/vieter/usage/builds/schedule/))
|
||||||
arch | Comma-separated list of architectures to build package on.
|
arch | Comma-separated list of architectures to build package on.
|
||||||
|
@ -149,12 +152,20 @@ Parameter | Description
|
||||||
kind | Kind of target; one of 'git', 'url'.
|
kind | Kind of target; one of 'git', 'url'.
|
||||||
url | URL of the Git repository.
|
url | URL of the Git repository.
|
||||||
branch | Branch of the Git repository.
|
branch | Branch of the Git repository.
|
||||||
|
path | Subdirectory inside Git repository to use.
|
||||||
repo | Vieter repository to publish built packages to.
|
repo | Vieter repository to publish built packages to.
|
||||||
schedule | Cron build schedule
|
schedule | Cron build schedule
|
||||||
arch | Comma-separated list of architectures to build package on.
|
arch | Comma-separated list of architectures to build package on.
|
||||||
|
|
||||||
## Remove a target
|
## Remove a target
|
||||||
|
|
||||||
|
```shell
|
||||||
|
curl \
|
||||||
|
-XDELETE \
|
||||||
|
-H 'X-Api-Key: secret' \
|
||||||
|
https://example.com/api/v1/targets/1
|
||||||
|
```
|
||||||
|
|
||||||
Remove a target from the server.
|
Remove a target from the server.
|
||||||
|
|
||||||
### HTTP Request
|
### HTTP Request
|
||||||
|
|
|
@ -11,6 +11,7 @@ includes:
|
||||||
- repository
|
- repository
|
||||||
- targets
|
- targets
|
||||||
- logs
|
- logs
|
||||||
|
- jobs
|
||||||
|
|
||||||
search: true
|
search: true
|
||||||
|
|
||||||
|
|
|
@ -32,11 +32,11 @@ configuration variable required for each command.
|
||||||
|
|
||||||
### `vieter server`
|
### `vieter server`
|
||||||
|
|
||||||
|
* `port`: HTTP port to run on
|
||||||
|
* Default: `8000`
|
||||||
* `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`,
|
* `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`,
|
||||||
`WARN`, `INFO` or `DEBUG`.
|
`WARN`, `INFO` or `DEBUG`.
|
||||||
* Default: `WARN`
|
* Default: `WARN`
|
||||||
* `log_file`: log file to write logs to.
|
|
||||||
* Default: `vieter.log` (in the current directory)
|
|
||||||
* `pkg_dir`: where Vieter should store the actual package archives.
|
* `pkg_dir`: where Vieter should store the actual package archives.
|
||||||
* `data_dir`: where Vieter stores the repositories, log file & database.
|
* `data_dir`: where Vieter stores the repositories, log file & database.
|
||||||
* `api_key`: the API key to use when authenticating requests.
|
* `api_key`: the API key to use when authenticating requests.
|
||||||
|
@ -44,9 +44,27 @@ configuration variable required for each command.
|
||||||
* Packages with architecture `any` are always added to this architecture.
|
* Packages with architecture `any` are always added to this architecture.
|
||||||
This prevents the server from being confused when an `any` package is
|
This prevents the server from being confused when an `any` package is
|
||||||
published as the very first package for a repository.
|
published as the very first package for a repository.
|
||||||
* Git repositories added without an `arch` value use this value instead.
|
* Targets added without an `arch` value use this value instead.
|
||||||
* `port`: HTTP port to run on
|
* `global_schedule`: build schedule for any target that does not have a
|
||||||
* Default: `8000`
|
schedule defined. For information about this syntax, see
|
||||||
|
[here](/usage/builds/schedule).
|
||||||
|
* Default: `0 3` (3AM every night)
|
||||||
|
* `base_image`: Docker image to use when building a package. Any Pacman-based
|
||||||
|
distro image should work, as long as `/etc/pacman.conf` is used &
|
||||||
|
`base-devel` exists in the repositories. Make sure that the image supports
|
||||||
|
the architecture of your cron daemon.
|
||||||
|
* Default: `archlinux:base-devel` (only works on `x86_64`). If you require
|
||||||
|
`aarch64` support, consider using
|
||||||
|
[`menci/archlinuxarm:base-devel`](https://hub.docker.com/r/menci/archlinuxarm)
|
||||||
|
([GitHub](https://github.com/Menci/docker-archlinuxarm)). This is the
|
||||||
|
image used for the Vieter CI builds.
|
||||||
|
* `max_log_age`: maximum age of logs (in days). Logs older than this will get
|
||||||
|
cleaned by the log removal daemon. If set to a negative value, no logs are
|
||||||
|
ever removed. The age of logs is determined by the time the build was
|
||||||
|
started.
|
||||||
|
* Default: `-1`
|
||||||
|
* `log_removal_schedule`: cron schedule defining when to clean old logs.
|
||||||
|
* Default: `0 0` (every day at midnight)
|
||||||
|
|
||||||
### `vieter cron`
|
### `vieter cron`
|
||||||
|
|
||||||
|
|
|
@ -23,15 +23,15 @@ guarantees about stability, so beware!
|
||||||
Thanks to the single-binary design of Vieter, this image can be used both for
|
Thanks to the single-binary design of Vieter, this image can be used both for
|
||||||
the repository server, the cron daemon and the agent.
|
the repository server, the cron daemon and the agent.
|
||||||
|
|
||||||
Below is an example compose file to set up both the repository server & the
|
Below is a minimal compose file to set up both the repository server & a build
|
||||||
cron daemon:
|
agent:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
version: '3'
|
version: '3'
|
||||||
|
|
||||||
services:
|
services:
|
||||||
server:
|
server:
|
||||||
image: 'chewingbever/vieter:dev'
|
image: 'chewingbever/vieter:0.5.0-rc.1'
|
||||||
restart: 'always'
|
restart: 'always'
|
||||||
|
|
||||||
environment:
|
environment:
|
||||||
|
@ -41,18 +41,19 @@ services:
|
||||||
- 'data:/data'
|
- 'data:/data'
|
||||||
|
|
||||||
cron:
|
cron:
|
||||||
image: 'chewingbever/vieter:dev'
|
image: 'chewingbever/vieter:0.5.0-rc.1'
|
||||||
restart: 'always'
|
restart: 'always'
|
||||||
|
# Required to connect to the Docker daemon
|
||||||
user: root
|
user: root
|
||||||
command: 'vieter cron'
|
command: 'vieter agent'
|
||||||
|
|
||||||
environment:
|
environment:
|
||||||
- 'VIETER_API_KEY=secret'
|
- 'VIETER_API_KEY=secret'
|
||||||
# MUST be public URL of Vieter repository
|
# MUST be public URL of Vieter repository
|
||||||
- 'VIETER_ADDRESS=https://example.com'
|
- 'VIETER_ADDRESS=https://example.com'
|
||||||
- 'VIETER_DEFAULT_ARCH=x86_64'
|
# Architecture for which the agent builds
|
||||||
|
- 'VIETER_ARCH=x86_64'
|
||||||
- 'VIETER_MAX_CONCURRENT_BUILDS=2'
|
- 'VIETER_MAX_CONCURRENT_BUILDS=2'
|
||||||
- 'VIETER_GLOBAL_SCHEDULE=0 3'
|
|
||||||
volumes:
|
volumes:
|
||||||
- '/var/run/docker.sock:/var/run/docker.sock'
|
- '/var/run/docker.sock:/var/run/docker.sock'
|
||||||
|
|
||||||
|
@ -63,14 +64,17 @@ volumes:
|
||||||
If you do not require the build system, the repository server can be used
|
If you do not require the build system, the repository server can be used
|
||||||
independently as well.
|
independently as well.
|
||||||
|
|
||||||
|
Of course, Vieter allows a lot more configuration than this. This compose file
|
||||||
|
is meant as a starting point for setting up your installation.
|
||||||
|
|
||||||
{{< hint info >}}
|
{{< hint info >}}
|
||||||
**Note**
|
**Note**
|
||||||
Builds are executed on the cron daemon's system using the host's Docker daemon.
|
Builds are executed on the agent's system using the host's Docker daemon. An
|
||||||
A cron daemon on a specific architecture will only build packages for that
|
agent for a specific `arch` will only build packages for that specific
|
||||||
specific architecture. Therefore, if you wish to build packages for both
|
architecture. Therefore, if you wish to build packages for both `x86_64` &
|
||||||
`x86_64` & `aarch64`, you'll have to deploy two cron daemons, one on each
|
`aarch64`, you'll have to deploy two agents, one on each architecture.
|
||||||
architecture. Afterwards, any Git repositories enabled for those two
|
Afterwards, any Git repositories enabled for those two architectures will build
|
||||||
architectures will build on both.
|
on both.
|
||||||
{{< /hint >}}
|
{{< /hint >}}
|
||||||
|
|
||||||
## Binary
|
## Binary
|
||||||
|
@ -99,9 +103,9 @@ latest official release or `vieter-git` for the latest development release.
|
||||||
### AUR
|
### AUR
|
||||||
|
|
||||||
If you prefer building the packages locally (or on your own Vieter instance),
|
If you prefer building the packages locally (or on your own Vieter instance),
|
||||||
there's the `[vieter](https://aur.archlinux.org/packages/vieter)` &
|
there's the [`vieter`](https://aur.archlinux.org/packages/vieter) &
|
||||||
`[vieter-git](https://aur.archlinux.org/packages/vieter-git)` packages on the
|
[`vieter-git`](https://aur.archlinux.org/packages/vieter-git) packages on the
|
||||||
AUR. These packages build using the `vlang-git` compiler package, so I can't
|
AUR. These packages build using the `vlang` compiler package, so I can't
|
||||||
guarantee that a compiler update won't temporarily break them.
|
guarantee that a compiler update won't temporarily break them.
|
||||||
|
|
||||||
## Building from source
|
## Building from source
|
||||||
|
|
|
@ -0,0 +1,23 @@
|
||||||
|
---
|
||||||
|
weight: 20
|
||||||
|
---
|
||||||
|
|
||||||
|
# Cleanup
|
||||||
|
|
||||||
|
Vieter stores the logs of every single package build. While this is great for
|
||||||
|
debugging why builds fail, it also causes an active or long-running Vieter
|
||||||
|
instance to accumulate thousands of logs.
|
||||||
|
|
||||||
|
To combat this, a log removal daemon can be enabled that periodically removes
|
||||||
|
old build logs. By starting your server with the `max_log_age` variable (see
|
||||||
|
[Configuration](/configuration#vieter-server)), a daemon will get enabled that
|
||||||
|
periodically removes logs older than this setting. By default, this will happen
|
||||||
|
every day at midnight, but this behavior can be changed using the
|
||||||
|
`log_removal_schedule` variable.
|
||||||
|
|
||||||
|
{{< hint info >}}
|
||||||
|
**Note**
|
||||||
|
The daemon will always run a removal of logs on startup. Therefore, it's
|
||||||
|
possible the daemon will be *very* active when first enabling this setting.
|
||||||
|
After the initial surge of logs to remove, it'll calm down again.
|
||||||
|
{{< /hint >}}
|
|
@ -1,3 +1,7 @@
|
||||||
|
---
|
||||||
|
weight: 10
|
||||||
|
---
|
||||||
|
|
||||||
# Cron schedule syntax
|
# Cron schedule syntax
|
||||||
|
|
||||||
The Vieter cron daemon uses a subset of the cron expression syntax to schedule
|
The Vieter cron daemon uses a subset of the cron expression syntax to schedule
|
||||||
|
|
|
@ -41,3 +41,8 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t
|
||||||
|
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// remove_build_log removes the build log with the given id from the server.
|
||||||
|
pub fn (c &Client) remove_build_log(id int) ! {
|
||||||
|
c.send_request<string>(.delete, '/api/v1/logs/$id', {})!
|
||||||
|
}
|
||||||
|
|
|
@ -138,6 +138,18 @@ pub fn cmd() cli.Command {
|
||||||
list(conf, filter, raw)!
|
list(conf, filter, raw)!
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
cli.Command{
|
||||||
|
name: 'remove'
|
||||||
|
required_args: 1
|
||||||
|
usage: 'id'
|
||||||
|
description: 'Remove a build log that matches the given id.'
|
||||||
|
execute: fn (cmd cli.Command) ! {
|
||||||
|
config_file := cmd.flags.get_string('config-file')!
|
||||||
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||||
|
|
||||||
|
remove(conf, cmd.args[0])!
|
||||||
|
}
|
||||||
|
},
|
||||||
cli.Command{
|
cli.Command{
|
||||||
name: 'info'
|
name: 'info'
|
||||||
required_args: 1
|
required_args: 1
|
||||||
|
@ -204,3 +216,9 @@ fn content(conf Config, id int) ! {
|
||||||
|
|
||||||
println(content)
|
println(content)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// remove removes a build log from the server's list.
|
||||||
|
fn remove(conf Config, id string) ! {
|
||||||
|
c := client.new(conf.address, conf.api_key)
|
||||||
|
c.remove_build_log(id.int())!
|
||||||
|
}
|
||||||
|
|
|
@ -20,7 +20,8 @@ fn main() {
|
||||||
mut app := cli.Command{
|
mut app := cli.Command{
|
||||||
name: 'vieter'
|
name: 'vieter'
|
||||||
description: 'Vieter is a lightweight implementation of an Arch repository server.'
|
description: 'Vieter is a lightweight implementation of an Arch repository server.'
|
||||||
version: '0.5.0-rc.1'
|
version: '0.5.0-rc.2'
|
||||||
|
posix_mode: true
|
||||||
flags: [
|
flags: [
|
||||||
cli.Flag{
|
cli.Flag{
|
||||||
flag: cli.FlagType.string
|
flag: cli.FlagType.string
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
module models
|
module models
|
||||||
|
|
||||||
import time
|
import time
|
||||||
|
import os
|
||||||
|
|
||||||
pub struct BuildLog {
|
pub struct BuildLog {
|
||||||
pub mut:
|
pub mut:
|
||||||
|
@ -28,6 +29,13 @@ pub fn (bl &BuildLog) str() string {
|
||||||
return str
|
return str
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// path returns the path to the log file, relative to the logs directory
|
||||||
|
pub fn (bl &BuildLog) path() string {
|
||||||
|
filename := bl.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
|
||||||
|
|
||||||
|
return os.join_path(bl.target_id.str(), bl.arch, filename)
|
||||||
|
}
|
||||||
|
|
||||||
[params]
|
[params]
|
||||||
pub struct BuildLogFilter {
|
pub struct BuildLogFilter {
|
||||||
pub mut:
|
pub mut:
|
||||||
|
|
|
@ -86,7 +86,7 @@ fn (mut app App) v1_post_log() web.Result {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store log in db
|
// Store log in db
|
||||||
log := BuildLog{
|
mut log := BuildLog{
|
||||||
target_id: target_id
|
target_id: target_id
|
||||||
start_time: start_time
|
start_time: start_time
|
||||||
end_time: end_time
|
end_time: end_time
|
||||||
|
@ -95,25 +95,20 @@ fn (mut app App) v1_post_log() web.Result {
|
||||||
}
|
}
|
||||||
|
|
||||||
// id of newly created log
|
// id of newly created log
|
||||||
log_id := app.db.add_build_log(log)
|
log.id = app.db.add_build_log(log)
|
||||||
|
log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
|
||||||
repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, target_id.str(), arch)
|
|
||||||
|
|
||||||
// Create the logs directory of it doesn't exist
|
// Create the logs directory of it doesn't exist
|
||||||
if !os.exists(repo_logs_dir) {
|
if !os.exists(os.dir(log_file_path)) {
|
||||||
os.mkdir_all(repo_logs_dir) or {
|
os.mkdir_all(os.dir(log_file_path)) or {
|
||||||
app.lerror("Couldn't create dir '$repo_logs_dir'.")
|
app.lerror('Error while creating log file: $err.msg()')
|
||||||
|
|
||||||
return app.status(.internal_server_error)
|
return app.status(.internal_server_error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stream log contents to correct file
|
|
||||||
file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
|
|
||||||
full_path := os.join_path_single(repo_logs_dir, file_name)
|
|
||||||
|
|
||||||
if length := app.req.header.get(.content_length) {
|
if length := app.req.header.get(.content_length) {
|
||||||
util.reader_to_file(mut app.reader, length.int(), full_path) or {
|
util.reader_to_file(mut app.reader, length.int(), log_file_path) or {
|
||||||
app.lerror('An error occured while receiving logs: $err.msg()')
|
app.lerror('An error occured while receiving logs: $err.msg()')
|
||||||
|
|
||||||
return app.status(.internal_server_error)
|
return app.status(.internal_server_error)
|
||||||
|
@ -122,5 +117,22 @@ fn (mut app App) v1_post_log() web.Result {
|
||||||
return app.status(.length_required)
|
return app.status(.length_required)
|
||||||
}
|
}
|
||||||
|
|
||||||
return app.json(.ok, new_data_response(log_id))
|
return app.json(.ok, new_data_response(log.id))
|
||||||
|
}
|
||||||
|
|
||||||
|
// v1_delete_log allows removing a build log from the system.
|
||||||
|
['/api/v1/logs/:id'; auth; delete]
|
||||||
|
fn (mut app App) v1_delete_log(id int) web.Result {
|
||||||
|
log := app.db.get_build_log(id) or { return app.status(.not_found) }
|
||||||
|
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
|
||||||
|
|
||||||
|
os.rm(full_path) or {
|
||||||
|
app.lerror('Failed to remove log file $full_path: $err.msg()')
|
||||||
|
|
||||||
|
return app.status(.internal_server_error)
|
||||||
|
}
|
||||||
|
|
||||||
|
app.db.delete_build_log(id)
|
||||||
|
|
||||||
|
return app.status(.ok)
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,14 +5,16 @@ import conf as vconf
|
||||||
|
|
||||||
struct Config {
|
struct Config {
|
||||||
pub:
|
pub:
|
||||||
log_level string = 'WARN'
|
port int = 8000
|
||||||
pkg_dir string
|
log_level string = 'WARN'
|
||||||
data_dir string
|
pkg_dir string
|
||||||
api_key string
|
data_dir string
|
||||||
default_arch string
|
api_key string
|
||||||
global_schedule string = '0 3'
|
default_arch string
|
||||||
port int = 8000
|
global_schedule string = '0 3'
|
||||||
base_image string = 'archlinux:base-devel'
|
base_image string = 'archlinux:base-devel'
|
||||||
|
max_log_age int = -1
|
||||||
|
log_removal_schedule string = '0 0'
|
||||||
}
|
}
|
||||||
|
|
||||||
// cmd returns the cli submodule that handles starting the server
|
// cmd returns the cli submodule that handles starting the server
|
||||||
|
|
|
@ -0,0 +1,62 @@
|
||||||
|
module server
|
||||||
|
|
||||||
|
import time
|
||||||
|
import models { BuildLog }
|
||||||
|
import os
|
||||||
|
import cron.expression { CronExpression }
|
||||||
|
|
||||||
|
const fallback_log_removal_frequency = 24 * time.hour
|
||||||
|
|
||||||
|
// log_removal_daemon removes old build logs every `log_removal_frequency`.
|
||||||
|
fn (mut app App) log_removal_daemon(schedule CronExpression) {
|
||||||
|
mut start_time := time.Time{}
|
||||||
|
|
||||||
|
for {
|
||||||
|
start_time = time.now()
|
||||||
|
|
||||||
|
mut too_old_timestamp := time.now().add_days(-app.conf.max_log_age)
|
||||||
|
|
||||||
|
app.linfo('Cleaning logs before $too_old_timestamp')
|
||||||
|
|
||||||
|
mut logs := []BuildLog{}
|
||||||
|
mut counter := 0
|
||||||
|
mut failed := u64(0)
|
||||||
|
|
||||||
|
// Remove old logs
|
||||||
|
for {
|
||||||
|
// The offset is used to skip logs that failed to remove. Besides
|
||||||
|
// this, we don't need to move the offset, because all previously
|
||||||
|
// oldest logs will have been removed.
|
||||||
|
logs = app.db.get_build_logs(before: too_old_timestamp, offset: failed, limit: 50)
|
||||||
|
|
||||||
|
for log in logs {
|
||||||
|
log_file_path := os.join_path(app.conf.data_dir, logs_dir_name, log.path())
|
||||||
|
|
||||||
|
os.rm(log_file_path) or {
|
||||||
|
app.lerror('Failed to remove log file $log_file_path: $err.msg()')
|
||||||
|
failed += 1
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
app.db.delete_build_log(log.id)
|
||||||
|
|
||||||
|
counter += 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if logs.len < 50 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
app.linfo('Cleaned $counter logs ($failed failed)')
|
||||||
|
|
||||||
|
// Sleep until the next cycle
|
||||||
|
next_time := schedule.next_from_now() or {
|
||||||
|
app.lerror("Log removal daemon couldn't calculate next time: $err.msg(); fallback to $server.fallback_log_removal_frequency")
|
||||||
|
|
||||||
|
start_time.add(server.fallback_log_removal_frequency)
|
||||||
|
}
|
||||||
|
|
||||||
|
time.sleep(next_time - time.now())
|
||||||
|
}
|
||||||
|
}
|
|
@ -55,6 +55,10 @@ pub fn server(conf Config) ! {
|
||||||
util.exit_with_message(1, 'Invalid global cron expression: $err.msg()')
|
util.exit_with_message(1, 'Invalid global cron expression: $err.msg()')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log_removal_ce := expression.parse_expression(conf.log_removal_schedule) or {
|
||||||
|
util.exit_with_message(1, 'Invalid log removal cron expression: $err.msg()')
|
||||||
|
}
|
||||||
|
|
||||||
// Configure logger
|
// Configure logger
|
||||||
log_level := log.level_from_tag(conf.log_level) or {
|
log_level := log.level_from_tag(conf.log_level) or {
|
||||||
util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
|
util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
|
||||||
|
@ -108,5 +112,9 @@ pub fn server(conf Config) ! {
|
||||||
util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()')
|
util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if conf.max_log_age > 0 {
|
||||||
|
go app.log_removal_daemon(log_removal_ce)
|
||||||
|
}
|
||||||
|
|
||||||
web.run(app, conf.port)
|
web.run(app, conf.port)
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,3 +12,4 @@ address = "http://localhost:8000"
|
||||||
api_update_frequency = 2
|
api_update_frequency = 2
|
||||||
image_rebuild_frequency = 1
|
image_rebuild_frequency = 1
|
||||||
max_concurrent_builds = 3
|
max_concurrent_builds = 3
|
||||||
|
max_log_age = 64
|
||||||
|
|
Loading…
Reference in New Issue