Compare commits

...

29 Commits

Author SHA1 Message Date
Jef Roosens 78fc3afcd3
feat(ci): also publish dev images as specific commit hash 2022-05-09 15:16:30 +02:00
Jef Roosens cae44fb593 Merge pull request 'integrate build logs API into build command & cron' (#171) from Chewing_Bever/vieter:build-logs into dev
Reviewed-on: vieter/vieter#171
2022-05-09 15:08:59 +02:00
Jef Roosens 3821ed29fd
refactor(docker): simplified loop expression 2022-05-09 15:05:53 +02:00
Jef Roosens 5a5f7f8346
refactor(docker): use builtin parse_rfc3339 function 2022-05-09 14:58:20 +02:00
Jef Roosens ea4c4fce16
feat(cron): upload logs after build 2022-05-09 08:51:10 +02:00
Jef Roosens e79d18100f
chore: ran `make fmt` 2022-05-09 08:51:10 +02:00
Jef Roosens 4b172cb5d8
feat(cli): `vieter build` now builds a single repo & uploads build logs 2022-05-09 08:51:10 +02:00
Jef Roosens 27aa215eff
feat(docker): added function to retrieve container logs 2022-05-09 08:51:10 +02:00
Jef Roosens 7e5f0c5a53 Merge pull request 'Build logs API & CLI + refactoring' (#169) from Chewing_Bever/vieter:build-logs into dev
Reviewed-on: vieter/vieter#169
2022-05-07 23:55:56 +02:00
Jef Roosens 30cce4fa72
chore: updated changelog 2022-05-07 22:13:35 +02:00
Jef Roosens 5f7d7c4780
doc: added documentation to all functions 2022-05-07 22:06:17 +02:00
Jef Roosens 5b016df85d
feat(cli): added commands for interacting with build logs 2022-05-07 21:50:20 +02:00
Jef Roosens fa6603bd45
feat(client): added client code for logs API 2022-05-07 19:38:28 +02:00
Jef Roosens 407b226955
refactor: moved client code into own module 2022-05-07 16:10:27 +02:00
Jef Roosens f42d3fd8b0
fix(server): prevent adding logs to non-existent repo 2022-05-07 15:44:59 +02:00
Jef Roosens 139142fcec
feat(server): added endpoint for content of build log 2022-05-07 15:41:49 +02:00
Jef Roosens 393e641a76
feat(server): allow filtering of builds per repo 2022-05-07 15:31:01 +02:00
Jef Roosens 7e01dbafec
feat(server): added endpoints for listing & uploading build logs 2022-05-07 15:10:07 +02:00
Jef Roosens 58c1ecd25e
db: added BuildLog & required methods 2022-05-07 14:16:30 +02:00
Jef Roosens 230920576d Merge pull request 'Release 0.3.0-alpha.1' (#164) from release-0.3.0-alpha.1 into main
Reviewed-on: vieter/vieter#164
2022-05-06 20:12:51 +02:00
Jef Roosens 356a34ab01
chore: bumped versions 2022-05-06 20:04:48 +02:00
Jef Roosens 1156e896f7 Merge pull request 'cron: filter out repos with wrong architecture' (#163) from Chewing_Bever/vieter:cron-check-arch into dev
Reviewed-on: vieter/vieter#163
2022-05-06 09:00:53 +02:00
Jef Roosens a3b6680153
cron: filter out repos with wrong architecture 2022-05-06 08:31:59 +02:00
Jef Roosens 7fdbcdf3e7
ci(arch): also change URL of downloaded PKGBUILD 2022-05-05 23:38:12 +02:00
Jef Roosens d4306133e0 Merge pull request 'Fix PKGBUILD' (#161) from Chewing_Bever/vieter:pkgbuild-fixes into dev
Reviewed-on: vieter/vieter#161
2022-05-05 23:36:42 +02:00
Jef Roosens 1990ade089
ci: fixed some steps running when not required 2022-05-05 23:30:54 +02:00
Jef Roosens e008133981
ci(arch): changed PKGBUILD to new URL 2022-05-05 23:29:08 +02:00
Jef Roosens 1a076a7a8c Merge pull request 'Sqlite backend & simplifying of config variables' (#158) from Chewing_Bever/vieter:sqlite-backend into dev
Reviewed-on: vieter/vieter#158
2022-05-05 23:18:43 +02:00
Jef Roosens 8c5652c230
ci: made build upload failable; updated ci for use with PRs 2022-05-05 23:11:18 +02:00
29 changed files with 686 additions and 201 deletions

View File

@ -23,8 +23,10 @@ pipeline:
- su builder
# Due to a bug with the V compiler, we can't just use the PKGBUILD from
# inside the repo
- curl -OL https://git.rustybever.be/Chewing_Bever/vieter/raw/branch/dev/PKGBUILD
- curl -OL https://git.rustybever.be/vieter/vieter/raw/branch/dev/PKGBUILD
- makepkg -s --noconfirm --needed
when:
event: push
publish:
image: 'curlimages/curl'
@ -33,3 +35,5 @@ pipeline:
- 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done'
secrets:
- vieter_api_key
when:
event: push

View File

@ -1,11 +1,8 @@
matrix:
PLATFORM:
- linux/amd64
- linux/arm64
# I just don't have a performant enough runner for this platform
# - linux/arm/v7
- 'linux/amd64'
- 'linux/arm64'
# These checks already get performed on the feature branches
platform: ${PLATFORM}
pipeline:
@ -15,9 +12,9 @@ pipeline:
commands:
- make
when:
event: push
event: [pull_request]
branch:
exclude: [main, dev]
exclude: [main]
prod:
image: 'chewingbever/vlang:latest'
@ -35,7 +32,7 @@ pipeline:
- strip -s pvieter
- du -h pvieter
when:
event: push
event: [push, pull_request]
upload:
image: 'chewingbever/vlang:latest'
@ -52,6 +49,7 @@ pipeline:
- >
curl
--silent
--fail
-XPUT
-T pvieter
-H "Host: $URL"
@ -60,4 +58,4 @@ pipeline:
-H "Authorization: AWS $S3_USERNAME:$SIGNATURE"
https://$URL$OBJ_PATH
when:
event: push
event: [push, pull_request]

View File

@ -1,29 +0,0 @@
# These builds are not important for the project, but might be valuable for
# fixing bugs in the V compiler.
platform: linux/amd64
branches:
exclude: [master, dev]
pipeline:
autofree:
image: 'chewingbever/vlang:latest'
pull: true
group: 'build'
commands:
- make autofree
- readelf -d afvieter
- du -h afvieter
when:
event: push
skip-unused:
image: 'chewingbever/vlang:latest'
pull: true
group: 'build'
commands:
- make skip-unused
- readelf -d suvieter
- du -h suvieter
when:
event: push

View File

@ -1,4 +1,4 @@
branches: 'dev'
branches: [ 'dev' ]
platform: 'linux/amd64'
depends_on:
- 'docker'
@ -14,3 +14,5 @@ pipeline:
commands:
- 'curl -XPOST -s --fail $WEBHOOK_APP'
- 'curl -XPOST -s --fail $WEBHOOK_CRON'
when:
event: push

View File

@ -1,30 +1,36 @@
branches: [main, dev]
platform: linux/amd64
platform: 'linux/amd64'
depends_on:
- build
pipeline:
dev:
image: woodpeckerci/plugin-docker-buildx
secrets: [ docker_username, docker_password ]
image: 'woodpeckerci/plugin-docker-buildx'
secrets:
- 'docker_username'
- 'docker_password'
settings:
repo: chewingbever/vieter
tag: dev
platforms: [ linux/arm64/v8, linux/amd64 ]
repo: 'chewingbever/vieter'
tags:
- 'dev'
- ${CI_COMMIT_SHA}
platforms: [ 'linux/arm64/v8', 'linux/amd64' ]
build_args_from_env:
- CI_COMMIT_SHA
- 'CI_COMMIT_SHA'
when:
event: push
branch: dev
release:
image: woodpeckerci/plugin-docker-buildx
secrets: [ docker_username, docker_password ]
image: 'woodpeckerci/plugin-docker-buildx'
secrets:
- 'docker_username'
- 'docker_password'
settings:
repo: chewingbever/vieter
repo: 'chewingbever/vieter'
auto_tag: true
platforms: [ linux/arm64/v8, linux/amd64 ]
platforms: [ 'linux/arm64/v8', 'linux/amd64' ]
build_args_from_env:
- CI_COMMIT_SHA
- 'CI_COMMIT_SHA'
when:
event: tag

View File

@ -1,6 +1,5 @@
# Yeah so this only works on tags so we'll worry about this later
platform: linux/amd64
branches: main
platform: 'linux/amd64'
branches: [ 'main' ]
depends_on:
- build

View File

@ -1,7 +1,7 @@
# These checks already get performed on the feature branches
branches:
exclude: [ main, dev ]
platform: linux/amd64
exclude: [ main ]
platform: 'linux/amd64'
pipeline:
lint:
@ -9,3 +9,5 @@ pipeline:
pull: true
commands:
- make lint
when:
event: [ pull_request ]

View File

@ -1,10 +1,10 @@
matrix:
PLATFORM:
- linux/amd64
- linux/arm64
- 'linux/amd64'
- 'linux/arm64'
branches:
exclude: [main, dev]
exclude: [ main ]
platform: ${PLATFORM}
pipeline:
@ -14,4 +14,4 @@ pipeline:
commands:
- make test
when:
event: push
event: [pull_request]

View File

@ -5,7 +5,14 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter)
## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev)
### Added
* Web API for adding & querying build logs
* CLI commands to access build logs API
## [0.3.0-alpha.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.1)
### Changed
@ -24,7 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* Binary no longer panics when an env var is missing
## [0.2.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.2.0)
## [0.2.0](https://git.rustybever.be/vieter/vieter/src/tag/0.2.0)
### Changed
@ -58,13 +65,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* Packages with unknown fields in .PKGINFO are now allowed
* Old packages are now properly removed
## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0)
## [0.1.0](https://git.rustybever.be/vieter/vieter/src/tag/0.1.0)
### Changed
* Improved logging
## [0.1.0-rc.1](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0-rc.1)
## [0.1.0-rc.1](https://git.rustybever.be/vieter/vieter/src/tag/0.1.0-rc.1)
### Added

View File

@ -4,12 +4,12 @@ pkgbase='vieter'
pkgname='vieter'
pkgver=0.2.0.r25.g20112b8
pkgrel=1
depends=('glibc' 'openssl' 'libarchive' 'gc')
depends=('glibc' 'openssl' 'libarchive' 'gc' 'sqlite')
makedepends=('git' 'gcc' 'vieter-v')
arch=('x86_64' 'aarch64' 'armv7')
url='https://git.rustybever.be/Chewing_Bever/vieter'
arch=('x86_64' 'aarch64')
url='https://git.rustybever.be/vieter/vieter'
license=('AGPL3')
source=($pkgname::git+https://git.rustybever.be/Chewing_Bever/vieter#branch=dev)
source=($pkgname::git+https://git.rustybever.be/vieter/vieter#branch=dev)
md5sums=('SKIP')
pkgver() {

View File

@ -3,9 +3,9 @@ module build
import docker
import encoding.base64
import time
import git
import os
import db
import client
const container_build_dir = '/build'
@ -73,10 +73,18 @@ pub fn create_build_image(base_image string) ?string {
return image.id
}
pub struct BuildResult {
pub:
start_time time.Time
end_time time.Time
exit_code int
logs string
}
// build_repo builds, packages & publishes a given Arch package based on the
// provided GitRepo. The base image ID should be of an image previously created
// by create_build_image.
pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ? {
// by create_build_image. It returns the logs of the container.
pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ?BuildResult {
build_arch := os.uname().machine
// TODO what to do with PKGBUILDs that build multiple packages?
@ -87,7 +95,7 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db
'source PKGBUILD',
// The build container checks whether the package is already
// present on the server
'curl --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
'curl -s --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done',
]
@ -107,43 +115,44 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db
id := docker.create_container(c) ?
docker.start_container(id) ?
mut data := docker.inspect_container(id) ?
// This loop waits until the container has stopped, so we can remove it after
for {
data := docker.inspect_container(id) ?
if !data.state.running {
break
}
for data.state.running {
time.sleep(1 * time.second)
data = docker.inspect_container(id) ?
}
logs := docker.get_container_logs(id) ?
docker.remove_container(id) ?
return BuildResult{
start_time: data.state.start_time
end_time: data.state.end_time
exit_code: data.state.exit_code
logs: logs
}
}
// build builds every Git repo in the server's list.
fn build(conf Config) ? {
fn build(conf Config, repo_id int) ? {
c := client.new(conf.address, conf.api_key)
repo := c.get_git_repo(repo_id) ?
build_arch := os.uname().machine
// We get the repos map from the Vieter instance
repos := git.get_repos(conf.address, conf.api_key) ?
// We filter out any repos that aren't allowed to be built on this
// architecture
filtered_repos := repos.filter(it.arch.map(it.value).contains(build_arch))
// No point in doing work if there's no repos present
if filtered_repos.len == 0 {
return
}
// First, we create a base image which has updated repos n stuff
println('Creating base image...')
image_id := create_build_image(conf.base_image) ?
for repo in filtered_repos {
build_repo(conf.address, conf.api_key, image_id, repo) ?
}
println('Running build...')
res := build_repo(conf.address, conf.api_key, image_id, repo) ?
// Finally, we remove the builder image
println('Removing build image...')
docker.remove_image(image_id) ?
println('Uploading logs to Vieter...')
c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code,
res.logs) ?
}

View File

@ -14,12 +14,16 @@ pub:
pub fn cmd() cli.Command {
return cli.Command{
name: 'build'
description: 'Run the build process.'
required_args: 1
usage: 'id'
description: 'Build the repository with the given ID.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
build(conf) ?
id := cmd.args[0].int()
build(conf, id) ?
}
}
}

View File

@ -0,0 +1,67 @@
module client
import net.http { Method }
import net.urllib
import response { Response }
import json
pub struct Client {
pub:
address string
api_key string
}
// new creates a new Client instance.
pub fn new(address string, api_key string) Client {
return Client{
address: address
api_key: api_key
}
}
// send_request_raw sends an HTTP request, returning the http.Response object.
// It encodes the params so that they're safe to pass as HTTP query parameters.
fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) ?http.Response {
mut full_url := '$c.address$url'
if params.len > 0 {
mut params_escaped := map[string]string{}
// Escape each query param
for k, v in params {
params_escaped[k] = urllib.query_escape(v)
}
params_str := params_escaped.keys().map('$it=${params[it]}').join('&')
full_url = '$full_url?$params_str'
}
mut req := http.new_request(method, full_url, body) ?
req.add_custom_header('X-Api-Key', c.api_key) ?
res := req.do() ?
return res
}
// send_request<T> just calls send_request_with_body<T> with an empty body.
fn (c &Client) send_request<T>(method Method, url string, params map[string]string) ?Response<T> {
return c.send_request_with_body<T>(method, url, params, '')
}
// send_request_with_body<T> calls send_request_raw_response & parses its
// output as a Response<T> object.
fn (c &Client) send_request_with_body<T>(method Method, url string, params map[string]string, body string) ?Response<T> {
res_text := c.send_request_raw_response(method, url, params, body) ?
data := json.decode(Response<T>, res_text) ?
return data
}
// send_request_raw_response returns the raw text response for an HTTP request.
fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) ?string {
res := c.send_request_raw(method, url, params, body) ?
return res.text
}

51
src/client/git.v 100644
View File

@ -0,0 +1,51 @@
module client
import db { GitRepo }
import net.http { Method }
import response { Response }
// get_git_repos returns the current list of repos.
pub fn (c &Client) get_git_repos() ?[]GitRepo {
data := c.send_request<[]GitRepo>(Method.get, '/api/repos', {}) ?
return data.data
}
// get_git_repo returns the repo for a specific ID.
pub fn (c &Client) get_git_repo(id int) ?GitRepo {
data := c.send_request<GitRepo>(Method.get, '/api/repos/$id', {}) ?
return data.data
}
// add_git_repo adds a new repo to the server.
pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []string) ?Response<string> {
mut params := {
'url': url
'branch': branch
'repo': repo
}
if arch.len > 0 {
params['arch'] = arch.join(',')
}
data := c.send_request<string>(Method.post, '/api/repos', params) ?
return data
}
// remove_git_repo removes the repo with the given ID from the server.
pub fn (c &Client) remove_git_repo(id int) ?Response<string> {
data := c.send_request<string>(Method.delete, '/api/repos/$id', {}) ?
return data
}
// patch_git_repo sends a PATCH request to the given repo with the params as
// payload.
pub fn (c &Client) patch_git_repo(id int, params map[string]string) ?Response<string> {
data := c.send_request<string>(Method.patch, '/api/repos/$id', params) ?
return data
}

53
src/client/logs.v 100644
View File

@ -0,0 +1,53 @@
module client
import db { BuildLog }
import net.http { Method }
import response { Response }
import time
// get_build_logs returns all build logs.
pub fn (c &Client) get_build_logs() ?Response<[]BuildLog> {
data := c.send_request<[]BuildLog>(Method.get, '/api/logs', {}) ?
return data
}
// get_build_logs_for_repo returns all build logs for a given repo.
pub fn (c &Client) get_build_logs_for_repo(repo_id int) ?Response<[]BuildLog> {
params := {
'repo': repo_id.str()
}
data := c.send_request<[]BuildLog>(Method.get, '/api/logs', params) ?
return data
}
// get_build_log returns a specific build log.
pub fn (c &Client) get_build_log(id int) ?Response<BuildLog> {
data := c.send_request<BuildLog>(Method.get, '/api/logs/$id', {}) ?
return data
}
// get_build_log_content returns the contents of the build log file.
pub fn (c &Client) get_build_log_content(id int) ?string {
data := c.send_request_raw_response(Method.get, '/api/logs/$id/content', {}, '') ?
return data
}
// add_build_log adds a new build log to the server.
pub fn (c &Client) add_build_log(repo_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response<string> {
params := {
'repo': repo_id.str()
'startTime': start_time.str()
'endTime': end_time.str()
'arch': arch
'exitCode': exit_code.str()
}
data := c.send_request_with_body<string>(Method.post, '/api/logs', params, content) ?
return data
}

View File

@ -0,0 +1 @@
module console

View File

@ -3,6 +3,7 @@ module git
import cli
import env
import cron.expression { parse_expression }
import client
struct Config {
address string [required]
@ -119,7 +120,8 @@ pub fn cmd() cli.Command {
// list prints out a list of all repositories.
fn list(conf Config) ? {
repos := get_repos(conf.address, conf.api_key) ?
c := client.new(conf.address, conf.api_key)
repos := c.get_git_repos() ?
for repo in repos {
println('$repo.id\t$repo.url\t$repo.branch\t$repo.repo')
@ -128,7 +130,8 @@ fn list(conf Config) ? {
// add adds a new repository to the server's list.
fn add(conf Config, url string, branch string, repo string) ? {
res := add_repo(conf.address, conf.api_key, url, branch, repo, []) ?
c := client.new(conf.address, conf.api_key)
res := c.add_git_repo(url, branch, repo, []) ?
println(res.message)
}
@ -139,7 +142,8 @@ fn remove(conf Config, id string) ? {
id_int := id.int()
if id_int != 0 {
res := remove_repo(conf.address, conf.api_key, id_int) ?
c := client.new(conf.address, conf.api_key)
res := c.remove_git_repo(id_int) ?
println(res.message)
}
}
@ -156,7 +160,8 @@ fn patch(conf Config, id string, params map[string]string) ? {
id_int := id.int()
if id_int != 0 {
res := patch_repo(conf.address, conf.api_key, id_int, params) ?
c := client.new(conf.address, conf.api_key)
res := c.patch_git_repo(id_int, params) ?
println(res.message)
}
@ -170,6 +175,7 @@ fn info(conf Config, id string) ? {
return
}
repo := get_repo(conf.address, conf.api_key, id_int) ?
c := client.new(conf.address, conf.api_key)
repo := c.get_git_repo(id_int) ?
println(repo)
}

View File

@ -0,0 +1,106 @@
module logs
import cli
import env
import client
import db
struct Config {
address string [required]
api_key string [required]
}
// cmd returns the cli module that handles the build repos API.
pub fn cmd() cli.Command {
return cli.Command{
name: 'logs'
description: 'Interact with the build logs API.'
commands: [
cli.Command{
name: 'list'
description: 'List the build logs. If a repo ID is provided, only list the build logs for that repo.'
flags: [
cli.Flag{
name: 'repo'
description: 'ID of the Git repo to restrict list to.'
flag: cli.FlagType.int
},
]
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
repo_id := cmd.flags.get_int('repo') ?
if repo_id == 0 { list(conf) ? } else { list_for_repo(conf, repo_id) ? }
}
},
cli.Command{
name: 'info'
required_args: 1
usage: 'id'
description: 'Show all info for a specific build log.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
id := cmd.args[0].int()
info(conf, id) ?
}
},
cli.Command{
name: 'content'
required_args: 1
usage: 'id'
description: 'Output the content of a build log to stdout.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
id := cmd.args[0].int()
content(conf, id) ?
}
},
]
}
}
// print_log_list prints a list of logs.
fn print_log_list(logs []db.BuildLog) {
for log in logs {
println('$log.id\t$log.start_time\t$log.exit_code')
}
}
// list prints a list of all build logs.
fn list(conf Config) ? {
c := client.new(conf.address, conf.api_key)
logs := c.get_build_logs() ?.data
print_log_list(logs)
}
// list prints a list of all build logs for a given repo.
fn list_for_repo(conf Config, repo_id int) ? {
c := client.new(conf.address, conf.api_key)
logs := c.get_build_logs_for_repo(repo_id) ?.data
print_log_list(logs)
}
// info print the detailed info for a given build log.
fn info(conf Config, id int) ? {
c := client.new(conf.address, conf.api_key)
log := c.get_build_log(id) ?.data
print(log)
}
// content outputs the contents of the log file for a given build log to
// stdout.
fn content(conf Config, id int) ? {
c := client.new(conf.address, conf.api_key)
content := c.get_build_log_content(id) ?
println(content)
}

View File

@ -3,6 +3,7 @@ module daemon
import time
import sync.stdatomic
import build
import os
const (
build_empty = 0
@ -77,13 +78,20 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) {
// 0 means success, 1 means failure
mut status := 0
build.build_repo(d.address, d.api_key, d.builder_images.last(), &sb.repo) or {
res := build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(),
&sb.repo) or {
d.ldebug('build_repo error: $err.msg()')
status = 1
build.BuildResult{}
}
if status == 0 {
d.linfo('finished build: $sb.repo.url $sb.repo.branch')
d.linfo('finished build: $sb.repo.url $sb.repo.branch; uploading logs...')
build_arch := os.uname().machine
d.client.add_build_log(sb.repo.id, res.start_time, res.end_time, build_arch, res.exit_code,
res.logs) or { d.lerror('Failed to upload logs for $sb.repo.url $sb.repo.arch') }
} else {
d.linfo('failed build: $sb.repo.url $sb.repo.branch')
}

View File

@ -1,6 +1,5 @@
module daemon
import git
import time
import log
import datatypes { MinHeap }
@ -9,6 +8,8 @@ import math
import build
import docker
import db
import os
import client
const (
// How many seconds to wait before retrying to update API if failed
@ -19,7 +20,6 @@ const (
struct ScheduledBuild {
pub:
repo_id string
repo db.GitRepo
timestamp time.Time
}
@ -31,8 +31,7 @@ fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool {
pub struct Daemon {
mut:
address string
api_key string
client client.Client
base_image string
builder_images []string
global_schedule CronExpression
@ -56,8 +55,7 @@ mut:
// populates the build queue for the first time.
pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon {
mut d := Daemon{
address: address
api_key: api_key
client: client.new(address, api_key)
base_image: base_image
global_schedule: global_schedule
api_update_frequency: api_update_frequency
@ -180,13 +178,17 @@ fn (mut d Daemon) schedule_build(repo db.GitRepo) {
fn (mut d Daemon) renew_repos() {
d.linfo('Renewing repos...')
mut new_repos := git.get_repos(d.address, d.api_key) or {
mut new_repos := d.client.get_git_repos() or {
d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...')
d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout)
return
}
// Filter out any repos that shouldn't run on this architecture
cur_arch := os.uname().machine
new_repos = new_repos.filter(it.arch.any(it.value == cur_arch))
d.repos = new_repos
d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency)

View File

@ -12,6 +12,7 @@ pub fn init(db_path string) ?VieterDb {
sql conn {
create table GitRepo
create table BuildLog
}
return VieterDb{

View File

@ -94,7 +94,7 @@ pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo {
// If a select statement fails, it returns a zeroed object. By
// checking one of the required fields, we can see whether the query
// returned a result or not.
if res.url == '' {
if res.id == 0 {
return none
}
@ -118,11 +118,9 @@ pub fn (db &VieterDb) delete_git_repo(repo_id int) {
// update_git_repo updates any non-array values for a given GitRepo.
pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) {
// sql db.conn {
// update GitRepo set repo
//}
mut values := []string{}
// TODO does this allow for SQL injection?
$for field in GitRepo.fields {
if field.name in params {
// Any fields that are array types require their own update method
@ -154,3 +152,11 @@ pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) {
}
}
}
// git_repo_exists is a utility function that checks whether a repo with the
// given id exists.
pub fn (db &VieterDb) git_repo_exists(repo_id int) bool {
db.get_git_repo(repo_id) or { return false }
return true
}

74
src/db/logs.v 100644
View File

@ -0,0 +1,74 @@
module db
import time
pub struct BuildLog {
pub:
id int [primary; sql: serial]
repo_id int [nonull]
start_time time.Time [nonull]
end_time time.Time [nonull]
arch string [nonull]
exit_code int [nonull]
}
// str returns a string representation.
pub fn (bl &BuildLog) str() string {
mut parts := [
'id: $bl.id',
'repo id: $bl.repo_id',
'start time: $bl.start_time',
'end time: $bl.end_time',
'arch: $bl.arch',
'exit code: $bl.exit_code',
]
str := parts.join('\n')
return str
}
// get_build_logs returns all BuildLog's in the database.
pub fn (db &VieterDb) get_build_logs() []BuildLog {
res := sql db.conn {
select from BuildLog order by id
}
return res
}
// get_build_logs_for_repo returns all BuildLog's in the database for a given
// repo.
pub fn (db &VieterDb) get_build_logs_for_repo(repo_id int) []BuildLog {
res := sql db.conn {
select from BuildLog where repo_id == repo_id order by id
}
return res
}
// get_build_log tries to return a specific BuildLog.
pub fn (db &VieterDb) get_build_log(id int) ?BuildLog {
res := sql db.conn {
select from BuildLog where id == id
}
if res.id == 0 {
return none
}
return res
}
// add_build_log inserts the given BuildLog into the database.
pub fn (db &VieterDb) add_build_log(log BuildLog) {
sql db.conn {
insert log into BuildLog
}
}
// delete_build_log delete the BuildLog with the given ID from the database.
pub fn (db &VieterDb) delete_build_log(id int) {
sql db.conn {
delete from BuildLog where id == id
}
}

View File

@ -2,6 +2,7 @@ module docker
import json
import net.urllib
import time
struct Container {
id string [json: Id]
@ -49,13 +50,21 @@ pub fn start_container(id string) ?bool {
}
struct ContainerInspect {
pub:
pub mut:
state ContainerState [json: State]
}
struct ContainerState {
pub:
running bool [json: Running]
running bool [json: Running]
status string [json: Status]
exit_code int [json: ExitCode]
// These use a rather specific format so they have to be parsed later
start_time_str string [json: StartedAt]
end_time_str string [json: FinishedAt]
pub mut:
start_time time.Time [skip]
end_time time.Time [skip]
}
// inspect_container returns the result of inspecting a container with a given
@ -67,7 +76,15 @@ pub fn inspect_container(id string) ?ContainerInspect {
return error('Failed to inspect container.')
}
return json.decode(ContainerInspect, res.text) or {}
mut data := json.decode(ContainerInspect, res.text) ?
data.state.start_time = time.parse_rfc3339(data.state.start_time_str) ?
if data.state.status == 'exited' {
data.state.end_time = time.parse_rfc3339(data.state.end_time_str) ?
}
return data
}
// remove_container removes a container with a given ID.
@ -76,3 +93,25 @@ pub fn remove_container(id string) ?bool {
return res.status_code == 204
}
// get_container_logs retrieves the logs for a Docker container, both stdout &
// stderr.
pub fn get_container_logs(id string) ?string {
res := request('GET', urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true') ?) ?
mut res_bytes := res.text.bytes()
// Docker uses a special "stream" format for their logs, so we have to
// clean up the data.
mut index := 0
for index < res_bytes.len {
// The reverse is required because V reads in the bytes differently
t := res_bytes[index + 4..index + 8].reverse()
len_length := unsafe { *(&u32(&t[0])) }
res_bytes.delete_many(index, 8)
index += int(len_length)
}
return res_bytes.bytestr()
}

View File

@ -1,77 +0,0 @@
module git
import json
import response { Response }
import net.http
import db
// send_request<T> is a convenience method for sending requests to the repos
// API. It mostly does string manipulation to create a query string containing
// the provided params.
fn send_request<T>(method http.Method, address string, url string, api_key string, params map[string]string) ?Response<T> {
mut full_url := '$address$url'
if params.len > 0 {
params_str := params.keys().map('$it=${params[it]}').join('&')
full_url = '$full_url?$params_str'
}
mut req := http.new_request(method, full_url, '') ?
req.add_custom_header('X-API-Key', api_key) ?
res := req.do() ?
data := json.decode(Response<T>, res.text) ?
return data
}
// get_repos returns the current list of repos.
pub fn get_repos(address string, api_key string) ?[]db.GitRepo {
data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', api_key,
{}) ?
return data.data
}
// get_repo returns the repo for a specific ID.
pub fn get_repo(address string, api_key string, id int) ?db.GitRepo {
data := send_request<db.GitRepo>(http.Method.get, address, '/api/repos/$id', api_key,
{}) ?
return data.data
}
// add_repo adds a new repo to the server.
pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response<string> {
mut params := {
'url': url
'branch': branch
'repo': repo
}
if arch.len > 0 {
params['arch'] = arch.join(',')
}
data := send_request<string>(http.Method.post, address, '/api/repos', api_key, params) ?
return data
}
// remove_repo removes the repo with the given ID from the server.
pub fn remove_repo(address string, api_key string, id int) ?Response<string> {
data := send_request<string>(http.Method.delete, address, '/api/repos/$id', api_key,
{}) ?
return data
}
// patch_repo sends a PATCH request to the given repo with the params as
// payload.
pub fn patch_repo(address string, api_key string, id int, params map[string]string) ?Response<string> {
data := send_request<string>(http.Method.patch, address, '/api/repos/$id', api_key,
params) ?
return data
}

View File

@ -4,14 +4,15 @@ import os
import server
import cli
import build
import git
import console.git
import console.logs
import cron
fn main() {
mut app := cli.Command{
name: 'vieter'
description: 'Vieter is a lightweight implementation of an Arch repository server.'
version: '0.2.0'
version: '0.3.0-alpha.1'
flags: [
cli.Flag{
flag: cli.FlagType.string
@ -27,6 +28,7 @@ fn main() {
build.cmd(),
git.cmd(),
cron.cmd(),
logs.cmd(),
]
}

View File

@ -60,7 +60,6 @@ fn (mut app App) delete_repo(id int) web.Result {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
// repos.delete(id)
app.db.delete_git_repo(id)
return app.json(http.Status.ok, new_response('Repo removed successfully.'))

136
src/server/logs.v 100644
View File

@ -0,0 +1,136 @@
module server
import web
import net.http
import net.urllib
import response { new_data_response, new_response }
import db
import time
import os
import util
// get_logs returns all build logs in the database. A 'repo' query param can
// optionally be added to limit the list of build logs to that repository.
['/api/logs'; get]
fn (mut app App) get_logs() web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
logs := if 'repo' in app.query {
app.db.get_build_logs_for_repo(app.query['repo'].int())
} else {
app.db.get_build_logs()
}
return app.json(http.Status.ok, new_data_response(logs))
}
// get_single_log returns the build log with the given id.
['/api/logs/:id'; get]
fn (mut app App) get_single_log(id int) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
log := app.db.get_build_log(id) or { return app.not_found() }
return app.json(http.Status.ok, new_data_response(log))
}
// get_log_content returns the actual build log file for the given id.
['/api/logs/:id/content'; get]
fn (mut app App) get_log_content(id int) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
log := app.db.get_build_log(id) or { return app.not_found() }
file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.repo_id.str(), log.arch,
file_name)
return app.file(full_path)
}
// parse_query_time unescapes an HTTP query parameter & tries to parse it as a
// time.Time struct.
fn parse_query_time(query string) ?time.Time {
unescaped := urllib.query_unescape(query) ?
t := time.parse(unescaped) ?
return t
}
// post_log adds a new log to the database.
['/api/logs'; post]
fn (mut app App) post_log() web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
// Parse query params
start_time := parse_query_time(app.query['startTime']) or {
return app.json(http.Status.bad_request, new_response('Invalid or missing start time.'))
}
end_time := parse_query_time(app.query['endTime']) or {
return app.json(http.Status.bad_request, new_response('Invalid or missing end time.'))
}
if 'exitCode' !in app.query {
return app.json(http.Status.bad_request, new_response('Missing exit code.'))
}
exit_code := app.query['exitCode'].int()
if 'arch' !in app.query {
return app.json(http.Status.bad_request, new_response("Missing parameter 'arch'."))
}
arch := app.query['arch']
repo_id := app.query['repo'].int()
if !app.db.git_repo_exists(repo_id) {
return app.json(http.Status.bad_request, new_response('Unknown Git repo.'))
}
// Store log in db
log := db.BuildLog{
repo_id: repo_id
start_time: start_time
end_time: end_time
arch: arch
exit_code: exit_code
}
app.db.add_build_log(log)
repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo_id.str(), arch)
// Create the logs directory of it doesn't exist
if !os.exists(repo_logs_dir) {
os.mkdir_all(repo_logs_dir) or {
app.lerror("Couldn't create dir '$repo_logs_dir'.")
return app.json(http.Status.internal_server_error, new_response('An error occured while processing the request.'))
}
}
// Stream log contents to correct file
file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
full_path := os.join_path_single(repo_logs_dir, file_name)
if length := app.req.header.get(.content_length) {
util.reader_to_file(mut app.reader, length.int(), full_path) or {
app.lerror('An error occured while receiving logs: $err.msg()')
return app.json(http.Status.internal_server_error, new_response('Failed to upload logs.'))
}
} else {
return app.status(http.Status.length_required)
}
return app.json(http.Status.ok, new_response('Logs added successfully.'))
}

View File

@ -12,6 +12,7 @@ const (
log_file_name = 'vieter.log'
repo_dir_name = 'repos'
db_file_name = 'vieter.sqlite'
logs_dir_name = 'logs'
)
struct App {
@ -37,6 +38,14 @@ pub fn server(conf Config) ? {
os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') }
logs_dir := os.join_path_single(conf.data_dir, server.logs_dir_name)
if !os.exists(logs_dir) {
os.mkdir(os.join_path_single(conf.data_dir, server.logs_dir_name)) or {
util.exit_with_message(1, 'Failed to create logs directory.')
}
}
mut logger := log.Log{
level: log_level
}