Add direct PKGBUILD link as target option #254
|
@ -10,12 +10,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
* Server port can now be configured
|
* Server port can now be configured
|
||||||
|
* Targets now have a 'kind' field describing whether it's a Git repository or a
|
||||||
|
URL to a PKGBUILD
|
||||||
|
* Targets with kind 'url' can provide a direct URL to a PKGBUILD instead of
|
||||||
|
providing a Git repository
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
* Moved all API routes under `/v1` namespace
|
* Moved all API routes under `/v1` namespace
|
||||||
* Renamed `vieter repos` to `vieter targets`
|
* Renamed `vieter repos` to `vieter targets`
|
||||||
* Renamed `/api/v1/repos` namespace to `/api/v1/targets`
|
* Renamed `/api/v1/repos` namespace to `/api/v1/targets`
|
||||||
|
* Branch name for 'git' targets is now optional; if not provided, the
|
||||||
|
repository will be cloned with the default branch
|
||||||
|
|
||||||
### Removed
|
### Removed
|
||||||
|
|
||||||
|
|
|
@ -24,6 +24,7 @@ curl \
|
||||||
"data": [
|
"data": [
|
||||||
{
|
{
|
||||||
"id": 1,
|
"id": 1,
|
||||||
|
"kind": "git",
|
||||||
"url": "https://aur.archlinux.org/discord-ptb.git",
|
"url": "https://aur.archlinux.org/discord-ptb.git",
|
||||||
"branch": "master",
|
"branch": "master",
|
||||||
"repo": "bur",
|
"repo": "bur",
|
||||||
|
@ -69,6 +70,7 @@ curl \
|
||||||
"message": "",
|
"message": "",
|
||||||
"data": {
|
"data": {
|
||||||
"id": 1,
|
"id": 1,
|
||||||
|
"kind": "git",
|
||||||
"url": "https://aur.archlinux.org/discord-ptb.git",
|
"url": "https://aur.archlinux.org/discord-ptb.git",
|
||||||
"branch": "master",
|
"branch": "master",
|
||||||
"repo": "bur",
|
"repo": "bur",
|
||||||
|
@ -108,6 +110,7 @@ Create a new target with the given data.
|
||||||
|
|
||||||
Parameter | Description
|
Parameter | Description
|
||||||
--------- | -----------
|
--------- | -----------
|
||||||
|
kind | Kind of target to add; one of 'git', 'url'.
|
||||||
url | URL of the Git repository.
|
url | URL of the Git repository.
|
||||||
branch | Branch of the Git repository.
|
branch | Branch of the Git repository.
|
||||||
repo | Vieter repository to publish built packages to.
|
repo | Vieter repository to publish built packages to.
|
||||||
|
@ -132,6 +135,7 @@ id | id of target to modify
|
||||||
|
|
||||||
Parameter | Description
|
Parameter | Description
|
||||||
--------- | -----------
|
--------- | -----------
|
||||||
|
kind | Kind of target; one of 'git', 'url'.
|
||||||
url | URL of the Git repository.
|
url | URL of the Git repository.
|
||||||
branch | Branch of the Git repository.
|
branch | Branch of the Git repository.
|
||||||
repo | Vieter repository to publish built packages to.
|
repo | Vieter repository to publish built packages to.
|
||||||
|
|
|
@ -20,24 +20,24 @@ pages](https://rustybever.be/man/vieter/vieter-targets.1.html) describe this in
|
||||||
greater detail, but the basic usage is as follows:
|
greater detail, but the basic usage is as follows:
|
||||||
|
|
||||||
```
|
```
|
||||||
vieter targets add some-url some-branch some-repository
|
vieter targets add some-url some-repository
|
||||||
```
|
```
|
||||||
|
|
||||||
Here, `some-url` is the URL of the Git repository containing the PKGBUILD. This
|
Here, `some-url` is the URL of the Git repository containing the PKGBUILD. This
|
||||||
URL is passed to `git clone`, meaning the repository should be public. Vieter
|
URL is passed to `git clone`, meaning the repository should be public. Vieter
|
||||||
expects the same format as an AUR Git repository, so you can directly use AUR
|
expects the same format as an AUR Git repository, so you can directly use AUR
|
||||||
URLs here.
|
URLs here. Alternatively, you can also provide the URL to a PKGBUILD file
|
||||||
|
instead. See
|
||||||
|
[vieter-targets-add(1)](https://rustybever.be/man/vieter/vieter-targets-add.1.html)
|
||||||
|
for more information.
|
||||||
|
|
||||||
`some-branch` is the branch of the Git repository the build should check out.
|
`some-repo` is the repository to which the built package archives should be
|
||||||
If you're using an AUR package, this should be `master`.
|
published.
|
||||||
|
|
||||||
Finally, `some-repo` is the repository to which the built package archives
|
|
||||||
should be published.
|
|
||||||
|
|
||||||
The above command intentionally leaves out a few parameters to make the CLI
|
The above command intentionally leaves out a few parameters to make the CLI
|
||||||
more useable. For information on how to modify all parameters using the CLI,
|
more useable. For information on how to modify all parameters using the CLI,
|
||||||
see
|
see
|
||||||
[vieter-targets-edit(1)](https://rustybever.be/man/vieter/vieter-targets-edit.1.html).
|
[vieter-targets(1)](https://rustybever.be/man/vieter/vieter-targets.1.html).
|
||||||
|
|
||||||
## Reading logs
|
## Reading logs
|
||||||
|
|
||||||
|
|
|
@ -90,10 +90,10 @@ pub:
|
||||||
logs string
|
logs string
|
||||||
}
|
}
|
||||||
|
|
||||||
// build_repo builds, packages & publishes a given Arch package based on the
|
// build_target builds, packages & publishes a given Arch package based on the
|
||||||
// provided target. The base image ID should be of an image previously created
|
// provided target. The base image ID should be of an image previously created
|
||||||
// by create_build_image. It returns the logs of the container.
|
// by create_build_image. It returns the logs of the container.
|
||||||
pub fn build_repo(address string, api_key string, base_image_id string, repo &Target) ?BuildResult {
|
pub fn build_target(address string, api_key string, base_image_id string, target &Target) ?BuildResult {
|
||||||
mut dd := docker.new_conn()?
|
mut dd := docker.new_conn()?
|
||||||
|
|
||||||
defer {
|
defer {
|
||||||
|
@ -101,7 +101,7 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Ta
|
||||||
}
|
}
|
||||||
|
|
||||||
build_arch := os.uname().machine
|
build_arch := os.uname().machine
|
||||||
build_script := create_build_script(address, repo, build_arch)
|
build_script := create_build_script(address, target, build_arch)
|
||||||
|
|
||||||
// We convert the build script into a base64 string, which then gets passed
|
// We convert the build script into a base64 string, which then gets passed
|
||||||
// to the container as an env var
|
// to the container as an env var
|
||||||
|
|
|
@ -4,8 +4,8 @@ echo -e '+ pacman -Syu --needed --noconfirm'
|
||||||
pacman -Syu --needed --noconfirm
|
pacman -Syu --needed --noconfirm
|
||||||
echo -e '+ su builder'
|
echo -e '+ su builder'
|
||||||
su builder
|
su builder
|
||||||
echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo'
|
echo -e '+ git clone --single-branch --depth 1 '\''https://examplerepo.com'\'' repo'
|
||||||
git clone --single-branch --depth 1 --branch main https://examplerepo.com repo
|
git clone --single-branch --depth 1 'https://examplerepo.com' repo
|
||||||
echo -e '+ cd repo'
|
echo -e '+ cd repo'
|
||||||
cd repo
|
cd repo
|
||||||
echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'
|
echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'
|
|
@ -0,0 +1,20 @@
|
||||||
|
echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf'
|
||||||
|
echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
|
||||||
|
echo -e '+ pacman -Syu --needed --noconfirm'
|
||||||
|
pacman -Syu --needed --noconfirm
|
||||||
|
echo -e '+ su builder'
|
||||||
|
su builder
|
||||||
|
echo -e '+ git clone --single-branch --depth 1 --branch main '\''https://examplerepo.com'\'' repo'
|
||||||
|
git clone --single-branch --depth 1 --branch main 'https://examplerepo.com' repo
|
||||||
|
echo -e '+ cd repo'
|
||||||
|
cd repo
|
||||||
|
echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'
|
||||||
|
makepkg --nobuild --syncdeps --needed --noconfirm
|
||||||
|
echo -e '+ source PKGBUILD'
|
||||||
|
source PKGBUILD
|
||||||
|
echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0'
|
||||||
|
curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
|
||||||
|
echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
|
||||||
|
[ "$(id -u)" == 0 ] && exit 0
|
||||||
|
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
|
||||||
|
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done
|
|
@ -0,0 +1,22 @@
|
||||||
|
echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf'
|
||||||
|
echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
|
||||||
|
echo -e '+ pacman -Syu --needed --noconfirm'
|
||||||
|
pacman -Syu --needed --noconfirm
|
||||||
|
echo -e '+ su builder'
|
||||||
|
su builder
|
||||||
|
echo -e '+ mkdir repo'
|
||||||
|
mkdir repo
|
||||||
|
echo -e '+ curl -o repo/PKGBUILD -L '\''https://examplerepo.com'\'''
|
||||||
|
curl -o repo/PKGBUILD -L 'https://examplerepo.com'
|
||||||
|
echo -e '+ cd repo'
|
||||||
|
cd repo
|
||||||
|
echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'
|
||||||
|
makepkg --nobuild --syncdeps --needed --noconfirm
|
||||||
|
echo -e '+ source PKGBUILD'
|
||||||
|
source PKGBUILD
|
||||||
|
echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0'
|
||||||
|
curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
|
||||||
|
echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
|
||||||
|
[ "$(id -u)" == 0 ] && exit 0
|
||||||
|
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
|
||||||
|
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done
|
|
@ -23,20 +23,45 @@ pub fn echo_commands(cmds []string) []string {
|
||||||
}
|
}
|
||||||
|
|
||||||
// create_build_script generates a shell script that builds a given Target.
|
// create_build_script generates a shell script that builds a given Target.
|
||||||
fn create_build_script(address string, repo &Target, build_arch string) string {
|
fn create_build_script(address string, target &Target, build_arch string) string {
|
||||||
repo_url := '$address/$repo.repo'
|
repo_url := '$address/$target.repo'
|
||||||
|
|
||||||
commands := echo_commands([
|
mut commands := [
|
||||||
// This will later be replaced by a proper setting for changing the
|
// This will later be replaced by a proper setting for changing the
|
||||||
// mirrorlist
|
// mirrorlist
|
||||||
"echo -e '[$repo.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf"
|
"echo -e '[$target.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf"
|
||||||
// We need to update the package list of the repo we just added above.
|
// We need to update the package list of the repo we just added above.
|
||||||
// This should however not pull in a lot of packages as long as the
|
// This should however not pull in a lot of packages as long as the
|
||||||
// builder image is rebuilt frequently.
|
// builder image is rebuilt frequently.
|
||||||
'pacman -Syu --needed --noconfirm',
|
'pacman -Syu --needed --noconfirm',
|
||||||
// makepkg can't run as root
|
// makepkg can't run as root
|
||||||
'su builder',
|
'su builder',
|
||||||
'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
|
]
|
||||||
|
|
||||||
|
commands << match target.kind {
|
||||||
|
'git' {
|
||||||
|
if target.branch == '' {
|
||||||
|
[
|
||||||
|
"git clone --single-branch --depth 1 '$target.url' repo",
|
||||||
|
]
|
||||||
|
} else {
|
||||||
|
[
|
||||||
|
"git clone --single-branch --depth 1 --branch $target.branch '$target.url' repo",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
'url' {
|
||||||
|
[
|
||||||
|
'mkdir repo',
|
||||||
|
"curl -o repo/PKGBUILD -L '$target.url'",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
panic("Invalid kind. This shouldn't be possible.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
commands << [
|
||||||
'cd repo',
|
'cd repo',
|
||||||
'makepkg --nobuild --syncdeps --needed --noconfirm',
|
'makepkg --nobuild --syncdeps --needed --noconfirm',
|
||||||
'source PKGBUILD',
|
'source PKGBUILD',
|
||||||
|
@ -49,7 +74,7 @@ fn create_build_script(address string, repo &Target, build_arch string) string {
|
||||||
// we're in root so we don't proceed.
|
// we're in root so we don't proceed.
|
||||||
'[ "\$(id -u)" == 0 ] && exit 0',
|
'[ "\$(id -u)" == 0 ] && exit 0',
|
||||||
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done',
|
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done',
|
||||||
])
|
]
|
||||||
|
|
||||||
return commands.join('\n')
|
return echo_commands(commands).join('\n')
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,15 +2,42 @@ module build
|
||||||
|
|
||||||
import models { Target }
|
import models { Target }
|
||||||
|
|
||||||
fn test_create_build_script() {
|
fn test_create_build_script_git_branch() {
|
||||||
target := Target{
|
target := Target{
|
||||||
id: 1
|
id: 1
|
||||||
|
kind: 'git'
|
||||||
url: 'https://examplerepo.com'
|
url: 'https://examplerepo.com'
|
||||||
branch: 'main'
|
branch: 'main'
|
||||||
repo: 'vieter'
|
repo: 'vieter'
|
||||||
}
|
}
|
||||||
build_script := create_build_script('https://example.com', target, 'x86_64')
|
build_script := create_build_script('https://example.com', target, 'x86_64')
|
||||||
expected := $embed_file('build_script.sh')
|
expected := $embed_file('build_script_git_branch.sh')
|
||||||
|
|
||||||
|
assert build_script == expected.to_string().trim_space()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_create_build_script_git() {
|
||||||
|
target := Target{
|
||||||
|
id: 1
|
||||||
|
kind: 'git'
|
||||||
|
url: 'https://examplerepo.com'
|
||||||
|
repo: 'vieter'
|
||||||
|
}
|
||||||
|
build_script := create_build_script('https://example.com', target, 'x86_64')
|
||||||
|
expected := $embed_file('build_script_git.sh')
|
||||||
|
|
||||||
|
assert build_script == expected.to_string().trim_space()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_create_build_script_url() {
|
||||||
|
target := Target{
|
||||||
|
id: 1
|
||||||
|
kind: 'url'
|
||||||
|
url: 'https://examplerepo.com'
|
||||||
|
repo: 'vieter'
|
||||||
|
}
|
||||||
|
build_script := create_build_script('https://example.com', target, 'x86_64')
|
||||||
|
expected := $embed_file('build_script_url.sh')
|
||||||
|
|
||||||
assert build_script == expected.to_string().trim_space()
|
assert build_script == expected.to_string().trim_space()
|
||||||
}
|
}
|
||||||
|
|
|
@ -40,18 +40,17 @@ pub fn (c &Client) get_target(id int) ?Target {
|
||||||
return data.data
|
return data.data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct NewTarget {
|
||||||
|
kind string
|
||||||
|
url string
|
||||||
|
branch string
|
||||||
|
repo string
|
||||||
|
arch []string
|
||||||
|
}
|
||||||
|
|
||||||
// add_target adds a new target to the server.
|
// add_target adds a new target to the server.
|
||||||
pub fn (c &Client) add_target(url string, branch string, repo string, arch []string) ?Response<string> {
|
pub fn (c &Client) add_target(t NewTarget) ?Response<string> {
|
||||||
mut params := {
|
params := models.params_from<NewTarget>(t)
|
||||||
'url': url
|
|
||||||
'branch': branch
|
|
||||||
'repo': repo
|
|
||||||
}
|
|
||||||
|
|
||||||
if arch.len > 0 {
|
|
||||||
params['arch'] = arch.join(',')
|
|
||||||
}
|
|
||||||
|
|
||||||
data := c.send_request<string>(Method.post, '/api/v1/targets', params)?
|
data := c.send_request<string>(Method.post, '/api/v1/targets', params)?
|
||||||
|
|
||||||
return data
|
return data
|
||||||
|
|
|
@ -6,9 +6,9 @@ import os
|
||||||
import build
|
import build
|
||||||
|
|
||||||
// build locally builds the target with the given id.
|
// build locally builds the target with the given id.
|
||||||
fn build(conf Config, repo_id int) ? {
|
fn build(conf Config, target_id int) ? {
|
||||||
c := client.new(conf.address, conf.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
repo := c.get_target(repo_id)?
|
target := c.get_target(target_id)?
|
||||||
|
|
||||||
build_arch := os.uname().machine
|
build_arch := os.uname().machine
|
||||||
|
|
||||||
|
@ -16,7 +16,7 @@ fn build(conf Config, repo_id int) ? {
|
||||||
image_id := build.create_build_image(conf.base_image)?
|
image_id := build.create_build_image(conf.base_image)?
|
||||||
|
|
||||||
println('Running build...')
|
println('Running build...')
|
||||||
res := build.build_repo(conf.address, conf.api_key, image_id, repo)?
|
res := build.build_target(conf.address, conf.api_key, image_id, target)?
|
||||||
|
|
||||||
println('Removing build image...')
|
println('Removing build image...')
|
||||||
|
|
||||||
|
@ -29,6 +29,6 @@ fn build(conf Config, repo_id int) ? {
|
||||||
dd.remove_image(image_id)?
|
dd.remove_image(image_id)?
|
||||||
|
|
||||||
println('Uploading logs to Vieter...')
|
println('Uploading logs to Vieter...')
|
||||||
c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code,
|
c.add_build_log(target.id, res.start_time, res.end_time, build_arch, res.exit_code,
|
||||||
res.logs)?
|
res.logs)?
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,7 +3,7 @@ module targets
|
||||||
import cli
|
import cli
|
||||||
import vieter.vconf
|
import vieter.vconf
|
||||||
import cron.expression { parse_expression }
|
import cron.expression { parse_expression }
|
||||||
import client
|
import client { NewTarget }
|
||||||
import console
|
import console
|
||||||
import models { TargetFilter }
|
import models { TargetFilter }
|
||||||
|
|
||||||
|
@ -65,14 +65,34 @@ pub fn cmd() cli.Command {
|
||||||
},
|
},
|
||||||
cli.Command{
|
cli.Command{
|
||||||
name: 'add'
|
name: 'add'
|
||||||
required_args: 3
|
required_args: 2
|
||||||
usage: 'url branch repo'
|
usage: 'url repo'
|
||||||
description: 'Add a new Git repository target.'
|
description: 'Add a new target with the given URL & target repo.'
|
||||||
|
flags: [
|
||||||
|
cli.Flag{
|
||||||
|
name: 'kind'
|
||||||
|
description: "Kind of target to add. Defaults to 'git' if not specified. One of 'git', 'url'."
|
||||||
|
flag: cli.FlagType.string
|
||||||
|
default_value: ['git']
|
||||||
|
},
|
||||||
|
cli.Flag{
|
||||||
|
name: 'branch'
|
||||||
|
description: "Which branch to clone; only applies to kind 'git'."
|
||||||
|
flag: cli.FlagType.string
|
||||||
|
},
|
||||||
|
]
|
||||||
execute: fn (cmd cli.Command) ? {
|
execute: fn (cmd cli.Command) ? {
|
||||||
config_file := cmd.flags.get_string('config-file')?
|
config_file := cmd.flags.get_string('config-file')?
|
||||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
||||||
|
|
||||||
add(conf, cmd.args[0], cmd.args[1], cmd.args[2])?
|
t := NewTarget{
|
||||||
|
kind: cmd.flags.get_string('kind')?
|
||||||
|
url: cmd.args[0]
|
||||||
|
repo: cmd.args[1]
|
||||||
|
branch: cmd.flags.get_string('branch') or { '' }
|
||||||
|
}
|
||||||
|
|
||||||
|
add(conf, t)?
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
cli.Command{
|
cli.Command{
|
||||||
|
@ -103,11 +123,11 @@ pub fn cmd() cli.Command {
|
||||||
name: 'edit'
|
name: 'edit'
|
||||||
required_args: 1
|
required_args: 1
|
||||||
usage: 'id'
|
usage: 'id'
|
||||||
description: 'Edit the Git repository target that matches the given id.'
|
description: 'Edit the target that matches the given id.'
|
||||||
flags: [
|
flags: [
|
||||||
cli.Flag{
|
cli.Flag{
|
||||||
name: 'url'
|
name: 'url'
|
||||||
description: 'URL of the Git repository.'
|
description: 'URL value. Meaning depends on kind of target.'
|
||||||
flag: cli.FlagType.string
|
flag: cli.FlagType.string
|
||||||
},
|
},
|
||||||
cli.Flag{
|
cli.Flag{
|
||||||
|
@ -130,6 +150,11 @@ pub fn cmd() cli.Command {
|
||||||
description: 'Cron schedule for repository.'
|
description: 'Cron schedule for repository.'
|
||||||
flag: cli.FlagType.string
|
flag: cli.FlagType.string
|
||||||
},
|
},
|
||||||
|
cli.Flag{
|
||||||
|
name: 'kind'
|
||||||
|
description: 'Kind of target.'
|
||||||
|
flag: cli.FlagType.string
|
||||||
|
},
|
||||||
]
|
]
|
||||||
execute: fn (cmd cli.Command) ? {
|
execute: fn (cmd cli.Command) ? {
|
||||||
config_file := cmd.flags.get_string('config-file')?
|
config_file := cmd.flags.get_string('config-file')?
|
||||||
|
@ -171,22 +196,21 @@ pub fn cmd() cli.Command {
|
||||||
fn list(conf Config, filter TargetFilter) ? {
|
fn list(conf Config, filter TargetFilter) ? {
|
||||||
c := client.new(conf.address, conf.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
repos := c.get_targets(filter)?
|
repos := c.get_targets(filter)?
|
||||||
data := repos.map([it.id.str(), it.url, it.branch, it.repo])
|
data := repos.map([it.id.str(), it.kind, it.url, it.repo])
|
||||||
|
|
||||||
println(console.pretty_table(['id', 'url', 'branch', 'repo'], data)?)
|
println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?)
|
||||||
}
|
}
|
||||||
|
|
||||||
// add adds a new repository to the server's list.
|
// add adds a new repository to the server's list.
|
||||||
fn add(conf Config, url string, branch string, repo string) ? {
|
fn add(conf Config, t &NewTarget) ? {
|
||||||
c := client.new(conf.address, conf.api_key)
|
c := client.new(conf.address, conf.api_key)
|
||||||
res := c.add_target(url, branch, repo, [])?
|
res := c.add_target(t)?
|
||||||
|
|
||||||
println(res.message)
|
println(res.message)
|
||||||
}
|
}
|
||||||
|
|
||||||
// remove removes a repository from the server's list.
|
// remove removes a repository from the server's list.
|
||||||
fn remove(conf Config, id string) ? {
|
fn remove(conf Config, id string) ? {
|
||||||
// id, _ := get_repo_by_prefix(conf, id_prefix) ?
|
|
||||||
id_int := id.int()
|
id_int := id.int()
|
||||||
|
|
||||||
if id_int != 0 {
|
if id_int != 0 {
|
||||||
|
|
|
@ -71,29 +71,31 @@ fn (mut d Daemon) start_build(sb ScheduledBuild) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// run_build actually starts the build process for a given repo.
|
// run_build actually starts the build process for a given target.
|
||||||
fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) {
|
fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) {
|
||||||
d.linfo('started build: $sb.repo.url $sb.repo.branch')
|
d.linfo('started build: $sb.target.url -> $sb.target.repo')
|
||||||
|
|
||||||
// 0 means success, 1 means failure
|
// 0 means success, 1 means failure
|
||||||
mut status := 0
|
mut status := 0
|
||||||
|
|
||||||
res := build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(),
|
res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(),
|
||||||
&sb.repo) or {
|
&sb.target) or {
|
||||||
d.ldebug('build_repo error: $err.msg()')
|
d.ldebug('build_target error: $err.msg()')
|
||||||
status = 1
|
status = 1
|
||||||
|
|
||||||
build.BuildResult{}
|
build.BuildResult{}
|
||||||
}
|
}
|
||||||
|
|
||||||
if status == 0 {
|
if status == 0 {
|
||||||
d.linfo('finished build: $sb.repo.url $sb.repo.branch; uploading logs...')
|
d.linfo('finished build: $sb.target.url -> $sb.target.repo; uploading logs...')
|
||||||
|
|
||||||
build_arch := os.uname().machine
|
build_arch := os.uname().machine
|
||||||
d.client.add_build_log(sb.repo.id, res.start_time, res.end_time, build_arch, res.exit_code,
|
d.client.add_build_log(sb.target.id, res.start_time, res.end_time, build_arch,
|
||||||
res.logs) or { d.lerror('Failed to upload logs for $sb.repo.url $sb.repo.arch') }
|
res.exit_code, res.logs) or {
|
||||||
|
d.lerror('Failed to upload logs for build: $sb.target.url -> $sb.target.repo')
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
d.linfo('failed build: $sb.repo.url $sb.repo.branch')
|
d.linfo('an error occured during build: $sb.target.url -> $sb.target.repo')
|
||||||
}
|
}
|
||||||
|
|
||||||
stdatomic.store_u64(&d.atomics[build_index], daemon.build_done)
|
stdatomic.store_u64(&d.atomics[build_index], daemon.build_done)
|
||||||
|
|
|
@ -20,7 +20,7 @@ const (
|
||||||
|
|
||||||
struct ScheduledBuild {
|
struct ScheduledBuild {
|
||||||
pub:
|
pub:
|
||||||
repo Target
|
target Target
|
||||||
timestamp time.Time
|
timestamp time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -37,9 +37,9 @@ mut:
|
||||||
global_schedule CronExpression
|
global_schedule CronExpression
|
||||||
api_update_frequency int
|
api_update_frequency int
|
||||||
image_rebuild_frequency int
|
image_rebuild_frequency int
|
||||||
// Repos currently loaded from API.
|
// Targets currently loaded from API.
|
||||||
repos []Target
|
targets []Target
|
||||||
// At what point to update the list of repositories.
|
// At what point to update the list of targets.
|
||||||
api_update_timestamp time.Time
|
api_update_timestamp time.Time
|
||||||
image_build_timestamp time.Time
|
image_build_timestamp time.Time
|
||||||
queue MinHeap<ScheduledBuild>
|
queue MinHeap<ScheduledBuild>
|
||||||
|
@ -51,7 +51,7 @@ mut:
|
||||||
logger shared log.Log
|
logger shared log.Log
|
||||||
}
|
}
|
||||||
|
|
||||||
// init_daemon initializes a new Daemon object. It renews the repositories &
|
// init_daemon initializes a new Daemon object. It renews the targets &
|
||||||
// populates the build queue for the first time.
|
// populates the build queue for the first time.
|
||||||
pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon {
|
pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon {
|
||||||
mut d := Daemon{
|
mut d := Daemon{
|
||||||
|
@ -65,8 +65,8 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st
|
||||||
logger: logger
|
logger: logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize the repos & queue
|
// Initialize the targets & queue
|
||||||
d.renew_repos()
|
d.renew_targets()
|
||||||
d.renew_queue()
|
d.renew_queue()
|
||||||
if !d.rebuild_base_image() {
|
if !d.rebuild_base_image() {
|
||||||
return error('The base image failed to build. The Vieter cron daemon cannot run without an initial builder image.')
|
return error('The base image failed to build. The Vieter cron daemon cannot run without an initial builder image.')
|
||||||
|
@ -76,21 +76,21 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st
|
||||||
}
|
}
|
||||||
|
|
||||||
// run starts the actual daemon process. It runs builds when possible &
|
// run starts the actual daemon process. It runs builds when possible &
|
||||||
// periodically refreshes the list of repositories to ensure we stay in sync.
|
// periodically refreshes the list of targets to ensure we stay in sync.
|
||||||
pub fn (mut d Daemon) run() {
|
pub fn (mut d Daemon) run() {
|
||||||
for {
|
for {
|
||||||
finished_builds := d.clean_finished_builds()
|
finished_builds := d.clean_finished_builds()
|
||||||
|
|
||||||
// Update the API's contents if needed & renew the queue
|
// Update the API's contents if needed & renew the queue
|
||||||
if time.now() >= d.api_update_timestamp {
|
if time.now() >= d.api_update_timestamp {
|
||||||
d.renew_repos()
|
d.renew_targets()
|
||||||
d.renew_queue()
|
d.renew_queue()
|
||||||
}
|
}
|
||||||
// The finished builds should only be rescheduled if the API contents
|
// The finished builds should only be rescheduled if the API contents
|
||||||
// haven't been renewed.
|
// haven't been renewed.
|
||||||
else {
|
else {
|
||||||
for sb in finished_builds {
|
for sb in finished_builds {
|
||||||
d.schedule_build(sb.repo)
|
d.schedule_build(sb.target)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -114,7 +114,7 @@ pub fn (mut d Daemon) run() {
|
||||||
// every second to clean up any finished builds & start new ones.
|
// every second to clean up any finished builds & start new ones.
|
||||||
mut delay := time.Duration(1 * time.second)
|
mut delay := time.Duration(1 * time.second)
|
||||||
|
|
||||||
// Sleep either until we have to refresh the repos or when the next
|
// Sleep either until we have to refresh the targets or when the next
|
||||||
// build has to start, with a minimum of 1 second.
|
// build has to start, with a minimum of 1 second.
|
||||||
if d.current_build_count() == 0 {
|
if d.current_build_count() == 0 {
|
||||||
now := time.now()
|
now := time.now()
|
||||||
|
@ -148,12 +148,13 @@ pub fn (mut d Daemon) run() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// schedule_build adds the next occurence of the given repo build to the queue.
|
// schedule_build adds the next occurence of the given targets build to the
|
||||||
fn (mut d Daemon) schedule_build(repo Target) {
|
// queue.
|
||||||
ce := if repo.schedule != '' {
|
fn (mut d Daemon) schedule_build(target Target) {
|
||||||
parse_expression(repo.schedule) or {
|
ce := if target.schedule != '' {
|
||||||
|
parse_expression(target.schedule) or {
|
||||||
// TODO This shouldn't return an error if the expression is empty.
|
// TODO This shouldn't return an error if the expression is empty.
|
||||||
d.lerror("Error while parsing cron expression '$repo.schedule' (id $repo.id): $err.msg()")
|
d.lerror("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()")
|
||||||
|
|
||||||
d.global_schedule
|
d.global_schedule
|
||||||
}
|
}
|
||||||
|
@ -161,41 +162,41 @@ fn (mut d Daemon) schedule_build(repo Target) {
|
||||||
d.global_schedule
|
d.global_schedule
|
||||||
}
|
}
|
||||||
|
|
||||||
// A repo that can't be scheduled will just be skipped for now
|
// A target that can't be scheduled will just be skipped for now
|
||||||
timestamp := ce.next_from_now() or {
|
timestamp := ce.next_from_now() or {
|
||||||
d.lerror("Couldn't calculate next timestamp from '$repo.schedule'; skipping")
|
d.lerror("Couldn't calculate next timestamp from '$target.schedule'; skipping")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
d.queue.insert(ScheduledBuild{
|
d.queue.insert(ScheduledBuild{
|
||||||
repo: repo
|
target: target
|
||||||
timestamp: timestamp
|
timestamp: timestamp
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// renew_repos requests the newest list of Git repos from the server & replaces
|
// renew_targets requests the newest list of targets from the server & replaces
|
||||||
// the old one.
|
// the old one.
|
||||||
fn (mut d Daemon) renew_repos() {
|
fn (mut d Daemon) renew_targets() {
|
||||||
d.linfo('Renewing repos...')
|
d.linfo('Renewing targets...')
|
||||||
|
|
||||||
mut new_repos := d.client.get_all_targets() or {
|
mut new_targets := d.client.get_all_targets() or {
|
||||||
d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...')
|
d.lerror('Failed to renew targets. Retrying in ${daemon.api_update_retry_timeout}s...')
|
||||||
d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout)
|
d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Filter out any repos that shouldn't run on this architecture
|
// Filter out any targets that shouldn't run on this architecture
|
||||||
cur_arch := os.uname().machine
|
cur_arch := os.uname().machine
|
||||||
new_repos = new_repos.filter(it.arch.any(it.value == cur_arch))
|
new_targets = new_targets.filter(it.arch.any(it.value == cur_arch))
|
||||||
|
|
||||||
d.repos = new_repos
|
d.targets = new_targets
|
||||||
|
|
||||||
d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency)
|
d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency)
|
||||||
}
|
}
|
||||||
|
|
||||||
// renew_queue replaces the old queue with a new one that reflects the newest
|
// renew_queue replaces the old queue with a new one that reflects the newest
|
||||||
// values in repos_map.
|
// values in targets.
|
||||||
fn (mut d Daemon) renew_queue() {
|
fn (mut d Daemon) renew_queue() {
|
||||||
d.linfo('Renewing queue...')
|
d.linfo('Renewing queue...')
|
||||||
mut new_queue := MinHeap<ScheduledBuild>{}
|
mut new_queue := MinHeap<ScheduledBuild>{}
|
||||||
|
@ -225,10 +226,10 @@ fn (mut d Daemon) renew_queue() {
|
||||||
|
|
||||||
d.queue = new_queue
|
d.queue = new_queue
|
||||||
|
|
||||||
// For each repository in repos_map, parse their cron expression (or use
|
// For each target in targets, parse their cron expression (or use the
|
||||||
// the default one if not present) & add them to the queue
|
// default one if not present) & add them to the queue
|
||||||
for repo in d.repos {
|
for target in d.targets {
|
||||||
d.schedule_build(repo)
|
d.schedule_build(target)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -16,9 +16,13 @@ const (
|
||||||
migrations_up = [
|
migrations_up = [
|
||||||
$embed_file('migrations/001-initial/up.sql'),
|
$embed_file('migrations/001-initial/up.sql'),
|
||||||
$embed_file('migrations/002-rename-to-targets/up.sql'),
|
$embed_file('migrations/002-rename-to-targets/up.sql'),
|
||||||
|
$embed_file('migrations/003-target-url-type/up.sql'),
|
||||||
|
]
|
||||||
|
migrations_down = [
|
||||||
|
$embed_file('migrations/001-initial/down.sql'),
|
||||||
|
$embed_file('migrations/002-rename-to-targets/down.sql'),
|
||||||
|
$embed_file('migrations/003-target-url-type/down.sql'),
|
||||||
]
|
]
|
||||||
migrations_down = [$embed_file('migrations/001-initial/down.sql'),
|
|
||||||
$embed_file('migrations/002-rename-to-targets/down.sql')]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// init initializes a database & adds the correct tables.
|
// init initializes a database & adds the correct tables.
|
||||||
|
|
|
@ -0,0 +1,4 @@
|
||||||
|
-- I'm not sure whether I should remove any non-git targets here. Keeping them
|
||||||
|
-- will result in invalid targets, but removing them means losing data.
|
||||||
|
ALTER TABLE Target DROP COLUMN kind;
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
ALTER TABLE Target ADD COLUMN kind TEXT NOT NULL DEFAULT 'git';
|
|
@ -1,5 +1,7 @@
|
||||||
module models
|
module models
|
||||||
|
|
||||||
|
pub const valid_kinds = ['git', 'url']
|
||||||
|
|
||||||
pub struct TargetArch {
|
pub struct TargetArch {
|
||||||
pub:
|
pub:
|
||||||
id int [primary; sql: serial]
|
id int [primary; sql: serial]
|
||||||
|
@ -15,10 +17,13 @@ pub fn (gra &TargetArch) str() string {
|
||||||
pub struct Target {
|
pub struct Target {
|
||||||
pub mut:
|
pub mut:
|
||||||
id int [primary; sql: serial]
|
id int [primary; sql: serial]
|
||||||
// URL of the Git repository
|
kind string [nonull]
|
||||||
|
// If kind is git: URL of the Git repository
|
||||||
|
// If kind is url: URL to PKGBUILD file
|
||||||
url string [nonull]
|
url string [nonull]
|
||||||
// Branch of the Git repository to use
|
// Branch of the Git repository to use; only applicable when kind is git.
|
||||||
branch string [nonull]
|
// If not provided, the repository is cloned with the default branch.
|
||||||
|
branch string
|
||||||
// Which repo the builder should publish packages to
|
// Which repo the builder should publish packages to
|
||||||
repo string [nonull]
|
repo string [nonull]
|
||||||
// Cron schedule describing how frequently to build the repo.
|
// Cron schedule describing how frequently to build the repo.
|
||||||
|
@ -32,6 +37,7 @@ pub mut:
|
||||||
pub fn (gr &Target) str() string {
|
pub fn (gr &Target) str() string {
|
||||||
mut parts := [
|
mut parts := [
|
||||||
'id: $gr.id',
|
'id: $gr.id',
|
||||||
|
'kind: $gr.kind',
|
||||||
'url: $gr.url',
|
'url: $gr.url',
|
||||||
'branch: $gr.branch',
|
'branch: $gr.branch',
|
||||||
'repo: $gr.repo',
|
'repo: $gr.repo',
|
||||||
|
|
|
@ -52,6 +52,11 @@ fn (mut app App) v1_post_target() web.Result {
|
||||||
return app.json(http.Status.bad_request, new_response(err.msg()))
|
return app.json(http.Status.bad_request, new_response(err.msg()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Ensure someone doesn't submit an invalid kind
|
||||||
|
if new_repo.kind !in models.valid_kinds {
|
||||||
|
return app.json(http.Status.bad_request, new_response('Invalid kind.'))
|
||||||
|
}
|
||||||
|
|
||||||
app.db.add_target(new_repo)
|
app.db.add_target(new_repo)
|
||||||
|
|
||||||
return app.json(http.Status.ok, new_response('Repo added successfully.'))
|
return app.json(http.Status.ok, new_response('Repo added successfully.'))
|
||||||
|
|
Loading…
Reference in New Issue