forked from vieter-v/vieter
Merge pull request 'Release 0.5.0: release candidate 1' (#316) from release-0.5.0-rc.1 into main
Reviewed-on: vieter-v/vieter#316
commit
8b72a9fc0f
|
|
@ -26,6 +26,7 @@ gdb.txt
|
|||
|
||||
# Generated docs
|
||||
_docs/
|
||||
docs/resources/_gen/
|
||||
/man/
|
||||
|
||||
# VLS logs
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ skip_clone: true
|
|||
|
||||
pipeline:
|
||||
build:
|
||||
image: 'menci/archlinuxarm:base-devel'
|
||||
image: 'git.rustybever.be/vieter-v/vieter-builder'
|
||||
commands:
|
||||
# Add the vieter repository so we can use the compiler
|
||||
- echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ skip_clone: true
|
|||
|
||||
pipeline:
|
||||
build:
|
||||
image: 'menci/archlinuxarm:base-devel'
|
||||
image: 'git.rustybever.be/vieter-v/vieter-builder'
|
||||
commands:
|
||||
# Add the vieter repository so we can use the compiler
|
||||
- echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
variables:
|
||||
- &vlang_image 'chewingbever/vlang:0.3'
|
||||
- &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2'
|
||||
|
||||
matrix:
|
||||
PLATFORM:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
variables:
|
||||
- &vlang_image 'chewingbever/vlang:0.3'
|
||||
- &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2'
|
||||
|
||||
platform: 'linux/amd64'
|
||||
branches:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
variables:
|
||||
- &vlang_image 'chewingbever/vlang:0.3'
|
||||
- &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2'
|
||||
|
||||
platform: 'linux/amd64'
|
||||
branches: [ 'main' ]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
variables:
|
||||
- &vlang_image 'chewingbever/vlang:0.3'
|
||||
- &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2'
|
||||
|
||||
# These checks already get performed on the feature branches
|
||||
branches:
|
||||
|
|
@ -7,10 +7,21 @@ branches:
|
|||
platform: 'linux/amd64'
|
||||
|
||||
pipeline:
|
||||
# vfmt seems to get confused if these aren't present
|
||||
install-modules:
|
||||
image: *vlang_image
|
||||
pull: true
|
||||
commands:
|
||||
- export VMODULES=$PWD/.vmodules
|
||||
- 'cd src && v install'
|
||||
when:
|
||||
event: [pull_request]
|
||||
|
||||
lint:
|
||||
image: *vlang_image
|
||||
pull: true
|
||||
commands:
|
||||
- export VMODULES=$PWD/.vmodules
|
||||
- make lint
|
||||
when:
|
||||
event: [ pull_request ]
|
||||
event: [pull_request]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
variables:
|
||||
- &vlang_image 'chewingbever/vlang:0.3'
|
||||
- &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2'
|
||||
|
||||
platform: 'linux/amd64'
|
||||
branches:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
variables:
|
||||
- &vlang_image 'chewingbever/vlang:0.3'
|
||||
- &vlang_image 'git.rustybever.be/chewing_bever/vlang:0.3.2'
|
||||
|
||||
matrix:
|
||||
PLATFORM:
|
||||
|
|
|
|||
27
CHANGELOG.md
27
CHANGELOG.md
|
|
@ -7,6 +7,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||
|
||||
## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
|
||||
|
||||
## [0.5.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0-rc.1)
|
||||
|
||||
### Added
|
||||
|
||||
* Allow specifying subdirectory inside Git repository
|
||||
* Added option to deploy using agent-server architecture instead of cron daemon
|
||||
* Allow scheduling builds on the server from the CLI tool instead of building
|
||||
them locally
|
||||
* Allow force-building packages, meaning the build won't check if the
|
||||
repository is already up to date
|
||||
|
||||
### Changed
|
||||
|
||||
* Migrated codebase to V 0.3.2
|
||||
* Cron expression parser now uses bitfields instead of bool arrays
|
||||
|
||||
### Fixed
|
||||
|
||||
* Arch value for target is now properly set if not provided
|
||||
* Allow NULL values for branch in database
|
||||
* Endpoint for adding targets now returns the correct id
|
||||
* CLI now correctly errors and doesn't error when sending requests
|
||||
* Fixed possible infinite loop when removing old build images
|
||||
* Check whether build image still exists before starting build
|
||||
* Don't run makepkg `prepare()` function twice
|
||||
* Don't buffer stdout in Docker containers
|
||||
|
||||
## [0.4.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.4.0)
|
||||
|
||||
### Added
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
FROM chewingbever/vlang:0.3 AS builder
|
||||
FROM git.rustybever.be/chewing_bever/vlang:0.3.2 AS builder
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
ARG CI_COMMIT_SHA
|
||||
|
|
@ -23,6 +23,7 @@ RUN if [ -n "${CI_COMMIT_SHA}" ]; then \
|
|||
"https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \
|
||||
chmod +x vieter ; \
|
||||
else \
|
||||
cd src && v install && cd .. && \
|
||||
LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static' make prod && \
|
||||
mv pvieter vieter ; \
|
||||
fi
|
||||
|
|
|
|||
6
Makefile
6
Makefile
|
|
@ -3,7 +3,7 @@ SRC_DIR := src
|
|||
SOURCES != find '$(SRC_DIR)' -iname '*.v'
|
||||
|
||||
V_PATH ?= v
|
||||
V := $(V_PATH) -showcc -gc boehm
|
||||
V := $(V_PATH) -showcc -gc boehm -W -d use_openssl
|
||||
|
||||
all: vieter
|
||||
|
||||
|
|
@ -92,9 +92,9 @@ clean:
|
|||
.PHONY: autofree
|
||||
autofree: afvieter
|
||||
afvieter: $(SOURCES)
|
||||
$(V_PATH) -showcc -autofree -o afvieter $(SRC_DIR)
|
||||
$(V) -showcc -autofree -o afvieter $(SRC_DIR)
|
||||
|
||||
.PHONY: skip-unused
|
||||
skip-unused: suvieter
|
||||
suvieter: $(SOURCES)
|
||||
$(V_PATH) -showcc -skip-unused -o suvieter $(SRC_DIR)
|
||||
$(V) -skip-unused -o suvieter $(SRC_DIR)
|
||||
|
|
|
|||
2
PKGBUILD
2
PKGBUILD
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
pkgbase='vieter'
|
||||
pkgname='vieter'
|
||||
pkgver='0.4.0'
|
||||
pkgver='0.5.0-rc.1'
|
||||
pkgrel=1
|
||||
pkgdesc="Lightweight Arch repository server & package build system"
|
||||
depends=('glibc' 'openssl' 'libarchive' 'sqlite')
|
||||
|
|
|
|||
|
|
@ -21,7 +21,8 @@ quicker.
|
|||
I chose [V](https://vlang.io/) as I've been very intrigued by this language for
|
||||
a while now. I wanted a fast language that I could code while relaxing, without
|
||||
having to exert too much mental effort & V seemed like the right choice for
|
||||
that.
|
||||
that. Sadly, this didn't quite turn out the way I expected, but I'm sticking
|
||||
with it anyways ;p
|
||||
|
||||
## Features
|
||||
|
||||
|
|
@ -49,7 +50,7 @@ update`.
|
|||
|
||||
I used to maintain a mirror that tracked the latest master, but nowadays, I
|
||||
maintain a Docker image containing the specific compiler version that Vieter
|
||||
builds with. Currently, this is V 0.3.
|
||||
builds with. Currently, this is V 0.3.2.
|
||||
|
||||
## Contributing
|
||||
|
||||
|
|
|
|||
|
|
@ -97,3 +97,25 @@ configuration variable required for each command.
|
|||
build`.
|
||||
* Default: `archlinux:base-devel`
|
||||
|
||||
### `vieter agent`
|
||||
|
||||
* `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`,
|
||||
`WARN`, `INFO` or `DEBUG`.
|
||||
* Default: `WARN`
|
||||
* `address`: *public* URL of the Vieter repository server to build for. From
|
||||
this server jobs are retrieved. All built packages are published to this
|
||||
server.
|
||||
* `api_key`: API key of the above server.
|
||||
* `data_dir`: directory to store log file in.
|
||||
* `max_concurrent_builds`: how many builds to run at the same time.
|
||||
* Default: `1`
|
||||
* `polling_frequency`: how often (in seconds) to poll the server for new
|
||||
builds. Note that the agent might poll more frequently when it's actively
|
||||
processing builds.
|
||||
* `image_rebuild_frequency`: Vieter periodically builds images that are then
|
||||
used as a basis for running build containers. This is to prevent each build
|
||||
from downloading an entire repository worth of dependencies. This setting
|
||||
defines how frequently (in minutes) to rebuild these images.
|
||||
* Default: `1440` (every 24 hours)
|
||||
* `arch`: architecture for which this agent should pull down builds (e.g.
|
||||
`x86_64`)
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ branch. This branch will be the most up to date, but does not give any
|
|||
guarantees about stability, so beware!
|
||||
|
||||
Thanks to the single-binary design of Vieter, this image can be used both for
|
||||
the repository server & the cron daemon.
|
||||
the repository server, the cron daemon and the agent.
|
||||
|
||||
Below is an example compose file to set up both the repository server & the
|
||||
cron daemon:
|
||||
|
|
@ -76,7 +76,7 @@ architectures will build on both.
|
|||
## Binary
|
||||
|
||||
On the
|
||||
[releases](https://git.rustybever.be/vieter/vieter/releases)
|
||||
[releases](https://git.rustybever.be/vieter-v/vieter/releases)
|
||||
page, you can find statically compiled binaries for all
|
||||
released versions. This is the same binary as used inside
|
||||
the Docker images.
|
||||
|
|
@ -106,5 +106,5 @@ guarantee that a compiler update won't temporarily break them.
|
|||
|
||||
## Building from source
|
||||
|
||||
The project [README](https://git.rustybever.be/vieter/vieter#building) contains
|
||||
instructions for building Vieter from source.
|
||||
The project [README](https://git.rustybever.be/vieter-v/vieter#building)
|
||||
contains instructions for building Vieter from source.
|
||||
|
|
|
|||
|
|
@ -37,6 +37,6 @@ Each section can consist of as many of these parts as necessary.
|
|||
## CLI tool
|
||||
|
||||
The Vieter binary contains a command that shows you the next matching times for
|
||||
a given expression. This can be useful to understand the syntax. For more
|
||||
a given expression. This can be useful for understanding the syntax. For more
|
||||
information, see
|
||||
[vieter-schedule(1)](https://rustybever.be/man/vieter/vieter-schedule.1.html).
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
module agent
|
||||
|
||||
import log
|
||||
import os
|
||||
import util
|
||||
|
||||
const log_file_name = 'vieter.agent.log'
|
||||
|
||||
// agent starts an agent service
|
||||
pub fn agent(conf Config) ! {
|
||||
log_level := log.level_from_tag(conf.log_level) or {
|
||||
return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
|
||||
}
|
||||
|
||||
mut logger := log.Log{
|
||||
level: log_level
|
||||
}
|
||||
|
||||
os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') }
|
||||
|
||||
log_file := os.join_path_single(conf.data_dir, agent.log_file_name)
|
||||
logger.set_full_logpath(log_file)
|
||||
logger.log_to_console_too()
|
||||
|
||||
mut d := agent_init(logger, conf)
|
||||
d.run()
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
module agent
|
||||
|
||||
import cli
|
||||
import conf as vconf
|
||||
|
||||
struct Config {
|
||||
pub:
|
||||
log_level string = 'WARN'
|
||||
// Architecture that the agent represents
|
||||
arch string
|
||||
api_key string
|
||||
address string
|
||||
data_dir string
|
||||
max_concurrent_builds int = 1
|
||||
polling_frequency int = 30
|
||||
image_rebuild_frequency int = 1440
|
||||
}
|
||||
|
||||
// cmd returns the cli module that handles the cron daemon.
|
||||
pub fn cmd() cli.Command {
|
||||
return cli.Command{
|
||||
name: 'agent'
|
||||
description: 'Start an agent daemon.'
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||
|
||||
agent(conf)!
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,197 @@
|
|||
module agent
|
||||
|
||||
import log
|
||||
import sync.stdatomic
|
||||
import build
|
||||
import models { BuildConfig }
|
||||
import client
|
||||
import time
|
||||
import os
|
||||
|
||||
const (
|
||||
build_empty = 0
|
||||
build_running = 1
|
||||
build_done = 2
|
||||
)
|
||||
|
||||
struct AgentDaemon {
|
||||
logger shared log.Log
|
||||
conf Config
|
||||
client client.Client
|
||||
mut:
|
||||
images ImageManager
|
||||
// Which builds are currently running; length is conf.max_concurrent_builds
|
||||
builds []BuildConfig
|
||||
// Atomic variables used to detect when a build has finished; length is
|
||||
// conf.max_concurrent_builds
|
||||
atomics []u64
|
||||
}
|
||||
|
||||
// agent_init initializes a new agent
|
||||
fn agent_init(logger log.Log, conf Config) AgentDaemon {
|
||||
mut d := AgentDaemon{
|
||||
logger: logger
|
||||
client: client.new(conf.address, conf.api_key)
|
||||
conf: conf
|
||||
images: new_image_manager(conf.image_rebuild_frequency * 60)
|
||||
builds: []BuildConfig{len: conf.max_concurrent_builds}
|
||||
atomics: []u64{len: conf.max_concurrent_builds}
|
||||
}
|
||||
|
||||
return d
|
||||
}
|
||||
|
||||
// run starts the actual agent daemon. This function will run forever.
|
||||
pub fn (mut d AgentDaemon) run() {
|
||||
// This is just so that the very first time the loop is ran, the jobs are
|
||||
// always polled
|
||||
mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency)
|
||||
mut sleep_time := 0 * time.second
|
||||
mut finished, mut empty, mut running := 0, 0, 0
|
||||
|
||||
for {
|
||||
if sleep_time > 0 {
|
||||
d.ldebug('Sleeping for $sleep_time')
|
||||
time.sleep(sleep_time)
|
||||
}
|
||||
|
||||
finished, empty = d.update_atomics()
|
||||
running = d.conf.max_concurrent_builds - finished - empty
|
||||
|
||||
// No new finished builds and no free slots, so there's nothing to be
|
||||
// done
|
||||
if finished + empty == 0 {
|
||||
sleep_time = 1 * time.second
|
||||
continue
|
||||
}
|
||||
|
||||
// Builds have finished, so old builder images might have freed up.
|
||||
// TODO this might query the docker daemon too frequently.
|
||||
if finished > 0 {
|
||||
d.images.clean_old_images()
|
||||
}
|
||||
|
||||
// The agent will always poll for new jobs after at most
|
||||
// `polling_frequency` seconds. However, when jobs have finished, the
|
||||
// agent will also poll for new jobs. This is because jobs are often
|
||||
// clustered together (especially when mostly using the global cron
|
||||
// schedule), so there's a much higher chance jobs are available.
|
||||
if finished > 0 || time.now() >= last_poll_time.add_seconds(d.conf.polling_frequency) {
|
||||
d.ldebug('Polling for new jobs')
|
||||
|
||||
new_configs := d.client.poll_jobs(d.conf.arch, finished + empty) or {
|
||||
d.lerror('Failed to poll jobs: $err.msg()')
|
||||
|
||||
// TODO pick a better delay here
|
||||
sleep_time = 5 * time.second
|
||||
continue
|
||||
}
|
||||
|
||||
d.ldebug('Received $new_configs.len jobs')
|
||||
|
||||
last_poll_time = time.now()
|
||||
|
||||
for config in new_configs {
|
||||
// Make sure a recent build base image is available for
|
||||
// building the config
|
||||
if !d.images.up_to_date(config.base_image) {
|
||||
d.linfo('Building builder image from base image $config.base_image')
|
||||
|
||||
// TODO handle this better than to just skip the config
|
||||
d.images.refresh_image(config.base_image) or {
|
||||
d.lerror(err.msg())
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// It's technically still possible that the build image is
|
||||
// removed in the very short period between building the
|
||||
// builder image and starting a build container with it. If
|
||||
// this happens, faith really just didn't want you to do this
|
||||
// build.
|
||||
|
||||
d.start_build(config)
|
||||
running++
|
||||
}
|
||||
}
|
||||
|
||||
// The agent is not doing anything, so we just wait until the next poll
|
||||
// time
|
||||
if running == 0 {
|
||||
sleep_time = last_poll_time.add_seconds(d.conf.polling_frequency) - time.now()
|
||||
} else {
|
||||
sleep_time = 1 * time.second
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// update_atomics checks for each build whether it's completed, and sets it to
|
||||
// empty again if so. The return value is a tuple `(finished, empty)` where
|
||||
// `finished` is how many builds were just finished and thus set to empty, and
|
||||
// `empty` is how many build slots were already empty. The amount of running
|
||||
// builds can then be calculated by substracting these two values from the
|
||||
// total allowed concurrent builds.
|
||||
fn (mut d AgentDaemon) update_atomics() (int, int) {
|
||||
mut finished := 0
|
||||
mut empty := 0
|
||||
|
||||
for i in 0 .. d.atomics.len {
|
||||
if stdatomic.load_u64(&d.atomics[i]) == agent.build_done {
|
||||
stdatomic.store_u64(&d.atomics[i], agent.build_empty)
|
||||
finished++
|
||||
} else if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty {
|
||||
empty++
|
||||
}
|
||||
}
|
||||
|
||||
return finished, empty
|
||||
}
|
||||
|
||||
// start_build starts a build for the given BuildConfig.
|
||||
fn (mut d AgentDaemon) start_build(config BuildConfig) bool {
|
||||
for i in 0 .. d.atomics.len {
|
||||
if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty {
|
||||
stdatomic.store_u64(&d.atomics[i], agent.build_running)
|
||||
d.builds[i] = config
|
||||
|
||||
go d.run_build(i, config)
|
||||
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// run_build actually starts the build process for a given target.
|
||||
fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) {
|
||||
d.linfo('started build: $config')
|
||||
|
||||
// 0 means success, 1 means failure
|
||||
mut status := 0
|
||||
|
||||
new_config := BuildConfig{
|
||||
...config
|
||||
base_image: d.images.get(config.base_image)
|
||||
}
|
||||
|
||||
res := build.build_config(d.client.address, d.client.api_key, new_config) or {
|
||||
d.ldebug('build_config error: $err.msg()')
|
||||
status = 1
|
||||
|
||||
build.BuildResult{}
|
||||
}
|
||||
|
||||
if status == 0 {
|
||||
d.linfo('Uploading build logs for $config')
|
||||
|
||||
// TODO use the arch value here
|
||||
build_arch := os.uname().machine
|
||||
d.client.add_build_log(config.target_id, res.start_time, res.end_time, build_arch,
|
||||
res.exit_code, res.logs) or { d.lerror('Failed to upload logs for $config') }
|
||||
} else {
|
||||
d.lwarn('an error occurred during build: $config')
|
||||
}
|
||||
|
||||
stdatomic.store_u64(&d.atomics[build_index], agent.build_done)
|
||||
}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
module agent
|
||||
|
||||
import time
|
||||
import docker
|
||||
import build
|
||||
|
||||
// An ImageManager is a utility that creates builder images from given base
|
||||
// images, updating these builder images if they've become too old. This
|
||||
// structure can manage images from any number of base images, paving the way
|
||||
// for configurable base images per target/repository.
|
||||
struct ImageManager {
|
||||
max_image_age int [required]
|
||||
mut:
|
||||
// For each base image, one or more builder images can exist at the same
|
||||
// time
|
||||
images map[string][]string [required]
|
||||
// For each base image, we track when its newest image was built
|
||||
timestamps map[string]time.Time [required]
|
||||
}
|
||||
|
||||
// new_image_manager initializes a new image manager.
|
||||
fn new_image_manager(max_image_age int) ImageManager {
|
||||
return ImageManager{
|
||||
max_image_age: max_image_age
|
||||
images: map[string][]string{}
|
||||
timestamps: map[string]time.Time{}
|
||||
}
|
||||
}
|
||||
|
||||
// get returns the name of the newest image for the given base image. Note that
|
||||
// this function should only be called *after* a first call to `refresh_image`.
|
||||
pub fn (m &ImageManager) get(base_image string) string {
|
||||
return m.images[base_image].last()
|
||||
}
|
||||
|
||||
// up_to_date returns true if the last known builder image exists and is up to
|
||||
// date. If this function returns true, the last builder image may be used to
|
||||
// perform a build.
|
||||
pub fn (mut m ImageManager) up_to_date(base_image string) bool {
|
||||
if base_image !in m.timestamps
|
||||
|| m.timestamps[base_image].add_seconds(m.max_image_age) <= time.now() {
|
||||
return false
|
||||
}
|
||||
|
||||
// It's possible the image has been removed by some external event, so we
|
||||
// check whether it actually exists as well.
|
||||
mut dd := docker.new_conn() or { return false }
|
||||
|
||||
defer {
|
||||
dd.close() or {}
|
||||
}
|
||||
|
||||
dd.image_inspect(m.images[base_image].last()) or {
|
||||
// Image doesn't exist, so we stop tracking it
|
||||
if err.code() == 404 {
|
||||
m.images[base_image].delete_last()
|
||||
m.timestamps.delete(base_image)
|
||||
}
|
||||
|
||||
// If the inspect fails, it's either because the image doesn't exist or
|
||||
// because of some other error. Either way, we can't know *for certain*
|
||||
// that the image exists, so we return false.
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// refresh_image builds a new builder image from the given base image. This
|
||||
// function should only be called if `up_to_date` returned false.
|
||||
fn (mut m ImageManager) refresh_image(base_image string) ! {
|
||||
// TODO use better image tags for built images
|
||||
new_image := build.create_build_image(base_image) or {
|
||||
return error('Failed to build builder image from base image $base_image')
|
||||
}
|
||||
|
||||
m.images[base_image] << new_image
|
||||
m.timestamps[base_image] = time.now()
|
||||
}
|
||||
|
||||
// clean_old_images removes all older builder images that are no longer in use.
|
||||
// The function will always leave at least one builder image, namely the newest
|
||||
// one.
|
||||
fn (mut m ImageManager) clean_old_images() {
|
||||
mut dd := docker.new_conn() or { return }
|
||||
|
||||
defer {
|
||||
dd.close() or {}
|
||||
}
|
||||
|
||||
mut i := 0
|
||||
|
||||
for image in m.images.keys() {
|
||||
i = 0
|
||||
|
||||
for i < m.images[image].len - 1 {
|
||||
// For each builder image, we try to remove it by calling the Docker
|
||||
// API. If the function returns an error or false, that means the image
|
||||
// wasn't deleted. Therefore, we move the index over. If the function
|
||||
// returns true, the array's length has decreased by one so we don't
|
||||
// move the index.
|
||||
dd.remove_image(m.images[image][i]) or {
|
||||
// The image was removed by an external event
|
||||
if err.code() == 404 {
|
||||
m.images[image].delete(i)
|
||||
}
|
||||
// The image couldn't be removed, so we need to keep track of
|
||||
// it
|
||||
else {
|
||||
i += 1
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
m.images[image].delete(i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
module agent
|
||||
|
||||
import log
|
||||
|
||||
// log a message with the given level
|
||||
pub fn (mut d AgentDaemon) log(msg string, level log.Level) {
|
||||
lock d.logger {
|
||||
d.logger.send_output(msg, level)
|
||||
}
|
||||
}
|
||||
|
||||
// lfatal create a log message with the fatal level
|
||||
pub fn (mut d AgentDaemon) lfatal(msg string) {
|
||||
d.log(msg, log.Level.fatal)
|
||||
}
|
||||
|
||||
// lerror create a log message with the error level
|
||||
pub fn (mut d AgentDaemon) lerror(msg string) {
|
||||
d.log(msg, log.Level.error)
|
||||
}
|
||||
|
||||
// lwarn create a log message with the warn level
|
||||
pub fn (mut d AgentDaemon) lwarn(msg string) {
|
||||
d.log(msg, log.Level.warn)
|
||||
}
|
||||
|
||||
// linfo create a log message with the info level
|
||||
pub fn (mut d AgentDaemon) linfo(msg string) {
|
||||
d.log(msg, log.Level.info)
|
||||
}
|
||||
|
||||
// ldebug create a log message with the debug level
|
||||
pub fn (mut d AgentDaemon) ldebug(msg string) {
|
||||
d.log(msg, log.Level.debug)
|
||||
}
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
module build
|
||||
|
||||
import vieter_v.docker
|
||||
import docker
|
||||
import encoding.base64
|
||||
import time
|
||||
import os
|
||||
import strings
|
||||
import util
|
||||
import models { Target }
|
||||
import models { BuildConfig, Target }
|
||||
|
||||
const (
|
||||
container_build_dir = '/build'
|
||||
|
|
@ -21,8 +21,8 @@ const (
|
|||
// system, install some necessary packages & creates a non-root user to run
|
||||
// makepkg with. The base image should be some Linux distribution that uses
|
||||
// Pacman as its package manager.
|
||||
pub fn create_build_image(base_image string) ?string {
|
||||
mut dd := docker.new_conn()?
|
||||
pub fn create_build_image(base_image string) !string {
|
||||
mut dd := docker.new_conn()!
|
||||
|
||||
defer {
|
||||
dd.close() or {}
|
||||
|
|
@ -57,15 +57,15 @@ pub fn create_build_image(base_image string) ?string {
|
|||
image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' }
|
||||
|
||||
// We pull the provided image
|
||||
dd.pull_image(image_name, image_tag)?
|
||||
dd.pull_image(image_name, image_tag)!
|
||||
|
||||
id := dd.container_create(c)?.id
|
||||
// id := docker.create_container(c)?
|
||||
dd.container_start(id)?
|
||||
id := dd.container_create(c)!.id
|
||||
// id := docker.create_container(c)!
|
||||
dd.container_start(id)!
|
||||
|
||||
// This loop waits until the container has stopped, so we can remove it after
|
||||
for {
|
||||
data := dd.container_inspect(id)?
|
||||
data := dd.container_inspect(id)!
|
||||
|
||||
if !data.state.running {
|
||||
break
|
||||
|
|
@ -79,8 +79,8 @@ pub fn create_build_image(base_image string) ?string {
|
|||
// TODO also add the base image's name into the image name to prevent
|
||||
// conflicts.
|
||||
tag := time.sys_mono_now().str()
|
||||
image := dd.create_image_from_container(id, 'vieter-build', tag)?
|
||||
dd.container_remove(id)?
|
||||
image := dd.create_image_from_container(id, 'vieter-build', tag)!
|
||||
dd.container_remove(id)!
|
||||
|
||||
return image.id
|
||||
}
|
||||
|
|
@ -93,25 +93,32 @@ pub:
|
|||
logs string
|
||||
}
|
||||
|
||||
// build_target builds, packages & publishes a given Arch package based on the
|
||||
// build_target builds the given target. Internally it calls `build_config`.
|
||||
pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool) !BuildResult {
|
||||
config := target.as_build_config(base_image_id, force)
|
||||
|
||||
return build_config(address, api_key, config)
|
||||
}
|
||||
|
||||
// build_config builds, packages & publishes a given Arch package based on the
|
||||
// provided target. The base image ID should be of an image previously created
|
||||
// by create_build_image. It returns the logs of the container.
|
||||
pub fn build_target(address string, api_key string, base_image_id string, target &Target) ?BuildResult {
|
||||
mut dd := docker.new_conn()?
|
||||
pub fn build_config(address string, api_key string, config BuildConfig) !BuildResult {
|
||||
mut dd := docker.new_conn()!
|
||||
|
||||
defer {
|
||||
dd.close() or {}
|
||||
}
|
||||
|
||||
build_arch := os.uname().machine
|
||||
build_script := create_build_script(address, target, build_arch)
|
||||
build_script := create_build_script(address, config, build_arch)
|
||||
|
||||
// We convert the build script into a base64 string, which then gets passed
|
||||
// to the container as an env var
|
||||
base64_script := base64.encode_str(build_script)
|
||||
|
||||
c := docker.NewContainer{
|
||||
image: '$base_image_id'
|
||||
image: '$config.base_image'
|
||||
env: [
|
||||
'BUILD_SCRIPT=$base64_script',
|
||||
'API_KEY=$api_key',
|
||||
|
|
@ -125,25 +132,25 @@ pub fn build_target(address string, api_key string, base_image_id string, target
|
|||
user: '0:0'
|
||||
}
|
||||
|
||||
id := dd.container_create(c)?.id
|
||||
dd.container_start(id)?
|
||||
id := dd.container_create(c)!.id
|
||||
dd.container_start(id)!
|
||||
|
||||
mut data := dd.container_inspect(id)?
|
||||
mut data := dd.container_inspect(id)!
|
||||
|
||||
// This loop waits until the container has stopped, so we can remove it after
|
||||
for data.state.running {
|
||||
time.sleep(1 * time.second)
|
||||
|
||||
data = dd.container_inspect(id)?
|
||||
data = dd.container_inspect(id)!
|
||||
}
|
||||
|
||||
mut logs_stream := dd.container_get_logs(id)?
|
||||
mut logs_stream := dd.container_get_logs(id)!
|
||||
|
||||
// Read in the entire stream
|
||||
mut logs_builder := strings.new_builder(10 * 1024)
|
||||
util.reader_to_writer(mut logs_stream, mut logs_builder)?
|
||||
util.reader_to_writer(mut logs_stream, mut logs_builder)!
|
||||
|
||||
dd.container_remove(id)?
|
||||
dd.container_remove(id)!
|
||||
|
||||
return BuildResult{
|
||||
start_time: data.state.start_time
|
||||
|
|
|
|||
|
|
@ -0,0 +1,220 @@
|
|||
module build
|
||||
|
||||
import models { BuildConfig, Target }
|
||||
import cron.expression { CronExpression, parse_expression }
|
||||
import time
|
||||
import datatypes { MinHeap }
|
||||
import util
|
||||
|
||||
struct BuildJob {
|
||||
pub mut:
|
||||
// Time at which this build job was created/queued
|
||||
created time.Time
|
||||
// Next timestamp from which point this job is allowed to be executed
|
||||
timestamp time.Time
|
||||
// Required for calculating next timestamp after having pop'ed a job
|
||||
ce CronExpression
|
||||
// Actual build config sent to the agent
|
||||
config BuildConfig
|
||||
// Whether this is a one-time job
|
||||
single bool
|
||||
}
|
||||
|
||||
// Allows BuildJob structs to be sorted according to their timestamp in
|
||||
// MinHeaps
|
||||
fn (r1 BuildJob) < (r2 BuildJob) bool {
|
||||
return r1.timestamp < r2.timestamp
|
||||
}
|
||||
|
||||
// The build job queue is responsible for managing the list of scheduled builds
|
||||
// for each architecture. Agents receive jobs from this queue.
|
||||
pub struct BuildJobQueue {
|
||||
// Schedule to use for targets without explicitely defined cron expression
|
||||
default_schedule CronExpression
|
||||
// Base image to use for targets without defined base image
|
||||
default_base_image string
|
||||
mut:
|
||||
mutex shared util.Dummy
|
||||
// For each architecture, a priority queue is tracked
|
||||
queues map[string]MinHeap<BuildJob>
|
||||
// When a target is removed from the server or edited, its previous build
|
||||
// configs will be invalid. This map allows for those to be simply skipped
|
||||
// by ignoring any build configs created before this timestamp.
|
||||
invalidated map[int]time.Time
|
||||
}
|
||||
|
||||
// new_job_queue initializes a new job queue
|
||||
pub fn new_job_queue(default_schedule CronExpression, default_base_image string) BuildJobQueue {
|
||||
return BuildJobQueue{
|
||||
default_schedule: default_schedule
|
||||
default_base_image: default_base_image
|
||||
invalidated: map[int]time.Time{}
|
||||
}
|
||||
}
|
||||
|
||||
// insert_all executes insert for each architecture of the given Target.
|
||||
pub fn (mut q BuildJobQueue) insert_all(target Target) ! {
|
||||
for arch in target.arch {
|
||||
q.insert(target: target, arch: arch.value)!
|
||||
}
|
||||
}
|
||||
|
||||
[params]
|
||||
pub struct InsertConfig {
|
||||
target Target [required]
|
||||
arch string [required]
|
||||
single bool
|
||||
force bool
|
||||
now bool
|
||||
}
|
||||
|
||||
// insert a new target's job into the queue for the given architecture. This
|
||||
// job will then be endlessly rescheduled after being pop'ed, unless removed
|
||||
// explicitely.
|
||||
pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! {
|
||||
lock q.mutex {
|
||||
if input.arch !in q.queues {
|
||||
q.queues[input.arch] = MinHeap<BuildJob>{}
|
||||
}
|
||||
|
||||
mut job := BuildJob{
|
||||
created: time.now()
|
||||
single: input.single
|
||||
config: input.target.as_build_config(q.default_base_image, input.force)
|
||||
}
|
||||
|
||||
if !input.now {
|
||||
ce := if input.target.schedule != '' {
|
||||
parse_expression(input.target.schedule) or {
|
||||
return error("Error while parsing cron expression '$input.target.schedule' (id $input.target.id): $err.msg()")
|
||||
}
|
||||
} else {
|
||||
q.default_schedule
|
||||
}
|
||||
|
||||
job.timestamp = ce.next_from_now()!
|
||||
job.ce = ce
|
||||
} else {
|
||||
job.timestamp = time.now()
|
||||
}
|
||||
|
||||
q.queues[input.arch].insert(job)
|
||||
}
|
||||
}
|
||||
|
||||
// reschedule the given job by calculating the next timestamp and re-adding it
|
||||
// to its respective queue. This function is called by the pop functions
|
||||
// *after* having pop'ed the job.
|
||||
fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) ! {
|
||||
new_timestamp := job.ce.next_from_now()!
|
||||
|
||||
new_job := BuildJob{
|
||||
...job
|
||||
created: time.now()
|
||||
timestamp: new_timestamp
|
||||
}
|
||||
|
||||
q.queues[arch].insert(new_job)
|
||||
}
|
||||
|
||||
// pop_invalid pops all invalid jobs.
|
||||
fn (mut q BuildJobQueue) pop_invalid(arch string) {
|
||||
for {
|
||||
job := q.queues[arch].peek() or { return }
|
||||
|
||||
if job.config.target_id in q.invalidated
|
||||
&& job.created < q.invalidated[job.config.target_id] {
|
||||
// This pop *should* never fail according to the source code
|
||||
q.queues[arch].pop() or {}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// peek shows the first job for the given architecture that's ready to be
|
||||
// executed, if present.
|
||||
pub fn (mut q BuildJobQueue) peek(arch string) ?BuildJob {
|
||||
// Even peek requires a write lock, because pop_invalid can modify the data
|
||||
// structure
|
||||
lock q.mutex {
|
||||
if arch !in q.queues {
|
||||
return none
|
||||
}
|
||||
|
||||
q.pop_invalid(arch)
|
||||
job := q.queues[arch].peek()?
|
||||
|
||||
if job.timestamp < time.now() {
|
||||
return job
|
||||
}
|
||||
}
|
||||
|
||||
return none
|
||||
}
|
||||
|
||||
// pop removes the first job for the given architecture that's ready to be
|
||||
// executed from the queue and returns it, if present.
|
||||
pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob {
|
||||
lock q.mutex {
|
||||
if arch !in q.queues {
|
||||
return none
|
||||
}
|
||||
|
||||
q.pop_invalid(arch)
|
||||
mut job := q.queues[arch].peek()?
|
||||
|
||||
if job.timestamp < time.now() {
|
||||
job = q.queues[arch].pop()?
|
||||
|
||||
if !job.single {
|
||||
// TODO how do we handle this properly? Is it even possible for a
|
||||
// cron expression to not return a next time if it's already been
|
||||
// used before?
|
||||
q.reschedule(job, arch) or {}
|
||||
}
|
||||
|
||||
return job
|
||||
}
|
||||
}
|
||||
|
||||
return none
|
||||
}
|
||||
|
||||
// pop_n tries to pop at most n available jobs for the given architecture.
|
||||
pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob {
|
||||
lock q.mutex {
|
||||
if arch !in q.queues {
|
||||
return []
|
||||
}
|
||||
|
||||
mut out := []BuildJob{}
|
||||
|
||||
for out.len < n {
|
||||
q.pop_invalid(arch)
|
||||
mut job := q.queues[arch].peek() or { break }
|
||||
|
||||
if job.timestamp < time.now() {
|
||||
job = q.queues[arch].pop() or { break }
|
||||
|
||||
if !job.single {
|
||||
// TODO idem
|
||||
q.reschedule(job, arch) or {}
|
||||
}
|
||||
|
||||
out << job
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
return []
|
||||
}
|
||||
|
||||
// invalidate a target's old build jobs.
|
||||
pub fn (mut q BuildJobQueue) invalidate(target_id int) {
|
||||
q.invalidated[target_id] = time.now()
|
||||
}
|
||||
|
|
@ -16,5 +16,5 @@ echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkg
|
|||
curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
|
||||
echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
|
||||
[ "$(id -u)" == 0 ] && exit 0
|
||||
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
|
||||
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done
|
||||
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
|
||||
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done
|
||||
|
|
@ -16,5 +16,5 @@ echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkg
|
|||
curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
|
||||
echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
|
||||
[ "$(id -u)" == 0 ] && exit 0
|
||||
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
|
||||
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done
|
||||
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
|
||||
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf'
|
||||
echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
|
||||
echo -e '+ pacman -Syu --needed --noconfirm'
|
||||
pacman -Syu --needed --noconfirm
|
||||
echo -e '+ su builder'
|
||||
su builder
|
||||
echo -e '+ git clone --single-branch --depth 1 '\''https://examplerepo.com'\'' repo'
|
||||
git clone --single-branch --depth 1 'https://examplerepo.com' repo
|
||||
echo -e '+ cd '\''repo/example/path'\'''
|
||||
cd 'repo/example/path'
|
||||
echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'
|
||||
makepkg --nobuild --syncdeps --needed --noconfirm
|
||||
echo -e '+ source PKGBUILD'
|
||||
source PKGBUILD
|
||||
echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0'
|
||||
curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
|
||||
echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
|
||||
[ "$(id -u)" == 0 ] && exit 0
|
||||
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
|
||||
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf'
|
||||
echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf
|
||||
echo -e '+ pacman -Syu --needed --noconfirm'
|
||||
pacman -Syu --needed --noconfirm
|
||||
echo -e '+ su builder'
|
||||
su builder
|
||||
echo -e '+ git clone --single-branch --depth 1 '\''https://examplerepo.com'\'' repo'
|
||||
git clone --single-branch --depth 1 'https://examplerepo.com' repo
|
||||
echo -e '+ cd '\''repo/example/path with spaces'\'''
|
||||
cd 'repo/example/path with spaces'
|
||||
echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm'
|
||||
makepkg --nobuild --syncdeps --needed --noconfirm
|
||||
echo -e '+ source PKGBUILD'
|
||||
source PKGBUILD
|
||||
echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0'
|
||||
curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
|
||||
echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
|
||||
[ "$(id -u)" == 0 ] && exit 0
|
||||
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
|
||||
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done
|
||||
|
|
@ -18,5 +18,5 @@ echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkg
|
|||
curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0
|
||||
echo -e '+ [ "$(id -u)" == 0 ] && exit 0'
|
||||
[ "$(id -u)" == 0 ] && exit 0
|
||||
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
|
||||
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done
|
||||
echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done'
|
||||
MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
module build
|
||||
|
||||
import models { Target }
|
||||
import models { BuildConfig }
|
||||
|
||||
// escape_shell_string escapes any characters that could be interpreted
|
||||
// incorrectly by a shell. The resulting value should be safe to use inside an
|
||||
|
|
@ -23,13 +23,13 @@ pub fn echo_commands(cmds []string) []string {
|
|||
}
|
||||
|
||||
// create_build_script generates a shell script that builds a given Target.
|
||||
fn create_build_script(address string, target &Target, build_arch string) string {
|
||||
repo_url := '$address/$target.repo'
|
||||
fn create_build_script(address string, config BuildConfig, build_arch string) string {
|
||||
repo_url := '$address/$config.repo'
|
||||
|
||||
mut commands := [
|
||||
// This will later be replaced by a proper setting for changing the
|
||||
// mirrorlist
|
||||
"echo -e '[$target.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf"
|
||||
"echo -e '[$config.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf"
|
||||
// We need to update the package list of the repo we just added above.
|
||||
// This should however not pull in a lot of packages as long as the
|
||||
// builder image is rebuilt frequently.
|
||||
|
|
@ -38,22 +38,22 @@ fn create_build_script(address string, target &Target, build_arch string) string
|
|||
'su builder',
|
||||
]
|
||||
|
||||
commands << match target.kind {
|
||||
commands << match config.kind {
|
||||
'git' {
|
||||
if target.branch == '' {
|
||||
if config.branch == '' {
|
||||
[
|
||||
"git clone --single-branch --depth 1 '$target.url' repo",
|
||||
"git clone --single-branch --depth 1 '$config.url' repo",
|
||||
]
|
||||
} else {
|
||||
[
|
||||
"git clone --single-branch --depth 1 --branch $target.branch '$target.url' repo",
|
||||
"git clone --single-branch --depth 1 --branch $config.branch '$config.url' repo",
|
||||
]
|
||||
}
|
||||
}
|
||||
'url' {
|
||||
[
|
||||
'mkdir repo',
|
||||
"curl -o repo/PKGBUILD -L '$target.url'",
|
||||
"curl -o repo/PKGBUILD -L '$config.url'",
|
||||
]
|
||||
}
|
||||
else {
|
||||
|
|
@ -61,19 +61,32 @@ fn create_build_script(address string, target &Target, build_arch string) string
|
|||
}
|
||||
}
|
||||
|
||||
commands << if config.path != '' {
|
||||
"cd 'repo/$config.path'"
|
||||
} else {
|
||||
'cd repo'
|
||||
}
|
||||
|
||||
commands << [
|
||||
'cd repo',
|
||||
'makepkg --nobuild --syncdeps --needed --noconfirm',
|
||||
'source PKGBUILD',
|
||||
]
|
||||
|
||||
if !config.force {
|
||||
// The build container checks whether the package is already present on
|
||||
// the server.
|
||||
'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
|
||||
// If the above curl command succeeds, we don't need to rebuild the
|
||||
// package. However, because we're in a su shell, the exit command will
|
||||
// drop us back into the root shell. Therefore, we must check whether
|
||||
// we're in root so we don't proceed.
|
||||
'[ "\$(id -u)" == 0 ] && exit 0',
|
||||
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done',
|
||||
commands << [
|
||||
'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
|
||||
// If the above curl command succeeds, we don't need to rebuild the
|
||||
// package. However, because we're in a su shell, the exit command will
|
||||
// drop us back into the root shell. Therefore, we must check whether
|
||||
// we're in root so we don't proceed.
|
||||
'[ "\$(id -u)" == 0 ] && exit 0',
|
||||
]
|
||||
}
|
||||
|
||||
commands << [
|
||||
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed --noextract && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done',
|
||||
]
|
||||
|
||||
return echo_commands(commands).join('\n')
|
||||
|
|
|
|||
|
|
@ -1,43 +1,75 @@
|
|||
module build
|
||||
|
||||
import models { Target }
|
||||
import models { BuildConfig }
|
||||
|
||||
fn test_create_build_script_git_branch() {
|
||||
target := Target{
|
||||
id: 1
|
||||
fn test_create_build_script_git() {
|
||||
config := BuildConfig{
|
||||
target_id: 1
|
||||
kind: 'git'
|
||||
url: 'https://examplerepo.com'
|
||||
branch: 'main'
|
||||
repo: 'vieter'
|
||||
base_image: 'not-used:latest'
|
||||
}
|
||||
build_script := create_build_script('https://example.com', target, 'x86_64')
|
||||
expected := $embed_file('build_script_git_branch.sh')
|
||||
|
||||
build_script := create_build_script('https://example.com', config, 'x86_64')
|
||||
expected := $embed_file('scripts/git.sh')
|
||||
|
||||
assert build_script == expected.to_string().trim_space()
|
||||
}
|
||||
|
||||
fn test_create_build_script_git() {
|
||||
target := Target{
|
||||
id: 1
|
||||
fn test_create_build_script_git_path() {
|
||||
mut config := BuildConfig{
|
||||
target_id: 1
|
||||
kind: 'git'
|
||||
url: 'https://examplerepo.com'
|
||||
repo: 'vieter'
|
||||
path: 'example/path'
|
||||
base_image: 'not-used:latest'
|
||||
}
|
||||
build_script := create_build_script('https://example.com', target, 'x86_64')
|
||||
expected := $embed_file('build_script_git.sh')
|
||||
|
||||
mut build_script := create_build_script('https://example.com', config, 'x86_64')
|
||||
mut expected := $embed_file('scripts/git_path.sh')
|
||||
|
||||
assert build_script == expected.to_string().trim_space()
|
||||
|
||||
config = BuildConfig{
|
||||
...config
|
||||
path: 'example/path with spaces'
|
||||
}
|
||||
|
||||
build_script = create_build_script('https://example.com', config, 'x86_64')
|
||||
expected = $embed_file('scripts/git_path_spaces.sh')
|
||||
|
||||
assert build_script == expected.to_string().trim_space()
|
||||
}
|
||||
|
||||
fn test_create_build_script_git_branch() {
|
||||
config := BuildConfig{
|
||||
target_id: 1
|
||||
kind: 'git'
|
||||
url: 'https://examplerepo.com'
|
||||
branch: 'main'
|
||||
repo: 'vieter'
|
||||
base_image: 'not-used:latest'
|
||||
}
|
||||
|
||||
build_script := create_build_script('https://example.com', config, 'x86_64')
|
||||
expected := $embed_file('scripts/git_branch.sh')
|
||||
|
||||
assert build_script == expected.to_string().trim_space()
|
||||
}
|
||||
|
||||
fn test_create_build_script_url() {
|
||||
target := Target{
|
||||
id: 1
|
||||
config := BuildConfig{
|
||||
target_id: 1
|
||||
kind: 'url'
|
||||
url: 'https://examplerepo.com'
|
||||
repo: 'vieter'
|
||||
base_image: 'not-used:latest'
|
||||
}
|
||||
build_script := create_build_script('https://example.com', target, 'x86_64')
|
||||
expected := $embed_file('build_script_url.sh')
|
||||
|
||||
build_script := create_build_script('https://example.com', config, 'x86_64')
|
||||
expected := $embed_file('scripts/url.sh')
|
||||
|
||||
assert build_script == expected.to_string().trim_space()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ module client
|
|||
|
||||
import net.http { Method }
|
||||
import net.urllib
|
||||
import web.response { Response }
|
||||
import web.response { Response, new_data_response }
|
||||
import json
|
||||
|
||||
pub struct Client {
|
||||
|
|
@ -21,7 +21,7 @@ pub fn new(address string, api_key string) Client {
|
|||
|
||||
// send_request_raw sends an HTTP request, returning the http.Response object.
|
||||
// It encodes the params so that they're safe to pass as HTTP query parameters.
|
||||
fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) ?http.Response {
|
||||
fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) !http.Response {
|
||||
mut full_url := '$c.address$url'
|
||||
|
||||
if params.len > 0 {
|
||||
|
|
@ -38,31 +38,53 @@ fn (c &Client) send_request_raw(method Method, url string, params map[string]str
|
|||
full_url = '$full_url?$params_str'
|
||||
}
|
||||
|
||||
mut req := http.new_request(method, full_url, body)?
|
||||
req.add_custom_header('X-Api-Key', c.api_key)?
|
||||
// Looking at the source code, this function doesn't actually fail, so I'm
|
||||
// not sure why it returns an optional
|
||||
mut req := http.new_request(method, full_url, body) or { return error('') }
|
||||
req.add_custom_header('X-Api-Key', c.api_key)!
|
||||
|
||||
res := req.do()?
|
||||
res := req.do()!
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// send_request<T> just calls send_request_with_body<T> with an empty body.
|
||||
fn (c &Client) send_request<T>(method Method, url string, params map[string]string) ?Response<T> {
|
||||
fn (c &Client) send_request<T>(method Method, url string, params map[string]string) !Response<T> {
|
||||
return c.send_request_with_body<T>(method, url, params, '')
|
||||
}
|
||||
|
||||
// send_request_with_body<T> calls send_request_raw_response & parses its
|
||||
// output as a Response<T> object.
|
||||
fn (c &Client) send_request_with_body<T>(method Method, url string, params map[string]string, body string) ?Response<T> {
|
||||
res_text := c.send_request_raw_response(method, url, params, body)?
|
||||
data := json.decode(Response<T>, res_text)?
|
||||
fn (c &Client) send_request_with_body<T>(method Method, url string, params map[string]string, body string) !Response<T> {
|
||||
res := c.send_request_raw(method, url, params, body)!
|
||||
status := res.status()
|
||||
|
||||
// Non-successful requests are expected to return either an empty body or
|
||||
// Response<string>
|
||||
if status.is_error() {
|
||||
// A non-successful status call will have an empty body
|
||||
if res.body == '' {
|
||||
return error('Error $res.status_code ($status.str()): (empty response)')
|
||||
}
|
||||
|
||||
data := json.decode(Response<string>, res.body)!
|
||||
|
||||
return error('Status $res.status_code ($status.str()): $data.message')
|
||||
}
|
||||
|
||||
// Just return an empty successful response
|
||||
if res.body == '' {
|
||||
return new_data_response(T{})
|
||||
}
|
||||
|
||||
data := json.decode(Response<T>, res.body)!
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
// send_request_raw_response returns the raw text response for an HTTP request.
|
||||
fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) ?string {
|
||||
res := c.send_request_raw(method, url, params, body)?
|
||||
fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) !string {
|
||||
res := c.send_request_raw(method, url, params, body)!
|
||||
|
||||
return res.body
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
module client
|
||||
|
||||
import models { BuildConfig }
|
||||
|
||||
// poll_jobs requests a list of new build jobs from the server.
|
||||
pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig {
|
||||
data := c.send_request<[]BuildConfig>(.get, '/api/v1/jobs/poll', {
|
||||
'arch': arch
|
||||
'max': max.str()
|
||||
})!
|
||||
|
||||
return data.data
|
||||
}
|
||||
|
||||
// queue_job adds a new one-time build job for the given target to the job
|
||||
// queue.
|
||||
pub fn (c &Client) queue_job(target_id int, arch string, force bool) ! {
|
||||
c.send_request<string>(.post, '/api/v1/jobs/queue', {
|
||||
'target': target_id.str()
|
||||
'arch': arch
|
||||
'force': force.str()
|
||||
})!
|
||||
}
|
||||
|
|
@ -6,40 +6,29 @@ import web.response { Response }
|
|||
import time
|
||||
|
||||
// get_build_logs returns all build logs.
|
||||
pub fn (c &Client) get_build_logs(filter BuildLogFilter) ?Response<[]BuildLog> {
|
||||
pub fn (c &Client) get_build_logs(filter BuildLogFilter) ![]BuildLog {
|
||||
params := models.params_from(filter)
|
||||
data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)?
|
||||
data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)!
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
// get_build_logs_for_target returns all build logs for a given target.
|
||||
pub fn (c &Client) get_build_logs_for_target(target_id int) ?Response<[]BuildLog> {
|
||||
params := {
|
||||
'repo': target_id.str()
|
||||
}
|
||||
|
||||
data := c.send_request<[]BuildLog>(Method.get, '/api/v1/logs', params)?
|
||||
|
||||
return data
|
||||
return data.data
|
||||
}
|
||||
|
||||
// get_build_log returns a specific build log.
|
||||
pub fn (c &Client) get_build_log(id int) ?Response<BuildLog> {
|
||||
data := c.send_request<BuildLog>(Method.get, '/api/v1/logs/$id', {})?
|
||||
pub fn (c &Client) get_build_log(id int) !BuildLog {
|
||||
data := c.send_request<BuildLog>(Method.get, '/api/v1/logs/$id', {})!
|
||||
|
||||
return data
|
||||
return data.data
|
||||
}
|
||||
|
||||
// get_build_log_content returns the contents of the build log file.
|
||||
pub fn (c &Client) get_build_log_content(id int) ?string {
|
||||
data := c.send_request_raw_response(Method.get, '/api/v1/logs/$id/content', {}, '')?
|
||||
pub fn (c &Client) get_build_log_content(id int) !string {
|
||||
data := c.send_request_raw_response(Method.get, '/api/v1/logs/$id/content', {}, '')!
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
// add_build_log adds a new build log to the server.
|
||||
pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response<int> {
|
||||
pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) !Response<int> {
|
||||
params := {
|
||||
'target': target_id.str()
|
||||
'startTime': start_time.unix_time().str()
|
||||
|
|
@ -48,7 +37,7 @@ pub fn (c &Client) add_build_log(target_id int, start_time time.Time, end_time t
|
|||
'exitCode': exit_code.str()
|
||||
}
|
||||
|
||||
data := c.send_request_with_body<int>(Method.post, '/api/v1/logs', params, content)?
|
||||
data := c.send_request_with_body<int>(Method.post, '/api/v1/logs', params, content)!
|
||||
|
||||
return data
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,24 +2,23 @@ module client
|
|||
|
||||
import models { Target, TargetFilter }
|
||||
import net.http { Method }
|
||||
import web.response { Response }
|
||||
|
||||
// get_targets returns a list of targets, given a filter object.
|
||||
pub fn (c &Client) get_targets(filter TargetFilter) ?[]Target {
|
||||
pub fn (c &Client) get_targets(filter TargetFilter) ![]Target {
|
||||
params := models.params_from(filter)
|
||||
data := c.send_request<[]Target>(Method.get, '/api/v1/targets', params)?
|
||||
data := c.send_request<[]Target>(Method.get, '/api/v1/targets', params)!
|
||||
|
||||
return data.data
|
||||
}
|
||||
|
||||
// get_all_targets retrieves *all* targs from the API using the default
|
||||
// limit.
|
||||
pub fn (c &Client) get_all_targets() ?[]Target {
|
||||
pub fn (c &Client) get_all_targets() ![]Target {
|
||||
mut targets := []Target{}
|
||||
mut offset := u64(0)
|
||||
|
||||
for {
|
||||
sub_targets := c.get_targets(offset: offset)?
|
||||
sub_targets := c.get_targets(offset: offset)!
|
||||
|
||||
if sub_targets.len == 0 {
|
||||
break
|
||||
|
|
@ -34,8 +33,8 @@ pub fn (c &Client) get_all_targets() ?[]Target {
|
|||
}
|
||||
|
||||
// get_target returns the target for a specific id.
|
||||
pub fn (c &Client) get_target(id int) ?Target {
|
||||
data := c.send_request<Target>(Method.get, '/api/v1/targets/$id', {})?
|
||||
pub fn (c &Client) get_target(id int) !Target {
|
||||
data := c.send_request<Target>(Method.get, '/api/v1/targets/$id', {})!
|
||||
|
||||
return data.data
|
||||
}
|
||||
|
|
@ -45,28 +44,29 @@ pub struct NewTarget {
|
|||
url string
|
||||
branch string
|
||||
repo string
|
||||
path string
|
||||
arch []string
|
||||
}
|
||||
|
||||
// add_target adds a new target to the server.
|
||||
pub fn (c &Client) add_target(t NewTarget) ?Response<int> {
|
||||
pub fn (c &Client) add_target(t NewTarget) !int {
|
||||
params := models.params_from<NewTarget>(t)
|
||||
data := c.send_request<int>(Method.post, '/api/v1/targets', params)?
|
||||
data := c.send_request<int>(Method.post, '/api/v1/targets', params)!
|
||||
|
||||
return data
|
||||
return data.data
|
||||
}
|
||||
|
||||
// remove_target removes the target with the given id from the server.
|
||||
pub fn (c &Client) remove_target(id int) ?Response<string> {
|
||||
data := c.send_request<string>(Method.delete, '/api/v1/targets/$id', {})?
|
||||
pub fn (c &Client) remove_target(id int) !string {
|
||||
data := c.send_request<string>(Method.delete, '/api/v1/targets/$id', {})!
|
||||
|
||||
return data
|
||||
return data.data
|
||||
}
|
||||
|
||||
// patch_target sends a PATCH request to the given target with the params as
|
||||
// payload.
|
||||
pub fn (c &Client) patch_target(id int, params map[string]string) ?Response<string> {
|
||||
data := c.send_request<string>(Method.patch, '/api/v1/targets/$id', params)?
|
||||
pub fn (c &Client) patch_target(id int, params map[string]string) !string {
|
||||
data := c.send_request<string>(Method.patch, '/api/v1/targets/$id', params)!
|
||||
|
||||
return data
|
||||
return data.data
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ module aur
|
|||
import cli
|
||||
import console
|
||||
import client
|
||||
import vieter_v.aur
|
||||
import vieter_v.conf as vconf
|
||||
import aur
|
||||
import conf as vconf
|
||||
|
||||
struct Config {
|
||||
address string [required]
|
||||
|
|
@ -21,12 +21,12 @@ pub fn cmd() cli.Command {
|
|||
name: 'search'
|
||||
description: 'Search for packages.'
|
||||
required_args: 1
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
c := aur.new()
|
||||
pkgs := c.search(cmd.args[0])?
|
||||
pkgs := c.search(cmd.args[0])!
|
||||
data := pkgs.map([it.name, it.description])
|
||||
|
||||
println(console.pretty_table(['name', 'description'], data)?)
|
||||
println(console.pretty_table(['name', 'description'], data)!)
|
||||
}
|
||||
},
|
||||
cli.Command{
|
||||
|
|
@ -34,12 +34,12 @@ pub fn cmd() cli.Command {
|
|||
usage: 'repo pkg-name [pkg-name...]'
|
||||
description: 'Add the given AUR package(s) to Vieter. Non-existent packages will be silently ignored.'
|
||||
required_args: 2
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
config_file := cmd.flags.get_string('config-file')?
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||
|
||||
c := aur.new()
|
||||
pkgs := c.info(cmd.args[1..])?
|
||||
pkgs := c.info(cmd.args[1..])!
|
||||
|
||||
vc := client.new(conf.address, conf.api_key)
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ pub fn tabbed_table(data [][]string) string {
|
|||
// pretty_table converts a list of string data into a pretty table. Many thanks
|
||||
// to @hungrybluedev in the Vlang Discord for providing this code!
|
||||
// https://ptb.discord.com/channels/592103645835821068/592106336838352923/970278787143045192
|
||||
pub fn pretty_table(header []string, data [][]string) ?string {
|
||||
pub fn pretty_table(header []string, data [][]string) !string {
|
||||
column_count := header.len
|
||||
|
||||
mut column_widths := []int{len: column_count, init: header[it].len}
|
||||
|
|
@ -26,7 +26,7 @@ pub fn pretty_table(header []string, data [][]string) ?string {
|
|||
}
|
||||
}
|
||||
|
||||
single_line_length := arrays.sum(column_widths)? + (column_count + 1) * 3 - 4
|
||||
single_line_length := arrays.sum(column_widths)! + (column_count + 1) * 3 - 4
|
||||
|
||||
horizontal_line := '+' + strings.repeat(`-`, single_line_length) + '+'
|
||||
mut buffer := strings.new_builder(data.len * single_line_length)
|
||||
|
|
@ -64,12 +64,12 @@ pub fn pretty_table(header []string, data [][]string) ?string {
|
|||
|
||||
// export_man_pages recursively generates all man pages for the given
|
||||
// cli.Command & writes them to the given directory.
|
||||
pub fn export_man_pages(cmd cli.Command, path string) ? {
|
||||
pub fn export_man_pages(cmd cli.Command, path string) ! {
|
||||
man := cmd.manpage()
|
||||
os.write_file(os.join_path_single(path, cmd.full_name().replace(' ', '-') + '.1'),
|
||||
man)?
|
||||
man)!
|
||||
|
||||
for sub_cmd in cmd.commands {
|
||||
export_man_pages(sub_cmd, path)?
|
||||
export_man_pages(sub_cmd, path)!
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
module logs
|
||||
|
||||
import cli
|
||||
import vieter_v.conf as vconf
|
||||
import conf as vconf
|
||||
import client
|
||||
import console
|
||||
import time
|
||||
|
|
@ -63,30 +63,30 @@ pub fn cmd() cli.Command {
|
|||
flag: cli.FlagType.string
|
||||
},
|
||||
]
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
config_file := cmd.flags.get_string('config-file')?
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||
|
||||
mut filter := BuildLogFilter{}
|
||||
|
||||
limit := cmd.flags.get_int('limit')?
|
||||
limit := cmd.flags.get_int('limit')!
|
||||
if limit != 0 {
|
||||
filter.limit = u64(limit)
|
||||
}
|
||||
|
||||
offset := cmd.flags.get_int('offset')?
|
||||
offset := cmd.flags.get_int('offset')!
|
||||
if offset != 0 {
|
||||
filter.offset = u64(offset)
|
||||
}
|
||||
|
||||
target_id := cmd.flags.get_int('target')?
|
||||
target_id := cmd.flags.get_int('target')!
|
||||
if target_id != 0 {
|
||||
filter.target = target_id
|
||||
}
|
||||
|
||||
tz_offset := time.offset()
|
||||
|
||||
if cmd.flags.get_bool('today')? {
|
||||
if cmd.flags.get_bool('today')! {
|
||||
today := time.now()
|
||||
|
||||
filter.after = time.new_time(time.Time{
|
||||
|
|
@ -98,12 +98,12 @@ pub fn cmd() cli.Command {
|
|||
}
|
||||
// The -today flag overwrites any of the other date flags.
|
||||
else {
|
||||
day_str := cmd.flags.get_string('day')?
|
||||
before_str := cmd.flags.get_string('before')?
|
||||
after_str := cmd.flags.get_string('after')?
|
||||
day_str := cmd.flags.get_string('day')!
|
||||
before_str := cmd.flags.get_string('before')!
|
||||
after_str := cmd.flags.get_string('after')!
|
||||
|
||||
if day_str != '' {
|
||||
day := time.parse_rfc3339(day_str)?
|
||||
day := time.parse_rfc3339(day_str)!
|
||||
day_utc := time.new_time(time.Time{
|
||||
year: day.year
|
||||
month: day.month
|
||||
|
|
@ -118,24 +118,24 @@ pub fn cmd() cli.Command {
|
|||
filter.before = day_utc.add_days(1)
|
||||
} else {
|
||||
if before_str != '' {
|
||||
filter.before = time.parse(before_str)?.add_seconds(-tz_offset)
|
||||
filter.before = time.parse(before_str)!.add_seconds(-tz_offset)
|
||||
}
|
||||
|
||||
if after_str != '' {
|
||||
filter.after = time.parse(after_str)?.add_seconds(-tz_offset)
|
||||
filter.after = time.parse(after_str)!.add_seconds(-tz_offset)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if cmd.flags.get_bool('failed')? {
|
||||
if cmd.flags.get_bool('failed')! {
|
||||
filter.exit_codes = [
|
||||
'!0',
|
||||
]
|
||||
}
|
||||
|
||||
raw := cmd.flags.get_bool('raw')?
|
||||
raw := cmd.flags.get_bool('raw')!
|
||||
|
||||
list(conf, filter, raw)?
|
||||
list(conf, filter, raw)!
|
||||
}
|
||||
},
|
||||
cli.Command{
|
||||
|
|
@ -143,12 +143,12 @@ pub fn cmd() cli.Command {
|
|||
required_args: 1
|
||||
usage: 'id'
|
||||
description: 'Show all info for a specific build log.'
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
config_file := cmd.flags.get_string('config-file')?
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||
|
||||
id := cmd.args[0].int()
|
||||
info(conf, id)?
|
||||
info(conf, id)!
|
||||
}
|
||||
},
|
||||
cli.Command{
|
||||
|
|
@ -156,12 +156,12 @@ pub fn cmd() cli.Command {
|
|||
required_args: 1
|
||||
usage: 'id'
|
||||
description: 'Output the content of a build log to stdout.'
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
config_file := cmd.flags.get_string('config-file')?
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||
|
||||
id := cmd.args[0].int()
|
||||
content(conf, id)?
|
||||
content(conf, id)!
|
||||
}
|
||||
},
|
||||
]
|
||||
|
|
@ -169,46 +169,38 @@ pub fn cmd() cli.Command {
|
|||
}
|
||||
|
||||
// print_log_list prints a list of logs.
|
||||
fn print_log_list(logs []BuildLog, raw bool) ? {
|
||||
fn print_log_list(logs []BuildLog, raw bool) ! {
|
||||
data := logs.map([it.id.str(), it.target_id.str(), it.start_time.local().str(),
|
||||
it.exit_code.str()])
|
||||
|
||||
if raw {
|
||||
println(console.tabbed_table(data))
|
||||
} else {
|
||||
println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)?)
|
||||
println(console.pretty_table(['id', 'target', 'start time', 'exit code'], data)!)
|
||||
}
|
||||
}
|
||||
|
||||
// list prints a list of all build logs.
|
||||
fn list(conf Config, filter BuildLogFilter, raw bool) ? {
|
||||
fn list(conf Config, filter BuildLogFilter, raw bool) ! {
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
logs := c.get_build_logs(filter)?.data
|
||||
logs := c.get_build_logs(filter)!
|
||||
|
||||
print_log_list(logs, raw)?
|
||||
}
|
||||
|
||||
// list prints a list of all build logs for a given target.
|
||||
fn list_for_target(conf Config, target_id int, raw bool) ? {
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
logs := c.get_build_logs_for_target(target_id)?.data
|
||||
|
||||
print_log_list(logs, raw)?
|
||||
print_log_list(logs, raw)!
|
||||
}
|
||||
|
||||
// info print the detailed info for a given build log.
|
||||
fn info(conf Config, id int) ? {
|
||||
fn info(conf Config, id int) ! {
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
log := c.get_build_log(id)?.data
|
||||
log := c.get_build_log(id)!
|
||||
|
||||
print(log)
|
||||
}
|
||||
|
||||
// content outputs the contents of the log file for a given build log to
|
||||
// stdout.
|
||||
fn content(conf Config, id int) ? {
|
||||
fn content(conf Config, id int) ! {
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
content := c.get_build_log_content(id)?
|
||||
content := c.get_build_log_content(id)!
|
||||
|
||||
println(content)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,11 +11,11 @@ pub fn cmd() cli.Command {
|
|||
description: 'Generate all man pages & save them in the given directory.'
|
||||
usage: 'dir'
|
||||
required_args: 1
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
root := cmd.root()
|
||||
os.mkdir_all(cmd.args[0])?
|
||||
os.mkdir_all(cmd.args[0])!
|
||||
|
||||
console.export_man_pages(root, cmd.args[0])?
|
||||
console.export_man_pages(root, cmd.args[0])!
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,11 +18,11 @@ pub fn cmd() cli.Command {
|
|||
default_value: ['5']
|
||||
},
|
||||
]
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
ce := parse_expression(cmd.args.join(' '))?
|
||||
count := cmd.flags.get_int('count')?
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
ce := parse_expression(cmd.args.join(' '))!
|
||||
count := cmd.flags.get_int('count')!
|
||||
|
||||
for t in ce.next_n(time.now(), count)? {
|
||||
for t in ce.next_n(time.now(), count)! {
|
||||
println(t)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,34 +1,34 @@
|
|||
module targets
|
||||
|
||||
import client
|
||||
import vieter_v.docker
|
||||
import docker
|
||||
import os
|
||||
import build
|
||||
|
||||
// build locally builds the target with the given id.
|
||||
fn build(conf Config, target_id int) ? {
|
||||
fn build(conf Config, target_id int, force bool) ! {
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
target := c.get_target(target_id)?
|
||||
target := c.get_target(target_id)!
|
||||
|
||||
build_arch := os.uname().machine
|
||||
|
||||
println('Creating base image...')
|
||||
image_id := build.create_build_image(conf.base_image)?
|
||||
image_id := build.create_build_image(conf.base_image)!
|
||||
|
||||
println('Running build...')
|
||||
res := build.build_target(conf.address, conf.api_key, image_id, target)?
|
||||
res := build.build_target(conf.address, conf.api_key, image_id, target, force)!
|
||||
|
||||
println('Removing build image...')
|
||||
|
||||
mut dd := docker.new_conn()?
|
||||
mut dd := docker.new_conn()!
|
||||
|
||||
defer {
|
||||
dd.close() or {}
|
||||
}
|
||||
|
||||
dd.remove_image(image_id)?
|
||||
dd.remove_image(image_id)!
|
||||
|
||||
println('Uploading logs to Vieter...')
|
||||
c.add_build_log(target.id, res.start_time, res.end_time, build_arch, res.exit_code,
|
||||
res.logs)?
|
||||
res.logs)!
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
module targets
|
||||
|
||||
import cli
|
||||
import vieter_v.conf as vconf
|
||||
import conf as vconf
|
||||
import cron.expression { parse_expression }
|
||||
import client { NewTarget }
|
||||
import console
|
||||
|
|
@ -13,7 +13,7 @@ struct Config {
|
|||
base_image string = 'archlinux:base-devel'
|
||||
}
|
||||
|
||||
// cmd returns the cli submodule that handles the repos API interaction
|
||||
// cmd returns the cli submodule that handles the targets API interaction
|
||||
pub fn cmd() cli.Command {
|
||||
return cli.Command{
|
||||
name: 'targets'
|
||||
|
|
@ -39,30 +39,30 @@ pub fn cmd() cli.Command {
|
|||
flag: cli.FlagType.string
|
||||
},
|
||||
]
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
config_file := cmd.flags.get_string('config-file')?
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||
|
||||
mut filter := TargetFilter{}
|
||||
|
||||
limit := cmd.flags.get_int('limit')?
|
||||
limit := cmd.flags.get_int('limit')!
|
||||
if limit != 0 {
|
||||
filter.limit = u64(limit)
|
||||
}
|
||||
|
||||
offset := cmd.flags.get_int('offset')?
|
||||
offset := cmd.flags.get_int('offset')!
|
||||
if offset != 0 {
|
||||
filter.offset = u64(offset)
|
||||
}
|
||||
|
||||
repo := cmd.flags.get_string('repo')?
|
||||
repo := cmd.flags.get_string('repo')!
|
||||
if repo != '' {
|
||||
filter.repo = repo
|
||||
}
|
||||
|
||||
raw := cmd.flags.get_bool('raw')?
|
||||
raw := cmd.flags.get_bool('raw')!
|
||||
|
||||
list(conf, filter, raw)?
|
||||
list(conf, filter, raw)!
|
||||
}
|
||||
},
|
||||
cli.Command{
|
||||
|
|
@ -82,21 +82,27 @@ pub fn cmd() cli.Command {
|
|||
description: "Which branch to clone; only applies to kind 'git'."
|
||||
flag: cli.FlagType.string
|
||||
},
|
||||
cli.Flag{
|
||||
name: 'path'
|
||||
description: 'Subdirectory inside Git repository to use.'
|
||||
flag: cli.FlagType.string
|
||||
},
|
||||
]
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
config_file := cmd.flags.get_string('config-file')?
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||
|
||||
t := NewTarget{
|
||||
kind: cmd.flags.get_string('kind')?
|
||||
kind: cmd.flags.get_string('kind')!
|
||||
url: cmd.args[0]
|
||||
repo: cmd.args[1]
|
||||
branch: cmd.flags.get_string('branch') or { '' }
|
||||
path: cmd.flags.get_string('path') or { '' }
|
||||
}
|
||||
|
||||
raw := cmd.flags.get_bool('raw')?
|
||||
raw := cmd.flags.get_bool('raw')!
|
||||
|
||||
add(conf, t, raw)?
|
||||
add(conf, t, raw)!
|
||||
}
|
||||
},
|
||||
cli.Command{
|
||||
|
|
@ -104,11 +110,11 @@ pub fn cmd() cli.Command {
|
|||
required_args: 1
|
||||
usage: 'id'
|
||||
description: 'Remove a target that matches the given id.'
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
config_file := cmd.flags.get_string('config-file')?
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||
|
||||
remove(conf, cmd.args[0])?
|
||||
remove(conf, cmd.args[0])!
|
||||
}
|
||||
},
|
||||
cli.Command{
|
||||
|
|
@ -116,11 +122,11 @@ pub fn cmd() cli.Command {
|
|||
required_args: 1
|
||||
usage: 'id'
|
||||
description: 'Show detailed information for the target matching the id.'
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
config_file := cmd.flags.get_string('config-file')?
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||
|
||||
info(conf, cmd.args[0])?
|
||||
info(conf, cmd.args[0])!
|
||||
}
|
||||
},
|
||||
cli.Command{
|
||||
|
|
@ -159,10 +165,15 @@ pub fn cmd() cli.Command {
|
|||
description: 'Kind of target.'
|
||||
flag: cli.FlagType.string
|
||||
},
|
||||
cli.Flag{
|
||||
name: 'path'
|
||||
description: 'Subdirectory inside Git repository to use.'
|
||||
flag: cli.FlagType.string
|
||||
},
|
||||
]
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
config_file := cmd.flags.get_string('config-file')?
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||
|
||||
found := cmd.flags.get_all_found()
|
||||
|
||||
|
|
@ -170,11 +181,11 @@ pub fn cmd() cli.Command {
|
|||
|
||||
for f in found {
|
||||
if f.name != 'config-file' {
|
||||
params[f.name] = f.get_string()?
|
||||
params[f.name] = f.get_string()!
|
||||
}
|
||||
}
|
||||
|
||||
patch(conf, cmd.args[0], params)?
|
||||
patch(conf, cmd.args[0], params)!
|
||||
}
|
||||
},
|
||||
cli.Command{
|
||||
|
|
@ -182,58 +193,82 @@ pub fn cmd() cli.Command {
|
|||
required_args: 1
|
||||
usage: 'id'
|
||||
description: 'Build the target with the given id & publish it.'
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
config_file := cmd.flags.get_string('config-file')?
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
||||
flags: [
|
||||
cli.Flag{
|
||||
name: 'force'
|
||||
description: 'Build the target without checking whether it needs to be renewed.'
|
||||
flag: cli.FlagType.bool
|
||||
},
|
||||
cli.Flag{
|
||||
name: 'remote'
|
||||
description: 'Schedule the build on the server instead of running it locally.'
|
||||
flag: cli.FlagType.bool
|
||||
},
|
||||
cli.Flag{
|
||||
name: 'arch'
|
||||
description: 'Architecture to schedule build for. Required when using -remote.'
|
||||
flag: cli.FlagType.string
|
||||
},
|
||||
]
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||
|
||||
build(conf, cmd.args[0].int())?
|
||||
remote := cmd.flags.get_bool('remote')!
|
||||
force := cmd.flags.get_bool('force')!
|
||||
target_id := cmd.args[0].int()
|
||||
|
||||
if remote {
|
||||
arch := cmd.flags.get_string('arch')!
|
||||
|
||||
if arch == '' {
|
||||
return error('When scheduling the build remotely, you have to specify an architecture.')
|
||||
}
|
||||
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
c.queue_job(target_id, arch, force)!
|
||||
} else {
|
||||
build(conf, target_id, force)!
|
||||
}
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
// get_repo_by_prefix tries to find the repo with the given prefix in its
|
||||
// ID. If multiple or none are found, an error is raised.
|
||||
|
||||
// list prints out a list of all repositories.
|
||||
fn list(conf Config, filter TargetFilter, raw bool) ? {
|
||||
fn list(conf Config, filter TargetFilter, raw bool) ! {
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
repos := c.get_targets(filter)?
|
||||
data := repos.map([it.id.str(), it.kind, it.url, it.repo])
|
||||
targets := c.get_targets(filter)!
|
||||
data := targets.map([it.id.str(), it.kind, it.url, it.repo])
|
||||
|
||||
if raw {
|
||||
println(console.tabbed_table(data))
|
||||
} else {
|
||||
println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?)
|
||||
println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)!)
|
||||
}
|
||||
}
|
||||
|
||||
// add adds a new repository to the server's list.
|
||||
fn add(conf Config, t &NewTarget, raw bool) ? {
|
||||
// add adds a new target to the server's list.
|
||||
fn add(conf Config, t &NewTarget, raw bool) ! {
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
res := c.add_target(t)?
|
||||
target_id := c.add_target(t)!
|
||||
|
||||
if raw {
|
||||
println(res.data)
|
||||
println(target_id)
|
||||
} else {
|
||||
println('Target added with id $res.data')
|
||||
println('Target added with id $target_id')
|
||||
}
|
||||
}
|
||||
|
||||
// remove removes a repository from the server's list.
|
||||
fn remove(conf Config, id string) ? {
|
||||
id_int := id.int()
|
||||
|
||||
if id_int != 0 {
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
res := c.remove_target(id_int)?
|
||||
println(res.message)
|
||||
}
|
||||
// remove removes a target from the server's list.
|
||||
fn remove(conf Config, id string) ! {
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
c.remove_target(id.int())!
|
||||
}
|
||||
|
||||
// patch patches a given repository with the provided params.
|
||||
fn patch(conf Config, id string, params map[string]string) ? {
|
||||
// patch patches a given target with the provided params.
|
||||
fn patch(conf Config, id string, params map[string]string) ! {
|
||||
// We check the cron expression first because it's useless to send an
|
||||
// invalid one to the server.
|
||||
if 'schedule' in params && params['schedule'] != '' {
|
||||
|
|
@ -242,24 +277,13 @@ fn patch(conf Config, id string, params map[string]string) ? {
|
|||
}
|
||||
}
|
||||
|
||||
id_int := id.int()
|
||||
if id_int != 0 {
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
res := c.patch_target(id_int, params)?
|
||||
|
||||
println(res.message)
|
||||
}
|
||||
}
|
||||
|
||||
// info shows detailed information for a given repo.
|
||||
fn info(conf Config, id string) ? {
|
||||
id_int := id.int()
|
||||
|
||||
if id_int == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
repo := c.get_target(id_int)?
|
||||
println(repo)
|
||||
c.patch_target(id.int(), params)!
|
||||
}
|
||||
|
||||
// info shows detailed information for a given target.
|
||||
fn info(conf Config, id string) ! {
|
||||
c := client.new(conf.address, conf.api_key)
|
||||
target := c.get_target(id.int())!
|
||||
println(target)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
module cron
|
||||
|
||||
import cli
|
||||
import vieter_v.conf as vconf
|
||||
import conf as vconf
|
||||
|
||||
struct Config {
|
||||
pub:
|
||||
|
|
@ -22,11 +22,11 @@ pub fn cmd() cli.Command {
|
|||
return cli.Command{
|
||||
name: 'cron'
|
||||
description: 'Start the cron service that periodically runs builds.'
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
config_file := cmd.flags.get_string('config-file')?
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||
|
||||
cron(conf)?
|
||||
cron(conf)!
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import os
|
|||
const log_file_name = 'vieter.cron.log'
|
||||
|
||||
// cron starts a cron daemon & starts periodically scheduling builds.
|
||||
pub fn cron(conf Config) ? {
|
||||
pub fn cron(conf Config) ! {
|
||||
// Configure logger
|
||||
log_level := log.level_from_tag(conf.log_level) or {
|
||||
return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
|
||||
|
|
@ -27,7 +27,7 @@ pub fn cron(conf Config) ? {
|
|||
}
|
||||
|
||||
mut d := daemon.init_daemon(logger, conf.address, conf.api_key, conf.base_image, ce,
|
||||
conf.max_concurrent_builds, conf.api_update_frequency, conf.image_rebuild_frequency)?
|
||||
conf.max_concurrent_builds, conf.api_update_frequency, conf.image_rebuild_frequency)!
|
||||
|
||||
d.run()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) {
|
|||
mut status := 0
|
||||
|
||||
res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(),
|
||||
&sb.target) or {
|
||||
&sb.target, false) or {
|
||||
d.ldebug('build_target error: $err.msg()')
|
||||
status = 1
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import datatypes { MinHeap }
|
|||
import cron.expression { CronExpression, parse_expression }
|
||||
import math
|
||||
import build
|
||||
import vieter_v.docker
|
||||
import docker
|
||||
import os
|
||||
import client
|
||||
import models { Target }
|
||||
|
|
@ -53,7 +53,7 @@ mut:
|
|||
|
||||
// init_daemon initializes a new Daemon object. It renews the targets &
|
||||
// populates the build queue for the first time.
|
||||
pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon {
|
||||
pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) !Daemon {
|
||||
mut d := Daemon{
|
||||
client: client.new(address, api_key)
|
||||
base_image: base_image
|
||||
|
|
@ -207,7 +207,7 @@ fn (mut d Daemon) renew_queue() {
|
|||
|
||||
// For some reason, using
|
||||
// ```v
|
||||
// for d.queue.len() > 0 && d.queue.peek() ?.timestamp < now {
|
||||
// for d.queue.len() > 0 && d.queue.peek() !.timestamp < now {
|
||||
//```
|
||||
// here causes the function to prematurely just exit, without any errors or anything, very weird
|
||||
// https://github.com/vlang/v/issues/14042
|
||||
|
|
|
|||
|
|
@ -3,33 +3,33 @@ module daemon
|
|||
import log
|
||||
|
||||
// log reate a log message with the given level
|
||||
pub fn (mut d Daemon) log(msg &string, level log.Level) {
|
||||
pub fn (mut d Daemon) log(msg string, level log.Level) {
|
||||
lock d.logger {
|
||||
d.logger.send_output(msg, level)
|
||||
}
|
||||
}
|
||||
|
||||
// lfatal create a log message with the fatal level
|
||||
pub fn (mut d Daemon) lfatal(msg &string) {
|
||||
pub fn (mut d Daemon) lfatal(msg string) {
|
||||
d.log(msg, log.Level.fatal)
|
||||
}
|
||||
|
||||
// lerror create a log message with the error level
|
||||
pub fn (mut d Daemon) lerror(msg &string) {
|
||||
pub fn (mut d Daemon) lerror(msg string) {
|
||||
d.log(msg, log.Level.error)
|
||||
}
|
||||
|
||||
// lwarn create a log message with the warn level
|
||||
pub fn (mut d Daemon) lwarn(msg &string) {
|
||||
pub fn (mut d Daemon) lwarn(msg string) {
|
||||
d.log(msg, log.Level.warn)
|
||||
}
|
||||
|
||||
// linfo create a log message with the info level
|
||||
pub fn (mut d Daemon) linfo(msg &string) {
|
||||
pub fn (mut d Daemon) linfo(msg string) {
|
||||
d.log(msg, log.Level.info)
|
||||
}
|
||||
|
||||
// ldebug create a log message with the debug level
|
||||
pub fn (mut d Daemon) ldebug(msg &string) {
|
||||
pub fn (mut d Daemon) ldebug(msg string) {
|
||||
d.log(msg, log.Level.debug)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ pub struct CronExpression {
|
|||
// next calculates the earliest time this cron expression is valid. It will
|
||||
// always pick a moment in the future, even if ref matches completely up to the
|
||||
// minute. This function conciously does not take gap years into account.
|
||||
pub fn (ce &CronExpression) next(ref time.Time) ?time.Time {
|
||||
pub fn (ce &CronExpression) next(ref time.Time) !time.Time {
|
||||
// If the given ref matches the next cron occurence up to the minute, it
|
||||
// will return that value. Because we always want to return a value in the
|
||||
// future, we artifically shift the ref 60 seconds to make sure we always
|
||||
|
|
@ -117,159 +117,20 @@ pub fn (ce &CronExpression) next(ref time.Time) ?time.Time {
|
|||
|
||||
// next_from_now returns the result of ce.next(ref) where ref is the result of
|
||||
// time.now().
|
||||
pub fn (ce &CronExpression) next_from_now() ?time.Time {
|
||||
pub fn (ce &CronExpression) next_from_now() !time.Time {
|
||||
return ce.next(time.now())
|
||||
}
|
||||
|
||||
// next_n returns the n next occurences of the expression, given a starting
|
||||
// time.
|
||||
pub fn (ce &CronExpression) next_n(ref time.Time, n int) ?[]time.Time {
|
||||
pub fn (ce &CronExpression) next_n(ref time.Time, n int) ![]time.Time {
|
||||
mut times := []time.Time{cap: n}
|
||||
|
||||
times << ce.next(ref)?
|
||||
times << ce.next(ref)!
|
||||
|
||||
for i in 1 .. n {
|
||||
times << ce.next(times[i - 1])?
|
||||
times << ce.next(times[i - 1])!
|
||||
}
|
||||
|
||||
return times
|
||||
}
|
||||
|
||||
// parse_range parses a given string into a range of sorted integers, if
|
||||
// possible.
|
||||
fn parse_range(s string, min int, max int, mut bitv []bool) ? {
|
||||
mut start := min
|
||||
mut end := max
|
||||
mut interval := 1
|
||||
|
||||
exps := s.split('/')
|
||||
|
||||
if exps.len > 2 {
|
||||
return error('Invalid expression.')
|
||||
}
|
||||
|
||||
if exps[0] != '*' {
|
||||
dash_parts := exps[0].split('-')
|
||||
|
||||
if dash_parts.len > 2 {
|
||||
return error('Invalid expression.')
|
||||
}
|
||||
|
||||
start = dash_parts[0].int()
|
||||
|
||||
// The builtin parsing functions return zero if the string can't be
|
||||
// parsed into a number, so we have to explicitely check whether they
|
||||
// actually entered zero or if it's an invalid number.
|
||||
if start == 0 && dash_parts[0] != '0' {
|
||||
return error('Invalid number.')
|
||||
}
|
||||
|
||||
// Check whether the start value is out of range
|
||||
if start < min || start > max {
|
||||
return error('Out of range.')
|
||||
}
|
||||
|
||||
if dash_parts.len == 2 {
|
||||
end = dash_parts[1].int()
|
||||
|
||||
if end == 0 && dash_parts[1] != '0' {
|
||||
return error('Invalid number.')
|
||||
}
|
||||
|
||||
if end < start || end > max {
|
||||
return error('Out of range.')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if exps.len > 1 {
|
||||
interval = exps[1].int()
|
||||
|
||||
// interval being zero is always invalid, but we want to check why
|
||||
// it's invalid for better error messages.
|
||||
if interval == 0 {
|
||||
if exps[1] != '0' {
|
||||
return error('Invalid number.')
|
||||
} else {
|
||||
return error('Step size zero not allowed.')
|
||||
}
|
||||
}
|
||||
|
||||
if interval > max - min {
|
||||
return error('Step size too large.')
|
||||
}
|
||||
}
|
||||
// Here, s solely consists of a number, so that's the only value we
|
||||
// should return.
|
||||
else if exps[0] != '*' && !exps[0].contains('-') {
|
||||
bitv[start - min] = true
|
||||
return
|
||||
}
|
||||
|
||||
for start <= end {
|
||||
bitv[start - min] = true
|
||||
start += interval
|
||||
}
|
||||
}
|
||||
|
||||
// bitv_to_ints converts a bit vector into an array containing the
|
||||
// corresponding values.
|
||||
fn bitv_to_ints(bitv []bool, min int) []int {
|
||||
mut out := []int{}
|
||||
|
||||
for i in 0 .. bitv.len {
|
||||
if bitv[i] {
|
||||
out << min + i
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// parse_part parses a given part of a cron expression & returns the
|
||||
// corresponding array of ints.
|
||||
fn parse_part(s string, min int, max int) ?[]int {
|
||||
mut bitv := []bool{len: max - min + 1, init: false}
|
||||
|
||||
for range in s.split(',') {
|
||||
parse_range(range, min, max, mut bitv)?
|
||||
}
|
||||
|
||||
return bitv_to_ints(bitv, min)
|
||||
}
|
||||
|
||||
// parse_expression parses an entire cron expression string into a
|
||||
// CronExpression object, if possible.
|
||||
pub fn parse_expression(exp string) ?CronExpression {
|
||||
// The filter allows for multiple spaces between parts
|
||||
mut parts := exp.split(' ').filter(it != '')
|
||||
|
||||
if parts.len < 2 || parts.len > 4 {
|
||||
return error('Expression must contain between 2 and 4 space-separated parts.')
|
||||
}
|
||||
|
||||
// For ease of use, we allow the user to only specify as many parts as they
|
||||
// need.
|
||||
for parts.len < 4 {
|
||||
parts << '*'
|
||||
}
|
||||
|
||||
mut part_results := [][]int{}
|
||||
|
||||
mins := [0, 0, 1, 1]
|
||||
maxs := [59, 23, 31, 12]
|
||||
|
||||
// This for loop allows us to more clearly propagate the error to the user.
|
||||
for i, min in mins {
|
||||
part_results << parse_part(parts[i], min, maxs[i]) or {
|
||||
return error('An error occurred with part $i: $err.msg()')
|
||||
}
|
||||
}
|
||||
|
||||
return CronExpression{
|
||||
minutes: part_results[0]
|
||||
hours: part_results[1]
|
||||
days: part_results[2]
|
||||
months: part_results[3]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,146 @@
|
|||
module expression
|
||||
|
||||
import bitfield
|
||||
|
||||
// parse_range parses a given string into a range of sorted integers. Its
|
||||
// result is a BitField with set bits for all numbers in the result.
|
||||
fn parse_range(s string, min int, max int) !bitfield.BitField {
|
||||
mut start := min
|
||||
mut end := max
|
||||
mut interval := 1
|
||||
mut bf := bitfield.new(max - min + 1)
|
||||
|
||||
exps := s.split('/')
|
||||
|
||||
if exps.len > 2 {
|
||||
return error('Invalid expression.')
|
||||
}
|
||||
|
||||
if exps[0] != '*' {
|
||||
dash_parts := exps[0].split('-')
|
||||
|
||||
if dash_parts.len > 2 {
|
||||
return error('Invalid expression.')
|
||||
}
|
||||
|
||||
start = dash_parts[0].int()
|
||||
|
||||
// The builtin parsing functions return zero if the string can't be
|
||||
// parsed into a number, so we have to explicitely check whether they
|
||||
// actually entered zero or if it's an invalid number.
|
||||
if start == 0 && dash_parts[0] != '0' {
|
||||
return error('Invalid number.')
|
||||
}
|
||||
|
||||
// Check whether the start value is out of range
|
||||
if start < min || start > max {
|
||||
return error('Out of range.')
|
||||
}
|
||||
|
||||
if dash_parts.len == 2 {
|
||||
end = dash_parts[1].int()
|
||||
|
||||
if end == 0 && dash_parts[1] != '0' {
|
||||
return error('Invalid number.')
|
||||
}
|
||||
|
||||
if end < start || end > max {
|
||||
return error('Out of range.')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if exps.len > 1 {
|
||||
interval = exps[1].int()
|
||||
|
||||
// interval being zero is always invalid, but we want to check why
|
||||
// it's invalid for better error messages.
|
||||
if interval == 0 {
|
||||
if exps[1] != '0' {
|
||||
return error('Invalid number.')
|
||||
} else {
|
||||
return error('Step size zero not allowed.')
|
||||
}
|
||||
}
|
||||
|
||||
if interval > max - min {
|
||||
return error('Step size too large.')
|
||||
}
|
||||
}
|
||||
// Here, s solely consists of a number, so that's the only value we
|
||||
// should return.
|
||||
else if exps[0] != '*' && !exps[0].contains('-') {
|
||||
bf.set_bit(start - min)
|
||||
return bf
|
||||
}
|
||||
|
||||
for start <= end {
|
||||
bf.set_bit(start - min)
|
||||
start += interval
|
||||
}
|
||||
|
||||
return bf
|
||||
}
|
||||
|
||||
// bf_to_ints takes a BitField and converts it into the expected list of actual
|
||||
// integers.
|
||||
fn bf_to_ints(bf bitfield.BitField, min int) []int {
|
||||
mut out := []int{}
|
||||
|
||||
for i in 0 .. bf.get_size() {
|
||||
if bf.get_bit(i) == 1 {
|
||||
out << min + i
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// parse_part parses a given part of a cron expression & returns the
|
||||
// corresponding array of ints.
|
||||
fn parse_part(s string, min int, max int) ![]int {
|
||||
mut bf := bitfield.new(max - min + 1)
|
||||
|
||||
for range in s.split(',') {
|
||||
bf2 := parse_range(range, min, max)!
|
||||
bf = bitfield.bf_or(bf, bf2)
|
||||
}
|
||||
|
||||
return bf_to_ints(bf, min)
|
||||
}
|
||||
|
||||
// parse_expression parses an entire cron expression string into a
|
||||
// CronExpression object, if possible.
|
||||
pub fn parse_expression(exp string) !CronExpression {
|
||||
// The filter allows for multiple spaces between parts
|
||||
mut parts := exp.split(' ').filter(it != '')
|
||||
|
||||
if parts.len < 2 || parts.len > 4 {
|
||||
return error('Expression must contain between 2 and 4 space-separated parts.')
|
||||
}
|
||||
|
||||
// For ease of use, we allow the user to only specify as many parts as they
|
||||
// need.
|
||||
for parts.len < 4 {
|
||||
parts << '*'
|
||||
}
|
||||
|
||||
mut part_results := [][]int{}
|
||||
|
||||
mins := [0, 0, 1, 1]
|
||||
maxs := [59, 23, 31, 12]
|
||||
|
||||
// This for loop allows us to more clearly propagate the error to the user.
|
||||
for i, min in mins {
|
||||
part_results << parse_part(parts[i], min, maxs[i]) or {
|
||||
return error('An error occurred with part $i: $err.msg()')
|
||||
}
|
||||
}
|
||||
|
||||
return CronExpression{
|
||||
minutes: part_results[0]
|
||||
hours: part_results[1]
|
||||
days: part_results[2]
|
||||
months: part_results[3]
|
||||
}
|
||||
}
|
||||
|
|
@ -3,96 +3,87 @@ module expression
|
|||
// parse_range_error returns the returned error message. If the result is '',
|
||||
// that means the function didn't error.
|
||||
fn parse_range_error(s string, min int, max int) string {
|
||||
mut bitv := []bool{len: max - min + 1, init: false}
|
||||
|
||||
parse_range(s, min, max, mut bitv) or { return err.msg }
|
||||
parse_range(s, min, max) or { return err.msg }
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
// =====parse_range=====
|
||||
fn test_range_star_range() ? {
|
||||
mut bitv := []bool{len: 6, init: false}
|
||||
parse_range('*', 0, 5, mut bitv)?
|
||||
fn test_range_star_range() ! {
|
||||
bf := parse_range('*', 0, 5)!
|
||||
|
||||
assert bitv == [true, true, true, true, true, true]
|
||||
assert bf_to_ints(bf, 0) == [0, 1, 2, 3, 4, 5]
|
||||
}
|
||||
|
||||
fn test_range_number() ? {
|
||||
mut bitv := []bool{len: 6, init: false}
|
||||
parse_range('4', 0, 5, mut bitv)?
|
||||
fn test_range_number() ! {
|
||||
bf := parse_range('4', 0, 5)!
|
||||
|
||||
assert bitv_to_ints(bitv, 0) == [4]
|
||||
assert bf_to_ints(bf, 0) == [4]
|
||||
}
|
||||
|
||||
fn test_range_number_too_large() ? {
|
||||
fn test_range_number_too_large() ! {
|
||||
assert parse_range_error('10', 0, 6) == 'Out of range.'
|
||||
}
|
||||
|
||||
fn test_range_number_too_small() ? {
|
||||
fn test_range_number_too_small() ! {
|
||||
assert parse_range_error('0', 2, 6) == 'Out of range.'
|
||||
}
|
||||
|
||||
fn test_range_number_invalid() ? {
|
||||
fn test_range_number_invalid() ! {
|
||||
assert parse_range_error('x', 0, 6) == 'Invalid number.'
|
||||
}
|
||||
|
||||
fn test_range_step_star_1() ? {
|
||||
mut bitv := []bool{len: 21, init: false}
|
||||
parse_range('*/4', 0, 20, mut bitv)?
|
||||
fn test_range_step_star_1() ! {
|
||||
bf := parse_range('*/4', 0, 20)!
|
||||
|
||||
assert bitv_to_ints(bitv, 0) == [0, 4, 8, 12, 16, 20]
|
||||
assert bf_to_ints(bf, 0) == [0, 4, 8, 12, 16, 20]
|
||||
}
|
||||
|
||||
fn test_range_step_star_2() ? {
|
||||
mut bitv := []bool{len: 8, init: false}
|
||||
parse_range('*/3', 1, 8, mut bitv)?
|
||||
fn test_range_step_star_2() ! {
|
||||
bf := parse_range('*/3', 1, 8)!
|
||||
|
||||
assert bitv_to_ints(bitv, 1) == [1, 4, 7]
|
||||
assert bf_to_ints(bf, 1) == [1, 4, 7]
|
||||
}
|
||||
|
||||
fn test_range_step_star_too_large() ? {
|
||||
fn test_range_step_star_too_large() ! {
|
||||
assert parse_range_error('*/21', 0, 20) == 'Step size too large.'
|
||||
}
|
||||
|
||||
fn test_range_step_zero() ? {
|
||||
fn test_range_step_zero() ! {
|
||||
assert parse_range_error('*/0', 0, 20) == 'Step size zero not allowed.'
|
||||
}
|
||||
|
||||
fn test_range_step_number() ? {
|
||||
mut bitv := []bool{len: 21, init: false}
|
||||
parse_range('5/4', 2, 22, mut bitv)?
|
||||
fn test_range_step_number() ! {
|
||||
bf := parse_range('5/4', 2, 22)!
|
||||
|
||||
assert bitv_to_ints(bitv, 2) == [5, 9, 13, 17, 21]
|
||||
assert bf_to_ints(bf, 2) == [5, 9, 13, 17, 21]
|
||||
}
|
||||
|
||||
fn test_range_step_number_too_large() ? {
|
||||
fn test_range_step_number_too_large() ! {
|
||||
assert parse_range_error('10/4', 0, 5) == 'Out of range.'
|
||||
}
|
||||
|
||||
fn test_range_step_number_too_small() ? {
|
||||
fn test_range_step_number_too_small() ! {
|
||||
assert parse_range_error('2/4', 5, 10) == 'Out of range.'
|
||||
}
|
||||
|
||||
fn test_range_dash() ? {
|
||||
mut bitv := []bool{len: 10, init: false}
|
||||
parse_range('4-8', 0, 9, mut bitv)?
|
||||
fn test_range_dash() ! {
|
||||
bf := parse_range('4-8', 0, 9)!
|
||||
|
||||
assert bitv_to_ints(bitv, 0) == [4, 5, 6, 7, 8]
|
||||
assert bf_to_ints(bf, 0) == [4, 5, 6, 7, 8]
|
||||
}
|
||||
|
||||
fn test_range_dash_step() ? {
|
||||
mut bitv := []bool{len: 10, init: false}
|
||||
parse_range('4-8/2', 0, 9, mut bitv)?
|
||||
fn test_range_dash_step() ! {
|
||||
bf := parse_range('4-8/2', 0, 9)!
|
||||
|
||||
assert bitv_to_ints(bitv, 0) == [4, 6, 8]
|
||||
assert bf_to_ints(bf, 0) == [4, 6, 8]
|
||||
}
|
||||
|
||||
// =====parse_part=====
|
||||
fn test_part_single() ? {
|
||||
assert parse_part('*', 0, 5)? == [0, 1, 2, 3, 4, 5]
|
||||
fn test_part_single() ! {
|
||||
assert parse_part('*', 0, 5)! == [0, 1, 2, 3, 4, 5]
|
||||
}
|
||||
|
||||
fn test_part_multiple() ? {
|
||||
assert parse_part('*/2,2/3', 1, 8)? == [1, 2, 3, 5, 7, 8]
|
||||
fn test_part_multiple() ! {
|
||||
assert parse_part('*/2,2/3', 1, 8)! == [1, 2, 3, 5, 7, 8]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,12 +2,12 @@ module expression
|
|||
|
||||
import time { parse }
|
||||
|
||||
fn util_test_time(exp string, t1_str string, t2_str string) ? {
|
||||
ce := parse_expression(exp)?
|
||||
t1 := parse(t1_str)?
|
||||
t2 := parse(t2_str)?
|
||||
fn util_test_time(exp string, t1_str string, t2_str string) ! {
|
||||
ce := parse_expression(exp)!
|
||||
t1 := parse(t1_str)!
|
||||
t2 := parse(t2_str)!
|
||||
|
||||
t3 := ce.next(t1)?
|
||||
t3 := ce.next(t1)!
|
||||
|
||||
assert t2.year == t3.year
|
||||
assert t2.month == t3.month
|
||||
|
|
@ -16,19 +16,19 @@ fn util_test_time(exp string, t1_str string, t2_str string) ? {
|
|||
assert t2.minute == t3.minute
|
||||
}
|
||||
|
||||
fn test_next_simple() ? {
|
||||
fn test_next_simple() ! {
|
||||
// Very simple
|
||||
util_test_time('0 3', '2002-01-01 00:00:00', '2002-01-01 03:00:00')?
|
||||
util_test_time('0 3', '2002-01-01 00:00:00', '2002-01-01 03:00:00')!
|
||||
|
||||
// Overlap to next day
|
||||
util_test_time('0 3', '2002-01-01 03:00:00', '2002-01-02 03:00:00')?
|
||||
util_test_time('0 3', '2002-01-01 04:00:00', '2002-01-02 03:00:00')?
|
||||
util_test_time('0 3', '2002-01-01 03:00:00', '2002-01-02 03:00:00')!
|
||||
util_test_time('0 3', '2002-01-01 04:00:00', '2002-01-02 03:00:00')!
|
||||
|
||||
util_test_time('0 3/4', '2002-01-01 04:00:00', '2002-01-01 07:00:00')?
|
||||
util_test_time('0 3/4', '2002-01-01 04:00:00', '2002-01-01 07:00:00')!
|
||||
|
||||
// Overlap to next month
|
||||
util_test_time('0 3', '2002-11-31 04:00:00', '2002-12-01 03:00:00')?
|
||||
util_test_time('0 3', '2002-11-31 04:00:00', '2002-12-01 03:00:00')!
|
||||
|
||||
// Overlap to next year
|
||||
util_test_time('0 3', '2002-12-31 04:00:00', '2003-01-01 03:00:00')?
|
||||
util_test_time('0 3', '2002-12-31 04:00:00', '2003-01-01 03:00:00')!
|
||||
}
|
||||
|
|
|
|||
10
src/db/db.v
10
src/db/db.v
|
|
@ -17,17 +17,21 @@ const (
|
|||
$embed_file('migrations/001-initial/up.sql'),
|
||||
$embed_file('migrations/002-rename-to-targets/up.sql'),
|
||||
$embed_file('migrations/003-target-url-type/up.sql'),
|
||||
$embed_file('migrations/004-nullable-branch/up.sql'),
|
||||
$embed_file('migrations/005-repo-path/up.sql'),
|
||||
]
|
||||
migrations_down = [
|
||||
$embed_file('migrations/001-initial/down.sql'),
|
||||
$embed_file('migrations/002-rename-to-targets/down.sql'),
|
||||
$embed_file('migrations/003-target-url-type/down.sql'),
|
||||
$embed_file('migrations/004-nullable-branch/down.sql'),
|
||||
$embed_file('migrations/005-repo-path/down.sql'),
|
||||
]
|
||||
)
|
||||
|
||||
// init initializes a database & adds the correct tables.
|
||||
pub fn init(db_path string) ?VieterDb {
|
||||
conn := sqlite.connect(db_path)?
|
||||
pub fn init(db_path string) !VieterDb {
|
||||
conn := sqlite.connect(db_path)!
|
||||
|
||||
sql conn {
|
||||
create table MigrationVersion
|
||||
|
|
@ -60,7 +64,7 @@ pub fn init(db_path string) ?VieterDb {
|
|||
res := conn.exec_none(part)
|
||||
|
||||
if res != sqlite.sqlite_done {
|
||||
return error('An error occurred while applying migration $version_num')
|
||||
return error('An error occurred while applying migration $version_num: SQLite error code $res')
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -84,6 +84,8 @@ pub fn (db &VieterDb) add_build_log(log BuildLog) int {
|
|||
insert log into BuildLog
|
||||
}
|
||||
|
||||
// Here, this does work because a log doesn't contain any foreign keys,
|
||||
// meaning the ORM only has to do a single add
|
||||
inserted_id := db.conn.last_id() as int
|
||||
|
||||
return inserted_id
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
-- This down won't really work because it'll throw NOT NULL errors, but I'm
|
||||
-- just putting it here for future reference (still not sure whether I'm even
|
||||
-- gonna use these)
|
||||
PRAGMA foreign_keys=off;
|
||||
|
||||
BEGIN TRANSACTION;
|
||||
|
||||
ALTER TABLE Target RENAME TO _Target_old;
|
||||
|
||||
CREATE TABLE Target (
|
||||
id INTEGER PRIMARY KEY,
|
||||
url TEXT NOT NULL,
|
||||
branch TEXT NOT NULL,
|
||||
repo TEXT NOT NULL,
|
||||
schedule TEXT,
|
||||
kind TEXT NOT NULL DEFAULT 'git'
|
||||
);
|
||||
|
||||
INSERT INTO Target (id, url, branch, repo, schedule, kind)
|
||||
SELECT id, url, branch, repo, schedule, kind FROM _Target_old;
|
||||
|
||||
DROP TABLE _Target_old;
|
||||
|
||||
COMMIT;
|
||||
|
||||
PRAGMA foreign_keys=on;
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
PRAGMA foreign_keys=off;
|
||||
|
||||
BEGIN TRANSACTION;
|
||||
|
||||
ALTER TABLE Target RENAME TO _Target_old;
|
||||
|
||||
CREATE TABLE Target (
|
||||
id INTEGER PRIMARY KEY,
|
||||
url TEXT NOT NULL,
|
||||
branch TEXT,
|
||||
repo TEXT NOT NULL,
|
||||
schedule TEXT,
|
||||
kind TEXT NOT NULL DEFAULT 'git'
|
||||
);
|
||||
|
||||
INSERT INTO Target (id, url, branch, repo, schedule, kind)
|
||||
SELECT id, url, branch, repo, schedule, kind FROM _Target_old;
|
||||
|
||||
DROP TABLE _Target_old;
|
||||
|
||||
COMMIT;
|
||||
|
||||
PRAGMA foreign_keys=on;
|
||||
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE Target DROP COLUMN path;
|
||||
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE Target ADD COLUMN path TEXT;
|
||||
|
|
@ -38,14 +38,17 @@ pub fn (db &VieterDb) get_target(target_id int) ?Target {
|
|||
}
|
||||
|
||||
// add_target inserts the given target into the database.
|
||||
pub fn (db &VieterDb) add_target(repo Target) int {
|
||||
pub fn (db &VieterDb) add_target(target Target) int {
|
||||
sql db.conn {
|
||||
insert repo into Target
|
||||
insert target into Target
|
||||
}
|
||||
|
||||
inserted_id := db.conn.last_id() as int
|
||||
// ID of inserted target is the largest id
|
||||
inserted_target := sql db.conn {
|
||||
select from Target order by id desc limit 1
|
||||
}
|
||||
|
||||
return inserted_id
|
||||
return inserted_target.id
|
||||
}
|
||||
|
||||
// delete_target deletes the target with the given id from the database.
|
||||
|
|
|
|||
|
|
@ -9,12 +9,18 @@ import console.schedule
|
|||
import console.man
|
||||
import console.aur
|
||||
import cron
|
||||
import agent
|
||||
|
||||
fn main() {
|
||||
// Stop buffering output so logs always show up immediately
|
||||
unsafe {
|
||||
C.setbuf(C.stdout, 0)
|
||||
}
|
||||
|
||||
mut app := cli.Command{
|
||||
name: 'vieter'
|
||||
description: 'Vieter is a lightweight implementation of an Arch repository server.'
|
||||
version: '0.4.0'
|
||||
version: '0.5.0-rc.1'
|
||||
flags: [
|
||||
cli.Flag{
|
||||
flag: cli.FlagType.string
|
||||
|
|
@ -40,6 +46,7 @@ fn main() {
|
|||
schedule.cmd(),
|
||||
man.cmd(),
|
||||
aur.cmd(),
|
||||
agent.cmd(),
|
||||
]
|
||||
}
|
||||
app.setup()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
module models
|
||||
|
||||
pub struct BuildConfig {
|
||||
pub:
|
||||
target_id int
|
||||
kind string
|
||||
url string
|
||||
branch string
|
||||
path string
|
||||
repo string
|
||||
base_image string
|
||||
force bool
|
||||
}
|
||||
|
||||
// str return a single-line string representation of a build log
|
||||
pub fn (c BuildConfig) str() string {
|
||||
return '{ target: $c.target_id, kind: $c.kind, url: $c.url, branch: $c.branch, path: $c.path, repo: $c.repo, base_image: $c.base_image, force: $c.force }'
|
||||
}
|
||||
|
|
@ -28,27 +28,45 @@ pub mut:
|
|||
repo string [nonull]
|
||||
// Cron schedule describing how frequently to build the repo.
|
||||
schedule string
|
||||
// Subdirectory in the Git repository to cd into
|
||||
path string
|
||||
// On which architectures the package is allowed to be built. In reality,
|
||||
// this controls which builders will periodically build the image.
|
||||
// this controls which agents will build this package when scheduled.
|
||||
arch []TargetArch [fkey: 'target_id']
|
||||
}
|
||||
|
||||
// str returns a string representation.
|
||||
pub fn (gr &Target) str() string {
|
||||
pub fn (t &Target) str() string {
|
||||
mut parts := [
|
||||
'id: $gr.id',
|
||||
'kind: $gr.kind',
|
||||
'url: $gr.url',
|
||||
'branch: $gr.branch',
|
||||
'repo: $gr.repo',
|
||||
'schedule: $gr.schedule',
|
||||
'arch: ${gr.arch.map(it.value).join(', ')}',
|
||||
'id: $t.id',
|
||||
'kind: $t.kind',
|
||||
'url: $t.url',
|
||||
'branch: $t.branch',
|
||||
'path: $t.path',
|
||||
'repo: $t.repo',
|
||||
'schedule: $t.schedule',
|
||||
'arch: ${t.arch.map(it.value).join(', ')}',
|
||||
]
|
||||
str := parts.join('\n')
|
||||
|
||||
return str
|
||||
}
|
||||
|
||||
// as_build_config converts a Target into a BuildConfig, given some extra
|
||||
// needed information.
|
||||
pub fn (t &Target) as_build_config(base_image string, force bool) BuildConfig {
|
||||
return BuildConfig{
|
||||
target_id: t.id
|
||||
kind: t.kind
|
||||
url: t.url
|
||||
branch: t.branch
|
||||
path: t.path
|
||||
repo: t.repo
|
||||
base_image: base_image
|
||||
force: force
|
||||
}
|
||||
}
|
||||
|
||||
[params]
|
||||
pub struct TargetFilter {
|
||||
pub mut:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
# package
|
||||
|
||||
This module handles both parsing the published Arch tarballs & the contents of
|
||||
their `.PKGINFO` files, as well as generating the contents of the database
|
||||
archives' `desc` & `files` files.
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
module package
|
||||
|
||||
// format_entry returns a string properly formatted to be added to a desc file.
|
||||
[inline]
|
||||
fn format_entry(key string, value string) string {
|
||||
return '\n%$key%\n$value\n'
|
||||
}
|
||||
|
||||
// full_name returns the properly formatted name for the package, including
|
||||
// version & architecture
|
||||
pub fn (pkg &Pkg) full_name() string {
|
||||
p := pkg.info
|
||||
return '$p.name-$p.version-$p.arch'
|
||||
}
|
||||
|
||||
// filename returns the correct filename of the package file
|
||||
pub fn (pkg &Pkg) filename() string {
|
||||
ext := match pkg.compression {
|
||||
0 { '.tar' }
|
||||
1 { '.tar.gz' }
|
||||
6 { '.tar.xz' }
|
||||
14 { '.tar.zst' }
|
||||
else { panic("Another compression code shouldn't be possible. Faulty code: $pkg.compression") }
|
||||
}
|
||||
|
||||
return '${pkg.full_name()}.pkg$ext'
|
||||
}
|
||||
|
||||
// to_desc returns a desc file valid string representation
|
||||
pub fn (pkg &Pkg) to_desc() !string {
|
||||
p := pkg.info
|
||||
|
||||
// filename
|
||||
mut desc := '%FILENAME%\n$pkg.filename()\n'
|
||||
|
||||
desc += format_entry('NAME', p.name)
|
||||
desc += format_entry('BASE', p.base)
|
||||
desc += format_entry('VERSION', p.version)
|
||||
|
||||
if p.description.len > 0 {
|
||||
desc += format_entry('DESC', p.description)
|
||||
}
|
||||
|
||||
if p.groups.len > 0 {
|
||||
desc += format_entry('GROUPS', p.groups.join_lines())
|
||||
}
|
||||
|
||||
desc += format_entry('CSIZE', p.csize.str())
|
||||
desc += format_entry('ISIZE', p.size.str())
|
||||
|
||||
sha256sum := pkg.checksum()!
|
||||
|
||||
desc += format_entry('SHA256SUM', sha256sum)
|
||||
|
||||
// TODO add pgpsig stuff
|
||||
|
||||
if p.url.len > 0 {
|
||||
desc += format_entry('URL', p.url)
|
||||
}
|
||||
|
||||
if p.licenses.len > 0 {
|
||||
desc += format_entry('LICENSE', p.licenses.join_lines())
|
||||
}
|
||||
|
||||
desc += format_entry('ARCH', p.arch)
|
||||
desc += format_entry('BUILDDATE', p.build_date.str())
|
||||
desc += format_entry('PACKAGER', p.packager)
|
||||
|
||||
if p.replaces.len > 0 {
|
||||
desc += format_entry('REPLACES', p.replaces.join_lines())
|
||||
}
|
||||
|
||||
if p.conflicts.len > 0 {
|
||||
desc += format_entry('CONFLICTS', p.conflicts.join_lines())
|
||||
}
|
||||
|
||||
if p.provides.len > 0 {
|
||||
desc += format_entry('PROVIDES', p.provides.join_lines())
|
||||
}
|
||||
|
||||
if p.depends.len > 0 {
|
||||
desc += format_entry('DEPENDS', p.depends.join_lines())
|
||||
}
|
||||
|
||||
if p.optdepends.len > 0 {
|
||||
desc += format_entry('OPTDEPENDS', p.optdepends.join_lines())
|
||||
}
|
||||
|
||||
if p.makedepends.len > 0 {
|
||||
desc += format_entry('MAKEDEPENDS', p.makedepends.join_lines())
|
||||
}
|
||||
|
||||
if p.checkdepends.len > 0 {
|
||||
desc += format_entry('CHECKDEPENDS', p.checkdepends.join_lines())
|
||||
}
|
||||
|
||||
return '$desc\n'
|
||||
}
|
||||
|
||||
// to_files returns a files file valid string representation
|
||||
pub fn (pkg &Pkg) to_files() string {
|
||||
return '%FILES%\n$pkg.files.join_lines()\n'
|
||||
}
|
||||
|
|
@ -43,12 +43,12 @@ pub mut:
|
|||
}
|
||||
|
||||
// checksum calculates the sha256 hash of the package
|
||||
pub fn (p &Pkg) checksum() ?string {
|
||||
pub fn (p &Pkg) checksum() !string {
|
||||
return util.hash_file(p.path)
|
||||
}
|
||||
|
||||
// parse_pkg_info_string parses a PkgInfo object from a string
|
||||
fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo {
|
||||
fn parse_pkg_info_string(pkg_info_str &string) !PkgInfo {
|
||||
mut pkg_info := PkgInfo{}
|
||||
|
||||
// Iterate over the entire string
|
||||
|
|
@ -101,7 +101,7 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo {
|
|||
|
||||
// read_pkg_archive extracts the file list & .PKGINFO contents from an archive
|
||||
// NOTE: this command only supports zstd-, xz- & gzip-compressed tarballs.
|
||||
pub fn read_pkg_archive(pkg_path string) ?Pkg {
|
||||
pub fn read_pkg_archive(pkg_path string) !Pkg {
|
||||
if !os.is_file(pkg_path) {
|
||||
return error("'$pkg_path' doesn't exist or isn't a file.")
|
||||
}
|
||||
|
|
@ -159,7 +159,7 @@ pub fn read_pkg_archive(pkg_path string) ?Pkg {
|
|||
|
||||
pkg_text := unsafe { buf.vstring_with_len(size).clone() }
|
||||
|
||||
pkg_info = parse_pkg_info_string(pkg_text)?
|
||||
pkg_info = parse_pkg_info_string(pkg_text)!
|
||||
} else {
|
||||
C.archive_read_data_skip(a)
|
||||
}
|
||||
|
|
@ -174,104 +174,3 @@ pub fn read_pkg_archive(pkg_path string) ?Pkg {
|
|||
compression: compression_code
|
||||
}
|
||||
}
|
||||
|
||||
// format_entry returns a string properly formatted to be added to a desc file.
|
||||
fn format_entry(key string, value string) string {
|
||||
return '\n%$key%\n$value\n'
|
||||
}
|
||||
|
||||
// full_name returns the properly formatted name for the package, including
|
||||
// version & architecture
|
||||
pub fn (pkg &Pkg) full_name() string {
|
||||
p := pkg.info
|
||||
return '$p.name-$p.version-$p.arch'
|
||||
}
|
||||
|
||||
// filename returns the correct filename of the package file
|
||||
pub fn (pkg &Pkg) filename() string {
|
||||
ext := match pkg.compression {
|
||||
0 { '.tar' }
|
||||
1 { '.tar.gz' }
|
||||
6 { '.tar.xz' }
|
||||
14 { '.tar.zst' }
|
||||
else { panic("Another compression code shouldn't be possible. Faulty code: $pkg.compression") }
|
||||
}
|
||||
|
||||
return '${pkg.full_name()}.pkg$ext'
|
||||
}
|
||||
|
||||
// to_desc returns a desc file valid string representation
|
||||
pub fn (pkg &Pkg) to_desc() ?string {
|
||||
p := pkg.info
|
||||
|
||||
// filename
|
||||
mut desc := '%FILENAME%\n$pkg.filename()\n'
|
||||
|
||||
desc += format_entry('NAME', p.name)
|
||||
desc += format_entry('BASE', p.base)
|
||||
desc += format_entry('VERSION', p.version)
|
||||
|
||||
if p.description.len > 0 {
|
||||
desc += format_entry('DESC', p.description)
|
||||
}
|
||||
|
||||
if p.groups.len > 0 {
|
||||
desc += format_entry('GROUPS', p.groups.join_lines())
|
||||
}
|
||||
|
||||
desc += format_entry('CSIZE', p.csize.str())
|
||||
desc += format_entry('ISIZE', p.size.str())
|
||||
|
||||
sha256sum := pkg.checksum()?
|
||||
|
||||
desc += format_entry('SHA256SUM', sha256sum)
|
||||
|
||||
// TODO add pgpsig stuff
|
||||
|
||||
if p.url.len > 0 {
|
||||
desc += format_entry('URL', p.url)
|
||||
}
|
||||
|
||||
if p.licenses.len > 0 {
|
||||
desc += format_entry('LICENSE', p.licenses.join_lines())
|
||||
}
|
||||
|
||||
desc += format_entry('ARCH', p.arch)
|
||||
desc += format_entry('BUILDDATE', p.build_date.str())
|
||||
desc += format_entry('PACKAGER', p.packager)
|
||||
|
||||
if p.replaces.len > 0 {
|
||||
desc += format_entry('REPLACES', p.replaces.join_lines())
|
||||
}
|
||||
|
||||
if p.conflicts.len > 0 {
|
||||
desc += format_entry('CONFLICTS', p.conflicts.join_lines())
|
||||
}
|
||||
|
||||
if p.provides.len > 0 {
|
||||
desc += format_entry('PROVIDES', p.provides.join_lines())
|
||||
}
|
||||
|
||||
if p.depends.len > 0 {
|
||||
desc += format_entry('DEPENDS', p.depends.join_lines())
|
||||
}
|
||||
|
||||
if p.optdepends.len > 0 {
|
||||
desc += format_entry('OPTDEPENDS', p.optdepends.join_lines())
|
||||
}
|
||||
|
||||
if p.makedepends.len > 0 {
|
||||
desc += format_entry('MAKEDEPENDS', p.makedepends.join_lines())
|
||||
}
|
||||
|
||||
if p.checkdepends.len > 0 {
|
||||
desc += format_entry('CHECKDEPENDS', p.checkdepends.join_lines())
|
||||
}
|
||||
|
||||
return '$desc\n'
|
||||
}
|
||||
|
||||
// to_files returns a files file valid string representation
|
||||
pub fn (pkg &Pkg) to_files() string {
|
||||
return '%FILES%\n$pkg.files.join_lines()\n'
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
# repo
|
||||
|
||||
This module manages the contents of the various repositories stored within a
|
||||
Vieter instance.
|
||||
|
||||
## Terminology
|
||||
|
||||
* Arch-repository (arch-repo): specific architecture of a given repository. This is what
|
||||
Pacman actually uses as a repository, and contains its own `.db` & `.files`
|
||||
files.
|
||||
* Repository (repo): a collection of arch-repositories. A single repository can
|
||||
contain packages of different architectures, with each package being stored
|
||||
in that specific architecture' arch-repository.
|
||||
* Repository group (repo-group): a collection of repositories. Each Vieter
|
||||
instance consists of a single repository group, which manages all underlying
|
||||
repositories & arch-repositories.
|
||||
|
||||
## Arch-repository layout
|
||||
|
||||
An arch-repository (aka a regular Pacman repository) consists of a directory
|
||||
with the following files (`{repo}` should be replaced with the name of the
|
||||
repository):
|
||||
|
||||
* One or more package directories. These directories follow the naming scheme
|
||||
`${pkgname}-${pkgver}-${pkgrel}`. Each of these directories contains two
|
||||
files, `desc` & `files`. The `desc` file is a list of the package's metadata,
|
||||
while `files` contains a list of all files that the package contains. The
|
||||
latter is used when using `pacman -F`.
|
||||
* `{repo}.db` & `{repo}.db.tar.gz`: the database file of the repository. This
|
||||
is just a compressed tarball of all package directories, but only their
|
||||
`desc` files. Both these files should have the same content (`repo-add`
|
||||
creates a symlink, but Vieter just serves the same file for both routes)
|
||||
* `{repo}.files` & `{repo}.files.tar.gz`: the same as the `.db` file, but this
|
||||
also contains the `files` files, instead of just the `desc` files.
|
||||
|
||||
## Filesystem layout
|
||||
|
||||
The repository part of Vieter consists of two directories. One is the `repos`
|
||||
directory inside the configured `data_dir`, while the other is the configured
|
||||
`pkg_dir`. `repos` contains only the repository group, while `pkg_dir` contains
|
||||
the actual package archives. `pkg_dir` is the directory that can take up a
|
||||
significant amount of memory, while `repos` solely consists of small text
|
||||
files.
|
||||
|
|
@ -29,7 +29,7 @@ pub:
|
|||
}
|
||||
|
||||
// new creates a new RepoGroupManager & creates the directories as needed
|
||||
pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupManager {
|
||||
pub fn new(repos_dir string, pkg_dir string, default_arch string) !RepoGroupManager {
|
||||
if !os.is_dir(repos_dir) {
|
||||
os.mkdir_all(repos_dir) or { return error('Failed to create repos directory: $err.msg()') }
|
||||
}
|
||||
|
|
@ -49,27 +49,27 @@ pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupMana
|
|||
// pkg archive. It's a wrapper around add_pkg_in_repo that parses the archive
|
||||
// file, passes the result to add_pkg_in_repo, and hard links the archive to
|
||||
// the right subdirectories in r.pkg_dir if it was successfully added.
|
||||
pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult {
|
||||
pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) !RepoAddResult {
|
||||
pkg := package.read_pkg_archive(pkg_path) or {
|
||||
return error('Failed to read package file: $err.msg()')
|
||||
}
|
||||
|
||||
archs := r.add_pkg_in_repo(repo, pkg)?
|
||||
archs := r.add_pkg_in_repo(repo, pkg)!
|
||||
|
||||
// If the add was successful, we move the file to the packages directory
|
||||
for arch in archs {
|
||||
repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch))
|
||||
dest_path := os.join_path_single(repo_pkg_path, pkg.filename())
|
||||
|
||||
os.mkdir_all(repo_pkg_path)?
|
||||
os.mkdir_all(repo_pkg_path)!
|
||||
|
||||
// We create hard links so that "any" arch packages aren't stored
|
||||
// multiple times
|
||||
os.link(pkg_path, dest_path)?
|
||||
os.link(pkg_path, dest_path)!
|
||||
}
|
||||
|
||||
// After linking, we can remove the original file
|
||||
os.rm(pkg_path)?
|
||||
os.rm(pkg_path)!
|
||||
|
||||
return RepoAddResult{
|
||||
name: pkg.info.name
|
||||
|
|
@ -85,11 +85,11 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re
|
|||
// r.default_arch. If this arch-repo doesn't exist yet, it is created. If the
|
||||
// architecture isn't 'any', the package is only added to the specific
|
||||
// architecture.
|
||||
fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]string {
|
||||
fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ![]string {
|
||||
// A package not of arch 'any' can be handled easily by adding it to the
|
||||
// respective repo
|
||||
if pkg.info.arch != 'any' {
|
||||
r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)?
|
||||
r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)!
|
||||
|
||||
return [pkg.info.arch]
|
||||
}
|
||||
|
|
@ -104,7 +104,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin
|
|||
// If this is the first package that's added to the repo, the directory
|
||||
// won't exist yet
|
||||
if os.exists(repo_dir) {
|
||||
arch_repos = os.ls(repo_dir)?
|
||||
arch_repos = os.ls(repo_dir)!
|
||||
}
|
||||
|
||||
// The default_arch should always be updated when a package with arch 'any'
|
||||
|
|
@ -118,7 +118,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin
|
|||
// not know which arch-repositories did succeed in adding the package, if
|
||||
// any.
|
||||
for arch in arch_repos {
|
||||
r.add_pkg_in_arch_repo(repo, arch, pkg)?
|
||||
r.add_pkg_in_arch_repo(repo, arch, pkg)!
|
||||
}
|
||||
|
||||
return arch_repos
|
||||
|
|
@ -128,24 +128,24 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin
|
|||
// arch-repo. It records the package's data in the arch-repo's desc & files
|
||||
// files, and afterwards updates the db & files archives to reflect these
|
||||
// changes.
|
||||
fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ? {
|
||||
fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ! {
|
||||
pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version')
|
||||
|
||||
// Remove the previous version of the package, if present
|
||||
r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false)?
|
||||
r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false)!
|
||||
|
||||
os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') }
|
||||
|
||||
os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()?) or {
|
||||
os.rmdir_all(pkg_dir)?
|
||||
os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()!) or {
|
||||
os.rmdir_all(pkg_dir)!
|
||||
|
||||
return error('Failed to write desc file.')
|
||||
}
|
||||
os.write_file(os.join_path_single(pkg_dir, 'files'), pkg.to_files()) or {
|
||||
os.rmdir_all(pkg_dir)?
|
||||
os.rmdir_all(pkg_dir)!
|
||||
|
||||
return error('Failed to write files file.')
|
||||
}
|
||||
|
||||
r.sync(repo, arch)?
|
||||
r.sync(repo, arch)!
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import os
|
|||
// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It
|
||||
// returns false if the package wasn't present in the database. It also
|
||||
// optionally re-syncs the repo archives.
|
||||
pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool {
|
||||
pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) !bool {
|
||||
repo_dir := os.join_path(r.repos_dir, repo, arch)
|
||||
|
||||
// If the repository doesn't exist yet, the result is automatically false
|
||||
|
|
@ -15,7 +15,7 @@ pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string,
|
|||
|
||||
// We iterate over every directory in the repo dir
|
||||
// TODO filter so we only check directories
|
||||
for d in os.ls(repo_dir)? {
|
||||
for d in os.ls(repo_dir)! {
|
||||
// Because a repository only allows a single version of each package,
|
||||
// we need only compare whether the name of the package is the same,
|
||||
// not the version.
|
||||
|
|
@ -25,22 +25,22 @@ pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string,
|
|||
// We lock the mutex here to prevent other routines from creating a
|
||||
// new archive while we remove an entry
|
||||
lock r.mutex {
|
||||
os.rmdir_all(os.join_path_single(repo_dir, d))?
|
||||
os.rmdir_all(os.join_path_single(repo_dir, d))!
|
||||
}
|
||||
|
||||
// Also remove the package archive
|
||||
repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch)
|
||||
|
||||
archives := os.ls(repo_pkg_dir)?.filter(it.split('-')#[..-3].join('-') == name)
|
||||
archives := os.ls(repo_pkg_dir)!.filter(it.split('-')#[..-3].join('-') == name)
|
||||
|
||||
for archive_name in archives {
|
||||
full_path := os.join_path_single(repo_pkg_dir, archive_name)
|
||||
os.rm(full_path)?
|
||||
os.rm(full_path)!
|
||||
}
|
||||
|
||||
// Sync the db archives if requested
|
||||
if sync {
|
||||
r.sync(repo, arch)?
|
||||
r.sync(repo, arch)!
|
||||
}
|
||||
|
||||
return true
|
||||
|
|
@ -51,7 +51,7 @@ pub fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string,
|
|||
}
|
||||
|
||||
// remove_arch_repo removes an arch-repo & its packages.
|
||||
pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) ?bool {
|
||||
pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) !bool {
|
||||
repo_dir := os.join_path(r.repos_dir, repo, arch)
|
||||
|
||||
// If the repository doesn't exist yet, the result is automatically false
|
||||
|
|
@ -59,16 +59,16 @@ pub fn (r &RepoGroupManager) remove_arch_repo(repo string, arch string) ?bool {
|
|||
return false
|
||||
}
|
||||
|
||||
os.rmdir_all(repo_dir)?
|
||||
os.rmdir_all(repo_dir)!
|
||||
|
||||
pkg_dir := os.join_path(r.pkg_dir, repo, arch)
|
||||
os.rmdir_all(pkg_dir)?
|
||||
os.rmdir_all(pkg_dir)!
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// remove_repo removes a repo & its packages.
|
||||
pub fn (r &RepoGroupManager) remove_repo(repo string) ?bool {
|
||||
pub fn (r &RepoGroupManager) remove_repo(repo string) !bool {
|
||||
repo_dir := os.join_path_single(r.repos_dir, repo)
|
||||
|
||||
// If the repository doesn't exist yet, the result is automatically false
|
||||
|
|
@ -76,10 +76,10 @@ pub fn (r &RepoGroupManager) remove_repo(repo string) ?bool {
|
|||
return false
|
||||
}
|
||||
|
||||
os.rmdir_all(repo_dir)?
|
||||
os.rmdir_all(repo_dir)!
|
||||
|
||||
pkg_dir := os.join_path_single(r.pkg_dir, repo)
|
||||
os.rmdir_all(pkg_dir)?
|
||||
os.rmdir_all(pkg_dir)!
|
||||
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri
|
|||
}
|
||||
|
||||
// sync regenerates the repository archive files.
|
||||
fn (r &RepoGroupManager) sync(repo string, arch string) ? {
|
||||
fn (r &RepoGroupManager) sync(repo string, arch string) ! {
|
||||
subrepo_path := os.join_path(r.repos_dir, repo, arch)
|
||||
|
||||
lock r.mutex {
|
||||
|
|
@ -54,7 +54,7 @@ fn (r &RepoGroupManager) sync(repo string, arch string) ? {
|
|||
C.archive_write_open_filename(a_files, &char(files_path.str))
|
||||
|
||||
// Iterate over each directory
|
||||
for d in os.ls(subrepo_path)?.filter(os.is_dir(os.join_path_single(subrepo_path,
|
||||
for d in os.ls(subrepo_path)!.filter(os.is_dir(os.join_path_single(subrepo_path,
|
||||
it))) {
|
||||
// desc
|
||||
mut inner_path := os.join_path_single(d, 'desc')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
module server
|
||||
|
||||
import web
|
||||
import web.response { new_data_response, new_response }
|
||||
|
||||
// v1_poll_job_queue allows agents to poll for new build jobs.
|
||||
['/api/v1/jobs/poll'; auth; get]
|
||||
fn (mut app App) v1_poll_job_queue() web.Result {
|
||||
arch := app.query['arch'] or {
|
||||
return app.json(.bad_request, new_response('Missing arch query arg.'))
|
||||
}
|
||||
|
||||
max_str := app.query['max'] or {
|
||||
return app.json(.bad_request, new_response('Missing max query arg.'))
|
||||
}
|
||||
max := max_str.int()
|
||||
|
||||
mut out := app.job_queue.pop_n(arch, max).map(it.config)
|
||||
|
||||
return app.json(.ok, new_data_response(out))
|
||||
}
|
||||
|
||||
// v1_queue_job allows queueing a new one-time build job for the given target.
|
||||
['/api/v1/jobs/queue'; auth; post]
|
||||
fn (mut app App) v1_queue_job() web.Result {
|
||||
target_id := app.query['target'] or {
|
||||
return app.json(.bad_request, new_response('Missing target query arg.'))
|
||||
}.int()
|
||||
|
||||
arch := app.query['arch'] or {
|
||||
return app.json(.bad_request, new_response('Missing arch query arg.'))
|
||||
}
|
||||
|
||||
if arch == '' {
|
||||
app.json(.bad_request, new_response('Empty arch query arg.'))
|
||||
}
|
||||
|
||||
force := 'force' in app.query
|
||||
|
||||
target := app.db.get_target(target_id) or {
|
||||
return app.json(.bad_request, new_response('Unknown target id.'))
|
||||
}
|
||||
|
||||
app.job_queue.insert(target: target, arch: arch, single: true, now: true, force: force) or {
|
||||
return app.status(.internal_server_error)
|
||||
}
|
||||
|
||||
return app.status(.ok)
|
||||
}
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
module server
|
||||
|
||||
import web
|
||||
import net.http
|
||||
import net.urllib
|
||||
import web.response { new_data_response, new_response }
|
||||
import db
|
||||
|
|
@ -15,7 +14,7 @@ import models { BuildLog, BuildLogFilter }
|
|||
['/api/v1/logs'; auth; get]
|
||||
fn (mut app App) v1_get_logs() web.Result {
|
||||
filter := models.from_params<BuildLogFilter>(app.query) or {
|
||||
return app.json(http.Status.bad_request, new_response('Invalid query parameters.'))
|
||||
return app.json(.bad_request, new_response('Invalid query parameters.'))
|
||||
}
|
||||
logs := app.db.get_build_logs(filter)
|
||||
|
||||
|
|
@ -25,7 +24,7 @@ fn (mut app App) v1_get_logs() web.Result {
|
|||
// v1_get_single_log returns the build log with the given id.
|
||||
['/api/v1/logs/:id'; auth; get]
|
||||
fn (mut app App) v1_get_single_log(id int) web.Result {
|
||||
log := app.db.get_build_log(id) or { return app.not_found() }
|
||||
log := app.db.get_build_log(id) or { return app.status(.not_found) }
|
||||
|
||||
return app.json(.ok, new_data_response(log))
|
||||
}
|
||||
|
|
@ -33,7 +32,7 @@ fn (mut app App) v1_get_single_log(id int) web.Result {
|
|||
// v1_get_log_content returns the actual build log file for the given id.
|
||||
['/api/v1/logs/:id/content'; auth; get]
|
||||
fn (mut app App) v1_get_log_content(id int) web.Result {
|
||||
log := app.db.get_build_log(id) or { return app.not_found() }
|
||||
log := app.db.get_build_log(id) or { return app.status(.not_found) }
|
||||
file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss')
|
||||
full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.target_id.str(), log.arch,
|
||||
file_name)
|
||||
|
|
@ -43,9 +42,9 @@ fn (mut app App) v1_get_log_content(id int) web.Result {
|
|||
|
||||
// parse_query_time unescapes an HTTP query parameter & tries to parse it as a
|
||||
// time.Time struct.
|
||||
fn parse_query_time(query string) ?time.Time {
|
||||
unescaped := urllib.query_unescape(query)?
|
||||
t := time.parse(unescaped)?
|
||||
fn parse_query_time(query string) !time.Time {
|
||||
unescaped := urllib.query_unescape(query)!
|
||||
t := time.parse(unescaped)!
|
||||
|
||||
return t
|
||||
}
|
||||
|
|
@ -57,25 +56,25 @@ fn (mut app App) v1_post_log() web.Result {
|
|||
start_time_int := app.query['startTime'].int()
|
||||
|
||||
if start_time_int == 0 {
|
||||
return app.json(http.Status.bad_request, new_response('Invalid or missing start time.'))
|
||||
return app.json(.bad_request, new_response('Invalid or missing start time.'))
|
||||
}
|
||||
start_time := time.unix(start_time_int)
|
||||
|
||||
end_time_int := app.query['endTime'].int()
|
||||
|
||||
if end_time_int == 0 {
|
||||
return app.json(http.Status.bad_request, new_response('Invalid or missing end time.'))
|
||||
return app.json(.bad_request, new_response('Invalid or missing end time.'))
|
||||
}
|
||||
end_time := time.unix(end_time_int)
|
||||
|
||||
if 'exitCode' !in app.query {
|
||||
return app.json(http.Status.bad_request, new_response('Missing exit code.'))
|
||||
return app.json(.bad_request, new_response('Missing exit code.'))
|
||||
}
|
||||
|
||||
exit_code := app.query['exitCode'].int()
|
||||
|
||||
if 'arch' !in app.query {
|
||||
return app.json(http.Status.bad_request, new_response("Missing parameter 'arch'."))
|
||||
return app.json(.bad_request, new_response("Missing parameter 'arch'."))
|
||||
}
|
||||
|
||||
arch := app.query['arch']
|
||||
|
|
@ -83,7 +82,7 @@ fn (mut app App) v1_post_log() web.Result {
|
|||
target_id := app.query['target'].int()
|
||||
|
||||
if !app.db.target_exists(target_id) {
|
||||
return app.json(http.Status.bad_request, new_response('Unknown target.'))
|
||||
return app.json(.bad_request, new_response('Unknown target.'))
|
||||
}
|
||||
|
||||
// Store log in db
|
||||
|
|
@ -105,7 +104,7 @@ fn (mut app App) v1_post_log() web.Result {
|
|||
os.mkdir_all(repo_logs_dir) or {
|
||||
app.lerror("Couldn't create dir '$repo_logs_dir'.")
|
||||
|
||||
return app.json(http.Status.internal_server_error, new_response('An error occured while processing the request.'))
|
||||
return app.status(.internal_server_error)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -117,10 +116,10 @@ fn (mut app App) v1_post_log() web.Result {
|
|||
util.reader_to_file(mut app.reader, length.int(), full_path) or {
|
||||
app.lerror('An error occured while receiving logs: $err.msg()')
|
||||
|
||||
return app.json(http.Status.internal_server_error, new_response('Failed to upload logs.'))
|
||||
return app.status(.internal_server_error)
|
||||
}
|
||||
} else {
|
||||
return app.status(http.Status.length_required)
|
||||
return app.status(.length_required)
|
||||
}
|
||||
|
||||
return app.json(.ok, new_data_response(log_id))
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
module server
|
||||
|
||||
import web
|
||||
import net.http
|
||||
import web.response { new_data_response, new_response }
|
||||
import db
|
||||
import models { Target, TargetArch, TargetFilter }
|
||||
|
|
@ -10,19 +9,19 @@ import models { Target, TargetArch, TargetFilter }
|
|||
['/api/v1/targets'; auth; get]
|
||||
fn (mut app App) v1_get_targets() web.Result {
|
||||
filter := models.from_params<TargetFilter>(app.query) or {
|
||||
return app.json(http.Status.bad_request, new_response('Invalid query parameters.'))
|
||||
return app.json(.bad_request, new_response('Invalid query parameters.'))
|
||||
}
|
||||
repos := app.db.get_targets(filter)
|
||||
targets := app.db.get_targets(filter)
|
||||
|
||||
return app.json(.ok, new_data_response(repos))
|
||||
return app.json(.ok, new_data_response(targets))
|
||||
}
|
||||
|
||||
// v1_get_single_target returns the information for a single target.
|
||||
['/api/v1/targets/:id'; auth; get]
|
||||
fn (mut app App) v1_get_single_target(id int) web.Result {
|
||||
repo := app.db.get_target(id) or { return app.not_found() }
|
||||
target := app.db.get_target(id) or { return app.status(.not_found) }
|
||||
|
||||
return app.json(.ok, new_data_response(repo))
|
||||
return app.json(.ok, new_data_response(target))
|
||||
}
|
||||
|
||||
// v1_post_target creates a new target from the provided query string.
|
||||
|
|
@ -30,30 +29,36 @@ fn (mut app App) v1_get_single_target(id int) web.Result {
|
|||
fn (mut app App) v1_post_target() web.Result {
|
||||
mut params := app.query.clone()
|
||||
|
||||
// If a repo is created without specifying the arch, we assume it's meant
|
||||
// If a target is created without specifying the arch, we assume it's meant
|
||||
// for the default architecture.
|
||||
if 'arch' !in params {
|
||||
if 'arch' !in params || params['arch'] == '' {
|
||||
params['arch'] = app.conf.default_arch
|
||||
}
|
||||
|
||||
new_repo := models.from_params<Target>(params) or {
|
||||
return app.json(http.Status.bad_request, new_response(err.msg()))
|
||||
mut new_target := models.from_params<Target>(params) or {
|
||||
return app.json(.bad_request, new_response(err.msg()))
|
||||
}
|
||||
|
||||
// Ensure someone doesn't submit an invalid kind
|
||||
if new_repo.kind !in models.valid_kinds {
|
||||
return app.json(http.Status.bad_request, new_response('Invalid kind.'))
|
||||
if new_target.kind !in models.valid_kinds {
|
||||
return app.json(.bad_request, new_response('Invalid kind.'))
|
||||
}
|
||||
|
||||
id := app.db.add_target(new_repo)
|
||||
id := app.db.add_target(new_target)
|
||||
new_target.id = id
|
||||
|
||||
return app.json(http.Status.ok, new_data_response(id))
|
||||
// Add the target to the job queue
|
||||
// TODO return better error here if it's the cron schedule that's incorrect
|
||||
app.job_queue.insert_all(new_target) or { return app.status(.internal_server_error) }
|
||||
|
||||
return app.json(.ok, new_data_response(id))
|
||||
}
|
||||
|
||||
// v1_delete_target removes a given target from the server's list.
|
||||
['/api/v1/targets/:id'; auth; delete]
|
||||
fn (mut app App) v1_delete_target(id int) web.Result {
|
||||
app.db.delete_target(id)
|
||||
app.job_queue.invalidate(id)
|
||||
|
||||
return app.status(.ok)
|
||||
}
|
||||
|
|
@ -69,5 +74,10 @@ fn (mut app App) v1_patch_target(id int) web.Result {
|
|||
app.db.update_target_archs(id, arch_objs)
|
||||
}
|
||||
|
||||
return app.status(.ok)
|
||||
target := app.db.get_target(id) or { return app.status(.internal_server_error) }
|
||||
|
||||
app.job_queue.invalidate(id)
|
||||
app.job_queue.insert_all(target) or { return app.status(.internal_server_error) }
|
||||
|
||||
return app.json(.ok, new_data_response(target))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,16 +1,18 @@
|
|||
module server
|
||||
|
||||
import cli
|
||||
import vieter_v.conf as vconf
|
||||
import conf as vconf
|
||||
|
||||
struct Config {
|
||||
pub:
|
||||
log_level string = 'WARN'
|
||||
pkg_dir string
|
||||
data_dir string
|
||||
api_key string
|
||||
default_arch string
|
||||
port int = 8000
|
||||
log_level string = 'WARN'
|
||||
pkg_dir string
|
||||
data_dir string
|
||||
api_key string
|
||||
default_arch string
|
||||
global_schedule string = '0 3'
|
||||
port int = 8000
|
||||
base_image string = 'archlinux:base-devel'
|
||||
}
|
||||
|
||||
// cmd returns the cli submodule that handles starting the server
|
||||
|
|
@ -18,11 +20,11 @@ pub fn cmd() cli.Command {
|
|||
return cli.Command{
|
||||
name: 'server'
|
||||
description: 'Start the Vieter server.'
|
||||
execute: fn (cmd cli.Command) ? {
|
||||
config_file := cmd.flags.get_string('config-file')?
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)?
|
||||
execute: fn (cmd cli.Command) ! {
|
||||
config_file := cmd.flags.get_string('config-file')!
|
||||
conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)!
|
||||
|
||||
server(conf)?
|
||||
server(conf)!
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import log
|
|||
import repo
|
||||
import util
|
||||
import db
|
||||
import build { BuildJobQueue }
|
||||
import cron.expression
|
||||
|
||||
const (
|
||||
log_file_name = 'vieter.log'
|
||||
|
|
@ -20,16 +22,39 @@ pub:
|
|||
conf Config [required; web_global]
|
||||
pub mut:
|
||||
repo repo.RepoGroupManager [required; web_global]
|
||||
db db.VieterDb
|
||||
// Keys are the various architectures for packages
|
||||
job_queue BuildJobQueue [required; web_global]
|
||||
db db.VieterDb
|
||||
}
|
||||
|
||||
// init_job_queue populates a fresh job queue with all the targets currently
|
||||
// stored in the database.
|
||||
fn (mut app App) init_job_queue() ! {
|
||||
// Initialize build queues
|
||||
mut targets := app.db.get_targets(limit: 25)
|
||||
mut i := u64(0)
|
||||
|
||||
for targets.len > 0 {
|
||||
for target in targets {
|
||||
app.job_queue.insert_all(target)!
|
||||
}
|
||||
|
||||
i += 25
|
||||
targets = app.db.get_targets(limit: 25, offset: i)
|
||||
}
|
||||
}
|
||||
|
||||
// server starts the web server & starts listening for requests
|
||||
pub fn server(conf Config) ? {
|
||||
pub fn server(conf Config) ! {
|
||||
// Prevent using 'any' as the default arch
|
||||
if conf.default_arch == 'any' {
|
||||
util.exit_with_message(1, "'any' is not allowed as the value for default_arch.")
|
||||
}
|
||||
|
||||
global_ce := expression.parse_expression(conf.global_schedule) or {
|
||||
util.exit_with_message(1, 'Invalid global cron expression: $err.msg()')
|
||||
}
|
||||
|
||||
// Configure logger
|
||||
log_level := log.level_from_tag(conf.log_level) or {
|
||||
util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
|
||||
|
|
@ -71,11 +96,17 @@ pub fn server(conf Config) ? {
|
|||
util.exit_with_message(1, 'Failed to initialize database: $err.msg()')
|
||||
}
|
||||
|
||||
web.run(&App{
|
||||
mut app := &App{
|
||||
logger: logger
|
||||
api_key: conf.api_key
|
||||
conf: conf
|
||||
repo: repo
|
||||
db: db
|
||||
}, conf.port)
|
||||
job_queue: build.new_job_queue(global_ce, conf.base_image)
|
||||
}
|
||||
app.init_job_queue() or {
|
||||
util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()')
|
||||
}
|
||||
|
||||
web.run(app, conf.port)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import io
|
|||
import os
|
||||
|
||||
// reader_to_writer tries to consume the entire reader & write it to the writer.
|
||||
pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ? {
|
||||
pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ! {
|
||||
mut buf := []u8{len: 10 * 1024}
|
||||
|
||||
for {
|
||||
|
|
@ -21,8 +21,8 @@ pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ? {
|
|||
}
|
||||
|
||||
// reader_to_file writes the contents of a BufferedReader to a file
|
||||
pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? {
|
||||
mut file := os.create(path)?
|
||||
pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ! {
|
||||
mut file := os.create(path)!
|
||||
defer {
|
||||
file.close()
|
||||
}
|
||||
|
|
@ -69,11 +69,11 @@ pub fn match_array_in_array<T>(a1 []T, a2 []T) int {
|
|||
|
||||
// read_until_separator consumes an io.Reader until it encounters some
|
||||
// separator array. The data read is stored inside the provided res array.
|
||||
pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ? {
|
||||
pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ! {
|
||||
mut buf := []u8{len: sep.len}
|
||||
|
||||
for {
|
||||
c := reader.read(mut buf)?
|
||||
c := reader.read(mut buf)!
|
||||
res << buf[..c]
|
||||
|
||||
match_len := match_array_in_array(buf[..c], sep)
|
||||
|
|
@ -84,7 +84,7 @@ pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ? {
|
|||
|
||||
if match_len > 0 {
|
||||
match_left := sep.len - match_len
|
||||
c2 := reader.read(mut buf[..match_left])?
|
||||
c2 := reader.read(mut buf[..match_left])!
|
||||
res << buf[..c2]
|
||||
|
||||
if buf[..c2] == sep[match_len..] {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ pub fn exit_with_message(code int, msg string) {
|
|||
}
|
||||
|
||||
// hash_file returns the sha256 hash of a given file
|
||||
pub fn hash_file(path &string) ?string {
|
||||
pub fn hash_file(path &string) !string {
|
||||
file := os.open(path) or { return error('Failed to open file.') }
|
||||
|
||||
mut sha256sum := sha256.new()
|
||||
|
|
@ -39,7 +39,7 @@ pub fn hash_file(path &string) ?string {
|
|||
|
||||
// This function never actually fails, but returns an option to follow
|
||||
// the Writer interface.
|
||||
sha256sum.write(buf[..bytes_read])?
|
||||
sha256sum.write(buf[..bytes_read])!
|
||||
}
|
||||
|
||||
return sha256sum.checksum().hex()
|
||||
|
|
|
|||
|
|
@ -3,33 +3,33 @@ module web
|
|||
import log
|
||||
|
||||
// log reate a log message with the given level
|
||||
pub fn (mut ctx Context) log(msg &string, level log.Level) {
|
||||
pub fn (mut ctx Context) log(msg string, level log.Level) {
|
||||
lock ctx.logger {
|
||||
ctx.logger.send_output(msg, level)
|
||||
}
|
||||
}
|
||||
|
||||
// lfatal create a log message with the fatal level
|
||||
pub fn (mut ctx Context) lfatal(msg &string) {
|
||||
pub fn (mut ctx Context) lfatal(msg string) {
|
||||
ctx.log(msg, log.Level.fatal)
|
||||
}
|
||||
|
||||
// lerror create a log message with the error level
|
||||
pub fn (mut ctx Context) lerror(msg &string) {
|
||||
pub fn (mut ctx Context) lerror(msg string) {
|
||||
ctx.log(msg, log.Level.error)
|
||||
}
|
||||
|
||||
// lwarn create a log message with the warn level
|
||||
pub fn (mut ctx Context) lwarn(msg &string) {
|
||||
pub fn (mut ctx Context) lwarn(msg string) {
|
||||
ctx.log(msg, log.Level.warn)
|
||||
}
|
||||
|
||||
// linfo create a log message with the info level
|
||||
pub fn (mut ctx Context) linfo(msg &string) {
|
||||
pub fn (mut ctx Context) linfo(msg string) {
|
||||
ctx.log(msg, log.Level.info)
|
||||
}
|
||||
|
||||
// ldebug create a log message with the debug level
|
||||
pub fn (mut ctx Context) ldebug(msg &string) {
|
||||
pub fn (mut ctx Context) ldebug(msg string) {
|
||||
ctx.log(msg, log.Level.debug)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import net.http
|
|||
const attrs_to_ignore = ['auth']
|
||||
|
||||
// Parsing function attributes for methods and path.
|
||||
fn parse_attrs(name string, attrs []string) ?([]http.Method, string) {
|
||||
fn parse_attrs(name string, attrs []string) !([]http.Method, string) {
|
||||
if attrs.len == 0 {
|
||||
return [http.Method.get], '/$name'
|
||||
}
|
||||
|
|
@ -61,7 +61,7 @@ fn parse_query_from_url(url urllib.URL) map[string]string {
|
|||
}
|
||||
|
||||
// Extract form data from an HTTP request.
|
||||
fn parse_form_from_request(request http.Request) ?(map[string]string, map[string][]http.FileData) {
|
||||
fn parse_form_from_request(request http.Request) !(map[string]string, map[string][]http.FileData) {
|
||||
mut form := map[string]string{}
|
||||
mut files := map[string][]http.FileData{}
|
||||
if request.method in methods_with_form {
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ pub:
|
|||
pub mut:
|
||||
// TCP connection to client.
|
||||
// But beware, do not store it for further use, after request processing web will close connection.
|
||||
conn &net.TcpConn
|
||||
conn &net.TcpConn = unsafe { nil }
|
||||
// Gives access to a shared logger object
|
||||
logger shared log.Log
|
||||
// time.ticks() from start of web connection handle.
|
||||
|
|
@ -67,20 +67,20 @@ struct Route {
|
|||
pub fn (ctx Context) before_request() {}
|
||||
|
||||
// send_string writes the given string to the TCP connection socket.
|
||||
fn (mut ctx Context) send_string(s string) ? {
|
||||
ctx.conn.write(s.bytes())?
|
||||
fn (mut ctx Context) send_string(s string) ! {
|
||||
ctx.conn.write(s.bytes())!
|
||||
}
|
||||
|
||||
// send_reader reads at most `size` bytes from the given reader & writes them
|
||||
// to the TCP connection socket. Internally, a 10KB buffer is used, to avoid
|
||||
// having to store all bytes in memory at once.
|
||||
fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ? {
|
||||
fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ! {
|
||||
mut buf := []u8{len: 10_000}
|
||||
mut bytes_left := size
|
||||
|
||||
// Repeat as long as the stream still has data
|
||||
for bytes_left > 0 {
|
||||
bytes_read := reader.read(mut buf)?
|
||||
bytes_read := reader.read(mut buf)!
|
||||
bytes_left -= u64(bytes_read)
|
||||
|
||||
mut to_write := bytes_read
|
||||
|
|
@ -96,20 +96,20 @@ fn (mut ctx Context) send_reader(mut reader io.Reader, size u64) ? {
|
|||
// send_custom_response sends the given http.Response to the client. It can be
|
||||
// used to overwrite the Context object & send a completely custom
|
||||
// http.Response instead.
|
||||
fn (mut ctx Context) send_custom_response(resp &http.Response) ? {
|
||||
ctx.send_string(resp.bytestr())?
|
||||
fn (mut ctx Context) send_custom_response(resp &http.Response) ! {
|
||||
ctx.send_string(resp.bytestr())!
|
||||
}
|
||||
|
||||
// send_response_header constructs a valid HTTP response with an empty body &
|
||||
// sends it to the client.
|
||||
pub fn (mut ctx Context) send_response_header() ? {
|
||||
pub fn (mut ctx Context) send_response_header() ! {
|
||||
mut resp := http.new_response(
|
||||
header: ctx.header.join(headers_close)
|
||||
)
|
||||
resp.header.add(.content_type, ctx.content_type)
|
||||
resp.set_status(ctx.status)
|
||||
|
||||
ctx.send_custom_response(resp)?
|
||||
ctx.send_custom_response(resp)!
|
||||
}
|
||||
|
||||
// send is a convenience function for sending the HTTP response with an empty
|
||||
|
|
@ -260,13 +260,6 @@ pub fn (mut ctx Context) redirect(url string) Result {
|
|||
return Result{}
|
||||
}
|
||||
|
||||
// not_found Send an not_found response
|
||||
pub fn (mut ctx Context) not_found() Result {
|
||||
ctx.send_custom_response(http_404) or {}
|
||||
|
||||
return Result{}
|
||||
}
|
||||
|
||||
interface DbInterface {
|
||||
db voidptr
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,11 +4,11 @@ data_dir = "data"
|
|||
pkg_dir = "data/pkgs"
|
||||
log_level = "DEBUG"
|
||||
default_arch = "x86_64"
|
||||
arch = "x86_64"
|
||||
|
||||
address = "http://localhost:8000"
|
||||
|
||||
global_schedule = '* *'
|
||||
# global_schedule = '* *'
|
||||
api_update_frequency = 2
|
||||
image_rebuild_frequency = 1
|
||||
max_concurrent_builds = 3
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue