Merge pull request 'basic builds' (#84) from basic-builds into dev

Reviewed-on: Chewing_Bever/vieter#84
main
Jef Roosens 2022-02-21 18:06:54 +01:00
commit d6e71e9a1c
10 changed files with 345 additions and 46 deletions

View File

@ -7,4 +7,5 @@ end_of_line = lf
insert_final_newline = true
[*.v]
indent_style = space
# vfmt wants it :(
indent_style = tab

View File

@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter)
## Added
* Very basic build system
* Build is triggered by separate cron container
* Packages build on cron container's system
* Packages are always rebuilt, even if they haven't changed
* Hardcoded planning of builds
* Builds are sequential
## Fixed
* Each package can now only have one version in the repository at once
(required by Pacman)
## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0)
### Changed

View File

@ -36,11 +36,14 @@ HEALTHCHECK --interval=30s \
CMD /bin/wget --spider http://localhost:8000/health || exit 1
RUN mkdir /data && \
chown -R www-data:www-data /data
chown -R www-data:www-data /data && \
mkdir -p '/var/spool/cron/crontabs' && \
echo '0 3 * * * /bin/vieter build' >> /var/spool/cron/crontabs/www-data && \
chown www-data:www-data /var/spool/cron/crontabs/www-data
WORKDIR /data
USER www-data:www-data
ENTRYPOINT ["/bin/dumb-init", "--"]
CMD ["/bin/vieter"]
CMD ["/bin/vieter", "server"]

View File

@ -34,7 +34,7 @@ c:
# Run the server in the default 'data' directory
.PHONY: run
run: vieter
API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter
API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter server
.PHONY: run-prod
run-prod: prod

84
src/build.v 100644
View File

@ -0,0 +1,84 @@
module main
import docker
import encoding.base64
import rand
import time
import os
import json
import git
const container_build_dir = '/build'
fn build(key string, repo_dir string) ? {
server_url := os.getenv_opt('VIETER_ADDRESS') or {
exit_with_message(1, 'No Vieter server address was provided.')
}
// Read in the repos from a json file
filename := os.join_path_single(repo_dir, 'repos.json')
txt := os.read_file(filename) ?
repos := json.decode([]git.GitRepo, txt) ?
mut commands := [
// Update repos & install required packages
'pacman -Syu --needed --noconfirm base-devel git'
// Add a non-root user to run makepkg
'groupadd -g 1000 builder',
'useradd -mg builder builder'
// Make sure they can use sudo without a password
"echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers"
// Create the directory for the builds & make it writeable for the
// build user
'mkdir /build',
'chown -R builder:builder /build',
]
// Each repo gets a unique UUID to avoid naming conflicts when cloning
mut uuids := []string{}
for repo in repos {
mut uuid := rand.uuid_v4()
// Just to be sure we don't have any collisions
for uuids.contains(uuid) {
uuid = rand.uuid_v4()
}
uuids << uuid
commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'"
commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $server_url/publish; done\''
}
// We convert the list of commands into a base64 string, which then gets
// passed to the container as an env var
cmds_str := base64.encode_str(commands.join('\n'))
c := docker.NewContainer{
image: 'archlinux:latest'
env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$key']
entrypoint: ['/bin/sh', '-c']
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e']
}
// First, we pull the latest archlinux image
docker.pull_image('archlinux', 'latest') ?
id := docker.create_container(c) ?
docker.start_container(id) ?
// This loop waits until the container has stopped, so we can remove it after
for {
data := docker.inspect_container(id) ?
if !data.state.running {
break
}
// Wait for 5 seconds
time.sleep(5000000000)
}
docker.remove_container(id) ?
}

View File

@ -0,0 +1,76 @@
module docker
import json
import net.urllib
struct Container {
id string [json: Id]
names []string [json: Names]
}
// containers returns a list of all currently running containers
pub fn containers() ?[]Container {
res := request('GET', urllib.parse('/v1.41/containers/json') ?) ?
return json.decode([]Container, res.text) or {}
}
pub struct NewContainer {
image string [json: Image]
entrypoint []string [json: Entrypoint]
cmd []string [json: Cmd]
env []string [json: Env]
}
struct CreatedContainer {
id string [json: Id]
}
// create_container creates a container defined by the given configuration. If
// successful, it returns the ID of the newly created container.
pub fn create_container(c &NewContainer) ?string {
res := request_with_json('POST', urllib.parse('/v1.41/containers/create') ?, c) ?
if res.status_code != 201 {
return error('Failed to create container.')
}
return json.decode(CreatedContainer, res.text) ?.id
}
// start_container starts a container with a given ID. It returns whether the
// container was started or not.
pub fn start_container(id string) ?bool {
res := request('POST', urllib.parse('/v1.41/containers/$id/start') ?) ?
return res.status_code == 204
}
struct ContainerInspect {
pub:
state ContainerState [json: State]
}
struct ContainerState {
pub:
running bool [json: Running]
}
// inspect_container returns the result of inspecting a container with a given
// ID.
pub fn inspect_container(id string) ?ContainerInspect {
res := request('GET', urllib.parse('/v1.41/containers/$id/json') ?) ?
if res.status_code != 200 {
return error('Failed to inspect container.')
}
return json.decode(ContainerInspect, res.text) or {}
}
// remove_container removes a container with a given ID.
pub fn remove_container(id string) ?bool {
res := request('DELETE', urllib.parse('/v1.41/containers/$id') ?) ?
return res.status_code == 204
}

View File

@ -0,0 +1,98 @@
module docker
import net.unix
import net.urllib
import net.http
import json
const socket = '/var/run/docker.sock'
const buf_len = 1024
fn send(req &string) ?http.Response {
// Open a connection to the socket
mut s := unix.connect_stream(docker.socket) or {
return error('Failed to connect to socket ${docker.socket}.')
}
defer {
// This or is required because otherwise, the V compiler segfaults for
// some reason
// https://github.com/vlang/v/issues/13534
s.close() or {}
}
// Write the request to the socket
s.write_string(req) or { return error('Failed to write request to socket ${docker.socket}.') }
s.wait_for_write() ?
mut c := 0
mut buf := []byte{len: docker.buf_len}
mut res := []byte{}
for {
c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') }
res << buf[..c]
if c < docker.buf_len {
break
}
}
// After reading the first part of the response, we parse it into an HTTP
// response. If it isn't chunked, we return early with the data.
parsed := http.parse_response(res.bytestr()) or {
return error('Failed to parse HTTP response from socket ${docker.socket}.')
}
if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' {
return parsed
}
// We loop until we've encountered the end of the chunked response
// A chunked HTTP response always ends with '0\r\n\r\n'.
for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] {
// Wait for the server to respond
s.wait_for_write() ?
for {
c = s.read(mut buf) or {
return error('Failed to read data from socket ${docker.socket}.')
}
res << buf[..c]
if c < docker.buf_len {
break
}
}
}
// Decode chunked response
return http.parse_response(res.bytestr())
}
fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response {
req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n'
return send(req)
}
fn request(method string, url urllib.URL) ?http.Response {
req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n'
return send(req)
}
// request_with_json<T> sends a request to the Docker socket with a given JSON
// payload
pub fn request_with_json<T>(method string, url urllib.URL, data &T) ?http.Response {
body := json.encode(data)
return request_with_body(method, url, 'application/json', body)
}
// pull_image pulls tries to pull the image for the given image & tag
pub fn pull_image(image string, tag string) ?http.Response {
return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?)
}

7
src/git.v 100644
View File

@ -0,0 +1,7 @@
module git
pub struct GitRepo {
pub:
url string [required]
branch string [required]
}

View File

@ -2,7 +2,6 @@ module main
import web
import os
import log
import io
import repo
@ -10,8 +9,6 @@ const port = 8000
const buf_size = 1_000_000
const db_name = 'pieter.db'
struct App {
web.Context
pub:
@ -54,50 +51,18 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? {
}
fn main() {
// Configure logger
log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' }
log_level := log.level_from_tag(log_level_str) or {
exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
}
log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' }
mut logger := log.Log{
level: log_level
}
logger.set_full_logpath(log_file)
logger.log_to_console_too()
defer {
logger.info('Flushing log file')
logger.flush()
logger.close()
}
// Configure web server
key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') }
repo_dir := os.getenv_opt('REPO_DIR') or {
exit_with_message(1, 'No repo directory was configured.')
}
pkg_dir := os.getenv_opt('PKG_DIR') or {
exit_with_message(1, 'No package directory was configured.')
}
dl_dir := os.getenv_opt('DOWNLOAD_DIR') or {
exit_with_message(1, 'No download directory was configured.')
if os.args.len == 1 {
exit_with_message(1, 'No action provided.')
}
// This also creates the directories if needed
repo := repo.new(repo_dir, pkg_dir) or {
logger.error(err.msg)
exit(1)
match os.args[1] {
'server' { server(key, repo_dir) }
'build' { build(key, repo_dir) ? }
else { exit_with_message(1, 'Unknown action: ${os.args[1]}') }
}
os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') }
web.run(&App{
logger: logger
api_key: key
dl_dir: dl_dir
repo: repo
}, port)
}

51
src/server.v 100644
View File

@ -0,0 +1,51 @@
module main
import web
import os
import log
import repo
fn server(key string, repo_dir string) {
// Configure logger
log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' }
log_level := log.level_from_tag(log_level_str) or {
exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
}
log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' }
mut logger := log.Log{
level: log_level
}
logger.set_full_logpath(log_file)
logger.log_to_console_too()
defer {
logger.info('Flushing log file')
logger.flush()
logger.close()
}
// Configure web server
pkg_dir := os.getenv_opt('PKG_DIR') or {
exit_with_message(1, 'No package directory was configured.')
}
dl_dir := os.getenv_opt('DOWNLOAD_DIR') or {
exit_with_message(1, 'No download directory was configured.')
}
// This also creates the directories if needed
repo := repo.new(repo_dir, pkg_dir) or {
logger.error(err.msg)
exit(1)
}
os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') }
web.run(&App{
logger: logger
api_key: key
dl_dir: dl_dir
repo: repo
}, port)
}