jenspots
/
vieter
forked from vieter-v/vieter
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Merge pull request 'Split docker, rename org & modules' (#259) from Chewing_Bever/vieter:split-docker into dev 

Reviewed-on: vieter-v/vieter#259
dev
Jef Roosens 2022-06-22 10:25:45 +02:00
parent 39eb03077e c8fc4c6a96
commit af4fbc4ccc
23 changed files with 51 additions and 499 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.woodpecker/.arch-rel.yml
View File

@ -23,7 +23,7 @@ pipeline:
- su builder
# Due to a bug with the V compiler, we can't just use the PKGBUILD from
# inside the repo
- curl -OL "https://git.rustybever.be/vieter/vieter/raw/tag/$CI_COMMIT_TAG/PKGBUILD"
- curl -OL "https://git.rustybever.be/vieter-v/vieter/raw/tag/$CI_COMMIT_TAG/PKGBUILD"
- makepkg -s --noconfirm --needed
when:
event: tag

2
.woodpecker/.arch.yml
View File

@ -23,7 +23,7 @@ pipeline:
- su builder
# Due to a bug with the V compiler, we can't just use the PKGBUILD from
# inside the repo
- curl -o PKGBUILD -L https://git.rustybever.be/vieter/vieter/raw/branch/dev/PKGBUILD.dev
- curl -o PKGBUILD -L https://git.rustybever.be/vieter-v/vieter/raw/branch/dev/PKGBUILD.dev
- makepkg -s --noconfirm --needed
when:
event: push

2
.woodpecker/.docs.yml
View File

@ -4,7 +4,7 @@ branches:
pipeline:
docs:
image: 'klakegg/hugo:alpine'
image: 'klakegg/hugo:ext-alpine'
group: 'generate'
commands:
- apk add git

10
.woodpecker/.test.yml
View File

@ -8,10 +8,20 @@ branches:
platform: ${PLATFORM}
pipeline:
install-modules:
image: 'chewingbever/vlang:latest'
pull: true
commands:
- export VMODULES=$PWD/.vmodules
- 'cd src && v install'
when:
event: [pull_request]
test:
image: 'chewingbever/vlang:latest'
pull: true
commands:
- export VMODULES=$PWD/.vmodules
- make test
when:
event: [pull_request]

16
CHANGELOG.md
View File

@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev)
## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev)
### Added
@ -28,11 +28,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* md5 hashes are no longer calculated for packages
## [0.3.0](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0)
## [0.3.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.3.0)
Nothing besides bumping the versions.
## [0.3.0-rc.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-rc.1)
## [0.3.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.3.0-rc.1)
### Added
@ -60,7 +60,7 @@ Nothing besides bumping the versions.
* `POST /api/logs` now correctly uses epoch timestamps instead of strings
## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2)
## [0.3.0-alpha.2](https://git.rustybever.be/vieter-v/vieter/src/tag/0.3.0-alpha.2)
### Added
@ -85,7 +85,7 @@ Nothing besides bumping the versions.
* `vieter-git` is the latest commit on the dev branch
* Full refactor of Docker socket code
## [0.3.0-alpha.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.1)
## [0.3.0-alpha.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.3.0-alpha.1)
### Changed
@ -104,7 +104,7 @@ Nothing besides bumping the versions.
* Binary no longer panics when an env var is missing
## [0.2.0](https://git.rustybever.be/vieter/vieter/src/tag/0.2.0)
## [0.2.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.2.0)
### Changed
@ -138,13 +138,13 @@ Nothing besides bumping the versions.
* Packages with unknown fields in .PKGINFO are now allowed
* Old packages are now properly removed
## [0.1.0](https://git.rustybever.be/vieter/vieter/src/tag/0.1.0)
## [0.1.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.1.0)
### Changed
* Improved logging
## [0.1.0-rc.1](https://git.rustybever.be/vieter/vieter/src/tag/0.1.0-rc.1)
## [0.1.0-rc.1](https://git.rustybever.be/vieter-v/vieter/src/tag/0.1.0-rc.1)
### Added

4
Makefile
View File

@ -87,12 +87,12 @@ test:
.PHONY: v
v: v/v
v/v:
git clone --single-branch https://git.rustybever.be/Chewing_Bever/v v
git clone --single-branch https://git.rustybever.be/vieter-v/v v
make -C v
.PHONY: clean
clean:
rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' 'docs/public'
rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' 'docs/public'
# =====EXPERIMENTAL=====

4
PKGBUILD
View File

@ -9,9 +9,9 @@ pkgdesc="Vieter is a lightweight implementation of an Arch repository server."
depends=('glibc' 'openssl' 'libarchive' 'sqlite')
makedepends=('git' 'vieter-v')
arch=('x86_64' 'aarch64')
url='https://git.rustybever.be/vieter/vieter'
url='https://git.rustybever.be/vieter-v/vieter'
license=('AGPL3')
source=("$pkgname::git+https://git.rustybever.be/vieter/vieter#tag=${pkgver//_/-}")
source=("$pkgname::git+https://git.rustybever.be/vieter-v/vieter#tag=${pkgver//_/-}")
md5sums=('SKIP')
prepare() {

4
PKGBUILD.dev
View File

@ -9,9 +9,9 @@ pkgdesc="Vieter is a lightweight implementation of an Arch repository server."
depends=('glibc' 'openssl' 'libarchive' 'sqlite')
makedepends=('git' 'vieter-v')
arch=('x86_64' 'aarch64')
url='https://git.rustybever.be/vieter/vieter'
url='https://git.rustybever.be/vieter-v/vieter'
license=('AGPL3')
source=("$pkgname::git+https://git.rustybever.be/vieter/vieter#branch=dev")
source=("$pkgname::git+https://git.rustybever.be/vieter-v/vieter#branch=dev")
md5sums=('SKIP')
provides=('vieter')
conflicts=('vieter')

6
README.md
View File

@ -49,9 +49,9 @@ update`.
### Compiler
Vieter compiles with the standard Vlang compiler. However, I do maintain a
[mirror](https://git.rustybever.be/vieter/v). This is to ensure my CI does not
break without reason, as I control when & how frequently the mirror is updated
to reflect the official repository.
[mirror](https://git.rustybever.be/vieter-v/v). This is to ensure my CI does
not break without reason, as I control when & how frequently the mirror is
updated to reflect the official repository.
If you encounter issues using the latest V compiler, try using my mirror
instead. `make v` will clone the repository & build the mirror. Afterwards,

4
docs/config.toml
View File

@ -38,7 +38,7 @@ enableGitInfo = true
weight = 20
[[menu.after]]
name = "Vieter"
url = "https://git.rustybever.be/vieter/vieter"
url = "https://git.rustybever.be/vieter-v/vieter"
weight = 30
[[menu.after]]
name = "Hugo Theme"
@ -70,7 +70,7 @@ enableGitInfo = true
# Set source repository location.
# Used for 'Last Modified' and 'Edit this page' links.
BookRepo = 'https://git.rustybever.be/vieter/vieter'
BookRepo = 'https://git.rustybever.be/vieter-v/vieter'
# (Optional, default 'commit') Specifies commit portion of the link to the page's last modified
# commit hash for 'doc' page type.

22
src/build/build.v
View File

@ -1,6 +1,6 @@
module build
import docker
import vieter_v.docker
import encoding.base64
import time
import os
@ -59,13 +59,13 @@ pub fn create_build_image(base_image string) ?string {
// We pull the provided image
dd.pull_image(image_name, image_tag)?
id := dd.create_container(c)?.id
id := dd.container_create(c)?.id
// id := docker.create_container(c)?
dd.start_container(id)?
dd.container_start(id)?
// This loop waits until the container has stopped, so we can remove it after
for {
data := dd.inspect_container(id)?
data := dd.container_inspect(id)?
if !data.state.running {
break
@ -80,7 +80,7 @@ pub fn create_build_image(base_image string) ?string {
// conflicts.
tag := time.sys_mono_now().str()
image := dd.create_image_from_container(id, 'vieter-build', tag)?
dd.remove_container(id)?
dd.container_remove(id)?
return image.id
}
@ -125,25 +125,25 @@ pub fn build_target(address string, api_key string, base_image_id string, target
user: '0:0'
}
id := dd.create_container(c)?.id
dd.start_container(id)?
id := dd.container_create(c)?.id
dd.container_start(id)?
mut data := dd.inspect_container(id)?
mut data := dd.container_inspect(id)?
// This loop waits until the container has stopped, so we can remove it after
for data.state.running {
time.sleep(1 * time.second)
data = dd.inspect_container(id)?
data = dd.container_inspect(id)?
}
mut logs_stream := dd.get_container_logs(id)?
mut logs_stream := dd.container_get_logs(id)?
// Read in the entire stream
mut logs_builder := strings.new_builder(10 * 1024)
util.reader_to_writer(mut logs_stream, mut logs_builder)?
dd.remove_container(id)?
dd.container_remove(id)?
return BuildResult{
start_time: data.state.start_time

2
src/console/logs/logs.v
View File

@ -1,7 +1,7 @@
module logs
import cli
import vieter.vconf
import vieter_v.conf as vconf
import client
import console
import time

2
src/console/targets/build.v
View File

@ -1,7 +1,7 @@
module targets
import client
import docker
import vieter_v.docker
import os
import build

2
src/console/targets/targets.v
View File

@ -1,7 +1,7 @@
module targets
import cli
import vieter.vconf
import vieter_v.conf as vconf
import cron.expression { parse_expression }
import client { NewTarget }
import console

2
src/cron/cli.v
View File

@ -1,7 +1,7 @@
module cron
import cli
import vieter.vconf
import vieter_v.conf as vconf
struct Config {
pub:

2
src/cron/daemon/daemon.v
View File

@ -6,7 +6,7 @@ import datatypes { MinHeap }
import cron.expression { CronExpression, parse_expression }
import math
import build
import docker
import vieter_v.docker
import os
import client
import models { Target }

3
src/docker/README.md
View File

@ -1,3 +0,0 @@
This module implements part of the Docker Engine API v1.41
([documentation](https://docs.docker.com/engine/api/v1.41/)) using socket-based
HTTP communication.

123
src/docker/containers.v
View File

@ -1,123 +0,0 @@
module docker
import json
import net.urllib
import time
import net.http { Method }
struct DockerError {
message string
}
pub struct NewContainer {
image string [json: Image]
entrypoint []string [json: Entrypoint]
cmd []string [json: Cmd]
env []string [json: Env]
work_dir string [json: WorkingDir]
user string [json: User]
}
struct CreatedContainer {
pub:
id string [json: Id]
warnings []string [json: Warnings]
}
// create_container creates a new container with the given config.
pub fn (mut d DockerConn) create_container(c NewContainer) ?CreatedContainer {
d.send_request_with_json(Method.post, urllib.parse('/v1.41/containers/create')?, c)?
head, res := d.read_response()?
if head.status_code != 201 {
data := json.decode(DockerError, res)?
return error(data.message)
}
data := json.decode(CreatedContainer, res)?
return data
}
// start_container starts the container with the given id.
pub fn (mut d DockerConn) start_container(id string) ? {
d.send_request(Method.post, urllib.parse('/v1.41/containers/$id/start')?)?
head, body := d.read_response()?
if head.status_code != 204 {
data := json.decode(DockerError, body)?
return error(data.message)
}
}
struct ContainerInspect {
pub mut:
state ContainerState [json: State]
}
struct ContainerState {
pub:
running bool [json: Running]
status string [json: Status]
exit_code int [json: ExitCode]
// These use a rather specific format so they have to be parsed later
start_time_str string [json: StartedAt]
end_time_str string [json: FinishedAt]
pub mut:
start_time time.Time [skip]
end_time time.Time [skip]
}
// inspect_container returns detailed information for a given container.
pub fn (mut d DockerConn) inspect_container(id string) ?ContainerInspect {
d.send_request(Method.get, urllib.parse('/v1.41/containers/$id/json')?)?
head, body := d.read_response()?
if head.status_code != 200 {
data := json.decode(DockerError, body)?
return error(data.message)
}
mut data := json.decode(ContainerInspect, body)?
// The Docker engine API *should* always return UTC time.
data.state.start_time = time.parse_rfc3339(data.state.start_time_str)?
if data.state.status == 'exited' {
data.state.end_time = time.parse_rfc3339(data.state.end_time_str)?
}
return data
}
// remove_container removes the container with the given id.
pub fn (mut d DockerConn) remove_container(id string) ? {
d.send_request(Method.delete, urllib.parse('/v1.41/containers/$id')?)?
head, body := d.read_response()?
if head.status_code != 204 {
data := json.decode(DockerError, body)?
return error(data.message)
}
}
// get_container_logs returns a reader object allowing access to the
// container's logs.
pub fn (mut d DockerConn) get_container_logs(id string) ?&StreamFormatReader {
d.send_request(Method.get, urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true')?)?
head := d.read_response_head()?
if head.status_code != 200 {
content_length := head.header.get(http.CommonHeader.content_length)?.int()
body := d.read_response_body(content_length)?
data := json.decode(DockerError, body)?
return error(data.message)
}
return d.get_stream_format_reader()
}

137
src/docker/docker.v
View File

@ -1,137 +0,0 @@
module docker
import net.unix
import io
import net.http
import strings
import net.urllib
import json
import util
const (
socket = '/var/run/docker.sock'
buf_len = 10 * 1024
http_separator = [u8(`\r`), `\n`, `\r`, `\n`]
http_chunk_separator = [u8(`\r`), `\n`]
)
pub struct DockerConn {
mut:
socket &unix.StreamConn
reader &io.BufferedReader
}
// new_conn creates a new connection to the Docker daemon.
pub fn new_conn() ?&DockerConn {
s := unix.connect_stream(docker.socket)?
d := &DockerConn{
socket: s
reader: io.new_buffered_reader(reader: s)
}
return d
}
// close closes the underlying socket connection.
pub fn (mut d DockerConn) close() ? {
d.socket.close()?
}
// send_request sends an HTTP request without body.
pub fn (mut d DockerConn) send_request(method http.Method, url urllib.URL) ? {
req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n'
d.socket.write_string(req)?
// When starting a new request, the reader needs to be reset.
d.reader = io.new_buffered_reader(reader: d.socket)
}
// send_request_with_body sends an HTTP request with the given body.
pub fn (mut d DockerConn) send_request_with_body(method http.Method, url urllib.URL, content_type string, body string) ? {
req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n'
d.socket.write_string(req)?
// When starting a new request, the reader needs to be reset.
d.reader = io.new_buffered_reader(reader: d.socket)
}
// send_request_with_json<T> is a convenience wrapper around
// send_request_with_body that encodes the input as JSON.
pub fn (mut d DockerConn) send_request_with_json<T>(method http.Method, url urllib.URL, data &T) ? {
body := json.encode(data)
return d.send_request_with_body(method, url, 'application/json', body)
}
// read_response_head consumes the socket's contents until it encounters
// '\r\n\r\n', after which it parses the response as an HTTP response.
// Importantly, this function never consumes the reader past the HTTP
// separator, so the body can be read fully later on.
pub fn (mut d DockerConn) read_response_head() ?http.Response {
mut res := []u8{}
util.read_until_separator(mut d.reader, mut res, docker.http_separator)?
return http.parse_response(res.bytestr())
}
// read_response_body reads `length` bytes from the stream. It can be used when
// the response encoding isn't chunked to fully read it.
pub fn (mut d DockerConn) read_response_body(length int) ?string {
if length == 0 {
return ''
}
mut buf := []u8{len: docker.buf_len}
mut c := 0
mut builder := strings.new_builder(docker.buf_len)
for builder.len < length {
c = d.reader.read(mut buf) or { break }
builder.write(buf[..c])?
}
return builder.str()
}
// read_response is a convenience function which always consumes the entire
// response & returns it. It should only be used when we're certain that the
// result isn't too large.
pub fn (mut d DockerConn) read_response() ?(http.Response, string) {
head := d.read_response_head()?
if head.header.get(http.CommonHeader.transfer_encoding) or { '' } == 'chunked' {
mut builder := strings.new_builder(1024)
mut body := d.get_chunked_response_reader()
util.reader_to_writer(mut body, mut builder)?
return head, builder.str()
}
content_length := head.header.get(http.CommonHeader.content_length)?.int()
res := d.read_response_body(content_length)?
return head, res
}
// get_chunked_response_reader returns a ChunkedResponseReader using the socket
// as reader.
pub fn (mut d DockerConn) get_chunked_response_reader() &ChunkedResponseReader {
r := new_chunked_response_reader(d.reader)
return r
}
// get_stream_format_reader returns a StreamFormatReader using the socket as
// reader.
pub fn (mut d DockerConn) get_stream_format_reader() &StreamFormatReader {
r := new_chunked_response_reader(d.reader)
r2 := new_stream_format_reader(r)
return r2
}

61
src/docker/images.v
View File

@ -1,61 +0,0 @@
module docker
import net.http { Method }
import net.urllib
import json
struct Image {
pub:
id string [json: Id]
}
// pull_image pulls the given image:tag.
pub fn (mut d DockerConn) pull_image(image string, tag string) ? {
d.send_request(Method.post, urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?)?
head := d.read_response_head()?
if head.status_code != 200 {
content_length := head.header.get(http.CommonHeader.content_length)?.int()
body := d.read_response_body(content_length)?
data := json.decode(DockerError, body)?
return error(data.message)
}
// Keep reading the body until the pull has completed
mut body := d.get_chunked_response_reader()
mut buf := []u8{len: 1024}
for {
body.read(mut buf) or { break }
}
}
// create_image_from_container creates a new image from a container.
pub fn (mut d DockerConn) create_image_from_container(id string, repo string, tag string) ?Image {
d.send_request(Method.post, urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag')?)?
head, body := d.read_response()?
if head.status_code != 201 {
data := json.decode(DockerError, body)?
return error(data.message)
}
data := json.decode(Image, body)?
return data
}
// remove_image removes the image with the given id.
pub fn (mut d DockerConn) remove_image(id string) ? {
d.send_request(Method.delete, urllib.parse('/v1.41/images/$id')?)?
head, body := d.read_response()?
if head.status_code != 200 {
data := json.decode(DockerError, body)?
return error(data.message)
}
}

135
src/docker/stream.v
View File

@ -1,135 +0,0 @@
module docker
import io
import util
import encoding.binary
import encoding.hex
// ChunkedResponseReader parses an underlying HTTP chunked response, exposing
// it as if it was a continuous stream of data.
struct ChunkedResponseReader {
mut:
reader io.BufferedReader
bytes_left_in_chunk u64
started bool
}
// new_chunked_response_reader creates a new ChunkedResponseReader on the heap
// with the provided reader.
pub fn new_chunked_response_reader(reader io.BufferedReader) &ChunkedResponseReader {
r := &ChunkedResponseReader{
reader: reader
}
return r
}
// read satisfies the io.Reader interface.
pub fn (mut r ChunkedResponseReader) read(mut buf []u8) ?int {
if r.bytes_left_in_chunk == 0 {
// An io.BufferedReader always returns none if its stream has
// ended.
r.bytes_left_in_chunk = r.read_chunk_size()?
}
mut c := 0
// Make sure we don't read more than we can safely read. This is to avoid
// the underlying reader from becoming out of sync with our parsing:
if buf.len > r.bytes_left_in_chunk {
c = r.reader.read(mut buf[..r.bytes_left_in_chunk])?
} else {
c = r.reader.read(mut buf)?
}
r.bytes_left_in_chunk -= u64(c)
return c
}
// read_chunk_size advances the reader & reads the size of the next HTTP chunk.
// This function should only be called if the previous chunk has been
// completely consumed.
fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 {
if r.started {
mut buf := []u8{len: 2}
// Each chunk ends with a `\r\n` which we want to skip first
r.reader.read(mut buf)?
}
r.started = true
mut res := []u8{}
util.read_until_separator(mut r.reader, mut res, http_chunk_separator)?
// The length of the next chunk is provided as a hexadecimal
mut num_data := hex.decode(res#[..-2].bytestr())?
for num_data.len < 8 {
num_data.insert(0, 0)
}
num := binary.big_endian_u64(num_data)
// This only occurs for the very last chunk, which always reports a size of
// 0.
if num == 0 {
return none
}
return num
}
// StreamFormatReader parses an underlying stream of Docker logs, removing the
// header bytes.
struct StreamFormatReader {
mut:
reader ChunkedResponseReader
bytes_left_in_chunk u32
}
// new_stream_format_reader creates a new StreamFormatReader using the given
// reader.
pub fn new_stream_format_reader(reader ChunkedResponseReader) &StreamFormatReader {
r := &StreamFormatReader{
reader: reader
}
return r
}
// read satisfies the io.Reader interface.
pub fn (mut r StreamFormatReader) read(mut buf []u8) ?int {
if r.bytes_left_in_chunk == 0 {
r.bytes_left_in_chunk = r.read_chunk_size()?
}
mut c := 0
if buf.len > r.bytes_left_in_chunk {
c = r.reader.read(mut buf[..r.bytes_left_in_chunk])?
} else {
c = r.reader.read(mut buf)?
}
r.bytes_left_in_chunk -= u32(c)
return c
}
// read_chunk_size advances the reader & reads the header bytes for the length
// of the next chunk.
fn (mut r StreamFormatReader) read_chunk_size() ?u32 {
mut buf := []u8{len: 8}
r.reader.read(mut buf)?
num := binary.big_endian_u32(buf[4..])
if num == 0 {
return none
}
return num
}

2
src/server/cli.v
View File

@ -1,7 +1,7 @@
module server
import cli
import vieter.vconf
import vieter_v.conf as vconf
struct Config {
pub:

3
src/v.mod
View File

@ -1,5 +1,6 @@
Module {
dependencies: [
'https://git.rustybever.be/vieter/vconf'
'https://git.rustybever.be/vieter-v/conf',
'https://git.rustybever.be/vieter-v/docker'
]
}