Chewing_Bever
/
vieter
forked from vieter-v/vieter
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Merge pull request 'Release 0.2.0' (#121) from release-0.2.0 into main 

Reviewed-on: #121
main 0.2.0
Jef Roosens 2022-04-11 09:41:43 +02:00
parent dec2e9d759 3b15066329
commit 4b6a661d71
42 changed files with 1607 additions and 668 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.editorconfig
View File

@ -1,4 +1,3 @@
# top-most EditorConfig file
root = true
# Unix-style newlines with a newline ending every file
@ -7,4 +6,5 @@ end_of_line = lf
insert_final_newline = true
[*.v]
indent_style = space
# vfmt wants it :(
indent_style = tab

7
.gitignore vendored
View File

@ -5,6 +5,8 @@ data/
vieter
dvieter
pvieter
dvieterctl
vieterctl
vieter.c
# Ignore testing files
@ -17,4 +19,7 @@ libarchive-*
test/
# V compiler directory
v-*/
v/
# gdb log file
gdb.txt

30
.woodpecker/.arch.yml 100644
View File

@ -0,0 +1,30 @@
matrix:
PLATFORM:
- linux/amd64
- linux/arm64
platform: ${PLATFORM}
branches: [dev]
pipeline:
build:
image: 'menci/archlinuxarm:base-devel'
commands:
# Update packages
- pacman -Syu --noconfirm
# Create non-root user to perform build & switch to their home
- groupadd -g 1000 builder
- useradd -mg builder builder
- chown -R builder:builder "$PWD"
- "echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers"
- su builder
# Build the package
- makepkg -s --noconfirm --needed
publish:
image: 'curlimages/curl'
commands:
# Publish the package
- 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done'
secrets:
- vieter_api_key

7
.woodpecker/.build.yml
View File

@ -2,7 +2,8 @@ matrix:
PLATFORM:
- linux/amd64
- linux/arm64
- linux/arm/v7
# I just don't have a performant enough runner for this platform
# - linux/arm/v7
# These checks already get performed on the feature branches
platform: ${PLATFORM}
@ -42,12 +43,12 @@ pipeline:
commands:
# https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f
- export URL=s3.rustybever.be
- export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')"
- export DATE="$(date -R --utc)"
- export CONTENT_TYPE='application/zstd'
- export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')"
- export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH"
- export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64`
- >
curl
--silent

18
.woodpecker/.builder.yml
View File

@ -1,18 +0,0 @@
branches: dev
platform: linux/amd64
pipeline:
publish:
image: woodpeckerci/plugin-docker-buildx
secrets: [ docker_username, docker_password ]
settings:
repo: chewingbever/vlang
tag: latest
dockerfile: Dockerfile.builder
platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
when:
event: push
path:
- Makefile
- Dockerfile.builder
- patches/*

15
.woodpecker/.deploy.yml
View File

@ -1,15 +0,0 @@
# Deploys the newest development image to my server
branches: [dev]
platform: linux/amd64
depends_on:
- docker
skip_clone: true
pipeline:
webhook:
image: chewingbever/vlang:latest
secrets:
- webhook
commands:
- curl -XPOST -s "$WEBHOOK"

7
.woodpecker/.docker.yml
View File

@ -1,7 +1,6 @@
branches: [main, dev]
platform: linux/amd64
depends_on:
- builder
- build
pipeline:
@ -10,9 +9,8 @@ pipeline:
secrets: [ docker_username, docker_password ]
settings:
repo: chewingbever/vieter
dockerfile: Dockerfile.ci
tag: dev
platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
platforms: [ linux/arm64/v8, linux/amd64 ]
build_args_from_env:
- CI_COMMIT_SHA
when:
@ -24,9 +22,8 @@ pipeline:
secrets: [ docker_username, docker_password ]
settings:
repo: chewingbever/vieter
dockerfile: Dockerfile.ci
auto_tag: true
platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
platforms: [ linux/arm64/v8, linux/amd64 ]
build_args_from_env:
- CI_COMMIT_SHA
when:

34
CHANGELOG.md
View File

@ -7,6 +7,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter)
## [0.2.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.2.0)
### Changed
* Better config system
* Support for both a config file & environment variables
* Each env var can now be provided from a file by appending it with `_FILE`
& passing the path to the file as value
* Revamped web framework
* All routes now return proper JSON where applicable & the correct status
codes
### Added
* Very basic build system
* Build is triggered by separate cron container
* Packages build on cron container's system
* A HEAD request is used to determine whether a package should be rebuilt
or not
* Hardcoded planning of builds
* Builds are sequential
* API for managing Git repositories to build
* CLI to list, add & remove Git repos to build
* Published packages on my Vieter instance
* Support for multiple repositories
* Support for multiple architectures per repository
### Fixed
* Each package can now only have one version in the repository at once
(required by Pacman)
* Packages with unknown fields in .PKGINFO are now allowed
* Old packages are now properly removed
## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0)
### Changed

49
Dockerfile
View File

@ -1,19 +1,56 @@
FROM chewingbever/vlang:latest AS builder
ARG TARGETPLATFORM
ARG CI_COMMIT_SHA
ARG DI_VER=1.2.5
WORKDIR /app
# Build dumb-init
RUN curl -Lo - "https://github.com/Yelp/dumb-init/archive/refs/tags/v${DI_VER}.tar.gz" | tar -xzf - && \
cd "dumb-init-${DI_VER}" && \
make SHELL=/bin/sh && \
mv dumb-init .. && \
cd ..
# Copy over source code & build production binary
COPY src ./src
COPY Makefile ./
ENV LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static'
RUN v -o pvieter -cflags "-O3" src
RUN if [ -n "${CI_COMMIT_SHA}" ]; then \
curl --fail \
-o vieter \
"https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \
chmod +x vieter ; \
else \
LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' make prod && \
mv pvieter vieter ; \
fi
FROM alpine:3.15
FROM busybox:1.35.0
ENV REPO_DIR=/data
ENV PATH=/bin \
VIETER_REPOS_DIR=/data/repos \
VIETER_PKG_DIR=/data/pkgs \
VIETER_DOWNLOAD_DIR=/data/downloads \
VIETER_REPOS_FILE=/data/repos.json
COPY --from=builder /app/pvieter /usr/local/bin/vieter
COPY --from=builder /app/dumb-init /app/vieter /bin/
ENTRYPOINT [ "/usr/local/bin/vieter" ]
HEALTHCHECK --interval=30s \
--timeout=3s \
--start-period=5s \
CMD /bin/wget --spider http://localhost:8000/health || exit 1
RUN mkdir /data && \
chown -R www-data:www-data /data && \
mkdir -p '/var/spool/cron/crontabs' && \
echo '0 3 * * * /bin/vieter build' | crontab -
WORKDIR /data
USER www-data:www-data
ENTRYPOINT ["/bin/dumb-init", "--"]
CMD ["/bin/vieter", "server"]

35
Dockerfile.builder
View File

@ -1,35 +0,0 @@
FROM alpine:3.12
ARG TARGETPLATFORM
WORKDIR /opt/vlang
ENV VVV /opt/vlang
ENV PATH /opt/vlang:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ENV VFLAGS -cc gcc
ENV V_PATH /opt/vlang/v
RUN ln -s /opt/vlang/v /usr/bin/v && \
apk --no-cache add \
git make gcc curl openssl \
musl-dev \
openssl-libs-static openssl-dev \
zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \
sqlite-static sqlite-dev \
libx11-dev glfw-dev freetype-dev \
libarchive-static libarchive-dev \
diffutils
COPY patches ./patches
COPY Makefile ./
RUN make v && \
mv v-*/* /opt/vlang && \
v -version
RUN if [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \
wget -O /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && \
chmod +x /usr/local/bin/mc ; \
fi
CMD ["v"]

46
Dockerfile.ci
View File

@ -1,46 +0,0 @@
# vim: ft=dockerfile
# This image just has the required tools to download the binaries
FROM chewingbever/vlang:latest AS builder
ARG TARGETPLATFORM
ARG CI_COMMIT_SHA
ARG DI_VER=1.2.5
WORKDIR /app
# Build dumb-init
RUN curl -Lo - "https://github.com/Yelp/dumb-init/archive/refs/tags/v${DI_VER}.tar.gz" | tar -xzf - && \
cd "dumb-init-${DI_VER}" && \
make SHELL=/bin/sh && \
mv dumb-init .. && \
cd ..
RUN curl --fail \
-o vieter \
"https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \
chmod +x vieter
FROM busybox:1.35.0
ENV PATH=/bin \
REPO_DIR=/data/repo \
PKG_DIR=/data/pkgs \
DOWNLOAD_DIR=/data/downloads
COPY --from=builder /app/dumb-init /app/vieter /bin/
HEALTHCHECK --interval=30s \
--timeout=3s \
--start-period=5s \
CMD /bin/wget --spider http://localhost:8000/health || exit 1
RUN mkdir /data && \
chown -R www-data:www-data /data
WORKDIR /data
USER www-data:www-data
ENTRYPOINT ["/bin/dumb-init", "--"]
CMD ["/bin/vieter"]

36
Makefile
View File

@ -2,9 +2,8 @@
SRC_DIR := src
SOURCES != find '$(SRC_DIR)' -iname '*.v'
V_RELEASE := weekly.2022.05
V_PATH ?= v-$(V_RELEASE)/v
V := $(V_PATH) -showcc
V_PATH ?= v/v
V := $(V_PATH) -showcc -gc boehm
all: vieter
@ -14,10 +13,17 @@ vieter: $(SOURCES)
$(V) -g -o vieter $(SRC_DIR)
# Debug build using gcc
# The debug build can't use the boehm garbage collector, as that is
# multi-threaded and causes issues when running vieter inside gdb.
.PHONY: debug
debug: dvieter
dvieter: $(SOURCES)
$(V) -keepc -cg -cc gcc -o dvieter $(SRC_DIR)
$(V_PATH) -showcc -keepc -cg -o dvieter $(SRC_DIR)
# Run the debug build inside gdb
.PHONY: gdb
gdb: dvieter
gdb --args './dvieter -f vieter.toml server'
# Optimised production build
.PHONY: prod
@ -30,22 +36,15 @@ pvieter: $(SOURCES)
c:
$(V) -o vieter.c $(SRC_DIR)
# =====EXECUTION=====
# Run the server in the default 'data' directory
.PHONY: run
run: vieter
API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter
./vieter -f vieter.toml server
.PHONY: run-prod
run-prod: prod
API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./pvieter
# Same as run, but restart when the source code changes
.PHONY: watch
watch:
API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG $(V) watch run vieter
./pvieter -f vieter.toml server
# =====OTHER=====
.PHONY: lint
@ -63,11 +62,10 @@ vet:
# Build & patch the V compiler
.PHONY: v
v: v-$(V_RELEASE)/v
v-$(V_RELEASE)/v:
curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf -
cd patches && sh patch.sh '../v-$(V_RELEASE)'
make -C 'v-$(V_RELEASE)'
v: v/v
v/v:
git clone --single-branch --branch patches https://git.rustybever.be/Chewing_Bever/vieter-v v
make -C v
clean:
rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'v-$(V_RELEASE)'
rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter'

34
PKGBUILD 100644
View File

@ -0,0 +1,34 @@
# Maintainer: Jef Roosens
pkgbase='vieter'
pkgname='vieter'
pkgver=0.1.0.rc1.r117.gc3ac00f
pkgrel=1
depends=('glibc' 'openssl' 'libarchive' 'gc')
makedepends=('git' 'gcc')
arch=('x86_64' 'aarch64' 'armv7')
url='https://git.rustybever.be/Chewing_Bever/vieter'
license=('AGPL3')
source=($pkgname::git+https://git.rustybever.be/Chewing_Bever/vieter#branch=dev)
md5sums=('SKIP')
pkgver() {
cd "$pkgname"
git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g'
}
build() {
cd "$pkgname"
# Build the compiler
CFLAGS= make v
make prod
}
package() {
pkgdesc="Vieter is a lightweight implementation of an Arch repository server."
install -dm755 "$pkgdir/usr/bin"
install -Dm755 "$pkgbase/pvieter" "$pkgdir/usr/bin/vieter"
}

60
README.md
View File

@ -1,10 +1,17 @@
# Vieter
Vieter is a re-implementation of the Pieter project. The goal is to create a
simple PKGBUILD-based build system, combined with a self-hosted Arch
repository. This would allow me to periodically re-build AUR packages (or
PKGBUILDs I created myself), & make sure I never have to compile anything on my
own systems, making my updates a lot quicker.
## Documentation
I host documentation for Vieter over at https://rustybever.be/docs/vieter/.
## Overview
Vieter is a restart of the Pieter project. The goal is to create a simple,
lightweight self-hostable Arch repository server, paired with a system that
periodically builds & publishes select Arch packages. This would allow me to
build AUR packages (or PKGBUILDs I created myself) "in the cloud" & make sure I
never have to compile anything on my own systems, making my updates a lot
quicker.
## Why V?
@ -16,31 +23,30 @@ that.
### Custom Compiler
Currently, this program only works with a very slightly modified version of the
V standard library, and therefore the compiler. The changes that are made to
the standard V release can be found in the [patches](/patches) directory. You
can obtain this modified version of the compiler by running `make v`, which
will download, patch & build the compiler. Afterwards, all make commands that
require the V compiler will use this new binary.
V standard library, and therefore the compiler. The source code for this fork
can be found [here](https://git.rustybever.be/Chewing_Bever/vieter-v). You can
obtain this modified version of the compiler by running `make v`, which will
clone & build the compiler. Afterwards, all make commands that require the V
compiler will use this new binary. I try to keep this fork as up to date with
upstream as possible.
## Features
The project will consist of a server-agent model, where one or more builder
nodes can register with the server. These agents communicate with the Docker
daemon to start builds, which are then uploaded to the server's repository. The
server also allows for non-agents to upload packages, as long as they have the
required secrets. This allows me to also develop non-git packages, such as my
terminal, & upload them to the servers using CI.
* Arch repository server
* Support for multiple repositories & multiple architectures
* Endpoints for publishing new packages
* API for managing repositories to build
* Build system
* Periodic rebuilding of packages
* Prevent unnecessary rebuilds
## Directory Structure
## Building
The data directory consists of three main directories:
In order to build Vieter, you'll need a couple of libraries:
* `downloads` - This is where packages are initially downloaded. Because vieter
moves files from this folder to the `pkgs` folder, these two folders should
best be on the same drive
* `pkgs` - This is where approved package files are stored.
* `repos` - Each repository gets a subfolder here. The subfolder contains the
uncompressed contents of the db file.
* Each repo subdirectory contains the compressed db & files archive for the
repository, alongside a directory called `files` which contains the
uncompressed contents.
* gc
* libarchive
* openssl
Before building Vieter, you'll have to build the compiler using `make v`.
Afterwards, run `make` to build the debug binary.

23
patches/parse_request_no_body.v
View File

@ -1,23 +0,0 @@
// Parse the header of a raw HTTP request into a Request object
pub fn parse_request_head(mut reader io.BufferedReader) ?Request {
// request line
mut line := reader.read_line() ?
method, target, version := parse_request_line(line) ?
// headers
mut header := new_header()
line = reader.read_line() ?
for line != '' {
key, value := parse_header(line) ?
header.add_custom(key, value) ?
line = reader.read_line() ?
}
header.coerce(canonicalize: true)
return Request{
method: method
url: target.str()
header: header
version: version
}
}

13
patches/patch.sh
View File

@ -1,13 +0,0 @@
#!/usr/bin/env sh
# This file patches the downloaded V version
# Should be run from within the directory it's in, as it uses relative paths to the files used for patching.
# $1 is the path to the downloaded V version
# Add parse_request_no_body
cat parse_request_no_body.v >> "$1"/vlib/net/http/request.v
# Make sha256 functions public
sed -i \
-e 's/\(fn (mut d Digest) checksum(\)/pub \1/' \
-e 's/\(fn (mut d Digest) write(\)/pub \1/' \
"$1"/vlib/crypto/sha256/sha256.v

3
src/archive.v → src/archive.c.v
View File

@ -15,6 +15,9 @@ fn C.archive_read_support_filter_zstd(&C.archive)
// Configure the archive to work with gzip compression
fn C.archive_read_support_filter_gzip(&C.archive)
// Configure the archive to work with xz compression
fn C.archive_read_support_filter_xz(&C.archive)
// Configure the archive to work with a tarball content
fn C.archive_read_support_format_tar(&C.archive)

133
src/build/build.v 100644
View File

@ -0,0 +1,133 @@
module build
import docker
import encoding.base64
import time
import git
import os
const container_build_dir = '/build'
const build_image_repo = 'vieter-build'
fn create_build_image(base_image string) ?string {
commands := [
// Update repos & install required packages
'pacman -Syu --needed --noconfirm base-devel git'
// Add a non-root user to run makepkg
'groupadd -g 1000 builder',
'useradd -mg builder builder'
// Make sure they can use sudo without a password
"echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers"
// Create the directory for the builds & make it writeable for the
// build user
'mkdir /build',
'chown -R builder:builder /build',
]
cmds_str := base64.encode_str(commands.join('\n'))
c := docker.NewContainer{
image: base_image
env: ['BUILD_SCRIPT=$cmds_str']
entrypoint: ['/bin/sh', '-c']
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e']
}
// This check is needed so the user can pass "archlinux" without passing a
// tag & make it still work
image_parts := base_image.split_nth(':', 2)
image_name := image_parts[0]
image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' }
// We pull the provided image
docker.pull_image(image_name, image_tag) ?
id := docker.create_container(c) ?
docker.start_container(id) ?
// This loop waits until the container has stopped, so we can remove it after
for {
data := docker.inspect_container(id) ?
if !data.state.running {
break
}
// Wait for 5 seconds
time.sleep(5000000000)
}
// Finally, we create the image from the container
// As the tag, we use the epoch value
tag := time.sys_mono_now().str()
image := docker.create_image_from_container(id, 'vieter-build', tag) ?
docker.remove_container(id) ?
return image.id
}
fn build(conf Config) ? {
build_arch := os.uname().machine
// We get the repos map from the Vieter instance
repos_map := git.get_repos(conf.address, conf.api_key) ?
// We filter out any repos that aren't allowed to be built on this
// architecture
filtered_repos := repos_map.keys().map(repos_map[it]).filter(it.arch.contains(build_arch))
// No point in doing work if there's no repos present
if filtered_repos.len == 0 {
return
}
// First, we create a base image which has updated repos n stuff
image_id := create_build_image(conf.base_image) ?
for repo in filtered_repos {
// TODO what to do with PKGBUILDs that build multiple packages?
commands := [
'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
'cd repo',
'makepkg --nobuild --nodeps',
'source PKGBUILD',
// The build container checks whether the package is already
// present on the server
'curl --head --fail $conf.address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/$repo.repo/publish; done',
]
// We convert the list of commands into a base64 string, which then gets
// passed to the container as an env var
cmds_str := base64.encode_str(commands.join('\n'))
c := docker.NewContainer{
image: '$image_id'
env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key']
entrypoint: ['/bin/sh', '-c']
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e']
work_dir: '/build'
user: 'builder:builder'
}
id := docker.create_container(c) ?
docker.start_container(id) ?
// This loop waits until the container has stopped, so we can remove it after
for {
data := docker.inspect_container(id) ?
if !data.state.running {
break
}
// Wait for 5 seconds
time.sleep(5000000000)
}
docker.remove_container(id) ?
}
// Finally, we remove the builder image
docker.remove_image(image_id) ?
}

25
src/build/cli.v 100644
View File

@ -0,0 +1,25 @@
module build
import cli
import env
pub struct Config {
pub:
api_key string
address string
base_image string = 'archlinux:base-devel'
}
// cmd returns the cli submodule that handles the build process
pub fn cmd() cli.Command {
return cli.Command{
name: 'build'
description: 'Run the build process.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
build(conf) ?
}
}
}

78
src/docker/containers.v 100644
View File

@ -0,0 +1,78 @@
module docker
import json
import net.urllib
struct Container {
id string [json: Id]
names []string [json: Names]
}
// containers returns a list of all currently running containers
pub fn containers() ?[]Container {
res := request('GET', urllib.parse('/v1.41/containers/json') ?) ?
return json.decode([]Container, res.text) or {}
}
pub struct NewContainer {
image string [json: Image]
entrypoint []string [json: Entrypoint]
cmd []string [json: Cmd]
env []string [json: Env]
work_dir string [json: WorkingDir]
user string [json: User]
}
struct CreatedContainer {
id string [json: Id]
}
// create_container creates a container defined by the given configuration. If
// successful, it returns the ID of the newly created container.
pub fn create_container(c &NewContainer) ?string {
res := request_with_json('POST', urllib.parse('/v1.41/containers/create') ?, c) ?
if res.status_code != 201 {
return error('Failed to create container.')
}
return json.decode(CreatedContainer, res.text) ?.id
}
// start_container starts a container with a given ID. It returns whether the
// container was started or not.
pub fn start_container(id string) ?bool {
res := request('POST', urllib.parse('/v1.41/containers/$id/start') ?) ?
return res.status_code == 204
}
struct ContainerInspect {
pub:
state ContainerState [json: State]
}
struct ContainerState {
pub:
running bool [json: Running]
}
// inspect_container returns the result of inspecting a container with a given
// ID.
pub fn inspect_container(id string) ?ContainerInspect {
res := request('GET', urllib.parse('/v1.41/containers/$id/json') ?) ?
if res.status_code != 200 {
return error('Failed to inspect container.')
}
return json.decode(ContainerInspect, res.text) or {}
}
// remove_container removes a container with a given ID.
pub fn remove_container(id string) ?bool {
res := request('DELETE', urllib.parse('/v1.41/containers/$id') ?) ?
return res.status_code == 204
}

93
src/docker/docker.v 100644
View File

@ -0,0 +1,93 @@
module docker
import net.unix
import net.urllib
import net.http
import json
const socket = '/var/run/docker.sock'
const buf_len = 1024
fn send(req &string) ?http.Response {
// Open a connection to the socket
mut s := unix.connect_stream(docker.socket) or {
return error('Failed to connect to socket ${docker.socket}.')
}
defer {
// This or is required because otherwise, the V compiler segfaults for
// some reason
// https://github.com/vlang/v/issues/13534
s.close() or {}
}
// Write the request to the socket
s.write_string(req) or { return error('Failed to write request to socket ${docker.socket}.') }
s.wait_for_write() ?
mut c := 0
mut buf := []byte{len: docker.buf_len}
mut res := []byte{}
for {
c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') }
res << buf[..c]
if c < docker.buf_len {
break
}
}
// After reading the first part of the response, we parse it into an HTTP
// response. If it isn't chunked, we return early with the data.
parsed := http.parse_response(res.bytestr()) or {
return error('Failed to parse HTTP response from socket ${docker.socket}.')
}
if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' {
return parsed
}
// We loop until we've encountered the end of the chunked response
// A chunked HTTP response always ends with '0\r\n\r\n'.
for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] {
// Wait for the server to respond
s.wait_for_write() ?
for {
c = s.read(mut buf) or {
return error('Failed to read data from socket ${docker.socket}.')
}
res << buf[..c]
if c < docker.buf_len {
break
}
}
}
// Decode chunked response
return http.parse_response(res.bytestr())
}
fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response {
req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n'
return send(req)
}
fn request(method string, url urllib.URL) ?http.Response {
req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n'
return send(req)
}
// request_with_json<T> sends a request to the Docker socket with a given JSON
// payload
pub fn request_with_json<T>(method string, url urllib.URL, data &T) ?http.Response {
body := json.encode(data)
return request_with_body(method, url, 'application/json', body)
}

34
src/docker/images.v 100644
View File

@ -0,0 +1,34 @@
module docker
import net.http
import net.urllib
import json
struct Image {
pub:
id string [json: Id]
}
// pull_image pulls tries to pull the image for the given image & tag
pub fn pull_image(image string, tag string) ?http.Response {
return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?)
}
// create_image_from_container creates a new image from a container with the
// given repo & tag, given the container's ID.
pub fn create_image_from_container(id string, repo string, tag string) ?Image {
res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag') ?) ?
if res.status_code != 201 {
return error('Failed to create image from container.')
}
return json.decode(Image, res.text) or {}
}
// remove_image removes the image with the given ID.
pub fn remove_image(id string) ?bool {
res := request('DELETE', urllib.parse('/v1.41/images/$id') ?) ?
return res.status_code == 200
}

82
src/env.v 100644
View File

@ -0,0 +1,82 @@
module env
import os
import toml
// The prefix that every environment variable should have
const prefix = 'VIETER_'
// The suffix an environment variable in order for it to be loaded from a file
// instead
const file_suffix = '_FILE'
fn get_env_var(field_name string) ?string {
env_var_name := '$env.prefix$field_name.to_upper()'
env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix'
env_var := os.getenv(env_var_name)
env_file := os.getenv(env_file_name)
// If both are missing, we return an empty string
if env_var == '' && env_file == '' {
return ''
}
// If they're both set, we report a conflict
if env_var != '' && env_file != '' {
return error('Only one of $env_var_name or $env_file_name can be defined.')
}
// If it's the env var itself, we return it.
// I'm pretty sure this also prevents variable ending in _FILE (e.g.
// VIETER_LOG_FILE) from being mistakingely read as an _FILE suffixed env
// var.
if env_var != '' {
return env_var
}
// Otherwise, we process the file
return os.read_file(env_file) or {
error('Failed to read file defined in $env_file_name: ${err.msg}.')
}
}
// load<T> attempts to create an object of type T from the given path to a toml
// file & environment variables. For each field, it will select either a value
// given from an environment variable, a value defined in the config file or a
// configured default if present, in that order.
pub fn load<T>(path string) ?T {
mut res := T{}
if os.exists(path) {
// We don't use reflect here because reflect also sets any fields not
// in the toml back to their zero value, which we don't want
doc := toml.parse_file(path) ?
$for field in T.fields {
s := doc.value(field.name)
// We currently only support strings
if s.type_name() == 'string' {
res.$(field.name) = s.string()
}
}
}
$for field in T.fields {
$if field.typ is string {
env_value := get_env_var(field.name) ?
// The value of the env var will always be chosen over the config
// file
if env_value != '' {
res.$(field.name) = env_value
}
// If there's no value from the toml file either, we try to find a
// default value
else if res.$(field.name) == '' {
return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.")
}
}
}
return res
}

147
src/git/cli.v 100644
View File

@ -0,0 +1,147 @@
module git
import cli
import env
struct Config {
address string [required]
api_key string [required]
}
// cmd returns the cli submodule that handles the repos API interaction
pub fn cmd() cli.Command {
return cli.Command{
name: 'repos'
description: 'Interact with the repos API.'
commands: [
cli.Command{
name: 'list'
description: 'List the current repos.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
list(conf) ?
}
},
cli.Command{
name: 'add'
required_args: 4
usage: 'url branch repo arch...'
description: 'Add a new repository.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
add(conf, cmd.args[0], cmd.args[1], cmd.args[2], cmd.args[3..]) ?
}
},
cli.Command{
name: 'remove'
required_args: 1
usage: 'id'
description: 'Remove a repository that matches the given ID prefix.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
remove(conf, cmd.args[0]) ?
}
},
cli.Command{
name: 'edit'
required_args: 1
usage: 'id'
description: 'Edit the repository that matches the given ID prefix.'
flags: [
cli.Flag{
name: 'url'
description: 'URL of the Git repository.'
flag: cli.FlagType.string
},
cli.Flag{
name: 'branch'
description: 'Branch of the Git repository.'
flag: cli.FlagType.string
},
cli.Flag{
name: 'repo'
description: 'Repo to publish builds to.'
flag: cli.FlagType.string
},
cli.Flag{
name: 'arch'
description: 'Comma-separated list of architectures to build on.'
flag: cli.FlagType.string
},
]
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
found := cmd.flags.get_all_found()
mut params := map[string]string{}
for f in found {
if f.name != 'config-file' {
params[f.name] = f.get_string() ?
}
}
patch(conf, cmd.args[0], params) ?
}
},
]
}
}
fn get_repo_id_by_prefix(conf Config, id_prefix string) ?string {
repos := get_repos(conf.address, conf.api_key) ?
mut res := []string{}
for id, _ in repos {
if id.starts_with(id_prefix) {
res << id
}
}
if res.len == 0 {
return error('No repo found for given prefix.')
}
if res.len > 1 {
return error('Multiple repos found for given prefix.')
}
return res[0]
}
fn list(conf Config) ? {
repos := get_repos(conf.address, conf.api_key) ?
for id, details in repos {
println('${id[..8]}\t$details.url\t$details.branch\t$details.repo\t$details.arch')
}
}
fn add(conf Config, url string, branch string, repo string, arch []string) ? {
res := add_repo(conf.address, conf.api_key, url, branch, repo, arch) ?
println(res.message)
}
fn remove(conf Config, id_prefix string) ? {
id := get_repo_id_by_prefix(conf, id_prefix) ?
res := remove_repo(conf.address, conf.api_key, id) ?
println(res.message)
}
fn patch(conf Config, id_prefix string, params map[string]string) ? {
id := get_repo_id_by_prefix(conf, id_prefix) ?
res := patch_repo(conf.address, conf.api_key, id, params) ?
println(res.message)
}

61
src/git/client.v 100644
View File

@ -0,0 +1,61 @@
module git
import json
import response { Response }
import net.http
fn send_request<T>(method http.Method, address string, url string, api_key string, params map[string]string) ?Response<T> {
mut full_url := '$address$url'
if params.len > 0 {
params_str := params.keys().map('$it=${params[it]}').join('&')
full_url = '$full_url?$params_str'
}
mut req := http.new_request(method, full_url, '') ?
req.add_custom_header('X-API-Key', api_key) ?
res := req.do() ?
data := json.decode(Response<T>, res.text) ?
return data
}
// get_repos returns the current list of repos.
pub fn get_repos(address string, api_key string) ?map[string]GitRepo {
data := send_request<map[string]GitRepo>(http.Method.get, address, '/api/repos', api_key,
{}) ?
return data.data
}
// add_repo adds a new repo to the server.
pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response<string> {
params := {
'url': url
'branch': branch
'repo': repo
'arch': arch.join(',')
}
data := send_request<string>(http.Method.post, address, '/api/repos', api_key, params) ?
return data
}
// remove_repo removes the repo with the given ID from the server.
pub fn remove_repo(address string, api_key string, id string) ?Response<string> {
data := send_request<string>(http.Method.delete, address, '/api/repos/$id', api_key,
{}) ?
return data
}
// patch_repo sends a PATCH request to the given repo with the params as
// payload.
pub fn patch_repo(address string, api_key string, id string, params map[string]string) ?Response<string> {
data := send_request<string>(http.Method.patch, address, '/api/repos/$id', api_key,
params) ?
return data
}

82
src/git/git.v 100644
View File

@ -0,0 +1,82 @@
module git
import os
import json
pub struct GitRepo {
pub mut:
// URL of the Git repository
url string
// Branch of the Git repository to use
branch string
// On which architectures the package is allowed to be built. In reality,
// this controls which builders will periodically build the image.
arch []string
// Which repo the builder should publish packages to
repo string
}
// patch_from_params patches a GitRepo from a map[string]string, usually
// provided from a web.App's params
pub fn (mut r GitRepo) patch_from_params(params map[string]string) {
$for field in GitRepo.fields {
if field.name in params {
$if field.typ is string {
r.$(field.name) = params[field.name]
// This specific type check is needed for the compiler to ensure
// our types are correct
} $else $if field.typ is []string {
r.$(field.name) = params[field.name].split(',')
}
}
}
}
// read_repos reads the provided path & parses it into a map of GitRepo's.
pub fn read_repos(path string) ?map[string]GitRepo {
if !os.exists(path) {
mut f := os.create(path) ?
defer {
f.close()
}
f.write_string('{}') ?
return {}
}
content := os.read_file(path) ?
res := json.decode(map[string]GitRepo, content) ?
return res
}
// write_repos writes a map of GitRepo's back to disk given the provided path.
pub fn write_repos(path string, repos &map[string]GitRepo) ? {
mut f := os.create(path) ?
defer {
f.close()
}
value := json.encode(repos)
f.write_string(value) ?
}
// repo_from_params creates a GitRepo from a map[string]string, usually
// provided from a web.App's params
pub fn repo_from_params(params map[string]string) ?GitRepo {
mut repo := GitRepo{}
// If we're creating a new GitRepo, we want all fields to be present before
// "patching".
$for field in GitRepo.fields {
if field.name !in params {
return error('Missing parameter: ${field.name}.')
}
}
repo.patch_from_params(params)
return repo
}

120
src/main.v
View File

@ -1,103 +1,33 @@
module main
import web
import os
import log
import io
import repo
const port = 8000
const buf_size = 1_000_000
const db_name = 'pieter.db'
struct App {
web.Context
pub:
api_key string [required; web_global]
dl_dir string [required; web_global]
pub mut:
repo repo.Repo [required; web_global]
}
[noreturn]
fn exit_with_message(code int, msg string) {
eprintln(msg)
exit(code)
}
fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? {
mut file := os.create(path) ?
defer {
file.close()
}
mut buf := []byte{len: buf_size}
mut bytes_left := length
// Repeat as long as the stream still has data
for bytes_left > 0 {
// TODO check if just breaking here is safe
bytes_read := reader.read(mut buf) or { break }
bytes_left -= bytes_read
mut to_write := bytes_read
for to_write > 0 {
// TODO don't just loop infinitely here
bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue }
to_write = to_write - bytes_written
}
}
}
import server
import cli
import build
import git
fn main() {
// Configure logger
log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' }
log_level := log.level_from_tag(log_level_str) or {
exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
}
log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' }
mut logger := log.Log{
level: log_level
mut app := cli.Command{
name: 'vieter'
description: 'Vieter is a lightweight implementation of an Arch repository server.'
version: '0.2.0'
flags: [
cli.Flag{
flag: cli.FlagType.string
name: 'config-file'
abbrev: 'f'
description: 'Location of Vieter config file; defaults to ~/.vieterrc.'
global: true
default_value: [os.expand_tilde_to_home('~/.vieterrc')]
},
]
commands: [
server.cmd(),
build.cmd(),
git.cmd(),
]
}
logger.set_full_logpath(log_file)
logger.log_to_console_too()
defer {
logger.info('Flushing log file')
logger.flush()
logger.close()
}
// Configure web server
key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') }
repo_dir := os.getenv_opt('REPO_DIR') or {
exit_with_message(1, 'No repo directory was configured.')
}
pkg_dir := os.getenv_opt('PKG_DIR') or {
exit_with_message(1, 'No package directory was configured.')
}
dl_dir := os.getenv_opt('DOWNLOAD_DIR') or {
exit_with_message(1, 'No download directory was configured.')
}
// This also creates the directories if needed
repo := repo.new(repo_dir, pkg_dir) or {
logger.error(err.msg)
exit(1)
}
os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') }
web.run(&App{
logger: logger
api_key: key
dl_dir: dl_dir
repo: repo
}, port)
app.setup()
app.parse(os.args)
}

16
src/package.v
View File

@ -72,14 +72,11 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo {
'pkgbase' { pkg_info.base = value }
'pkgver' { pkg_info.version = value }
'pkgdesc' { pkg_info.description = value }
'csize' { continue }
'size' { pkg_info.size = value.int() }
'url' { pkg_info.url = value }
'arch' { pkg_info.arch = value }
'builddate' { pkg_info.build_date = value.int() }
'packager' { pkg_info.packager = value }
'md5sum' { continue }
'sha256sum' { continue }
'pgpsig' { pkg_info.pgpsig = value }
'pgpsigsize' { pkg_info.pgpsigsize = value.int() }
// Array values
@ -92,16 +89,19 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo {
'optdepend' { pkg_info.optdepends << value }
'makedepend' { pkg_info.makedepends << value }
'checkdepend' { pkg_info.checkdepends << value }
else { return error("Invalid key '$key'.") }
// There's no real point in trying to exactly manage which fields
// are allowed, so we just ignore any we don't explicitely need for
// in the db file
else { continue }
}
}
return pkg_info
}
// read_pkg extracts the file list & .PKGINFO contents from an archive
// NOTE: this command currently only supports zstd-compressed tarballs
pub fn read_pkg(pkg_path string) ?Pkg {
// read_pkg_archive extracts the file list & .PKGINFO contents from an archive
// NOTE: this command only supports zstd-, xz- & gzip-compressed tarballs.
pub fn read_pkg_archive(pkg_path string) ?Pkg {
if !os.is_file(pkg_path) {
return error("'$pkg_path' doesn't exist or isn't a file.")
}
@ -112,6 +112,7 @@ pub fn read_pkg(pkg_path string) ?Pkg {
// Sinds 2020, all newly built Arch packages use zstd
C.archive_read_support_filter_zstd(a)
C.archive_read_support_filter_gzip(a)
C.archive_read_support_filter_xz(a)
// The content should always be a tarball
C.archive_read_support_format_tar(a)
@ -190,6 +191,7 @@ pub fn (pkg &Pkg) filename() string {
ext := match pkg.compression {
0 { '.tar' }
1 { '.tar.gz' }
6 { '.tar.xz' }
14 { '.tar.zst' }
else { panic("Another compression code shouldn't be possible. Faulty code: $pkg.compression") }
}

198
src/repo/repo.v
View File

@ -2,22 +2,23 @@ module repo
import os
import package
import util
// Dummy struct to work around the fact that you can only share structs, maps &
// arrays
pub struct Dummy {
x int
}
// This struct manages a single repository.
pub struct Repo {
// Manages a group of repositories. Each repository contains one or more
// arch-repositories, each of which represent a specific architecture.
pub struct RepoGroupManager {
mut:
mutex shared Dummy
mutex shared util.Dummy
pub:
// Where to store repository files
repo_dir string [required]
// Where to find packages; packages are expected to all be in the same directory
// Where to store repositories' files
repos_dir string [required]
// Where packages are stored; each arch-repository gets its own
// subdirectory
pkg_dir string [required]
// The default architecture to use for a repository. Whenever a package of
// arch "any" is added to a repo, it will also be added to this
// architecture.
default_arch string [required]
}
pub struct RepoAddResult {
@ -26,55 +27,117 @@ pub:
pkg &package.Pkg [required]
}
// new creates a new Repo & creates the directories as needed
pub fn new(repo_dir string, pkg_dir string) ?Repo {
if !os.is_dir(repo_dir) {
os.mkdir_all(repo_dir) or { return error('Failed to create repo directory: $err.msg') }
// new creates a new RepoGroupManager & creates the directories as needed
pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupManager {
if !os.is_dir(repos_dir) {
os.mkdir_all(repos_dir) or { return error('Failed to create repos directory: $err.msg') }
}
if !os.is_dir(pkg_dir) {
os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg') }
}
return Repo{
repo_dir: repo_dir
return RepoGroupManager{
repos_dir: repos_dir
pkg_dir: pkg_dir
default_arch: default_arch
}
}
// add_from_path adds a package from an arbitrary path & moves it into the pkgs
// directory if necessary.
pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult {
pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') }
added := r.add(pkg) ?
// If the add was successful, we move the file to the packages directory
if added {
dest_path := os.real_path(os.join_path_single(r.pkg_dir, pkg.filename()))
// Only move the file if it's not already in the package directory
if dest_path != os.real_path(pkg_path) {
os.mv(pkg_path, dest_path) ?
}
// add_pkg_from_path adds a package to a given repo, given the file path to the
// pkg archive. It's a wrapper around add_pkg_in_repo that parses the archive
// file, passes the result to add_pkg_in_repo, and hard links the archive to
// the right subdirectories in r.pkg_dir if it was successfully added.
pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult {
pkg := package.read_pkg_archive(pkg_path) or {
return error('Failed to read package file: $err.msg')
}
added := r.add_pkg_in_repo(repo, pkg) ?
// If the add was successful, we move the file to the packages directory
for arch in added {
repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch))
dest_path := os.join_path_single(repo_pkg_path, pkg.filename())
os.mkdir_all(repo_pkg_path) ?
// We create hard links so that "any" arch packages aren't stored
// multiple times
os.link(pkg_path, dest_path) ?
}
// After linking, we can remove the original file
os.rm(pkg_path) ?
return RepoAddResult{
added: added
added: added.len > 0
pkg: &pkg
}
}
// add adds a given Pkg to the repository
fn (r &Repo) add(pkg &package.Pkg) ?bool {
pkg_dir := r.pkg_path(pkg)
// We can't add the same package twice
if os.exists(pkg_dir) {
return false
// add_pkg_in_repo adds a package to a given repo. This function is responsible
// for inspecting the package architecture. If said architecture is 'any', the
// package is added to each arch-repository within the given repo. A package of
// architecture 'any' is always added to the arch-repo defined by
// r.default_arch. If this arch-repo doesn't exist yet, it is created. If the
// architecture isn't 'any', the package is only added to the specific
// architecture.
fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]string {
// A package not of arch 'any' can be handled easily by adding it to the
// respective repo
if pkg.info.arch != 'any' {
if r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) ? {
return [pkg.info.arch]
} else {
return []
}
}
os.mkdir(pkg_dir) or { return error('Failed to create package directory.') }
mut arch_repos := []string{}
// If it is an "any" package, the package gets added to every currently
// present arch-repo. It will always get added to the r.default_arch repo,
// even if no or multiple others are present.
repo_dir := os.join_path_single(r.repos_dir, repo)
// If this is the first package that's added to the repo, the directory
// won't exist yet
if os.exists(repo_dir) {
arch_repos = os.ls(repo_dir) ?
}
// The default_arch should always be updated when a package with arch 'any'
// is added.
if !arch_repos.contains(r.default_arch) {
arch_repos << r.default_arch
}
mut added := []string{}
// We add the package to each repository. If any of the repositories
// return true, the result of the function is also true.
for arch in arch_repos {
if r.add_pkg_in_arch_repo(repo, arch, pkg) ? {
added << arch
}
}
return added
}
// add_pkg_in_arch_repo is the function that actually adds a package to a given
// arch-repo. It records the package's data in the arch-repo's desc & files
// files, and afterwards updates the db & files archives to reflect these
// changes. The function returns false if the package was already present in
// the repo, and true otherwise.
fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool {
pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version')
// Remove the previous version of the package, if present
r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false) ?
os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') }
os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or {
os.rmdir_all(pkg_dir) ?
@ -87,12 +150,55 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool {
return error('Failed to write files file.')
}
r.sync() ?
r.sync(repo, arch) ?
return true
}
// Returns the path where the given package's desc & files files are stored
fn (r &Repo) pkg_path(pkg &package.Pkg) string {
return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version')
// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It
// returns false if the package wasn't present in the database. It also
// optionally re-syncs the repo archives.
fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool {
repo_dir := os.join_path(r.repos_dir, repo, arch)
// If the repository doesn't exist yet, the result is automatically false
if !os.exists(repo_dir) {
return false
}
// We iterate over every directory in the repo dir
// TODO filter so we only check directories
for d in os.ls(repo_dir) ? {
// Because a repository only allows a single version of each package,
// we need only compare whether the name of the package is the same,
// not the version.
name := d.split('-')#[..-2].join('-')
if name == pkg_name {
// We lock the mutex here to prevent other routines from creating a
// new archive while we remove an entry
lock r.mutex {
os.rmdir_all(os.join_path_single(repo_dir, d)) ?
}
// Also remove the package archive
repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch)
archives := os.ls(repo_pkg_dir) ?.filter(it.split('-')#[..-3].join('-') == name)
for archive_name in archives {
full_path := os.join_path_single(repo_pkg_dir, archive_name)
os.rm(full_path) ?
}
// Sync the db archives if requested
if sync {
r.sync(repo, arch) ?
}
return true
}
}
return false
}

15
src/repo/sync.v
View File

@ -30,8 +30,9 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri
}
// Re-generate the repo archive files
fn (r &Repo) sync() ? {
// TODO also write files archive
fn (r &RepoGroupManager) sync(repo string, arch string) ? {
subrepo_path := os.join_path(r.repos_dir, repo, arch)
lock r.mutex {
a_db := C.archive_write_new()
a_files := C.archive_write_new()
@ -44,18 +45,18 @@ fn (r &Repo) sync() ? {
C.archive_write_add_filter_gzip(a_files)
C.archive_write_set_format_pax_restricted(a_files)
db_path := os.join_path_single(r.repo_dir, 'vieter.db.tar.gz')
files_path := os.join_path_single(r.repo_dir, 'vieter.files.tar.gz')
db_path := os.join_path_single(subrepo_path, '${repo}.db.tar.gz')
files_path := os.join_path_single(subrepo_path, '${repo}.files.tar.gz')
C.archive_write_open_filename(a_db, &char(db_path.str))
C.archive_write_open_filename(a_files, &char(files_path.str))
// Iterate over each directory
for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir,
for d in os.ls(subrepo_path) ?.filter(os.is_dir(os.join_path_single(subrepo_path,
it))) {
// desc
mut inner_path := os.join_path_single(d, 'desc')
mut actual_path := os.join_path_single(r.repo_dir, inner_path)
mut actual_path := os.join_path_single(subrepo_path, inner_path)
archive_add_entry(a_db, entry, actual_path, inner_path)
archive_add_entry(a_files, entry, actual_path, inner_path)
@ -64,7 +65,7 @@ fn (r &Repo) sync() ? {
// files
inner_path = os.join_path_single(d, 'files')
actual_path = os.join_path_single(r.repo_dir, inner_path)
actual_path = os.join_path_single(subrepo_path, inner_path)
archive_add_entry(a_files, entry, actual_path, inner_path)

34
src/response.v 100644
View File

@ -0,0 +1,34 @@
module response
pub struct Response<T> {
pub:
message string
data T
}
// new_response constructs a new Response<String> object with the given message
// & an empty data field.
pub fn new_response(message string) Response<string> {
return Response<string>{
message: message
data: ''
}
}
// new_data_response<T> constructs a new Response<T> object with the given data
// & an empty message field.
pub fn new_data_response<T>(data T) Response<T> {
return Response<T>{
message: ''
data: data
}
}
// new_full_response<T> constructs a new Response<T> object with the given
// message & data.
pub fn new_full_response<T>(message string, data T) Response<T> {
return Response<T>{
message: message
data: data
}
}

103
src/routes.v
View File

@ -1,103 +0,0 @@
module main
import web
import os
import repo
import time
import rand
const prefixes = ['B', 'KB', 'MB', 'GB']
// pretty_bytes converts a byte count to human-readable version
fn pretty_bytes(bytes int) string {
mut i := 0
mut n := f32(bytes)
for n >= 1024 {
i++
n /= 1024
}
return '${n:.2}${prefixes[i]}'
}
fn is_pkg_name(s string) bool {
return s.contains('.pkg')
}
// healthcheck just returns a string, but can be used to quickly check if the
// server is still responsive.
['/health'; get]
pub fn (mut app App) healthcheck() web.Result {
return app.text('Healthy')
}
// get_root handles a GET request for a file on the root
['/:filename'; get]
fn (mut app App) get_root(filename string) web.Result {
mut full_path := ''
if filename.ends_with('.db') || filename.ends_with('.files') {
full_path = os.join_path_single(app.repo.repo_dir, '${filename}.tar.gz')
} else if filename.ends_with('.db.tar.gz') || filename.ends_with('.files.tar.gz') {
full_path = os.join_path_single(app.repo.repo_dir, '$filename')
} else {
full_path = os.join_path_single(app.repo.pkg_dir, filename)
}
return app.file(full_path)
}
['/publish'; post]
fn (mut app App) put_package() web.Result {
if !app.is_authorized() {
return app.text('Unauthorized.')
}
mut pkg_path := ''
if length := app.req.header.get(.content_length) {
// Generate a random filename for the temp file
pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4())
for os.exists(pkg_path) {
pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4())
}
app.ldebug("Uploading $length bytes (${pretty_bytes(length.int())}) to '$pkg_path'.")
// This is used to time how long it takes to upload a file
mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true })
reader_to_file(mut app.reader, length.int(), pkg_path) or {
app.lwarn("Failed to upload '$pkg_path'")
return app.text('Failed to upload file.')
}
sw.stop()
app.ldebug("Upload of '$pkg_path' completed in ${sw.elapsed().seconds():.3}s.")
} else {
app.lwarn('Tried to upload package without specifying a Content-Length.')
return app.text("Content-Type header isn't set.")
}
res := app.repo.add_from_path(pkg_path) or {
app.lerror('Error while adding package: $err.msg')
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }
return app.text('Failed to add package.')
}
if !res.added {
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }
app.lwarn("Duplicate package '$res.pkg.full_name()'.")
return app.text('File already exists.')
}
app.linfo("Added '$res.pkg.full_name()' to repository.")
return app.text('Package added successfully.')
}

4
src/auth.v → src/server/auth.v
View File

@ -1,4 +1,4 @@
module main
module server
import net.http
@ -7,5 +7,5 @@ fn (mut app App) is_authorized() bool {
return false
}
return x_header.trim_space() == app.api_key
return x_header.trim_space() == app.conf.api_key
}

30
src/server/cli.v 100644
View File

@ -0,0 +1,30 @@
module server
import cli
import env
struct Config {
pub:
log_level string = 'WARN'
log_file string = 'vieter.log'
pkg_dir string
download_dir string
api_key string
repos_dir string
repos_file string
default_arch string
}
// cmd returns the cli submodule that handles starting the server
pub fn cmd() cli.Command {
return cli.Command{
name: 'server'
description: 'Start the Vieter server.'
execute: fn (cmd cli.Command) ? {
config_file := cmd.flags.get_string('config-file') ?
conf := env.load<Config>(config_file) ?
server(conf) ?
}
}
}

141
src/server/git.v 100644
View File

@ -0,0 +1,141 @@
module server
import web
import git
import net.http
import rand
import response { new_data_response, new_response }
const repos_file = 'repos.json'
['/api/repos'; get]
fn (mut app App) get_repos() web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
repos := rlock app.git_mutex {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file: $err.msg')
return app.status(http.Status.internal_server_error)
}
}
return app.json(http.Status.ok, new_data_response(repos))
}
['/api/repos/:id'; get]
fn (mut app App) get_single_repo(id string) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
repos := rlock app.git_mutex {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
return app.status(http.Status.internal_server_error)
}
}
if id !in repos {
return app.not_found()
}
repo := repos[id]
return app.json(http.Status.ok, new_data_response(repo))
}
['/api/repos'; post]
fn (mut app App) post_repo() web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
new_repo := git.repo_from_params(app.query) or {
return app.json(http.Status.bad_request, new_response(err.msg))
}
id := rand.uuid_v4()
mut repos := rlock app.git_mutex {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
return app.status(http.Status.internal_server_error)
}
}
// We need to check for duplicates
for _, repo in repos {
if repo == new_repo {
return app.json(http.Status.bad_request, new_response('Duplicate repository.'))
}
}
repos[id] = new_repo
lock app.git_mutex {
git.write_repos(app.conf.repos_file, &repos) or {
return app.status(http.Status.internal_server_error)
}
}
return app.json(http.Status.ok, new_response('Repo added successfully.'))
}
['/api/repos/:id'; delete]
fn (mut app App) delete_repo(id string) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
mut repos := rlock app.git_mutex {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
return app.status(http.Status.internal_server_error)
}
}
if id !in repos {
return app.not_found()
}
repos.delete(id)
lock app.git_mutex {
git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) }
}
return app.json(http.Status.ok, new_response('Repo removed successfully.'))
}
['/api/repos/:id'; patch]
fn (mut app App) patch_repo(id string) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
mut repos := rlock app.git_mutex {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
return app.status(http.Status.internal_server_error)
}
}
if id !in repos {
return app.not_found()
}
repos[id].patch_from_params(app.query)
lock app.git_mutex {
git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) }
}
return app.json(http.Status.ok, new_response('Repo updated successfully.'))
}

108
src/server/routes.v 100644
View File

@ -0,0 +1,108 @@
module server
import web
import os
import repo
import time
import rand
import util
import net.http
import response { new_response }
// healthcheck just returns a string, but can be used to quickly check if the
// server is still responsive.
['/health'; get]
pub fn (mut app App) healthcheck() web.Result {
return app.json(http.Status.ok, new_response('Healthy.'))
}
['/:repo/:arch/:filename'; get; head]
fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result {
mut full_path := ''
db_exts := ['.db', '.files', '.db.tar.gz', '.files.tar.gz']
// There's no point in having the ability to serve db archives with wrong
// filenames
if db_exts.any(filename == '$repo$it') {
full_path = os.join_path(app.repo.repos_dir, repo, arch, filename)
// repo-add does this using symlinks, but we just change the requested
// path
if !full_path.ends_with('.tar.gz') {
full_path += '.tar.gz'
}
} else if filename.contains('.pkg') {
full_path = os.join_path(app.repo.pkg_dir, repo, arch, filename)
}
// Default behavior is to return the desc file for the package, if present.
// This can then also be used by the build system to properly check whether
// a package is present in an arch-repo.
else {
full_path = os.join_path(app.repo.repos_dir, repo, arch, filename, 'desc')
}
// Scuffed way to respond to HEAD requests
if app.req.method == http.Method.head {
if os.exists(full_path) {
return app.status(http.Status.ok)
}
return app.not_found()
}
return app.file(full_path)
}
['/:repo/publish'; post]
fn (mut app App) put_package(repo string) web.Result {
if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
}
mut pkg_path := ''
if length := app.req.header.get(.content_length) {
// Generate a random filename for the temp file
pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4())
app.ldebug("Uploading $length bytes (${util.pretty_bytes(length.int())}) to '$pkg_path'.")
// This is used to time how long it takes to upload a file
mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true })
util.reader_to_file(mut app.reader, length.int(), pkg_path) or {
app.lwarn("Failed to upload '$pkg_path'")
return app.json(http.Status.internal_server_error, new_response('Failed to upload file.'))
}
sw.stop()
app.ldebug("Upload of '$pkg_path' completed in ${sw.elapsed().seconds():.3}s.")
} else {
app.lwarn('Tried to upload package without specifying a Content-Length.')
// length required
return app.status(http.Status.length_required)
}
res := app.repo.add_pkg_from_path(repo, pkg_path) or {
app.lerror('Error while adding package: $err.msg')
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }
return app.json(http.Status.internal_server_error, new_response('Failed to add package.'))
}
if !res.added {
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }
app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo'.")
return app.json(http.Status.bad_request, new_response('File already exists.'))
}
app.linfo("Added '$res.pkg.full_name()' to repo '$repo ($res.pkg.info.arch)'.")
return app.json(http.Status.ok, new_response('Package added successfully.'))
}

61
src/server/server.v 100644
View File

@ -0,0 +1,61 @@
module server
import web
import os
import log
import repo
import util
const port = 8000
struct App {
web.Context
pub:
conf Config [required; web_global]
pub mut:
repo repo.RepoGroupManager [required; web_global]
// This is used to claim the file lock on the repos file
git_mutex shared util.Dummy
}
// server starts the web server & starts listening for requests
pub fn server(conf Config) ? {
// Prevent using 'any' as the default arch
if conf.default_arch == 'any' {
util.exit_with_message(1, "'any' is not allowed as the value for default_arch.")
}
// Configure logger
log_level := log.level_from_tag(conf.log_level) or {
util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
}
mut logger := log.Log{
level: log_level
}
logger.set_full_logpath(conf.log_file)
logger.log_to_console_too()
defer {
logger.info('Flushing log file')
logger.flush()
logger.close()
}
// This also creates the directories if needed
repo := repo.new(conf.repos_dir, conf.pkg_dir, conf.default_arch) or {
logger.error(err.msg)
exit(1)
}
os.mkdir_all(conf.download_dir) or {
util.exit_with_message(1, 'Failed to create download directory.')
}
web.run(&App{
logger: logger
conf: conf
repo: repo
}, server.port)
}

59
src/util.v
View File

@ -1,9 +1,55 @@
module util
import os
import io
import crypto.md5
import crypto.sha256
const reader_buf_size = 1_000_000
const prefixes = ['B', 'KB', 'MB', 'GB']
// Dummy struct to work around the fact that you can only share structs, maps &
// arrays
pub struct Dummy {
x int
}
// exit_with_message exits the program with a given status code after having
// first printed a specific message to STDERR
[noreturn]
pub fn exit_with_message(code int, msg string) {
eprintln(msg)
exit(code)
}
// reader_to_file writes the contents of a BufferedReader to a file
pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? {
mut file := os.create(path) ?
defer {
file.close()
}
mut buf := []byte{len: util.reader_buf_size}
mut bytes_left := length
// Repeat as long as the stream still has data
for bytes_left > 0 {
// TODO check if just breaking here is safe
bytes_read := reader.read(mut buf) or { break }
bytes_left -= bytes_read
mut to_write := bytes_read
for to_write > 0 {
// TODO don't just loop infinitely here
bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue }
to_write = to_write - bytes_written
}
}
}
// hash_file returns the md5 & sha256 hash of a given file
// TODO actually implement sha256
pub fn hash_file(path &string) ?(string, string) {
@ -32,3 +78,16 @@ pub fn hash_file(path &string) ?(string, string) {
return md5sum.checksum().hex(), sha256sum.checksum().hex()
}
// pretty_bytes converts a byte count to human-readable version
pub fn pretty_bytes(bytes int) string {
mut i := 0
mut n := f32(bytes)
for n >= 1024 {
i++
n /= 1024
}
return '${n:.2}${util.prefixes[i]}'
}

6
src/web/parse.v
View File

@ -34,7 +34,7 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) {
}
if x.len > 0 {
return IError(http.UnexpectedExtraAttributeError{
msg: 'Encountered unexpected extra attributes: $x'
attributes: x
})
}
if methods.len == 0 {
@ -49,8 +49,8 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) {
fn parse_query_from_url(url urllib.URL) map[string]string {
mut query := map[string]string{}
for k, v in url.query().data {
query[k] = v.data[0]
for v in url.query().data {
query[v.key] = v.value
}
return query
}

210
src/web/web.v
View File

@ -12,9 +12,6 @@ import time
import json
import log
// A type which don't get filtered inside templates
pub type RawHtml = string
// A dummy structure that returns from routes to indicate that you actually sent something to a user
[noinit]
pub struct Result {}
@ -22,7 +19,7 @@ pub struct Result {}
pub const (
methods_with_form = [http.Method.post, .put, .patch]
headers_close = http.new_custom_header_from_map({
'Server': 'VWeb'
'Server': 'VWeb'
http.CommonHeader.connection.str(): 'close'
}) or { panic('should never fail') }
@ -141,8 +138,8 @@ pub const (
// It has fields for the query, form, files.
pub struct Context {
mut:
content_type string = 'text/plain'
status string = '200 OK'
content_type string = 'text/plain'
status http.Status = http.Status.ok
pub:
// HTTP Request
req http.Request
@ -186,24 +183,14 @@ struct Route {
path string
}
// Defining this method is optional.
// init_server is called at server start.
// You can use it for initializing globals.
pub fn (ctx Context) init_server() {
eprintln('init_server() has been deprecated, please init your web app in `fn main()`')
}
// Defining this method is optional.
// before_request is called before every request (aka middleware).
// Probably you can use it for check user session cookie or add header.
pub fn (ctx Context) before_request() {}
pub struct Cookie {
name string
value string
expires time.Time
secure bool
http_only bool
// send_string
fn send_string(mut conn net.TcpConn, s string) ? {
conn.write(s.bytes()) ?
}
// send_response_to_client sends a response to the client
@ -225,34 +212,27 @@ pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bo
text: res
}
resp.set_version(.v1_1)
resp.set_status(http.status_from_int(ctx.status.int()))
resp.set_status(ctx.status)
send_string(mut ctx.conn, resp.bytestr()) or { return false }
return true
}
// html HTTP_OK with s as payload with content-type `text/html`
pub fn (mut ctx Context) html(s string) Result {
ctx.send_response_to_client('text/html', s)
return Result{}
}
// text responds to a request with some plaintext.
pub fn (mut ctx Context) text(status http.Status, s string) Result {
ctx.status = status
// text HTTP_OK with s as payload with content-type `text/plain`
pub fn (mut ctx Context) text(s string) Result {
ctx.send_response_to_client('text/plain', s)
return Result{}
}
// json<T> HTTP_OK with json_s as payload with content-type `application/json`
pub fn (mut ctx Context) json<T>(j T) Result {
pub fn (mut ctx Context) json<T>(status http.Status, j T) Result {
ctx.status = status
json_s := json.encode(j)
ctx.send_response_to_client('application/json', json_s)
return Result{}
}
// json_pretty<T> Response HTTP_OK with a pretty-printed JSON result
pub fn (mut ctx Context) json_pretty<T>(j T) Result {
json_s := json.encode_pretty(j)
ctx.send_response_to_client('application/json', json_s)
return Result{}
}
@ -302,7 +282,7 @@ pub fn (mut ctx Context) file(f_path string) Result {
header: header.join(web.headers_close)
}
resp.set_version(.v1_1)
resp.set_status(http.status_from_int(ctx.status.int()))
resp.set_status(ctx.status)
send_string(mut ctx.conn, resp.bytestr()) or { return Result{} }
mut buf := []byte{len: 1_000_000}
@ -328,10 +308,10 @@ pub fn (mut ctx Context) file(f_path string) Result {
return Result{}
}
// ok Response HTTP_OK with s as payload
pub fn (mut ctx Context) ok(s string) Result {
ctx.send_response_to_client(ctx.content_type, s)
return Result{}
// status responds with an empty textual response, essentially only returning
// the given status code.
pub fn (mut ctx Context) status(status http.Status) Result {
return ctx.text(status, '')
}
// server_error Response a server error
@ -361,64 +341,7 @@ pub fn (mut ctx Context) redirect(url string) Result {
// not_found Send an not_found response
pub fn (mut ctx Context) not_found() Result {
if ctx.done {
return Result{}
}
ctx.done = true
send_string(mut ctx.conn, web.http_404.bytestr()) or {}
return Result{}
}
// set_cookie Sets a cookie
pub fn (mut ctx Context) set_cookie(cookie Cookie) {
mut cookie_data := []string{}
mut secure := if cookie.secure { 'Secure;' } else { '' }
secure += if cookie.http_only { ' HttpOnly' } else { ' ' }
cookie_data << secure
if cookie.expires.unix > 0 {
cookie_data << 'expires=$cookie.expires.utc_string()'
}
data := cookie_data.join(' ')
ctx.add_header('Set-Cookie', '$cookie.name=$cookie.value; $data')
}
// set_content_type Sets the response content type
pub fn (mut ctx Context) set_content_type(typ string) {
ctx.content_type = typ
}
// set_cookie_with_expire_date Sets a cookie with a `expire_data`
pub fn (mut ctx Context) set_cookie_with_expire_date(key string, val string, expire_date time.Time) {
ctx.add_header('Set-Cookie', '$key=$val; Secure; HttpOnly; expires=$expire_date.utc_string()')
}
// get_cookie Gets a cookie by a key
pub fn (ctx &Context) get_cookie(key string) ?string { // TODO refactor
mut cookie_header := ctx.get_header('cookie')
if cookie_header == '' {
cookie_header = ctx.get_header('Cookie')
}
cookie_header = ' ' + cookie_header
// println('cookie_header="$cookie_header"')
// println(ctx.req.header)
cookie := if cookie_header.contains(';') {
cookie_header.find_between(' $key=', ';')
} else {
cookie_header.find_between(' $key=', '\r')
}
if cookie != '' {
return cookie.trim_space()
}
return error('Cookie not found')
}
// set_status Sets the response status
pub fn (mut ctx Context) set_status(code int, desc string) {
if code < 100 || code > 599 {
ctx.status = '500 Internal Server Error'
} else {
ctx.status = '$code $desc'
}
return ctx.status(http.Status.not_found)
}
// add_header Adds an header to the response with key and val
@ -560,12 +483,6 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T, routes map[string]Route) {
// Calling middleware...
app.before_request()
// Static handling
if serve_if_static<T>(mut app, url) {
// successfully served a static file
return
}
// Route matching
$for method in T.methods {
$if method.return_type is Result {
@ -661,83 +578,6 @@ fn route_matches(url_words []string, route_words []string) ?[]string {
return params
}
// serve_if_static<T> checks if request is for a static file and serves it
// returns true if we served a static file, false otherwise
[manualfree]
fn serve_if_static<T>(mut app T, url urllib.URL) bool {
// TODO: handle url parameters properly - for now, ignore them
static_file := app.static_files[url.path]
mime_type := app.static_mime_types[url.path]
if static_file == '' || mime_type == '' {
return false
}
data := os.read_file(static_file) or {
send_string(mut app.conn, web.http_404.bytestr()) or {}
return true
}
app.send_response_to_client(mime_type, data)
unsafe { data.free() }
return true
}
// scan_static_directory makes a static route for each file in a directory
fn (mut ctx Context) scan_static_directory(directory_path string, mount_path string) {
files := os.ls(directory_path) or { panic(err) }
if files.len > 0 {
for file in files {
full_path := os.join_path(directory_path, file)
if os.is_dir(full_path) {
ctx.scan_static_directory(full_path, mount_path + '/' + file)
} else if file.contains('.') && !file.starts_with('.') && !file.ends_with('.') {
ext := os.file_ext(file)
// Rudimentary guard against adding files not in mime_types.
// Use serve_static directly to add non-standard mime types.
if ext in web.mime_types {
ctx.serve_static(mount_path + '/' + file, full_path)
}
}
}
}
}
// handle_static Handles a directory static
// If `root` is set the mount path for the dir will be in '/'
pub fn (mut ctx Context) handle_static(directory_path string, root bool) bool {
if ctx.done || !os.exists(directory_path) {
return false
}
dir_path := directory_path.trim_space().trim_right('/')
mut mount_path := ''
if dir_path != '.' && os.is_dir(dir_path) && !root {
// Mount point hygene, "./assets" => "/assets".
mount_path = '/' + dir_path.trim_left('.').trim('/')
}
ctx.scan_static_directory(dir_path, mount_path)
return true
}
// mount_static_folder_at - makes all static files in `directory_path` and inside it, available at http://server/mount_path
// For example: suppose you have called .mount_static_folder_at('/var/share/myassets', '/assets'),
// and you have a file /var/share/myassets/main.css .
// => That file will be available at URL: http://server/assets/main.css .
pub fn (mut ctx Context) mount_static_folder_at(directory_path string, mount_path string) bool {
if ctx.done || mount_path.len < 1 || mount_path[0] != `/` || !os.exists(directory_path) {
return false
}
dir_path := directory_path.trim_right('/')
ctx.scan_static_directory(dir_path, mount_path[1..])
return true
}
// serve_static Serves a file static
// `url` is the access path on the site, `file_path` is the real path to the file, `mime_type` is the file type
pub fn (mut ctx Context) serve_static(url string, file_path string) {
ctx.static_files[url] = file_path
// ctx.static_mime_types[url] = mime_type
ext := os.file_ext(file_path)
ctx.static_mime_types[url] = web.mime_types[ext]
}
// ip Returns the ip address from the current user
pub fn (ctx &Context) ip() string {
mut ip := ctx.req.header.get(.x_forwarded_for) or { '' }
@ -760,16 +600,6 @@ pub fn (mut ctx Context) error(s string) {
ctx.form_error = s
}
// not_found Returns an empty result
pub fn not_found() Result {
return Result{}
}
// send_string
fn send_string(mut conn net.TcpConn, s string) ? {
conn.write(s.bytes()) ?
}
// filter Do not delete.
// It used by `vlib/v/gen/c/str_intp.v:130` for string interpolation inside web templates
// TODO: move it to template render

4
test.py
View File

@ -38,7 +38,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len):
Generates a random .PKGINFO
"""
name = "-".join(random_words(words, name_min_len, name_max_len))
ver = "0.1.0-1" # doesn't matter what it is anyways
ver = "0.1.0-3" # doesn't matter what it is anyways
# TODO add random dependencies (all types)
@ -97,7 +97,7 @@ async def upload_random_package(tar_path, sem):
async with sem:
with open(tar_path, 'rb') as f:
async with aiohttp.ClientSession() as s:
async with s.post("http://localhost:8000/publish", data=f.read(), headers={"x-api-key": "test"}) as r:
async with s.post("http://localhost:8000/vieter/publish", data=f.read(), headers={"x-api-key": "test"}) as r:
return await check_output(r)

10
vieter.toml 100644
View File

@ -0,0 +1,10 @@
# This file contains settings used during development
api_key = "test"
download_dir = "data/downloads"
repos_dir = "data/repos"
pkg_dir = "data/pkgs"
log_level = "DEBUG"
repos_file = "data/repos.json"
default_arch = "x86_64"
address = "http://localhost:8000"