forked from vieter-v/vieter
Merge pull request 'Release 0.2.0' (#121) from release-0.2.0 into main
Reviewed-on: Chewing_Bever/vieter#121main 0.2.0
commit
4b6a661d71
|
|
@ -1,4 +1,3 @@
|
||||||
# top-most EditorConfig file
|
|
||||||
root = true
|
root = true
|
||||||
|
|
||||||
# Unix-style newlines with a newline ending every file
|
# Unix-style newlines with a newline ending every file
|
||||||
|
|
@ -7,4 +6,5 @@ end_of_line = lf
|
||||||
insert_final_newline = true
|
insert_final_newline = true
|
||||||
|
|
||||||
[*.v]
|
[*.v]
|
||||||
indent_style = space
|
# vfmt wants it :(
|
||||||
|
indent_style = tab
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,8 @@ data/
|
||||||
vieter
|
vieter
|
||||||
dvieter
|
dvieter
|
||||||
pvieter
|
pvieter
|
||||||
|
dvieterctl
|
||||||
|
vieterctl
|
||||||
vieter.c
|
vieter.c
|
||||||
|
|
||||||
# Ignore testing files
|
# Ignore testing files
|
||||||
|
|
@ -17,4 +19,7 @@ libarchive-*
|
||||||
test/
|
test/
|
||||||
|
|
||||||
# V compiler directory
|
# V compiler directory
|
||||||
v-*/
|
v/
|
||||||
|
|
||||||
|
# gdb log file
|
||||||
|
gdb.txt
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
matrix:
|
||||||
|
PLATFORM:
|
||||||
|
- linux/amd64
|
||||||
|
- linux/arm64
|
||||||
|
|
||||||
|
platform: ${PLATFORM}
|
||||||
|
branches: [dev]
|
||||||
|
|
||||||
|
pipeline:
|
||||||
|
build:
|
||||||
|
image: 'menci/archlinuxarm:base-devel'
|
||||||
|
commands:
|
||||||
|
# Update packages
|
||||||
|
- pacman -Syu --noconfirm
|
||||||
|
# Create non-root user to perform build & switch to their home
|
||||||
|
- groupadd -g 1000 builder
|
||||||
|
- useradd -mg builder builder
|
||||||
|
- chown -R builder:builder "$PWD"
|
||||||
|
- "echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers"
|
||||||
|
- su builder
|
||||||
|
# Build the package
|
||||||
|
- makepkg -s --noconfirm --needed
|
||||||
|
|
||||||
|
publish:
|
||||||
|
image: 'curlimages/curl'
|
||||||
|
commands:
|
||||||
|
# Publish the package
|
||||||
|
- 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done'
|
||||||
|
secrets:
|
||||||
|
- vieter_api_key
|
||||||
|
|
@ -2,7 +2,8 @@ matrix:
|
||||||
PLATFORM:
|
PLATFORM:
|
||||||
- linux/amd64
|
- linux/amd64
|
||||||
- linux/arm64
|
- linux/arm64
|
||||||
- linux/arm/v7
|
# I just don't have a performant enough runner for this platform
|
||||||
|
# - linux/arm/v7
|
||||||
|
|
||||||
# These checks already get performed on the feature branches
|
# These checks already get performed on the feature branches
|
||||||
platform: ${PLATFORM}
|
platform: ${PLATFORM}
|
||||||
|
|
@ -42,12 +43,12 @@ pipeline:
|
||||||
commands:
|
commands:
|
||||||
# https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f
|
# https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f
|
||||||
- export URL=s3.rustybever.be
|
- export URL=s3.rustybever.be
|
||||||
- export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')"
|
|
||||||
- export DATE="$(date -R --utc)"
|
- export DATE="$(date -R --utc)"
|
||||||
- export CONTENT_TYPE='application/zstd'
|
- export CONTENT_TYPE='application/zstd'
|
||||||
|
|
||||||
|
- export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')"
|
||||||
- export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH"
|
- export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH"
|
||||||
- export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64`
|
- export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64`
|
||||||
|
|
||||||
- >
|
- >
|
||||||
curl
|
curl
|
||||||
--silent
|
--silent
|
||||||
|
|
|
||||||
|
|
@ -1,18 +0,0 @@
|
||||||
branches: dev
|
|
||||||
platform: linux/amd64
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
publish:
|
|
||||||
image: woodpeckerci/plugin-docker-buildx
|
|
||||||
secrets: [ docker_username, docker_password ]
|
|
||||||
settings:
|
|
||||||
repo: chewingbever/vlang
|
|
||||||
tag: latest
|
|
||||||
dockerfile: Dockerfile.builder
|
|
||||||
platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
|
|
||||||
when:
|
|
||||||
event: push
|
|
||||||
path:
|
|
||||||
- Makefile
|
|
||||||
- Dockerfile.builder
|
|
||||||
- patches/*
|
|
||||||
|
|
@ -1,15 +0,0 @@
|
||||||
# Deploys the newest development image to my server
|
|
||||||
branches: [dev]
|
|
||||||
platform: linux/amd64
|
|
||||||
depends_on:
|
|
||||||
- docker
|
|
||||||
|
|
||||||
skip_clone: true
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
webhook:
|
|
||||||
image: chewingbever/vlang:latest
|
|
||||||
secrets:
|
|
||||||
- webhook
|
|
||||||
commands:
|
|
||||||
- curl -XPOST -s "$WEBHOOK"
|
|
||||||
|
|
@ -1,7 +1,6 @@
|
||||||
branches: [main, dev]
|
branches: [main, dev]
|
||||||
platform: linux/amd64
|
platform: linux/amd64
|
||||||
depends_on:
|
depends_on:
|
||||||
- builder
|
|
||||||
- build
|
- build
|
||||||
|
|
||||||
pipeline:
|
pipeline:
|
||||||
|
|
@ -10,9 +9,8 @@ pipeline:
|
||||||
secrets: [ docker_username, docker_password ]
|
secrets: [ docker_username, docker_password ]
|
||||||
settings:
|
settings:
|
||||||
repo: chewingbever/vieter
|
repo: chewingbever/vieter
|
||||||
dockerfile: Dockerfile.ci
|
|
||||||
tag: dev
|
tag: dev
|
||||||
platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
|
platforms: [ linux/arm64/v8, linux/amd64 ]
|
||||||
build_args_from_env:
|
build_args_from_env:
|
||||||
- CI_COMMIT_SHA
|
- CI_COMMIT_SHA
|
||||||
when:
|
when:
|
||||||
|
|
@ -24,9 +22,8 @@ pipeline:
|
||||||
secrets: [ docker_username, docker_password ]
|
secrets: [ docker_username, docker_password ]
|
||||||
settings:
|
settings:
|
||||||
repo: chewingbever/vieter
|
repo: chewingbever/vieter
|
||||||
dockerfile: Dockerfile.ci
|
|
||||||
auto_tag: true
|
auto_tag: true
|
||||||
platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
|
platforms: [ linux/arm64/v8, linux/amd64 ]
|
||||||
build_args_from_env:
|
build_args_from_env:
|
||||||
- CI_COMMIT_SHA
|
- CI_COMMIT_SHA
|
||||||
when:
|
when:
|
||||||
|
|
|
||||||
34
CHANGELOG.md
34
CHANGELOG.md
|
|
@ -7,6 +7,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||||
|
|
||||||
## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter)
|
## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter)
|
||||||
|
|
||||||
|
## [0.2.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.2.0)
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
* Better config system
|
||||||
|
* Support for both a config file & environment variables
|
||||||
|
* Each env var can now be provided from a file by appending it with `_FILE`
|
||||||
|
& passing the path to the file as value
|
||||||
|
* Revamped web framework
|
||||||
|
* All routes now return proper JSON where applicable & the correct status
|
||||||
|
codes
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
* Very basic build system
|
||||||
|
* Build is triggered by separate cron container
|
||||||
|
* Packages build on cron container's system
|
||||||
|
* A HEAD request is used to determine whether a package should be rebuilt
|
||||||
|
or not
|
||||||
|
* Hardcoded planning of builds
|
||||||
|
* Builds are sequential
|
||||||
|
* API for managing Git repositories to build
|
||||||
|
* CLI to list, add & remove Git repos to build
|
||||||
|
* Published packages on my Vieter instance
|
||||||
|
* Support for multiple repositories
|
||||||
|
* Support for multiple architectures per repository
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
* Each package can now only have one version in the repository at once
|
||||||
|
(required by Pacman)
|
||||||
|
* Packages with unknown fields in .PKGINFO are now allowed
|
||||||
|
* Old packages are now properly removed
|
||||||
|
|
||||||
## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0)
|
## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0)
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
|
||||||
49
Dockerfile
49
Dockerfile
|
|
@ -1,19 +1,56 @@
|
||||||
FROM chewingbever/vlang:latest AS builder
|
FROM chewingbever/vlang:latest AS builder
|
||||||
|
|
||||||
|
ARG TARGETPLATFORM
|
||||||
|
ARG CI_COMMIT_SHA
|
||||||
|
ARG DI_VER=1.2.5
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Build dumb-init
|
||||||
|
RUN curl -Lo - "https://github.com/Yelp/dumb-init/archive/refs/tags/v${DI_VER}.tar.gz" | tar -xzf - && \
|
||||||
|
cd "dumb-init-${DI_VER}" && \
|
||||||
|
make SHELL=/bin/sh && \
|
||||||
|
mv dumb-init .. && \
|
||||||
|
cd ..
|
||||||
|
|
||||||
# Copy over source code & build production binary
|
# Copy over source code & build production binary
|
||||||
COPY src ./src
|
COPY src ./src
|
||||||
COPY Makefile ./
|
COPY Makefile ./
|
||||||
|
|
||||||
ENV LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static'
|
RUN if [ -n "${CI_COMMIT_SHA}" ]; then \
|
||||||
RUN v -o pvieter -cflags "-O3" src
|
curl --fail \
|
||||||
|
-o vieter \
|
||||||
|
"https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \
|
||||||
|
chmod +x vieter ; \
|
||||||
|
else \
|
||||||
|
LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' make prod && \
|
||||||
|
mv pvieter vieter ; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
FROM alpine:3.15
|
FROM busybox:1.35.0
|
||||||
|
|
||||||
ENV REPO_DIR=/data
|
ENV PATH=/bin \
|
||||||
|
VIETER_REPOS_DIR=/data/repos \
|
||||||
|
VIETER_PKG_DIR=/data/pkgs \
|
||||||
|
VIETER_DOWNLOAD_DIR=/data/downloads \
|
||||||
|
VIETER_REPOS_FILE=/data/repos.json
|
||||||
|
|
||||||
COPY --from=builder /app/pvieter /usr/local/bin/vieter
|
COPY --from=builder /app/dumb-init /app/vieter /bin/
|
||||||
|
|
||||||
ENTRYPOINT [ "/usr/local/bin/vieter" ]
|
HEALTHCHECK --interval=30s \
|
||||||
|
--timeout=3s \
|
||||||
|
--start-period=5s \
|
||||||
|
CMD /bin/wget --spider http://localhost:8000/health || exit 1
|
||||||
|
|
||||||
|
RUN mkdir /data && \
|
||||||
|
chown -R www-data:www-data /data && \
|
||||||
|
mkdir -p '/var/spool/cron/crontabs' && \
|
||||||
|
echo '0 3 * * * /bin/vieter build' | crontab -
|
||||||
|
|
||||||
|
WORKDIR /data
|
||||||
|
|
||||||
|
USER www-data:www-data
|
||||||
|
|
||||||
|
ENTRYPOINT ["/bin/dumb-init", "--"]
|
||||||
|
CMD ["/bin/vieter", "server"]
|
||||||
|
|
|
||||||
|
|
@ -1,35 +0,0 @@
|
||||||
FROM alpine:3.12
|
|
||||||
|
|
||||||
ARG TARGETPLATFORM
|
|
||||||
|
|
||||||
WORKDIR /opt/vlang
|
|
||||||
|
|
||||||
ENV VVV /opt/vlang
|
|
||||||
ENV PATH /opt/vlang:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
|
||||||
ENV VFLAGS -cc gcc
|
|
||||||
ENV V_PATH /opt/vlang/v
|
|
||||||
|
|
||||||
RUN ln -s /opt/vlang/v /usr/bin/v && \
|
|
||||||
apk --no-cache add \
|
|
||||||
git make gcc curl openssl \
|
|
||||||
musl-dev \
|
|
||||||
openssl-libs-static openssl-dev \
|
|
||||||
zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \
|
|
||||||
sqlite-static sqlite-dev \
|
|
||||||
libx11-dev glfw-dev freetype-dev \
|
|
||||||
libarchive-static libarchive-dev \
|
|
||||||
diffutils
|
|
||||||
|
|
||||||
COPY patches ./patches
|
|
||||||
COPY Makefile ./
|
|
||||||
|
|
||||||
RUN make v && \
|
|
||||||
mv v-*/* /opt/vlang && \
|
|
||||||
v -version
|
|
||||||
|
|
||||||
RUN if [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \
|
|
||||||
wget -O /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && \
|
|
||||||
chmod +x /usr/local/bin/mc ; \
|
|
||||||
fi
|
|
||||||
|
|
||||||
CMD ["v"]
|
|
||||||
|
|
@ -1,46 +0,0 @@
|
||||||
# vim: ft=dockerfile
|
|
||||||
# This image just has the required tools to download the binaries
|
|
||||||
FROM chewingbever/vlang:latest AS builder
|
|
||||||
|
|
||||||
ARG TARGETPLATFORM
|
|
||||||
ARG CI_COMMIT_SHA
|
|
||||||
ARG DI_VER=1.2.5
|
|
||||||
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# Build dumb-init
|
|
||||||
RUN curl -Lo - "https://github.com/Yelp/dumb-init/archive/refs/tags/v${DI_VER}.tar.gz" | tar -xzf - && \
|
|
||||||
cd "dumb-init-${DI_VER}" && \
|
|
||||||
make SHELL=/bin/sh && \
|
|
||||||
mv dumb-init .. && \
|
|
||||||
cd ..
|
|
||||||
|
|
||||||
RUN curl --fail \
|
|
||||||
-o vieter \
|
|
||||||
"https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \
|
|
||||||
chmod +x vieter
|
|
||||||
|
|
||||||
|
|
||||||
FROM busybox:1.35.0
|
|
||||||
|
|
||||||
ENV PATH=/bin \
|
|
||||||
REPO_DIR=/data/repo \
|
|
||||||
PKG_DIR=/data/pkgs \
|
|
||||||
DOWNLOAD_DIR=/data/downloads
|
|
||||||
|
|
||||||
COPY --from=builder /app/dumb-init /app/vieter /bin/
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=30s \
|
|
||||||
--timeout=3s \
|
|
||||||
--start-period=5s \
|
|
||||||
CMD /bin/wget --spider http://localhost:8000/health || exit 1
|
|
||||||
|
|
||||||
RUN mkdir /data && \
|
|
||||||
chown -R www-data:www-data /data
|
|
||||||
|
|
||||||
WORKDIR /data
|
|
||||||
|
|
||||||
USER www-data:www-data
|
|
||||||
|
|
||||||
ENTRYPOINT ["/bin/dumb-init", "--"]
|
|
||||||
CMD ["/bin/vieter"]
|
|
||||||
36
Makefile
36
Makefile
|
|
@ -2,9 +2,8 @@
|
||||||
SRC_DIR := src
|
SRC_DIR := src
|
||||||
SOURCES != find '$(SRC_DIR)' -iname '*.v'
|
SOURCES != find '$(SRC_DIR)' -iname '*.v'
|
||||||
|
|
||||||
V_RELEASE := weekly.2022.05
|
V_PATH ?= v/v
|
||||||
V_PATH ?= v-$(V_RELEASE)/v
|
V := $(V_PATH) -showcc -gc boehm
|
||||||
V := $(V_PATH) -showcc
|
|
||||||
|
|
||||||
all: vieter
|
all: vieter
|
||||||
|
|
||||||
|
|
@ -14,10 +13,17 @@ vieter: $(SOURCES)
|
||||||
$(V) -g -o vieter $(SRC_DIR)
|
$(V) -g -o vieter $(SRC_DIR)
|
||||||
|
|
||||||
# Debug build using gcc
|
# Debug build using gcc
|
||||||
|
# The debug build can't use the boehm garbage collector, as that is
|
||||||
|
# multi-threaded and causes issues when running vieter inside gdb.
|
||||||
.PHONY: debug
|
.PHONY: debug
|
||||||
debug: dvieter
|
debug: dvieter
|
||||||
dvieter: $(SOURCES)
|
dvieter: $(SOURCES)
|
||||||
$(V) -keepc -cg -cc gcc -o dvieter $(SRC_DIR)
|
$(V_PATH) -showcc -keepc -cg -o dvieter $(SRC_DIR)
|
||||||
|
|
||||||
|
# Run the debug build inside gdb
|
||||||
|
.PHONY: gdb
|
||||||
|
gdb: dvieter
|
||||||
|
gdb --args './dvieter -f vieter.toml server'
|
||||||
|
|
||||||
# Optimised production build
|
# Optimised production build
|
||||||
.PHONY: prod
|
.PHONY: prod
|
||||||
|
|
@ -30,22 +36,15 @@ pvieter: $(SOURCES)
|
||||||
c:
|
c:
|
||||||
$(V) -o vieter.c $(SRC_DIR)
|
$(V) -o vieter.c $(SRC_DIR)
|
||||||
|
|
||||||
|
|
||||||
# =====EXECUTION=====
|
# =====EXECUTION=====
|
||||||
# Run the server in the default 'data' directory
|
# Run the server in the default 'data' directory
|
||||||
.PHONY: run
|
.PHONY: run
|
||||||
run: vieter
|
run: vieter
|
||||||
API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter
|
./vieter -f vieter.toml server
|
||||||
|
|
||||||
.PHONY: run-prod
|
.PHONY: run-prod
|
||||||
run-prod: prod
|
run-prod: prod
|
||||||
API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./pvieter
|
./pvieter -f vieter.toml server
|
||||||
|
|
||||||
# Same as run, but restart when the source code changes
|
|
||||||
.PHONY: watch
|
|
||||||
watch:
|
|
||||||
API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG $(V) watch run vieter
|
|
||||||
|
|
||||||
|
|
||||||
# =====OTHER=====
|
# =====OTHER=====
|
||||||
.PHONY: lint
|
.PHONY: lint
|
||||||
|
|
@ -63,11 +62,10 @@ vet:
|
||||||
|
|
||||||
# Build & patch the V compiler
|
# Build & patch the V compiler
|
||||||
.PHONY: v
|
.PHONY: v
|
||||||
v: v-$(V_RELEASE)/v
|
v: v/v
|
||||||
v-$(V_RELEASE)/v:
|
v/v:
|
||||||
curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf -
|
git clone --single-branch --branch patches https://git.rustybever.be/Chewing_Bever/vieter-v v
|
||||||
cd patches && sh patch.sh '../v-$(V_RELEASE)'
|
make -C v
|
||||||
make -C 'v-$(V_RELEASE)'
|
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'v-$(V_RELEASE)'
|
rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter'
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
# Maintainer: Jef Roosens
|
||||||
|
|
||||||
|
pkgbase='vieter'
|
||||||
|
pkgname='vieter'
|
||||||
|
pkgver=0.1.0.rc1.r117.gc3ac00f
|
||||||
|
pkgrel=1
|
||||||
|
depends=('glibc' 'openssl' 'libarchive' 'gc')
|
||||||
|
makedepends=('git' 'gcc')
|
||||||
|
arch=('x86_64' 'aarch64' 'armv7')
|
||||||
|
url='https://git.rustybever.be/Chewing_Bever/vieter'
|
||||||
|
license=('AGPL3')
|
||||||
|
source=($pkgname::git+https://git.rustybever.be/Chewing_Bever/vieter#branch=dev)
|
||||||
|
md5sums=('SKIP')
|
||||||
|
|
||||||
|
pkgver() {
|
||||||
|
cd "$pkgname"
|
||||||
|
git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g'
|
||||||
|
}
|
||||||
|
|
||||||
|
build() {
|
||||||
|
cd "$pkgname"
|
||||||
|
|
||||||
|
# Build the compiler
|
||||||
|
CFLAGS= make v
|
||||||
|
|
||||||
|
make prod
|
||||||
|
}
|
||||||
|
|
||||||
|
package() {
|
||||||
|
pkgdesc="Vieter is a lightweight implementation of an Arch repository server."
|
||||||
|
install -dm755 "$pkgdir/usr/bin"
|
||||||
|
|
||||||
|
install -Dm755 "$pkgbase/pvieter" "$pkgdir/usr/bin/vieter"
|
||||||
|
}
|
||||||
60
README.md
60
README.md
|
|
@ -1,10 +1,17 @@
|
||||||
# Vieter
|
# Vieter
|
||||||
|
|
||||||
Vieter is a re-implementation of the Pieter project. The goal is to create a
|
## Documentation
|
||||||
simple PKGBUILD-based build system, combined with a self-hosted Arch
|
|
||||||
repository. This would allow me to periodically re-build AUR packages (or
|
I host documentation for Vieter over at https://rustybever.be/docs/vieter/.
|
||||||
PKGBUILDs I created myself), & make sure I never have to compile anything on my
|
|
||||||
own systems, making my updates a lot quicker.
|
## Overview
|
||||||
|
|
||||||
|
Vieter is a restart of the Pieter project. The goal is to create a simple,
|
||||||
|
lightweight self-hostable Arch repository server, paired with a system that
|
||||||
|
periodically builds & publishes select Arch packages. This would allow me to
|
||||||
|
build AUR packages (or PKGBUILDs I created myself) "in the cloud" & make sure I
|
||||||
|
never have to compile anything on my own systems, making my updates a lot
|
||||||
|
quicker.
|
||||||
|
|
||||||
## Why V?
|
## Why V?
|
||||||
|
|
||||||
|
|
@ -16,31 +23,30 @@ that.
|
||||||
### Custom Compiler
|
### Custom Compiler
|
||||||
|
|
||||||
Currently, this program only works with a very slightly modified version of the
|
Currently, this program only works with a very slightly modified version of the
|
||||||
V standard library, and therefore the compiler. The changes that are made to
|
V standard library, and therefore the compiler. The source code for this fork
|
||||||
the standard V release can be found in the [patches](/patches) directory. You
|
can be found [here](https://git.rustybever.be/Chewing_Bever/vieter-v). You can
|
||||||
can obtain this modified version of the compiler by running `make v`, which
|
obtain this modified version of the compiler by running `make v`, which will
|
||||||
will download, patch & build the compiler. Afterwards, all make commands that
|
clone & build the compiler. Afterwards, all make commands that require the V
|
||||||
require the V compiler will use this new binary.
|
compiler will use this new binary. I try to keep this fork as up to date with
|
||||||
|
upstream as possible.
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
The project will consist of a server-agent model, where one or more builder
|
* Arch repository server
|
||||||
nodes can register with the server. These agents communicate with the Docker
|
* Support for multiple repositories & multiple architectures
|
||||||
daemon to start builds, which are then uploaded to the server's repository. The
|
* Endpoints for publishing new packages
|
||||||
server also allows for non-agents to upload packages, as long as they have the
|
* API for managing repositories to build
|
||||||
required secrets. This allows me to also develop non-git packages, such as my
|
* Build system
|
||||||
terminal, & upload them to the servers using CI.
|
* Periodic rebuilding of packages
|
||||||
|
* Prevent unnecessary rebuilds
|
||||||
|
|
||||||
## Directory Structure
|
## Building
|
||||||
|
|
||||||
The data directory consists of three main directories:
|
In order to build Vieter, you'll need a couple of libraries:
|
||||||
|
|
||||||
* `downloads` - This is where packages are initially downloaded. Because vieter
|
* gc
|
||||||
moves files from this folder to the `pkgs` folder, these two folders should
|
* libarchive
|
||||||
best be on the same drive
|
* openssl
|
||||||
* `pkgs` - This is where approved package files are stored.
|
|
||||||
* `repos` - Each repository gets a subfolder here. The subfolder contains the
|
Before building Vieter, you'll have to build the compiler using `make v`.
|
||||||
uncompressed contents of the db file.
|
Afterwards, run `make` to build the debug binary.
|
||||||
* Each repo subdirectory contains the compressed db & files archive for the
|
|
||||||
repository, alongside a directory called `files` which contains the
|
|
||||||
uncompressed contents.
|
|
||||||
|
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
// Parse the header of a raw HTTP request into a Request object
|
|
||||||
pub fn parse_request_head(mut reader io.BufferedReader) ?Request {
|
|
||||||
// request line
|
|
||||||
mut line := reader.read_line() ?
|
|
||||||
method, target, version := parse_request_line(line) ?
|
|
||||||
|
|
||||||
// headers
|
|
||||||
mut header := new_header()
|
|
||||||
line = reader.read_line() ?
|
|
||||||
for line != '' {
|
|
||||||
key, value := parse_header(line) ?
|
|
||||||
header.add_custom(key, value) ?
|
|
||||||
line = reader.read_line() ?
|
|
||||||
}
|
|
||||||
header.coerce(canonicalize: true)
|
|
||||||
|
|
||||||
return Request{
|
|
||||||
method: method
|
|
||||||
url: target.str()
|
|
||||||
header: header
|
|
||||||
version: version
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,13 +0,0 @@
|
||||||
#!/usr/bin/env sh
|
|
||||||
# This file patches the downloaded V version
|
|
||||||
# Should be run from within the directory it's in, as it uses relative paths to the files used for patching.
|
|
||||||
# $1 is the path to the downloaded V version
|
|
||||||
|
|
||||||
# Add parse_request_no_body
|
|
||||||
cat parse_request_no_body.v >> "$1"/vlib/net/http/request.v
|
|
||||||
|
|
||||||
# Make sha256 functions public
|
|
||||||
sed -i \
|
|
||||||
-e 's/\(fn (mut d Digest) checksum(\)/pub \1/' \
|
|
||||||
-e 's/\(fn (mut d Digest) write(\)/pub \1/' \
|
|
||||||
"$1"/vlib/crypto/sha256/sha256.v
|
|
||||||
|
|
@ -15,6 +15,9 @@ fn C.archive_read_support_filter_zstd(&C.archive)
|
||||||
// Configure the archive to work with gzip compression
|
// Configure the archive to work with gzip compression
|
||||||
fn C.archive_read_support_filter_gzip(&C.archive)
|
fn C.archive_read_support_filter_gzip(&C.archive)
|
||||||
|
|
||||||
|
// Configure the archive to work with xz compression
|
||||||
|
fn C.archive_read_support_filter_xz(&C.archive)
|
||||||
|
|
||||||
// Configure the archive to work with a tarball content
|
// Configure the archive to work with a tarball content
|
||||||
fn C.archive_read_support_format_tar(&C.archive)
|
fn C.archive_read_support_format_tar(&C.archive)
|
||||||
|
|
||||||
|
|
@ -0,0 +1,133 @@
|
||||||
|
module build
|
||||||
|
|
||||||
|
import docker
|
||||||
|
import encoding.base64
|
||||||
|
import time
|
||||||
|
import git
|
||||||
|
import os
|
||||||
|
|
||||||
|
const container_build_dir = '/build'
|
||||||
|
|
||||||
|
const build_image_repo = 'vieter-build'
|
||||||
|
|
||||||
|
fn create_build_image(base_image string) ?string {
|
||||||
|
commands := [
|
||||||
|
// Update repos & install required packages
|
||||||
|
'pacman -Syu --needed --noconfirm base-devel git'
|
||||||
|
// Add a non-root user to run makepkg
|
||||||
|
'groupadd -g 1000 builder',
|
||||||
|
'useradd -mg builder builder'
|
||||||
|
// Make sure they can use sudo without a password
|
||||||
|
"echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers"
|
||||||
|
// Create the directory for the builds & make it writeable for the
|
||||||
|
// build user
|
||||||
|
'mkdir /build',
|
||||||
|
'chown -R builder:builder /build',
|
||||||
|
]
|
||||||
|
cmds_str := base64.encode_str(commands.join('\n'))
|
||||||
|
|
||||||
|
c := docker.NewContainer{
|
||||||
|
image: base_image
|
||||||
|
env: ['BUILD_SCRIPT=$cmds_str']
|
||||||
|
entrypoint: ['/bin/sh', '-c']
|
||||||
|
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e']
|
||||||
|
}
|
||||||
|
|
||||||
|
// This check is needed so the user can pass "archlinux" without passing a
|
||||||
|
// tag & make it still work
|
||||||
|
image_parts := base_image.split_nth(':', 2)
|
||||||
|
image_name := image_parts[0]
|
||||||
|
image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' }
|
||||||
|
|
||||||
|
// We pull the provided image
|
||||||
|
docker.pull_image(image_name, image_tag) ?
|
||||||
|
|
||||||
|
id := docker.create_container(c) ?
|
||||||
|
docker.start_container(id) ?
|
||||||
|
|
||||||
|
// This loop waits until the container has stopped, so we can remove it after
|
||||||
|
for {
|
||||||
|
data := docker.inspect_container(id) ?
|
||||||
|
|
||||||
|
if !data.state.running {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for 5 seconds
|
||||||
|
time.sleep(5000000000)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finally, we create the image from the container
|
||||||
|
// As the tag, we use the epoch value
|
||||||
|
tag := time.sys_mono_now().str()
|
||||||
|
image := docker.create_image_from_container(id, 'vieter-build', tag) ?
|
||||||
|
docker.remove_container(id) ?
|
||||||
|
|
||||||
|
return image.id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build(conf Config) ? {
|
||||||
|
build_arch := os.uname().machine
|
||||||
|
|
||||||
|
// We get the repos map from the Vieter instance
|
||||||
|
repos_map := git.get_repos(conf.address, conf.api_key) ?
|
||||||
|
|
||||||
|
// We filter out any repos that aren't allowed to be built on this
|
||||||
|
// architecture
|
||||||
|
filtered_repos := repos_map.keys().map(repos_map[it]).filter(it.arch.contains(build_arch))
|
||||||
|
|
||||||
|
// No point in doing work if there's no repos present
|
||||||
|
if filtered_repos.len == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// First, we create a base image which has updated repos n stuff
|
||||||
|
image_id := create_build_image(conf.base_image) ?
|
||||||
|
|
||||||
|
for repo in filtered_repos {
|
||||||
|
// TODO what to do with PKGBUILDs that build multiple packages?
|
||||||
|
commands := [
|
||||||
|
'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo',
|
||||||
|
'cd repo',
|
||||||
|
'makepkg --nobuild --nodeps',
|
||||||
|
'source PKGBUILD',
|
||||||
|
// The build container checks whether the package is already
|
||||||
|
// present on the server
|
||||||
|
'curl --head --fail $conf.address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0',
|
||||||
|
'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/$repo.repo/publish; done',
|
||||||
|
]
|
||||||
|
|
||||||
|
// We convert the list of commands into a base64 string, which then gets
|
||||||
|
// passed to the container as an env var
|
||||||
|
cmds_str := base64.encode_str(commands.join('\n'))
|
||||||
|
|
||||||
|
c := docker.NewContainer{
|
||||||
|
image: '$image_id'
|
||||||
|
env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key']
|
||||||
|
entrypoint: ['/bin/sh', '-c']
|
||||||
|
cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e']
|
||||||
|
work_dir: '/build'
|
||||||
|
user: 'builder:builder'
|
||||||
|
}
|
||||||
|
|
||||||
|
id := docker.create_container(c) ?
|
||||||
|
docker.start_container(id) ?
|
||||||
|
|
||||||
|
// This loop waits until the container has stopped, so we can remove it after
|
||||||
|
for {
|
||||||
|
data := docker.inspect_container(id) ?
|
||||||
|
|
||||||
|
if !data.state.running {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for 5 seconds
|
||||||
|
time.sleep(5000000000)
|
||||||
|
}
|
||||||
|
|
||||||
|
docker.remove_container(id) ?
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finally, we remove the builder image
|
||||||
|
docker.remove_image(image_id) ?
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
module build
|
||||||
|
|
||||||
|
import cli
|
||||||
|
import env
|
||||||
|
|
||||||
|
pub struct Config {
|
||||||
|
pub:
|
||||||
|
api_key string
|
||||||
|
address string
|
||||||
|
base_image string = 'archlinux:base-devel'
|
||||||
|
}
|
||||||
|
|
||||||
|
// cmd returns the cli submodule that handles the build process
|
||||||
|
pub fn cmd() cli.Command {
|
||||||
|
return cli.Command{
|
||||||
|
name: 'build'
|
||||||
|
description: 'Run the build process.'
|
||||||
|
execute: fn (cmd cli.Command) ? {
|
||||||
|
config_file := cmd.flags.get_string('config-file') ?
|
||||||
|
conf := env.load<Config>(config_file) ?
|
||||||
|
|
||||||
|
build(conf) ?
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,78 @@
|
||||||
|
module docker
|
||||||
|
|
||||||
|
import json
|
||||||
|
import net.urllib
|
||||||
|
|
||||||
|
struct Container {
|
||||||
|
id string [json: Id]
|
||||||
|
names []string [json: Names]
|
||||||
|
}
|
||||||
|
|
||||||
|
// containers returns a list of all currently running containers
|
||||||
|
pub fn containers() ?[]Container {
|
||||||
|
res := request('GET', urllib.parse('/v1.41/containers/json') ?) ?
|
||||||
|
|
||||||
|
return json.decode([]Container, res.text) or {}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct NewContainer {
|
||||||
|
image string [json: Image]
|
||||||
|
entrypoint []string [json: Entrypoint]
|
||||||
|
cmd []string [json: Cmd]
|
||||||
|
env []string [json: Env]
|
||||||
|
work_dir string [json: WorkingDir]
|
||||||
|
user string [json: User]
|
||||||
|
}
|
||||||
|
|
||||||
|
struct CreatedContainer {
|
||||||
|
id string [json: Id]
|
||||||
|
}
|
||||||
|
|
||||||
|
// create_container creates a container defined by the given configuration. If
|
||||||
|
// successful, it returns the ID of the newly created container.
|
||||||
|
pub fn create_container(c &NewContainer) ?string {
|
||||||
|
res := request_with_json('POST', urllib.parse('/v1.41/containers/create') ?, c) ?
|
||||||
|
|
||||||
|
if res.status_code != 201 {
|
||||||
|
return error('Failed to create container.')
|
||||||
|
}
|
||||||
|
|
||||||
|
return json.decode(CreatedContainer, res.text) ?.id
|
||||||
|
}
|
||||||
|
|
||||||
|
// start_container starts a container with a given ID. It returns whether the
|
||||||
|
// container was started or not.
|
||||||
|
pub fn start_container(id string) ?bool {
|
||||||
|
res := request('POST', urllib.parse('/v1.41/containers/$id/start') ?) ?
|
||||||
|
|
||||||
|
return res.status_code == 204
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ContainerInspect {
|
||||||
|
pub:
|
||||||
|
state ContainerState [json: State]
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ContainerState {
|
||||||
|
pub:
|
||||||
|
running bool [json: Running]
|
||||||
|
}
|
||||||
|
|
||||||
|
// inspect_container returns the result of inspecting a container with a given
|
||||||
|
// ID.
|
||||||
|
pub fn inspect_container(id string) ?ContainerInspect {
|
||||||
|
res := request('GET', urllib.parse('/v1.41/containers/$id/json') ?) ?
|
||||||
|
|
||||||
|
if res.status_code != 200 {
|
||||||
|
return error('Failed to inspect container.')
|
||||||
|
}
|
||||||
|
|
||||||
|
return json.decode(ContainerInspect, res.text) or {}
|
||||||
|
}
|
||||||
|
|
||||||
|
// remove_container removes a container with a given ID.
|
||||||
|
pub fn remove_container(id string) ?bool {
|
||||||
|
res := request('DELETE', urllib.parse('/v1.41/containers/$id') ?) ?
|
||||||
|
|
||||||
|
return res.status_code == 204
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,93 @@
|
||||||
|
module docker
|
||||||
|
|
||||||
|
import net.unix
|
||||||
|
import net.urllib
|
||||||
|
import net.http
|
||||||
|
import json
|
||||||
|
|
||||||
|
const socket = '/var/run/docker.sock'
|
||||||
|
|
||||||
|
const buf_len = 1024
|
||||||
|
|
||||||
|
fn send(req &string) ?http.Response {
|
||||||
|
// Open a connection to the socket
|
||||||
|
mut s := unix.connect_stream(docker.socket) or {
|
||||||
|
return error('Failed to connect to socket ${docker.socket}.')
|
||||||
|
}
|
||||||
|
|
||||||
|
defer {
|
||||||
|
// This or is required because otherwise, the V compiler segfaults for
|
||||||
|
// some reason
|
||||||
|
// https://github.com/vlang/v/issues/13534
|
||||||
|
s.close() or {}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write the request to the socket
|
||||||
|
s.write_string(req) or { return error('Failed to write request to socket ${docker.socket}.') }
|
||||||
|
|
||||||
|
s.wait_for_write() ?
|
||||||
|
|
||||||
|
mut c := 0
|
||||||
|
mut buf := []byte{len: docker.buf_len}
|
||||||
|
mut res := []byte{}
|
||||||
|
|
||||||
|
for {
|
||||||
|
c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') }
|
||||||
|
res << buf[..c]
|
||||||
|
|
||||||
|
if c < docker.buf_len {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// After reading the first part of the response, we parse it into an HTTP
|
||||||
|
// response. If it isn't chunked, we return early with the data.
|
||||||
|
parsed := http.parse_response(res.bytestr()) or {
|
||||||
|
return error('Failed to parse HTTP response from socket ${docker.socket}.')
|
||||||
|
}
|
||||||
|
|
||||||
|
if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' {
|
||||||
|
return parsed
|
||||||
|
}
|
||||||
|
|
||||||
|
// We loop until we've encountered the end of the chunked response
|
||||||
|
// A chunked HTTP response always ends with '0\r\n\r\n'.
|
||||||
|
for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] {
|
||||||
|
// Wait for the server to respond
|
||||||
|
s.wait_for_write() ?
|
||||||
|
|
||||||
|
for {
|
||||||
|
c = s.read(mut buf) or {
|
||||||
|
return error('Failed to read data from socket ${docker.socket}.')
|
||||||
|
}
|
||||||
|
res << buf[..c]
|
||||||
|
|
||||||
|
if c < docker.buf_len {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decode chunked response
|
||||||
|
return http.parse_response(res.bytestr())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response {
|
||||||
|
req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n'
|
||||||
|
|
||||||
|
return send(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request(method string, url urllib.URL) ?http.Response {
|
||||||
|
req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n'
|
||||||
|
|
||||||
|
return send(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// request_with_json<T> sends a request to the Docker socket with a given JSON
|
||||||
|
// payload
|
||||||
|
pub fn request_with_json<T>(method string, url urllib.URL, data &T) ?http.Response {
|
||||||
|
body := json.encode(data)
|
||||||
|
|
||||||
|
return request_with_body(method, url, 'application/json', body)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
module docker
|
||||||
|
|
||||||
|
import net.http
|
||||||
|
import net.urllib
|
||||||
|
import json
|
||||||
|
|
||||||
|
struct Image {
|
||||||
|
pub:
|
||||||
|
id string [json: Id]
|
||||||
|
}
|
||||||
|
|
||||||
|
// pull_image pulls tries to pull the image for the given image & tag
|
||||||
|
pub fn pull_image(image string, tag string) ?http.Response {
|
||||||
|
return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?)
|
||||||
|
}
|
||||||
|
|
||||||
|
// create_image_from_container creates a new image from a container with the
|
||||||
|
// given repo & tag, given the container's ID.
|
||||||
|
pub fn create_image_from_container(id string, repo string, tag string) ?Image {
|
||||||
|
res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag') ?) ?
|
||||||
|
|
||||||
|
if res.status_code != 201 {
|
||||||
|
return error('Failed to create image from container.')
|
||||||
|
}
|
||||||
|
|
||||||
|
return json.decode(Image, res.text) or {}
|
||||||
|
}
|
||||||
|
|
||||||
|
// remove_image removes the image with the given ID.
|
||||||
|
pub fn remove_image(id string) ?bool {
|
||||||
|
res := request('DELETE', urllib.parse('/v1.41/images/$id') ?) ?
|
||||||
|
|
||||||
|
return res.status_code == 200
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,82 @@
|
||||||
|
module env
|
||||||
|
|
||||||
|
import os
|
||||||
|
import toml
|
||||||
|
|
||||||
|
// The prefix that every environment variable should have
|
||||||
|
const prefix = 'VIETER_'
|
||||||
|
|
||||||
|
// The suffix an environment variable in order for it to be loaded from a file
|
||||||
|
// instead
|
||||||
|
const file_suffix = '_FILE'
|
||||||
|
|
||||||
|
fn get_env_var(field_name string) ?string {
|
||||||
|
env_var_name := '$env.prefix$field_name.to_upper()'
|
||||||
|
env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix'
|
||||||
|
env_var := os.getenv(env_var_name)
|
||||||
|
env_file := os.getenv(env_file_name)
|
||||||
|
|
||||||
|
// If both are missing, we return an empty string
|
||||||
|
if env_var == '' && env_file == '' {
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
|
||||||
|
// If they're both set, we report a conflict
|
||||||
|
if env_var != '' && env_file != '' {
|
||||||
|
return error('Only one of $env_var_name or $env_file_name can be defined.')
|
||||||
|
}
|
||||||
|
|
||||||
|
// If it's the env var itself, we return it.
|
||||||
|
// I'm pretty sure this also prevents variable ending in _FILE (e.g.
|
||||||
|
// VIETER_LOG_FILE) from being mistakingely read as an _FILE suffixed env
|
||||||
|
// var.
|
||||||
|
if env_var != '' {
|
||||||
|
return env_var
|
||||||
|
}
|
||||||
|
|
||||||
|
// Otherwise, we process the file
|
||||||
|
return os.read_file(env_file) or {
|
||||||
|
error('Failed to read file defined in $env_file_name: ${err.msg}.')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// load<T> attempts to create an object of type T from the given path to a toml
|
||||||
|
// file & environment variables. For each field, it will select either a value
|
||||||
|
// given from an environment variable, a value defined in the config file or a
|
||||||
|
// configured default if present, in that order.
|
||||||
|
pub fn load<T>(path string) ?T {
|
||||||
|
mut res := T{}
|
||||||
|
|
||||||
|
if os.exists(path) {
|
||||||
|
// We don't use reflect here because reflect also sets any fields not
|
||||||
|
// in the toml back to their zero value, which we don't want
|
||||||
|
doc := toml.parse_file(path) ?
|
||||||
|
|
||||||
|
$for field in T.fields {
|
||||||
|
s := doc.value(field.name)
|
||||||
|
|
||||||
|
// We currently only support strings
|
||||||
|
if s.type_name() == 'string' {
|
||||||
|
res.$(field.name) = s.string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$for field in T.fields {
|
||||||
|
$if field.typ is string {
|
||||||
|
env_value := get_env_var(field.name) ?
|
||||||
|
|
||||||
|
// The value of the env var will always be chosen over the config
|
||||||
|
// file
|
||||||
|
if env_value != '' {
|
||||||
|
res.$(field.name) = env_value
|
||||||
|
}
|
||||||
|
// If there's no value from the toml file either, we try to find a
|
||||||
|
// default value
|
||||||
|
else if res.$(field.name) == '' {
|
||||||
|
return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,147 @@
|
||||||
|
module git
|
||||||
|
|
||||||
|
import cli
|
||||||
|
import env
|
||||||
|
|
||||||
|
struct Config {
|
||||||
|
address string [required]
|
||||||
|
api_key string [required]
|
||||||
|
}
|
||||||
|
|
||||||
|
// cmd returns the cli submodule that handles the repos API interaction
|
||||||
|
pub fn cmd() cli.Command {
|
||||||
|
return cli.Command{
|
||||||
|
name: 'repos'
|
||||||
|
description: 'Interact with the repos API.'
|
||||||
|
commands: [
|
||||||
|
cli.Command{
|
||||||
|
name: 'list'
|
||||||
|
description: 'List the current repos.'
|
||||||
|
execute: fn (cmd cli.Command) ? {
|
||||||
|
config_file := cmd.flags.get_string('config-file') ?
|
||||||
|
conf := env.load<Config>(config_file) ?
|
||||||
|
|
||||||
|
list(conf) ?
|
||||||
|
}
|
||||||
|
},
|
||||||
|
cli.Command{
|
||||||
|
name: 'add'
|
||||||
|
required_args: 4
|
||||||
|
usage: 'url branch repo arch...'
|
||||||
|
description: 'Add a new repository.'
|
||||||
|
execute: fn (cmd cli.Command) ? {
|
||||||
|
config_file := cmd.flags.get_string('config-file') ?
|
||||||
|
conf := env.load<Config>(config_file) ?
|
||||||
|
|
||||||
|
add(conf, cmd.args[0], cmd.args[1], cmd.args[2], cmd.args[3..]) ?
|
||||||
|
}
|
||||||
|
},
|
||||||
|
cli.Command{
|
||||||
|
name: 'remove'
|
||||||
|
required_args: 1
|
||||||
|
usage: 'id'
|
||||||
|
description: 'Remove a repository that matches the given ID prefix.'
|
||||||
|
execute: fn (cmd cli.Command) ? {
|
||||||
|
config_file := cmd.flags.get_string('config-file') ?
|
||||||
|
conf := env.load<Config>(config_file) ?
|
||||||
|
|
||||||
|
remove(conf, cmd.args[0]) ?
|
||||||
|
}
|
||||||
|
},
|
||||||
|
cli.Command{
|
||||||
|
name: 'edit'
|
||||||
|
required_args: 1
|
||||||
|
usage: 'id'
|
||||||
|
description: 'Edit the repository that matches the given ID prefix.'
|
||||||
|
flags: [
|
||||||
|
cli.Flag{
|
||||||
|
name: 'url'
|
||||||
|
description: 'URL of the Git repository.'
|
||||||
|
flag: cli.FlagType.string
|
||||||
|
},
|
||||||
|
cli.Flag{
|
||||||
|
name: 'branch'
|
||||||
|
description: 'Branch of the Git repository.'
|
||||||
|
flag: cli.FlagType.string
|
||||||
|
},
|
||||||
|
cli.Flag{
|
||||||
|
name: 'repo'
|
||||||
|
description: 'Repo to publish builds to.'
|
||||||
|
flag: cli.FlagType.string
|
||||||
|
},
|
||||||
|
cli.Flag{
|
||||||
|
name: 'arch'
|
||||||
|
description: 'Comma-separated list of architectures to build on.'
|
||||||
|
flag: cli.FlagType.string
|
||||||
|
},
|
||||||
|
]
|
||||||
|
execute: fn (cmd cli.Command) ? {
|
||||||
|
config_file := cmd.flags.get_string('config-file') ?
|
||||||
|
conf := env.load<Config>(config_file) ?
|
||||||
|
|
||||||
|
found := cmd.flags.get_all_found()
|
||||||
|
|
||||||
|
mut params := map[string]string{}
|
||||||
|
|
||||||
|
for f in found {
|
||||||
|
if f.name != 'config-file' {
|
||||||
|
params[f.name] = f.get_string() ?
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
patch(conf, cmd.args[0], params) ?
|
||||||
|
}
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_repo_id_by_prefix(conf Config, id_prefix string) ?string {
|
||||||
|
repos := get_repos(conf.address, conf.api_key) ?
|
||||||
|
|
||||||
|
mut res := []string{}
|
||||||
|
|
||||||
|
for id, _ in repos {
|
||||||
|
if id.starts_with(id_prefix) {
|
||||||
|
res << id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if res.len == 0 {
|
||||||
|
return error('No repo found for given prefix.')
|
||||||
|
}
|
||||||
|
|
||||||
|
if res.len > 1 {
|
||||||
|
return error('Multiple repos found for given prefix.')
|
||||||
|
}
|
||||||
|
|
||||||
|
return res[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn list(conf Config) ? {
|
||||||
|
repos := get_repos(conf.address, conf.api_key) ?
|
||||||
|
|
||||||
|
for id, details in repos {
|
||||||
|
println('${id[..8]}\t$details.url\t$details.branch\t$details.repo\t$details.arch')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn add(conf Config, url string, branch string, repo string, arch []string) ? {
|
||||||
|
res := add_repo(conf.address, conf.api_key, url, branch, repo, arch) ?
|
||||||
|
|
||||||
|
println(res.message)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove(conf Config, id_prefix string) ? {
|
||||||
|
id := get_repo_id_by_prefix(conf, id_prefix) ?
|
||||||
|
res := remove_repo(conf.address, conf.api_key, id) ?
|
||||||
|
|
||||||
|
println(res.message)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn patch(conf Config, id_prefix string, params map[string]string) ? {
|
||||||
|
id := get_repo_id_by_prefix(conf, id_prefix) ?
|
||||||
|
res := patch_repo(conf.address, conf.api_key, id, params) ?
|
||||||
|
|
||||||
|
println(res.message)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,61 @@
|
||||||
|
module git
|
||||||
|
|
||||||
|
import json
|
||||||
|
import response { Response }
|
||||||
|
import net.http
|
||||||
|
|
||||||
|
fn send_request<T>(method http.Method, address string, url string, api_key string, params map[string]string) ?Response<T> {
|
||||||
|
mut full_url := '$address$url'
|
||||||
|
|
||||||
|
if params.len > 0 {
|
||||||
|
params_str := params.keys().map('$it=${params[it]}').join('&')
|
||||||
|
|
||||||
|
full_url = '$full_url?$params_str'
|
||||||
|
}
|
||||||
|
|
||||||
|
mut req := http.new_request(method, full_url, '') ?
|
||||||
|
req.add_custom_header('X-API-Key', api_key) ?
|
||||||
|
|
||||||
|
res := req.do() ?
|
||||||
|
data := json.decode(Response<T>, res.text) ?
|
||||||
|
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
// get_repos returns the current list of repos.
|
||||||
|
pub fn get_repos(address string, api_key string) ?map[string]GitRepo {
|
||||||
|
data := send_request<map[string]GitRepo>(http.Method.get, address, '/api/repos', api_key,
|
||||||
|
{}) ?
|
||||||
|
|
||||||
|
return data.data
|
||||||
|
}
|
||||||
|
|
||||||
|
// add_repo adds a new repo to the server.
|
||||||
|
pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response<string> {
|
||||||
|
params := {
|
||||||
|
'url': url
|
||||||
|
'branch': branch
|
||||||
|
'repo': repo
|
||||||
|
'arch': arch.join(',')
|
||||||
|
}
|
||||||
|
data := send_request<string>(http.Method.post, address, '/api/repos', api_key, params) ?
|
||||||
|
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
// remove_repo removes the repo with the given ID from the server.
|
||||||
|
pub fn remove_repo(address string, api_key string, id string) ?Response<string> {
|
||||||
|
data := send_request<string>(http.Method.delete, address, '/api/repos/$id', api_key,
|
||||||
|
{}) ?
|
||||||
|
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
// patch_repo sends a PATCH request to the given repo with the params as
|
||||||
|
// payload.
|
||||||
|
pub fn patch_repo(address string, api_key string, id string, params map[string]string) ?Response<string> {
|
||||||
|
data := send_request<string>(http.Method.patch, address, '/api/repos/$id', api_key,
|
||||||
|
params) ?
|
||||||
|
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,82 @@
|
||||||
|
module git
|
||||||
|
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
|
||||||
|
pub struct GitRepo {
|
||||||
|
pub mut:
|
||||||
|
// URL of the Git repository
|
||||||
|
url string
|
||||||
|
// Branch of the Git repository to use
|
||||||
|
branch string
|
||||||
|
// On which architectures the package is allowed to be built. In reality,
|
||||||
|
// this controls which builders will periodically build the image.
|
||||||
|
arch []string
|
||||||
|
// Which repo the builder should publish packages to
|
||||||
|
repo string
|
||||||
|
}
|
||||||
|
|
||||||
|
// patch_from_params patches a GitRepo from a map[string]string, usually
|
||||||
|
// provided from a web.App's params
|
||||||
|
pub fn (mut r GitRepo) patch_from_params(params map[string]string) {
|
||||||
|
$for field in GitRepo.fields {
|
||||||
|
if field.name in params {
|
||||||
|
$if field.typ is string {
|
||||||
|
r.$(field.name) = params[field.name]
|
||||||
|
// This specific type check is needed for the compiler to ensure
|
||||||
|
// our types are correct
|
||||||
|
} $else $if field.typ is []string {
|
||||||
|
r.$(field.name) = params[field.name].split(',')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// read_repos reads the provided path & parses it into a map of GitRepo's.
|
||||||
|
pub fn read_repos(path string) ?map[string]GitRepo {
|
||||||
|
if !os.exists(path) {
|
||||||
|
mut f := os.create(path) ?
|
||||||
|
|
||||||
|
defer {
|
||||||
|
f.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
f.write_string('{}') ?
|
||||||
|
|
||||||
|
return {}
|
||||||
|
}
|
||||||
|
|
||||||
|
content := os.read_file(path) ?
|
||||||
|
res := json.decode(map[string]GitRepo, content) ?
|
||||||
|
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
// write_repos writes a map of GitRepo's back to disk given the provided path.
|
||||||
|
pub fn write_repos(path string, repos &map[string]GitRepo) ? {
|
||||||
|
mut f := os.create(path) ?
|
||||||
|
|
||||||
|
defer {
|
||||||
|
f.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
value := json.encode(repos)
|
||||||
|
f.write_string(value) ?
|
||||||
|
}
|
||||||
|
|
||||||
|
// repo_from_params creates a GitRepo from a map[string]string, usually
|
||||||
|
// provided from a web.App's params
|
||||||
|
pub fn repo_from_params(params map[string]string) ?GitRepo {
|
||||||
|
mut repo := GitRepo{}
|
||||||
|
|
||||||
|
// If we're creating a new GitRepo, we want all fields to be present before
|
||||||
|
// "patching".
|
||||||
|
$for field in GitRepo.fields {
|
||||||
|
if field.name !in params {
|
||||||
|
return error('Missing parameter: ${field.name}.')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
repo.patch_from_params(params)
|
||||||
|
|
||||||
|
return repo
|
||||||
|
}
|
||||||
120
src/main.v
120
src/main.v
|
|
@ -1,103 +1,33 @@
|
||||||
module main
|
module main
|
||||||
|
|
||||||
import web
|
|
||||||
import os
|
import os
|
||||||
import log
|
import server
|
||||||
import io
|
import cli
|
||||||
import repo
|
import build
|
||||||
|
import git
|
||||||
const port = 8000
|
|
||||||
|
|
||||||
const buf_size = 1_000_000
|
|
||||||
|
|
||||||
const db_name = 'pieter.db'
|
|
||||||
|
|
||||||
struct App {
|
|
||||||
web.Context
|
|
||||||
pub:
|
|
||||||
api_key string [required; web_global]
|
|
||||||
dl_dir string [required; web_global]
|
|
||||||
pub mut:
|
|
||||||
repo repo.Repo [required; web_global]
|
|
||||||
}
|
|
||||||
|
|
||||||
[noreturn]
|
|
||||||
fn exit_with_message(code int, msg string) {
|
|
||||||
eprintln(msg)
|
|
||||||
exit(code)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? {
|
|
||||||
mut file := os.create(path) ?
|
|
||||||
defer {
|
|
||||||
file.close()
|
|
||||||
}
|
|
||||||
|
|
||||||
mut buf := []byte{len: buf_size}
|
|
||||||
mut bytes_left := length
|
|
||||||
|
|
||||||
// Repeat as long as the stream still has data
|
|
||||||
for bytes_left > 0 {
|
|
||||||
// TODO check if just breaking here is safe
|
|
||||||
bytes_read := reader.read(mut buf) or { break }
|
|
||||||
bytes_left -= bytes_read
|
|
||||||
|
|
||||||
mut to_write := bytes_read
|
|
||||||
|
|
||||||
for to_write > 0 {
|
|
||||||
// TODO don't just loop infinitely here
|
|
||||||
bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue }
|
|
||||||
|
|
||||||
to_write = to_write - bytes_written
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
// Configure logger
|
mut app := cli.Command{
|
||||||
log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' }
|
name: 'vieter'
|
||||||
log_level := log.level_from_tag(log_level_str) or {
|
description: 'Vieter is a lightweight implementation of an Arch repository server.'
|
||||||
exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
|
version: '0.2.0'
|
||||||
}
|
flags: [
|
||||||
log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' }
|
cli.Flag{
|
||||||
|
flag: cli.FlagType.string
|
||||||
mut logger := log.Log{
|
name: 'config-file'
|
||||||
level: log_level
|
abbrev: 'f'
|
||||||
|
description: 'Location of Vieter config file; defaults to ~/.vieterrc.'
|
||||||
|
global: true
|
||||||
|
default_value: [os.expand_tilde_to_home('~/.vieterrc')]
|
||||||
|
},
|
||||||
|
]
|
||||||
|
commands: [
|
||||||
|
server.cmd(),
|
||||||
|
build.cmd(),
|
||||||
|
git.cmd(),
|
||||||
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.set_full_logpath(log_file)
|
app.setup()
|
||||||
logger.log_to_console_too()
|
app.parse(os.args)
|
||||||
|
|
||||||
defer {
|
|
||||||
logger.info('Flushing log file')
|
|
||||||
logger.flush()
|
|
||||||
logger.close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Configure web server
|
|
||||||
key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') }
|
|
||||||
repo_dir := os.getenv_opt('REPO_DIR') or {
|
|
||||||
exit_with_message(1, 'No repo directory was configured.')
|
|
||||||
}
|
|
||||||
pkg_dir := os.getenv_opt('PKG_DIR') or {
|
|
||||||
exit_with_message(1, 'No package directory was configured.')
|
|
||||||
}
|
|
||||||
dl_dir := os.getenv_opt('DOWNLOAD_DIR') or {
|
|
||||||
exit_with_message(1, 'No download directory was configured.')
|
|
||||||
}
|
|
||||||
|
|
||||||
// This also creates the directories if needed
|
|
||||||
repo := repo.new(repo_dir, pkg_dir) or {
|
|
||||||
logger.error(err.msg)
|
|
||||||
exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') }
|
|
||||||
|
|
||||||
web.run(&App{
|
|
||||||
logger: logger
|
|
||||||
api_key: key
|
|
||||||
dl_dir: dl_dir
|
|
||||||
repo: repo
|
|
||||||
}, port)
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -72,14 +72,11 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo {
|
||||||
'pkgbase' { pkg_info.base = value }
|
'pkgbase' { pkg_info.base = value }
|
||||||
'pkgver' { pkg_info.version = value }
|
'pkgver' { pkg_info.version = value }
|
||||||
'pkgdesc' { pkg_info.description = value }
|
'pkgdesc' { pkg_info.description = value }
|
||||||
'csize' { continue }
|
|
||||||
'size' { pkg_info.size = value.int() }
|
'size' { pkg_info.size = value.int() }
|
||||||
'url' { pkg_info.url = value }
|
'url' { pkg_info.url = value }
|
||||||
'arch' { pkg_info.arch = value }
|
'arch' { pkg_info.arch = value }
|
||||||
'builddate' { pkg_info.build_date = value.int() }
|
'builddate' { pkg_info.build_date = value.int() }
|
||||||
'packager' { pkg_info.packager = value }
|
'packager' { pkg_info.packager = value }
|
||||||
'md5sum' { continue }
|
|
||||||
'sha256sum' { continue }
|
|
||||||
'pgpsig' { pkg_info.pgpsig = value }
|
'pgpsig' { pkg_info.pgpsig = value }
|
||||||
'pgpsigsize' { pkg_info.pgpsigsize = value.int() }
|
'pgpsigsize' { pkg_info.pgpsigsize = value.int() }
|
||||||
// Array values
|
// Array values
|
||||||
|
|
@ -92,16 +89,19 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo {
|
||||||
'optdepend' { pkg_info.optdepends << value }
|
'optdepend' { pkg_info.optdepends << value }
|
||||||
'makedepend' { pkg_info.makedepends << value }
|
'makedepend' { pkg_info.makedepends << value }
|
||||||
'checkdepend' { pkg_info.checkdepends << value }
|
'checkdepend' { pkg_info.checkdepends << value }
|
||||||
else { return error("Invalid key '$key'.") }
|
// There's no real point in trying to exactly manage which fields
|
||||||
|
// are allowed, so we just ignore any we don't explicitely need for
|
||||||
|
// in the db file
|
||||||
|
else { continue }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return pkg_info
|
return pkg_info
|
||||||
}
|
}
|
||||||
|
|
||||||
// read_pkg extracts the file list & .PKGINFO contents from an archive
|
// read_pkg_archive extracts the file list & .PKGINFO contents from an archive
|
||||||
// NOTE: this command currently only supports zstd-compressed tarballs
|
// NOTE: this command only supports zstd-, xz- & gzip-compressed tarballs.
|
||||||
pub fn read_pkg(pkg_path string) ?Pkg {
|
pub fn read_pkg_archive(pkg_path string) ?Pkg {
|
||||||
if !os.is_file(pkg_path) {
|
if !os.is_file(pkg_path) {
|
||||||
return error("'$pkg_path' doesn't exist or isn't a file.")
|
return error("'$pkg_path' doesn't exist or isn't a file.")
|
||||||
}
|
}
|
||||||
|
|
@ -112,6 +112,7 @@ pub fn read_pkg(pkg_path string) ?Pkg {
|
||||||
// Sinds 2020, all newly built Arch packages use zstd
|
// Sinds 2020, all newly built Arch packages use zstd
|
||||||
C.archive_read_support_filter_zstd(a)
|
C.archive_read_support_filter_zstd(a)
|
||||||
C.archive_read_support_filter_gzip(a)
|
C.archive_read_support_filter_gzip(a)
|
||||||
|
C.archive_read_support_filter_xz(a)
|
||||||
// The content should always be a tarball
|
// The content should always be a tarball
|
||||||
C.archive_read_support_format_tar(a)
|
C.archive_read_support_format_tar(a)
|
||||||
|
|
||||||
|
|
@ -190,6 +191,7 @@ pub fn (pkg &Pkg) filename() string {
|
||||||
ext := match pkg.compression {
|
ext := match pkg.compression {
|
||||||
0 { '.tar' }
|
0 { '.tar' }
|
||||||
1 { '.tar.gz' }
|
1 { '.tar.gz' }
|
||||||
|
6 { '.tar.xz' }
|
||||||
14 { '.tar.zst' }
|
14 { '.tar.zst' }
|
||||||
else { panic("Another compression code shouldn't be possible. Faulty code: $pkg.compression") }
|
else { panic("Another compression code shouldn't be possible. Faulty code: $pkg.compression") }
|
||||||
}
|
}
|
||||||
|
|
|
||||||
190
src/repo/repo.v
190
src/repo/repo.v
|
|
@ -2,22 +2,23 @@ module repo
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import package
|
import package
|
||||||
|
import util
|
||||||
|
|
||||||
// Dummy struct to work around the fact that you can only share structs, maps &
|
// Manages a group of repositories. Each repository contains one or more
|
||||||
// arrays
|
// arch-repositories, each of which represent a specific architecture.
|
||||||
pub struct Dummy {
|
pub struct RepoGroupManager {
|
||||||
x int
|
|
||||||
}
|
|
||||||
|
|
||||||
// This struct manages a single repository.
|
|
||||||
pub struct Repo {
|
|
||||||
mut:
|
mut:
|
||||||
mutex shared Dummy
|
mutex shared util.Dummy
|
||||||
pub:
|
pub:
|
||||||
// Where to store repository files
|
// Where to store repositories' files
|
||||||
repo_dir string [required]
|
repos_dir string [required]
|
||||||
// Where to find packages; packages are expected to all be in the same directory
|
// Where packages are stored; each arch-repository gets its own
|
||||||
|
// subdirectory
|
||||||
pkg_dir string [required]
|
pkg_dir string [required]
|
||||||
|
// The default architecture to use for a repository. Whenever a package of
|
||||||
|
// arch "any" is added to a repo, it will also be added to this
|
||||||
|
// architecture.
|
||||||
|
default_arch string [required]
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct RepoAddResult {
|
pub struct RepoAddResult {
|
||||||
|
|
@ -26,55 +27,117 @@ pub:
|
||||||
pkg &package.Pkg [required]
|
pkg &package.Pkg [required]
|
||||||
}
|
}
|
||||||
|
|
||||||
// new creates a new Repo & creates the directories as needed
|
// new creates a new RepoGroupManager & creates the directories as needed
|
||||||
pub fn new(repo_dir string, pkg_dir string) ?Repo {
|
pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupManager {
|
||||||
if !os.is_dir(repo_dir) {
|
if !os.is_dir(repos_dir) {
|
||||||
os.mkdir_all(repo_dir) or { return error('Failed to create repo directory: $err.msg') }
|
os.mkdir_all(repos_dir) or { return error('Failed to create repos directory: $err.msg') }
|
||||||
}
|
}
|
||||||
|
|
||||||
if !os.is_dir(pkg_dir) {
|
if !os.is_dir(pkg_dir) {
|
||||||
os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg') }
|
os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg') }
|
||||||
}
|
}
|
||||||
|
|
||||||
return Repo{
|
return RepoGroupManager{
|
||||||
repo_dir: repo_dir
|
repos_dir: repos_dir
|
||||||
pkg_dir: pkg_dir
|
pkg_dir: pkg_dir
|
||||||
|
default_arch: default_arch
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// add_from_path adds a package from an arbitrary path & moves it into the pkgs
|
// add_pkg_from_path adds a package to a given repo, given the file path to the
|
||||||
// directory if necessary.
|
// pkg archive. It's a wrapper around add_pkg_in_repo that parses the archive
|
||||||
pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult {
|
// file, passes the result to add_pkg_in_repo, and hard links the archive to
|
||||||
pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') }
|
// the right subdirectories in r.pkg_dir if it was successfully added.
|
||||||
|
pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult {
|
||||||
|
pkg := package.read_pkg_archive(pkg_path) or {
|
||||||
|
return error('Failed to read package file: $err.msg')
|
||||||
|
}
|
||||||
|
|
||||||
added := r.add(pkg) ?
|
added := r.add_pkg_in_repo(repo, pkg) ?
|
||||||
|
|
||||||
// If the add was successful, we move the file to the packages directory
|
// If the add was successful, we move the file to the packages directory
|
||||||
if added {
|
for arch in added {
|
||||||
dest_path := os.real_path(os.join_path_single(r.pkg_dir, pkg.filename()))
|
repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch))
|
||||||
|
dest_path := os.join_path_single(repo_pkg_path, pkg.filename())
|
||||||
|
|
||||||
// Only move the file if it's not already in the package directory
|
os.mkdir_all(repo_pkg_path) ?
|
||||||
if dest_path != os.real_path(pkg_path) {
|
|
||||||
os.mv(pkg_path, dest_path) ?
|
// We create hard links so that "any" arch packages aren't stored
|
||||||
}
|
// multiple times
|
||||||
|
os.link(pkg_path, dest_path) ?
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// After linking, we can remove the original file
|
||||||
|
os.rm(pkg_path) ?
|
||||||
|
|
||||||
return RepoAddResult{
|
return RepoAddResult{
|
||||||
added: added
|
added: added.len > 0
|
||||||
pkg: &pkg
|
pkg: &pkg
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// add adds a given Pkg to the repository
|
// add_pkg_in_repo adds a package to a given repo. This function is responsible
|
||||||
fn (r &Repo) add(pkg &package.Pkg) ?bool {
|
// for inspecting the package architecture. If said architecture is 'any', the
|
||||||
pkg_dir := r.pkg_path(pkg)
|
// package is added to each arch-repository within the given repo. A package of
|
||||||
|
// architecture 'any' is always added to the arch-repo defined by
|
||||||
// We can't add the same package twice
|
// r.default_arch. If this arch-repo doesn't exist yet, it is created. If the
|
||||||
if os.exists(pkg_dir) {
|
// architecture isn't 'any', the package is only added to the specific
|
||||||
return false
|
// architecture.
|
||||||
|
fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]string {
|
||||||
|
// A package not of arch 'any' can be handled easily by adding it to the
|
||||||
|
// respective repo
|
||||||
|
if pkg.info.arch != 'any' {
|
||||||
|
if r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) ? {
|
||||||
|
return [pkg.info.arch]
|
||||||
|
} else {
|
||||||
|
return []
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
os.mkdir(pkg_dir) or { return error('Failed to create package directory.') }
|
mut arch_repos := []string{}
|
||||||
|
|
||||||
|
// If it is an "any" package, the package gets added to every currently
|
||||||
|
// present arch-repo. It will always get added to the r.default_arch repo,
|
||||||
|
// even if no or multiple others are present.
|
||||||
|
repo_dir := os.join_path_single(r.repos_dir, repo)
|
||||||
|
|
||||||
|
// If this is the first package that's added to the repo, the directory
|
||||||
|
// won't exist yet
|
||||||
|
if os.exists(repo_dir) {
|
||||||
|
arch_repos = os.ls(repo_dir) ?
|
||||||
|
}
|
||||||
|
|
||||||
|
// The default_arch should always be updated when a package with arch 'any'
|
||||||
|
// is added.
|
||||||
|
if !arch_repos.contains(r.default_arch) {
|
||||||
|
arch_repos << r.default_arch
|
||||||
|
}
|
||||||
|
|
||||||
|
mut added := []string{}
|
||||||
|
|
||||||
|
// We add the package to each repository. If any of the repositories
|
||||||
|
// return true, the result of the function is also true.
|
||||||
|
for arch in arch_repos {
|
||||||
|
if r.add_pkg_in_arch_repo(repo, arch, pkg) ? {
|
||||||
|
added << arch
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return added
|
||||||
|
}
|
||||||
|
|
||||||
|
// add_pkg_in_arch_repo is the function that actually adds a package to a given
|
||||||
|
// arch-repo. It records the package's data in the arch-repo's desc & files
|
||||||
|
// files, and afterwards updates the db & files archives to reflect these
|
||||||
|
// changes. The function returns false if the package was already present in
|
||||||
|
// the repo, and true otherwise.
|
||||||
|
fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool {
|
||||||
|
pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version')
|
||||||
|
|
||||||
|
// Remove the previous version of the package, if present
|
||||||
|
r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false) ?
|
||||||
|
|
||||||
|
os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') }
|
||||||
|
|
||||||
os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or {
|
os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or {
|
||||||
os.rmdir_all(pkg_dir) ?
|
os.rmdir_all(pkg_dir) ?
|
||||||
|
|
@ -87,12 +150,55 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool {
|
||||||
return error('Failed to write files file.')
|
return error('Failed to write files file.')
|
||||||
}
|
}
|
||||||
|
|
||||||
r.sync() ?
|
r.sync(repo, arch) ?
|
||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns the path where the given package's desc & files files are stored
|
// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It
|
||||||
fn (r &Repo) pkg_path(pkg &package.Pkg) string {
|
// returns false if the package wasn't present in the database. It also
|
||||||
return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version')
|
// optionally re-syncs the repo archives.
|
||||||
|
fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool {
|
||||||
|
repo_dir := os.join_path(r.repos_dir, repo, arch)
|
||||||
|
|
||||||
|
// If the repository doesn't exist yet, the result is automatically false
|
||||||
|
if !os.exists(repo_dir) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// We iterate over every directory in the repo dir
|
||||||
|
// TODO filter so we only check directories
|
||||||
|
for d in os.ls(repo_dir) ? {
|
||||||
|
// Because a repository only allows a single version of each package,
|
||||||
|
// we need only compare whether the name of the package is the same,
|
||||||
|
// not the version.
|
||||||
|
name := d.split('-')#[..-2].join('-')
|
||||||
|
|
||||||
|
if name == pkg_name {
|
||||||
|
// We lock the mutex here to prevent other routines from creating a
|
||||||
|
// new archive while we remove an entry
|
||||||
|
lock r.mutex {
|
||||||
|
os.rmdir_all(os.join_path_single(repo_dir, d)) ?
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also remove the package archive
|
||||||
|
repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch)
|
||||||
|
|
||||||
|
archives := os.ls(repo_pkg_dir) ?.filter(it.split('-')#[..-3].join('-') == name)
|
||||||
|
|
||||||
|
for archive_name in archives {
|
||||||
|
full_path := os.join_path_single(repo_pkg_dir, archive_name)
|
||||||
|
os.rm(full_path) ?
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sync the db archives if requested
|
||||||
|
if sync {
|
||||||
|
r.sync(repo, arch) ?
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -30,8 +30,9 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri
|
||||||
}
|
}
|
||||||
|
|
||||||
// Re-generate the repo archive files
|
// Re-generate the repo archive files
|
||||||
fn (r &Repo) sync() ? {
|
fn (r &RepoGroupManager) sync(repo string, arch string) ? {
|
||||||
// TODO also write files archive
|
subrepo_path := os.join_path(r.repos_dir, repo, arch)
|
||||||
|
|
||||||
lock r.mutex {
|
lock r.mutex {
|
||||||
a_db := C.archive_write_new()
|
a_db := C.archive_write_new()
|
||||||
a_files := C.archive_write_new()
|
a_files := C.archive_write_new()
|
||||||
|
|
@ -44,18 +45,18 @@ fn (r &Repo) sync() ? {
|
||||||
C.archive_write_add_filter_gzip(a_files)
|
C.archive_write_add_filter_gzip(a_files)
|
||||||
C.archive_write_set_format_pax_restricted(a_files)
|
C.archive_write_set_format_pax_restricted(a_files)
|
||||||
|
|
||||||
db_path := os.join_path_single(r.repo_dir, 'vieter.db.tar.gz')
|
db_path := os.join_path_single(subrepo_path, '${repo}.db.tar.gz')
|
||||||
files_path := os.join_path_single(r.repo_dir, 'vieter.files.tar.gz')
|
files_path := os.join_path_single(subrepo_path, '${repo}.files.tar.gz')
|
||||||
|
|
||||||
C.archive_write_open_filename(a_db, &char(db_path.str))
|
C.archive_write_open_filename(a_db, &char(db_path.str))
|
||||||
C.archive_write_open_filename(a_files, &char(files_path.str))
|
C.archive_write_open_filename(a_files, &char(files_path.str))
|
||||||
|
|
||||||
// Iterate over each directory
|
// Iterate over each directory
|
||||||
for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir,
|
for d in os.ls(subrepo_path) ?.filter(os.is_dir(os.join_path_single(subrepo_path,
|
||||||
it))) {
|
it))) {
|
||||||
// desc
|
// desc
|
||||||
mut inner_path := os.join_path_single(d, 'desc')
|
mut inner_path := os.join_path_single(d, 'desc')
|
||||||
mut actual_path := os.join_path_single(r.repo_dir, inner_path)
|
mut actual_path := os.join_path_single(subrepo_path, inner_path)
|
||||||
|
|
||||||
archive_add_entry(a_db, entry, actual_path, inner_path)
|
archive_add_entry(a_db, entry, actual_path, inner_path)
|
||||||
archive_add_entry(a_files, entry, actual_path, inner_path)
|
archive_add_entry(a_files, entry, actual_path, inner_path)
|
||||||
|
|
@ -64,7 +65,7 @@ fn (r &Repo) sync() ? {
|
||||||
|
|
||||||
// files
|
// files
|
||||||
inner_path = os.join_path_single(d, 'files')
|
inner_path = os.join_path_single(d, 'files')
|
||||||
actual_path = os.join_path_single(r.repo_dir, inner_path)
|
actual_path = os.join_path_single(subrepo_path, inner_path)
|
||||||
|
|
||||||
archive_add_entry(a_files, entry, actual_path, inner_path)
|
archive_add_entry(a_files, entry, actual_path, inner_path)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
module response
|
||||||
|
|
||||||
|
pub struct Response<T> {
|
||||||
|
pub:
|
||||||
|
message string
|
||||||
|
data T
|
||||||
|
}
|
||||||
|
|
||||||
|
// new_response constructs a new Response<String> object with the given message
|
||||||
|
// & an empty data field.
|
||||||
|
pub fn new_response(message string) Response<string> {
|
||||||
|
return Response<string>{
|
||||||
|
message: message
|
||||||
|
data: ''
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// new_data_response<T> constructs a new Response<T> object with the given data
|
||||||
|
// & an empty message field.
|
||||||
|
pub fn new_data_response<T>(data T) Response<T> {
|
||||||
|
return Response<T>{
|
||||||
|
message: ''
|
||||||
|
data: data
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// new_full_response<T> constructs a new Response<T> object with the given
|
||||||
|
// message & data.
|
||||||
|
pub fn new_full_response<T>(message string, data T) Response<T> {
|
||||||
|
return Response<T>{
|
||||||
|
message: message
|
||||||
|
data: data
|
||||||
|
}
|
||||||
|
}
|
||||||
103
src/routes.v
103
src/routes.v
|
|
@ -1,103 +0,0 @@
|
||||||
module main
|
|
||||||
|
|
||||||
import web
|
|
||||||
import os
|
|
||||||
import repo
|
|
||||||
import time
|
|
||||||
import rand
|
|
||||||
|
|
||||||
const prefixes = ['B', 'KB', 'MB', 'GB']
|
|
||||||
|
|
||||||
// pretty_bytes converts a byte count to human-readable version
|
|
||||||
fn pretty_bytes(bytes int) string {
|
|
||||||
mut i := 0
|
|
||||||
mut n := f32(bytes)
|
|
||||||
|
|
||||||
for n >= 1024 {
|
|
||||||
i++
|
|
||||||
n /= 1024
|
|
||||||
}
|
|
||||||
|
|
||||||
return '${n:.2}${prefixes[i]}'
|
|
||||||
}
|
|
||||||
|
|
||||||
fn is_pkg_name(s string) bool {
|
|
||||||
return s.contains('.pkg')
|
|
||||||
}
|
|
||||||
|
|
||||||
// healthcheck just returns a string, but can be used to quickly check if the
|
|
||||||
// server is still responsive.
|
|
||||||
['/health'; get]
|
|
||||||
pub fn (mut app App) healthcheck() web.Result {
|
|
||||||
return app.text('Healthy')
|
|
||||||
}
|
|
||||||
|
|
||||||
// get_root handles a GET request for a file on the root
|
|
||||||
['/:filename'; get]
|
|
||||||
fn (mut app App) get_root(filename string) web.Result {
|
|
||||||
mut full_path := ''
|
|
||||||
|
|
||||||
if filename.ends_with('.db') || filename.ends_with('.files') {
|
|
||||||
full_path = os.join_path_single(app.repo.repo_dir, '${filename}.tar.gz')
|
|
||||||
} else if filename.ends_with('.db.tar.gz') || filename.ends_with('.files.tar.gz') {
|
|
||||||
full_path = os.join_path_single(app.repo.repo_dir, '$filename')
|
|
||||||
} else {
|
|
||||||
full_path = os.join_path_single(app.repo.pkg_dir, filename)
|
|
||||||
}
|
|
||||||
|
|
||||||
return app.file(full_path)
|
|
||||||
}
|
|
||||||
|
|
||||||
['/publish'; post]
|
|
||||||
fn (mut app App) put_package() web.Result {
|
|
||||||
if !app.is_authorized() {
|
|
||||||
return app.text('Unauthorized.')
|
|
||||||
}
|
|
||||||
|
|
||||||
mut pkg_path := ''
|
|
||||||
|
|
||||||
if length := app.req.header.get(.content_length) {
|
|
||||||
// Generate a random filename for the temp file
|
|
||||||
pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4())
|
|
||||||
|
|
||||||
for os.exists(pkg_path) {
|
|
||||||
pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4())
|
|
||||||
}
|
|
||||||
|
|
||||||
app.ldebug("Uploading $length bytes (${pretty_bytes(length.int())}) to '$pkg_path'.")
|
|
||||||
|
|
||||||
// This is used to time how long it takes to upload a file
|
|
||||||
mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true })
|
|
||||||
|
|
||||||
reader_to_file(mut app.reader, length.int(), pkg_path) or {
|
|
||||||
app.lwarn("Failed to upload '$pkg_path'")
|
|
||||||
|
|
||||||
return app.text('Failed to upload file.')
|
|
||||||
}
|
|
||||||
|
|
||||||
sw.stop()
|
|
||||||
app.ldebug("Upload of '$pkg_path' completed in ${sw.elapsed().seconds():.3}s.")
|
|
||||||
} else {
|
|
||||||
app.lwarn('Tried to upload package without specifying a Content-Length.')
|
|
||||||
return app.text("Content-Type header isn't set.")
|
|
||||||
}
|
|
||||||
|
|
||||||
res := app.repo.add_from_path(pkg_path) or {
|
|
||||||
app.lerror('Error while adding package: $err.msg')
|
|
||||||
|
|
||||||
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }
|
|
||||||
|
|
||||||
return app.text('Failed to add package.')
|
|
||||||
}
|
|
||||||
if !res.added {
|
|
||||||
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }
|
|
||||||
|
|
||||||
app.lwarn("Duplicate package '$res.pkg.full_name()'.")
|
|
||||||
|
|
||||||
return app.text('File already exists.')
|
|
||||||
}
|
|
||||||
|
|
||||||
app.linfo("Added '$res.pkg.full_name()' to repository.")
|
|
||||||
|
|
||||||
return app.text('Package added successfully.')
|
|
||||||
}
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
module main
|
module server
|
||||||
|
|
||||||
import net.http
|
import net.http
|
||||||
|
|
||||||
|
|
@ -7,5 +7,5 @@ fn (mut app App) is_authorized() bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
return x_header.trim_space() == app.api_key
|
return x_header.trim_space() == app.conf.api_key
|
||||||
}
|
}
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
module server
|
||||||
|
|
||||||
|
import cli
|
||||||
|
import env
|
||||||
|
|
||||||
|
struct Config {
|
||||||
|
pub:
|
||||||
|
log_level string = 'WARN'
|
||||||
|
log_file string = 'vieter.log'
|
||||||
|
pkg_dir string
|
||||||
|
download_dir string
|
||||||
|
api_key string
|
||||||
|
repos_dir string
|
||||||
|
repos_file string
|
||||||
|
default_arch string
|
||||||
|
}
|
||||||
|
|
||||||
|
// cmd returns the cli submodule that handles starting the server
|
||||||
|
pub fn cmd() cli.Command {
|
||||||
|
return cli.Command{
|
||||||
|
name: 'server'
|
||||||
|
description: 'Start the Vieter server.'
|
||||||
|
execute: fn (cmd cli.Command) ? {
|
||||||
|
config_file := cmd.flags.get_string('config-file') ?
|
||||||
|
conf := env.load<Config>(config_file) ?
|
||||||
|
|
||||||
|
server(conf) ?
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,141 @@
|
||||||
|
module server
|
||||||
|
|
||||||
|
import web
|
||||||
|
import git
|
||||||
|
import net.http
|
||||||
|
import rand
|
||||||
|
import response { new_data_response, new_response }
|
||||||
|
|
||||||
|
const repos_file = 'repos.json'
|
||||||
|
|
||||||
|
['/api/repos'; get]
|
||||||
|
fn (mut app App) get_repos() web.Result {
|
||||||
|
if !app.is_authorized() {
|
||||||
|
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
repos := rlock app.git_mutex {
|
||||||
|
git.read_repos(app.conf.repos_file) or {
|
||||||
|
app.lerror('Failed to read repos file: $err.msg')
|
||||||
|
|
||||||
|
return app.status(http.Status.internal_server_error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return app.json(http.Status.ok, new_data_response(repos))
|
||||||
|
}
|
||||||
|
|
||||||
|
['/api/repos/:id'; get]
|
||||||
|
fn (mut app App) get_single_repo(id string) web.Result {
|
||||||
|
if !app.is_authorized() {
|
||||||
|
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
repos := rlock app.git_mutex {
|
||||||
|
git.read_repos(app.conf.repos_file) or {
|
||||||
|
app.lerror('Failed to read repos file.')
|
||||||
|
|
||||||
|
return app.status(http.Status.internal_server_error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if id !in repos {
|
||||||
|
return app.not_found()
|
||||||
|
}
|
||||||
|
|
||||||
|
repo := repos[id]
|
||||||
|
|
||||||
|
return app.json(http.Status.ok, new_data_response(repo))
|
||||||
|
}
|
||||||
|
|
||||||
|
['/api/repos'; post]
|
||||||
|
fn (mut app App) post_repo() web.Result {
|
||||||
|
if !app.is_authorized() {
|
||||||
|
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
new_repo := git.repo_from_params(app.query) or {
|
||||||
|
return app.json(http.Status.bad_request, new_response(err.msg))
|
||||||
|
}
|
||||||
|
|
||||||
|
id := rand.uuid_v4()
|
||||||
|
|
||||||
|
mut repos := rlock app.git_mutex {
|
||||||
|
git.read_repos(app.conf.repos_file) or {
|
||||||
|
app.lerror('Failed to read repos file.')
|
||||||
|
|
||||||
|
return app.status(http.Status.internal_server_error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// We need to check for duplicates
|
||||||
|
for _, repo in repos {
|
||||||
|
if repo == new_repo {
|
||||||
|
return app.json(http.Status.bad_request, new_response('Duplicate repository.'))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
repos[id] = new_repo
|
||||||
|
|
||||||
|
lock app.git_mutex {
|
||||||
|
git.write_repos(app.conf.repos_file, &repos) or {
|
||||||
|
return app.status(http.Status.internal_server_error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return app.json(http.Status.ok, new_response('Repo added successfully.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
['/api/repos/:id'; delete]
|
||||||
|
fn (mut app App) delete_repo(id string) web.Result {
|
||||||
|
if !app.is_authorized() {
|
||||||
|
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
mut repos := rlock app.git_mutex {
|
||||||
|
git.read_repos(app.conf.repos_file) or {
|
||||||
|
app.lerror('Failed to read repos file.')
|
||||||
|
|
||||||
|
return app.status(http.Status.internal_server_error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if id !in repos {
|
||||||
|
return app.not_found()
|
||||||
|
}
|
||||||
|
|
||||||
|
repos.delete(id)
|
||||||
|
|
||||||
|
lock app.git_mutex {
|
||||||
|
git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) }
|
||||||
|
}
|
||||||
|
|
||||||
|
return app.json(http.Status.ok, new_response('Repo removed successfully.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
['/api/repos/:id'; patch]
|
||||||
|
fn (mut app App) patch_repo(id string) web.Result {
|
||||||
|
if !app.is_authorized() {
|
||||||
|
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
mut repos := rlock app.git_mutex {
|
||||||
|
git.read_repos(app.conf.repos_file) or {
|
||||||
|
app.lerror('Failed to read repos file.')
|
||||||
|
|
||||||
|
return app.status(http.Status.internal_server_error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if id !in repos {
|
||||||
|
return app.not_found()
|
||||||
|
}
|
||||||
|
|
||||||
|
repos[id].patch_from_params(app.query)
|
||||||
|
|
||||||
|
lock app.git_mutex {
|
||||||
|
git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) }
|
||||||
|
}
|
||||||
|
|
||||||
|
return app.json(http.Status.ok, new_response('Repo updated successfully.'))
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,108 @@
|
||||||
|
module server
|
||||||
|
|
||||||
|
import web
|
||||||
|
import os
|
||||||
|
import repo
|
||||||
|
import time
|
||||||
|
import rand
|
||||||
|
import util
|
||||||
|
import net.http
|
||||||
|
import response { new_response }
|
||||||
|
|
||||||
|
// healthcheck just returns a string, but can be used to quickly check if the
|
||||||
|
// server is still responsive.
|
||||||
|
['/health'; get]
|
||||||
|
pub fn (mut app App) healthcheck() web.Result {
|
||||||
|
return app.json(http.Status.ok, new_response('Healthy.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
['/:repo/:arch/:filename'; get; head]
|
||||||
|
fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result {
|
||||||
|
mut full_path := ''
|
||||||
|
|
||||||
|
db_exts := ['.db', '.files', '.db.tar.gz', '.files.tar.gz']
|
||||||
|
|
||||||
|
// There's no point in having the ability to serve db archives with wrong
|
||||||
|
// filenames
|
||||||
|
if db_exts.any(filename == '$repo$it') {
|
||||||
|
full_path = os.join_path(app.repo.repos_dir, repo, arch, filename)
|
||||||
|
|
||||||
|
// repo-add does this using symlinks, but we just change the requested
|
||||||
|
// path
|
||||||
|
if !full_path.ends_with('.tar.gz') {
|
||||||
|
full_path += '.tar.gz'
|
||||||
|
}
|
||||||
|
} else if filename.contains('.pkg') {
|
||||||
|
full_path = os.join_path(app.repo.pkg_dir, repo, arch, filename)
|
||||||
|
}
|
||||||
|
// Default behavior is to return the desc file for the package, if present.
|
||||||
|
// This can then also be used by the build system to properly check whether
|
||||||
|
// a package is present in an arch-repo.
|
||||||
|
else {
|
||||||
|
full_path = os.join_path(app.repo.repos_dir, repo, arch, filename, 'desc')
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scuffed way to respond to HEAD requests
|
||||||
|
if app.req.method == http.Method.head {
|
||||||
|
if os.exists(full_path) {
|
||||||
|
return app.status(http.Status.ok)
|
||||||
|
}
|
||||||
|
|
||||||
|
return app.not_found()
|
||||||
|
}
|
||||||
|
|
||||||
|
return app.file(full_path)
|
||||||
|
}
|
||||||
|
|
||||||
|
['/:repo/publish'; post]
|
||||||
|
fn (mut app App) put_package(repo string) web.Result {
|
||||||
|
if !app.is_authorized() {
|
||||||
|
return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
mut pkg_path := ''
|
||||||
|
|
||||||
|
if length := app.req.header.get(.content_length) {
|
||||||
|
// Generate a random filename for the temp file
|
||||||
|
pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4())
|
||||||
|
|
||||||
|
app.ldebug("Uploading $length bytes (${util.pretty_bytes(length.int())}) to '$pkg_path'.")
|
||||||
|
|
||||||
|
// This is used to time how long it takes to upload a file
|
||||||
|
mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true })
|
||||||
|
|
||||||
|
util.reader_to_file(mut app.reader, length.int(), pkg_path) or {
|
||||||
|
app.lwarn("Failed to upload '$pkg_path'")
|
||||||
|
|
||||||
|
return app.json(http.Status.internal_server_error, new_response('Failed to upload file.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
sw.stop()
|
||||||
|
app.ldebug("Upload of '$pkg_path' completed in ${sw.elapsed().seconds():.3}s.")
|
||||||
|
} else {
|
||||||
|
app.lwarn('Tried to upload package without specifying a Content-Length.')
|
||||||
|
|
||||||
|
// length required
|
||||||
|
return app.status(http.Status.length_required)
|
||||||
|
}
|
||||||
|
|
||||||
|
res := app.repo.add_pkg_from_path(repo, pkg_path) or {
|
||||||
|
app.lerror('Error while adding package: $err.msg')
|
||||||
|
|
||||||
|
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }
|
||||||
|
|
||||||
|
return app.json(http.Status.internal_server_error, new_response('Failed to add package.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
if !res.added {
|
||||||
|
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }
|
||||||
|
|
||||||
|
app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo'.")
|
||||||
|
|
||||||
|
return app.json(http.Status.bad_request, new_response('File already exists.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
app.linfo("Added '$res.pkg.full_name()' to repo '$repo ($res.pkg.info.arch)'.")
|
||||||
|
|
||||||
|
return app.json(http.Status.ok, new_response('Package added successfully.'))
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,61 @@
|
||||||
|
module server
|
||||||
|
|
||||||
|
import web
|
||||||
|
import os
|
||||||
|
import log
|
||||||
|
import repo
|
||||||
|
import util
|
||||||
|
|
||||||
|
const port = 8000
|
||||||
|
|
||||||
|
struct App {
|
||||||
|
web.Context
|
||||||
|
pub:
|
||||||
|
conf Config [required; web_global]
|
||||||
|
pub mut:
|
||||||
|
repo repo.RepoGroupManager [required; web_global]
|
||||||
|
// This is used to claim the file lock on the repos file
|
||||||
|
git_mutex shared util.Dummy
|
||||||
|
}
|
||||||
|
|
||||||
|
// server starts the web server & starts listening for requests
|
||||||
|
pub fn server(conf Config) ? {
|
||||||
|
// Prevent using 'any' as the default arch
|
||||||
|
if conf.default_arch == 'any' {
|
||||||
|
util.exit_with_message(1, "'any' is not allowed as the value for default_arch.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Configure logger
|
||||||
|
log_level := log.level_from_tag(conf.log_level) or {
|
||||||
|
util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
|
||||||
|
}
|
||||||
|
|
||||||
|
mut logger := log.Log{
|
||||||
|
level: log_level
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.set_full_logpath(conf.log_file)
|
||||||
|
logger.log_to_console_too()
|
||||||
|
|
||||||
|
defer {
|
||||||
|
logger.info('Flushing log file')
|
||||||
|
logger.flush()
|
||||||
|
logger.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// This also creates the directories if needed
|
||||||
|
repo := repo.new(conf.repos_dir, conf.pkg_dir, conf.default_arch) or {
|
||||||
|
logger.error(err.msg)
|
||||||
|
exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
os.mkdir_all(conf.download_dir) or {
|
||||||
|
util.exit_with_message(1, 'Failed to create download directory.')
|
||||||
|
}
|
||||||
|
|
||||||
|
web.run(&App{
|
||||||
|
logger: logger
|
||||||
|
conf: conf
|
||||||
|
repo: repo
|
||||||
|
}, server.port)
|
||||||
|
}
|
||||||
59
src/util.v
59
src/util.v
|
|
@ -1,9 +1,55 @@
|
||||||
module util
|
module util
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import io
|
||||||
import crypto.md5
|
import crypto.md5
|
||||||
import crypto.sha256
|
import crypto.sha256
|
||||||
|
|
||||||
|
const reader_buf_size = 1_000_000
|
||||||
|
|
||||||
|
const prefixes = ['B', 'KB', 'MB', 'GB']
|
||||||
|
|
||||||
|
// Dummy struct to work around the fact that you can only share structs, maps &
|
||||||
|
// arrays
|
||||||
|
pub struct Dummy {
|
||||||
|
x int
|
||||||
|
}
|
||||||
|
|
||||||
|
// exit_with_message exits the program with a given status code after having
|
||||||
|
// first printed a specific message to STDERR
|
||||||
|
[noreturn]
|
||||||
|
pub fn exit_with_message(code int, msg string) {
|
||||||
|
eprintln(msg)
|
||||||
|
exit(code)
|
||||||
|
}
|
||||||
|
|
||||||
|
// reader_to_file writes the contents of a BufferedReader to a file
|
||||||
|
pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? {
|
||||||
|
mut file := os.create(path) ?
|
||||||
|
defer {
|
||||||
|
file.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
mut buf := []byte{len: util.reader_buf_size}
|
||||||
|
mut bytes_left := length
|
||||||
|
|
||||||
|
// Repeat as long as the stream still has data
|
||||||
|
for bytes_left > 0 {
|
||||||
|
// TODO check if just breaking here is safe
|
||||||
|
bytes_read := reader.read(mut buf) or { break }
|
||||||
|
bytes_left -= bytes_read
|
||||||
|
|
||||||
|
mut to_write := bytes_read
|
||||||
|
|
||||||
|
for to_write > 0 {
|
||||||
|
// TODO don't just loop infinitely here
|
||||||
|
bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue }
|
||||||
|
|
||||||
|
to_write = to_write - bytes_written
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// hash_file returns the md5 & sha256 hash of a given file
|
// hash_file returns the md5 & sha256 hash of a given file
|
||||||
// TODO actually implement sha256
|
// TODO actually implement sha256
|
||||||
pub fn hash_file(path &string) ?(string, string) {
|
pub fn hash_file(path &string) ?(string, string) {
|
||||||
|
|
@ -32,3 +78,16 @@ pub fn hash_file(path &string) ?(string, string) {
|
||||||
|
|
||||||
return md5sum.checksum().hex(), sha256sum.checksum().hex()
|
return md5sum.checksum().hex(), sha256sum.checksum().hex()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pretty_bytes converts a byte count to human-readable version
|
||||||
|
pub fn pretty_bytes(bytes int) string {
|
||||||
|
mut i := 0
|
||||||
|
mut n := f32(bytes)
|
||||||
|
|
||||||
|
for n >= 1024 {
|
||||||
|
i++
|
||||||
|
n /= 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
return '${n:.2}${util.prefixes[i]}'
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) {
|
||||||
}
|
}
|
||||||
if x.len > 0 {
|
if x.len > 0 {
|
||||||
return IError(http.UnexpectedExtraAttributeError{
|
return IError(http.UnexpectedExtraAttributeError{
|
||||||
msg: 'Encountered unexpected extra attributes: $x'
|
attributes: x
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if methods.len == 0 {
|
if methods.len == 0 {
|
||||||
|
|
@ -49,8 +49,8 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) {
|
||||||
|
|
||||||
fn parse_query_from_url(url urllib.URL) map[string]string {
|
fn parse_query_from_url(url urllib.URL) map[string]string {
|
||||||
mut query := map[string]string{}
|
mut query := map[string]string{}
|
||||||
for k, v in url.query().data {
|
for v in url.query().data {
|
||||||
query[k] = v.data[0]
|
query[v.key] = v.value
|
||||||
}
|
}
|
||||||
return query
|
return query
|
||||||
}
|
}
|
||||||
|
|
|
||||||
206
src/web/web.v
206
src/web/web.v
|
|
@ -12,9 +12,6 @@ import time
|
||||||
import json
|
import json
|
||||||
import log
|
import log
|
||||||
|
|
||||||
// A type which don't get filtered inside templates
|
|
||||||
pub type RawHtml = string
|
|
||||||
|
|
||||||
// A dummy structure that returns from routes to indicate that you actually sent something to a user
|
// A dummy structure that returns from routes to indicate that you actually sent something to a user
|
||||||
[noinit]
|
[noinit]
|
||||||
pub struct Result {}
|
pub struct Result {}
|
||||||
|
|
@ -142,7 +139,7 @@ pub const (
|
||||||
pub struct Context {
|
pub struct Context {
|
||||||
mut:
|
mut:
|
||||||
content_type string = 'text/plain'
|
content_type string = 'text/plain'
|
||||||
status string = '200 OK'
|
status http.Status = http.Status.ok
|
||||||
pub:
|
pub:
|
||||||
// HTTP Request
|
// HTTP Request
|
||||||
req http.Request
|
req http.Request
|
||||||
|
|
@ -186,24 +183,14 @@ struct Route {
|
||||||
path string
|
path string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Defining this method is optional.
|
|
||||||
// init_server is called at server start.
|
|
||||||
// You can use it for initializing globals.
|
|
||||||
pub fn (ctx Context) init_server() {
|
|
||||||
eprintln('init_server() has been deprecated, please init your web app in `fn main()`')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Defining this method is optional.
|
// Defining this method is optional.
|
||||||
// before_request is called before every request (aka middleware).
|
// before_request is called before every request (aka middleware).
|
||||||
// Probably you can use it for check user session cookie or add header.
|
// Probably you can use it for check user session cookie or add header.
|
||||||
pub fn (ctx Context) before_request() {}
|
pub fn (ctx Context) before_request() {}
|
||||||
|
|
||||||
pub struct Cookie {
|
// send_string
|
||||||
name string
|
fn send_string(mut conn net.TcpConn, s string) ? {
|
||||||
value string
|
conn.write(s.bytes()) ?
|
||||||
expires time.Time
|
|
||||||
secure bool
|
|
||||||
http_only bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// send_response_to_client sends a response to the client
|
// send_response_to_client sends a response to the client
|
||||||
|
|
@ -225,34 +212,27 @@ pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bo
|
||||||
text: res
|
text: res
|
||||||
}
|
}
|
||||||
resp.set_version(.v1_1)
|
resp.set_version(.v1_1)
|
||||||
resp.set_status(http.status_from_int(ctx.status.int()))
|
resp.set_status(ctx.status)
|
||||||
send_string(mut ctx.conn, resp.bytestr()) or { return false }
|
send_string(mut ctx.conn, resp.bytestr()) or { return false }
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// html HTTP_OK with s as payload with content-type `text/html`
|
// text responds to a request with some plaintext.
|
||||||
pub fn (mut ctx Context) html(s string) Result {
|
pub fn (mut ctx Context) text(status http.Status, s string) Result {
|
||||||
ctx.send_response_to_client('text/html', s)
|
ctx.status = status
|
||||||
return Result{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// text HTTP_OK with s as payload with content-type `text/plain`
|
|
||||||
pub fn (mut ctx Context) text(s string) Result {
|
|
||||||
ctx.send_response_to_client('text/plain', s)
|
ctx.send_response_to_client('text/plain', s)
|
||||||
|
|
||||||
return Result{}
|
return Result{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// json<T> HTTP_OK with json_s as payload with content-type `application/json`
|
// json<T> HTTP_OK with json_s as payload with content-type `application/json`
|
||||||
pub fn (mut ctx Context) json<T>(j T) Result {
|
pub fn (mut ctx Context) json<T>(status http.Status, j T) Result {
|
||||||
|
ctx.status = status
|
||||||
|
|
||||||
json_s := json.encode(j)
|
json_s := json.encode(j)
|
||||||
ctx.send_response_to_client('application/json', json_s)
|
ctx.send_response_to_client('application/json', json_s)
|
||||||
return Result{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// json_pretty<T> Response HTTP_OK with a pretty-printed JSON result
|
|
||||||
pub fn (mut ctx Context) json_pretty<T>(j T) Result {
|
|
||||||
json_s := json.encode_pretty(j)
|
|
||||||
ctx.send_response_to_client('application/json', json_s)
|
|
||||||
return Result{}
|
return Result{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -302,7 +282,7 @@ pub fn (mut ctx Context) file(f_path string) Result {
|
||||||
header: header.join(web.headers_close)
|
header: header.join(web.headers_close)
|
||||||
}
|
}
|
||||||
resp.set_version(.v1_1)
|
resp.set_version(.v1_1)
|
||||||
resp.set_status(http.status_from_int(ctx.status.int()))
|
resp.set_status(ctx.status)
|
||||||
send_string(mut ctx.conn, resp.bytestr()) or { return Result{} }
|
send_string(mut ctx.conn, resp.bytestr()) or { return Result{} }
|
||||||
|
|
||||||
mut buf := []byte{len: 1_000_000}
|
mut buf := []byte{len: 1_000_000}
|
||||||
|
|
@ -328,10 +308,10 @@ pub fn (mut ctx Context) file(f_path string) Result {
|
||||||
return Result{}
|
return Result{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ok Response HTTP_OK with s as payload
|
// status responds with an empty textual response, essentially only returning
|
||||||
pub fn (mut ctx Context) ok(s string) Result {
|
// the given status code.
|
||||||
ctx.send_response_to_client(ctx.content_type, s)
|
pub fn (mut ctx Context) status(status http.Status) Result {
|
||||||
return Result{}
|
return ctx.text(status, '')
|
||||||
}
|
}
|
||||||
|
|
||||||
// server_error Response a server error
|
// server_error Response a server error
|
||||||
|
|
@ -361,64 +341,7 @@ pub fn (mut ctx Context) redirect(url string) Result {
|
||||||
|
|
||||||
// not_found Send an not_found response
|
// not_found Send an not_found response
|
||||||
pub fn (mut ctx Context) not_found() Result {
|
pub fn (mut ctx Context) not_found() Result {
|
||||||
if ctx.done {
|
return ctx.status(http.Status.not_found)
|
||||||
return Result{}
|
|
||||||
}
|
|
||||||
ctx.done = true
|
|
||||||
send_string(mut ctx.conn, web.http_404.bytestr()) or {}
|
|
||||||
return Result{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// set_cookie Sets a cookie
|
|
||||||
pub fn (mut ctx Context) set_cookie(cookie Cookie) {
|
|
||||||
mut cookie_data := []string{}
|
|
||||||
mut secure := if cookie.secure { 'Secure;' } else { '' }
|
|
||||||
secure += if cookie.http_only { ' HttpOnly' } else { ' ' }
|
|
||||||
cookie_data << secure
|
|
||||||
if cookie.expires.unix > 0 {
|
|
||||||
cookie_data << 'expires=$cookie.expires.utc_string()'
|
|
||||||
}
|
|
||||||
data := cookie_data.join(' ')
|
|
||||||
ctx.add_header('Set-Cookie', '$cookie.name=$cookie.value; $data')
|
|
||||||
}
|
|
||||||
|
|
||||||
// set_content_type Sets the response content type
|
|
||||||
pub fn (mut ctx Context) set_content_type(typ string) {
|
|
||||||
ctx.content_type = typ
|
|
||||||
}
|
|
||||||
|
|
||||||
// set_cookie_with_expire_date Sets a cookie with a `expire_data`
|
|
||||||
pub fn (mut ctx Context) set_cookie_with_expire_date(key string, val string, expire_date time.Time) {
|
|
||||||
ctx.add_header('Set-Cookie', '$key=$val; Secure; HttpOnly; expires=$expire_date.utc_string()')
|
|
||||||
}
|
|
||||||
|
|
||||||
// get_cookie Gets a cookie by a key
|
|
||||||
pub fn (ctx &Context) get_cookie(key string) ?string { // TODO refactor
|
|
||||||
mut cookie_header := ctx.get_header('cookie')
|
|
||||||
if cookie_header == '' {
|
|
||||||
cookie_header = ctx.get_header('Cookie')
|
|
||||||
}
|
|
||||||
cookie_header = ' ' + cookie_header
|
|
||||||
// println('cookie_header="$cookie_header"')
|
|
||||||
// println(ctx.req.header)
|
|
||||||
cookie := if cookie_header.contains(';') {
|
|
||||||
cookie_header.find_between(' $key=', ';')
|
|
||||||
} else {
|
|
||||||
cookie_header.find_between(' $key=', '\r')
|
|
||||||
}
|
|
||||||
if cookie != '' {
|
|
||||||
return cookie.trim_space()
|
|
||||||
}
|
|
||||||
return error('Cookie not found')
|
|
||||||
}
|
|
||||||
|
|
||||||
// set_status Sets the response status
|
|
||||||
pub fn (mut ctx Context) set_status(code int, desc string) {
|
|
||||||
if code < 100 || code > 599 {
|
|
||||||
ctx.status = '500 Internal Server Error'
|
|
||||||
} else {
|
|
||||||
ctx.status = '$code $desc'
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// add_header Adds an header to the response with key and val
|
// add_header Adds an header to the response with key and val
|
||||||
|
|
@ -560,12 +483,6 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T, routes map[string]Route) {
|
||||||
// Calling middleware...
|
// Calling middleware...
|
||||||
app.before_request()
|
app.before_request()
|
||||||
|
|
||||||
// Static handling
|
|
||||||
if serve_if_static<T>(mut app, url) {
|
|
||||||
// successfully served a static file
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Route matching
|
// Route matching
|
||||||
$for method in T.methods {
|
$for method in T.methods {
|
||||||
$if method.return_type is Result {
|
$if method.return_type is Result {
|
||||||
|
|
@ -661,83 +578,6 @@ fn route_matches(url_words []string, route_words []string) ?[]string {
|
||||||
return params
|
return params
|
||||||
}
|
}
|
||||||
|
|
||||||
// serve_if_static<T> checks if request is for a static file and serves it
|
|
||||||
// returns true if we served a static file, false otherwise
|
|
||||||
[manualfree]
|
|
||||||
fn serve_if_static<T>(mut app T, url urllib.URL) bool {
|
|
||||||
// TODO: handle url parameters properly - for now, ignore them
|
|
||||||
static_file := app.static_files[url.path]
|
|
||||||
mime_type := app.static_mime_types[url.path]
|
|
||||||
if static_file == '' || mime_type == '' {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
data := os.read_file(static_file) or {
|
|
||||||
send_string(mut app.conn, web.http_404.bytestr()) or {}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
app.send_response_to_client(mime_type, data)
|
|
||||||
unsafe { data.free() }
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// scan_static_directory makes a static route for each file in a directory
|
|
||||||
fn (mut ctx Context) scan_static_directory(directory_path string, mount_path string) {
|
|
||||||
files := os.ls(directory_path) or { panic(err) }
|
|
||||||
if files.len > 0 {
|
|
||||||
for file in files {
|
|
||||||
full_path := os.join_path(directory_path, file)
|
|
||||||
if os.is_dir(full_path) {
|
|
||||||
ctx.scan_static_directory(full_path, mount_path + '/' + file)
|
|
||||||
} else if file.contains('.') && !file.starts_with('.') && !file.ends_with('.') {
|
|
||||||
ext := os.file_ext(file)
|
|
||||||
// Rudimentary guard against adding files not in mime_types.
|
|
||||||
// Use serve_static directly to add non-standard mime types.
|
|
||||||
if ext in web.mime_types {
|
|
||||||
ctx.serve_static(mount_path + '/' + file, full_path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// handle_static Handles a directory static
|
|
||||||
// If `root` is set the mount path for the dir will be in '/'
|
|
||||||
pub fn (mut ctx Context) handle_static(directory_path string, root bool) bool {
|
|
||||||
if ctx.done || !os.exists(directory_path) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
dir_path := directory_path.trim_space().trim_right('/')
|
|
||||||
mut mount_path := ''
|
|
||||||
if dir_path != '.' && os.is_dir(dir_path) && !root {
|
|
||||||
// Mount point hygene, "./assets" => "/assets".
|
|
||||||
mount_path = '/' + dir_path.trim_left('.').trim('/')
|
|
||||||
}
|
|
||||||
ctx.scan_static_directory(dir_path, mount_path)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// mount_static_folder_at - makes all static files in `directory_path` and inside it, available at http://server/mount_path
|
|
||||||
// For example: suppose you have called .mount_static_folder_at('/var/share/myassets', '/assets'),
|
|
||||||
// and you have a file /var/share/myassets/main.css .
|
|
||||||
// => That file will be available at URL: http://server/assets/main.css .
|
|
||||||
pub fn (mut ctx Context) mount_static_folder_at(directory_path string, mount_path string) bool {
|
|
||||||
if ctx.done || mount_path.len < 1 || mount_path[0] != `/` || !os.exists(directory_path) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
dir_path := directory_path.trim_right('/')
|
|
||||||
ctx.scan_static_directory(dir_path, mount_path[1..])
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// serve_static Serves a file static
|
|
||||||
// `url` is the access path on the site, `file_path` is the real path to the file, `mime_type` is the file type
|
|
||||||
pub fn (mut ctx Context) serve_static(url string, file_path string) {
|
|
||||||
ctx.static_files[url] = file_path
|
|
||||||
// ctx.static_mime_types[url] = mime_type
|
|
||||||
ext := os.file_ext(file_path)
|
|
||||||
ctx.static_mime_types[url] = web.mime_types[ext]
|
|
||||||
}
|
|
||||||
|
|
||||||
// ip Returns the ip address from the current user
|
// ip Returns the ip address from the current user
|
||||||
pub fn (ctx &Context) ip() string {
|
pub fn (ctx &Context) ip() string {
|
||||||
mut ip := ctx.req.header.get(.x_forwarded_for) or { '' }
|
mut ip := ctx.req.header.get(.x_forwarded_for) or { '' }
|
||||||
|
|
@ -760,16 +600,6 @@ pub fn (mut ctx Context) error(s string) {
|
||||||
ctx.form_error = s
|
ctx.form_error = s
|
||||||
}
|
}
|
||||||
|
|
||||||
// not_found Returns an empty result
|
|
||||||
pub fn not_found() Result {
|
|
||||||
return Result{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// send_string
|
|
||||||
fn send_string(mut conn net.TcpConn, s string) ? {
|
|
||||||
conn.write(s.bytes()) ?
|
|
||||||
}
|
|
||||||
|
|
||||||
// filter Do not delete.
|
// filter Do not delete.
|
||||||
// It used by `vlib/v/gen/c/str_intp.v:130` for string interpolation inside web templates
|
// It used by `vlib/v/gen/c/str_intp.v:130` for string interpolation inside web templates
|
||||||
// TODO: move it to template render
|
// TODO: move it to template render
|
||||||
|
|
|
||||||
4
test.py
4
test.py
|
|
@ -38,7 +38,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len):
|
||||||
Generates a random .PKGINFO
|
Generates a random .PKGINFO
|
||||||
"""
|
"""
|
||||||
name = "-".join(random_words(words, name_min_len, name_max_len))
|
name = "-".join(random_words(words, name_min_len, name_max_len))
|
||||||
ver = "0.1.0-1" # doesn't matter what it is anyways
|
ver = "0.1.0-3" # doesn't matter what it is anyways
|
||||||
|
|
||||||
# TODO add random dependencies (all types)
|
# TODO add random dependencies (all types)
|
||||||
|
|
||||||
|
|
@ -97,7 +97,7 @@ async def upload_random_package(tar_path, sem):
|
||||||
async with sem:
|
async with sem:
|
||||||
with open(tar_path, 'rb') as f:
|
with open(tar_path, 'rb') as f:
|
||||||
async with aiohttp.ClientSession() as s:
|
async with aiohttp.ClientSession() as s:
|
||||||
async with s.post("http://localhost:8000/publish", data=f.read(), headers={"x-api-key": "test"}) as r:
|
async with s.post("http://localhost:8000/vieter/publish", data=f.read(), headers={"x-api-key": "test"}) as r:
|
||||||
return await check_output(r)
|
return await check_output(r)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
# This file contains settings used during development
|
||||||
|
api_key = "test"
|
||||||
|
download_dir = "data/downloads"
|
||||||
|
repos_dir = "data/repos"
|
||||||
|
pkg_dir = "data/pkgs"
|
||||||
|
log_level = "DEBUG"
|
||||||
|
repos_file = "data/repos.json"
|
||||||
|
default_arch = "x86_64"
|
||||||
|
|
||||||
|
address = "http://localhost:8000"
|
||||||
Loading…
Reference in New Issue