diff --git a/.editorconfig b/.editorconfig index e23a3c7..355c6bf 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,3 +1,4 @@ +# top-most EditorConfig file root = true # Unix-style newlines with a newline ending every file @@ -6,5 +7,4 @@ end_of_line = lf insert_final_newline = true [*.v] -# vfmt wants it :( -indent_style = tab +indent_style = space diff --git a/.gitignore b/.gitignore index 7847b3f..df4fcc9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,6 @@ data/ vieter dvieter pvieter -dvieterctl -vieterctl vieter.c # Ignore testing files @@ -19,7 +17,4 @@ libarchive-* test/ # V compiler directory -v/ - -# gdb log file -gdb.txt +v-*/ diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml deleted file mode 100644 index ab3c6ea..0000000 --- a/.woodpecker/.arch.yml +++ /dev/null @@ -1,30 +0,0 @@ -matrix: - PLATFORM: - - linux/amd64 - - linux/arm64 - -platform: ${PLATFORM} -branches: [dev] - -pipeline: - build: - image: 'menci/archlinuxarm:base-devel' - commands: - # Update packages - - pacman -Syu --noconfirm - # Create non-root user to perform build & switch to their home - - groupadd -g 1000 builder - - useradd -mg builder builder - - chown -R builder:builder "$PWD" - - "echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers" - - su builder - # Build the package - - makepkg -s --noconfirm --needed - - publish: - image: 'curlimages/curl' - commands: - # Publish the package - - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done' - secrets: - - vieter_api_key diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index e68c4c9..66caa8c 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -2,8 +2,7 @@ matrix: PLATFORM: - linux/amd64 - linux/arm64 - # I just don't have a performant enough runner for this platform - # - linux/arm/v7 + - linux/arm/v7 # These checks already get performed on the feature branches platform: ${PLATFORM} @@ -43,12 +42,12 @@ pipeline: commands: # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f - export URL=s3.rustybever.be + - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')" - export DATE="$(date -R --utc)" - export CONTENT_TYPE='application/zstd' - - - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')" - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH" - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64` + - > curl --silent diff --git a/.woodpecker/.builder.yml b/.woodpecker/.builder.yml new file mode 100644 index 0000000..e94a846 --- /dev/null +++ b/.woodpecker/.builder.yml @@ -0,0 +1,18 @@ +branches: dev +platform: linux/amd64 + +pipeline: + publish: + image: woodpeckerci/plugin-docker-buildx + secrets: [ docker_username, docker_password ] + settings: + repo: chewingbever/vlang + tag: latest + dockerfile: Dockerfile.builder + platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + when: + event: push + path: + - Makefile + - Dockerfile.builder + - patches/* diff --git a/.woodpecker/.deploy.yml b/.woodpecker/.deploy.yml new file mode 100644 index 0000000..56e6c2f --- /dev/null +++ b/.woodpecker/.deploy.yml @@ -0,0 +1,15 @@ +# Deploys the newest development image to my server +branches: [dev] +platform: linux/amd64 +depends_on: + - docker + +skip_clone: true + +pipeline: + webhook: + image: chewingbever/vlang:latest + secrets: + - webhook + commands: + - curl -XPOST -s "$WEBHOOK" diff --git a/.woodpecker/.docker.yml b/.woodpecker/.docker.yml index 9b605f3..9dfdf08 100644 --- a/.woodpecker/.docker.yml +++ b/.woodpecker/.docker.yml @@ -1,6 +1,7 @@ branches: [main, dev] platform: linux/amd64 depends_on: + - builder - build pipeline: @@ -9,8 +10,9 @@ pipeline: secrets: [ docker_username, docker_password ] settings: repo: chewingbever/vieter + dockerfile: Dockerfile.ci tag: dev - platforms: [ linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] build_args_from_env: - CI_COMMIT_SHA when: @@ -22,8 +24,9 @@ pipeline: secrets: [ docker_username, docker_password ] settings: repo: chewingbever/vieter + dockerfile: Dockerfile.ci auto_tag: true - platforms: [ linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] build_args_from_env: - CI_COMMIT_SHA when: diff --git a/CHANGELOG.md b/CHANGELOG.md index 8738952..3824088 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,40 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter) -## [0.2.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.2.0) - -### Changed - -* Better config system - * Support for both a config file & environment variables - * Each env var can now be provided from a file by appending it with `_FILE` - & passing the path to the file as value -* Revamped web framework - * All routes now return proper JSON where applicable & the correct status - codes - -### Added - -* Very basic build system - * Build is triggered by separate cron container - * Packages build on cron container's system - * A HEAD request is used to determine whether a package should be rebuilt - or not - * Hardcoded planning of builds - * Builds are sequential -* API for managing Git repositories to build -* CLI to list, add & remove Git repos to build -* Published packages on my Vieter instance -* Support for multiple repositories -* Support for multiple architectures per repository - -### Fixed - -* Each package can now only have one version in the repository at once - (required by Pacman) -* Packages with unknown fields in .PKGINFO are now allowed -* Old packages are now properly removed - ## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0) ### Changed diff --git a/Dockerfile b/Dockerfile index 58087ad..5564e34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,56 +1,19 @@ FROM chewingbever/vlang:latest AS builder -ARG TARGETPLATFORM -ARG CI_COMMIT_SHA -ARG DI_VER=1.2.5 - WORKDIR /app -# Build dumb-init -RUN curl -Lo - "https://github.com/Yelp/dumb-init/archive/refs/tags/v${DI_VER}.tar.gz" | tar -xzf - && \ - cd "dumb-init-${DI_VER}" && \ - make SHELL=/bin/sh && \ - mv dumb-init .. && \ - cd .. - # Copy over source code & build production binary COPY src ./src COPY Makefile ./ -RUN if [ -n "${CI_COMMIT_SHA}" ]; then \ - curl --fail \ - -o vieter \ - "https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \ - chmod +x vieter ; \ - else \ - LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' make prod && \ - mv pvieter vieter ; \ - fi +ENV LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' +RUN v -o pvieter -cflags "-O3" src -FROM busybox:1.35.0 +FROM alpine:3.15 -ENV PATH=/bin \ - VIETER_REPOS_DIR=/data/repos \ - VIETER_PKG_DIR=/data/pkgs \ - VIETER_DOWNLOAD_DIR=/data/downloads \ - VIETER_REPOS_FILE=/data/repos.json +ENV REPO_DIR=/data -COPY --from=builder /app/dumb-init /app/vieter /bin/ +COPY --from=builder /app/pvieter /usr/local/bin/vieter -HEALTHCHECK --interval=30s \ - --timeout=3s \ - --start-period=5s \ - CMD /bin/wget --spider http://localhost:8000/health || exit 1 - -RUN mkdir /data && \ - chown -R www-data:www-data /data && \ - mkdir -p '/var/spool/cron/crontabs' && \ - echo '0 3 * * * /bin/vieter build' | crontab - - -WORKDIR /data - -USER www-data:www-data - -ENTRYPOINT ["/bin/dumb-init", "--"] -CMD ["/bin/vieter", "server"] +ENTRYPOINT [ "/usr/local/bin/vieter" ] diff --git a/Dockerfile.builder b/Dockerfile.builder new file mode 100644 index 0000000..3e40d89 --- /dev/null +++ b/Dockerfile.builder @@ -0,0 +1,35 @@ +FROM alpine:3.12 + +ARG TARGETPLATFORM + +WORKDIR /opt/vlang + +ENV VVV /opt/vlang +ENV PATH /opt/vlang:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV VFLAGS -cc gcc +ENV V_PATH /opt/vlang/v + +RUN ln -s /opt/vlang/v /usr/bin/v && \ + apk --no-cache add \ + git make gcc curl openssl \ + musl-dev \ + openssl-libs-static openssl-dev \ + zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \ + sqlite-static sqlite-dev \ + libx11-dev glfw-dev freetype-dev \ + libarchive-static libarchive-dev \ + diffutils + +COPY patches ./patches +COPY Makefile ./ + +RUN make v && \ + mv v-*/* /opt/vlang && \ + v -version + +RUN if [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \ + wget -O /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && \ + chmod +x /usr/local/bin/mc ; \ +fi + +CMD ["v"] diff --git a/Dockerfile.ci b/Dockerfile.ci new file mode 100644 index 0000000..24f2bef --- /dev/null +++ b/Dockerfile.ci @@ -0,0 +1,46 @@ +# vim: ft=dockerfile +# This image just has the required tools to download the binaries +FROM chewingbever/vlang:latest AS builder + +ARG TARGETPLATFORM +ARG CI_COMMIT_SHA +ARG DI_VER=1.2.5 + +WORKDIR /app + +# Build dumb-init +RUN curl -Lo - "https://github.com/Yelp/dumb-init/archive/refs/tags/v${DI_VER}.tar.gz" | tar -xzf - && \ + cd "dumb-init-${DI_VER}" && \ + make SHELL=/bin/sh && \ + mv dumb-init .. && \ + cd .. + +RUN curl --fail \ + -o vieter \ + "https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \ + chmod +x vieter + + +FROM busybox:1.35.0 + +ENV PATH=/bin \ + REPO_DIR=/data/repo \ + PKG_DIR=/data/pkgs \ + DOWNLOAD_DIR=/data/downloads + +COPY --from=builder /app/dumb-init /app/vieter /bin/ + +HEALTHCHECK --interval=30s \ + --timeout=3s \ + --start-period=5s \ + CMD /bin/wget --spider http://localhost:8000/health || exit 1 + +RUN mkdir /data && \ + chown -R www-data:www-data /data + +WORKDIR /data + +USER www-data:www-data + +ENTRYPOINT ["/bin/dumb-init", "--"] +CMD ["/bin/vieter"] diff --git a/Makefile b/Makefile index 76ab7b5..8674ca0 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,9 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' -V_PATH ?= v/v -V := $(V_PATH) -showcc -gc boehm +V_RELEASE := weekly.2022.05 +V_PATH ?= v-$(V_RELEASE)/v +V := $(V_PATH) -showcc all: vieter @@ -13,17 +14,10 @@ vieter: $(SOURCES) $(V) -g -o vieter $(SRC_DIR) # Debug build using gcc -# The debug build can't use the boehm garbage collector, as that is -# multi-threaded and causes issues when running vieter inside gdb. .PHONY: debug debug: dvieter dvieter: $(SOURCES) - $(V_PATH) -showcc -keepc -cg -o dvieter $(SRC_DIR) - -# Run the debug build inside gdb -.PHONY: gdb -gdb: dvieter - gdb --args './dvieter -f vieter.toml server' + $(V) -keepc -cg -cc gcc -o dvieter $(SRC_DIR) # Optimised production build .PHONY: prod @@ -36,15 +30,22 @@ pvieter: $(SOURCES) c: $(V) -o vieter.c $(SRC_DIR) + # =====EXECUTION===== # Run the server in the default 'data' directory .PHONY: run run: vieter - ./vieter -f vieter.toml server + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter .PHONY: run-prod run-prod: prod - ./pvieter -f vieter.toml server + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./pvieter + +# Same as run, but restart when the source code changes +.PHONY: watch +watch: + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG $(V) watch run vieter + # =====OTHER===== .PHONY: lint @@ -62,10 +63,11 @@ vet: # Build & patch the V compiler .PHONY: v -v: v/v -v/v: - git clone --single-branch --branch patches https://git.rustybever.be/Chewing_Bever/vieter-v v - make -C v +v: v-$(V_RELEASE)/v +v-$(V_RELEASE)/v: + curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf - + cd patches && sh patch.sh '../v-$(V_RELEASE)' + make -C 'v-$(V_RELEASE)' clean: - rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' + rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'v-$(V_RELEASE)' diff --git a/PKGBUILD b/PKGBUILD deleted file mode 100644 index 5011ab1..0000000 --- a/PKGBUILD +++ /dev/null @@ -1,34 +0,0 @@ -# Maintainer: Jef Roosens - -pkgbase='vieter' -pkgname='vieter' -pkgver=0.1.0.rc1.r117.gc3ac00f -pkgrel=1 -depends=('glibc' 'openssl' 'libarchive' 'gc') -makedepends=('git' 'gcc') -arch=('x86_64' 'aarch64' 'armv7') -url='https://git.rustybever.be/Chewing_Bever/vieter' -license=('AGPL3') -source=($pkgname::git+https://git.rustybever.be/Chewing_Bever/vieter#branch=dev) -md5sums=('SKIP') - -pkgver() { - cd "$pkgname" - git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g' -} - -build() { - cd "$pkgname" - - # Build the compiler - CFLAGS= make v - - make prod -} - -package() { - pkgdesc="Vieter is a lightweight implementation of an Arch repository server." - install -dm755 "$pkgdir/usr/bin" - - install -Dm755 "$pkgbase/pvieter" "$pkgdir/usr/bin/vieter" -} diff --git a/README.md b/README.md index cd78f74..fbfa259 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,10 @@ # Vieter -## Documentation - -I host documentation for Vieter over at https://rustybever.be/docs/vieter/. - -## Overview - -Vieter is a restart of the Pieter project. The goal is to create a simple, -lightweight self-hostable Arch repository server, paired with a system that -periodically builds & publishes select Arch packages. This would allow me to -build AUR packages (or PKGBUILDs I created myself) "in the cloud" & make sure I -never have to compile anything on my own systems, making my updates a lot -quicker. +Vieter is a re-implementation of the Pieter project. The goal is to create a +simple PKGBUILD-based build system, combined with a self-hosted Arch +repository. This would allow me to periodically re-build AUR packages (or +PKGBUILDs I created myself), & make sure I never have to compile anything on my +own systems, making my updates a lot quicker. ## Why V? @@ -23,30 +16,31 @@ that. ### Custom Compiler Currently, this program only works with a very slightly modified version of the -V standard library, and therefore the compiler. The source code for this fork -can be found [here](https://git.rustybever.be/Chewing_Bever/vieter-v). You can -obtain this modified version of the compiler by running `make v`, which will -clone & build the compiler. Afterwards, all make commands that require the V -compiler will use this new binary. I try to keep this fork as up to date with -upstream as possible. +V standard library, and therefore the compiler. The changes that are made to +the standard V release can be found in the [patches](/patches) directory. You +can obtain this modified version of the compiler by running `make v`, which +will download, patch & build the compiler. Afterwards, all make commands that +require the V compiler will use this new binary. ## Features -* Arch repository server - * Support for multiple repositories & multiple architectures - * Endpoints for publishing new packages - * API for managing repositories to build -* Build system - * Periodic rebuilding of packages - * Prevent unnecessary rebuilds +The project will consist of a server-agent model, where one or more builder +nodes can register with the server. These agents communicate with the Docker +daemon to start builds, which are then uploaded to the server's repository. The +server also allows for non-agents to upload packages, as long as they have the +required secrets. This allows me to also develop non-git packages, such as my +terminal, & upload them to the servers using CI. -## Building +## Directory Structure -In order to build Vieter, you'll need a couple of libraries: +The data directory consists of three main directories: -* gc -* libarchive -* openssl - -Before building Vieter, you'll have to build the compiler using `make v`. -Afterwards, run `make` to build the debug binary. +* `downloads` - This is where packages are initially downloaded. Because vieter + moves files from this folder to the `pkgs` folder, these two folders should + best be on the same drive +* `pkgs` - This is where approved package files are stored. +* `repos` - Each repository gets a subfolder here. The subfolder contains the + uncompressed contents of the db file. + * Each repo subdirectory contains the compressed db & files archive for the + repository, alongside a directory called `files` which contains the + uncompressed contents. diff --git a/patches/parse_request_no_body.v b/patches/parse_request_no_body.v new file mode 100644 index 0000000..c00a51c --- /dev/null +++ b/patches/parse_request_no_body.v @@ -0,0 +1,23 @@ +// Parse the header of a raw HTTP request into a Request object +pub fn parse_request_head(mut reader io.BufferedReader) ?Request { + // request line + mut line := reader.read_line() ? + method, target, version := parse_request_line(line) ? + + // headers + mut header := new_header() + line = reader.read_line() ? + for line != '' { + key, value := parse_header(line) ? + header.add_custom(key, value) ? + line = reader.read_line() ? + } + header.coerce(canonicalize: true) + + return Request{ + method: method + url: target.str() + header: header + version: version + } +} diff --git a/patches/patch.sh b/patches/patch.sh new file mode 100755 index 0000000..9813976 --- /dev/null +++ b/patches/patch.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh +# This file patches the downloaded V version +# Should be run from within the directory it's in, as it uses relative paths to the files used for patching. +# $1 is the path to the downloaded V version + +# Add parse_request_no_body +cat parse_request_no_body.v >> "$1"/vlib/net/http/request.v + +# Make sha256 functions public +sed -i \ + -e 's/\(fn (mut d Digest) checksum(\)/pub \1/' \ + -e 's/\(fn (mut d Digest) write(\)/pub \1/' \ + "$1"/vlib/crypto/sha256/sha256.v diff --git a/src/archive.c.v b/src/archive.v similarity index 96% rename from src/archive.c.v rename to src/archive.v index 1f0d1dd..8d1314f 100644 --- a/src/archive.c.v +++ b/src/archive.v @@ -15,9 +15,6 @@ fn C.archive_read_support_filter_zstd(&C.archive) // Configure the archive to work with gzip compression fn C.archive_read_support_filter_gzip(&C.archive) -// Configure the archive to work with xz compression -fn C.archive_read_support_filter_xz(&C.archive) - // Configure the archive to work with a tarball content fn C.archive_read_support_format_tar(&C.archive) diff --git a/src/server/auth.v b/src/auth.v similarity index 73% rename from src/server/auth.v rename to src/auth.v index 8bc9d55..eab63c8 100644 --- a/src/server/auth.v +++ b/src/auth.v @@ -1,4 +1,4 @@ -module server +module main import net.http @@ -7,5 +7,5 @@ fn (mut app App) is_authorized() bool { return false } - return x_header.trim_space() == app.conf.api_key + return x_header.trim_space() == app.api_key } diff --git a/src/build/build.v b/src/build/build.v deleted file mode 100644 index 942ce8a..0000000 --- a/src/build/build.v +++ /dev/null @@ -1,133 +0,0 @@ -module build - -import docker -import encoding.base64 -import time -import git -import os - -const container_build_dir = '/build' - -const build_image_repo = 'vieter-build' - -fn create_build_image(base_image string) ?string { - commands := [ - // Update repos & install required packages - 'pacman -Syu --needed --noconfirm base-devel git' - // Add a non-root user to run makepkg - 'groupadd -g 1000 builder', - 'useradd -mg builder builder' - // Make sure they can use sudo without a password - "echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers" - // Create the directory for the builds & make it writeable for the - // build user - 'mkdir /build', - 'chown -R builder:builder /build', - ] - cmds_str := base64.encode_str(commands.join('\n')) - - c := docker.NewContainer{ - image: base_image - env: ['BUILD_SCRIPT=$cmds_str'] - entrypoint: ['/bin/sh', '-c'] - cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] - } - - // This check is needed so the user can pass "archlinux" without passing a - // tag & make it still work - image_parts := base_image.split_nth(':', 2) - image_name := image_parts[0] - image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' } - - // We pull the provided image - docker.pull_image(image_name, image_tag) ? - - id := docker.create_container(c) ? - docker.start_container(id) ? - - // This loop waits until the container has stopped, so we can remove it after - for { - data := docker.inspect_container(id) ? - - if !data.state.running { - break - } - - // Wait for 5 seconds - time.sleep(5000000000) - } - - // Finally, we create the image from the container - // As the tag, we use the epoch value - tag := time.sys_mono_now().str() - image := docker.create_image_from_container(id, 'vieter-build', tag) ? - docker.remove_container(id) ? - - return image.id -} - -fn build(conf Config) ? { - build_arch := os.uname().machine - - // We get the repos map from the Vieter instance - repos_map := git.get_repos(conf.address, conf.api_key) ? - - // We filter out any repos that aren't allowed to be built on this - // architecture - filtered_repos := repos_map.keys().map(repos_map[it]).filter(it.arch.contains(build_arch)) - - // No point in doing work if there's no repos present - if filtered_repos.len == 0 { - return - } - - // First, we create a base image which has updated repos n stuff - image_id := create_build_image(conf.base_image) ? - - for repo in filtered_repos { - // TODO what to do with PKGBUILDs that build multiple packages? - commands := [ - 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', - 'cd repo', - 'makepkg --nobuild --nodeps', - 'source PKGBUILD', - // The build container checks whether the package is already - // present on the server - 'curl --head --fail $conf.address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', - 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $conf.address/$repo.repo/publish; done', - ] - - // We convert the list of commands into a base64 string, which then gets - // passed to the container as an env var - cmds_str := base64.encode_str(commands.join('\n')) - - c := docker.NewContainer{ - image: '$image_id' - env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$conf.api_key'] - entrypoint: ['/bin/sh', '-c'] - cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/bash -e'] - work_dir: '/build' - user: 'builder:builder' - } - - id := docker.create_container(c) ? - docker.start_container(id) ? - - // This loop waits until the container has stopped, so we can remove it after - for { - data := docker.inspect_container(id) ? - - if !data.state.running { - break - } - - // Wait for 5 seconds - time.sleep(5000000000) - } - - docker.remove_container(id) ? - } - - // Finally, we remove the builder image - docker.remove_image(image_id) ? -} diff --git a/src/build/cli.v b/src/build/cli.v deleted file mode 100644 index 0131396..0000000 --- a/src/build/cli.v +++ /dev/null @@ -1,25 +0,0 @@ -module build - -import cli -import env - -pub struct Config { -pub: - api_key string - address string - base_image string = 'archlinux:base-devel' -} - -// cmd returns the cli submodule that handles the build process -pub fn cmd() cli.Command { - return cli.Command{ - name: 'build' - description: 'Run the build process.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? - - build(conf) ? - } - } -} diff --git a/src/docker/containers.v b/src/docker/containers.v deleted file mode 100644 index d0f5a4d..0000000 --- a/src/docker/containers.v +++ /dev/null @@ -1,78 +0,0 @@ -module docker - -import json -import net.urllib - -struct Container { - id string [json: Id] - names []string [json: Names] -} - -// containers returns a list of all currently running containers -pub fn containers() ?[]Container { - res := request('GET', urllib.parse('/v1.41/containers/json') ?) ? - - return json.decode([]Container, res.text) or {} -} - -pub struct NewContainer { - image string [json: Image] - entrypoint []string [json: Entrypoint] - cmd []string [json: Cmd] - env []string [json: Env] - work_dir string [json: WorkingDir] - user string [json: User] -} - -struct CreatedContainer { - id string [json: Id] -} - -// create_container creates a container defined by the given configuration. If -// successful, it returns the ID of the newly created container. -pub fn create_container(c &NewContainer) ?string { - res := request_with_json('POST', urllib.parse('/v1.41/containers/create') ?, c) ? - - if res.status_code != 201 { - return error('Failed to create container.') - } - - return json.decode(CreatedContainer, res.text) ?.id -} - -// start_container starts a container with a given ID. It returns whether the -// container was started or not. -pub fn start_container(id string) ?bool { - res := request('POST', urllib.parse('/v1.41/containers/$id/start') ?) ? - - return res.status_code == 204 -} - -struct ContainerInspect { -pub: - state ContainerState [json: State] -} - -struct ContainerState { -pub: - running bool [json: Running] -} - -// inspect_container returns the result of inspecting a container with a given -// ID. -pub fn inspect_container(id string) ?ContainerInspect { - res := request('GET', urllib.parse('/v1.41/containers/$id/json') ?) ? - - if res.status_code != 200 { - return error('Failed to inspect container.') - } - - return json.decode(ContainerInspect, res.text) or {} -} - -// remove_container removes a container with a given ID. -pub fn remove_container(id string) ?bool { - res := request('DELETE', urllib.parse('/v1.41/containers/$id') ?) ? - - return res.status_code == 204 -} diff --git a/src/docker/docker.v b/src/docker/docker.v deleted file mode 100644 index a6f7640..0000000 --- a/src/docker/docker.v +++ /dev/null @@ -1,93 +0,0 @@ -module docker - -import net.unix -import net.urllib -import net.http -import json - -const socket = '/var/run/docker.sock' - -const buf_len = 1024 - -fn send(req &string) ?http.Response { - // Open a connection to the socket - mut s := unix.connect_stream(docker.socket) or { - return error('Failed to connect to socket ${docker.socket}.') - } - - defer { - // This or is required because otherwise, the V compiler segfaults for - // some reason - // https://github.com/vlang/v/issues/13534 - s.close() or {} - } - - // Write the request to the socket - s.write_string(req) or { return error('Failed to write request to socket ${docker.socket}.') } - - s.wait_for_write() ? - - mut c := 0 - mut buf := []byte{len: docker.buf_len} - mut res := []byte{} - - for { - c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } - res << buf[..c] - - if c < docker.buf_len { - break - } - } - - // After reading the first part of the response, we parse it into an HTTP - // response. If it isn't chunked, we return early with the data. - parsed := http.parse_response(res.bytestr()) or { - return error('Failed to parse HTTP response from socket ${docker.socket}.') - } - - if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' { - return parsed - } - - // We loop until we've encountered the end of the chunked response - // A chunked HTTP response always ends with '0\r\n\r\n'. - for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] { - // Wait for the server to respond - s.wait_for_write() ? - - for { - c = s.read(mut buf) or { - return error('Failed to read data from socket ${docker.socket}.') - } - res << buf[..c] - - if c < docker.buf_len { - break - } - } - } - - // Decode chunked response - return http.parse_response(res.bytestr()) -} - -fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' - - return send(req) -} - -fn request(method string, url urllib.URL) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' - - return send(req) -} - -// request_with_json sends a request to the Docker socket with a given JSON -// payload -pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Response { - body := json.encode(data) - - return request_with_body(method, url, 'application/json', body) -} diff --git a/src/docker/images.v b/src/docker/images.v deleted file mode 100644 index e94ceca..0000000 --- a/src/docker/images.v +++ /dev/null @@ -1,34 +0,0 @@ -module docker - -import net.http -import net.urllib -import json - -struct Image { -pub: - id string [json: Id] -} - -// pull_image pulls tries to pull the image for the given image & tag -pub fn pull_image(image string, tag string) ?http.Response { - return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) -} - -// create_image_from_container creates a new image from a container with the -// given repo & tag, given the container's ID. -pub fn create_image_from_container(id string, repo string, tag string) ?Image { - res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag') ?) ? - - if res.status_code != 201 { - return error('Failed to create image from container.') - } - - return json.decode(Image, res.text) or {} -} - -// remove_image removes the image with the given ID. -pub fn remove_image(id string) ?bool { - res := request('DELETE', urllib.parse('/v1.41/images/$id') ?) ? - - return res.status_code == 200 -} diff --git a/src/env.v b/src/env.v deleted file mode 100644 index cbde67e..0000000 --- a/src/env.v +++ /dev/null @@ -1,82 +0,0 @@ -module env - -import os -import toml - -// The prefix that every environment variable should have -const prefix = 'VIETER_' - -// The suffix an environment variable in order for it to be loaded from a file -// instead -const file_suffix = '_FILE' - -fn get_env_var(field_name string) ?string { - env_var_name := '$env.prefix$field_name.to_upper()' - env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix' - env_var := os.getenv(env_var_name) - env_file := os.getenv(env_file_name) - - // If both are missing, we return an empty string - if env_var == '' && env_file == '' { - return '' - } - - // If they're both set, we report a conflict - if env_var != '' && env_file != '' { - return error('Only one of $env_var_name or $env_file_name can be defined.') - } - - // If it's the env var itself, we return it. - // I'm pretty sure this also prevents variable ending in _FILE (e.g. - // VIETER_LOG_FILE) from being mistakingely read as an _FILE suffixed env - // var. - if env_var != '' { - return env_var - } - - // Otherwise, we process the file - return os.read_file(env_file) or { - error('Failed to read file defined in $env_file_name: ${err.msg}.') - } -} - -// load attempts to create an object of type T from the given path to a toml -// file & environment variables. For each field, it will select either a value -// given from an environment variable, a value defined in the config file or a -// configured default if present, in that order. -pub fn load(path string) ?T { - mut res := T{} - - if os.exists(path) { - // We don't use reflect here because reflect also sets any fields not - // in the toml back to their zero value, which we don't want - doc := toml.parse_file(path) ? - - $for field in T.fields { - s := doc.value(field.name) - - // We currently only support strings - if s.type_name() == 'string' { - res.$(field.name) = s.string() - } - } - } - - $for field in T.fields { - $if field.typ is string { - env_value := get_env_var(field.name) ? - - // The value of the env var will always be chosen over the config - // file - if env_value != '' { - res.$(field.name) = env_value - } - // If there's no value from the toml file either, we try to find a - // default value - else if res.$(field.name) == '' { - return error("Missing config variable '$field.name' with no provided default. Either add it to the config file or provide it using an environment variable.") - } - } - } - return res -} diff --git a/src/git/cli.v b/src/git/cli.v deleted file mode 100644 index 463f1ba..0000000 --- a/src/git/cli.v +++ /dev/null @@ -1,147 +0,0 @@ -module git - -import cli -import env - -struct Config { - address string [required] - api_key string [required] -} - -// cmd returns the cli submodule that handles the repos API interaction -pub fn cmd() cli.Command { - return cli.Command{ - name: 'repos' - description: 'Interact with the repos API.' - commands: [ - cli.Command{ - name: 'list' - description: 'List the current repos.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? - - list(conf) ? - } - }, - cli.Command{ - name: 'add' - required_args: 4 - usage: 'url branch repo arch...' - description: 'Add a new repository.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? - - add(conf, cmd.args[0], cmd.args[1], cmd.args[2], cmd.args[3..]) ? - } - }, - cli.Command{ - name: 'remove' - required_args: 1 - usage: 'id' - description: 'Remove a repository that matches the given ID prefix.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? - - remove(conf, cmd.args[0]) ? - } - }, - cli.Command{ - name: 'edit' - required_args: 1 - usage: 'id' - description: 'Edit the repository that matches the given ID prefix.' - flags: [ - cli.Flag{ - name: 'url' - description: 'URL of the Git repository.' - flag: cli.FlagType.string - }, - cli.Flag{ - name: 'branch' - description: 'Branch of the Git repository.' - flag: cli.FlagType.string - }, - cli.Flag{ - name: 'repo' - description: 'Repo to publish builds to.' - flag: cli.FlagType.string - }, - cli.Flag{ - name: 'arch' - description: 'Comma-separated list of architectures to build on.' - flag: cli.FlagType.string - }, - ] - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? - - found := cmd.flags.get_all_found() - - mut params := map[string]string{} - - for f in found { - if f.name != 'config-file' { - params[f.name] = f.get_string() ? - } - } - - patch(conf, cmd.args[0], params) ? - } - }, - ] - } -} - -fn get_repo_id_by_prefix(conf Config, id_prefix string) ?string { - repos := get_repos(conf.address, conf.api_key) ? - - mut res := []string{} - - for id, _ in repos { - if id.starts_with(id_prefix) { - res << id - } - } - - if res.len == 0 { - return error('No repo found for given prefix.') - } - - if res.len > 1 { - return error('Multiple repos found for given prefix.') - } - - return res[0] -} - -fn list(conf Config) ? { - repos := get_repos(conf.address, conf.api_key) ? - - for id, details in repos { - println('${id[..8]}\t$details.url\t$details.branch\t$details.repo\t$details.arch') - } -} - -fn add(conf Config, url string, branch string, repo string, arch []string) ? { - res := add_repo(conf.address, conf.api_key, url, branch, repo, arch) ? - - println(res.message) -} - -fn remove(conf Config, id_prefix string) ? { - id := get_repo_id_by_prefix(conf, id_prefix) ? - res := remove_repo(conf.address, conf.api_key, id) ? - - println(res.message) -} - -fn patch(conf Config, id_prefix string, params map[string]string) ? { - id := get_repo_id_by_prefix(conf, id_prefix) ? - res := patch_repo(conf.address, conf.api_key, id, params) ? - - println(res.message) -} diff --git a/src/git/client.v b/src/git/client.v deleted file mode 100644 index e4a39ac..0000000 --- a/src/git/client.v +++ /dev/null @@ -1,61 +0,0 @@ -module git - -import json -import response { Response } -import net.http - -fn send_request(method http.Method, address string, url string, api_key string, params map[string]string) ?Response { - mut full_url := '$address$url' - - if params.len > 0 { - params_str := params.keys().map('$it=${params[it]}').join('&') - - full_url = '$full_url?$params_str' - } - - mut req := http.new_request(method, full_url, '') ? - req.add_custom_header('X-API-Key', api_key) ? - - res := req.do() ? - data := json.decode(Response, res.text) ? - - return data -} - -// get_repos returns the current list of repos. -pub fn get_repos(address string, api_key string) ?map[string]GitRepo { - data := send_request(http.Method.get, address, '/api/repos', api_key, - {}) ? - - return data.data -} - -// add_repo adds a new repo to the server. -pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response { - params := { - 'url': url - 'branch': branch - 'repo': repo - 'arch': arch.join(',') - } - data := send_request(http.Method.post, address, '/api/repos', api_key, params) ? - - return data -} - -// remove_repo removes the repo with the given ID from the server. -pub fn remove_repo(address string, api_key string, id string) ?Response { - data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, - {}) ? - - return data -} - -// patch_repo sends a PATCH request to the given repo with the params as -// payload. -pub fn patch_repo(address string, api_key string, id string, params map[string]string) ?Response { - data := send_request(http.Method.patch, address, '/api/repos/$id', api_key, - params) ? - - return data -} diff --git a/src/git/git.v b/src/git/git.v deleted file mode 100644 index eaec895..0000000 --- a/src/git/git.v +++ /dev/null @@ -1,82 +0,0 @@ -module git - -import os -import json - -pub struct GitRepo { -pub mut: - // URL of the Git repository - url string - // Branch of the Git repository to use - branch string - // On which architectures the package is allowed to be built. In reality, - // this controls which builders will periodically build the image. - arch []string - // Which repo the builder should publish packages to - repo string -} - -// patch_from_params patches a GitRepo from a map[string]string, usually -// provided from a web.App's params -pub fn (mut r GitRepo) patch_from_params(params map[string]string) { - $for field in GitRepo.fields { - if field.name in params { - $if field.typ is string { - r.$(field.name) = params[field.name] - // This specific type check is needed for the compiler to ensure - // our types are correct - } $else $if field.typ is []string { - r.$(field.name) = params[field.name].split(',') - } - } - } -} - -// read_repos reads the provided path & parses it into a map of GitRepo's. -pub fn read_repos(path string) ?map[string]GitRepo { - if !os.exists(path) { - mut f := os.create(path) ? - - defer { - f.close() - } - - f.write_string('{}') ? - - return {} - } - - content := os.read_file(path) ? - res := json.decode(map[string]GitRepo, content) ? - - return res -} - -// write_repos writes a map of GitRepo's back to disk given the provided path. -pub fn write_repos(path string, repos &map[string]GitRepo) ? { - mut f := os.create(path) ? - - defer { - f.close() - } - - value := json.encode(repos) - f.write_string(value) ? -} - -// repo_from_params creates a GitRepo from a map[string]string, usually -// provided from a web.App's params -pub fn repo_from_params(params map[string]string) ?GitRepo { - mut repo := GitRepo{} - - // If we're creating a new GitRepo, we want all fields to be present before - // "patching". - $for field in GitRepo.fields { - if field.name !in params { - return error('Missing parameter: ${field.name}.') - } - } - repo.patch_from_params(params) - - return repo -} diff --git a/src/main.v b/src/main.v index 3025389..cddd9ae 100644 --- a/src/main.v +++ b/src/main.v @@ -1,33 +1,103 @@ module main +import web import os -import server -import cli -import build -import git +import log +import io +import repo -fn main() { - mut app := cli.Command{ - name: 'vieter' - description: 'Vieter is a lightweight implementation of an Arch repository server.' - version: '0.2.0' - flags: [ - cli.Flag{ - flag: cli.FlagType.string - name: 'config-file' - abbrev: 'f' - description: 'Location of Vieter config file; defaults to ~/.vieterrc.' - global: true - default_value: [os.expand_tilde_to_home('~/.vieterrc')] - }, - ] - commands: [ - server.cmd(), - build.cmd(), - git.cmd(), - ] +const port = 8000 + +const buf_size = 1_000_000 + +const db_name = 'pieter.db' + +struct App { + web.Context +pub: + api_key string [required; web_global] + dl_dir string [required; web_global] +pub mut: + repo repo.Repo [required; web_global] +} + +[noreturn] +fn exit_with_message(code int, msg string) { + eprintln(msg) + exit(code) +} + +fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { + mut file := os.create(path) ? + defer { + file.close() } - app.setup() - app.parse(os.args) + mut buf := []byte{len: buf_size} + mut bytes_left := length + + // Repeat as long as the stream still has data + for bytes_left > 0 { + // TODO check if just breaking here is safe + bytes_read := reader.read(mut buf) or { break } + bytes_left -= bytes_read + + mut to_write := bytes_read + + for to_write > 0 { + // TODO don't just loop infinitely here + bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue } + + to_write = to_write - bytes_written + } + } +} + +fn main() { + // Configure logger + log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } + log_level := log.level_from_tag(log_level_str) or { + exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + } + log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } + + mut logger := log.Log{ + level: log_level + } + + logger.set_full_logpath(log_file) + logger.log_to_console_too() + + defer { + logger.info('Flushing log file') + logger.flush() + logger.close() + } + + // Configure web server + key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } + repo_dir := os.getenv_opt('REPO_DIR') or { + exit_with_message(1, 'No repo directory was configured.') + } + pkg_dir := os.getenv_opt('PKG_DIR') or { + exit_with_message(1, 'No package directory was configured.') + } + dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { + exit_with_message(1, 'No download directory was configured.') + } + + // This also creates the directories if needed + repo := repo.new(repo_dir, pkg_dir) or { + logger.error(err.msg) + exit(1) + } + + os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } + + web.run(&App{ + logger: logger + api_key: key + dl_dir: dl_dir + repo: repo + }, port) } diff --git a/src/package.v b/src/package.v index a6be636..4e3b97f 100644 --- a/src/package.v +++ b/src/package.v @@ -72,11 +72,14 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { 'pkgbase' { pkg_info.base = value } 'pkgver' { pkg_info.version = value } 'pkgdesc' { pkg_info.description = value } + 'csize' { continue } 'size' { pkg_info.size = value.int() } 'url' { pkg_info.url = value } 'arch' { pkg_info.arch = value } 'builddate' { pkg_info.build_date = value.int() } 'packager' { pkg_info.packager = value } + 'md5sum' { continue } + 'sha256sum' { continue } 'pgpsig' { pkg_info.pgpsig = value } 'pgpsigsize' { pkg_info.pgpsigsize = value.int() } // Array values @@ -89,19 +92,16 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { 'optdepend' { pkg_info.optdepends << value } 'makedepend' { pkg_info.makedepends << value } 'checkdepend' { pkg_info.checkdepends << value } - // There's no real point in trying to exactly manage which fields - // are allowed, so we just ignore any we don't explicitely need for - // in the db file - else { continue } + else { return error("Invalid key '$key'.") } } } return pkg_info } -// read_pkg_archive extracts the file list & .PKGINFO contents from an archive -// NOTE: this command only supports zstd-, xz- & gzip-compressed tarballs. -pub fn read_pkg_archive(pkg_path string) ?Pkg { +// read_pkg extracts the file list & .PKGINFO contents from an archive +// NOTE: this command currently only supports zstd-compressed tarballs +pub fn read_pkg(pkg_path string) ?Pkg { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") } @@ -112,7 +112,6 @@ pub fn read_pkg_archive(pkg_path string) ?Pkg { // Sinds 2020, all newly built Arch packages use zstd C.archive_read_support_filter_zstd(a) C.archive_read_support_filter_gzip(a) - C.archive_read_support_filter_xz(a) // The content should always be a tarball C.archive_read_support_format_tar(a) @@ -191,7 +190,6 @@ pub fn (pkg &Pkg) filename() string { ext := match pkg.compression { 0 { '.tar' } 1 { '.tar.gz' } - 6 { '.tar.xz' } 14 { '.tar.zst' } else { panic("Another compression code shouldn't be possible. Faulty code: $pkg.compression") } } diff --git a/src/repo/repo.v b/src/repo/repo.v index f439f58..554b744 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -2,23 +2,22 @@ module repo import os import package -import util -// Manages a group of repositories. Each repository contains one or more -// arch-repositories, each of which represent a specific architecture. -pub struct RepoGroupManager { +// Dummy struct to work around the fact that you can only share structs, maps & +// arrays +pub struct Dummy { + x int +} + +// This struct manages a single repository. +pub struct Repo { mut: - mutex shared util.Dummy + mutex shared Dummy pub: - // Where to store repositories' files - repos_dir string [required] - // Where packages are stored; each arch-repository gets its own - // subdirectory + // Where to store repository files + repo_dir string [required] + // Where to find packages; packages are expected to all be in the same directory pkg_dir string [required] - // The default architecture to use for a repository. Whenever a package of - // arch "any" is added to a repo, it will also be added to this - // architecture. - default_arch string [required] } pub struct RepoAddResult { @@ -27,117 +26,55 @@ pub: pkg &package.Pkg [required] } -// new creates a new RepoGroupManager & creates the directories as needed -pub fn new(repos_dir string, pkg_dir string, default_arch string) ?RepoGroupManager { - if !os.is_dir(repos_dir) { - os.mkdir_all(repos_dir) or { return error('Failed to create repos directory: $err.msg') } +// new creates a new Repo & creates the directories as needed +pub fn new(repo_dir string, pkg_dir string) ?Repo { + if !os.is_dir(repo_dir) { + os.mkdir_all(repo_dir) or { return error('Failed to create repo directory: $err.msg') } } if !os.is_dir(pkg_dir) { os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg') } } - return RepoGroupManager{ - repos_dir: repos_dir + return Repo{ + repo_dir: repo_dir pkg_dir: pkg_dir - default_arch: default_arch } } -// add_pkg_from_path adds a package to a given repo, given the file path to the -// pkg archive. It's a wrapper around add_pkg_in_repo that parses the archive -// file, passes the result to add_pkg_in_repo, and hard links the archive to -// the right subdirectories in r.pkg_dir if it was successfully added. -pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult { - pkg := package.read_pkg_archive(pkg_path) or { - return error('Failed to read package file: $err.msg') - } +// add_from_path adds a package from an arbitrary path & moves it into the pkgs +// directory if necessary. +pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult { + pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') } - added := r.add_pkg_in_repo(repo, pkg) ? + added := r.add(pkg) ? // If the add was successful, we move the file to the packages directory - for arch in added { - repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch)) - dest_path := os.join_path_single(repo_pkg_path, pkg.filename()) + if added { + dest_path := os.real_path(os.join_path_single(r.pkg_dir, pkg.filename())) - os.mkdir_all(repo_pkg_path) ? - - // We create hard links so that "any" arch packages aren't stored - // multiple times - os.link(pkg_path, dest_path) ? + // Only move the file if it's not already in the package directory + if dest_path != os.real_path(pkg_path) { + os.mv(pkg_path, dest_path) ? + } } - // After linking, we can remove the original file - os.rm(pkg_path) ? - return RepoAddResult{ - added: added.len > 0 + added: added pkg: &pkg } } -// add_pkg_in_repo adds a package to a given repo. This function is responsible -// for inspecting the package architecture. If said architecture is 'any', the -// package is added to each arch-repository within the given repo. A package of -// architecture 'any' is always added to the arch-repo defined by -// r.default_arch. If this arch-repo doesn't exist yet, it is created. If the -// architecture isn't 'any', the package is only added to the specific -// architecture. -fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]string { - // A package not of arch 'any' can be handled easily by adding it to the - // respective repo - if pkg.info.arch != 'any' { - if r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) ? { - return [pkg.info.arch] - } else { - return [] - } +// add adds a given Pkg to the repository +fn (r &Repo) add(pkg &package.Pkg) ?bool { + pkg_dir := r.pkg_path(pkg) + + // We can't add the same package twice + if os.exists(pkg_dir) { + return false } - mut arch_repos := []string{} - - // If it is an "any" package, the package gets added to every currently - // present arch-repo. It will always get added to the r.default_arch repo, - // even if no or multiple others are present. - repo_dir := os.join_path_single(r.repos_dir, repo) - - // If this is the first package that's added to the repo, the directory - // won't exist yet - if os.exists(repo_dir) { - arch_repos = os.ls(repo_dir) ? - } - - // The default_arch should always be updated when a package with arch 'any' - // is added. - if !arch_repos.contains(r.default_arch) { - arch_repos << r.default_arch - } - - mut added := []string{} - - // We add the package to each repository. If any of the repositories - // return true, the result of the function is also true. - for arch in arch_repos { - if r.add_pkg_in_arch_repo(repo, arch, pkg) ? { - added << arch - } - } - - return added -} - -// add_pkg_in_arch_repo is the function that actually adds a package to a given -// arch-repo. It records the package's data in the arch-repo's desc & files -// files, and afterwards updates the db & files archives to reflect these -// changes. The function returns false if the package was already present in -// the repo, and true otherwise. -fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool { - pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version') - - // Remove the previous version of the package, if present - r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false) ? - - os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') } + os.mkdir(pkg_dir) or { return error('Failed to create package directory.') } os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or { os.rmdir_all(pkg_dir) ? @@ -150,55 +87,12 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac return error('Failed to write files file.') } - r.sync(repo, arch) ? + r.sync() ? return true } -// remove_pkg_from_arch_repo removes a package from an arch-repo's database. It -// returns false if the package wasn't present in the database. It also -// optionally re-syncs the repo archives. -fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg_name string, sync bool) ?bool { - repo_dir := os.join_path(r.repos_dir, repo, arch) - - // If the repository doesn't exist yet, the result is automatically false - if !os.exists(repo_dir) { - return false - } - - // We iterate over every directory in the repo dir - // TODO filter so we only check directories - for d in os.ls(repo_dir) ? { - // Because a repository only allows a single version of each package, - // we need only compare whether the name of the package is the same, - // not the version. - name := d.split('-')#[..-2].join('-') - - if name == pkg_name { - // We lock the mutex here to prevent other routines from creating a - // new archive while we remove an entry - lock r.mutex { - os.rmdir_all(os.join_path_single(repo_dir, d)) ? - } - - // Also remove the package archive - repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch) - - archives := os.ls(repo_pkg_dir) ?.filter(it.split('-')#[..-3].join('-') == name) - - for archive_name in archives { - full_path := os.join_path_single(repo_pkg_dir, archive_name) - os.rm(full_path) ? - } - - // Sync the db archives if requested - if sync { - r.sync(repo, arch) ? - } - - return true - } - } - - return false +// Returns the path where the given package's desc & files files are stored +fn (r &Repo) pkg_path(pkg &package.Pkg) string { + return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version') } diff --git a/src/repo/sync.v b/src/repo/sync.v index e2b7aac..d6080e0 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -30,9 +30,8 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri } // Re-generate the repo archive files -fn (r &RepoGroupManager) sync(repo string, arch string) ? { - subrepo_path := os.join_path(r.repos_dir, repo, arch) - +fn (r &Repo) sync() ? { + // TODO also write files archive lock r.mutex { a_db := C.archive_write_new() a_files := C.archive_write_new() @@ -45,18 +44,18 @@ fn (r &RepoGroupManager) sync(repo string, arch string) ? { C.archive_write_add_filter_gzip(a_files) C.archive_write_set_format_pax_restricted(a_files) - db_path := os.join_path_single(subrepo_path, '${repo}.db.tar.gz') - files_path := os.join_path_single(subrepo_path, '${repo}.files.tar.gz') + db_path := os.join_path_single(r.repo_dir, 'vieter.db.tar.gz') + files_path := os.join_path_single(r.repo_dir, 'vieter.files.tar.gz') C.archive_write_open_filename(a_db, &char(db_path.str)) C.archive_write_open_filename(a_files, &char(files_path.str)) // Iterate over each directory - for d in os.ls(subrepo_path) ?.filter(os.is_dir(os.join_path_single(subrepo_path, + for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, it))) { // desc mut inner_path := os.join_path_single(d, 'desc') - mut actual_path := os.join_path_single(subrepo_path, inner_path) + mut actual_path := os.join_path_single(r.repo_dir, inner_path) archive_add_entry(a_db, entry, actual_path, inner_path) archive_add_entry(a_files, entry, actual_path, inner_path) @@ -65,7 +64,7 @@ fn (r &RepoGroupManager) sync(repo string, arch string) ? { // files inner_path = os.join_path_single(d, 'files') - actual_path = os.join_path_single(subrepo_path, inner_path) + actual_path = os.join_path_single(r.repo_dir, inner_path) archive_add_entry(a_files, entry, actual_path, inner_path) diff --git a/src/response.v b/src/response.v deleted file mode 100644 index a06a589..0000000 --- a/src/response.v +++ /dev/null @@ -1,34 +0,0 @@ -module response - -pub struct Response { -pub: - message string - data T -} - -// new_response constructs a new Response object with the given message -// & an empty data field. -pub fn new_response(message string) Response { - return Response{ - message: message - data: '' - } -} - -// new_data_response constructs a new Response object with the given data -// & an empty message field. -pub fn new_data_response(data T) Response { - return Response{ - message: '' - data: data - } -} - -// new_full_response constructs a new Response object with the given -// message & data. -pub fn new_full_response(message string, data T) Response { - return Response{ - message: message - data: data - } -} diff --git a/src/routes.v b/src/routes.v new file mode 100644 index 0000000..8b7ddeb --- /dev/null +++ b/src/routes.v @@ -0,0 +1,103 @@ +module main + +import web +import os +import repo +import time +import rand + +const prefixes = ['B', 'KB', 'MB', 'GB'] + +// pretty_bytes converts a byte count to human-readable version +fn pretty_bytes(bytes int) string { + mut i := 0 + mut n := f32(bytes) + + for n >= 1024 { + i++ + n /= 1024 + } + + return '${n:.2}${prefixes[i]}' +} + +fn is_pkg_name(s string) bool { + return s.contains('.pkg') +} + +// healthcheck just returns a string, but can be used to quickly check if the +// server is still responsive. +['/health'; get] +pub fn (mut app App) healthcheck() web.Result { + return app.text('Healthy') +} + +// get_root handles a GET request for a file on the root +['/:filename'; get] +fn (mut app App) get_root(filename string) web.Result { + mut full_path := '' + + if filename.ends_with('.db') || filename.ends_with('.files') { + full_path = os.join_path_single(app.repo.repo_dir, '${filename}.tar.gz') + } else if filename.ends_with('.db.tar.gz') || filename.ends_with('.files.tar.gz') { + full_path = os.join_path_single(app.repo.repo_dir, '$filename') + } else { + full_path = os.join_path_single(app.repo.pkg_dir, filename) + } + + return app.file(full_path) +} + +['/publish'; post] +fn (mut app App) put_package() web.Result { + if !app.is_authorized() { + return app.text('Unauthorized.') + } + + mut pkg_path := '' + + if length := app.req.header.get(.content_length) { + // Generate a random filename for the temp file + pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) + + for os.exists(pkg_path) { + pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) + } + + app.ldebug("Uploading $length bytes (${pretty_bytes(length.int())}) to '$pkg_path'.") + + // This is used to time how long it takes to upload a file + mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) + + reader_to_file(mut app.reader, length.int(), pkg_path) or { + app.lwarn("Failed to upload '$pkg_path'") + + return app.text('Failed to upload file.') + } + + sw.stop() + app.ldebug("Upload of '$pkg_path' completed in ${sw.elapsed().seconds():.3}s.") + } else { + app.lwarn('Tried to upload package without specifying a Content-Length.') + return app.text("Content-Type header isn't set.") + } + + res := app.repo.add_from_path(pkg_path) or { + app.lerror('Error while adding package: $err.msg') + + os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } + + return app.text('Failed to add package.') + } + if !res.added { + os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } + + app.lwarn("Duplicate package '$res.pkg.full_name()'.") + + return app.text('File already exists.') + } + + app.linfo("Added '$res.pkg.full_name()' to repository.") + + return app.text('Package added successfully.') +} diff --git a/src/server/cli.v b/src/server/cli.v deleted file mode 100644 index bea223d..0000000 --- a/src/server/cli.v +++ /dev/null @@ -1,30 +0,0 @@ -module server - -import cli -import env - -struct Config { -pub: - log_level string = 'WARN' - log_file string = 'vieter.log' - pkg_dir string - download_dir string - api_key string - repos_dir string - repos_file string - default_arch string -} - -// cmd returns the cli submodule that handles starting the server -pub fn cmd() cli.Command { - return cli.Command{ - name: 'server' - description: 'Start the Vieter server.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? - - server(conf) ? - } - } -} diff --git a/src/server/git.v b/src/server/git.v deleted file mode 100644 index 2a682d8..0000000 --- a/src/server/git.v +++ /dev/null @@ -1,141 +0,0 @@ -module server - -import web -import git -import net.http -import rand -import response { new_data_response, new_response } - -const repos_file = 'repos.json' - -['/api/repos'; get] -fn (mut app App) get_repos() web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - - repos := rlock app.git_mutex { - git.read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file: $err.msg') - - return app.status(http.Status.internal_server_error) - } - } - - return app.json(http.Status.ok, new_data_response(repos)) -} - -['/api/repos/:id'; get] -fn (mut app App) get_single_repo(id string) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - - repos := rlock app.git_mutex { - git.read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file.') - - return app.status(http.Status.internal_server_error) - } - } - - if id !in repos { - return app.not_found() - } - - repo := repos[id] - - return app.json(http.Status.ok, new_data_response(repo)) -} - -['/api/repos'; post] -fn (mut app App) post_repo() web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - - new_repo := git.repo_from_params(app.query) or { - return app.json(http.Status.bad_request, new_response(err.msg)) - } - - id := rand.uuid_v4() - - mut repos := rlock app.git_mutex { - git.read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file.') - - return app.status(http.Status.internal_server_error) - } - } - - // We need to check for duplicates - for _, repo in repos { - if repo == new_repo { - return app.json(http.Status.bad_request, new_response('Duplicate repository.')) - } - } - - repos[id] = new_repo - - lock app.git_mutex { - git.write_repos(app.conf.repos_file, &repos) or { - return app.status(http.Status.internal_server_error) - } - } - - return app.json(http.Status.ok, new_response('Repo added successfully.')) -} - -['/api/repos/:id'; delete] -fn (mut app App) delete_repo(id string) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - - mut repos := rlock app.git_mutex { - git.read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file.') - - return app.status(http.Status.internal_server_error) - } - } - - if id !in repos { - return app.not_found() - } - - repos.delete(id) - - lock app.git_mutex { - git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } - } - - return app.json(http.Status.ok, new_response('Repo removed successfully.')) -} - -['/api/repos/:id'; patch] -fn (mut app App) patch_repo(id string) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - - mut repos := rlock app.git_mutex { - git.read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file.') - - return app.status(http.Status.internal_server_error) - } - } - - if id !in repos { - return app.not_found() - } - - repos[id].patch_from_params(app.query) - - lock app.git_mutex { - git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } - } - - return app.json(http.Status.ok, new_response('Repo updated successfully.')) -} diff --git a/src/server/routes.v b/src/server/routes.v deleted file mode 100644 index 138f253..0000000 --- a/src/server/routes.v +++ /dev/null @@ -1,108 +0,0 @@ -module server - -import web -import os -import repo -import time -import rand -import util -import net.http -import response { new_response } - -// healthcheck just returns a string, but can be used to quickly check if the -// server is still responsive. -['/health'; get] -pub fn (mut app App) healthcheck() web.Result { - return app.json(http.Status.ok, new_response('Healthy.')) -} - -['/:repo/:arch/:filename'; get; head] -fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result { - mut full_path := '' - - db_exts := ['.db', '.files', '.db.tar.gz', '.files.tar.gz'] - - // There's no point in having the ability to serve db archives with wrong - // filenames - if db_exts.any(filename == '$repo$it') { - full_path = os.join_path(app.repo.repos_dir, repo, arch, filename) - - // repo-add does this using symlinks, but we just change the requested - // path - if !full_path.ends_with('.tar.gz') { - full_path += '.tar.gz' - } - } else if filename.contains('.pkg') { - full_path = os.join_path(app.repo.pkg_dir, repo, arch, filename) - } - // Default behavior is to return the desc file for the package, if present. - // This can then also be used by the build system to properly check whether - // a package is present in an arch-repo. - else { - full_path = os.join_path(app.repo.repos_dir, repo, arch, filename, 'desc') - } - - // Scuffed way to respond to HEAD requests - if app.req.method == http.Method.head { - if os.exists(full_path) { - return app.status(http.Status.ok) - } - - return app.not_found() - } - - return app.file(full_path) -} - -['/:repo/publish'; post] -fn (mut app App) put_package(repo string) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - - mut pkg_path := '' - - if length := app.req.header.get(.content_length) { - // Generate a random filename for the temp file - pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4()) - - app.ldebug("Uploading $length bytes (${util.pretty_bytes(length.int())}) to '$pkg_path'.") - - // This is used to time how long it takes to upload a file - mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) - - util.reader_to_file(mut app.reader, length.int(), pkg_path) or { - app.lwarn("Failed to upload '$pkg_path'") - - return app.json(http.Status.internal_server_error, new_response('Failed to upload file.')) - } - - sw.stop() - app.ldebug("Upload of '$pkg_path' completed in ${sw.elapsed().seconds():.3}s.") - } else { - app.lwarn('Tried to upload package without specifying a Content-Length.') - - // length required - return app.status(http.Status.length_required) - } - - res := app.repo.add_pkg_from_path(repo, pkg_path) or { - app.lerror('Error while adding package: $err.msg') - - os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } - - return app.json(http.Status.internal_server_error, new_response('Failed to add package.')) - } - - if !res.added { - os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } - - app.lwarn("Duplicate package '$res.pkg.full_name()' in repo '$repo'.") - - return app.json(http.Status.bad_request, new_response('File already exists.')) - } - - app.linfo("Added '$res.pkg.full_name()' to repo '$repo ($res.pkg.info.arch)'.") - - return app.json(http.Status.ok, new_response('Package added successfully.')) -} diff --git a/src/server/server.v b/src/server/server.v deleted file mode 100644 index 5bf9a87..0000000 --- a/src/server/server.v +++ /dev/null @@ -1,61 +0,0 @@ -module server - -import web -import os -import log -import repo -import util - -const port = 8000 - -struct App { - web.Context -pub: - conf Config [required; web_global] -pub mut: - repo repo.RepoGroupManager [required; web_global] - // This is used to claim the file lock on the repos file - git_mutex shared util.Dummy -} - -// server starts the web server & starts listening for requests -pub fn server(conf Config) ? { - // Prevent using 'any' as the default arch - if conf.default_arch == 'any' { - util.exit_with_message(1, "'any' is not allowed as the value for default_arch.") - } - - // Configure logger - log_level := log.level_from_tag(conf.log_level) or { - util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') - } - - mut logger := log.Log{ - level: log_level - } - - logger.set_full_logpath(conf.log_file) - logger.log_to_console_too() - - defer { - logger.info('Flushing log file') - logger.flush() - logger.close() - } - - // This also creates the directories if needed - repo := repo.new(conf.repos_dir, conf.pkg_dir, conf.default_arch) or { - logger.error(err.msg) - exit(1) - } - - os.mkdir_all(conf.download_dir) or { - util.exit_with_message(1, 'Failed to create download directory.') - } - - web.run(&App{ - logger: logger - conf: conf - repo: repo - }, server.port) -} diff --git a/src/util.v b/src/util.v index 49c9d22..f81a256 100644 --- a/src/util.v +++ b/src/util.v @@ -1,55 +1,9 @@ module util import os -import io import crypto.md5 import crypto.sha256 -const reader_buf_size = 1_000_000 - -const prefixes = ['B', 'KB', 'MB', 'GB'] - -// Dummy struct to work around the fact that you can only share structs, maps & -// arrays -pub struct Dummy { - x int -} - -// exit_with_message exits the program with a given status code after having -// first printed a specific message to STDERR -[noreturn] -pub fn exit_with_message(code int, msg string) { - eprintln(msg) - exit(code) -} - -// reader_to_file writes the contents of a BufferedReader to a file -pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { - mut file := os.create(path) ? - defer { - file.close() - } - - mut buf := []byte{len: util.reader_buf_size} - mut bytes_left := length - - // Repeat as long as the stream still has data - for bytes_left > 0 { - // TODO check if just breaking here is safe - bytes_read := reader.read(mut buf) or { break } - bytes_left -= bytes_read - - mut to_write := bytes_read - - for to_write > 0 { - // TODO don't just loop infinitely here - bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue } - - to_write = to_write - bytes_written - } - } -} - // hash_file returns the md5 & sha256 hash of a given file // TODO actually implement sha256 pub fn hash_file(path &string) ?(string, string) { @@ -78,16 +32,3 @@ pub fn hash_file(path &string) ?(string, string) { return md5sum.checksum().hex(), sha256sum.checksum().hex() } - -// pretty_bytes converts a byte count to human-readable version -pub fn pretty_bytes(bytes int) string { - mut i := 0 - mut n := f32(bytes) - - for n >= 1024 { - i++ - n /= 1024 - } - - return '${n:.2}${util.prefixes[i]}' -} diff --git a/src/web/parse.v b/src/web/parse.v index 2eeef5e..554c9f0 100644 --- a/src/web/parse.v +++ b/src/web/parse.v @@ -34,7 +34,7 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) { } if x.len > 0 { return IError(http.UnexpectedExtraAttributeError{ - attributes: x + msg: 'Encountered unexpected extra attributes: $x' }) } if methods.len == 0 { @@ -49,8 +49,8 @@ fn parse_attrs(name string, attrs []string) ?([]http.Method, string) { fn parse_query_from_url(url urllib.URL) map[string]string { mut query := map[string]string{} - for v in url.query().data { - query[v.key] = v.value + for k, v in url.query().data { + query[k] = v.data[0] } return query } diff --git a/src/web/web.v b/src/web/web.v index 000c6a6..b1e8b92 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -12,6 +12,9 @@ import time import json import log +// A type which don't get filtered inside templates +pub type RawHtml = string + // A dummy structure that returns from routes to indicate that you actually sent something to a user [noinit] pub struct Result {} @@ -19,7 +22,7 @@ pub struct Result {} pub const ( methods_with_form = [http.Method.post, .put, .patch] headers_close = http.new_custom_header_from_map({ - 'Server': 'VWeb' + 'Server': 'VWeb' http.CommonHeader.connection.str(): 'close' }) or { panic('should never fail') } @@ -138,8 +141,8 @@ pub const ( // It has fields for the query, form, files. pub struct Context { mut: - content_type string = 'text/plain' - status http.Status = http.Status.ok + content_type string = 'text/plain' + status string = '200 OK' pub: // HTTP Request req http.Request @@ -183,14 +186,24 @@ struct Route { path string } +// Defining this method is optional. +// init_server is called at server start. +// You can use it for initializing globals. +pub fn (ctx Context) init_server() { + eprintln('init_server() has been deprecated, please init your web app in `fn main()`') +} + // Defining this method is optional. // before_request is called before every request (aka middleware). // Probably you can use it for check user session cookie or add header. pub fn (ctx Context) before_request() {} -// send_string -fn send_string(mut conn net.TcpConn, s string) ? { - conn.write(s.bytes()) ? +pub struct Cookie { + name string + value string + expires time.Time + secure bool + http_only bool } // send_response_to_client sends a response to the client @@ -212,27 +225,34 @@ pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bo text: res } resp.set_version(.v1_1) - resp.set_status(ctx.status) + resp.set_status(http.status_from_int(ctx.status.int())) send_string(mut ctx.conn, resp.bytestr()) or { return false } return true } -// text responds to a request with some plaintext. -pub fn (mut ctx Context) text(status http.Status, s string) Result { - ctx.status = status +// html HTTP_OK with s as payload with content-type `text/html` +pub fn (mut ctx Context) html(s string) Result { + ctx.send_response_to_client('text/html', s) + return Result{} +} +// text HTTP_OK with s as payload with content-type `text/plain` +pub fn (mut ctx Context) text(s string) Result { ctx.send_response_to_client('text/plain', s) - return Result{} } // json HTTP_OK with json_s as payload with content-type `application/json` -pub fn (mut ctx Context) json(status http.Status, j T) Result { - ctx.status = status - +pub fn (mut ctx Context) json(j T) Result { json_s := json.encode(j) ctx.send_response_to_client('application/json', json_s) + return Result{} +} +// json_pretty Response HTTP_OK with a pretty-printed JSON result +pub fn (mut ctx Context) json_pretty(j T) Result { + json_s := json.encode_pretty(j) + ctx.send_response_to_client('application/json', json_s) return Result{} } @@ -282,7 +302,7 @@ pub fn (mut ctx Context) file(f_path string) Result { header: header.join(web.headers_close) } resp.set_version(.v1_1) - resp.set_status(ctx.status) + resp.set_status(http.status_from_int(ctx.status.int())) send_string(mut ctx.conn, resp.bytestr()) or { return Result{} } mut buf := []byte{len: 1_000_000} @@ -308,10 +328,10 @@ pub fn (mut ctx Context) file(f_path string) Result { return Result{} } -// status responds with an empty textual response, essentially only returning -// the given status code. -pub fn (mut ctx Context) status(status http.Status) Result { - return ctx.text(status, '') +// ok Response HTTP_OK with s as payload +pub fn (mut ctx Context) ok(s string) Result { + ctx.send_response_to_client(ctx.content_type, s) + return Result{} } // server_error Response a server error @@ -341,7 +361,64 @@ pub fn (mut ctx Context) redirect(url string) Result { // not_found Send an not_found response pub fn (mut ctx Context) not_found() Result { - return ctx.status(http.Status.not_found) + if ctx.done { + return Result{} + } + ctx.done = true + send_string(mut ctx.conn, web.http_404.bytestr()) or {} + return Result{} +} + +// set_cookie Sets a cookie +pub fn (mut ctx Context) set_cookie(cookie Cookie) { + mut cookie_data := []string{} + mut secure := if cookie.secure { 'Secure;' } else { '' } + secure += if cookie.http_only { ' HttpOnly' } else { ' ' } + cookie_data << secure + if cookie.expires.unix > 0 { + cookie_data << 'expires=$cookie.expires.utc_string()' + } + data := cookie_data.join(' ') + ctx.add_header('Set-Cookie', '$cookie.name=$cookie.value; $data') +} + +// set_content_type Sets the response content type +pub fn (mut ctx Context) set_content_type(typ string) { + ctx.content_type = typ +} + +// set_cookie_with_expire_date Sets a cookie with a `expire_data` +pub fn (mut ctx Context) set_cookie_with_expire_date(key string, val string, expire_date time.Time) { + ctx.add_header('Set-Cookie', '$key=$val; Secure; HttpOnly; expires=$expire_date.utc_string()') +} + +// get_cookie Gets a cookie by a key +pub fn (ctx &Context) get_cookie(key string) ?string { // TODO refactor + mut cookie_header := ctx.get_header('cookie') + if cookie_header == '' { + cookie_header = ctx.get_header('Cookie') + } + cookie_header = ' ' + cookie_header + // println('cookie_header="$cookie_header"') + // println(ctx.req.header) + cookie := if cookie_header.contains(';') { + cookie_header.find_between(' $key=', ';') + } else { + cookie_header.find_between(' $key=', '\r') + } + if cookie != '' { + return cookie.trim_space() + } + return error('Cookie not found') +} + +// set_status Sets the response status +pub fn (mut ctx Context) set_status(code int, desc string) { + if code < 100 || code > 599 { + ctx.status = '500 Internal Server Error' + } else { + ctx.status = '$code $desc' + } } // add_header Adds an header to the response with key and val @@ -483,6 +560,12 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { // Calling middleware... app.before_request() + // Static handling + if serve_if_static(mut app, url) { + // successfully served a static file + return + } + // Route matching $for method in T.methods { $if method.return_type is Result { @@ -578,6 +661,83 @@ fn route_matches(url_words []string, route_words []string) ?[]string { return params } +// serve_if_static checks if request is for a static file and serves it +// returns true if we served a static file, false otherwise +[manualfree] +fn serve_if_static(mut app T, url urllib.URL) bool { + // TODO: handle url parameters properly - for now, ignore them + static_file := app.static_files[url.path] + mime_type := app.static_mime_types[url.path] + if static_file == '' || mime_type == '' { + return false + } + data := os.read_file(static_file) or { + send_string(mut app.conn, web.http_404.bytestr()) or {} + return true + } + app.send_response_to_client(mime_type, data) + unsafe { data.free() } + return true +} + +// scan_static_directory makes a static route for each file in a directory +fn (mut ctx Context) scan_static_directory(directory_path string, mount_path string) { + files := os.ls(directory_path) or { panic(err) } + if files.len > 0 { + for file in files { + full_path := os.join_path(directory_path, file) + if os.is_dir(full_path) { + ctx.scan_static_directory(full_path, mount_path + '/' + file) + } else if file.contains('.') && !file.starts_with('.') && !file.ends_with('.') { + ext := os.file_ext(file) + // Rudimentary guard against adding files not in mime_types. + // Use serve_static directly to add non-standard mime types. + if ext in web.mime_types { + ctx.serve_static(mount_path + '/' + file, full_path) + } + } + } + } +} + +// handle_static Handles a directory static +// If `root` is set the mount path for the dir will be in '/' +pub fn (mut ctx Context) handle_static(directory_path string, root bool) bool { + if ctx.done || !os.exists(directory_path) { + return false + } + dir_path := directory_path.trim_space().trim_right('/') + mut mount_path := '' + if dir_path != '.' && os.is_dir(dir_path) && !root { + // Mount point hygene, "./assets" => "/assets". + mount_path = '/' + dir_path.trim_left('.').trim('/') + } + ctx.scan_static_directory(dir_path, mount_path) + return true +} + +// mount_static_folder_at - makes all static files in `directory_path` and inside it, available at http://server/mount_path +// For example: suppose you have called .mount_static_folder_at('/var/share/myassets', '/assets'), +// and you have a file /var/share/myassets/main.css . +// => That file will be available at URL: http://server/assets/main.css . +pub fn (mut ctx Context) mount_static_folder_at(directory_path string, mount_path string) bool { + if ctx.done || mount_path.len < 1 || mount_path[0] != `/` || !os.exists(directory_path) { + return false + } + dir_path := directory_path.trim_right('/') + ctx.scan_static_directory(dir_path, mount_path[1..]) + return true +} + +// serve_static Serves a file static +// `url` is the access path on the site, `file_path` is the real path to the file, `mime_type` is the file type +pub fn (mut ctx Context) serve_static(url string, file_path string) { + ctx.static_files[url] = file_path + // ctx.static_mime_types[url] = mime_type + ext := os.file_ext(file_path) + ctx.static_mime_types[url] = web.mime_types[ext] +} + // ip Returns the ip address from the current user pub fn (ctx &Context) ip() string { mut ip := ctx.req.header.get(.x_forwarded_for) or { '' } @@ -600,6 +760,16 @@ pub fn (mut ctx Context) error(s string) { ctx.form_error = s } +// not_found Returns an empty result +pub fn not_found() Result { + return Result{} +} + +// send_string +fn send_string(mut conn net.TcpConn, s string) ? { + conn.write(s.bytes()) ? +} + // filter Do not delete. // It used by `vlib/v/gen/c/str_intp.v:130` for string interpolation inside web templates // TODO: move it to template render diff --git a/test.py b/test.py index 46dac52..5721310 100644 --- a/test.py +++ b/test.py @@ -38,7 +38,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len): Generates a random .PKGINFO """ name = "-".join(random_words(words, name_min_len, name_max_len)) - ver = "0.1.0-3" # doesn't matter what it is anyways + ver = "0.1.0-1" # doesn't matter what it is anyways # TODO add random dependencies (all types) @@ -97,7 +97,7 @@ async def upload_random_package(tar_path, sem): async with sem: with open(tar_path, 'rb') as f: async with aiohttp.ClientSession() as s: - async with s.post("http://localhost:8000/vieter/publish", data=f.read(), headers={"x-api-key": "test"}) as r: + async with s.post("http://localhost:8000/publish", data=f.read(), headers={"x-api-key": "test"}) as r: return await check_output(r) diff --git a/vieter.toml b/vieter.toml deleted file mode 100644 index 8e0447b..0000000 --- a/vieter.toml +++ /dev/null @@ -1,10 +0,0 @@ -# This file contains settings used during development -api_key = "test" -download_dir = "data/downloads" -repos_dir = "data/repos" -pkg_dir = "data/pkgs" -log_level = "DEBUG" -repos_file = "data/repos.json" -default_arch = "x86_64" - -address = "http://localhost:8000"