diff --git a/.dockerignore b/.dockerignore index 643e6db..3909ae0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,4 @@ !src/ !Makefile +!patches/ diff --git a/.gitignore b/.gitignore index 71064b1..df4fcc9 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ vieter.log # External lib; gets added by Makefile libarchive-* test/ + +# V compiler directory +v-*/ diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 4b87f8b..66caa8c 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -5,26 +5,58 @@ matrix: - linux/arm/v7 # These checks already get performed on the feature branches -branches: - exclude: [ main, dev ] platform: ${PLATFORM} pipeline: # The default build isn't needed, as alpine switches to gcc for the compiler anyways debug: image: 'chewingbever/vlang:latest' + pull: true group: 'build' commands: - make debug + when: + event: push prod: image: 'chewingbever/vlang:latest' + pull: true environment: - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static group: 'build' commands: - make prod - # Make sure the binary is actually static + # Make sure the binary is actually statically built - readelf -d pvieter - du -h pvieter - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' + # This removes so much, it's amazing + - strip -s pvieter + - du -h pvieter + when: + event: push + + upload: + image: 'chewingbever/vlang:latest' + secrets: [ s3_username, s3_password ] + commands: + # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f + - export URL=s3.rustybever.be + - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')" + - export DATE="$(date -R --utc)" + - export CONTENT_TYPE='application/zstd' + - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH" + - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64` + + - > + curl + --silent + -XPUT + -T pvieter + -H "Host: $URL" + -H "Date: $DATE" + -H "Content-Type: $CONTENT_TYPE" + -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" + https://$URL$OBJ_PATH + when: + event: push diff --git a/.woodpecker/.builder.yml b/.woodpecker/.builder.yml index f827566..75f5267 100644 --- a/.woodpecker/.builder.yml +++ b/.woodpecker/.builder.yml @@ -12,4 +12,7 @@ pipeline: platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] when: event: push - path: Dockerfile.builder + path: + - Dockerfile.builder + - patches/* + - Makefile diff --git a/.woodpecker/.image.yml b/.woodpecker/.image.yml index 7f583a8..3d0b310 100644 --- a/.woodpecker/.image.yml +++ b/.woodpecker/.image.yml @@ -2,12 +2,14 @@ # because it has to be the target of the pull request branches: dev platform: linux/amd64 +depends_on: [ build ] pipeline: dryrun: image: woodpeckerci/plugin-docker-buildx secrets: [ docker_username, docker_password ] settings: + dockerfile: Dockerfile.ci repo: chewingbever/vieter tag: dev platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] diff --git a/.woodpecker/.lint.yml b/.woodpecker/.lint.yml index 1ccd077..ce000cd 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/.lint.yml @@ -6,12 +6,8 @@ platform: linux/amd64 pipeline: lint: image: 'chewingbever/vlang:latest' + pull: true group: lint commands: - make lint - - vet: - image: 'chewingbever/vlang:latest' - group: lint - commands: - make vet diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 9e5f39a..2e6b459 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -6,9 +6,12 @@ pipeline: image: woodpeckerci/plugin-docker-buildx secrets: [ docker_username, docker_password ] settings: + dockerfile: Dockerfile.ci repo: chewingbever/vieter tag: dev platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + build_args_from_env: + - CI_COMMIT_SHA when: event: push branch: dev @@ -28,3 +31,4 @@ pipeline: depends_on: - builder + - build diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml new file mode 100644 index 0000000..bc43b90 --- /dev/null +++ b/.woodpecker/.release.yml @@ -0,0 +1,39 @@ +# Yeah so this only works on tags so we'll worry about this later +platform: linux/amd64 +branches: main + +# We need the entire repo in order for the release names to work +skip_clone: true + +pipeline: + prepare: + image: 'chewingbever/vlang:latest' + pull: true + secrets: [ s3_username, s3_password ] + commands: + - git clone "$CI_REPO_REMOTE" . + - git checkout "$CI_COMMIT_BRANCH" + # Write the title to a file that the plugin can then read + - echo "$(git describe --tags --abbrev=0 2> /dev/null || echo '0.0.0')-$(git rev-list --count ^dev)" > title + - cat title + - mc alias set s3/ https://s3.rustybever.be "$S3_USERNAME" "$S3_PASSWORD" + - mc cp -r "s3/vieter/commits/$CI_COMMIT_SHA" assets + + release: + image: 'plugins/gitea-release' + secrets: + - gitea_release_api_key + settings: + base_url: https://git.rustybever.be + files: assets/* + checksums: + - md5 + - sha256 + prerelease: true + # This should get read in as a file + title: title + when: + event: push + +depends_on: + - build diff --git a/Dockerfile.builder b/Dockerfile.builder index 406386b..1f44b4c 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -1,6 +1,6 @@ -FROM alpine:3.12 +FROM alpine:3.15 -ARG V_RELEASE=weekly.2022.03 +ARG TARGETPLATFORM WORKDIR /opt/vlang @@ -10,7 +10,7 @@ ENV VFLAGS -cc gcc RUN ln -s /opt/vlang/v /usr/bin/v && \ apk --no-cache add \ - curl git make gcc \ + git make gcc curl openssl \ musl-dev \ openssl-libs-static openssl-dev \ zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \ @@ -19,15 +19,16 @@ RUN ln -s /opt/vlang/v /usr/bin/v && \ libarchive-static libarchive-dev \ diffutils -RUN curl -Lo - "https://github.com/vlang/v/archive/refs/tags/${V_RELEASE}.tar.gz" | tar xzf - && \ - mv "v-${V_RELEASE}"/* /opt/vlang && \ - make && v -version -# RUN git clone \ -# 'https://github.com/ChewingBever/v/' \ -# -b vweb-streaming \ -# --single-branch \ -# '/opt/vlang' && \ -# rm -rf '/vlang-local' && \ -# make && v -version +COPY patches ./patches +COPY Makefile ./ + +RUN make v && \ + mv v-*/* /opt/vlang && \ + v -version + +RUN if [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \ + wget -O /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && \ + chmod +x /usr/local/bin/mc ; \ +fi CMD ["v"] diff --git a/Dockerfile.ci b/Dockerfile.ci new file mode 100644 index 0000000..0a3f8cd --- /dev/null +++ b/Dockerfile.ci @@ -0,0 +1,20 @@ +# vim: ft=dockerfile +# This image just has the required tools to download the binaries +FROM chewingbever/vlang:latest AS builder + +ARG TARGETPLATFORM +ARG CI_COMMIT_SHA + +WORKDIR /app + +RUN curl --fail \ + -o vieter \ + "https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \ + chmod +x vieter + + +FROM scratch + +COPY --from=builder /app/vieter /bin/vieter + +ENTRYPOINT ["/bin/vieter"] diff --git a/Makefile b/Makefile index 9a5d6fa..aea3f6f 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,11 @@ LARCHIVE_VER := 3.5.2 LARCHIVE_DIR := libarchive-$(LARCHIVE_VER) LARCHIVE_LIB := $(LARCHIVE_DIR)/libarchive/libarchive.so +V_RELEASE := weekly.2022.04 + # Custom V command for linking libarchive # V := LDFLAGS=$(PWD)/$(LARCHIVE_LIB) v -cflags '-I$(PWD)/$(LARCHIVE_DIR) -I $(PWD)/$(LARCHIVE_DIR)' -V := v -showcc +V := v-$(V_RELEASE)/v -showcc all: vieter @@ -64,24 +66,13 @@ fmt: vet: $(V) vet -W $(SRC_DIR) -# Pulls & builds my personal build of the v compiler, required for this project to function -.PHONY: customv -customv: - rm -rf v-jjr - git clone \ - -b vweb-streaming \ - --single-branch \ - https://github.com/ChewingBever/v jjr-v - '$(MAKE)' -C jjr-v - - -# =====LIBARCHIVE===== -.PHONY: libarchive -libarchive: $(LARCHIVE_LIB) -$(LARCHIVE_LIB): - curl -o - "https://libarchive.org/downloads/libarchive-${LARCHIVE_VER}.tar.gz" | tar xzf - - cd "libarchive-${LARCHIVE_VER}" && cmake . - '$(MAKE)' -C "libarchive-${LARCHIVE_VER}" +# Build & patch the V compiler +.PHONY: v +v: v-$(V_RELEASE)/v +v-$(V_RELEASE)/v: + curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf - + cd patches && ./patch.sh '../v-$(V_RELEASE)' + '$(MAKE)' -C 'v-$(V_RELEASE)' clean: - rm -rf '$(LARCHIVE_DIR)' 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' + rm -rf '$(LARCHIVE_DIR)' 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'v-$(V_RELEASE)' diff --git a/README.md b/README.md index 5d49577..b51ab7f 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,12 @@ terminal, & upload them to the servers using CI. The data directory consists of three main directories: -* `downloads` - This is where packages are initially downloaded. Because vieter moves files from this folder to the `pkgs` folder, these two folders should best be on the same drive +* `downloads` - This is where packages are initially downloaded. Because vieter + moves files from this folder to the `pkgs` folder, these two folders should + best be on the same drive * `pkgs` - This is where approved package files are stored. -* `repos` - Each repository gets a subfolder here. The subfolder contains the uncompressed contents of the db file. - * Each repo subdirectory contains the compressed db & files archive for the repository, alongside a directory called `files` which contains the uncompressed contents. +* `repos` - Each repository gets a subfolder here. The subfolder contains the + uncompressed contents of the db file. + * Each repo subdirectory contains the compressed db & files archive for the + repository, alongside a directory called `files` which contains the + uncompressed contents. diff --git a/patches/parse_request_no_body.v b/patches/parse_request_no_body.v new file mode 100644 index 0000000..c00a51c --- /dev/null +++ b/patches/parse_request_no_body.v @@ -0,0 +1,23 @@ +// Parse the header of a raw HTTP request into a Request object +pub fn parse_request_head(mut reader io.BufferedReader) ?Request { + // request line + mut line := reader.read_line() ? + method, target, version := parse_request_line(line) ? + + // headers + mut header := new_header() + line = reader.read_line() ? + for line != '' { + key, value := parse_header(line) ? + header.add_custom(key, value) ? + line = reader.read_line() ? + } + header.coerce(canonicalize: true) + + return Request{ + method: method + url: target.str() + header: header + version: version + } +} diff --git a/patches/patch.sh b/patches/patch.sh new file mode 100755 index 0000000..a7378e7 --- /dev/null +++ b/patches/patch.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh +# This file patches the downloaded V version +# Should be run from within the directory it's in, as it uses relative paths to the files used for patching. +# $1 is the path to the downloaded V version + +# Add parse_request_no_body +cat parse_request_no_body.v >> "$1"/vlib/net/http/request.v diff --git a/src/package.v b/src/package.v index c229e3f..7db5849 100644 --- a/src/package.v +++ b/src/package.v @@ -143,16 +143,17 @@ pub fn read_pkg(pkg_path string) ?Pkg { // TODO can this unsafe block be avoided? buf = unsafe { malloc(size) } + defer { + unsafe { + free(buf) + } + } C.archive_read_data(a, buf, size) unsafe { println(cstring_to_vstring(buf)) } pkg_info = parse_pkg_info_string(unsafe { cstring_to_vstring(buf) }) ? - - unsafe { - free(buf) - } } else { C.archive_read_data_skip(a) }