Switched to patch-based builder, based on weekly

.dockerignore

@@ -2,3 +2,4 @@
 
 !src/
 !Makefile
+!patches/

.gitignore

@@ -15,3 +15,6 @@ vieter.log
 # External lib; gets added by Makefile
 libarchive-*
 test/
+
+# V compiler directory
+v-*/

.woodpecker/.build.yml

@@ -5,26 +5,58 @@ matrix:
     - linux/arm/v7
 
 # These checks already get performed on the feature branches
-branches:
-  exclude: [ main, dev ]
 platform: ${PLATFORM}
 
 pipeline:
   # The default build isn't needed, as alpine switches to gcc for the compiler anyways
   debug:
     image: 'chewingbever/vlang:latest'
+    pull: true
     group: 'build'
     commands:
       - make debug
+    when:
+      event: push
 
   prod:
     image: 'chewingbever/vlang:latest'
+    pull: true
     environment:
       - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static
     group: 'build'
     commands:
       - make prod
-      # Make sure the binary is actually static
+      # Make sure the binary is actually statically built
       - readelf -d pvieter
       - du -h pvieter
       - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]'
+      # This removes so much, it's amazing
+      - strip -s pvieter
+      - du -h pvieter
+    when:
+      event: push
+
+  upload:
+    image: 'chewingbever/vlang:latest'
+    secrets: [ s3_username, s3_password ]
+    commands:
+      # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f
+      - export URL=s3.rustybever.be
+      - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')"
+      - export DATE="$(date -R --utc)"
+      - export CONTENT_TYPE='application/zstd'
+      - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH"
+      - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64`
+
+      - >
+        curl 
+        --silent
+        -XPUT
+        -T pvieter
+        -H "Host: $URL"
+        -H "Date: $DATE"
+        -H "Content-Type: $CONTENT_TYPE"
+        -H "Authorization: AWS $S3_USERNAME:$SIGNATURE"
+        https://$URL$OBJ_PATH
+    when:
+      event: push

.woodpecker/.builder.yml

@@ -12,4 +12,7 @@ pipeline:
       platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
     when:
       event: push
-      path: Dockerfile.builder
+      path:
+        - Dockerfile.builder
+        - patches/*
+        - Makefile

.woodpecker/.image.yml

@@ -2,12 +2,14 @@
 # because it has to be the target of the pull request
 branches: dev
 platform: linux/amd64
+depends_on: [ build ]
 
 pipeline:
   dryrun:
     image: woodpeckerci/plugin-docker-buildx
     secrets: [ docker_username, docker_password ]
     settings:
+      dockerfile: Dockerfile.ci
       repo: chewingbever/vieter
       tag: dev
       platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]

.woodpecker/.lint.yml

@@ -6,12 +6,8 @@ platform: linux/amd64
 pipeline:
   lint:
     image: 'chewingbever/vlang:latest'
+    pull: true
     group: lint
     commands:
       - make lint
-
-  vet:
-    image: 'chewingbever/vlang:latest'
-    group: lint
-    commands:
       - make vet

.woodpecker/.publish.yml

@@ -6,9 +6,12 @@ pipeline:
     image: woodpeckerci/plugin-docker-buildx
     secrets: [ docker_username, docker_password ]
     settings:
+      dockerfile: Dockerfile.ci
       repo: chewingbever/vieter
       tag: dev
       platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
+      build_args_from_env:
+        - CI_COMMIT_SHA
     when:
       event: push
       branch: dev
@@ -28,3 +31,4 @@ pipeline:
 
 depends_on:
   - builder
+  - build

.woodpecker/.release.yml

@@ -0,0 +1,39 @@
+# Yeah so this only works on tags so we'll worry about this later
+platform: linux/amd64
+branches: main
+
+# We need the entire repo in order for the release names to work
+skip_clone: true
+
+pipeline:
+  prepare:
+    image: 'chewingbever/vlang:latest'
+    pull: true
+    secrets: [ s3_username, s3_password ]
+    commands:
+      - git clone "$CI_REPO_REMOTE" .
+      - git checkout "$CI_COMMIT_BRANCH"
+      # Write the title to a file that the plugin can then read
+      - echo "$(git describe --tags --abbrev=0 2> /dev/null || echo '0.0.0')-$(git rev-list --count ^dev)" > title
+      - cat title
+      - mc alias set s3/ https://s3.rustybever.be "$S3_USERNAME" "$S3_PASSWORD"
+      - mc cp -r "s3/vieter/commits/$CI_COMMIT_SHA" assets
+
+  release:
+    image: 'plugins/gitea-release'
+    secrets:
+      - gitea_release_api_key
+    settings:
+      base_url: https://git.rustybever.be
+      files: assets/*
+      checksums:
+        - md5
+        - sha256
+      prerelease: true
+      # This should get read in as a file
+      title: title
+    when:
+      event: push
+
+depends_on:
+  - build

Dockerfile.builder

@@ -1,6 +1,6 @@
-FROM alpine:3.12
+FROM alpine:3.15
 
-ARG V_RELEASE=weekly.2022.03
+ARG TARGETPLATFORM
 
 WORKDIR /opt/vlang
 
@@ -10,7 +10,7 @@ ENV VFLAGS -cc gcc
 
 RUN ln -s /opt/vlang/v /usr/bin/v && \
   apk --no-cache add \
-    curl git make gcc \
+    git make gcc curl openssl \
     musl-dev \
     openssl-libs-static openssl-dev \
     zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \
@@ -19,15 +19,16 @@ RUN ln -s /opt/vlang/v /usr/bin/v && \
     libarchive-static libarchive-dev \
     diffutils
 
-RUN curl -Lo - "https://github.com/vlang/v/archive/refs/tags/${V_RELEASE}.tar.gz" | tar xzf - && \
+COPY patches ./patches
-  mv "v-${V_RELEASE}"/* /opt/vlang && \
+COPY Makefile ./
-  make && v -version
+
-# RUN git clone \
+RUN make v && \
-#       'https://github.com/ChewingBever/v/' \
+  mv v-*/* /opt/vlang && \
-#       -b vweb-streaming \
+  v -version
-#       --single-branch \
+
-#       '/opt/vlang' && \
+RUN if [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \
-#     rm -rf '/vlang-local' && \
+  wget -O /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && \
-#     make && v -version
+  chmod +x /usr/local/bin/mc ; \
+fi
 
 CMD ["v"]

Dockerfile.ci

@@ -0,0 +1,20 @@
+# vim: ft=dockerfile
+# This image just has the required tools to download the binaries
+FROM chewingbever/vlang:latest AS builder
+
+ARG TARGETPLATFORM
+ARG CI_COMMIT_SHA
+
+WORKDIR /app
+
+RUN curl --fail \
+    -o vieter \
+    "https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \
+    chmod +x vieter
+
+
+FROM scratch
+
+COPY --from=builder /app/vieter /bin/vieter
+
+ENTRYPOINT ["/bin/vieter"]

Makefile

@@ -6,9 +6,11 @@ LARCHIVE_VER := 3.5.2
 LARCHIVE_DIR := libarchive-$(LARCHIVE_VER)
 LARCHIVE_LIB := $(LARCHIVE_DIR)/libarchive/libarchive.so
 
+V_RELEASE := weekly.2022.04
+
 # Custom V command for linking libarchive
 # V := LDFLAGS=$(PWD)/$(LARCHIVE_LIB) v -cflags '-I$(PWD)/$(LARCHIVE_DIR) -I $(PWD)/$(LARCHIVE_DIR)'
-V := v -showcc
+V := v-$(V_RELEASE)/v -showcc
 
 all: vieter
 
@@ -64,24 +66,13 @@ fmt:
 vet:
 	$(V) vet -W $(SRC_DIR)
 
-# Pulls & builds my personal build of the v compiler, required for this project to function
+# Build & patch the V compiler
-.PHONY: customv
+.PHONY: v
-customv:
+v: v-$(V_RELEASE)/v
-	rm -rf v-jjr
+v-$(V_RELEASE)/v:
-	git clone \
+	curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf -
-		-b vweb-streaming \
+	cd patches && ./patch.sh '../v-$(V_RELEASE)'
-		--single-branch \
+	'$(MAKE)' -C 'v-$(V_RELEASE)'
-		https://github.com/ChewingBever/v jjr-v
-	'$(MAKE)' -C jjr-v
-
-
-# =====LIBARCHIVE=====
-.PHONY: libarchive
-libarchive: $(LARCHIVE_LIB)
-$(LARCHIVE_LIB):
-	curl -o - "https://libarchive.org/downloads/libarchive-${LARCHIVE_VER}.tar.gz" | tar xzf -
-	cd "libarchive-${LARCHIVE_VER}" && cmake .
-	'$(MAKE)' -C "libarchive-${LARCHIVE_VER}"
 
 clean:
-	rm -rf '$(LARCHIVE_DIR)' 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c'
+	rm -rf '$(LARCHIVE_DIR)' 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'v-$(V_RELEASE)'

README.md

@@ -33,7 +33,12 @@ terminal, & upload them to the servers using CI.
 
 The data directory consists of three main directories:
 
-* `downloads` - This is where packages are initially downloaded. Because vieter moves files from this folder to the `pkgs` folder, these two folders should best be on the same drive
+* `downloads` - This is where packages are initially downloaded. Because vieter
+  moves files from this folder to the `pkgs` folder, these two folders should
+  best be on the same drive
 * `pkgs` - This is where approved package files are stored.
-* `repos` - Each repository gets a subfolder here. The subfolder contains the uncompressed contents of the db file.
+* `repos` - Each repository gets a subfolder here. The subfolder contains the
-    * Each repo subdirectory contains the compressed db & files archive for the repository, alongside a directory called `files` which contains the uncompressed contents.
+  uncompressed contents of the db file.
+    * Each repo subdirectory contains the compressed db & files archive for the
+      repository, alongside a directory called `files` which contains the
+      uncompressed contents.

patches/parse_request_no_body.v

@@ -0,0 +1,23 @@
+// Parse the header of a raw HTTP request into a Request object
+pub fn parse_request_head(mut reader io.BufferedReader) ?Request {
+	// request line
+	mut line := reader.read_line() ?
+	method, target, version := parse_request_line(line) ?
+
+	// headers
+	mut header := new_header()
+	line = reader.read_line() ?
+	for line != '' {
+		key, value := parse_header(line) ?
+		header.add_custom(key, value) ?
+		line = reader.read_line() ?
+	}
+	header.coerce(canonicalize: true)
+
+	return Request{
+		method: method
+		url: target.str()
+		header: header
+		version: version
+	}
+}

patches/patch.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+# This file patches the downloaded V version
+# Should be run from within the directory it's in, as it uses relative paths to the files used for patching.
+# $1 is the path to the downloaded V version
+
+# Add parse_request_no_body
+cat parse_request_no_body.v >> "$1"/vlib/net/http/request.v

src/package.v

@@ -143,16 +143,17 @@ pub fn read_pkg(pkg_path string) ?Pkg {
 
 			// TODO can this unsafe block be avoided?
 			buf = unsafe { malloc(size) }
+			defer {
+				unsafe {
+					free(buf)
+				}
+			}
 			C.archive_read_data(a, buf, size)
 
 			unsafe {
 				println(cstring_to_vstring(buf))
 			}
 			pkg_info = parse_pkg_info_string(unsafe { cstring_to_vstring(buf) }) ?
-
-			unsafe {
-				free(buf)
-			}
 		} else {
 			C.archive_read_data_skip(a)
 		}