Switched to patch-based builder, based on weekly

main
Jef Roosens 2022-01-27 19:18:11 +01:00
commit 78e343fae2
Signed by untrusted user: Jef Roosens
GPG Key ID: 955C0660072F691F
15 changed files with 177 additions and 49 deletions

View File

@ -2,3 +2,4 @@
!src/
!Makefile
!patches/

3
.gitignore vendored
View File

@ -15,3 +15,6 @@ vieter.log
# External lib; gets added by Makefile
libarchive-*
test/
# V compiler directory
v-*/

View File

@ -5,26 +5,58 @@ matrix:
- linux/arm/v7
# These checks already get performed on the feature branches
branches:
exclude: [ main, dev ]
platform: ${PLATFORM}
pipeline:
# The default build isn't needed, as alpine switches to gcc for the compiler anyways
debug:
image: 'chewingbever/vlang:latest'
pull: true
group: 'build'
commands:
- make debug
when:
event: push
prod:
image: 'chewingbever/vlang:latest'
pull: true
environment:
- LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static
group: 'build'
commands:
- make prod
# Make sure the binary is actually static
# Make sure the binary is actually statically built
- readelf -d pvieter
- du -h pvieter
- '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]'
# This removes so much, it's amazing
- strip -s pvieter
- du -h pvieter
when:
event: push
upload:
image: 'chewingbever/vlang:latest'
secrets: [ s3_username, s3_password ]
commands:
# https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f
- export URL=s3.rustybever.be
- export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')"
- export DATE="$(date -R --utc)"
- export CONTENT_TYPE='application/zstd'
- export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH"
- export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64`
- >
curl
--silent
-XPUT
-T pvieter
-H "Host: $URL"
-H "Date: $DATE"
-H "Content-Type: $CONTENT_TYPE"
-H "Authorization: AWS $S3_USERNAME:$SIGNATURE"
https://$URL$OBJ_PATH
when:
event: push

View File

@ -12,4 +12,7 @@ pipeline:
platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
when:
event: push
path: Dockerfile.builder
path:
- Dockerfile.builder
- patches/*
- Makefile

View File

@ -2,12 +2,14 @@
# because it has to be the target of the pull request
branches: dev
platform: linux/amd64
depends_on: [ build ]
pipeline:
dryrun:
image: woodpeckerci/plugin-docker-buildx
secrets: [ docker_username, docker_password ]
settings:
dockerfile: Dockerfile.ci
repo: chewingbever/vieter
tag: dev
platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]

View File

@ -6,12 +6,8 @@ platform: linux/amd64
pipeline:
lint:
image: 'chewingbever/vlang:latest'
pull: true
group: lint
commands:
- make lint
vet:
image: 'chewingbever/vlang:latest'
group: lint
commands:
- make vet

View File

@ -6,9 +6,12 @@ pipeline:
image: woodpeckerci/plugin-docker-buildx
secrets: [ docker_username, docker_password ]
settings:
dockerfile: Dockerfile.ci
repo: chewingbever/vieter
tag: dev
platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ]
build_args_from_env:
- CI_COMMIT_SHA
when:
event: push
branch: dev
@ -28,3 +31,4 @@ pipeline:
depends_on:
- builder
- build

View File

@ -0,0 +1,39 @@
# Yeah so this only works on tags so we'll worry about this later
platform: linux/amd64
branches: main
# We need the entire repo in order for the release names to work
skip_clone: true
pipeline:
prepare:
image: 'chewingbever/vlang:latest'
pull: true
secrets: [ s3_username, s3_password ]
commands:
- git clone "$CI_REPO_REMOTE" .
- git checkout "$CI_COMMIT_BRANCH"
# Write the title to a file that the plugin can then read
- echo "$(git describe --tags --abbrev=0 2> /dev/null || echo '0.0.0')-$(git rev-list --count ^dev)" > title
- cat title
- mc alias set s3/ https://s3.rustybever.be "$S3_USERNAME" "$S3_PASSWORD"
- mc cp -r "s3/vieter/commits/$CI_COMMIT_SHA" assets
release:
image: 'plugins/gitea-release'
secrets:
- gitea_release_api_key
settings:
base_url: https://git.rustybever.be
files: assets/*
checksums:
- md5
- sha256
prerelease: true
# This should get read in as a file
title: title
when:
event: push
depends_on:
- build

View File

@ -1,6 +1,6 @@
FROM alpine:3.12
FROM alpine:3.15
ARG V_RELEASE=weekly.2022.03
ARG TARGETPLATFORM
WORKDIR /opt/vlang
@ -10,7 +10,7 @@ ENV VFLAGS -cc gcc
RUN ln -s /opt/vlang/v /usr/bin/v && \
apk --no-cache add \
curl git make gcc \
git make gcc curl openssl \
musl-dev \
openssl-libs-static openssl-dev \
zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \
@ -19,15 +19,16 @@ RUN ln -s /opt/vlang/v /usr/bin/v && \
libarchive-static libarchive-dev \
diffutils
RUN curl -Lo - "https://github.com/vlang/v/archive/refs/tags/${V_RELEASE}.tar.gz" | tar xzf - && \
mv "v-${V_RELEASE}"/* /opt/vlang && \
make && v -version
# RUN git clone \
# 'https://github.com/ChewingBever/v/' \
# -b vweb-streaming \
# --single-branch \
# '/opt/vlang' && \
# rm -rf '/vlang-local' && \
# make && v -version
COPY patches ./patches
COPY Makefile ./
RUN make v && \
mv v-*/* /opt/vlang && \
v -version
RUN if [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \
wget -O /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && \
chmod +x /usr/local/bin/mc ; \
fi
CMD ["v"]

20
Dockerfile.ci 100644
View File

@ -0,0 +1,20 @@
# vim: ft=dockerfile
# This image just has the required tools to download the binaries
FROM chewingbever/vlang:latest AS builder
ARG TARGETPLATFORM
ARG CI_COMMIT_SHA
WORKDIR /app
RUN curl --fail \
-o vieter \
"https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \
chmod +x vieter
FROM scratch
COPY --from=builder /app/vieter /bin/vieter
ENTRYPOINT ["/bin/vieter"]

View File

@ -6,9 +6,11 @@ LARCHIVE_VER := 3.5.2
LARCHIVE_DIR := libarchive-$(LARCHIVE_VER)
LARCHIVE_LIB := $(LARCHIVE_DIR)/libarchive/libarchive.so
V_RELEASE := weekly.2022.04
# Custom V command for linking libarchive
# V := LDFLAGS=$(PWD)/$(LARCHIVE_LIB) v -cflags '-I$(PWD)/$(LARCHIVE_DIR) -I $(PWD)/$(LARCHIVE_DIR)'
V := v -showcc
V := v-$(V_RELEASE)/v -showcc
all: vieter
@ -64,24 +66,13 @@ fmt:
vet:
$(V) vet -W $(SRC_DIR)
# Pulls & builds my personal build of the v compiler, required for this project to function
.PHONY: customv
customv:
rm -rf v-jjr
git clone \
-b vweb-streaming \
--single-branch \
https://github.com/ChewingBever/v jjr-v
'$(MAKE)' -C jjr-v
# =====LIBARCHIVE=====
.PHONY: libarchive
libarchive: $(LARCHIVE_LIB)
$(LARCHIVE_LIB):
curl -o - "https://libarchive.org/downloads/libarchive-${LARCHIVE_VER}.tar.gz" | tar xzf -
cd "libarchive-${LARCHIVE_VER}" && cmake .
'$(MAKE)' -C "libarchive-${LARCHIVE_VER}"
# Build & patch the V compiler
.PHONY: v
v: v-$(V_RELEASE)/v
v-$(V_RELEASE)/v:
curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf -
cd patches && ./patch.sh '../v-$(V_RELEASE)'
'$(MAKE)' -C 'v-$(V_RELEASE)'
clean:
rm -rf '$(LARCHIVE_DIR)' 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c'
rm -rf '$(LARCHIVE_DIR)' 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'v-$(V_RELEASE)'

View File

@ -33,7 +33,12 @@ terminal, & upload them to the servers using CI.
The data directory consists of three main directories:
* `downloads` - This is where packages are initially downloaded. Because vieter moves files from this folder to the `pkgs` folder, these two folders should best be on the same drive
* `downloads` - This is where packages are initially downloaded. Because vieter
moves files from this folder to the `pkgs` folder, these two folders should
best be on the same drive
* `pkgs` - This is where approved package files are stored.
* `repos` - Each repository gets a subfolder here. The subfolder contains the uncompressed contents of the db file.
* Each repo subdirectory contains the compressed db & files archive for the repository, alongside a directory called `files` which contains the uncompressed contents.
* `repos` - Each repository gets a subfolder here. The subfolder contains the
uncompressed contents of the db file.
* Each repo subdirectory contains the compressed db & files archive for the
repository, alongside a directory called `files` which contains the
uncompressed contents.

View File

@ -0,0 +1,23 @@
// Parse the header of a raw HTTP request into a Request object
pub fn parse_request_head(mut reader io.BufferedReader) ?Request {
// request line
mut line := reader.read_line() ?
method, target, version := parse_request_line(line) ?
// headers
mut header := new_header()
line = reader.read_line() ?
for line != '' {
key, value := parse_header(line) ?
header.add_custom(key, value) ?
line = reader.read_line() ?
}
header.coerce(canonicalize: true)
return Request{
method: method
url: target.str()
header: header
version: version
}
}

7
patches/patch.sh 100755
View File

@ -0,0 +1,7 @@
#!/usr/bin/env sh
# This file patches the downloaded V version
# Should be run from within the directory it's in, as it uses relative paths to the files used for patching.
# $1 is the path to the downloaded V version
# Add parse_request_no_body
cat parse_request_no_body.v >> "$1"/vlib/net/http/request.v

View File

@ -143,16 +143,17 @@ pub fn read_pkg(pkg_path string) ?Pkg {
// TODO can this unsafe block be avoided?
buf = unsafe { malloc(size) }
defer {
unsafe {
free(buf)
}
}
C.archive_read_data(a, buf, size)
unsafe {
println(cstring_to_vstring(buf))
}
pkg_info = parse_pkg_info_string(unsafe { cstring_to_vstring(buf) }) ?
unsafe {
free(buf)
}
} else {
C.archive_read_data_skip(a)
}