From 68cb239dfe3a3e7c7c0d8abcf72e260344394e6d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 13 Jan 2022 15:29:43 +0100 Subject: [PATCH 001/111] Make sure CI works as little as needed --- .woodpecker/.build.yml | 4 ++++ .woodpecker/.lint.yml | 4 ++++ .woodpecker/.publish.yml | 8 ++------ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 40a50e1..4830d10 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -1,3 +1,7 @@ +# These checks already get performed on the feature branches +branches: + exclude: [ main, dev ] + pipeline: vieter: image: 'chewingbever/vlang:latest' diff --git a/.woodpecker/.lint.yml b/.woodpecker/.lint.yml index c72c13a..ae47c32 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/.lint.yml @@ -1,3 +1,7 @@ +# These checks already get performed on the feature branches +branches: + exclude: [ main, dev ] + pipeline: lint: image: 'chewingbever/vlang:latest' diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 1e52db3..34a3675 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -1,3 +1,5 @@ +branches: [main, dev] + pipeline: dev: image: plugins/docker @@ -21,9 +23,3 @@ pipeline: when: event: tag branch: main - -branches: [main, dev] - -depends_on: - - lint - - build From 5616e7a4e2ddab8ac7e4d2795b64bd12705c4af0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 13 Jan 2022 19:20:14 +0100 Subject: [PATCH 002/111] Added function to read .PKGINFO into struct --- src/archive/archive.v | 2 +- src/main.v | 18 ++++----- src/repo/pkg.v | 90 +++++++++++++++++++++++++++++++++++++++++++ src/{ => repo}/repo.v | 1 + 4 files changed, 101 insertions(+), 10 deletions(-) create mode 100644 src/repo/pkg.v rename src/{ => repo}/repo.v (98%) diff --git a/src/archive/archive.v b/src/archive/archive.v index ca911c7..011333f 100644 --- a/src/archive/archive.v +++ b/src/archive/archive.v @@ -2,7 +2,7 @@ module archive import os -pub fn get_pkg_info(pkg_path string) ?string { +pub fn pkg_info_string(pkg_path string) ?string { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") } diff --git a/src/main.v b/src/main.v index af6b06c..0b27736 100644 --- a/src/main.v +++ b/src/main.v @@ -54,7 +54,7 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } } -fn main() { +fn main2() { // Configure logger log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } log_level := log.level_from_tag(log_level_str) or { @@ -102,11 +102,11 @@ fn main() { }, port) } -// fn main() { -// // archive.list_filenames() -// info := archive.get_pkg_info('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { -// eprintln(err.msg) -// return -// } -// println(info) -// } +fn main() { + // archive.list_filenames() + info := repo.get_pkg_info('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { + eprintln(err.msg) + return + } + println(info) +} diff --git a/src/repo/pkg.v b/src/repo/pkg.v new file mode 100644 index 0000000..b45839b --- /dev/null +++ b/src/repo/pkg.v @@ -0,0 +1,90 @@ +module repo + +import archive +import time + +struct PkgInfo { +mut: + // Single values + name string + base string + version string + description string + size i64 + csize i64 + url string + arch string + build_date i64 + packager string + md5sum string + sha256sum string + pgpsig string + pgpsigsize i64 + + // Array values + groups []string + licenses []string + replaces []string + depends []string + conflicts []string + provides []string + optdepends []string + makedepends []string + checkdepends []string +} + +pub fn get_pkg_info(pkg_path string) ?PkgInfo { + pkg_info_str := archive.pkg_info_string(pkg_path) ? + mut pkg_info := PkgInfo{} + + mut i := 0 + mut j := 0 + + // Iterate over the entire string + for line in pkg_info_str.split_into_lines() { + // Skip any comment lines + if line.starts_with('#') { + continue + } + parts := line.split_nth('=', 2) + + if parts.len < 2 { + return error('Invalid line detected.') + } + + value := parts[1].trim_space() + key := parts[0].trim_space() + + match key { + // Single values + 'pkgname' { pkg_info.name = value } + 'pkgbase' { pkg_info.base = value } + 'pkgver' { pkg_info.version = value } + 'pkgdesc' { pkg_info.description = value } + 'csize' { pkg_info.csize = value.int() } + 'size' { pkg_info.size = value.int() } + 'url' { pkg_info.url = value } + 'arch' { pkg_info.arch = value } + 'builddate' { pkg_info.build_date = value.int() } + 'packager' { pkg_info.packager = value } + 'md5sum' { pkg_info.md5sum = value } + 'sha256sum' { pkg_info.sha256sum = value } + 'pgpsig' { pkg_info.pgpsig = value } + 'pgpsigsize' { pkg_info.pgpsigsize = value.int() } + + // Array values + 'group' { pkg_info.groups << value } + 'license' { pkg_info.licenses << value } + 'replaces' { pkg_info.replaces << value } + 'depend' { pkg_info.depends << value } + 'conflict' { pkg_info.conflicts << value } + 'provides' { pkg_info.provides << value } + 'optdepend' { pkg_info.optdepends << value } + 'makedepend' { pkg_info.makedepends << value } + 'checkdepend' { pkg_info.checkdepends << value } + else { return error("Invalid key '$key'.") } + } + } + + return pkg_info +} diff --git a/src/repo.v b/src/repo/repo.v similarity index 98% rename from src/repo.v rename to src/repo/repo.v index 6b307e8..494f8c9 100644 --- a/src/repo.v +++ b/src/repo/repo.v @@ -1,6 +1,7 @@ module repo import os +import archive const pkgs_subpath = 'pkgs' From d4b7a25c06da5c900e40b19652d90b20d4635900 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 13 Jan 2022 21:47:14 +0100 Subject: [PATCH 003/111] Lots of restructuring for repo backend --- .gitignore | 1 + README.md | 9 +++++ src/archive/archive.v | 15 ++++++-- src/main.v | 88 ++++++++++++++++++++++--------------------- src/{repo => }/pkg.v | 76 +++++++++++++++++++++++-------------- src/{repo => }/repo.v | 28 ++++++++++---- src/routes.v | 79 ++++++++++++++++++++------------------ 7 files changed, 176 insertions(+), 120 deletions(-) rename src/{repo => }/pkg.v (63%) rename src/{repo => }/repo.v (51%) diff --git a/.gitignore b/.gitignore index 8c67f97..71064b1 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ vieter.log # External lib; gets added by Makefile libarchive-* +test/ diff --git a/README.md b/README.md index e3c79a2..5d49577 100644 --- a/README.md +++ b/README.md @@ -28,3 +28,12 @@ daemon to start builds, which are then uploaded to the server's repository. The server also allows for non-agents to upload packages, as long as they have the required secrets. This allows me to also develop non-git packages, such as my terminal, & upload them to the servers using CI. + +## Directory Structure + +The data directory consists of three main directories: + +* `downloads` - This is where packages are initially downloaded. Because vieter moves files from this folder to the `pkgs` folder, these two folders should best be on the same drive +* `pkgs` - This is where approved package files are stored. +* `repos` - Each repository gets a subfolder here. The subfolder contains the uncompressed contents of the db file. + * Each repo subdirectory contains the compressed db & files archive for the repository, alongside a directory called `files` which contains the uncompressed contents. diff --git a/src/archive/archive.v b/src/archive/archive.v index 011333f..2983350 100644 --- a/src/archive/archive.v +++ b/src/archive/archive.v @@ -2,7 +2,8 @@ module archive import os -pub fn pkg_info_string(pkg_path string) ?string { +// Returns the .PKGINFO file's contents & the list of files. +pub fn pkg_info(pkg_path string) ?(string, []string) { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") } @@ -26,18 +27,24 @@ pub fn pkg_info_string(pkg_path string) ?string { // We iterate over every header in search of the .PKGINFO one mut buf := voidptr(0) + mut files := []string{} for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { - if C.strcmp(C.archive_entry_pathname(entry), c'.PKGINFO') == 0 { + pathname := C.archive_entry_pathname(entry) + + unsafe { + files << cstring_to_vstring(pathname) + } + + if C.strcmp(pathname, c'.PKGINFO') == 0 { size := C.archive_entry_size(entry) // TODO can this unsafe block be avoided? buf = unsafe { malloc(size) } C.archive_read_data(a, voidptr(buf), size) - break } else { C.archive_read_data_skip(a) } } - return unsafe { cstring_to_vstring(&char(buf)) } + return unsafe { cstring_to_vstring(&char(buf)) }, files } diff --git a/src/main.v b/src/main.v index 0b27736..e6b5bc6 100644 --- a/src/main.v +++ b/src/main.v @@ -4,8 +4,9 @@ import web import os import log import io -import repo +import pkg import archive +import repo const port = 8000 @@ -54,59 +55,62 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } } -fn main2() { - // Configure logger - log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } - log_level := log.level_from_tag(log_level_str) or { - exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') - } - log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } +// fn main2() { +// // Configure logger +// log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } +// log_level := log.level_from_tag(log_level_str) or { +// exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') +// } +// log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } - mut logger := log.Log{ - level: log_level - } +// mut logger := log.Log{ +// level: log_level +// } - logger.set_full_logpath(log_file) - logger.log_to_console_too() +// logger.set_full_logpath(log_file) +// logger.log_to_console_too() - defer { - logger.info('Flushing log file') - logger.flush() - logger.close() - } +// defer { +// logger.info('Flushing log file') +// logger.flush() +// logger.close() +// } - // Configure web server - key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } - repo_dir := os.getenv_opt('REPO_DIR') or { - exit_with_message(1, 'No repo directory was configured.') - } +// // Configure web server +// key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } +// repo_dir := os.getenv_opt('REPO_DIR') or { +// exit_with_message(1, 'No repo directory was configured.') +// } - repo := repo.Repo{ - dir: repo_dir - name: db_name - } +// repo := repo.Repo{ +// dir: repo_dir +// name: db_name +// } - // We create the upload directory during startup - if !os.is_dir(repo.pkg_dir()) { - os.mkdir_all(repo.pkg_dir()) or { - exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.") - } +// // We create the upload directory during startup +// if !os.is_dir(repo.pkg_dir()) { +// os.mkdir_all(repo.pkg_dir()) or { +// exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.") +// } - logger.info("Created package directory '$repo.pkg_dir()'.") - } +// logger.info("Created package directory '$repo.pkg_dir()'.") +// } - web.run(&App{ - logger: logger - api_key: key - repo: repo - }, port) -} +// web.run(&App{ +// logger: logger +// api_key: key +// repo: repo +// }, port) +// } fn main() { // archive.list_filenames() - info := repo.get_pkg_info('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { + res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { eprintln(err.msg) return } - println(info) + // println(info) + println(res.info) + print(res.files) + println(res.info.to_desc()) } diff --git a/src/repo/pkg.v b/src/pkg.v similarity index 63% rename from src/repo/pkg.v rename to src/pkg.v index b45839b..1bb1abc 100644 --- a/src/repo/pkg.v +++ b/src/pkg.v @@ -1,45 +1,46 @@ -module repo +module pkg import archive import time +struct Pkg { +pub: + info PkgInfo [required] + files []string [required] +} + struct PkgInfo { mut: // Single values - name string - base string - version string + name string + base string + version string description string - size i64 - csize i64 - url string - arch string - build_date i64 - packager string - md5sum string - sha256sum string - pgpsig string - pgpsigsize i64 - + size i64 + csize i64 + url string + arch string + build_date i64 + packager string + md5sum string + sha256sum string + pgpsig string + pgpsigsize i64 // Array values - groups []string - licenses []string - replaces []string - depends []string - conflicts []string - provides []string - optdepends []string - makedepends []string + groups []string + licenses []string + replaces []string + depends []string + conflicts []string + provides []string + optdepends []string + makedepends []string checkdepends []string } -pub fn get_pkg_info(pkg_path string) ?PkgInfo { - pkg_info_str := archive.pkg_info_string(pkg_path) ? +fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { mut pkg_info := PkgInfo{} - mut i := 0 - mut j := 0 - // Iterate over the entire string for line in pkg_info_str.split_into_lines() { // Skip any comment lines @@ -71,7 +72,6 @@ pub fn get_pkg_info(pkg_path string) ?PkgInfo { 'sha256sum' { pkg_info.sha256sum = value } 'pgpsig' { pkg_info.pgpsig = value } 'pgpsigsize' { pkg_info.pgpsigsize = value.int() } - // Array values 'group' { pkg_info.groups << value } 'license' { pkg_info.licenses << value } @@ -88,3 +88,21 @@ pub fn get_pkg_info(pkg_path string) ?PkgInfo { return pkg_info } + +pub fn read_pkg(pkg_path string) ?Pkg { + pkg_info_str, files := archive.pkg_info(pkg_path) ? + pkg_info := parse_pkg_info_string(pkg_info_str) ? + + return Pkg{ + info: pkg_info + files: files + } +} + +// Represent a PkgInfo struct as a desc file +pub fn (p &PkgInfo) to_desc() string { + // TODO calculate md5 & sha256 instead of believing the file + mut desc := '' + + return desc +} diff --git a/src/repo/repo.v b/src/repo.v similarity index 51% rename from src/repo/repo.v rename to src/repo.v index 494f8c9..ae76cc9 100644 --- a/src/repo/repo.v +++ b/src/repo.v @@ -11,23 +11,35 @@ pub struct Dummy { x int } -// Handles management of a repository. Package files are stored in '$dir/pkgs' -// & moved there if necessary. +// This struct manages a single repository. pub struct Repo { mut: mutex shared Dummy pub: - dir string [required] - name string [required] + // Where to store repository files; should exist + repo_dir string [required] + // Where to find packages; packages are expected to all be in the same directory + pkg_dir string [required] } -pub fn (r &Repo) pkg_dir() string { - return os.join_path_single(r.dir, repo.pkgs_subpath) +// Returns whether the repository contains the given package. +pub fn (r &Repo) contains(pkg string) bool { + return os.exists(os.join_path(r.repo_dir, 'files', pkg)) +} + +// Adds the given package to the repo. If false, the package was already +// present in the repository. +pub fn (r &Repo) add(pkg string) ?bool { + return false +} + +// Re-generate the db & files archives. +fn (r &Repo) genenerate() ? { } // Returns path to the given package, prepended with the repo's path. pub fn (r &Repo) pkg_path(pkg string) string { - return os.join_path(r.dir, repo.pkgs_subpath, pkg) + return os.join_path_single(r.pkg_dir, pkg) } pub fn (r &Repo) exists(pkg string) bool { @@ -36,7 +48,7 @@ pub fn (r &Repo) exists(pkg string) bool { // Returns the full path to the database file pub fn (r &Repo) db_path() string { - return os.join_path_single(r.dir, '${r.name}.tar.gz') + return os.join_path_single(r.repo_dir, 'repo.tar.gz') } pub fn (r &Repo) add_package(pkg_path string) ? { diff --git a/src/routes.v b/src/routes.v index 0b570ca..c7df286 100644 --- a/src/routes.v +++ b/src/routes.v @@ -28,62 +28,67 @@ fn (mut app App) get_root(filename string) web.Result { mut full_path := '' if is_pkg_name(filename) { - full_path = os.join_path_single(app.repo.pkg_dir(), filename) + full_path = os.join_path_single(app.repo.pkg_dir, filename) } else { - full_path = os.join_path_single(app.repo.dir, filename) + full_path = os.join_path_single(app.repo.repo_dir, filename) } return app.file(full_path) } -['/pkgs/:pkg'; put] -fn (mut app App) put_package(pkg string) web.Result { - if !app.is_authorized() { - return app.text('Unauthorized.') - } +// ['/pkgs/:pkg'; put] +// fn (mut app App) put_package(pkg string) web.Result { +// if !app.is_authorized() { +// return app.text('Unauthorized.') +// } - if !is_pkg_name(pkg) { - app.lwarn("Invalid package name '$pkg'.") +// if !is_pkg_name(pkg) { +// app.lwarn("Invalid package name '$pkg'.") - return app.text('Invalid filename.') - } +// return app.text('Invalid filename.') +// } - if app.repo.exists(pkg) { - app.lwarn("Duplicate package '$pkg'") +// if app.repo.exists(pkg) { +// app.lwarn("Duplicate package '$pkg'") - return app.text('File already exists.') - } +// return app.text('File already exists.') +// } - pkg_path := app.repo.pkg_path(pkg) +// pkg_path := app.repo.pkg_path(pkg) - if length := app.req.header.get(.content_length) { - app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.") +// if length := app.req.header.get(.content_length) { +// app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.") - // This is used to time how long it takes to upload a file - mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) +// // This is used to time how long it takes to upload a file +// mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) - reader_to_file(mut app.reader, length.int(), pkg_path) or { - app.lwarn("Failed to upload package '$pkg'") +// reader_to_file(mut app.reader, length.int(), pkg_path) or { +// app.lwarn("Failed to upload package '$pkg'") - return app.text('Failed to upload file.') - } +// return app.text('Failed to upload file.') +// } - sw.stop() - app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.") - } else { - app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.") - return app.text("Content-Type header isn't set.") - } +// sw.stop() +// app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.") +// } else { +// app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.") +// return app.text("Content-Type header isn't set.") +// } - app.repo.add_package(pkg_path) or { - app.lwarn("Failed to add package '$pkg' to database.") +// app.repo.add_package(pkg_path) or { +// app.lwarn("Failed to add package '$pkg' to database.") - os.rm(pkg_path) or { println('Failed to remove $pkg_path') } +// os.rm(pkg_path) or { println('Failed to remove $pkg_path') } - return app.text('Failed to add package to repo.') - } +// return app.text('Failed to add package to repo.') +// } - app.linfo("Added '$pkg' to repository.") +// app.linfo("Added '$pkg' to repository.") - return app.text('Package added successfully.') +// return app.text('Package added successfully.') +// } + +['/add'; put] +pub fn (mut app App) add_package() web.Result { + return app.text('') } From 0ed2c9f8544286281c6747d53921209723469407 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 13 Jan 2022 23:28:18 +0100 Subject: [PATCH 004/111] Ignore certain files from archive --- src/archive/archive.v | 7 +++++-- src/main.v | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/archive/archive.v b/src/archive/archive.v index 2983350..88649ee 100644 --- a/src/archive/archive.v +++ b/src/archive/archive.v @@ -31,8 +31,11 @@ pub fn pkg_info(pkg_path string) ?(string, []string) { for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { pathname := C.archive_entry_pathname(entry) - unsafe { - files << cstring_to_vstring(pathname) + ignored_names := [c'.BUILDINFO', c'.INSTALL', c'.MTREE', c'.PKGINFO', c'.CHANGELOG'] + if ignored_names.all(C.strcmp(it, pathname) != 0) { + unsafe { + files << cstring_to_vstring(pathname) + } } if C.strcmp(pathname, c'.PKGINFO') == 0 { diff --git a/src/main.v b/src/main.v index e6b5bc6..9d9e541 100644 --- a/src/main.v +++ b/src/main.v @@ -105,7 +105,7 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { fn main() { // archive.list_filenames() - res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { + res := pkg.read_pkg('test/homebank-5.5.1-1-x86_64.pkg.tar.zst') or { eprintln(err.msg) return } From 12217b51466f70f82d08c90b121ebde10a552549 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 11:20:24 +0100 Subject: [PATCH 005/111] Added builder Dockerfile --- Dockerfile.builder | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 Dockerfile.builder diff --git a/Dockerfile.builder b/Dockerfile.builder new file mode 100644 index 0000000..a4882e6 --- /dev/null +++ b/Dockerfile.builder @@ -0,0 +1,33 @@ +FROM alpine:3.12 + +LABEL maintainer="spytheman " + +WORKDIR /opt/vlang + +ENV VVV /opt/vlang +ENV PATH /opt/vlang:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV VFLAGS -cc gcc + +RUN mkdir -p /opt/vlang && ln -s /opt/vlang/v /usr/bin/v + +RUN apk --no-cache add \ + git make upx gcc bash \ + musl-dev \ + openssl-libs-static openssl-dev \ + sqlite-static sqlite-dev \ + libx11-dev glfw-dev freetype-dev \ + libarchive-static libarchive-dev \ + diffutils + +## RUN apk --no-cache add --virtual sdl2deps sdl2-dev sdl2_ttf-dev sdl2_mixer-dev sdl2_image-dev +COPY . /vlang-local + +RUN git clone \ + 'https://github.com/ChewingBever/v/' \ + -b vweb-streaming \ + --single-branch \ + '/opt/vlang' && \ + rm -rf '/vlang-local' && \ + make && v -version + +CMD ["v"] From 8c80aa65f98a0a71294a2df82143ac3859569558 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 11:23:12 +0100 Subject: [PATCH 006/111] Added CI builder config --- .woodpecker/.builder.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .woodpecker/.builder.yml diff --git a/.woodpecker/.builder.yml b/.woodpecker/.builder.yml new file mode 100644 index 0000000..f5a3d0a --- /dev/null +++ b/.woodpecker/.builder.yml @@ -0,0 +1,13 @@ +branches: dev + +pipeline: + publish: + image: plugins/docker + secrets: [ docker_username, docker_password ] + settings: + repo: chewingbever/vlang + tag: latest + when: + event: push + branch: dev + path: Dockerfile.builder From e73d226abc3c949ec94a624e813fde7cd243c91a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 11:25:48 +0100 Subject: [PATCH 007/111] Removed unnecessary constraint --- .woodpecker/.builder.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.woodpecker/.builder.yml b/.woodpecker/.builder.yml index f5a3d0a..d42eda9 100644 --- a/.woodpecker/.builder.yml +++ b/.woodpecker/.builder.yml @@ -9,5 +9,4 @@ pipeline: tag: latest when: event: push - branch: dev path: Dockerfile.builder From 0f8fab6c2fcb662326aeb145ffaa5c5e920b279d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 11:27:25 +0100 Subject: [PATCH 008/111] Added extra event constraint --- .woodpecker/.builder.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker/.builder.yml b/.woodpecker/.builder.yml index d42eda9..3cafc5c 100644 --- a/.woodpecker/.builder.yml +++ b/.woodpecker/.builder.yml @@ -1,4 +1,5 @@ branches: dev +event: push pipeline: publish: From 9f01cb39a37d9d9db19625dd9b4a55afeb5f41f9 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 11:31:13 +0100 Subject: [PATCH 009/111] Set plugin to correct Dockerfile --- .woodpecker/.builder.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.builder.yml b/.woodpecker/.builder.yml index 3cafc5c..ec4a36e 100644 --- a/.woodpecker/.builder.yml +++ b/.woodpecker/.builder.yml @@ -1,5 +1,4 @@ branches: dev -event: push pipeline: publish: @@ -8,6 +7,7 @@ pipeline: settings: repo: chewingbever/vlang tag: latest + dockerfile: Dockerfile.builder when: event: push path: Dockerfile.builder From 8549c4be8167cb36c68336e0babef233b85fc99c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 11:42:11 +0100 Subject: [PATCH 010/111] Test --- .woodpecker/.builder.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker/.builder.yml b/.woodpecker/.builder.yml index ec4a36e..4fb4ae8 100644 --- a/.woodpecker/.builder.yml +++ b/.woodpecker/.builder.yml @@ -4,6 +4,7 @@ pipeline: publish: image: plugins/docker secrets: [ docker_username, docker_password ] + settings: repo: chewingbever/vlang tag: latest From e0a5f9379901892521d1d9001be67f94f4332aad Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 11:43:10 +0100 Subject: [PATCH 011/111] Proper test --- .woodpecker/.builder.yml | 1 - Dockerfile.builder | 1 - 2 files changed, 2 deletions(-) diff --git a/.woodpecker/.builder.yml b/.woodpecker/.builder.yml index 4fb4ae8..ec4a36e 100644 --- a/.woodpecker/.builder.yml +++ b/.woodpecker/.builder.yml @@ -4,7 +4,6 @@ pipeline: publish: image: plugins/docker secrets: [ docker_username, docker_password ] - settings: repo: chewingbever/vlang tag: latest diff --git a/Dockerfile.builder b/Dockerfile.builder index a4882e6..b8e45aa 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -19,7 +19,6 @@ RUN apk --no-cache add \ libarchive-static libarchive-dev \ diffutils -## RUN apk --no-cache add --virtual sdl2deps sdl2-dev sdl2_ttf-dev sdl2_mixer-dev sdl2_image-dev COPY . /vlang-local RUN git clone \ From 8ab3af123832c5fe762a0975e50d50cbf0e993b4 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 18:32:00 +0100 Subject: [PATCH 012/111] Switched to static compilation in CI --- .woodpecker/.build.yml | 6 ++++++ Dockerfile.builder | 1 + Makefile | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 4830d10..520df9b 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -6,17 +6,23 @@ pipeline: vieter: image: 'chewingbever/vlang:latest' group: 'build' + environment: + - LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' commands: - make vieter debug: image: 'chewingbever/vlang:latest' + environment: + - LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' group: 'build' commands: - make debug prod: image: 'chewingbever/vlang:latest' + environment: + - LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' group: 'build' commands: - make prod diff --git a/Dockerfile.builder b/Dockerfile.builder index b8e45aa..d45fe83 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -14,6 +14,7 @@ RUN apk --no-cache add \ git make upx gcc bash \ musl-dev \ openssl-libs-static openssl-dev \ + zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \ sqlite-static sqlite-dev \ libx11-dev glfw-dev freetype-dev \ libarchive-static libarchive-dev \ diff --git a/Makefile b/Makefile index 38593be..bf77b62 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ run: vieter .PHONY: run-prod run-prod: prod - API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG ./vieter-prod + API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG ./pvieter # Same as run, but restart when the source code changes .PHONY: watch From f82b1528e05595332eb0d764388ba58c9aad5c1b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 18:35:22 +0100 Subject: [PATCH 013/111] Builds always pull now --- .woodpecker/.build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 520df9b..674293a 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -5,6 +5,7 @@ branches: pipeline: vieter: image: 'chewingbever/vlang:latest' + pull: true group: 'build' environment: - LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' @@ -13,6 +14,7 @@ pipeline: debug: image: 'chewingbever/vlang:latest' + pull: true environment: - LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' group: 'build' @@ -21,6 +23,7 @@ pipeline: prod: image: 'chewingbever/vlang:latest' + pull: true environment: - LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' group: 'build' From 15acdb7f8593942bfa170a7a4e7346be05effb7f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 18:36:36 +0100 Subject: [PATCH 014/111] The quotes did it --- .woodpecker/.build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 674293a..bca2dbc 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -8,7 +8,7 @@ pipeline: pull: true group: 'build' environment: - - LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' + - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static commands: - make vieter @@ -16,7 +16,7 @@ pipeline: image: 'chewingbever/vlang:latest' pull: true environment: - - LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' + - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static group: 'build' commands: - make debug @@ -25,7 +25,7 @@ pipeline: image: 'chewingbever/vlang:latest' pull: true environment: - - LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' + - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static group: 'build' commands: - make prod From b5d8e7120d4dabd0ab22c2816c9d32edd2035405 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 18:38:23 +0100 Subject: [PATCH 015/111] Only prod needs to be static --- .woodpecker/.build.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index bca2dbc..cc66bc7 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -5,25 +5,18 @@ branches: pipeline: vieter: image: 'chewingbever/vlang:latest' - pull: true group: 'build' - environment: - - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static commands: - make vieter debug: image: 'chewingbever/vlang:latest' - pull: true - environment: - - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static group: 'build' commands: - make debug prod: image: 'chewingbever/vlang:latest' - pull: true environment: - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static group: 'build' From 3f9e9a82dcd89d34e2270b7d3cf02e615b836bb5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 19:59:30 +0100 Subject: [PATCH 016/111] Attempt at building multi-platform builder --- .woodpecker/.build.yml | 1 + .woodpecker/.builder.yml | 6 ++++-- .woodpecker/.lint.yml | 1 + .woodpecker/.publish.yml | 1 + Dockerfile.builder | 22 +++++++++++----------- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index cc66bc7..12610e4 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -1,6 +1,7 @@ # These checks already get performed on the feature branches branches: exclude: [ main, dev ] +platform: linux/amd64 pipeline: vieter: diff --git a/.woodpecker/.builder.yml b/.woodpecker/.builder.yml index ec4a36e..8a1329a 100644 --- a/.woodpecker/.builder.yml +++ b/.woodpecker/.builder.yml @@ -1,13 +1,15 @@ -branches: dev +# branches: dev +platform: linux/amd64 pipeline: publish: - image: plugins/docker + image: woodpeckerci/plugin-docker-buildx secrets: [ docker_username, docker_password ] settings: repo: chewingbever/vlang tag: latest dockerfile: Dockerfile.builder + platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] when: event: push path: Dockerfile.builder diff --git a/.woodpecker/.lint.yml b/.woodpecker/.lint.yml index ae47c32..380b3d4 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/.lint.yml @@ -1,6 +1,7 @@ # These checks already get performed on the feature branches branches: exclude: [ main, dev ] +platform: linux/amd64 pipeline: lint: diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 34a3675..fa958ca 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -1,4 +1,5 @@ branches: [main, dev] +platform: linux/amd64 pipeline: dev: diff --git a/Dockerfile.builder b/Dockerfile.builder index d45fe83..4691c8e 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -8,17 +8,17 @@ ENV VVV /opt/vlang ENV PATH /opt/vlang:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ENV VFLAGS -cc gcc -RUN mkdir -p /opt/vlang && ln -s /opt/vlang/v /usr/bin/v - -RUN apk --no-cache add \ - git make upx gcc bash \ - musl-dev \ - openssl-libs-static openssl-dev \ - zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \ - sqlite-static sqlite-dev \ - libx11-dev glfw-dev freetype-dev \ - libarchive-static libarchive-dev \ - diffutils +RUN mkdir -p /opt/vlang && \ + ln -s /opt/vlang/v /usr/bin/v && \ + apk --no-cache add \ + git make upx gcc bash \ + musl-dev \ + openssl-libs-static openssl-dev \ + zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \ + sqlite-static sqlite-dev \ + libx11-dev glfw-dev freetype-dev \ + libarchive-static libarchive-dev \ + diffutils COPY . /vlang-local From 0b6c20140d7a1e0ab16d21ff1be7f9946d69ead3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 20:05:02 +0100 Subject: [PATCH 017/111] Removed upx dep --- Dockerfile.builder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.builder b/Dockerfile.builder index 4691c8e..6635f3b 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -11,7 +11,7 @@ ENV VFLAGS -cc gcc RUN mkdir -p /opt/vlang && \ ln -s /opt/vlang/v /usr/bin/v && \ apk --no-cache add \ - git make upx gcc bash \ + git make gcc bash \ musl-dev \ openssl-libs-static openssl-dev \ zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \ From 3ee796f0a57740e2a4c0207b372dabeb45e27560 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 20:10:40 +0100 Subject: [PATCH 018/111] Removed bash from builder --- Dockerfile.builder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.builder b/Dockerfile.builder index 6635f3b..48048f8 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -11,7 +11,7 @@ ENV VFLAGS -cc gcc RUN mkdir -p /opt/vlang && \ ln -s /opt/vlang/v /usr/bin/v && \ apk --no-cache add \ - git make gcc bash \ + git make gcc \ musl-dev \ openssl-libs-static openssl-dev \ zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \ From 3acce6df0f9596825ac44b67ca79d9fc90772f5b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 20:30:56 +0100 Subject: [PATCH 019/111] Added arm64 tests (hopefully) --- .woodpecker/.build_arm64.yml | 25 +++++++++++++++++++++++++ .woodpecker/.builder.yml | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 .woodpecker/.build_arm64.yml diff --git a/.woodpecker/.build_arm64.yml b/.woodpecker/.build_arm64.yml new file mode 100644 index 0000000..33d5f3f --- /dev/null +++ b/.woodpecker/.build_arm64.yml @@ -0,0 +1,25 @@ +# These checks already get performed on the feature branches +branches: + exclude: [ main, dev ] +platform: linux/arm64 + +pipeline: + vieter: + image: 'chewingbever/vlang:latest' + group: 'build' + commands: + - make vieter + + debug: + image: 'chewingbever/vlang:latest' + group: 'build' + commands: + - make debug + + prod: + image: 'chewingbever/vlang:latest' + environment: + - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static + group: 'build' + commands: + - make prod diff --git a/.woodpecker/.builder.yml b/.woodpecker/.builder.yml index 8a1329a..f827566 100644 --- a/.woodpecker/.builder.yml +++ b/.woodpecker/.builder.yml @@ -1,4 +1,4 @@ -# branches: dev +branches: dev platform: linux/amd64 pipeline: From d3dc0abc5bdca34390ad4dd85145b1941897849b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 20:38:36 +0100 Subject: [PATCH 020/111] Added check for static binary --- .woodpecker/.build_arm64.yml | 2 ++ .woodpecker/.publish.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.woodpecker/.build_arm64.yml b/.woodpecker/.build_arm64.yml index 33d5f3f..3d8843b 100644 --- a/.woodpecker/.build_arm64.yml +++ b/.woodpecker/.build_arm64.yml @@ -23,3 +23,5 @@ pipeline: group: 'build' commands: - make prod + # Make sure the binary is actually static + - [ "$(readelf -d pvieter) | grep NEEDED | wc -l" = 0 ] diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index fa958ca..002fdbd 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -24,3 +24,6 @@ pipeline: when: event: tag branch: main + +depends_on: + builder From d1f867e511f7c3b5ab86e26c32903c2c44e85d74 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 20:39:06 +0100 Subject: [PATCH 021/111] Sorry for the typo --- .woodpecker/.build_arm64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.build_arm64.yml b/.woodpecker/.build_arm64.yml index 3d8843b..fa3e2fb 100644 --- a/.woodpecker/.build_arm64.yml +++ b/.woodpecker/.build_arm64.yml @@ -24,4 +24,4 @@ pipeline: commands: - make prod # Make sure the binary is actually static - - [ "$(readelf -d pvieter) | grep NEEDED | wc -l" = 0 ] + - '[ "$(readelf -d pvieter) | grep NEEDED | wc -l" = 0 ]' From 641a9783a629959907089db02bc938ee911c755d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 20:39:44 +0100 Subject: [PATCH 022/111] Another one --- .woodpecker/.publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 002fdbd..d23f63c 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -26,4 +26,4 @@ pipeline: branch: main depends_on: - builder + - builder From 9af37819f3189f623c0bc42504bd3bf399807b0c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 20:41:42 +0100 Subject: [PATCH 023/111] Added static test to amd64 builds as well --- .woodpecker/.build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 12610e4..f94e1ab 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -23,3 +23,5 @@ pipeline: group: 'build' commands: - make prod + # Make sure the binary is actually static + - '[ "$(readelf -d pvieter) | grep NEEDED | wc -l" = 0 ]' From 8a8c7fab838a261793f1a80a88079a77bd65748f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 20:43:11 +0100 Subject: [PATCH 024/111] Of course I made another typo --- .woodpecker/.build.yml | 3 ++- .woodpecker/.build_arm64.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index f94e1ab..83dcb0e 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -24,4 +24,5 @@ pipeline: commands: - make prod # Make sure the binary is actually static - - '[ "$(readelf -d pvieter) | grep NEEDED | wc -l" = 0 ]' + - readelf -d pvieter + - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' diff --git a/.woodpecker/.build_arm64.yml b/.woodpecker/.build_arm64.yml index fa3e2fb..704bc48 100644 --- a/.woodpecker/.build_arm64.yml +++ b/.woodpecker/.build_arm64.yml @@ -24,4 +24,5 @@ pipeline: commands: - make prod # Make sure the binary is actually static - - '[ "$(readelf -d pvieter) | grep NEEDED | wc -l" = 0 ]' + - readelf -d pvieter + - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' From b7d2d0e97d782dc8aefaddfaaec81a6d39538643 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 21:02:34 +0100 Subject: [PATCH 025/111] Added v vet to CI --- .woodpecker/.lint.yml | 7 +++++++ Makefile | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/.woodpecker/.lint.yml b/.woodpecker/.lint.yml index 380b3d4..1ccd077 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/.lint.yml @@ -6,5 +6,12 @@ platform: linux/amd64 pipeline: lint: image: 'chewingbever/vlang:latest' + group: lint commands: - make lint + + vet: + image: 'chewingbever/vlang:latest' + group: lint + commands: + - make vet diff --git a/Makefile b/Makefile index bf77b62..103aca6 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,10 @@ lint: fmt: $(V) fmt -w $(SRC_DIR) +.PHONY: vet +vet: + $(V) vet -W $(SRC_DIR) + # Pulls & builds my personal build of the v compiler, required for this project to function .PHONY: customv customv: From e13081a4aa6f71e7d133d446db1af00204aaa534 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 21:50:43 +0100 Subject: [PATCH 026/111] Only support zstd-compressed tarballs --- .woodpecker/.build.yml | 1 + .woodpecker/.build_arm64.yml | 1 + src/archive/archive.v | 6 ++++-- src/archive/bindings.v | 6 ++++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 83dcb0e..5ece461 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -25,4 +25,5 @@ pipeline: - make prod # Make sure the binary is actually static - readelf -d pvieter + - du -h pvieter - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' diff --git a/.woodpecker/.build_arm64.yml b/.woodpecker/.build_arm64.yml index 704bc48..3b919b4 100644 --- a/.woodpecker/.build_arm64.yml +++ b/.woodpecker/.build_arm64.yml @@ -25,4 +25,5 @@ pipeline: - make prod # Make sure the binary is actually static - readelf -d pvieter + - du -h pvieter - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' diff --git a/src/archive/archive.v b/src/archive/archive.v index 88649ee..c280a42 100644 --- a/src/archive/archive.v +++ b/src/archive/archive.v @@ -12,8 +12,10 @@ pub fn pkg_info(pkg_path string) ?(string, []string) { entry := C.archive_entry_new() mut r := 0 - C.archive_read_support_filter_all(a) - C.archive_read_support_format_all(a) + // Sinds 2020, all newly built Arch packages use zstd + C.archive_read_support_filter_zstd(a) + // The content should always be a tarball + C.archive_read_support_format_tar(a) // TODO find out where does this 10240 come from r = C.archive_read_open_filename(a, &char(pkg_path.str), 10240) diff --git a/src/archive/bindings.v b/src/archive/bindings.v index 678d715..f507fac 100644 --- a/src/archive/bindings.v +++ b/src/archive/bindings.v @@ -8,8 +8,10 @@ struct C.archive {} // Create a new archive struct fn C.archive_read_new() &C.archive -fn C.archive_read_support_filter_all(&C.archive) -fn C.archive_read_support_format_all(&C.archive) +// Configure the archive to work with zstd compression +fn C.archive_read_support_filter_zstd(&C.archive) +// Configure the archive to work with a tarball content +fn C.archive_read_support_format_tar(&C.archive) // Open an archive for reading fn C.archive_read_open_filename(&C.archive, &char, int) int From d538e25da3ff17b56707ea35efc6b685289683d5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 21:52:40 +0100 Subject: [PATCH 027/111] CI? You awake? --- src/archive/bindings.v | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/archive/bindings.v b/src/archive/bindings.v index f507fac..26af13f 100644 --- a/src/archive/bindings.v +++ b/src/archive/bindings.v @@ -8,9 +8,11 @@ struct C.archive {} // Create a new archive struct fn C.archive_read_new() &C.archive + // Configure the archive to work with zstd compression fn C.archive_read_support_filter_zstd(&C.archive) -// Configure the archive to work with a tarball content + +// Configure the archive to work with a tarball content fn C.archive_read_support_format_tar(&C.archive) // Open an archive for reading From 1f8f16fe8dd7e33da2d6c35aaeebb86003b6d96f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 22:20:58 +0100 Subject: [PATCH 028/111] Moved archive read code into pkg.v --- src/archive/archive.v | 55 --------------------- src/archive/bindings.v | 50 -------------------- src/pkg.v | 105 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 102 insertions(+), 108 deletions(-) delete mode 100644 src/archive/archive.v delete mode 100644 src/archive/bindings.v diff --git a/src/archive/archive.v b/src/archive/archive.v deleted file mode 100644 index c280a42..0000000 --- a/src/archive/archive.v +++ /dev/null @@ -1,55 +0,0 @@ -module archive - -import os - -// Returns the .PKGINFO file's contents & the list of files. -pub fn pkg_info(pkg_path string) ?(string, []string) { - if !os.is_file(pkg_path) { - return error("'$pkg_path' doesn't exist or isn't a file.") - } - - a := C.archive_read_new() - entry := C.archive_entry_new() - mut r := 0 - - // Sinds 2020, all newly built Arch packages use zstd - C.archive_read_support_filter_zstd(a) - // The content should always be a tarball - C.archive_read_support_format_tar(a) - - // TODO find out where does this 10240 come from - r = C.archive_read_open_filename(a, &char(pkg_path.str), 10240) - defer { - C.archive_read_free(a) - } - - if r != C.ARCHIVE_OK { - return error('Failed to open package.') - } - - // We iterate over every header in search of the .PKGINFO one - mut buf := voidptr(0) - mut files := []string{} - for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { - pathname := C.archive_entry_pathname(entry) - - ignored_names := [c'.BUILDINFO', c'.INSTALL', c'.MTREE', c'.PKGINFO', c'.CHANGELOG'] - if ignored_names.all(C.strcmp(it, pathname) != 0) { - unsafe { - files << cstring_to_vstring(pathname) - } - } - - if C.strcmp(pathname, c'.PKGINFO') == 0 { - size := C.archive_entry_size(entry) - - // TODO can this unsafe block be avoided? - buf = unsafe { malloc(size) } - C.archive_read_data(a, voidptr(buf), size) - } else { - C.archive_read_data_skip(a) - } - } - - return unsafe { cstring_to_vstring(&char(buf)) }, files -} diff --git a/src/archive/bindings.v b/src/archive/bindings.v deleted file mode 100644 index 26af13f..0000000 --- a/src/archive/bindings.v +++ /dev/null @@ -1,50 +0,0 @@ -module archive - -#flag -larchive - -#include "archive.h" - -struct C.archive {} - -// Create a new archive struct -fn C.archive_read_new() &C.archive - -// Configure the archive to work with zstd compression -fn C.archive_read_support_filter_zstd(&C.archive) - -// Configure the archive to work with a tarball content -fn C.archive_read_support_format_tar(&C.archive) - -// Open an archive for reading -fn C.archive_read_open_filename(&C.archive, &char, int) int - -// Go to next entry header in archive -fn C.archive_read_next_header(&C.archive, &&C.archive_entry) int - -// Skip reading the current entry -fn C.archive_read_data_skip(&C.archive) - -// Free an archive -fn C.archive_read_free(&C.archive) int - -// Read an archive entry's contents into a pointer -fn C.archive_read_data(&C.archive, voidptr, int) - -#include "archive_entry.h" - -struct C.archive_entry {} - -// Create a new archive_entry struct -fn C.archive_entry_new() &C.archive_entry - -// Get the filename of the given entry -fn C.archive_entry_pathname(&C.archive_entry) &char - -// Get an entry's file size -// Note: this function actually returns an i64, but as this can't be used as an arugment to malloc, we'll just roll with it & assume an entry is never bigger than 4 gigs -fn C.archive_entry_size(&C.archive_entry) int - -#include - -// Compare two C strings; 0 means they're equal -fn C.strcmp(&char, &char) int diff --git a/src/pkg.v b/src/pkg.v index 1bb1abc..cd1f2a6 100644 --- a/src/pkg.v +++ b/src/pkg.v @@ -1,14 +1,65 @@ module pkg -import archive import time +import os +#flag -larchive + +#include "archive.h" + +struct C.archive {} + +// Create a new archive struct +fn C.archive_read_new() &C.archive + +// Configure the archive to work with zstd compression +fn C.archive_read_support_filter_zstd(&C.archive) + +// Configure the archive to work with a tarball content +fn C.archive_read_support_format_tar(&C.archive) + +// Open an archive for reading +fn C.archive_read_open_filename(&C.archive, &char, int) int + +// Go to next entry header in archive +fn C.archive_read_next_header(&C.archive, &&C.archive_entry) int + +// Skip reading the current entry +fn C.archive_read_data_skip(&C.archive) + +// Free an archive +fn C.archive_read_free(&C.archive) int + +// Read an archive entry's contents into a pointer +fn C.archive_read_data(&C.archive, voidptr, int) + +#include "archive_entry.h" + +struct C.archive_entry {} + +// Create a new archive_entry struct +fn C.archive_entry_new() &C.archive_entry + +// Get the filename of the given entry +fn C.archive_entry_pathname(&C.archive_entry) &char + +// Get an entry's file size +// Note: this function actually returns an i64, but as this can't be used as an arugment to malloc, we'll just roll with it & assume an entry is never bigger than 4 gigs +fn C.archive_entry_size(&C.archive_entry) int + +#include + +// Compare two C strings; 0 means they're equal +fn C.strcmp(&char, &char) int + +// Represents a read archive struct Pkg { pub: info PkgInfo [required] files []string [required] } +// Represents the contents of a .PKGINFO file struct PkgInfo { mut: // Single values @@ -89,9 +140,57 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { return pkg_info } +// Extracts the file list & .PKGINFO contents from an archive +// NOTE: this command currently only supports zstd-compressed tarballs pub fn read_pkg(pkg_path string) ?Pkg { - pkg_info_str, files := archive.pkg_info(pkg_path) ? - pkg_info := parse_pkg_info_string(pkg_info_str) ? + if !os.is_file(pkg_path) { + return error("'$pkg_path' doesn't exist or isn't a file.") + } + + a := C.archive_read_new() + entry := C.archive_entry_new() + mut r := 0 + + // Sinds 2020, all newly built Arch packages use zstd + C.archive_read_support_filter_zstd(a) + // The content should always be a tarball + C.archive_read_support_format_tar(a) + + // TODO find out where does this 10240 come from + r = C.archive_read_open_filename(a, &char(pkg_path.str), 10240) + defer { + C.archive_read_free(a) + } + + if r != C.ARCHIVE_OK { + return error('Failed to open package.') + } + + // We iterate over every header in search of the .PKGINFO one + mut buf := voidptr(0) + mut files := []string{} + for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { + pathname := C.archive_entry_pathname(entry) + + ignored_names := [c'.BUILDINFO', c'.INSTALL', c'.MTREE', c'.PKGINFO', c'.CHANGELOG'] + if ignored_names.all(C.strcmp(it, pathname) != 0) { + unsafe { + files << cstring_to_vstring(pathname) + } + } + + if C.strcmp(pathname, c'.PKGINFO') == 0 { + size := C.archive_entry_size(entry) + + // TODO can this unsafe block be avoided? + buf = unsafe { malloc(size) } + C.archive_read_data(a, voidptr(buf), size) + } else { + C.archive_read_data_skip(a) + } + } + + pkg_info := parse_pkg_info_string(unsafe { cstring_to_vstring(&char(buf)) }) ? return Pkg{ info: pkg_info From b86c6b5e16b59f536d0d2fd06e0b2663135e00f3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 22:46:04 +0100 Subject: [PATCH 029/111] Code now passes vet --- src/main.v | 5 +--- src/pkg.v | 7 +++--- src/repo.v | 13 ++++++----- src/routes.v | 3 +++ src/web/logging.v | 6 +++++ src/web/web.v | 58 +++++++++++++++++++++++++---------------------- 6 files changed, 52 insertions(+), 40 deletions(-) diff --git a/src/main.v b/src/main.v index 9d9e541..ca35c9d 100644 --- a/src/main.v +++ b/src/main.v @@ -5,7 +5,6 @@ import os import log import io import pkg -import archive import repo const port = 8000 @@ -110,7 +109,5 @@ fn main() { return } // println(info) - println(res.info) - print(res.files) - println(res.info.to_desc()) + print(res) } diff --git a/src/pkg.v b/src/pkg.v index cd1f2a6..bcd2724 100644 --- a/src/pkg.v +++ b/src/pkg.v @@ -89,6 +89,7 @@ mut: checkdepends []string } +// parse_pkg_info_string parses a PkgInfo object from a string fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { mut pkg_info := PkgInfo{} @@ -140,7 +141,7 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { return pkg_info } -// Extracts the file list & .PKGINFO contents from an archive +// read_pkg extracts the file list & .PKGINFO contents from an archive // NOTE: this command currently only supports zstd-compressed tarballs pub fn read_pkg(pkg_path string) ?Pkg { if !os.is_file(pkg_path) { @@ -166,9 +167,9 @@ pub fn read_pkg(pkg_path string) ?Pkg { return error('Failed to open package.') } - // We iterate over every header in search of the .PKGINFO one mut buf := voidptr(0) mut files := []string{} + for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { pathname := C.archive_entry_pathname(entry) @@ -198,7 +199,7 @@ pub fn read_pkg(pkg_path string) ?Pkg { } } -// Represent a PkgInfo struct as a desc file +// to_desc returns a desc file valid string representation pub fn (p &PkgInfo) to_desc() string { // TODO calculate md5 & sha256 instead of believing the file mut desc := '' diff --git a/src/repo.v b/src/repo.v index ae76cc9..f3ebfd4 100644 --- a/src/repo.v +++ b/src/repo.v @@ -1,7 +1,6 @@ module repo import os -import archive const pkgs_subpath = 'pkgs' @@ -22,35 +21,37 @@ pub: pkg_dir string [required] } -// Returns whether the repository contains the given package. +// contains returns whether the repository contains the given package. pub fn (r &Repo) contains(pkg string) bool { return os.exists(os.join_path(r.repo_dir, 'files', pkg)) } -// Adds the given package to the repo. If false, the package was already +// add adds the given package to the repo. If false, the package was already // present in the repository. pub fn (r &Repo) add(pkg string) ?bool { return false } -// Re-generate the db & files archives. +// generate re-generates the db & files archives. fn (r &Repo) genenerate() ? { } -// Returns path to the given package, prepended with the repo's path. +// pkg_path returns path to the given package, prepended with the repo's path. pub fn (r &Repo) pkg_path(pkg string) string { return os.join_path_single(r.pkg_dir, pkg) } +// exists checks whether a package file exists pub fn (r &Repo) exists(pkg string) bool { return os.exists(r.pkg_path(pkg)) } -// Returns the full path to the database file +// db_path returns the full path to the database file pub fn (r &Repo) db_path() string { return os.join_path_single(r.repo_dir, 'repo.tar.gz') } +// add_package adds a package to the repository pub fn (r &Repo) add_package(pkg_path string) ? { mut res := os.Result{} diff --git a/src/routes.v b/src/routes.v index c7df286..101e1c5 100644 --- a/src/routes.v +++ b/src/routes.v @@ -7,6 +7,7 @@ import time const prefixes = ['B', 'KB', 'MB', 'GB'] +// pretty_bytes converts a byte count to human-readable version fn pretty_bytes(bytes int) string { mut i := 0 mut n := f32(bytes) @@ -23,6 +24,7 @@ fn is_pkg_name(s string) bool { return s.contains('.pkg') } +// get_root handles a GET request for a file on the root ['/:filename'; get] fn (mut app App) get_root(filename string) web.Result { mut full_path := '' @@ -88,6 +90,7 @@ fn (mut app App) get_root(filename string) web.Result { // return app.text('Package added successfully.') // } +// add_package PUT a new package to the server ['/add'; put] pub fn (mut app App) add_package() web.Result { return app.text('') diff --git a/src/web/logging.v b/src/web/logging.v index 66426f2..fc697ff 100644 --- a/src/web/logging.v +++ b/src/web/logging.v @@ -2,28 +2,34 @@ module web import log +// log reate a log message with the given level pub fn (mut ctx Context) log(msg &string, level log.Level) { lock ctx.logger { ctx.logger.send_output(msg, level) } } +// lfatal create a log message with the fatal level pub fn (mut ctx Context) lfatal(msg &string) { ctx.log(msg, log.Level.fatal) } +// lerror create a log message with the error level pub fn (mut ctx Context) lerror(msg &string) { ctx.log(msg, log.Level.error) } +// lwarn create a log message with the warn level pub fn (mut ctx Context) lwarn(msg &string) { ctx.log(msg, log.Level.warn) } +// linfo create a log message with the info level pub fn (mut ctx Context) linfo(msg &string) { ctx.log(msg, log.Level.info) } +// ldebug create a log message with the debug level pub fn (mut ctx Context) ldebug(msg &string) { ctx.log(msg, log.Level.debug) } diff --git a/src/web/web.v b/src/web/web.v index 404bd08..2db8f18 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -187,14 +187,14 @@ struct Route { } // Defining this method is optional. -// This method called at server start. +// init_server is called at server start. // You can use it for initializing globals. pub fn (ctx Context) init_server() { eprintln('init_server() has been deprecated, please init your web app in `fn main()`') } // Defining this method is optional. -// This method called before every request (aka middleware). +// before_request is called before every request (aka middleware). // Probably you can use it for check user session cookie or add header. pub fn (ctx Context) before_request() {} @@ -206,7 +206,7 @@ pub struct Cookie { http_only bool } -// web intern function +// send_response_to_client sends a response to the client [manualfree] pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bool { if ctx.done { @@ -230,33 +230,33 @@ pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bo return true } -// Response HTTP_OK with s as payload with content-type `text/html` +// html HTTP_OK with s as payload with content-type `text/html` pub fn (mut ctx Context) html(s string) Result { ctx.send_response_to_client('text/html', s) return Result{} } -// Response HTTP_OK with s as payload with content-type `text/plain` +// text HTTP_OK with s as payload with content-type `text/plain` pub fn (mut ctx Context) text(s string) Result { ctx.send_response_to_client('text/plain', s) return Result{} } -// Response HTTP_OK with json_s as payload with content-type `application/json` +// json HTTP_OK with json_s as payload with content-type `application/json` pub fn (mut ctx Context) json(j T) Result { json_s := json.encode(j) ctx.send_response_to_client('application/json', json_s) return Result{} } -// Response HTTP_OK with a pretty-printed JSON result +// json_pretty Response HTTP_OK with a pretty-printed JSON result pub fn (mut ctx Context) json_pretty(j T) Result { json_s := json.encode_pretty(j) ctx.send_response_to_client('application/json', json_s) return Result{} } -// Response HTTP_OK with file as payload +// file Response HTTP_OK with file as payload // This function manually implements responses because it needs to stream the file contents pub fn (mut ctx Context) file(f_path string) Result { if ctx.done { @@ -329,13 +329,13 @@ pub fn (mut ctx Context) file(f_path string) Result { return Result{} } -// Response HTTP_OK with s as payload +// ok Response HTTP_OK with s as payload pub fn (mut ctx Context) ok(s string) Result { ctx.send_response_to_client(ctx.content_type, s) return Result{} } -// Response a server error +// server_error Response a server error pub fn (mut ctx Context) server_error(ecode int) Result { $if debug { eprintln('> ctx.server_error ecode: $ecode') @@ -347,7 +347,7 @@ pub fn (mut ctx Context) server_error(ecode int) Result { return Result{} } -// Redirect to an url +// redirect Redirect to an url pub fn (mut ctx Context) redirect(url string) Result { if ctx.done { return Result{} @@ -360,7 +360,7 @@ pub fn (mut ctx Context) redirect(url string) Result { return Result{} } -// Send an not_found response +// not_found Send an not_found response pub fn (mut ctx Context) not_found() Result { if ctx.done { return Result{} @@ -370,7 +370,7 @@ pub fn (mut ctx Context) not_found() Result { return Result{} } -// Sets a cookie +// set_cookie Sets a cookie pub fn (mut ctx Context) set_cookie(cookie Cookie) { mut cookie_data := []string{} mut secure := if cookie.secure { 'Secure;' } else { '' } @@ -383,17 +383,17 @@ pub fn (mut ctx Context) set_cookie(cookie Cookie) { ctx.add_header('Set-Cookie', '$cookie.name=$cookie.value; $data') } -// Sets the response content type +// set_content_type Sets the response content type pub fn (mut ctx Context) set_content_type(typ string) { ctx.content_type = typ } -// Sets a cookie with a `expire_data` +// set_cookie_with_expire_date Sets a cookie with a `expire_data` pub fn (mut ctx Context) set_cookie_with_expire_date(key string, val string, expire_date time.Time) { ctx.add_header('Set-Cookie', '$key=$val; Secure; HttpOnly; expires=$expire_date.utc_string()') } -// Gets a cookie by a key +// get_cookie Gets a cookie by a key pub fn (ctx &Context) get_cookie(key string) ?string { // TODO refactor mut cookie_header := ctx.get_header('cookie') if cookie_header == '' { @@ -413,7 +413,7 @@ pub fn (ctx &Context) get_cookie(key string) ?string { // TODO refactor return error('Cookie not found') } -// Sets the response status +// set_status Sets the response status pub fn (mut ctx Context) set_status(code int, desc string) { if code < 100 || code > 599 { ctx.status = '500 Internal Server Error' @@ -422,12 +422,12 @@ pub fn (mut ctx Context) set_status(code int, desc string) { } } -// Adds an header to the response with key and val +// add_header Adds an header to the response with key and val pub fn (mut ctx Context) add_header(key string, val string) { ctx.header.add_custom(key, val) or {} } -// Returns the header data from the key +// get_header Returns the header data from the key pub fn (ctx &Context) get_header(key string) string { return ctx.req.header.get_custom(key) or { '' } } @@ -436,7 +436,7 @@ interface DbInterface { db voidptr } -// run_app +// run runs the app [manualfree] pub fn run(global_app &T, port int) { mut l := net.listen_tcp(.ip6, ':$port') or { panic('failed to listen $err.code $err') } @@ -478,6 +478,7 @@ pub fn run(global_app &T, port int) { } } +// handle_conn handles a connection [manualfree] fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { conn.set_read_timeout(30 * time.second) @@ -615,6 +616,7 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { conn.write(web.http_404.bytes()) or {} } +// route_matches returns wether a route matches fn route_matches(url_words []string, route_words []string) ?[]string { // URL path should be at least as long as the route path // except for the catchall route (`/:path...`) @@ -657,7 +659,7 @@ fn route_matches(url_words []string, route_words []string) ?[]string { return params } -// check if request is for a static file and serves it +// serve_if_static checks if request is for a static file and serves it // returns true if we served a static file, false otherwise [manualfree] fn serve_if_static(mut app T, url urllib.URL) bool { @@ -676,6 +678,7 @@ fn serve_if_static(mut app T, url urllib.URL) bool { return true } +// scan_static_directory makes a static route for each file in a directory fn (mut ctx Context) scan_static_directory(directory_path string, mount_path string) { files := os.ls(directory_path) or { panic(err) } if files.len > 0 { @@ -695,7 +698,7 @@ fn (mut ctx Context) scan_static_directory(directory_path string, mount_path str } } -// Handles a directory static +// handle_static Handles a directory static // If `root` is set the mount path for the dir will be in '/' pub fn (mut ctx Context) handle_static(directory_path string, root bool) bool { if ctx.done || !os.exists(directory_path) { @@ -724,7 +727,7 @@ pub fn (mut ctx Context) mount_static_folder_at(directory_path string, mount_pat return true } -// Serves a file static +// serve_static Serves a file static // `url` is the access path on the site, `file_path` is the real path to the file, `mime_type` is the file type pub fn (mut ctx Context) serve_static(url string, file_path string) { ctx.static_files[url] = file_path @@ -733,7 +736,7 @@ pub fn (mut ctx Context) serve_static(url string, file_path string) { ctx.static_mime_types[url] = web.mime_types[ext] } -// Returns the ip address from the current user +// ip Returns the ip address from the current user pub fn (ctx &Context) ip() string { mut ip := ctx.req.header.get(.x_forwarded_for) or { '' } if ip == '' { @@ -749,22 +752,23 @@ pub fn (ctx &Context) ip() string { return ip } -// Set s to the form error +// error Set s to the form error pub fn (mut ctx Context) error(s string) { println('web error: $s') ctx.form_error = s } -// Returns an empty result +// not_found Returns an empty result pub fn not_found() Result { return Result{} } +// send_string fn send_string(mut conn net.TcpConn, s string) ? { conn.write(s.bytes()) ? } -// Do not delete. +// filter Do not delete. // It used by `vlib/v/gen/c/str_intp.v:130` for string interpolation inside web templates // TODO: move it to template render fn filter(s string) string { From 5b031b12ce50076501c75123403056de80878a2b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 10:31:11 +0100 Subject: [PATCH 030/111] Added missing compiler flags to Dockerfile --- .woodpecker/.publish.yml | 9 +++++---- Dockerfile | 7 ++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index d23f63c..f94832a 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -1,13 +1,14 @@ -branches: [main, dev] +# branches: [main, dev] platform: linux/amd64 pipeline: dev: - image: plugins/docker + image: woodpeckerci/plugin-docker-buildx secrets: [ docker_username, docker_password ] settings: repo: chewingbever/vieter tag: dev + platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] when: event: push branch: dev @@ -25,5 +26,5 @@ pipeline: event: tag branch: main -depends_on: - - builder +# depends_on: +# - builder diff --git a/Dockerfile b/Dockerfile index 556ca58..4ca6c87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,17 +5,14 @@ WORKDIR /app # Copy over source code & build production binary COPY src ./src COPY Makefile ./ -RUN make prod +RUN LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' \ + make prod FROM alpine:3.15 ENV REPO_DIR=/data -RUN apk update && \ - apk add --no-cache \ - libarchive - COPY --from=builder /app/pvieter /usr/local/bin/vieter ENTRYPOINT [ "/usr/local/bin/vieter" ] From 72699725d03e4ac5e053276987422413614152fe Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 10:31:54 +0100 Subject: [PATCH 031/111] Missed a constraint --- .woodpecker/.publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index f94832a..dd0674f 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -11,7 +11,7 @@ pipeline: platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] when: event: push - branch: dev + # branch: dev release: image: plugins/docker From ce1e46e2d60f7822a932d6194887da16952ae203 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 10:36:23 +0100 Subject: [PATCH 032/111] Changed to ENV call --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4ca6c87..e24f909 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,9 @@ WORKDIR /app # Copy over source code & build production binary COPY src ./src COPY Makefile ./ -RUN LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' \ - make prod + +ENV LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' +RUN make prod FROM alpine:3.15 From 964fefc7ee6c4952c7fd2fd58f02d1bae05f03e2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 10:39:06 +0100 Subject: [PATCH 033/111] Removed arm/v7 --- .woodpecker/.publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index dd0674f..3a4b150 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -8,7 +8,7 @@ pipeline: settings: repo: chewingbever/vieter tag: dev - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm64/v8, linux/amd64 ] when: event: push # branch: dev From 759dfa6b08b5d543a01155b1771bd2da53f68c4d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 11:16:09 +0100 Subject: [PATCH 034/111] Changed up Dockerfile optimisation --- Dockerfile | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e24f909..f244847 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ COPY src ./src COPY Makefile ./ ENV LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' -RUN make prod +RUN v -o pvieter -clags "-O3" src FROM alpine:3.15 diff --git a/Makefile b/Makefile index 103aca6..dc2273b 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ LARCHIVE_LIB := $(LARCHIVE_DIR)/libarchive/libarchive.so # Custom V command for linking libarchive # V := LDFLAGS=$(PWD)/$(LARCHIVE_LIB) v -cflags '-I$(PWD)/$(LARCHIVE_DIR) -I $(PWD)/$(LARCHIVE_DIR)' -V := v +V := v -showcc all: vieter From 87f7caea2b9a8d6679e8dead6b4ff5331d5f1ba5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 11:18:05 +0100 Subject: [PATCH 035/111] Another typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f244847..5564e34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ COPY src ./src COPY Makefile ./ ENV LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' -RUN v -o pvieter -clags "-O3" src +RUN v -o pvieter -cflags "-O3" src FROM alpine:3.15 From 2dd9ea4553e0f5ae7fbe77c876948be646487e14 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 11:35:29 +0100 Subject: [PATCH 036/111] Re-added necessary constraints --- .woodpecker/.publish.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 3a4b150..6399d7f 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -1,4 +1,4 @@ -# branches: [main, dev] +branches: [main, dev] platform: linux/amd64 pipeline: @@ -11,20 +11,20 @@ pipeline: platforms: [ linux/arm64/v8, linux/amd64 ] when: event: push - # branch: dev + branch: dev release: - image: plugins/docker + image: woodpeckerci/plugin-docker-buildx secrets: [ docker_username, docker_password ] settings: repo: chewingbever/vieter tag: - latest - $CI_COMMIT_TAG - mtu: 1000 + platforms: [ linux/arm64/v8, linux/amd64 ] when: event: tag branch: main -# depends_on: -# - builder +depends_on: + - builder From e1ca4085ffb0387d44e0b09994e7f1cb4a2dfe20 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 15:28:58 +0100 Subject: [PATCH 037/111] Let's test matrix builds using platforms --- .woodpecker/.build.yml | 7 ++++++- .woodpecker/.build_arm64.yml => build_arm64.yml | 0 2 files changed, 6 insertions(+), 1 deletion(-) rename .woodpecker/.build_arm64.yml => build_arm64.yml (100%) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 5ece461..2f9221c 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -1,7 +1,12 @@ +matrix: + PLATFORM: + - linux/amd64 + - linux/arm64 + # These checks already get performed on the feature branches branches: exclude: [ main, dev ] -platform: linux/amd64 +platform: $PLATFORM pipeline: vieter: diff --git a/.woodpecker/.build_arm64.yml b/build_arm64.yml similarity index 100% rename from .woodpecker/.build_arm64.yml rename to build_arm64.yml From 7b6c1c254d6f50d7d49c6fb78c35a2eb7b26de64 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 15:30:09 +0100 Subject: [PATCH 038/111] Maybe it needs the brackets? --- .woodpecker/.build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 2f9221c..b4aabd2 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -6,7 +6,7 @@ matrix: # These checks already get performed on the feature branches branches: exclude: [ main, dev ] -platform: $PLATFORM +platform: ${PLATFORM} pipeline: vieter: From 22ae53b5c277d76aa7414ef0f258c9ef139e5316 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 15:35:12 +0100 Subject: [PATCH 039/111] Added arm/v7 to CI --- .woodpecker/.build.yml | 1 + build_arm64.yml | 29 ----------------------------- 2 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 build_arm64.yml diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index b4aabd2..67c9ee3 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -2,6 +2,7 @@ matrix: PLATFORM: - linux/amd64 - linux/arm64 + - linux/arm/v7 # These checks already get performed on the feature branches branches: diff --git a/build_arm64.yml b/build_arm64.yml deleted file mode 100644 index 3b919b4..0000000 --- a/build_arm64.yml +++ /dev/null @@ -1,29 +0,0 @@ -# These checks already get performed on the feature branches -branches: - exclude: [ main, dev ] -platform: linux/arm64 - -pipeline: - vieter: - image: 'chewingbever/vlang:latest' - group: 'build' - commands: - - make vieter - - debug: - image: 'chewingbever/vlang:latest' - group: 'build' - commands: - - make debug - - prod: - image: 'chewingbever/vlang:latest' - environment: - - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static - group: 'build' - commands: - - make prod - # Make sure the binary is actually static - - readelf -d pvieter - - du -h pvieter - - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' From 45647755e08c5af7842313e4f1f7f4960684a0e4 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 15:49:19 +0100 Subject: [PATCH 040/111] added arm/v7 to release image archs --- .woodpecker/.build.yml | 7 +------ .woodpecker/.publish.yml | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 67c9ee3..4b87f8b 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -10,12 +10,7 @@ branches: platform: ${PLATFORM} pipeline: - vieter: - image: 'chewingbever/vlang:latest' - group: 'build' - commands: - - make vieter - + # The default build isn't needed, as alpine switches to gcc for the compiler anyways debug: image: 'chewingbever/vlang:latest' group: 'build' diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 6399d7f..9e5f39a 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -8,7 +8,7 @@ pipeline: settings: repo: chewingbever/vieter tag: dev - platforms: [ linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] when: event: push branch: dev @@ -21,7 +21,7 @@ pipeline: tag: - latest - $CI_COMMIT_TAG - platforms: [ linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] when: event: tag branch: main From 3dc355f786a17d2277637bfc7ca486ee0ace7fb5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 18 Jan 2022 20:27:00 +0100 Subject: [PATCH 041/111] Wrote PkgInfo to desc generator --- src/main.v | 2 +- src/pkg.v | 83 ++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 76 insertions(+), 9 deletions(-) diff --git a/src/main.v b/src/main.v index ca35c9d..2fcf1dc 100644 --- a/src/main.v +++ b/src/main.v @@ -109,5 +109,5 @@ fn main() { return } // println(info) - print(res) + print(res.info.to_desc()) } diff --git a/src/pkg.v b/src/pkg.v index bcd2724..8d23b6a 100644 --- a/src/pkg.v +++ b/src/pkg.v @@ -114,7 +114,7 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { 'pkgbase' { pkg_info.base = value } 'pkgver' { pkg_info.version = value } 'pkgdesc' { pkg_info.description = value } - 'csize' { pkg_info.csize = value.int() } + 'csize' { continue } 'size' { pkg_info.size = value.int() } 'url' { pkg_info.url = value } 'arch' { pkg_info.arch = value } @@ -191,7 +191,8 @@ pub fn read_pkg(pkg_path string) ?Pkg { } } - pkg_info := parse_pkg_info_string(unsafe { cstring_to_vstring(&char(buf)) }) ? + mut pkg_info := parse_pkg_info_string(unsafe { cstring_to_vstring(&char(buf)) }) ? + pkg_info.csize = i64(os.file_size(pkg_path)) return Pkg{ info: pkg_info @@ -199,10 +200,76 @@ pub fn read_pkg(pkg_path string) ?Pkg { } } -// to_desc returns a desc file valid string representation -pub fn (p &PkgInfo) to_desc() string { - // TODO calculate md5 & sha256 instead of believing the file - mut desc := '' - - return desc +fn format_entry(key string, value string) string { + return '\n%$key%\n$value\n' +} + +// to_desc returns a desc file valid string representation +// TODO calculate md5 & sha256 instead of believing the file +pub fn (p &PkgInfo) to_desc() string { + // filename + mut desc := '%FILENAME%\n$p.name-$p.version-${p.arch}.pkg.tar.zst\n' + + desc += format_entry('NAME', p.name) + desc += format_entry('BASE', p.base) + desc += format_entry('VERSION', p.version) + + if p.description.len > 0 { + desc += format_entry('DESC', p.description) + } + + if p.groups.len > 0 { + desc += format_entry('GROUPS', p.groups.join_lines()) + } + + desc += format_entry('CSIZE', p.csize.str()) + desc += format_entry('ISIZE', p.size.str()) + + // TODO calculate these + desc += format_entry('MD5SUM', p.md5sum) + desc += format_entry('SHA256SUM', p.sha256sum) + + // TODO add pgpsig stuff + + if p.url.len > 0 { + desc += format_entry('URL', p.url) + } + + if p.licenses.len > 0 { + desc += format_entry('LICENSE', p.licenses.join_lines()) + } + + desc += format_entry('ARCH', p.arch) + desc += format_entry('BUILDDATE', p.build_date.str()) + desc += format_entry('PACKAGER', p.packager) + + if p.replaces.len > 0 { + desc += format_entry('REPLACES', p.replaces.join_lines()) + } + + if p.conflicts.len > 0 { + desc += format_entry('CONFLICTS', p.conflicts.join_lines()) + } + + if p.provides.len > 0 { + desc += format_entry('PROVIDES', p.provides.join_lines()) + } + + if p.depends.len > 0 { + desc += format_entry('DEPENDS', p.depends.join_lines()) + } + + if p.optdepends.len > 0 { + desc += format_entry('OPTDEPENDS', p.optdepends.join_lines()) + } + + if p.makedepends.len > 0 { + desc += format_entry('MAKEDEPENDS', p.makedepends.join_lines()) + } + + if p.checkdepends.len > 0 { + desc += format_entry('CHECKDEPENDS', p.checkdepends.join_lines()) + } + + return '$desc\n' } From cb16091f659c9e90c4ea0fe6f143d2acad582357 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 18 Jan 2022 21:41:09 +0100 Subject: [PATCH 042/111] Non-working hashing code [CI SKIP] --- .woodpecker/.publish.yml | 14 +++++++++++++- src/pkg.v | 10 +++++++--- src/util.v | 30 ++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 src/util.v diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 9e5f39a..6d3b34e 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -1,7 +1,19 @@ -branches: [main, dev] +# branches: [main, dev] platform: linux/amd64 pipeline: + dryrun: + image: woodpeckerci/plugin-docker-buildx + secrets: [ docker_username, docker_password ] + settings: + repo: chewingbever/vieter + tag: dev + platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + dry_run: true + when: + event: pull_request + branch: dev + dev: image: woodpeckerci/plugin-docker-buildx secrets: [ docker_username, docker_password ] diff --git a/src/pkg.v b/src/pkg.v index 8d23b6a..8b1849c 100644 --- a/src/pkg.v +++ b/src/pkg.v @@ -2,6 +2,7 @@ module pkg import time import os +import util #flag -larchive @@ -120,8 +121,8 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { 'arch' { pkg_info.arch = value } 'builddate' { pkg_info.build_date = value.int() } 'packager' { pkg_info.packager = value } - 'md5sum' { pkg_info.md5sum = value } - 'sha256sum' { pkg_info.sha256sum = value } + 'md5sum' { continue } + 'sha256sum' { continue } 'pgpsig' { pkg_info.pgpsig = value } 'pgpsigsize' { pkg_info.pgpsigsize = value.int() } // Array values @@ -192,7 +193,11 @@ pub fn read_pkg(pkg_path string) ?Pkg { } mut pkg_info := parse_pkg_info_string(unsafe { cstring_to_vstring(&char(buf)) }) ? + pkg_info.csize = i64(os.file_size(pkg_path)) + pkg_info.md5sum, pkg_info.sha256sum = util.hash_file(pkg_path) or { + return error('Failed to hash package.') + } return Pkg{ info: pkg_info @@ -225,7 +230,6 @@ pub fn (p &PkgInfo) to_desc() string { desc += format_entry('CSIZE', p.csize.str()) desc += format_entry('ISIZE', p.size.str()) - // TODO calculate these desc += format_entry('MD5SUM', p.md5sum) desc += format_entry('SHA256SUM', p.sha256sum) diff --git a/src/util.v b/src/util.v new file mode 100644 index 0000000..cb17c3f --- /dev/null +++ b/src/util.v @@ -0,0 +1,30 @@ +module util + +import os +import crypto.md5 +import crypto.sha256 + +// hash_file returns the md5 & sha256 hash of a given file +pub fn hash_file(path &string) ?(string, string) { + file := os.open(path) or { return error('Failed to open file.') } + + mut md5sum := md5.new() + mut sha256sum := sha256.new() + + buf_size := i32(1_000_000) + mut buf := []byte{len: buf_size} + mut bytes_left := os.file_size(path) + + for bytes_left > 0 { + // TODO check if just breaking here is safe + bytes_read := file.read(mut buf) or { break } + bytes_left -= u64(bytes_read) + + if bytes_left > buf_size { + md5sum.write(buf) + sha256sum.write(buf) + } + } + + return md5sum.sum(buf).hex(), sha256sum.sum(buf).hex() +} From 9bd14b4cbf584c524ffe1bd1ecebe2bed90a5cb3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 18 Jan 2022 22:52:14 +0100 Subject: [PATCH 043/111] Yeah this doesn't work [CI SKIP] --- src/main.v | 2 +- src/util.v | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main.v b/src/main.v index 2fcf1dc..8a73822 100644 --- a/src/main.v +++ b/src/main.v @@ -104,7 +104,7 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { fn main() { // archive.list_filenames() - res := pkg.read_pkg('test/homebank-5.5.1-1-x86_64.pkg.tar.zst') or { + res := pkg.read_pkg('test/homebank-5.5.3-1-x86_64.pkg.tar.zst') or { eprintln(err.msg) return } diff --git a/src/util.v b/src/util.v index cb17c3f..19d174f 100644 --- a/src/util.v +++ b/src/util.v @@ -5,13 +5,14 @@ import crypto.md5 import crypto.sha256 // hash_file returns the md5 & sha256 hash of a given file +// TODO actually implement sha256 pub fn hash_file(path &string) ?(string, string) { file := os.open(path) or { return error('Failed to open file.') } mut md5sum := md5.new() mut sha256sum := sha256.new() - buf_size := i32(1_000_000) + buf_size := int(1_000_000) mut buf := []byte{len: buf_size} mut bytes_left := os.file_size(path) @@ -21,10 +22,12 @@ pub fn hash_file(path &string) ?(string, string) { bytes_left -= u64(bytes_read) if bytes_left > buf_size { - md5sum.write(buf) - sha256sum.write(buf) + // For now we'll assume that this always works + md5sum.write(buf) or {} + // sha256sum.write(buf) or {} } } - return md5sum.sum(buf).hex(), sha256sum.sum(buf).hex() + // return md5sum.sum(buf).hex(), sha256sum.sum(buf).hex() + return md5sum.sum(buf).hex(), '' } From df3310944e5f0127ae39f586ae5b38b5714ebc3c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 17:15:37 +0100 Subject: [PATCH 044/111] Working md5 checksum --- src/main.v | 2 +- src/util.v | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main.v b/src/main.v index 8a73822..e81ed92 100644 --- a/src/main.v +++ b/src/main.v @@ -104,7 +104,7 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { fn main() { // archive.list_filenames() - res := pkg.read_pkg('test/homebank-5.5.3-1-x86_64.pkg.tar.zst') or { + res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { eprintln(err.msg) return } diff --git a/src/util.v b/src/util.v index 19d174f..066a58d 100644 --- a/src/util.v +++ b/src/util.v @@ -2,7 +2,7 @@ module util import os import crypto.md5 -import crypto.sha256 +// import crypto.sha256 // hash_file returns the md5 & sha256 hash of a given file // TODO actually implement sha256 @@ -10,7 +10,7 @@ pub fn hash_file(path &string) ?(string, string) { file := os.open(path) or { return error('Failed to open file.') } mut md5sum := md5.new() - mut sha256sum := sha256.new() + // mut sha256sum := sha256.new() buf_size := int(1_000_000) mut buf := []byte{len: buf_size} @@ -18,16 +18,16 @@ pub fn hash_file(path &string) ?(string, string) { for bytes_left > 0 { // TODO check if just breaking here is safe - bytes_read := file.read(mut buf) or { break } + bytes_read := file.read(mut buf) or { return error('Failed to read from file.') } bytes_left -= u64(bytes_read) - if bytes_left > buf_size { - // For now we'll assume that this always works - md5sum.write(buf) or {} - // sha256sum.write(buf) or {} + // For now we'll assume that this always works + md5sum.write(buf[..bytes_read]) or { + return error('Failed to update checksum. This should never happen.') } + // sha256sum.write(buf) or {} } // return md5sum.sum(buf).hex(), sha256sum.sum(buf).hex() - return md5sum.sum(buf).hex(), '' + return md5sum.checksum().hex(), '' } From 4bc3bfb8cf7a2744fd06905c7cfa983c6a2ff849 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 17:22:36 +0100 Subject: [PATCH 045/111] Moved checksum calculation to desc function --- src/main.v | 3 ++- src/pkg.v | 29 +++++++++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/main.v b/src/main.v index e81ed92..835b822 100644 --- a/src/main.v +++ b/src/main.v @@ -109,5 +109,6 @@ fn main() { return } // println(info) - print(res.info.to_desc()) + println('hey') + print(res.to_desc()) } diff --git a/src/pkg.v b/src/pkg.v index 8b1849c..2598179 100644 --- a/src/pkg.v +++ b/src/pkg.v @@ -56,6 +56,7 @@ fn C.strcmp(&char, &char) int // Represents a read archive struct Pkg { pub: + path string [required] info PkgInfo [required] files []string [required] } @@ -74,10 +75,10 @@ mut: arch string build_date i64 packager string - md5sum string - sha256sum string - pgpsig string - pgpsigsize i64 + // md5sum string + // sha256sum string + pgpsig string + pgpsigsize i64 // Array values groups []string licenses []string @@ -90,6 +91,10 @@ mut: checkdepends []string } +pub fn (p &Pkg) checksum() ?(string, string) { + return util.hash_file(p.path) +} + // parse_pkg_info_string parses a PkgInfo object from a string fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { mut pkg_info := PkgInfo{} @@ -195,11 +200,9 @@ pub fn read_pkg(pkg_path string) ?Pkg { mut pkg_info := parse_pkg_info_string(unsafe { cstring_to_vstring(&char(buf)) }) ? pkg_info.csize = i64(os.file_size(pkg_path)) - pkg_info.md5sum, pkg_info.sha256sum = util.hash_file(pkg_path) or { - return error('Failed to hash package.') - } return Pkg{ + path: pkg_path info: pkg_info files: files } @@ -211,7 +214,9 @@ fn format_entry(key string, value string) string { // to_desc returns a desc file valid string representation // TODO calculate md5 & sha256 instead of believing the file -pub fn (p &PkgInfo) to_desc() string { +pub fn (pkg &Pkg) to_desc() string { + p := pkg.info + // filename mut desc := '%FILENAME%\n$p.name-$p.version-${p.arch}.pkg.tar.zst\n' @@ -230,8 +235,12 @@ pub fn (p &PkgInfo) to_desc() string { desc += format_entry('CSIZE', p.csize.str()) desc += format_entry('ISIZE', p.size.str()) - desc += format_entry('MD5SUM', p.md5sum) - desc += format_entry('SHA256SUM', p.sha256sum) + md5sum, sha256sum := pkg.checksum() or { '', '' } + + desc += format_entry('MD5SUM', md5sum) + + // TODO add this + // desc += format_entry('SHA256SUM', sha256sum) // TODO add pgpsig stuff From 57fe767a7052ac0d115800d60e1d14a5d522cf8e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 18:54:33 +0100 Subject: [PATCH 046/111] Working add to repo functionality --- src/main.v | 17 ++++--- src/{pkg.v => package.v} | 21 ++++++-- src/repo.v | 107 ++++++++++++++++++++++++--------------- 3 files changed, 94 insertions(+), 51 deletions(-) rename src/{pkg.v => package.v} (94%) diff --git a/src/main.v b/src/main.v index 835b822..dc515ea 100644 --- a/src/main.v +++ b/src/main.v @@ -4,7 +4,6 @@ import web import os import log import io -import pkg import repo const port = 8000 @@ -103,12 +102,16 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { // } fn main() { + r := repo.new('data/repo', 'data/pkgs') or { return } + print(r.add_from_path('test/homebank-5.5.1-1-x86_64.pkg.tar.zst') or { panic('you fialed') }) + // archive.list_filenames() - res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { - eprintln(err.msg) - return - } + // res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { + // eprintln(err.msg) + // return + // } // println(info) - println('hey') - print(res.to_desc()) + // println('hey') + // print(res.to_desc()) + // print(res.to_files()) } diff --git a/src/pkg.v b/src/package.v similarity index 94% rename from src/pkg.v rename to src/package.v index 2598179..12480b3 100644 --- a/src/pkg.v +++ b/src/package.v @@ -1,4 +1,4 @@ -module pkg +module package import time import os @@ -45,7 +45,9 @@ fn C.archive_entry_new() &C.archive_entry fn C.archive_entry_pathname(&C.archive_entry) &char // Get an entry's file size -// Note: this function actually returns an i64, but as this can't be used as an arugment to malloc, we'll just roll with it & assume an entry is never bigger than 4 gigs +// Note: this function actually returns an i64, but as this can't be used as an +// arugment to malloc, we'll just roll with it & assume an entry is never +// bigger than 4 gigs fn C.archive_entry_size(&C.archive_entry) int #include @@ -63,7 +65,7 @@ pub: // Represents the contents of a .PKGINFO file struct PkgInfo { -mut: +pub mut: // Single values name string base string @@ -212,13 +214,19 @@ fn format_entry(key string, value string) string { return '\n%$key%\n$value\n' } +pub fn (pkg &Pkg) filename() string { + p := pkg.info + + return '$p.name-$p.version-${p.arch}.pkg.tar.zst' +} + // to_desc returns a desc file valid string representation // TODO calculate md5 & sha256 instead of believing the file pub fn (pkg &Pkg) to_desc() string { p := pkg.info // filename - mut desc := '%FILENAME%\n$p.name-$p.version-${p.arch}.pkg.tar.zst\n' + mut desc := '%FILENAME%\n$pkg.filename()\n' desc += format_entry('NAME', p.name) desc += format_entry('BASE', p.base) @@ -286,3 +294,8 @@ pub fn (pkg &Pkg) to_desc() string { return '$desc\n' } + +// to_files returns a files file valid string representation +pub fn (pkg &Pkg) to_files() string { + return '%FILES%\n$pkg.files.join_lines()\n' +} diff --git a/src/repo.v b/src/repo.v index f3ebfd4..c7b7db5 100644 --- a/src/repo.v +++ b/src/repo.v @@ -1,8 +1,13 @@ module repo import os +import package -const pkgs_subpath = 'pkgs' +// subpath where the uncompressed version of the files archive is stored +const files_subpath = 'files' + +// subpath where the uncompressed version of the repo archive is stored +const repo_subpath = 'repo' // Dummy struct to work around the fact that you can only share structs, maps & // arrays @@ -21,46 +26,68 @@ pub: pkg_dir string [required] } -// contains returns whether the repository contains the given package. -pub fn (r &Repo) contains(pkg string) bool { - return os.exists(os.join_path(r.repo_dir, 'files', pkg)) -} - -// add adds the given package to the repo. If false, the package was already -// present in the repository. -pub fn (r &Repo) add(pkg string) ?bool { - return false -} - -// generate re-generates the db & files archives. -fn (r &Repo) genenerate() ? { -} - -// pkg_path returns path to the given package, prepended with the repo's path. -pub fn (r &Repo) pkg_path(pkg string) string { - return os.join_path_single(r.pkg_dir, pkg) -} - -// exists checks whether a package file exists -pub fn (r &Repo) exists(pkg string) bool { - return os.exists(r.pkg_path(pkg)) -} - -// db_path returns the full path to the database file -pub fn (r &Repo) db_path() string { - return os.join_path_single(r.repo_dir, 'repo.tar.gz') -} - -// add_package adds a package to the repository -pub fn (r &Repo) add_package(pkg_path string) ? { - mut res := os.Result{} - - lock r.mutex { - res = os.execute("repo-add '$r.db_path()' '$pkg_path'") +pub fn new(repo_dir string, pkg_dir string) ?Repo { + if !os.is_dir(repo_dir) { + os.mkdir_all(repo_dir) or { return error('Failed to create repo directory.') } } - if res.exit_code != 0 { - println(res.output) - return error('repo-add failed.') + if !os.is_dir(pkg_dir) { + os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') } + } + + return Repo{ + repo_dir: repo_dir + pkg_dir: pkg_dir } } + +// add_from_path adds a package from an arbitrary path & moves it into the pkgs +// directory if necessary. +pub fn (r &Repo) add_from_path(pkg_path string) ?bool { + pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file.') } + + added := r.add(pkg) ? + + // If the add was successful, we move the file to the packages directory + if added { + dest_path := os.real_path(os.join_path_single(r.pkg_dir, pkg.filename())) + + // Only move the file if it's not already in the package directory + if dest_path != os.real_path(pkg_path) { + os.mv(pkg_path, dest_path) ? + } + } + + return true +} + +// add adds a given Pkg to the repository +fn (r &Repo) add(pkg &package.Pkg) ?bool { + pkg_dir := r.pkg_path(pkg) + + // We can't add the same package twice + if os.exists(pkg_dir) { + return false + } + + os.mkdir(pkg_dir) or { return error('Failed to create package directory.') } + + os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or { + os.rmdir_all(pkg_dir) ? + + return error('Failed to write desc file.') + } + os.write_file(os.join_path_single(pkg_dir, 'files'), pkg.to_files()) or { + os.rmdir_all(pkg_dir) ? + + return error('Failed to write files file.') + } + // TODO generate database archive + + return true +} + +// Returns the path where the given package's desc & files files are stored +fn (r &Repo) pkg_path(pkg &package.Pkg) string { + return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version') +} From d04ba06ad35362a1b43a976988e4712bee6469c7 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 21:24:46 +0100 Subject: [PATCH 047/111] Finally enabled web server again --- Makefile | 2 +- src/main.v | 121 ++++++++++++++++++++++++++------------------------ src/package.v | 3 +- src/repo.v | 1 + src/routes.v | 75 +++++++++++++++---------------- 5 files changed, 103 insertions(+), 99 deletions(-) diff --git a/Makefile b/Makefile index dc2273b..9a5d6fa 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ c: # Run the server in the default 'data' directory .PHONY: run run: vieter - API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG ./vieter + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter .PHONY: run-prod run-prod: prod diff --git a/src/main.v b/src/main.v index dc515ea..ed6383e 100644 --- a/src/main.v +++ b/src/main.v @@ -16,6 +16,7 @@ struct App { web.Context pub: api_key string [required; web_global] + dl_dir string [required; web_global] pub mut: repo repo.Repo [required; web_global] } @@ -53,65 +54,67 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } } -// fn main2() { -// // Configure logger -// log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } -// log_level := log.level_from_tag(log_level_str) or { -// exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') -// } -// log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } - -// mut logger := log.Log{ -// level: log_level -// } - -// logger.set_full_logpath(log_file) -// logger.log_to_console_too() - -// defer { -// logger.info('Flushing log file') -// logger.flush() -// logger.close() -// } - -// // Configure web server -// key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } -// repo_dir := os.getenv_opt('REPO_DIR') or { -// exit_with_message(1, 'No repo directory was configured.') -// } - -// repo := repo.Repo{ -// dir: repo_dir -// name: db_name -// } - -// // We create the upload directory during startup -// if !os.is_dir(repo.pkg_dir()) { -// os.mkdir_all(repo.pkg_dir()) or { -// exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.") -// } - -// logger.info("Created package directory '$repo.pkg_dir()'.") -// } - -// web.run(&App{ -// logger: logger -// api_key: key -// repo: repo -// }, port) -// } - fn main() { - r := repo.new('data/repo', 'data/pkgs') or { return } - print(r.add_from_path('test/homebank-5.5.1-1-x86_64.pkg.tar.zst') or { panic('you fialed') }) + // Configure logger + log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } + log_level := log.level_from_tag(log_level_str) or { + exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + } + log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } - // archive.list_filenames() - // res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { - // eprintln(err.msg) - // return - // } - // println(info) - // println('hey') - // print(res.to_desc()) - // print(res.to_files()) + mut logger := log.Log{ + level: log_level + } + + logger.set_full_logpath(log_file) + logger.log_to_console_too() + + defer { + logger.info('Flushing log file') + logger.flush() + logger.close() + } + + // Configure web server + key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } + repo_dir := os.getenv_opt('REPO_DIR') or { + exit_with_message(1, 'No repo directory was configured.') + } + pkg_dir := os.getenv_opt('PKG_DIR') or { + exit_with_message(1, 'No package directory was configured.') + } + dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { + exit_with_message(1, 'No download directory was configured.') + } + + // This also creates the directories if needed + repo := repo.new(repo_dir, pkg_dir) or { + exit_with_message(1, 'Failed to create required directories.') + } + + os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } + + web.run(&App{ + logger: logger + api_key: key + dl_dir: dl_dir + repo: repo + }, port) } + +// fn main() { +// r := repo.new('data/repo', 'data/pkgs') or { return } +// print(r.add_from_path('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { +// panic('you fialed') +// }) + +// // archive.list_filenames() +// // res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { +// // eprintln(err.msg) +// // return +// // } +// // println(info) +// // println('hey') +// // print(res.to_desc()) +// // print(res.to_files()) +// } diff --git a/src/package.v b/src/package.v index 12480b3..6340021 100644 --- a/src/package.v +++ b/src/package.v @@ -1,6 +1,5 @@ module package -import time import os import util @@ -93,6 +92,7 @@ pub mut: checkdepends []string } +// checksum calculates the md5 & sha256 hash of the package pub fn (p &Pkg) checksum() ?(string, string) { return util.hash_file(p.path) } @@ -214,6 +214,7 @@ fn format_entry(key string, value string) string { return '\n%$key%\n$value\n' } +// filename returns the correct filename of the package file pub fn (pkg &Pkg) filename() string { p := pkg.info diff --git a/src/repo.v b/src/repo.v index c7b7db5..389ebce 100644 --- a/src/repo.v +++ b/src/repo.v @@ -26,6 +26,7 @@ pub: pkg_dir string [required] } +// new creates a new Repo & creates the directories as needed pub fn new(repo_dir string, pkg_dir string) ?Repo { if !os.is_dir(repo_dir) { os.mkdir_all(repo_dir) or { return error('Failed to create repo directory.') } diff --git a/src/routes.v b/src/routes.v index 101e1c5..c7e38ba 100644 --- a/src/routes.v +++ b/src/routes.v @@ -4,6 +4,7 @@ import web import os import repo import time +import rand const prefixes = ['B', 'KB', 'MB', 'GB'] @@ -38,57 +39,55 @@ fn (mut app App) get_root(filename string) web.Result { return app.file(full_path) } -// ['/pkgs/:pkg'; put] -// fn (mut app App) put_package(pkg string) web.Result { -// if !app.is_authorized() { -// return app.text('Unauthorized.') -// } +['/publish'; post] +fn (mut app App) put_package() web.Result { + if !app.is_authorized() { + return app.text('Unauthorized.') + } -// if !is_pkg_name(pkg) { -// app.lwarn("Invalid package name '$pkg'.") + mut pkg_path := '' -// return app.text('Invalid filename.') -// } + if length := app.req.header.get(.content_length) { + // Generate a random filename for the temp file + pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) -// if app.repo.exists(pkg) { -// app.lwarn("Duplicate package '$pkg'") + for os.exists(pkg_path) { + pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) + } -// return app.text('File already exists.') -// } + app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to '$pkg_path'.") -// pkg_path := app.repo.pkg_path(pkg) + // This is used to time how long it takes to upload a file + mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) -// if length := app.req.header.get(.content_length) { -// app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.") + reader_to_file(mut app.reader, length.int(), pkg_path) or { + app.lwarn("Failed to upload '$pkg_path'") -// // This is used to time how long it takes to upload a file -// mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) + return app.text('Failed to upload file.') + } -// reader_to_file(mut app.reader, length.int(), pkg_path) or { -// app.lwarn("Failed to upload package '$pkg'") + sw.stop() + app.ldebug("Upload of '$pkg_path' completed in ${sw.elapsed().seconds():.3}s.") + } else { + app.lwarn('Tried to upload package without specifying a Content-Length.') + return app.text("Content-Type header isn't set.") + } -// return app.text('Failed to upload file.') -// } + added := app.repo.add_from_path(pkg_path) or { + app.lerror('Error while adding package.') -// sw.stop() -// app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.") -// } else { -// app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.") -// return app.text("Content-Type header isn't set.") -// } + return app.text('Failed to add package.') + } + if !added { + app.lwarn('Duplicate package.') -// app.repo.add_package(pkg_path) or { -// app.lwarn("Failed to add package '$pkg' to database.") + return app.text('File already exists.') + } -// os.rm(pkg_path) or { println('Failed to remove $pkg_path') } + app.linfo("Added '$pkg_path' to repository.") -// return app.text('Failed to add package to repo.') -// } - -// app.linfo("Added '$pkg' to repository.") - -// return app.text('Package added successfully.') -// } + return app.text('Package added successfully.') +} // add_package PUT a new package to the server ['/add'; put] From 735492dcbd591e7525974d273ebcbe3cd90c0003 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 21:30:56 +0100 Subject: [PATCH 048/111] Be nice, CI --- src/package.v | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/package.v b/src/package.v index 6340021..cf727a9 100644 --- a/src/package.v +++ b/src/package.v @@ -244,7 +244,7 @@ pub fn (pkg &Pkg) to_desc() string { desc += format_entry('CSIZE', p.csize.str()) desc += format_entry('ISIZE', p.size.str()) - md5sum, sha256sum := pkg.checksum() or { '', '' } + md5sum, _ := pkg.checksum() or { '', '' } desc += format_entry('MD5SUM', md5sum) From 3fc0a2f9a984fc92b48e702920efe0e0f4572a4d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 23:06:46 +0100 Subject: [PATCH 049/111] Moved image dryruns to own pipeline --- .woodpecker/.image.yml | 17 +++++++++++++++++ .woodpecker/.publish.yml | 14 +------------- 2 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 .woodpecker/.image.yml diff --git a/.woodpecker/.image.yml b/.woodpecker/.image.yml new file mode 100644 index 0000000..6b78861 --- /dev/null +++ b/.woodpecker/.image.yml @@ -0,0 +1,17 @@ +# These checks already get performed on the feature branches +branches: + exclude: [ main, dev ] +platform: linux/amd64 + +pipeline: + dryrun: + image: woodpeckerci/plugin-docker-buildx + secrets: [ docker_username, docker_password ] + settings: + repo: chewingbever/vieter + tag: dev + platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + dry_run: true + when: + event: pull_request + branch: dev diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 6d3b34e..9e5f39a 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -1,19 +1,7 @@ -# branches: [main, dev] +branches: [main, dev] platform: linux/amd64 pipeline: - dryrun: - image: woodpeckerci/plugin-docker-buildx - secrets: [ docker_username, docker_password ] - settings: - repo: chewingbever/vieter - tag: dev - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] - dry_run: true - when: - event: pull_request - branch: dev - dev: image: woodpeckerci/plugin-docker-buildx secrets: [ docker_username, docker_password ] From 944bc412753d77c7d9e633441dd8fd832702a176 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 23:45:12 +0100 Subject: [PATCH 050/111] Split libarchive bindings into separate file --- src/libarchive.v | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ src/package.v | 51 ----------------------------------------------- 2 files changed, 52 insertions(+), 51 deletions(-) create mode 100644 src/libarchive.v diff --git a/src/libarchive.v b/src/libarchive.v new file mode 100644 index 0000000..bd90f09 --- /dev/null +++ b/src/libarchive.v @@ -0,0 +1,52 @@ +// Bindings for the libarchive library + +#flag -larchive + +#include "archive.h" + +struct C.archive {} + +// Create a new archive struct +fn C.archive_read_new() &C.archive + +// Configure the archive to work with zstd compression +fn C.archive_read_support_filter_zstd(&C.archive) + +// Configure the archive to work with a tarball content +fn C.archive_read_support_format_tar(&C.archive) + +// Open an archive for reading +fn C.archive_read_open_filename(&C.archive, &char, int) int + +// Go to next entry header in archive +fn C.archive_read_next_header(&C.archive, &&C.archive_entry) int + +// Skip reading the current entry +fn C.archive_read_data_skip(&C.archive) + +// Free an archive +fn C.archive_read_free(&C.archive) int + +// Read an archive entry's contents into a pointer +fn C.archive_read_data(&C.archive, voidptr, int) + +#include "archive_entry.h" + +struct C.archive_entry {} + +// Create a new archive_entry struct +fn C.archive_entry_new() &C.archive_entry + +// Get the filename of the given entry +fn C.archive_entry_pathname(&C.archive_entry) &char + +// Get an entry's file size +// Note: this function actually returns an i64, but as this can't be used as an +// arugment to malloc, we'll just roll with it & assume an entry is never +// bigger than 4 gigs +fn C.archive_entry_size(&C.archive_entry) int + +#include + +// Compare two C strings; 0 means they're equal +fn C.strcmp(&char, &char) int diff --git a/src/package.v b/src/package.v index cf727a9..9e8e503 100644 --- a/src/package.v +++ b/src/package.v @@ -3,57 +3,6 @@ module package import os import util -#flag -larchive - -#include "archive.h" - -struct C.archive {} - -// Create a new archive struct -fn C.archive_read_new() &C.archive - -// Configure the archive to work with zstd compression -fn C.archive_read_support_filter_zstd(&C.archive) - -// Configure the archive to work with a tarball content -fn C.archive_read_support_format_tar(&C.archive) - -// Open an archive for reading -fn C.archive_read_open_filename(&C.archive, &char, int) int - -// Go to next entry header in archive -fn C.archive_read_next_header(&C.archive, &&C.archive_entry) int - -// Skip reading the current entry -fn C.archive_read_data_skip(&C.archive) - -// Free an archive -fn C.archive_read_free(&C.archive) int - -// Read an archive entry's contents into a pointer -fn C.archive_read_data(&C.archive, voidptr, int) - -#include "archive_entry.h" - -struct C.archive_entry {} - -// Create a new archive_entry struct -fn C.archive_entry_new() &C.archive_entry - -// Get the filename of the given entry -fn C.archive_entry_pathname(&C.archive_entry) &char - -// Get an entry's file size -// Note: this function actually returns an i64, but as this can't be used as an -// arugment to malloc, we'll just roll with it & assume an entry is never -// bigger than 4 gigs -fn C.archive_entry_size(&C.archive_entry) int - -#include - -// Compare two C strings; 0 means they're equal -fn C.strcmp(&char, &char) int - // Represents a read archive struct Pkg { pub: From a37d40d27859faa30659e11fd9d0b0977d8dff99 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 20 Jan 2022 00:08:26 +0100 Subject: [PATCH 051/111] Added start of libarchive bindings for writing --- src/{libarchive.v => archive.v} | 35 ++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) rename src/{libarchive.v => archive.v} (50%) diff --git a/src/libarchive.v b/src/archive.v similarity index 50% rename from src/libarchive.v rename to src/archive.v index bd90f09..ee72e3a 100644 --- a/src/libarchive.v +++ b/src/archive.v @@ -6,7 +6,7 @@ struct C.archive {} -// Create a new archive struct +// Create a new archive struct for reading fn C.archive_read_new() &C.archive // Configure the archive to work with zstd compression @@ -30,6 +30,27 @@ fn C.archive_read_free(&C.archive) int // Read an archive entry's contents into a pointer fn C.archive_read_data(&C.archive, voidptr, int) +// Create a new archive struct for writing +fn C.archive_write_new() &C.archive + +// Sets the filter for the archive to gzip +fn C.archive_write_add_filter_gzip(&C.archive) + +// Sets to archive to "pax restricted" mode. Libarchive's "pax restricted" +// format is a tar format that uses pax extensions only when absolutely +// necessary. Most of the time, it will write plain ustar entries. This is the +// recommended tar format for most uses. You should explicitly use ustar format +// only when you have to create archives that will be readable on older +// systems; you should explicitly request pax format only when you need to +// preserve as many attributes as possible. +fn C.archive_write_set_format_pax_restricted(&C.archive) + +// Opens up the filename for writing +fn C.archive_write_open_filename(&C.archive, &char) + +// Write an entry to the archive file +fn C.archive_write_header(&C.archive, &C.archive_entry) + #include "archive_entry.h" struct C.archive_entry {} @@ -46,6 +67,18 @@ fn C.archive_entry_pathname(&C.archive_entry) &char // bigger than 4 gigs fn C.archive_entry_size(&C.archive_entry) int +// Set the pathname for the entry +fn C.archive_entry_set_pathname(&C.archive_entry, &char) + +// Sets the file size of the entry +fn C.archive_entry_set_size(&C.archive_entry, i64) + +// Sets the file type for an entry +fn C.archive_entry_set_filetype(&C.archive_entry, u32) + +// Sets the file permissions for an entry +fn C.archive_entry_set_perm(&C.archive_entry, int) + #include // Compare two C strings; 0 means they're equal From b6e9fa5b02cd5d24b952c933dde3874cf1d0e5d3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 20 Jan 2022 17:11:03 +0100 Subject: [PATCH 052/111] First working version of creating archive --- src/archive.v | 15 +++++++++++++++ src/repo.v | 52 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/archive.v b/src/archive.v index ee72e3a..7ded6a1 100644 --- a/src/archive.v +++ b/src/archive.v @@ -51,6 +51,15 @@ fn C.archive_write_open_filename(&C.archive, &char) // Write an entry to the archive file fn C.archive_write_header(&C.archive, &C.archive_entry) +// Write the data in the buffer to the archive +fn C.archive_write_data(&C.archive, voidptr, int) + +// Close an archive for writing +fn C.archive_write_close(&C.archive) + +// Free the write archive +fn C.archive_write_free(&C.archive) + #include "archive_entry.h" struct C.archive_entry {} @@ -79,6 +88,12 @@ fn C.archive_entry_set_filetype(&C.archive_entry, u32) // Sets the file permissions for an entry fn C.archive_entry_set_perm(&C.archive_entry, int) +// Clears out an entry struct +fn C.archive_entry_clear(&C.archive_entry) + +// Copy over a stat struct to the archive entry +fn C.archive_entry_copy_stat(&C.archive_entry, &C.stat) + #include // Compare two C strings; 0 means they're equal diff --git a/src/repo.v b/src/repo.v index 389ebce..ff84909 100644 --- a/src/repo.v +++ b/src/repo.v @@ -59,7 +59,7 @@ pub fn (r &Repo) add_from_path(pkg_path string) ?bool { } } - return true + return added } // add adds a given Pkg to the repository @@ -83,7 +83,8 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool { return error('Failed to write files file.') } - // TODO generate database archive + + r.sync() ? return true } @@ -92,3 +93,50 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool { fn (r &Repo) pkg_path(pkg &package.Pkg) string { return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version') } + +// Re-generate the repo archive files +fn (r &Repo) sync() ? { + a := C.archive_write_new() + entry := C.archive_entry_new() + st := C.stat{} + buf := [8192]byte{} + + // This makes the archive a gzip-compressed tarball + C.archive_write_add_filter_gzip(a) + C.archive_write_set_format_pax_restricted(a) + + repo_path := os.join_path_single(r.repo_dir, 'repo.db') + + C.archive_write_open_filename(a, &char(repo_path.str)) + + // Iterate over each directory + for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, it))) { + inner_path := os.join_path_single(d, 'desc') + actual_path := os.join_path_single(r.repo_dir, inner_path) + + unsafe { + C.stat(&char(actual_path.str), &st) + } + + C.archive_entry_set_pathname(entry, &char(inner_path.str)) + // C.archive_entry_copy_stat(entry, &st) + C.archive_entry_set_size(entry, st.st_size) + C.archive_entry_set_filetype(entry, C.AE_IFREG) + C.archive_entry_set_perm(entry, 0o644) + C.archive_write_header(a, entry) + + fd := C.open(&char(actual_path.str), C.O_RDONLY) + mut len := C.read(fd, &buf, sizeof(buf)) + + for len > 0 { + C.archive_write_data(a, &buf, len) + len = C.read(fd, &buf, sizeof(buf)) + } + C.close(fd) + + C.archive_entry_clear(entry) + } + + C.archive_write_close(a) + C.archive_write_free(a) +} From 01d961f68e0e4700e4a85597384361534234a9fc Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 20 Jan 2022 17:45:04 +0100 Subject: [PATCH 053/111] File stats are now correctly copied over to archive --- src/archive.v | 2 +- src/repo.v | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/archive.v b/src/archive.v index 7ded6a1..bdc1846 100644 --- a/src/archive.v +++ b/src/archive.v @@ -92,7 +92,7 @@ fn C.archive_entry_set_perm(&C.archive_entry, int) fn C.archive_entry_clear(&C.archive_entry) // Copy over a stat struct to the archive entry -fn C.archive_entry_copy_stat(&C.archive_entry, &C.stat) +fn C.archive_entry_copy_stat(entry &C.archive_entry, const_stat &C.stat) #include diff --git a/src/repo.v b/src/repo.v index ff84909..6ddde2a 100644 --- a/src/repo.v +++ b/src/repo.v @@ -119,10 +119,10 @@ fn (r &Repo) sync() ? { } C.archive_entry_set_pathname(entry, &char(inner_path.str)) - // C.archive_entry_copy_stat(entry, &st) - C.archive_entry_set_size(entry, st.st_size) - C.archive_entry_set_filetype(entry, C.AE_IFREG) - C.archive_entry_set_perm(entry, 0o644) + C.archive_entry_copy_stat(entry, &st) + // C.archive_entry_set_size(entry, st.st_size) + // C.archive_entry_set_filetype(entry, C.AE_IFREG) + // C.archive_entry_set_perm(entry, 0o644) C.archive_write_header(a, entry) fd := C.open(&char(actual_path.str), C.O_RDONLY) From f1a17cab22816bbfa1f0f9ab3f3fe883f8e3a9f7 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 20 Jan 2022 19:39:38 +0100 Subject: [PATCH 054/111] Failed attempt at fixing memory bug --- src/package.v | 14 +++++++--- src/repo.v | 75 ++++++++++++++++++++++++++------------------------- src/routes.v | 2 +- 3 files changed, 51 insertions(+), 40 deletions(-) diff --git a/src/package.v b/src/package.v index 9e8e503..c229e3f 100644 --- a/src/package.v +++ b/src/package.v @@ -126,6 +126,7 @@ pub fn read_pkg(pkg_path string) ?Pkg { mut buf := voidptr(0) mut files := []string{} + mut pkg_info := PkgInfo{} for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { pathname := C.archive_entry_pathname(entry) @@ -142,14 +143,21 @@ pub fn read_pkg(pkg_path string) ?Pkg { // TODO can this unsafe block be avoided? buf = unsafe { malloc(size) } - C.archive_read_data(a, voidptr(buf), size) + C.archive_read_data(a, buf, size) + + unsafe { + println(cstring_to_vstring(buf)) + } + pkg_info = parse_pkg_info_string(unsafe { cstring_to_vstring(buf) }) ? + + unsafe { + free(buf) + } } else { C.archive_read_data_skip(a) } } - mut pkg_info := parse_pkg_info_string(unsafe { cstring_to_vstring(&char(buf)) }) ? - pkg_info.csize = i64(os.file_size(pkg_path)) return Pkg{ diff --git a/src/repo.v b/src/repo.v index 6ddde2a..cca6b84 100644 --- a/src/repo.v +++ b/src/repo.v @@ -45,7 +45,7 @@ pub fn new(repo_dir string, pkg_dir string) ?Repo { // add_from_path adds a package from an arbitrary path & moves it into the pkgs // directory if necessary. pub fn (r &Repo) add_from_path(pkg_path string) ?bool { - pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file.') } + pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') } added := r.add(pkg) ? @@ -96,47 +96,50 @@ fn (r &Repo) pkg_path(pkg &package.Pkg) string { // Re-generate the repo archive files fn (r &Repo) sync() ? { - a := C.archive_write_new() - entry := C.archive_entry_new() - st := C.stat{} - buf := [8192]byte{} + lock r.mutex { + a := C.archive_write_new() + entry := C.archive_entry_new() + st := C.stat{} + buf := [8192]byte{} - // This makes the archive a gzip-compressed tarball - C.archive_write_add_filter_gzip(a) - C.archive_write_set_format_pax_restricted(a) + // This makes the archive a gzip-compressed tarball + C.archive_write_add_filter_gzip(a) + C.archive_write_set_format_pax_restricted(a) - repo_path := os.join_path_single(r.repo_dir, 'repo.db') + repo_path := os.join_path_single(r.repo_dir, 'repo.db') - C.archive_write_open_filename(a, &char(repo_path.str)) + C.archive_write_open_filename(a, &char(repo_path.str)) - // Iterate over each directory - for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, it))) { - inner_path := os.join_path_single(d, 'desc') - actual_path := os.join_path_single(r.repo_dir, inner_path) + // Iterate over each directory + for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, + it))) { + inner_path := os.join_path_single(d, 'desc') + actual_path := os.join_path_single(r.repo_dir, inner_path) - unsafe { - C.stat(&char(actual_path.str), &st) + unsafe { + C.stat(&char(actual_path.str), &st) + } + + C.archive_entry_set_pathname(entry, &char(inner_path.str)) + C.archive_entry_copy_stat(entry, &st) + // C.archive_entry_set_size(entry, st.st_size) + // C.archive_entry_set_filetype(entry, C.AE_IFREG) + // C.archive_entry_set_perm(entry, 0o644) + C.archive_write_header(a, entry) + + fd := C.open(&char(actual_path.str), C.O_RDONLY) + mut len := C.read(fd, &buf, sizeof(buf)) + + for len > 0 { + C.archive_write_data(a, &buf, len) + len = C.read(fd, &buf, sizeof(buf)) + } + C.close(fd) + + C.archive_entry_clear(entry) } - C.archive_entry_set_pathname(entry, &char(inner_path.str)) - C.archive_entry_copy_stat(entry, &st) - // C.archive_entry_set_size(entry, st.st_size) - // C.archive_entry_set_filetype(entry, C.AE_IFREG) - // C.archive_entry_set_perm(entry, 0o644) - C.archive_write_header(a, entry) - - fd := C.open(&char(actual_path.str), C.O_RDONLY) - mut len := C.read(fd, &buf, sizeof(buf)) - - for len > 0 { - C.archive_write_data(a, &buf, len) - len = C.read(fd, &buf, sizeof(buf)) - } - C.close(fd) - - C.archive_entry_clear(entry) + C.archive_write_close(a) + C.archive_write_free(a) } - - C.archive_write_close(a) - C.archive_write_free(a) } diff --git a/src/routes.v b/src/routes.v index c7e38ba..8ad4a4a 100644 --- a/src/routes.v +++ b/src/routes.v @@ -74,7 +74,7 @@ fn (mut app App) put_package() web.Result { } added := app.repo.add_from_path(pkg_path) or { - app.lerror('Error while adding package.') + app.lerror('Error while adding package: $err.msg') return app.text('Failed to add package.') } From e817535e86b7c9b255d941267653c7f1c3a2d0e3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 20 Jan 2022 20:42:18 +0100 Subject: [PATCH 055/111] Removed some commented-out code --- src/main.v | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/main.v b/src/main.v index ed6383e..97788e7 100644 --- a/src/main.v +++ b/src/main.v @@ -101,20 +101,3 @@ fn main() { repo: repo }, port) } - -// fn main() { -// r := repo.new('data/repo', 'data/pkgs') or { return } -// print(r.add_from_path('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { -// panic('you fialed') -// }) - -// // archive.list_filenames() -// // res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { -// // eprintln(err.msg) -// // return -// // } -// // println(info) -// // println('hey') -// // print(res.to_desc()) -// // print(res.to_files()) -// } From ad7b42758ed447aeb13ff95a43775983e03e798f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 20 Jan 2022 20:47:15 +0100 Subject: [PATCH 056/111] Hopefully fixed image dryrun ci setting --- .woodpecker/.image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.woodpecker/.image.yml b/.woodpecker/.image.yml index 6b78861..7f583a8 100644 --- a/.woodpecker/.image.yml +++ b/.woodpecker/.image.yml @@ -1,6 +1,6 @@ -# These checks already get performed on the feature branches -branches: - exclude: [ main, dev ] +# Because the only step here is a pull_request event, the branch should be dev +# because it has to be the target of the pull request +branches: dev platform: linux/amd64 pipeline: From ed169abfff84f2d4e1a5057201fbc7e976d2d893 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 20 Jan 2022 21:52:19 +0100 Subject: [PATCH 057/111] Builder dockerfile now builds from gh releases [CI SKIP] --- Dockerfile.builder | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Dockerfile.builder b/Dockerfile.builder index 48048f8..406386b 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -1,6 +1,6 @@ FROM alpine:3.12 -LABEL maintainer="spytheman " +ARG V_RELEASE=weekly.2022.03 WORKDIR /opt/vlang @@ -8,10 +8,9 @@ ENV VVV /opt/vlang ENV PATH /opt/vlang:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ENV VFLAGS -cc gcc -RUN mkdir -p /opt/vlang && \ - ln -s /opt/vlang/v /usr/bin/v && \ +RUN ln -s /opt/vlang/v /usr/bin/v && \ apk --no-cache add \ - git make gcc \ + curl git make gcc \ musl-dev \ openssl-libs-static openssl-dev \ zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \ @@ -20,14 +19,15 @@ RUN mkdir -p /opt/vlang && \ libarchive-static libarchive-dev \ diffutils -COPY . /vlang-local - -RUN git clone \ - 'https://github.com/ChewingBever/v/' \ - -b vweb-streaming \ - --single-branch \ - '/opt/vlang' && \ - rm -rf '/vlang-local' && \ - make && v -version +RUN curl -Lo - "https://github.com/vlang/v/archive/refs/tags/${V_RELEASE}.tar.gz" | tar xzf - && \ + mv "v-${V_RELEASE}"/* /opt/vlang && \ + make && v -version +# RUN git clone \ +# 'https://github.com/ChewingBever/v/' \ +# -b vweb-streaming \ +# --single-branch \ +# '/opt/vlang' && \ +# rm -rf '/vlang-local' && \ +# make && v -version CMD ["v"] From 0700eb16fbfe1e58e1d87d6120581fab2c8e32db Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 19:01:46 +0100 Subject: [PATCH 058/111] Added static binary upload --- .woodpecker/.build.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 4b87f8b..fdfd3da 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -28,3 +28,16 @@ pipeline: - readelf -d pvieter - du -h pvieter - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' + # This removes so much, it's amazing + - strip -s pvieter + - du -h pvieter + + upload: + image: 'alpine:latest' + secrets: [ s3_username, s3_password ] + commands: + # https://min.io/download#/linux + - wget https://dl.min.io/client/mc/release/linux-amd64/mc + - chmod +x mc + - ./mc alias set s3 https://s3.rustybever.be "$S3_USERNAME" "$S3_PASSWORD" + - ./mc cp pvieter "s3/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:')" From 81a0881454d378e1bcdbd18bb7993fbe54fadea0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 19:22:19 +0100 Subject: [PATCH 059/111] Switched to cross-platform curl-based solution --- .woodpecker/.build.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index fdfd3da..2a243dd 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -36,8 +36,24 @@ pipeline: image: 'alpine:latest' secrets: [ s3_username, s3_password ] commands: - # https://min.io/download#/linux - - wget https://dl.min.io/client/mc/release/linux-amd64/mc - - chmod +x mc - - ./mc alias set s3 https://s3.rustybever.be "$S3_USERNAME" "$S3_PASSWORD" - - ./mc cp pvieter "s3/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:')" + # We should just add this to the builder image instead + - apk add curl openssl + + # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f + - export URL=s3.rustybever.be + - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:')" + - export DATE=`date -R --utc` + - export CONTENT_TYPE='application/zstd' + - export SIG_STRING="PUT\n\n${CONTENT_TYPE}\n${DATE}\n${OBJ_PATH}" + - export SIGNATURE=`echo -en ${SIG_STRING} | openssl sha1 -hmac ${S3_PASSWORD} -binary | base64` + + - > + curl \ + --silent \ + -XPUT \ + -T pvieter \ + -H "Host: $URL" \ + -H "Date: ${DATE}" \ + -H "Content-Type: ${CONTENT_TYPE}" \ + -H "Authorization: AWS ${S3_USERNAME}:${SIGNATURE}" \ + https://$URL${OBJ_PATH} From 32fc1317b052228d8c88680c19a162f3b30b599b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 19:26:52 +0100 Subject: [PATCH 060/111] Squashing some bugs --- .woodpecker/.build.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 2a243dd..15dbdb2 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -42,18 +42,18 @@ pipeline: # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f - export URL=s3.rustybever.be - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:')" - - export DATE=`date -R --utc` + - export DATE="$(date -R --utc`)" - export CONTENT_TYPE='application/zstd' - - export SIG_STRING="PUT\n\n${CONTENT_TYPE}\n${DATE}\n${OBJ_PATH}" - - export SIGNATURE=`echo -en ${SIG_STRING} | openssl sha1 -hmac ${S3_PASSWORD} -binary | base64` + - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH" + - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64` - > - curl \ - --silent \ - -XPUT \ - -T pvieter \ - -H "Host: $URL" \ - -H "Date: ${DATE}" \ - -H "Content-Type: ${CONTENT_TYPE}" \ - -H "Authorization: AWS ${S3_USERNAME}:${SIGNATURE}" \ - https://$URL${OBJ_PATH} + curl + --silent + -XPUT + -T pvieter + -H "Host: $URL" + -H "Date: $DATE" + -H "Content-Type: $CONTENT_TYPE" + -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" + https://$URL$OBJ_PATH From 2ce92e237d679570dae6c3d9646754cd5559773f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 19:29:01 +0100 Subject: [PATCH 061/111] CI? --- .woodpecker/.build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 15dbdb2..1b6e83f 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -24,7 +24,7 @@ pipeline: group: 'build' commands: - make prod - # Make sure the binary is actually static + # Make sure the binary is actually statically built - readelf -d pvieter - du -h pvieter - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' From df648ba129762b1754e0c808c07d84df89b50520 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 19:30:46 +0100 Subject: [PATCH 062/111] Stewie just said that! --- .woodpecker/.build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 1b6e83f..6803f01 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -42,7 +42,7 @@ pipeline: # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f - export URL=s3.rustybever.be - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:')" - - export DATE="$(date -R --utc`)" + - export DATE="$(date -R --utc)" - export CONTENT_TYPE='application/zstd' - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH" - export SIGNATURE=`echo -en $SIG_STRING | openssl sha1 -hmac $S3_PASSWORD -binary | base64` From da57da8ceed554c7712d188fa882f82297601481 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 19:36:42 +0100 Subject: [PATCH 063/111] Of course I forgot the g --- .woodpecker/.build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 6803f01..f3ae8b6 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -41,7 +41,7 @@ pipeline: # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f - export URL=s3.rustybever.be - - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:')" + - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')" - export DATE="$(date -R --utc)" - export CONTENT_TYPE='application/zstd' - export SIG_STRING="PUT\n\n$CONTENT_TYPE\n$DATE\n$OBJ_PATH" From a810d5b3b0e3416fe9d60b39cf580426662f7017 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 21:26:25 +0100 Subject: [PATCH 064/111] Added gitea release section to ci --- .woodpecker/.build.yml | 2 -- .woodpecker/.release.yml | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 .woodpecker/.release.yml diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index f3ae8b6..ecf1d73 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -5,8 +5,6 @@ matrix: - linux/arm/v7 # These checks already get performed on the feature branches -branches: - exclude: [ main, dev ] platform: ${PLATFORM} pipeline: diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml new file mode 100644 index 0000000..622b6e4 --- /dev/null +++ b/.woodpecker/.release.yml @@ -0,0 +1,33 @@ +platform: linux/amd64 + +pipeline: + prepare: + image: 'chewingbever/vlang:latest' + secrets: [ s3_username, s3_password ] + commands: + # Write the title to a file that the plugin can then read + - echo "$(git describe --tags --abbrev=0 2> /dev/null || echo '0.0.0')-$(git rev-list --count ^dev)" > title + - cat title + - wget https://dl.min.io/client/mc/release/linux-amd64/mc + - chmod +x mc + - ./mc alias set s3/ https://s3.rustybever.be "$S3_USERNAME" "$S3_PASSWORD" + - ./mc cp "s3/vieter/commits/$CI" assets + + release: + image: 'plugins/gitea-release' + secrets: + - gitea_release_api_key + settings: + base_url: https://git.rustybever.be + files: assets/* + checksums: + - md5 + - sha256 + prerelease: true + # This should get read in as a file + title: title + when: + event: push + +depends_on: + - build From 3d07213a917bf53455eaac6be90cd4f595869c4d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 21:27:55 +0100 Subject: [PATCH 065/111] Wrong env var used --- .woodpecker/.release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml index 622b6e4..26bf522 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.release.yml @@ -11,7 +11,7 @@ pipeline: - wget https://dl.min.io/client/mc/release/linux-amd64/mc - chmod +x mc - ./mc alias set s3/ https://s3.rustybever.be "$S3_USERNAME" "$S3_PASSWORD" - - ./mc cp "s3/vieter/commits/$CI" assets + - ./mc cp "s3/vieter/commits/$CI_COMMIT_SHA" assets release: image: 'plugins/gitea-release' From fd4439eb429bf2e44b949a7ebf5ef6dad5737f69 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 21:54:03 +0100 Subject: [PATCH 066/111] Possible releases fix --- .woodpecker/.build.yml | 4 ++-- .woodpecker/.release.yml | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index ecf1d73..c067a6c 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -1,8 +1,8 @@ matrix: PLATFORM: - linux/amd64 - - linux/arm64 - - linux/arm/v7 + # - linux/arm64 + # - linux/arm/v7 # These checks already get performed on the feature branches platform: ${PLATFORM} diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml index 26bf522..184f5eb 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.release.yml @@ -1,17 +1,23 @@ platform: linux/amd64 +# We need the entire repo in order for the release names to work +clone: + skip: true + pipeline: prepare: image: 'chewingbever/vlang:latest' secrets: [ s3_username, s3_password ] commands: + - git clone "$CI_REPO_REMOTE" + - git checkout "$CI_COMMIT_BRANCH" # Write the title to a file that the plugin can then read - echo "$(git describe --tags --abbrev=0 2> /dev/null || echo '0.0.0')-$(git rev-list --count ^dev)" > title - cat title - wget https://dl.min.io/client/mc/release/linux-amd64/mc - chmod +x mc - ./mc alias set s3/ https://s3.rustybever.be "$S3_USERNAME" "$S3_PASSWORD" - - ./mc cp "s3/vieter/commits/$CI_COMMIT_SHA" assets + - ./mc cp -r "s3/vieter/commits/$CI_COMMIT_SHA" assets release: image: 'plugins/gitea-release' From 21ef11ec336fe3591bbe7b170f0646f455691102 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 21:55:11 +0100 Subject: [PATCH 067/111] We go agane --- .woodpecker/.release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml index 184f5eb..ee6cc17 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.release.yml @@ -2,7 +2,8 @@ platform: linux/amd64 # We need the entire repo in order for the release names to work clone: - skip: true + git: + skip: true pipeline: prepare: From 8aeabe3476d224f73dc8bc4e200590b8cd69a384 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 21:58:04 +0100 Subject: [PATCH 068/111] finally --- .woodpecker/.release.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml index ee6cc17..d3ea0b2 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.release.yml @@ -1,9 +1,7 @@ platform: linux/amd64 # We need the entire repo in order for the release names to work -clone: - git: - skip: true +skip_clone: true pipeline: prepare: From 7552e0e86fc4eebd0e4694fedfe63971e5e7e27b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 22:02:22 +0100 Subject: [PATCH 069/111] Some more debugging --- .woodpecker/.release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml index d3ea0b2..46651b9 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.release.yml @@ -8,7 +8,7 @@ pipeline: image: 'chewingbever/vlang:latest' secrets: [ s3_username, s3_password ] commands: - - git clone "$CI_REPO_REMOTE" + - git clone "$CI_REPO_REMOTE" . - git checkout "$CI_COMMIT_BRANCH" # Write the title to a file that the plugin can then read - echo "$(git describe --tags --abbrev=0 2> /dev/null || echo '0.0.0')-$(git rev-list --count ^dev)" > title From 20ec4c12043eec5bd50fedb852011c7eeaa4b074 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 22:12:30 +0100 Subject: [PATCH 070/111] Let's give up for now --- .woodpecker/.build.yml | 6 ++++-- .woodpecker/.lint.yml | 6 +----- .woodpecker/.release.yml | 2 ++ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index c067a6c..96fa16b 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -1,8 +1,8 @@ matrix: PLATFORM: - linux/amd64 - # - linux/arm64 - # - linux/arm/v7 + - linux/arm64 + - linux/arm/v7 # These checks already get performed on the feature branches platform: ${PLATFORM} @@ -11,12 +11,14 @@ pipeline: # The default build isn't needed, as alpine switches to gcc for the compiler anyways debug: image: 'chewingbever/vlang:latest' + pull: true group: 'build' commands: - make debug prod: image: 'chewingbever/vlang:latest' + pull: true environment: - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static group: 'build' diff --git a/.woodpecker/.lint.yml b/.woodpecker/.lint.yml index 1ccd077..ce000cd 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/.lint.yml @@ -6,12 +6,8 @@ platform: linux/amd64 pipeline: lint: image: 'chewingbever/vlang:latest' + pull: true group: lint commands: - make lint - - vet: - image: 'chewingbever/vlang:latest' - group: lint - commands: - make vet diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml index 46651b9..9da27be 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.release.yml @@ -1,4 +1,6 @@ +# Yeah so this only works on tags so we'll worry about this later platform: linux/amd64 +branches: main # We need the entire repo in order for the release names to work skip_clone: true From 4a71fd1f7f5beaa30c2ffad4e4ed7ffcf31a0bad Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 22:14:29 +0100 Subject: [PATCH 071/111] Builds should only run on push --- .woodpecker/.build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 96fa16b..6e522d3 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -3,6 +3,8 @@ matrix: - linux/amd64 - linux/arm64 - linux/arm/v7 +when: + event: push # These checks already get performed on the feature branches platform: ${PLATFORM} From 2b2f3987d7a4edb3f1a7de9d373fe552b9c3569e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 22:17:38 +0100 Subject: [PATCH 072/111] Same thing, different style --- .woodpecker/.build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 6e522d3..c00e0a7 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -3,8 +3,6 @@ matrix: - linux/amd64 - linux/arm64 - linux/arm/v7 -when: - event: push # These checks already get performed on the feature branches platform: ${PLATFORM} @@ -33,6 +31,8 @@ pipeline: # This removes so much, it's amazing - strip -s pvieter - du -h pvieter + when: + event: push upload: image: 'alpine:latest' @@ -59,3 +59,5 @@ pipeline: -H "Content-Type: $CONTENT_TYPE" -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" https://$URL$OBJ_PATH + when: + event: push From 46ad12797d4660c721dd7a424e5e771cb8340acf Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 22:18:19 +0100 Subject: [PATCH 073/111] You know what they say --- .woodpecker/.build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index c00e0a7..516fb20 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -15,6 +15,8 @@ pipeline: group: 'build' commands: - make debug + when: + event: push prod: image: 'chewingbever/vlang:latest' From a1dd6c0c3634a452d963d364a750afe0ac2ae47a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 22:37:27 +0100 Subject: [PATCH 074/111] Moving some more tools into builder image [CI SKIP] --- .woodpecker/.release.yml | 1 + Dockerfile.builder | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml index 9da27be..2e32c7f 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.release.yml @@ -8,6 +8,7 @@ skip_clone: true pipeline: prepare: image: 'chewingbever/vlang:latest' + pull: true secrets: [ s3_username, s3_password ] commands: - git clone "$CI_REPO_REMOTE" . diff --git a/Dockerfile.builder b/Dockerfile.builder index 48048f8..ff2761f 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -11,14 +11,17 @@ ENV VFLAGS -cc gcc RUN mkdir -p /opt/vlang && \ ln -s /opt/vlang/v /usr/bin/v && \ apk --no-cache add \ - git make gcc \ + git make gcc curl openssl \ musl-dev \ openssl-libs-static openssl-dev \ zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \ sqlite-static sqlite-dev \ libx11-dev glfw-dev freetype-dev \ libarchive-static libarchive-dev \ - diffutils + diffutils && \ + # yes yes I know this is amd64, it's okay + wget -O /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && \ + chmod +x /usr/local/bin/mc COPY . /vlang-local From 7443a06325ce13a7912a76af91ce90e5e38235b9 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 22:40:44 +0100 Subject: [PATCH 075/111] Poke the CI --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5d49577..b51ab7f 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,12 @@ terminal, & upload them to the servers using CI. The data directory consists of three main directories: -* `downloads` - This is where packages are initially downloaded. Because vieter moves files from this folder to the `pkgs` folder, these two folders should best be on the same drive +* `downloads` - This is where packages are initially downloaded. Because vieter + moves files from this folder to the `pkgs` folder, these two folders should + best be on the same drive * `pkgs` - This is where approved package files are stored. -* `repos` - Each repository gets a subfolder here. The subfolder contains the uncompressed contents of the db file. - * Each repo subdirectory contains the compressed db & files archive for the repository, alongside a directory called `files` which contains the uncompressed contents. +* `repos` - Each repository gets a subfolder here. The subfolder contains the + uncompressed contents of the db file. + * Each repo subdirectory contains the compressed db & files archive for the + repository, alongside a directory called `files` which contains the + uncompressed contents. From 9a102a08fadb05baa6168176bfdf61fcd2939e59 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 22:41:46 +0100 Subject: [PATCH 076/111] Come on --- Dockerfile.builder | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile.builder b/Dockerfile.builder index ff2761f..bc90473 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -1,7 +1,5 @@ FROM alpine:3.12 -LABEL maintainer="spytheman " - WORKDIR /opt/vlang ENV VVV /opt/vlang From 4c42838c93630874dc6b6893cfa039a1b5f15a8b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 22:52:21 +0100 Subject: [PATCH 077/111] A test, as they say --- Dockerfile.builder | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile.builder b/Dockerfile.builder index bc90473..2186c89 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -6,6 +6,8 @@ ENV VVV /opt/vlang ENV PATH /opt/vlang:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ENV VFLAGS -cc gcc +RUN echo "$TARGETPLATFORM" && exit 1 + RUN mkdir -p /opt/vlang && \ ln -s /opt/vlang/v /usr/bin/v && \ apk --no-cache add \ From 3414f32718694b5426d43b313598c2710e978c6a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 22:58:09 +0100 Subject: [PATCH 078/111] Possible fix --- Dockerfile.builder | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Dockerfile.builder b/Dockerfile.builder index 2186c89..b075dda 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -6,8 +6,6 @@ ENV VVV /opt/vlang ENV PATH /opt/vlang:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ENV VFLAGS -cc gcc -RUN echo "$TARGETPLATFORM" && exit 1 - RUN mkdir -p /opt/vlang && \ ln -s /opt/vlang/v /usr/bin/v && \ apk --no-cache add \ @@ -18,10 +16,13 @@ RUN mkdir -p /opt/vlang && \ sqlite-static sqlite-dev \ libx11-dev glfw-dev freetype-dev \ libarchive-static libarchive-dev \ - diffutils && \ - # yes yes I know this is amd64, it's okay - wget -O /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && \ - chmod +x /usr/local/bin/mc + diffutils + +RUN if [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \ + wget -O /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && \ + chmod +x /usr/local/bin/mc ; \ +fi + COPY . /vlang-local From 3f77c49c0137c0e497569a491d9df07cb8a5e0f4 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 23:03:15 +0100 Subject: [PATCH 079/111] Forgot to add build arg --- Dockerfile.builder | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile.builder b/Dockerfile.builder index b075dda..75b1e3c 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -1,5 +1,7 @@ FROM alpine:3.12 +ARG TARGETPLATFORM + WORKDIR /opt/vlang ENV VVV /opt/vlang From d2af65cbce2942119ca61605f082ad99324587d6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 22 Jan 2022 23:15:36 +0100 Subject: [PATCH 080/111] It's in the builder now [CI SKIP] --- .woodpecker/.build.yml | 3 --- .woodpecker/.release.yml | 6 ++---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 516fb20..b91f7cc 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -40,9 +40,6 @@ pipeline: image: 'alpine:latest' secrets: [ s3_username, s3_password ] commands: - # We should just add this to the builder image instead - - apk add curl openssl - # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f - export URL=s3.rustybever.be - export OBJ_PATH="/vieter/commits/$CI_COMMIT_SHA/vieter-$(echo '${PLATFORM}' | sed 's:/:-:g')" diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml index 2e32c7f..bc43b90 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.release.yml @@ -16,10 +16,8 @@ pipeline: # Write the title to a file that the plugin can then read - echo "$(git describe --tags --abbrev=0 2> /dev/null || echo '0.0.0')-$(git rev-list --count ^dev)" > title - cat title - - wget https://dl.min.io/client/mc/release/linux-amd64/mc - - chmod +x mc - - ./mc alias set s3/ https://s3.rustybever.be "$S3_USERNAME" "$S3_PASSWORD" - - ./mc cp -r "s3/vieter/commits/$CI_COMMIT_SHA" assets + - mc alias set s3/ https://s3.rustybever.be "$S3_USERNAME" "$S3_PASSWORD" + - mc cp -r "s3/vieter/commits/$CI_COMMIT_SHA" assets release: image: 'plugins/gitea-release' From 3fc22bf078319ab5499cd37946ceb16fd53901b9 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 23 Jan 2022 10:53:58 +0100 Subject: [PATCH 081/111] Switchd to hyper-minimal docker image --- .woodpecker/.image.yml | 2 ++ .woodpecker/.publish.yml | 3 +++ .woodpecker/.release.yml | 1 + Dockerfile.ci | 20 ++++++++++++++++++++ 4 files changed, 26 insertions(+) create mode 100644 Dockerfile.ci diff --git a/.woodpecker/.image.yml b/.woodpecker/.image.yml index 7f583a8..dcf40f9 100644 --- a/.woodpecker/.image.yml +++ b/.woodpecker/.image.yml @@ -2,12 +2,14 @@ # because it has to be the target of the pull request branches: dev platform: linux/amd64 +depends_on: build pipeline: dryrun: image: woodpeckerci/plugin-docker-buildx secrets: [ docker_username, docker_password ] settings: + dockerfile: Dockerfile.ci repo: chewingbever/vieter tag: dev platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 9e5f39a..1e70eaf 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -6,9 +6,12 @@ pipeline: image: woodpeckerci/plugin-docker-buildx secrets: [ docker_username, docker_password ] settings: + dockerfile: Dockerfile.ci repo: chewingbever/vieter tag: dev platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + args: + - CI_COMMIT_SHA when: event: push branch: dev diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml index bc43b90..a220c75 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.release.yml @@ -1,6 +1,7 @@ # Yeah so this only works on tags so we'll worry about this later platform: linux/amd64 branches: main +depends_on: build # We need the entire repo in order for the release names to work skip_clone: true diff --git a/Dockerfile.ci b/Dockerfile.ci new file mode 100644 index 0000000..387d2ef --- /dev/null +++ b/Dockerfile.ci @@ -0,0 +1,20 @@ +# vim: ft=dockerfile +# This image just has the required tools to download the binaries +FROM chewingbever/vlang:latest AS builder + +ARG TARGETPLATFORM +ARG CI_COMMIT_SHA + +WORKDIR /app + +RUN curl \ + -o vieter \ + "https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \ + chmod +x vieter + + +FROM scratch + +COPY --from=builder /app/vieter /bin/vieter + +ENTRYPOINT ["/bin/vieter"] From 702fb0d13bd2eacbeaab08b4c899bdd51a1ad64c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 23 Jan 2022 10:55:03 +0100 Subject: [PATCH 082/111] Oops --- .woodpecker/.image.yml | 2 +- .woodpecker/.release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.woodpecker/.image.yml b/.woodpecker/.image.yml index dcf40f9..3d0b310 100644 --- a/.woodpecker/.image.yml +++ b/.woodpecker/.image.yml @@ -2,7 +2,7 @@ # because it has to be the target of the pull request branches: dev platform: linux/amd64 -depends_on: build +depends_on: [ build ] pipeline: dryrun: diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml index a220c75..7dfd624 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.release.yml @@ -1,7 +1,7 @@ # Yeah so this only works on tags so we'll worry about this later platform: linux/amd64 branches: main -depends_on: build +depends_on: [ build ] # We need the entire repo in order for the release names to work skip_clone: true From c3657e178eb1c8bddde17c3255182d2149a84acb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 23 Jan 2022 10:56:22 +0100 Subject: [PATCH 083/111] Oops again --- .woodpecker/.publish.yml | 1 + .woodpecker/.release.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 1e70eaf..08eadb9 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -31,3 +31,4 @@ pipeline: depends_on: - builder + - build diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml index 7dfd624..bc43b90 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.release.yml @@ -1,7 +1,6 @@ # Yeah so this only works on tags so we'll worry about this later platform: linux/amd64 branches: main -depends_on: [ build ] # We need the entire repo in order for the release names to work skip_clone: true From 015af3626cc9e89b4abc3cae3a5ab6c4e6f7083a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 23 Jan 2022 10:58:49 +0100 Subject: [PATCH 084/111] Used wrong image for upload step --- .woodpecker/.build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index b91f7cc..66caa8c 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -37,7 +37,7 @@ pipeline: event: push upload: - image: 'alpine:latest' + image: 'chewingbever/vlang:latest' secrets: [ s3_username, s3_password ] commands: # https://gist.github.com/JustinTimperio/7c7115f87b775618637d67ac911e595f From c7d1a78b84139544bf4f38b17f32ebcf55d1f804 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 23 Jan 2022 11:13:47 +0100 Subject: [PATCH 085/111] This *should* work --- .woodpecker/.publish.yml | 2 +- Dockerfile.ci | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 08eadb9..cbcc8f0 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -10,7 +10,7 @@ pipeline: repo: chewingbever/vieter tag: dev platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] - args: + args-from-env: - CI_COMMIT_SHA when: event: push diff --git a/Dockerfile.ci b/Dockerfile.ci index 387d2ef..0a3f8cd 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -7,7 +7,7 @@ ARG CI_COMMIT_SHA WORKDIR /app -RUN curl \ +RUN curl --fail \ -o vieter \ "https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \ chmod +x vieter From e79e607721d92da3e41b49a17a56b9d233f84de7 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 23 Jan 2022 11:33:07 +0100 Subject: [PATCH 086/111] Possible fix --- .woodpecker/.publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index cbcc8f0..5655314 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -10,8 +10,8 @@ pipeline: repo: chewingbever/vieter tag: dev platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] - args-from-env: - - CI_COMMIT_SHA + args: + - "CI_COMMIT_SHA=${CI_COMMIT_SHA}" when: event: push branch: dev From 434c8d62edb789221dc06d68730138c0e8420fbc Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 23 Jan 2022 13:58:29 +0100 Subject: [PATCH 087/111] A new challenger approaches --- .woodpecker/.publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 5655314..e6da2b6 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -10,8 +10,8 @@ pipeline: repo: chewingbever/vieter tag: dev platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] - args: - - "CI_COMMIT_SHA=${CI_COMMIT_SHA}" + build-args-from-env: + - CI_COMMIT_SHA when: event: push branch: dev From bca53f6ea4c815fd73e0d63c0548b15aee491a76 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 23 Jan 2022 18:43:52 +0100 Subject: [PATCH 088/111] I can be so dumb --- .woodpecker/.publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index e6da2b6..2e6b459 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -10,7 +10,7 @@ pipeline: repo: chewingbever/vieter tag: dev platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] - build-args-from-env: + build_args_from_env: - CI_COMMIT_SHA when: event: push From 4084e3c857b5976eb68658ca0d94ae6d1cbbdb93 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 27 Jan 2022 19:24:35 +0100 Subject: [PATCH 089/111] Added way to specify V path in CI --- Dockerfile.builder | 1 + Makefile | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile.builder b/Dockerfile.builder index 1f44b4c..55fa092 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -7,6 +7,7 @@ WORKDIR /opt/vlang ENV VVV /opt/vlang ENV PATH /opt/vlang:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ENV VFLAGS -cc gcc +ENV V_PATH /opt/vlang/v RUN ln -s /opt/vlang/v /usr/bin/v && \ apk --no-cache add \ diff --git a/Makefile b/Makefile index aea3f6f..4301d90 100644 --- a/Makefile +++ b/Makefile @@ -7,10 +7,11 @@ LARCHIVE_DIR := libarchive-$(LARCHIVE_VER) LARCHIVE_LIB := $(LARCHIVE_DIR)/libarchive/libarchive.so V_RELEASE := weekly.2022.04 +V_PATH ?= v-$(V_RELEASE)/v # Custom V command for linking libarchive # V := LDFLAGS=$(PWD)/$(LARCHIVE_LIB) v -cflags '-I$(PWD)/$(LARCHIVE_DIR) -I $(PWD)/$(LARCHIVE_DIR)' -V := v-$(V_RELEASE)/v -showcc +V := $(V_PATH) -showcc all: vieter From e366787e34714cdd8b629c8d95eea2d01c0a8ed5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 27 Jan 2022 19:38:36 +0100 Subject: [PATCH 090/111] I'm just trying some stuff --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4301d90..c9d0108 100644 --- a/Makefile +++ b/Makefile @@ -72,7 +72,7 @@ vet: v: v-$(V_RELEASE)/v v-$(V_RELEASE)/v: curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf - - cd patches && ./patch.sh '../v-$(V_RELEASE)' + cd patches && sh patch.sh '../v-$(V_RELEASE)' '$(MAKE)' -C 'v-$(V_RELEASE)' clean: From afb72e2df2fb30108d87fb02375f22352b02cfbe Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 27 Jan 2022 19:46:46 +0100 Subject: [PATCH 091/111] Another change to test --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c9d0108..db63021 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ v: v-$(V_RELEASE)/v v-$(V_RELEASE)/v: curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf - cd patches && sh patch.sh '../v-$(V_RELEASE)' - '$(MAKE)' -C 'v-$(V_RELEASE)' + make -C 'v-$(V_RELEASE)' clean: rm -rf '$(LARCHIVE_DIR)' 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'v-$(V_RELEASE)' From 459387ed4ceac096b42cd9f262a0bc1d463b8042 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 27 Jan 2022 19:59:13 +0100 Subject: [PATCH 092/111] Buildx really be trippin --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index db63021..f1c0cc6 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ v: v-$(V_RELEASE)/v v-$(V_RELEASE)/v: curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf - cd patches && sh patch.sh '../v-$(V_RELEASE)' - make -C 'v-$(V_RELEASE)' + cd 'v-$(V_RELEASE)' && make clean: rm -rf '$(LARCHIVE_DIR)' 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'v-$(V_RELEASE)' From 1ebba52161c1531fab5d0b2394d9c082d073c460 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 27 Jan 2022 20:05:06 +0100 Subject: [PATCH 093/111] Will it really be this? --- Dockerfile.builder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.builder b/Dockerfile.builder index 55fa092..3e40d89 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -1,4 +1,4 @@ -FROM alpine:3.15 +FROM alpine:3.12 ARG TARGETPLATFORM From d6575882cf5d35543ab5934ea7e0e776762bcd4a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 27 Jan 2022 20:18:41 +0100 Subject: [PATCH 094/111] Clone instead of curl --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f1c0cc6..b51ec35 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,8 @@ vet: .PHONY: v v: v-$(V_RELEASE)/v v-$(V_RELEASE)/v: - curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf - + git clone --depth 1 --single-branch --branch '$(V_RELEASE)' https://github.com/vlang/v 'v-$(V_RELEASE)' + # curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf - cd patches && sh patch.sh '../v-$(V_RELEASE)' cd 'v-$(V_RELEASE)' && make From 65cc4f5a50f23bd471fc040d48a0ac7b41460fc5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 27 Jan 2022 20:44:44 +0100 Subject: [PATCH 095/111] Back to curl --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b51ec35..db63021 100644 --- a/Makefile +++ b/Makefile @@ -71,10 +71,9 @@ vet: .PHONY: v v: v-$(V_RELEASE)/v v-$(V_RELEASE)/v: - git clone --depth 1 --single-branch --branch '$(V_RELEASE)' https://github.com/vlang/v 'v-$(V_RELEASE)' - # curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf - + curl -Lo - 'https://github.com/vlang/v/archive/refs/tags/$(V_RELEASE).tar.gz' | tar xzf - cd patches && sh patch.sh '../v-$(V_RELEASE)' - cd 'v-$(V_RELEASE)' && make + make -C 'v-$(V_RELEASE)' clean: rm -rf '$(LARCHIVE_DIR)' 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'v-$(V_RELEASE)' From 8e6b1e233f6dabffdfa5f7facfe4d2090ac2e375 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 27 Jan 2022 21:04:23 +0100 Subject: [PATCH 096/111] Let's test this new builder --- src/main.v | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main.v b/src/main.v index 97788e7..c3c5ed9 100644 --- a/src/main.v +++ b/src/main.v @@ -28,7 +28,6 @@ fn exit_with_message(code int, msg string) { } fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { - // Open up a file for writing to mut file := os.create(path) ? defer { file.close() From 83f1e3aa3d04f3c52133b8c06511507f994d07bb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 27 Jan 2022 21:47:25 +0100 Subject: [PATCH 097/111] Little bit of cleanup [CI SKIP] --- .woodpecker/.builder.yml | 1 - Makefile | 12 +++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.woodpecker/.builder.yml b/.woodpecker/.builder.yml index 75f5267..55d83d5 100644 --- a/.woodpecker/.builder.yml +++ b/.woodpecker/.builder.yml @@ -15,4 +15,3 @@ pipeline: path: - Dockerfile.builder - patches/* - - Makefile diff --git a/Makefile b/Makefile index db63021..c363c8a 100644 --- a/Makefile +++ b/Makefile @@ -2,15 +2,8 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' -LARCHIVE_VER := 3.5.2 -LARCHIVE_DIR := libarchive-$(LARCHIVE_VER) -LARCHIVE_LIB := $(LARCHIVE_DIR)/libarchive/libarchive.so - V_RELEASE := weekly.2022.04 V_PATH ?= v-$(V_RELEASE)/v - -# Custom V command for linking libarchive -# V := LDFLAGS=$(PWD)/$(LARCHIVE_LIB) v -cflags '-I$(PWD)/$(LARCHIVE_DIR) -I $(PWD)/$(LARCHIVE_DIR)' V := $(V_PATH) -showcc all: vieter @@ -32,6 +25,7 @@ prod: pvieter pvieter: $(SOURCES) $(V) -o pvieter -prod $(SRC_DIR) +# Only generate C code .PHONY: c c: $(V) -o vieter.c $(SRC_DIR) @@ -45,12 +39,12 @@ run: vieter .PHONY: run-prod run-prod: prod - API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG ./pvieter + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./pvieter # Same as run, but restart when the source code changes .PHONY: watch watch: - API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG $(V) watch run vieter + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG $(V) watch run vieter # =====OTHER===== From 797b63f8eaafb58a4dbbe0fc3a44dc1e6a947e00 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 27 Jan 2022 22:14:08 +0100 Subject: [PATCH 098/111] Started a changelog [CI SKIP] --- .woodpecker/.image.yml | 5 +++++ .woodpecker/.publish.yml | 7 +++++-- CHANGELOG.md | 13 +++++++++++++ README.md | 8 +++++--- 4 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 CHANGELOG.md diff --git a/.woodpecker/.image.yml b/.woodpecker/.image.yml index 3d0b310..890edd0 100644 --- a/.woodpecker/.image.yml +++ b/.woodpecker/.image.yml @@ -14,6 +14,11 @@ pipeline: tag: dev platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] dry_run: true + build_args_from_env: + - CI_COMMIT_SHA when: event: pull_request branch: dev + +depends_on: + - build diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 2e6b459..b9bf78f 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -6,8 +6,8 @@ pipeline: image: woodpeckerci/plugin-docker-buildx secrets: [ docker_username, docker_password ] settings: - dockerfile: Dockerfile.ci repo: chewingbever/vieter + dockerfile: Dockerfile.ci tag: dev platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] build_args_from_env: @@ -21,10 +21,13 @@ pipeline: secrets: [ docker_username, docker_password ] settings: repo: chewingbever/vieter + dockerfile: Dockerfile.ci tag: - latest - - $CI_COMMIT_TAG + - ${CI_COMMIT_TAG} platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + build_args_from_env: + - CI_COMMIT_SHA when: event: tag branch: main diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..017cbff --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter) + +### Added + +* Ability to publish packages +* Re-wrote repo-add in V diff --git a/README.md b/README.md index b51ab7f..fbfa259 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,11 @@ that. ### Custom Compiler Currently, this program only works with a very slightly modified version of the -V standard library, and therefore the compiler. The code for this can be found -[here](https://github.com/ChewingBever/v). For CI purposes & ease of use, you -can also clone & build that repo locally by running `make customv`. +V standard library, and therefore the compiler. The changes that are made to +the standard V release can be found in the [patches](/patches) directory. You +can obtain this modified version of the compiler by running `make v`, which +will download, patch & build the compiler. Afterwards, all make commands that +require the V compiler will use this new binary. ## Features From 4ac4e65d9dcdae7437ee40aec650cae542e117a0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 27 Jan 2022 22:34:12 +0100 Subject: [PATCH 099/111] Added AGPLv3 license [CI SKIP] --- LICENSE | 661 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 661 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0ad25db --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. From ae579f83b5ecc587e3760c0b35bebae0f438e63d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 27 Jan 2022 22:39:03 +0100 Subject: [PATCH 100/111] Added editorconfig file [CI SKIP] --- .editorconfig | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..355c6bf --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true + +[*.v] +indent_style = space From 4c56351132026bc003b3d09ee0a47f8198098375 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 30 Jan 2022 14:13:37 +0100 Subject: [PATCH 101/111] Started stress testing script [CI SKIP] --- src/archive.v | 3 ++ src/package.v | 14 +++--- src/repo.v | 2 + src/routes.v | 8 ++++ test.py | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 test.py diff --git a/src/archive.v b/src/archive.v index bdc1846..3eaad5c 100644 --- a/src/archive.v +++ b/src/archive.v @@ -12,6 +12,9 @@ fn C.archive_read_new() &C.archive // Configure the archive to work with zstd compression fn C.archive_read_support_filter_zstd(&C.archive) +// Configure the archive to work with gzip compression +fn C.archive_read_support_filter_gzip(&C.archive) + // Configure the archive to work with a tarball content fn C.archive_read_support_format_tar(&C.archive) diff --git a/src/package.v b/src/package.v index 7db5849..2d783bd 100644 --- a/src/package.v +++ b/src/package.v @@ -111,19 +111,21 @@ pub fn read_pkg(pkg_path string) ?Pkg { // Sinds 2020, all newly built Arch packages use zstd C.archive_read_support_filter_zstd(a) + C.archive_read_support_filter_gzip(a) // The content should always be a tarball C.archive_read_support_format_tar(a) // TODO find out where does this 10240 come from r = C.archive_read_open_filename(a, &char(pkg_path.str), 10240) - defer { - C.archive_read_free(a) - } if r != C.ARCHIVE_OK { return error('Failed to open package.') } + defer { + C.archive_read_free(a) + } + mut buf := voidptr(0) mut files := []string{} mut pkg_info := PkgInfo{} @@ -150,10 +152,8 @@ pub fn read_pkg(pkg_path string) ?Pkg { } C.archive_read_data(a, buf, size) - unsafe { - println(cstring_to_vstring(buf)) - } - pkg_info = parse_pkg_info_string(unsafe { cstring_to_vstring(buf) }) ? + pkg_text := unsafe { cstring_to_vstring(buf) } + pkg_info = parse_pkg_info_string(pkg_text) ? } else { C.archive_read_data_skip(a) } diff --git a/src/repo.v b/src/repo.v index cca6b84..fcf4808 100644 --- a/src/repo.v +++ b/src/repo.v @@ -96,6 +96,7 @@ fn (r &Repo) pkg_path(pkg &package.Pkg) string { // Re-generate the repo archive files fn (r &Repo) sync() ? { + // TODO also write files archive lock r.mutex { a := C.archive_write_new() entry := C.archive_entry_new() @@ -106,6 +107,7 @@ fn (r &Repo) sync() ? { C.archive_write_add_filter_gzip(a) C.archive_write_set_format_pax_restricted(a) + // TODO add symlink to .tar.gz version repo_path := os.join_path_single(r.repo_dir, 'repo.db') C.archive_write_open_filename(a, &char(repo_path.str)) diff --git a/src/routes.v b/src/routes.v index 8ad4a4a..c1e888b 100644 --- a/src/routes.v +++ b/src/routes.v @@ -76,9 +76,17 @@ fn (mut app App) put_package() web.Result { added := app.repo.add_from_path(pkg_path) or { app.lerror('Error while adding package: $err.msg') + os.rm(pkg_path) or { + app.lerror("Failed to remove download '$pkg_path'.") + } + return app.text('Failed to add package.') } if !added { + os.rm(pkg_path) or { + app.lerror("Failed to remove download '$pkg_path'.") + } + app.lwarn('Duplicate package.') return app.text('File already exists.') diff --git a/test.py b/test.py new file mode 100644 index 0000000..3a0b51c --- /dev/null +++ b/test.py @@ -0,0 +1,116 @@ +import random +import tempfile +import tarfile +from pathlib import Path +import uuid +import argparse +import asyncio +import aiohttp + + +# A list of words the program can choose from +WORDS = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", + "hotel", "india", "juliet", "kilo", "lima", "mike", "november", + "oscar", "papa", "quebec", "romeo", "sierra", "tango", "uniform", + "victor", "whiskey", "xray", "yankee", "zulu"] +SEED = 2022 + + +def random_words(max_len=None, min_len=1): + """ + Returns a random list of words, with a length randomly choosen between + min_len and max_len. If max_len is None, it is equal to the length of + WORDS. + """ + if max_len is None: + max_len = len(WORDS) + + k = random.randint(min_len, max_len) + + return random.choices(WORDS, k=k) + +def random_lists(n, max_len=None, min_len=1): + return [random_words(max_len, min_len) for _ in range(n)] + +def create_random_pkginfo(): + """ + Generates a random .PKGINFO + """ + name = "-".join(random_words(3)) + ver = "0.1.0" # doesn't matter what it is anyways + + # TODO add random dependencies (all types) + + data = { + "pkgname": name, + "pkgbase": name, + "pkgver": ver, + "arch": "x86_64" + } + + return "\n".join(f"{key} = {value}" for key, value in data.items()) + +def create_random_package(tmpdir): + """ + Creates a random, but valid Arch package, using the provided tmpdir. Output + is the path to the created package tarball. + """ + + sub_path = tmpdir / uuid.uuid4().hex + sub_path.mkdir() + + tar_path = sub_path / "archive.pkg.tar.gz" + + def remove_prefix(tar_info): + tar_info.name = tar_info.name[len(str(sub_path)):] + + return tar_info + + with tarfile.open(tar_path, "w") as tar: + # Add random .PKGINFO file + pkginfo_file = sub_path / ".PKGINFO" + pkginfo_file.write_text(create_random_pkginfo()) + tar.add(pkginfo_file, filter=remove_prefix) + + # Create random files + for words in random_lists(10, max_len=5): + path = sub_path / 'usr' / ('/'.join(words) + ".txt") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(' '.join(words)) + + tar.add(path, filter=remove_prefix) + + return tar_path + + +async def upload_random_package(dir, sem): + tar_path = create_random_package(dir) + + async with sem: + with open(tar_path, 'rb') as f: + async with aiohttp.ClientSession() as s: + async with s.post("http://localhost:8000/publish", data=f.read(), headers={"x-api-key": "test"}) as r: + print(r.text) + + +async def main(): + parser = argparse.ArgumentParser(description="Test vieter by uploading random package files.") + + parser.add_argument("count", help="How many packages to upload.", default=1, type=int) + parser.add_argument("-p", "--parallel", help="How many uploads to run in parallel.", default=1, type=int) + parser.add_argument("-s", "--seed", help="Seed for the randomizer.", default=SEED, type=int) + # parser.add_argument("-d", "--dir", help="Directory to create ") + + args = parser.parse_args() + + sem = asyncio.BoundedSemaphore(args.parallel) + random.seed(args.seed) + + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdir = Path(tmpdirname) + + await asyncio.gather(*(upload_random_package(tmpdir, sem) for _ in range(args.count))) + + +if __name__ == "__main__": + asyncio.run(main()) From 766e097be8d6765783de996952ac011217da5ca8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 30 Jan 2022 18:02:45 +0100 Subject: [PATCH 102/111] Overcomplicated the test script because of course [CI SKIP] --- src/package.v | 3 +-- test.py | 59 ++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/package.v b/src/package.v index 2d783bd..91e8db3 100644 --- a/src/package.v +++ b/src/package.v @@ -107,7 +107,6 @@ pub fn read_pkg(pkg_path string) ?Pkg { a := C.archive_read_new() entry := C.archive_entry_new() - mut r := 0 // Sinds 2020, all newly built Arch packages use zstd C.archive_read_support_filter_zstd(a) @@ -116,7 +115,7 @@ pub fn read_pkg(pkg_path string) ?Pkg { C.archive_read_support_format_tar(a) // TODO find out where does this 10240 come from - r = C.archive_read_open_filename(a, &char(pkg_path.str), 10240) + r := C.archive_read_open_filename(a, &char(pkg_path.str), 10240) if r != C.ARCHIVE_OK { return error('Failed to open package.') diff --git a/test.py b/test.py index 3a0b51c..17445b7 100644 --- a/test.py +++ b/test.py @@ -6,6 +6,7 @@ import uuid import argparse import asyncio import aiohttp +import sys # A list of words the program can choose from @@ -16,27 +17,27 @@ WORDS = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", SEED = 2022 -def random_words(max_len=None, min_len=1): +def random_words(words, min_len, max_len=None): """ Returns a random list of words, with a length randomly choosen between min_len and max_len. If max_len is None, it is equal to the length of - WORDS. + words. """ if max_len is None: - max_len = len(WORDS) + max_len = len(words) k = random.randint(min_len, max_len) - return random.choices(WORDS, k=k) + return random.choices(words, k=k) -def random_lists(n, max_len=None, min_len=1): - return [random_words(max_len, min_len) for _ in range(n)] +def random_lists(words, n, min_len, max_len=None): + return [random_words(words, min_len, max_len) for _ in range(n)] -def create_random_pkginfo(): +def create_random_pkginfo(words, name_min_len, name_max_len): """ Generates a random .PKGINFO """ - name = "-".join(random_words(3)) + name = "-".join(random_words(words, name_min_len, name_max_len)) ver = "0.1.0" # doesn't matter what it is anyways # TODO add random dependencies (all types) @@ -50,7 +51,7 @@ def create_random_pkginfo(): return "\n".join(f"{key} = {value}" for key, value in data.items()) -def create_random_package(tmpdir): +def create_random_package(tmpdir, words, pkg_name_min_len, pkg_name_max_len, min_files, max_files, min_filename_len, max_filename_len): """ Creates a random, but valid Arch package, using the provided tmpdir. Output is the path to the created package tarball. @@ -69,11 +70,13 @@ def create_random_package(tmpdir): with tarfile.open(tar_path, "w") as tar: # Add random .PKGINFO file pkginfo_file = sub_path / ".PKGINFO" - pkginfo_file.write_text(create_random_pkginfo()) + pkginfo_file.write_text(create_random_pkginfo(words, pkg_name_min_len, pkg_name_max_len)) tar.add(pkginfo_file, filter=remove_prefix) # Create random files - for words in random_lists(10, max_len=5): + file_count = random.randint(min_files, max_files) + + for words in random_lists(words, file_count, min_filename_len, max_filename_len): path = sub_path / 'usr' / ('/'.join(words) + ".txt") path.parent.mkdir(parents=True, exist_ok=True) path.write_text(' '.join(words)) @@ -83,14 +86,19 @@ def create_random_package(tmpdir): return tar_path -async def upload_random_package(dir, sem): - tar_path = create_random_package(dir) +async def check_output(r): + good = {"File already exists.", "Package added successfully."} + txt = await r.text() + return (txt in good, txt) + + +async def upload_random_package(tar_path, sem): async with sem: with open(tar_path, 'rb') as f: async with aiohttp.ClientSession() as s: async with s.post("http://localhost:8000/publish", data=f.read(), headers={"x-api-key": "test"}) as r: - print(r.text) + return await check_output(r) async def main(): @@ -99,6 +107,14 @@ async def main(): parser.add_argument("count", help="How many packages to upload.", default=1, type=int) parser.add_argument("-p", "--parallel", help="How many uploads to run in parallel.", default=1, type=int) parser.add_argument("-s", "--seed", help="Seed for the randomizer.", default=SEED, type=int) + parser.add_argument("--min-files", help="Minimum amount of files to add to an archive.", default=5, type=int) + parser.add_argument("--max-files", help="Max amount of files to add to an archive.", default=10, type=int) + parser.add_argument("--min-filename-length", help="Minimum amount of words to use for generating filenames.", default=1, type=int) + parser.add_argument("--max-filename-length", help="Max amount of words to use for generating filenames.", default=5, type=int) + parser.add_argument("--min-pkg-name-length", help="Minimum amount of words to use for creating package name.", default=1, type=int) + parser.add_argument("--max-pkg-name-length", help="Max amount of words to use for creating package name.", default=3, type=int) + parser.add_argument("--words", help="Words to use for randomizing.", default=WORDS, type=lambda s: s.split(',')) + # parser.add_argument("--words", help="Words to use for randomizing.", default=WORDS, type=) # parser.add_argument("-d", "--dir", help="Directory to create ") args = parser.parse_args() @@ -106,10 +122,23 @@ async def main(): sem = asyncio.BoundedSemaphore(args.parallel) random.seed(args.seed) + with tempfile.TemporaryDirectory() as tmpdirname: tmpdir = Path(tmpdirname) - await asyncio.gather(*(upload_random_package(tmpdir, sem) for _ in range(args.count))) + # We generate the tars in advance because they're not async anyways + print("Generating tarballs...") + tars = { + create_random_package(tmpdir, args.words, args.min_pkg_name_length, args.max_pkg_name_length, args.min_files, args.max_files, args.min_filename_length, args.max_filename_length) + for _ in range(args.count) + } + + print("Sending requests...") + res = await asyncio.gather(*(upload_random_package(tar, sem) for tar in tars)) + + # Generate status report + if any(not x[0] for x in res): + sys.exit(1) if __name__ == "__main__": From b0212b81620f8c43d691e5d50d2c4f180a20d8c9 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 30 Jan 2022 21:03:32 +0100 Subject: [PATCH 103/111] Fixed the memory bug! thanks spy --- src/package.v | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/package.v b/src/package.v index 91e8db3..a4d1d6c 100644 --- a/src/package.v +++ b/src/package.v @@ -125,7 +125,6 @@ pub fn read_pkg(pkg_path string) ?Pkg { C.archive_read_free(a) } - mut buf := voidptr(0) mut files := []string{} mut pkg_info := PkgInfo{} @@ -143,7 +142,7 @@ pub fn read_pkg(pkg_path string) ?Pkg { size := C.archive_entry_size(entry) // TODO can this unsafe block be avoided? - buf = unsafe { malloc(size) } + buf := unsafe { malloc(size) } defer { unsafe { free(buf) @@ -151,7 +150,8 @@ pub fn read_pkg(pkg_path string) ?Pkg { } C.archive_read_data(a, buf, size) - pkg_text := unsafe { cstring_to_vstring(buf) } + pkg_text := unsafe { buf.vstring_with_len(size).clone() } + pkg_info = parse_pkg_info_string(pkg_text) ? } else { C.archive_read_data_skip(a) From 2e344eecc7e86872f3f1de2a87f3195147f13504 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 30 Jan 2022 21:07:37 +0100 Subject: [PATCH 104/111] Fixed some faulty ci settings --- .woodpecker/.image.yml | 3 --- .woodpecker/.publish.yml | 7 +++---- .woodpecker/.release.yml | 5 ++--- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.woodpecker/.image.yml b/.woodpecker/.image.yml index 890edd0..27072a0 100644 --- a/.woodpecker/.image.yml +++ b/.woodpecker/.image.yml @@ -19,6 +19,3 @@ pipeline: when: event: pull_request branch: dev - -depends_on: - - build diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index b9bf78f..22183c5 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -1,5 +1,8 @@ branches: [main, dev] platform: linux/amd64 +depends_on: + - builder + - build pipeline: dev: @@ -31,7 +34,3 @@ pipeline: when: event: tag branch: main - -depends_on: - - builder - - build diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml index bc43b90..c7527cb 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.release.yml @@ -1,6 +1,8 @@ # Yeah so this only works on tags so we'll worry about this later platform: linux/amd64 branches: main +depends_on: + - build # We need the entire repo in order for the release names to work skip_clone: true @@ -34,6 +36,3 @@ pipeline: title: title when: event: push - -depends_on: - - build From e26e2746de9b2c9f7de95fcb4d23e49d253482ed Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 30 Jan 2022 23:54:05 +0100 Subject: [PATCH 105/111] Added creation of files archive --- src/package.v | 6 ++-- src/{ => repo}/repo.v | 52 ---------------------------- src/repo/sync.v | 80 +++++++++++++++++++++++++++++++++++++++++++ src/routes.v | 8 ++--- 4 files changed, 85 insertions(+), 61 deletions(-) rename src/{ => repo}/repo.v (63%) create mode 100644 src/repo/sync.v diff --git a/src/package.v b/src/package.v index a4d1d6c..da3988f 100644 --- a/src/package.v +++ b/src/package.v @@ -115,7 +115,7 @@ pub fn read_pkg(pkg_path string) ?Pkg { C.archive_read_support_format_tar(a) // TODO find out where does this 10240 come from - r := C.archive_read_open_filename(a, &char(pkg_path.str), 10240) + r := C.archive_read_open_filename(a, &char(pkg_path.str), 10240) if r != C.ARCHIVE_OK { return error('Failed to open package.') @@ -142,7 +142,7 @@ pub fn read_pkg(pkg_path string) ?Pkg { size := C.archive_entry_size(entry) // TODO can this unsafe block be avoided? - buf := unsafe { malloc(size) } + buf := unsafe { malloc(size) } defer { unsafe { free(buf) @@ -150,7 +150,7 @@ pub fn read_pkg(pkg_path string) ?Pkg { } C.archive_read_data(a, buf, size) - pkg_text := unsafe { buf.vstring_with_len(size).clone() } + pkg_text := unsafe { buf.vstring_with_len(size).clone() } pkg_info = parse_pkg_info_string(pkg_text) ? } else { diff --git a/src/repo.v b/src/repo/repo.v similarity index 63% rename from src/repo.v rename to src/repo/repo.v index fcf4808..ded30ba 100644 --- a/src/repo.v +++ b/src/repo/repo.v @@ -93,55 +93,3 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool { fn (r &Repo) pkg_path(pkg &package.Pkg) string { return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version') } - -// Re-generate the repo archive files -fn (r &Repo) sync() ? { - // TODO also write files archive - lock r.mutex { - a := C.archive_write_new() - entry := C.archive_entry_new() - st := C.stat{} - buf := [8192]byte{} - - // This makes the archive a gzip-compressed tarball - C.archive_write_add_filter_gzip(a) - C.archive_write_set_format_pax_restricted(a) - - // TODO add symlink to .tar.gz version - repo_path := os.join_path_single(r.repo_dir, 'repo.db') - - C.archive_write_open_filename(a, &char(repo_path.str)) - - // Iterate over each directory - for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, - it))) { - inner_path := os.join_path_single(d, 'desc') - actual_path := os.join_path_single(r.repo_dir, inner_path) - - unsafe { - C.stat(&char(actual_path.str), &st) - } - - C.archive_entry_set_pathname(entry, &char(inner_path.str)) - C.archive_entry_copy_stat(entry, &st) - // C.archive_entry_set_size(entry, st.st_size) - // C.archive_entry_set_filetype(entry, C.AE_IFREG) - // C.archive_entry_set_perm(entry, 0o644) - C.archive_write_header(a, entry) - - fd := C.open(&char(actual_path.str), C.O_RDONLY) - mut len := C.read(fd, &buf, sizeof(buf)) - - for len > 0 { - C.archive_write_data(a, &buf, len) - len = C.read(fd, &buf, sizeof(buf)) - } - C.close(fd) - - C.archive_entry_clear(entry) - } - - C.archive_write_close(a) - C.archive_write_free(a) - } -} diff --git a/src/repo/sync.v b/src/repo/sync.v new file mode 100644 index 0000000..0bb5545 --- /dev/null +++ b/src/repo/sync.v @@ -0,0 +1,80 @@ +module repo + +import os + +fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &string, inner_path &string) { + st := C.stat{} + + unsafe { + C.stat(&char(file_path.str), &st) + } + + C.archive_entry_set_pathname(entry, &char(inner_path.str)) + C.archive_entry_copy_stat(entry, &st) + C.archive_write_header(archive, entry) + + mut fd := C.open(&char(file_path.str), C.O_RDONLY) + defer { + C.close(fd) + } + + // Write the file to the archive + buf := [8192]byte{} + mut len := C.read(fd, &buf, sizeof(buf)) + + for len > 0 { + C.archive_write_data(archive, &buf, len) + + len = C.read(fd, &buf, sizeof(buf)) + } +} + +// Re-generate the repo archive files +fn (r &Repo) sync() ? { + // TODO also write files archive + lock r.mutex { + a_db := C.archive_write_new() + a_files := C.archive_write_new() + + entry := C.archive_entry_new() + + // This makes the archive a gzip-compressed tarball + C.archive_write_add_filter_gzip(a_db) + C.archive_write_set_format_pax_restricted(a_db) + C.archive_write_add_filter_gzip(a_files) + C.archive_write_set_format_pax_restricted(a_files) + + // TODO add symlink to .tar.gz version + db_path := os.join_path_single(r.repo_dir, 'repo.db') + files_path := os.join_path_single(r.repo_dir, 'repo.files') + + C.archive_write_open_filename(a_db, &char(db_path.str)) + C.archive_write_open_filename(a_files, &char(files_path.str)) + + // Iterate over each directory + for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, + it))) { + // desc + mut inner_path := os.join_path_single(d, 'desc') + mut actual_path := os.join_path_single(r.repo_dir, inner_path) + + archive_add_entry(a_db, entry, actual_path, inner_path) + archive_add_entry(a_files, entry, actual_path, inner_path) + + C.archive_entry_clear(entry) + + // files + inner_path = os.join_path_single(d, 'files') + actual_path = os.join_path_single(r.repo_dir, inner_path) + + archive_add_entry(a_files, entry, actual_path, inner_path) + + C.archive_entry_clear(entry) + } + + C.archive_write_close(a_db) + C.archive_write_free(a_db) + C.archive_write_close(a_files) + C.archive_write_free(a_files) + } +} diff --git a/src/routes.v b/src/routes.v index c1e888b..6218898 100644 --- a/src/routes.v +++ b/src/routes.v @@ -76,16 +76,12 @@ fn (mut app App) put_package() web.Result { added := app.repo.add_from_path(pkg_path) or { app.lerror('Error while adding package: $err.msg') - os.rm(pkg_path) or { - app.lerror("Failed to remove download '$pkg_path'.") - } + os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path'.") } return app.text('Failed to add package.') } if !added { - os.rm(pkg_path) or { - app.lerror("Failed to remove download '$pkg_path'.") - } + os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path'.") } app.lwarn('Duplicate package.') From 21c8e28e64483c29748a36a72f6748cc3f085f3d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 31 Jan 2022 11:11:45 +0100 Subject: [PATCH 106/111] Added healthcheck route; upgraded Docker image --- Dockerfile.ci | 32 +++++++++++++++++++++++++++++--- src/routes.v | 5 +++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 0a3f8cd..24f2bef 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -4,17 +4,43 @@ FROM chewingbever/vlang:latest AS builder ARG TARGETPLATFORM ARG CI_COMMIT_SHA +ARG DI_VER=1.2.5 WORKDIR /app +# Build dumb-init +RUN curl -Lo - "https://github.com/Yelp/dumb-init/archive/refs/tags/v${DI_VER}.tar.gz" | tar -xzf - && \ + cd "dumb-init-${DI_VER}" && \ + make SHELL=/bin/sh && \ + mv dumb-init .. && \ + cd .. + RUN curl --fail \ -o vieter \ "https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \ chmod +x vieter -FROM scratch +FROM busybox:1.35.0 -COPY --from=builder /app/vieter /bin/vieter +ENV PATH=/bin \ + REPO_DIR=/data/repo \ + PKG_DIR=/data/pkgs \ + DOWNLOAD_DIR=/data/downloads -ENTRYPOINT ["/bin/vieter"] +COPY --from=builder /app/dumb-init /app/vieter /bin/ + +HEALTHCHECK --interval=30s \ + --timeout=3s \ + --start-period=5s \ + CMD /bin/wget --spider http://localhost:8000/health || exit 1 + +RUN mkdir /data && \ + chown -R www-data:www-data /data + +WORKDIR /data + +USER www-data:www-data + +ENTRYPOINT ["/bin/dumb-init", "--"] +CMD ["/bin/vieter"] diff --git a/src/routes.v b/src/routes.v index 6218898..5dfd34e 100644 --- a/src/routes.v +++ b/src/routes.v @@ -25,6 +25,11 @@ fn is_pkg_name(s string) bool { return s.contains('.pkg') } +['/health'; get] +pub fn (mut app App) healthcheck() web.Result { + return app.text('Healthy') +} + // get_root handles a GET request for a file on the root ['/:filename'; get] fn (mut app App) get_root(filename string) web.Result { From cb492cadc6ba24321ecc7153d21e00c7c5838057 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 31 Jan 2022 14:19:37 +0100 Subject: [PATCH 107/111] Added builder patch for public sha256 functions --- patches/patch.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/patches/patch.sh b/patches/patch.sh index a7378e7..9813976 100755 --- a/patches/patch.sh +++ b/patches/patch.sh @@ -5,3 +5,9 @@ # Add parse_request_no_body cat parse_request_no_body.v >> "$1"/vlib/net/http/request.v + +# Make sha256 functions public +sed -i \ + -e 's/\(fn (mut d Digest) checksum(\)/pub \1/' \ + -e 's/\(fn (mut d Digest) write(\)/pub \1/' \ + "$1"/vlib/crypto/sha256/sha256.v From 3c0b7156c12f8d31f541e9f669ae10fc167da42c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 31 Jan 2022 14:43:57 +0100 Subject: [PATCH 108/111] Sha256sum gets calculated as well now (closes #44) --- src/package.v | 6 ++---- src/util.v | 13 +++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/package.v b/src/package.v index da3988f..b9bcae8 100644 --- a/src/package.v +++ b/src/package.v @@ -201,12 +201,10 @@ pub fn (pkg &Pkg) to_desc() string { desc += format_entry('CSIZE', p.csize.str()) desc += format_entry('ISIZE', p.size.str()) - md5sum, _ := pkg.checksum() or { '', '' } + md5sum, sha256sum := pkg.checksum() or { '', '' } desc += format_entry('MD5SUM', md5sum) - - // TODO add this - // desc += format_entry('SHA256SUM', sha256sum) + desc += format_entry('SHA256SUM', sha256sum) // TODO add pgpsig stuff diff --git a/src/util.v b/src/util.v index 066a58d..f81a256 100644 --- a/src/util.v +++ b/src/util.v @@ -2,7 +2,7 @@ module util import os import crypto.md5 -// import crypto.sha256 +import crypto.sha256 // hash_file returns the md5 & sha256 hash of a given file // TODO actually implement sha256 @@ -10,7 +10,7 @@ pub fn hash_file(path &string) ?(string, string) { file := os.open(path) or { return error('Failed to open file.') } mut md5sum := md5.new() - // mut sha256sum := sha256.new() + mut sha256sum := sha256.new() buf_size := int(1_000_000) mut buf := []byte{len: buf_size} @@ -23,11 +23,12 @@ pub fn hash_file(path &string) ?(string, string) { // For now we'll assume that this always works md5sum.write(buf[..bytes_read]) or { - return error('Failed to update checksum. This should never happen.') + return error('Failed to update md5 checksum. This should never happen.') + } + sha256sum.write(buf[..bytes_read]) or { + return error('Failed to update sha256 checksum. This should never happen.') } - // sha256sum.write(buf) or {} } - // return md5sum.sum(buf).hex(), sha256sum.sum(buf).hex() - return md5sum.checksum().hex(), '' + return md5sum.checksum().hex(), sha256sum.checksum().hex() } From 8a0beffbe2b37d18b6078ce8a38c64fabbbc4f58 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 31 Jan 2022 21:14:43 +0100 Subject: [PATCH 109/111] Bumped v release; files are now served properly --- .woodpecker/.builder.yml | 1 + Makefile | 4 ++-- src/repo/sync.v | 5 ++--- src/routes.v | 10 ++++++---- src/web/web.v | 1 - 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.woodpecker/.builder.yml b/.woodpecker/.builder.yml index 55d83d5..e94a846 100644 --- a/.woodpecker/.builder.yml +++ b/.woodpecker/.builder.yml @@ -13,5 +13,6 @@ pipeline: when: event: push path: + - Makefile - Dockerfile.builder - patches/* diff --git a/Makefile b/Makefile index c363c8a..8674ca0 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' -V_RELEASE := weekly.2022.04 +V_RELEASE := weekly.2022.05 V_PATH ?= v-$(V_RELEASE)/v V := $(V_PATH) -showcc @@ -70,4 +70,4 @@ v-$(V_RELEASE)/v: make -C 'v-$(V_RELEASE)' clean: - rm -rf '$(LARCHIVE_DIR)' 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'v-$(V_RELEASE)' + rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'v-$(V_RELEASE)' diff --git a/src/repo/sync.v b/src/repo/sync.v index 0bb5545..d6080e0 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -44,9 +44,8 @@ fn (r &Repo) sync() ? { C.archive_write_add_filter_gzip(a_files) C.archive_write_set_format_pax_restricted(a_files) - // TODO add symlink to .tar.gz version - db_path := os.join_path_single(r.repo_dir, 'repo.db') - files_path := os.join_path_single(r.repo_dir, 'repo.files') + db_path := os.join_path_single(r.repo_dir, 'vieter.db.tar.gz') + files_path := os.join_path_single(r.repo_dir, 'vieter.files.tar.gz') C.archive_write_open_filename(a_db, &char(db_path.str)) C.archive_write_open_filename(a_files, &char(files_path.str)) diff --git a/src/routes.v b/src/routes.v index 5dfd34e..45e7fa5 100644 --- a/src/routes.v +++ b/src/routes.v @@ -27,7 +27,7 @@ fn is_pkg_name(s string) bool { ['/health'; get] pub fn (mut app App) healthcheck() web.Result { - return app.text('Healthy') + return app.text('Healthy') } // get_root handles a GET request for a file on the root @@ -35,10 +35,12 @@ pub fn (mut app App) healthcheck() web.Result { fn (mut app App) get_root(filename string) web.Result { mut full_path := '' - if is_pkg_name(filename) { - full_path = os.join_path_single(app.repo.pkg_dir, filename) + if filename.ends_with('.db') || filename.ends_with('.files') { + full_path = os.join_path_single(app.repo.repo_dir, '${filename}.tar.gz') + } else if filename.ends_with('.db.tar.gz') || filename.ends_with('.files.tar.gz') { + full_path = os.join_path_single(app.repo.repo_dir, '$filename') } else { - full_path = os.join_path_single(app.repo.repo_dir, filename) + full_path = os.join_path_single(app.repo.pkg_dir, filename) } return app.file(full_path) diff --git a/src/web/web.v b/src/web/web.v index 2db8f18..bbb909f 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -262,7 +262,6 @@ pub fn (mut ctx Context) file(f_path string) Result { if ctx.done { return Result{} } - ctx.done = true if !os.is_file(f_path) { return ctx.not_found() From 33e695bdf30a2e1fd77a6c1226d01cf7a81e71f5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 31 Jan 2022 22:59:12 +0100 Subject: [PATCH 110/111] Filenames now use the correct compression scheme (closes #56) --- src/archive.v | 3 +++ src/package.v | 22 ++++++++++++++++++---- test.py | 2 +- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/archive.v b/src/archive.v index 3eaad5c..8d1314f 100644 --- a/src/archive.v +++ b/src/archive.v @@ -63,6 +63,9 @@ fn C.archive_write_close(&C.archive) // Free the write archive fn C.archive_write_free(&C.archive) +// Returns the name of the filter +fn C.archive_filter_code(&C.archive, int) int + #include "archive_entry.h" struct C.archive_entry {} diff --git a/src/package.v b/src/package.v index b9bcae8..03103c9 100644 --- a/src/package.v +++ b/src/package.v @@ -6,9 +6,10 @@ import util // Represents a read archive struct Pkg { pub: - path string [required] - info PkgInfo [required] - files []string [required] + path string [required] + info PkgInfo [required] + files []string [required] + compression int [required] } // Represents the contents of a .PKGINFO file @@ -125,6 +126,11 @@ pub fn read_pkg(pkg_path string) ?Pkg { C.archive_read_free(a) } + // 0: no compression (just a tarball) + // 1: gzip + // 14: zstd + compression_code := C.archive_filter_code(a, 0) + mut files := []string{} mut pkg_info := PkgInfo{} @@ -164,6 +170,7 @@ pub fn read_pkg(pkg_path string) ?Pkg { path: pkg_path info: pkg_info files: files + compression: compression_code } } @@ -175,7 +182,14 @@ fn format_entry(key string, value string) string { pub fn (pkg &Pkg) filename() string { p := pkg.info - return '$p.name-$p.version-${p.arch}.pkg.tar.zst' + ext := match pkg.compression { + 0 { '.tar' } + 1 { '.tar.gz' } + 14 { '.tar.zst' } + else { panic("Another compression code shouldn't be possible. Faulty code: $pkg.compression") } + } + + return '$p.name-$p.version-${p.arch}.pkg$ext' } // to_desc returns a desc file valid string representation diff --git a/test.py b/test.py index 17445b7..2d1e4f5 100644 --- a/test.py +++ b/test.py @@ -67,7 +67,7 @@ def create_random_package(tmpdir, words, pkg_name_min_len, pkg_name_max_len, min return tar_info - with tarfile.open(tar_path, "w") as tar: + with tarfile.open(tar_path, "w:gz") as tar: # Add random .PKGINFO file pkginfo_file = sub_path / ".PKGINFO" pkginfo_file.write_text(create_random_pkginfo(words, pkg_name_min_len, pkg_name_max_len)) From aa5ee55059c87f9814b7bf22e23e2cb9781f4644 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 31 Jan 2022 23:39:04 +0100 Subject: [PATCH 111/111] Tweaked CI a bit --- .woodpecker/{.publish.yml => .docker.yml} | 4 +--- .woodpecker/{.release.yml => .gitea.yml} | 12 ++++-------- .woodpecker/.image.yml | 21 --------------------- test.py | 2 +- 4 files changed, 6 insertions(+), 33 deletions(-) rename .woodpecker/{.publish.yml => .docker.yml} (93%) rename .woodpecker/{.release.yml => .gitea.yml} (67%) delete mode 100644 .woodpecker/.image.yml diff --git a/.woodpecker/.publish.yml b/.woodpecker/.docker.yml similarity index 93% rename from .woodpecker/.publish.yml rename to .woodpecker/.docker.yml index 22183c5..f23b3df 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.docker.yml @@ -25,9 +25,7 @@ pipeline: settings: repo: chewingbever/vieter dockerfile: Dockerfile.ci - tag: - - latest - - ${CI_COMMIT_TAG} + auto_tag: true platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] build_args_from_env: - CI_COMMIT_SHA diff --git a/.woodpecker/.release.yml b/.woodpecker/.gitea.yml similarity index 67% rename from .woodpecker/.release.yml rename to .woodpecker/.gitea.yml index c7527cb..e6062eb 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.gitea.yml @@ -13,13 +13,10 @@ pipeline: pull: true secrets: [ s3_username, s3_password ] commands: - - git clone "$CI_REPO_REMOTE" . - - git checkout "$CI_COMMIT_BRANCH" - # Write the title to a file that the plugin can then read - - echo "$(git describe --tags --abbrev=0 2> /dev/null || echo '0.0.0')-$(git rev-list --count ^dev)" > title - - cat title - mc alias set s3/ https://s3.rustybever.be "$S3_USERNAME" "$S3_PASSWORD" - mc cp -r "s3/vieter/commits/$CI_COMMIT_SHA" assets + when: + event: tag release: image: 'plugins/gitea-release' @@ -32,7 +29,6 @@ pipeline: - md5 - sha256 prerelease: true - # This should get read in as a file - title: title + title: ${CI_COMMIT_TAG} when: - event: push + event: tag diff --git a/.woodpecker/.image.yml b/.woodpecker/.image.yml deleted file mode 100644 index 27072a0..0000000 --- a/.woodpecker/.image.yml +++ /dev/null @@ -1,21 +0,0 @@ -# Because the only step here is a pull_request event, the branch should be dev -# because it has to be the target of the pull request -branches: dev -platform: linux/amd64 -depends_on: [ build ] - -pipeline: - dryrun: - image: woodpeckerci/plugin-docker-buildx - secrets: [ docker_username, docker_password ] - settings: - dockerfile: Dockerfile.ci - repo: chewingbever/vieter - tag: dev - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] - dry_run: true - build_args_from_env: - - CI_COMMIT_SHA - when: - event: pull_request - branch: dev diff --git a/test.py b/test.py index 2d1e4f5..5721310 100644 --- a/test.py +++ b/test.py @@ -38,7 +38,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len): Generates a random .PKGINFO """ name = "-".join(random_words(words, name_min_len, name_max_len)) - ver = "0.1.0" # doesn't matter what it is anyways + ver = "0.1.0-1" # doesn't matter what it is anyways # TODO add random dependencies (all types)