From e13081a4aa6f71e7d133d446db1af00204aaa534 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 21:50:43 +0100 Subject: [PATCH 01/24] Only support zstd-compressed tarballs --- .woodpecker/.build.yml | 1 + .woodpecker/.build_arm64.yml | 1 + src/archive/archive.v | 6 ++++-- src/archive/bindings.v | 6 ++++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 83dcb0e5..5ece4617 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -25,4 +25,5 @@ pipeline: - make prod # Make sure the binary is actually static - readelf -d pvieter + - du -h pvieter - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' diff --git a/.woodpecker/.build_arm64.yml b/.woodpecker/.build_arm64.yml index 704bc483..3b919b40 100644 --- a/.woodpecker/.build_arm64.yml +++ b/.woodpecker/.build_arm64.yml @@ -25,4 +25,5 @@ pipeline: - make prod # Make sure the binary is actually static - readelf -d pvieter + - du -h pvieter - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' diff --git a/src/archive/archive.v b/src/archive/archive.v index 88649eea..c280a422 100644 --- a/src/archive/archive.v +++ b/src/archive/archive.v @@ -12,8 +12,10 @@ pub fn pkg_info(pkg_path string) ?(string, []string) { entry := C.archive_entry_new() mut r := 0 - C.archive_read_support_filter_all(a) - C.archive_read_support_format_all(a) + // Sinds 2020, all newly built Arch packages use zstd + C.archive_read_support_filter_zstd(a) + // The content should always be a tarball + C.archive_read_support_format_tar(a) // TODO find out where does this 10240 come from r = C.archive_read_open_filename(a, &char(pkg_path.str), 10240) diff --git a/src/archive/bindings.v b/src/archive/bindings.v index 678d715f..f507facf 100644 --- a/src/archive/bindings.v +++ b/src/archive/bindings.v @@ -8,8 +8,10 @@ struct C.archive {} // Create a new archive struct fn C.archive_read_new() &C.archive -fn C.archive_read_support_filter_all(&C.archive) -fn C.archive_read_support_format_all(&C.archive) +// Configure the archive to work with zstd compression +fn C.archive_read_support_filter_zstd(&C.archive) +// Configure the archive to work with a tarball content +fn C.archive_read_support_format_tar(&C.archive) // Open an archive for reading fn C.archive_read_open_filename(&C.archive, &char, int) int From d538e25da3ff17b56707ea35efc6b685289683d5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 21:52:40 +0100 Subject: [PATCH 02/24] CI? You awake? --- src/archive/bindings.v | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/archive/bindings.v b/src/archive/bindings.v index f507facf..26af13ff 100644 --- a/src/archive/bindings.v +++ b/src/archive/bindings.v @@ -8,9 +8,11 @@ struct C.archive {} // Create a new archive struct fn C.archive_read_new() &C.archive + // Configure the archive to work with zstd compression fn C.archive_read_support_filter_zstd(&C.archive) -// Configure the archive to work with a tarball content + +// Configure the archive to work with a tarball content fn C.archive_read_support_format_tar(&C.archive) // Open an archive for reading From 1f8f16fe8dd7e33da2d6c35aaeebb86003b6d96f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 22:20:58 +0100 Subject: [PATCH 03/24] Moved archive read code into pkg.v --- src/archive/archive.v | 55 --------------------- src/archive/bindings.v | 50 -------------------- src/pkg.v | 105 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 102 insertions(+), 108 deletions(-) delete mode 100644 src/archive/archive.v delete mode 100644 src/archive/bindings.v diff --git a/src/archive/archive.v b/src/archive/archive.v deleted file mode 100644 index c280a422..00000000 --- a/src/archive/archive.v +++ /dev/null @@ -1,55 +0,0 @@ -module archive - -import os - -// Returns the .PKGINFO file's contents & the list of files. -pub fn pkg_info(pkg_path string) ?(string, []string) { - if !os.is_file(pkg_path) { - return error("'$pkg_path' doesn't exist or isn't a file.") - } - - a := C.archive_read_new() - entry := C.archive_entry_new() - mut r := 0 - - // Sinds 2020, all newly built Arch packages use zstd - C.archive_read_support_filter_zstd(a) - // The content should always be a tarball - C.archive_read_support_format_tar(a) - - // TODO find out where does this 10240 come from - r = C.archive_read_open_filename(a, &char(pkg_path.str), 10240) - defer { - C.archive_read_free(a) - } - - if r != C.ARCHIVE_OK { - return error('Failed to open package.') - } - - // We iterate over every header in search of the .PKGINFO one - mut buf := voidptr(0) - mut files := []string{} - for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { - pathname := C.archive_entry_pathname(entry) - - ignored_names := [c'.BUILDINFO', c'.INSTALL', c'.MTREE', c'.PKGINFO', c'.CHANGELOG'] - if ignored_names.all(C.strcmp(it, pathname) != 0) { - unsafe { - files << cstring_to_vstring(pathname) - } - } - - if C.strcmp(pathname, c'.PKGINFO') == 0 { - size := C.archive_entry_size(entry) - - // TODO can this unsafe block be avoided? - buf = unsafe { malloc(size) } - C.archive_read_data(a, voidptr(buf), size) - } else { - C.archive_read_data_skip(a) - } - } - - return unsafe { cstring_to_vstring(&char(buf)) }, files -} diff --git a/src/archive/bindings.v b/src/archive/bindings.v deleted file mode 100644 index 26af13ff..00000000 --- a/src/archive/bindings.v +++ /dev/null @@ -1,50 +0,0 @@ -module archive - -#flag -larchive - -#include "archive.h" - -struct C.archive {} - -// Create a new archive struct -fn C.archive_read_new() &C.archive - -// Configure the archive to work with zstd compression -fn C.archive_read_support_filter_zstd(&C.archive) - -// Configure the archive to work with a tarball content -fn C.archive_read_support_format_tar(&C.archive) - -// Open an archive for reading -fn C.archive_read_open_filename(&C.archive, &char, int) int - -// Go to next entry header in archive -fn C.archive_read_next_header(&C.archive, &&C.archive_entry) int - -// Skip reading the current entry -fn C.archive_read_data_skip(&C.archive) - -// Free an archive -fn C.archive_read_free(&C.archive) int - -// Read an archive entry's contents into a pointer -fn C.archive_read_data(&C.archive, voidptr, int) - -#include "archive_entry.h" - -struct C.archive_entry {} - -// Create a new archive_entry struct -fn C.archive_entry_new() &C.archive_entry - -// Get the filename of the given entry -fn C.archive_entry_pathname(&C.archive_entry) &char - -// Get an entry's file size -// Note: this function actually returns an i64, but as this can't be used as an arugment to malloc, we'll just roll with it & assume an entry is never bigger than 4 gigs -fn C.archive_entry_size(&C.archive_entry) int - -#include - -// Compare two C strings; 0 means they're equal -fn C.strcmp(&char, &char) int diff --git a/src/pkg.v b/src/pkg.v index 1bb1abc2..cd1f2a65 100644 --- a/src/pkg.v +++ b/src/pkg.v @@ -1,14 +1,65 @@ module pkg -import archive import time +import os +#flag -larchive + +#include "archive.h" + +struct C.archive {} + +// Create a new archive struct +fn C.archive_read_new() &C.archive + +// Configure the archive to work with zstd compression +fn C.archive_read_support_filter_zstd(&C.archive) + +// Configure the archive to work with a tarball content +fn C.archive_read_support_format_tar(&C.archive) + +// Open an archive for reading +fn C.archive_read_open_filename(&C.archive, &char, int) int + +// Go to next entry header in archive +fn C.archive_read_next_header(&C.archive, &&C.archive_entry) int + +// Skip reading the current entry +fn C.archive_read_data_skip(&C.archive) + +// Free an archive +fn C.archive_read_free(&C.archive) int + +// Read an archive entry's contents into a pointer +fn C.archive_read_data(&C.archive, voidptr, int) + +#include "archive_entry.h" + +struct C.archive_entry {} + +// Create a new archive_entry struct +fn C.archive_entry_new() &C.archive_entry + +// Get the filename of the given entry +fn C.archive_entry_pathname(&C.archive_entry) &char + +// Get an entry's file size +// Note: this function actually returns an i64, but as this can't be used as an arugment to malloc, we'll just roll with it & assume an entry is never bigger than 4 gigs +fn C.archive_entry_size(&C.archive_entry) int + +#include + +// Compare two C strings; 0 means they're equal +fn C.strcmp(&char, &char) int + +// Represents a read archive struct Pkg { pub: info PkgInfo [required] files []string [required] } +// Represents the contents of a .PKGINFO file struct PkgInfo { mut: // Single values @@ -89,9 +140,57 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { return pkg_info } +// Extracts the file list & .PKGINFO contents from an archive +// NOTE: this command currently only supports zstd-compressed tarballs pub fn read_pkg(pkg_path string) ?Pkg { - pkg_info_str, files := archive.pkg_info(pkg_path) ? - pkg_info := parse_pkg_info_string(pkg_info_str) ? + if !os.is_file(pkg_path) { + return error("'$pkg_path' doesn't exist or isn't a file.") + } + + a := C.archive_read_new() + entry := C.archive_entry_new() + mut r := 0 + + // Sinds 2020, all newly built Arch packages use zstd + C.archive_read_support_filter_zstd(a) + // The content should always be a tarball + C.archive_read_support_format_tar(a) + + // TODO find out where does this 10240 come from + r = C.archive_read_open_filename(a, &char(pkg_path.str), 10240) + defer { + C.archive_read_free(a) + } + + if r != C.ARCHIVE_OK { + return error('Failed to open package.') + } + + // We iterate over every header in search of the .PKGINFO one + mut buf := voidptr(0) + mut files := []string{} + for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { + pathname := C.archive_entry_pathname(entry) + + ignored_names := [c'.BUILDINFO', c'.INSTALL', c'.MTREE', c'.PKGINFO', c'.CHANGELOG'] + if ignored_names.all(C.strcmp(it, pathname) != 0) { + unsafe { + files << cstring_to_vstring(pathname) + } + } + + if C.strcmp(pathname, c'.PKGINFO') == 0 { + size := C.archive_entry_size(entry) + + // TODO can this unsafe block be avoided? + buf = unsafe { malloc(size) } + C.archive_read_data(a, voidptr(buf), size) + } else { + C.archive_read_data_skip(a) + } + } + + pkg_info := parse_pkg_info_string(unsafe { cstring_to_vstring(&char(buf)) }) ? return Pkg{ info: pkg_info From b86c6b5e16b59f536d0d2fd06e0b2663135e00f3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 14 Jan 2022 22:46:04 +0100 Subject: [PATCH 04/24] Code now passes vet --- src/main.v | 5 +--- src/pkg.v | 7 +++--- src/repo.v | 13 ++++++----- src/routes.v | 3 +++ src/web/logging.v | 6 +++++ src/web/web.v | 58 +++++++++++++++++++++++++---------------------- 6 files changed, 52 insertions(+), 40 deletions(-) diff --git a/src/main.v b/src/main.v index 9d9e5419..ca35c9de 100644 --- a/src/main.v +++ b/src/main.v @@ -5,7 +5,6 @@ import os import log import io import pkg -import archive import repo const port = 8000 @@ -110,7 +109,5 @@ fn main() { return } // println(info) - println(res.info) - print(res.files) - println(res.info.to_desc()) + print(res) } diff --git a/src/pkg.v b/src/pkg.v index cd1f2a65..bcd27246 100644 --- a/src/pkg.v +++ b/src/pkg.v @@ -89,6 +89,7 @@ mut: checkdepends []string } +// parse_pkg_info_string parses a PkgInfo object from a string fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { mut pkg_info := PkgInfo{} @@ -140,7 +141,7 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { return pkg_info } -// Extracts the file list & .PKGINFO contents from an archive +// read_pkg extracts the file list & .PKGINFO contents from an archive // NOTE: this command currently only supports zstd-compressed tarballs pub fn read_pkg(pkg_path string) ?Pkg { if !os.is_file(pkg_path) { @@ -166,9 +167,9 @@ pub fn read_pkg(pkg_path string) ?Pkg { return error('Failed to open package.') } - // We iterate over every header in search of the .PKGINFO one mut buf := voidptr(0) mut files := []string{} + for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { pathname := C.archive_entry_pathname(entry) @@ -198,7 +199,7 @@ pub fn read_pkg(pkg_path string) ?Pkg { } } -// Represent a PkgInfo struct as a desc file +// to_desc returns a desc file valid string representation pub fn (p &PkgInfo) to_desc() string { // TODO calculate md5 & sha256 instead of believing the file mut desc := '' diff --git a/src/repo.v b/src/repo.v index ae76cc9a..f3ebfd42 100644 --- a/src/repo.v +++ b/src/repo.v @@ -1,7 +1,6 @@ module repo import os -import archive const pkgs_subpath = 'pkgs' @@ -22,35 +21,37 @@ pub: pkg_dir string [required] } -// Returns whether the repository contains the given package. +// contains returns whether the repository contains the given package. pub fn (r &Repo) contains(pkg string) bool { return os.exists(os.join_path(r.repo_dir, 'files', pkg)) } -// Adds the given package to the repo. If false, the package was already +// add adds the given package to the repo. If false, the package was already // present in the repository. pub fn (r &Repo) add(pkg string) ?bool { return false } -// Re-generate the db & files archives. +// generate re-generates the db & files archives. fn (r &Repo) genenerate() ? { } -// Returns path to the given package, prepended with the repo's path. +// pkg_path returns path to the given package, prepended with the repo's path. pub fn (r &Repo) pkg_path(pkg string) string { return os.join_path_single(r.pkg_dir, pkg) } +// exists checks whether a package file exists pub fn (r &Repo) exists(pkg string) bool { return os.exists(r.pkg_path(pkg)) } -// Returns the full path to the database file +// db_path returns the full path to the database file pub fn (r &Repo) db_path() string { return os.join_path_single(r.repo_dir, 'repo.tar.gz') } +// add_package adds a package to the repository pub fn (r &Repo) add_package(pkg_path string) ? { mut res := os.Result{} diff --git a/src/routes.v b/src/routes.v index c7df2868..101e1c58 100644 --- a/src/routes.v +++ b/src/routes.v @@ -7,6 +7,7 @@ import time const prefixes = ['B', 'KB', 'MB', 'GB'] +// pretty_bytes converts a byte count to human-readable version fn pretty_bytes(bytes int) string { mut i := 0 mut n := f32(bytes) @@ -23,6 +24,7 @@ fn is_pkg_name(s string) bool { return s.contains('.pkg') } +// get_root handles a GET request for a file on the root ['/:filename'; get] fn (mut app App) get_root(filename string) web.Result { mut full_path := '' @@ -88,6 +90,7 @@ fn (mut app App) get_root(filename string) web.Result { // return app.text('Package added successfully.') // } +// add_package PUT a new package to the server ['/add'; put] pub fn (mut app App) add_package() web.Result { return app.text('') diff --git a/src/web/logging.v b/src/web/logging.v index 66426f28..fc697ffc 100644 --- a/src/web/logging.v +++ b/src/web/logging.v @@ -2,28 +2,34 @@ module web import log +// log reate a log message with the given level pub fn (mut ctx Context) log(msg &string, level log.Level) { lock ctx.logger { ctx.logger.send_output(msg, level) } } +// lfatal create a log message with the fatal level pub fn (mut ctx Context) lfatal(msg &string) { ctx.log(msg, log.Level.fatal) } +// lerror create a log message with the error level pub fn (mut ctx Context) lerror(msg &string) { ctx.log(msg, log.Level.error) } +// lwarn create a log message with the warn level pub fn (mut ctx Context) lwarn(msg &string) { ctx.log(msg, log.Level.warn) } +// linfo create a log message with the info level pub fn (mut ctx Context) linfo(msg &string) { ctx.log(msg, log.Level.info) } +// ldebug create a log message with the debug level pub fn (mut ctx Context) ldebug(msg &string) { ctx.log(msg, log.Level.debug) } diff --git a/src/web/web.v b/src/web/web.v index 404bd085..2db8f18b 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -187,14 +187,14 @@ struct Route { } // Defining this method is optional. -// This method called at server start. +// init_server is called at server start. // You can use it for initializing globals. pub fn (ctx Context) init_server() { eprintln('init_server() has been deprecated, please init your web app in `fn main()`') } // Defining this method is optional. -// This method called before every request (aka middleware). +// before_request is called before every request (aka middleware). // Probably you can use it for check user session cookie or add header. pub fn (ctx Context) before_request() {} @@ -206,7 +206,7 @@ pub struct Cookie { http_only bool } -// web intern function +// send_response_to_client sends a response to the client [manualfree] pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bool { if ctx.done { @@ -230,33 +230,33 @@ pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bo return true } -// Response HTTP_OK with s as payload with content-type `text/html` +// html HTTP_OK with s as payload with content-type `text/html` pub fn (mut ctx Context) html(s string) Result { ctx.send_response_to_client('text/html', s) return Result{} } -// Response HTTP_OK with s as payload with content-type `text/plain` +// text HTTP_OK with s as payload with content-type `text/plain` pub fn (mut ctx Context) text(s string) Result { ctx.send_response_to_client('text/plain', s) return Result{} } -// Response HTTP_OK with json_s as payload with content-type `application/json` +// json HTTP_OK with json_s as payload with content-type `application/json` pub fn (mut ctx Context) json(j T) Result { json_s := json.encode(j) ctx.send_response_to_client('application/json', json_s) return Result{} } -// Response HTTP_OK with a pretty-printed JSON result +// json_pretty Response HTTP_OK with a pretty-printed JSON result pub fn (mut ctx Context) json_pretty(j T) Result { json_s := json.encode_pretty(j) ctx.send_response_to_client('application/json', json_s) return Result{} } -// Response HTTP_OK with file as payload +// file Response HTTP_OK with file as payload // This function manually implements responses because it needs to stream the file contents pub fn (mut ctx Context) file(f_path string) Result { if ctx.done { @@ -329,13 +329,13 @@ pub fn (mut ctx Context) file(f_path string) Result { return Result{} } -// Response HTTP_OK with s as payload +// ok Response HTTP_OK with s as payload pub fn (mut ctx Context) ok(s string) Result { ctx.send_response_to_client(ctx.content_type, s) return Result{} } -// Response a server error +// server_error Response a server error pub fn (mut ctx Context) server_error(ecode int) Result { $if debug { eprintln('> ctx.server_error ecode: $ecode') @@ -347,7 +347,7 @@ pub fn (mut ctx Context) server_error(ecode int) Result { return Result{} } -// Redirect to an url +// redirect Redirect to an url pub fn (mut ctx Context) redirect(url string) Result { if ctx.done { return Result{} @@ -360,7 +360,7 @@ pub fn (mut ctx Context) redirect(url string) Result { return Result{} } -// Send an not_found response +// not_found Send an not_found response pub fn (mut ctx Context) not_found() Result { if ctx.done { return Result{} @@ -370,7 +370,7 @@ pub fn (mut ctx Context) not_found() Result { return Result{} } -// Sets a cookie +// set_cookie Sets a cookie pub fn (mut ctx Context) set_cookie(cookie Cookie) { mut cookie_data := []string{} mut secure := if cookie.secure { 'Secure;' } else { '' } @@ -383,17 +383,17 @@ pub fn (mut ctx Context) set_cookie(cookie Cookie) { ctx.add_header('Set-Cookie', '$cookie.name=$cookie.value; $data') } -// Sets the response content type +// set_content_type Sets the response content type pub fn (mut ctx Context) set_content_type(typ string) { ctx.content_type = typ } -// Sets a cookie with a `expire_data` +// set_cookie_with_expire_date Sets a cookie with a `expire_data` pub fn (mut ctx Context) set_cookie_with_expire_date(key string, val string, expire_date time.Time) { ctx.add_header('Set-Cookie', '$key=$val; Secure; HttpOnly; expires=$expire_date.utc_string()') } -// Gets a cookie by a key +// get_cookie Gets a cookie by a key pub fn (ctx &Context) get_cookie(key string) ?string { // TODO refactor mut cookie_header := ctx.get_header('cookie') if cookie_header == '' { @@ -413,7 +413,7 @@ pub fn (ctx &Context) get_cookie(key string) ?string { // TODO refactor return error('Cookie not found') } -// Sets the response status +// set_status Sets the response status pub fn (mut ctx Context) set_status(code int, desc string) { if code < 100 || code > 599 { ctx.status = '500 Internal Server Error' @@ -422,12 +422,12 @@ pub fn (mut ctx Context) set_status(code int, desc string) { } } -// Adds an header to the response with key and val +// add_header Adds an header to the response with key and val pub fn (mut ctx Context) add_header(key string, val string) { ctx.header.add_custom(key, val) or {} } -// Returns the header data from the key +// get_header Returns the header data from the key pub fn (ctx &Context) get_header(key string) string { return ctx.req.header.get_custom(key) or { '' } } @@ -436,7 +436,7 @@ interface DbInterface { db voidptr } -// run_app +// run runs the app [manualfree] pub fn run(global_app &T, port int) { mut l := net.listen_tcp(.ip6, ':$port') or { panic('failed to listen $err.code $err') } @@ -478,6 +478,7 @@ pub fn run(global_app &T, port int) { } } +// handle_conn handles a connection [manualfree] fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { conn.set_read_timeout(30 * time.second) @@ -615,6 +616,7 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { conn.write(web.http_404.bytes()) or {} } +// route_matches returns wether a route matches fn route_matches(url_words []string, route_words []string) ?[]string { // URL path should be at least as long as the route path // except for the catchall route (`/:path...`) @@ -657,7 +659,7 @@ fn route_matches(url_words []string, route_words []string) ?[]string { return params } -// check if request is for a static file and serves it +// serve_if_static checks if request is for a static file and serves it // returns true if we served a static file, false otherwise [manualfree] fn serve_if_static(mut app T, url urllib.URL) bool { @@ -676,6 +678,7 @@ fn serve_if_static(mut app T, url urllib.URL) bool { return true } +// scan_static_directory makes a static route for each file in a directory fn (mut ctx Context) scan_static_directory(directory_path string, mount_path string) { files := os.ls(directory_path) or { panic(err) } if files.len > 0 { @@ -695,7 +698,7 @@ fn (mut ctx Context) scan_static_directory(directory_path string, mount_path str } } -// Handles a directory static +// handle_static Handles a directory static // If `root` is set the mount path for the dir will be in '/' pub fn (mut ctx Context) handle_static(directory_path string, root bool) bool { if ctx.done || !os.exists(directory_path) { @@ -724,7 +727,7 @@ pub fn (mut ctx Context) mount_static_folder_at(directory_path string, mount_pat return true } -// Serves a file static +// serve_static Serves a file static // `url` is the access path on the site, `file_path` is the real path to the file, `mime_type` is the file type pub fn (mut ctx Context) serve_static(url string, file_path string) { ctx.static_files[url] = file_path @@ -733,7 +736,7 @@ pub fn (mut ctx Context) serve_static(url string, file_path string) { ctx.static_mime_types[url] = web.mime_types[ext] } -// Returns the ip address from the current user +// ip Returns the ip address from the current user pub fn (ctx &Context) ip() string { mut ip := ctx.req.header.get(.x_forwarded_for) or { '' } if ip == '' { @@ -749,22 +752,23 @@ pub fn (ctx &Context) ip() string { return ip } -// Set s to the form error +// error Set s to the form error pub fn (mut ctx Context) error(s string) { println('web error: $s') ctx.form_error = s } -// Returns an empty result +// not_found Returns an empty result pub fn not_found() Result { return Result{} } +// send_string fn send_string(mut conn net.TcpConn, s string) ? { conn.write(s.bytes()) ? } -// Do not delete. +// filter Do not delete. // It used by `vlib/v/gen/c/str_intp.v:130` for string interpolation inside web templates // TODO: move it to template render fn filter(s string) string { From e1ca4085ffb0387d44e0b09994e7f1cb4a2dfe20 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 15:28:58 +0100 Subject: [PATCH 05/24] Let's test matrix builds using platforms --- .woodpecker/.build.yml | 7 ++++++- .woodpecker/.build_arm64.yml => build_arm64.yml | 0 2 files changed, 6 insertions(+), 1 deletion(-) rename .woodpecker/.build_arm64.yml => build_arm64.yml (100%) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 5ece4617..2f9221c0 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -1,7 +1,12 @@ +matrix: + PLATFORM: + - linux/amd64 + - linux/arm64 + # These checks already get performed on the feature branches branches: exclude: [ main, dev ] -platform: linux/amd64 +platform: $PLATFORM pipeline: vieter: diff --git a/.woodpecker/.build_arm64.yml b/build_arm64.yml similarity index 100% rename from .woodpecker/.build_arm64.yml rename to build_arm64.yml From 7b6c1c254d6f50d7d49c6fb78c35a2eb7b26de64 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 15:30:09 +0100 Subject: [PATCH 06/24] Maybe it needs the brackets? --- .woodpecker/.build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 2f9221c0..b4aabd2f 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -6,7 +6,7 @@ matrix: # These checks already get performed on the feature branches branches: exclude: [ main, dev ] -platform: $PLATFORM +platform: ${PLATFORM} pipeline: vieter: From 22ae53b5c277d76aa7414ef0f258c9ef139e5316 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 15:35:12 +0100 Subject: [PATCH 07/24] Added arm/v7 to CI --- .woodpecker/.build.yml | 1 + build_arm64.yml | 29 ----------------------------- 2 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 build_arm64.yml diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index b4aabd2f..67c9ee3f 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -2,6 +2,7 @@ matrix: PLATFORM: - linux/amd64 - linux/arm64 + - linux/arm/v7 # These checks already get performed on the feature branches branches: diff --git a/build_arm64.yml b/build_arm64.yml deleted file mode 100644 index 3b919b40..00000000 --- a/build_arm64.yml +++ /dev/null @@ -1,29 +0,0 @@ -# These checks already get performed on the feature branches -branches: - exclude: [ main, dev ] -platform: linux/arm64 - -pipeline: - vieter: - image: 'chewingbever/vlang:latest' - group: 'build' - commands: - - make vieter - - debug: - image: 'chewingbever/vlang:latest' - group: 'build' - commands: - - make debug - - prod: - image: 'chewingbever/vlang:latest' - environment: - - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static - group: 'build' - commands: - - make prod - # Make sure the binary is actually static - - readelf -d pvieter - - du -h pvieter - - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' From 45647755e08c5af7842313e4f1f7f4960684a0e4 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 15 Jan 2022 15:49:19 +0100 Subject: [PATCH 08/24] added arm/v7 to release image archs --- .woodpecker/.build.yml | 7 +------ .woodpecker/.publish.yml | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 67c9ee3f..4b87f8b3 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -10,12 +10,7 @@ branches: platform: ${PLATFORM} pipeline: - vieter: - image: 'chewingbever/vlang:latest' - group: 'build' - commands: - - make vieter - + # The default build isn't needed, as alpine switches to gcc for the compiler anyways debug: image: 'chewingbever/vlang:latest' group: 'build' diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 6399d7f8..9e5f39a3 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -8,7 +8,7 @@ pipeline: settings: repo: chewingbever/vieter tag: dev - platforms: [ linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] when: event: push branch: dev @@ -21,7 +21,7 @@ pipeline: tag: - latest - $CI_COMMIT_TAG - platforms: [ linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] when: event: tag branch: main From 3dc355f786a17d2277637bfc7ca486ee0ace7fb5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 18 Jan 2022 20:27:00 +0100 Subject: [PATCH 09/24] Wrote PkgInfo to desc generator --- src/main.v | 2 +- src/pkg.v | 83 ++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 76 insertions(+), 9 deletions(-) diff --git a/src/main.v b/src/main.v index ca35c9de..2fcf1dc5 100644 --- a/src/main.v +++ b/src/main.v @@ -109,5 +109,5 @@ fn main() { return } // println(info) - print(res) + print(res.info.to_desc()) } diff --git a/src/pkg.v b/src/pkg.v index bcd27246..8d23b6ac 100644 --- a/src/pkg.v +++ b/src/pkg.v @@ -114,7 +114,7 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { 'pkgbase' { pkg_info.base = value } 'pkgver' { pkg_info.version = value } 'pkgdesc' { pkg_info.description = value } - 'csize' { pkg_info.csize = value.int() } + 'csize' { continue } 'size' { pkg_info.size = value.int() } 'url' { pkg_info.url = value } 'arch' { pkg_info.arch = value } @@ -191,7 +191,8 @@ pub fn read_pkg(pkg_path string) ?Pkg { } } - pkg_info := parse_pkg_info_string(unsafe { cstring_to_vstring(&char(buf)) }) ? + mut pkg_info := parse_pkg_info_string(unsafe { cstring_to_vstring(&char(buf)) }) ? + pkg_info.csize = i64(os.file_size(pkg_path)) return Pkg{ info: pkg_info @@ -199,10 +200,76 @@ pub fn read_pkg(pkg_path string) ?Pkg { } } -// to_desc returns a desc file valid string representation -pub fn (p &PkgInfo) to_desc() string { - // TODO calculate md5 & sha256 instead of believing the file - mut desc := '' - - return desc +fn format_entry(key string, value string) string { + return '\n%$key%\n$value\n' +} + +// to_desc returns a desc file valid string representation +// TODO calculate md5 & sha256 instead of believing the file +pub fn (p &PkgInfo) to_desc() string { + // filename + mut desc := '%FILENAME%\n$p.name-$p.version-${p.arch}.pkg.tar.zst\n' + + desc += format_entry('NAME', p.name) + desc += format_entry('BASE', p.base) + desc += format_entry('VERSION', p.version) + + if p.description.len > 0 { + desc += format_entry('DESC', p.description) + } + + if p.groups.len > 0 { + desc += format_entry('GROUPS', p.groups.join_lines()) + } + + desc += format_entry('CSIZE', p.csize.str()) + desc += format_entry('ISIZE', p.size.str()) + + // TODO calculate these + desc += format_entry('MD5SUM', p.md5sum) + desc += format_entry('SHA256SUM', p.sha256sum) + + // TODO add pgpsig stuff + + if p.url.len > 0 { + desc += format_entry('URL', p.url) + } + + if p.licenses.len > 0 { + desc += format_entry('LICENSE', p.licenses.join_lines()) + } + + desc += format_entry('ARCH', p.arch) + desc += format_entry('BUILDDATE', p.build_date.str()) + desc += format_entry('PACKAGER', p.packager) + + if p.replaces.len > 0 { + desc += format_entry('REPLACES', p.replaces.join_lines()) + } + + if p.conflicts.len > 0 { + desc += format_entry('CONFLICTS', p.conflicts.join_lines()) + } + + if p.provides.len > 0 { + desc += format_entry('PROVIDES', p.provides.join_lines()) + } + + if p.depends.len > 0 { + desc += format_entry('DEPENDS', p.depends.join_lines()) + } + + if p.optdepends.len > 0 { + desc += format_entry('OPTDEPENDS', p.optdepends.join_lines()) + } + + if p.makedepends.len > 0 { + desc += format_entry('MAKEDEPENDS', p.makedepends.join_lines()) + } + + if p.checkdepends.len > 0 { + desc += format_entry('CHECKDEPENDS', p.checkdepends.join_lines()) + } + + return '$desc\n' } From cb16091f659c9e90c4ea0fe6f143d2acad582357 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 18 Jan 2022 21:41:09 +0100 Subject: [PATCH 10/24] Non-working hashing code [CI SKIP] --- .woodpecker/.publish.yml | 14 +++++++++++++- src/pkg.v | 10 +++++++--- src/util.v | 30 ++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 src/util.v diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 9e5f39a3..6d3b34ef 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -1,7 +1,19 @@ -branches: [main, dev] +# branches: [main, dev] platform: linux/amd64 pipeline: + dryrun: + image: woodpeckerci/plugin-docker-buildx + secrets: [ docker_username, docker_password ] + settings: + repo: chewingbever/vieter + tag: dev + platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + dry_run: true + when: + event: pull_request + branch: dev + dev: image: woodpeckerci/plugin-docker-buildx secrets: [ docker_username, docker_password ] diff --git a/src/pkg.v b/src/pkg.v index 8d23b6ac..8b1849cb 100644 --- a/src/pkg.v +++ b/src/pkg.v @@ -2,6 +2,7 @@ module pkg import time import os +import util #flag -larchive @@ -120,8 +121,8 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { 'arch' { pkg_info.arch = value } 'builddate' { pkg_info.build_date = value.int() } 'packager' { pkg_info.packager = value } - 'md5sum' { pkg_info.md5sum = value } - 'sha256sum' { pkg_info.sha256sum = value } + 'md5sum' { continue } + 'sha256sum' { continue } 'pgpsig' { pkg_info.pgpsig = value } 'pgpsigsize' { pkg_info.pgpsigsize = value.int() } // Array values @@ -192,7 +193,11 @@ pub fn read_pkg(pkg_path string) ?Pkg { } mut pkg_info := parse_pkg_info_string(unsafe { cstring_to_vstring(&char(buf)) }) ? + pkg_info.csize = i64(os.file_size(pkg_path)) + pkg_info.md5sum, pkg_info.sha256sum = util.hash_file(pkg_path) or { + return error('Failed to hash package.') + } return Pkg{ info: pkg_info @@ -225,7 +230,6 @@ pub fn (p &PkgInfo) to_desc() string { desc += format_entry('CSIZE', p.csize.str()) desc += format_entry('ISIZE', p.size.str()) - // TODO calculate these desc += format_entry('MD5SUM', p.md5sum) desc += format_entry('SHA256SUM', p.sha256sum) diff --git a/src/util.v b/src/util.v new file mode 100644 index 00000000..cb17c3f6 --- /dev/null +++ b/src/util.v @@ -0,0 +1,30 @@ +module util + +import os +import crypto.md5 +import crypto.sha256 + +// hash_file returns the md5 & sha256 hash of a given file +pub fn hash_file(path &string) ?(string, string) { + file := os.open(path) or { return error('Failed to open file.') } + + mut md5sum := md5.new() + mut sha256sum := sha256.new() + + buf_size := i32(1_000_000) + mut buf := []byte{len: buf_size} + mut bytes_left := os.file_size(path) + + for bytes_left > 0 { + // TODO check if just breaking here is safe + bytes_read := file.read(mut buf) or { break } + bytes_left -= u64(bytes_read) + + if bytes_left > buf_size { + md5sum.write(buf) + sha256sum.write(buf) + } + } + + return md5sum.sum(buf).hex(), sha256sum.sum(buf).hex() +} From 9bd14b4cbf584c524ffe1bd1ecebe2bed90a5cb3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 18 Jan 2022 22:52:14 +0100 Subject: [PATCH 11/24] Yeah this doesn't work [CI SKIP] --- src/main.v | 2 +- src/util.v | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main.v b/src/main.v index 2fcf1dc5..8a73822c 100644 --- a/src/main.v +++ b/src/main.v @@ -104,7 +104,7 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { fn main() { // archive.list_filenames() - res := pkg.read_pkg('test/homebank-5.5.1-1-x86_64.pkg.tar.zst') or { + res := pkg.read_pkg('test/homebank-5.5.3-1-x86_64.pkg.tar.zst') or { eprintln(err.msg) return } diff --git a/src/util.v b/src/util.v index cb17c3f6..19d174f0 100644 --- a/src/util.v +++ b/src/util.v @@ -5,13 +5,14 @@ import crypto.md5 import crypto.sha256 // hash_file returns the md5 & sha256 hash of a given file +// TODO actually implement sha256 pub fn hash_file(path &string) ?(string, string) { file := os.open(path) or { return error('Failed to open file.') } mut md5sum := md5.new() mut sha256sum := sha256.new() - buf_size := i32(1_000_000) + buf_size := int(1_000_000) mut buf := []byte{len: buf_size} mut bytes_left := os.file_size(path) @@ -21,10 +22,12 @@ pub fn hash_file(path &string) ?(string, string) { bytes_left -= u64(bytes_read) if bytes_left > buf_size { - md5sum.write(buf) - sha256sum.write(buf) + // For now we'll assume that this always works + md5sum.write(buf) or {} + // sha256sum.write(buf) or {} } } - return md5sum.sum(buf).hex(), sha256sum.sum(buf).hex() + // return md5sum.sum(buf).hex(), sha256sum.sum(buf).hex() + return md5sum.sum(buf).hex(), '' } From df3310944e5f0127ae39f586ae5b38b5714ebc3c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 17:15:37 +0100 Subject: [PATCH 12/24] Working md5 checksum --- src/main.v | 2 +- src/util.v | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main.v b/src/main.v index 8a73822c..e81ed92e 100644 --- a/src/main.v +++ b/src/main.v @@ -104,7 +104,7 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { fn main() { // archive.list_filenames() - res := pkg.read_pkg('test/homebank-5.5.3-1-x86_64.pkg.tar.zst') or { + res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { eprintln(err.msg) return } diff --git a/src/util.v b/src/util.v index 19d174f0..066a58df 100644 --- a/src/util.v +++ b/src/util.v @@ -2,7 +2,7 @@ module util import os import crypto.md5 -import crypto.sha256 +// import crypto.sha256 // hash_file returns the md5 & sha256 hash of a given file // TODO actually implement sha256 @@ -10,7 +10,7 @@ pub fn hash_file(path &string) ?(string, string) { file := os.open(path) or { return error('Failed to open file.') } mut md5sum := md5.new() - mut sha256sum := sha256.new() + // mut sha256sum := sha256.new() buf_size := int(1_000_000) mut buf := []byte{len: buf_size} @@ -18,16 +18,16 @@ pub fn hash_file(path &string) ?(string, string) { for bytes_left > 0 { // TODO check if just breaking here is safe - bytes_read := file.read(mut buf) or { break } + bytes_read := file.read(mut buf) or { return error('Failed to read from file.') } bytes_left -= u64(bytes_read) - if bytes_left > buf_size { - // For now we'll assume that this always works - md5sum.write(buf) or {} - // sha256sum.write(buf) or {} + // For now we'll assume that this always works + md5sum.write(buf[..bytes_read]) or { + return error('Failed to update checksum. This should never happen.') } + // sha256sum.write(buf) or {} } // return md5sum.sum(buf).hex(), sha256sum.sum(buf).hex() - return md5sum.sum(buf).hex(), '' + return md5sum.checksum().hex(), '' } From 4bc3bfb8cf7a2744fd06905c7cfa983c6a2ff849 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 17:22:36 +0100 Subject: [PATCH 13/24] Moved checksum calculation to desc function --- src/main.v | 3 ++- src/pkg.v | 29 +++++++++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/main.v b/src/main.v index e81ed92e..835b8224 100644 --- a/src/main.v +++ b/src/main.v @@ -109,5 +109,6 @@ fn main() { return } // println(info) - print(res.info.to_desc()) + println('hey') + print(res.to_desc()) } diff --git a/src/pkg.v b/src/pkg.v index 8b1849cb..2598179a 100644 --- a/src/pkg.v +++ b/src/pkg.v @@ -56,6 +56,7 @@ fn C.strcmp(&char, &char) int // Represents a read archive struct Pkg { pub: + path string [required] info PkgInfo [required] files []string [required] } @@ -74,10 +75,10 @@ mut: arch string build_date i64 packager string - md5sum string - sha256sum string - pgpsig string - pgpsigsize i64 + // md5sum string + // sha256sum string + pgpsig string + pgpsigsize i64 // Array values groups []string licenses []string @@ -90,6 +91,10 @@ mut: checkdepends []string } +pub fn (p &Pkg) checksum() ?(string, string) { + return util.hash_file(p.path) +} + // parse_pkg_info_string parses a PkgInfo object from a string fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { mut pkg_info := PkgInfo{} @@ -195,11 +200,9 @@ pub fn read_pkg(pkg_path string) ?Pkg { mut pkg_info := parse_pkg_info_string(unsafe { cstring_to_vstring(&char(buf)) }) ? pkg_info.csize = i64(os.file_size(pkg_path)) - pkg_info.md5sum, pkg_info.sha256sum = util.hash_file(pkg_path) or { - return error('Failed to hash package.') - } return Pkg{ + path: pkg_path info: pkg_info files: files } @@ -211,7 +214,9 @@ fn format_entry(key string, value string) string { // to_desc returns a desc file valid string representation // TODO calculate md5 & sha256 instead of believing the file -pub fn (p &PkgInfo) to_desc() string { +pub fn (pkg &Pkg) to_desc() string { + p := pkg.info + // filename mut desc := '%FILENAME%\n$p.name-$p.version-${p.arch}.pkg.tar.zst\n' @@ -230,8 +235,12 @@ pub fn (p &PkgInfo) to_desc() string { desc += format_entry('CSIZE', p.csize.str()) desc += format_entry('ISIZE', p.size.str()) - desc += format_entry('MD5SUM', p.md5sum) - desc += format_entry('SHA256SUM', p.sha256sum) + md5sum, sha256sum := pkg.checksum() or { '', '' } + + desc += format_entry('MD5SUM', md5sum) + + // TODO add this + // desc += format_entry('SHA256SUM', sha256sum) // TODO add pgpsig stuff From 57fe767a7052ac0d115800d60e1d14a5d522cf8e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 18:54:33 +0100 Subject: [PATCH 14/24] Working add to repo functionality --- src/main.v | 17 ++++--- src/{pkg.v => package.v} | 21 ++++++-- src/repo.v | 107 ++++++++++++++++++++++++--------------- 3 files changed, 94 insertions(+), 51 deletions(-) rename src/{pkg.v => package.v} (94%) diff --git a/src/main.v b/src/main.v index 835b8224..dc515eaf 100644 --- a/src/main.v +++ b/src/main.v @@ -4,7 +4,6 @@ import web import os import log import io -import pkg import repo const port = 8000 @@ -103,12 +102,16 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { // } fn main() { + r := repo.new('data/repo', 'data/pkgs') or { return } + print(r.add_from_path('test/homebank-5.5.1-1-x86_64.pkg.tar.zst') or { panic('you fialed') }) + // archive.list_filenames() - res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { - eprintln(err.msg) - return - } + // res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { + // eprintln(err.msg) + // return + // } // println(info) - println('hey') - print(res.to_desc()) + // println('hey') + // print(res.to_desc()) + // print(res.to_files()) } diff --git a/src/pkg.v b/src/package.v similarity index 94% rename from src/pkg.v rename to src/package.v index 2598179a..12480b3f 100644 --- a/src/pkg.v +++ b/src/package.v @@ -1,4 +1,4 @@ -module pkg +module package import time import os @@ -45,7 +45,9 @@ fn C.archive_entry_new() &C.archive_entry fn C.archive_entry_pathname(&C.archive_entry) &char // Get an entry's file size -// Note: this function actually returns an i64, but as this can't be used as an arugment to malloc, we'll just roll with it & assume an entry is never bigger than 4 gigs +// Note: this function actually returns an i64, but as this can't be used as an +// arugment to malloc, we'll just roll with it & assume an entry is never +// bigger than 4 gigs fn C.archive_entry_size(&C.archive_entry) int #include @@ -63,7 +65,7 @@ pub: // Represents the contents of a .PKGINFO file struct PkgInfo { -mut: +pub mut: // Single values name string base string @@ -212,13 +214,19 @@ fn format_entry(key string, value string) string { return '\n%$key%\n$value\n' } +pub fn (pkg &Pkg) filename() string { + p := pkg.info + + return '$p.name-$p.version-${p.arch}.pkg.tar.zst' +} + // to_desc returns a desc file valid string representation // TODO calculate md5 & sha256 instead of believing the file pub fn (pkg &Pkg) to_desc() string { p := pkg.info // filename - mut desc := '%FILENAME%\n$p.name-$p.version-${p.arch}.pkg.tar.zst\n' + mut desc := '%FILENAME%\n$pkg.filename()\n' desc += format_entry('NAME', p.name) desc += format_entry('BASE', p.base) @@ -286,3 +294,8 @@ pub fn (pkg &Pkg) to_desc() string { return '$desc\n' } + +// to_files returns a files file valid string representation +pub fn (pkg &Pkg) to_files() string { + return '%FILES%\n$pkg.files.join_lines()\n' +} diff --git a/src/repo.v b/src/repo.v index f3ebfd42..c7b7db5c 100644 --- a/src/repo.v +++ b/src/repo.v @@ -1,8 +1,13 @@ module repo import os +import package -const pkgs_subpath = 'pkgs' +// subpath where the uncompressed version of the files archive is stored +const files_subpath = 'files' + +// subpath where the uncompressed version of the repo archive is stored +const repo_subpath = 'repo' // Dummy struct to work around the fact that you can only share structs, maps & // arrays @@ -21,46 +26,68 @@ pub: pkg_dir string [required] } -// contains returns whether the repository contains the given package. -pub fn (r &Repo) contains(pkg string) bool { - return os.exists(os.join_path(r.repo_dir, 'files', pkg)) -} - -// add adds the given package to the repo. If false, the package was already -// present in the repository. -pub fn (r &Repo) add(pkg string) ?bool { - return false -} - -// generate re-generates the db & files archives. -fn (r &Repo) genenerate() ? { -} - -// pkg_path returns path to the given package, prepended with the repo's path. -pub fn (r &Repo) pkg_path(pkg string) string { - return os.join_path_single(r.pkg_dir, pkg) -} - -// exists checks whether a package file exists -pub fn (r &Repo) exists(pkg string) bool { - return os.exists(r.pkg_path(pkg)) -} - -// db_path returns the full path to the database file -pub fn (r &Repo) db_path() string { - return os.join_path_single(r.repo_dir, 'repo.tar.gz') -} - -// add_package adds a package to the repository -pub fn (r &Repo) add_package(pkg_path string) ? { - mut res := os.Result{} - - lock r.mutex { - res = os.execute("repo-add '$r.db_path()' '$pkg_path'") +pub fn new(repo_dir string, pkg_dir string) ?Repo { + if !os.is_dir(repo_dir) { + os.mkdir_all(repo_dir) or { return error('Failed to create repo directory.') } } - if res.exit_code != 0 { - println(res.output) - return error('repo-add failed.') + if !os.is_dir(pkg_dir) { + os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') } + } + + return Repo{ + repo_dir: repo_dir + pkg_dir: pkg_dir } } + +// add_from_path adds a package from an arbitrary path & moves it into the pkgs +// directory if necessary. +pub fn (r &Repo) add_from_path(pkg_path string) ?bool { + pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file.') } + + added := r.add(pkg) ? + + // If the add was successful, we move the file to the packages directory + if added { + dest_path := os.real_path(os.join_path_single(r.pkg_dir, pkg.filename())) + + // Only move the file if it's not already in the package directory + if dest_path != os.real_path(pkg_path) { + os.mv(pkg_path, dest_path) ? + } + } + + return true +} + +// add adds a given Pkg to the repository +fn (r &Repo) add(pkg &package.Pkg) ?bool { + pkg_dir := r.pkg_path(pkg) + + // We can't add the same package twice + if os.exists(pkg_dir) { + return false + } + + os.mkdir(pkg_dir) or { return error('Failed to create package directory.') } + + os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or { + os.rmdir_all(pkg_dir) ? + + return error('Failed to write desc file.') + } + os.write_file(os.join_path_single(pkg_dir, 'files'), pkg.to_files()) or { + os.rmdir_all(pkg_dir) ? + + return error('Failed to write files file.') + } + // TODO generate database archive + + return true +} + +// Returns the path where the given package's desc & files files are stored +fn (r &Repo) pkg_path(pkg &package.Pkg) string { + return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version') +} From d04ba06ad35362a1b43a976988e4712bee6469c7 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 21:24:46 +0100 Subject: [PATCH 15/24] Finally enabled web server again --- Makefile | 2 +- src/main.v | 121 ++++++++++++++++++++++++++------------------------ src/package.v | 3 +- src/repo.v | 1 + src/routes.v | 75 +++++++++++++++---------------- 5 files changed, 103 insertions(+), 99 deletions(-) diff --git a/Makefile b/Makefile index dc2273bf..9a5d6fae 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ c: # Run the server in the default 'data' directory .PHONY: run run: vieter - API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG ./vieter + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter .PHONY: run-prod run-prod: prod diff --git a/src/main.v b/src/main.v index dc515eaf..ed6383e0 100644 --- a/src/main.v +++ b/src/main.v @@ -16,6 +16,7 @@ struct App { web.Context pub: api_key string [required; web_global] + dl_dir string [required; web_global] pub mut: repo repo.Repo [required; web_global] } @@ -53,65 +54,67 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } } -// fn main2() { -// // Configure logger -// log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } -// log_level := log.level_from_tag(log_level_str) or { -// exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') -// } -// log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } - -// mut logger := log.Log{ -// level: log_level -// } - -// logger.set_full_logpath(log_file) -// logger.log_to_console_too() - -// defer { -// logger.info('Flushing log file') -// logger.flush() -// logger.close() -// } - -// // Configure web server -// key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } -// repo_dir := os.getenv_opt('REPO_DIR') or { -// exit_with_message(1, 'No repo directory was configured.') -// } - -// repo := repo.Repo{ -// dir: repo_dir -// name: db_name -// } - -// // We create the upload directory during startup -// if !os.is_dir(repo.pkg_dir()) { -// os.mkdir_all(repo.pkg_dir()) or { -// exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.") -// } - -// logger.info("Created package directory '$repo.pkg_dir()'.") -// } - -// web.run(&App{ -// logger: logger -// api_key: key -// repo: repo -// }, port) -// } - fn main() { - r := repo.new('data/repo', 'data/pkgs') or { return } - print(r.add_from_path('test/homebank-5.5.1-1-x86_64.pkg.tar.zst') or { panic('you fialed') }) + // Configure logger + log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } + log_level := log.level_from_tag(log_level_str) or { + exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + } + log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } - // archive.list_filenames() - // res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { - // eprintln(err.msg) - // return - // } - // println(info) - // println('hey') - // print(res.to_desc()) - // print(res.to_files()) + mut logger := log.Log{ + level: log_level + } + + logger.set_full_logpath(log_file) + logger.log_to_console_too() + + defer { + logger.info('Flushing log file') + logger.flush() + logger.close() + } + + // Configure web server + key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } + repo_dir := os.getenv_opt('REPO_DIR') or { + exit_with_message(1, 'No repo directory was configured.') + } + pkg_dir := os.getenv_opt('PKG_DIR') or { + exit_with_message(1, 'No package directory was configured.') + } + dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { + exit_with_message(1, 'No download directory was configured.') + } + + // This also creates the directories if needed + repo := repo.new(repo_dir, pkg_dir) or { + exit_with_message(1, 'Failed to create required directories.') + } + + os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } + + web.run(&App{ + logger: logger + api_key: key + dl_dir: dl_dir + repo: repo + }, port) } + +// fn main() { +// r := repo.new('data/repo', 'data/pkgs') or { return } +// print(r.add_from_path('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { +// panic('you fialed') +// }) + +// // archive.list_filenames() +// // res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { +// // eprintln(err.msg) +// // return +// // } +// // println(info) +// // println('hey') +// // print(res.to_desc()) +// // print(res.to_files()) +// } diff --git a/src/package.v b/src/package.v index 12480b3f..63400217 100644 --- a/src/package.v +++ b/src/package.v @@ -1,6 +1,5 @@ module package -import time import os import util @@ -93,6 +92,7 @@ pub mut: checkdepends []string } +// checksum calculates the md5 & sha256 hash of the package pub fn (p &Pkg) checksum() ?(string, string) { return util.hash_file(p.path) } @@ -214,6 +214,7 @@ fn format_entry(key string, value string) string { return '\n%$key%\n$value\n' } +// filename returns the correct filename of the package file pub fn (pkg &Pkg) filename() string { p := pkg.info diff --git a/src/repo.v b/src/repo.v index c7b7db5c..389ebce3 100644 --- a/src/repo.v +++ b/src/repo.v @@ -26,6 +26,7 @@ pub: pkg_dir string [required] } +// new creates a new Repo & creates the directories as needed pub fn new(repo_dir string, pkg_dir string) ?Repo { if !os.is_dir(repo_dir) { os.mkdir_all(repo_dir) or { return error('Failed to create repo directory.') } diff --git a/src/routes.v b/src/routes.v index 101e1c58..c7e38ba1 100644 --- a/src/routes.v +++ b/src/routes.v @@ -4,6 +4,7 @@ import web import os import repo import time +import rand const prefixes = ['B', 'KB', 'MB', 'GB'] @@ -38,57 +39,55 @@ fn (mut app App) get_root(filename string) web.Result { return app.file(full_path) } -// ['/pkgs/:pkg'; put] -// fn (mut app App) put_package(pkg string) web.Result { -// if !app.is_authorized() { -// return app.text('Unauthorized.') -// } +['/publish'; post] +fn (mut app App) put_package() web.Result { + if !app.is_authorized() { + return app.text('Unauthorized.') + } -// if !is_pkg_name(pkg) { -// app.lwarn("Invalid package name '$pkg'.") + mut pkg_path := '' -// return app.text('Invalid filename.') -// } + if length := app.req.header.get(.content_length) { + // Generate a random filename for the temp file + pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) -// if app.repo.exists(pkg) { -// app.lwarn("Duplicate package '$pkg'") + for os.exists(pkg_path) { + pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) + } -// return app.text('File already exists.') -// } + app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to '$pkg_path'.") -// pkg_path := app.repo.pkg_path(pkg) + // This is used to time how long it takes to upload a file + mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) -// if length := app.req.header.get(.content_length) { -// app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.") + reader_to_file(mut app.reader, length.int(), pkg_path) or { + app.lwarn("Failed to upload '$pkg_path'") -// // This is used to time how long it takes to upload a file -// mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) + return app.text('Failed to upload file.') + } -// reader_to_file(mut app.reader, length.int(), pkg_path) or { -// app.lwarn("Failed to upload package '$pkg'") + sw.stop() + app.ldebug("Upload of '$pkg_path' completed in ${sw.elapsed().seconds():.3}s.") + } else { + app.lwarn('Tried to upload package without specifying a Content-Length.') + return app.text("Content-Type header isn't set.") + } -// return app.text('Failed to upload file.') -// } + added := app.repo.add_from_path(pkg_path) or { + app.lerror('Error while adding package.') -// sw.stop() -// app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.") -// } else { -// app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.") -// return app.text("Content-Type header isn't set.") -// } + return app.text('Failed to add package.') + } + if !added { + app.lwarn('Duplicate package.') -// app.repo.add_package(pkg_path) or { -// app.lwarn("Failed to add package '$pkg' to database.") + return app.text('File already exists.') + } -// os.rm(pkg_path) or { println('Failed to remove $pkg_path') } + app.linfo("Added '$pkg_path' to repository.") -// return app.text('Failed to add package to repo.') -// } - -// app.linfo("Added '$pkg' to repository.") - -// return app.text('Package added successfully.') -// } + return app.text('Package added successfully.') +} // add_package PUT a new package to the server ['/add'; put] From 735492dcbd591e7525974d273ebcbe3cd90c0003 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 21:30:56 +0100 Subject: [PATCH 16/24] Be nice, CI --- src/package.v | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/package.v b/src/package.v index 63400217..cf727a99 100644 --- a/src/package.v +++ b/src/package.v @@ -244,7 +244,7 @@ pub fn (pkg &Pkg) to_desc() string { desc += format_entry('CSIZE', p.csize.str()) desc += format_entry('ISIZE', p.size.str()) - md5sum, sha256sum := pkg.checksum() or { '', '' } + md5sum, _ := pkg.checksum() or { '', '' } desc += format_entry('MD5SUM', md5sum) From 3fc0a2f9a984fc92b48e702920efe0e0f4572a4d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 23:06:46 +0100 Subject: [PATCH 17/24] Moved image dryruns to own pipeline --- .woodpecker/.image.yml | 17 +++++++++++++++++ .woodpecker/.publish.yml | 14 +------------- 2 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 .woodpecker/.image.yml diff --git a/.woodpecker/.image.yml b/.woodpecker/.image.yml new file mode 100644 index 00000000..6b788613 --- /dev/null +++ b/.woodpecker/.image.yml @@ -0,0 +1,17 @@ +# These checks already get performed on the feature branches +branches: + exclude: [ main, dev ] +platform: linux/amd64 + +pipeline: + dryrun: + image: woodpeckerci/plugin-docker-buildx + secrets: [ docker_username, docker_password ] + settings: + repo: chewingbever/vieter + tag: dev + platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + dry_run: true + when: + event: pull_request + branch: dev diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 6d3b34ef..9e5f39a3 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -1,19 +1,7 @@ -# branches: [main, dev] +branches: [main, dev] platform: linux/amd64 pipeline: - dryrun: - image: woodpeckerci/plugin-docker-buildx - secrets: [ docker_username, docker_password ] - settings: - repo: chewingbever/vieter - tag: dev - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] - dry_run: true - when: - event: pull_request - branch: dev - dev: image: woodpeckerci/plugin-docker-buildx secrets: [ docker_username, docker_password ] From 944bc412753d77c7d9e633441dd8fd832702a176 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 19 Jan 2022 23:45:12 +0100 Subject: [PATCH 18/24] Split libarchive bindings into separate file --- src/libarchive.v | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ src/package.v | 51 ----------------------------------------------- 2 files changed, 52 insertions(+), 51 deletions(-) create mode 100644 src/libarchive.v diff --git a/src/libarchive.v b/src/libarchive.v new file mode 100644 index 00000000..bd90f099 --- /dev/null +++ b/src/libarchive.v @@ -0,0 +1,52 @@ +// Bindings for the libarchive library + +#flag -larchive + +#include "archive.h" + +struct C.archive {} + +// Create a new archive struct +fn C.archive_read_new() &C.archive + +// Configure the archive to work with zstd compression +fn C.archive_read_support_filter_zstd(&C.archive) + +// Configure the archive to work with a tarball content +fn C.archive_read_support_format_tar(&C.archive) + +// Open an archive for reading +fn C.archive_read_open_filename(&C.archive, &char, int) int + +// Go to next entry header in archive +fn C.archive_read_next_header(&C.archive, &&C.archive_entry) int + +// Skip reading the current entry +fn C.archive_read_data_skip(&C.archive) + +// Free an archive +fn C.archive_read_free(&C.archive) int + +// Read an archive entry's contents into a pointer +fn C.archive_read_data(&C.archive, voidptr, int) + +#include "archive_entry.h" + +struct C.archive_entry {} + +// Create a new archive_entry struct +fn C.archive_entry_new() &C.archive_entry + +// Get the filename of the given entry +fn C.archive_entry_pathname(&C.archive_entry) &char + +// Get an entry's file size +// Note: this function actually returns an i64, but as this can't be used as an +// arugment to malloc, we'll just roll with it & assume an entry is never +// bigger than 4 gigs +fn C.archive_entry_size(&C.archive_entry) int + +#include + +// Compare two C strings; 0 means they're equal +fn C.strcmp(&char, &char) int diff --git a/src/package.v b/src/package.v index cf727a99..9e8e5038 100644 --- a/src/package.v +++ b/src/package.v @@ -3,57 +3,6 @@ module package import os import util -#flag -larchive - -#include "archive.h" - -struct C.archive {} - -// Create a new archive struct -fn C.archive_read_new() &C.archive - -// Configure the archive to work with zstd compression -fn C.archive_read_support_filter_zstd(&C.archive) - -// Configure the archive to work with a tarball content -fn C.archive_read_support_format_tar(&C.archive) - -// Open an archive for reading -fn C.archive_read_open_filename(&C.archive, &char, int) int - -// Go to next entry header in archive -fn C.archive_read_next_header(&C.archive, &&C.archive_entry) int - -// Skip reading the current entry -fn C.archive_read_data_skip(&C.archive) - -// Free an archive -fn C.archive_read_free(&C.archive) int - -// Read an archive entry's contents into a pointer -fn C.archive_read_data(&C.archive, voidptr, int) - -#include "archive_entry.h" - -struct C.archive_entry {} - -// Create a new archive_entry struct -fn C.archive_entry_new() &C.archive_entry - -// Get the filename of the given entry -fn C.archive_entry_pathname(&C.archive_entry) &char - -// Get an entry's file size -// Note: this function actually returns an i64, but as this can't be used as an -// arugment to malloc, we'll just roll with it & assume an entry is never -// bigger than 4 gigs -fn C.archive_entry_size(&C.archive_entry) int - -#include - -// Compare two C strings; 0 means they're equal -fn C.strcmp(&char, &char) int - // Represents a read archive struct Pkg { pub: From a37d40d27859faa30659e11fd9d0b0977d8dff99 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 20 Jan 2022 00:08:26 +0100 Subject: [PATCH 19/24] Added start of libarchive bindings for writing --- src/{libarchive.v => archive.v} | 35 ++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) rename src/{libarchive.v => archive.v} (50%) diff --git a/src/libarchive.v b/src/archive.v similarity index 50% rename from src/libarchive.v rename to src/archive.v index bd90f099..ee72e3ac 100644 --- a/src/libarchive.v +++ b/src/archive.v @@ -6,7 +6,7 @@ struct C.archive {} -// Create a new archive struct +// Create a new archive struct for reading fn C.archive_read_new() &C.archive // Configure the archive to work with zstd compression @@ -30,6 +30,27 @@ fn C.archive_read_free(&C.archive) int // Read an archive entry's contents into a pointer fn C.archive_read_data(&C.archive, voidptr, int) +// Create a new archive struct for writing +fn C.archive_write_new() &C.archive + +// Sets the filter for the archive to gzip +fn C.archive_write_add_filter_gzip(&C.archive) + +// Sets to archive to "pax restricted" mode. Libarchive's "pax restricted" +// format is a tar format that uses pax extensions only when absolutely +// necessary. Most of the time, it will write plain ustar entries. This is the +// recommended tar format for most uses. You should explicitly use ustar format +// only when you have to create archives that will be readable on older +// systems; you should explicitly request pax format only when you need to +// preserve as many attributes as possible. +fn C.archive_write_set_format_pax_restricted(&C.archive) + +// Opens up the filename for writing +fn C.archive_write_open_filename(&C.archive, &char) + +// Write an entry to the archive file +fn C.archive_write_header(&C.archive, &C.archive_entry) + #include "archive_entry.h" struct C.archive_entry {} @@ -46,6 +67,18 @@ fn C.archive_entry_pathname(&C.archive_entry) &char // bigger than 4 gigs fn C.archive_entry_size(&C.archive_entry) int +// Set the pathname for the entry +fn C.archive_entry_set_pathname(&C.archive_entry, &char) + +// Sets the file size of the entry +fn C.archive_entry_set_size(&C.archive_entry, i64) + +// Sets the file type for an entry +fn C.archive_entry_set_filetype(&C.archive_entry, u32) + +// Sets the file permissions for an entry +fn C.archive_entry_set_perm(&C.archive_entry, int) + #include // Compare two C strings; 0 means they're equal From b6e9fa5b02cd5d24b952c933dde3874cf1d0e5d3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 20 Jan 2022 17:11:03 +0100 Subject: [PATCH 20/24] First working version of creating archive --- src/archive.v | 15 +++++++++++++++ src/repo.v | 52 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/archive.v b/src/archive.v index ee72e3ac..7ded6a1b 100644 --- a/src/archive.v +++ b/src/archive.v @@ -51,6 +51,15 @@ fn C.archive_write_open_filename(&C.archive, &char) // Write an entry to the archive file fn C.archive_write_header(&C.archive, &C.archive_entry) +// Write the data in the buffer to the archive +fn C.archive_write_data(&C.archive, voidptr, int) + +// Close an archive for writing +fn C.archive_write_close(&C.archive) + +// Free the write archive +fn C.archive_write_free(&C.archive) + #include "archive_entry.h" struct C.archive_entry {} @@ -79,6 +88,12 @@ fn C.archive_entry_set_filetype(&C.archive_entry, u32) // Sets the file permissions for an entry fn C.archive_entry_set_perm(&C.archive_entry, int) +// Clears out an entry struct +fn C.archive_entry_clear(&C.archive_entry) + +// Copy over a stat struct to the archive entry +fn C.archive_entry_copy_stat(&C.archive_entry, &C.stat) + #include // Compare two C strings; 0 means they're equal diff --git a/src/repo.v b/src/repo.v index 389ebce3..ff849097 100644 --- a/src/repo.v +++ b/src/repo.v @@ -59,7 +59,7 @@ pub fn (r &Repo) add_from_path(pkg_path string) ?bool { } } - return true + return added } // add adds a given Pkg to the repository @@ -83,7 +83,8 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool { return error('Failed to write files file.') } - // TODO generate database archive + + r.sync() ? return true } @@ -92,3 +93,50 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool { fn (r &Repo) pkg_path(pkg &package.Pkg) string { return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version') } + +// Re-generate the repo archive files +fn (r &Repo) sync() ? { + a := C.archive_write_new() + entry := C.archive_entry_new() + st := C.stat{} + buf := [8192]byte{} + + // This makes the archive a gzip-compressed tarball + C.archive_write_add_filter_gzip(a) + C.archive_write_set_format_pax_restricted(a) + + repo_path := os.join_path_single(r.repo_dir, 'repo.db') + + C.archive_write_open_filename(a, &char(repo_path.str)) + + // Iterate over each directory + for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, it))) { + inner_path := os.join_path_single(d, 'desc') + actual_path := os.join_path_single(r.repo_dir, inner_path) + + unsafe { + C.stat(&char(actual_path.str), &st) + } + + C.archive_entry_set_pathname(entry, &char(inner_path.str)) + // C.archive_entry_copy_stat(entry, &st) + C.archive_entry_set_size(entry, st.st_size) + C.archive_entry_set_filetype(entry, C.AE_IFREG) + C.archive_entry_set_perm(entry, 0o644) + C.archive_write_header(a, entry) + + fd := C.open(&char(actual_path.str), C.O_RDONLY) + mut len := C.read(fd, &buf, sizeof(buf)) + + for len > 0 { + C.archive_write_data(a, &buf, len) + len = C.read(fd, &buf, sizeof(buf)) + } + C.close(fd) + + C.archive_entry_clear(entry) + } + + C.archive_write_close(a) + C.archive_write_free(a) +} From 01d961f68e0e4700e4a85597384361534234a9fc Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 20 Jan 2022 17:45:04 +0100 Subject: [PATCH 21/24] File stats are now correctly copied over to archive --- src/archive.v | 2 +- src/repo.v | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/archive.v b/src/archive.v index 7ded6a1b..bdc18462 100644 --- a/src/archive.v +++ b/src/archive.v @@ -92,7 +92,7 @@ fn C.archive_entry_set_perm(&C.archive_entry, int) fn C.archive_entry_clear(&C.archive_entry) // Copy over a stat struct to the archive entry -fn C.archive_entry_copy_stat(&C.archive_entry, &C.stat) +fn C.archive_entry_copy_stat(entry &C.archive_entry, const_stat &C.stat) #include diff --git a/src/repo.v b/src/repo.v index ff849097..6ddde2aa 100644 --- a/src/repo.v +++ b/src/repo.v @@ -119,10 +119,10 @@ fn (r &Repo) sync() ? { } C.archive_entry_set_pathname(entry, &char(inner_path.str)) - // C.archive_entry_copy_stat(entry, &st) - C.archive_entry_set_size(entry, st.st_size) - C.archive_entry_set_filetype(entry, C.AE_IFREG) - C.archive_entry_set_perm(entry, 0o644) + C.archive_entry_copy_stat(entry, &st) + // C.archive_entry_set_size(entry, st.st_size) + // C.archive_entry_set_filetype(entry, C.AE_IFREG) + // C.archive_entry_set_perm(entry, 0o644) C.archive_write_header(a, entry) fd := C.open(&char(actual_path.str), C.O_RDONLY) From f1a17cab22816bbfa1f0f9ab3f3fe883f8e3a9f7 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 20 Jan 2022 19:39:38 +0100 Subject: [PATCH 22/24] Failed attempt at fixing memory bug --- src/package.v | 14 +++++++--- src/repo.v | 75 ++++++++++++++++++++++++++------------------------- src/routes.v | 2 +- 3 files changed, 51 insertions(+), 40 deletions(-) diff --git a/src/package.v b/src/package.v index 9e8e5038..c229e3ff 100644 --- a/src/package.v +++ b/src/package.v @@ -126,6 +126,7 @@ pub fn read_pkg(pkg_path string) ?Pkg { mut buf := voidptr(0) mut files := []string{} + mut pkg_info := PkgInfo{} for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { pathname := C.archive_entry_pathname(entry) @@ -142,14 +143,21 @@ pub fn read_pkg(pkg_path string) ?Pkg { // TODO can this unsafe block be avoided? buf = unsafe { malloc(size) } - C.archive_read_data(a, voidptr(buf), size) + C.archive_read_data(a, buf, size) + + unsafe { + println(cstring_to_vstring(buf)) + } + pkg_info = parse_pkg_info_string(unsafe { cstring_to_vstring(buf) }) ? + + unsafe { + free(buf) + } } else { C.archive_read_data_skip(a) } } - mut pkg_info := parse_pkg_info_string(unsafe { cstring_to_vstring(&char(buf)) }) ? - pkg_info.csize = i64(os.file_size(pkg_path)) return Pkg{ diff --git a/src/repo.v b/src/repo.v index 6ddde2aa..cca6b848 100644 --- a/src/repo.v +++ b/src/repo.v @@ -45,7 +45,7 @@ pub fn new(repo_dir string, pkg_dir string) ?Repo { // add_from_path adds a package from an arbitrary path & moves it into the pkgs // directory if necessary. pub fn (r &Repo) add_from_path(pkg_path string) ?bool { - pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file.') } + pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') } added := r.add(pkg) ? @@ -96,47 +96,50 @@ fn (r &Repo) pkg_path(pkg &package.Pkg) string { // Re-generate the repo archive files fn (r &Repo) sync() ? { - a := C.archive_write_new() - entry := C.archive_entry_new() - st := C.stat{} - buf := [8192]byte{} + lock r.mutex { + a := C.archive_write_new() + entry := C.archive_entry_new() + st := C.stat{} + buf := [8192]byte{} - // This makes the archive a gzip-compressed tarball - C.archive_write_add_filter_gzip(a) - C.archive_write_set_format_pax_restricted(a) + // This makes the archive a gzip-compressed tarball + C.archive_write_add_filter_gzip(a) + C.archive_write_set_format_pax_restricted(a) - repo_path := os.join_path_single(r.repo_dir, 'repo.db') + repo_path := os.join_path_single(r.repo_dir, 'repo.db') - C.archive_write_open_filename(a, &char(repo_path.str)) + C.archive_write_open_filename(a, &char(repo_path.str)) - // Iterate over each directory - for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, it))) { - inner_path := os.join_path_single(d, 'desc') - actual_path := os.join_path_single(r.repo_dir, inner_path) + // Iterate over each directory + for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, + it))) { + inner_path := os.join_path_single(d, 'desc') + actual_path := os.join_path_single(r.repo_dir, inner_path) - unsafe { - C.stat(&char(actual_path.str), &st) + unsafe { + C.stat(&char(actual_path.str), &st) + } + + C.archive_entry_set_pathname(entry, &char(inner_path.str)) + C.archive_entry_copy_stat(entry, &st) + // C.archive_entry_set_size(entry, st.st_size) + // C.archive_entry_set_filetype(entry, C.AE_IFREG) + // C.archive_entry_set_perm(entry, 0o644) + C.archive_write_header(a, entry) + + fd := C.open(&char(actual_path.str), C.O_RDONLY) + mut len := C.read(fd, &buf, sizeof(buf)) + + for len > 0 { + C.archive_write_data(a, &buf, len) + len = C.read(fd, &buf, sizeof(buf)) + } + C.close(fd) + + C.archive_entry_clear(entry) } - C.archive_entry_set_pathname(entry, &char(inner_path.str)) - C.archive_entry_copy_stat(entry, &st) - // C.archive_entry_set_size(entry, st.st_size) - // C.archive_entry_set_filetype(entry, C.AE_IFREG) - // C.archive_entry_set_perm(entry, 0o644) - C.archive_write_header(a, entry) - - fd := C.open(&char(actual_path.str), C.O_RDONLY) - mut len := C.read(fd, &buf, sizeof(buf)) - - for len > 0 { - C.archive_write_data(a, &buf, len) - len = C.read(fd, &buf, sizeof(buf)) - } - C.close(fd) - - C.archive_entry_clear(entry) + C.archive_write_close(a) + C.archive_write_free(a) } - - C.archive_write_close(a) - C.archive_write_free(a) } diff --git a/src/routes.v b/src/routes.v index c7e38ba1..8ad4a4a0 100644 --- a/src/routes.v +++ b/src/routes.v @@ -74,7 +74,7 @@ fn (mut app App) put_package() web.Result { } added := app.repo.add_from_path(pkg_path) or { - app.lerror('Error while adding package.') + app.lerror('Error while adding package: $err.msg') return app.text('Failed to add package.') } From e817535e86b7c9b255d941267653c7f1c3a2d0e3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 20 Jan 2022 20:42:18 +0100 Subject: [PATCH 23/24] Removed some commented-out code --- src/main.v | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/main.v b/src/main.v index ed6383e0..97788e7e 100644 --- a/src/main.v +++ b/src/main.v @@ -101,20 +101,3 @@ fn main() { repo: repo }, port) } - -// fn main() { -// r := repo.new('data/repo', 'data/pkgs') or { return } -// print(r.add_from_path('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { -// panic('you fialed') -// }) - -// // archive.list_filenames() -// // res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { -// // eprintln(err.msg) -// // return -// // } -// // println(info) -// // println('hey') -// // print(res.to_desc()) -// // print(res.to_files()) -// } From ad7b42758ed447aeb13ff95a43775983e03e798f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 20 Jan 2022 20:47:15 +0100 Subject: [PATCH 24/24] Hopefully fixed image dryrun ci setting --- .woodpecker/.image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.woodpecker/.image.yml b/.woodpecker/.image.yml index 6b788613..7f583a8a 100644 --- a/.woodpecker/.image.yml +++ b/.woodpecker/.image.yml @@ -1,6 +1,6 @@ -# These checks already get performed on the feature branches -branches: - exclude: [ main, dev ] +# Because the only step here is a pull_request event, the branch should be dev +# because it has to be the target of the pull request +branches: dev platform: linux/amd64 pipeline: