diff --git a/.gitignore b/.gitignore index 8c67f97b..71064b15 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ vieter.log # External lib; gets added by Makefile libarchive-* +test/ diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 4830d103..cc66bc7f 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -17,6 +17,8 @@ pipeline: prod: image: 'chewingbever/vlang:latest' + environment: + - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static group: 'build' commands: - make prod diff --git a/Dockerfile.builder b/Dockerfile.builder index b8e45aa7..d45fe830 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -14,6 +14,7 @@ RUN apk --no-cache add \ git make upx gcc bash \ musl-dev \ openssl-libs-static openssl-dev \ + zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \ sqlite-static sqlite-dev \ libx11-dev glfw-dev freetype-dev \ libarchive-static libarchive-dev \ diff --git a/Makefile b/Makefile index 38593bee..bf77b629 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ run: vieter .PHONY: run-prod run-prod: prod - API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG ./vieter-prod + API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG ./pvieter # Same as run, but restart when the source code changes .PHONY: watch diff --git a/README.md b/README.md index e3c79a24..5d495770 100644 --- a/README.md +++ b/README.md @@ -28,3 +28,12 @@ daemon to start builds, which are then uploaded to the server's repository. The server also allows for non-agents to upload packages, as long as they have the required secrets. This allows me to also develop non-git packages, such as my terminal, & upload them to the servers using CI. + +## Directory Structure + +The data directory consists of three main directories: + +* `downloads` - This is where packages are initially downloaded. Because vieter moves files from this folder to the `pkgs` folder, these two folders should best be on the same drive +* `pkgs` - This is where approved package files are stored. +* `repos` - Each repository gets a subfolder here. The subfolder contains the uncompressed contents of the db file. + * Each repo subdirectory contains the compressed db & files archive for the repository, alongside a directory called `files` which contains the uncompressed contents. diff --git a/src/archive/archive.v b/src/archive/archive.v index ca911c70..88649eea 100644 --- a/src/archive/archive.v +++ b/src/archive/archive.v @@ -2,7 +2,8 @@ module archive import os -pub fn get_pkg_info(pkg_path string) ?string { +// Returns the .PKGINFO file's contents & the list of files. +pub fn pkg_info(pkg_path string) ?(string, []string) { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") } @@ -26,18 +27,27 @@ pub fn get_pkg_info(pkg_path string) ?string { // We iterate over every header in search of the .PKGINFO one mut buf := voidptr(0) + mut files := []string{} for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { - if C.strcmp(C.archive_entry_pathname(entry), c'.PKGINFO') == 0 { + pathname := C.archive_entry_pathname(entry) + + ignored_names := [c'.BUILDINFO', c'.INSTALL', c'.MTREE', c'.PKGINFO', c'.CHANGELOG'] + if ignored_names.all(C.strcmp(it, pathname) != 0) { + unsafe { + files << cstring_to_vstring(pathname) + } + } + + if C.strcmp(pathname, c'.PKGINFO') == 0 { size := C.archive_entry_size(entry) // TODO can this unsafe block be avoided? buf = unsafe { malloc(size) } C.archive_read_data(a, voidptr(buf), size) - break } else { C.archive_read_data_skip(a) } } - return unsafe { cstring_to_vstring(&char(buf)) } + return unsafe { cstring_to_vstring(&char(buf)) }, files } diff --git a/src/main.v b/src/main.v index af6b06c5..9d9e5419 100644 --- a/src/main.v +++ b/src/main.v @@ -4,8 +4,9 @@ import web import os import log import io -import repo +import pkg import archive +import repo const port = 8000 @@ -54,59 +55,62 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } } -fn main() { - // Configure logger - log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } - log_level := log.level_from_tag(log_level_str) or { - exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') - } - log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } - - mut logger := log.Log{ - level: log_level - } - - logger.set_full_logpath(log_file) - logger.log_to_console_too() - - defer { - logger.info('Flushing log file') - logger.flush() - logger.close() - } - - // Configure web server - key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } - repo_dir := os.getenv_opt('REPO_DIR') or { - exit_with_message(1, 'No repo directory was configured.') - } - - repo := repo.Repo{ - dir: repo_dir - name: db_name - } - - // We create the upload directory during startup - if !os.is_dir(repo.pkg_dir()) { - os.mkdir_all(repo.pkg_dir()) or { - exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.") - } - - logger.info("Created package directory '$repo.pkg_dir()'.") - } - - web.run(&App{ - logger: logger - api_key: key - repo: repo - }, port) -} - -// fn main() { -// // archive.list_filenames() -// info := archive.get_pkg_info('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { -// eprintln(err.msg) -// return +// fn main2() { +// // Configure logger +// log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } +// log_level := log.level_from_tag(log_level_str) or { +// exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') // } -// println(info) +// log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } + +// mut logger := log.Log{ +// level: log_level +// } + +// logger.set_full_logpath(log_file) +// logger.log_to_console_too() + +// defer { +// logger.info('Flushing log file') +// logger.flush() +// logger.close() +// } + +// // Configure web server +// key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } +// repo_dir := os.getenv_opt('REPO_DIR') or { +// exit_with_message(1, 'No repo directory was configured.') +// } + +// repo := repo.Repo{ +// dir: repo_dir +// name: db_name +// } + +// // We create the upload directory during startup +// if !os.is_dir(repo.pkg_dir()) { +// os.mkdir_all(repo.pkg_dir()) or { +// exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.") +// } + +// logger.info("Created package directory '$repo.pkg_dir()'.") +// } + +// web.run(&App{ +// logger: logger +// api_key: key +// repo: repo +// }, port) // } + +fn main() { + // archive.list_filenames() + res := pkg.read_pkg('test/homebank-5.5.1-1-x86_64.pkg.tar.zst') or { + eprintln(err.msg) + return + } + // println(info) + println(res.info) + print(res.files) + println(res.info.to_desc()) +} diff --git a/src/pkg.v b/src/pkg.v new file mode 100644 index 00000000..1bb1abc2 --- /dev/null +++ b/src/pkg.v @@ -0,0 +1,108 @@ +module pkg + +import archive +import time + +struct Pkg { +pub: + info PkgInfo [required] + files []string [required] +} + +struct PkgInfo { +mut: + // Single values + name string + base string + version string + description string + size i64 + csize i64 + url string + arch string + build_date i64 + packager string + md5sum string + sha256sum string + pgpsig string + pgpsigsize i64 + // Array values + groups []string + licenses []string + replaces []string + depends []string + conflicts []string + provides []string + optdepends []string + makedepends []string + checkdepends []string +} + +fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { + mut pkg_info := PkgInfo{} + + // Iterate over the entire string + for line in pkg_info_str.split_into_lines() { + // Skip any comment lines + if line.starts_with('#') { + continue + } + parts := line.split_nth('=', 2) + + if parts.len < 2 { + return error('Invalid line detected.') + } + + value := parts[1].trim_space() + key := parts[0].trim_space() + + match key { + // Single values + 'pkgname' { pkg_info.name = value } + 'pkgbase' { pkg_info.base = value } + 'pkgver' { pkg_info.version = value } + 'pkgdesc' { pkg_info.description = value } + 'csize' { pkg_info.csize = value.int() } + 'size' { pkg_info.size = value.int() } + 'url' { pkg_info.url = value } + 'arch' { pkg_info.arch = value } + 'builddate' { pkg_info.build_date = value.int() } + 'packager' { pkg_info.packager = value } + 'md5sum' { pkg_info.md5sum = value } + 'sha256sum' { pkg_info.sha256sum = value } + 'pgpsig' { pkg_info.pgpsig = value } + 'pgpsigsize' { pkg_info.pgpsigsize = value.int() } + // Array values + 'group' { pkg_info.groups << value } + 'license' { pkg_info.licenses << value } + 'replaces' { pkg_info.replaces << value } + 'depend' { pkg_info.depends << value } + 'conflict' { pkg_info.conflicts << value } + 'provides' { pkg_info.provides << value } + 'optdepend' { pkg_info.optdepends << value } + 'makedepend' { pkg_info.makedepends << value } + 'checkdepend' { pkg_info.checkdepends << value } + else { return error("Invalid key '$key'.") } + } + } + + return pkg_info +} + +pub fn read_pkg(pkg_path string) ?Pkg { + pkg_info_str, files := archive.pkg_info(pkg_path) ? + pkg_info := parse_pkg_info_string(pkg_info_str) ? + + return Pkg{ + info: pkg_info + files: files + } +} + +// Represent a PkgInfo struct as a desc file +pub fn (p &PkgInfo) to_desc() string { + // TODO calculate md5 & sha256 instead of believing the file + mut desc := '' + + return desc +} diff --git a/src/repo.v b/src/repo.v index 6b307e89..ae76cc9a 100644 --- a/src/repo.v +++ b/src/repo.v @@ -1,6 +1,7 @@ module repo import os +import archive const pkgs_subpath = 'pkgs' @@ -10,23 +11,35 @@ pub struct Dummy { x int } -// Handles management of a repository. Package files are stored in '$dir/pkgs' -// & moved there if necessary. +// This struct manages a single repository. pub struct Repo { mut: mutex shared Dummy pub: - dir string [required] - name string [required] + // Where to store repository files; should exist + repo_dir string [required] + // Where to find packages; packages are expected to all be in the same directory + pkg_dir string [required] } -pub fn (r &Repo) pkg_dir() string { - return os.join_path_single(r.dir, repo.pkgs_subpath) +// Returns whether the repository contains the given package. +pub fn (r &Repo) contains(pkg string) bool { + return os.exists(os.join_path(r.repo_dir, 'files', pkg)) +} + +// Adds the given package to the repo. If false, the package was already +// present in the repository. +pub fn (r &Repo) add(pkg string) ?bool { + return false +} + +// Re-generate the db & files archives. +fn (r &Repo) genenerate() ? { } // Returns path to the given package, prepended with the repo's path. pub fn (r &Repo) pkg_path(pkg string) string { - return os.join_path(r.dir, repo.pkgs_subpath, pkg) + return os.join_path_single(r.pkg_dir, pkg) } pub fn (r &Repo) exists(pkg string) bool { @@ -35,7 +48,7 @@ pub fn (r &Repo) exists(pkg string) bool { // Returns the full path to the database file pub fn (r &Repo) db_path() string { - return os.join_path_single(r.dir, '${r.name}.tar.gz') + return os.join_path_single(r.repo_dir, 'repo.tar.gz') } pub fn (r &Repo) add_package(pkg_path string) ? { diff --git a/src/routes.v b/src/routes.v index 0b570ca0..c7df2868 100644 --- a/src/routes.v +++ b/src/routes.v @@ -28,62 +28,67 @@ fn (mut app App) get_root(filename string) web.Result { mut full_path := '' if is_pkg_name(filename) { - full_path = os.join_path_single(app.repo.pkg_dir(), filename) + full_path = os.join_path_single(app.repo.pkg_dir, filename) } else { - full_path = os.join_path_single(app.repo.dir, filename) + full_path = os.join_path_single(app.repo.repo_dir, filename) } return app.file(full_path) } -['/pkgs/:pkg'; put] -fn (mut app App) put_package(pkg string) web.Result { - if !app.is_authorized() { - return app.text('Unauthorized.') - } +// ['/pkgs/:pkg'; put] +// fn (mut app App) put_package(pkg string) web.Result { +// if !app.is_authorized() { +// return app.text('Unauthorized.') +// } - if !is_pkg_name(pkg) { - app.lwarn("Invalid package name '$pkg'.") +// if !is_pkg_name(pkg) { +// app.lwarn("Invalid package name '$pkg'.") - return app.text('Invalid filename.') - } +// return app.text('Invalid filename.') +// } - if app.repo.exists(pkg) { - app.lwarn("Duplicate package '$pkg'") +// if app.repo.exists(pkg) { +// app.lwarn("Duplicate package '$pkg'") - return app.text('File already exists.') - } +// return app.text('File already exists.') +// } - pkg_path := app.repo.pkg_path(pkg) +// pkg_path := app.repo.pkg_path(pkg) - if length := app.req.header.get(.content_length) { - app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.") +// if length := app.req.header.get(.content_length) { +// app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.") - // This is used to time how long it takes to upload a file - mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) +// // This is used to time how long it takes to upload a file +// mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) - reader_to_file(mut app.reader, length.int(), pkg_path) or { - app.lwarn("Failed to upload package '$pkg'") +// reader_to_file(mut app.reader, length.int(), pkg_path) or { +// app.lwarn("Failed to upload package '$pkg'") - return app.text('Failed to upload file.') - } +// return app.text('Failed to upload file.') +// } - sw.stop() - app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.") - } else { - app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.") - return app.text("Content-Type header isn't set.") - } +// sw.stop() +// app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.") +// } else { +// app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.") +// return app.text("Content-Type header isn't set.") +// } - app.repo.add_package(pkg_path) or { - app.lwarn("Failed to add package '$pkg' to database.") +// app.repo.add_package(pkg_path) or { +// app.lwarn("Failed to add package '$pkg' to database.") - os.rm(pkg_path) or { println('Failed to remove $pkg_path') } +// os.rm(pkg_path) or { println('Failed to remove $pkg_path') } - return app.text('Failed to add package to repo.') - } +// return app.text('Failed to add package to repo.') +// } - app.linfo("Added '$pkg' to repository.") +// app.linfo("Added '$pkg' to repository.") - return app.text('Package added successfully.') +// return app.text('Package added successfully.') +// } + +['/add'; put] +pub fn (mut app App) add_package() web.Result { + return app.text('') }