diff --git a/.gitignore b/.gitignore index 71064b1..8c67f97 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,3 @@ vieter.log # External lib; gets added by Makefile libarchive-* -test/ diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index cc66bc7..4830d10 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -17,8 +17,6 @@ pipeline: prod: image: 'chewingbever/vlang:latest' - environment: - - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static group: 'build' commands: - make prod diff --git a/Dockerfile.builder b/Dockerfile.builder index d45fe83..b8e45aa 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -14,7 +14,6 @@ RUN apk --no-cache add \ git make upx gcc bash \ musl-dev \ openssl-libs-static openssl-dev \ - zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \ sqlite-static sqlite-dev \ libx11-dev glfw-dev freetype-dev \ libarchive-static libarchive-dev \ diff --git a/Makefile b/Makefile index bf77b62..38593be 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ run: vieter .PHONY: run-prod run-prod: prod - API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG ./pvieter + API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG ./vieter-prod # Same as run, but restart when the source code changes .PHONY: watch diff --git a/README.md b/README.md index 5d49577..e3c79a2 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,3 @@ daemon to start builds, which are then uploaded to the server's repository. The server also allows for non-agents to upload packages, as long as they have the required secrets. This allows me to also develop non-git packages, such as my terminal, & upload them to the servers using CI. - -## Directory Structure - -The data directory consists of three main directories: - -* `downloads` - This is where packages are initially downloaded. Because vieter moves files from this folder to the `pkgs` folder, these two folders should best be on the same drive -* `pkgs` - This is where approved package files are stored. -* `repos` - Each repository gets a subfolder here. The subfolder contains the uncompressed contents of the db file. - * Each repo subdirectory contains the compressed db & files archive for the repository, alongside a directory called `files` which contains the uncompressed contents. diff --git a/src/archive/archive.v b/src/archive/archive.v index 88649ee..ca911c7 100644 --- a/src/archive/archive.v +++ b/src/archive/archive.v @@ -2,8 +2,7 @@ module archive import os -// Returns the .PKGINFO file's contents & the list of files. -pub fn pkg_info(pkg_path string) ?(string, []string) { +pub fn get_pkg_info(pkg_path string) ?string { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") } @@ -27,27 +26,18 @@ pub fn pkg_info(pkg_path string) ?(string, []string) { // We iterate over every header in search of the .PKGINFO one mut buf := voidptr(0) - mut files := []string{} for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { - pathname := C.archive_entry_pathname(entry) - - ignored_names := [c'.BUILDINFO', c'.INSTALL', c'.MTREE', c'.PKGINFO', c'.CHANGELOG'] - if ignored_names.all(C.strcmp(it, pathname) != 0) { - unsafe { - files << cstring_to_vstring(pathname) - } - } - - if C.strcmp(pathname, c'.PKGINFO') == 0 { + if C.strcmp(C.archive_entry_pathname(entry), c'.PKGINFO') == 0 { size := C.archive_entry_size(entry) // TODO can this unsafe block be avoided? buf = unsafe { malloc(size) } C.archive_read_data(a, voidptr(buf), size) + break } else { C.archive_read_data_skip(a) } } - return unsafe { cstring_to_vstring(&char(buf)) }, files + return unsafe { cstring_to_vstring(&char(buf)) } } diff --git a/src/main.v b/src/main.v index 9d9e541..af6b06c 100644 --- a/src/main.v +++ b/src/main.v @@ -4,9 +4,8 @@ import web import os import log import io -import pkg -import archive import repo +import archive const port = 8000 @@ -55,62 +54,59 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } } -// fn main2() { -// // Configure logger -// log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } -// log_level := log.level_from_tag(log_level_str) or { -// exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') -// } -// log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } - -// mut logger := log.Log{ -// level: log_level -// } - -// logger.set_full_logpath(log_file) -// logger.log_to_console_too() - -// defer { -// logger.info('Flushing log file') -// logger.flush() -// logger.close() -// } - -// // Configure web server -// key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } -// repo_dir := os.getenv_opt('REPO_DIR') or { -// exit_with_message(1, 'No repo directory was configured.') -// } - -// repo := repo.Repo{ -// dir: repo_dir -// name: db_name -// } - -// // We create the upload directory during startup -// if !os.is_dir(repo.pkg_dir()) { -// os.mkdir_all(repo.pkg_dir()) or { -// exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.") -// } - -// logger.info("Created package directory '$repo.pkg_dir()'.") -// } - -// web.run(&App{ -// logger: logger -// api_key: key -// repo: repo -// }, port) -// } - fn main() { - // archive.list_filenames() - res := pkg.read_pkg('test/homebank-5.5.1-1-x86_64.pkg.tar.zst') or { - eprintln(err.msg) - return + // Configure logger + log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } + log_level := log.level_from_tag(log_level_str) or { + exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') } - // println(info) - println(res.info) - print(res.files) - println(res.info.to_desc()) + log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } + + mut logger := log.Log{ + level: log_level + } + + logger.set_full_logpath(log_file) + logger.log_to_console_too() + + defer { + logger.info('Flushing log file') + logger.flush() + logger.close() + } + + // Configure web server + key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } + repo_dir := os.getenv_opt('REPO_DIR') or { + exit_with_message(1, 'No repo directory was configured.') + } + + repo := repo.Repo{ + dir: repo_dir + name: db_name + } + + // We create the upload directory during startup + if !os.is_dir(repo.pkg_dir()) { + os.mkdir_all(repo.pkg_dir()) or { + exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.") + } + + logger.info("Created package directory '$repo.pkg_dir()'.") + } + + web.run(&App{ + logger: logger + api_key: key + repo: repo + }, port) } + +// fn main() { +// // archive.list_filenames() +// info := archive.get_pkg_info('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { +// eprintln(err.msg) +// return +// } +// println(info) +// } diff --git a/src/pkg.v b/src/pkg.v deleted file mode 100644 index 1bb1abc..0000000 --- a/src/pkg.v +++ /dev/null @@ -1,108 +0,0 @@ -module pkg - -import archive -import time - -struct Pkg { -pub: - info PkgInfo [required] - files []string [required] -} - -struct PkgInfo { -mut: - // Single values - name string - base string - version string - description string - size i64 - csize i64 - url string - arch string - build_date i64 - packager string - md5sum string - sha256sum string - pgpsig string - pgpsigsize i64 - // Array values - groups []string - licenses []string - replaces []string - depends []string - conflicts []string - provides []string - optdepends []string - makedepends []string - checkdepends []string -} - -fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { - mut pkg_info := PkgInfo{} - - // Iterate over the entire string - for line in pkg_info_str.split_into_lines() { - // Skip any comment lines - if line.starts_with('#') { - continue - } - parts := line.split_nth('=', 2) - - if parts.len < 2 { - return error('Invalid line detected.') - } - - value := parts[1].trim_space() - key := parts[0].trim_space() - - match key { - // Single values - 'pkgname' { pkg_info.name = value } - 'pkgbase' { pkg_info.base = value } - 'pkgver' { pkg_info.version = value } - 'pkgdesc' { pkg_info.description = value } - 'csize' { pkg_info.csize = value.int() } - 'size' { pkg_info.size = value.int() } - 'url' { pkg_info.url = value } - 'arch' { pkg_info.arch = value } - 'builddate' { pkg_info.build_date = value.int() } - 'packager' { pkg_info.packager = value } - 'md5sum' { pkg_info.md5sum = value } - 'sha256sum' { pkg_info.sha256sum = value } - 'pgpsig' { pkg_info.pgpsig = value } - 'pgpsigsize' { pkg_info.pgpsigsize = value.int() } - // Array values - 'group' { pkg_info.groups << value } - 'license' { pkg_info.licenses << value } - 'replaces' { pkg_info.replaces << value } - 'depend' { pkg_info.depends << value } - 'conflict' { pkg_info.conflicts << value } - 'provides' { pkg_info.provides << value } - 'optdepend' { pkg_info.optdepends << value } - 'makedepend' { pkg_info.makedepends << value } - 'checkdepend' { pkg_info.checkdepends << value } - else { return error("Invalid key '$key'.") } - } - } - - return pkg_info -} - -pub fn read_pkg(pkg_path string) ?Pkg { - pkg_info_str, files := archive.pkg_info(pkg_path) ? - pkg_info := parse_pkg_info_string(pkg_info_str) ? - - return Pkg{ - info: pkg_info - files: files - } -} - -// Represent a PkgInfo struct as a desc file -pub fn (p &PkgInfo) to_desc() string { - // TODO calculate md5 & sha256 instead of believing the file - mut desc := '' - - return desc -} diff --git a/src/repo.v b/src/repo.v index ae76cc9..6b307e8 100644 --- a/src/repo.v +++ b/src/repo.v @@ -1,7 +1,6 @@ module repo import os -import archive const pkgs_subpath = 'pkgs' @@ -11,35 +10,23 @@ pub struct Dummy { x int } -// This struct manages a single repository. +// Handles management of a repository. Package files are stored in '$dir/pkgs' +// & moved there if necessary. pub struct Repo { mut: mutex shared Dummy pub: - // Where to store repository files; should exist - repo_dir string [required] - // Where to find packages; packages are expected to all be in the same directory - pkg_dir string [required] + dir string [required] + name string [required] } -// Returns whether the repository contains the given package. -pub fn (r &Repo) contains(pkg string) bool { - return os.exists(os.join_path(r.repo_dir, 'files', pkg)) -} - -// Adds the given package to the repo. If false, the package was already -// present in the repository. -pub fn (r &Repo) add(pkg string) ?bool { - return false -} - -// Re-generate the db & files archives. -fn (r &Repo) genenerate() ? { +pub fn (r &Repo) pkg_dir() string { + return os.join_path_single(r.dir, repo.pkgs_subpath) } // Returns path to the given package, prepended with the repo's path. pub fn (r &Repo) pkg_path(pkg string) string { - return os.join_path_single(r.pkg_dir, pkg) + return os.join_path(r.dir, repo.pkgs_subpath, pkg) } pub fn (r &Repo) exists(pkg string) bool { @@ -48,7 +35,7 @@ pub fn (r &Repo) exists(pkg string) bool { // Returns the full path to the database file pub fn (r &Repo) db_path() string { - return os.join_path_single(r.repo_dir, 'repo.tar.gz') + return os.join_path_single(r.dir, '${r.name}.tar.gz') } pub fn (r &Repo) add_package(pkg_path string) ? { diff --git a/src/routes.v b/src/routes.v index c7df286..0b570ca 100644 --- a/src/routes.v +++ b/src/routes.v @@ -28,67 +28,62 @@ fn (mut app App) get_root(filename string) web.Result { mut full_path := '' if is_pkg_name(filename) { - full_path = os.join_path_single(app.repo.pkg_dir, filename) + full_path = os.join_path_single(app.repo.pkg_dir(), filename) } else { - full_path = os.join_path_single(app.repo.repo_dir, filename) + full_path = os.join_path_single(app.repo.dir, filename) } return app.file(full_path) } -// ['/pkgs/:pkg'; put] -// fn (mut app App) put_package(pkg string) web.Result { -// if !app.is_authorized() { -// return app.text('Unauthorized.') -// } +['/pkgs/:pkg'; put] +fn (mut app App) put_package(pkg string) web.Result { + if !app.is_authorized() { + return app.text('Unauthorized.') + } -// if !is_pkg_name(pkg) { -// app.lwarn("Invalid package name '$pkg'.") + if !is_pkg_name(pkg) { + app.lwarn("Invalid package name '$pkg'.") -// return app.text('Invalid filename.') -// } + return app.text('Invalid filename.') + } -// if app.repo.exists(pkg) { -// app.lwarn("Duplicate package '$pkg'") + if app.repo.exists(pkg) { + app.lwarn("Duplicate package '$pkg'") -// return app.text('File already exists.') -// } + return app.text('File already exists.') + } -// pkg_path := app.repo.pkg_path(pkg) + pkg_path := app.repo.pkg_path(pkg) -// if length := app.req.header.get(.content_length) { -// app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.") + if length := app.req.header.get(.content_length) { + app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.") -// // This is used to time how long it takes to upload a file -// mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) + // This is used to time how long it takes to upload a file + mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) -// reader_to_file(mut app.reader, length.int(), pkg_path) or { -// app.lwarn("Failed to upload package '$pkg'") + reader_to_file(mut app.reader, length.int(), pkg_path) or { + app.lwarn("Failed to upload package '$pkg'") -// return app.text('Failed to upload file.') -// } + return app.text('Failed to upload file.') + } -// sw.stop() -// app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.") -// } else { -// app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.") -// return app.text("Content-Type header isn't set.") -// } + sw.stop() + app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.") + } else { + app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.") + return app.text("Content-Type header isn't set.") + } -// app.repo.add_package(pkg_path) or { -// app.lwarn("Failed to add package '$pkg' to database.") + app.repo.add_package(pkg_path) or { + app.lwarn("Failed to add package '$pkg' to database.") -// os.rm(pkg_path) or { println('Failed to remove $pkg_path') } + os.rm(pkg_path) or { println('Failed to remove $pkg_path') } -// return app.text('Failed to add package to repo.') -// } + return app.text('Failed to add package to repo.') + } -// app.linfo("Added '$pkg' to repository.") + app.linfo("Added '$pkg' to repository.") -// return app.text('Package added successfully.') -// } - -['/add'; put] -pub fn (mut app App) add_package() web.Result { - return app.text('') + return app.text('Package added successfully.') }