From d4b7a25c06da5c900e40b19652d90b20d4635900 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 13 Jan 2022 21:47:14 +0100 Subject: [PATCH] Lots of restructuring for repo backend --- .gitignore | 1 + README.md | 9 +++++ src/archive/archive.v | 15 ++++++-- src/main.v | 88 ++++++++++++++++++++++--------------------- src/{repo => }/pkg.v | 76 +++++++++++++++++++++++-------------- src/{repo => }/repo.v | 28 ++++++++++---- src/routes.v | 79 ++++++++++++++++++++------------------ 7 files changed, 176 insertions(+), 120 deletions(-) rename src/{repo => }/pkg.v (63%) rename src/{repo => }/repo.v (51%) diff --git a/.gitignore b/.gitignore index 8c67f97..71064b1 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ vieter.log # External lib; gets added by Makefile libarchive-* +test/ diff --git a/README.md b/README.md index e3c79a2..5d49577 100644 --- a/README.md +++ b/README.md @@ -28,3 +28,12 @@ daemon to start builds, which are then uploaded to the server's repository. The server also allows for non-agents to upload packages, as long as they have the required secrets. This allows me to also develop non-git packages, such as my terminal, & upload them to the servers using CI. + +## Directory Structure + +The data directory consists of three main directories: + +* `downloads` - This is where packages are initially downloaded. Because vieter moves files from this folder to the `pkgs` folder, these two folders should best be on the same drive +* `pkgs` - This is where approved package files are stored. +* `repos` - Each repository gets a subfolder here. The subfolder contains the uncompressed contents of the db file. + * Each repo subdirectory contains the compressed db & files archive for the repository, alongside a directory called `files` which contains the uncompressed contents. diff --git a/src/archive/archive.v b/src/archive/archive.v index 011333f..2983350 100644 --- a/src/archive/archive.v +++ b/src/archive/archive.v @@ -2,7 +2,8 @@ module archive import os -pub fn pkg_info_string(pkg_path string) ?string { +// Returns the .PKGINFO file's contents & the list of files. +pub fn pkg_info(pkg_path string) ?(string, []string) { if !os.is_file(pkg_path) { return error("'$pkg_path' doesn't exist or isn't a file.") } @@ -26,18 +27,24 @@ pub fn pkg_info_string(pkg_path string) ?string { // We iterate over every header in search of the .PKGINFO one mut buf := voidptr(0) + mut files := []string{} for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { - if C.strcmp(C.archive_entry_pathname(entry), c'.PKGINFO') == 0 { + pathname := C.archive_entry_pathname(entry) + + unsafe { + files << cstring_to_vstring(pathname) + } + + if C.strcmp(pathname, c'.PKGINFO') == 0 { size := C.archive_entry_size(entry) // TODO can this unsafe block be avoided? buf = unsafe { malloc(size) } C.archive_read_data(a, voidptr(buf), size) - break } else { C.archive_read_data_skip(a) } } - return unsafe { cstring_to_vstring(&char(buf)) } + return unsafe { cstring_to_vstring(&char(buf)) }, files } diff --git a/src/main.v b/src/main.v index 0b27736..e6b5bc6 100644 --- a/src/main.v +++ b/src/main.v @@ -4,8 +4,9 @@ import web import os import log import io -import repo +import pkg import archive +import repo const port = 8000 @@ -54,59 +55,62 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } } -fn main2() { - // Configure logger - log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } - log_level := log.level_from_tag(log_level_str) or { - exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') - } - log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } +// fn main2() { +// // Configure logger +// log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } +// log_level := log.level_from_tag(log_level_str) or { +// exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') +// } +// log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } - mut logger := log.Log{ - level: log_level - } +// mut logger := log.Log{ +// level: log_level +// } - logger.set_full_logpath(log_file) - logger.log_to_console_too() +// logger.set_full_logpath(log_file) +// logger.log_to_console_too() - defer { - logger.info('Flushing log file') - logger.flush() - logger.close() - } +// defer { +// logger.info('Flushing log file') +// logger.flush() +// logger.close() +// } - // Configure web server - key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } - repo_dir := os.getenv_opt('REPO_DIR') or { - exit_with_message(1, 'No repo directory was configured.') - } +// // Configure web server +// key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } +// repo_dir := os.getenv_opt('REPO_DIR') or { +// exit_with_message(1, 'No repo directory was configured.') +// } - repo := repo.Repo{ - dir: repo_dir - name: db_name - } +// repo := repo.Repo{ +// dir: repo_dir +// name: db_name +// } - // We create the upload directory during startup - if !os.is_dir(repo.pkg_dir()) { - os.mkdir_all(repo.pkg_dir()) or { - exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.") - } +// // We create the upload directory during startup +// if !os.is_dir(repo.pkg_dir()) { +// os.mkdir_all(repo.pkg_dir()) or { +// exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.") +// } - logger.info("Created package directory '$repo.pkg_dir()'.") - } +// logger.info("Created package directory '$repo.pkg_dir()'.") +// } - web.run(&App{ - logger: logger - api_key: key - repo: repo - }, port) -} +// web.run(&App{ +// logger: logger +// api_key: key +// repo: repo +// }, port) +// } fn main() { // archive.list_filenames() - info := repo.get_pkg_info('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { + res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { eprintln(err.msg) return } - println(info) + // println(info) + println(res.info) + print(res.files) + println(res.info.to_desc()) } diff --git a/src/repo/pkg.v b/src/pkg.v similarity index 63% rename from src/repo/pkg.v rename to src/pkg.v index b45839b..1bb1abc 100644 --- a/src/repo/pkg.v +++ b/src/pkg.v @@ -1,45 +1,46 @@ -module repo +module pkg import archive import time +struct Pkg { +pub: + info PkgInfo [required] + files []string [required] +} + struct PkgInfo { mut: // Single values - name string - base string - version string + name string + base string + version string description string - size i64 - csize i64 - url string - arch string - build_date i64 - packager string - md5sum string - sha256sum string - pgpsig string - pgpsigsize i64 - + size i64 + csize i64 + url string + arch string + build_date i64 + packager string + md5sum string + sha256sum string + pgpsig string + pgpsigsize i64 // Array values - groups []string - licenses []string - replaces []string - depends []string - conflicts []string - provides []string - optdepends []string - makedepends []string + groups []string + licenses []string + replaces []string + depends []string + conflicts []string + provides []string + optdepends []string + makedepends []string checkdepends []string } -pub fn get_pkg_info(pkg_path string) ?PkgInfo { - pkg_info_str := archive.pkg_info_string(pkg_path) ? +fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { mut pkg_info := PkgInfo{} - mut i := 0 - mut j := 0 - // Iterate over the entire string for line in pkg_info_str.split_into_lines() { // Skip any comment lines @@ -71,7 +72,6 @@ pub fn get_pkg_info(pkg_path string) ?PkgInfo { 'sha256sum' { pkg_info.sha256sum = value } 'pgpsig' { pkg_info.pgpsig = value } 'pgpsigsize' { pkg_info.pgpsigsize = value.int() } - // Array values 'group' { pkg_info.groups << value } 'license' { pkg_info.licenses << value } @@ -88,3 +88,21 @@ pub fn get_pkg_info(pkg_path string) ?PkgInfo { return pkg_info } + +pub fn read_pkg(pkg_path string) ?Pkg { + pkg_info_str, files := archive.pkg_info(pkg_path) ? + pkg_info := parse_pkg_info_string(pkg_info_str) ? + + return Pkg{ + info: pkg_info + files: files + } +} + +// Represent a PkgInfo struct as a desc file +pub fn (p &PkgInfo) to_desc() string { + // TODO calculate md5 & sha256 instead of believing the file + mut desc := '' + + return desc +} diff --git a/src/repo/repo.v b/src/repo.v similarity index 51% rename from src/repo/repo.v rename to src/repo.v index 494f8c9..ae76cc9 100644 --- a/src/repo/repo.v +++ b/src/repo.v @@ -11,23 +11,35 @@ pub struct Dummy { x int } -// Handles management of a repository. Package files are stored in '$dir/pkgs' -// & moved there if necessary. +// This struct manages a single repository. pub struct Repo { mut: mutex shared Dummy pub: - dir string [required] - name string [required] + // Where to store repository files; should exist + repo_dir string [required] + // Where to find packages; packages are expected to all be in the same directory + pkg_dir string [required] } -pub fn (r &Repo) pkg_dir() string { - return os.join_path_single(r.dir, repo.pkgs_subpath) +// Returns whether the repository contains the given package. +pub fn (r &Repo) contains(pkg string) bool { + return os.exists(os.join_path(r.repo_dir, 'files', pkg)) +} + +// Adds the given package to the repo. If false, the package was already +// present in the repository. +pub fn (r &Repo) add(pkg string) ?bool { + return false +} + +// Re-generate the db & files archives. +fn (r &Repo) genenerate() ? { } // Returns path to the given package, prepended with the repo's path. pub fn (r &Repo) pkg_path(pkg string) string { - return os.join_path(r.dir, repo.pkgs_subpath, pkg) + return os.join_path_single(r.pkg_dir, pkg) } pub fn (r &Repo) exists(pkg string) bool { @@ -36,7 +48,7 @@ pub fn (r &Repo) exists(pkg string) bool { // Returns the full path to the database file pub fn (r &Repo) db_path() string { - return os.join_path_single(r.dir, '${r.name}.tar.gz') + return os.join_path_single(r.repo_dir, 'repo.tar.gz') } pub fn (r &Repo) add_package(pkg_path string) ? { diff --git a/src/routes.v b/src/routes.v index 0b570ca..c7df286 100644 --- a/src/routes.v +++ b/src/routes.v @@ -28,62 +28,67 @@ fn (mut app App) get_root(filename string) web.Result { mut full_path := '' if is_pkg_name(filename) { - full_path = os.join_path_single(app.repo.pkg_dir(), filename) + full_path = os.join_path_single(app.repo.pkg_dir, filename) } else { - full_path = os.join_path_single(app.repo.dir, filename) + full_path = os.join_path_single(app.repo.repo_dir, filename) } return app.file(full_path) } -['/pkgs/:pkg'; put] -fn (mut app App) put_package(pkg string) web.Result { - if !app.is_authorized() { - return app.text('Unauthorized.') - } +// ['/pkgs/:pkg'; put] +// fn (mut app App) put_package(pkg string) web.Result { +// if !app.is_authorized() { +// return app.text('Unauthorized.') +// } - if !is_pkg_name(pkg) { - app.lwarn("Invalid package name '$pkg'.") +// if !is_pkg_name(pkg) { +// app.lwarn("Invalid package name '$pkg'.") - return app.text('Invalid filename.') - } +// return app.text('Invalid filename.') +// } - if app.repo.exists(pkg) { - app.lwarn("Duplicate package '$pkg'") +// if app.repo.exists(pkg) { +// app.lwarn("Duplicate package '$pkg'") - return app.text('File already exists.') - } +// return app.text('File already exists.') +// } - pkg_path := app.repo.pkg_path(pkg) +// pkg_path := app.repo.pkg_path(pkg) - if length := app.req.header.get(.content_length) { - app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.") +// if length := app.req.header.get(.content_length) { +// app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.") - // This is used to time how long it takes to upload a file - mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) +// // This is used to time how long it takes to upload a file +// mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) - reader_to_file(mut app.reader, length.int(), pkg_path) or { - app.lwarn("Failed to upload package '$pkg'") +// reader_to_file(mut app.reader, length.int(), pkg_path) or { +// app.lwarn("Failed to upload package '$pkg'") - return app.text('Failed to upload file.') - } +// return app.text('Failed to upload file.') +// } - sw.stop() - app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.") - } else { - app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.") - return app.text("Content-Type header isn't set.") - } +// sw.stop() +// app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.") +// } else { +// app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.") +// return app.text("Content-Type header isn't set.") +// } - app.repo.add_package(pkg_path) or { - app.lwarn("Failed to add package '$pkg' to database.") +// app.repo.add_package(pkg_path) or { +// app.lwarn("Failed to add package '$pkg' to database.") - os.rm(pkg_path) or { println('Failed to remove $pkg_path') } +// os.rm(pkg_path) or { println('Failed to remove $pkg_path') } - return app.text('Failed to add package to repo.') - } +// return app.text('Failed to add package to repo.') +// } - app.linfo("Added '$pkg' to repository.") +// app.linfo("Added '$pkg' to repository.") - return app.text('Package added successfully.') +// return app.text('Package added successfully.') +// } + +['/add'; put] +pub fn (mut app App) add_package() web.Result { + return app.text('') }