Lots of restructuring for repo backend
ci/woodpecker/push/publish unknown status Details
ci/woodpecker/push/lint Pipeline was successful Details
ci/woodpecker/push/build Pipeline was successful Details

pull/26/head
Jef Roosens 2022-01-13 21:47:14 +01:00
parent 5616e7a4e2
commit d4b7a25c06
Signed by: Jef Roosens
GPG Key ID: 955C0660072F691F
7 changed files with 176 additions and 120 deletions

1
.gitignore vendored
View File

@ -14,3 +14,4 @@ vieter.log
# External lib; gets added by Makefile # External lib; gets added by Makefile
libarchive-* libarchive-*
test/

View File

@ -28,3 +28,12 @@ daemon to start builds, which are then uploaded to the server's repository. The
server also allows for non-agents to upload packages, as long as they have the server also allows for non-agents to upload packages, as long as they have the
required secrets. This allows me to also develop non-git packages, such as my required secrets. This allows me to also develop non-git packages, such as my
terminal, & upload them to the servers using CI. terminal, & upload them to the servers using CI.
## Directory Structure
The data directory consists of three main directories:
* `downloads` - This is where packages are initially downloaded. Because vieter moves files from this folder to the `pkgs` folder, these two folders should best be on the same drive
* `pkgs` - This is where approved package files are stored.
* `repos` - Each repository gets a subfolder here. The subfolder contains the uncompressed contents of the db file.
* Each repo subdirectory contains the compressed db & files archive for the repository, alongside a directory called `files` which contains the uncompressed contents.

View File

@ -2,7 +2,8 @@ module archive
import os import os
pub fn pkg_info_string(pkg_path string) ?string { // Returns the .PKGINFO file's contents & the list of files.
pub fn pkg_info(pkg_path string) ?(string, []string) {
if !os.is_file(pkg_path) { if !os.is_file(pkg_path) {
return error("'$pkg_path' doesn't exist or isn't a file.") return error("'$pkg_path' doesn't exist or isn't a file.")
} }
@ -26,18 +27,24 @@ pub fn pkg_info_string(pkg_path string) ?string {
// We iterate over every header in search of the .PKGINFO one // We iterate over every header in search of the .PKGINFO one
mut buf := voidptr(0) mut buf := voidptr(0)
mut files := []string{}
for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK {
if C.strcmp(C.archive_entry_pathname(entry), c'.PKGINFO') == 0 { pathname := C.archive_entry_pathname(entry)
unsafe {
files << cstring_to_vstring(pathname)
}
if C.strcmp(pathname, c'.PKGINFO') == 0 {
size := C.archive_entry_size(entry) size := C.archive_entry_size(entry)
// TODO can this unsafe block be avoided? // TODO can this unsafe block be avoided?
buf = unsafe { malloc(size) } buf = unsafe { malloc(size) }
C.archive_read_data(a, voidptr(buf), size) C.archive_read_data(a, voidptr(buf), size)
break
} else { } else {
C.archive_read_data_skip(a) C.archive_read_data_skip(a)
} }
} }
return unsafe { cstring_to_vstring(&char(buf)) } return unsafe { cstring_to_vstring(&char(buf)) }, files
} }

View File

@ -4,8 +4,9 @@ import web
import os import os
import log import log
import io import io
import repo import pkg
import archive import archive
import repo
const port = 8000 const port = 8000
@ -54,59 +55,62 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? {
} }
} }
fn main2() { // fn main2() {
// Configure logger // // Configure logger
log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } // log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' }
log_level := log.level_from_tag(log_level_str) or { // log_level := log.level_from_tag(log_level_str) or {
exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') // exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
} // }
log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } // log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' }
mut logger := log.Log{ // mut logger := log.Log{
level: log_level // level: log_level
} // }
logger.set_full_logpath(log_file) // logger.set_full_logpath(log_file)
logger.log_to_console_too() // logger.log_to_console_too()
defer { // defer {
logger.info('Flushing log file') // logger.info('Flushing log file')
logger.flush() // logger.flush()
logger.close() // logger.close()
} // }
// Configure web server // // Configure web server
key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } // key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') }
repo_dir := os.getenv_opt('REPO_DIR') or { // repo_dir := os.getenv_opt('REPO_DIR') or {
exit_with_message(1, 'No repo directory was configured.') // exit_with_message(1, 'No repo directory was configured.')
} // }
repo := repo.Repo{ // repo := repo.Repo{
dir: repo_dir // dir: repo_dir
name: db_name // name: db_name
} // }
// We create the upload directory during startup // // We create the upload directory during startup
if !os.is_dir(repo.pkg_dir()) { // if !os.is_dir(repo.pkg_dir()) {
os.mkdir_all(repo.pkg_dir()) or { // os.mkdir_all(repo.pkg_dir()) or {
exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.") // exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.")
} // }
logger.info("Created package directory '$repo.pkg_dir()'.") // logger.info("Created package directory '$repo.pkg_dir()'.")
} // }
web.run(&App{ // web.run(&App{
logger: logger // logger: logger
api_key: key // api_key: key
repo: repo // repo: repo
}, port) // }, port)
} // }
fn main() { fn main() {
// archive.list_filenames() // archive.list_filenames()
info := repo.get_pkg_info('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or { res := pkg.read_pkg('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or {
eprintln(err.msg) eprintln(err.msg)
return return
} }
println(info) // println(info)
println(res.info)
print(res.files)
println(res.info.to_desc())
} }

View File

@ -1,45 +1,46 @@
module repo module pkg
import archive import archive
import time import time
struct Pkg {
pub:
info PkgInfo [required]
files []string [required]
}
struct PkgInfo { struct PkgInfo {
mut: mut:
// Single values // Single values
name string name string
base string base string
version string version string
description string description string
size i64 size i64
csize i64 csize i64
url string url string
arch string arch string
build_date i64 build_date i64
packager string packager string
md5sum string md5sum string
sha256sum string sha256sum string
pgpsig string pgpsig string
pgpsigsize i64 pgpsigsize i64
// Array values // Array values
groups []string groups []string
licenses []string licenses []string
replaces []string replaces []string
depends []string depends []string
conflicts []string conflicts []string
provides []string provides []string
optdepends []string optdepends []string
makedepends []string makedepends []string
checkdepends []string checkdepends []string
} }
pub fn get_pkg_info(pkg_path string) ?PkgInfo { fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo {
pkg_info_str := archive.pkg_info_string(pkg_path) ?
mut pkg_info := PkgInfo{} mut pkg_info := PkgInfo{}
mut i := 0
mut j := 0
// Iterate over the entire string // Iterate over the entire string
for line in pkg_info_str.split_into_lines() { for line in pkg_info_str.split_into_lines() {
// Skip any comment lines // Skip any comment lines
@ -71,7 +72,6 @@ pub fn get_pkg_info(pkg_path string) ?PkgInfo {
'sha256sum' { pkg_info.sha256sum = value } 'sha256sum' { pkg_info.sha256sum = value }
'pgpsig' { pkg_info.pgpsig = value } 'pgpsig' { pkg_info.pgpsig = value }
'pgpsigsize' { pkg_info.pgpsigsize = value.int() } 'pgpsigsize' { pkg_info.pgpsigsize = value.int() }
// Array values // Array values
'group' { pkg_info.groups << value } 'group' { pkg_info.groups << value }
'license' { pkg_info.licenses << value } 'license' { pkg_info.licenses << value }
@ -88,3 +88,21 @@ pub fn get_pkg_info(pkg_path string) ?PkgInfo {
return pkg_info return pkg_info
} }
pub fn read_pkg(pkg_path string) ?Pkg {
pkg_info_str, files := archive.pkg_info(pkg_path) ?
pkg_info := parse_pkg_info_string(pkg_info_str) ?
return Pkg{
info: pkg_info
files: files
}
}
// Represent a PkgInfo struct as a desc file
pub fn (p &PkgInfo) to_desc() string {
// TODO calculate md5 & sha256 instead of believing the file
mut desc := ''
return desc
}

View File

@ -11,23 +11,35 @@ pub struct Dummy {
x int x int
} }
// Handles management of a repository. Package files are stored in '$dir/pkgs' // This struct manages a single repository.
// & moved there if necessary.
pub struct Repo { pub struct Repo {
mut: mut:
mutex shared Dummy mutex shared Dummy
pub: pub:
dir string [required] // Where to store repository files; should exist
name string [required] repo_dir string [required]
// Where to find packages; packages are expected to all be in the same directory
pkg_dir string [required]
} }
pub fn (r &Repo) pkg_dir() string { // Returns whether the repository contains the given package.
return os.join_path_single(r.dir, repo.pkgs_subpath) pub fn (r &Repo) contains(pkg string) bool {
return os.exists(os.join_path(r.repo_dir, 'files', pkg))
}
// Adds the given package to the repo. If false, the package was already
// present in the repository.
pub fn (r &Repo) add(pkg string) ?bool {
return false
}
// Re-generate the db & files archives.
fn (r &Repo) genenerate() ? {
} }
// Returns path to the given package, prepended with the repo's path. // Returns path to the given package, prepended with the repo's path.
pub fn (r &Repo) pkg_path(pkg string) string { pub fn (r &Repo) pkg_path(pkg string) string {
return os.join_path(r.dir, repo.pkgs_subpath, pkg) return os.join_path_single(r.pkg_dir, pkg)
} }
pub fn (r &Repo) exists(pkg string) bool { pub fn (r &Repo) exists(pkg string) bool {
@ -36,7 +48,7 @@ pub fn (r &Repo) exists(pkg string) bool {
// Returns the full path to the database file // Returns the full path to the database file
pub fn (r &Repo) db_path() string { pub fn (r &Repo) db_path() string {
return os.join_path_single(r.dir, '${r.name}.tar.gz') return os.join_path_single(r.repo_dir, 'repo.tar.gz')
} }
pub fn (r &Repo) add_package(pkg_path string) ? { pub fn (r &Repo) add_package(pkg_path string) ? {

View File

@ -28,62 +28,67 @@ fn (mut app App) get_root(filename string) web.Result {
mut full_path := '' mut full_path := ''
if is_pkg_name(filename) { if is_pkg_name(filename) {
full_path = os.join_path_single(app.repo.pkg_dir(), filename) full_path = os.join_path_single(app.repo.pkg_dir, filename)
} else { } else {
full_path = os.join_path_single(app.repo.dir, filename) full_path = os.join_path_single(app.repo.repo_dir, filename)
} }
return app.file(full_path) return app.file(full_path)
} }
['/pkgs/:pkg'; put] // ['/pkgs/:pkg'; put]
fn (mut app App) put_package(pkg string) web.Result { // fn (mut app App) put_package(pkg string) web.Result {
if !app.is_authorized() { // if !app.is_authorized() {
return app.text('Unauthorized.') // return app.text('Unauthorized.')
} // }
if !is_pkg_name(pkg) { // if !is_pkg_name(pkg) {
app.lwarn("Invalid package name '$pkg'.") // app.lwarn("Invalid package name '$pkg'.")
return app.text('Invalid filename.') // return app.text('Invalid filename.')
} // }
if app.repo.exists(pkg) { // if app.repo.exists(pkg) {
app.lwarn("Duplicate package '$pkg'") // app.lwarn("Duplicate package '$pkg'")
return app.text('File already exists.') // return app.text('File already exists.')
} // }
pkg_path := app.repo.pkg_path(pkg) // pkg_path := app.repo.pkg_path(pkg)
if length := app.req.header.get(.content_length) { // if length := app.req.header.get(.content_length) {
app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.") // app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.")
// This is used to time how long it takes to upload a file // // This is used to time how long it takes to upload a file
mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) // mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true })
reader_to_file(mut app.reader, length.int(), pkg_path) or { // reader_to_file(mut app.reader, length.int(), pkg_path) or {
app.lwarn("Failed to upload package '$pkg'") // app.lwarn("Failed to upload package '$pkg'")
return app.text('Failed to upload file.') // return app.text('Failed to upload file.')
} // }
sw.stop() // sw.stop()
app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.") // app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.")
} else { // } else {
app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.") // app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.")
return app.text("Content-Type header isn't set.") // return app.text("Content-Type header isn't set.")
} // }
app.repo.add_package(pkg_path) or { // app.repo.add_package(pkg_path) or {
app.lwarn("Failed to add package '$pkg' to database.") // app.lwarn("Failed to add package '$pkg' to database.")
os.rm(pkg_path) or { println('Failed to remove $pkg_path') } // os.rm(pkg_path) or { println('Failed to remove $pkg_path') }
return app.text('Failed to add package to repo.') // return app.text('Failed to add package to repo.')
} // }
app.linfo("Added '$pkg' to repository.") // app.linfo("Added '$pkg' to repository.")
return app.text('Package added successfully.') // return app.text('Package added successfully.')
// }
['/add'; put]
pub fn (mut app App) add_package() web.Result {
return app.text('')
} }