Very alpha support for multiple & multi-arch repos

cron
Jef Roosens 2022-03-27 16:33:06 +02:00
parent 013ce511d7
commit a47cace296
Signed by untrusted user: Jef Roosens
GPG Key ID: B75D4F293C7052DB
8 changed files with 83 additions and 47 deletions

View File

@ -25,7 +25,7 @@ dvieter: $(SOURCES)
gdb: dvieter gdb: dvieter
VIETER_API_KEY=test \ VIETER_API_KEY=test \
VIETER_DOWNLOAD_DIR=data/downloads \ VIETER_DOWNLOAD_DIR=data/downloads \
VIETER_REPO_DIR=data/repo \ VIETER_DATA_DIR=data/repo \
VIETER_PKG_DIR=data/pkgs \ VIETER_PKG_DIR=data/pkgs \
VIETER_LOG_LEVEL=DEBUG \ VIETER_LOG_LEVEL=DEBUG \
VIETER_REPOS_FILE=data/repos.json \ VIETER_REPOS_FILE=data/repos.json \
@ -60,7 +60,7 @@ cli-prod:
run: vieter run: vieter
VIETER_API_KEY=test \ VIETER_API_KEY=test \
VIETER_DOWNLOAD_DIR=data/downloads \ VIETER_DOWNLOAD_DIR=data/downloads \
VIETER_REPO_DIR=data/repo \ VIETER_DATA_DIR=data/repo \
VIETER_PKG_DIR=data/pkgs \ VIETER_PKG_DIR=data/pkgs \
VIETER_LOG_LEVEL=DEBUG \ VIETER_LOG_LEVEL=DEBUG \
VIETER_REPOS_FILE=data/repos.json \ VIETER_REPOS_FILE=data/repos.json \

View File

@ -16,7 +16,7 @@ pub:
pkg_dir string pkg_dir string
download_dir string download_dir string
api_key string api_key string
repo_dir string data_dir string
repos_file string repos_file string
} }

View File

@ -101,7 +101,7 @@ fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo {
// read_pkg extracts the file list & .PKGINFO contents from an archive // read_pkg extracts the file list & .PKGINFO contents from an archive
// NOTE: this command currently only supports zstd-compressed tarballs // NOTE: this command currently only supports zstd-compressed tarballs
pub fn read_pkg(pkg_path string) ?Pkg { pub fn read_pkg_archive(pkg_path string) ?Pkg {
if !os.is_file(pkg_path) { if !os.is_file(pkg_path) {
return error("'$pkg_path' doesn't exist or isn't a file.") return error("'$pkg_path' doesn't exist or isn't a file.")
} }

View File

@ -4,15 +4,20 @@ import os
import package import package
import util import util
// This struct manages a single repository. // Manages a group of repositories. Each repository contains one or more
pub struct Repo { // arch-repositories, each of which represent a specific architecture.
pub struct RepoGroupManager {
mut: mut:
mutex shared util.Dummy mutex shared util.Dummy
pub: pub:
// Where to store repository files // Where to store repositories' files
repo_dir string [required] data_dir string [required]
// Where to find packages; packages are expected to all be in the same directory // Where packages are stored; each architecture gets its own subdirectory
pkg_dir string [required] pkg_dir string [required]
// The default architecture to use for a repository. In reality, this value
// is only required when a package with architecture "any" is added as the
// first package to a repository.
default_arch string [required]
} }
pub struct RepoAddResult { pub struct RepoAddResult {
@ -21,28 +26,29 @@ pub:
pkg &package.Pkg [required] pkg &package.Pkg [required]
} }
// new creates a new Repo & creates the directories as needed // new creates a new RepoGroupManager & creates the directories as needed
pub fn new(repo_dir string, pkg_dir string) ?Repo { pub fn new(data_dir string, pkg_dir string, default_arch string) ?RepoGroupManager {
if !os.is_dir(repo_dir) { if !os.is_dir(data_dir) {
os.mkdir_all(repo_dir) or { return error('Failed to create repo directory: $err.msg') } os.mkdir_all(data_dir) or { return error('Failed to create repo directory: $err.msg') }
} }
if !os.is_dir(pkg_dir) { if !os.is_dir(pkg_dir) {
os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg') } os.mkdir_all(pkg_dir) or { return error('Failed to create package directory: $err.msg') }
} }
return Repo{ return RepoGroupManager{
repo_dir: repo_dir data_dir: data_dir
pkg_dir: pkg_dir pkg_dir: pkg_dir
default_arch: default_arch
} }
} }
// add_from_path adds a package from an arbitrary path & moves it into the pkgs // add_from_path adds a package from an arbitrary path & moves it into the pkgs
// directory if necessary. // directory if necessary.
pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult { pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?RepoAddResult {
pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') } pkg := package.read_pkg_archive(pkg_path) or { return error('Failed to read package file: $err.msg') }
added := r.add(pkg) ? added := r.add_pkg_in_repo(repo, pkg) ?
// If the add was successful, we move the file to the packages directory // If the add was successful, we move the file to the packages directory
if added { if added {
@ -60,9 +66,33 @@ pub fn (r &Repo) add_from_path(pkg_path string) ?RepoAddResult {
} }
} }
// add adds a given Pkg to the repository fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?bool {
fn (r &Repo) add(pkg &package.Pkg) ?bool { if pkg.info.arch == "any" {
pkg_dir := r.pkg_path(pkg) repo_dir := os.join_path_single(r.data_dir, repo)
// We get a listing of all currently present arch-repos in the given repo
mut arch_repos := os.ls(repo_dir) ?.filter(os.is_dir(os.join_path_single(repo_dir, it)))
if arch_repos.len == 0 {
arch_repos << r.default_arch
}
for arch in arch_repos {
r.add_pkg_in_arch_repo(repo, arch, pkg) ?
}
}else{
r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) ?
}
// TODO properly handle this
return true
}
// add_pkg_in_repo adds the given package to the specified repo. A repo is an
// arbitrary subdirectory of r.repo_dir, but in practice, it will always be an
// architecture-specific version of some sub-repository.
fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &package.Pkg) ?bool {
pkg_dir := os.join_path(r.data_dir, repo, arch, '$pkg.info.name-$pkg.info.version')
// We can't add the same package twice // We can't add the same package twice
if os.exists(pkg_dir) { if os.exists(pkg_dir) {
@ -70,9 +100,9 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool {
} }
// We remove the older package version first, if present // We remove the older package version first, if present
r.remove(pkg.info.name, false) ? r.remove_pkg_from_arch_repo(repo, arch, pkg, false) ?
os.mkdir(pkg_dir) or { return error('Failed to create package directory.') } os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') }
os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or { os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or {
os.rmdir_all(pkg_dir) ? os.rmdir_all(pkg_dir) ?
@ -85,27 +115,35 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool {
return error('Failed to write files file.') return error('Failed to write files file.')
} }
r.sync() ? r.sync(repo, arch) ?
return true return true
} }
// remove removes a package from the database. It returns false if the package // remove removes a package from the database. It returns false if the package
// wasn't present in the database. // wasn't present in the database.
fn (r &Repo) remove(pkg_name string, sync bool) ?bool { fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg &package.Pkg, sync bool) ?bool {
repo_dir := os.join_path(r.data_dir, repo, arch)
// If the repository doesn't exist yet, the result is automatically false
if !os.exists(repo_dir) {
return false
}
// We iterate over every directory in the repo dir // We iterate over every directory in the repo dir
for d in os.ls(r.repo_dir) ? { // TODO filter so we only check directories
for d in os.ls(repo_dir) ? {
name := d.split('-')#[..-2].join('-') name := d.split('-')#[..-2].join('-')
if name == pkg_name { if name == pkg.info.name {
// We lock the mutex here to prevent other routines from creating a // We lock the mutex here to prevent other routines from creating a
// new archive while we removed an entry // new archive while we removed an entry
lock r.mutex { lock r.mutex {
os.rmdir_all(os.join_path_single(r.repo_dir, d)) ? os.rmdir_all(os.join_path_single(repo_dir, d)) ?
} }
if sync { if sync {
r.sync() ? r.sync(repo, arch) ?
} }
return true return true
@ -114,8 +152,3 @@ fn (r &Repo) remove(pkg_name string, sync bool) ?bool {
return false return false
} }
// Returns the path where the given package's desc & files files are stored
fn (r &Repo) pkg_path(pkg &package.Pkg) string {
return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version')
}

View File

@ -30,8 +30,9 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri
} }
// Re-generate the repo archive files // Re-generate the repo archive files
fn (r &Repo) sync() ? { fn (r &RepoGroupManager) sync(repo string, arch string) ? {
// TODO also write files archive subrepo_path := os.join_path(r.data_dir, repo, arch)
lock r.mutex { lock r.mutex {
a_db := C.archive_write_new() a_db := C.archive_write_new()
a_files := C.archive_write_new() a_files := C.archive_write_new()
@ -44,18 +45,18 @@ fn (r &Repo) sync() ? {
C.archive_write_add_filter_gzip(a_files) C.archive_write_add_filter_gzip(a_files)
C.archive_write_set_format_pax_restricted(a_files) C.archive_write_set_format_pax_restricted(a_files)
db_path := os.join_path_single(r.repo_dir, 'vieter.db.tar.gz') db_path := os.join_path_single(subrepo_path, 'vieter.db.tar.gz')
files_path := os.join_path_single(r.repo_dir, 'vieter.files.tar.gz') files_path := os.join_path_single(subrepo_path, 'vieter.files.tar.gz')
C.archive_write_open_filename(a_db, &char(db_path.str)) C.archive_write_open_filename(a_db, &char(db_path.str))
C.archive_write_open_filename(a_files, &char(files_path.str)) C.archive_write_open_filename(a_files, &char(files_path.str))
// Iterate over each directory // Iterate over each directory
for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, for d in os.ls(subrepo_path) ?.filter(os.is_dir(os.join_path_single(subrepo_path,
it))) { it))) {
// desc // desc
mut inner_path := os.join_path_single(d, 'desc') mut inner_path := os.join_path_single(d, 'desc')
mut actual_path := os.join_path_single(r.repo_dir, inner_path) mut actual_path := os.join_path_single(subrepo_path, inner_path)
archive_add_entry(a_db, entry, actual_path, inner_path) archive_add_entry(a_db, entry, actual_path, inner_path)
archive_add_entry(a_files, entry, actual_path, inner_path) archive_add_entry(a_files, entry, actual_path, inner_path)
@ -64,7 +65,7 @@ fn (r &Repo) sync() ? {
// files // files
inner_path = os.join_path_single(d, 'files') inner_path = os.join_path_single(d, 'files')
actual_path = os.join_path_single(r.repo_dir, inner_path) actual_path = os.join_path_single(subrepo_path, inner_path)
archive_add_entry(a_files, entry, actual_path, inner_path) archive_add_entry(a_files, entry, actual_path, inner_path)

View File

@ -8,6 +8,8 @@ import rand
import util import util
import net.http import net.http
const default_repo = "vieter"
// healthcheck just returns a string, but can be used to quickly check if the // healthcheck just returns a string, but can be used to quickly check if the
// server is still responsive. // server is still responsive.
['/health'; get] ['/health'; get]
@ -21,9 +23,9 @@ fn (mut app App) get_root(filename string) web.Result {
mut full_path := '' mut full_path := ''
if filename.ends_with('.db') || filename.ends_with('.files') { if filename.ends_with('.db') || filename.ends_with('.files') {
full_path = os.join_path_single(app.repo.repo_dir, '${filename}.tar.gz') full_path = os.join_path_single(app.repo.data_dir, '${filename}.tar.gz')
} else if filename.ends_with('.db.tar.gz') || filename.ends_with('.files.tar.gz') { } else if filename.ends_with('.db.tar.gz') || filename.ends_with('.files.tar.gz') {
full_path = os.join_path_single(app.repo.repo_dir, '$filename') full_path = os.join_path_single(app.repo.data_dir, '$filename')
} else { } else {
full_path = os.join_path_single(app.repo.pkg_dir, filename) full_path = os.join_path_single(app.repo.pkg_dir, filename)
} }
@ -74,7 +76,7 @@ fn (mut app App) put_package() web.Result {
return app.text("Content-Type header isn't set.") return app.text("Content-Type header isn't set.")
} }
res := app.repo.add_from_path(pkg_path) or { res := app.repo.add_pkg_from_path(default_repo, pkg_path) or {
app.lerror('Error while adding package: $err.msg') app.lerror('Error while adding package: $err.msg')
os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") } os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path': $err.msg") }

View File

@ -14,7 +14,7 @@ struct App {
pub: pub:
conf env.ServerConfig [required; web_global] conf env.ServerConfig [required; web_global]
pub mut: pub mut:
repo repo.Repo [required; web_global] repo repo.RepoGroupManager [required; web_global]
// This is used to claim the file lock on the repos file // This is used to claim the file lock on the repos file
git_mutex shared util.Dummy git_mutex shared util.Dummy
} }
@ -42,7 +42,7 @@ pub fn server() ? {
} }
// This also creates the directories if needed // This also creates the directories if needed
repo := repo.new(conf.repo_dir, conf.pkg_dir) or { repo := repo.new(conf.data_dir, conf.pkg_dir, "x86_64") or {
logger.error(err.msg) logger.error(err.msg)
exit(1) exit(1)
} }

View File

@ -46,7 +46,7 @@ def create_random_pkginfo(words, name_min_len, name_max_len):
"pkgname": name, "pkgname": name,
"pkgbase": name, "pkgbase": name,
"pkgver": ver, "pkgver": ver,
"arch": "x86_64" "arch": "any"
} }
return "\n".join(f"{key} = {value}" for key, value in data.items()) return "\n".join(f"{key} = {value}" for key, value in data.items())