forked from vieter-v/vieter
Merge pull request 'First part of repo-add replacement' (#26) from repo-add into dev
Reviewed-on: #26main
commit
ed75dea2d6
|
|
@ -14,3 +14,4 @@ vieter.log
|
|||
|
||||
# External lib; gets added by Makefile
|
||||
libarchive-*
|
||||
test/
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ pipeline:
|
|||
|
||||
prod:
|
||||
image: 'chewingbever/vlang:latest'
|
||||
environment:
|
||||
- LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static
|
||||
group: 'build'
|
||||
commands:
|
||||
- make prod
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ RUN apk --no-cache add \
|
|||
git make upx gcc bash \
|
||||
musl-dev \
|
||||
openssl-libs-static openssl-dev \
|
||||
zlib-static bzip2-static xz-dev expat-static zstd-static lz4-static \
|
||||
sqlite-static sqlite-dev \
|
||||
libx11-dev glfw-dev freetype-dev \
|
||||
libarchive-static libarchive-dev \
|
||||
|
|
|
|||
2
Makefile
2
Makefile
|
|
@ -42,7 +42,7 @@ run: vieter
|
|||
|
||||
.PHONY: run-prod
|
||||
run-prod: prod
|
||||
API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG ./vieter-prod
|
||||
API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG ./pvieter
|
||||
|
||||
# Same as run, but restart when the source code changes
|
||||
.PHONY: watch
|
||||
|
|
|
|||
|
|
@ -28,3 +28,12 @@ daemon to start builds, which are then uploaded to the server's repository. The
|
|||
server also allows for non-agents to upload packages, as long as they have the
|
||||
required secrets. This allows me to also develop non-git packages, such as my
|
||||
terminal, & upload them to the servers using CI.
|
||||
|
||||
## Directory Structure
|
||||
|
||||
The data directory consists of three main directories:
|
||||
|
||||
* `downloads` - This is where packages are initially downloaded. Because vieter moves files from this folder to the `pkgs` folder, these two folders should best be on the same drive
|
||||
* `pkgs` - This is where approved package files are stored.
|
||||
* `repos` - Each repository gets a subfolder here. The subfolder contains the uncompressed contents of the db file.
|
||||
* Each repo subdirectory contains the compressed db & files archive for the repository, alongside a directory called `files` which contains the uncompressed contents.
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@ module archive
|
|||
|
||||
import os
|
||||
|
||||
pub fn get_pkg_info(pkg_path string) ?string {
|
||||
// Returns the .PKGINFO file's contents & the list of files.
|
||||
pub fn pkg_info(pkg_path string) ?(string, []string) {
|
||||
if !os.is_file(pkg_path) {
|
||||
return error("'$pkg_path' doesn't exist or isn't a file.")
|
||||
}
|
||||
|
|
@ -26,18 +27,27 @@ pub fn get_pkg_info(pkg_path string) ?string {
|
|||
|
||||
// We iterate over every header in search of the .PKGINFO one
|
||||
mut buf := voidptr(0)
|
||||
mut files := []string{}
|
||||
for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK {
|
||||
if C.strcmp(C.archive_entry_pathname(entry), c'.PKGINFO') == 0 {
|
||||
pathname := C.archive_entry_pathname(entry)
|
||||
|
||||
ignored_names := [c'.BUILDINFO', c'.INSTALL', c'.MTREE', c'.PKGINFO', c'.CHANGELOG']
|
||||
if ignored_names.all(C.strcmp(it, pathname) != 0) {
|
||||
unsafe {
|
||||
files << cstring_to_vstring(pathname)
|
||||
}
|
||||
}
|
||||
|
||||
if C.strcmp(pathname, c'.PKGINFO') == 0 {
|
||||
size := C.archive_entry_size(entry)
|
||||
|
||||
// TODO can this unsafe block be avoided?
|
||||
buf = unsafe { malloc(size) }
|
||||
C.archive_read_data(a, voidptr(buf), size)
|
||||
break
|
||||
} else {
|
||||
C.archive_read_data_skip(a)
|
||||
}
|
||||
}
|
||||
|
||||
return unsafe { cstring_to_vstring(&char(buf)) }
|
||||
return unsafe { cstring_to_vstring(&char(buf)) }, files
|
||||
}
|
||||
|
|
|
|||
114
src/main.v
114
src/main.v
|
|
@ -4,8 +4,9 @@ import web
|
|||
import os
|
||||
import log
|
||||
import io
|
||||
import repo
|
||||
import pkg
|
||||
import archive
|
||||
import repo
|
||||
|
||||
const port = 8000
|
||||
|
||||
|
|
@ -54,59 +55,62 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? {
|
|||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
// Configure logger
|
||||
log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' }
|
||||
log_level := log.level_from_tag(log_level_str) or {
|
||||
exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
|
||||
}
|
||||
log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' }
|
||||
|
||||
mut logger := log.Log{
|
||||
level: log_level
|
||||
}
|
||||
|
||||
logger.set_full_logpath(log_file)
|
||||
logger.log_to_console_too()
|
||||
|
||||
defer {
|
||||
logger.info('Flushing log file')
|
||||
logger.flush()
|
||||
logger.close()
|
||||
}
|
||||
|
||||
// Configure web server
|
||||
key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') }
|
||||
repo_dir := os.getenv_opt('REPO_DIR') or {
|
||||
exit_with_message(1, 'No repo directory was configured.')
|
||||
}
|
||||
|
||||
repo := repo.Repo{
|
||||
dir: repo_dir
|
||||
name: db_name
|
||||
}
|
||||
|
||||
// We create the upload directory during startup
|
||||
if !os.is_dir(repo.pkg_dir()) {
|
||||
os.mkdir_all(repo.pkg_dir()) or {
|
||||
exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.")
|
||||
}
|
||||
|
||||
logger.info("Created package directory '$repo.pkg_dir()'.")
|
||||
}
|
||||
|
||||
web.run(&App{
|
||||
logger: logger
|
||||
api_key: key
|
||||
repo: repo
|
||||
}, port)
|
||||
}
|
||||
|
||||
// fn main() {
|
||||
// // archive.list_filenames()
|
||||
// info := archive.get_pkg_info('test/jjr-joplin-desktop-2.6.10-4-x86_64.pkg.tar.zst') or {
|
||||
// eprintln(err.msg)
|
||||
// return
|
||||
// fn main2() {
|
||||
// // Configure logger
|
||||
// log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' }
|
||||
// log_level := log.level_from_tag(log_level_str) or {
|
||||
// exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.')
|
||||
// }
|
||||
// println(info)
|
||||
// log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' }
|
||||
|
||||
// mut logger := log.Log{
|
||||
// level: log_level
|
||||
// }
|
||||
|
||||
// logger.set_full_logpath(log_file)
|
||||
// logger.log_to_console_too()
|
||||
|
||||
// defer {
|
||||
// logger.info('Flushing log file')
|
||||
// logger.flush()
|
||||
// logger.close()
|
||||
// }
|
||||
|
||||
// // Configure web server
|
||||
// key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') }
|
||||
// repo_dir := os.getenv_opt('REPO_DIR') or {
|
||||
// exit_with_message(1, 'No repo directory was configured.')
|
||||
// }
|
||||
|
||||
// repo := repo.Repo{
|
||||
// dir: repo_dir
|
||||
// name: db_name
|
||||
// }
|
||||
|
||||
// // We create the upload directory during startup
|
||||
// if !os.is_dir(repo.pkg_dir()) {
|
||||
// os.mkdir_all(repo.pkg_dir()) or {
|
||||
// exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.")
|
||||
// }
|
||||
|
||||
// logger.info("Created package directory '$repo.pkg_dir()'.")
|
||||
// }
|
||||
|
||||
// web.run(&App{
|
||||
// logger: logger
|
||||
// api_key: key
|
||||
// repo: repo
|
||||
// }, port)
|
||||
// }
|
||||
|
||||
fn main() {
|
||||
// archive.list_filenames()
|
||||
res := pkg.read_pkg('test/homebank-5.5.1-1-x86_64.pkg.tar.zst') or {
|
||||
eprintln(err.msg)
|
||||
return
|
||||
}
|
||||
// println(info)
|
||||
println(res.info)
|
||||
print(res.files)
|
||||
println(res.info.to_desc())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,108 @@
|
|||
module pkg
|
||||
|
||||
import archive
|
||||
import time
|
||||
|
||||
struct Pkg {
|
||||
pub:
|
||||
info PkgInfo [required]
|
||||
files []string [required]
|
||||
}
|
||||
|
||||
struct PkgInfo {
|
||||
mut:
|
||||
// Single values
|
||||
name string
|
||||
base string
|
||||
version string
|
||||
description string
|
||||
size i64
|
||||
csize i64
|
||||
url string
|
||||
arch string
|
||||
build_date i64
|
||||
packager string
|
||||
md5sum string
|
||||
sha256sum string
|
||||
pgpsig string
|
||||
pgpsigsize i64
|
||||
// Array values
|
||||
groups []string
|
||||
licenses []string
|
||||
replaces []string
|
||||
depends []string
|
||||
conflicts []string
|
||||
provides []string
|
||||
optdepends []string
|
||||
makedepends []string
|
||||
checkdepends []string
|
||||
}
|
||||
|
||||
fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo {
|
||||
mut pkg_info := PkgInfo{}
|
||||
|
||||
// Iterate over the entire string
|
||||
for line in pkg_info_str.split_into_lines() {
|
||||
// Skip any comment lines
|
||||
if line.starts_with('#') {
|
||||
continue
|
||||
}
|
||||
parts := line.split_nth('=', 2)
|
||||
|
||||
if parts.len < 2 {
|
||||
return error('Invalid line detected.')
|
||||
}
|
||||
|
||||
value := parts[1].trim_space()
|
||||
key := parts[0].trim_space()
|
||||
|
||||
match key {
|
||||
// Single values
|
||||
'pkgname' { pkg_info.name = value }
|
||||
'pkgbase' { pkg_info.base = value }
|
||||
'pkgver' { pkg_info.version = value }
|
||||
'pkgdesc' { pkg_info.description = value }
|
||||
'csize' { pkg_info.csize = value.int() }
|
||||
'size' { pkg_info.size = value.int() }
|
||||
'url' { pkg_info.url = value }
|
||||
'arch' { pkg_info.arch = value }
|
||||
'builddate' { pkg_info.build_date = value.int() }
|
||||
'packager' { pkg_info.packager = value }
|
||||
'md5sum' { pkg_info.md5sum = value }
|
||||
'sha256sum' { pkg_info.sha256sum = value }
|
||||
'pgpsig' { pkg_info.pgpsig = value }
|
||||
'pgpsigsize' { pkg_info.pgpsigsize = value.int() }
|
||||
// Array values
|
||||
'group' { pkg_info.groups << value }
|
||||
'license' { pkg_info.licenses << value }
|
||||
'replaces' { pkg_info.replaces << value }
|
||||
'depend' { pkg_info.depends << value }
|
||||
'conflict' { pkg_info.conflicts << value }
|
||||
'provides' { pkg_info.provides << value }
|
||||
'optdepend' { pkg_info.optdepends << value }
|
||||
'makedepend' { pkg_info.makedepends << value }
|
||||
'checkdepend' { pkg_info.checkdepends << value }
|
||||
else { return error("Invalid key '$key'.") }
|
||||
}
|
||||
}
|
||||
|
||||
return pkg_info
|
||||
}
|
||||
|
||||
pub fn read_pkg(pkg_path string) ?Pkg {
|
||||
pkg_info_str, files := archive.pkg_info(pkg_path) ?
|
||||
pkg_info := parse_pkg_info_string(pkg_info_str) ?
|
||||
|
||||
return Pkg{
|
||||
info: pkg_info
|
||||
files: files
|
||||
}
|
||||
}
|
||||
|
||||
// Represent a PkgInfo struct as a desc file
|
||||
pub fn (p &PkgInfo) to_desc() string {
|
||||
// TODO calculate md5 & sha256 instead of believing the file
|
||||
mut desc := ''
|
||||
|
||||
return desc
|
||||
}
|
||||
29
src/repo.v
29
src/repo.v
|
|
@ -1,6 +1,7 @@
|
|||
module repo
|
||||
|
||||
import os
|
||||
import archive
|
||||
|
||||
const pkgs_subpath = 'pkgs'
|
||||
|
||||
|
|
@ -10,23 +11,35 @@ pub struct Dummy {
|
|||
x int
|
||||
}
|
||||
|
||||
// Handles management of a repository. Package files are stored in '$dir/pkgs'
|
||||
// & moved there if necessary.
|
||||
// This struct manages a single repository.
|
||||
pub struct Repo {
|
||||
mut:
|
||||
mutex shared Dummy
|
||||
pub:
|
||||
dir string [required]
|
||||
name string [required]
|
||||
// Where to store repository files; should exist
|
||||
repo_dir string [required]
|
||||
// Where to find packages; packages are expected to all be in the same directory
|
||||
pkg_dir string [required]
|
||||
}
|
||||
|
||||
pub fn (r &Repo) pkg_dir() string {
|
||||
return os.join_path_single(r.dir, repo.pkgs_subpath)
|
||||
// Returns whether the repository contains the given package.
|
||||
pub fn (r &Repo) contains(pkg string) bool {
|
||||
return os.exists(os.join_path(r.repo_dir, 'files', pkg))
|
||||
}
|
||||
|
||||
// Adds the given package to the repo. If false, the package was already
|
||||
// present in the repository.
|
||||
pub fn (r &Repo) add(pkg string) ?bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Re-generate the db & files archives.
|
||||
fn (r &Repo) genenerate() ? {
|
||||
}
|
||||
|
||||
// Returns path to the given package, prepended with the repo's path.
|
||||
pub fn (r &Repo) pkg_path(pkg string) string {
|
||||
return os.join_path(r.dir, repo.pkgs_subpath, pkg)
|
||||
return os.join_path_single(r.pkg_dir, pkg)
|
||||
}
|
||||
|
||||
pub fn (r &Repo) exists(pkg string) bool {
|
||||
|
|
@ -35,7 +48,7 @@ pub fn (r &Repo) exists(pkg string) bool {
|
|||
|
||||
// Returns the full path to the database file
|
||||
pub fn (r &Repo) db_path() string {
|
||||
return os.join_path_single(r.dir, '${r.name}.tar.gz')
|
||||
return os.join_path_single(r.repo_dir, 'repo.tar.gz')
|
||||
}
|
||||
|
||||
pub fn (r &Repo) add_package(pkg_path string) ? {
|
||||
|
|
|
|||
79
src/routes.v
79
src/routes.v
|
|
@ -28,62 +28,67 @@ fn (mut app App) get_root(filename string) web.Result {
|
|||
mut full_path := ''
|
||||
|
||||
if is_pkg_name(filename) {
|
||||
full_path = os.join_path_single(app.repo.pkg_dir(), filename)
|
||||
full_path = os.join_path_single(app.repo.pkg_dir, filename)
|
||||
} else {
|
||||
full_path = os.join_path_single(app.repo.dir, filename)
|
||||
full_path = os.join_path_single(app.repo.repo_dir, filename)
|
||||
}
|
||||
|
||||
return app.file(full_path)
|
||||
}
|
||||
|
||||
['/pkgs/:pkg'; put]
|
||||
fn (mut app App) put_package(pkg string) web.Result {
|
||||
if !app.is_authorized() {
|
||||
return app.text('Unauthorized.')
|
||||
}
|
||||
// ['/pkgs/:pkg'; put]
|
||||
// fn (mut app App) put_package(pkg string) web.Result {
|
||||
// if !app.is_authorized() {
|
||||
// return app.text('Unauthorized.')
|
||||
// }
|
||||
|
||||
if !is_pkg_name(pkg) {
|
||||
app.lwarn("Invalid package name '$pkg'.")
|
||||
// if !is_pkg_name(pkg) {
|
||||
// app.lwarn("Invalid package name '$pkg'.")
|
||||
|
||||
return app.text('Invalid filename.')
|
||||
}
|
||||
// return app.text('Invalid filename.')
|
||||
// }
|
||||
|
||||
if app.repo.exists(pkg) {
|
||||
app.lwarn("Duplicate package '$pkg'")
|
||||
// if app.repo.exists(pkg) {
|
||||
// app.lwarn("Duplicate package '$pkg'")
|
||||
|
||||
return app.text('File already exists.')
|
||||
}
|
||||
// return app.text('File already exists.')
|
||||
// }
|
||||
|
||||
pkg_path := app.repo.pkg_path(pkg)
|
||||
// pkg_path := app.repo.pkg_path(pkg)
|
||||
|
||||
if length := app.req.header.get(.content_length) {
|
||||
app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.")
|
||||
// if length := app.req.header.get(.content_length) {
|
||||
// app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.")
|
||||
|
||||
// This is used to time how long it takes to upload a file
|
||||
mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true })
|
||||
// // This is used to time how long it takes to upload a file
|
||||
// mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true })
|
||||
|
||||
reader_to_file(mut app.reader, length.int(), pkg_path) or {
|
||||
app.lwarn("Failed to upload package '$pkg'")
|
||||
// reader_to_file(mut app.reader, length.int(), pkg_path) or {
|
||||
// app.lwarn("Failed to upload package '$pkg'")
|
||||
|
||||
return app.text('Failed to upload file.')
|
||||
}
|
||||
// return app.text('Failed to upload file.')
|
||||
// }
|
||||
|
||||
sw.stop()
|
||||
app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.")
|
||||
} else {
|
||||
app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.")
|
||||
return app.text("Content-Type header isn't set.")
|
||||
}
|
||||
// sw.stop()
|
||||
// app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.")
|
||||
// } else {
|
||||
// app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.")
|
||||
// return app.text("Content-Type header isn't set.")
|
||||
// }
|
||||
|
||||
app.repo.add_package(pkg_path) or {
|
||||
app.lwarn("Failed to add package '$pkg' to database.")
|
||||
// app.repo.add_package(pkg_path) or {
|
||||
// app.lwarn("Failed to add package '$pkg' to database.")
|
||||
|
||||
os.rm(pkg_path) or { println('Failed to remove $pkg_path') }
|
||||
// os.rm(pkg_path) or { println('Failed to remove $pkg_path') }
|
||||
|
||||
return app.text('Failed to add package to repo.')
|
||||
}
|
||||
// return app.text('Failed to add package to repo.')
|
||||
// }
|
||||
|
||||
app.linfo("Added '$pkg' to repository.")
|
||||
// app.linfo("Added '$pkg' to repository.")
|
||||
|
||||
return app.text('Package added successfully.')
|
||||
// return app.text('Package added successfully.')
|
||||
// }
|
||||
|
||||
['/add'; put]
|
||||
pub fn (mut app App) add_package() web.Result {
|
||||
return app.text('')
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue