diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 4b87f8b..83dcb0e 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -1,16 +1,15 @@ -matrix: - PLATFORM: - - linux/amd64 - - linux/arm64 - - linux/arm/v7 - # These checks already get performed on the feature branches branches: exclude: [ main, dev ] -platform: ${PLATFORM} +platform: linux/amd64 pipeline: - # The default build isn't needed, as alpine switches to gcc for the compiler anyways + vieter: + image: 'chewingbever/vlang:latest' + group: 'build' + commands: + - make vieter + debug: image: 'chewingbever/vlang:latest' group: 'build' @@ -26,5 +25,4 @@ pipeline: - make prod # Make sure the binary is actually static - readelf -d pvieter - - du -h pvieter - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' diff --git a/.woodpecker/.build_arm64.yml b/.woodpecker/.build_arm64.yml new file mode 100644 index 0000000..704bc48 --- /dev/null +++ b/.woodpecker/.build_arm64.yml @@ -0,0 +1,28 @@ +# These checks already get performed on the feature branches +branches: + exclude: [ main, dev ] +platform: linux/arm64 + +pipeline: + vieter: + image: 'chewingbever/vlang:latest' + group: 'build' + commands: + - make vieter + + debug: + image: 'chewingbever/vlang:latest' + group: 'build' + commands: + - make debug + + prod: + image: 'chewingbever/vlang:latest' + environment: + - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static + group: 'build' + commands: + - make prod + # Make sure the binary is actually static + - readelf -d pvieter + - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' diff --git a/.woodpecker/.image.yml b/.woodpecker/.image.yml deleted file mode 100644 index 7f583a8..0000000 --- a/.woodpecker/.image.yml +++ /dev/null @@ -1,17 +0,0 @@ -# Because the only step here is a pull_request event, the branch should be dev -# because it has to be the target of the pull request -branches: dev -platform: linux/amd64 - -pipeline: - dryrun: - image: woodpeckerci/plugin-docker-buildx - secrets: [ docker_username, docker_password ] - settings: - repo: chewingbever/vieter - tag: dev - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] - dry_run: true - when: - event: pull_request - branch: dev diff --git a/.woodpecker/.publish.yml b/.woodpecker/.publish.yml index 9e5f39a..6399d7f 100644 --- a/.woodpecker/.publish.yml +++ b/.woodpecker/.publish.yml @@ -8,7 +8,7 @@ pipeline: settings: repo: chewingbever/vieter tag: dev - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm64/v8, linux/amd64 ] when: event: push branch: dev @@ -21,7 +21,7 @@ pipeline: tag: - latest - $CI_COMMIT_TAG - platforms: [ linux/arm/v7, linux/arm64/v8, linux/amd64 ] + platforms: [ linux/arm64/v8, linux/amd64 ] when: event: tag branch: main diff --git a/Makefile b/Makefile index 9a5d6fa..dc2273b 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ c: # Run the server in the default 'data' directory .PHONY: run run: vieter - API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter + API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG ./vieter .PHONY: run-prod run-prod: prod diff --git a/src/archive.v b/src/archive.v deleted file mode 100644 index bdc1846..0000000 --- a/src/archive.v +++ /dev/null @@ -1,100 +0,0 @@ -// Bindings for the libarchive library - -#flag -larchive - -#include "archive.h" - -struct C.archive {} - -// Create a new archive struct for reading -fn C.archive_read_new() &C.archive - -// Configure the archive to work with zstd compression -fn C.archive_read_support_filter_zstd(&C.archive) - -// Configure the archive to work with a tarball content -fn C.archive_read_support_format_tar(&C.archive) - -// Open an archive for reading -fn C.archive_read_open_filename(&C.archive, &char, int) int - -// Go to next entry header in archive -fn C.archive_read_next_header(&C.archive, &&C.archive_entry) int - -// Skip reading the current entry -fn C.archive_read_data_skip(&C.archive) - -// Free an archive -fn C.archive_read_free(&C.archive) int - -// Read an archive entry's contents into a pointer -fn C.archive_read_data(&C.archive, voidptr, int) - -// Create a new archive struct for writing -fn C.archive_write_new() &C.archive - -// Sets the filter for the archive to gzip -fn C.archive_write_add_filter_gzip(&C.archive) - -// Sets to archive to "pax restricted" mode. Libarchive's "pax restricted" -// format is a tar format that uses pax extensions only when absolutely -// necessary. Most of the time, it will write plain ustar entries. This is the -// recommended tar format for most uses. You should explicitly use ustar format -// only when you have to create archives that will be readable on older -// systems; you should explicitly request pax format only when you need to -// preserve as many attributes as possible. -fn C.archive_write_set_format_pax_restricted(&C.archive) - -// Opens up the filename for writing -fn C.archive_write_open_filename(&C.archive, &char) - -// Write an entry to the archive file -fn C.archive_write_header(&C.archive, &C.archive_entry) - -// Write the data in the buffer to the archive -fn C.archive_write_data(&C.archive, voidptr, int) - -// Close an archive for writing -fn C.archive_write_close(&C.archive) - -// Free the write archive -fn C.archive_write_free(&C.archive) - -#include "archive_entry.h" - -struct C.archive_entry {} - -// Create a new archive_entry struct -fn C.archive_entry_new() &C.archive_entry - -// Get the filename of the given entry -fn C.archive_entry_pathname(&C.archive_entry) &char - -// Get an entry's file size -// Note: this function actually returns an i64, but as this can't be used as an -// arugment to malloc, we'll just roll with it & assume an entry is never -// bigger than 4 gigs -fn C.archive_entry_size(&C.archive_entry) int - -// Set the pathname for the entry -fn C.archive_entry_set_pathname(&C.archive_entry, &char) - -// Sets the file size of the entry -fn C.archive_entry_set_size(&C.archive_entry, i64) - -// Sets the file type for an entry -fn C.archive_entry_set_filetype(&C.archive_entry, u32) - -// Sets the file permissions for an entry -fn C.archive_entry_set_perm(&C.archive_entry, int) - -// Clears out an entry struct -fn C.archive_entry_clear(&C.archive_entry) - -// Copy over a stat struct to the archive entry -fn C.archive_entry_copy_stat(entry &C.archive_entry, const_stat &C.stat) - -#include - -// Compare two C strings; 0 means they're equal -fn C.strcmp(&char, &char) int diff --git a/src/archive/archive.v b/src/archive/archive.v new file mode 100644 index 0000000..88649ee --- /dev/null +++ b/src/archive/archive.v @@ -0,0 +1,53 @@ +module archive + +import os + +// Returns the .PKGINFO file's contents & the list of files. +pub fn pkg_info(pkg_path string) ?(string, []string) { + if !os.is_file(pkg_path) { + return error("'$pkg_path' doesn't exist or isn't a file.") + } + + a := C.archive_read_new() + entry := C.archive_entry_new() + mut r := 0 + + C.archive_read_support_filter_all(a) + C.archive_read_support_format_all(a) + + // TODO find out where does this 10240 come from + r = C.archive_read_open_filename(a, &char(pkg_path.str), 10240) + defer { + C.archive_read_free(a) + } + + if r != C.ARCHIVE_OK { + return error('Failed to open package.') + } + + // We iterate over every header in search of the .PKGINFO one + mut buf := voidptr(0) + mut files := []string{} + for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { + pathname := C.archive_entry_pathname(entry) + + ignored_names := [c'.BUILDINFO', c'.INSTALL', c'.MTREE', c'.PKGINFO', c'.CHANGELOG'] + if ignored_names.all(C.strcmp(it, pathname) != 0) { + unsafe { + files << cstring_to_vstring(pathname) + } + } + + if C.strcmp(pathname, c'.PKGINFO') == 0 { + size := C.archive_entry_size(entry) + + // TODO can this unsafe block be avoided? + buf = unsafe { malloc(size) } + C.archive_read_data(a, voidptr(buf), size) + } else { + C.archive_read_data_skip(a) + } + } + + return unsafe { cstring_to_vstring(&char(buf)) }, files +} diff --git a/src/archive/bindings.v b/src/archive/bindings.v new file mode 100644 index 0000000..678d715 --- /dev/null +++ b/src/archive/bindings.v @@ -0,0 +1,46 @@ +module archive + +#flag -larchive + +#include "archive.h" + +struct C.archive {} + +// Create a new archive struct +fn C.archive_read_new() &C.archive +fn C.archive_read_support_filter_all(&C.archive) +fn C.archive_read_support_format_all(&C.archive) + +// Open an archive for reading +fn C.archive_read_open_filename(&C.archive, &char, int) int + +// Go to next entry header in archive +fn C.archive_read_next_header(&C.archive, &&C.archive_entry) int + +// Skip reading the current entry +fn C.archive_read_data_skip(&C.archive) + +// Free an archive +fn C.archive_read_free(&C.archive) int + +// Read an archive entry's contents into a pointer +fn C.archive_read_data(&C.archive, voidptr, int) + +#include "archive_entry.h" + +struct C.archive_entry {} + +// Create a new archive_entry struct +fn C.archive_entry_new() &C.archive_entry + +// Get the filename of the given entry +fn C.archive_entry_pathname(&C.archive_entry) &char + +// Get an entry's file size +// Note: this function actually returns an i64, but as this can't be used as an arugment to malloc, we'll just roll with it & assume an entry is never bigger than 4 gigs +fn C.archive_entry_size(&C.archive_entry) int + +#include + +// Compare two C strings; 0 means they're equal +fn C.strcmp(&char, &char) int diff --git a/src/main.v b/src/main.v index 97788e7..9d9e541 100644 --- a/src/main.v +++ b/src/main.v @@ -4,6 +4,8 @@ import web import os import log import io +import pkg +import archive import repo const port = 8000 @@ -16,7 +18,6 @@ struct App { web.Context pub: api_key string [required; web_global] - dl_dir string [required; web_global] pub mut: repo repo.Repo [required; web_global] } @@ -54,50 +55,62 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } } +// fn main2() { +// // Configure logger +// log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } +// log_level := log.level_from_tag(log_level_str) or { +// exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') +// } +// log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } + +// mut logger := log.Log{ +// level: log_level +// } + +// logger.set_full_logpath(log_file) +// logger.log_to_console_too() + +// defer { +// logger.info('Flushing log file') +// logger.flush() +// logger.close() +// } + +// // Configure web server +// key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } +// repo_dir := os.getenv_opt('REPO_DIR') or { +// exit_with_message(1, 'No repo directory was configured.') +// } + +// repo := repo.Repo{ +// dir: repo_dir +// name: db_name +// } + +// // We create the upload directory during startup +// if !os.is_dir(repo.pkg_dir()) { +// os.mkdir_all(repo.pkg_dir()) or { +// exit_with_message(2, "Failed to create repo directory '$repo.pkg_dir()'.") +// } + +// logger.info("Created package directory '$repo.pkg_dir()'.") +// } + +// web.run(&App{ +// logger: logger +// api_key: key +// repo: repo +// }, port) +// } + fn main() { - // Configure logger - log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } - log_level := log.level_from_tag(log_level_str) or { - exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + // archive.list_filenames() + res := pkg.read_pkg('test/homebank-5.5.1-1-x86_64.pkg.tar.zst') or { + eprintln(err.msg) + return } - log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } - - mut logger := log.Log{ - level: log_level - } - - logger.set_full_logpath(log_file) - logger.log_to_console_too() - - defer { - logger.info('Flushing log file') - logger.flush() - logger.close() - } - - // Configure web server - key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } - repo_dir := os.getenv_opt('REPO_DIR') or { - exit_with_message(1, 'No repo directory was configured.') - } - pkg_dir := os.getenv_opt('PKG_DIR') or { - exit_with_message(1, 'No package directory was configured.') - } - dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { - exit_with_message(1, 'No download directory was configured.') - } - - // This also creates the directories if needed - repo := repo.new(repo_dir, pkg_dir) or { - exit_with_message(1, 'Failed to create required directories.') - } - - os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } - - web.run(&App{ - logger: logger - api_key: key - dl_dir: dl_dir - repo: repo - }, port) + // println(info) + println(res.info) + print(res.files) + println(res.info.to_desc()) } diff --git a/src/package.v b/src/package.v deleted file mode 100644 index c229e3f..0000000 --- a/src/package.v +++ /dev/null @@ -1,259 +0,0 @@ -module package - -import os -import util - -// Represents a read archive -struct Pkg { -pub: - path string [required] - info PkgInfo [required] - files []string [required] -} - -// Represents the contents of a .PKGINFO file -struct PkgInfo { -pub mut: - // Single values - name string - base string - version string - description string - size i64 - csize i64 - url string - arch string - build_date i64 - packager string - // md5sum string - // sha256sum string - pgpsig string - pgpsigsize i64 - // Array values - groups []string - licenses []string - replaces []string - depends []string - conflicts []string - provides []string - optdepends []string - makedepends []string - checkdepends []string -} - -// checksum calculates the md5 & sha256 hash of the package -pub fn (p &Pkg) checksum() ?(string, string) { - return util.hash_file(p.path) -} - -// parse_pkg_info_string parses a PkgInfo object from a string -fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { - mut pkg_info := PkgInfo{} - - // Iterate over the entire string - for line in pkg_info_str.split_into_lines() { - // Skip any comment lines - if line.starts_with('#') { - continue - } - parts := line.split_nth('=', 2) - - if parts.len < 2 { - return error('Invalid line detected.') - } - - value := parts[1].trim_space() - key := parts[0].trim_space() - - match key { - // Single values - 'pkgname' { pkg_info.name = value } - 'pkgbase' { pkg_info.base = value } - 'pkgver' { pkg_info.version = value } - 'pkgdesc' { pkg_info.description = value } - 'csize' { continue } - 'size' { pkg_info.size = value.int() } - 'url' { pkg_info.url = value } - 'arch' { pkg_info.arch = value } - 'builddate' { pkg_info.build_date = value.int() } - 'packager' { pkg_info.packager = value } - 'md5sum' { continue } - 'sha256sum' { continue } - 'pgpsig' { pkg_info.pgpsig = value } - 'pgpsigsize' { pkg_info.pgpsigsize = value.int() } - // Array values - 'group' { pkg_info.groups << value } - 'license' { pkg_info.licenses << value } - 'replaces' { pkg_info.replaces << value } - 'depend' { pkg_info.depends << value } - 'conflict' { pkg_info.conflicts << value } - 'provides' { pkg_info.provides << value } - 'optdepend' { pkg_info.optdepends << value } - 'makedepend' { pkg_info.makedepends << value } - 'checkdepend' { pkg_info.checkdepends << value } - else { return error("Invalid key '$key'.") } - } - } - - return pkg_info -} - -// read_pkg extracts the file list & .PKGINFO contents from an archive -// NOTE: this command currently only supports zstd-compressed tarballs -pub fn read_pkg(pkg_path string) ?Pkg { - if !os.is_file(pkg_path) { - return error("'$pkg_path' doesn't exist or isn't a file.") - } - - a := C.archive_read_new() - entry := C.archive_entry_new() - mut r := 0 - - // Sinds 2020, all newly built Arch packages use zstd - C.archive_read_support_filter_zstd(a) - // The content should always be a tarball - C.archive_read_support_format_tar(a) - - // TODO find out where does this 10240 come from - r = C.archive_read_open_filename(a, &char(pkg_path.str), 10240) - defer { - C.archive_read_free(a) - } - - if r != C.ARCHIVE_OK { - return error('Failed to open package.') - } - - mut buf := voidptr(0) - mut files := []string{} - mut pkg_info := PkgInfo{} - - for C.archive_read_next_header(a, &entry) == C.ARCHIVE_OK { - pathname := C.archive_entry_pathname(entry) - - ignored_names := [c'.BUILDINFO', c'.INSTALL', c'.MTREE', c'.PKGINFO', c'.CHANGELOG'] - if ignored_names.all(C.strcmp(it, pathname) != 0) { - unsafe { - files << cstring_to_vstring(pathname) - } - } - - if C.strcmp(pathname, c'.PKGINFO') == 0 { - size := C.archive_entry_size(entry) - - // TODO can this unsafe block be avoided? - buf = unsafe { malloc(size) } - C.archive_read_data(a, buf, size) - - unsafe { - println(cstring_to_vstring(buf)) - } - pkg_info = parse_pkg_info_string(unsafe { cstring_to_vstring(buf) }) ? - - unsafe { - free(buf) - } - } else { - C.archive_read_data_skip(a) - } - } - - pkg_info.csize = i64(os.file_size(pkg_path)) - - return Pkg{ - path: pkg_path - info: pkg_info - files: files - } -} - -fn format_entry(key string, value string) string { - return '\n%$key%\n$value\n' -} - -// filename returns the correct filename of the package file -pub fn (pkg &Pkg) filename() string { - p := pkg.info - - return '$p.name-$p.version-${p.arch}.pkg.tar.zst' -} - -// to_desc returns a desc file valid string representation -// TODO calculate md5 & sha256 instead of believing the file -pub fn (pkg &Pkg) to_desc() string { - p := pkg.info - - // filename - mut desc := '%FILENAME%\n$pkg.filename()\n' - - desc += format_entry('NAME', p.name) - desc += format_entry('BASE', p.base) - desc += format_entry('VERSION', p.version) - - if p.description.len > 0 { - desc += format_entry('DESC', p.description) - } - - if p.groups.len > 0 { - desc += format_entry('GROUPS', p.groups.join_lines()) - } - - desc += format_entry('CSIZE', p.csize.str()) - desc += format_entry('ISIZE', p.size.str()) - - md5sum, _ := pkg.checksum() or { '', '' } - - desc += format_entry('MD5SUM', md5sum) - - // TODO add this - // desc += format_entry('SHA256SUM', sha256sum) - - // TODO add pgpsig stuff - - if p.url.len > 0 { - desc += format_entry('URL', p.url) - } - - if p.licenses.len > 0 { - desc += format_entry('LICENSE', p.licenses.join_lines()) - } - - desc += format_entry('ARCH', p.arch) - desc += format_entry('BUILDDATE', p.build_date.str()) - desc += format_entry('PACKAGER', p.packager) - - if p.replaces.len > 0 { - desc += format_entry('REPLACES', p.replaces.join_lines()) - } - - if p.conflicts.len > 0 { - desc += format_entry('CONFLICTS', p.conflicts.join_lines()) - } - - if p.provides.len > 0 { - desc += format_entry('PROVIDES', p.provides.join_lines()) - } - - if p.depends.len > 0 { - desc += format_entry('DEPENDS', p.depends.join_lines()) - } - - if p.optdepends.len > 0 { - desc += format_entry('OPTDEPENDS', p.optdepends.join_lines()) - } - - if p.makedepends.len > 0 { - desc += format_entry('MAKEDEPENDS', p.makedepends.join_lines()) - } - - if p.checkdepends.len > 0 { - desc += format_entry('CHECKDEPENDS', p.checkdepends.join_lines()) - } - - return '$desc\n' -} - -// to_files returns a files file valid string representation -pub fn (pkg &Pkg) to_files() string { - return '%FILES%\n$pkg.files.join_lines()\n' -} diff --git a/src/pkg.v b/src/pkg.v new file mode 100644 index 0000000..1bb1abc --- /dev/null +++ b/src/pkg.v @@ -0,0 +1,108 @@ +module pkg + +import archive +import time + +struct Pkg { +pub: + info PkgInfo [required] + files []string [required] +} + +struct PkgInfo { +mut: + // Single values + name string + base string + version string + description string + size i64 + csize i64 + url string + arch string + build_date i64 + packager string + md5sum string + sha256sum string + pgpsig string + pgpsigsize i64 + // Array values + groups []string + licenses []string + replaces []string + depends []string + conflicts []string + provides []string + optdepends []string + makedepends []string + checkdepends []string +} + +fn parse_pkg_info_string(pkg_info_str &string) ?PkgInfo { + mut pkg_info := PkgInfo{} + + // Iterate over the entire string + for line in pkg_info_str.split_into_lines() { + // Skip any comment lines + if line.starts_with('#') { + continue + } + parts := line.split_nth('=', 2) + + if parts.len < 2 { + return error('Invalid line detected.') + } + + value := parts[1].trim_space() + key := parts[0].trim_space() + + match key { + // Single values + 'pkgname' { pkg_info.name = value } + 'pkgbase' { pkg_info.base = value } + 'pkgver' { pkg_info.version = value } + 'pkgdesc' { pkg_info.description = value } + 'csize' { pkg_info.csize = value.int() } + 'size' { pkg_info.size = value.int() } + 'url' { pkg_info.url = value } + 'arch' { pkg_info.arch = value } + 'builddate' { pkg_info.build_date = value.int() } + 'packager' { pkg_info.packager = value } + 'md5sum' { pkg_info.md5sum = value } + 'sha256sum' { pkg_info.sha256sum = value } + 'pgpsig' { pkg_info.pgpsig = value } + 'pgpsigsize' { pkg_info.pgpsigsize = value.int() } + // Array values + 'group' { pkg_info.groups << value } + 'license' { pkg_info.licenses << value } + 'replaces' { pkg_info.replaces << value } + 'depend' { pkg_info.depends << value } + 'conflict' { pkg_info.conflicts << value } + 'provides' { pkg_info.provides << value } + 'optdepend' { pkg_info.optdepends << value } + 'makedepend' { pkg_info.makedepends << value } + 'checkdepend' { pkg_info.checkdepends << value } + else { return error("Invalid key '$key'.") } + } + } + + return pkg_info +} + +pub fn read_pkg(pkg_path string) ?Pkg { + pkg_info_str, files := archive.pkg_info(pkg_path) ? + pkg_info := parse_pkg_info_string(pkg_info_str) ? + + return Pkg{ + info: pkg_info + files: files + } +} + +// Represent a PkgInfo struct as a desc file +pub fn (p &PkgInfo) to_desc() string { + // TODO calculate md5 & sha256 instead of believing the file + mut desc := '' + + return desc +} diff --git a/src/repo.v b/src/repo.v index cca6b84..ae76cc9 100644 --- a/src/repo.v +++ b/src/repo.v @@ -1,13 +1,9 @@ module repo import os -import package +import archive -// subpath where the uncompressed version of the files archive is stored -const files_subpath = 'files' - -// subpath where the uncompressed version of the repo archive is stored -const repo_subpath = 'repo' +const pkgs_subpath = 'pkgs' // Dummy struct to work around the fact that you can only share structs, maps & // arrays @@ -26,120 +22,44 @@ pub: pkg_dir string [required] } -// new creates a new Repo & creates the directories as needed -pub fn new(repo_dir string, pkg_dir string) ?Repo { - if !os.is_dir(repo_dir) { - os.mkdir_all(repo_dir) or { return error('Failed to create repo directory.') } - } - - if !os.is_dir(pkg_dir) { - os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') } - } - - return Repo{ - repo_dir: repo_dir - pkg_dir: pkg_dir - } +// Returns whether the repository contains the given package. +pub fn (r &Repo) contains(pkg string) bool { + return os.exists(os.join_path(r.repo_dir, 'files', pkg)) } -// add_from_path adds a package from an arbitrary path & moves it into the pkgs -// directory if necessary. -pub fn (r &Repo) add_from_path(pkg_path string) ?bool { - pkg := package.read_pkg(pkg_path) or { return error('Failed to read package file: $err.msg') } - - added := r.add(pkg) ? - - // If the add was successful, we move the file to the packages directory - if added { - dest_path := os.real_path(os.join_path_single(r.pkg_dir, pkg.filename())) - - // Only move the file if it's not already in the package directory - if dest_path != os.real_path(pkg_path) { - os.mv(pkg_path, dest_path) ? - } - } - - return added +// Adds the given package to the repo. If false, the package was already +// present in the repository. +pub fn (r &Repo) add(pkg string) ?bool { + return false } -// add adds a given Pkg to the repository -fn (r &Repo) add(pkg &package.Pkg) ?bool { - pkg_dir := r.pkg_path(pkg) - - // We can't add the same package twice - if os.exists(pkg_dir) { - return false - } - - os.mkdir(pkg_dir) or { return error('Failed to create package directory.') } - - os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or { - os.rmdir_all(pkg_dir) ? - - return error('Failed to write desc file.') - } - os.write_file(os.join_path_single(pkg_dir, 'files'), pkg.to_files()) or { - os.rmdir_all(pkg_dir) ? - - return error('Failed to write files file.') - } - - r.sync() ? - - return true +// Re-generate the db & files archives. +fn (r &Repo) genenerate() ? { } -// Returns the path where the given package's desc & files files are stored -fn (r &Repo) pkg_path(pkg &package.Pkg) string { - return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version') +// Returns path to the given package, prepended with the repo's path. +pub fn (r &Repo) pkg_path(pkg string) string { + return os.join_path_single(r.pkg_dir, pkg) } -// Re-generate the repo archive files -fn (r &Repo) sync() ? { +pub fn (r &Repo) exists(pkg string) bool { + return os.exists(r.pkg_path(pkg)) +} + +// Returns the full path to the database file +pub fn (r &Repo) db_path() string { + return os.join_path_single(r.repo_dir, 'repo.tar.gz') +} + +pub fn (r &Repo) add_package(pkg_path string) ? { + mut res := os.Result{} + lock r.mutex { - a := C.archive_write_new() - entry := C.archive_entry_new() - st := C.stat{} - buf := [8192]byte{} + res = os.execute("repo-add '$r.db_path()' '$pkg_path'") + } - // This makes the archive a gzip-compressed tarball - C.archive_write_add_filter_gzip(a) - C.archive_write_set_format_pax_restricted(a) - - repo_path := os.join_path_single(r.repo_dir, 'repo.db') - - C.archive_write_open_filename(a, &char(repo_path.str)) - - // Iterate over each directory - for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir, - it))) { - inner_path := os.join_path_single(d, 'desc') - actual_path := os.join_path_single(r.repo_dir, inner_path) - - unsafe { - C.stat(&char(actual_path.str), &st) - } - - C.archive_entry_set_pathname(entry, &char(inner_path.str)) - C.archive_entry_copy_stat(entry, &st) - // C.archive_entry_set_size(entry, st.st_size) - // C.archive_entry_set_filetype(entry, C.AE_IFREG) - // C.archive_entry_set_perm(entry, 0o644) - C.archive_write_header(a, entry) - - fd := C.open(&char(actual_path.str), C.O_RDONLY) - mut len := C.read(fd, &buf, sizeof(buf)) - - for len > 0 { - C.archive_write_data(a, &buf, len) - len = C.read(fd, &buf, sizeof(buf)) - } - C.close(fd) - - C.archive_entry_clear(entry) - } - - C.archive_write_close(a) - C.archive_write_free(a) + if res.exit_code != 0 { + println(res.output) + return error('repo-add failed.') } } diff --git a/src/routes.v b/src/routes.v index 8ad4a4a..c7df286 100644 --- a/src/routes.v +++ b/src/routes.v @@ -4,11 +4,9 @@ import web import os import repo import time -import rand const prefixes = ['B', 'KB', 'MB', 'GB'] -// pretty_bytes converts a byte count to human-readable version fn pretty_bytes(bytes int) string { mut i := 0 mut n := f32(bytes) @@ -25,7 +23,6 @@ fn is_pkg_name(s string) bool { return s.contains('.pkg') } -// get_root handles a GET request for a file on the root ['/:filename'; get] fn (mut app App) get_root(filename string) web.Result { mut full_path := '' @@ -39,57 +36,58 @@ fn (mut app App) get_root(filename string) web.Result { return app.file(full_path) } -['/publish'; post] -fn (mut app App) put_package() web.Result { - if !app.is_authorized() { - return app.text('Unauthorized.') - } +// ['/pkgs/:pkg'; put] +// fn (mut app App) put_package(pkg string) web.Result { +// if !app.is_authorized() { +// return app.text('Unauthorized.') +// } - mut pkg_path := '' +// if !is_pkg_name(pkg) { +// app.lwarn("Invalid package name '$pkg'.") - if length := app.req.header.get(.content_length) { - // Generate a random filename for the temp file - pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) +// return app.text('Invalid filename.') +// } - for os.exists(pkg_path) { - pkg_path = os.join_path_single(app.dl_dir, rand.uuid_v4()) - } +// if app.repo.exists(pkg) { +// app.lwarn("Duplicate package '$pkg'") - app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to '$pkg_path'.") +// return app.text('File already exists.') +// } - // This is used to time how long it takes to upload a file - mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) +// pkg_path := app.repo.pkg_path(pkg) - reader_to_file(mut app.reader, length.int(), pkg_path) or { - app.lwarn("Failed to upload '$pkg_path'") +// if length := app.req.header.get(.content_length) { +// app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.") - return app.text('Failed to upload file.') - } +// // This is used to time how long it takes to upload a file +// mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) - sw.stop() - app.ldebug("Upload of '$pkg_path' completed in ${sw.elapsed().seconds():.3}s.") - } else { - app.lwarn('Tried to upload package without specifying a Content-Length.') - return app.text("Content-Type header isn't set.") - } +// reader_to_file(mut app.reader, length.int(), pkg_path) or { +// app.lwarn("Failed to upload package '$pkg'") - added := app.repo.add_from_path(pkg_path) or { - app.lerror('Error while adding package: $err.msg') +// return app.text('Failed to upload file.') +// } - return app.text('Failed to add package.') - } - if !added { - app.lwarn('Duplicate package.') +// sw.stop() +// app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.") +// } else { +// app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.") +// return app.text("Content-Type header isn't set.") +// } - return app.text('File already exists.') - } +// app.repo.add_package(pkg_path) or { +// app.lwarn("Failed to add package '$pkg' to database.") - app.linfo("Added '$pkg_path' to repository.") +// os.rm(pkg_path) or { println('Failed to remove $pkg_path') } - return app.text('Package added successfully.') -} +// return app.text('Failed to add package to repo.') +// } + +// app.linfo("Added '$pkg' to repository.") + +// return app.text('Package added successfully.') +// } -// add_package PUT a new package to the server ['/add'; put] pub fn (mut app App) add_package() web.Result { return app.text('') diff --git a/src/util.v b/src/util.v deleted file mode 100644 index 066a58d..0000000 --- a/src/util.v +++ /dev/null @@ -1,33 +0,0 @@ -module util - -import os -import crypto.md5 -// import crypto.sha256 - -// hash_file returns the md5 & sha256 hash of a given file -// TODO actually implement sha256 -pub fn hash_file(path &string) ?(string, string) { - file := os.open(path) or { return error('Failed to open file.') } - - mut md5sum := md5.new() - // mut sha256sum := sha256.new() - - buf_size := int(1_000_000) - mut buf := []byte{len: buf_size} - mut bytes_left := os.file_size(path) - - for bytes_left > 0 { - // TODO check if just breaking here is safe - bytes_read := file.read(mut buf) or { return error('Failed to read from file.') } - bytes_left -= u64(bytes_read) - - // For now we'll assume that this always works - md5sum.write(buf[..bytes_read]) or { - return error('Failed to update checksum. This should never happen.') - } - // sha256sum.write(buf) or {} - } - - // return md5sum.sum(buf).hex(), sha256sum.sum(buf).hex() - return md5sum.checksum().hex(), '' -} diff --git a/src/web/logging.v b/src/web/logging.v index fc697ff..66426f2 100644 --- a/src/web/logging.v +++ b/src/web/logging.v @@ -2,34 +2,28 @@ module web import log -// log reate a log message with the given level pub fn (mut ctx Context) log(msg &string, level log.Level) { lock ctx.logger { ctx.logger.send_output(msg, level) } } -// lfatal create a log message with the fatal level pub fn (mut ctx Context) lfatal(msg &string) { ctx.log(msg, log.Level.fatal) } -// lerror create a log message with the error level pub fn (mut ctx Context) lerror(msg &string) { ctx.log(msg, log.Level.error) } -// lwarn create a log message with the warn level pub fn (mut ctx Context) lwarn(msg &string) { ctx.log(msg, log.Level.warn) } -// linfo create a log message with the info level pub fn (mut ctx Context) linfo(msg &string) { ctx.log(msg, log.Level.info) } -// ldebug create a log message with the debug level pub fn (mut ctx Context) ldebug(msg &string) { ctx.log(msg, log.Level.debug) } diff --git a/src/web/web.v b/src/web/web.v index 2db8f18..404bd08 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -187,14 +187,14 @@ struct Route { } // Defining this method is optional. -// init_server is called at server start. +// This method called at server start. // You can use it for initializing globals. pub fn (ctx Context) init_server() { eprintln('init_server() has been deprecated, please init your web app in `fn main()`') } // Defining this method is optional. -// before_request is called before every request (aka middleware). +// This method called before every request (aka middleware). // Probably you can use it for check user session cookie or add header. pub fn (ctx Context) before_request() {} @@ -206,7 +206,7 @@ pub struct Cookie { http_only bool } -// send_response_to_client sends a response to the client +// web intern function [manualfree] pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bool { if ctx.done { @@ -230,33 +230,33 @@ pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bo return true } -// html HTTP_OK with s as payload with content-type `text/html` +// Response HTTP_OK with s as payload with content-type `text/html` pub fn (mut ctx Context) html(s string) Result { ctx.send_response_to_client('text/html', s) return Result{} } -// text HTTP_OK with s as payload with content-type `text/plain` +// Response HTTP_OK with s as payload with content-type `text/plain` pub fn (mut ctx Context) text(s string) Result { ctx.send_response_to_client('text/plain', s) return Result{} } -// json HTTP_OK with json_s as payload with content-type `application/json` +// Response HTTP_OK with json_s as payload with content-type `application/json` pub fn (mut ctx Context) json(j T) Result { json_s := json.encode(j) ctx.send_response_to_client('application/json', json_s) return Result{} } -// json_pretty Response HTTP_OK with a pretty-printed JSON result +// Response HTTP_OK with a pretty-printed JSON result pub fn (mut ctx Context) json_pretty(j T) Result { json_s := json.encode_pretty(j) ctx.send_response_to_client('application/json', json_s) return Result{} } -// file Response HTTP_OK with file as payload +// Response HTTP_OK with file as payload // This function manually implements responses because it needs to stream the file contents pub fn (mut ctx Context) file(f_path string) Result { if ctx.done { @@ -329,13 +329,13 @@ pub fn (mut ctx Context) file(f_path string) Result { return Result{} } -// ok Response HTTP_OK with s as payload +// Response HTTP_OK with s as payload pub fn (mut ctx Context) ok(s string) Result { ctx.send_response_to_client(ctx.content_type, s) return Result{} } -// server_error Response a server error +// Response a server error pub fn (mut ctx Context) server_error(ecode int) Result { $if debug { eprintln('> ctx.server_error ecode: $ecode') @@ -347,7 +347,7 @@ pub fn (mut ctx Context) server_error(ecode int) Result { return Result{} } -// redirect Redirect to an url +// Redirect to an url pub fn (mut ctx Context) redirect(url string) Result { if ctx.done { return Result{} @@ -360,7 +360,7 @@ pub fn (mut ctx Context) redirect(url string) Result { return Result{} } -// not_found Send an not_found response +// Send an not_found response pub fn (mut ctx Context) not_found() Result { if ctx.done { return Result{} @@ -370,7 +370,7 @@ pub fn (mut ctx Context) not_found() Result { return Result{} } -// set_cookie Sets a cookie +// Sets a cookie pub fn (mut ctx Context) set_cookie(cookie Cookie) { mut cookie_data := []string{} mut secure := if cookie.secure { 'Secure;' } else { '' } @@ -383,17 +383,17 @@ pub fn (mut ctx Context) set_cookie(cookie Cookie) { ctx.add_header('Set-Cookie', '$cookie.name=$cookie.value; $data') } -// set_content_type Sets the response content type +// Sets the response content type pub fn (mut ctx Context) set_content_type(typ string) { ctx.content_type = typ } -// set_cookie_with_expire_date Sets a cookie with a `expire_data` +// Sets a cookie with a `expire_data` pub fn (mut ctx Context) set_cookie_with_expire_date(key string, val string, expire_date time.Time) { ctx.add_header('Set-Cookie', '$key=$val; Secure; HttpOnly; expires=$expire_date.utc_string()') } -// get_cookie Gets a cookie by a key +// Gets a cookie by a key pub fn (ctx &Context) get_cookie(key string) ?string { // TODO refactor mut cookie_header := ctx.get_header('cookie') if cookie_header == '' { @@ -413,7 +413,7 @@ pub fn (ctx &Context) get_cookie(key string) ?string { // TODO refactor return error('Cookie not found') } -// set_status Sets the response status +// Sets the response status pub fn (mut ctx Context) set_status(code int, desc string) { if code < 100 || code > 599 { ctx.status = '500 Internal Server Error' @@ -422,12 +422,12 @@ pub fn (mut ctx Context) set_status(code int, desc string) { } } -// add_header Adds an header to the response with key and val +// Adds an header to the response with key and val pub fn (mut ctx Context) add_header(key string, val string) { ctx.header.add_custom(key, val) or {} } -// get_header Returns the header data from the key +// Returns the header data from the key pub fn (ctx &Context) get_header(key string) string { return ctx.req.header.get_custom(key) or { '' } } @@ -436,7 +436,7 @@ interface DbInterface { db voidptr } -// run runs the app +// run_app [manualfree] pub fn run(global_app &T, port int) { mut l := net.listen_tcp(.ip6, ':$port') or { panic('failed to listen $err.code $err') } @@ -478,7 +478,6 @@ pub fn run(global_app &T, port int) { } } -// handle_conn handles a connection [manualfree] fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { conn.set_read_timeout(30 * time.second) @@ -616,7 +615,6 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { conn.write(web.http_404.bytes()) or {} } -// route_matches returns wether a route matches fn route_matches(url_words []string, route_words []string) ?[]string { // URL path should be at least as long as the route path // except for the catchall route (`/:path...`) @@ -659,7 +657,7 @@ fn route_matches(url_words []string, route_words []string) ?[]string { return params } -// serve_if_static checks if request is for a static file and serves it +// check if request is for a static file and serves it // returns true if we served a static file, false otherwise [manualfree] fn serve_if_static(mut app T, url urllib.URL) bool { @@ -678,7 +676,6 @@ fn serve_if_static(mut app T, url urllib.URL) bool { return true } -// scan_static_directory makes a static route for each file in a directory fn (mut ctx Context) scan_static_directory(directory_path string, mount_path string) { files := os.ls(directory_path) or { panic(err) } if files.len > 0 { @@ -698,7 +695,7 @@ fn (mut ctx Context) scan_static_directory(directory_path string, mount_path str } } -// handle_static Handles a directory static +// Handles a directory static // If `root` is set the mount path for the dir will be in '/' pub fn (mut ctx Context) handle_static(directory_path string, root bool) bool { if ctx.done || !os.exists(directory_path) { @@ -727,7 +724,7 @@ pub fn (mut ctx Context) mount_static_folder_at(directory_path string, mount_pat return true } -// serve_static Serves a file static +// Serves a file static // `url` is the access path on the site, `file_path` is the real path to the file, `mime_type` is the file type pub fn (mut ctx Context) serve_static(url string, file_path string) { ctx.static_files[url] = file_path @@ -736,7 +733,7 @@ pub fn (mut ctx Context) serve_static(url string, file_path string) { ctx.static_mime_types[url] = web.mime_types[ext] } -// ip Returns the ip address from the current user +// Returns the ip address from the current user pub fn (ctx &Context) ip() string { mut ip := ctx.req.header.get(.x_forwarded_for) or { '' } if ip == '' { @@ -752,23 +749,22 @@ pub fn (ctx &Context) ip() string { return ip } -// error Set s to the form error +// Set s to the form error pub fn (mut ctx Context) error(s string) { println('web error: $s') ctx.form_error = s } -// not_found Returns an empty result +// Returns an empty result pub fn not_found() Result { return Result{} } -// send_string fn send_string(mut conn net.TcpConn, s string) ? { conn.write(s.bytes()) ? } -// filter Do not delete. +// Do not delete. // It used by `vlib/v/gen/c/str_intp.v:130` for string interpolation inside web templates // TODO: move it to template render fn filter(s string) string {