diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index 93c9840..742095c 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -25,8 +25,6 @@ pipeline: # inside the repo - curl -OL https://git.rustybever.be/Chewing_Bever/vieter/raw/branch/dev/PKGBUILD - makepkg -s --noconfirm --needed - when: - event: push publish: image: 'curlimages/curl' @@ -35,5 +33,3 @@ pipeline: - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done' secrets: - vieter_api_key - when: - event: push diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index a3b66ea..b41a39d 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -1,8 +1,11 @@ matrix: PLATFORM: - - 'linux/amd64' - - 'linux/arm64' + - linux/amd64 + - linux/arm64 + # I just don't have a performant enough runner for this platform + # - linux/arm/v7 +# These checks already get performed on the feature branches platform: ${PLATFORM} pipeline: @@ -12,15 +15,15 @@ pipeline: commands: - make when: - event: [push, pull_request] + event: push branch: - exclude: [main] + exclude: [main, dev] prod: image: 'chewingbever/vlang:latest' pull: true environment: - - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static + - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static commands: # Apparently this -D is *very* important - CFLAGS='-DGC_THREADS=1' make prod @@ -32,7 +35,7 @@ pipeline: - strip -s pvieter - du -h pvieter when: - event: [push, pull_request] + event: push upload: image: 'chewingbever/vlang:latest' @@ -49,7 +52,6 @@ pipeline: - > curl --silent - --fail -XPUT -T pvieter -H "Host: $URL" @@ -58,4 +60,4 @@ pipeline: -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" https://$URL$OBJ_PATH when: - event: [push, pull_request] + event: push diff --git a/.woodpecker/.build_experimental.yml b/.woodpecker/.build_experimental.yml new file mode 100644 index 0000000..0129d2b --- /dev/null +++ b/.woodpecker/.build_experimental.yml @@ -0,0 +1,29 @@ +# These builds are not important for the project, but might be valuable for +# fixing bugs in the V compiler. + +platform: linux/amd64 +branches: + exclude: [master, dev] + +pipeline: + autofree: + image: 'chewingbever/vlang:latest' + pull: true + group: 'build' + commands: + - make autofree + - readelf -d afvieter + - du -h afvieter + when: + event: push + + skip-unused: + image: 'chewingbever/vlang:latest' + pull: true + group: 'build' + commands: + - make skip-unused + - readelf -d suvieter + - du -h suvieter + when: + event: push diff --git a/.woodpecker/.deploy.yml b/.woodpecker/.deploy.yml index 8e68641..dd77fb9 100644 --- a/.woodpecker/.deploy.yml +++ b/.woodpecker/.deploy.yml @@ -1,4 +1,4 @@ -branches: [ 'dev' ] +branches: 'dev' platform: 'linux/amd64' depends_on: - 'docker' @@ -14,5 +14,3 @@ pipeline: commands: - 'curl -XPOST -s --fail $WEBHOOK_APP' - 'curl -XPOST -s --fail $WEBHOOK_CRON' - when: - event: push diff --git a/.woodpecker/.docker.yml b/.woodpecker/.docker.yml index bab869b..9b605f3 100644 --- a/.woodpecker/.docker.yml +++ b/.woodpecker/.docker.yml @@ -1,34 +1,30 @@ branches: [main, dev] -platform: 'linux/amd64' +platform: linux/amd64 depends_on: - build pipeline: dev: - image: 'woodpeckerci/plugin-docker-buildx' - secrets: - - 'docker_username' - - 'docker_password' + image: woodpeckerci/plugin-docker-buildx + secrets: [ docker_username, docker_password ] settings: - repo: 'chewingbever/vieter' - tag: 'dev' - platforms: [ 'linux/arm64/v8', 'linux/amd64' ] + repo: chewingbever/vieter + tag: dev + platforms: [ linux/arm64/v8, linux/amd64 ] build_args_from_env: - - 'CI_COMMIT_SHA' + - CI_COMMIT_SHA when: event: push branch: dev release: - image: 'woodpeckerci/plugin-docker-buildx' - secrets: - - 'docker_username' - - 'docker_password' + image: woodpeckerci/plugin-docker-buildx + secrets: [ docker_username, docker_password ] settings: - repo: 'chewingbever/vieter' + repo: chewingbever/vieter auto_tag: true - platforms: [ 'linux/arm64/v8', 'linux/amd64' ] + platforms: [ linux/arm64/v8, linux/amd64 ] build_args_from_env: - - 'CI_COMMIT_SHA' + - CI_COMMIT_SHA when: event: tag diff --git a/.woodpecker/.gitea.yml b/.woodpecker/.gitea.yml index 1877050..c492d34 100644 --- a/.woodpecker/.gitea.yml +++ b/.woodpecker/.gitea.yml @@ -1,5 +1,6 @@ -platform: 'linux/amd64' -branches: [ 'main' ] +# Yeah so this only works on tags so we'll worry about this later +platform: linux/amd64 +branches: main depends_on: - build diff --git a/.woodpecker/.lint.yml b/.woodpecker/.lint.yml index b13aea4..b1c16fd 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/.lint.yml @@ -1,7 +1,7 @@ # These checks already get performed on the feature branches branches: - exclude: [ main ] -platform: 'linux/amd64' + exclude: [ main, dev ] +platform: linux/amd64 pipeline: lint: @@ -9,5 +9,3 @@ pipeline: pull: true commands: - make lint - when: - event: [ push, pull_request ] diff --git a/.woodpecker/.test.yml b/.woodpecker/.test.yml index 0337570..3800cc1 100644 --- a/.woodpecker/.test.yml +++ b/.woodpecker/.test.yml @@ -1,10 +1,10 @@ matrix: PLATFORM: - - 'linux/amd64' - - 'linux/arm64' + - linux/amd64 + - linux/arm64 branches: - exclude: [ main ] + exclude: [main, dev] platform: ${PLATFORM} pipeline: @@ -14,4 +14,4 @@ pipeline: commands: - make test when: - event: [push, pull_request] + event: push diff --git a/CHANGELOG.md b/CHANGELOG.md index f40ce52..abbdc74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed * Switched from compiler fork to fully vanilla compiler mirror -* `download_dir`, `repos_file` & `repos_dir` config values have been replaced - with `data_dir` -* Storage of metadata (e.g. Git repositories) is now done using Sqlite ### Added diff --git a/Dockerfile b/Dockerfile index 2ba6181..58087ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN if [ -n "${CI_COMMIT_SHA}" ]; then \ "https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \ chmod +x vieter ; \ else \ - LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static' make prod && \ + LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' make prod && \ mv pvieter vieter ; \ fi @@ -31,8 +31,10 @@ RUN if [ -n "${CI_COMMIT_SHA}" ]; then \ FROM busybox:1.35.0 ENV PATH=/bin \ - VIETER_DATA_DIR=/data \ - VIETER_PKG_DIR=/data/pkgs + VIETER_REPOS_DIR=/data/repos \ + VIETER_PKG_DIR=/data/pkgs \ + VIETER_DOWNLOAD_DIR=/data/downloads \ + VIETER_REPOS_FILE=/data/repos.json COPY --from=builder /app/dumb-init /app/vieter /bin/ diff --git a/docs/content/configuration.md b/docs/content/configuration.md index ded40cb..df92844 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -35,10 +35,18 @@ passed to them. Each mode requires a different configuration. * `log_file`: log file to write logs to. Defaults to `vieter.log` in the current directory. * `pkg_dir`: where Vieter should store the actual package archives. -* `data_dir`: where Vieter stores the repositories, log file & database. +* `download_dir`: where Vieter should initially download uploaded files. * `api_key`: the API key to use when authenticating requests. +* `repo_dir`: where Vieter should store the contents of the repository. +* `repos_file`: JSON file where the list of Git repositories is saved * `default_arch`: architecture to always add packages of arch `any` to. +{{< hint info >}} +**Note** +Because Vieter hard links files between `download_dir` & `pkg_dir`, they need +to be on the same file system. +{{< /hint >}} + ### Builder * `api_key`: the API key to use when authenticating requests. @@ -54,18 +62,3 @@ passed to them. Each mode requires a different configuration. * `api_key`: the API key to use when authenticating requests. * `address`: Base your URL of your Vieter instance, e.g. https://example.com - -### Cron - -* `log_level`: defines how much logs to show. Valid values are one of `FATAL`, - `ERROR`, `WARN`, `INFO` or `DEBUG`. Defaults to `WARN` -* `api_key`: the API key to use when authenticating requests. -* `address`: Base your URL of your Vieter instance, e.g. https://example.com. - This *must* be the publicly facing URL of your Vieter instance. -* `data_dir`: where Vieter stores the log file. -* `base_image`: Docker image from which to create the builder images. -* `max_concurrent_builds`: amount of builds to run at once. -* `api_update_frequency`: how frequenty to check for changes in the repo list. -* `image_rebuild+frequency`: how frequently to rebuild the builder image -* `global_schedule`: cron schedule to use for any repo without an individual - schedule diff --git a/src/build/build.v b/src/build/build.v index 15a5eb8..bc604fa 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -5,7 +5,6 @@ import encoding.base64 import time import git import os -import db const container_build_dir = '/build' @@ -76,7 +75,7 @@ pub fn create_build_image(base_image string) ?string { // build_repo builds, packages & publishes a given Arch package based on the // provided GitRepo. The base image ID should be of an image previously created // by create_build_image. -pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ? { +pub fn build_repo(address string, api_key string, base_image_id string, repo &git.GitRepo) ? { build_arch := os.uname().machine // TODO what to do with PKGBUILDs that build multiple packages? @@ -126,11 +125,11 @@ fn build(conf Config) ? { build_arch := os.uname().machine // We get the repos map from the Vieter instance - repos := git.get_repos(conf.address, conf.api_key) ? + repos_map := git.get_repos(conf.address, conf.api_key) ? // We filter out any repos that aren't allowed to be built on this // architecture - filtered_repos := repos.filter(it.arch.map(it.value).contains(build_arch)) + filtered_repos := repos_map.keys().map(repos_map[it]).filter(it.arch.contains(build_arch)) // No point in doing work if there's no repos present if filtered_repos.len == 0 { diff --git a/src/cron/cli.v b/src/cron/cli.v index 15bc986..24cbe2c 100644 --- a/src/cron/cli.v +++ b/src/cron/cli.v @@ -6,9 +6,9 @@ import env struct Config { pub: log_level string = 'WARN' + log_file string = 'vieter.log' api_key string address string - data_dir string base_image string = 'archlinux:base-devel' max_concurrent_builds int = 1 api_update_frequency int = 15 diff --git a/src/cron/cron.v b/src/cron/cron.v index e356faa..e10e4dd 100644 --- a/src/cron/cron.v +++ b/src/cron/cron.v @@ -3,9 +3,6 @@ module cron import log import cron.daemon import cron.expression -import os - -const log_file_name = 'vieter.cron.log' // cron starts a cron daemon & starts periodically scheduling builds. pub fn cron(conf Config) ? { @@ -18,8 +15,7 @@ pub fn cron(conf Config) ? { level: log_level } - log_file := os.join_path_single(conf.data_dir, cron.log_file_name) - logger.set_full_logpath(log_file) + logger.set_full_logpath(conf.log_file) logger.log_to_console_too() ce := expression.parse_expression(conf.global_schedule) or { diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index e54a39e..067d191 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -4,11 +4,11 @@ import time import sync.stdatomic import build -const ( - build_empty = 0 - build_running = 1 - build_done = 2 -) +const build_empty = 0 + +const build_running = 1 + +const build_done = 2 // clean_finished_builds removes finished builds from the build slots & returns // them. diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 35cca5f..088a24f 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -8,19 +8,17 @@ import cron.expression { CronExpression, parse_expression } import math import build import docker -import db -const ( - // How many seconds to wait before retrying to update API if failed - api_update_retry_timeout = 5 - // How many seconds to wait before retrying to rebuild image if failed - rebuild_base_image_retry_timout = 30 -) +// How many seconds to wait before retrying to update API if failed +const api_update_retry_timeout = 5 + +// How many seconds to wait before retrying to rebuild image if failed +const rebuild_base_image_retry_timout = 30 struct ScheduledBuild { pub: repo_id string - repo db.GitRepo + repo git.GitRepo timestamp time.Time } @@ -39,7 +37,7 @@ mut: api_update_frequency int image_rebuild_frequency int // Repos currently loaded from API. - repos []db.GitRepo + repos_map map[string]git.GitRepo // At what point to update the list of repositories. api_update_timestamp time.Time image_build_timestamp time.Time @@ -92,7 +90,7 @@ pub fn (mut d Daemon) run() { // haven't been renewed. else { for sb in finished_builds { - d.schedule_build(sb.repo) + d.schedule_build(sb.repo_id, sb.repo) } } @@ -151,11 +149,11 @@ pub fn (mut d Daemon) run() { } // schedule_build adds the next occurence of the given repo build to the queue. -fn (mut d Daemon) schedule_build(repo db.GitRepo) { +fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) { ce := if repo.schedule != '' { parse_expression(repo.schedule) or { // TODO This shouldn't return an error if the expression is empty. - d.lerror("Error while parsing cron expression '$repo.schedule' (id $repo.id): $err.msg()") + d.lerror("Error while parsing cron expression '$repo.schedule' ($repo_id): $err.msg()") d.global_schedule } @@ -170,6 +168,7 @@ fn (mut d Daemon) schedule_build(repo db.GitRepo) { } d.queue.insert(ScheduledBuild{ + repo_id: repo_id repo: repo timestamp: timestamp }) @@ -187,7 +186,7 @@ fn (mut d Daemon) renew_repos() { return } - d.repos = new_repos + d.repos_map = new_repos.move() d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency) } @@ -225,8 +224,8 @@ fn (mut d Daemon) renew_queue() { // For each repository in repos_map, parse their cron expression (or use // the default one if not present) & add them to the queue - for repo in d.repos { - d.schedule_build(repo) + for id, repo in d.repos_map { + d.schedule_build(id, repo) } } diff --git a/src/db/db.v b/src/db/db.v deleted file mode 100644 index a75c34c..0000000 --- a/src/db/db.v +++ /dev/null @@ -1,20 +0,0 @@ -module db - -import sqlite - -struct VieterDb { - conn sqlite.DB -} - -// init initializes a database & adds the correct tables. -pub fn init(db_path string) ?VieterDb { - conn := sqlite.connect(db_path) ? - - sql conn { - create table GitRepo - } - - return VieterDb{ - conn: conn - } -} diff --git a/src/db/git.v b/src/db/git.v deleted file mode 100644 index c40086b..0000000 --- a/src/db/git.v +++ /dev/null @@ -1,154 +0,0 @@ -module db - -pub struct GitRepoArch { -pub: - id int [primary; sql: serial] - repo_id int [nonull] - value string [nonull] -} - -// str returns a string representation. -pub fn (gra &GitRepoArch) str() string { - return gra.value -} - -pub struct GitRepo { -pub mut: - id int [optional; primary; sql: serial] - // URL of the Git repository - url string [nonull] - // Branch of the Git repository to use - branch string [nonull] - // Which repo the builder should publish packages to - repo string [nonull] - // Cron schedule describing how frequently to build the repo. - schedule string [optional] - // On which architectures the package is allowed to be built. In reality, - // this controls which builders will periodically build the image. - arch []GitRepoArch [fkey: 'repo_id'] -} - -// str returns a string representation. -pub fn (gr &GitRepo) str() string { - mut parts := [ - 'id: $gr.id', - 'url: $gr.url', - 'branch: $gr.branch', - 'repo: $gr.repo', - 'schedule: $gr.schedule', - 'arch: ${gr.arch.map(it.value).join(', ')}', - ] - str := parts.join('\n') - - return str -} - -// patch_from_params patches a GitRepo from a map[string]string, usually -// provided from a web.App's params -pub fn (mut r GitRepo) patch_from_params(params map[string]string) { - $for field in GitRepo.fields { - if field.name in params { - $if field.typ is string { - r.$(field.name) = params[field.name] - // This specific type check is needed for the compiler to ensure - // our types are correct - } $else $if field.typ is []GitRepoArch { - r.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) - } - } - } -} - -// git_repo_from_params creates a GitRepo from a map[string]string, usually -// provided from a web.App's params -pub fn git_repo_from_params(params map[string]string) ?GitRepo { - mut repo := GitRepo{} - - // If we're creating a new GitRepo, we want all fields to be present before - // "patching". - $for field in GitRepo.fields { - if field.name !in params && !field.attrs.contains('optional') { - return error('Missing parameter: ${field.name}.') - } - } - repo.patch_from_params(params) - - return repo -} - -// get_git_repos returns all GitRepo's in the database. -pub fn (db &VieterDb) get_git_repos() []GitRepo { - res := sql db.conn { - select from GitRepo order by id - } - - return res -} - -// get_git_repo tries to return a specific GitRepo. -pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo { - res := sql db.conn { - select from GitRepo where id == repo_id - } - - // If a select statement fails, it returns a zeroed object. By - // checking one of the required fields, we can see whether the query - // returned a result or not. - if res.url == '' { - return none - } - - return res -} - -// add_git_repo inserts the given GitRepo into the database. -pub fn (db &VieterDb) add_git_repo(repo GitRepo) { - sql db.conn { - insert repo into GitRepo - } -} - -// delete_git_repo deletes the repo with the given ID from the database. -pub fn (db &VieterDb) delete_git_repo(repo_id int) { - sql db.conn { - delete from GitRepo where id == repo_id - delete from GitRepoArch where repo_id == repo_id - } -} - -// update_git_repo updates any non-array values for a given GitRepo. -pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) { - mut values := []string{} - - // TODO does this allow for SQL injection? - $for field in GitRepo.fields { - if field.name in params { - // Any fields that are array types require their own update method - $if field.typ is string { - values << "$field.name = '${params[field.name]}'" - } - } - } - values_str := values.join(', ') - query := 'update GitRepo set $values_str where id == $repo_id' - - db.conn.exec_none(query) -} - -// update_git_repo_archs updates a given GitRepo's arch value. -pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { - archs_with_id := archs.map(GitRepoArch{ - ...it - repo_id: repo_id - }) - - sql db.conn { - delete from GitRepoArch where repo_id == repo_id - } - - for arch in archs_with_id { - sql db.conn { - insert arch into GitRepoArch - } - } -} diff --git a/src/git/cli.v b/src/git/cli.v index 634b778..0eff55f 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -116,13 +116,34 @@ pub fn cmd() cli.Command { // get_repo_by_prefix tries to find the repo with the given prefix in its // ID. If multiple or none are found, an error is raised. +fn get_repo_by_prefix(conf Config, id_prefix string) ?(string, GitRepo) { + repos := get_repos(conf.address, conf.api_key) ? + + mut res := map[string]GitRepo{} + + for id, repo in repos { + if id.starts_with(id_prefix) { + res[id] = repo + } + } + + if res.len == 0 { + return error('No repo found for given prefix.') + } + + if res.len > 1 { + return error('Multiple repos found for given prefix.') + } + + return res.keys()[0], res[res.keys()[0]] +} // list prints out a list of all repositories. fn list(conf Config) ? { repos := get_repos(conf.address, conf.api_key) ? - for repo in repos { - println('$repo.id\t$repo.url\t$repo.branch\t$repo.repo') + for id, details in repos { + println('${id[..8]}\t$details.url\t$details.branch\t$details.repo') } } @@ -134,18 +155,15 @@ fn add(conf Config, url string, branch string, repo string) ? { } // remove removes a repository from the server's list. -fn remove(conf Config, id string) ? { - // id, _ := get_repo_by_prefix(conf, id_prefix) ? - id_int := id.int() +fn remove(conf Config, id_prefix string) ? { + id, _ := get_repo_by_prefix(conf, id_prefix) ? + res := remove_repo(conf.address, conf.api_key, id) ? - if id_int != 0 { - res := remove_repo(conf.address, conf.api_key, id_int) ? - println(res.message) - } + println(res.message) } // patch patches a given repository with the provided params. -fn patch(conf Config, id string, params map[string]string) ? { +fn patch(conf Config, id_prefix string, params map[string]string) ? { // We check the cron expression first because it's useless to send an // invalid one to the server. if 'schedule' in params && params['schedule'] != '' { @@ -154,22 +172,20 @@ fn patch(conf Config, id string, params map[string]string) ? { } } - id_int := id.int() - if id_int != 0 { - res := patch_repo(conf.address, conf.api_key, id_int, params) ? + id, _ := get_repo_by_prefix(conf, id_prefix) ? + res := patch_repo(conf.address, conf.api_key, id, params) ? - println(res.message) - } + println(res.message) } // info shows detailed information for a given repo. -fn info(conf Config, id string) ? { - id_int := id.int() +fn info(conf Config, id_prefix string) ? { + id, repo := get_repo_by_prefix(conf, id_prefix) ? - if id_int == 0 { - return + println('id: $id') + + $for field in GitRepo.fields { + val := repo.$(field.name) + println('$field.name: $val') } - - repo := get_repo(conf.address, conf.api_key, id_int) ? - println(repo) } diff --git a/src/git/client.v b/src/git/client.v index b5f8e9f..0ed19b5 100644 --- a/src/git/client.v +++ b/src/git/client.v @@ -3,7 +3,6 @@ module git import json import response { Response } import net.http -import db // send_request is a convenience method for sending requests to the repos // API. It mostly does string manipulation to create a query string containing @@ -27,16 +26,8 @@ fn send_request(method http.Method, address string, url string, api_key strin } // get_repos returns the current list of repos. -pub fn get_repos(address string, api_key string) ?[]db.GitRepo { - data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', api_key, - {}) ? - - return data.data -} - -// get_repo returns the repo for a specific ID. -pub fn get_repo(address string, api_key string, id int) ?db.GitRepo { - data := send_request(http.Method.get, address, '/api/repos/$id', api_key, +pub fn get_repos(address string, api_key string) ?map[string]GitRepo { + data := send_request(http.Method.get, address, '/api/repos', api_key, {}) ? return data.data @@ -60,7 +51,7 @@ pub fn add_repo(address string, api_key string, url string, branch string, repo } // remove_repo removes the repo with the given ID from the server. -pub fn remove_repo(address string, api_key string, id int) ?Response { +pub fn remove_repo(address string, api_key string, id string) ?Response { data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, {}) ? @@ -69,7 +60,7 @@ pub fn remove_repo(address string, api_key string, id int) ?Response { // patch_repo sends a PATCH request to the given repo with the params as // payload. -pub fn patch_repo(address string, api_key string, id int, params map[string]string) ?Response { +pub fn patch_repo(address string, api_key string, id string, params map[string]string) ?Response { data := send_request(http.Method.patch, address, '/api/repos/$id', api_key, params) ? diff --git a/src/git/git.v b/src/git/git.v new file mode 100644 index 0000000..2023f34 --- /dev/null +++ b/src/git/git.v @@ -0,0 +1,84 @@ +module git + +import os +import json + +pub struct GitRepo { +pub mut: + // URL of the Git repository + url string + // Branch of the Git repository to use + branch string + // On which architectures the package is allowed to be built. In reality, + // this controls which builders will periodically build the image. + arch []string + // Which repo the builder should publish packages to + repo string + // Cron schedule describing how frequently to build the repo. + schedule string [optional] +} + +// patch_from_params patches a GitRepo from a map[string]string, usually +// provided from a web.App's params +pub fn (mut r GitRepo) patch_from_params(params map[string]string) { + $for field in GitRepo.fields { + if field.name in params { + $if field.typ is string { + r.$(field.name) = params[field.name] + // This specific type check is needed for the compiler to ensure + // our types are correct + } $else $if field.typ is []string { + r.$(field.name) = params[field.name].split(',') + } + } + } +} + +// read_repos reads the provided path & parses it into a map of GitRepo's. +pub fn read_repos(path string) ?map[string]GitRepo { + if !os.exists(path) { + mut f := os.create(path) ? + + defer { + f.close() + } + + f.write_string('{}') ? + + return {} + } + + content := os.read_file(path) ? + res := json.decode(map[string]GitRepo, content) ? + + return res +} + +// write_repos writes a map of GitRepo's back to disk given the provided path. +pub fn write_repos(path string, repos &map[string]GitRepo) ? { + mut f := os.create(path) ? + + defer { + f.close() + } + + value := json.encode(repos) + f.write_string(value) ? +} + +// repo_from_params creates a GitRepo from a map[string]string, usually +// provided from a web.App's params +pub fn repo_from_params(params map[string]string) ?GitRepo { + mut repo := GitRepo{} + + // If we're creating a new GitRepo, we want all fields to be present before + // "patching". + $for field in GitRepo.fields { + if field.name !in params && !field.attrs.contains('optional') { + return error('Missing parameter: ${field.name}.') + } + } + repo.patch_from_params(params) + + return repo +} diff --git a/src/server/cli.v b/src/server/cli.v index 4d39666..bea223d 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -6,9 +6,12 @@ import env struct Config { pub: log_level string = 'WARN' + log_file string = 'vieter.log' pkg_dir string - data_dir string + download_dir string api_key string + repos_dir string + repos_file string default_arch string } diff --git a/src/server/git.v b/src/server/git.v index c5cbc0a..c136d98 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -1,9 +1,12 @@ module server import web +import git import net.http +import rand import response { new_data_response, new_response } -import db + +const repos_file = 'repos.json' // get_repos returns the current list of repos. ['/api/repos'; get] @@ -12,19 +15,37 @@ fn (mut app App) get_repos() web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - repos := app.db.get_git_repos() + repos := rlock app.git_mutex { + git.read_repos(app.conf.repos_file) or { + app.lerror('Failed to read repos file: $err.msg()') + + return app.status(http.Status.internal_server_error) + } + } return app.json(http.Status.ok, new_data_response(repos)) } // get_single_repo returns the information for a single repo. ['/api/repos/:id'; get] -fn (mut app App) get_single_repo(id int) web.Result { +fn (mut app App) get_single_repo(id string) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - repo := app.db.get_git_repo(id) or { return app.not_found() } + repos := rlock app.git_mutex { + git.read_repos(app.conf.repos_file) or { + app.lerror('Failed to read repos file.') + + return app.status(http.Status.internal_server_error) + } + } + + if id !in repos { + return app.not_found() + } + + repo := repos[id] return app.json(http.Status.ok, new_data_response(repo)) } @@ -44,40 +65,89 @@ fn (mut app App) post_repo() web.Result { params['arch'] = app.conf.default_arch } - new_repo := db.git_repo_from_params(params) or { + new_repo := git.repo_from_params(params) or { return app.json(http.Status.bad_request, new_response(err.msg())) } - app.db.add_git_repo(new_repo) + id := rand.uuid_v4() + + mut repos := rlock app.git_mutex { + git.read_repos(app.conf.repos_file) or { + app.lerror('Failed to read repos file.') + + return app.status(http.Status.internal_server_error) + } + } + + // We need to check for duplicates + for _, repo in repos { + if repo == new_repo { + return app.json(http.Status.bad_request, new_response('Duplicate repository.')) + } + } + + repos[id] = new_repo + + lock app.git_mutex { + git.write_repos(app.conf.repos_file, &repos) or { + return app.status(http.Status.internal_server_error) + } + } return app.json(http.Status.ok, new_response('Repo added successfully.')) } // delete_repo removes a given repo from the server's list. ['/api/repos/:id'; delete] -fn (mut app App) delete_repo(id int) web.Result { +fn (mut app App) delete_repo(id string) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - app.db.delete_git_repo(id) + mut repos := rlock app.git_mutex { + git.read_repos(app.conf.repos_file) or { + app.lerror('Failed to read repos file.') + + return app.status(http.Status.internal_server_error) + } + } + + if id !in repos { + return app.not_found() + } + + repos.delete(id) + + lock app.git_mutex { + git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } + } return app.json(http.Status.ok, new_response('Repo removed successfully.')) } // patch_repo updates a repo's data with the given query params. ['/api/repos/:id'; patch] -fn (mut app App) patch_repo(id int) web.Result { +fn (mut app App) patch_repo(id string) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - app.db.update_git_repo(id, app.query) + mut repos := rlock app.git_mutex { + git.read_repos(app.conf.repos_file) or { + app.lerror('Failed to read repos file.') - if 'arch' in app.query { - arch_objs := app.query['arch'].split(',').map(db.GitRepoArch{ value: it }) + return app.status(http.Status.internal_server_error) + } + } - app.db.update_git_repo_archs(id, arch_objs) + if id !in repos { + return app.not_found() + } + + repos[id].patch_from_params(app.query) + + lock app.git_mutex { + git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } } return app.json(http.Status.ok, new_response('Repo updated successfully.')) diff --git a/src/server/routes.v b/src/server/routes.v index fbf37df..f27afb4 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -68,7 +68,7 @@ fn (mut app App) put_package(repo string) web.Result { if length := app.req.header.get(.content_length) { // Generate a random filename for the temp file - pkg_path = os.join_path_single(app.repo.pkg_dir, rand.uuid_v4()) + pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4()) app.ldebug("Uploading $length bytes (${util.pretty_bytes(length.int())}) to '$pkg_path'.") diff --git a/src/server/server.v b/src/server/server.v index b2a2ad2..c4317c5 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -5,14 +5,8 @@ import os import log import repo import util -import db -const ( - port = 8000 - log_file_name = 'vieter.log' - repo_dir_name = 'repos' - db_file_name = 'vieter.sqlite' -) +const port = 8000 struct App { web.Context @@ -20,7 +14,8 @@ pub: conf Config [required; web_global] pub mut: repo repo.RepoGroupManager [required; web_global] - db db.VieterDb + // This is used to claim the file lock on the repos file + git_mutex shared util.Dummy } // server starts the web server & starts listening for requests @@ -35,14 +30,11 @@ pub fn server(conf Config) ? { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') } - os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') } - mut logger := log.Log{ level: log_level } - log_file := os.join_path_single(conf.data_dir, server.log_file_name) - logger.set_full_logpath(log_file) + logger.set_full_logpath(conf.log_file) logger.log_to_console_too() defer { @@ -51,20 +43,19 @@ pub fn server(conf Config) ? { logger.close() } - repo_dir := os.join_path_single(conf.data_dir, server.repo_dir_name) // This also creates the directories if needed - repo := repo.new(repo_dir, conf.pkg_dir, conf.default_arch) or { + repo := repo.new(conf.repos_dir, conf.pkg_dir, conf.default_arch) or { logger.error(err.msg()) exit(1) } - db_file := os.join_path_single(conf.data_dir, server.db_file_name) - db := db.init(db_file) or { util.exit_with_message(1, 'Failed to initialize database.') } + os.mkdir_all(conf.download_dir) or { + util.exit_with_message(1, 'Failed to create download directory.') + } web.run(&App{ logger: logger conf: conf repo: repo - db: db }, server.port) } diff --git a/vieter.toml b/vieter.toml index d3922a4..fc86d77 100644 --- a/vieter.toml +++ b/vieter.toml @@ -1,8 +1,10 @@ # This file contains settings used during development api_key = "test" -data_dir = "data" +download_dir = "data/downloads" +repos_dir = "data/repos" pkg_dir = "data/pkgs" log_level = "DEBUG" +repos_file = "data/repos.json" default_arch = "x86_64" address = "http://localhost:8000"