From 891a206116dd33b1e4640d6a9867a1a897ce7b21 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 1 May 2022 22:47:00 +0200 Subject: [PATCH 1/8] feat(server): partially migrated repos API to sqlite --- .woodpecker/.build.yml | 2 +- src/db/db.v | 19 +++++++ src/db/git.v | 98 ++++++++++++++++++++++++++++++++++++ src/server/git.v | 109 ++++++++++++++++++++++------------------- src/server/server.v | 5 ++ 5 files changed, 181 insertions(+), 52 deletions(-) create mode 100644 src/db/db.v create mode 100644 src/db/git.v diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index b41a39d..f9cab00 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -23,7 +23,7 @@ pipeline: image: 'chewingbever/vlang:latest' pull: true environment: - - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static + - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static commands: # Apparently this -D is *very* important - CFLAGS='-DGC_THREADS=1' make prod diff --git a/src/db/db.v b/src/db/db.v new file mode 100644 index 0000000..b62fa3b --- /dev/null +++ b/src/db/db.v @@ -0,0 +1,19 @@ +module db + +import sqlite + +struct VieterDb { + conn sqlite.DB +} + +pub fn init(db_path string) ?VieterDb { + conn := sqlite.connect(db_path) ? + + sql conn { + create table GitRepo + } + + return VieterDb{ + conn: conn + } +} diff --git a/src/db/git.v b/src/db/git.v new file mode 100644 index 0000000..fca46c6 --- /dev/null +++ b/src/db/git.v @@ -0,0 +1,98 @@ +module db + +struct GitRepoArch { +pub: + id int [primary; sql: serial] + repo_id int + value string +} + +pub struct GitRepo { +pub mut: + id int [optional; primary; sql: serial] + // URL of the Git repository + url string [nonull] + // Branch of the Git repository to use + branch string [nonull] + // Which repo the builder should publish packages to + repo string [nonull] + // Cron schedule describing how frequently to build the repo. + schedule string [optional] + // On which architectures the package is allowed to be built. In reality, + // this controls which builders will periodically build the image. + arch []GitRepoArch [fkey: 'repo_id'] +} + +// patch_from_params patches a GitRepo from a map[string]string, usually +// provided from a web.App's params +pub fn (mut r GitRepo) patch_from_params(params map[string]string) { + $for field in GitRepo.fields { + if field.name in params { + $if field.typ is string { + r.$(field.name) = params[field.name] + // This specific type check is needed for the compiler to ensure + // our types are correct + } $else $if field.typ is []GitRepoArch { + r.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) + } + } + } +} + +// repo_from_params creates a GitRepo from a map[string]string, usually +// provided from a web.App's params +pub fn git_repo_from_params(params map[string]string) ?GitRepo { + mut repo := GitRepo{} + + // If we're creating a new GitRepo, we want all fields to be present before + // "patching". + $for field in GitRepo.fields { + if field.name !in params && !field.attrs.contains('optional') { + return error('Missing parameter: ${field.name}.') + } + } + repo.patch_from_params(params) + + return repo +} + +pub fn (db &VieterDb) get_git_repos() []GitRepo { + res := sql db.conn { + select from GitRepo + } + + return res +} + +pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo { + res := sql db.conn { + select from GitRepo where id == repo_id + } + + // If a select statement fails, it returns a zeroed object. By + // checking one of the required fields, we can see whether the query + // returned a result or not. + if res.url == '' { + return none + } + + return res +} + +pub fn (db &VieterDb) add_git_repo(repo GitRepo) { + sql db.conn { + insert repo into GitRepo + } +} + +pub fn (db &VieterDb) delete_git_repo(repo_id int) { + sql db.conn { + delete from GitRepo where id == repo_id + } +} + +pub fn (db &VieterDb) update_git_repo(repo GitRepo) { + /* sql db.conn { */ + /* update GitRepo set repo */ + /* } */ +} diff --git a/src/server/git.v b/src/server/git.v index c136d98..6485eca 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -5,6 +5,7 @@ import git import net.http import rand import response { new_data_response, new_response } +import db const repos_file = 'repos.json' @@ -15,37 +16,39 @@ fn (mut app App) get_repos() web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - repos := rlock app.git_mutex { - git.read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file: $err.msg()') + repos := app.db.get_git_repos() + // repos := rlock app.git_mutex { + // git.read_repos(app.conf.repos_file) or { + // app.lerror('Failed to read repos file: $err.msg()') - return app.status(http.Status.internal_server_error) - } - } + // return app.status(http.Status.internal_server_error) + // } + //} return app.json(http.Status.ok, new_data_response(repos)) } // get_single_repo returns the information for a single repo. ['/api/repos/:id'; get] -fn (mut app App) get_single_repo(id string) web.Result { +fn (mut app App) get_single_repo(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - repos := rlock app.git_mutex { - git.read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file.') + // repos := rlock app.git_mutex { + // git.read_repos(app.conf.repos_file) or { + // app.lerror('Failed to read repos file.') - return app.status(http.Status.internal_server_error) - } - } + // return app.status(http.Status.internal_server_error) + // } + //} - if id !in repos { - return app.not_found() - } + // if id !in repos { + // return app.not_found() + //} - repo := repos[id] + // repo := repos[id] + repo := app.db.get_git_repo(id) or { return app.not_found() } return app.json(http.Status.ok, new_data_response(repo)) } @@ -65,62 +68,66 @@ fn (mut app App) post_repo() web.Result { params['arch'] = app.conf.default_arch } - new_repo := git.repo_from_params(params) or { + new_repo := db.git_repo_from_params(params) or { return app.json(http.Status.bad_request, new_response(err.msg())) } - id := rand.uuid_v4() + app.db.add_git_repo(new_repo) - mut repos := rlock app.git_mutex { - git.read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file.') + // id := rand.uuid_v4() - return app.status(http.Status.internal_server_error) - } - } + // mut repos := rlock app.git_mutex { + // git.read_repos(app.conf.repos_file) or { + // app.lerror('Failed to read repos file.') - // We need to check for duplicates - for _, repo in repos { - if repo == new_repo { - return app.json(http.Status.bad_request, new_response('Duplicate repository.')) - } - } + // return app.status(http.Status.internal_server_error) + // } + //} + // repos := app.db.get_git_repos() - repos[id] = new_repo + //// We need to check for duplicates + // for _, repo in repos { + // if repo == new_repo { + // return app.json(http.Status.bad_request, new_response('Duplicate repository.')) + // } + //} - lock app.git_mutex { - git.write_repos(app.conf.repos_file, &repos) or { - return app.status(http.Status.internal_server_error) - } - } + // repos[id] = new_repo + + // lock app.git_mutex { + // git.write_repos(app.conf.repos_file, &repos) or { + // return app.status(http.Status.internal_server_error) + // } + //} return app.json(http.Status.ok, new_response('Repo added successfully.')) } // delete_repo removes a given repo from the server's list. ['/api/repos/:id'; delete] -fn (mut app App) delete_repo(id string) web.Result { +fn (mut app App) delete_repo(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - mut repos := rlock app.git_mutex { - git.read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file.') + /* mut repos := rlock app.git_mutex { */ + /* git.read_repos(app.conf.repos_file) or { */ + /* app.lerror('Failed to read repos file.') */ - return app.status(http.Status.internal_server_error) - } - } + /* return app.status(http.Status.internal_server_error) */ + /* } */ + /* } */ - if id !in repos { - return app.not_found() - } + /* if id !in repos { */ + /* return app.not_found() */ + /* } */ - repos.delete(id) + /* repos.delete(id) */ + app.db.delete_git_repo(id) - lock app.git_mutex { - git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } - } +/* lock app.git_mutex { */ +/* git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } */ +/* } */ return app.json(http.Status.ok, new_response('Repo removed successfully.')) } diff --git a/src/server/server.v b/src/server/server.v index c4317c5..751ea9c 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -5,6 +5,7 @@ import os import log import repo import util +import db const port = 8000 @@ -16,6 +17,7 @@ pub mut: repo repo.RepoGroupManager [required; web_global] // This is used to claim the file lock on the repos file git_mutex shared util.Dummy + db db.VieterDb } // server starts the web server & starts listening for requests @@ -53,9 +55,12 @@ pub fn server(conf Config) ? { util.exit_with_message(1, 'Failed to create download directory.') } + db := db.init('test.db') or { util.exit_with_message(1, 'Failed to initialize database.') } + web.run(&App{ logger: logger conf: conf repo: repo + db: db }, server.port) } From 0a2488a4dfc54a32f26e8f4341090daa6c466a73 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 2 May 2022 20:59:53 +0200 Subject: [PATCH 2/8] feat(server): migrated repo patch to sqlite --- src/db/git.v | 49 +++++++++++++++++++++++++++++++++++++++++++++--- src/server/git.v | 34 ++++++++++++++++++++------------- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/src/db/git.v b/src/db/git.v index fca46c6..29606e4 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -1,6 +1,6 @@ module db -struct GitRepoArch { +pub struct GitRepoArch { pub: id int [primary; sql: serial] repo_id int @@ -58,7 +58,7 @@ pub fn git_repo_from_params(params map[string]string) ?GitRepo { pub fn (db &VieterDb) get_git_repos() []GitRepo { res := sql db.conn { - select from GitRepo + select from GitRepo order by id } return res @@ -88,11 +88,54 @@ pub fn (db &VieterDb) add_git_repo(repo GitRepo) { pub fn (db &VieterDb) delete_git_repo(repo_id int) { sql db.conn { delete from GitRepo where id == repo_id + delete from GitRepoArch where repo_id == repo_id } } -pub fn (db &VieterDb) update_git_repo(repo GitRepo) { +pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) { /* sql db.conn { */ /* update GitRepo set repo */ /* } */ + mut values := []string{} + + $for field in GitRepo.fields { + if field.name in params { + // Any fields that are array types require their own update method + $if field.typ is string { + values << "${field.name} = '${params[field.name]}'" + /* r.$(field.name) = params[field.name] */ + // This specific type check is needed for the compiler to ensure + // our types are correct + } + /* $else $if field.typ is []GitRepoArch { */ + /* r.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) */ + /* } */ + } + } + + values_str := values.join(', ') + query := "update GitRepo set $values_str where id == $repo_id" + println(query) + db.conn.exec_none(query) + +} + +pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { +archs_with_id := archs.map(GitRepoArch{ + ...it + repo_id: repo_id + }) + + sql db.conn { + // Remove all old values + delete from GitRepoArch where repo_id == repo_id + // Insert all the new ones + /* insert archs_with_id into GitRepoArch */ + } + + for arch in archs_with_id { + sql db.conn { + insert arch into GitRepoArch + } + } } diff --git a/src/server/git.v b/src/server/git.v index 6485eca..69cce81 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -134,28 +134,36 @@ fn (mut app App) delete_repo(id int) web.Result { // patch_repo updates a repo's data with the given query params. ['/api/repos/:id'; patch] -fn (mut app App) patch_repo(id string) web.Result { +fn (mut app App) patch_repo(id int) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - mut repos := rlock app.git_mutex { - git.read_repos(app.conf.repos_file) or { - app.lerror('Failed to read repos file.') + app.db.update_git_repo(id, app.query) - return app.status(http.Status.internal_server_error) + if 'arch' in app.query { + arch_objs := app.query['arch'].split(',').map(db.GitRepoArch{value: it}) + + app.db.update_git_repo_archs(id, arch_objs) } - } - if id !in repos { - return app.not_found() - } +/* mut repos := rlock app.git_mutex { */ +/* git.read_repos(app.conf.repos_file) or { */ +/* app.lerror('Failed to read repos file.') */ - repos[id].patch_from_params(app.query) +/* return app.status(http.Status.internal_server_error) */ +/* } */ +/* } */ - lock app.git_mutex { - git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } - } +/* if id !in repos { */ +/* return app.not_found() */ +/* } */ + +/* repos[id].patch_from_params(app.query) */ + +/* lock app.git_mutex { */ +/* git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } */ +/* } */ return app.json(http.Status.ok, new_response('Repo updated successfully.')) } From 7419144f97a955c9d5d67816d483e8da55690d47 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 3 May 2022 16:16:56 +0200 Subject: [PATCH 3/8] feat: removed git.GitRepo type feat(cli): updated to new GitRepo format --- src/build/build.v | 7 +- src/cron/daemon/daemon.v | 18 +++--- src/db/git.v | 53 +++++++++------ src/git/cli.v | 61 +++++++----------- src/git/client.v | 16 +++-- src/git/git.v | 136 +++++++++++++++++++-------------------- src/server/git.v | 60 ++++++++--------- 7 files changed, 179 insertions(+), 172 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index bc604fa..15a5eb8 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -5,6 +5,7 @@ import encoding.base64 import time import git import os +import db const container_build_dir = '/build' @@ -75,7 +76,7 @@ pub fn create_build_image(base_image string) ?string { // build_repo builds, packages & publishes a given Arch package based on the // provided GitRepo. The base image ID should be of an image previously created // by create_build_image. -pub fn build_repo(address string, api_key string, base_image_id string, repo &git.GitRepo) ? { +pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ? { build_arch := os.uname().machine // TODO what to do with PKGBUILDs that build multiple packages? @@ -125,11 +126,11 @@ fn build(conf Config) ? { build_arch := os.uname().machine // We get the repos map from the Vieter instance - repos_map := git.get_repos(conf.address, conf.api_key) ? + repos := git.get_repos(conf.address, conf.api_key) ? // We filter out any repos that aren't allowed to be built on this // architecture - filtered_repos := repos_map.keys().map(repos_map[it]).filter(it.arch.contains(build_arch)) + filtered_repos := repos.filter(it.arch.map(it.value).contains(build_arch)) // No point in doing work if there's no repos present if filtered_repos.len == 0 { diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 088a24f..1747494 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -8,6 +8,7 @@ import cron.expression { CronExpression, parse_expression } import math import build import docker +import db // How many seconds to wait before retrying to update API if failed const api_update_retry_timeout = 5 @@ -18,7 +19,7 @@ const rebuild_base_image_retry_timout = 30 struct ScheduledBuild { pub: repo_id string - repo git.GitRepo + repo db.GitRepo timestamp time.Time } @@ -37,7 +38,7 @@ mut: api_update_frequency int image_rebuild_frequency int // Repos currently loaded from API. - repos_map map[string]git.GitRepo + repos []db.GitRepo // At what point to update the list of repositories. api_update_timestamp time.Time image_build_timestamp time.Time @@ -90,7 +91,7 @@ pub fn (mut d Daemon) run() { // haven't been renewed. else { for sb in finished_builds { - d.schedule_build(sb.repo_id, sb.repo) + d.schedule_build(sb.repo) } } @@ -149,11 +150,11 @@ pub fn (mut d Daemon) run() { } // schedule_build adds the next occurence of the given repo build to the queue. -fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) { +fn (mut d Daemon) schedule_build(repo db.GitRepo) { ce := if repo.schedule != '' { parse_expression(repo.schedule) or { // TODO This shouldn't return an error if the expression is empty. - d.lerror("Error while parsing cron expression '$repo.schedule' ($repo_id): $err.msg()") + d.lerror("Error while parsing cron expression '$repo.schedule' (id $repo.id): $err.msg()") d.global_schedule } @@ -168,7 +169,6 @@ fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) { } d.queue.insert(ScheduledBuild{ - repo_id: repo_id repo: repo timestamp: timestamp }) @@ -186,7 +186,7 @@ fn (mut d Daemon) renew_repos() { return } - d.repos_map = new_repos.move() + d.repos = new_repos d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency) } @@ -224,8 +224,8 @@ fn (mut d Daemon) renew_queue() { // For each repository in repos_map, parse their cron expression (or use // the default one if not present) & add them to the queue - for id, repo in d.repos_map { - d.schedule_build(id, repo) + for repo in d.repos { + d.schedule_build(repo) } } diff --git a/src/db/git.v b/src/db/git.v index 29606e4..ac35ff4 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -3,8 +3,12 @@ module db pub struct GitRepoArch { pub: id int [primary; sql: serial] - repo_id int - value string + repo_id int [nonull] + value string [nonull] +} + +pub fn (gra &GitRepoArch) str() string { + return gra.value } pub struct GitRepo { @@ -23,6 +27,20 @@ pub mut: arch []GitRepoArch [fkey: 'repo_id'] } +pub fn (gr &GitRepo) str() string { + mut parts := [ + "id: $gr.id", + "url: $gr.url", + "branch: $gr.branch", + "repo: $gr.repo", + "schedule: $gr.schedule", + "arch: ${gr.arch.map(it.value).join(', ')}" + ] + str := parts.join('\n') + + return str +} + // patch_from_params patches a GitRepo from a map[string]string, usually // provided from a web.App's params pub fn (mut r GitRepo) patch_from_params(params map[string]string) { @@ -93,44 +111,39 @@ pub fn (db &VieterDb) delete_git_repo(repo_id int) { } pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) { - /* sql db.conn { */ - /* update GitRepo set repo */ - /* } */ + // sql db.conn { + // update GitRepo set repo + //} mut values := []string{} $for field in GitRepo.fields { if field.name in params { // Any fields that are array types require their own update method $if field.typ is string { - values << "${field.name} = '${params[field.name]}'" - /* r.$(field.name) = params[field.name] */ + values << "$field.name = '${params[field.name]}'" + // r.$(field.name) = params[field.name] // This specific type check is needed for the compiler to ensure // our types are correct } - /* $else $if field.typ is []GitRepoArch { */ - /* r.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) */ - /* } */ + //$else $if field.typ is []GitRepoArch { + // r.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) + //} } } - values_str := values.join(', ') - query := "update GitRepo set $values_str where id == $repo_id" + query := 'update GitRepo set $values_str where id == $repo_id' println(query) db.conn.exec_none(query) - } pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { -archs_with_id := archs.map(GitRepoArch{ - ...it - repo_id: repo_id - }) + archs_with_id := archs.map(GitRepoArch{ + ...it + repo_id: repo_id + }) sql db.conn { - // Remove all old values delete from GitRepoArch where repo_id == repo_id - // Insert all the new ones - /* insert archs_with_id into GitRepoArch */ } for arch in archs_with_id { diff --git a/src/git/cli.v b/src/git/cli.v index 0eff55f..1839492 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -3,6 +3,7 @@ module git import cli import env import cron.expression { parse_expression } +import db { GitRepo, GitRepoArch } struct Config { address string [required] @@ -116,34 +117,13 @@ pub fn cmd() cli.Command { // get_repo_by_prefix tries to find the repo with the given prefix in its // ID. If multiple or none are found, an error is raised. -fn get_repo_by_prefix(conf Config, id_prefix string) ?(string, GitRepo) { - repos := get_repos(conf.address, conf.api_key) ? - - mut res := map[string]GitRepo{} - - for id, repo in repos { - if id.starts_with(id_prefix) { - res[id] = repo - } - } - - if res.len == 0 { - return error('No repo found for given prefix.') - } - - if res.len > 1 { - return error('Multiple repos found for given prefix.') - } - - return res.keys()[0], res[res.keys()[0]] -} // list prints out a list of all repositories. fn list(conf Config) ? { repos := get_repos(conf.address, conf.api_key) ? - for id, details in repos { - println('${id[..8]}\t$details.url\t$details.branch\t$details.repo') + for repo in repos { + println('${repo.id}\t$repo.url\t$repo.branch\t$repo.repo') } } @@ -155,15 +135,18 @@ fn add(conf Config, url string, branch string, repo string) ? { } // remove removes a repository from the server's list. -fn remove(conf Config, id_prefix string) ? { - id, _ := get_repo_by_prefix(conf, id_prefix) ? - res := remove_repo(conf.address, conf.api_key, id) ? +fn remove(conf Config, id string) ? { + // id, _ := get_repo_by_prefix(conf, id_prefix) ? + id_int := id.int() - println(res.message) + if id_int != 0 { + res := remove_repo(conf.address, conf.api_key, id_int) ? + println(res.message) + } } // patch patches a given repository with the provided params. -fn patch(conf Config, id_prefix string, params map[string]string) ? { +fn patch(conf Config, id string, params map[string]string) ? { // We check the cron expression first because it's useless to send an // invalid one to the server. if 'schedule' in params && params['schedule'] != '' { @@ -172,20 +155,22 @@ fn patch(conf Config, id_prefix string, params map[string]string) ? { } } - id, _ := get_repo_by_prefix(conf, id_prefix) ? - res := patch_repo(conf.address, conf.api_key, id, params) ? + id_int := id.int() + if id_int != 0 { + res := patch_repo(conf.address, conf.api_key, id_int, params) ? - println(res.message) + println(res.message) + } } // info shows detailed information for a given repo. -fn info(conf Config, id_prefix string) ? { - id, repo := get_repo_by_prefix(conf, id_prefix) ? +fn info(conf Config, id string) ? { + id_int := id.int() - println('id: $id') - - $for field in GitRepo.fields { - val := repo.$(field.name) - println('$field.name: $val') + if id_int == 0 { + return } + + repo := get_repo(conf.address, conf.api_key, id_int) ? + println(repo) } diff --git a/src/git/client.v b/src/git/client.v index 0ed19b5..d4c5282 100644 --- a/src/git/client.v +++ b/src/git/client.v @@ -3,6 +3,7 @@ module git import json import response { Response } import net.http +import db // send_request is a convenience method for sending requests to the repos // API. It mostly does string manipulation to create a query string containing @@ -26,8 +27,15 @@ fn send_request(method http.Method, address string, url string, api_key strin } // get_repos returns the current list of repos. -pub fn get_repos(address string, api_key string) ?map[string]GitRepo { - data := send_request(http.Method.get, address, '/api/repos', api_key, +pub fn get_repos(address string, api_key string) ?[]db.GitRepo { + data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', + api_key, {}) ? + + return data.data +} + +pub fn get_repo(address string, api_key string, id int) ?db.GitRepo { + data := send_request(http.Method.get, address, '/api/repos/$id', api_key, {}) ? return data.data @@ -51,7 +59,7 @@ pub fn add_repo(address string, api_key string, url string, branch string, repo } // remove_repo removes the repo with the given ID from the server. -pub fn remove_repo(address string, api_key string, id string) ?Response { +pub fn remove_repo(address string, api_key string, id int) ?Response { data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, {}) ? @@ -60,7 +68,7 @@ pub fn remove_repo(address string, api_key string, id string) ?Response // patch_repo sends a PATCH request to the given repo with the params as // payload. -pub fn patch_repo(address string, api_key string, id string, params map[string]string) ?Response { +pub fn patch_repo(address string, api_key string, id int, params map[string]string) ?Response { data := send_request(http.Method.patch, address, '/api/repos/$id', api_key, params) ? diff --git a/src/git/git.v b/src/git/git.v index 2023f34..7c1c83c 100644 --- a/src/git/git.v +++ b/src/git/git.v @@ -1,84 +1,84 @@ module git -import os -import json +/* import os */ +/* import json */ -pub struct GitRepo { -pub mut: - // URL of the Git repository - url string - // Branch of the Git repository to use - branch string - // On which architectures the package is allowed to be built. In reality, - // this controls which builders will periodically build the image. - arch []string - // Which repo the builder should publish packages to - repo string - // Cron schedule describing how frequently to build the repo. - schedule string [optional] -} +/* pub struct GitRepo { */ +/* pub mut: */ +/* // URL of the Git repository */ +/* url string */ +/* // Branch of the Git repository to use */ +/* branch string */ +/* // On which architectures the package is allowed to be built. In reality, */ +/* // this controls which builders will periodically build the image. */ +/* arch []string */ +/* // Which repo the builder should publish packages to */ +/* repo string */ +/* // Cron schedule describing how frequently to build the repo. */ +/* schedule string [optional] */ +/* } */ -// patch_from_params patches a GitRepo from a map[string]string, usually -// provided from a web.App's params -pub fn (mut r GitRepo) patch_from_params(params map[string]string) { - $for field in GitRepo.fields { - if field.name in params { - $if field.typ is string { - r.$(field.name) = params[field.name] - // This specific type check is needed for the compiler to ensure - // our types are correct - } $else $if field.typ is []string { - r.$(field.name) = params[field.name].split(',') - } - } - } -} +/* // patch_from_params patches a GitRepo from a map[string]string, usually */ +/* // provided from a web.App's params */ +/* pub fn (mut r GitRepo) patch_from_params(params map[string]string) { */ +/* $for field in GitRepo.fields { */ +/* if field.name in params { */ +/* $if field.typ is string { */ +/* r.$(field.name) = params[field.name] */ +/* // This specific type check is needed for the compiler to ensure */ +/* // our types are correct */ +/* } $else $if field.typ is []string { */ +/* r.$(field.name) = params[field.name].split(',') */ +/* } */ +/* } */ +/* } */ +/* } */ -// read_repos reads the provided path & parses it into a map of GitRepo's. -pub fn read_repos(path string) ?map[string]GitRepo { - if !os.exists(path) { - mut f := os.create(path) ? +/* // read_repos reads the provided path & parses it into a map of GitRepo's. */ +/* pub fn read_repos(path string) ?map[string]GitRepo { */ +/* if !os.exists(path) { */ +/* mut f := os.create(path) ? */ - defer { - f.close() - } +/* defer { */ +/* f.close() */ +/* } */ - f.write_string('{}') ? +/* f.write_string('{}') ? */ - return {} - } +/* return {} */ +/* } */ - content := os.read_file(path) ? - res := json.decode(map[string]GitRepo, content) ? +/* content := os.read_file(path) ? */ +/* res := json.decode(map[string]GitRepo, content) ? */ - return res -} +/* return res */ +/* } */ -// write_repos writes a map of GitRepo's back to disk given the provided path. -pub fn write_repos(path string, repos &map[string]GitRepo) ? { - mut f := os.create(path) ? +/* // write_repos writes a map of GitRepo's back to disk given the provided path. */ +/* pub fn write_repos(path string, repos &map[string]GitRepo) ? { */ +/* mut f := os.create(path) ? */ - defer { - f.close() - } +/* defer { */ +/* f.close() */ +/* } */ - value := json.encode(repos) - f.write_string(value) ? -} +/* value := json.encode(repos) */ +/* f.write_string(value) ? */ +/* } */ -// repo_from_params creates a GitRepo from a map[string]string, usually -// provided from a web.App's params -pub fn repo_from_params(params map[string]string) ?GitRepo { - mut repo := GitRepo{} +/* // repo_from_params creates a GitRepo from a map[string]string, usually */ +/* // provided from a web.App's params */ +/* pub fn repo_from_params(params map[string]string) ?GitRepo { */ +/* mut repo := GitRepo{} */ - // If we're creating a new GitRepo, we want all fields to be present before - // "patching". - $for field in GitRepo.fields { - if field.name !in params && !field.attrs.contains('optional') { - return error('Missing parameter: ${field.name}.') - } - } - repo.patch_from_params(params) +/* // If we're creating a new GitRepo, we want all fields to be present before */ +/* // "patching". */ +/* $for field in GitRepo.fields { */ +/* if field.name !in params && !field.attrs.contains('optional') { */ +/* return error('Missing parameter: ${field.name}.') */ +/* } */ +/* } */ +/* repo.patch_from_params(params) */ - return repo -} +/* return repo */ +/* } */ diff --git a/src/server/git.v b/src/server/git.v index 69cce81..0389d5f 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -110,24 +110,24 @@ fn (mut app App) delete_repo(id int) web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - /* mut repos := rlock app.git_mutex { */ - /* git.read_repos(app.conf.repos_file) or { */ - /* app.lerror('Failed to read repos file.') */ + // mut repos := rlock app.git_mutex { + // git.read_repos(app.conf.repos_file) or { + // app.lerror('Failed to read repos file.') - /* return app.status(http.Status.internal_server_error) */ - /* } */ - /* } */ + // return app.status(http.Status.internal_server_error) + // } + //} - /* if id !in repos { */ - /* return app.not_found() */ - /* } */ + // if id !in repos { + // return app.not_found() + //} - /* repos.delete(id) */ + // repos.delete(id) app.db.delete_git_repo(id) -/* lock app.git_mutex { */ -/* git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } */ -/* } */ + // lock app.git_mutex { + // git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } + // } return app.json(http.Status.ok, new_response('Repo removed successfully.')) } @@ -141,29 +141,29 @@ fn (mut app App) patch_repo(id int) web.Result { app.db.update_git_repo(id, app.query) - if 'arch' in app.query { - arch_objs := app.query['arch'].split(',').map(db.GitRepoArch{value: it}) + if 'arch' in app.query { + arch_objs := app.query['arch'].split(',').map(db.GitRepoArch{ value: it }) - app.db.update_git_repo_archs(id, arch_objs) - } + app.db.update_git_repo_archs(id, arch_objs) + } -/* mut repos := rlock app.git_mutex { */ -/* git.read_repos(app.conf.repos_file) or { */ -/* app.lerror('Failed to read repos file.') */ + // mut repos := rlock app.git_mutex { + // git.read_repos(app.conf.repos_file) or { + // app.lerror('Failed to read repos file.') -/* return app.status(http.Status.internal_server_error) */ -/* } */ -/* } */ + // return app.status(http.Status.internal_server_error) + // } + // } -/* if id !in repos { */ -/* return app.not_found() */ -/* } */ + // if id !in repos { + // return app.not_found() + // } -/* repos[id].patch_from_params(app.query) */ + // repos[id].patch_from_params(app.query) -/* lock app.git_mutex { */ -/* git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } */ -/* } */ + // lock app.git_mutex { + // git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } + // } return app.json(http.Status.ok, new_response('Repo updated successfully.')) } From c8182737903bf69e859e33bd276d76066dc0bcce Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 3 May 2022 16:54:12 +0200 Subject: [PATCH 4/8] feat: simplified config down to pkg_dir & data_dir BREAKING: downloads are now stored inside the root of pkg_dir, the log file is always stored in the root of data_dir --- Dockerfile | 8 +++----- src/cron/cli.v | 2 +- src/cron/cron.v | 6 +++++- src/cron/daemon/build.v | 10 +++++----- src/server/cli.v | 5 +---- src/server/routes.v | 2 +- src/server/server.v | 22 ++++++++++++++-------- vieter.toml | 4 +--- 8 files changed, 31 insertions(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index 58087ad..2ba6181 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN if [ -n "${CI_COMMIT_SHA}" ]; then \ "https://s3.rustybever.be/vieter/commits/${CI_COMMIT_SHA}/vieter-$(echo "${TARGETPLATFORM}" | sed 's:/:-:g')" && \ chmod +x vieter ; \ else \ - LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static' make prod && \ + LDFLAGS='-lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3 -static' make prod && \ mv pvieter vieter ; \ fi @@ -31,10 +31,8 @@ RUN if [ -n "${CI_COMMIT_SHA}" ]; then \ FROM busybox:1.35.0 ENV PATH=/bin \ - VIETER_REPOS_DIR=/data/repos \ - VIETER_PKG_DIR=/data/pkgs \ - VIETER_DOWNLOAD_DIR=/data/downloads \ - VIETER_REPOS_FILE=/data/repos.json + VIETER_DATA_DIR=/data \ + VIETER_PKG_DIR=/data/pkgs COPY --from=builder /app/dumb-init /app/vieter /bin/ diff --git a/src/cron/cli.v b/src/cron/cli.v index 24cbe2c..9536c37c 100644 --- a/src/cron/cli.v +++ b/src/cron/cli.v @@ -6,9 +6,9 @@ import env struct Config { pub: log_level string = 'WARN' - log_file string = 'vieter.log' api_key string address string + data_dir string base_image string = 'archlinux:base-devel' max_concurrent_builds int = 1 api_update_frequency int = 15 diff --git a/src/cron/cron.v b/src/cron/cron.v index e10e4dd..e356faa 100644 --- a/src/cron/cron.v +++ b/src/cron/cron.v @@ -3,6 +3,9 @@ module cron import log import cron.daemon import cron.expression +import os + +const log_file_name = 'vieter.cron.log' // cron starts a cron daemon & starts periodically scheduling builds. pub fn cron(conf Config) ? { @@ -15,7 +18,8 @@ pub fn cron(conf Config) ? { level: log_level } - logger.set_full_logpath(conf.log_file) + log_file := os.join_path_single(conf.data_dir, cron.log_file_name) + logger.set_full_logpath(log_file) logger.log_to_console_too() ce := expression.parse_expression(conf.global_schedule) or { diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index 067d191..e54a39e 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -4,11 +4,11 @@ import time import sync.stdatomic import build -const build_empty = 0 - -const build_running = 1 - -const build_done = 2 +const ( + build_empty = 0 + build_running = 1 + build_done = 2 +) // clean_finished_builds removes finished builds from the build slots & returns // them. diff --git a/src/server/cli.v b/src/server/cli.v index bea223d..4d39666 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -6,12 +6,9 @@ import env struct Config { pub: log_level string = 'WARN' - log_file string = 'vieter.log' pkg_dir string - download_dir string + data_dir string api_key string - repos_dir string - repos_file string default_arch string } diff --git a/src/server/routes.v b/src/server/routes.v index f27afb4..fbf37df 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -68,7 +68,7 @@ fn (mut app App) put_package(repo string) web.Result { if length := app.req.header.get(.content_length) { // Generate a random filename for the temp file - pkg_path = os.join_path_single(app.conf.download_dir, rand.uuid_v4()) + pkg_path = os.join_path_single(app.repo.pkg_dir, rand.uuid_v4()) app.ldebug("Uploading $length bytes (${util.pretty_bytes(length.int())}) to '$pkg_path'.") diff --git a/src/server/server.v b/src/server/server.v index 751ea9c..2883942 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -7,7 +7,12 @@ import repo import util import db -const port = 8000 +const ( + port = 8000 + log_file_name = 'vieter.log' + repo_dir_name = 'repos' + db_file_name = 'vieter.sqlite' +) struct App { web.Context @@ -32,11 +37,14 @@ pub fn server(conf Config) ? { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') } + os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') } + mut logger := log.Log{ level: log_level } - logger.set_full_logpath(conf.log_file) + log_file := os.join_path_single(conf.data_dir, server.log_file_name) + logger.set_full_logpath(log_file) logger.log_to_console_too() defer { @@ -45,17 +53,15 @@ pub fn server(conf Config) ? { logger.close() } + repo_dir := os.join_path_single(conf.data_dir, server.repo_dir_name) // This also creates the directories if needed - repo := repo.new(conf.repos_dir, conf.pkg_dir, conf.default_arch) or { + repo := repo.new(repo_dir, conf.pkg_dir, conf.default_arch) or { logger.error(err.msg()) exit(1) } - os.mkdir_all(conf.download_dir) or { - util.exit_with_message(1, 'Failed to create download directory.') - } - - db := db.init('test.db') or { util.exit_with_message(1, 'Failed to initialize database.') } + db_file := os.join_path_single(conf.data_dir, server.db_file_name) + db := db.init(db_file) or { util.exit_with_message(1, 'Failed to initialize database.') } web.run(&App{ logger: logger diff --git a/vieter.toml b/vieter.toml index fc86d77..d3922a4 100644 --- a/vieter.toml +++ b/vieter.toml @@ -1,10 +1,8 @@ # This file contains settings used during development api_key = "test" -download_dir = "data/downloads" -repos_dir = "data/repos" +data_dir = "data" pkg_dir = "data/pkgs" log_level = "DEBUG" -repos_file = "data/repos.json" default_arch = "x86_64" address = "http://localhost:8000" From 204144cee8fdf0aa1dcc55cf1990a5449b83fc41 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 3 May 2022 16:55:50 +0200 Subject: [PATCH 5/8] refactor: removed commented code & ran formatter --- src/cron/daemon/daemon.v | 11 +++--- src/db/git.v | 20 ++++------ src/git/cli.v | 3 +- src/git/client.v | 4 +- src/git/git.v | 84 ---------------------------------------- src/server/git.v | 84 ---------------------------------------- src/server/server.v | 4 +- 7 files changed, 17 insertions(+), 193 deletions(-) delete mode 100644 src/git/git.v diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 1747494..35cca5f 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -10,11 +10,12 @@ import build import docker import db -// How many seconds to wait before retrying to update API if failed -const api_update_retry_timeout = 5 - -// How many seconds to wait before retrying to rebuild image if failed -const rebuild_base_image_retry_timout = 30 +const ( + // How many seconds to wait before retrying to update API if failed + api_update_retry_timeout = 5 + // How many seconds to wait before retrying to rebuild image if failed + rebuild_base_image_retry_timout = 30 +) struct ScheduledBuild { pub: diff --git a/src/db/git.v b/src/db/git.v index ac35ff4..f4a66f0 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -29,12 +29,12 @@ pub mut: pub fn (gr &GitRepo) str() string { mut parts := [ - "id: $gr.id", - "url: $gr.url", - "branch: $gr.branch", - "repo: $gr.repo", - "schedule: $gr.schedule", - "arch: ${gr.arch.map(it.value).join(', ')}" + 'id: $gr.id', + 'url: $gr.url', + 'branch: $gr.branch', + 'repo: $gr.repo', + 'schedule: $gr.schedule', + 'arch: ${gr.arch.map(it.value).join(', ')}', ] str := parts.join('\n') @@ -121,18 +121,12 @@ pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) { // Any fields that are array types require their own update method $if field.typ is string { values << "$field.name = '${params[field.name]}'" - // r.$(field.name) = params[field.name] - // This specific type check is needed for the compiler to ensure - // our types are correct } - //$else $if field.typ is []GitRepoArch { - // r.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) - //} } } values_str := values.join(', ') query := 'update GitRepo set $values_str where id == $repo_id' - println(query) + db.conn.exec_none(query) } diff --git a/src/git/cli.v b/src/git/cli.v index 1839492..634b778 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -3,7 +3,6 @@ module git import cli import env import cron.expression { parse_expression } -import db { GitRepo, GitRepoArch } struct Config { address string [required] @@ -123,7 +122,7 @@ fn list(conf Config) ? { repos := get_repos(conf.address, conf.api_key) ? for repo in repos { - println('${repo.id}\t$repo.url\t$repo.branch\t$repo.repo') + println('$repo.id\t$repo.url\t$repo.branch\t$repo.repo') } } diff --git a/src/git/client.v b/src/git/client.v index d4c5282..f34d2ff 100644 --- a/src/git/client.v +++ b/src/git/client.v @@ -28,8 +28,8 @@ fn send_request(method http.Method, address string, url string, api_key strin // get_repos returns the current list of repos. pub fn get_repos(address string, api_key string) ?[]db.GitRepo { - data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', - api_key, {}) ? + data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', api_key, + {}) ? return data.data } diff --git a/src/git/git.v b/src/git/git.v deleted file mode 100644 index 7c1c83c..0000000 --- a/src/git/git.v +++ /dev/null @@ -1,84 +0,0 @@ -module git - -/* import os */ -/* import json */ - -/* pub struct GitRepo { */ -/* pub mut: */ -/* // URL of the Git repository */ -/* url string */ -/* // Branch of the Git repository to use */ -/* branch string */ -/* // On which architectures the package is allowed to be built. In reality, */ -/* // this controls which builders will periodically build the image. */ -/* arch []string */ -/* // Which repo the builder should publish packages to */ -/* repo string */ -/* // Cron schedule describing how frequently to build the repo. */ -/* schedule string [optional] */ -/* } */ - -/* // patch_from_params patches a GitRepo from a map[string]string, usually */ -/* // provided from a web.App's params */ -/* pub fn (mut r GitRepo) patch_from_params(params map[string]string) { */ -/* $for field in GitRepo.fields { */ -/* if field.name in params { */ -/* $if field.typ is string { */ -/* r.$(field.name) = params[field.name] */ -/* // This specific type check is needed for the compiler to ensure */ -/* // our types are correct */ -/* } $else $if field.typ is []string { */ -/* r.$(field.name) = params[field.name].split(',') */ -/* } */ -/* } */ -/* } */ -/* } */ - -/* // read_repos reads the provided path & parses it into a map of GitRepo's. */ -/* pub fn read_repos(path string) ?map[string]GitRepo { */ -/* if !os.exists(path) { */ -/* mut f := os.create(path) ? */ - -/* defer { */ -/* f.close() */ -/* } */ - -/* f.write_string('{}') ? */ - -/* return {} */ -/* } */ - -/* content := os.read_file(path) ? */ -/* res := json.decode(map[string]GitRepo, content) ? */ - -/* return res */ -/* } */ - -/* // write_repos writes a map of GitRepo's back to disk given the provided path. */ -/* pub fn write_repos(path string, repos &map[string]GitRepo) ? { */ -/* mut f := os.create(path) ? */ - -/* defer { */ -/* f.close() */ -/* } */ - -/* value := json.encode(repos) */ -/* f.write_string(value) ? */ -/* } */ - -/* // repo_from_params creates a GitRepo from a map[string]string, usually */ -/* // provided from a web.App's params */ -/* pub fn repo_from_params(params map[string]string) ?GitRepo { */ -/* mut repo := GitRepo{} */ - -/* // If we're creating a new GitRepo, we want all fields to be present before */ -/* // "patching". */ -/* $for field in GitRepo.fields { */ -/* if field.name !in params && !field.attrs.contains('optional') { */ -/* return error('Missing parameter: ${field.name}.') */ -/* } */ -/* } */ -/* repo.patch_from_params(params) */ - -/* return repo */ -/* } */ diff --git a/src/server/git.v b/src/server/git.v index 0389d5f..6c852b8 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -1,14 +1,10 @@ module server import web -import git import net.http -import rand import response { new_data_response, new_response } import db -const repos_file = 'repos.json' - // get_repos returns the current list of repos. ['/api/repos'; get] fn (mut app App) get_repos() web.Result { @@ -17,13 +13,6 @@ fn (mut app App) get_repos() web.Result { } repos := app.db.get_git_repos() - // repos := rlock app.git_mutex { - // git.read_repos(app.conf.repos_file) or { - // app.lerror('Failed to read repos file: $err.msg()') - - // return app.status(http.Status.internal_server_error) - // } - //} return app.json(http.Status.ok, new_data_response(repos)) } @@ -35,19 +24,6 @@ fn (mut app App) get_single_repo(id int) web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - // repos := rlock app.git_mutex { - // git.read_repos(app.conf.repos_file) or { - // app.lerror('Failed to read repos file.') - - // return app.status(http.Status.internal_server_error) - // } - //} - - // if id !in repos { - // return app.not_found() - //} - - // repo := repos[id] repo := app.db.get_git_repo(id) or { return app.not_found() } return app.json(http.Status.ok, new_data_response(repo)) @@ -74,32 +50,6 @@ fn (mut app App) post_repo() web.Result { app.db.add_git_repo(new_repo) - // id := rand.uuid_v4() - - // mut repos := rlock app.git_mutex { - // git.read_repos(app.conf.repos_file) or { - // app.lerror('Failed to read repos file.') - - // return app.status(http.Status.internal_server_error) - // } - //} - // repos := app.db.get_git_repos() - - //// We need to check for duplicates - // for _, repo in repos { - // if repo == new_repo { - // return app.json(http.Status.bad_request, new_response('Duplicate repository.')) - // } - //} - - // repos[id] = new_repo - - // lock app.git_mutex { - // git.write_repos(app.conf.repos_file, &repos) or { - // return app.status(http.Status.internal_server_error) - // } - //} - return app.json(http.Status.ok, new_response('Repo added successfully.')) } @@ -110,25 +60,9 @@ fn (mut app App) delete_repo(id int) web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - // mut repos := rlock app.git_mutex { - // git.read_repos(app.conf.repos_file) or { - // app.lerror('Failed to read repos file.') - - // return app.status(http.Status.internal_server_error) - // } - //} - - // if id !in repos { - // return app.not_found() - //} - // repos.delete(id) app.db.delete_git_repo(id) - // lock app.git_mutex { - // git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } - // } - return app.json(http.Status.ok, new_response('Repo removed successfully.')) } @@ -147,23 +81,5 @@ fn (mut app App) patch_repo(id int) web.Result { app.db.update_git_repo_archs(id, arch_objs) } - // mut repos := rlock app.git_mutex { - // git.read_repos(app.conf.repos_file) or { - // app.lerror('Failed to read repos file.') - - // return app.status(http.Status.internal_server_error) - // } - // } - - // if id !in repos { - // return app.not_found() - // } - - // repos[id].patch_from_params(app.query) - - // lock app.git_mutex { - // git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } - // } - return app.json(http.Status.ok, new_response('Repo updated successfully.')) } diff --git a/src/server/server.v b/src/server/server.v index 2883942..b2a2ad2 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -20,9 +20,7 @@ pub: conf Config [required; web_global] pub mut: repo repo.RepoGroupManager [required; web_global] - // This is used to claim the file lock on the repos file - git_mutex shared util.Dummy - db db.VieterDb + db db.VieterDb } // server starts the web server & starts listening for requests From 5781796e99543f2f16d24bd210acb26d937d3684 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 3 May 2022 19:55:52 +0200 Subject: [PATCH 6/8] doc: added docstrings to all db/git functions --- CHANGELOG.md | 3 +++ src/cron/cli.v | 2 +- src/db/db.v | 1 + src/db/git.v | 10 +++++++++- src/git/client.v | 1 + 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abbdc74..f40ce52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed * Switched from compiler fork to fully vanilla compiler mirror +* `download_dir`, `repos_file` & `repos_dir` config values have been replaced + with `data_dir` +* Storage of metadata (e.g. Git repositories) is now done using Sqlite ### Added diff --git a/src/cron/cli.v b/src/cron/cli.v index 9536c37c..15bc986 100644 --- a/src/cron/cli.v +++ b/src/cron/cli.v @@ -8,7 +8,7 @@ pub: log_level string = 'WARN' api_key string address string - data_dir string + data_dir string base_image string = 'archlinux:base-devel' max_concurrent_builds int = 1 api_update_frequency int = 15 diff --git a/src/db/db.v b/src/db/db.v index b62fa3b..a75c34c 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -6,6 +6,7 @@ struct VieterDb { conn sqlite.DB } +// init initializes a database & adds the correct tables. pub fn init(db_path string) ?VieterDb { conn := sqlite.connect(db_path) ? diff --git a/src/db/git.v b/src/db/git.v index f4a66f0..2fb2a92 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -7,6 +7,7 @@ pub: value string [nonull] } +// str returns a string representation. pub fn (gra &GitRepoArch) str() string { return gra.value } @@ -27,6 +28,7 @@ pub mut: arch []GitRepoArch [fkey: 'repo_id'] } +// str returns a string representation. pub fn (gr &GitRepo) str() string { mut parts := [ 'id: $gr.id', @@ -57,7 +59,7 @@ pub fn (mut r GitRepo) patch_from_params(params map[string]string) { } } -// repo_from_params creates a GitRepo from a map[string]string, usually +// git_repo_from_params creates a GitRepo from a map[string]string, usually // provided from a web.App's params pub fn git_repo_from_params(params map[string]string) ?GitRepo { mut repo := GitRepo{} @@ -74,6 +76,7 @@ pub fn git_repo_from_params(params map[string]string) ?GitRepo { return repo } +// get_git_repos returns all GitRepo's in the database. pub fn (db &VieterDb) get_git_repos() []GitRepo { res := sql db.conn { select from GitRepo order by id @@ -82,6 +85,7 @@ pub fn (db &VieterDb) get_git_repos() []GitRepo { return res } +// get_git_repo tries to return a specific GitRepo. pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo { res := sql db.conn { select from GitRepo where id == repo_id @@ -97,12 +101,14 @@ pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo { return res } +// add_git_repo inserts the given GitRepo into the database. pub fn (db &VieterDb) add_git_repo(repo GitRepo) { sql db.conn { insert repo into GitRepo } } +// delete_git_repo deletes the repo with the given ID from the database. pub fn (db &VieterDb) delete_git_repo(repo_id int) { sql db.conn { delete from GitRepo where id == repo_id @@ -110,6 +116,7 @@ pub fn (db &VieterDb) delete_git_repo(repo_id int) { } } +// update_git_repo updates any non-array values for a given GitRepo. pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) { // sql db.conn { // update GitRepo set repo @@ -130,6 +137,7 @@ pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) { db.conn.exec_none(query) } +// update_git_repo_archs updates a given GitRepo's arch value. pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { archs_with_id := archs.map(GitRepoArch{ ...it diff --git a/src/git/client.v b/src/git/client.v index f34d2ff..b5f8e9f 100644 --- a/src/git/client.v +++ b/src/git/client.v @@ -34,6 +34,7 @@ pub fn get_repos(address string, api_key string) ?[]db.GitRepo { return data.data } +// get_repo returns the repo for a specific ID. pub fn get_repo(address string, api_key string, id int) ?db.GitRepo { data := send_request(http.Method.get, address, '/api/repos/$id', api_key, {}) ? From b6d5bd3228dd16169e539efcfe516ef0f109c1fd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 3 May 2022 20:13:28 +0200 Subject: [PATCH 7/8] doc: listed new config variables in docs --- docs/content/configuration.md | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/docs/content/configuration.md b/docs/content/configuration.md index df92844..ded40cb 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -35,18 +35,10 @@ passed to them. Each mode requires a different configuration. * `log_file`: log file to write logs to. Defaults to `vieter.log` in the current directory. * `pkg_dir`: where Vieter should store the actual package archives. -* `download_dir`: where Vieter should initially download uploaded files. +* `data_dir`: where Vieter stores the repositories, log file & database. * `api_key`: the API key to use when authenticating requests. -* `repo_dir`: where Vieter should store the contents of the repository. -* `repos_file`: JSON file where the list of Git repositories is saved * `default_arch`: architecture to always add packages of arch `any` to. -{{< hint info >}} -**Note** -Because Vieter hard links files between `download_dir` & `pkg_dir`, they need -to be on the same file system. -{{< /hint >}} - ### Builder * `api_key`: the API key to use when authenticating requests. @@ -62,3 +54,18 @@ to be on the same file system. * `api_key`: the API key to use when authenticating requests. * `address`: Base your URL of your Vieter instance, e.g. https://example.com + +### Cron + +* `log_level`: defines how much logs to show. Valid values are one of `FATAL`, + `ERROR`, `WARN`, `INFO` or `DEBUG`. Defaults to `WARN` +* `api_key`: the API key to use when authenticating requests. +* `address`: Base your URL of your Vieter instance, e.g. https://example.com. + This *must* be the publicly facing URL of your Vieter instance. +* `data_dir`: where Vieter stores the log file. +* `base_image`: Docker image from which to create the builder images. +* `max_concurrent_builds`: amount of builds to run at once. +* `api_update_frequency`: how frequenty to check for changes in the repo list. +* `image_rebuild+frequency`: how frequently to rebuild the builder image +* `global_schedule`: cron schedule to use for any repo without an individual + schedule From 8c5652c2301a55e0c44c07653f628ae7349e6bdf Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 5 May 2022 09:35:19 +0200 Subject: [PATCH 8/8] ci: made build upload failable; updated ci for use with PRs --- .woodpecker/.arch.yml | 4 ++++ .woodpecker/.build.yml | 16 +++++++--------- .woodpecker/.build_experimental.yml | 29 ----------------------------- .woodpecker/.deploy.yml | 4 +++- .woodpecker/.docker.yml | 28 ++++++++++++++++------------ .woodpecker/.gitea.yml | 5 ++--- .woodpecker/.lint.yml | 6 ++++-- .woodpecker/.test.yml | 8 ++++---- src/db/git.v | 4 +--- src/server/git.v | 1 - 10 files changed, 41 insertions(+), 64 deletions(-) delete mode 100644 .woodpecker/.build_experimental.yml diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index 742095c..93c9840 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -25,6 +25,8 @@ pipeline: # inside the repo - curl -OL https://git.rustybever.be/Chewing_Bever/vieter/raw/branch/dev/PKGBUILD - makepkg -s --noconfirm --needed + when: + event: push publish: image: 'curlimages/curl' @@ -33,3 +35,5 @@ pipeline: - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done' secrets: - vieter_api_key + when: + event: push diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index f9cab00..a3b66ea 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -1,11 +1,8 @@ matrix: PLATFORM: - - linux/amd64 - - linux/arm64 - # I just don't have a performant enough runner for this platform - # - linux/arm/v7 + - 'linux/amd64' + - 'linux/arm64' -# These checks already get performed on the feature branches platform: ${PLATFORM} pipeline: @@ -15,9 +12,9 @@ pipeline: commands: - make when: - event: push + event: [push, pull_request] branch: - exclude: [main, dev] + exclude: [main] prod: image: 'chewingbever/vlang:latest' @@ -35,7 +32,7 @@ pipeline: - strip -s pvieter - du -h pvieter when: - event: push + event: [push, pull_request] upload: image: 'chewingbever/vlang:latest' @@ -52,6 +49,7 @@ pipeline: - > curl --silent + --fail -XPUT -T pvieter -H "Host: $URL" @@ -60,4 +58,4 @@ pipeline: -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" https://$URL$OBJ_PATH when: - event: push + event: [push, pull_request] diff --git a/.woodpecker/.build_experimental.yml b/.woodpecker/.build_experimental.yml deleted file mode 100644 index 0129d2b..0000000 --- a/.woodpecker/.build_experimental.yml +++ /dev/null @@ -1,29 +0,0 @@ -# These builds are not important for the project, but might be valuable for -# fixing bugs in the V compiler. - -platform: linux/amd64 -branches: - exclude: [master, dev] - -pipeline: - autofree: - image: 'chewingbever/vlang:latest' - pull: true - group: 'build' - commands: - - make autofree - - readelf -d afvieter - - du -h afvieter - when: - event: push - - skip-unused: - image: 'chewingbever/vlang:latest' - pull: true - group: 'build' - commands: - - make skip-unused - - readelf -d suvieter - - du -h suvieter - when: - event: push diff --git a/.woodpecker/.deploy.yml b/.woodpecker/.deploy.yml index dd77fb9..8e68641 100644 --- a/.woodpecker/.deploy.yml +++ b/.woodpecker/.deploy.yml @@ -1,4 +1,4 @@ -branches: 'dev' +branches: [ 'dev' ] platform: 'linux/amd64' depends_on: - 'docker' @@ -14,3 +14,5 @@ pipeline: commands: - 'curl -XPOST -s --fail $WEBHOOK_APP' - 'curl -XPOST -s --fail $WEBHOOK_CRON' + when: + event: push diff --git a/.woodpecker/.docker.yml b/.woodpecker/.docker.yml index 9b605f3..bab869b 100644 --- a/.woodpecker/.docker.yml +++ b/.woodpecker/.docker.yml @@ -1,30 +1,34 @@ branches: [main, dev] -platform: linux/amd64 +platform: 'linux/amd64' depends_on: - build pipeline: dev: - image: woodpeckerci/plugin-docker-buildx - secrets: [ docker_username, docker_password ] + image: 'woodpeckerci/plugin-docker-buildx' + secrets: + - 'docker_username' + - 'docker_password' settings: - repo: chewingbever/vieter - tag: dev - platforms: [ linux/arm64/v8, linux/amd64 ] + repo: 'chewingbever/vieter' + tag: 'dev' + platforms: [ 'linux/arm64/v8', 'linux/amd64' ] build_args_from_env: - - CI_COMMIT_SHA + - 'CI_COMMIT_SHA' when: event: push branch: dev release: - image: woodpeckerci/plugin-docker-buildx - secrets: [ docker_username, docker_password ] + image: 'woodpeckerci/plugin-docker-buildx' + secrets: + - 'docker_username' + - 'docker_password' settings: - repo: chewingbever/vieter + repo: 'chewingbever/vieter' auto_tag: true - platforms: [ linux/arm64/v8, linux/amd64 ] + platforms: [ 'linux/arm64/v8', 'linux/amd64' ] build_args_from_env: - - CI_COMMIT_SHA + - 'CI_COMMIT_SHA' when: event: tag diff --git a/.woodpecker/.gitea.yml b/.woodpecker/.gitea.yml index c492d34..1877050 100644 --- a/.woodpecker/.gitea.yml +++ b/.woodpecker/.gitea.yml @@ -1,6 +1,5 @@ -# Yeah so this only works on tags so we'll worry about this later -platform: linux/amd64 -branches: main +platform: 'linux/amd64' +branches: [ 'main' ] depends_on: - build diff --git a/.woodpecker/.lint.yml b/.woodpecker/.lint.yml index b1c16fd..b13aea4 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/.lint.yml @@ -1,7 +1,7 @@ # These checks already get performed on the feature branches branches: - exclude: [ main, dev ] -platform: linux/amd64 + exclude: [ main ] +platform: 'linux/amd64' pipeline: lint: @@ -9,3 +9,5 @@ pipeline: pull: true commands: - make lint + when: + event: [ push, pull_request ] diff --git a/.woodpecker/.test.yml b/.woodpecker/.test.yml index 3800cc1..0337570 100644 --- a/.woodpecker/.test.yml +++ b/.woodpecker/.test.yml @@ -1,10 +1,10 @@ matrix: PLATFORM: - - linux/amd64 - - linux/arm64 + - 'linux/amd64' + - 'linux/arm64' branches: - exclude: [main, dev] + exclude: [ main ] platform: ${PLATFORM} pipeline: @@ -14,4 +14,4 @@ pipeline: commands: - make test when: - event: push + event: [push, pull_request] diff --git a/src/db/git.v b/src/db/git.v index 2fb2a92..c40086b 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -118,11 +118,9 @@ pub fn (db &VieterDb) delete_git_repo(repo_id int) { // update_git_repo updates any non-array values for a given GitRepo. pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) { - // sql db.conn { - // update GitRepo set repo - //} mut values := []string{} + // TODO does this allow for SQL injection? $for field in GitRepo.fields { if field.name in params { // Any fields that are array types require their own update method diff --git a/src/server/git.v b/src/server/git.v index 6c852b8..c5cbc0a 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -60,7 +60,6 @@ fn (mut app App) delete_repo(id int) web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - // repos.delete(id) app.db.delete_git_repo(id) return app.json(http.Status.ok, new_response('Repo removed successfully.'))