From 7419144f97a955c9d5d67816d483e8da55690d47 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 3 May 2022 16:16:56 +0200 Subject: [PATCH] feat: removed git.GitRepo type feat(cli): updated to new GitRepo format --- src/build/build.v | 7 +- src/cron/daemon/daemon.v | 18 +++--- src/db/git.v | 53 +++++++++------ src/git/cli.v | 61 +++++++----------- src/git/client.v | 16 +++-- src/git/git.v | 136 +++++++++++++++++++-------------------- src/server/git.v | 60 ++++++++--------- 7 files changed, 179 insertions(+), 172 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index bc604fa..15a5eb8 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -5,6 +5,7 @@ import encoding.base64 import time import git import os +import db const container_build_dir = '/build' @@ -75,7 +76,7 @@ pub fn create_build_image(base_image string) ?string { // build_repo builds, packages & publishes a given Arch package based on the // provided GitRepo. The base image ID should be of an image previously created // by create_build_image. -pub fn build_repo(address string, api_key string, base_image_id string, repo &git.GitRepo) ? { +pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ? { build_arch := os.uname().machine // TODO what to do with PKGBUILDs that build multiple packages? @@ -125,11 +126,11 @@ fn build(conf Config) ? { build_arch := os.uname().machine // We get the repos map from the Vieter instance - repos_map := git.get_repos(conf.address, conf.api_key) ? + repos := git.get_repos(conf.address, conf.api_key) ? // We filter out any repos that aren't allowed to be built on this // architecture - filtered_repos := repos_map.keys().map(repos_map[it]).filter(it.arch.contains(build_arch)) + filtered_repos := repos.filter(it.arch.map(it.value).contains(build_arch)) // No point in doing work if there's no repos present if filtered_repos.len == 0 { diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 088a24f..1747494 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -8,6 +8,7 @@ import cron.expression { CronExpression, parse_expression } import math import build import docker +import db // How many seconds to wait before retrying to update API if failed const api_update_retry_timeout = 5 @@ -18,7 +19,7 @@ const rebuild_base_image_retry_timout = 30 struct ScheduledBuild { pub: repo_id string - repo git.GitRepo + repo db.GitRepo timestamp time.Time } @@ -37,7 +38,7 @@ mut: api_update_frequency int image_rebuild_frequency int // Repos currently loaded from API. - repos_map map[string]git.GitRepo + repos []db.GitRepo // At what point to update the list of repositories. api_update_timestamp time.Time image_build_timestamp time.Time @@ -90,7 +91,7 @@ pub fn (mut d Daemon) run() { // haven't been renewed. else { for sb in finished_builds { - d.schedule_build(sb.repo_id, sb.repo) + d.schedule_build(sb.repo) } } @@ -149,11 +150,11 @@ pub fn (mut d Daemon) run() { } // schedule_build adds the next occurence of the given repo build to the queue. -fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) { +fn (mut d Daemon) schedule_build(repo db.GitRepo) { ce := if repo.schedule != '' { parse_expression(repo.schedule) or { // TODO This shouldn't return an error if the expression is empty. - d.lerror("Error while parsing cron expression '$repo.schedule' ($repo_id): $err.msg()") + d.lerror("Error while parsing cron expression '$repo.schedule' (id $repo.id): $err.msg()") d.global_schedule } @@ -168,7 +169,6 @@ fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) { } d.queue.insert(ScheduledBuild{ - repo_id: repo_id repo: repo timestamp: timestamp }) @@ -186,7 +186,7 @@ fn (mut d Daemon) renew_repos() { return } - d.repos_map = new_repos.move() + d.repos = new_repos d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency) } @@ -224,8 +224,8 @@ fn (mut d Daemon) renew_queue() { // For each repository in repos_map, parse their cron expression (or use // the default one if not present) & add them to the queue - for id, repo in d.repos_map { - d.schedule_build(id, repo) + for repo in d.repos { + d.schedule_build(repo) } } diff --git a/src/db/git.v b/src/db/git.v index 29606e4..ac35ff4 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -3,8 +3,12 @@ module db pub struct GitRepoArch { pub: id int [primary; sql: serial] - repo_id int - value string + repo_id int [nonull] + value string [nonull] +} + +pub fn (gra &GitRepoArch) str() string { + return gra.value } pub struct GitRepo { @@ -23,6 +27,20 @@ pub mut: arch []GitRepoArch [fkey: 'repo_id'] } +pub fn (gr &GitRepo) str() string { + mut parts := [ + "id: $gr.id", + "url: $gr.url", + "branch: $gr.branch", + "repo: $gr.repo", + "schedule: $gr.schedule", + "arch: ${gr.arch.map(it.value).join(', ')}" + ] + str := parts.join('\n') + + return str +} + // patch_from_params patches a GitRepo from a map[string]string, usually // provided from a web.App's params pub fn (mut r GitRepo) patch_from_params(params map[string]string) { @@ -93,44 +111,39 @@ pub fn (db &VieterDb) delete_git_repo(repo_id int) { } pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) { - /* sql db.conn { */ - /* update GitRepo set repo */ - /* } */ + // sql db.conn { + // update GitRepo set repo + //} mut values := []string{} $for field in GitRepo.fields { if field.name in params { // Any fields that are array types require their own update method $if field.typ is string { - values << "${field.name} = '${params[field.name]}'" - /* r.$(field.name) = params[field.name] */ + values << "$field.name = '${params[field.name]}'" + // r.$(field.name) = params[field.name] // This specific type check is needed for the compiler to ensure // our types are correct } - /* $else $if field.typ is []GitRepoArch { */ - /* r.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) */ - /* } */ + //$else $if field.typ is []GitRepoArch { + // r.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) + //} } } - values_str := values.join(', ') - query := "update GitRepo set $values_str where id == $repo_id" + query := 'update GitRepo set $values_str where id == $repo_id' println(query) db.conn.exec_none(query) - } pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { -archs_with_id := archs.map(GitRepoArch{ - ...it - repo_id: repo_id - }) + archs_with_id := archs.map(GitRepoArch{ + ...it + repo_id: repo_id + }) sql db.conn { - // Remove all old values delete from GitRepoArch where repo_id == repo_id - // Insert all the new ones - /* insert archs_with_id into GitRepoArch */ } for arch in archs_with_id { diff --git a/src/git/cli.v b/src/git/cli.v index 0eff55f..1839492 100644 --- a/src/git/cli.v +++ b/src/git/cli.v @@ -3,6 +3,7 @@ module git import cli import env import cron.expression { parse_expression } +import db { GitRepo, GitRepoArch } struct Config { address string [required] @@ -116,34 +117,13 @@ pub fn cmd() cli.Command { // get_repo_by_prefix tries to find the repo with the given prefix in its // ID. If multiple or none are found, an error is raised. -fn get_repo_by_prefix(conf Config, id_prefix string) ?(string, GitRepo) { - repos := get_repos(conf.address, conf.api_key) ? - - mut res := map[string]GitRepo{} - - for id, repo in repos { - if id.starts_with(id_prefix) { - res[id] = repo - } - } - - if res.len == 0 { - return error('No repo found for given prefix.') - } - - if res.len > 1 { - return error('Multiple repos found for given prefix.') - } - - return res.keys()[0], res[res.keys()[0]] -} // list prints out a list of all repositories. fn list(conf Config) ? { repos := get_repos(conf.address, conf.api_key) ? - for id, details in repos { - println('${id[..8]}\t$details.url\t$details.branch\t$details.repo') + for repo in repos { + println('${repo.id}\t$repo.url\t$repo.branch\t$repo.repo') } } @@ -155,15 +135,18 @@ fn add(conf Config, url string, branch string, repo string) ? { } // remove removes a repository from the server's list. -fn remove(conf Config, id_prefix string) ? { - id, _ := get_repo_by_prefix(conf, id_prefix) ? - res := remove_repo(conf.address, conf.api_key, id) ? +fn remove(conf Config, id string) ? { + // id, _ := get_repo_by_prefix(conf, id_prefix) ? + id_int := id.int() - println(res.message) + if id_int != 0 { + res := remove_repo(conf.address, conf.api_key, id_int) ? + println(res.message) + } } // patch patches a given repository with the provided params. -fn patch(conf Config, id_prefix string, params map[string]string) ? { +fn patch(conf Config, id string, params map[string]string) ? { // We check the cron expression first because it's useless to send an // invalid one to the server. if 'schedule' in params && params['schedule'] != '' { @@ -172,20 +155,22 @@ fn patch(conf Config, id_prefix string, params map[string]string) ? { } } - id, _ := get_repo_by_prefix(conf, id_prefix) ? - res := patch_repo(conf.address, conf.api_key, id, params) ? + id_int := id.int() + if id_int != 0 { + res := patch_repo(conf.address, conf.api_key, id_int, params) ? - println(res.message) + println(res.message) + } } // info shows detailed information for a given repo. -fn info(conf Config, id_prefix string) ? { - id, repo := get_repo_by_prefix(conf, id_prefix) ? +fn info(conf Config, id string) ? { + id_int := id.int() - println('id: $id') - - $for field in GitRepo.fields { - val := repo.$(field.name) - println('$field.name: $val') + if id_int == 0 { + return } + + repo := get_repo(conf.address, conf.api_key, id_int) ? + println(repo) } diff --git a/src/git/client.v b/src/git/client.v index 0ed19b5..d4c5282 100644 --- a/src/git/client.v +++ b/src/git/client.v @@ -3,6 +3,7 @@ module git import json import response { Response } import net.http +import db // send_request is a convenience method for sending requests to the repos // API. It mostly does string manipulation to create a query string containing @@ -26,8 +27,15 @@ fn send_request(method http.Method, address string, url string, api_key strin } // get_repos returns the current list of repos. -pub fn get_repos(address string, api_key string) ?map[string]GitRepo { - data := send_request(http.Method.get, address, '/api/repos', api_key, +pub fn get_repos(address string, api_key string) ?[]db.GitRepo { + data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', + api_key, {}) ? + + return data.data +} + +pub fn get_repo(address string, api_key string, id int) ?db.GitRepo { + data := send_request(http.Method.get, address, '/api/repos/$id', api_key, {}) ? return data.data @@ -51,7 +59,7 @@ pub fn add_repo(address string, api_key string, url string, branch string, repo } // remove_repo removes the repo with the given ID from the server. -pub fn remove_repo(address string, api_key string, id string) ?Response { +pub fn remove_repo(address string, api_key string, id int) ?Response { data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, {}) ? @@ -60,7 +68,7 @@ pub fn remove_repo(address string, api_key string, id string) ?Response // patch_repo sends a PATCH request to the given repo with the params as // payload. -pub fn patch_repo(address string, api_key string, id string, params map[string]string) ?Response { +pub fn patch_repo(address string, api_key string, id int, params map[string]string) ?Response { data := send_request(http.Method.patch, address, '/api/repos/$id', api_key, params) ? diff --git a/src/git/git.v b/src/git/git.v index 2023f34..7c1c83c 100644 --- a/src/git/git.v +++ b/src/git/git.v @@ -1,84 +1,84 @@ module git -import os -import json +/* import os */ +/* import json */ -pub struct GitRepo { -pub mut: - // URL of the Git repository - url string - // Branch of the Git repository to use - branch string - // On which architectures the package is allowed to be built. In reality, - // this controls which builders will periodically build the image. - arch []string - // Which repo the builder should publish packages to - repo string - // Cron schedule describing how frequently to build the repo. - schedule string [optional] -} +/* pub struct GitRepo { */ +/* pub mut: */ +/* // URL of the Git repository */ +/* url string */ +/* // Branch of the Git repository to use */ +/* branch string */ +/* // On which architectures the package is allowed to be built. In reality, */ +/* // this controls which builders will periodically build the image. */ +/* arch []string */ +/* // Which repo the builder should publish packages to */ +/* repo string */ +/* // Cron schedule describing how frequently to build the repo. */ +/* schedule string [optional] */ +/* } */ -// patch_from_params patches a GitRepo from a map[string]string, usually -// provided from a web.App's params -pub fn (mut r GitRepo) patch_from_params(params map[string]string) { - $for field in GitRepo.fields { - if field.name in params { - $if field.typ is string { - r.$(field.name) = params[field.name] - // This specific type check is needed for the compiler to ensure - // our types are correct - } $else $if field.typ is []string { - r.$(field.name) = params[field.name].split(',') - } - } - } -} +/* // patch_from_params patches a GitRepo from a map[string]string, usually */ +/* // provided from a web.App's params */ +/* pub fn (mut r GitRepo) patch_from_params(params map[string]string) { */ +/* $for field in GitRepo.fields { */ +/* if field.name in params { */ +/* $if field.typ is string { */ +/* r.$(field.name) = params[field.name] */ +/* // This specific type check is needed for the compiler to ensure */ +/* // our types are correct */ +/* } $else $if field.typ is []string { */ +/* r.$(field.name) = params[field.name].split(',') */ +/* } */ +/* } */ +/* } */ +/* } */ -// read_repos reads the provided path & parses it into a map of GitRepo's. -pub fn read_repos(path string) ?map[string]GitRepo { - if !os.exists(path) { - mut f := os.create(path) ? +/* // read_repos reads the provided path & parses it into a map of GitRepo's. */ +/* pub fn read_repos(path string) ?map[string]GitRepo { */ +/* if !os.exists(path) { */ +/* mut f := os.create(path) ? */ - defer { - f.close() - } +/* defer { */ +/* f.close() */ +/* } */ - f.write_string('{}') ? +/* f.write_string('{}') ? */ - return {} - } +/* return {} */ +/* } */ - content := os.read_file(path) ? - res := json.decode(map[string]GitRepo, content) ? +/* content := os.read_file(path) ? */ +/* res := json.decode(map[string]GitRepo, content) ? */ - return res -} +/* return res */ +/* } */ -// write_repos writes a map of GitRepo's back to disk given the provided path. -pub fn write_repos(path string, repos &map[string]GitRepo) ? { - mut f := os.create(path) ? +/* // write_repos writes a map of GitRepo's back to disk given the provided path. */ +/* pub fn write_repos(path string, repos &map[string]GitRepo) ? { */ +/* mut f := os.create(path) ? */ - defer { - f.close() - } +/* defer { */ +/* f.close() */ +/* } */ - value := json.encode(repos) - f.write_string(value) ? -} +/* value := json.encode(repos) */ +/* f.write_string(value) ? */ +/* } */ -// repo_from_params creates a GitRepo from a map[string]string, usually -// provided from a web.App's params -pub fn repo_from_params(params map[string]string) ?GitRepo { - mut repo := GitRepo{} +/* // repo_from_params creates a GitRepo from a map[string]string, usually */ +/* // provided from a web.App's params */ +/* pub fn repo_from_params(params map[string]string) ?GitRepo { */ +/* mut repo := GitRepo{} */ - // If we're creating a new GitRepo, we want all fields to be present before - // "patching". - $for field in GitRepo.fields { - if field.name !in params && !field.attrs.contains('optional') { - return error('Missing parameter: ${field.name}.') - } - } - repo.patch_from_params(params) +/* // If we're creating a new GitRepo, we want all fields to be present before */ +/* // "patching". */ +/* $for field in GitRepo.fields { */ +/* if field.name !in params && !field.attrs.contains('optional') { */ +/* return error('Missing parameter: ${field.name}.') */ +/* } */ +/* } */ +/* repo.patch_from_params(params) */ - return repo -} +/* return repo */ +/* } */ diff --git a/src/server/git.v b/src/server/git.v index 69cce81..0389d5f 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -110,24 +110,24 @@ fn (mut app App) delete_repo(id int) web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } - /* mut repos := rlock app.git_mutex { */ - /* git.read_repos(app.conf.repos_file) or { */ - /* app.lerror('Failed to read repos file.') */ + // mut repos := rlock app.git_mutex { + // git.read_repos(app.conf.repos_file) or { + // app.lerror('Failed to read repos file.') - /* return app.status(http.Status.internal_server_error) */ - /* } */ - /* } */ + // return app.status(http.Status.internal_server_error) + // } + //} - /* if id !in repos { */ - /* return app.not_found() */ - /* } */ + // if id !in repos { + // return app.not_found() + //} - /* repos.delete(id) */ + // repos.delete(id) app.db.delete_git_repo(id) -/* lock app.git_mutex { */ -/* git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } */ -/* } */ + // lock app.git_mutex { + // git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } + // } return app.json(http.Status.ok, new_response('Repo removed successfully.')) } @@ -141,29 +141,29 @@ fn (mut app App) patch_repo(id int) web.Result { app.db.update_git_repo(id, app.query) - if 'arch' in app.query { - arch_objs := app.query['arch'].split(',').map(db.GitRepoArch{value: it}) + if 'arch' in app.query { + arch_objs := app.query['arch'].split(',').map(db.GitRepoArch{ value: it }) - app.db.update_git_repo_archs(id, arch_objs) - } + app.db.update_git_repo_archs(id, arch_objs) + } -/* mut repos := rlock app.git_mutex { */ -/* git.read_repos(app.conf.repos_file) or { */ -/* app.lerror('Failed to read repos file.') */ + // mut repos := rlock app.git_mutex { + // git.read_repos(app.conf.repos_file) or { + // app.lerror('Failed to read repos file.') -/* return app.status(http.Status.internal_server_error) */ -/* } */ -/* } */ + // return app.status(http.Status.internal_server_error) + // } + // } -/* if id !in repos { */ -/* return app.not_found() */ -/* } */ + // if id !in repos { + // return app.not_found() + // } -/* repos[id].patch_from_params(app.query) */ + // repos[id].patch_from_params(app.query) -/* lock app.git_mutex { */ -/* git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } */ -/* } */ + // lock app.git_mutex { + // git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } + // } return app.json(http.Status.ok, new_response('Repo updated successfully.')) }