Compare commits

..

No commits in common. "ac91cfebb268bccf5681bcaa4d8ed0aca8450177" and "8174327a34f6c61cb9f6a577a844df7ebc9d778d" have entirely different histories.

7 changed files with 162 additions and 203 deletions

View File

@ -5,7 +5,6 @@ import encoding.base64
import time import time
import git import git
import os import os
import db
const container_build_dir = '/build' const container_build_dir = '/build'
@ -76,7 +75,7 @@ pub fn create_build_image(base_image string) ?string {
// build_repo builds, packages & publishes a given Arch package based on the // build_repo builds, packages & publishes a given Arch package based on the
// provided GitRepo. The base image ID should be of an image previously created // provided GitRepo. The base image ID should be of an image previously created
// by create_build_image. // by create_build_image.
pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ? { pub fn build_repo(address string, api_key string, base_image_id string, repo &git.GitRepo) ? {
build_arch := os.uname().machine build_arch := os.uname().machine
// TODO what to do with PKGBUILDs that build multiple packages? // TODO what to do with PKGBUILDs that build multiple packages?
@ -126,11 +125,11 @@ fn build(conf Config) ? {
build_arch := os.uname().machine build_arch := os.uname().machine
// We get the repos map from the Vieter instance // We get the repos map from the Vieter instance
repos := git.get_repos(conf.address, conf.api_key) ? repos_map := git.get_repos(conf.address, conf.api_key) ?
// We filter out any repos that aren't allowed to be built on this // We filter out any repos that aren't allowed to be built on this
// architecture // architecture
filtered_repos := repos.filter(it.arch.map(it.value).contains(build_arch)) filtered_repos := repos_map.keys().map(repos_map[it]).filter(it.arch.contains(build_arch))
// No point in doing work if there's no repos present // No point in doing work if there's no repos present
if filtered_repos.len == 0 { if filtered_repos.len == 0 {

View File

@ -8,7 +8,6 @@ import cron.expression { CronExpression, parse_expression }
import math import math
import build import build
import docker import docker
import db
// How many seconds to wait before retrying to update API if failed // How many seconds to wait before retrying to update API if failed
const api_update_retry_timeout = 5 const api_update_retry_timeout = 5
@ -19,7 +18,7 @@ const rebuild_base_image_retry_timout = 30
struct ScheduledBuild { struct ScheduledBuild {
pub: pub:
repo_id string repo_id string
repo db.GitRepo repo git.GitRepo
timestamp time.Time timestamp time.Time
} }
@ -38,7 +37,7 @@ mut:
api_update_frequency int api_update_frequency int
image_rebuild_frequency int image_rebuild_frequency int
// Repos currently loaded from API. // Repos currently loaded from API.
repos []db.GitRepo repos_map map[string]git.GitRepo
// At what point to update the list of repositories. // At what point to update the list of repositories.
api_update_timestamp time.Time api_update_timestamp time.Time
image_build_timestamp time.Time image_build_timestamp time.Time
@ -91,7 +90,7 @@ pub fn (mut d Daemon) run() {
// haven't been renewed. // haven't been renewed.
else { else {
for sb in finished_builds { for sb in finished_builds {
d.schedule_build(sb.repo) d.schedule_build(sb.repo_id, sb.repo)
} }
} }
@ -150,11 +149,11 @@ pub fn (mut d Daemon) run() {
} }
// schedule_build adds the next occurence of the given repo build to the queue. // schedule_build adds the next occurence of the given repo build to the queue.
fn (mut d Daemon) schedule_build(repo db.GitRepo) { fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) {
ce := if repo.schedule != '' { ce := if repo.schedule != '' {
parse_expression(repo.schedule) or { parse_expression(repo.schedule) or {
// TODO This shouldn't return an error if the expression is empty. // TODO This shouldn't return an error if the expression is empty.
d.lerror("Error while parsing cron expression '$repo.schedule' (id $repo.id): $err.msg()") d.lerror("Error while parsing cron expression '$repo.schedule' ($repo_id): $err.msg()")
d.global_schedule d.global_schedule
} }
@ -169,6 +168,7 @@ fn (mut d Daemon) schedule_build(repo db.GitRepo) {
} }
d.queue.insert(ScheduledBuild{ d.queue.insert(ScheduledBuild{
repo_id: repo_id
repo: repo repo: repo
timestamp: timestamp timestamp: timestamp
}) })
@ -186,7 +186,7 @@ fn (mut d Daemon) renew_repos() {
return return
} }
d.repos = new_repos d.repos_map = new_repos.move()
d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency) d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency)
} }
@ -224,8 +224,8 @@ fn (mut d Daemon) renew_queue() {
// For each repository in repos_map, parse their cron expression (or use // For each repository in repos_map, parse their cron expression (or use
// the default one if not present) & add them to the queue // the default one if not present) & add them to the queue
for repo in d.repos { for id, repo in d.repos_map {
d.schedule_build(repo) d.schedule_build(id, repo)
} }
} }

View File

@ -1,14 +1,10 @@
module db module db
pub struct GitRepoArch { struct GitRepoArch {
pub: pub:
id int [primary; sql: serial] id int [primary; sql: serial]
repo_id int [nonull] repo_id int
value string [nonull] value string
}
pub fn (gra &GitRepoArch) str() string {
return gra.value
} }
pub struct GitRepo { pub struct GitRepo {
@ -27,20 +23,6 @@ pub mut:
arch []GitRepoArch [fkey: 'repo_id'] arch []GitRepoArch [fkey: 'repo_id']
} }
pub fn (gr &GitRepo) str() string {
mut parts := [
"id: $gr.id",
"url: $gr.url",
"branch: $gr.branch",
"repo: $gr.repo",
"schedule: $gr.schedule",
"arch: ${gr.arch.map(it.value).join(', ')}"
]
str := parts.join('\n')
return str
}
// patch_from_params patches a GitRepo from a map[string]string, usually // patch_from_params patches a GitRepo from a map[string]string, usually
// provided from a web.App's params // provided from a web.App's params
pub fn (mut r GitRepo) patch_from_params(params map[string]string) { pub fn (mut r GitRepo) patch_from_params(params map[string]string) {
@ -76,7 +58,7 @@ pub fn git_repo_from_params(params map[string]string) ?GitRepo {
pub fn (db &VieterDb) get_git_repos() []GitRepo { pub fn (db &VieterDb) get_git_repos() []GitRepo {
res := sql db.conn { res := sql db.conn {
select from GitRepo order by id select from GitRepo
} }
return res return res
@ -106,49 +88,28 @@ pub fn (db &VieterDb) add_git_repo(repo GitRepo) {
pub fn (db &VieterDb) delete_git_repo(repo_id int) { pub fn (db &VieterDb) delete_git_repo(repo_id int) {
sql db.conn { sql db.conn {
delete from GitRepo where id == repo_id delete from GitRepo where id == repo_id
delete from GitRepoArch where repo_id == repo_id
} }
} }
pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) { pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) {
// sql db.conn { /* sql db.conn { */
// update GitRepo set repo /* update GitRepo set repo */
//} /* } */
mut values := []string{} mut values := []string{}
$for field in GitRepo.fields { $for field in GitRepo.fields {
if field.name in params { if field.name in params {
// Any fields that are array types require their own update method
$if field.typ is string { $if field.typ is string {
values << "$field.name = '${params[field.name]}'" values << "${field.name} = '${params[field.name]}'"
// r.$(field.name) = params[field.name] /* r.$(field.name) = params[field.name] */
// This specific type check is needed for the compiler to ensure // This specific type check is needed for the compiler to ensure
// our types are correct // our types are correct
} }
//$else $if field.typ is []GitRepoArch { /* $else $if field.typ is []GitRepoArch { */
// r.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) /* r.$(field.name) = params[field.name].split(',').map(GitRepoArch{ value: it }) */
//} /* } */
}
}
values_str := values.join(', ')
query := 'update GitRepo set $values_str where id == $repo_id'
println(query)
db.conn.exec_none(query)
}
pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) {
archs_with_id := archs.map(GitRepoArch{
...it
repo_id: repo_id
})
sql db.conn {
delete from GitRepoArch where repo_id == repo_id
}
for arch in archs_with_id {
sql db.conn {
insert arch into GitRepoArch
} }
} }
} }

View File

@ -3,7 +3,6 @@ module git
import cli import cli
import env import env
import cron.expression { parse_expression } import cron.expression { parse_expression }
import db { GitRepo, GitRepoArch }
struct Config { struct Config {
address string [required] address string [required]
@ -117,13 +116,34 @@ pub fn cmd() cli.Command {
// get_repo_by_prefix tries to find the repo with the given prefix in its // get_repo_by_prefix tries to find the repo with the given prefix in its
// ID. If multiple or none are found, an error is raised. // ID. If multiple or none are found, an error is raised.
fn get_repo_by_prefix(conf Config, id_prefix string) ?(string, GitRepo) {
repos := get_repos(conf.address, conf.api_key) ?
mut res := map[string]GitRepo{}
for id, repo in repos {
if id.starts_with(id_prefix) {
res[id] = repo
}
}
if res.len == 0 {
return error('No repo found for given prefix.')
}
if res.len > 1 {
return error('Multiple repos found for given prefix.')
}
return res.keys()[0], res[res.keys()[0]]
}
// list prints out a list of all repositories. // list prints out a list of all repositories.
fn list(conf Config) ? { fn list(conf Config) ? {
repos := get_repos(conf.address, conf.api_key) ? repos := get_repos(conf.address, conf.api_key) ?
for repo in repos { for id, details in repos {
println('${repo.id}\t$repo.url\t$repo.branch\t$repo.repo') println('${id[..8]}\t$details.url\t$details.branch\t$details.repo')
} }
} }
@ -135,18 +155,15 @@ fn add(conf Config, url string, branch string, repo string) ? {
} }
// remove removes a repository from the server's list. // remove removes a repository from the server's list.
fn remove(conf Config, id string) ? { fn remove(conf Config, id_prefix string) ? {
// id, _ := get_repo_by_prefix(conf, id_prefix) ? id, _ := get_repo_by_prefix(conf, id_prefix) ?
id_int := id.int() res := remove_repo(conf.address, conf.api_key, id) ?
if id_int != 0 { println(res.message)
res := remove_repo(conf.address, conf.api_key, id_int) ?
println(res.message)
}
} }
// patch patches a given repository with the provided params. // patch patches a given repository with the provided params.
fn patch(conf Config, id string, params map[string]string) ? { fn patch(conf Config, id_prefix string, params map[string]string) ? {
// We check the cron expression first because it's useless to send an // We check the cron expression first because it's useless to send an
// invalid one to the server. // invalid one to the server.
if 'schedule' in params && params['schedule'] != '' { if 'schedule' in params && params['schedule'] != '' {
@ -155,22 +172,20 @@ fn patch(conf Config, id string, params map[string]string) ? {
} }
} }
id_int := id.int() id, _ := get_repo_by_prefix(conf, id_prefix) ?
if id_int != 0 { res := patch_repo(conf.address, conf.api_key, id, params) ?
res := patch_repo(conf.address, conf.api_key, id_int, params) ?
println(res.message) println(res.message)
}
} }
// info shows detailed information for a given repo. // info shows detailed information for a given repo.
fn info(conf Config, id string) ? { fn info(conf Config, id_prefix string) ? {
id_int := id.int() id, repo := get_repo_by_prefix(conf, id_prefix) ?
if id_int == 0 { println('id: $id')
return
$for field in GitRepo.fields {
val := repo.$(field.name)
println('$field.name: $val')
} }
repo := get_repo(conf.address, conf.api_key, id_int) ?
println(repo)
} }

View File

@ -3,7 +3,6 @@ module git
import json import json
import response { Response } import response { Response }
import net.http import net.http
import db
// send_request<T> is a convenience method for sending requests to the repos // send_request<T> is a convenience method for sending requests to the repos
// API. It mostly does string manipulation to create a query string containing // API. It mostly does string manipulation to create a query string containing
@ -27,15 +26,8 @@ fn send_request<T>(method http.Method, address string, url string, api_key strin
} }
// get_repos returns the current list of repos. // get_repos returns the current list of repos.
pub fn get_repos(address string, api_key string) ?[]db.GitRepo { pub fn get_repos(address string, api_key string) ?map[string]GitRepo {
data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', data := send_request<map[string]GitRepo>(http.Method.get, address, '/api/repos', api_key,
api_key, {}) ?
return data.data
}
pub fn get_repo(address string, api_key string, id int) ?db.GitRepo {
data := send_request<db.GitRepo>(http.Method.get, address, '/api/repos/$id', api_key,
{}) ? {}) ?
return data.data return data.data
@ -59,7 +51,7 @@ pub fn add_repo(address string, api_key string, url string, branch string, repo
} }
// remove_repo removes the repo with the given ID from the server. // remove_repo removes the repo with the given ID from the server.
pub fn remove_repo(address string, api_key string, id int) ?Response<string> { pub fn remove_repo(address string, api_key string, id string) ?Response<string> {
data := send_request<string>(http.Method.delete, address, '/api/repos/$id', api_key, data := send_request<string>(http.Method.delete, address, '/api/repos/$id', api_key,
{}) ? {}) ?
@ -68,7 +60,7 @@ pub fn remove_repo(address string, api_key string, id int) ?Response<string> {
// patch_repo sends a PATCH request to the given repo with the params as // patch_repo sends a PATCH request to the given repo with the params as
// payload. // payload.
pub fn patch_repo(address string, api_key string, id int, params map[string]string) ?Response<string> { pub fn patch_repo(address string, api_key string, id string, params map[string]string) ?Response<string> {
data := send_request<string>(http.Method.patch, address, '/api/repos/$id', api_key, data := send_request<string>(http.Method.patch, address, '/api/repos/$id', api_key,
params) ? params) ?

View File

@ -1,84 +1,84 @@
module git module git
/* import os */ import os
/* import json */ import json
/* pub struct GitRepo { */ pub struct GitRepo {
/* pub mut: */ pub mut:
/* // URL of the Git repository */ // URL of the Git repository
/* url string */ url string
/* // Branch of the Git repository to use */ // Branch of the Git repository to use
/* branch string */ branch string
/* // On which architectures the package is allowed to be built. In reality, */ // On which architectures the package is allowed to be built. In reality,
/* // this controls which builders will periodically build the image. */ // this controls which builders will periodically build the image.
/* arch []string */ arch []string
/* // Which repo the builder should publish packages to */ // Which repo the builder should publish packages to
/* repo string */ repo string
/* // Cron schedule describing how frequently to build the repo. */ // Cron schedule describing how frequently to build the repo.
/* schedule string [optional] */ schedule string [optional]
/* } */ }
/* // patch_from_params patches a GitRepo from a map[string]string, usually */ // patch_from_params patches a GitRepo from a map[string]string, usually
/* // provided from a web.App's params */ // provided from a web.App's params
/* pub fn (mut r GitRepo) patch_from_params(params map[string]string) { */ pub fn (mut r GitRepo) patch_from_params(params map[string]string) {
/* $for field in GitRepo.fields { */ $for field in GitRepo.fields {
/* if field.name in params { */ if field.name in params {
/* $if field.typ is string { */ $if field.typ is string {
/* r.$(field.name) = params[field.name] */ r.$(field.name) = params[field.name]
/* // This specific type check is needed for the compiler to ensure */ // This specific type check is needed for the compiler to ensure
/* // our types are correct */ // our types are correct
/* } $else $if field.typ is []string { */ } $else $if field.typ is []string {
/* r.$(field.name) = params[field.name].split(',') */ r.$(field.name) = params[field.name].split(',')
/* } */ }
/* } */ }
/* } */ }
/* } */ }
/* // read_repos reads the provided path & parses it into a map of GitRepo's. */ // read_repos reads the provided path & parses it into a map of GitRepo's.
/* pub fn read_repos(path string) ?map[string]GitRepo { */ pub fn read_repos(path string) ?map[string]GitRepo {
/* if !os.exists(path) { */ if !os.exists(path) {
/* mut f := os.create(path) ? */ mut f := os.create(path) ?
/* defer { */ defer {
/* f.close() */ f.close()
/* } */ }
/* f.write_string('{}') ? */ f.write_string('{}') ?
/* return {} */ return {}
/* } */ }
/* content := os.read_file(path) ? */ content := os.read_file(path) ?
/* res := json.decode(map[string]GitRepo, content) ? */ res := json.decode(map[string]GitRepo, content) ?
/* return res */ return res
/* } */ }
/* // write_repos writes a map of GitRepo's back to disk given the provided path. */ // write_repos writes a map of GitRepo's back to disk given the provided path.
/* pub fn write_repos(path string, repos &map[string]GitRepo) ? { */ pub fn write_repos(path string, repos &map[string]GitRepo) ? {
/* mut f := os.create(path) ? */ mut f := os.create(path) ?
/* defer { */ defer {
/* f.close() */ f.close()
/* } */ }
/* value := json.encode(repos) */ value := json.encode(repos)
/* f.write_string(value) ? */ f.write_string(value) ?
/* } */ }
/* // repo_from_params creates a GitRepo from a map[string]string, usually */ // repo_from_params creates a GitRepo from a map[string]string, usually
/* // provided from a web.App's params */ // provided from a web.App's params
/* pub fn repo_from_params(params map[string]string) ?GitRepo { */ pub fn repo_from_params(params map[string]string) ?GitRepo {
/* mut repo := GitRepo{} */ mut repo := GitRepo{}
/* // If we're creating a new GitRepo, we want all fields to be present before */ // If we're creating a new GitRepo, we want all fields to be present before
/* // "patching". */ // "patching".
/* $for field in GitRepo.fields { */ $for field in GitRepo.fields {
/* if field.name !in params && !field.attrs.contains('optional') { */ if field.name !in params && !field.attrs.contains('optional') {
/* return error('Missing parameter: ${field.name}.') */ return error('Missing parameter: ${field.name}.')
/* } */ }
/* } */ }
/* repo.patch_from_params(params) */ repo.patch_from_params(params)
/* return repo */ return repo
/* } */ }

View File

@ -110,60 +110,52 @@ fn (mut app App) delete_repo(id int) web.Result {
return app.json(http.Status.unauthorized, new_response('Unauthorized.')) return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
} }
// mut repos := rlock app.git_mutex { /* mut repos := rlock app.git_mutex { */
// git.read_repos(app.conf.repos_file) or { /* git.read_repos(app.conf.repos_file) or { */
// app.lerror('Failed to read repos file.') /* app.lerror('Failed to read repos file.') */
// return app.status(http.Status.internal_server_error) /* return app.status(http.Status.internal_server_error) */
// } /* } */
//} /* } */
// if id !in repos { /* if id !in repos { */
// return app.not_found() /* return app.not_found() */
//} /* } */
// repos.delete(id) /* repos.delete(id) */
app.db.delete_git_repo(id) app.db.delete_git_repo(id)
// lock app.git_mutex { /* lock app.git_mutex { */
// git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } /* git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) } */
// } /* } */
return app.json(http.Status.ok, new_response('Repo removed successfully.')) return app.json(http.Status.ok, new_response('Repo removed successfully.'))
} }
// patch_repo updates a repo's data with the given query params. // patch_repo updates a repo's data with the given query params.
['/api/repos/:id'; patch] ['/api/repos/:id'; patch]
fn (mut app App) patch_repo(id int) web.Result { fn (mut app App) patch_repo(id string) web.Result {
if !app.is_authorized() { if !app.is_authorized() {
return app.json(http.Status.unauthorized, new_response('Unauthorized.')) return app.json(http.Status.unauthorized, new_response('Unauthorized.'))
} }
app.db.update_git_repo(id, app.query) mut repos := rlock app.git_mutex {
git.read_repos(app.conf.repos_file) or {
app.lerror('Failed to read repos file.')
if 'arch' in app.query { return app.status(http.Status.internal_server_error)
arch_objs := app.query['arch'].split(',').map(db.GitRepoArch{ value: it }) }
app.db.update_git_repo_archs(id, arch_objs)
} }
// mut repos := rlock app.git_mutex { if id !in repos {
// git.read_repos(app.conf.repos_file) or { return app.not_found()
// app.lerror('Failed to read repos file.') }
// return app.status(http.Status.internal_server_error) repos[id].patch_from_params(app.query)
// }
// }
// if id !in repos { lock app.git_mutex {
// return app.not_found() git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) }
// } }
// repos[id].patch_from_params(app.query)
// lock app.git_mutex {
// git.write_repos(app.conf.repos_file, &repos) or { return app.server_error(500) }
// }
return app.json(http.Status.ok, new_response('Repo updated successfully.')) return app.json(http.Status.ok, new_response('Repo updated successfully.'))
} }