diff --git a/CHANGELOG.md b/CHANGELOG.md index c55e16b..d2dd760 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Migrated codebase to V 0.3.2 * Cron expression parser now uses bitfields instead of bool arrays -* Added option to deploy using agent-server architecture instead of cron daemon -* Allow force-building packages, meaning the build won't check if the - repository is already up to date -* Allow scheduling builds on the server from the CLI tool instead of building - them locally ### Fixed @@ -24,7 +19,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * CLI no longer exits with non-zero status code when removing/patching target * Allow NULL values for branch in database -* Endpoint for adding targets now returns the correct id ## [0.4.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.4.0) diff --git a/README.md b/README.md index 637d4c1..b9fff69 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,7 @@ quicker. I chose [V](https://vlang.io/) as I've been very intrigued by this language for a while now. I wanted a fast language that I could code while relaxing, without having to exert too much mental effort & V seemed like the right choice for -that. Sadly, this didn't quite turn out the way I expected, but I'm sticking -with it anyways ;p +that. ## Features @@ -50,7 +49,7 @@ update`. I used to maintain a mirror that tracked the latest master, but nowadays, I maintain a Docker image containing the specific compiler version that Vieter -builds with. Currently, this is V 0.3.2. +builds with. Currently, this is V 0.3. ## Contributing diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 95bf713..af941a2 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -17,7 +17,7 @@ If a variable is both present in the config file & as an environment variable, the value in the environment variable is used. {{< hint info >}} -**Note** +**Note** All environment variables can also be provided from a file by appending them with `_FILE`. This for example allows you to provide the API key from a Docker secrets file. @@ -97,25 +97,3 @@ configuration variable required for each command. build`. * Default: `archlinux:base-devel` -### `vieter agent` - -* `log_level`: log verbosity level. Value should be one of `FATAL`, `ERROR`, - `WARN`, `INFO` or `DEBUG`. - * Default: `WARN` -* `address`: *public* URL of the Vieter repository server to build for. From - this server jobs are retrieved. All built packages are published to this - server. -* `api_key`: API key of the above server. -* `data_dir`: directory to store log file in. -* `max_concurrent_builds`: how many builds to run at the same time. - * Default: `1` -* `polling_frequency`: how often (in seconds) to poll the server for new - builds. Note that the agent might poll more frequently when it's actively - processing builds. -* `image_rebuild_frequency`: Vieter periodically builds images that are then - used as a basis for running build containers. This is to prevent each build - from downloading an entire repository worth of dependencies. This setting - defines how frequently (in minutes) to rebuild these images. - * Default: `1440` (every 24 hours) -* `arch`: architecture for which this agent should pull down builds (e.g. - `x86_64`) diff --git a/docs/content/installation.md b/docs/content/installation.md index 21eda64..87b9cba 100644 --- a/docs/content/installation.md +++ b/docs/content/installation.md @@ -21,7 +21,7 @@ branch. This branch will be the most up to date, but does not give any guarantees about stability, so beware! Thanks to the single-binary design of Vieter, this image can be used both for -the repository server, the cron daemon and the agent. +the repository server & the cron daemon. Below is an example compose file to set up both the repository server & the cron daemon: @@ -76,7 +76,7 @@ architectures will build on both. ## Binary On the -[releases](https://git.rustybever.be/vieter-v/vieter/releases) +[releases](https://git.rustybever.be/vieter/vieter/releases) page, you can find statically compiled binaries for all released versions. This is the same binary as used inside the Docker images. @@ -106,5 +106,5 @@ guarantee that a compiler update won't temporarily break them. ## Building from source -The project [README](https://git.rustybever.be/vieter-v/vieter#building) -contains instructions for building Vieter from source. +The project [README](https://git.rustybever.be/vieter/vieter#building) contains +instructions for building Vieter from source. diff --git a/docs/content/usage/builds/schedule.md b/docs/content/usage/builds/schedule.md index de59e25..38f76a4 100644 --- a/docs/content/usage/builds/schedule.md +++ b/docs/content/usage/builds/schedule.md @@ -37,6 +37,6 @@ Each section can consist of as many of these parts as necessary. ## CLI tool The Vieter binary contains a command that shows you the next matching times for -a given expression. This can be useful for understanding the syntax. For more +a given expression. This can be useful to understand the syntax. For more information, see [vieter-schedule(1)](https://rustybever.be/man/vieter/vieter-schedule.1.html). diff --git a/src/agent/agent.v b/src/agent/agent.v deleted file mode 100644 index 69b9947..0000000 --- a/src/agent/agent.v +++ /dev/null @@ -1,27 +0,0 @@ -module agent - -import log -import os -import util - -const log_file_name = 'vieter.agent.log' - -// agent starts an agent service -pub fn agent(conf Config) ! { - log_level := log.level_from_tag(conf.log_level) or { - return error('Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') - } - - mut logger := log.Log{ - level: log_level - } - - os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') } - - log_file := os.join_path_single(conf.data_dir, agent.log_file_name) - logger.set_full_logpath(log_file) - logger.log_to_console_too() - - mut d := agent_init(logger, conf) - d.run() -} diff --git a/src/agent/cli.v b/src/agent/cli.v deleted file mode 100644 index 1535e17..0000000 --- a/src/agent/cli.v +++ /dev/null @@ -1,31 +0,0 @@ -module agent - -import cli -import conf as vconf - -struct Config { -pub: - log_level string = 'WARN' - // Architecture that the agent represents - arch string - api_key string - address string - data_dir string - max_concurrent_builds int = 1 - polling_frequency int = 30 - image_rebuild_frequency int = 1440 -} - -// cmd returns the cli module that handles the cron daemon. -pub fn cmd() cli.Command { - return cli.Command{ - name: 'agent' - description: 'Start an agent daemon.' - execute: fn (cmd cli.Command) ! { - config_file := cmd.flags.get_string('config-file')! - conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - - agent(conf)! - } - } -} diff --git a/src/agent/daemon.v b/src/agent/daemon.v deleted file mode 100644 index 0647733..0000000 --- a/src/agent/daemon.v +++ /dev/null @@ -1,178 +0,0 @@ -module agent - -import log -import sync.stdatomic -import build { BuildConfig } -import client -import time -import os - -const ( - build_empty = 0 - build_running = 1 - build_done = 2 -) - -struct AgentDaemon { - logger shared log.Log - conf Config - client client.Client -mut: - images ImageManager - // Which builds are currently running; length is conf.max_concurrent_builds - builds []BuildConfig - // Atomic variables used to detect when a build has finished; length is - // conf.max_concurrent_builds - atomics []u64 -} - -// agent_init initializes a new agent -fn agent_init(logger log.Log, conf Config) AgentDaemon { - mut d := AgentDaemon{ - logger: logger - client: client.new(conf.address, conf.api_key) - conf: conf - images: new_image_manager(conf.image_rebuild_frequency * 60) - builds: []BuildConfig{len: conf.max_concurrent_builds} - atomics: []u64{len: conf.max_concurrent_builds} - } - - return d -} - -// run starts the actual agent daemon. This function will run forever. -pub fn (mut d AgentDaemon) run() { - // This is just so that the very first time the loop is ran, the jobs are - // always polled - mut last_poll_time := time.now().add_seconds(-d.conf.polling_frequency) - mut sleep_time := 1 * time.second - mut finished, mut empty := 0, 0 - - for { - finished, empty = d.update_atomics() - - // No new finished builds and no free slots, so there's nothing to be - // done - if finished + empty == 0 { - time.sleep(1 * time.second) - continue - } - - // Builds have finished, so old builder images might have freed up. - // TODO this might query the docker daemon too frequently. - if finished > 0 { - d.images.clean_old_images() - } - - // The agent will always poll for new jobs after at most - // `polling_frequency` seconds. However, when jobs have finished, the - // agent will also poll for new jobs. This is because jobs are often - // clustered together (especially when mostly using the global cron - // schedule), so there's a much higher chance jobs are available. - if finished > 0 || time.now() >= last_poll_time.add_seconds(d.conf.polling_frequency) { - new_configs := d.client.poll_jobs(d.conf.arch, finished + empty) or { - d.lerror('Failed to poll jobs: $err.msg()') - - // TODO pick a better delay here - time.sleep(5 * time.second) - continue - } - last_poll_time = time.now() - - for config in new_configs { - // TODO handle this better than to just skip the config - // Make sure a recent build base image is available for - // building the config - d.images.refresh_image(config.base_image) or { - d.lerror(err.msg()) - continue - } - d.start_build(config) - } - - // No new jobs were scheduled and the agent isn't doing anything, - // so we just wait until the next polling period. - if new_configs.len == 0 && finished + empty == d.conf.max_concurrent_builds { - sleep_time = time.now() - last_poll_time - } - } - // The agent is not doing anything, so we just wait until the next poll - // time - else if finished + empty == d.conf.max_concurrent_builds { - sleep_time = time.now() - last_poll_time - } - - time.sleep(sleep_time) - } -} - -// update_atomics checks for each build whether it's completed, and sets it to -// empty again if so. The return value is a tuple `(finished, empty)` where -// `finished` is how many builds were just finished and thus set to empty, and -// `empty` is how many build slots were already empty. The amount of running -// builds can then be calculated by substracting these two values from the -// total allowed concurrent builds. -fn (mut d AgentDaemon) update_atomics() (int, int) { - mut finished := 0 - mut empty := 0 - - for i in 0 .. d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == agent.build_done { - stdatomic.store_u64(&d.atomics[i], agent.build_empty) - finished++ - } else if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { - empty++ - } - } - - return finished, empty -} - -// start_build starts a build for the given BuildConfig. -fn (mut d AgentDaemon) start_build(config BuildConfig) bool { - for i in 0 .. d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == agent.build_empty { - stdatomic.store_u64(&d.atomics[i], agent.build_running) - d.builds[i] = config - - go d.run_build(i, config) - - return true - } - } - - return false -} - -// run_build actually starts the build process for a given target. -fn (mut d AgentDaemon) run_build(build_index int, config BuildConfig) { - d.linfo('started build: $config') - - // 0 means success, 1 means failure - mut status := 0 - - new_config := BuildConfig{ - ...config - base_image: d.images.get(config.base_image) - } - - res := build.build_config(d.client.address, d.client.api_key, new_config) or { - d.ldebug('build_config error: $err.msg()') - status = 1 - - build.BuildResult{} - } - - if status == 0 { - d.linfo('Uploading build logs for $config') - - // TODO use the arch value here - build_arch := os.uname().machine - d.client.add_build_log(config.target_id, res.start_time, res.end_time, build_arch, - res.exit_code, res.logs) or { d.lerror('Failed to upload logs for $config') } - } else { - d.lwarn('an error occurred during build: $config') - } - - stdatomic.store_u64(&d.atomics[build_index], agent.build_done) -} diff --git a/src/agent/images.v b/src/agent/images.v deleted file mode 100644 index 185192e..0000000 --- a/src/agent/images.v +++ /dev/null @@ -1,79 +0,0 @@ -module agent - -import time -import docker -import build - -// An ImageManager is a utility that creates builder images from given base -// images, updating these builder images if they've become too old. This -// structure can manage images from any number of base images, paving the way -// for configurable base images per target/repository. -struct ImageManager { - max_image_age int [required] -mut: - // For each base image, one or more builder images can exist at the same - // time - images map[string][]string [required] - // For each base image, we track when its newest image was built - timestamps map[string]time.Time [required] -} - -// new_image_manager initializes a new image manager. -fn new_image_manager(max_image_age int) ImageManager { - return ImageManager{ - max_image_age: max_image_age - images: map[string][]string{} - timestamps: map[string]time.Time{} - } -} - -// get returns the name of the newest image for the given base image. Note that -// this function should only be called *after* a first call to `refresh_image`. -pub fn (m &ImageManager) get(base_image string) string { - return m.images[base_image].last() -} - -// refresh_image builds a new builder image from the given base image if the -// previous builder image is too old or non-existent. This function will do -// nothing if these conditions aren't met, so it's safe to call it every time -// you want to ensure an image is up to date. -fn (mut m ImageManager) refresh_image(base_image string) ! { - if base_image in m.timestamps - && m.timestamps[base_image].add_seconds(m.max_image_age) > time.now() { - return - } - - // TODO use better image tags for built images - new_image := build.create_build_image(base_image) or { - return error('Failed to build builder image from base image $base_image') - } - - m.images[base_image] << new_image - m.timestamps[base_image] = time.now() -} - -// clean_old_images removes all older builder images that are no longer in use. -// The function will always leave at least one builder image, namely the newest -// one. -fn (mut m ImageManager) clean_old_images() { - mut dd := docker.new_conn() or { return } - - defer { - dd.close() or {} - } - - mut i := 0 - - for image in m.images.keys() { - i = 0 - - for i < m.images[image].len - 1 { - // For each builder image, we try to remove it by calling the Docker - // API. If the function returns an error or false, that means the image - // wasn't deleted. Therefore, we move the index over. If the function - // returns true, the array's length has decreased by one so we don't - // move the index. - dd.remove_image(m.images[image][i]) or { i += 1 } - } - } -} diff --git a/src/agent/log.v b/src/agent/log.v deleted file mode 100644 index cd59207..0000000 --- a/src/agent/log.v +++ /dev/null @@ -1,35 +0,0 @@ -module agent - -import log - -// log a message with the given level -pub fn (mut d AgentDaemon) log(msg string, level log.Level) { - lock d.logger { - d.logger.send_output(msg, level) - } -} - -// lfatal create a log message with the fatal level -pub fn (mut d AgentDaemon) lfatal(msg string) { - d.log(msg, log.Level.fatal) -} - -// lerror create a log message with the error level -pub fn (mut d AgentDaemon) lerror(msg string) { - d.log(msg, log.Level.error) -} - -// lwarn create a log message with the warn level -pub fn (mut d AgentDaemon) lwarn(msg string) { - d.log(msg, log.Level.warn) -} - -// linfo create a log message with the info level -pub fn (mut d AgentDaemon) linfo(msg string) { - d.log(msg, log.Level.info) -} - -// ldebug create a log message with the debug level -pub fn (mut d AgentDaemon) ldebug(msg string) { - d.log(msg, log.Level.debug) -} diff --git a/src/build/build.v b/src/build/build.v index 3d916bf..247df6e 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -16,22 +16,6 @@ const ( '/usr/local/bin', '/usr/bin/site_perl', '/usr/bin/vendor_perl', '/usr/bin/core_perl'] ) -pub struct BuildConfig { -pub: - target_id int - kind string - url string - branch string - repo string - base_image string - force bool -} - -// str return a single-line string representation of a build log -pub fn (c BuildConfig) str() string { - return '{ target: $c.target_id, kind: $c.kind, url: $c.url, branch: $c.branch, repo: $c.repo, base_image: $c.base_image, force: $c.force }' -} - // create_build_image creates a builder image given some base image which can // then be used to build & package Arch images. It mostly just updates the // system, install some necessary packages & creates a non-root user to run @@ -109,25 +93,10 @@ pub: logs string } -// build_target builds the given target. Internally it calls `build_config`. -pub fn build_target(address string, api_key string, base_image_id string, target &Target, force bool) !BuildResult { - config := BuildConfig{ - target_id: target.id - kind: target.kind - url: target.url - branch: target.branch - repo: target.repo - base_image: base_image_id - force: force - } - - return build_config(address, api_key, config) -} - -// build_config builds, packages & publishes a given Arch package based on the +// build_target builds, packages & publishes a given Arch package based on the // provided target. The base image ID should be of an image previously created // by create_build_image. It returns the logs of the container. -pub fn build_config(address string, api_key string, config BuildConfig) !BuildResult { +pub fn build_target(address string, api_key string, base_image_id string, target &Target) !BuildResult { mut dd := docker.new_conn()! defer { @@ -135,14 +104,14 @@ pub fn build_config(address string, api_key string, config BuildConfig) !BuildRe } build_arch := os.uname().machine - build_script := create_build_script(address, config, build_arch) + build_script := create_build_script(address, target, build_arch) // We convert the build script into a base64 string, which then gets passed // to the container as an env var base64_script := base64.encode_str(build_script) c := docker.NewContainer{ - image: '$config.base_image' + image: '$base_image_id' env: [ 'BUILD_SCRIPT=$base64_script', 'API_KEY=$api_key', diff --git a/src/build/queue.v b/src/build/queue.v deleted file mode 100644 index 7902173..0000000 --- a/src/build/queue.v +++ /dev/null @@ -1,229 +0,0 @@ -module build - -import models { Target } -import cron.expression { CronExpression, parse_expression } -import time -import datatypes { MinHeap } -import util - -struct BuildJob { -pub mut: - // Time at which this build job was created/queued - created time.Time - // Next timestamp from which point this job is allowed to be executed - timestamp time.Time - // Required for calculating next timestamp after having pop'ed a job - ce CronExpression - // Actual build config sent to the agent - config BuildConfig - // Whether this is a one-time job - single bool -} - -// Allows BuildJob structs to be sorted according to their timestamp in -// MinHeaps -fn (r1 BuildJob) < (r2 BuildJob) bool { - return r1.timestamp < r2.timestamp -} - -// The build job queue is responsible for managing the list of scheduled builds -// for each architecture. Agents receive jobs from this queue. -pub struct BuildJobQueue { - // Schedule to use for targets without explicitely defined cron expression - default_schedule CronExpression - // Base image to use for targets without defined base image - default_base_image string -mut: - mutex shared util.Dummy - // For each architecture, a priority queue is tracked - queues map[string]MinHeap - // When a target is removed from the server or edited, its previous build - // configs will be invalid. This map allows for those to be simply skipped - // by ignoring any build configs created before this timestamp. - invalidated map[int]time.Time -} - -// new_job_queue initializes a new job queue -pub fn new_job_queue(default_schedule CronExpression, default_base_image string) BuildJobQueue { - return BuildJobQueue{ - default_schedule: default_schedule - default_base_image: default_base_image - invalidated: map[int]time.Time{} - } -} - -// insert_all executes insert for each architecture of the given Target. -pub fn (mut q BuildJobQueue) insert_all(target Target) ! { - for arch in target.arch { - q.insert(target: target, arch: arch.value)! - } -} - -[params] -pub struct InsertConfig { - target Target [required] - arch string [required] - single bool - force bool - now bool -} - -// insert a new target's job into the queue for the given architecture. This -// job will then be endlessly rescheduled after being pop'ed, unless removed -// explicitely. -pub fn (mut q BuildJobQueue) insert(input InsertConfig) ! { - lock q.mutex { - if input.arch !in q.queues { - q.queues[input.arch] = MinHeap{} - } - - mut job := BuildJob{ - created: time.now() - single: input.single - config: BuildConfig{ - target_id: input.target.id - kind: input.target.kind - url: input.target.url - branch: input.target.branch - repo: input.target.repo - // TODO make this configurable - base_image: q.default_base_image - force: input.force - } - } - - if !input.now { - ce := if input.target.schedule != '' { - parse_expression(input.target.schedule) or { - return error("Error while parsing cron expression '$input.target.schedule' (id $input.target.id): $err.msg()") - } - } else { - q.default_schedule - } - - job.timestamp = ce.next_from_now()! - job.ce = ce - } else { - job.timestamp = time.now() - } - - q.queues[input.arch].insert(job) - } -} - -// reschedule the given job by calculating the next timestamp and re-adding it -// to its respective queue. This function is called by the pop functions -// *after* having pop'ed the job. -fn (mut q BuildJobQueue) reschedule(job BuildJob, arch string) ! { - new_timestamp := job.ce.next_from_now()! - - new_job := BuildJob{ - ...job - created: time.now() - timestamp: new_timestamp - } - - q.queues[arch].insert(new_job) -} - -// pop_invalid pops all invalid jobs. -fn (mut q BuildJobQueue) pop_invalid(arch string) { - for { - job := q.queues[arch].peek() or { return } - - if job.config.target_id in q.invalidated - && job.created < q.invalidated[job.config.target_id] { - // This pop *should* never fail according to the source code - q.queues[arch].pop() or {} - } else { - break - } - } -} - -// peek shows the first job for the given architecture that's ready to be -// executed, if present. -pub fn (mut q BuildJobQueue) peek(arch string) ?BuildJob { - // Even peek requires a write lock, because pop_invalid can modify the data - // structure - lock q.mutex { - if arch !in q.queues { - return none - } - - q.pop_invalid(arch) - job := q.queues[arch].peek()? - - if job.timestamp < time.now() { - return job - } - } - - return none -} - -// pop removes the first job for the given architecture that's ready to be -// executed from the queue and returns it, if present. -pub fn (mut q BuildJobQueue) pop(arch string) ?BuildJob { - lock q.mutex { - if arch !in q.queues { - return none - } - - q.pop_invalid(arch) - mut job := q.queues[arch].peek()? - - if job.timestamp < time.now() { - job = q.queues[arch].pop()? - - if !job.single { - // TODO how do we handle this properly? Is it even possible for a - // cron expression to not return a next time if it's already been - // used before? - q.reschedule(job, arch) or {} - } - - return job - } - } - - return none -} - -// pop_n tries to pop at most n available jobs for the given architecture. -pub fn (mut q BuildJobQueue) pop_n(arch string, n int) []BuildJob { - lock q.mutex { - if arch !in q.queues { - return [] - } - - mut out := []BuildJob{} - - for out.len < n { - q.pop_invalid(arch) - mut job := q.queues[arch].peek() or { break } - - if job.timestamp < time.now() { - job = q.queues[arch].pop() or { break } - - if !job.single { - // TODO idem - q.reschedule(job, arch) or {} - } - - out << job - } else { - break - } - } - - return out - } - - return [] -} - -// invalidate a target's old build jobs. -pub fn (mut q BuildJobQueue) invalidate(target_id int) { - q.invalidated[target_id] = time.now() -} diff --git a/src/build/shell.v b/src/build/shell.v index ac61e07..e573d53 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -1,5 +1,7 @@ module build +import models { Target } + // escape_shell_string escapes any characters that could be interpreted // incorrectly by a shell. The resulting value should be safe to use inside an // echo statement. @@ -21,13 +23,13 @@ pub fn echo_commands(cmds []string) []string { } // create_build_script generates a shell script that builds a given Target. -fn create_build_script(address string, config BuildConfig, build_arch string) string { - repo_url := '$address/$config.repo' +fn create_build_script(address string, target &Target, build_arch string) string { + repo_url := '$address/$target.repo' mut commands := [ // This will later be replaced by a proper setting for changing the // mirrorlist - "echo -e '[$config.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" + "echo -e '[$target.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" // We need to update the package list of the repo we just added above. // This should however not pull in a lot of packages as long as the // builder image is rebuilt frequently. @@ -36,22 +38,22 @@ fn create_build_script(address string, config BuildConfig, build_arch string) st 'su builder', ] - commands << match config.kind { + commands << match target.kind { 'git' { - if config.branch == '' { + if target.branch == '' { [ - "git clone --single-branch --depth 1 '$config.url' repo", + "git clone --single-branch --depth 1 '$target.url' repo", ] } else { [ - "git clone --single-branch --depth 1 --branch $config.branch '$config.url' repo", + "git clone --single-branch --depth 1 --branch $target.branch '$target.url' repo", ] } } 'url' { [ 'mkdir repo', - "curl -o repo/PKGBUILD -L '$config.url'", + "curl -o repo/PKGBUILD -L '$target.url'", ] } else { @@ -63,22 +65,14 @@ fn create_build_script(address string, config BuildConfig, build_arch string) st 'cd repo', 'makepkg --nobuild --syncdeps --needed --noconfirm', 'source PKGBUILD', - ] - - if !config.force { // The build container checks whether the package is already present on // the server. - commands << [ - 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', - // If the above curl command succeeds, we don't need to rebuild the - // package. However, because we're in a su shell, the exit command will - // drop us back into the root shell. Therefore, we must check whether - // we're in root so we don't proceed. - '[ "\$(id -u)" == 0 ] && exit 0', - ] - } - - commands << [ + 'curl -s --head --fail $repo_url/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + // If the above curl command succeeds, we don't need to rebuild the + // package. However, because we're in a su shell, the exit command will + // drop us back into the root shell. Therefore, we must check whether + // we're in root so we don't proceed. + '[ "\$(id -u)" == 0 ] && exit 0', 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', ] diff --git a/src/build/shell_test.v b/src/build/shell_test.v index 8bb22d9..341df88 100644 --- a/src/build/shell_test.v +++ b/src/build/shell_test.v @@ -1,46 +1,42 @@ module build +import models { Target } + fn test_create_build_script_git_branch() { - config := BuildConfig{ - target_id: 1 + target := Target{ + id: 1 kind: 'git' url: 'https://examplerepo.com' branch: 'main' repo: 'vieter' - base_image: 'not-used:latest' } - - build_script := create_build_script('https://example.com', config, 'x86_64') + build_script := create_build_script('https://example.com', target, 'x86_64') expected := $embed_file('build_script_git_branch.sh') assert build_script == expected.to_string().trim_space() } fn test_create_build_script_git() { - config := BuildConfig{ - target_id: 1 + target := Target{ + id: 1 kind: 'git' url: 'https://examplerepo.com' repo: 'vieter' - base_image: 'not-used:latest' } - - build_script := create_build_script('https://example.com', config, 'x86_64') + build_script := create_build_script('https://example.com', target, 'x86_64') expected := $embed_file('build_script_git.sh') assert build_script == expected.to_string().trim_space() } fn test_create_build_script_url() { - config := BuildConfig{ - target_id: 1 + target := Target{ + id: 1 kind: 'url' url: 'https://examplerepo.com' repo: 'vieter' - base_image: 'not-used:latest' } - - build_script := create_build_script('https://example.com', config, 'x86_64') + build_script := create_build_script('https://example.com', target, 'x86_64') expected := $embed_file('build_script_url.sh') assert build_script == expected.to_string().trim_space() diff --git a/src/client/jobs.v b/src/client/jobs.v deleted file mode 100644 index 440affa..0000000 --- a/src/client/jobs.v +++ /dev/null @@ -1,26 +0,0 @@ -module client - -import build { BuildConfig } -import web.response { Response } - -// poll_jobs requests a list of new build jobs from the server. -pub fn (c &Client) poll_jobs(arch string, max int) ![]BuildConfig { - data := c.send_request<[]BuildConfig>(.get, '/api/v1/jobs/poll', { - 'arch': arch - 'max': max.str() - })! - - return data.data -} - -// queue_job adds a new one-time build job for the given target to the job -// queue. -pub fn (c &Client) queue_job(target_id int, arch string, force bool) !Response { - data := c.send_request(.post, '/api/v1/jobs/queue', { - 'target': target_id.str() - 'arch': arch - 'force': force.str() - })! - - return data -} diff --git a/src/console/targets/build.v b/src/console/targets/build.v index e18077d..9368558 100644 --- a/src/console/targets/build.v +++ b/src/console/targets/build.v @@ -6,7 +6,7 @@ import os import build // build locally builds the target with the given id. -fn build(conf Config, target_id int, force bool) ! { +fn build(conf Config, target_id int) ! { c := client.new(conf.address, conf.api_key) target := c.get_target(target_id)! @@ -16,7 +16,7 @@ fn build(conf Config, target_id int, force bool) ! { image_id := build.create_build_image(conf.base_image)! println('Running build...') - res := build.build_target(conf.address, conf.api_key, image_id, target, force)! + res := build.build_target(conf.address, conf.api_key, image_id, target)! println('Removing build image...') diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index b527896..4179363 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -182,44 +182,11 @@ pub fn cmd() cli.Command { required_args: 1 usage: 'id' description: 'Build the target with the given id & publish it.' - flags: [ - cli.Flag{ - name: 'force' - description: 'Build the target without checking whether it needs to be renewed.' - flag: cli.FlagType.bool - }, - cli.Flag{ - name: 'remote' - description: 'Schedule the build on the server instead of running it locally.' - flag: cli.FlagType.bool - }, - cli.Flag{ - name: 'arch' - description: 'Architecture to schedule build for. Required when using -remote.' - flag: cli.FlagType.string - }, - ] execute: fn (cmd cli.Command) ! { config_file := cmd.flags.get_string('config-file')! conf := vconf.load(prefix: 'VIETER_', default_path: config_file)! - remote := cmd.flags.get_bool('remote')! - force := cmd.flags.get_bool('force')! - target_id := cmd.args[0].int() - - if remote { - arch := cmd.flags.get_string('arch')! - - if arch == '' { - return error('When scheduling the build remotely, you have to specify an architecture.') - } - - c := client.new(conf.address, conf.api_key) - res := c.queue_job(target_id, arch, force)! - println(res.message) - } else { - build(conf, target_id, force)! - } + build(conf, cmd.args[0].int())! } }, ] diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index 42edc92..beed9fc 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -79,7 +79,7 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { mut status := 0 res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(), - &sb.target, false) or { + &sb.target) or { d.ldebug('build_target error: $err.msg()') status = 1 diff --git a/src/db/logs.v b/src/db/logs.v index 2745467..923dde2 100644 --- a/src/db/logs.v +++ b/src/db/logs.v @@ -84,8 +84,6 @@ pub fn (db &VieterDb) add_build_log(log BuildLog) int { insert log into BuildLog } - // Here, this does work because a log doesn't contain any foreign keys, - // meaning the ORM only has to do a single add inserted_id := db.conn.last_id() as int return inserted_id diff --git a/src/db/targets.v b/src/db/targets.v index 41e56df..a705ebb 100644 --- a/src/db/targets.v +++ b/src/db/targets.v @@ -38,17 +38,14 @@ pub fn (db &VieterDb) get_target(target_id int) ?Target { } // add_target inserts the given target into the database. -pub fn (db &VieterDb) add_target(target Target) int { +pub fn (db &VieterDb) add_target(repo Target) int { sql db.conn { - insert target into Target + insert repo into Target } - // ID of inserted target is the largest id - inserted_target := sql db.conn { - select from Target order by id desc limit 1 - } + inserted_id := db.conn.last_id() as int - return inserted_target.id + return inserted_id } // delete_target deletes the target with the given id from the database. diff --git a/src/main.v b/src/main.v index 34387bf..fc09f7e 100644 --- a/src/main.v +++ b/src/main.v @@ -9,7 +9,6 @@ import console.schedule import console.man import console.aur import cron -import agent fn main() { mut app := cli.Command{ @@ -41,7 +40,6 @@ fn main() { schedule.cmd(), man.cmd(), aur.cmd(), - agent.cmd(), ] } app.setup() diff --git a/src/server/api_jobs.v b/src/server/api_jobs.v deleted file mode 100644 index 7795351..0000000 --- a/src/server/api_jobs.v +++ /dev/null @@ -1,49 +0,0 @@ -module server - -import web -import web.response { new_data_response, new_response } - -// v1_poll_job_queue allows agents to poll for new build jobs. -['/api/v1/jobs/poll'; auth; get] -fn (mut app App) v1_poll_job_queue() web.Result { - arch := app.query['arch'] or { - return app.json(.bad_request, new_response('Missing arch query arg.')) - } - - max_str := app.query['max'] or { - return app.json(.bad_request, new_response('Missing max query arg.')) - } - max := max_str.int() - - mut out := app.job_queue.pop_n(arch, max).map(it.config) - - return app.json(.ok, new_data_response(out)) -} - -// v1_queue_job allows queueing a new one-time build job for the given target. -['/api/v1/jobs/queue'; auth; post] -fn (mut app App) v1_queue_job() web.Result { - target_id := app.query['target'] or { - return app.json(.bad_request, new_response('Missing target query arg.')) - }.int() - - arch := app.query['arch'] or { - return app.json(.bad_request, new_response('Missing arch query arg.')) - } - - if arch == '' { - app.json(.bad_request, new_response('Empty arch query arg.')) - } - - force := 'force' in app.query - - target := app.db.get_target(target_id) or { - return app.json(.bad_request, new_response('Unknown target id.')) - } - - app.job_queue.insert(target: target, arch: arch, single: true, now: true, force: force) or { - return app.status(.internal_server_error) - } - - return app.status(.ok) -} diff --git a/src/server/api_targets.v b/src/server/api_targets.v index dc39d37..16db7e9 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -12,17 +12,17 @@ fn (mut app App) v1_get_targets() web.Result { filter := models.from_params(app.query) or { return app.json(http.Status.bad_request, new_response('Invalid query parameters.')) } - targets := app.db.get_targets(filter) + repos := app.db.get_targets(filter) - return app.json(.ok, new_data_response(targets)) + return app.json(.ok, new_data_response(repos)) } // v1_get_single_target returns the information for a single target. ['/api/v1/targets/:id'; auth; get] fn (mut app App) v1_get_single_target(id int) web.Result { - target := app.db.get_target(id) or { return app.not_found() } + repo := app.db.get_target(id) or { return app.not_found() } - return app.json(.ok, new_data_response(target)) + return app.json(.ok, new_data_response(repo)) } // v1_post_target creates a new target from the provided query string. @@ -30,27 +30,22 @@ fn (mut app App) v1_get_single_target(id int) web.Result { fn (mut app App) v1_post_target() web.Result { mut params := app.query.clone() - // If a target is created without specifying the arch, we assume it's meant + // If a repo is created without specifying the arch, we assume it's meant // for the default architecture. if 'arch' !in params || params['arch'] == '' { params['arch'] = app.conf.default_arch } - mut new_target := models.from_params(params) or { + new_repo := models.from_params(params) or { return app.json(http.Status.bad_request, new_response(err.msg())) } // Ensure someone doesn't submit an invalid kind - if new_target.kind !in models.valid_kinds { + if new_repo.kind !in models.valid_kinds { return app.json(http.Status.bad_request, new_response('Invalid kind.')) } - id := app.db.add_target(new_target) - new_target.id = id - - // Add the target to the job queue - // TODO return better error here if it's the cron schedule that's incorrect - app.job_queue.insert_all(new_target) or { return app.status(.internal_server_error) } + id := app.db.add_target(new_repo) return app.json(.ok, new_data_response(id)) } @@ -59,7 +54,6 @@ fn (mut app App) v1_post_target() web.Result { ['/api/v1/targets/:id'; auth; delete] fn (mut app App) v1_delete_target(id int) web.Result { app.db.delete_target(id) - app.job_queue.invalidate(id) return app.json(.ok, new_response('')) } @@ -75,10 +69,7 @@ fn (mut app App) v1_patch_target(id int) web.Result { app.db.update_target_archs(id, arch_objs) } - target := app.db.get_target(id) or { return app.status(.internal_server_error) } + repo := app.db.get_target(id) or { return app.status(.internal_server_error) } - app.job_queue.invalidate(id) - app.job_queue.insert_all(target) or { return app.status(.internal_server_error) } - - return app.json(.ok, new_data_response(target)) + return app.json(.ok, new_data_response(repo)) } diff --git a/src/server/cli.v b/src/server/cli.v index 2fede6c..a9644f3 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -5,14 +5,12 @@ import conf as vconf struct Config { pub: - log_level string = 'WARN' - pkg_dir string - data_dir string - api_key string - default_arch string - global_schedule string = '0 3' - port int = 8000 - base_image string = 'archlinux:base-devel' + log_level string = 'WARN' + pkg_dir string + data_dir string + api_key string + default_arch string + port int = 8000 } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/server.v b/src/server/server.v index 74b1f37..d5f6135 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -6,8 +6,6 @@ import log import repo import util import db -import build { BuildJobQueue } -import cron.expression const ( log_file_name = 'vieter.log' @@ -22,26 +20,7 @@ pub: conf Config [required; web_global] pub mut: repo repo.RepoGroupManager [required; web_global] - // Keys are the various architectures for packages - job_queue BuildJobQueue [required; web_global] - db db.VieterDb -} - -// init_job_queue populates a fresh job queue with all the targets currently -// stored in the database. -fn (mut app App) init_job_queue() ! { - // Initialize build queues - mut targets := app.db.get_targets(limit: 25) - mut i := u64(0) - - for targets.len > 0 { - for target in targets { - app.job_queue.insert_all(target)! - } - - i += 25 - targets = app.db.get_targets(limit: 25, offset: i) - } + db db.VieterDb } // server starts the web server & starts listening for requests @@ -51,10 +30,6 @@ pub fn server(conf Config) ! { util.exit_with_message(1, "'any' is not allowed as the value for default_arch.") } - global_ce := expression.parse_expression(conf.global_schedule) or { - util.exit_with_message(1, 'Invalid global cron expression: $err.msg()') - } - // Configure logger log_level := log.level_from_tag(conf.log_level) or { util.exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') @@ -96,17 +71,11 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Failed to initialize database: $err.msg()') } - mut app := &App{ + web.run(&App{ logger: logger api_key: conf.api_key conf: conf repo: repo db: db - job_queue: build.new_job_queue(global_ce, conf.base_image) - } - app.init_job_queue() or { - util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()') - } - - web.run(app, conf.port) + }, conf.port) } diff --git a/vieter.toml b/vieter.toml index 74a7397..d3922a4 100644 --- a/vieter.toml +++ b/vieter.toml @@ -4,11 +4,11 @@ data_dir = "data" pkg_dir = "data/pkgs" log_level = "DEBUG" default_arch = "x86_64" -arch = "x86_64" address = "http://localhost:8000" -# global_schedule = '* *' +global_schedule = '* *' api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 +