diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index 6b8f8f2f..742095c0 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -23,10 +23,8 @@ pipeline: - su builder # Due to a bug with the V compiler, we can't just use the PKGBUILD from # inside the repo - - curl -OL https://git.rustybever.be/vieter/vieter/raw/branch/dev/PKGBUILD + - curl -OL https://git.rustybever.be/Chewing_Bever/vieter/raw/branch/dev/PKGBUILD - makepkg -s --noconfirm --needed - when: - event: push publish: image: 'curlimages/curl' @@ -35,5 +33,3 @@ pipeline: - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done' secrets: - vieter_api_key - when: - event: push diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 16981293..c89348b1 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -1,7 +1,7 @@ matrix: PLATFORM: - - 'linux/amd64' - - 'linux/arm64' + - linux/amd64 + - linux/arm64 platform: ${PLATFORM} @@ -12,9 +12,9 @@ pipeline: commands: - make when: - event: [pull_request] + event: push branch: - exclude: [main] + exclude: [main, dev] prod: image: 'chewingbever/vlang:latest' @@ -32,7 +32,7 @@ pipeline: - strip -s pvieter - du -h pvieter when: - event: [push, pull_request] + event: push upload: image: 'chewingbever/vlang:latest' @@ -49,7 +49,6 @@ pipeline: - > curl --silent - --fail -XPUT -T pvieter -H "Host: $URL" @@ -58,4 +57,4 @@ pipeline: -H "Authorization: AWS $S3_USERNAME:$SIGNATURE" https://$URL$OBJ_PATH when: - event: [push, pull_request] + event: push diff --git a/.woodpecker/.build_experimental.yml b/.woodpecker/.build_experimental.yml new file mode 100644 index 00000000..0129d2b4 --- /dev/null +++ b/.woodpecker/.build_experimental.yml @@ -0,0 +1,29 @@ +# These builds are not important for the project, but might be valuable for +# fixing bugs in the V compiler. + +platform: linux/amd64 +branches: + exclude: [master, dev] + +pipeline: + autofree: + image: 'chewingbever/vlang:latest' + pull: true + group: 'build' + commands: + - make autofree + - readelf -d afvieter + - du -h afvieter + when: + event: push + + skip-unused: + image: 'chewingbever/vlang:latest' + pull: true + group: 'build' + commands: + - make skip-unused + - readelf -d suvieter + - du -h suvieter + when: + event: push diff --git a/.woodpecker/.deploy.yml b/.woodpecker/.deploy.yml index 8e68641e..dd77fb98 100644 --- a/.woodpecker/.deploy.yml +++ b/.woodpecker/.deploy.yml @@ -1,4 +1,4 @@ -branches: [ 'dev' ] +branches: 'dev' platform: 'linux/amd64' depends_on: - 'docker' @@ -14,5 +14,3 @@ pipeline: commands: - 'curl -XPOST -s --fail $WEBHOOK_APP' - 'curl -XPOST -s --fail $WEBHOOK_CRON' - when: - event: push diff --git a/.woodpecker/.docker.yml b/.woodpecker/.docker.yml index f31490a1..9b605f3f 100644 --- a/.woodpecker/.docker.yml +++ b/.woodpecker/.docker.yml @@ -1,36 +1,30 @@ branches: [main, dev] -platform: 'linux/amd64' +platform: linux/amd64 depends_on: - build pipeline: dev: - image: 'woodpeckerci/plugin-docker-buildx' - secrets: - - 'docker_username' - - 'docker_password' + image: woodpeckerci/plugin-docker-buildx + secrets: [ docker_username, docker_password ] settings: - repo: 'chewingbever/vieter' - tags: - - 'dev' - - ${CI_COMMIT_SHA} - platforms: [ 'linux/arm64/v8', 'linux/amd64' ] + repo: chewingbever/vieter + tag: dev + platforms: [ linux/arm64/v8, linux/amd64 ] build_args_from_env: - - 'CI_COMMIT_SHA' + - CI_COMMIT_SHA when: event: push branch: dev release: - image: 'woodpeckerci/plugin-docker-buildx' - secrets: - - 'docker_username' - - 'docker_password' + image: woodpeckerci/plugin-docker-buildx + secrets: [ docker_username, docker_password ] settings: - repo: 'chewingbever/vieter' + repo: chewingbever/vieter auto_tag: true - platforms: [ 'linux/arm64/v8', 'linux/amd64' ] + platforms: [ linux/arm64/v8, linux/amd64 ] build_args_from_env: - - 'CI_COMMIT_SHA' + - CI_COMMIT_SHA when: event: tag diff --git a/.woodpecker/.gitea.yml b/.woodpecker/.gitea.yml index 18770502..c492d345 100644 --- a/.woodpecker/.gitea.yml +++ b/.woodpecker/.gitea.yml @@ -1,5 +1,6 @@ -platform: 'linux/amd64' -branches: [ 'main' ] +# Yeah so this only works on tags so we'll worry about this later +platform: linux/amd64 +branches: main depends_on: - build diff --git a/.woodpecker/.lint.yml b/.woodpecker/.lint.yml index e70648d8..b1c16fd8 100644 --- a/.woodpecker/.lint.yml +++ b/.woodpecker/.lint.yml @@ -1,7 +1,7 @@ # These checks already get performed on the feature branches branches: - exclude: [ main ] -platform: 'linux/amd64' + exclude: [ main, dev ] +platform: linux/amd64 pipeline: lint: @@ -9,5 +9,3 @@ pipeline: pull: true commands: - make lint - when: - event: [ pull_request ] diff --git a/.woodpecker/.test.yml b/.woodpecker/.test.yml index 6b7b646d..3800cc1d 100644 --- a/.woodpecker/.test.yml +++ b/.woodpecker/.test.yml @@ -1,10 +1,10 @@ matrix: PLATFORM: - - 'linux/amd64' - - 'linux/arm64' + - linux/amd64 + - linux/arm64 branches: - exclude: [ main ] + exclude: [main, dev] platform: ${PLATFORM} pipeline: @@ -14,4 +14,4 @@ pipeline: commands: - make test when: - event: [pull_request] + event: push diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d9eb4d6..f40ce526 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,14 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev) - -### Added - -* Web API for adding & querying build logs -* CLI commands to access build logs API - -## [0.3.0-alpha.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.1) +## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter) ### Changed @@ -31,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Binary no longer panics when an env var is missing -## [0.2.0](https://git.rustybever.be/vieter/vieter/src/tag/0.2.0) +## [0.2.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.2.0) ### Changed @@ -65,13 +58,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Packages with unknown fields in .PKGINFO are now allowed * Old packages are now properly removed -## [0.1.0](https://git.rustybever.be/vieter/vieter/src/tag/0.1.0) +## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0) ### Changed * Improved logging -## [0.1.0-rc.1](https://git.rustybever.be/vieter/vieter/src/tag/0.1.0-rc.1) +## [0.1.0-rc.1](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0-rc.1) ### Added diff --git a/PKGBUILD b/PKGBUILD index 87c575ff..eb866834 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -4,12 +4,12 @@ pkgbase='vieter' pkgname='vieter' pkgver=0.2.0.r25.g20112b8 pkgrel=1 -depends=('glibc' 'openssl' 'libarchive' 'gc' 'sqlite') +depends=('glibc' 'openssl' 'libarchive' 'gc') makedepends=('git' 'gcc' 'vieter-v') -arch=('x86_64' 'aarch64') -url='https://git.rustybever.be/vieter/vieter' +arch=('x86_64' 'aarch64' 'armv7') +url='https://git.rustybever.be/Chewing_Bever/vieter' license=('AGPL3') -source=($pkgname::git+https://git.rustybever.be/vieter/vieter#branch=dev) +source=($pkgname::git+https://git.rustybever.be/Chewing_Bever/vieter#branch=dev) md5sums=('SKIP') pkgver() { diff --git a/src/build/build.v b/src/build/build.v index 0a978aa0..15a5eb81 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -3,9 +3,9 @@ module build import docker import encoding.base64 import time +import git import os import db -import client const container_build_dir = '/build' @@ -73,18 +73,10 @@ pub fn create_build_image(base_image string) ?string { return image.id } -pub struct BuildResult { -pub: - start_time time.Time - end_time time.Time - exit_code int - logs string -} - // build_repo builds, packages & publishes a given Arch package based on the // provided GitRepo. The base image ID should be of an image previously created -// by create_build_image. It returns the logs of the container. -pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ?BuildResult { +// by create_build_image. +pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ? { build_arch := os.uname().machine // TODO what to do with PKGBUILDs that build multiple packages? @@ -95,7 +87,7 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db 'source PKGBUILD', // The build container checks whether the package is already // present on the server - 'curl -s --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + 'curl --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done', ] @@ -115,44 +107,43 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db id := docker.create_container(c) ? docker.start_container(id) ? - mut data := docker.inspect_container(id) ? - // This loop waits until the container has stopped, so we can remove it after - for data.state.running { + for { + data := docker.inspect_container(id) ? + + if !data.state.running { + break + } + time.sleep(1 * time.second) - - data = docker.inspect_container(id) ? } - logs := docker.get_container_logs(id) ? - docker.remove_container(id) ? - - return BuildResult{ - start_time: data.state.start_time - end_time: data.state.end_time - exit_code: data.state.exit_code - logs: logs - } } // build builds every Git repo in the server's list. -fn build(conf Config, repo_id int) ? { - c := client.new(conf.address, conf.api_key) - repo := c.get_git_repo(repo_id) ? - +fn build(conf Config) ? { build_arch := os.uname().machine - println('Creating base image...') + // We get the repos map from the Vieter instance + repos := git.get_repos(conf.address, conf.api_key) ? + + // We filter out any repos that aren't allowed to be built on this + // architecture + filtered_repos := repos.filter(it.arch.map(it.value).contains(build_arch)) + + // No point in doing work if there's no repos present + if filtered_repos.len == 0 { + return + } + + // First, we create a base image which has updated repos n stuff image_id := create_build_image(conf.base_image) ? - println('Running build...') - res := build_repo(conf.address, conf.api_key, image_id, repo) ? + for repo in filtered_repos { + build_repo(conf.address, conf.api_key, image_id, repo) ? + } - println('Removing build image...') + // Finally, we remove the builder image docker.remove_image(image_id) ? - - println('Uploading logs to Vieter...') - c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code, - res.logs) ? } diff --git a/src/build/cli.v b/src/build/cli.v index 5247e871..01313960 100644 --- a/src/build/cli.v +++ b/src/build/cli.v @@ -14,16 +14,12 @@ pub: pub fn cmd() cli.Command { return cli.Command{ name: 'build' - required_args: 1 - usage: 'id' - description: 'Build the repository with the given ID.' + description: 'Run the build process.' execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file') ? conf := env.load(config_file) ? - id := cmd.args[0].int() - - build(conf, id) ? + build(conf) ? } } } diff --git a/src/client/client.v b/src/client/client.v deleted file mode 100644 index 25224a51..00000000 --- a/src/client/client.v +++ /dev/null @@ -1,67 +0,0 @@ -module client - -import net.http { Method } -import net.urllib -import response { Response } -import json - -pub struct Client { -pub: - address string - api_key string -} - -// new creates a new Client instance. -pub fn new(address string, api_key string) Client { - return Client{ - address: address - api_key: api_key - } -} - -// send_request_raw sends an HTTP request, returning the http.Response object. -// It encodes the params so that they're safe to pass as HTTP query parameters. -fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) ?http.Response { - mut full_url := '$c.address$url' - - if params.len > 0 { - mut params_escaped := map[string]string{} - - // Escape each query param - for k, v in params { - params_escaped[k] = urllib.query_escape(v) - } - - params_str := params_escaped.keys().map('$it=${params[it]}').join('&') - - full_url = '$full_url?$params_str' - } - - mut req := http.new_request(method, full_url, body) ? - req.add_custom_header('X-Api-Key', c.api_key) ? - - res := req.do() ? - - return res -} - -// send_request just calls send_request_with_body with an empty body. -fn (c &Client) send_request(method Method, url string, params map[string]string) ?Response { - return c.send_request_with_body(method, url, params, '') -} - -// send_request_with_body calls send_request_raw_response & parses its -// output as a Response object. -fn (c &Client) send_request_with_body(method Method, url string, params map[string]string, body string) ?Response { - res_text := c.send_request_raw_response(method, url, params, body) ? - data := json.decode(Response, res_text) ? - - return data -} - -// send_request_raw_response returns the raw text response for an HTTP request. -fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) ?string { - res := c.send_request_raw(method, url, params, body) ? - - return res.text -} diff --git a/src/client/git.v b/src/client/git.v deleted file mode 100644 index b09d4c22..00000000 --- a/src/client/git.v +++ /dev/null @@ -1,51 +0,0 @@ -module client - -import db { GitRepo } -import net.http { Method } -import response { Response } - -// get_git_repos returns the current list of repos. -pub fn (c &Client) get_git_repos() ?[]GitRepo { - data := c.send_request<[]GitRepo>(Method.get, '/api/repos', {}) ? - - return data.data -} - -// get_git_repo returns the repo for a specific ID. -pub fn (c &Client) get_git_repo(id int) ?GitRepo { - data := c.send_request(Method.get, '/api/repos/$id', {}) ? - - return data.data -} - -// add_git_repo adds a new repo to the server. -pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []string) ?Response { - mut params := { - 'url': url - 'branch': branch - 'repo': repo - } - - if arch.len > 0 { - params['arch'] = arch.join(',') - } - - data := c.send_request(Method.post, '/api/repos', params) ? - - return data -} - -// remove_git_repo removes the repo with the given ID from the server. -pub fn (c &Client) remove_git_repo(id int) ?Response { - data := c.send_request(Method.delete, '/api/repos/$id', {}) ? - - return data -} - -// patch_git_repo sends a PATCH request to the given repo with the params as -// payload. -pub fn (c &Client) patch_git_repo(id int, params map[string]string) ?Response { - data := c.send_request(Method.patch, '/api/repos/$id', params) ? - - return data -} diff --git a/src/client/logs.v b/src/client/logs.v deleted file mode 100644 index 8c532138..00000000 --- a/src/client/logs.v +++ /dev/null @@ -1,53 +0,0 @@ -module client - -import db { BuildLog } -import net.http { Method } -import response { Response } -import time - -// get_build_logs returns all build logs. -pub fn (c &Client) get_build_logs() ?Response<[]BuildLog> { - data := c.send_request<[]BuildLog>(Method.get, '/api/logs', {}) ? - - return data -} - -// get_build_logs_for_repo returns all build logs for a given repo. -pub fn (c &Client) get_build_logs_for_repo(repo_id int) ?Response<[]BuildLog> { - params := { - 'repo': repo_id.str() - } - - data := c.send_request<[]BuildLog>(Method.get, '/api/logs', params) ? - - return data -} - -// get_build_log returns a specific build log. -pub fn (c &Client) get_build_log(id int) ?Response { - data := c.send_request(Method.get, '/api/logs/$id', {}) ? - - return data -} - -// get_build_log_content returns the contents of the build log file. -pub fn (c &Client) get_build_log_content(id int) ?string { - data := c.send_request_raw_response(Method.get, '/api/logs/$id/content', {}, '') ? - - return data -} - -// add_build_log adds a new build log to the server. -pub fn (c &Client) add_build_log(repo_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { - params := { - 'repo': repo_id.str() - 'startTime': start_time.str() - 'endTime': end_time.str() - 'arch': arch - 'exitCode': exit_code.str() - } - - data := c.send_request_with_body(Method.post, '/api/logs', params, content) ? - - return data -} diff --git a/src/console/console.v b/src/console/console.v deleted file mode 100644 index 6f296bdd..00000000 --- a/src/console/console.v +++ /dev/null @@ -1 +0,0 @@ -module console diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v deleted file mode 100644 index e3c7d14a..00000000 --- a/src/console/logs/logs.v +++ /dev/null @@ -1,106 +0,0 @@ -module logs - -import cli -import env -import client -import db - -struct Config { - address string [required] - api_key string [required] -} - -// cmd returns the cli module that handles the build repos API. -pub fn cmd() cli.Command { - return cli.Command{ - name: 'logs' - description: 'Interact with the build logs API.' - commands: [ - cli.Command{ - name: 'list' - description: 'List the build logs. If a repo ID is provided, only list the build logs for that repo.' - flags: [ - cli.Flag{ - name: 'repo' - description: 'ID of the Git repo to restrict list to.' - flag: cli.FlagType.int - }, - ] - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? - - repo_id := cmd.flags.get_int('repo') ? - - if repo_id == 0 { list(conf) ? } else { list_for_repo(conf, repo_id) ? } - } - }, - cli.Command{ - name: 'info' - required_args: 1 - usage: 'id' - description: 'Show all info for a specific build log.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? - - id := cmd.args[0].int() - info(conf, id) ? - } - }, - cli.Command{ - name: 'content' - required_args: 1 - usage: 'id' - description: 'Output the content of a build log to stdout.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? - - id := cmd.args[0].int() - content(conf, id) ? - } - }, - ] - } -} - -// print_log_list prints a list of logs. -fn print_log_list(logs []db.BuildLog) { - for log in logs { - println('$log.id\t$log.start_time\t$log.exit_code') - } -} - -// list prints a list of all build logs. -fn list(conf Config) ? { - c := client.new(conf.address, conf.api_key) - logs := c.get_build_logs() ?.data - - print_log_list(logs) -} - -// list prints a list of all build logs for a given repo. -fn list_for_repo(conf Config, repo_id int) ? { - c := client.new(conf.address, conf.api_key) - logs := c.get_build_logs_for_repo(repo_id) ?.data - - print_log_list(logs) -} - -// info print the detailed info for a given build log. -fn info(conf Config, id int) ? { - c := client.new(conf.address, conf.api_key) - log := c.get_build_log(id) ?.data - - print(log) -} - -// content outputs the contents of the log file for a given build log to -// stdout. -fn content(conf Config, id int) ? { - c := client.new(conf.address, conf.api_key) - content := c.get_build_log_content(id) ? - - println(content) -} diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index aa08f9f3..e54a39e0 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -3,7 +3,6 @@ module daemon import time import sync.stdatomic import build -import os const ( build_empty = 0 @@ -78,20 +77,13 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { // 0 means success, 1 means failure mut status := 0 - res := build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(), - &sb.repo) or { + build.build_repo(d.address, d.api_key, d.builder_images.last(), &sb.repo) or { d.ldebug('build_repo error: $err.msg()') status = 1 - - build.BuildResult{} } if status == 0 { - d.linfo('finished build: $sb.repo.url $sb.repo.branch; uploading logs...') - - build_arch := os.uname().machine - d.client.add_build_log(sb.repo.id, res.start_time, res.end_time, build_arch, res.exit_code, - res.logs) or { d.lerror('Failed to upload logs for $sb.repo.url $sb.repo.arch') } + d.linfo('finished build: $sb.repo.url $sb.repo.branch') } else { d.linfo('failed build: $sb.repo.url $sb.repo.branch') } diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index ade8fcbf..35cca5f1 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -1,5 +1,6 @@ module daemon +import git import time import log import datatypes { MinHeap } @@ -8,8 +9,6 @@ import math import build import docker import db -import os -import client const ( // How many seconds to wait before retrying to update API if failed @@ -20,6 +19,7 @@ const ( struct ScheduledBuild { pub: + repo_id string repo db.GitRepo timestamp time.Time } @@ -31,7 +31,8 @@ fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool { pub struct Daemon { mut: - client client.Client + address string + api_key string base_image string builder_images []string global_schedule CronExpression @@ -55,7 +56,8 @@ mut: // populates the build queue for the first time. pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon { mut d := Daemon{ - client: client.new(address, api_key) + address: address + api_key: api_key base_image: base_image global_schedule: global_schedule api_update_frequency: api_update_frequency @@ -178,17 +180,13 @@ fn (mut d Daemon) schedule_build(repo db.GitRepo) { fn (mut d Daemon) renew_repos() { d.linfo('Renewing repos...') - mut new_repos := d.client.get_git_repos() or { + mut new_repos := git.get_repos(d.address, d.api_key) or { d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...') d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) return } - // Filter out any repos that shouldn't run on this architecture - cur_arch := os.uname().machine - new_repos = new_repos.filter(it.arch.any(it.value == cur_arch)) - d.repos = new_repos d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency) diff --git a/src/db/db.v b/src/db/db.v index 5ec240de..a75c34c4 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -12,7 +12,6 @@ pub fn init(db_path string) ?VieterDb { sql conn { create table GitRepo - create table BuildLog } return VieterDb{ diff --git a/src/db/git.v b/src/db/git.v index 9a475a54..2fb2a92e 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -94,7 +94,7 @@ pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo { // If a select statement fails, it returns a zeroed object. By // checking one of the required fields, we can see whether the query // returned a result or not. - if res.id == 0 { + if res.url == '' { return none } @@ -118,9 +118,11 @@ pub fn (db &VieterDb) delete_git_repo(repo_id int) { // update_git_repo updates any non-array values for a given GitRepo. pub fn (db &VieterDb) update_git_repo(repo_id int, params map[string]string) { + // sql db.conn { + // update GitRepo set repo + //} mut values := []string{} - // TODO does this allow for SQL injection? $for field in GitRepo.fields { if field.name in params { // Any fields that are array types require their own update method @@ -152,11 +154,3 @@ pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { } } } - -// git_repo_exists is a utility function that checks whether a repo with the -// given id exists. -pub fn (db &VieterDb) git_repo_exists(repo_id int) bool { - db.get_git_repo(repo_id) or { return false } - - return true -} diff --git a/src/db/logs.v b/src/db/logs.v deleted file mode 100644 index 817db78b..00000000 --- a/src/db/logs.v +++ /dev/null @@ -1,74 +0,0 @@ -module db - -import time - -pub struct BuildLog { -pub: - id int [primary; sql: serial] - repo_id int [nonull] - start_time time.Time [nonull] - end_time time.Time [nonull] - arch string [nonull] - exit_code int [nonull] -} - -// str returns a string representation. -pub fn (bl &BuildLog) str() string { - mut parts := [ - 'id: $bl.id', - 'repo id: $bl.repo_id', - 'start time: $bl.start_time', - 'end time: $bl.end_time', - 'arch: $bl.arch', - 'exit code: $bl.exit_code', - ] - str := parts.join('\n') - - return str -} - -// get_build_logs returns all BuildLog's in the database. -pub fn (db &VieterDb) get_build_logs() []BuildLog { - res := sql db.conn { - select from BuildLog order by id - } - - return res -} - -// get_build_logs_for_repo returns all BuildLog's in the database for a given -// repo. -pub fn (db &VieterDb) get_build_logs_for_repo(repo_id int) []BuildLog { - res := sql db.conn { - select from BuildLog where repo_id == repo_id order by id - } - - return res -} - -// get_build_log tries to return a specific BuildLog. -pub fn (db &VieterDb) get_build_log(id int) ?BuildLog { - res := sql db.conn { - select from BuildLog where id == id - } - - if res.id == 0 { - return none - } - - return res -} - -// add_build_log inserts the given BuildLog into the database. -pub fn (db &VieterDb) add_build_log(log BuildLog) { - sql db.conn { - insert log into BuildLog - } -} - -// delete_build_log delete the BuildLog with the given ID from the database. -pub fn (db &VieterDb) delete_build_log(id int) { - sql db.conn { - delete from BuildLog where id == id - } -} diff --git a/src/docker/containers.v b/src/docker/containers.v index 2258f3bd..d0f5a4d7 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -2,7 +2,6 @@ module docker import json import net.urllib -import time struct Container { id string [json: Id] @@ -50,21 +49,13 @@ pub fn start_container(id string) ?bool { } struct ContainerInspect { -pub mut: +pub: state ContainerState [json: State] } struct ContainerState { pub: - running bool [json: Running] - status string [json: Status] - exit_code int [json: ExitCode] - // These use a rather specific format so they have to be parsed later - start_time_str string [json: StartedAt] - end_time_str string [json: FinishedAt] -pub mut: - start_time time.Time [skip] - end_time time.Time [skip] + running bool [json: Running] } // inspect_container returns the result of inspecting a container with a given @@ -76,15 +67,7 @@ pub fn inspect_container(id string) ?ContainerInspect { return error('Failed to inspect container.') } - mut data := json.decode(ContainerInspect, res.text) ? - - data.state.start_time = time.parse_rfc3339(data.state.start_time_str) ? - - if data.state.status == 'exited' { - data.state.end_time = time.parse_rfc3339(data.state.end_time_str) ? - } - - return data + return json.decode(ContainerInspect, res.text) or {} } // remove_container removes a container with a given ID. @@ -93,25 +76,3 @@ pub fn remove_container(id string) ?bool { return res.status_code == 204 } - -// get_container_logs retrieves the logs for a Docker container, both stdout & -// stderr. -pub fn get_container_logs(id string) ?string { - res := request('GET', urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true') ?) ? - mut res_bytes := res.text.bytes() - - // Docker uses a special "stream" format for their logs, so we have to - // clean up the data. - mut index := 0 - - for index < res_bytes.len { - // The reverse is required because V reads in the bytes differently - t := res_bytes[index + 4..index + 8].reverse() - len_length := unsafe { *(&u32(&t[0])) } - - res_bytes.delete_many(index, 8) - index += int(len_length) - } - - return res_bytes.bytestr() -} diff --git a/src/console/git/git.v b/src/git/cli.v similarity index 90% rename from src/console/git/git.v rename to src/git/cli.v index 3bf78d10..634b7782 100644 --- a/src/console/git/git.v +++ b/src/git/cli.v @@ -3,7 +3,6 @@ module git import cli import env import cron.expression { parse_expression } -import client struct Config { address string [required] @@ -120,8 +119,7 @@ pub fn cmd() cli.Command { // list prints out a list of all repositories. fn list(conf Config) ? { - c := client.new(conf.address, conf.api_key) - repos := c.get_git_repos() ? + repos := get_repos(conf.address, conf.api_key) ? for repo in repos { println('$repo.id\t$repo.url\t$repo.branch\t$repo.repo') @@ -130,8 +128,7 @@ fn list(conf Config) ? { // add adds a new repository to the server's list. fn add(conf Config, url string, branch string, repo string) ? { - c := client.new(conf.address, conf.api_key) - res := c.add_git_repo(url, branch, repo, []) ? + res := add_repo(conf.address, conf.api_key, url, branch, repo, []) ? println(res.message) } @@ -142,8 +139,7 @@ fn remove(conf Config, id string) ? { id_int := id.int() if id_int != 0 { - c := client.new(conf.address, conf.api_key) - res := c.remove_git_repo(id_int) ? + res := remove_repo(conf.address, conf.api_key, id_int) ? println(res.message) } } @@ -160,8 +156,7 @@ fn patch(conf Config, id string, params map[string]string) ? { id_int := id.int() if id_int != 0 { - c := client.new(conf.address, conf.api_key) - res := c.patch_git_repo(id_int, params) ? + res := patch_repo(conf.address, conf.api_key, id_int, params) ? println(res.message) } @@ -175,7 +170,6 @@ fn info(conf Config, id string) ? { return } - c := client.new(conf.address, conf.api_key) - repo := c.get_git_repo(id_int) ? + repo := get_repo(conf.address, conf.api_key, id_int) ? println(repo) } diff --git a/src/git/client.v b/src/git/client.v new file mode 100644 index 00000000..b5f8e9fd --- /dev/null +++ b/src/git/client.v @@ -0,0 +1,77 @@ +module git + +import json +import response { Response } +import net.http +import db + +// send_request is a convenience method for sending requests to the repos +// API. It mostly does string manipulation to create a query string containing +// the provided params. +fn send_request(method http.Method, address string, url string, api_key string, params map[string]string) ?Response { + mut full_url := '$address$url' + + if params.len > 0 { + params_str := params.keys().map('$it=${params[it]}').join('&') + + full_url = '$full_url?$params_str' + } + + mut req := http.new_request(method, full_url, '') ? + req.add_custom_header('X-API-Key', api_key) ? + + res := req.do() ? + data := json.decode(Response, res.text) ? + + return data +} + +// get_repos returns the current list of repos. +pub fn get_repos(address string, api_key string) ?[]db.GitRepo { + data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', api_key, + {}) ? + + return data.data +} + +// get_repo returns the repo for a specific ID. +pub fn get_repo(address string, api_key string, id int) ?db.GitRepo { + data := send_request(http.Method.get, address, '/api/repos/$id', api_key, + {}) ? + + return data.data +} + +// add_repo adds a new repo to the server. +pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response { + mut params := { + 'url': url + 'branch': branch + 'repo': repo + } + + if arch.len > 0 { + params['arch'] = arch.join(',') + } + + data := send_request(http.Method.post, address, '/api/repos', api_key, params) ? + + return data +} + +// remove_repo removes the repo with the given ID from the server. +pub fn remove_repo(address string, api_key string, id int) ?Response { + data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, + {}) ? + + return data +} + +// patch_repo sends a PATCH request to the given repo with the params as +// payload. +pub fn patch_repo(address string, api_key string, id int, params map[string]string) ?Response { + data := send_request(http.Method.patch, address, '/api/repos/$id', api_key, + params) ? + + return data +} diff --git a/src/main.v b/src/main.v index 41d0d331..37cabc3a 100644 --- a/src/main.v +++ b/src/main.v @@ -4,15 +4,14 @@ import os import server import cli import build -import console.git -import console.logs +import git import cron fn main() { mut app := cli.Command{ name: 'vieter' description: 'Vieter is a lightweight implementation of an Arch repository server.' - version: '0.3.0-alpha.1' + version: '0.2.0' flags: [ cli.Flag{ flag: cli.FlagType.string @@ -28,7 +27,6 @@ fn main() { build.cmd(), git.cmd(), cron.cmd(), - logs.cmd(), ] } diff --git a/src/server/git.v b/src/server/git.v index c5cbc0a6..6c852b84 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -60,6 +60,7 @@ fn (mut app App) delete_repo(id int) web.Result { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) } + // repos.delete(id) app.db.delete_git_repo(id) return app.json(http.Status.ok, new_response('Repo removed successfully.')) diff --git a/src/server/logs.v b/src/server/logs.v deleted file mode 100644 index b048dc4f..00000000 --- a/src/server/logs.v +++ /dev/null @@ -1,136 +0,0 @@ -module server - -import web -import net.http -import net.urllib -import response { new_data_response, new_response } -import db -import time -import os -import util - -// get_logs returns all build logs in the database. A 'repo' query param can -// optionally be added to limit the list of build logs to that repository. -['/api/logs'; get] -fn (mut app App) get_logs() web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - - logs := if 'repo' in app.query { - app.db.get_build_logs_for_repo(app.query['repo'].int()) - } else { - app.db.get_build_logs() - } - - return app.json(http.Status.ok, new_data_response(logs)) -} - -// get_single_log returns the build log with the given id. -['/api/logs/:id'; get] -fn (mut app App) get_single_log(id int) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - - log := app.db.get_build_log(id) or { return app.not_found() } - - return app.json(http.Status.ok, new_data_response(log)) -} - -// get_log_content returns the actual build log file for the given id. -['/api/logs/:id/content'; get] -fn (mut app App) get_log_content(id int) web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - - log := app.db.get_build_log(id) or { return app.not_found() } - file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') - full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.repo_id.str(), log.arch, - file_name) - - return app.file(full_path) -} - -// parse_query_time unescapes an HTTP query parameter & tries to parse it as a -// time.Time struct. -fn parse_query_time(query string) ?time.Time { - unescaped := urllib.query_unescape(query) ? - t := time.parse(unescaped) ? - - return t -} - -// post_log adds a new log to the database. -['/api/logs'; post] -fn (mut app App) post_log() web.Result { - if !app.is_authorized() { - return app.json(http.Status.unauthorized, new_response('Unauthorized.')) - } - - // Parse query params - start_time := parse_query_time(app.query['startTime']) or { - return app.json(http.Status.bad_request, new_response('Invalid or missing start time.')) - } - - end_time := parse_query_time(app.query['endTime']) or { - return app.json(http.Status.bad_request, new_response('Invalid or missing end time.')) - } - - if 'exitCode' !in app.query { - return app.json(http.Status.bad_request, new_response('Missing exit code.')) - } - - exit_code := app.query['exitCode'].int() - - if 'arch' !in app.query { - return app.json(http.Status.bad_request, new_response("Missing parameter 'arch'.")) - } - - arch := app.query['arch'] - - repo_id := app.query['repo'].int() - - if !app.db.git_repo_exists(repo_id) { - return app.json(http.Status.bad_request, new_response('Unknown Git repo.')) - } - - // Store log in db - log := db.BuildLog{ - repo_id: repo_id - start_time: start_time - end_time: end_time - arch: arch - exit_code: exit_code - } - - app.db.add_build_log(log) - - repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo_id.str(), arch) - - // Create the logs directory of it doesn't exist - if !os.exists(repo_logs_dir) { - os.mkdir_all(repo_logs_dir) or { - app.lerror("Couldn't create dir '$repo_logs_dir'.") - - return app.json(http.Status.internal_server_error, new_response('An error occured while processing the request.')) - } - } - - // Stream log contents to correct file - file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss') - full_path := os.join_path_single(repo_logs_dir, file_name) - - if length := app.req.header.get(.content_length) { - util.reader_to_file(mut app.reader, length.int(), full_path) or { - app.lerror('An error occured while receiving logs: $err.msg()') - - return app.json(http.Status.internal_server_error, new_response('Failed to upload logs.')) - } - } else { - return app.status(http.Status.length_required) - } - - return app.json(http.Status.ok, new_response('Logs added successfully.')) -} diff --git a/src/server/server.v b/src/server/server.v index 090aa76e..b2a2ad29 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -12,7 +12,6 @@ const ( log_file_name = 'vieter.log' repo_dir_name = 'repos' db_file_name = 'vieter.sqlite' - logs_dir_name = 'logs' ) struct App { @@ -38,14 +37,6 @@ pub fn server(conf Config) ? { os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') } - logs_dir := os.join_path_single(conf.data_dir, server.logs_dir_name) - - if !os.exists(logs_dir) { - os.mkdir(os.join_path_single(conf.data_dir, server.logs_dir_name)) or { - util.exit_with_message(1, 'Failed to create logs directory.') - } - } - mut logger := log.Log{ level: log_level }