diff --git a/.woodpecker/.arch-rel.yml b/.woodpecker/.arch-rel.yml new file mode 100644 index 0000000..b8f4c7a --- /dev/null +++ b/.woodpecker/.arch-rel.yml @@ -0,0 +1,39 @@ +matrix: + PLATFORM: + - linux/amd64 + - linux/arm64 + +platform: ${PLATFORM} +branches: [main] +skip_clone: true + +pipeline: + build: + image: 'menci/archlinuxarm:base-devel' + commands: + # Add the vieter repository so we can use the compiler + - echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf + # Update packages + - pacman -Syu --noconfirm + # Create non-root user to perform build & switch to their home + - groupadd -g 1000 builder + - useradd -mg builder builder + - chown -R builder:builder "$PWD" + - "echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers" + - su builder + # Due to a bug with the V compiler, we can't just use the PKGBUILD from + # inside the repo + - curl -OL "https://git.rustybever.be/vieter/vieter/raw/tag/$CI_COMMIT_TAG/PKGBUILD" + - makepkg -s --noconfirm --needed + when: + event: tag + + publish: + image: 'curlimages/curl' + commands: + # Publish the package + - 'for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $VIETER_API_KEY" https://arch.r8r.be/vieter/publish; done' + secrets: + - vieter_api_key + when: + event: tag diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index 6b8f8f2..b2a59ba 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -23,7 +23,7 @@ pipeline: - su builder # Due to a bug with the V compiler, we can't just use the PKGBUILD from # inside the repo - - curl -OL https://git.rustybever.be/vieter/vieter/raw/branch/dev/PKGBUILD + - curl -o PKGBUILD -L https://git.rustybever.be/vieter/vieter/raw/branch/dev/PKGBUILD.dev - makepkg -s --noconfirm --needed when: event: push diff --git a/.woodpecker/.docker.yml b/.woodpecker/.docker.yml index bab869b..f31490a 100644 --- a/.woodpecker/.docker.yml +++ b/.woodpecker/.docker.yml @@ -11,7 +11,9 @@ pipeline: - 'docker_password' settings: repo: 'chewingbever/vieter' - tag: 'dev' + tags: + - 'dev' + - ${CI_COMMIT_SHA} platforms: [ 'linux/arm64/v8', 'linux/amd64' ] build_args_from_env: - 'CI_COMMIT_SHA' diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bbe4f0..7dc0a00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,33 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased](https://git.rustybever.be/vieter/vieter/src/branch/dev) + +## [0.3.0-alpha.2](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.2) + +### Added + +* Web API for adding & querying build logs +* CLI commands to access build logs API +* Cron build logs are uploaded to above API +* Proper ASCII table output in CLI +* `vieter repos build id` command to run builds locally + +### Removed + +* `vieter build` command + * This command was used alongside cron for periodic builds, but this has + been replaced by `vieter cron` + +### Changed + +* `vieter build` command now only builds a single repository & uploads the + build logs +* Official Arch packages are now split between `vieter` & `vieter-git` + * `vieter` is the latest release + * `vieter-git` is the latest commit on the dev branch +* Full refactor of Docker socket code + ## [0.3.0-alpha.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.1) ### Changed diff --git a/Dockerfile b/Dockerfile index 2ba6181..5997adc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,15 +36,8 @@ ENV PATH=/bin \ COPY --from=builder /app/dumb-init /app/vieter /bin/ -HEALTHCHECK --interval=30s \ - --timeout=3s \ - --start-period=5s \ - CMD /bin/wget --spider http://localhost:8000/health || exit 1 - RUN mkdir /data && \ - chown -R www-data:www-data /data && \ - mkdir -p '/var/spool/cron/crontabs' && \ - echo '0 3 * * * /bin/vieter build' | crontab - + chown -R www-data:www-data /data WORKDIR /data diff --git a/PKGBUILD b/PKGBUILD index 87c575f..49fcf54 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -1,23 +1,18 @@ +# vim: ft=bash # Maintainer: Jef Roosens pkgbase='vieter' pkgname='vieter' -pkgver=0.2.0.r25.g20112b8 +pkgver='0.3.0_alpha.2' pkgrel=1 -depends=('glibc' 'openssl' 'libarchive' 'gc' 'sqlite') -makedepends=('git' 'gcc' 'vieter-v') +depends=('glibc' 'openssl' 'libarchive' 'sqlite') +makedepends=('git' 'vieter-v') arch=('x86_64' 'aarch64') url='https://git.rustybever.be/vieter/vieter' license=('AGPL3') -source=($pkgname::git+https://git.rustybever.be/vieter/vieter#branch=dev) +source=("$pkgname::git+https://git.rustybever.be/vieter/vieter#tag=${pkgver//_/-}") md5sums=('SKIP') -pkgver() { - cd "$pkgname" - - git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g' -} - build() { cd "$pkgname" @@ -28,5 +23,5 @@ package() { pkgdesc="Vieter is a lightweight implementation of an Arch repository server." install -dm755 "$pkgdir/usr/bin" - install -Dm755 "$pkgbase/pvieter" "$pkgdir/usr/bin/vieter" + install -Dm755 "$pkgname/pvieter" "$pkgdir/usr/bin/vieter" } diff --git a/PKGBUILD.dev b/PKGBUILD.dev new file mode 100644 index 0000000..d0176d8 --- /dev/null +++ b/PKGBUILD.dev @@ -0,0 +1,35 @@ +# vim: ft=bash +# Maintainer: Jef Roosens + +pkgbase='vieter-git' +pkgname='vieter-git' +pkgver=0.2.0.r25.g20112b8 +pkgrel=1 +depends=('glibc' 'openssl' 'libarchive' 'sqlite') +makedepends=('git' 'vieter-v') +arch=('x86_64' 'aarch64') +url='https://git.rustybever.be/vieter/vieter' +license=('AGPL3') +source=("$pkgname::git+https://git.rustybever.be/vieter/vieter#branch=dev") +md5sums=('SKIP') +provides=('vieter') +conflicts=('vieter') + +pkgver() { + cd "$pkgname" + + git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g' +} + +build() { + cd "$pkgname" + + make prod +} + +package() { + pkgdesc="Vieter is a lightweight implementation of an Arch repository server." + + install -dm755 "$pkgdir/usr/bin" + install -Dm755 "$pkgname/pvieter" "$pkgdir/usr/bin/vieter" +} diff --git a/README.md b/README.md index 08f1e75..138282a 100644 --- a/README.md +++ b/README.md @@ -55,3 +55,13 @@ clone my compiler in the `v` directory & build it. Afterwards, you can use this compiler with make by prepending all make commands with `V_PATH=v/v`. If you do encounter this issue, please let me know so I can update my mirror & the codebase to fix it! + +## Contributing + +If you wish to contribute to the project, please take note of the following: + +* Rebase instead of merging whenever possible, e.g. when updating your branch + with the dev branch. +* Please follow the + [Conventional Commits](https://www.conventionalcommits.org/) style for your + commit messages. diff --git a/docs/content/api.md b/docs/content/api.md index 7c395eb..0fbb694 100644 --- a/docs/content/api.md +++ b/docs/content/api.md @@ -56,6 +56,11 @@ Vieter only supports uploading archives compressed using either gzip, zstd or xz at the moment. {{< /hint >}} +### `GET /health` + +This endpoint's only use is to be used with healthchecks. It returns a JSON +response with the message "Healthy.". + ## API All API routes require the API key to provided using the `X-Api-Key` header. diff --git a/docs/content/builder.md b/docs/content/builder.md index 6a1bc3a..659717d 100644 --- a/docs/content/builder.md +++ b/docs/content/builder.md @@ -15,7 +15,7 @@ repositories. After the image has been created, each repository returned by previously created image as a base. Each container goes through the following steps: 1. The repository is cloned -2. `makepkg --nobuild --nodeps` is ran to update the `pkgver` variable inside +2. `makepkg --nobuild --syncdeps --needed --noconfirm` is ran to update the `pkgver` variable inside the `PKGBUILD` file 3. A HEAD request is sent to the Vieter server to check whether the specific version of the package is already present. If it is, the container exits. diff --git a/src/build/build.v b/src/build/build.v index 15a5eb8..fab6c35 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -3,13 +3,15 @@ module build import docker import encoding.base64 import time -import git import os import db +import strings +import util -const container_build_dir = '/build' - -const build_image_repo = 'vieter-build' +const ( + container_build_dir = '/build' + build_image_repo = 'vieter-build' +) // create_build_image creates a builder image given some base image which can // then be used to build & package Arch images. It mostly just updates the @@ -17,6 +19,12 @@ const build_image_repo = 'vieter-build' // makepkg with. The base image should be some Linux distribution that uses // Pacman as its package manager. pub fn create_build_image(base_image string) ?string { + mut dd := docker.new_conn()? + + defer { + dd.close() or {} + } + commands := [ // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' @@ -46,14 +54,15 @@ pub fn create_build_image(base_image string) ?string { image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' } // We pull the provided image - docker.pull_image(image_name, image_tag) ? + dd.pull_image(image_name, image_tag)? - id := docker.create_container(c) ? - docker.start_container(id) ? + id := dd.create_container(c)?.id + // id := docker.create_container(c)? + dd.start_container(id)? // This loop waits until the container has stopped, so we can remove it after for { - data := docker.inspect_container(id) ? + data := dd.inspect_container(id)? if !data.state.running { break @@ -67,27 +76,41 @@ pub fn create_build_image(base_image string) ?string { // TODO also add the base image's name into the image name to prevent // conflicts. tag := time.sys_mono_now().str() - image := docker.create_image_from_container(id, 'vieter-build', tag) ? - docker.remove_container(id) ? + image := dd.create_image_from_container(id, 'vieter-build', tag)? + dd.remove_container(id)? return image.id } +pub struct BuildResult { +pub: + start_time time.Time + end_time time.Time + exit_code int + logs string +} + // build_repo builds, packages & publishes a given Arch package based on the // provided GitRepo. The base image ID should be of an image previously created -// by create_build_image. -pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ? { +// by create_build_image. It returns the logs of the container. +pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ?BuildResult { + mut dd := docker.new_conn()? + + defer { + dd.close() or {} + } + build_arch := os.uname().machine // TODO what to do with PKGBUILDs that build multiple packages? commands := [ 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', 'cd repo', - 'makepkg --nobuild --nodeps', + 'makepkg --nobuild --syncdeps --needed --noconfirm', 'source PKGBUILD', // The build container checks whether the package is already // present on the server - 'curl --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', + 'curl -s --head --fail $address/$repo.repo/$build_arch/\$pkgname-\$pkgver-\$pkgrel && exit 0', 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $address/$repo.repo/publish; done', ] @@ -104,46 +127,30 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db user: 'builder:builder' } - id := docker.create_container(c) ? - docker.start_container(id) ? + id := dd.create_container(c)?.id + dd.start_container(id)? + + mut data := dd.inspect_container(id)? // This loop waits until the container has stopped, so we can remove it after - for { - data := docker.inspect_container(id) ? - - if !data.state.running { - break - } - + for data.state.running { time.sleep(1 * time.second) + + data = dd.inspect_container(id)? } - docker.remove_container(id) ? -} - -// build builds every Git repo in the server's list. -fn build(conf Config) ? { - build_arch := os.uname().machine - - // We get the repos map from the Vieter instance - repos := git.get_repos(conf.address, conf.api_key) ? - - // We filter out any repos that aren't allowed to be built on this - // architecture - filtered_repos := repos.filter(it.arch.map(it.value).contains(build_arch)) - - // No point in doing work if there's no repos present - if filtered_repos.len == 0 { - return - } - - // First, we create a base image which has updated repos n stuff - image_id := create_build_image(conf.base_image) ? - - for repo in filtered_repos { - build_repo(conf.address, conf.api_key, image_id, repo) ? - } - - // Finally, we remove the builder image - docker.remove_image(image_id) ? + mut logs_stream := dd.get_container_logs(id)? + + // Read in the entire stream + mut logs_builder := strings.new_builder(10 * 1024) + util.reader_to_writer(mut logs_stream, mut logs_builder)? + + dd.remove_container(id)? + + return BuildResult{ + start_time: data.state.start_time + end_time: data.state.end_time + exit_code: data.state.exit_code + logs: logs_builder.str() + } } diff --git a/src/build/cli.v b/src/build/cli.v deleted file mode 100644 index 0131396..0000000 --- a/src/build/cli.v +++ /dev/null @@ -1,25 +0,0 @@ -module build - -import cli -import env - -pub struct Config { -pub: - api_key string - address string - base_image string = 'archlinux:base-devel' -} - -// cmd returns the cli submodule that handles the build process -pub fn cmd() cli.Command { - return cli.Command{ - name: 'build' - description: 'Run the build process.' - execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? - - build(conf) ? - } - } -} diff --git a/src/client/client.v b/src/client/client.v new file mode 100644 index 0000000..3b28073 --- /dev/null +++ b/src/client/client.v @@ -0,0 +1,67 @@ +module client + +import net.http { Method } +import net.urllib +import response { Response } +import json + +pub struct Client { +pub: + address string + api_key string +} + +// new creates a new Client instance. +pub fn new(address string, api_key string) Client { + return Client{ + address: address + api_key: api_key + } +} + +// send_request_raw sends an HTTP request, returning the http.Response object. +// It encodes the params so that they're safe to pass as HTTP query parameters. +fn (c &Client) send_request_raw(method Method, url string, params map[string]string, body string) ?http.Response { + mut full_url := '$c.address$url' + + if params.len > 0 { + mut params_escaped := map[string]string{} + + // Escape each query param + for k, v in params { + params_escaped[k] = urllib.query_escape(v) + } + + params_str := params_escaped.keys().map('$it=${params[it]}').join('&') + + full_url = '$full_url?$params_str' + } + + mut req := http.new_request(method, full_url, body)? + req.add_custom_header('X-Api-Key', c.api_key)? + + res := req.do()? + + return res +} + +// send_request just calls send_request_with_body with an empty body. +fn (c &Client) send_request(method Method, url string, params map[string]string) ?Response { + return c.send_request_with_body(method, url, params, '') +} + +// send_request_with_body calls send_request_raw_response & parses its +// output as a Response object. +fn (c &Client) send_request_with_body(method Method, url string, params map[string]string, body string) ?Response { + res_text := c.send_request_raw_response(method, url, params, body)? + data := json.decode(Response, res_text)? + + return data +} + +// send_request_raw_response returns the raw text response for an HTTP request. +fn (c &Client) send_request_raw_response(method Method, url string, params map[string]string, body string) ?string { + res := c.send_request_raw(method, url, params, body)? + + return res.text +} diff --git a/src/client/git.v b/src/client/git.v new file mode 100644 index 0000000..280caab --- /dev/null +++ b/src/client/git.v @@ -0,0 +1,51 @@ +module client + +import db { GitRepo } +import net.http { Method } +import response { Response } + +// get_git_repos returns the current list of repos. +pub fn (c &Client) get_git_repos() ?[]GitRepo { + data := c.send_request<[]GitRepo>(Method.get, '/api/repos', {})? + + return data.data +} + +// get_git_repo returns the repo for a specific ID. +pub fn (c &Client) get_git_repo(id int) ?GitRepo { + data := c.send_request(Method.get, '/api/repos/$id', {})? + + return data.data +} + +// add_git_repo adds a new repo to the server. +pub fn (c &Client) add_git_repo(url string, branch string, repo string, arch []string) ?Response { + mut params := { + 'url': url + 'branch': branch + 'repo': repo + } + + if arch.len > 0 { + params['arch'] = arch.join(',') + } + + data := c.send_request(Method.post, '/api/repos', params)? + + return data +} + +// remove_git_repo removes the repo with the given ID from the server. +pub fn (c &Client) remove_git_repo(id int) ?Response { + data := c.send_request(Method.delete, '/api/repos/$id', {})? + + return data +} + +// patch_git_repo sends a PATCH request to the given repo with the params as +// payload. +pub fn (c &Client) patch_git_repo(id int, params map[string]string) ?Response { + data := c.send_request(Method.patch, '/api/repos/$id', params)? + + return data +} diff --git a/src/client/logs.v b/src/client/logs.v new file mode 100644 index 0000000..cdacab9 --- /dev/null +++ b/src/client/logs.v @@ -0,0 +1,53 @@ +module client + +import db { BuildLog } +import net.http { Method } +import response { Response } +import time + +// get_build_logs returns all build logs. +pub fn (c &Client) get_build_logs() ?Response<[]BuildLog> { + data := c.send_request<[]BuildLog>(Method.get, '/api/logs', {})? + + return data +} + +// get_build_logs_for_repo returns all build logs for a given repo. +pub fn (c &Client) get_build_logs_for_repo(repo_id int) ?Response<[]BuildLog> { + params := { + 'repo': repo_id.str() + } + + data := c.send_request<[]BuildLog>(Method.get, '/api/logs', params)? + + return data +} + +// get_build_log returns a specific build log. +pub fn (c &Client) get_build_log(id int) ?Response { + data := c.send_request(Method.get, '/api/logs/$id', {})? + + return data +} + +// get_build_log_content returns the contents of the build log file. +pub fn (c &Client) get_build_log_content(id int) ?string { + data := c.send_request_raw_response(Method.get, '/api/logs/$id/content', {}, '')? + + return data +} + +// add_build_log adds a new build log to the server. +pub fn (c &Client) add_build_log(repo_id int, start_time time.Time, end_time time.Time, arch string, exit_code int, content string) ?Response { + params := { + 'repo': repo_id.str() + 'startTime': start_time.str() + 'endTime': end_time.str() + 'arch': arch + 'exitCode': exit_code.str() + } + + data := c.send_request_with_body(Method.post, '/api/logs', params, content)? + + return data +} diff --git a/src/console/console.v b/src/console/console.v new file mode 100644 index 0000000..dfb2fac --- /dev/null +++ b/src/console/console.v @@ -0,0 +1,56 @@ +module console + +import arrays +import strings + +// pretty_table converts a list of string data into a pretty table. Many thanks +// to @hungrybluedev in the Vlang Discord for providing this code! +// https://ptb.discord.com/channels/592103645835821068/592106336838352923/970278787143045192 +pub fn pretty_table(header []string, data [][]string) ?string { + column_count := header.len + + mut column_widths := []int{len: column_count, init: header[it].len} + + for values in data { + for col, value in values { + if column_widths[col] < value.len { + column_widths[col] = value.len + } + } + } + + single_line_length := arrays.sum(column_widths)? + (column_count + 1) * 3 - 4 + + horizontal_line := '+' + strings.repeat(`-`, single_line_length) + '+' + mut buffer := strings.new_builder(data.len * single_line_length) + + buffer.writeln(horizontal_line) + + buffer.write_string('| ') + for col, head in header { + if col != 0 { + buffer.write_string(' | ') + } + buffer.write_string(head) + buffer.write_string(strings.repeat(` `, column_widths[col] - head.len)) + } + buffer.writeln(' |') + + buffer.writeln(horizontal_line) + + for values in data { + buffer.write_string('| ') + for col, value in values { + if col != 0 { + buffer.write_string(' | ') + } + buffer.write_string(value) + buffer.write_string(strings.repeat(` `, column_widths[col] - value.len)) + } + buffer.writeln(' |') + } + + buffer.writeln(horizontal_line) + + return buffer.str() +} diff --git a/src/console/git/build.v b/src/console/git/build.v new file mode 100644 index 0000000..fac760d --- /dev/null +++ b/src/console/git/build.v @@ -0,0 +1,34 @@ +module git + +import client +import docker +import os +import build + +// build builds every Git repo in the server's list. +fn build(conf Config, repo_id int) ? { + c := client.new(conf.address, conf.api_key) + repo := c.get_git_repo(repo_id)? + + build_arch := os.uname().machine + + println('Creating base image...') + image_id := build.create_build_image(conf.base_image)? + + println('Running build...') + res := build.build_repo(conf.address, conf.api_key, image_id, repo)? + + println('Removing build image...') + + mut dd := docker.new_conn()? + + defer { + dd.close() or {} + } + + dd.remove_image(image_id)? + + println('Uploading logs to Vieter...') + c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code, + res.logs)? +} diff --git a/src/git/cli.v b/src/console/git/git.v similarity index 64% rename from src/git/cli.v rename to src/console/git/git.v index 634b778..06d5f80 100644 --- a/src/git/cli.v +++ b/src/console/git/git.v @@ -3,10 +3,13 @@ module git import cli import env import cron.expression { parse_expression } +import client +import console struct Config { - address string [required] - api_key string [required] + address string [required] + api_key string [required] + base_image string = 'archlinux:base-devel' } // cmd returns the cli submodule that handles the repos API interaction @@ -19,10 +22,10 @@ pub fn cmd() cli.Command { name: 'list' description: 'List the current repos.' execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? + config_file := cmd.flags.get_string('config-file')? + conf := env.load(config_file)? - list(conf) ? + list(conf)? } }, cli.Command{ @@ -31,10 +34,10 @@ pub fn cmd() cli.Command { usage: 'url branch repo' description: 'Add a new repository.' execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? + config_file := cmd.flags.get_string('config-file')? + conf := env.load(config_file)? - add(conf, cmd.args[0], cmd.args[1], cmd.args[2]) ? + add(conf, cmd.args[0], cmd.args[1], cmd.args[2])? } }, cli.Command{ @@ -43,10 +46,10 @@ pub fn cmd() cli.Command { usage: 'id' description: 'Remove a repository that matches the given ID prefix.' execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? + config_file := cmd.flags.get_string('config-file')? + conf := env.load(config_file)? - remove(conf, cmd.args[0]) ? + remove(conf, cmd.args[0])? } }, cli.Command{ @@ -55,10 +58,10 @@ pub fn cmd() cli.Command { usage: 'id' description: 'Show detailed information for the repo matching the ID prefix.' execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? + config_file := cmd.flags.get_string('config-file')? + conf := env.load(config_file)? - info(conf, cmd.args[0]) ? + info(conf, cmd.args[0])? } }, cli.Command{ @@ -94,8 +97,8 @@ pub fn cmd() cli.Command { }, ] execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? + config_file := cmd.flags.get_string('config-file')? + conf := env.load(config_file)? found := cmd.flags.get_all_found() @@ -103,11 +106,23 @@ pub fn cmd() cli.Command { for f in found { if f.name != 'config-file' { - params[f.name] = f.get_string() ? + params[f.name] = f.get_string()? } } - patch(conf, cmd.args[0], params) ? + patch(conf, cmd.args[0], params)? + } + }, + cli.Command{ + name: 'build' + required_args: 1 + usage: 'id' + description: 'Build the repo with the given id & publish it.' + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := env.load(config_file)? + + build(conf, cmd.args[0].int())? } }, ] @@ -119,16 +134,17 @@ pub fn cmd() cli.Command { // list prints out a list of all repositories. fn list(conf Config) ? { - repos := get_repos(conf.address, conf.api_key) ? + c := client.new(conf.address, conf.api_key) + repos := c.get_git_repos()? + data := repos.map([it.id.str(), it.url, it.branch, it.repo]) - for repo in repos { - println('$repo.id\t$repo.url\t$repo.branch\t$repo.repo') - } + println(console.pretty_table(['id', 'url', 'branch', 'repo'], data)?) } // add adds a new repository to the server's list. fn add(conf Config, url string, branch string, repo string) ? { - res := add_repo(conf.address, conf.api_key, url, branch, repo, []) ? + c := client.new(conf.address, conf.api_key) + res := c.add_git_repo(url, branch, repo, [])? println(res.message) } @@ -139,7 +155,8 @@ fn remove(conf Config, id string) ? { id_int := id.int() if id_int != 0 { - res := remove_repo(conf.address, conf.api_key, id_int) ? + c := client.new(conf.address, conf.api_key) + res := c.remove_git_repo(id_int)? println(res.message) } } @@ -156,7 +173,8 @@ fn patch(conf Config, id string, params map[string]string) ? { id_int := id.int() if id_int != 0 { - res := patch_repo(conf.address, conf.api_key, id_int, params) ? + c := client.new(conf.address, conf.api_key) + res := c.patch_git_repo(id_int, params)? println(res.message) } @@ -170,6 +188,7 @@ fn info(conf Config, id string) ? { return } - repo := get_repo(conf.address, conf.api_key, id_int) ? + c := client.new(conf.address, conf.api_key) + repo := c.get_git_repo(id_int)? println(repo) } diff --git a/src/console/logs/logs.v b/src/console/logs/logs.v new file mode 100644 index 0000000..6400e80 --- /dev/null +++ b/src/console/logs/logs.v @@ -0,0 +1,108 @@ +module logs + +import cli +import env +import client +import db +import console + +struct Config { + address string [required] + api_key string [required] +} + +// cmd returns the cli module that handles the build repos API. +pub fn cmd() cli.Command { + return cli.Command{ + name: 'logs' + description: 'Interact with the build logs API.' + commands: [ + cli.Command{ + name: 'list' + description: 'List the build logs. If a repo ID is provided, only list the build logs for that repo.' + flags: [ + cli.Flag{ + name: 'repo' + description: 'ID of the Git repo to restrict list to.' + flag: cli.FlagType.int + }, + ] + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := env.load(config_file)? + + repo_id := cmd.flags.get_int('repo')? + + if repo_id == 0 { list(conf)? } else { list_for_repo(conf, repo_id)? } + } + }, + cli.Command{ + name: 'info' + required_args: 1 + usage: 'id' + description: 'Show all info for a specific build log.' + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := env.load(config_file)? + + id := cmd.args[0].int() + info(conf, id)? + } + }, + cli.Command{ + name: 'content' + required_args: 1 + usage: 'id' + description: 'Output the content of a build log to stdout.' + execute: fn (cmd cli.Command) ? { + config_file := cmd.flags.get_string('config-file')? + conf := env.load(config_file)? + + id := cmd.args[0].int() + content(conf, id)? + } + }, + ] + } +} + +// print_log_list prints a list of logs. +fn print_log_list(logs []db.BuildLog) ? { + data := logs.map([it.id.str(), it.repo_id.str(), it.start_time.str(), + it.exit_code.str()]) + + println(console.pretty_table(['id', 'repo', 'start time', 'exit code'], data)?) +} + +// list prints a list of all build logs. +fn list(conf Config) ? { + c := client.new(conf.address, conf.api_key) + logs := c.get_build_logs()?.data + + print_log_list(logs)? +} + +// list prints a list of all build logs for a given repo. +fn list_for_repo(conf Config, repo_id int) ? { + c := client.new(conf.address, conf.api_key) + logs := c.get_build_logs_for_repo(repo_id)?.data + + print_log_list(logs)? +} + +// info print the detailed info for a given build log. +fn info(conf Config, id int) ? { + c := client.new(conf.address, conf.api_key) + log := c.get_build_log(id)?.data + + print(log) +} + +// content outputs the contents of the log file for a given build log to +// stdout. +fn content(conf Config, id int) ? { + c := client.new(conf.address, conf.api_key) + content := c.get_build_log_content(id)? + + println(content) +} diff --git a/src/cron/cli.v b/src/cron/cli.v index 15bc986..9703c66 100644 --- a/src/cron/cli.v +++ b/src/cron/cli.v @@ -23,10 +23,10 @@ pub fn cmd() cli.Command { name: 'cron' description: 'Start the cron service that periodically runs builds.' execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? + config_file := cmd.flags.get_string('config-file')? + conf := env.load(config_file)? - cron(conf) ? + cron(conf)? } } } diff --git a/src/cron/cron.v b/src/cron/cron.v index e356faa..5f128cf 100644 --- a/src/cron/cron.v +++ b/src/cron/cron.v @@ -27,7 +27,7 @@ pub fn cron(conf Config) ? { } mut d := daemon.init_daemon(logger, conf.address, conf.api_key, conf.base_image, ce, - conf.max_concurrent_builds, conf.api_update_frequency, conf.image_rebuild_frequency) ? + conf.max_concurrent_builds, conf.api_update_frequency, conf.image_rebuild_frequency)? d.run() } diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index e54a39e..aa08f9f 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -3,6 +3,7 @@ module daemon import time import sync.stdatomic import build +import os const ( build_empty = 0 @@ -77,13 +78,20 @@ fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { // 0 means success, 1 means failure mut status := 0 - build.build_repo(d.address, d.api_key, d.builder_images.last(), &sb.repo) or { + res := build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(), + &sb.repo) or { d.ldebug('build_repo error: $err.msg()') status = 1 + + build.BuildResult{} } if status == 0 { - d.linfo('finished build: $sb.repo.url $sb.repo.branch') + d.linfo('finished build: $sb.repo.url $sb.repo.branch; uploading logs...') + + build_arch := os.uname().machine + d.client.add_build_log(sb.repo.id, res.start_time, res.end_time, build_arch, res.exit_code, + res.logs) or { d.lerror('Failed to upload logs for $sb.repo.url $sb.repo.arch') } } else { d.linfo('failed build: $sb.repo.url $sb.repo.branch') } diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index ffa2f6e..da3b46e 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -1,6 +1,5 @@ module daemon -import git import time import log import datatypes { MinHeap } @@ -10,6 +9,7 @@ import build import docker import db import os +import client const ( // How many seconds to wait before retrying to update API if failed @@ -31,8 +31,7 @@ fn (r1 ScheduledBuild) < (r2 ScheduledBuild) bool { pub struct Daemon { mut: - address string - api_key string + client client.Client base_image string builder_images []string global_schedule CronExpression @@ -56,8 +55,7 @@ mut: // populates the build queue for the first time. pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon { mut d := Daemon{ - address: address - api_key: api_key + client: client.new(address, api_key) base_image: base_image global_schedule: global_schedule api_update_frequency: api_update_frequency @@ -180,7 +178,7 @@ fn (mut d Daemon) schedule_build(repo db.GitRepo) { fn (mut d Daemon) renew_repos() { d.linfo('Renewing repos...') - mut new_repos := git.get_repos(d.address, d.api_key) or { + mut new_repos := d.client.get_git_repos() or { d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...') d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) @@ -255,14 +253,21 @@ fn (mut d Daemon) rebuild_base_image() bool { fn (mut d Daemon) clean_old_base_images() { mut i := 0 + mut dd := docker.new_conn() or { + d.lerror('Failed to connect to Docker socket.') + return + } + + defer { + dd.close() or {} + } + for i < d.builder_images.len - 1 { // For each builder image, we try to remove it by calling the Docker // API. If the function returns an error or false, that means the image // wasn't deleted. Therefore, we move the index over. If the function // returns true, the array's length has decreased by one so we don't // move the index. - if !docker.remove_image(d.builder_images[i]) or { false } { - i += 1 - } + dd.remove_image(d.builder_images[i]) or { i += 1 } } } diff --git a/src/cron/expression/expression.v b/src/cron/expression/expression.v index 124337f..5eae332 100644 --- a/src/cron/expression/expression.v +++ b/src/cron/expression/expression.v @@ -218,7 +218,7 @@ fn parse_part(s string, min int, max int) ?[]int { mut bitv := []bool{len: max - min + 1, init: false} for range in s.split(',') { - parse_range(range, min, max, mut bitv) ? + parse_range(range, min, max, mut bitv)? } return bitv_to_ints(bitv, min) diff --git a/src/cron/expression/expression_parse_test.v b/src/cron/expression/expression_parse_test.v index 18531c0..4eebc49 100644 --- a/src/cron/expression/expression_parse_test.v +++ b/src/cron/expression/expression_parse_test.v @@ -13,14 +13,14 @@ fn parse_range_error(s string, min int, max int) string { // =====parse_range===== fn test_range_star_range() ? { mut bitv := []bool{len: 6, init: false} - parse_range('*', 0, 5, mut bitv) ? + parse_range('*', 0, 5, mut bitv)? assert bitv == [true, true, true, true, true, true] } fn test_range_number() ? { mut bitv := []bool{len: 6, init: false} - parse_range('4', 0, 5, mut bitv) ? + parse_range('4', 0, 5, mut bitv)? assert bitv_to_ints(bitv, 0) == [4] } @@ -39,14 +39,14 @@ fn test_range_number_invalid() ? { fn test_range_step_star_1() ? { mut bitv := []bool{len: 21, init: false} - parse_range('*/4', 0, 20, mut bitv) ? + parse_range('*/4', 0, 20, mut bitv)? assert bitv_to_ints(bitv, 0) == [0, 4, 8, 12, 16, 20] } fn test_range_step_star_2() ? { mut bitv := []bool{len: 8, init: false} - parse_range('*/3', 1, 8, mut bitv) ? + parse_range('*/3', 1, 8, mut bitv)? assert bitv_to_ints(bitv, 1) == [1, 4, 7] } @@ -61,7 +61,7 @@ fn test_range_step_zero() ? { fn test_range_step_number() ? { mut bitv := []bool{len: 21, init: false} - parse_range('5/4', 2, 22, mut bitv) ? + parse_range('5/4', 2, 22, mut bitv)? assert bitv_to_ints(bitv, 2) == [5, 9, 13, 17, 21] } @@ -76,23 +76,23 @@ fn test_range_step_number_too_small() ? { fn test_range_dash() ? { mut bitv := []bool{len: 10, init: false} - parse_range('4-8', 0, 9, mut bitv) ? + parse_range('4-8', 0, 9, mut bitv)? assert bitv_to_ints(bitv, 0) == [4, 5, 6, 7, 8] } fn test_range_dash_step() ? { mut bitv := []bool{len: 10, init: false} - parse_range('4-8/2', 0, 9, mut bitv) ? + parse_range('4-8/2', 0, 9, mut bitv)? assert bitv_to_ints(bitv, 0) == [4, 6, 8] } // =====parse_part===== fn test_part_single() ? { - assert parse_part('*', 0, 5) ? == [0, 1, 2, 3, 4, 5] + assert parse_part('*', 0, 5)? == [0, 1, 2, 3, 4, 5] } fn test_part_multiple() ? { - assert parse_part('*/2,2/3', 1, 8) ? == [1, 2, 3, 5, 7, 8] + assert parse_part('*/2,2/3', 1, 8)? == [1, 2, 3, 5, 7, 8] } diff --git a/src/cron/expression/expression_test.v b/src/cron/expression/expression_test.v index ef0283a..9e25e92 100644 --- a/src/cron/expression/expression_test.v +++ b/src/cron/expression/expression_test.v @@ -3,11 +3,11 @@ module expression import time { parse } fn util_test_time(exp string, t1_str string, t2_str string) ? { - ce := parse_expression(exp) ? - t1 := parse(t1_str) ? - t2 := parse(t2_str) ? + ce := parse_expression(exp)? + t1 := parse(t1_str)? + t2 := parse(t2_str)? - t3 := ce.next(t1) ? + t3 := ce.next(t1)? assert t2.year == t3.year assert t2.month == t3.month @@ -18,17 +18,17 @@ fn util_test_time(exp string, t1_str string, t2_str string) ? { fn test_next_simple() ? { // Very simple - util_test_time('0 3', '2002-01-01 00:00:00', '2002-01-01 03:00:00') ? + util_test_time('0 3', '2002-01-01 00:00:00', '2002-01-01 03:00:00')? // Overlap to next day - util_test_time('0 3', '2002-01-01 03:00:00', '2002-01-02 03:00:00') ? - util_test_time('0 3', '2002-01-01 04:00:00', '2002-01-02 03:00:00') ? + util_test_time('0 3', '2002-01-01 03:00:00', '2002-01-02 03:00:00')? + util_test_time('0 3', '2002-01-01 04:00:00', '2002-01-02 03:00:00')? - util_test_time('0 3/4', '2002-01-01 04:00:00', '2002-01-01 07:00:00') ? + util_test_time('0 3/4', '2002-01-01 04:00:00', '2002-01-01 07:00:00')? // Overlap to next month - util_test_time('0 3', '2002-11-31 04:00:00', '2002-12-01 03:00:00') ? + util_test_time('0 3', '2002-11-31 04:00:00', '2002-12-01 03:00:00')? // Overlap to next year - util_test_time('0 3', '2002-12-31 04:00:00', '2003-01-01 03:00:00') ? + util_test_time('0 3', '2002-12-31 04:00:00', '2003-01-01 03:00:00')? } diff --git a/src/db/db.v b/src/db/db.v index a75c34c..7c1acf1 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -8,10 +8,11 @@ struct VieterDb { // init initializes a database & adds the correct tables. pub fn init(db_path string) ?VieterDb { - conn := sqlite.connect(db_path) ? + conn := sqlite.connect(db_path)? sql conn { create table GitRepo + create table BuildLog } return VieterDb{ diff --git a/src/db/git.v b/src/db/git.v index c40086b..9a475a5 100644 --- a/src/db/git.v +++ b/src/db/git.v @@ -94,7 +94,7 @@ pub fn (db &VieterDb) get_git_repo(repo_id int) ?GitRepo { // If a select statement fails, it returns a zeroed object. By // checking one of the required fields, we can see whether the query // returned a result or not. - if res.url == '' { + if res.id == 0 { return none } @@ -152,3 +152,11 @@ pub fn (db &VieterDb) update_git_repo_archs(repo_id int, archs []GitRepoArch) { } } } + +// git_repo_exists is a utility function that checks whether a repo with the +// given id exists. +pub fn (db &VieterDb) git_repo_exists(repo_id int) bool { + db.get_git_repo(repo_id) or { return false } + + return true +} diff --git a/src/db/logs.v b/src/db/logs.v new file mode 100644 index 0000000..817db78 --- /dev/null +++ b/src/db/logs.v @@ -0,0 +1,74 @@ +module db + +import time + +pub struct BuildLog { +pub: + id int [primary; sql: serial] + repo_id int [nonull] + start_time time.Time [nonull] + end_time time.Time [nonull] + arch string [nonull] + exit_code int [nonull] +} + +// str returns a string representation. +pub fn (bl &BuildLog) str() string { + mut parts := [ + 'id: $bl.id', + 'repo id: $bl.repo_id', + 'start time: $bl.start_time', + 'end time: $bl.end_time', + 'arch: $bl.arch', + 'exit code: $bl.exit_code', + ] + str := parts.join('\n') + + return str +} + +// get_build_logs returns all BuildLog's in the database. +pub fn (db &VieterDb) get_build_logs() []BuildLog { + res := sql db.conn { + select from BuildLog order by id + } + + return res +} + +// get_build_logs_for_repo returns all BuildLog's in the database for a given +// repo. +pub fn (db &VieterDb) get_build_logs_for_repo(repo_id int) []BuildLog { + res := sql db.conn { + select from BuildLog where repo_id == repo_id order by id + } + + return res +} + +// get_build_log tries to return a specific BuildLog. +pub fn (db &VieterDb) get_build_log(id int) ?BuildLog { + res := sql db.conn { + select from BuildLog where id == id + } + + if res.id == 0 { + return none + } + + return res +} + +// add_build_log inserts the given BuildLog into the database. +pub fn (db &VieterDb) add_build_log(log BuildLog) { + sql db.conn { + insert log into BuildLog + } +} + +// delete_build_log delete the BuildLog with the given ID from the database. +pub fn (db &VieterDb) delete_build_log(id int) { + sql db.conn { + delete from BuildLog where id == id + } +} diff --git a/src/docker/README.md b/src/docker/README.md new file mode 100644 index 0000000..4cc8971 --- /dev/null +++ b/src/docker/README.md @@ -0,0 +1,3 @@ +This module implements part of the Docker Engine API v1.41 +([documentation](https://docs.docker.com/engine/api/v1.41/)) using socket-based +HTTP communication. diff --git a/src/docker/containers.v b/src/docker/containers.v index d0f5a4d..0bc59bb 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -2,17 +2,11 @@ module docker import json import net.urllib +import time +import net.http { Method } -struct Container { - id string [json: Id] - names []string [json: Names] -} - -// containers returns a list of all currently running containers -pub fn containers() ?[]Container { - res := request('GET', urllib.parse('/v1.41/containers/json') ?) ? - - return json.decode([]Container, res.text) or {} +struct DockerError { + message string } pub struct NewContainer { @@ -25,54 +19,104 @@ pub struct NewContainer { } struct CreatedContainer { - id string [json: Id] +pub: + id string [json: Id] + warnings []string [json: Warnings] } -// create_container creates a container defined by the given configuration. If -// successful, it returns the ID of the newly created container. -pub fn create_container(c &NewContainer) ?string { - res := request_with_json('POST', urllib.parse('/v1.41/containers/create') ?, c) ? +// create_container creates a new container with the given config. +pub fn (mut d DockerConn) create_container(c NewContainer) ?CreatedContainer { + d.send_request_with_json(Method.post, urllib.parse('/v1.41/containers/create')?, c)? + head, res := d.read_response()? - if res.status_code != 201 { - return error('Failed to create container.') + if head.status_code != 201 { + data := json.decode(DockerError, res)? + + return error(data.message) } - return json.decode(CreatedContainer, res.text) ?.id + data := json.decode(CreatedContainer, res)? + + return data } -// start_container starts a container with a given ID. It returns whether the -// container was started or not. -pub fn start_container(id string) ?bool { - res := request('POST', urllib.parse('/v1.41/containers/$id/start') ?) ? +// start_container starts the container with the given id. +pub fn (mut d DockerConn) start_container(id string) ? { + d.send_request(Method.post, urllib.parse('/v1.41/containers/$id/start')?)? + head, body := d.read_response()? - return res.status_code == 204 + if head.status_code != 204 { + data := json.decode(DockerError, body)? + + return error(data.message) + } } struct ContainerInspect { -pub: +pub mut: state ContainerState [json: State] } struct ContainerState { pub: - running bool [json: Running] + running bool [json: Running] + status string [json: Status] + exit_code int [json: ExitCode] + // These use a rather specific format so they have to be parsed later + start_time_str string [json: StartedAt] + end_time_str string [json: FinishedAt] +pub mut: + start_time time.Time [skip] + end_time time.Time [skip] } -// inspect_container returns the result of inspecting a container with a given -// ID. -pub fn inspect_container(id string) ?ContainerInspect { - res := request('GET', urllib.parse('/v1.41/containers/$id/json') ?) ? +// inspect_container returns detailed information for a given container. +pub fn (mut d DockerConn) inspect_container(id string) ?ContainerInspect { + d.send_request(Method.get, urllib.parse('/v1.41/containers/$id/json')?)? + head, body := d.read_response()? - if res.status_code != 200 { - return error('Failed to inspect container.') + if head.status_code != 200 { + data := json.decode(DockerError, body)? + + return error(data.message) } - return json.decode(ContainerInspect, res.text) or {} + mut data := json.decode(ContainerInspect, body)? + + data.state.start_time = time.parse_rfc3339(data.state.start_time_str)? + + if data.state.status == 'exited' { + data.state.end_time = time.parse_rfc3339(data.state.end_time_str)? + } + + return data } -// remove_container removes a container with a given ID. -pub fn remove_container(id string) ?bool { - res := request('DELETE', urllib.parse('/v1.41/containers/$id') ?) ? +// remove_container removes the container with the given id. +pub fn (mut d DockerConn) remove_container(id string) ? { + d.send_request(Method.delete, urllib.parse('/v1.41/containers/$id')?)? + head, body := d.read_response()? - return res.status_code == 204 + if head.status_code != 204 { + data := json.decode(DockerError, body)? + + return error(data.message) + } +} + +// get_container_logs returns a reader object allowing access to the +// container's logs. +pub fn (mut d DockerConn) get_container_logs(id string) ?&StreamFormatReader { + d.send_request(Method.get, urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true')?)? + head := d.read_response_head()? + + if head.status_code != 200 { + content_length := head.header.get(http.CommonHeader.content_length)?.int() + body := d.read_response_body(content_length)? + data := json.decode(DockerError, body)? + + return error(data.message) + } + + return d.get_stream_format_reader() } diff --git a/src/docker/docker.v b/src/docker/docker.v index 5deef83..ccc6bed 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -1,97 +1,137 @@ module docker import net.unix -import net.urllib +import io import net.http +import strings +import net.urllib import json +import util -const socket = '/var/run/docker.sock' +const ( + socket = '/var/run/docker.sock' + buf_len = 10 * 1024 + http_separator = [u8(`\r`), `\n`, `\r`, `\n`] + http_chunk_separator = [u8(`\r`), `\n`] +) -const buf_len = 1024 +pub struct DockerConn { +mut: + socket &unix.StreamConn + reader &io.BufferedReader +} -// send writes a request to the Docker socket, waits for a response & returns -// it. -fn send(req &string) ?http.Response { - // Open a connection to the socket - mut s := unix.connect_stream(docker.socket) or { - return error('Failed to connect to socket ${docker.socket}.') +// new_conn creates a new connection to the Docker daemon. +pub fn new_conn() ?&DockerConn { + s := unix.connect_stream(docker.socket)? + + d := &DockerConn{ + socket: s + reader: io.new_buffered_reader(reader: s) } - defer { - // This or is required because otherwise, the V compiler segfaults for - // some reason - // https://github.com/vlang/v/issues/13534 - s.close() or {} - } + return d +} - // Write the request to the socket - s.write_string(req) or { return error('Failed to write request to socket ${docker.socket}.') } +// close closes the underlying socket connection. +pub fn (mut d DockerConn) close() ? { + d.socket.close()? +} - s.wait_for_write() ? +// send_request sends an HTTP request without body. +pub fn (mut d DockerConn) send_request(method http.Method, url urllib.URL) ? { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' - mut c := 0 - mut buf := []u8{len: docker.buf_len} + d.socket.write_string(req)? + + // When starting a new request, the reader needs to be reset. + d.reader = io.new_buffered_reader(reader: d.socket) +} + +// send_request_with_body sends an HTTP request with the given body. +pub fn (mut d DockerConn) send_request_with_body(method http.Method, url urllib.URL, content_type string, body string) ? { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' + + d.socket.write_string(req)? + + // When starting a new request, the reader needs to be reset. + d.reader = io.new_buffered_reader(reader: d.socket) +} + +// send_request_with_json is a convenience wrapper around +// send_request_with_body that encodes the input as JSON. +pub fn (mut d DockerConn) send_request_with_json(method http.Method, url urllib.URL, data &T) ? { + body := json.encode(data) + + return d.send_request_with_body(method, url, 'application/json', body) +} + +// read_response_head consumes the socket's contents until it encounters +// '\r\n\r\n', after which it parses the response as an HTTP response. +// Importantly, this function never consumes the reader past the HTTP +// separator, so the body can be read fully later on. +pub fn (mut d DockerConn) read_response_head() ?http.Response { mut res := []u8{} - for { - c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } - res << buf[..c] + util.read_until_separator(mut d.reader, mut res, docker.http_separator)? - if c < docker.buf_len { - break - } - } - - // After reading the first part of the response, we parse it into an HTTP - // response. If it isn't chunked, we return early with the data. - parsed := http.parse_response(res.bytestr()) or { - return error('Failed to parse HTTP response from socket ${docker.socket}.') - } - - if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' { - return parsed - } - - // We loop until we've encountered the end of the chunked response - // A chunked HTTP response always ends with '0\r\n\r\n'. - for res.len < 5 || res#[-5..] != [u8(`0`), `\r`, `\n`, `\r`, `\n`] { - // Wait for the server to respond - s.wait_for_write() ? - - for { - c = s.read(mut buf) or { - return error('Failed to read data from socket ${docker.socket}.') - } - res << buf[..c] - - if c < docker.buf_len { - break - } - } - } - - // Decode chunked response return http.parse_response(res.bytestr()) } -// request_with_body sends a request to the Docker socket with the given body. -fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' +// read_response_body reads `length` bytes from the stream. It can be used when +// the response encoding isn't chunked to fully read it. +pub fn (mut d DockerConn) read_response_body(length int) ?string { + if length == 0 { + return '' + } - return send(req) + mut buf := []u8{len: docker.buf_len} + mut c := 0 + mut builder := strings.new_builder(docker.buf_len) + + for builder.len < length { + c = d.reader.read(mut buf) or { break } + + builder.write(buf[..c])? + } + + return builder.str() } -// request sends a request to the Docker socket with an empty body. -fn request(method string, url urllib.URL) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' +// read_response is a convenience function which always consumes the entire +// response & returns it. It should only be used when we're certain that the +// result isn't too large. +pub fn (mut d DockerConn) read_response() ?(http.Response, string) { + head := d.read_response_head()? - return send(req) + if head.header.get(http.CommonHeader.transfer_encoding) or { '' } == 'chunked' { + mut builder := strings.new_builder(1024) + mut body := d.get_chunked_response_reader() + + util.reader_to_writer(mut body, mut builder)? + + return head, builder.str() + } + + content_length := head.header.get(http.CommonHeader.content_length)?.int() + res := d.read_response_body(content_length)? + + return head, res } -// request_with_json sends a request to the Docker socket with a given JSON -// payload -pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Response { - body := json.encode(data) +// get_chunked_response_reader returns a ChunkedResponseReader using the socket +// as reader. +pub fn (mut d DockerConn) get_chunked_response_reader() &ChunkedResponseReader { + r := new_chunked_response_reader(d.reader) - return request_with_body(method, url, 'application/json', body) + return r +} + +// get_stream_format_reader returns a StreamFormatReader using the socket as +// reader. +pub fn (mut d DockerConn) get_stream_format_reader() &StreamFormatReader { + r := new_chunked_response_reader(d.reader) + r2 := new_stream_format_reader(r) + + return r2 } diff --git a/src/docker/images.v b/src/docker/images.v index e94ceca..6161565 100644 --- a/src/docker/images.v +++ b/src/docker/images.v @@ -1,6 +1,6 @@ module docker -import net.http +import net.http { Method } import net.urllib import json @@ -9,26 +9,53 @@ pub: id string [json: Id] } -// pull_image pulls tries to pull the image for the given image & tag -pub fn pull_image(image string, tag string) ?http.Response { - return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) -} +// pull_image pulls the given image:tag. +pub fn (mut d DockerConn) pull_image(image string, tag string) ? { + d.send_request(Method.post, urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?)? + head := d.read_response_head()? -// create_image_from_container creates a new image from a container with the -// given repo & tag, given the container's ID. -pub fn create_image_from_container(id string, repo string, tag string) ?Image { - res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag') ?) ? + if head.status_code != 200 { + content_length := head.header.get(http.CommonHeader.content_length)?.int() + body := d.read_response_body(content_length)? + data := json.decode(DockerError, body)? - if res.status_code != 201 { - return error('Failed to create image from container.') + return error(data.message) } - return json.decode(Image, res.text) or {} + // Keep reading the body until the pull has completed + mut body := d.get_chunked_response_reader() + + mut buf := []u8{len: 1024} + + for { + body.read(mut buf) or { break } + } } -// remove_image removes the image with the given ID. -pub fn remove_image(id string) ?bool { - res := request('DELETE', urllib.parse('/v1.41/images/$id') ?) ? +// create_image_from_container creates a new image from a container. +pub fn (mut d DockerConn) create_image_from_container(id string, repo string, tag string) ?Image { + d.send_request(Method.post, urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag')?)? + head, body := d.read_response()? - return res.status_code == 200 + if head.status_code != 201 { + data := json.decode(DockerError, body)? + + return error(data.message) + } + + data := json.decode(Image, body)? + + return data +} + +// remove_image removes the image with the given id. +pub fn (mut d DockerConn) remove_image(id string) ? { + d.send_request(Method.delete, urllib.parse('/v1.41/images/$id')?)? + head, body := d.read_response()? + + if head.status_code != 200 { + data := json.decode(DockerError, body)? + + return error(data.message) + } } diff --git a/src/docker/stream.v b/src/docker/stream.v new file mode 100644 index 0000000..001f4b3 --- /dev/null +++ b/src/docker/stream.v @@ -0,0 +1,135 @@ +module docker + +import io +import util +import encoding.binary +import encoding.hex + +// ChunkedResponseReader parses an underlying HTTP chunked response, exposing +// it as if it was a continuous stream of data. +struct ChunkedResponseReader { +mut: + reader io.BufferedReader + bytes_left_in_chunk u64 + started bool +} + +// new_chunked_response_reader creates a new ChunkedResponseReader on the heap +// with the provided reader. +pub fn new_chunked_response_reader(reader io.BufferedReader) &ChunkedResponseReader { + r := &ChunkedResponseReader{ + reader: reader + } + + return r +} + +// read satisfies the io.Reader interface. +pub fn (mut r ChunkedResponseReader) read(mut buf []u8) ?int { + if r.bytes_left_in_chunk == 0 { + // An io.BufferedReader always returns none if its stream has + // ended. + r.bytes_left_in_chunk = r.read_chunk_size()? + } + + mut c := 0 + + // Make sure we don't read more than we can safely read. This is to avoid + // the underlying reader from becoming out of sync with our parsing: + if buf.len > r.bytes_left_in_chunk { + c = r.reader.read(mut buf[..r.bytes_left_in_chunk])? + } else { + c = r.reader.read(mut buf)? + } + + r.bytes_left_in_chunk -= u64(c) + + return c +} + +// read_chunk_size advances the reader & reads the size of the next HTTP chunk. +// This function should only be called if the previous chunk has been +// completely consumed. +fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { + if r.started { + mut buf := []u8{len: 2} + + // Each chunk ends with a `\r\n` which we want to skip first + r.reader.read(mut buf)? + } + + r.started = true + + mut res := []u8{} + util.read_until_separator(mut r.reader, mut res, http_chunk_separator)? + + // The length of the next chunk is provided as a hexadecimal + mut num_data := hex.decode(res#[..-2].bytestr())? + + for num_data.len < 8 { + num_data.insert(0, 0) + } + + num := binary.big_endian_u64(num_data) + + // This only occurs for the very last chunk, which always reports a size of + // 0. + if num == 0 { + return none + } + + return num +} + +// StreamFormatReader parses an underlying stream of Docker logs, removing the +// header bytes. +struct StreamFormatReader { +mut: + reader ChunkedResponseReader + bytes_left_in_chunk u32 +} + +// new_stream_format_reader creates a new StreamFormatReader using the given +// reader. +pub fn new_stream_format_reader(reader ChunkedResponseReader) &StreamFormatReader { + r := &StreamFormatReader{ + reader: reader + } + + return r +} + +// read satisfies the io.Reader interface. +pub fn (mut r StreamFormatReader) read(mut buf []u8) ?int { + if r.bytes_left_in_chunk == 0 { + r.bytes_left_in_chunk = r.read_chunk_size()? + } + + mut c := 0 + + if buf.len > r.bytes_left_in_chunk { + c = r.reader.read(mut buf[..r.bytes_left_in_chunk])? + } else { + c = r.reader.read(mut buf)? + } + + r.bytes_left_in_chunk -= u32(c) + + return c +} + +// read_chunk_size advances the reader & reads the header bytes for the length +// of the next chunk. +fn (mut r StreamFormatReader) read_chunk_size() ?u32 { + mut buf := []u8{len: 8} + + r.reader.read(mut buf)? + + num := binary.big_endian_u32(buf[4..]) + + if num == 0 { + return none + } + + return num +} diff --git a/src/env/README.md b/src/env/README.md new file mode 100644 index 0000000..135e8fa --- /dev/null +++ b/src/env/README.md @@ -0,0 +1,7 @@ +This module provides a framework for parsing a configuration, defined as a +struct, from both a TOML configuration file & environment variables. Some +notable features are: + +* Overwrite values in config file using environment variables +* Allow default values in config struct +* Read environment variable value from file diff --git a/src/env/env.v b/src/env/env.v index b2b5f44..5ed1955 100644 --- a/src/env/env.v +++ b/src/env/env.v @@ -3,13 +3,19 @@ module env import os import toml -// The prefix that every environment variable should have -const prefix = 'VIETER_' - -// The suffix an environment variable in order for it to be loaded from a file -// instead -const file_suffix = '_FILE' +const ( + // The prefix that every environment variable should have + prefix = 'VIETER_' + // The suffix an environment variable in order for it to be loaded from a file + // instead + file_suffix = '_FILE' +) +// get_env_var tries to read the contents of the given environment variable. It +// looks for either `${env.prefix}${field_name.to_upper()}` or +// `${env.prefix}${field_name.to_upper()}${env.file_suffix}`, returning the +// contents of the file instead if the latter. If both or neither exist, the +// function returns an error. fn get_env_var(field_name string) ?string { env_var_name := '$env.prefix$field_name.to_upper()' env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix' @@ -50,7 +56,7 @@ pub fn load(path string) ?T { if os.exists(path) { // We don't use reflect here because reflect also sets any fields not // in the toml back to their zero value, which we don't want - doc := toml.parse_file(path) ? + doc := toml.parse_file(path)? $for field in T.fields { s := doc.value(field.name) @@ -66,7 +72,7 @@ pub fn load(path string) ?T { } $for field in T.fields { - env_value := get_env_var(field.name) ? + env_value := get_env_var(field.name)? // The value of an env var will always take precedence over the toml // file. diff --git a/src/git/client.v b/src/git/client.v deleted file mode 100644 index b5f8e9f..0000000 --- a/src/git/client.v +++ /dev/null @@ -1,77 +0,0 @@ -module git - -import json -import response { Response } -import net.http -import db - -// send_request is a convenience method for sending requests to the repos -// API. It mostly does string manipulation to create a query string containing -// the provided params. -fn send_request(method http.Method, address string, url string, api_key string, params map[string]string) ?Response { - mut full_url := '$address$url' - - if params.len > 0 { - params_str := params.keys().map('$it=${params[it]}').join('&') - - full_url = '$full_url?$params_str' - } - - mut req := http.new_request(method, full_url, '') ? - req.add_custom_header('X-API-Key', api_key) ? - - res := req.do() ? - data := json.decode(Response, res.text) ? - - return data -} - -// get_repos returns the current list of repos. -pub fn get_repos(address string, api_key string) ?[]db.GitRepo { - data := send_request<[]db.GitRepo>(http.Method.get, address, '/api/repos', api_key, - {}) ? - - return data.data -} - -// get_repo returns the repo for a specific ID. -pub fn get_repo(address string, api_key string, id int) ?db.GitRepo { - data := send_request(http.Method.get, address, '/api/repos/$id', api_key, - {}) ? - - return data.data -} - -// add_repo adds a new repo to the server. -pub fn add_repo(address string, api_key string, url string, branch string, repo string, arch []string) ?Response { - mut params := { - 'url': url - 'branch': branch - 'repo': repo - } - - if arch.len > 0 { - params['arch'] = arch.join(',') - } - - data := send_request(http.Method.post, address, '/api/repos', api_key, params) ? - - return data -} - -// remove_repo removes the repo with the given ID from the server. -pub fn remove_repo(address string, api_key string, id int) ?Response { - data := send_request(http.Method.delete, address, '/api/repos/$id', api_key, - {}) ? - - return data -} - -// patch_repo sends a PATCH request to the given repo with the params as -// payload. -pub fn patch_repo(address string, api_key string, id int, params map[string]string) ?Response { - data := send_request(http.Method.patch, address, '/api/repos/$id', api_key, - params) ? - - return data -} diff --git a/src/main.v b/src/main.v index 4ba6d30..6df45dc 100644 --- a/src/main.v +++ b/src/main.v @@ -3,15 +3,15 @@ module main import os import server import cli -import build -import git +import console.git +import console.logs import cron fn main() { mut app := cli.Command{ name: 'vieter' description: 'Vieter is a lightweight implementation of an Arch repository server.' - version: '0.3.0-alpha.1' + version: '0.3.0-alpha.2' flags: [ cli.Flag{ flag: cli.FlagType.string @@ -24,12 +24,12 @@ fn main() { ] commands: [ server.cmd(), - build.cmd(), git.cmd(), cron.cmd(), + logs.cmd(), ] } - app.setup() app.parse(os.args) + return } diff --git a/src/package/package.v b/src/package/package.v index a1042b5..273322f 100644 --- a/src/package/package.v +++ b/src/package/package.v @@ -159,7 +159,7 @@ pub fn read_pkg_archive(pkg_path string) ?Pkg { pkg_text := unsafe { buf.vstring_with_len(size).clone() } - pkg_info = parse_pkg_info_string(pkg_text) ? + pkg_info = parse_pkg_info_string(pkg_text)? } else { C.archive_read_data_skip(a) } diff --git a/src/repo/repo.v b/src/repo/repo.v index e27e232..817ec30 100644 --- a/src/repo/repo.v +++ b/src/repo/repo.v @@ -53,22 +53,22 @@ pub fn (r &RepoGroupManager) add_pkg_from_path(repo string, pkg_path string) ?Re return error('Failed to read package file: $err.msg()') } - added := r.add_pkg_in_repo(repo, pkg) ? + added := r.add_pkg_in_repo(repo, pkg)? // If the add was successful, we move the file to the packages directory for arch in added { repo_pkg_path := os.real_path(os.join_path(r.pkg_dir, repo, arch)) dest_path := os.join_path_single(repo_pkg_path, pkg.filename()) - os.mkdir_all(repo_pkg_path) ? + os.mkdir_all(repo_pkg_path)? // We create hard links so that "any" arch packages aren't stored // multiple times - os.link(pkg_path, dest_path) ? + os.link(pkg_path, dest_path)? } // After linking, we can remove the original file - os.rm(pkg_path) ? + os.rm(pkg_path)? return RepoAddResult{ added: added.len > 0 @@ -87,7 +87,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin // A package not of arch 'any' can be handled easily by adding it to the // respective repo if pkg.info.arch != 'any' { - if r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg) ? { + if r.add_pkg_in_arch_repo(repo, pkg.info.arch, pkg)? { return [pkg.info.arch] } else { return [] @@ -104,7 +104,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin // If this is the first package that's added to the repo, the directory // won't exist yet if os.exists(repo_dir) { - arch_repos = os.ls(repo_dir) ? + arch_repos = os.ls(repo_dir)? } // The default_arch should always be updated when a package with arch 'any' @@ -118,7 +118,7 @@ fn (r &RepoGroupManager) add_pkg_in_repo(repo string, pkg &package.Pkg) ?[]strin // We add the package to each repository. If any of the repositories // return true, the result of the function is also true. for arch in arch_repos { - if r.add_pkg_in_arch_repo(repo, arch, pkg) ? { + if r.add_pkg_in_arch_repo(repo, arch, pkg)? { added << arch } } @@ -135,22 +135,22 @@ fn (r &RepoGroupManager) add_pkg_in_arch_repo(repo string, arch string, pkg &pac pkg_dir := os.join_path(r.repos_dir, repo, arch, '$pkg.info.name-$pkg.info.version') // Remove the previous version of the package, if present - r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false) ? + r.remove_pkg_from_arch_repo(repo, arch, pkg.info.name, false)? os.mkdir_all(pkg_dir) or { return error('Failed to create package directory.') } os.write_file(os.join_path_single(pkg_dir, 'desc'), pkg.to_desc()) or { - os.rmdir_all(pkg_dir) ? + os.rmdir_all(pkg_dir)? return error('Failed to write desc file.') } os.write_file(os.join_path_single(pkg_dir, 'files'), pkg.to_files()) or { - os.rmdir_all(pkg_dir) ? + os.rmdir_all(pkg_dir)? return error('Failed to write files file.') } - r.sync(repo, arch) ? + r.sync(repo, arch)? return true } @@ -168,7 +168,7 @@ fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg // We iterate over every directory in the repo dir // TODO filter so we only check directories - for d in os.ls(repo_dir) ? { + for d in os.ls(repo_dir)? { // Because a repository only allows a single version of each package, // we need only compare whether the name of the package is the same, // not the version. @@ -178,22 +178,22 @@ fn (r &RepoGroupManager) remove_pkg_from_arch_repo(repo string, arch string, pkg // We lock the mutex here to prevent other routines from creating a // new archive while we remove an entry lock r.mutex { - os.rmdir_all(os.join_path_single(repo_dir, d)) ? + os.rmdir_all(os.join_path_single(repo_dir, d))? } // Also remove the package archive repo_pkg_dir := os.join_path(r.pkg_dir, repo, arch) - archives := os.ls(repo_pkg_dir) ?.filter(it.split('-')#[..-3].join('-') == name) + archives := os.ls(repo_pkg_dir)?.filter(it.split('-')#[..-3].join('-') == name) for archive_name in archives { full_path := os.join_path_single(repo_pkg_dir, archive_name) - os.rm(full_path) ? + os.rm(full_path)? } // Sync the db archives if requested if sync { - r.sync(repo, arch) ? + r.sync(repo, arch)? } return true diff --git a/src/repo/sync.v b/src/repo/sync.v index 9c5e7ed..73d21c8 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -54,7 +54,7 @@ fn (r &RepoGroupManager) sync(repo string, arch string) ? { C.archive_write_open_filename(a_files, &char(files_path.str)) // Iterate over each directory - for d in os.ls(subrepo_path) ?.filter(os.is_dir(os.join_path_single(subrepo_path, + for d in os.ls(subrepo_path)?.filter(os.is_dir(os.join_path_single(subrepo_path, it))) { // desc mut inner_path := os.join_path_single(d, 'desc') diff --git a/src/server/cli.v b/src/server/cli.v index 4d39666..556efcf 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -18,10 +18,10 @@ pub fn cmd() cli.Command { name: 'server' description: 'Start the Vieter server.' execute: fn (cmd cli.Command) ? { - config_file := cmd.flags.get_string('config-file') ? - conf := env.load(config_file) ? + config_file := cmd.flags.get_string('config-file')? + conf := env.load(config_file)? - server(conf) ? + server(conf)? } } } diff --git a/src/server/logs.v b/src/server/logs.v new file mode 100644 index 0000000..21331e5 --- /dev/null +++ b/src/server/logs.v @@ -0,0 +1,136 @@ +module server + +import web +import net.http +import net.urllib +import response { new_data_response, new_response } +import db +import time +import os +import util + +// get_logs returns all build logs in the database. A 'repo' query param can +// optionally be added to limit the list of build logs to that repository. +['/api/logs'; get] +fn (mut app App) get_logs() web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + logs := if 'repo' in app.query { + app.db.get_build_logs_for_repo(app.query['repo'].int()) + } else { + app.db.get_build_logs() + } + + return app.json(http.Status.ok, new_data_response(logs)) +} + +// get_single_log returns the build log with the given id. +['/api/logs/:id'; get] +fn (mut app App) get_single_log(id int) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + log := app.db.get_build_log(id) or { return app.not_found() } + + return app.json(http.Status.ok, new_data_response(log)) +} + +// get_log_content returns the actual build log file for the given id. +['/api/logs/:id/content'; get] +fn (mut app App) get_log_content(id int) web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + log := app.db.get_build_log(id) or { return app.not_found() } + file_name := log.start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + full_path := os.join_path(app.conf.data_dir, logs_dir_name, log.repo_id.str(), log.arch, + file_name) + + return app.file(full_path) +} + +// parse_query_time unescapes an HTTP query parameter & tries to parse it as a +// time.Time struct. +fn parse_query_time(query string) ?time.Time { + unescaped := urllib.query_unescape(query)? + t := time.parse(unescaped)? + + return t +} + +// post_log adds a new log to the database. +['/api/logs'; post] +fn (mut app App) post_log() web.Result { + if !app.is_authorized() { + return app.json(http.Status.unauthorized, new_response('Unauthorized.')) + } + + // Parse query params + start_time := parse_query_time(app.query['startTime']) or { + return app.json(http.Status.bad_request, new_response('Invalid or missing start time.')) + } + + end_time := parse_query_time(app.query['endTime']) or { + return app.json(http.Status.bad_request, new_response('Invalid or missing end time.')) + } + + if 'exitCode' !in app.query { + return app.json(http.Status.bad_request, new_response('Missing exit code.')) + } + + exit_code := app.query['exitCode'].int() + + if 'arch' !in app.query { + return app.json(http.Status.bad_request, new_response("Missing parameter 'arch'.")) + } + + arch := app.query['arch'] + + repo_id := app.query['repo'].int() + + if !app.db.git_repo_exists(repo_id) { + return app.json(http.Status.bad_request, new_response('Unknown Git repo.')) + } + + // Store log in db + log := db.BuildLog{ + repo_id: repo_id + start_time: start_time + end_time: end_time + arch: arch + exit_code: exit_code + } + + app.db.add_build_log(log) + + repo_logs_dir := os.join_path(app.conf.data_dir, logs_dir_name, repo_id.str(), arch) + + // Create the logs directory of it doesn't exist + if !os.exists(repo_logs_dir) { + os.mkdir_all(repo_logs_dir) or { + app.lerror("Couldn't create dir '$repo_logs_dir'.") + + return app.json(http.Status.internal_server_error, new_response('An error occured while processing the request.')) + } + } + + // Stream log contents to correct file + file_name := start_time.custom_format('YYYY-MM-DD_HH-mm-ss') + full_path := os.join_path_single(repo_logs_dir, file_name) + + if length := app.req.header.get(.content_length) { + util.reader_to_file(mut app.reader, length.int(), full_path) or { + app.lerror('An error occured while receiving logs: $err.msg()') + + return app.json(http.Status.internal_server_error, new_response('Failed to upload logs.')) + } + } else { + return app.status(http.Status.length_required) + } + + return app.json(http.Status.ok, new_response('Logs added successfully.')) +} diff --git a/src/server/server.v b/src/server/server.v index b2a2ad2..090aa76 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -12,6 +12,7 @@ const ( log_file_name = 'vieter.log' repo_dir_name = 'repos' db_file_name = 'vieter.sqlite' + logs_dir_name = 'logs' ) struct App { @@ -37,6 +38,14 @@ pub fn server(conf Config) ? { os.mkdir_all(conf.data_dir) or { util.exit_with_message(1, 'Failed to create data directory.') } + logs_dir := os.join_path_single(conf.data_dir, server.logs_dir_name) + + if !os.exists(logs_dir) { + os.mkdir(os.join_path_single(conf.data_dir, server.logs_dir_name)) or { + util.exit_with_message(1, 'Failed to create logs directory.') + } + } + mut logger := log.Log{ level: log_level } diff --git a/src/util/README.md b/src/util/README.md new file mode 100644 index 0000000..529e412 --- /dev/null +++ b/src/util/README.md @@ -0,0 +1,2 @@ +This module defines a few useful functions used throughout the codebase that +don't specifically fit inside a module. diff --git a/src/util/stream.v b/src/util/stream.v new file mode 100644 index 0000000..06397aa --- /dev/null +++ b/src/util/stream.v @@ -0,0 +1,95 @@ +// Functions for interacting with `io.Reader` & `io.Writer` objects. +module util + +import io +import os + +// reader_to_writer tries to consume the entire reader & write it to the writer. +pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ? { + mut buf := []u8{len: 10 * 1024} + + for { + bytes_read := reader.read(mut buf) or { break } + mut bytes_written := 0 + + for bytes_written < bytes_read { + c := writer.write(buf[bytes_written..bytes_read]) or { break } + + bytes_written += c + } + } +} + +// reader_to_file writes the contents of a BufferedReader to a file +pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { + mut file := os.create(path)? + defer { + file.close() + } + + mut buf := []u8{len: reader_buf_size} + mut bytes_left := length + + // Repeat as long as the stream still has data + for bytes_left > 0 { + // TODO check if just breaking here is safe + bytes_read := reader.read(mut buf) or { break } + bytes_left -= bytes_read + + mut to_write := bytes_read + + for to_write > 0 { + // TODO don't just loop infinitely here + bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue } + // file.flush() + + to_write = to_write - bytes_written + } + } +} + +// match_array_in_array returns how many elements of a2 overlap with a1. For +// example, if a1 = "abcd" & a2 = "cd", the result will be 2. If the match is +// not at the end of a1, the result is 0. +pub fn match_array_in_array(a1 []T, a2 []T) int { + mut i := 0 + mut match_len := 0 + + for i + match_len < a1.len { + if a1[i + match_len] == a2[match_len] { + match_len += 1 + } else { + i += match_len + 1 + match_len = 0 + } + } + + return match_len +} + +// read_until_separator consumes an io.Reader until it encounters some +// separator array. The data read is stored inside the provided res array. +pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ? { + mut buf := []u8{len: sep.len} + + for { + c := reader.read(mut buf)? + res << buf[..c] + + match_len := match_array_in_array(buf[..c], sep) + + if match_len == sep.len { + break + } + + if match_len > 0 { + match_left := sep.len - match_len + c2 := reader.read(mut buf[..match_left])? + res << buf[..c2] + + if buf[..c2] == sep[match_len..] { + break + } + } + } +} diff --git a/src/util/util.v b/src/util/util.v index c1af30e..266bcb5 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -1,13 +1,13 @@ module util import os -import io import crypto.md5 import crypto.sha256 -const reader_buf_size = 1_000_000 - -const prefixes = ['B', 'KB', 'MB', 'GB'] +const ( + reader_buf_size = 1_000_000 + prefixes = ['B', 'KB', 'MB', 'GB'] +) // Dummy struct to work around the fact that you can only share structs, maps & // arrays @@ -23,34 +23,6 @@ pub fn exit_with_message(code int, msg string) { exit(code) } -// reader_to_file writes the contents of a BufferedReader to a file -pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { - mut file := os.create(path) ? - defer { - file.close() - } - - mut buf := []u8{len: util.reader_buf_size} - mut bytes_left := length - - // Repeat as long as the stream still has data - for bytes_left > 0 { - // TODO check if just breaking here is safe - bytes_read := reader.read(mut buf) or { break } - bytes_left -= bytes_read - - mut to_write := bytes_read - - for to_write > 0 { - // TODO don't just loop infinitely here - bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue } - // file.flush() - - to_write = to_write - bytes_written - } - } -} - // hash_file returns the md5 & sha256 hash of a given file // TODO actually implement sha256 pub fn hash_file(path &string) ?(string, string) { diff --git a/src/web/web.v b/src/web/web.v index 3e7b047..f237e24 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -190,7 +190,7 @@ pub fn (ctx Context) before_request() {} // send_string fn send_string(mut conn net.TcpConn, s string) ? { - conn.write(s.bytes()) ? + conn.write(s.bytes())? } // send_response_to_client sends a response to the client