diff --git a/.editorconfig b/.editorconfig index 630e4fa..355c6bf 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,5 +7,4 @@ end_of_line = lf insert_final_newline = true [*.v] -# vfmt wants it :( -indent_style = tab +indent_style = space diff --git a/CHANGELOG.md b/CHANGELOG.md index 1166377..3824088 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,20 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/vieter) -## Added - -* Very basic build system - * Build is triggered by separate cron container - * Packages build on cron container's system - * Packages are always rebuilt, even if they haven't changed - * Hardcoded planning of builds - * Builds are sequential - -## Fixed - -* Each package can now only have one version in the repository at once - (required by Pacman) - ## [0.1.0](https://git.rustybever.be/Chewing_Bever/vieter/src/tag/0.1.0) ### Changed diff --git a/Dockerfile.ci b/Dockerfile.ci index e0dd5da..24f2bef 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -36,14 +36,11 @@ HEALTHCHECK --interval=30s \ CMD /bin/wget --spider http://localhost:8000/health || exit 1 RUN mkdir /data && \ - chown -R www-data:www-data /data && \ - mkdir -p '/var/spool/cron/crontabs' && \ - echo '0 3 * * * /bin/vieter build' >> /var/spool/cron/crontabs/www-data && \ - chown www-data:www-data /var/spool/cron/crontabs/www-data + chown -R www-data:www-data /data WORKDIR /data USER www-data:www-data ENTRYPOINT ["/bin/dumb-init", "--"] -CMD ["/bin/vieter", "server"] +CMD ["/bin/vieter"] diff --git a/Makefile b/Makefile index d69e48e..d7efc45 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ c: # Run the server in the default 'data' directory .PHONY: run run: vieter - API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter server + API_KEY=test DOWNLOAD_DIR=data/downloads REPO_DIR=data/repo PKG_DIR=data/pkgs LOG_LEVEL=DEBUG ./vieter .PHONY: run-prod run-prod: prod diff --git a/src/build.v b/src/build.v deleted file mode 100644 index 00cf9ed..0000000 --- a/src/build.v +++ /dev/null @@ -1,84 +0,0 @@ -module main - -import docker -import encoding.base64 -import rand -import time -import os -import json -import git - -const container_build_dir = '/build' - -fn build(key string, repo_dir string) ? { - server_url := os.getenv_opt('VIETER_ADDRESS') or { - exit_with_message(1, 'No Vieter server address was provided.') - } - - // Read in the repos from a json file - filename := os.join_path_single(repo_dir, 'repos.json') - txt := os.read_file(filename) ? - repos := json.decode([]git.GitRepo, txt) ? - - mut commands := [ - // Update repos & install required packages - 'pacman -Syu --needed --noconfirm base-devel git' - // Add a non-root user to run makepkg - 'groupadd -g 1000 builder', - 'useradd -mg builder builder' - // Make sure they can use sudo without a password - "echo 'builder ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers" - // Create the directory for the builds & make it writeable for the - // build user - 'mkdir /build', - 'chown -R builder:builder /build', - ] - - // Each repo gets a unique UUID to avoid naming conflicts when cloning - mut uuids := []string{} - - for repo in repos { - mut uuid := rand.uuid_v4() - - // Just to be sure we don't have any collisions - for uuids.contains(uuid) { - uuid = rand.uuid_v4() - } - - uuids << uuid - - commands << "su builder -c 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url /build/$uuid'" - commands << 'su builder -c \'cd /build/$uuid && makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\${pkg}" -H "X-API-KEY: \$API_KEY" $server_url/publish; done\'' - } - - // We convert the list of commands into a base64 string, which then gets - // passed to the container as an env var - cmds_str := base64.encode_str(commands.join('\n')) - - c := docker.NewContainer{ - image: 'archlinux:latest' - env: ['BUILD_SCRIPT=$cmds_str', 'API_KEY=$key'] - entrypoint: ['/bin/sh', '-c'] - cmd: ['echo \$BUILD_SCRIPT | base64 -d | /bin/sh -e'] - } - - // First, we pull the latest archlinux image - docker.pull_image('archlinux', 'latest') ? - - id := docker.create_container(c) ? - docker.start_container(id) ? - - // This loop waits until the container has stopped, so we can remove it after - for { - data := docker.inspect_container(id) ? - - if !data.state.running { - break - } - - // Wait for 5 seconds - time.sleep(5000000000) - } - - docker.remove_container(id) ? -} diff --git a/src/docker/containers.v b/src/docker/containers.v deleted file mode 100644 index a6df345..0000000 --- a/src/docker/containers.v +++ /dev/null @@ -1,76 +0,0 @@ -module docker - -import json -import net.urllib - -struct Container { - id string [json: Id] - names []string [json: Names] -} - -// containers returns a list of all currently running containers -pub fn containers() ?[]Container { - res := request('GET', urllib.parse('/v1.41/containers/json') ?) ? - - return json.decode([]Container, res.text) or {} -} - -pub struct NewContainer { - image string [json: Image] - entrypoint []string [json: Entrypoint] - cmd []string [json: Cmd] - env []string [json: Env] -} - -struct CreatedContainer { - id string [json: Id] -} - -// create_container creates a container defined by the given configuration. If -// successful, it returns the ID of the newly created container. -pub fn create_container(c &NewContainer) ?string { - res := request_with_json('POST', urllib.parse('/v1.41/containers/create') ?, c) ? - - if res.status_code != 201 { - return error('Failed to create container.') - } - - return json.decode(CreatedContainer, res.text) ?.id -} - -// start_container starts a container with a given ID. It returns whether the -// container was started or not. -pub fn start_container(id string) ?bool { - res := request('POST', urllib.parse('/v1.41/containers/$id/start') ?) ? - - return res.status_code == 204 -} - -struct ContainerInspect { -pub: - state ContainerState [json: State] -} - -struct ContainerState { -pub: - running bool [json: Running] -} - -// inspect_container returns the result of inspecting a container with a given -// ID. -pub fn inspect_container(id string) ?ContainerInspect { - res := request('GET', urllib.parse('/v1.41/containers/$id/json') ?) ? - - if res.status_code != 200 { - return error('Failed to inspect container.') - } - - return json.decode(ContainerInspect, res.text) or {} -} - -// remove_container removes a container with a given ID. -pub fn remove_container(id string) ?bool { - res := request('DELETE', urllib.parse('/v1.41/containers/$id') ?) ? - - return res.status_code == 204 -} diff --git a/src/docker/docker.v b/src/docker/docker.v deleted file mode 100644 index e0dbf7d..0000000 --- a/src/docker/docker.v +++ /dev/null @@ -1,98 +0,0 @@ -module docker - -import net.unix -import net.urllib -import net.http -import json - -const socket = '/var/run/docker.sock' - -const buf_len = 1024 - -fn send(req &string) ?http.Response { - // Open a connection to the socket - mut s := unix.connect_stream(docker.socket) or { - return error('Failed to connect to socket ${docker.socket}.') - } - - defer { - // This or is required because otherwise, the V compiler segfaults for - // some reason - // https://github.com/vlang/v/issues/13534 - s.close() or {} - } - - // Write the request to the socket - s.write_string(req) or { return error('Failed to write request to socket ${docker.socket}.') } - - s.wait_for_write() ? - - mut c := 0 - mut buf := []byte{len: docker.buf_len} - mut res := []byte{} - - for { - c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } - res << buf[..c] - - if c < docker.buf_len { - break - } - } - - // After reading the first part of the response, we parse it into an HTTP - // response. If it isn't chunked, we return early with the data. - parsed := http.parse_response(res.bytestr()) or { - return error('Failed to parse HTTP response from socket ${docker.socket}.') - } - - if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' { - return parsed - } - - // We loop until we've encountered the end of the chunked response - // A chunked HTTP response always ends with '0\r\n\r\n'. - for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] { - // Wait for the server to respond - s.wait_for_write() ? - - for { - c = s.read(mut buf) or { - return error('Failed to read data from socket ${docker.socket}.') - } - res << buf[..c] - - if c < docker.buf_len { - break - } - } - } - - // Decode chunked response - return http.parse_response(res.bytestr()) -} - -fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' - - return send(req) -} - -fn request(method string, url urllib.URL) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' - - return send(req) -} - -// request_with_json sends a request to the Docker socket with a given JSON -// payload -pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Response { - body := json.encode(data) - - return request_with_body(method, url, 'application/json', body) -} - -// pull_image pulls tries to pull the image for the given image & tag -pub fn pull_image(image string, tag string) ?http.Response { - return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag') ?) -} diff --git a/src/git.v b/src/git.v deleted file mode 100644 index 76d80d7..0000000 --- a/src/git.v +++ /dev/null @@ -1,7 +0,0 @@ -module git - -pub struct GitRepo { -pub: - url string [required] - branch string [required] -} diff --git a/src/main.v b/src/main.v index cc66b59..cddd9ae 100644 --- a/src/main.v +++ b/src/main.v @@ -2,6 +2,7 @@ module main import web import os +import log import io import repo @@ -9,6 +10,8 @@ const port = 8000 const buf_size = 1_000_000 +const db_name = 'pieter.db' + struct App { web.Context pub: @@ -51,18 +54,50 @@ fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { } fn main() { + // Configure logger + log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } + log_level := log.level_from_tag(log_level_str) or { + exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') + } + log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } + + mut logger := log.Log{ + level: log_level + } + + logger.set_full_logpath(log_file) + logger.log_to_console_too() + + defer { + logger.info('Flushing log file') + logger.flush() + logger.close() + } + + // Configure web server key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } repo_dir := os.getenv_opt('REPO_DIR') or { exit_with_message(1, 'No repo directory was configured.') } - - if os.args.len == 1 { - exit_with_message(1, 'No action provided.') + pkg_dir := os.getenv_opt('PKG_DIR') or { + exit_with_message(1, 'No package directory was configured.') + } + dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { + exit_with_message(1, 'No download directory was configured.') } - match os.args[1] { - 'server' { server(key, repo_dir) } - 'build' { build(key, repo_dir) ? } - else { exit_with_message(1, 'Unknown action: ${os.args[1]}') } + // This also creates the directories if needed + repo := repo.new(repo_dir, pkg_dir) or { + logger.error(err.msg) + exit(1) } + + os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } + + web.run(&App{ + logger: logger + api_key: key + dl_dir: dl_dir + repo: repo + }, port) } diff --git a/src/server.v b/src/server.v deleted file mode 100644 index e0f22ec..0000000 --- a/src/server.v +++ /dev/null @@ -1,51 +0,0 @@ -module main - -import web -import os -import log -import repo - -fn server(key string, repo_dir string) { - // Configure logger - log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } - log_level := log.level_from_tag(log_level_str) or { - exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') - } - log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } - - mut logger := log.Log{ - level: log_level - } - - logger.set_full_logpath(log_file) - logger.log_to_console_too() - - defer { - logger.info('Flushing log file') - logger.flush() - logger.close() - } - - // Configure web server - pkg_dir := os.getenv_opt('PKG_DIR') or { - exit_with_message(1, 'No package directory was configured.') - } - dl_dir := os.getenv_opt('DOWNLOAD_DIR') or { - exit_with_message(1, 'No download directory was configured.') - } - - // This also creates the directories if needed - repo := repo.new(repo_dir, pkg_dir) or { - logger.error(err.msg) - exit(1) - } - - os.mkdir_all(dl_dir) or { exit_with_message(1, 'Failed to create download directory.') } - - web.run(&App{ - logger: logger - api_key: key - dl_dir: dl_dir - repo: repo - }, port) -}