From 10377632d5ba45c625005538bc081ded6e906842 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 15 Apr 2022 11:38:06 +0200 Subject: [PATCH 01/28] Make vieter compiler with -skip-unused --- .gitignore | 4 ++-- .woodpecker/.build.yml | 2 ++ src/server/git.v | 10 +++++----- src/server/routes.v | 6 +++--- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 7847b3f..a3f6afc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,8 @@ data/ vieter dvieter pvieter -dvieterctl -vieterctl +suvieter +afvieter vieter.c # Ignore testing files diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index e68c4c9..66ae5a0 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -18,6 +18,8 @@ pipeline: - make debug when: event: push + branch: + exclude: [main, dev] prod: image: 'chewingbever/vlang:latest' diff --git a/src/server/git.v b/src/server/git.v index a9d6f50..a7655fe 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -8,7 +8,7 @@ import response { new_data_response, new_response } const repos_file = 'repos.json' -['/api/repos'; get] +['/api/repos'; get; markused] fn (mut app App) get_repos() web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) @@ -25,7 +25,7 @@ fn (mut app App) get_repos() web.Result { return app.json(http.Status.ok, new_data_response(repos)) } -['/api/repos/:id'; get] +['/api/repos/:id'; get; markused] fn (mut app App) get_single_repo(id string) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) @@ -48,7 +48,7 @@ fn (mut app App) get_single_repo(id string) web.Result { return app.json(http.Status.ok, new_data_response(repo)) } -['/api/repos'; post] +['/api/repos'; post; markused] fn (mut app App) post_repo() web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) @@ -86,7 +86,7 @@ fn (mut app App) post_repo() web.Result { return app.json(http.Status.ok, new_response('Repo added successfully.')) } -['/api/repos/:id'; delete] +['/api/repos/:id'; delete; markused] fn (mut app App) delete_repo(id string) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) @@ -113,7 +113,7 @@ fn (mut app App) delete_repo(id string) web.Result { return app.json(http.Status.ok, new_response('Repo removed successfully.')) } -['/api/repos/:id'; patch] +['/api/repos/:id'; patch; markused] fn (mut app App) patch_repo(id string) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) diff --git a/src/server/routes.v b/src/server/routes.v index 4f6c4f0..b048f76 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -11,12 +11,12 @@ import response { new_response } // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. -['/health'; get] +['/health'; get; markused] pub fn (mut app App) healthcheck() web.Result { return app.json(http.Status.ok, new_response('Healthy.')) } -['/:repo/:arch/:filename'; get; head] +['/:repo/:arch/:filename'; get; head; markused] fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result { mut full_path := '' @@ -54,7 +54,7 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re return app.file(full_path) } -['/:repo/publish'; post] +['/:repo/publish'; post; markused] fn (mut app App) put_package(repo string) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) From c6813f1e0dacb205df2b5adf15a36dbbea0fd880 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 15 Apr 2022 11:46:14 +0200 Subject: [PATCH 02/28] Make skip-unused build prod; better ci information --- .woodpecker/.build_experimental.yml | 3 +++ Makefile | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.woodpecker/.build_experimental.yml b/.woodpecker/.build_experimental.yml index 032a42b..faea53d 100644 --- a/.woodpecker/.build_experimental.yml +++ b/.woodpecker/.build_experimental.yml @@ -38,5 +38,8 @@ pipeline: - make skip-unused - readelf -d suvieter - du -h suvieter + - '[ "$(readelf -d suvieter | grep NEEDED | wc -l)" = 0 ]' + - strip -s suvieter + - du -h suvieter when: event: push diff --git a/Makefile b/Makefile index 2f39983..6f39a40 100644 --- a/Makefile +++ b/Makefile @@ -87,4 +87,4 @@ afvieter: $(SOURCES) .PHONY: skip-unused skip-unused: suvieter suvieter: $(SOURCES) - $(V_PATH) -showcc -skip-unused -o suvieter $(SRC_DIR) + $(V_PATH) -showcc -skip-unused -prod -o suvieter $(SRC_DIR) From 2c62f6bda1dd2e0e9d27f8afce57fd11f15ec629 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 15 Apr 2022 11:54:56 +0200 Subject: [PATCH 03/28] Added boehm garbage collector to skip-unused --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6f39a40..ab67823 100644 --- a/Makefile +++ b/Makefile @@ -75,7 +75,7 @@ v/v: make -C v clean: - rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst + rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' # =====EXPERIMENTAL===== @@ -87,4 +87,4 @@ afvieter: $(SOURCES) .PHONY: skip-unused skip-unused: suvieter suvieter: $(SOURCES) - $(V_PATH) -showcc -skip-unused -prod -o suvieter $(SRC_DIR) + $(V_PATH) -showcc -gc boehm -skip-unused -prod -o suvieter $(SRC_DIR) From 32d542102fd03fabc0d9cfdae5882525c9027ef7 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 17 Apr 2022 16:41:47 +0200 Subject: [PATCH 04/28] Removed references to byte; enabled -skip-unused by default --- .woodpecker/.build_experimental.yml | 27 --------------------------- Makefile | 9 ++------- src/docker/docker.v | 6 +++--- src/repo/sync.v | 2 +- src/util/util.v | 4 ++-- src/web/web.v | 2 +- 6 files changed, 9 insertions(+), 41 deletions(-) diff --git a/.woodpecker/.build_experimental.yml b/.woodpecker/.build_experimental.yml index faea53d..2c0e20b 100644 --- a/.woodpecker/.build_experimental.yml +++ b/.woodpecker/.build_experimental.yml @@ -16,30 +16,3 @@ pipeline: - du -h afvieter when: event: push - - skip-unused: - image: 'chewingbever/vlang:latest' - pull: true - group: 'build' - commands: - - make skip-unused - - readelf -d suvieter - - du -h suvieter - when: - event: push - - skip-unused-static: - image: 'chewingbever/vlang:latest' - pull: true - environment: - - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static - group: 'build' - commands: - - make skip-unused - - readelf -d suvieter - - du -h suvieter - - '[ "$(readelf -d suvieter | grep NEEDED | wc -l)" = 0 ]' - - strip -s suvieter - - du -h suvieter - when: - event: push diff --git a/Makefile b/Makefile index ab67823..cc9c829 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' V_PATH ?= v -V := $(V_PATH) -showcc -gc boehm +V := $(V_PATH) -showcc -gc boehm -skip-unused all: vieter @@ -19,7 +19,7 @@ vieter: $(SOURCES) .PHONY: debug debug: dvieter dvieter: $(SOURCES) - $(V_PATH) -showcc -keepc -cg -o dvieter $(SRC_DIR) + $(V_PATH) -showcc -keepc -cg -skip-unused -o dvieter $(SRC_DIR) # Run the debug build inside gdb .PHONY: gdb @@ -83,8 +83,3 @@ clean: autofree: afvieter afvieter: $(SOURCES) $(V_PATH) -showcc -autofree -o afvieter $(SRC_DIR) - -.PHONY: skip-unused -skip-unused: suvieter -suvieter: $(SOURCES) - $(V_PATH) -showcc -gc boehm -skip-unused -prod -o suvieter $(SRC_DIR) diff --git a/src/docker/docker.v b/src/docker/docker.v index a6f7640..07ceb8e 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -28,8 +28,8 @@ fn send(req &string) ?http.Response { s.wait_for_write() ? mut c := 0 - mut buf := []byte{len: docker.buf_len} - mut res := []byte{} + mut buf := []u8{len: docker.buf_len} + mut res := []u8{} for { c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } @@ -52,7 +52,7 @@ fn send(req &string) ?http.Response { // We loop until we've encountered the end of the chunked response // A chunked HTTP response always ends with '0\r\n\r\n'. - for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] { + for res.len < 5 || res#[-5..] != [u8(`0`), `\r`, `\n`, `\r`, `\n`] { // Wait for the server to respond s.wait_for_write() ? diff --git a/src/repo/sync.v b/src/repo/sync.v index e2b7aac..12756b7 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -19,7 +19,7 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri } // Write the file to the archive - buf := [8192]byte{} + buf := [8192]u8{} mut len := C.read(fd, &buf, sizeof(buf)) for len > 0 { diff --git a/src/util/util.v b/src/util/util.v index 228f584..c1af30e 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -30,7 +30,7 @@ pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { file.close() } - mut buf := []byte{len: util.reader_buf_size} + mut buf := []u8{len: util.reader_buf_size} mut bytes_left := length // Repeat as long as the stream still has data @@ -60,7 +60,7 @@ pub fn hash_file(path &string) ?(string, string) { mut sha256sum := sha256.new() buf_size := int(1_000_000) - mut buf := []byte{len: buf_size} + mut buf := []u8{len: buf_size} mut bytes_left := os.file_size(path) for bytes_left > 0 { diff --git a/src/web/web.v b/src/web/web.v index 688f854..3e7b047 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -285,7 +285,7 @@ pub fn (mut ctx Context) file(f_path string) Result { resp.set_status(ctx.status) send_string(mut ctx.conn, resp.bytestr()) or { return Result{} } - mut buf := []byte{len: 1_000_000} + mut buf := []u8{len: 1_000_000} mut bytes_left := file_size // Repeat as long as the stream still has data From 05efe15dd95b686c81ffd48d3b62536005ad3da1 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 09:07:16 +0200 Subject: [PATCH 05/28] Updated PKGBUILD to use vieter-v [CI SKIP] --- .woodpecker/.arch.yml | 2 ++ PKGBUILD | 9 +++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index ab3c6ea..e37dc1a 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -10,6 +10,8 @@ pipeline: build: image: 'menci/archlinuxarm:base-devel' commands: + # Add the vieter repository so we can use the compiler + - echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf # Update packages - pacman -Syu --noconfirm # Create non-root user to perform build & switch to their home diff --git a/PKGBUILD b/PKGBUILD index 0c558b4..3f8c480 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -2,10 +2,10 @@ pkgbase='vieter' pkgname='vieter' -pkgver=0.2.0.r24.g9a56bd0 +pkgver=0.2.0.r25.g20112b8 pkgrel=1 depends=('glibc' 'openssl' 'libarchive' 'gc') -makedepends=('git' 'gcc') +makedepends=('git' 'gcc' 'vieter-v') arch=('x86_64' 'aarch64' 'armv7') url='https://git.rustybever.be/Chewing_Bever/vieter' license=('AGPL3') @@ -20,10 +20,7 @@ pkgver() { build() { cd "$pkgname" - # Build the compiler - CFLAGS= make v - - V_PATH=v/v make prod + make prod } package() { From 15d21e3f1e60e5206aa83adb788735de95fd5e60 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 09:52:23 +0200 Subject: [PATCH 06/28] cron: basic working loop; removed -skip-unused --- Makefile | 2 +- src/cron/daemon/build.v | 36 +++++++++++++++++++------- src/cron/daemon/daemon.v | 43 +++++++++++++++++++++----------- src/cron/expression/expression.v | 2 +- src/server/git.v | 10 ++++---- src/server/routes.v | 6 ++--- 6 files changed, 66 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index cc9c829..8ddabbe 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' V_PATH ?= v -V := $(V_PATH) -showcc -gc boehm -skip-unused +V := $(V_PATH) -showcc -gc boehm all: vieter diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index e7e5ac3..73ba183 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -1,9 +1,25 @@ module daemon -import git import time import sync.stdatomic +const build_empty = 0 +const build_running = 1 +const build_done = 2 + +// reschedule_builds looks for any builds with status code 2 & re-adds them to +// the queue. +fn (mut d Daemon) reschedule_builds() ? { + for i in 0..d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == build_done { + stdatomic.store_u64(&d.atomics[i], build_empty) + sb := d.builds[i] + + d.schedule_build(sb.repo_id, sb.repo) ? + } + } +} + // update_builds starts as many builds as possible. fn (mut d Daemon) update_builds() ? { now := time.now() @@ -13,7 +29,7 @@ fn (mut d Daemon) update_builds() ? { sb := d.queue.pop() ? // If this build couldn't be scheduled, no more will be possible. - if !d.start_build(sb.repo_id)? { + if !d.start_build(sb)? { break } } else { @@ -22,13 +38,14 @@ fn (mut d Daemon) update_builds() ? { } } -// start_build starts a build for the given repo_id. -fn (mut d Daemon) start_build(repo_id string) ?bool { +// start_build starts a build for the given ScheduledBuild object. +fn (mut d Daemon) start_build(sb ScheduledBuild) ?bool { for i in 0..d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == 0 { - stdatomic.store_u64(&d.atomics[i], 1) + if stdatomic.load_u64(&d.atomics[i]) == build_empty { + stdatomic.store_u64(&d.atomics[i], build_running) + d.builds[i] = sb - go d.run_build(i, d.repos_map[repo_id]) + go d.run_build(i, sb) return true } @@ -37,9 +54,10 @@ fn (mut d Daemon) start_build(repo_id string) ?bool { return false } -fn (mut d Daemon) run_build(build_index int, repo git.GitRepo) ? { +// run_build actually starts the build process for a given repo. +fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) ? { time.sleep(10 * time.second) - stdatomic.store_u64(&d.atomics[build_index], 2) + stdatomic.store_u64(&d.atomics[build_index], build_done) } diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index fc917e4..816bc15 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -30,7 +30,7 @@ mut: api_update_timestamp time.Time queue MinHeap // Which builds are currently running - builds []git.GitRepo + builds []ScheduledBuild // Atomic variables used to detect when a build has finished; length is the // same as builds atomics []u64 @@ -47,7 +47,7 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st global_schedule: global_schedule api_update_frequency: api_update_frequency atomics: []u64{len: max_concurrent_builds} - builds: []git.GitRepo{len: max_concurrent_builds} + builds: []ScheduledBuild{len: max_concurrent_builds} logger: logger } @@ -62,14 +62,37 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st // periodically refreshes the list of repositories to ensure we stay in sync. pub fn (mut d Daemon) run() ? { for { + println('1') + // Cleans up finished builds, opening up spots for new builds + d.reschedule_builds() ? + println('2') + // Schedules new builds when possible d.update_builds() ? + println(d.queue) println(d.atomics) - time.sleep(60 * time.second) + time.sleep(10 * time.second) } } +// schedule_build adds the next occurence of the given repo build to the queue. +fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) ? { + ce := parse_expression(repo.schedule) or { + d.lerror("Error while parsing cron expression '$repo.schedule' ($repo_id): $err.msg()") + + d.global_schedule + } + // A repo that can't be scheduled will just be skipped for now + timestamp := ce.next_from_now() ? + + d.queue.insert(ScheduledBuild{ + repo_id: repo_id + repo: repo + timestamp: timestamp + }) +} + fn (mut d Daemon) renew_repos() ? { mut new_repos := git.get_repos(d.address, d.api_key) ? @@ -101,19 +124,11 @@ fn (mut d Daemon) renew_queue() ? { } } + d.queue = new_queue + // For each repository in repos_map, parse their cron expression (or use // the default one if not present) & add them to the queue for id, repo in d.repos_map { - ce := parse_expression(repo.schedule) or { d.global_schedule } - // A repo that can't be scheduled will just be skipped for now - timestamp := ce.next(now) or { continue } - - new_queue.insert(ScheduledBuild{ - repo_id: id - repo: repo - timestamp: timestamp - }) + d.schedule_build(id, repo) ? } - - d.queue = new_queue } diff --git a/src/cron/expression/expression.v b/src/cron/expression/expression.v index c122585..6e11da2 100644 --- a/src/cron/expression/expression.v +++ b/src/cron/expression/expression.v @@ -114,7 +114,7 @@ pub fn (ce &CronExpression) next(ref time.Time) ?time.Time { }) } -fn (ce &CronExpression) next_from_now() ?time.Time { +pub fn (ce &CronExpression) next_from_now() ?time.Time { return ce.next(time.now()) } diff --git a/src/server/git.v b/src/server/git.v index a7655fe..a9d6f50 100644 --- a/src/server/git.v +++ b/src/server/git.v @@ -8,7 +8,7 @@ import response { new_data_response, new_response } const repos_file = 'repos.json' -['/api/repos'; get; markused] +['/api/repos'; get] fn (mut app App) get_repos() web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) @@ -25,7 +25,7 @@ fn (mut app App) get_repos() web.Result { return app.json(http.Status.ok, new_data_response(repos)) } -['/api/repos/:id'; get; markused] +['/api/repos/:id'; get] fn (mut app App) get_single_repo(id string) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) @@ -48,7 +48,7 @@ fn (mut app App) get_single_repo(id string) web.Result { return app.json(http.Status.ok, new_data_response(repo)) } -['/api/repos'; post; markused] +['/api/repos'; post] fn (mut app App) post_repo() web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) @@ -86,7 +86,7 @@ fn (mut app App) post_repo() web.Result { return app.json(http.Status.ok, new_response('Repo added successfully.')) } -['/api/repos/:id'; delete; markused] +['/api/repos/:id'; delete] fn (mut app App) delete_repo(id string) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) @@ -113,7 +113,7 @@ fn (mut app App) delete_repo(id string) web.Result { return app.json(http.Status.ok, new_response('Repo removed successfully.')) } -['/api/repos/:id'; patch; markused] +['/api/repos/:id'; patch] fn (mut app App) patch_repo(id string) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) diff --git a/src/server/routes.v b/src/server/routes.v index b048f76..4f6c4f0 100644 --- a/src/server/routes.v +++ b/src/server/routes.v @@ -11,12 +11,12 @@ import response { new_response } // healthcheck just returns a string, but can be used to quickly check if the // server is still responsive. -['/health'; get; markused] +['/health'; get] pub fn (mut app App) healthcheck() web.Result { return app.json(http.Status.ok, new_response('Healthy.')) } -['/:repo/:arch/:filename'; get; head; markused] +['/:repo/:arch/:filename'; get; head] fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Result { mut full_path := '' @@ -54,7 +54,7 @@ fn (mut app App) get_repo_file(repo string, arch string, filename string) web.Re return app.file(full_path) } -['/:repo/publish'; post; markused] +['/:repo/publish'; post] fn (mut app App) put_package(repo string) web.Result { if !app.is_authorized() { return app.json(http.Status.unauthorized, new_response('Unauthorized.')) From 0dca7374dbf0804f168f8a774850e753e06a675a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 09:58:54 +0200 Subject: [PATCH 07/28] Fixed some errors in CI --- .woodpecker/.build.yml | 3 +-- .woodpecker/.build_experimental.yml | 11 +++++++++++ Makefile | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 66ae5a0..c612737 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -9,13 +9,12 @@ matrix: platform: ${PLATFORM} pipeline: - # The default build isn't needed, as alpine switches to gcc for the compiler anyways debug: image: 'chewingbever/vlang:latest' pull: true group: 'build' commands: - - make debug + - make when: event: push branch: diff --git a/.woodpecker/.build_experimental.yml b/.woodpecker/.build_experimental.yml index 2c0e20b..0129d2b 100644 --- a/.woodpecker/.build_experimental.yml +++ b/.woodpecker/.build_experimental.yml @@ -16,3 +16,14 @@ pipeline: - du -h afvieter when: event: push + + skip-unused: + image: 'chewingbever/vlang:latest' + pull: true + group: 'build' + commands: + - make skip-unused + - readelf -d suvieter + - du -h suvieter + when: + event: push diff --git a/Makefile b/Makefile index 8ddabbe..fb044fa 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ vieter: $(SOURCES) .PHONY: debug debug: dvieter dvieter: $(SOURCES) - $(V_PATH) -showcc -keepc -cg -skip-unused -o dvieter $(SRC_DIR) + $(V_PATH) -showcc -keepc -cg -o dvieter $(SRC_DIR) # Run the debug build inside gdb .PHONY: gdb @@ -83,3 +83,8 @@ clean: autofree: afvieter afvieter: $(SOURCES) $(V_PATH) -showcc -autofree -o afvieter $(SRC_DIR) + +.PHONY: skip-unused +skip-unused: suvieter +skip-unused: $(SOURCES) + $(V_PATH) -showcc -skip-unused -o afvieter $(SRC_DIR) From dca7ccd649d58b3c51d812dbbffd70c628442438 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 15:51:39 +0200 Subject: [PATCH 08/28] ci: switch back to building debug --- .woodpecker/.build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index c612737..264872f 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -14,7 +14,7 @@ pipeline: pull: true group: 'build' commands: - - make + - make debug when: event: push branch: From efa67bf930411d8c8226f5424e888458a6323837 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 16:12:37 +0200 Subject: [PATCH 09/28] Compile boehm gc dynamically for debug --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index fb044fa..b665d4e 100644 --- a/Makefile +++ b/Makefile @@ -9,9 +9,10 @@ all: vieter # =====COMPILATION===== -# Regular binary +# We force the boehm gc to be compiled dynamically because otherwise, our CI +# build breaks. vieter: $(SOURCES) - $(V) -g -o vieter $(SRC_DIR) + $(V) -d dynamic_boehm -g -o vieter $(SRC_DIR) # Debug build using gcc # The debug build can't use the boehm garbage collector, as that is From 1b1b9d2edd7f049030f921036273733c73fbe508 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 16:13:24 +0200 Subject: [PATCH 10/28] ci: build regular binary again --- .woodpecker/.build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index 264872f..c612737 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -14,7 +14,7 @@ pipeline: pull: true group: 'build' commands: - - make debug + - make when: event: push branch: From 7df9d921f450bb59ac961d49b38e218063983825 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 16:15:01 +0200 Subject: [PATCH 11/28] Add dynamic boehm compiler to tests as well --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b665d4e..5480124 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ vet: .PHONY: test test: - $(V) test $(SRC_DIR) + $(V) -d dynamic_boehm test $(SRC_DIR) # Build & patch the V compiler .PHONY: v From 4be25ff356979ad4cf21c6d13862111f8c458c9d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 16:26:46 +0200 Subject: [PATCH 12/28] ci: added dynamic boehm prod build to experimental builds --- .woodpecker/.build_experimental.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.woodpecker/.build_experimental.yml b/.woodpecker/.build_experimental.yml index 0129d2b..3968e05 100644 --- a/.woodpecker/.build_experimental.yml +++ b/.woodpecker/.build_experimental.yml @@ -27,3 +27,22 @@ pipeline: - du -h suvieter when: event: push + + dynamic-boehm-prod: + image: 'chewingbever/vlang:latest' + pull: true + group: 'build' + environment: + - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static + - VFLAGS=-cc gcc -d dynamic_boehm + commands: + - make prod + # Make sure the binary is actually statically built + - readelf -d pvieter + - du -h pvieter + - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' + # This removes so much, it's amazing + - strip -s pvieter + - du -h pvieter + when: + event: push From 8e40481022506bec558b35730d7f8b11eaad7349 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 16:48:35 +0200 Subject: [PATCH 13/28] Use dynamic_boehm everywhere --- .woodpecker/.build_experimental.yml | 19 ------------------- Makefile | 6 +++--- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/.woodpecker/.build_experimental.yml b/.woodpecker/.build_experimental.yml index 3968e05..0129d2b 100644 --- a/.woodpecker/.build_experimental.yml +++ b/.woodpecker/.build_experimental.yml @@ -27,22 +27,3 @@ pipeline: - du -h suvieter when: event: push - - dynamic-boehm-prod: - image: 'chewingbever/vlang:latest' - pull: true - group: 'build' - environment: - - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static - - VFLAGS=-cc gcc -d dynamic_boehm - commands: - - make prod - # Make sure the binary is actually statically built - - readelf -d pvieter - - du -h pvieter - - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' - # This removes so much, it's amazing - - strip -s pvieter - - du -h pvieter - when: - event: push diff --git a/Makefile b/Makefile index 5480124..b9f369a 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' V_PATH ?= v -V := $(V_PATH) -showcc -gc boehm +V := $(V_PATH) -showcc -gc boehm -d dynamic_boehm all: vieter @@ -12,7 +12,7 @@ all: vieter # We force the boehm gc to be compiled dynamically because otherwise, our CI # build breaks. vieter: $(SOURCES) - $(V) -d dynamic_boehm -g -o vieter $(SRC_DIR) + $(V) -g -o vieter $(SRC_DIR) # Debug build using gcc # The debug build can't use the boehm garbage collector, as that is @@ -66,7 +66,7 @@ vet: .PHONY: test test: - $(V) -d dynamic_boehm test $(SRC_DIR) + $(V) test $(SRC_DIR) # Build & patch the V compiler .PHONY: v From 392a95bbbe9dfe56b6fad42cc0d6960058733fba Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 16:49:39 +0200 Subject: [PATCH 14/28] ci: run buils sequentially [CI SKIP] --- .woodpecker/.build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index c612737..e7341fd 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -12,7 +12,6 @@ pipeline: debug: image: 'chewingbever/vlang:latest' pull: true - group: 'build' commands: - make when: @@ -25,7 +24,6 @@ pipeline: pull: true environment: - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static - group: 'build' commands: - make prod # Make sure the binary is actually statically built From 8d1fbd306f93a10a763c42c7d512410d50835be5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 29 Apr 2022 10:34:12 +0200 Subject: [PATCH 15/28] Added docs command & notice in README [CI SKIP] --- .gitignore | 3 +++ Makefile | 8 +++++++- README.md | 4 +++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index a3f6afc..6a06eb2 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ v/ # gdb log file gdb.txt + +# Generated docs +_docs/ diff --git a/Makefile b/Makefile index b9f369a..db77c2e 100644 --- a/Makefile +++ b/Makefile @@ -75,8 +75,14 @@ v/v: git clone --single-branch https://git.rustybever.be/Chewing_Bever/v v make -C v +.PHONY: clean clean: - rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' + rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' + +.PHONY: docs +docs: + rm -rf '$(SRC_DIR)/_docs' + cd '$(SRC_DIR)' && v doc -all -f html -m -readme . # =====EXPERIMENTAL===== diff --git a/README.md b/README.md index 96b104d..08f1e75 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ ## Documentation -I host documentation for Vieter over at https://rustybever.be/docs/vieter/. +I host documentation for Vieter over at https://rustybever.be/docs/vieter/. API +documentation for the current codebase can be found at +https://rustybever.be/api-docs/vieter/. ## Overview From 46c068e07fc5192bef450b9ed27001297cbeacd9 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 30 Apr 2022 10:40:29 +0200 Subject: [PATCH 16/28] daemon: worked on daemon loop --- src/cron/cli.v | 5 +++-- src/cron/daemon/build.v | 23 +++++++++++++---------- src/cron/daemon/daemon.v | 38 +++++++++++++++++++++++++++++++++----- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/cron/cli.v b/src/cron/cli.v index f4b20ec..3b836dd 100644 --- a/src/cron/cli.v +++ b/src/cron/cli.v @@ -11,8 +11,9 @@ pub: address string base_image string = 'archlinux:base-devel' max_concurrent_builds int = 1 - api_update_frequency int = 60 - global_schedule string + api_update_frequency int = 15 + // Replicates the behavior of the original cron system + global_schedule string = '0 3' } // cmd returns the cli module that handles the cron daemon. diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index 73ba183..c5ef428 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -4,15 +4,17 @@ import time import sync.stdatomic const build_empty = 0 + const build_running = 1 + const build_done = 2 // reschedule_builds looks for any builds with status code 2 & re-adds them to // the queue. fn (mut d Daemon) reschedule_builds() ? { - for i in 0..d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == build_done { - stdatomic.store_u64(&d.atomics[i], build_empty) + for i in 0 .. d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == daemon.build_done { + stdatomic.store_u64(&d.atomics[i], daemon.build_empty) sb := d.builds[i] d.schedule_build(sb.repo_id, sb.repo) ? @@ -29,7 +31,8 @@ fn (mut d Daemon) update_builds() ? { sb := d.queue.pop() ? // If this build couldn't be scheduled, no more will be possible. - if !d.start_build(sb)? { + // TODO a build that couldn't be scheduled should be re-added to the queue. + if !d.start_build(sb) { break } } else { @@ -39,10 +42,10 @@ fn (mut d Daemon) update_builds() ? { } // start_build starts a build for the given ScheduledBuild object. -fn (mut d Daemon) start_build(sb ScheduledBuild) ?bool { - for i in 0..d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == build_empty { - stdatomic.store_u64(&d.atomics[i], build_running) +fn (mut d Daemon) start_build(sb ScheduledBuild) bool { + for i in 0 .. d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == daemon.build_empty { + stdatomic.store_u64(&d.atomics[i], daemon.build_running) d.builds[i] = sb go d.run_build(i, sb) @@ -56,8 +59,8 @@ fn (mut d Daemon) start_build(sb ScheduledBuild) ?bool { // run_build actually starts the build process for a given repo. fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) ? { + d.linfo('build $sb.repo.url') time.sleep(10 * time.second) - stdatomic.store_u64(&d.atomics[build_index], build_done) + stdatomic.store_u64(&d.atomics[build_index], daemon.build_done) } - diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 816bc15..7253e94 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -5,6 +5,8 @@ import time import log import datatypes { MinHeap } import cron.expression { CronExpression, parse_expression } +import math +import arrays struct ScheduledBuild { pub: @@ -62,23 +64,47 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st // periodically refreshes the list of repositories to ensure we stay in sync. pub fn (mut d Daemon) run() ? { for { - println('1') + // Update the API's contents if needed & renew the queue + if time.now() >= d.api_update_timestamp { + d.renew_repos() ? + d.renew_queue() ? + } + // Cleans up finished builds, opening up spots for new builds d.reschedule_builds() ? - println('2') + + // TODO rebuild builder image when needed + // Schedules new builds when possible d.update_builds() ? - println(d.queue) - println(d.atomics) + // Sleep either until we have to refresh the repos or when the next + // build has to start, with a minimum of 1 second. + now := time.now() - time.sleep(10 * time.second) + mut delay := d.api_update_timestamp - now + + if d.queue.len() > 0 { + time_until_next_job := d.queue.peek() ?.timestamp - now + + delay = math.min(delay, time_until_next_job) + } + + d.ldebug('Sleeping for ${delay}...') + + // TODO if there are builds active, the sleep time should be much lower to clean up the builds when they're finished. + + // We sleep for at least one second. This is to prevent the program + // from looping agressively when a cronjob can be scheduled, but + // there's no spots free for it to be started. + time.sleep(math.max(delay, 1 * time.second)) } } // schedule_build adds the next occurence of the given repo build to the queue. fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) ? { ce := parse_expression(repo.schedule) or { + // TODO This shouldn't return an error if the expression is empty. d.lerror("Error while parsing cron expression '$repo.schedule' ($repo_id): $err.msg()") d.global_schedule @@ -94,6 +120,7 @@ fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) ? { } fn (mut d Daemon) renew_repos() ? { + d.ldebug('Renewing repos...') mut new_repos := git.get_repos(d.address, d.api_key) ? d.repos_map = new_repos.move() @@ -104,6 +131,7 @@ fn (mut d Daemon) renew_repos() ? { // renew_queue replaces the old queue with a new one that reflects the newest // values in repos_map. fn (mut d Daemon) renew_queue() ? { + d.ldebug('Renewing queue...') mut new_queue := MinHeap{} // Move any jobs that should have already started from the old queue onto From b27041d5a789f3aea2faab732ad7cb4924df5aa7 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 30 Apr 2022 10:40:57 +0200 Subject: [PATCH 17/28] make: removed some typos from Makefile --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index db77c2e..55f15ae 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ dvieter: $(SOURCES) # Run the debug build inside gdb .PHONY: gdb gdb: dvieter - gdb --args './dvieter -f vieter.toml server' + gdb --args ./dvieter -f vieter.toml server # Optimised production build .PHONY: prod @@ -93,5 +93,5 @@ afvieter: $(SOURCES) .PHONY: skip-unused skip-unused: suvieter -skip-unused: $(SOURCES) - $(V_PATH) -showcc -skip-unused -o afvieter $(SRC_DIR) +suvieter: $(SOURCES) + $(V_PATH) -showcc -skip-unused -o suvieter $(SRC_DIR) From 20707f6af14ebce35476f7ddffa07b60c415ea86 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 15 Apr 2022 11:38:06 +0200 Subject: [PATCH 18/28] chore(ci): change debug build used chore(ci): removed skip-unused-static experimental build chore: updated Makefile --- .gitignore | 4 ++-- .woodpecker/.build.yml | 5 +++-- .woodpecker/.build_experimental.yml | 13 ------------- Makefile | 4 ++-- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 7847b3f..a3f6afc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,8 @@ data/ vieter dvieter pvieter -dvieterctl -vieterctl +suvieter +afvieter vieter.c # Ignore testing files diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index e68c4c9..c612737 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -9,15 +9,16 @@ matrix: platform: ${PLATFORM} pipeline: - # The default build isn't needed, as alpine switches to gcc for the compiler anyways debug: image: 'chewingbever/vlang:latest' pull: true group: 'build' commands: - - make debug + - make when: event: push + branch: + exclude: [main, dev] prod: image: 'chewingbever/vlang:latest' diff --git a/.woodpecker/.build_experimental.yml b/.woodpecker/.build_experimental.yml index 032a42b..0129d2b 100644 --- a/.woodpecker/.build_experimental.yml +++ b/.woodpecker/.build_experimental.yml @@ -27,16 +27,3 @@ pipeline: - du -h suvieter when: event: push - - skip-unused-static: - image: 'chewingbever/vlang:latest' - pull: true - environment: - - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static - group: 'build' - commands: - - make skip-unused - - readelf -d suvieter - - du -h suvieter - when: - event: push diff --git a/Makefile b/Makefile index 2f39983..fb97ec2 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ dvieter: $(SOURCES) # Run the debug build inside gdb .PHONY: gdb gdb: dvieter - gdb --args './dvieter -f vieter.toml server' + gdb --args ./dvieter -f vieter.toml server # Optimised production build .PHONY: prod @@ -75,7 +75,7 @@ v/v: make -C v clean: - rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst + rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' # =====EXPERIMENTAL===== From 7722d5a7e41cdaed0569acdfd51e21e9acb45734 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 30 Apr 2022 11:43:06 +0200 Subject: [PATCH 19/28] fix: replace byte with u8 BREAKING: the V compiler removed the byte type alias in favor of u8. --- src/docker/docker.v | 6 +++--- src/repo/sync.v | 2 +- src/util/util.v | 4 ++-- src/web/web.v | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/docker/docker.v b/src/docker/docker.v index a6f7640..07ceb8e 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -28,8 +28,8 @@ fn send(req &string) ?http.Response { s.wait_for_write() ? mut c := 0 - mut buf := []byte{len: docker.buf_len} - mut res := []byte{} + mut buf := []u8{len: docker.buf_len} + mut res := []u8{} for { c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } @@ -52,7 +52,7 @@ fn send(req &string) ?http.Response { // We loop until we've encountered the end of the chunked response // A chunked HTTP response always ends with '0\r\n\r\n'. - for res.len < 5 || res#[-5..] != [byte(`0`), `\r`, `\n`, `\r`, `\n`] { + for res.len < 5 || res#[-5..] != [u8(`0`), `\r`, `\n`, `\r`, `\n`] { // Wait for the server to respond s.wait_for_write() ? diff --git a/src/repo/sync.v b/src/repo/sync.v index e2b7aac..12756b7 100644 --- a/src/repo/sync.v +++ b/src/repo/sync.v @@ -19,7 +19,7 @@ fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &stri } // Write the file to the archive - buf := [8192]byte{} + buf := [8192]u8{} mut len := C.read(fd, &buf, sizeof(buf)) for len > 0 { diff --git a/src/util/util.v b/src/util/util.v index 228f584..c1af30e 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -30,7 +30,7 @@ pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { file.close() } - mut buf := []byte{len: util.reader_buf_size} + mut buf := []u8{len: util.reader_buf_size} mut bytes_left := length // Repeat as long as the stream still has data @@ -60,7 +60,7 @@ pub fn hash_file(path &string) ?(string, string) { mut sha256sum := sha256.new() buf_size := int(1_000_000) - mut buf := []byte{len: buf_size} + mut buf := []u8{len: buf_size} mut bytes_left := os.file_size(path) for bytes_left > 0 { diff --git a/src/web/web.v b/src/web/web.v index 688f854..3e7b047 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -285,7 +285,7 @@ pub fn (mut ctx Context) file(f_path string) Result { resp.set_status(ctx.status) send_string(mut ctx.conn, resp.bytestr()) or { return Result{} } - mut buf := []byte{len: 1_000_000} + mut buf := []u8{len: 1_000_000} mut bytes_left := file_size // Repeat as long as the stream still has data From 4d26797453c432297073e512942c2216c35c5edc Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 09:07:16 +0200 Subject: [PATCH 20/28] chore(ci): Updated PKGBUILD to use vieter-v package --- .woodpecker/.arch.yml | 2 ++ PKGBUILD | 9 +++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.woodpecker/.arch.yml b/.woodpecker/.arch.yml index ab3c6ea..e37dc1a 100644 --- a/.woodpecker/.arch.yml +++ b/.woodpecker/.arch.yml @@ -10,6 +10,8 @@ pipeline: build: image: 'menci/archlinuxarm:base-devel' commands: + # Add the vieter repository so we can use the compiler + - echo -e '[vieter]\nServer = https://arch.r8r.be/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf # Update packages - pacman -Syu --noconfirm # Create non-root user to perform build & switch to their home diff --git a/PKGBUILD b/PKGBUILD index 0c558b4..3f8c480 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -2,10 +2,10 @@ pkgbase='vieter' pkgname='vieter' -pkgver=0.2.0.r24.g9a56bd0 +pkgver=0.2.0.r25.g20112b8 pkgrel=1 depends=('glibc' 'openssl' 'libarchive' 'gc') -makedepends=('git' 'gcc') +makedepends=('git' 'gcc' 'vieter-v') arch=('x86_64' 'aarch64' 'armv7') url='https://git.rustybever.be/Chewing_Bever/vieter' license=('AGPL3') @@ -20,10 +20,7 @@ pkgver() { build() { cd "$pkgname" - # Build the compiler - CFLAGS= make v - - V_PATH=v/v make prod + make prod } package() { From 6f9e1b5f3cf02a5f60c419da6c71523a790d19bf Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 30 Apr 2022 11:31:14 +0200 Subject: [PATCH 21/28] feat(cron): start of working loop --- src/cron/daemon/build.v | 36 +++++++++++++++++++------- src/cron/daemon/daemon.v | 43 +++++++++++++++++++++----------- src/cron/expression/expression.v | 2 +- 3 files changed, 57 insertions(+), 24 deletions(-) diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index e7e5ac3..73ba183 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -1,9 +1,25 @@ module daemon -import git import time import sync.stdatomic +const build_empty = 0 +const build_running = 1 +const build_done = 2 + +// reschedule_builds looks for any builds with status code 2 & re-adds them to +// the queue. +fn (mut d Daemon) reschedule_builds() ? { + for i in 0..d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == build_done { + stdatomic.store_u64(&d.atomics[i], build_empty) + sb := d.builds[i] + + d.schedule_build(sb.repo_id, sb.repo) ? + } + } +} + // update_builds starts as many builds as possible. fn (mut d Daemon) update_builds() ? { now := time.now() @@ -13,7 +29,7 @@ fn (mut d Daemon) update_builds() ? { sb := d.queue.pop() ? // If this build couldn't be scheduled, no more will be possible. - if !d.start_build(sb.repo_id)? { + if !d.start_build(sb)? { break } } else { @@ -22,13 +38,14 @@ fn (mut d Daemon) update_builds() ? { } } -// start_build starts a build for the given repo_id. -fn (mut d Daemon) start_build(repo_id string) ?bool { +// start_build starts a build for the given ScheduledBuild object. +fn (mut d Daemon) start_build(sb ScheduledBuild) ?bool { for i in 0..d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == 0 { - stdatomic.store_u64(&d.atomics[i], 1) + if stdatomic.load_u64(&d.atomics[i]) == build_empty { + stdatomic.store_u64(&d.atomics[i], build_running) + d.builds[i] = sb - go d.run_build(i, d.repos_map[repo_id]) + go d.run_build(i, sb) return true } @@ -37,9 +54,10 @@ fn (mut d Daemon) start_build(repo_id string) ?bool { return false } -fn (mut d Daemon) run_build(build_index int, repo git.GitRepo) ? { +// run_build actually starts the build process for a given repo. +fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) ? { time.sleep(10 * time.second) - stdatomic.store_u64(&d.atomics[build_index], 2) + stdatomic.store_u64(&d.atomics[build_index], build_done) } diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index fc917e4..816bc15 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -30,7 +30,7 @@ mut: api_update_timestamp time.Time queue MinHeap // Which builds are currently running - builds []git.GitRepo + builds []ScheduledBuild // Atomic variables used to detect when a build has finished; length is the // same as builds atomics []u64 @@ -47,7 +47,7 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st global_schedule: global_schedule api_update_frequency: api_update_frequency atomics: []u64{len: max_concurrent_builds} - builds: []git.GitRepo{len: max_concurrent_builds} + builds: []ScheduledBuild{len: max_concurrent_builds} logger: logger } @@ -62,14 +62,37 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st // periodically refreshes the list of repositories to ensure we stay in sync. pub fn (mut d Daemon) run() ? { for { + println('1') + // Cleans up finished builds, opening up spots for new builds + d.reschedule_builds() ? + println('2') + // Schedules new builds when possible d.update_builds() ? + println(d.queue) println(d.atomics) - time.sleep(60 * time.second) + time.sleep(10 * time.second) } } +// schedule_build adds the next occurence of the given repo build to the queue. +fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) ? { + ce := parse_expression(repo.schedule) or { + d.lerror("Error while parsing cron expression '$repo.schedule' ($repo_id): $err.msg()") + + d.global_schedule + } + // A repo that can't be scheduled will just be skipped for now + timestamp := ce.next_from_now() ? + + d.queue.insert(ScheduledBuild{ + repo_id: repo_id + repo: repo + timestamp: timestamp + }) +} + fn (mut d Daemon) renew_repos() ? { mut new_repos := git.get_repos(d.address, d.api_key) ? @@ -101,19 +124,11 @@ fn (mut d Daemon) renew_queue() ? { } } + d.queue = new_queue + // For each repository in repos_map, parse their cron expression (or use // the default one if not present) & add them to the queue for id, repo in d.repos_map { - ce := parse_expression(repo.schedule) or { d.global_schedule } - // A repo that can't be scheduled will just be skipped for now - timestamp := ce.next(now) or { continue } - - new_queue.insert(ScheduledBuild{ - repo_id: id - repo: repo - timestamp: timestamp - }) + d.schedule_build(id, repo) ? } - - d.queue = new_queue } diff --git a/src/cron/expression/expression.v b/src/cron/expression/expression.v index c122585..6e11da2 100644 --- a/src/cron/expression/expression.v +++ b/src/cron/expression/expression.v @@ -114,7 +114,7 @@ pub fn (ce &CronExpression) next(ref time.Time) ?time.Time { }) } -fn (ce &CronExpression) next_from_now() ?time.Time { +pub fn (ce &CronExpression) next_from_now() ?time.Time { return ce.next(time.now()) } From 5ee6d553ac268d2a79dced18d421b15461ff7f11 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 16:12:37 +0200 Subject: [PATCH 22/28] Compile boehm gc dynamically for debug --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index fb97ec2..a0101a5 100644 --- a/Makefile +++ b/Makefile @@ -9,9 +9,10 @@ all: vieter # =====COMPILATION===== -# Regular binary +# We force the boehm gc to be compiled dynamically because otherwise, our CI +# build breaks. vieter: $(SOURCES) - $(V) -g -o vieter $(SRC_DIR) + $(V) -d dynamic_boehm -g -o vieter $(SRC_DIR) # Debug build using gcc # The debug build can't use the boehm garbage collector, as that is From 16151643e64034a6af15fd1c71ef33e411d17ff5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 16:15:01 +0200 Subject: [PATCH 23/28] Add dynamic boehm compiler to tests as well --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a0101a5..165939f 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ vet: .PHONY: test test: - $(V) test $(SRC_DIR) + $(V) -d dynamic_boehm test $(SRC_DIR) # Build & patch the V compiler .PHONY: v From ead4c5f4b73f3a458602771ecfb748eefbc0d541 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 16:26:46 +0200 Subject: [PATCH 24/28] ci: added dynamic boehm prod build to experimental builds --- .woodpecker/.build_experimental.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.woodpecker/.build_experimental.yml b/.woodpecker/.build_experimental.yml index 0129d2b..3968e05 100644 --- a/.woodpecker/.build_experimental.yml +++ b/.woodpecker/.build_experimental.yml @@ -27,3 +27,22 @@ pipeline: - du -h suvieter when: event: push + + dynamic-boehm-prod: + image: 'chewingbever/vlang:latest' + pull: true + group: 'build' + environment: + - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static + - VFLAGS=-cc gcc -d dynamic_boehm + commands: + - make prod + # Make sure the binary is actually statically built + - readelf -d pvieter + - du -h pvieter + - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' + # This removes so much, it's amazing + - strip -s pvieter + - du -h pvieter + when: + event: push From d5409201c74203911dc85e14235a6db88a230abc Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 16:48:35 +0200 Subject: [PATCH 25/28] Use dynamic_boehm everywhere --- .woodpecker/.build_experimental.yml | 19 ------------------- Makefile | 6 +++--- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/.woodpecker/.build_experimental.yml b/.woodpecker/.build_experimental.yml index 3968e05..0129d2b 100644 --- a/.woodpecker/.build_experimental.yml +++ b/.woodpecker/.build_experimental.yml @@ -27,22 +27,3 @@ pipeline: - du -h suvieter when: event: push - - dynamic-boehm-prod: - image: 'chewingbever/vlang:latest' - pull: true - group: 'build' - environment: - - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static - - VFLAGS=-cc gcc -d dynamic_boehm - commands: - - make prod - # Make sure the binary is actually statically built - - readelf -d pvieter - - du -h pvieter - - '[ "$(readelf -d pvieter | grep NEEDED | wc -l)" = 0 ]' - # This removes so much, it's amazing - - strip -s pvieter - - du -h pvieter - when: - event: push diff --git a/Makefile b/Makefile index 165939f..c7f4384 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SRC_DIR := src SOURCES != find '$(SRC_DIR)' -iname '*.v' V_PATH ?= v -V := $(V_PATH) -showcc -gc boehm +V := $(V_PATH) -showcc -gc boehm -d dynamic_boehm all: vieter @@ -12,7 +12,7 @@ all: vieter # We force the boehm gc to be compiled dynamically because otherwise, our CI # build breaks. vieter: $(SOURCES) - $(V) -d dynamic_boehm -g -o vieter $(SRC_DIR) + $(V) -g -o vieter $(SRC_DIR) # Debug build using gcc # The debug build can't use the boehm garbage collector, as that is @@ -66,7 +66,7 @@ vet: .PHONY: test test: - $(V) -d dynamic_boehm test $(SRC_DIR) + $(V) test $(SRC_DIR) # Build & patch the V compiler .PHONY: v From 488a172586d31d88d184eb58dea8c157b4547f8a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 21 Apr 2022 16:49:39 +0200 Subject: [PATCH 26/28] ci: run buils sequentially [CI SKIP] --- .woodpecker/.build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.woodpecker/.build.yml b/.woodpecker/.build.yml index c612737..e7341fd 100644 --- a/.woodpecker/.build.yml +++ b/.woodpecker/.build.yml @@ -12,7 +12,6 @@ pipeline: debug: image: 'chewingbever/vlang:latest' pull: true - group: 'build' commands: - make when: @@ -25,7 +24,6 @@ pipeline: pull: true environment: - LDFLAGS=-lz -lbz2 -llzma -lexpat -lzstd -llz4 -static - group: 'build' commands: - make prod # Make sure the binary is actually statically built From ed02d82a0d939acfd8a6ced18cd4b3dffc22d7f8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 29 Apr 2022 10:34:12 +0200 Subject: [PATCH 27/28] Added docs command & notice in README [CI SKIP] --- .gitignore | 3 +++ Makefile | 8 +++++++- README.md | 4 +++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index a3f6afc..6a06eb2 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ v/ # gdb log file gdb.txt + +# Generated docs +_docs/ diff --git a/Makefile b/Makefile index c7f4384..55f15ae 100644 --- a/Makefile +++ b/Makefile @@ -75,8 +75,14 @@ v/v: git clone --single-branch https://git.rustybever.be/Chewing_Bever/v v make -C v +.PHONY: clean clean: - rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' + rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' + +.PHONY: docs +docs: + rm -rf '$(SRC_DIR)/_docs' + cd '$(SRC_DIR)' && v doc -all -f html -m -readme . # =====EXPERIMENTAL===== diff --git a/README.md b/README.md index 96b104d..08f1e75 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ ## Documentation -I host documentation for Vieter over at https://rustybever.be/docs/vieter/. +I host documentation for Vieter over at https://rustybever.be/docs/vieter/. API +documentation for the current codebase can be found at +https://rustybever.be/api-docs/vieter/. ## Overview From cc061a38daae7ffda27135ee9cbe5e01595cbfb0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 30 Apr 2022 10:40:29 +0200 Subject: [PATCH 28/28] daemon: worked on daemon loop --- src/cron/cli.v | 5 +++-- src/cron/daemon/build.v | 23 +++++++++++++---------- src/cron/daemon/daemon.v | 38 +++++++++++++++++++++++++++++++++----- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/cron/cli.v b/src/cron/cli.v index f4b20ec..3b836dd 100644 --- a/src/cron/cli.v +++ b/src/cron/cli.v @@ -11,8 +11,9 @@ pub: address string base_image string = 'archlinux:base-devel' max_concurrent_builds int = 1 - api_update_frequency int = 60 - global_schedule string + api_update_frequency int = 15 + // Replicates the behavior of the original cron system + global_schedule string = '0 3' } // cmd returns the cli module that handles the cron daemon. diff --git a/src/cron/daemon/build.v b/src/cron/daemon/build.v index 73ba183..c5ef428 100644 --- a/src/cron/daemon/build.v +++ b/src/cron/daemon/build.v @@ -4,15 +4,17 @@ import time import sync.stdatomic const build_empty = 0 + const build_running = 1 + const build_done = 2 // reschedule_builds looks for any builds with status code 2 & re-adds them to // the queue. fn (mut d Daemon) reschedule_builds() ? { - for i in 0..d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == build_done { - stdatomic.store_u64(&d.atomics[i], build_empty) + for i in 0 .. d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == daemon.build_done { + stdatomic.store_u64(&d.atomics[i], daemon.build_empty) sb := d.builds[i] d.schedule_build(sb.repo_id, sb.repo) ? @@ -29,7 +31,8 @@ fn (mut d Daemon) update_builds() ? { sb := d.queue.pop() ? // If this build couldn't be scheduled, no more will be possible. - if !d.start_build(sb)? { + // TODO a build that couldn't be scheduled should be re-added to the queue. + if !d.start_build(sb) { break } } else { @@ -39,10 +42,10 @@ fn (mut d Daemon) update_builds() ? { } // start_build starts a build for the given ScheduledBuild object. -fn (mut d Daemon) start_build(sb ScheduledBuild) ?bool { - for i in 0..d.atomics.len { - if stdatomic.load_u64(&d.atomics[i]) == build_empty { - stdatomic.store_u64(&d.atomics[i], build_running) +fn (mut d Daemon) start_build(sb ScheduledBuild) bool { + for i in 0 .. d.atomics.len { + if stdatomic.load_u64(&d.atomics[i]) == daemon.build_empty { + stdatomic.store_u64(&d.atomics[i], daemon.build_running) d.builds[i] = sb go d.run_build(i, sb) @@ -56,8 +59,8 @@ fn (mut d Daemon) start_build(sb ScheduledBuild) ?bool { // run_build actually starts the build process for a given repo. fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) ? { + d.linfo('build $sb.repo.url') time.sleep(10 * time.second) - stdatomic.store_u64(&d.atomics[build_index], build_done) + stdatomic.store_u64(&d.atomics[build_index], daemon.build_done) } - diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index 816bc15..7253e94 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -5,6 +5,8 @@ import time import log import datatypes { MinHeap } import cron.expression { CronExpression, parse_expression } +import math +import arrays struct ScheduledBuild { pub: @@ -62,23 +64,47 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st // periodically refreshes the list of repositories to ensure we stay in sync. pub fn (mut d Daemon) run() ? { for { - println('1') + // Update the API's contents if needed & renew the queue + if time.now() >= d.api_update_timestamp { + d.renew_repos() ? + d.renew_queue() ? + } + // Cleans up finished builds, opening up spots for new builds d.reschedule_builds() ? - println('2') + + // TODO rebuild builder image when needed + // Schedules new builds when possible d.update_builds() ? - println(d.queue) - println(d.atomics) + // Sleep either until we have to refresh the repos or when the next + // build has to start, with a minimum of 1 second. + now := time.now() - time.sleep(10 * time.second) + mut delay := d.api_update_timestamp - now + + if d.queue.len() > 0 { + time_until_next_job := d.queue.peek() ?.timestamp - now + + delay = math.min(delay, time_until_next_job) + } + + d.ldebug('Sleeping for ${delay}...') + + // TODO if there are builds active, the sleep time should be much lower to clean up the builds when they're finished. + + // We sleep for at least one second. This is to prevent the program + // from looping agressively when a cronjob can be scheduled, but + // there's no spots free for it to be started. + time.sleep(math.max(delay, 1 * time.second)) } } // schedule_build adds the next occurence of the given repo build to the queue. fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) ? { ce := parse_expression(repo.schedule) or { + // TODO This shouldn't return an error if the expression is empty. d.lerror("Error while parsing cron expression '$repo.schedule' ($repo_id): $err.msg()") d.global_schedule @@ -94,6 +120,7 @@ fn (mut d Daemon) schedule_build(repo_id string, repo git.GitRepo) ? { } fn (mut d Daemon) renew_repos() ? { + d.ldebug('Renewing repos...') mut new_repos := git.get_repos(d.address, d.api_key) ? d.repos_map = new_repos.move() @@ -104,6 +131,7 @@ fn (mut d Daemon) renew_repos() ? { // renew_queue replaces the old queue with a new one that reflects the newest // values in repos_map. fn (mut d Daemon) renew_queue() ? { + d.ldebug('Renewing queue...') mut new_queue := MinHeap{} // Move any jobs that should have already started from the old queue onto