From c0f58ddc77e2db11dcf7d54d07210934015aaffd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 28 Dec 2022 16:09:00 +0100 Subject: [PATCH 1/3] feat(server): add metric collection --- src/server/api_metrics.v | 16 ++++++++++++++++ src/server/server.v | 2 ++ src/v.mod | 3 ++- src/web/web.v | 22 ++++++++++++++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 src/server/api_metrics.v diff --git a/src/server/api_metrics.v b/src/server/api_metrics.v new file mode 100644 index 0000000..af1b134 --- /dev/null +++ b/src/server/api_metrics.v @@ -0,0 +1,16 @@ +module server + +import metrics +import web + +['/api/v1/metrics'; get] +fn (mut app App) v1_metrics() web.Result { + mut exporter := metrics.new_prometheus_exporter([0.01, 0.05, 0.1, 0.5, 1, 100]) + exporter.load(app.collector) + + // TODO stream to connection instead + body := exporter.export_to_string() or { + return app.status(.internal_server_error) + } + return app.body(.ok, 'text/plain', body) +} diff --git a/src/server/server.v b/src/server/server.v index 178f657..9571b7b 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -8,6 +8,7 @@ import util import db import build { BuildJobQueue } import cron.expression +import metrics const ( log_file_name = 'vieter.log' @@ -107,6 +108,7 @@ pub fn server(conf Config) ! { repo: repo db: db job_queue: build.new_job_queue(global_ce, conf.base_image) + collector: metrics.new_default_collector() } app.init_job_queue() or { util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()') diff --git a/src/v.mod b/src/v.mod index 710c976..461af6a 100644 --- a/src/v.mod +++ b/src/v.mod @@ -2,6 +2,7 @@ Module { dependencies: [ 'https://git.rustybever.be/vieter-v/conf', 'https://git.rustybever.be/vieter-v/docker', - 'https://git.rustybever.be/vieter-v/aur' + 'https://git.rustybever.be/vieter-v/aur', + 'https://git.rustybever.be/vieter-v/metrics' ] } diff --git a/src/web/web.v b/src/web/web.v index 565baff..95c91ed 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -11,6 +11,7 @@ import net.urllib import time import json import log +import metrics // The Context struct represents the Context which hold the HTTP request and response. // It has fields for the query, form, files. @@ -27,6 +28,8 @@ pub mut: conn &net.TcpConn = unsafe { nil } // Gives access to a shared logger object logger shared log.Log + // Used to collect metrics on the web server + collector &metrics.MetricsCollector // time.ticks() from start of web connection handle. // You can use it to determine how much time is spent on your request. page_gen_start i64 @@ -145,6 +148,14 @@ pub fn (ctx &Context) is_authenticated() bool { return false } +pub fn (mut ctx Context) body(status http.Status, content_type string, body string) Result { + ctx.status = status + ctx.content_type = content_type + ctx.send_response(body) + + return Result{} +} + // json HTTP_OK with json_s as payload with content-type `application/json` pub fn (mut ctx Context) json(status http.Status, j T) Result { ctx.status = status @@ -319,6 +330,16 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { app.logger.flush() } + // Record how long request took to process + labels := [ + ['method', app.req.method.str()]!, + ['path', app.req.url]!, + ['status', app.status.int().str()]! + ] + app.collector.counter_increment(name: 'http_requests_total', labels: labels) + app.collector.histogram_record(time.ticks() - app.page_gen_start, name: 'http_requests_time_ms', labels: labels) + /* app.collector.histogram_ */ + unsafe { free(app) } @@ -384,6 +405,7 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { static_mime_types: app.static_mime_types reader: reader logger: app.logger + collector: app.collector api_key: app.api_key } From 4ca2521937bc57bda4b6e45080dcd497687d8ec0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 28 Dec 2022 17:39:45 +0100 Subject: [PATCH 2/3] feat(server): ability to disable metrics --- CHANGELOG.md | 2 ++ src/server/api_metrics.v | 13 ++++++++----- src/server/cli.v | 1 + src/server/server.v | 8 +++++++- src/web/web.v | 9 ++++++--- vieter.toml | 1 + 6 files changed, 25 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e4e228..72c5440 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/vieter-v/vieter/src/branch/dev) +* Metrics endpoint for Prometheus integration + ## [0.5.0](https://git.rustybever.be/vieter-v/vieter/src/tag/0.5.0) ### Added diff --git a/src/server/api_metrics.v b/src/server/api_metrics.v index af1b134..8d6f654 100644 --- a/src/server/api_metrics.v +++ b/src/server/api_metrics.v @@ -3,14 +3,17 @@ module server import metrics import web -['/api/v1/metrics'; get] +// v1_metrics serves a Prometheus-compatible metrics endpoint. +['/api/v1/metrics'; get; markused] fn (mut app App) v1_metrics() web.Result { + if !app.conf.collect_metrics { + return app.status(.not_found) + } + mut exporter := metrics.new_prometheus_exporter([0.01, 0.05, 0.1, 0.5, 1, 100]) exporter.load(app.collector) - + // TODO stream to connection instead - body := exporter.export_to_string() or { - return app.status(.internal_server_error) - } + body := exporter.export_to_string() or { return app.status(.internal_server_error) } return app.body(.ok, 'text/plain', body) } diff --git a/src/server/cli.v b/src/server/cli.v index 21fb15e..9a8b144 100644 --- a/src/server/cli.v +++ b/src/server/cli.v @@ -15,6 +15,7 @@ pub: base_image string = 'archlinux:base-devel' max_log_age int [empty_default] log_removal_schedule string = '0 0' + collect_metrics bool [empty_default] } // cmd returns the cli submodule that handles starting the server diff --git a/src/server/server.v b/src/server/server.v index 9571b7b..76e7ad6 100644 --- a/src/server/server.v +++ b/src/server/server.v @@ -101,14 +101,20 @@ pub fn server(conf Config) ! { util.exit_with_message(1, 'Failed to initialize database: $err.msg()') } + collector := if conf.collect_metrics { + &metrics.MetricsCollector(metrics.new_default_collector()) + } else { + &metrics.MetricsCollector(metrics.new_null_collector()) + } + mut app := &App{ logger: logger api_key: conf.api_key conf: conf repo: repo db: db + collector: collector job_queue: build.new_job_queue(global_ce, conf.base_image) - collector: metrics.new_default_collector() } app.init_job_queue() or { util.exit_with_message(1, 'Failed to inialize job queue: $err.msg()') diff --git a/src/web/web.v b/src/web/web.v index 95c91ed..c44057e 100644 --- a/src/web/web.v +++ b/src/web/web.v @@ -148,6 +148,7 @@ pub fn (ctx &Context) is_authenticated() bool { return false } +// body sends the given body as an HTTP response. pub fn (mut ctx Context) body(status http.Status, content_type string, body string) Result { ctx.status = status ctx.content_type = content_type @@ -334,11 +335,13 @@ fn handle_conn(mut conn net.TcpConn, mut app T, routes map[string]Route) { labels := [ ['method', app.req.method.str()]!, ['path', app.req.url]!, - ['status', app.status.int().str()]! + ['status', app.status.int().str()]!, ] app.collector.counter_increment(name: 'http_requests_total', labels: labels) - app.collector.histogram_record(time.ticks() - app.page_gen_start, name: 'http_requests_time_ms', labels: labels) - /* app.collector.histogram_ */ + app.collector.histogram_record(time.ticks() - app.page_gen_start, + name: 'http_requests_time_ms' + labels: labels + ) unsafe { free(app) diff --git a/vieter.toml b/vieter.toml index 1f839f0..31eadc0 100644 --- a/vieter.toml +++ b/vieter.toml @@ -13,3 +13,4 @@ api_update_frequency = 2 image_rebuild_frequency = 1 max_concurrent_builds = 3 max_log_age = 64 +collect_metrics = true From 4ed4ef4a27b904180aa103b20c94fc32d2c87920 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 3 Jan 2023 09:29:55 +0100 Subject: [PATCH 3/3] chore: generate man pages using debug build --- .woodpecker/man.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.woodpecker/man.yml b/.woodpecker/man.yml index 8c6ca06..8102443 100644 --- a/.woodpecker/man.yml +++ b/.woodpecker/man.yml @@ -8,15 +8,21 @@ branches: depends_on: - build -skip_clone: true - pipeline: - generate: + install-modules: image: *vlang_image pull: true commands: - - curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64" - - chmod +x vieter + - export VMODULES=$PWD/.vmodules + - 'cd src && v install' + + generate: + image: *vlang_image + commands: + # - curl -o vieter -L "https://s3.rustybever.be/vieter/commits/$CI_COMMIT_SHA/vieter-linux-amd64" + # - chmod +x vieter + - export VMODULES=$PWD/.vmodules + - make - ./vieter man man - cd man