set up otel stack

plays/boomhut.yml

@@ -23,3 +23,13 @@
   roles:
     - 'any.software.papermc-podman'
   tags: papermc
+
+- hosts: boomhut
+  become: true
+  roles:
+    - role: 'any.software.otel-collector'
+      vars:
+        otel_metrics_endpoint: 'http://10.0.0.1:4000/v1/otlp'
+        otel_logs_endpoint: 'http://10.0.0.1:4000/v1/otlp'
+        otel_traces_endpoint: 'http://10.0.0.1:4000/v1/otlp'
+  tags: otel

plays/pearl.yml

@@ -37,3 +37,13 @@
   roles:
     - 'any.software.lander'
   tags: lander
+
+- hosts: pearl
+  become: true
+  roles:
+    - role: 'any.software.otel-collector'
+      vars:
+        otel_metrics_endpoint: 'http://10.0.0.1:4000/v1/otlp'
+        otel_logs_endpoint: 'http://10.0.0.1:4000/v1/otlp'
+        otel_traces_endpoint: 'http://10.0.0.1:4000/v1/otlp'
+  tags: otel

roles/any.software.grafana/files/grafana.Caddyfile

@@ -0,0 +1,3 @@
+grafana.roosens.me {
+    reverse_proxy localhost:3000
+}

roles/any.software.grafana/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: 'restart grafana'
+  ansible.builtin.service:
+    name: 'grafana-server'
+    state: 'restarted'

roles/any.software.grafana/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: any.tools.caddy

roles/any.software.grafana/tasks/main.yml

@@ -0,0 +1,44 @@
+---
+- name: Ensure Grafana is installed
+  ansible.builtin.apt:
+    deb: 'https://dl.grafana.com/grafana/release/12.3.3/grafana_12.3.3_21957728731_linux_amd64.deb'
+  notify: 'restart grafana'
+
+- name: Ensure data directory permissions are correct
+  ansible.builtin.file:
+    path: '{{ item }}'
+    state: 'directory'
+    owner: 'grafana'
+    group: 'grafana'
+  loop:
+    - '{{ grafana_data_dir }}'
+    - '{{ grafana_logs_dir }}'
+
+- name: Ensure configuration directory is present
+  ansible.builtin.file:
+    path: '/etc/grafana'
+    state: directory
+    mode: '0755'
+
+- name: Ensure config file is present
+  ansible.builtin.template:
+    src: 'grafana.ini.j2'
+    dest: '/etc/grafana/grafana.ini'
+    mode: '0644'
+    owner: 'root'
+    group: 'root'
+  notify: 'restart grafana'
+
+- name: Ensure service is enabled
+  ansible.builtin.service:
+    name: 'grafana-server'
+    enabled: true
+
+- name: Ensure Caddyfile is present
+  ansible.builtin.copy:
+    src: 'grafana.Caddyfile'
+    dest: '/etc/caddy/grafana.Caddyfile'
+    owner: root
+    group: root
+    mode: '0644'
+  notify: reload caddy

roles/any.software.greptimedb-podman/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: 'restart greptimedb'
+  ansible.builtin.service:
+    name: 'greptimedb'
+    state: 'restarted'
+
+    scope: 'user'
+    daemon_reload: true

roles/any.software.greptimedb-podman/tasks/main.yml

@@ -0,0 +1,39 @@
+---
+- name: Ensure systemd directory is present
+  ansible.builtin.file:
+    path: '/home/debian/.config/containers/systemd'
+    state: 'directory'
+    mode: '0755'
+    owner: 'debian'
+    group: 'debian'
+
+- name: Ensure data directory permissions are correct
+  ansible.builtin.file:
+    path: '{{ greptimedb_data_dir }}'
+    state: directory
+    owner: 'debian'
+    group: 'debian'
+  become: true
+
+- name: Ensure Quadlet files are present
+  ansible.builtin.template:
+    src: "greptimedb.container.j2"
+    dest: "/home/debian/.config/containers/systemd/greptimedb.container"
+    mode: '0755'
+    owner: 'debian'
+    group: 'debian'
+  notify: 'restart greptimedb'
+
+# - name: Ensure stack is deployed
+#   ansible.builtin.shell:
+#     chdir: '/etc/miniflux'
+#     cmd: 'docker compose up -d --remove-orphans'
+#   when: 'res.changed'
+
+# - name: Ensure backup script is present
+#   ansible.builtin.copy:
+#     src: 'miniflux.backup.sh'
+#     dest: '/etc/backups/miniflux.backup.sh'
+#     owner: 'root'
+#     group: 'root'
+#     mode: '0644'

roles/any.software.greptimedb-podman/templates/greptimedb.container.j2

@@ -0,0 +1,21 @@
+# vim: ft=systemd
+[Container]
+Image=docker.io/greptime/greptimedb:v1.0.0-rc.1
+
+Exec=standalone start --http-addr 0.0.0.0:4000 --rpc-bind-addr 0.0.0.0:4001 --mysql-addr 0.0.0.0:4002 --postgres-addr 0.0.0.0:4003
+
+Volume={{ greptimedb_data_dir }}:/greptimedb_data
+
+PublishPort=0.0.0.0:4000:4000
+PublishPort=0.0.0.0:4001:4001
+PublishPort=0.0.0.0:4002:4002
+PublishPort=0.0.0.0:4003:4003
+
+User=0
+
+[Service]
+Restart=always
+
+[Install]
+WantedBy=default.target
+WantedBy=multi-user.target

roles/any.software.otel-collector/defaults/main.yml

@@ -0,0 +1,3 @@
+otel_metrics_endpoint: 'http://127.0.0.1:4000/v1/otlp'
+otel_logs_endpoint: 'http://127.0.0.1:4000/v1/otlp'
+otel_traces_endpoint: 'http://127.0.0.1:4000/v1/otlp'

roles/any.software.otel-collector/files/run_as_root.conf

@@ -0,0 +1,2 @@
+[Service]
+User=0

roles/any.software.otel-collector/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: 'restart otelcol-contrib'
+  ansible.builtin.service:
+    name: 'otelcol-contrib'
+    state: 'restarted'
+    daemon_reload: true

roles/any.software.otel-collector/tasks/main.yml

@@ -0,0 +1,42 @@
+---
+- name: Ensure collector is installed
+  ansible.builtin.apt:
+    deb: 'https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.146.1/otelcol-contrib_0.146.1_linux_amd64.deb'
+  notify: 'restart otelcol-contrib'
+
+- name: Ensure configuration directory is present
+  ansible.builtin.file:
+    path: '/etc/otelcol-contrib'
+    state: directory
+    mode: '0755'
+
+- name: Ensure config file is present
+  ansible.builtin.template:
+    src: 'config.yaml.j2'
+    dest: '/etc/otelcol-contrib/config.yaml'
+    mode: '0644'
+    owner: 'root'
+    group: 'root'
+  notify: 'restart otelcol-contrib'
+
+- name: Ensure Systemd override directory is present
+  ansible.builtin.file:
+    path: '/etc/systemd/system/otelcol-contrib.service.d'
+    state: 'directory'
+    mode: '0644'
+    owner: 'root'
+    group: 'root'
+
+- name: Ensure Systemd override file is present
+  ansible.builtin.copy:
+    src: 'run_as_root.conf'
+    dest: '/etc/systemd/system/otelcol-contrib.service.d/run_as_root.conf'
+    mode: '0644'
+    owner: 'root'
+    group: 'root'
+  notify: 'restart otelcol-contrib'
+
+- name: Ensure service is enabled
+  ansible.builtin.service:
+    name: 'otelcol-contrib'
+    enabled: true

roles/any.software.otel-collector/templates/config.yaml.j2

@@ -0,0 +1,90 @@
+# https://uptrace.dev/opentelemetry/collector#installation
+# Receivers configure how data gets into the Collector
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+  hostmetrics:
+    collection_interval: 10s
+    scrapers:
+      cpu:
+      memory:
+      disk:
+      filesystem:
+      network:
+      load:
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: 'caddy'
+          scrape_interval: 1m
+          static_configs:
+            - targets: ['localhost:2019']
+        - job_name: 'miniflux'
+          scrape_interval: 30s
+          static_configs:
+            - targets: ['localhost:8002']
+        - job_name: 'restic-rest'
+          scrape_interval: 30s
+          static_configs:
+            - targets: ['localhost:8000']
+
+# Processors specify what happens with the received data
+processors:
+  resourcedetection:
+    detectors: [env, system]
+  # cumulativetodelta:
+  batch:
+    send_batch_size: 10000
+    timeout: 10s
+
+exporters:
+  otlphttp/traces:
+    endpoint: '{{ otel_traces_endpoint }}'
+    # auth:
+    #   authenticator: basicauth/client
+    headers:
+      # x-greptime-db-name: '<your_db_name>'
+      x-greptime-pipeline-name: 'greptime_trace_v1'
+    tls:
+      insecure: true
+  otlphttp/logs:
+    endpoint: '{{ otel_logs_endpoint }}'
+    # auth:
+    #   authenticator: basicauth/client
+    headers:
+      # x-greptime-db-name: '<your_db_name>'
+      # x-greptime-log-table-name: '<table_name>'
+      # x-greptime-pipeline-name: '<pipeline_name>'
+    tls:
+      insecure: true
+
+  otlphttp/metrics:
+    endpoint: '{{ otel_metrics_endpoint }}'
+    # auth:
+    #   authenticator: basicauth/client
+    headers:
+      x-greptime-otlp-metric-promote-all-resource-attrs: 'true'
+      # x-greptime-db-name: '<your_db_name>'
+    tls:
+      insecure: true
+
+# Service pipelines pull the configured receivers, processors, and exporters together
+# into pipelines that process data
+#
+# Note: Receivers, processors, and exporters not used in pipelines are silently ignored
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp, hostmetrics, prometheus]
+      processors: [batch, resourcedetection]
+      exporters: [otlphttp/metrics]
+    traces:
+      receivers: [otlp]
+      processors: [batch, resourcedetection]
+      exporters: [otlphttp/traces]
+    logs:
+      receivers: [otlp]
+      processors: [batch, resourcedetection]
+      exporters: [otlphttp/logs]

roles/any.tools.otel-cli/tasks/main.yml

@@ -0,0 +1,4 @@
+---
+- name: Ensure otel-cli is installed
+  ansible.builtin.apt:
+    deb: 'https://github.com/equinix-labs/otel-cli/releases/download/v{{ otel_cli_version }}/otel-cli_{{ otel_cli_version }}_linux_amd64.deb'