add new and improved backup-scripts system

roles/any.tools.backup-scripts/defaults/main.yml

@@ -0,0 +1,47 @@
+# List of backup jobs to configure. Each entry creates a systemd service and
+# timer. Required keys per entry vary by type:
+#
+# All types:
+#   name:             (required) unique identifier, used in unit and script filenames
+#   type:             (required) backup template to use: btrfs-subvolume, podman-postgres, postgres
+#   user:             (optional) user to run the backup as; defaults to root
+#   group:            (optional) group to run the backup as; defaults to backups
+#   timer_delay_sec:  (optional) RandomizedDelaySec for the timer; defaults to 30 minutes
+#
+# btrfs-subvolume:
+#   path:             (required) path to the btrfs subvolume to back up
+#
+# podman-postgres:
+#   container:        (required) name of the podman container running postgres
+#   pg_user:          (required) postgres user to connect as
+#   database:         (required) postgres database to dump
+#
+# postgres:
+#   pwd:              (required) working directory for podman compose
+#   user:             (required) postgres user to connect as
+#   database:         (required) postgres database to dump
+backups: []
+
+# Restic REST server URL to publish backups to
+backup_restic_repository: "rest:http://localhost:8000/backups"
+
+# Path to the file containing the Restic repository password
+backup_restic_password_file: "/etc/backups/restic_backups_passwd"
+
+# Directory where backup scripts are stored
+backup_scripts_dir: "/etc/backups"
+
+# Hour at which all backup timers fire (24h)
+backup_timer_hour: "02"
+
+# Minute at which all backup timers fire
+backup_timer_minute: "00"
+
+# Randomized delay from start time that services are started
+backup_timer_delay_sec: "1800"
+
+# OpenTelemetry collector endpoint for backup tracing
+backup_otel_endpoint: "http://localhost:4318"
+
+# OpenTelemetry service name used to identify backup spans
+backup_otel_service_name: "backups"

roles/any.tools.backup-scripts/files/restic_backups_passwd

@@ -0,0 +1,7 @@
+$ANSIBLE_VAULT;1.1;AES256
+33666438313237356564363136333933633035303531653464643766373434623834663736386463
+3464643731366237633334616536613864396162353264360a316130333032316437393333396466
+34356638393834316235633062646330336438376135346666663064303831666632353834663465
+6636663930356138640a323433613263393939303833616637336436366630386133386338613736
+34353433643539306238663638656539373731616238656635353561356632366332623532396465
+3936373534643966616131616161633234663430633233653435

roles/any.tools.backup-scripts/handlers/main.yml

@@ -0,0 +1,3 @@
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true

roles/any.tools.backup-scripts/tasks/main.yml

@@ -0,0 +1,69 @@
+- name: Ensure backup scripts directory is present
+  ansible.builtin.file:
+    path: "{{ backup_scripts_dir }}"
+    state: directory
+    mode: "0755"
+
+- name: Ensure backups group exists
+  ansible.builtin.group:
+    name: backups
+    system: true
+    state: present
+
+- name: Ensure Restic backups password file is present
+  ansible.builtin.copy:
+    src: "restic_backups_passwd"
+    dest: "{{ backup_restic_password_file }}"
+    owner: root
+    group: backups
+    mode: "0640"
+
+- name: Ensure all backup scripts are present
+  ansible.builtin.template:
+    src: "{{ item.type }}.backup.sh.j2"
+    dest: "{{ backup_scripts_dir }}/{{ item.name }}.backup.sh"
+    owner: root
+    group: backups
+    mode: "0750"
+  loop: "{{ backups }}"
+
+- name: Ensure backup users are in the backups group
+  ansible.builtin.user:
+    name: "{{ item.user }}"
+    groups: backups
+    append: true
+  loop: "{{ backups }}"
+  when: item.user is defined
+
+- name: Ensure systemd service unit is present for each backup
+  ansible.builtin.template:
+    src: "backup.service.j2"
+    dest: "/etc/systemd/system/backup-{{ item.name }}.service"
+    owner: root
+    group: root
+    mode: "0644"
+  loop: "{{ backups }}"
+  notify: Reload systemd
+
+- name: Ensure systemd timer unit is present for each backup
+  ansible.builtin.template:
+    src: "backup.timer.j2"
+    dest: "/etc/systemd/system/backup-{{ item.name }}.timer"
+    owner: root
+    group: root
+    mode: "0644"
+  loop: "{{ backups }}"
+  notify: Reload systemd
+
+- name: Ensure backup timers are enabled and started
+  ansible.builtin.systemd:
+    name: "backup-{{ item.name }}.timer"
+    enabled: true
+    state: started
+    daemon_reload: true
+  loop: "{{ backups }}"
+
+- name: Remove legacy backup cronjob if present
+  ansible.builtin.cron:
+    name: "Perform nightly backups"
+    state: absent

roles/any.tools.backup-scripts/templates/backup.service.j2

@@ -0,0 +1,20 @@
+[Unit]
+Description=Backup: {{ item.name }}
+After=network.target
+
+[Service]
+Type=oneshot
+User={{ item.user | default('root') }}
+Group={{ item.group | default('backups') }}
+
+Environment="RESTIC_REPOSITORY={{ backup_restic_repository }}"
+Environment="RESTIC_PASSWORD_FILE={{ backup_restic_password_file }}"
+Environment="OTEL_EXPORTER_OTLP_ENDPOINT={{ backup_otel_endpoint }}"
+Environment="OTEL_SERVICE_NAME={{ backup_otel_service_name }}"
+
+ExecStart=/usr/bin/otel-cli exec \
+    --name "{{ item.name }}" \
+    --attrs 'backup.type={{ item.type }}' -- /usr/bin/bash {{ backup_scripts_dir }}/{{ item.name }}.backup.sh
+
+[Install]
+WantedBy=multi-user.target

roles/any.tools.backup-scripts/templates/backup.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Timer for backup: {{ item.name }}
+
+[Timer]
+OnCalendar=*-*-* {{ backup_timer_hour }}:{{ backup_timer_minute }}:00
+RandomizedDelaySec={{ backup_timer_delay_sec | default('0') }}
+Persistent=true
+
+[Install]
+WantedBy=timers.target

roles/any.tools.backup-scripts/templates/btrfs-subvolume.backup.sh.j2

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+data_dir='{{ item.path }}'
+snapshot_dir="${data_dir}.snapshot"
+
+# Read-only snapshot for atomic backup
+otel-cli exec --name "btrfs subvolume snapshot" -- btrfs subvolume snapshot -r "$data_dir" "$snapshot_dir" || exit $?
+
+otel-cli exec --name "restic backup directory" -- /usr/local/bin/restic backup "$snapshot_dir"
+
+# Always remove snapshot subvolume, even if restic fails
+otel-cli exec --name "btrfs subvolume delete" -- btrfs subvolume delete "$snapshot_dir"

roles/any.tools.backup-scripts/templates/podman-postgres.backup.sh.j2

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+podman exec -i {{ item.container }} pg_dump -U {{ item.pg_user }} {{ item.database }} |
+    /usr/bin/gzip --rsyncable |
+    /usr/local/bin/restic backup --stdin --stdin-filename {{ item.name }}.sql.gz