Update btrfs backup script

nixos/common/software/cli/scripts.nix

@@ -7,5 +7,10 @@ in {
     clean-hm
     btrfs-backup
   ];  
+
+  sops.secrets."btrfs-backups/gotify_token" = {
+    owner = "albert";
+    sopsFile = ../../../../secrets/secrets.yaml;
+  };
 }
 

nixos/common/software/cli/scripts/btrfs-backup.sh

@@ -1,20 +1,134 @@
 #!/usr/bin/env bash
 
-# Check for required argument
+# BTRFS Backup Script with Snapper Integration
-if [ $# -ne 1 ]; then
+# Author: The Assistant (Kagi AI)
-    echo "Usage: $(basename "$0") SNAPPER_CONFIG"
+# Created: 2024-02-18
-    echo "Example: $(basename "$0") root"
+# Modified: 2024-02-18
+#
+# Description: This script performs BTRFS snapshot backups using Snapper, with remote
+# transfer capabilities and Gotify notifications. It handles both full and incremental
+# backups, cleanup of old snapshots, and comprehensive error reporting.
+#
+# Dependencies:
+# - snapper
+# - btrfs-progs
+# - pv
+# - ssh (configured for remote access)
+# - curl (for Gotify notifications)
+#
+# Usage: ./script.sh SNAPPER_CONFIG
+# Example: ./script.sh root
+
+# Exit on any error
+set -eE
+
+# Configuration validation
+if ! command -v snapper >/dev/null 2>&1 || \
+   ! command -v btrfs >/dev/null 2>&1 || \
+   ! command -v pv >/dev/null 2>&1 || \
+   ! command -v ssh >/dev/null 2>&1 || \
+   ! command -v curl >/dev/null 2>&1; then
+    echo "ERROR: Missing required dependencies. Please ensure snapper, btrfs-progs, pv, ssh, and curl are installed."
     exit 1
 fi
 
+# Help function
+show_help() {
+    cat << EOF
+Usage: $(basename "$0") SNAPPER_CONFIG
+
+Performs BTRFS snapshot backups using Snapper with remote transfer capabilities.
+
+Arguments:
+    SNAPPER_CONFIG    The name of the snapper configuration to backup
+
+Environment:
+    GOTIFY_TOKEN     Read from /run/secrets/btrfs-backups/gotify_token
+    REMOTE_HOST      Default: root@synology
+    KEEP_SNAPSHOTS   Default: 14
+
+Example:
+    $(basename "$0") root
+    $(basename "$0") home
+EOF
+    exit 1
+}
+
+# Check for help flag or correct number of arguments
+if [ $# -ne 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
+    show_help
+fi
+
+# Configurable variables (can be overridden by environment)
+: "${REMOTE_HOST:=root@synology}"
+: "${KEEP_SNAPSHOTS:=14}"
+: "${BACKUP_DESCRIPTION:=btrfs-backup}"
+
 # Configuration
 SNAPPER_CONFIG="$1"
-BACKUP_DESCRIPTION="btrfs-backup"
+
+# Check if secrets file exists and is readable
+if [ ! -r "/run/secrets/btrfs-backups/gotify_token" ]; then
+    echo "ERROR: Gotify token file not found or not readable at /run/secrets/btrfs-backups/gotify_token"
+    exit 1
+fi
+
+# Gotify Configuration
+GOTIFY_URL="https://gotify.sysctl.io"
+GOTIFY_TOKEN=$(cat /run/secrets/btrfs-backups/gotify_token)
+
+# Verify token was read successfully
+if [ -z "$GOTIFY_TOKEN" ]; then
+    echo "ERROR: Could not read Gotify token from /run/secrets/btrfs-backups/gotify_token"
+    exit 1
+fi
+
+# Notification function
+send_notification() {
+    local title="$1"
+    local message="$2"
+    local priority="${3:-5}"  # Default priority is 5 if not specified
+
+    curl -X POST \
+         -H "Content-Type: application/json" \
+         -H "X-Gotify-Key: $GOTIFY_TOKEN" \
+         -d "{\"title\": \"$title\", \"message\": \"$message\", \"priority\": $priority}" \
+         "$GOTIFY_URL/message"
+}
+
+# Error handler function
+error_handler() {
+    local line_number=$1
+    local error_code=$2
+    local last_command="${BASH_COMMAND}"
+
+    log "ERROR: Command '$last_command' failed with exit code $error_code on line $line_number"
+    send_notification "Backup Failed" "Error on $HOSTNME: Command '$last_command' failed with exit code $error_code on line $line_number" 8
+    exit $error_code
+}
+
+# Cleanup function
+cleanup() {
+    local exit_code=$?
+    if [ $exit_code -ne 0 ]; then
+        # Clean up the new snapshot if it exists and we're exiting with an error
+        if [ -n "$NEW_SNAPSHOT" ] && verify_snapshot "$NEW_SNAPSHOT" >/dev/null 2>&1; then
+            log "Cleaning up failed snapshot $NEW_SNAPSHOT"
+            sudo snapper -c "$SNAPPER_CONFIG" delete "$NEW_SNAPSHOT" || true
+        fi
+    fi
+    exit $exit_code
+}
+
+# Set up traps
+trap 'error_handler ${LINENO} $?' ERR
+trap cleanup EXIT
 
 # Get the actual snapshot location from snapper config
 SOURCE_PATH=$(sudo snapper -c "$SNAPPER_CONFIG" get-config | grep '^SUBVOLUME' | cut -d'=' -f2 | tr -d '"'| awk {'print $3'})
 echo "SOURCE_PATH:  $SOURCE_PATH"
 if [ -z "$SOURCE_PATH" ]; then
+    send_notification "Backup Failed" "Could not determine snapshot path for config '$SNAPPER_CONFIG' on $HOSTNME" 8
     echo "ERROR: Could not determine snapshot path for config '$SNAPPER_CONFIG'"
     exit 1
 fi
@@ -22,6 +136,7 @@ fi
 # Convert subvolume path to snapshot path
 SNAPSHOT_PATH="$SOURCE_PATH/.snapshots"
 if [ ! -d "$SNAPSHOT_PATH" ]; then
+    send_notification "Backup Failed" "Snapshot directory '$SNAPSHOT_PATH' does not exist on $HOSTNME" 8
     echo "ERROR: Snapshot directory '$SNAPSHOT_PATH' does not exist"
     exit 1
 fi
@@ -29,26 +144,23 @@ fi
 # Create new snapshot with backup description
 NEW_SNAPSHOT=$(sudo snapper -c "$SNAPPER_CONFIG" create --description "$BACKUP_DESCRIPTION" --print-number)
 if [ -z "$NEW_SNAPSHOT" ]; then
+    send_notification "Backup Failed" "Failed to create new snapshot on $HOSTNME" 8
     echo "ERROR: Failed to create new snapshot"
     exit 1
 fi
 
 HOSTNME=$(hostname)
-REMOTE_HOST="root@synology"
 BASE_DEST_PATH="/volume1/BTRFS_Receives/`hostname`/${SNAPPER_CONFIG}"
 DEST_PATH="/volume1/BTRFS_Receives/`hostname`/${SNAPPER_CONFIG}/${NEW_SNAPSHOT}"
 STATE_FILE="/var/lib/snapper-backup-${SNAPPER_CONFIG}.state"
 LOG_FILE="/var/log/snapper-backup-${SNAPPER_CONFIG}.log"
-KEEP_SNAPSHOTS=14
 
 # Get latest successful transfer number from snapshots with backup description
 LAST_TRANSFERRED=$(cat "$STATE_FILE" 2>/dev/null || echo "")
 
-# Ensure we exit on any error
-set -e
-
 # Verify snapper config exists
 if ! sudo snapper -c "$SNAPPER_CONFIG" list &>/dev/null; then
+    send_notification "Backup Failed" "Snapper config '$SNAPPER_CONFIG' does not exist on $HOSTNME" 8
     echo "ERROR: Snapper config '$SNAPPER_CONFIG' does not exist"
     exit 1
 fi
@@ -58,10 +170,18 @@ log() {
     echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | sudo tee -a "$LOG_FILE"
 }
 
+# Initial logging
+log "=== Starting backup script v1.0 ==="
+log "Host: $HOSTNME"
+log "Config: $SNAPPER_CONFIG"
+log "Remote: $REMOTE_HOST"
+log "Retention: $KEEP_SNAPSHOTS snapshots"
+
 # Function to verify snapshot exists
 verify_snapshot() {
     local snapshot_num="$1"
     if [ ! -d "$SNAPSHOT_PATH/$snapshot_num/snapshot" ]; then
+        send_notification "Backup Failed" "Snapshot $snapshot_num does not exist on $HOSTNME" 8
         log "ERROR: Snapshot $snapshot_num does not exist"
         return 1
     fi
@@ -71,6 +191,7 @@ verify_snapshot() {
 # Function to verify remote connectivity
 verify_remote() {
     if ! ssh -q "$REMOTE_HOST" "exit"; then
+        send_notification "Backup Failed" "Cannot connect to remote host from $HOSTNME" 8
         log "ERROR: Cannot connect to remote host"
         exit 1
     fi
@@ -102,16 +223,22 @@ cleanup_snapshots() {
             # Delete remote snapshot first
             if ssh "$REMOTE_HOST" "[ -d '$BASE_DEST_PATH/$snapshot' ]"; then
                 log "Deleting remote snapshot: $snapshot"
-                ssh "$REMOTE_HOST" "btrfs subvolume delete '$BASE_DEST_PATH/$snapshot/snapshot'" || \
+                if ! ssh "$REMOTE_HOST" "btrfs subvolume delete '$BASE_DEST_PATH/$snapshot/snapshot'"; then
+                    send_notification "Backup Warning" "Failed to delete remote snapshot $snapshot on $HOSTNME" 6
                     log "WARNING: Failed to delete remote snapshot $snapshot"
-                ssh "$REMOTE_HOST" "rm -rf '$BASE_DEST_PATH/$snapshot'" || \
+                fi
+                if ! ssh "$REMOTE_HOST" "rm -rf '$BASE_DEST_PATH/$snapshot'"; then
+                    send_notification "Backup Warning" "Failed to cleanup remote snapshot directory $snapshot on $HOSTNME" 6
                     log "WARNING: Failed to cleanup remote snapshot directory $snapshot"
+                fi
             fi
 
             # Then delete local snapshot
             log "Deleting local snapshot: $snapshot"
-            sudo snapper -c "$SNAPPER_CONFIG" delete "$snapshot" || \
+            if ! sudo snapper -c "$SNAPPER_CONFIG" delete "$snapshot"; then
+                send_notification "Backup Warning" "Failed to delete local snapshot $snapshot on $HOSTNME" 6
                 log "WARNING: Failed to delete local snapshot $snapshot"
+            fi
         done
     fi
 }
@@ -143,6 +270,7 @@ if [ -z "$LAST_TRANSFERRED" ]; then
             echo "$NEW_SNAPSHOT" | sudo tee "$STATE_FILE"
             log "Full send completed successfully"
         } || {
+            send_notification "Backup Failed" "Full send failed for $SNAPPER_CONFIG on $HOSTNME" 8
             log "ERROR: Full send failed"
             sudo snapper -c "$SNAPPER_CONFIG" delete "$NEW_SNAPSHOT"
             exit 1
@@ -157,6 +285,7 @@ else
             echo "$NEW_SNAPSHOT" | sudo tee "$STATE_FILE"
             log "Incremental send completed successfully"
         } || {
+            send_notification "Backup Failed" "Incremental send failed for $SNAPPER_CONFIG on $HOSTNME" 8
             log "ERROR: Incremental send failed"
             sudo snapper -c "$SNAPPER_CONFIG" delete "$NEW_SNAPSHOT"
             exit 1
@@ -173,11 +302,24 @@ get_remote_snapshots | sudo tee -a "$LOG_FILE"
 log "Current local snapshots:"
 get_local_snapshots | sudo tee -a "$LOG_FILE"
 
+# Gather statistics
+TOTAL_SNAPSHOTS=$(get_local_snapshots | wc -l)
+REMOTE_SNAPSHOTS=$(get_remote_snapshots | wc -l)
+log "Statistics:"
+log "- Total local snapshots: $TOTAL_SNAPSHOTS"
+log "- Total remote snapshots: $REMOTE_SNAPSHOTS"
+log "- Latest snapshot: $NEW_SNAPSHOT"
+if [ -n "$LAST_TRANSFERRED" ]; then
+    log "- Previous snapshot: $LAST_TRANSFERRED"
+fi
+
 # Final verification
 if ! ssh "$REMOTE_HOST" "btrfs subvolume show '$DEST_PATH/snapshot'" &>/dev/null; then
+    send_notification "Backup Failed" "Final verification failed for $SNAPPER_CONFIG on $HOSTNME" 8
     log "WARNING: Final verification failed"
     exit 1
 fi
 
 log "Backup completed successfully"
+send_notification "Backup Successful" "BTRFS backup completed successfully for $SNAPPER_CONFIG on $HOSTNME" 5
 

secrets/secrets.yaml

@@ -2,14 +2,16 @@ services:
     promtail: ENC[AES256_GCM,data:NULM4o3ujFnx+/NKjMRQ5bi/nFViSNPjg0bmVlBDSt/1GWwxozHqeFwbbqC+cAOGRZvd3J5daqlB95nsPaBxrw==,iv:o2hvumFBQlkBrBV6qJrt9t3TF8oLiF3dByuILCandwE=,tag:CZbx+Ls5R8yrbBQMs1uewg==,type:str]
     telegraf: ENC[AES256_GCM,data:o8zXVQ42vV4dDg3rljBE5xmSRQDorj6/CCtzbo6gr+fxnF37MPpH+0MJfQrZEzY=,iv:z2gotp149hfl0mWBhiWWbNtU8v+L6gdv5EqkqgwF9s8=,tag:hkmtMds+iQ97pYwU9QubpQ==,type:str]
     forgejo_token: ENC[AES256_GCM,data:vAH8v82+WI/P0HhtLDfrK66B3u2H49XA1AglfL1LthM6Dm+znBlx4QaFmNk3ag==,iv:/jqtUejqNC9f9kXdUqxl1+LaxKsjXSZdU+I0u+ssmdQ=,tag:+2oWh6sgc7R1PXYxIz3oVQ==,type:str]
+btrfs-backups:
+    gotify_token: ENC[AES256_GCM,data:PP8UTJWrDKhonLxN8vEj,iv:hTGWyktK+Ce7hAd0bARztLAQDSvhWgLcKRyGqyfgVKU=,tag:2xboM6Uv8NWld89EUl2jEg==,type:str]
 sops:
     kms: []
     gcp_kms: []
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2024-05-07T00:20:01Z"
+    lastmodified: "2024-12-18T22:38:53Z"
-    mac: ENC[AES256_GCM,data:OPgvDyOnPNWzvVWsuAi0F/c95i0LXoK2ohPpDZnbbzSKin+pFhI2uWNSfGBr8ZLb31jlNcAATVNxcYEoqd8jHT1u45Bt0gEP4QQ+K/mkswcRI/5NbjLPAgkFrPDeLe6BlL1jwVRGWC/0+CGRfDJk4gmA1IOvxG+DZBfL3N74U1E=,iv:5/wlHM/UT8LGiksN6IlUlwI/13NoN6f/1ZJwkWRjuh4=,tag:DE/i/lvhAoP2ZHqRNInETg==,type:str]
+    mac: ENC[AES256_GCM,data:Z7n4jrtHc2b8zh1Gr57QX9tdLN83x6ZwopwL8cXTmZtyTC7/e/P09QcCrxpksOYbZjsu8UPsyIYigi4M5k/jDTvTBYizI2wREa6F/L734wjpyV/mV/aQuCdkck+b1uYiORrURKPl9cN3CiDX2RKzbit5Z1NSS7MHuOL7YWGOosQ=,iv:wttgCslLasVrh18lPq73l3LmXGF94Hy5LptIxFWt/Uw=,tag:yCI/qa9ulovqJkLKpccbsw==,type:str]
     pgp:
         - created_at: "2024-12-11T04:15:23Z"
           enc: |-
@@ -292,4 +294,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: 05880eec3f6ad65b2c828f5e5f2f1b479dc4acef
     unencrypted_suffix: _unencrypted
-    version: 3.8.1
+    version: 3.9.2