diff --git a/nixos/common/software/cli/scripts.nix b/nixos/common/software/cli/scripts.nix index f7863895..aee0fa9e 100644 --- a/nixos/common/software/cli/scripts.nix +++ b/nixos/common/software/cli/scripts.nix @@ -7,5 +7,10 @@ in { clean-hm btrfs-backup ]; + + sops.secrets."btrfs-backups/gotify_token" = { + owner = "albert"; + sopsFile = ../../../../secrets/secrets.yaml; + }; } diff --git a/nixos/common/software/cli/scripts/btrfs-backup.sh b/nixos/common/software/cli/scripts/btrfs-backup.sh index 015af71c..633479b8 100755 --- a/nixos/common/software/cli/scripts/btrfs-backup.sh +++ b/nixos/common/software/cli/scripts/btrfs-backup.sh @@ -1,20 +1,134 @@ #!/usr/bin/env bash -# Check for required argument -if [ $# -ne 1 ]; then - echo "Usage: $(basename "$0") SNAPPER_CONFIG" - echo "Example: $(basename "$0") root" +# BTRFS Backup Script with Snapper Integration +# Author: The Assistant (Kagi AI) +# Created: 2024-02-18 +# Modified: 2024-02-18 +# +# Description: This script performs BTRFS snapshot backups using Snapper, with remote +# transfer capabilities and Gotify notifications. It handles both full and incremental +# backups, cleanup of old snapshots, and comprehensive error reporting. +# +# Dependencies: +# - snapper +# - btrfs-progs +# - pv +# - ssh (configured for remote access) +# - curl (for Gotify notifications) +# +# Usage: ./script.sh SNAPPER_CONFIG +# Example: ./script.sh root + +# Exit on any error +set -eE + +# Configuration validation +if ! command -v snapper >/dev/null 2>&1 || \ + ! command -v btrfs >/dev/null 2>&1 || \ + ! command -v pv >/dev/null 2>&1 || \ + ! command -v ssh >/dev/null 2>&1 || \ + ! command -v curl >/dev/null 2>&1; then + echo "ERROR: Missing required dependencies. Please ensure snapper, btrfs-progs, pv, ssh, and curl are installed." exit 1 fi +# Help function +show_help() { + cat << EOF +Usage: $(basename "$0") SNAPPER_CONFIG + +Performs BTRFS snapshot backups using Snapper with remote transfer capabilities. + +Arguments: + SNAPPER_CONFIG The name of the snapper configuration to backup + +Environment: + GOTIFY_TOKEN Read from /run/secrets/btrfs-backups/gotify_token + REMOTE_HOST Default: root@synology + KEEP_SNAPSHOTS Default: 14 + +Example: + $(basename "$0") root + $(basename "$0") home +EOF + exit 1 +} + +# Check for help flag or correct number of arguments +if [ $# -ne 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + show_help +fi + +# Configurable variables (can be overridden by environment) +: "${REMOTE_HOST:=root@synology}" +: "${KEEP_SNAPSHOTS:=14}" +: "${BACKUP_DESCRIPTION:=btrfs-backup}" + # Configuration SNAPPER_CONFIG="$1" -BACKUP_DESCRIPTION="btrfs-backup" + +# Check if secrets file exists and is readable +if [ ! -r "/run/secrets/btrfs-backups/gotify_token" ]; then + echo "ERROR: Gotify token file not found or not readable at /run/secrets/btrfs-backups/gotify_token" + exit 1 +fi + +# Gotify Configuration +GOTIFY_URL="https://gotify.sysctl.io" +GOTIFY_TOKEN=$(cat /run/secrets/btrfs-backups/gotify_token) + +# Verify token was read successfully +if [ -z "$GOTIFY_TOKEN" ]; then + echo "ERROR: Could not read Gotify token from /run/secrets/btrfs-backups/gotify_token" + exit 1 +fi + +# Notification function +send_notification() { + local title="$1" + local message="$2" + local priority="${3:-5}" # Default priority is 5 if not specified + + curl -X POST \ + -H "Content-Type: application/json" \ + -H "X-Gotify-Key: $GOTIFY_TOKEN" \ + -d "{\"title\": \"$title\", \"message\": \"$message\", \"priority\": $priority}" \ + "$GOTIFY_URL/message" +} + +# Error handler function +error_handler() { + local line_number=$1 + local error_code=$2 + local last_command="${BASH_COMMAND}" + + log "ERROR: Command '$last_command' failed with exit code $error_code on line $line_number" + send_notification "Backup Failed" "Error on $HOSTNME: Command '$last_command' failed with exit code $error_code on line $line_number" 8 + exit $error_code +} + +# Cleanup function +cleanup() { + local exit_code=$? + if [ $exit_code -ne 0 ]; then + # Clean up the new snapshot if it exists and we're exiting with an error + if [ -n "$NEW_SNAPSHOT" ] && verify_snapshot "$NEW_SNAPSHOT" >/dev/null 2>&1; then + log "Cleaning up failed snapshot $NEW_SNAPSHOT" + sudo snapper -c "$SNAPPER_CONFIG" delete "$NEW_SNAPSHOT" || true + fi + fi + exit $exit_code +} + +# Set up traps +trap 'error_handler ${LINENO} $?' ERR +trap cleanup EXIT # Get the actual snapshot location from snapper config SOURCE_PATH=$(sudo snapper -c "$SNAPPER_CONFIG" get-config | grep '^SUBVOLUME' | cut -d'=' -f2 | tr -d '"'| awk {'print $3'}) echo "SOURCE_PATH: $SOURCE_PATH" if [ -z "$SOURCE_PATH" ]; then + send_notification "Backup Failed" "Could not determine snapshot path for config '$SNAPPER_CONFIG' on $HOSTNME" 8 echo "ERROR: Could not determine snapshot path for config '$SNAPPER_CONFIG'" exit 1 fi @@ -22,6 +136,7 @@ fi # Convert subvolume path to snapshot path SNAPSHOT_PATH="$SOURCE_PATH/.snapshots" if [ ! -d "$SNAPSHOT_PATH" ]; then + send_notification "Backup Failed" "Snapshot directory '$SNAPSHOT_PATH' does not exist on $HOSTNME" 8 echo "ERROR: Snapshot directory '$SNAPSHOT_PATH' does not exist" exit 1 fi @@ -29,26 +144,23 @@ fi # Create new snapshot with backup description NEW_SNAPSHOT=$(sudo snapper -c "$SNAPPER_CONFIG" create --description "$BACKUP_DESCRIPTION" --print-number) if [ -z "$NEW_SNAPSHOT" ]; then + send_notification "Backup Failed" "Failed to create new snapshot on $HOSTNME" 8 echo "ERROR: Failed to create new snapshot" exit 1 fi HOSTNME=$(hostname) -REMOTE_HOST="root@synology" BASE_DEST_PATH="/volume1/BTRFS_Receives/`hostname`/${SNAPPER_CONFIG}" DEST_PATH="/volume1/BTRFS_Receives/`hostname`/${SNAPPER_CONFIG}/${NEW_SNAPSHOT}" STATE_FILE="/var/lib/snapper-backup-${SNAPPER_CONFIG}.state" LOG_FILE="/var/log/snapper-backup-${SNAPPER_CONFIG}.log" -KEEP_SNAPSHOTS=14 # Get latest successful transfer number from snapshots with backup description LAST_TRANSFERRED=$(cat "$STATE_FILE" 2>/dev/null || echo "") -# Ensure we exit on any error -set -e - # Verify snapper config exists if ! sudo snapper -c "$SNAPPER_CONFIG" list &>/dev/null; then + send_notification "Backup Failed" "Snapper config '$SNAPPER_CONFIG' does not exist on $HOSTNME" 8 echo "ERROR: Snapper config '$SNAPPER_CONFIG' does not exist" exit 1 fi @@ -58,10 +170,18 @@ log() { echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | sudo tee -a "$LOG_FILE" } +# Initial logging +log "=== Starting backup script v1.0 ===" +log "Host: $HOSTNME" +log "Config: $SNAPPER_CONFIG" +log "Remote: $REMOTE_HOST" +log "Retention: $KEEP_SNAPSHOTS snapshots" + # Function to verify snapshot exists verify_snapshot() { local snapshot_num="$1" if [ ! -d "$SNAPSHOT_PATH/$snapshot_num/snapshot" ]; then + send_notification "Backup Failed" "Snapshot $snapshot_num does not exist on $HOSTNME" 8 log "ERROR: Snapshot $snapshot_num does not exist" return 1 fi @@ -71,6 +191,7 @@ verify_snapshot() { # Function to verify remote connectivity verify_remote() { if ! ssh -q "$REMOTE_HOST" "exit"; then + send_notification "Backup Failed" "Cannot connect to remote host from $HOSTNME" 8 log "ERROR: Cannot connect to remote host" exit 1 fi @@ -102,16 +223,22 @@ cleanup_snapshots() { # Delete remote snapshot first if ssh "$REMOTE_HOST" "[ -d '$BASE_DEST_PATH/$snapshot' ]"; then log "Deleting remote snapshot: $snapshot" - ssh "$REMOTE_HOST" "btrfs subvolume delete '$BASE_DEST_PATH/$snapshot/snapshot'" || \ + if ! ssh "$REMOTE_HOST" "btrfs subvolume delete '$BASE_DEST_PATH/$snapshot/snapshot'"; then + send_notification "Backup Warning" "Failed to delete remote snapshot $snapshot on $HOSTNME" 6 log "WARNING: Failed to delete remote snapshot $snapshot" - ssh "$REMOTE_HOST" "rm -rf '$BASE_DEST_PATH/$snapshot'" || \ + fi + if ! ssh "$REMOTE_HOST" "rm -rf '$BASE_DEST_PATH/$snapshot'"; then + send_notification "Backup Warning" "Failed to cleanup remote snapshot directory $snapshot on $HOSTNME" 6 log "WARNING: Failed to cleanup remote snapshot directory $snapshot" + fi fi # Then delete local snapshot log "Deleting local snapshot: $snapshot" - sudo snapper -c "$SNAPPER_CONFIG" delete "$snapshot" || \ + if ! sudo snapper -c "$SNAPPER_CONFIG" delete "$snapshot"; then + send_notification "Backup Warning" "Failed to delete local snapshot $snapshot on $HOSTNME" 6 log "WARNING: Failed to delete local snapshot $snapshot" + fi done fi } @@ -143,6 +270,7 @@ if [ -z "$LAST_TRANSFERRED" ]; then echo "$NEW_SNAPSHOT" | sudo tee "$STATE_FILE" log "Full send completed successfully" } || { + send_notification "Backup Failed" "Full send failed for $SNAPPER_CONFIG on $HOSTNME" 8 log "ERROR: Full send failed" sudo snapper -c "$SNAPPER_CONFIG" delete "$NEW_SNAPSHOT" exit 1 @@ -157,6 +285,7 @@ else echo "$NEW_SNAPSHOT" | sudo tee "$STATE_FILE" log "Incremental send completed successfully" } || { + send_notification "Backup Failed" "Incremental send failed for $SNAPPER_CONFIG on $HOSTNME" 8 log "ERROR: Incremental send failed" sudo snapper -c "$SNAPPER_CONFIG" delete "$NEW_SNAPSHOT" exit 1 @@ -173,11 +302,24 @@ get_remote_snapshots | sudo tee -a "$LOG_FILE" log "Current local snapshots:" get_local_snapshots | sudo tee -a "$LOG_FILE" +# Gather statistics +TOTAL_SNAPSHOTS=$(get_local_snapshots | wc -l) +REMOTE_SNAPSHOTS=$(get_remote_snapshots | wc -l) +log "Statistics:" +log "- Total local snapshots: $TOTAL_SNAPSHOTS" +log "- Total remote snapshots: $REMOTE_SNAPSHOTS" +log "- Latest snapshot: $NEW_SNAPSHOT" +if [ -n "$LAST_TRANSFERRED" ]; then + log "- Previous snapshot: $LAST_TRANSFERRED" +fi + # Final verification if ! ssh "$REMOTE_HOST" "btrfs subvolume show '$DEST_PATH/snapshot'" &>/dev/null; then + send_notification "Backup Failed" "Final verification failed for $SNAPPER_CONFIG on $HOSTNME" 8 log "WARNING: Final verification failed" exit 1 fi log "Backup completed successfully" +send_notification "Backup Successful" "BTRFS backup completed successfully for $SNAPPER_CONFIG on $HOSTNME" 5 diff --git a/secrets/secrets.yaml b/secrets/secrets.yaml index 5d2bfd0d..5a9d5043 100644 --- a/secrets/secrets.yaml +++ b/secrets/secrets.yaml @@ -2,14 +2,16 @@ services: promtail: ENC[AES256_GCM,data:NULM4o3ujFnx+/NKjMRQ5bi/nFViSNPjg0bmVlBDSt/1GWwxozHqeFwbbqC+cAOGRZvd3J5daqlB95nsPaBxrw==,iv:o2hvumFBQlkBrBV6qJrt9t3TF8oLiF3dByuILCandwE=,tag:CZbx+Ls5R8yrbBQMs1uewg==,type:str] telegraf: ENC[AES256_GCM,data:o8zXVQ42vV4dDg3rljBE5xmSRQDorj6/CCtzbo6gr+fxnF37MPpH+0MJfQrZEzY=,iv:z2gotp149hfl0mWBhiWWbNtU8v+L6gdv5EqkqgwF9s8=,tag:hkmtMds+iQ97pYwU9QubpQ==,type:str] forgejo_token: ENC[AES256_GCM,data:vAH8v82+WI/P0HhtLDfrK66B3u2H49XA1AglfL1LthM6Dm+znBlx4QaFmNk3ag==,iv:/jqtUejqNC9f9kXdUqxl1+LaxKsjXSZdU+I0u+ssmdQ=,tag:+2oWh6sgc7R1PXYxIz3oVQ==,type:str] +btrfs-backups: + gotify_token: ENC[AES256_GCM,data:PP8UTJWrDKhonLxN8vEj,iv:hTGWyktK+Ce7hAd0bARztLAQDSvhWgLcKRyGqyfgVKU=,tag:2xboM6Uv8NWld89EUl2jEg==,type:str] sops: kms: [] gcp_kms: [] azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-05-07T00:20:01Z" - mac: ENC[AES256_GCM,data:OPgvDyOnPNWzvVWsuAi0F/c95i0LXoK2ohPpDZnbbzSKin+pFhI2uWNSfGBr8ZLb31jlNcAATVNxcYEoqd8jHT1u45Bt0gEP4QQ+K/mkswcRI/5NbjLPAgkFrPDeLe6BlL1jwVRGWC/0+CGRfDJk4gmA1IOvxG+DZBfL3N74U1E=,iv:5/wlHM/UT8LGiksN6IlUlwI/13NoN6f/1ZJwkWRjuh4=,tag:DE/i/lvhAoP2ZHqRNInETg==,type:str] + lastmodified: "2024-12-18T22:38:53Z" + mac: ENC[AES256_GCM,data:Z7n4jrtHc2b8zh1Gr57QX9tdLN83x6ZwopwL8cXTmZtyTC7/e/P09QcCrxpksOYbZjsu8UPsyIYigi4M5k/jDTvTBYizI2wREa6F/L734wjpyV/mV/aQuCdkck+b1uYiORrURKPl9cN3CiDX2RKzbit5Z1NSS7MHuOL7YWGOosQ=,iv:wttgCslLasVrh18lPq73l3LmXGF94Hy5LptIxFWt/Uw=,tag:yCI/qa9ulovqJkLKpccbsw==,type:str] pgp: - created_at: "2024-12-11T04:15:23Z" enc: |- @@ -292,4 +294,4 @@ sops: -----END PGP MESSAGE----- fp: 05880eec3f6ad65b2c828f5e5f2f1b479dc4acef unencrypted_suffix: _unencrypted - version: 3.8.1 + version: 3.9.2