From fac2d9011693075d32c7498a04c0a52c0c7e55db Mon Sep 17 00:00:00 2001 From: albert Date: Wed, 5 Jun 2024 12:46:44 +0900 Subject: [PATCH] rearrange backups --- README.md | 3 + nixos/hosts/backups-rpi4/cron.nix | 7 +- .../backups-rpi4/scripts/rsync-data-vol.sh | 24 +++--- .../hosts/backups-rpi4/scripts/rsync-media.sh | 76 +++++++++---------- .../backups-rpi4/scripts/rsync-sysctl.sh | 24 +++--- 5 files changed, 69 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index b0922a04..9dc67122 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,9 @@ nix develop -c /etc/nixos/git/docs/setup.sh ## Homelab general * [-] High Priority: * [x] Fix mailserver / roundcube failing to authenticate + * [ ] Rearrange backups: + * Move /mnt/sdb/{Data,Backups} to /mnt/sda + * Move /mnt/sda/Movies to /mnt/sdb * [ ] Fixes: * [ ] Migrate Forgejo sqlite db to MySQL * [ ] Alternatively, figure out why sqlite is taking so long to load diff --git a/nixos/hosts/backups-rpi4/cron.nix b/nixos/hosts/backups-rpi4/cron.nix index d45b651b..7c026183 100644 --- a/nixos/hosts/backups-rpi4/cron.nix +++ b/nixos/hosts/backups-rpi4/cron.nix @@ -3,9 +3,10 @@ enable = true; systemCronJobs = [ # Backups from nuc-docker01 - See "backup-script.nix" - ''0 04 * * * root rsync-media &> /root/logs/media/rsync_`date "+\%Y-\%m-\%d"`.log'' - ''0 02 * * 3 root rsync-data-vol &> /root/logs/data-vol/rsync_`date "+\%Y-\%m-\%d"`.log'' - ''0 22 * * * root rsync-sysctl &> /root/logs/sysctl/rsync_`date "+\%Y-\%m-\%d"`.log'' + # TODO: Re-enable when done + # ''0 04 * * * root rsync-media &> /root/logs/media/rsync_`date "+\%Y-\%m-\%d"`.log'' + # ''0 02 * * 3 root rsync-data-vol &> /root/logs/data-vol/rsync_`date "+\%Y-\%m-\%d"`.log'' + # ''0 22 * * * root rsync-sysctl &> /root/logs/sysctl/rsync_`date "+\%Y-\%m-\%d"`.log'' ]; }; } diff --git a/nixos/hosts/backups-rpi4/scripts/rsync-data-vol.sh b/nixos/hosts/backups-rpi4/scripts/rsync-data-vol.sh index 1413f202..383e141f 100644 --- a/nixos/hosts/backups-rpi4/scripts/rsync-data-vol.sh +++ b/nixos/hosts/backups-rpi4/scripts/rsync-data-vol.sh @@ -5,21 +5,21 @@ echo "Sending start message" payload="{\"message\": \"$BACKUP_NAME Backup has started\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Started\"}" curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload" -drive1=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGHYP8C | awk {'print $1'}` -drive2=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGXW93C | awk {'print $1'}` -drive3=`lsblk -o NAME,SERIAL | grep -v luks | grep 5QH05G3F | awk {'print $1'}` +drivea=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGXW93C | awk {'print $1'}` +driveb=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGHYP8C | awk {'print $1'}` +drivec=`lsblk -o NAME,SERIAL | grep -v luks | grep 5QH05G3F | awk {'print $1'}` -mount1=`mount -l | grep $drive1`; mounted1=`echo $?` -mount2=`mount -l | grep $drive2`; mounted2=`echo $?` -mount3=`mount -l | grep $drive3`; mounted3=`echo $?` +mounta=`mount -l | grep $drivea`; mounteda=`echo $?` +mountb=`mount -l | grep $driveb`; mountedb=`echo $?` +mountc=`mount -l | grep $drivec`; mountedc=`echo $?` echo "Drive status" -echo "Drive 1: $drive1 / $mount1 / $mounted1" -echo "Drive 2: $drive2 / $mount2 / $mounted2" -echo "Drive 3: $drive3 / $mount3 / $mounted3" +echo "Drive 2: $drivea / $mounta / $mounteda" +echo "Drive 1: $driveb / $mountb / $mountedb" +echo "Drive 3: $drivec / $mountc / $mountedc" # If any drive isn't found, exit: -if [[ $mounted1 != 0 ]] || [[ $mounted2 != 0 ]] || [[ $mounted3 != 0 ]] ; then +if [[ $mounteda != 0 ]] || [[ $mountedb != 0 ]] || [[ $mountedc != 0 ]] ; then echo "One or more drives is not mounted. Exiting" payload="{\"message\": \"$BACKUP_NAME Backup stopped. One or more local drives are not mounted.\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Stopped\"}" curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload" @@ -50,8 +50,8 @@ if [[ $check != 1 ]] ; then fi echo -e "======= Time started : `date`" -echo -e "rsync -avr --delete --exclude=Downloads/ --exclude='Data/Docker/sysctl.io/' root@nuc-docker01:/Storage/Data/ /mnt/$drive1/Data/" -rsync -avr --delete --exclude=Downloads/ --exclude='Data/Docker/sysctl.io/' root@nuc-docker01:/Storage/Data/ /mnt/$drive1/Data/ +echo -e "rsync -avr --delete --exclude=Downloads/ --exclude='Data/Docker/sysctl.io/' root@nuc-docker01:/Storage/Data/ /mnt/$drivea/Data/" +rsync -avr --delete --exclude=Downloads/ --exclude='Data/Docker/sysctl.io/' root@nuc-docker01:/Storage/Data/ /mnt/$drivea/Data/ echo -e "======= Time complete: `date`" echo "Sending completion message" diff --git a/nixos/hosts/backups-rpi4/scripts/rsync-media.sh b/nixos/hosts/backups-rpi4/scripts/rsync-media.sh index ad5a53f2..88744c29 100644 --- a/nixos/hosts/backups-rpi4/scripts/rsync-media.sh +++ b/nixos/hosts/backups-rpi4/scripts/rsync-media.sh @@ -5,21 +5,21 @@ echo "Sending start message" payload="{\"message\": \"$BACKUP_NAME Backup has started\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Started\"}" curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload" -drive1=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGHYP8C | awk {'print $1'}` -drive2=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGXW93C | awk {'print $1'}` -drive3=`lsblk -o NAME,SERIAL | grep -v luks | grep 5QH05G3F | awk {'print $1'}` +driveb=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGHYP8C | awk {'print $1'}` +drivea=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGXW93C | awk {'print $1'}` +drivec=`lsblk -o NAME,SERIAL | grep -v luks | grep 5QH05G3F | awk {'print $1'}` -mount1=`mount -l | grep $drive1`; mounted1=`echo $?` -mount2=`mount -l | grep $drive2`; mounted2=`echo $?` -mount3=`mount -l | grep $drive3`; mounted3=`echo $?` +mountb=`mount -l | grep $driveb`; mountedb=`echo $?` +mounta=`mount -l | grep $drivea`; mounteda=`echo $?` +mount3=`mount -l | grep $drivec`; mountedc=`echo $?` echo "Drive status" -echo "Drive 1: $drive1 / $mount1 / $mounted1" -echo "Drive 2: $drive2 / $mount2 / $mounted2" -echo "Drive 3: $drive3 / $mount3 / $mounted3" +echo "Drive 2: $drivea / $mounta / $mounteda" +echo "Drive 1: $driveb / $mountb / $mountedb" +echo "Drive 3: $drivec / $mount3 / $mountedc" # If any drive isn't found, exit: -if [[ $mounted1 != 0 ]] || [[ $mounted2 != 0 ]] || [[ $mounted3 != 0 ]] ; then +if [[ $mounteda != 0 ]] || [[ $mountedb != 0 ]] || [[ $mountedc != 0 ]] ; then echo "One or more drives is not mounted. Exiting" payload="{\"message\": \"$BACKUP_NAME Backup stopped. One or more local drives are not mounted.\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Stopped\"}" curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload" @@ -50,60 +50,60 @@ if [[ $check != 1 ]] ; then fi echo -e "======= Time started : `date`" -echo -e "rsync -avr --delete --exclude=luks.img /Storage/Backups/Manual/Computers/ /mnt/$drive1/Backups/Manual/Computers/" -rsync -avr --delete --exclude=luks.img root@nuc-docker01:/Storage/Backups/Manual/Computers/ /mnt/$drive1/Backups/Manual/Computers/ +echo -e "rsync -avr --delete --exclude=luks.img /Storage/Backups/Manual/Computers/ /mnt/$drivea/Backups/Manual/Computers/" +rsync -avr --delete --exclude=luks.img root@nuc-docker01:/Storage/Backups/Manual/Computers/ /mnt/$drivea/Backups/Manual/Computers/ echo -e "======= Time: `date`" -echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Backups/Backblaze/ /mnt/$drive1/Backups/Backblaze/" -rsync -avr --delete root@nuc-docker01:/Storage/Backups/Backblaze/ /mnt/$drive1/Backups/Backblaze/ +echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Backups/Backblaze/ /mnt/$drivea/Backups/Backblaze/" +rsync -avr --delete root@nuc-docker01:/Storage/Backups/Backblaze/ /mnt/$drivea/Backups/Backblaze/ echo -e "======= Time: `date`" -echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Backups/Logs/ /mnt/$drive1/Backups/Logs/" -rsync -avr --delete root@nuc-docker01:/Storage/Backups/Logs/ /mnt/$drive1/Backups/Logs/ +echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Backups/Logs/ /mnt/$drivea/Backups/Logs/" +rsync -avr --delete root@nuc-docker01:/Storage/Backups/Logs/ /mnt/$drivea/Backups/Logs/ echo -e "======= Time: `date`" -echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Backups/Docker/ /mnt/$drive1/Backups/Docker/" -rsync -avr --delete root@nuc-docker01:/Storage/Backups/Docker/ /mnt/$drive1/Backups/Docker/ +echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Backups/Docker/ /mnt/$drivea/Backups/Docker/" +rsync -avr --delete root@nuc-docker01:/Storage/Backups/Docker/ /mnt/$drivea/Backups/Docker/ echo -e "======= Time: `date`" -echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Pictures/ /mnt/$drive1/Pictures/" -rsync -avr --delete root@nuc-docker01:/Storage/Media/Pictures/ /mnt/$drive1/Pictures/ +echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Pictures/ /mnt/$driveb/Pictures/" +rsync -avr --delete root@nuc-docker01:/Storage/Media/Pictures/ /mnt/$driveb/Pictures/ echo -e "======= Time: `date`" -echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Books/ /mnt/$drive1/Books/" -rsync -avr --delete root@nuc-docker01:/Storage/Media/Books/ /mnt/$drive1/Books/ +echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Books/ /mnt/$driveb/Books/" +rsync -avr --delete root@nuc-docker01:/Storage/Media/Books/ /mnt/$driveb/Books/ echo -e "======= Time: `date`" -echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Music/ /mnt/$drive1/Music/" -rsync -avr --delete root@nuc-docker01:/Storage/Media/Music/ /mnt/$drive1/Music/ +echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Music/ /mnt/$driveb/Music/" +rsync -avr --delete root@nuc-docker01:/Storage/Media/Music/ /mnt/$driveb/Music/ # echo -e "======= Time: `date`" -# echo -e "rsync -avr --delete --exclude=Downloads/ /Storage/Data/ /mnt/$drive1/Data/" -# rsync -avr --delete --exclude=Downloads/ --exclude=Docker/sysctl* root@nuc-docker01:/Storage/Data/ /mnt/$drive1/Data/ +# echo -e "rsync -avr --delete --exclude=Downloads/ /Storage/Data/ /mnt/$driveb/Data/" +# rsync -avr --delete --exclude=Downloads/ --exclude=Docker/sysctl* root@nuc-docker01:/Storage/Data/ /mnt/$driveb/Data/ echo -e "======= Time: `date`" -echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Movies/ /mnt/$drive2/Movies/" -rsync -avr --delete root@nuc-docker01:/Storage/Media/Movies/ /mnt/$drive2/Movies/ +echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Movies/ /mnt/$driveb/Movies/" +rsync -avr --delete root@nuc-docker01:/Storage/Media/Movies/ /mnt/$driveb/Movies/ echo -e "======= Time: `date`" -echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[0-9]* /mnt/$drive1/TV_Shows/" -rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[0-9]* /mnt/$drive1/TV_Shows/ +echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[0-9]* /mnt/$driveb/TV_Shows/" +rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[0-9]* /mnt/$driveb/TV_Shows/ echo -e "======= Time: `date`" -echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[A-K]* /mnt/$drive1/TV_Shows/" -rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[A-E]* /mnt/$drive1/TV_Shows/ +echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[A-K]* /mnt/$driveb/TV_Shows/" +rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[A-E]* /mnt/$driveb/TV_Shows/ echo -e "======= Time: `date`" -echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[L-R]* /mnt/$drive2/TV_Shows/" -rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[F-R]* /mnt/$drive2/TV_Shows/ +echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[L-R]* /mnt/$drivea/TV_Shows/" +rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[F-R]* /mnt/$drivea/TV_Shows/ echo -e "======= Time: `date`" -echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[S-Z]* /mnt/$drive3/TV_Shows/" -rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[S-Z]* /mnt/$drive3/TV_Shows/ +echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[S-Z]* /mnt/$drivec/TV_Shows/" +rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[S-Z]* /mnt/$drivec/TV_Shows/ echo -e "======= Time: `date`" -echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/YouTube /mnt/$drive3/YouTube/" -rsync -avr --delete root@nuc-docker01:/Storage/Media/YouTube/ /mnt/$drive3/YouTube/ +echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/YouTube /mnt/$drivec/YouTube/" +rsync -avr --delete root@nuc-docker01:/Storage/Media/YouTube/ /mnt/$drivec/YouTube/ echo -e "======= Time complete: `date`" echo "Sending completion message" diff --git a/nixos/hosts/backups-rpi4/scripts/rsync-sysctl.sh b/nixos/hosts/backups-rpi4/scripts/rsync-sysctl.sh index 8024e089..2f092d88 100644 --- a/nixos/hosts/backups-rpi4/scripts/rsync-sysctl.sh +++ b/nixos/hosts/backups-rpi4/scripts/rsync-sysctl.sh @@ -20,21 +20,21 @@ if [[ $count -ge 4 ]] ; then exit fi -drive1=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGHYP8C | awk {'print $1'}` -drive2=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGXW93C | awk {'print $1'}` -drive3=`lsblk -o NAME,SERIAL | grep -v luks | grep 5QH05G3F | awk {'print $1'}` +drivea=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGXW93C | awk {'print $1'}` +driveb=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGHYP8C | awk {'print $1'}` +drivec=`lsblk -o NAME,SERIAL | grep -v luks | grep 5QH05G3F | awk {'print $1'}` -mount1=`mount -l | grep $drive1`; mounted1=`echo $?` -mount2=`mount -l | grep $drive2`; mounted2=`echo $?` -mount3=`mount -l | grep $drive3`; mounted3=`echo $?` +mounta=`mount -l | grep $drivea`; mounteda=`echo $?` +mountb=`mount -l | grep $driveb`; mountedb=`echo $?` +mountc=`mount -l | grep $drivec`; mountedc=`echo $?` echo "Drive status" -echo "Drive 1: $drive1 / $mount1 / $mounted1" -echo "Drive 2: $drive2 / $mount2 / $mounted2" -echo "Drive 3: $drive3 / $mount3 / $mounted3" +echo "Drive 2: $drivea / $mounta / $mounteda" +echo "Drive 1: $driveb / $mountb / $mountedb" +echo "Drive 3: $drivec / $mountc / $mountedc" # If any drive isn't found, exit: -if [[ $mounted1 != 0 ]] || [[ $mounted2 != 0 ]] || [[ $mounted3 != 0 ]] ; then +if [[ $mounteda != 0 ]] || [[ $mountedb != 0 ]] || [[ $mountedc != 0 ]] ; then echo "One or more drives is not mounted. Exiting" payload="{\"message\": \"$BACKUP_NAME Backup stopped. One or more local drives are not mounted.\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Stopped\"}" curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload" @@ -43,8 +43,8 @@ fi # Start the backups echo -e "======= Time started : `date`" -echo -e "rsync -avr --delete --exclude=Downloads/ root@framework-server:/Storage/Data/Docker/sysctl.io/ /mnt/$drive1/Data/Docker/sysctl.io/" -rsync -avr --delete --exclude=Downloads/ root@framework-server:/Storage/Data/Docker/sysctl.io/ /mnt/$drive1/Data/Docker/sysctl.io/ +echo -e "rsync -avr --delete --exclude=Downloads/ root@framework-server:/Storage/Data/Docker/sysctl.io/ /mnt/$driveb/Data/Docker/sysctl.io/" +rsync -avr --delete --exclude=Downloads/ root@framework-server:/Storage/Data/Docker/sysctl.io/ /mnt/$driveb/Data/Docker/sysctl.io/ echo -e "======= Time complete: `date`" echo "Sending completion message"