albert/nix
albert/nix
1
0
Fork 0
Code Issues Pull requests Activity Actions

rearrange backups

Browse source
This commit is contained in:
albert 2024-06-05 12:46:44 +09:00
parent 6fb0723cbd
commit fac2d90116
Signed by: albert
GPG key ID: 3895DD267CA11BA9
5 changed files with 69 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
README.md
View file

@ -66,6 +66,9 @@ nix develop -c /etc/nixos/git/docs/setup.sh
## Homelab general ## Homelab general
* [-] High Priority: * [-] High Priority:
* [x] Fix mailserver / roundcube failing to authenticate * [x] Fix mailserver / roundcube failing to authenticate
* [ ] Rearrange backups:
* Move /mnt/sdb/{Data,Backups} to /mnt/sda
* Move /mnt/sda/Movies to /mnt/sdb
* [ ] Fixes: * [ ] Fixes:
* [ ] Migrate Forgejo sqlite db to MySQL * [ ] Migrate Forgejo sqlite db to MySQL
* [ ] Alternatively, figure out why sqlite is taking so long to load * [ ] Alternatively, figure out why sqlite is taking so long to load

7
nixos/hosts/backups-rpi4/cron.nix
View file

@ -3,9 +3,10 @@
enable = true; enable = true;
systemCronJobs = [ systemCronJobs = [
# Backups from nuc-docker01 - See "backup-script.nix" # Backups from nuc-docker01 - See "backup-script.nix"
''0 04 * * * root rsync-media &> /root/logs/media/rsync_`date "+\%Y-\%m-\%d"`.log'' # TODO: Re-enable when done
''0 02 * * 3 root rsync-data-vol &> /root/logs/data-vol/rsync_`date "+\%Y-\%m-\%d"`.log'' # ''0 04 * * * root rsync-media &> /root/logs/media/rsync_`date "+\%Y-\%m-\%d"`.log''
''0 22 * * * root rsync-sysctl &> /root/logs/sysctl/rsync_`date "+\%Y-\%m-\%d"`.log'' # ''0 02 * * 3 root rsync-data-vol &> /root/logs/data-vol/rsync_`date "+\%Y-\%m-\%d"`.log''
# ''0 22 * * * root rsync-sysctl &> /root/logs/sysctl/rsync_`date "+\%Y-\%m-\%d"`.log''
]; ];
}; };
} }

24
nixos/hosts/backups-rpi4/scripts/rsync-data-vol.sh
View file

@ -5,21 +5,21 @@ echo "Sending start message"
payload="{\"message\": \"$BACKUP_NAME Backup has started\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Started\"}" payload="{\"message\": \"$BACKUP_NAME Backup has started\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Started\"}"
curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload" curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload"
drive1=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGHYP8C | awk {'print $1'}` drivea=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGXW93C | awk {'print $1'}`
drive2=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGXW93C | awk {'print $1'}` driveb=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGHYP8C | awk {'print $1'}`
drive3=`lsblk -o NAME,SERIAL | grep -v luks | grep 5QH05G3F | awk {'print $1'}` drivec=`lsblk -o NAME,SERIAL | grep -v luks | grep 5QH05G3F | awk {'print $1'}`
mount1=`mount -l | grep $drive1`; mounted1=`echo $?` mounta=`mount -l | grep $drivea`; mounteda=`echo $?`
mount2=`mount -l | grep $drive2`; mounted2=`echo $?` mountb=`mount -l | grep $driveb`; mountedb=`echo $?`
mount3=`mount -l | grep $drive3`; mounted3=`echo $?` mountc=`mount -l | grep $drivec`; mountedc=`echo $?`
echo "Drive status" echo "Drive status"
echo "Drive 1: $drive1 / $mount1 / $mounted1" echo "Drive 2: $drivea / $mounta / $mounteda"
echo "Drive 2: $drive2 / $mount2 / $mounted2" echo "Drive 1: $driveb / $mountb / $mountedb"
echo "Drive 3: $drive3 / $mount3 / $mounted3" echo "Drive 3: $drivec / $mountc / $mountedc"
# If any drive isn't found, exit: # If any drive isn't found, exit:
if [[ $mounted1 != 0 ]] || [[ $mounted2 != 0 ]] || [[ $mounted3 != 0 ]] ; then if [[ $mounteda != 0 ]] || [[ $mountedb != 0 ]] || [[ $mountedc != 0 ]] ; then
echo "One or more drives is not mounted. Exiting" echo "One or more drives is not mounted. Exiting"
payload="{\"message\": \"$BACKUP_NAME Backup stopped. One or more local drives are not mounted.\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Stopped\"}" payload="{\"message\": \"$BACKUP_NAME Backup stopped. One or more local drives are not mounted.\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Stopped\"}"
curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload" curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload"
@ -50,8 +50,8 @@ if [[ $check != 1 ]] ; then
fi fi
echo -e "======= Time started : `date`" echo -e "======= Time started : `date`"
echo -e "rsync -avr --delete --exclude=Downloads/ --exclude='Data/Docker/sysctl.io/' root@nuc-docker01:/Storage/Data/ /mnt/$drive1/Data/" echo -e "rsync -avr --delete --exclude=Downloads/ --exclude='Data/Docker/sysctl.io/' root@nuc-docker01:/Storage/Data/ /mnt/$drivea/Data/"
rsync -avr --delete --exclude=Downloads/ --exclude='Data/Docker/sysctl.io/' root@nuc-docker01:/Storage/Data/ /mnt/$drive1/Data/ rsync -avr --delete --exclude=Downloads/ --exclude='Data/Docker/sysctl.io/' root@nuc-docker01:/Storage/Data/ /mnt/$drivea/Data/
echo -e "======= Time complete: `date`" echo -e "======= Time complete: `date`"
echo "Sending completion message" echo "Sending completion message"

76
nixos/hosts/backups-rpi4/scripts/rsync-media.sh
View file

@ -5,21 +5,21 @@ echo "Sending start message"
payload="{\"message\": \"$BACKUP_NAME Backup has started\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Started\"}" payload="{\"message\": \"$BACKUP_NAME Backup has started\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Started\"}"
curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload" curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload"
drive1=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGHYP8C | awk {'print $1'}` driveb=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGHYP8C | awk {'print $1'}`
drive2=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGXW93C | awk {'print $1'}` drivea=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGXW93C | awk {'print $1'}`
drive3=`lsblk -o NAME,SERIAL | grep -v luks | grep 5QH05G3F | awk {'print $1'}` drivec=`lsblk -o NAME,SERIAL | grep -v luks | grep 5QH05G3F | awk {'print $1'}`
mount1=`mount -l | grep $drive1`; mounted1=`echo $?` mountb=`mount -l | grep $driveb`; mountedb=`echo $?`
mount2=`mount -l | grep $drive2`; mounted2=`echo $?` mounta=`mount -l | grep $drivea`; mounteda=`echo $?`
mount3=`mount -l | grep $drive3`; mounted3=`echo $?` mount3=`mount -l | grep $drivec`; mountedc=`echo $?`
echo "Drive status" echo "Drive status"
echo "Drive 1: $drive1 / $mount1 / $mounted1" echo "Drive 2: $drivea / $mounta / $mounteda"
echo "Drive 2: $drive2 / $mount2 / $mounted2" echo "Drive 1: $driveb / $mountb / $mountedb"
echo "Drive 3: $drive3 / $mount3 / $mounted3" echo "Drive 3: $drivec / $mount3 / $mountedc"
# If any drive isn't found, exit: # If any drive isn't found, exit:
if [[ $mounted1 != 0 ]] || [[ $mounted2 != 0 ]] || [[ $mounted3 != 0 ]] ; then if [[ $mounteda != 0 ]] || [[ $mountedb != 0 ]] || [[ $mountedc != 0 ]] ; then
echo "One or more drives is not mounted. Exiting" echo "One or more drives is not mounted. Exiting"
payload="{\"message\": \"$BACKUP_NAME Backup stopped. One or more local drives are not mounted.\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Stopped\"}" payload="{\"message\": \"$BACKUP_NAME Backup stopped. One or more local drives are not mounted.\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Stopped\"}"
curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload" curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload"
@ -50,60 +50,60 @@ if [[ $check != 1 ]] ; then
fi fi
echo -e "======= Time started : `date`" echo -e "======= Time started : `date`"
echo -e "rsync -avr --delete --exclude=luks.img /Storage/Backups/Manual/Computers/ /mnt/$drive1/Backups/Manual/Computers/" echo -e "rsync -avr --delete --exclude=luks.img /Storage/Backups/Manual/Computers/ /mnt/$drivea/Backups/Manual/Computers/"
rsync -avr --delete --exclude=luks.img root@nuc-docker01:/Storage/Backups/Manual/Computers/ /mnt/$drive1/Backups/Manual/Computers/ rsync -avr --delete --exclude=luks.img root@nuc-docker01:/Storage/Backups/Manual/Computers/ /mnt/$drivea/Backups/Manual/Computers/
echo -e "======= Time: `date`" echo -e "======= Time: `date`"
echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Backups/Backblaze/ /mnt/$drive1/Backups/Backblaze/" echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Backups/Backblaze/ /mnt/$drivea/Backups/Backblaze/"
rsync -avr --delete root@nuc-docker01:/Storage/Backups/Backblaze/ /mnt/$drive1/Backups/Backblaze/ rsync -avr --delete root@nuc-docker01:/Storage/Backups/Backblaze/ /mnt/$drivea/Backups/Backblaze/
echo -e "======= Time: `date`" echo -e "======= Time: `date`"
echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Backups/Logs/ /mnt/$drive1/Backups/Logs/" echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Backups/Logs/ /mnt/$drivea/Backups/Logs/"
rsync -avr --delete root@nuc-docker01:/Storage/Backups/Logs/ /mnt/$drive1/Backups/Logs/ rsync -avr --delete root@nuc-docker01:/Storage/Backups/Logs/ /mnt/$drivea/Backups/Logs/
echo -e "======= Time: `date`" echo -e "======= Time: `date`"
echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Backups/Docker/ /mnt/$drive1/Backups/Docker/" echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Backups/Docker/ /mnt/$drivea/Backups/Docker/"
rsync -avr --delete root@nuc-docker01:/Storage/Backups/Docker/ /mnt/$drive1/Backups/Docker/ rsync -avr --delete root@nuc-docker01:/Storage/Backups/Docker/ /mnt/$drivea/Backups/Docker/
echo -e "======= Time: `date`" echo -e "======= Time: `date`"
echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Pictures/ /mnt/$drive1/Pictures/" echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Pictures/ /mnt/$driveb/Pictures/"
rsync -avr --delete root@nuc-docker01:/Storage/Media/Pictures/ /mnt/$drive1/Pictures/ rsync -avr --delete root@nuc-docker01:/Storage/Media/Pictures/ /mnt/$driveb/Pictures/
echo -e "======= Time: `date`" echo -e "======= Time: `date`"
echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Books/ /mnt/$drive1/Books/" echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Books/ /mnt/$driveb/Books/"
rsync -avr --delete root@nuc-docker01:/Storage/Media/Books/ /mnt/$drive1/Books/ rsync -avr --delete root@nuc-docker01:/Storage/Media/Books/ /mnt/$driveb/Books/
echo -e "======= Time: `date`" echo -e "======= Time: `date`"
echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Music/ /mnt/$drive1/Music/" echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Music/ /mnt/$driveb/Music/"
rsync -avr --delete root@nuc-docker01:/Storage/Media/Music/ /mnt/$drive1/Music/ rsync -avr --delete root@nuc-docker01:/Storage/Media/Music/ /mnt/$driveb/Music/
# echo -e "======= Time: `date`" # echo -e "======= Time: `date`"
# echo -e "rsync -avr --delete --exclude=Downloads/ /Storage/Data/ /mnt/$drive1/Data/" # echo -e "rsync -avr --delete --exclude=Downloads/ /Storage/Data/ /mnt/$driveb/Data/"
# rsync -avr --delete --exclude=Downloads/ --exclude=Docker/sysctl* root@nuc-docker01:/Storage/Data/ /mnt/$drive1/Data/ # rsync -avr --delete --exclude=Downloads/ --exclude=Docker/sysctl* root@nuc-docker01:/Storage/Data/ /mnt/$driveb/Data/
echo -e "======= Time: `date`" echo -e "======= Time: `date`"
echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Movies/ /mnt/$drive2/Movies/" echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/Movies/ /mnt/$driveb/Movies/"
rsync -avr --delete root@nuc-docker01:/Storage/Media/Movies/ /mnt/$drive2/Movies/ rsync -avr --delete root@nuc-docker01:/Storage/Media/Movies/ /mnt/$driveb/Movies/
echo -e "======= Time: `date`" echo -e "======= Time: `date`"
echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[0-9]* /mnt/$drive1/TV_Shows/" echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[0-9]* /mnt/$driveb/TV_Shows/"
rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[0-9]* /mnt/$drive1/TV_Shows/ rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[0-9]* /mnt/$driveb/TV_Shows/
echo -e "======= Time: `date`" echo -e "======= Time: `date`"
echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[A-K]* /mnt/$drive1/TV_Shows/" echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[A-K]* /mnt/$driveb/TV_Shows/"
rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[A-E]* /mnt/$drive1/TV_Shows/ rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[A-E]* /mnt/$driveb/TV_Shows/
echo -e "======= Time: `date`" echo -e "======= Time: `date`"
echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[L-R]* /mnt/$drive2/TV_Shows/" echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[L-R]* /mnt/$drivea/TV_Shows/"
rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[F-R]* /mnt/$drive2/TV_Shows/ rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[F-R]* /mnt/$drivea/TV_Shows/
echo -e "======= Time: `date`" echo -e "======= Time: `date`"
echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[S-Z]* /mnt/$drive3/TV_Shows/" echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[S-Z]* /mnt/$drivec/TV_Shows/"
rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[S-Z]* /mnt/$drive3/TV_Shows/ rsync -avr --delete root@nuc-docker01:/Storage/Media/TV_Shows/[S-Z]* /mnt/$drivec/TV_Shows/
echo -e "======= Time: `date`" echo -e "======= Time: `date`"
echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/YouTube /mnt/$drive3/YouTube/" echo -e "rsync -avr --delete root@nuc-docker01:/Storage/Media/YouTube /mnt/$drivec/YouTube/"
rsync -avr --delete root@nuc-docker01:/Storage/Media/YouTube/ /mnt/$drive3/YouTube/ rsync -avr --delete root@nuc-docker01:/Storage/Media/YouTube/ /mnt/$drivec/YouTube/
echo -e "======= Time complete: `date`" echo -e "======= Time complete: `date`"
echo "Sending completion message" echo "Sending completion message"

24
nixos/hosts/backups-rpi4/scripts/rsync-sysctl.sh
View file

@ -20,21 +20,21 @@ if [[ $count -ge 4 ]] ; then
exit exit
fi fi
drive1=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGHYP8C | awk {'print $1'}` drivea=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGXW93C | awk {'print $1'}`
drive2=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGXW93C | awk {'print $1'}` driveb=`lsblk -o NAME,SERIAL | grep -v luks | grep 9RGHYP8C | awk {'print $1'}`
drive3=`lsblk -o NAME,SERIAL | grep -v luks | grep 5QH05G3F | awk {'print $1'}` drivec=`lsblk -o NAME,SERIAL | grep -v luks | grep 5QH05G3F | awk {'print $1'}`
mount1=`mount -l | grep $drive1`; mounted1=`echo $?` mounta=`mount -l | grep $drivea`; mounteda=`echo $?`
mount2=`mount -l | grep $drive2`; mounted2=`echo $?` mountb=`mount -l | grep $driveb`; mountedb=`echo $?`
mount3=`mount -l | grep $drive3`; mounted3=`echo $?` mountc=`mount -l | grep $drivec`; mountedc=`echo $?`
echo "Drive status" echo "Drive status"
echo "Drive 1: $drive1 / $mount1 / $mounted1" echo "Drive 2: $drivea / $mounta / $mounteda"
echo "Drive 2: $drive2 / $mount2 / $mounted2" echo "Drive 1: $driveb / $mountb / $mountedb"
echo "Drive 3: $drive3 / $mount3 / $mounted3" echo "Drive 3: $drivec / $mountc / $mountedc"
# If any drive isn't found, exit: # If any drive isn't found, exit:
if [[ $mounted1 != 0 ]] || [[ $mounted2 != 0 ]] || [[ $mounted3 != 0 ]] ; then if [[ $mounteda != 0 ]] || [[ $mountedb != 0 ]] || [[ $mountedc != 0 ]] ; then
echo "One or more drives is not mounted. Exiting" echo "One or more drives is not mounted. Exiting"
payload="{\"message\": \"$BACKUP_NAME Backup stopped. One or more local drives are not mounted.\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Stopped\"}" payload="{\"message\": \"$BACKUP_NAME Backup stopped. One or more local drives are not mounted.\nDate: `date`\", \"priority\": 2,\"title\": \"japan-rpi4: $BACKUP_NAME Backup Stopped\"}"
curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload" curl -X "POST" "https://gotify.sysctl.io/message" -H "accept: application/json" -H "Content-Type: application/json" -H "X-Gotify-Key: $GOTIFY_TOKEN" -d "$payload"
@ -43,8 +43,8 @@ fi
# Start the backups # Start the backups
echo -e "======= Time started : `date`" echo -e "======= Time started : `date`"
echo -e "rsync -avr --delete --exclude=Downloads/ root@framework-server:/Storage/Data/Docker/sysctl.io/ /mnt/$drive1/Data/Docker/sysctl.io/" echo -e "rsync -avr --delete --exclude=Downloads/ root@framework-server:/Storage/Data/Docker/sysctl.io/ /mnt/$driveb/Data/Docker/sysctl.io/"
rsync -avr --delete --exclude=Downloads/ root@framework-server:/Storage/Data/Docker/sysctl.io/ /mnt/$drive1/Data/Docker/sysctl.io/ rsync -avr --delete --exclude=Downloads/ root@framework-server:/Storage/Data/Docker/sysctl.io/ /mnt/$driveb/Data/Docker/sysctl.io/
echo -e "======= Time complete: `date`" echo -e "======= Time complete: `date`"
echo "Sending completion message" echo "Sending completion message"