Bump rdiff-backup to 2.2.4

* show_config
* check_destination_dir

weitere kleine optische Anpassungen

Dockerfile

@@ -1,8 +1,22 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
-MAINTAINER Vitali Graf <info@vitaligraf.de>
+LABEL maintainer="Vitali Graf <info@vitaligraf.de>"
 
-RUN apt-get update && apt-get install openssh-client rdiff-backup python-pyxattr python-pylibacl --no-install-recommends -y && apt-get clean && rm -rf /var/lib/apt/lists/*
+ARG GIT_BRANCH=unspecified
+ARG GIT_COMMIT=unspecified
+ARG GIT_AUTHOR=unspecified
+ARG GIT_DATE=unspecified
+
+LABEL git.commit="$GIT_COMMIT"
+LABEL git.branch="$GIT_BRANCH"
+LABEL git.author="$GIT_AUTHOR"
+LABEL git.date="$GIT_DATE"
+
+RUN apt-get update \
+ && apt-get install openssh-client python3-pip python3-setuptools python3-pylibacl python3-pyxattr --no-install-recommends -y \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* \
+ && pip3 install rdiff-backup==2.2.4
 
 COPY *.sh /
 

example_conf.sh

@@ -4,9 +4,12 @@ REMOTE_HOST='example.com'
 # Mit welchem User soll backup gemacht werden. root ist empfohlen
 REMOTE_USER='root'
 
-# Wechle Ordner sollen gesichert werden als BASH-Array z.B. ("/home" "/var/www")
+# Welche Ordner sollen gesichert werden als BASH-Array z.B. ("/home" "/var/www")
 FOLDERS=()
 
+# Welche Docker volumes sollen gesichert werden als BASH-Array z.B. ("vol-1" "redis-vol-1")
+DOCKER_VOLUME=()
+
 # Wie lange sollen Backups gesichert bleiben z.B. 8W
 # ==================================================
 # Remove the incremental backup  information  in  the  destination
@@ -39,6 +42,5 @@ MYSQL_PASS='secret'
 # Dieser Ordner wird zusammen mit $FOLDERS gesichert
 MYSQL_DUMP_DIR='/var/backups/sqldump'
 
-
 # Die Optionen die beim Backup eingesetzt werden
 RDIFF_BACKUP_OPTS="--exclude-sockets --exclude-device-files --exclude-fifos --exclude-other-filesystems"

reset-conf.sh

@@ -1,6 +1,7 @@
 REMOTE_HOST=''
 REMOTE_USER='root'
 FOLDERS=()
+DOCKER_VOLUME=()
 DELETE_OLD_FILES='8W'
 
 MYSQL_DB=()

start.sh

@@ -4,6 +4,15 @@
 
 # source job.conf
 BACKUP_DIR=/backups
+
+FILES=ROOT
+VOLS=VOL
+CONTAINERS=CON
+
+BACKUP_DIR_FILES=$BACKUP_DIR/$FILES
+BACKUP_DIR_VOLS=$BACKUP_DIR/$VOLS
+BACKUP_DIR_CONTAINERS=$BACKUP_DIR/$CONTAINERS
+
 CONFIG_DIR=/config
 SSH_CF_DIR=/root/.ssh
 
@@ -34,19 +43,49 @@ gen_config(){
 }
 
 move_to_ROOT(){
-	read_config $1
 	# prüfen ob Ordner ROOT, VOL, CONT existieren
 	# wenn nicht ROOT anlegen und alle bisher vorhandene Ordner dorthin verschieben
-	[ ! -d "$BACKUP_DIR/${REMOTE_HOST}/ROOT" ] && mkdir -p "$BACKUP_DIR/${REMOTE_HOST}/ROOT"
+	[ ! -d "$BACKUP_DIR_FILES" ] && mkdir -p "$BACKUP_DIR_FILES" || return
 	
 	
 	### find /home/vitali/find_tests/ -maxdepth 1 -mindepth 1 -type d -name "CONT" -prune -or -name "VOL" -prune -or -name ROOT -prune -or -name "*" -exec mv -t /home/vitali/find_tests/ROOT/ {} +
 	
-	find "$BACKUP_DIR/${REMOTE_HOST}" -maxdepth 1 -mindepth 1 -type d -name "CONT" -prune -or -name "VOL" -prune -or -name "ROOT" -prune -or -name "*" -exec mv -t "$BACKUP_DIR/${REMOTE_HOST}/ROOT/" {} +
+	find "$BACKUP_DIR" -maxdepth 1 -mindepth 1 -type d -name "$CONTAINERS" -prune -or -name "$VOLS" -prune -or -name "$FILES" -prune -or -name "*" -exec mv -t "$BACKUP_DIR_FILES/" {} +
 	
 	# anschliessend VOL und CONT anlegen
-	[ ! -d "$BACKUP_DIR/${REMOTE_HOST}/VOL" ] && mkdir -p "$BACKUP_DIR/${REMOTE_HOST}/VOL" 
+	[ ! -d "$BACKUP_DIR_VOLS" ] && mkdir -p "$BACKUP_DIR_VOLS" 
-	[ ! -d "$BACKUP_DIR/${REMOTE_HOST}/CONT" ] && mkdir -p "$BACKUP_DIR/${REMOTE_HOST}/CONT" 
+	[ ! -d "$BACKUP_DIR_CONTAINERS" ] && mkdir -p "$BACKUP_DIR_CONTAINERS" 
+}
+
+## Einzelne Backup-Jobs
+b_files(){
+	## einzelne Dateien und Ordner mit rdiff-backup sichern
+	# prüfen ob das backup-ordner existiert ...
+	if [ ! -d "$2" ]
+	then
+		# ... und gegebenfals anlegen
+		mkdir -p "$2"
+	fi
+		
+	# die eigentliche Sicherung ...	
+	rdiff-backup $RDIFF_BACKUP_OPTS "$1" "$2"
+	
+	# alle Sicherungen die älter als $DELETE_OLD_FILES löschen
+	echo remove backup increments older than: $DELETE_OLD_FILES
+	rdiff-backup --remove-older-than $DELETE_OLD_FILES --force "$2"
+}
+
+export_sql(){
+	## Export einer MySQL-DB als SQL-Script in ein später (!) zu sicherndes Ordner
+	echo dumb database: "$1"
+	
+	if [ "$MYSQL_CONTAINER" != '' ]
+	then
+		echo from container: "$MYSQL_CONTAINER"
+		ssh ${REMOTE_USER}@${REMOTE_HOST} "docker exec -i \"$MYSQL_CONTAINER\" mysqldump -u \"$MYSQL_USER\" \"-p$MYSQL_PASS\" --events --routines --triggers --single-transaction --add-drop-database -B \"$1\" > \"$MYSQL_DUMP_DIR/$1.sql\""
+	else
+		ssh ${REMOTE_USER}@${REMOTE_HOST} "mysqldump -u \"$MYSQL_USER\" \"-p$MYSQL_PASS\" --events --routines --triggers --single-transaction --add-drop-database -B \"$1\" > \"$MYSQL_DUMP_DIR/$1.sql\""
+	fi
 }
 
 backup(){
@@ -55,8 +94,8 @@ backup(){
 	echo Start backup $1 
 	echo ==================================================
 	
-	# neue Konfiguration lesen
+	# Backupverzeichnisse einrichten
-	#source "$CONFIG_DIR/$1"
+	move_to_ROOT
 	
 	# SCHRITT 1:
 	# sichern der MySQL-Datenbanken auf dem remote host
@@ -68,106 +107,161 @@ backup(){
 		
 		for db in "${MYSQL_DB[@]}"
 		do
-			echo dumb database: "$db"
+			# lokaler Export der DB als SQL
-			if [ "$MYSQL_CONTAINER" != '' ]
+			export_sql "$db"
-			then
-				echo from container: "$MYSQL_CONTAINER"
-				ssh ${REMOTE_USER}@${REMOTE_HOST} "docker exec -i \"$MYSQL_CONTAINER\" mysqldump -u \"$MYSQL_USER\" \"-p$MYSQL_PASS\" --events --routines --triggers --single-transaction --add-drop-database -B \"$db\" > \"$MYSQL_DUMP_DIR/$db.sql\""
-			else
-				ssh ${REMOTE_USER}@${REMOTE_HOST} "mysqldump -u \"$MYSQL_USER\" \"-p$MYSQL_PASS\" --events --routines --triggers --single-transaction --add-drop-database -B \"$db\" > \"$MYSQL_DUMP_DIR/$db.sql\""
-			fi
 		done
 	
-		# Ordner mit SQL-Dateien in die Sicherung aufnehmen
+		# Ordner mit SQL-Dateien sofort (!) sichern
 		if [ "$MYSQL_CONTAINER" != '' ]
 		then
-			FOLDERS=("$MYSQL_DUMP_DIR")
+			b_files "${REMOTE_USER}@${REMOTE_HOST}::${MYSQL_DUMP_DIR}" "$BACKUP_DIR_CONTAINERS/${MYSQL_CONTAINER}"
 		else
-			FOLDERS+=("$MYSQL_DUMP_DIR")
+			b_files "${REMOTE_USER}@${REMOTE_HOST}::${MYSQL_DUMP_DIR}" "${BACKUP_DIR_FILES}/${MYSQL_DUMP_DIR}"
 		fi
+		
+		# die SQL-Dateien vom remote host wieder löschen
+		[ "$MYSQL_DUMP_DIR" != "" ] && [ "$MYSQL_DUMP_DIR" != "/" ] && ssh ${REMOTE_USER}@${REMOTE_HOST} "rm -rf \"$MYSQL_DUMP_DIR/\"*.sql"
+		echo
 	fi
 
 	# SCHRITT 2:
 	# die Sicherung der Ordner (inklusive der SQL-Dump)
 	for dir in "${FOLDERS[@]}"
 	do
-		# prüfen ob das backup-ordner existiert ...
-		if [ ! -d "$BACKUP_DIR/${REMOTE_HOST}${dir}" ]
-		then
-			# ... und gegebenfals anlegen
-			mkdir -p "$BACKUP_DIR/${REMOTE_HOST}${dir}"
-		fi
-		
 		# die eigentliche Sicherung ...	
 		echo backup file[s]: "$dir"
-		rdiff-backup $RDIFF_BACKUP_OPTS "${REMOTE_USER}@${REMOTE_HOST}::${dir}" "$BACKUP_DIR/${REMOTE_HOST}${dir}"
+		[ "$dir" != "" ] && b_files "${REMOTE_USER}@${REMOTE_HOST}::${dir}" "${BACKUP_DIR_FILES}/${dir}"
-		
-		# alle Sicherungen die älter als $DELETE_OLD_FILES löschen
-		rdiff-backup --remove-older-than $DELETE_OLD_FILES --force "$BACKUP_DIR/${REMOTE_HOST}${dir}"
 		
 		# Liste der Sicherungen anzeigen
 		if [ "$2" == "--list" ]; then
-			rdiff-backup -l "$BACKUP_DIR/${REMOTE_HOST}${dir}"
+			rdiff-backup -l "${BACKUP_DIR_FILES}/${dir}"
+		fi
+		echo
+	done
+	
+	# SCHRIT 3:
+	# die Sicherung der Docker volumes
+	for vol in "${DOCKER_VOLUME[@]}"
+	do
+		echo backup Docker volume: "$vol"
+		# herausfinden wo die volume gespeicher ist
+		dir=$(ssh ${REMOTE_USER}@${REMOTE_HOST} docker volume inspect -f "{{.Mountpoint}}" "$vol")
+		
+		# und die eigentliche Sicherung ...	
+		[ "$dir" != "" ] && b_files "${REMOTE_USER}@${REMOTE_HOST}::${dir}" "${BACKUP_DIR_VOLS}/${vol}"
+		
+		# Liste der Sicherungen anzeigen
+		if [ "$2" == "--list" ]; then
+			rdiff-backup -l "${BACKUP_DIR_VOLS}/${vol}"
 		fi
 		
 		echo
 	done
 	
-	# SCHRITT 3: 
-	# die SQL-Dateien vom remote host wieder löschen 
-	if [  ${#MYSQL_DB[@]} -gt 0 ]
-	then
-		[ "$MYSQL_DUMP_DIR" != "" ] && [ "$MYSQL_DUMP_DIR" != "/" ] && ssh ${REMOTE_USER}@${REMOTE_HOST} "rm -rf \"$MYSQL_DUMP_DIR/\"*.sql"
-	fi
 	echo Done backup $1 
 	echo
 }
 
+## Einzelne Restore-Jobs
+
+r_files(){
+	## einzelne Dateien und Ordner mit rdiff-backup wiederherstellen
+	if [ "$3" != "" ]; then
+		rdiff-backup --force -r "$3" "$1" "$2"
+	else
+		rdiff-backup --force -r now "$1" "$2"
+	fi
+}
+
+import_sql(){
+	## Import einer MySQL-DB von einem SQL-Script aus einem vorher (!) wiederherstellen Ordner
+			
+	if [ "$MYSQL_CONTAINER" != '' ]
+	then
+		FROM_FILE="${BACKUP_DIR_CONTAINERS}/${MYSQL_CONTAINER}/${1}.sql"
+	else
+		FROM_FILE="${BACKUP_DIR_FILES}/${MYSQL_DUMP_DIR}/${1}.sql"
+	fi
+	
+	r_files "$FROM_FILE" "${REMOTE_USER}@${REMOTE_HOST}::${MYSQL_DUMP_DIR}/${1}.sql" "$2"
+			
+	if [ "$MYSQL_CONTAINER" != '' ]
+	then
+		ssh ${REMOTE_USER}@${REMOTE_HOST} "docker exec -i \"$MYSQL_CONTAINER\" mysql -u \"$MYSQL_USER\" \"-p$MYSQL_PASS\" < \"$MYSQL_DUMP_DIR/${1}.sql\""
+	else
+		ssh ${REMOTE_USER}@${REMOTE_HOST} "mysql -u \"$MYSQL_USER\" \"-p$MYSQL_PASS\" < \"$MYSQL_DUMP_DIR/${1}.sql\""
+	fi
+	
+	# TODO: was davon muss noch sein?
+	[ "$MYSQL_DUMP_DIR" != "" ] && [ "$MYSQL_DUMP_DIR" != "/" ] && ssh ${REMOTE_USER}@${REMOTE_HOST} "rm -rf \"$MYSQL_DUMP_DIR/${1}.sql\""
+}
+
 restore(){
 	read_config "$1"
 	
-	echo Start restore $1 
+	echo Start restore $1 from $2
 	echo ==================================================
 	
-	# neue Konfiguration lesen
-	#source "$CONFIG_DIR/$1"
 	# SCHRITT 1:
 	# Alle Dateien und Ordner aus dem letzten Backup wiederherstellen
 	
 	if [  ${#MYSQL_DB[@]} -gt 0 ]
 	then
-		FOLDERS+=("$MYSQL_DUMP_DIR")
+		#FOLDERS+=("$MYSQL_DUMP_DIR")
+		
+		## jede SQL-Datei einzeln hochladen
+		
+		for db in "${MYSQL_DB[@]}"
+		do
+			echo restore database: "$db" from "$2"
+			import_sql "$db" "$2"
+		done
 	fi
 	
 	for dir in "${FOLDERS[@]}"
 	do
 		# die eigentliche Wiederherstellung ...	
-		echo restore file[s]: "$dir" "$2"
+		echo restore file[s]: "${dir}" from "$2"
 	
-		if [ "$2" != "" ]; then
+		r_files "${BACKUP_DIR_FILES}/${dir}" "${REMOTE_USER}@${REMOTE_HOST}::${dir}" "$2"
-			rdiff-backup --force -r "$2" "$BACKUP_DIR/${REMOTE_HOST}${dir}" "${REMOTE_USER}@${REMOTE_HOST}::${dir}"
+		echo
-		else
-			rdiff-backup --force -r now "$BACKUP_DIR/${REMOTE_HOST}${dir}" "${REMOTE_USER}@${REMOTE_HOST}::${dir}"
-		fi
 	done
-	# SCHRITT 2:
-	# Datenbanken wiederherstellen 
 	
-	for db in "${MYSQL_DB[@]}"
+	for vol in "${DOCKER_VOLUME[@]}"
 	do
-		echo restore database: "$db"
+		# die eigentliche Wiederherstellung ...	
-		ssh ${REMOTE_USER}@${REMOTE_HOST} "mysql -u \"$MYSQL_USER\" \"-p$MYSQL_PASS\" < \"$MYSQL_DUMP_DIR/$db.sql\""
+		echo restore Docker volume: "${vol}" from "$2"
-		[ "$MYSQL_DUMP_DIR" != "" ] && [ "$MYSQL_DUMP_DIR" != "/" ] && ssh ${REMOTE_USER}@${REMOTE_HOST} "rm -rf \"$MYSQL_DUMP_DIR/$db.sql\""
-	done
 		
+		# herausfinden wo das volume gespeicher ist
+		if [ dir=$(ssh ${REMOTE_USER}@${REMOTE_HOST} docker volume inspect -f "{{.Mountpoint}}" "$vol") ]
+		then
+			echo Volume don\'t exist.
+			echo Create one: docker volume create "${vol}"
+			
+			ssh ${REMOTE_USER}@${REMOTE_HOST} docker volume create "${vol}" 
+			dir=$(ssh ${REMOTE_USER}@${REMOTE_HOST} docker volume inspect -f "{{.Mountpoint}}" "$vol")
+		fi
+	
+		[ "$dir" != "" ] && r_files "${BACKUP_DIR_VOLS}/${vol}" "${REMOTE_USER}@${REMOTE_HOST}::${dir}" "$2"
+		echo
+	done
 	
 	echo Done restore $1 
 	echo
 }
 
-test_function(){
+show_config(){
 	read_config "$1"
-	echo $REMOTE_HOST
+	echo REMOTE_HOST="$REMOTE_HOST"
+	echo REMOTE_USER="$REMOTE_USER"
+	echo FOLDERS="${FOLDERS[@]}"
+	echo DOCKER_VOLUME="${DOCKER_VOLUME[@]}"
+	echo DELETE_OLD_FILES="$DELETE_OLD_FILES"
+	echo MYSQL_DB="${MYSQL_DB[@]}"
+	echo MYSQL_USER="$MYSQL_USER"
+	echo MYSQL_PASS='################'
+	echo MYSQL_DUMP_DIR="$MYSQL_DUMP_DIR"
+	echo RDIFF_BACKUP_OPTS="$RDIFF_BACKUP_OPTS"
 }
 
 
@@ -177,19 +271,33 @@ list(){
 	echo Show backups of $1 
 	echo ==================================================
 	
-	# neue Konfiguration lesen
-	#source "$CONFIG_DIR/$1"
-	
 	if [ ${#MYSQL_DB[@]} -gt 0 ]
 	then
-		FOLDERS+=("$MYSQL_DUMP_DIR")
+		echo show backups 'for' MySQL databases:
+		
+		if [ "$MYSQL_CONTAINER" != '' ]
+		then
+			rdiff-backup -l  "$BACKUP_DIR_CONTAINERS/${MYSQL_CONTAINER}"
+		else
+			rdiff-backup -l "${BACKUP_DIR_FILES}/${MYSQL_DUMP_DIR}"
+		fi
+		
+		echo
 	fi
 	
 	for dir in "${FOLDERS[@]}"
 	do
-		echo show backups for file[s]: "$dir"
+		echo show backups 'for' file[s]: "$dir"
 		
-		rdiff-backup -l "$BACKUP_DIR/${REMOTE_HOST}${dir}"
+		rdiff-backup -l "${BACKUP_DIR_FILES}/${dir}"
+		echo
+	done
+	
+	for vol in "${DOCKER_VOLUME[@]}"
+	do
+		echo show backups 'for' volume: "${vol}"
+		
+		rdiff-backup -l "${BACKUP_DIR_VOLS}/${vol}"
 		echo
 	done
 	
@@ -197,6 +305,46 @@ list(){
 	echo
 }
 
+check_destination_dir(){
+	read_config "$1"
+	
+	echo Check backups of $1 
+	echo ==================================================
+	
+	if [ ${#MYSQL_DB[@]} -gt 0 ]
+	then
+		echo check backups 'for' MySQL databases:
+		
+		if [ "$MYSQL_CONTAINER" != '' ]
+		then
+			rdiff-backup --check-destination-dir  "$BACKUP_DIR_CONTAINERS/${MYSQL_CONTAINER}"
+		else
+			rdiff-backup --check-destination-dir "${BACKUP_DIR_FILES}/${MYSQL_DUMP_DIR}"
+		fi
+		
+		echo
+	fi
+	
+	for dir in "${FOLDERS[@]}"
+	do
+		echo check backups 'for' file[s]: "$dir"
+		
+		rdiff-backup --check-destination-dir "${BACKUP_DIR_FILES}/${dir}"
+		echo
+	done
+	
+	for vol in "${DOCKER_VOLUME[@]}"
+	do
+		echo check backups 'for' volume: "${vol}"
+		
+		rdiff-backup --check-destination-dir "${BACKUP_DIR_VOLS}/${vol}"
+		echo
+	done
+	
+	echo Done check $1 
+	echo
+}
+
 if [ "$1" == "--gen-config" ]; then
 	gen_config
 	
@@ -216,11 +364,14 @@ elif [ "$1" == "--bash" ]; then
 	shift
 	/bin/bash "$@"
 	
-elif [ "$1" == "--testit" ]; then
+elif [ "$1" == "--check-destination-dir" ]; then
 	shift
-	test_function "$@"
+	check_destination_dir "$@"
+	
+elif [ "$1" == "--show-config" ]; then
+	shift
+	show_config "$@"
 	
 else
 	echo "usage ?"
 fi	
-