diff --git a/example_conf.sh b/example_conf.sh index 8c77501..93e9167 100644 --- a/example_conf.sh +++ b/example_conf.sh @@ -7,6 +7,9 @@ REMOTE_USER='root' # Wechle Ordner sollen gesichert werden als BASH-Array z.B. ("/home" "/var/www") FOLDERS=() +# Wechle Docker volumes sollen gesichert werden als BASH-Array z.B. ("vol-1" "redis-vol-1") +DOCKER_VOLUME=() + # Wie lange sollen Backups gesichert bleiben z.B. 8W # ================================================== # Remove the incremental backup information in the destination @@ -25,6 +28,10 @@ DELETE_OLD_FILES='8W' # Leere Array () => es werden keine MySQL-Datenbanken gesichert MYSQL_DB=() +# in welchem Docker-Container läuft MySQL-Dienst? +# leer lassen wenn es direkt auf dem $REMOTE_HOST läuft +MYSQL_CONTAINER='' + # MySQL User MYSQL_USER='root' @@ -35,6 +42,5 @@ MYSQL_PASS='secret' # Dieser Ordner wird zusammen mit $FOLDERS gesichert MYSQL_DUMP_DIR='/var/backups/sqldump' - # Die Optionen die beim Backup eingesetzt werden -RDIFF_BACKUP_OPTS="--exclude-sockets --exclude-device-files --exclude-fifos --exclude-other-filesystems" \ No newline at end of file +RDIFF_BACKUP_OPTS="--exclude-sockets --exclude-device-files --exclude-fifos --exclude-other-filesystems" diff --git a/reset-conf.sh b/reset-conf.sh index 057154b..60fd795 100644 --- a/reset-conf.sh +++ b/reset-conf.sh @@ -1,9 +1,11 @@ REMOTE_HOST='' REMOTE_USER='root' FOLDERS=() +DOCKER_VOLUME=() DELETE_OLD_FILES='8W' MYSQL_DB=() +MYSQL_CONTAINER='' MYSQL_USER='root' MYSQL_PASS='secret' MYSQL_DUMP_DIR="/var/backups/sqldump" diff --git a/start.sh b/start.sh index 6a12b43..d84c1ff 100755 --- a/start.sh +++ b/start.sh @@ -4,6 +4,15 @@ # source job.conf BACKUP_DIR=/backups + +FILES=ROOT +VOLS=VOL +CONTAINERS=CON + +BACKUP_DIR_FILES=$BACKUP_DIR/$FILES +BACKUP_DIR_VOLS=$BACKUP_DIR/$VOLS +BACKUP_DIR_CONTAINERS=$BACKUP_DIR/$CONTAINERS + CONFIG_DIR=/config SSH_CF_DIR=/root/.ssh @@ -16,6 +25,7 @@ read_config(){ exit 1 fi + source "$CONFIG_DIR/$1" } gen_config(){ @@ -32,14 +42,60 @@ gen_config(){ [ ! -f "$CONFIG_DIR/example_conf.sh" ] && cp /example_conf.sh "$CONFIG_DIR/example_conf.sh" } +move_to_ROOT(){ + # prüfen ob Ordner ROOT, VOL, CONT existieren + # wenn nicht ROOT anlegen und alle bisher vorhandene Ordner dorthin verschieben + [ ! -d "$BACKUP_DIR_FILES" ] && mkdir -p "$BACKUP_DIR_FILES" || return + + + ### find /home/vitali/find_tests/ -maxdepth 1 -mindepth 1 -type d -name "CONT" -prune -or -name "VOL" -prune -or -name ROOT -prune -or -name "*" -exec mv -t /home/vitali/find_tests/ROOT/ {} + + + find "$BACKUP_DIR" -maxdepth 1 -mindepth 1 -type d -name "$CONTAINERS" -prune -or -name "$VOLS" -prune -or -name "$FILES" -prune -or -name "*" -exec mv -t "$BACKUP_DIR_FILES/" {} + + + # anschliessend VOL und CONT anlegen + [ ! -d "$BACKUP_DIR_VOLS" ] && mkdir -p "$BACKUP_DIR_VOLS" + [ ! -d "$BACKUP_DIR_CONTAINERS" ] && mkdir -p "$BACKUP_DIR_CONTAINERS" +} + +## Einzelne Backup-Jobs +b_files(){ + ## einzelne Dateien und Ordner mit rdiff-backup sichern + # prüfen ob das backup-ordner existiert ... + if [ ! -d "$2" ] + then + # ... und gegebenfals anlegen + mkdir -p "$2" + fi + + # die eigentliche Sicherung ... + rdiff-backup $RDIFF_BACKUP_OPTS "$1" "$2" + + # alle Sicherungen die älter als $DELETE_OLD_FILES löschen + echo remove backup increments older than: $DELETE_OLD_FILES + rdiff-backup --remove-older-than $DELETE_OLD_FILES --force "$2" +} + +export_sql(){ + ## Export einer MySQL-DB als SQL-Script in ein später (!) zu sicherndes Ordner + echo dumb database: "$1" + + if [ "$MYSQL_CONTAINER" != '' ] + then + echo from container: "$MYSQL_CONTAINER" + ssh ${REMOTE_USER}@${REMOTE_HOST} "docker exec -i \"$MYSQL_CONTAINER\" mysqldump -u \"$MYSQL_USER\" \"-p$MYSQL_PASS\" --events --routines --triggers --single-transaction --add-drop-database -B \"$1\" > \"$MYSQL_DUMP_DIR/$1.sql\"" + else + ssh ${REMOTE_USER}@${REMOTE_HOST} "mysqldump -u \"$MYSQL_USER\" \"-p$MYSQL_PASS\" --events --routines --triggers --single-transaction --add-drop-database -B \"$1\" > \"$MYSQL_DUMP_DIR/$1.sql\"" + fi +} + backup(){ read_config "$1" echo Start backup $1 echo ================================================== - # neue Konfiguration lesen - source "$CONFIG_DIR/$1" + # Backupverzeichnisse einrichten + move_to_ROOT # SCHRITT 1: # sichern der MySQL-Datenbanken auf dem remote host @@ -51,92 +107,162 @@ backup(){ for db in "${MYSQL_DB[@]}" do - echo dumb database: "$db" - ssh ${REMOTE_USER}@${REMOTE_HOST} "mysqldump -u \"$MYSQL_USER\" \"-p$MYSQL_PASS\" --events --routines --triggers --single-transaction --add-drop-database -B \"$db\" > \"$MYSQL_DUMP_DIR/$db.sql\"" + # lokaler Export der DB als SQL + export_sql "$db" done - # Ordner mit SQL-Dateien in die Sicherung aufnehmen - FOLDERS+=("$MYSQL_DUMP_DIR") + # Ordner mit SQL-Dateien sofort (!) sichern + if [ "$MYSQL_CONTAINER" != '' ] + then + b_files "${REMOTE_USER}@${REMOTE_HOST}::${MYSQL_DUMP_DIR}" "$BACKUP_DIR_CONTAINERS/${MYSQL_CONTAINER}" + else + b_files "${REMOTE_USER}@${REMOTE_HOST}::${MYSQL_DUMP_DIR}" "${BACKUP_DIR_FILES}/${MYSQL_DUMP_DIR}" + fi + + # die SQL-Dateien vom remote host wieder löschen + [ "$MYSQL_DUMP_DIR" != "" ] && [ "$MYSQL_DUMP_DIR" != "/" ] && ssh ${REMOTE_USER}@${REMOTE_HOST} "rm -rf \"$MYSQL_DUMP_DIR/\"*.sql" fi # SCHRITT 2: # die Sicherung der Ordner (inklusive der SQL-Dump) for dir in "${FOLDERS[@]}" do - # prüfen ob das backup-ordner existiert ... - if [ ! -d "$BACKUP_DIR/${REMOTE_HOST}${dir}" ]; - then - # ... und gegebenfals anlegen - mkdir -p "$BACKUP_DIR/${REMOTE_HOST}${dir}" - fi - # die eigentliche Sicherung ... echo backup file[s]: "$dir" - rdiff-backup $RDIFF_BACKUP_OPTS "${REMOTE_USER}@${REMOTE_HOST}::${dir}" "$BACKUP_DIR/${REMOTE_HOST}${dir}" - - # alle Sicherungen die älter als $DELETE_OLD_FILES löschen - rdiff-backup --remove-older-than $DELETE_OLD_FILES --force "$BACKUP_DIR/${REMOTE_HOST}${dir}" + [ "$dir" != "" ] && b_files "${REMOTE_USER}@${REMOTE_HOST}::${dir}" "${BACKUP_DIR_FILES}/${dir}" # Liste der Sicherungen anzeigen if [ "$2" == "--list" ]; then - rdiff-backup -l "$BACKUP_DIR/${REMOTE_HOST}${dir}" + rdiff-backup -l "${BACKUP_DIR_FILES}/${dir}" + fi + echo + done + + # SCHRIT 3: + # die Sicherung der Docker volumes + for vol in "${DOCKER_VOLUME[@]}" + do + echo backup Docker volume: "$vol" + # herausfinden wo die volume gespeicher ist + dir=$(ssh ${REMOTE_USER}@${REMOTE_HOST} docker volume inspect -f "{{.Mountpoint}}" "$vol") + + # und die eigentliche Sicherung ... + [ "$dir" != "" ] && b_files "${REMOTE_USER}@${REMOTE_HOST}::${dir}" "${BACKUP_DIR_VOLS}/${vol}" + + # Liste der Sicherungen anzeigen + if [ "$2" == "--list" ]; then + rdiff-backup -l "${BACKUP_DIR_VOLS}/${vol}" fi echo done - # SCHRITT 3: - # die SQL-Dateien vom remote host wieder löschen - if [ ${#MYSQL_DB[@]} -gt 0 ] - then - [ "$MYSQL_DUMP_DIR" != "" ] && [ "$MYSQL_DUMP_DIR" != "/" ] && ssh ${REMOTE_USER}@${REMOTE_HOST} "rm -rf \"$MYSQL_DUMP_DIR/\"*.sql" - fi echo Done backup $1 echo } +## Einzelne Restore-Jobs + +r_files(){ + ## einzelne Dateien und Ordner mit rdiff-backup wiederherstellen + if [ "$3" != "" ]; then + rdiff-backup --force -r "$3" "$1" "$2" + else + rdiff-backup --force -r now "$1" "$2" + fi +} + +import_sql(){ + ## Import einer MySQL-DB von einem SQL-Script aus einem vorher (!) wiederherstellen Ordner + + if [ "$MYSQL_CONTAINER" != '' ] + then + FROM_FILE="${BACKUP_DIR_CONTAINERS}/${MYSQL_CONTAINER}/${1}.sql" + else + FROM_FILE="${BACKUP_DIR_FILES}/${MYSQL_DUMP_DIR}/${1}.sql" + fi + + r_files "$FROM_FILE" "${REMOTE_USER}@${REMOTE_HOST}::${MYSQL_DUMP_DIR}/${1}.sql" "$2" + + if [ "$MYSQL_CONTAINER" != '' ] + then + ssh ${REMOTE_USER}@${REMOTE_HOST} "docker exec -i \"$MYSQL_CONTAINER\" mysql -u \"$MYSQL_USER\" \"-p$MYSQL_PASS\" < \"$MYSQL_DUMP_DIR/${1}.sql\"" + else + ssh ${REMOTE_USER}@${REMOTE_HOST} "mysql -u \"$MYSQL_USER\" \"-p$MYSQL_PASS\" < \"$MYSQL_DUMP_DIR/${1}.sql\"" + fi + + # TODO: was davon muss noch sein? + [ "$MYSQL_DUMP_DIR" != "" ] && [ "$MYSQL_DUMP_DIR" != "/" ] && ssh ${REMOTE_USER}@${REMOTE_HOST} "rm -rf \"$MYSQL_DUMP_DIR/${1}.sql\"" +} + restore(){ read_config "$1" - echo Start restore $1 + echo Start restore $1 from $2 echo ================================================== - # neue Konfiguration lesen - source "$CONFIG_DIR/$1" # SCHRITT 1: # Alle Dateien und Ordner aus dem letzten Backup wiederherstellen if [ ${#MYSQL_DB[@]} -gt 0 ] then - FOLDERS+=("$MYSQL_DUMP_DIR") + #FOLDERS+=("$MYSQL_DUMP_DIR") + + ## jede SQL-Datei einzeln hochladen + + for db in "${MYSQL_DB[@]}" + do + echo restore database: "$db" from "$2" + import_sql "$db" "$2" + done fi for dir in "${FOLDERS[@]}" do # die eigentliche Wiederherstellung ... - echo restore file[s]: "$dir" "$2" - - if [ "$2" != "" ]; then - rdiff-backup --force -r "$2" "$BACKUP_DIR/${REMOTE_HOST}${dir}" "${REMOTE_USER}@${REMOTE_HOST}::${dir}" - else - rdiff-backup --force -r now "$BACKUP_DIR/${REMOTE_HOST}${dir}" "${REMOTE_USER}@${REMOTE_HOST}::${dir}" - fi - done - # SCHRITT 2: - # Datenbanken wiederherstellen - - for db in "${MYSQL_DB[@]}" - do - echo restore database: "$db" - ssh ${REMOTE_USER}@${REMOTE_HOST} "mysql -u \"$MYSQL_USER\" \"-p$MYSQL_PASS\" < \"$MYSQL_DUMP_DIR/$db.sql\"" - [ "$MYSQL_DUMP_DIR" != "" ] && [ "$MYSQL_DUMP_DIR" != "/" ] && ssh ${REMOTE_USER}@${REMOTE_HOST} "rm -rf \"$MYSQL_DUMP_DIR/$db.sql\"" + echo restore file[s]: "${dir}" from "$2" + + r_files "${BACKUP_DIR_FILES}/${dir}" "${REMOTE_USER}@${REMOTE_HOST}::${dir}" "$2" + echo done + for vol in "${DOCKER_VOLUME[@]}" + do + # die eigentliche Wiederherstellung ... + echo restore Docker volume: "${vol}" from "$2" + + # herausfinden wo das volume gespeicher ist + if [ dir=$(ssh ${REMOTE_USER}@${REMOTE_HOST} docker volume inspect -f "{{.Mountpoint}}" "$vol") ] + then + echo Volume don\'t exist. + echo Create one: docker volume create "${vol}" + + ssh ${REMOTE_USER}@${REMOTE_HOST} docker volume create "${vol}" + dir=$(ssh ${REMOTE_USER}@${REMOTE_HOST} docker volume inspect -f "{{.Mountpoint}}" "$vol") + fi + + [ "$dir" != "" ] && r_files "${BACKUP_DIR_VOLS}/${vol}" "${REMOTE_USER}@${REMOTE_HOST}::${dir}" "$2" + echo + done echo Done restore $1 echo } +show_config(){ + read_config "$1" + echo REMOTE_HOST="$REMOTE_HOST" + echo REMOTE_USER="$REMOTE_USER" + echo FOLDERS="${FOLDERS[@]}" + echo DOCKER_VOLUME="${DOCKER_VOLUME[@]}" + echo DELETE_OLD_FILES="$DELETE_OLD_FILES" + echo MYSQL_DB="${MYSQL_DB[@]}" + echo MYSQL_USER="$MYSQL_USER" + echo MYSQL_PASS='################' + echo MYSQL_DUMP_DIR="$MYSQL_DUMP_DIR" + echo RDIFF_BACKUP_OPTS="$RDIFF_BACKUP_OPTS" +} + list(){ read_config "$1" @@ -144,19 +270,33 @@ list(){ echo Show backups of $1 echo ================================================== - # neue Konfiguration lesen - source "$CONFIG_DIR/$1" - - if [ ${#MYSQL_DB[@]} -gt 0 ] + if [ ${#MYSQL_DB[@]} -gt 0 ] then - FOLDERS+=("$MYSQL_DUMP_DIR") + echo show backups 'for' MySQL databases: + + if [ "$MYSQL_CONTAINER" != '' ] + then + rdiff-backup -l "$BACKUP_DIR_CONTAINERS/${MYSQL_CONTAINER}" + else + rdiff-backup -l "${BACKUP_DIR_FILES}/${MYSQL_DUMP_DIR}" + fi + + echo fi for dir in "${FOLDERS[@]}" do - echo show backups for file[s]: "$dir" + echo show backups 'for' file[s]: "$dir" - rdiff-backup -l "$BACKUP_DIR/${REMOTE_HOST}${dir}" + rdiff-backup -l "${BACKUP_DIR_FILES}/${dir}" + echo + done + + for vol in "${DOCKER_VOLUME[@]}" + do + echo show backups 'for' volume: "${vol}" + + rdiff-backup -l "${BACKUP_DIR_VOLS}/${vol}" echo done @@ -164,6 +304,46 @@ list(){ echo } +check_destination_dir(){ + read_config "$1" + + echo Check backups of $1 + echo ================================================== + + if [ ${#MYSQL_DB[@]} -gt 0 ] + then + echo check backups 'for' MySQL databases: + + if [ "$MYSQL_CONTAINER" != '' ] + then + rdiff-backup --check-destination-dir "$BACKUP_DIR_CONTAINERS/${MYSQL_CONTAINER}" + else + rdiff-backup --check-destination-dir "${BACKUP_DIR_FILES}/${MYSQL_DUMP_DIR}" + fi + + echo + fi + + for dir in "${FOLDERS[@]}" + do + echo check backups 'for' file[s]: "$dir" + + rdiff-backup --check-destination-dir "${BACKUP_DIR_FILES}/${dir}" + echo + done + + for vol in "${DOCKER_VOLUME[@]}" + do + echo check backups 'for' volume: "${vol}" + + rdiff-backup --check-destination-dir "${BACKUP_DIR_VOLS}/${vol}" + echo + done + + echo Done check $1 + echo +} + if [ "$1" == "--gen-config" ]; then gen_config @@ -183,7 +363,14 @@ elif [ "$1" == "--bash" ]; then shift /bin/bash "$@" +elif [ "$1" == "--check-destination-dir" ]; then + shift + check_destination_dir "$@" + +elif [ "$1" == "--show-config" ]; then + shift + show_config "$@" + else echo "usage ?" -fi - +fi \ No newline at end of file