From 41580f9a0d6bf011f0adb952668e58bc73a24b5b Mon Sep 17 00:00:00 2001 From: mj-saunders Date: Fri, 4 Nov 2022 16:55:16 +0400 Subject: [PATCH] Refactor Simplify things by placing the server wrap code into it's own function in the main script. Move to predominantly lowercase and local variables. Introduce usage of separate config files for each server. --- .gitignore | 2 + README.md | 15 +++ backup.sh | 287 ++++++++++++++++++++++++---------------- server-wrap.sh | 55 -------- server_config/README.md | 38 ++++++ 5 files changed, 226 insertions(+), 171 deletions(-) create mode 100644 .gitignore create mode 100644 README.md delete mode 100755 server-wrap.sh create mode 100644 server_config/README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e9443bb --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +server_config/* +!server_config/README* diff --git a/README.md b/README.md new file mode 100644 index 0000000..e93ad47 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# BorgBackup Reverse + +If you find yourself in a situation where your only backup solution is remote +to local, this may provide a semi-sane method to achieve this without having to +store any credentials on your remote machines. + + +## Description + +TODO + + +## Usage + +TODO diff --git a/backup.sh b/backup.sh index 8fb59af..e813102 100755 --- a/backup.sh +++ b/backup.sh @@ -4,124 +4,179 @@ set -e [ -n "$DEBUG" ] && set -x -display_usage() { - echo -e "Usage: $0 create|extract repo-name\n" +# NOTE: +# The repo variable is used in a few different ways - it should be the name of: +# 1. an existing borg repository +# 2. a config file in $borg_dir/server_config/ +# 3. an entry in your ssh config + +main() +{ + local borg_old=0 + + while getopts "cp:h" option; do + case "$option" in + c) borg_old=1 + ;; + p) BORG_REPO_PATH=$OPTARG + ;; + h) + display_usage 0 + ;; + *) + echo "Invalid option" + display_usage 1 + ;; + esac + done + shift "$((OPTIND - 1))" + + if [ $# -ne 2 ]; then + display_usage 1 + fi + if [ `id -u` -eq 0 ]; then + echo "This script must not be run as root!" + display_usage 1 + fi + if [ -z ${BORG_REPO_PATH} ]; then + echo "[ERROR] BORG_REPO_PATH is empty or unset." + display_usage 1 + fi + + local mode=$1 + local repo=$2 + local borg_dir="${HOME}/.borg" + + if [ $mode != "create" ] && [ $mode != "extract" ]; then + display_usage 1 + fi + + # Server config + # The config file is expected to provide several arrays: + # include, exclude and volatile + # TODO: Move config to more standard location + #local config_file="${borg_dir}/server_config/${repo}" + local config_file="server_config/${repo}" + if [[ ! -f ${config_file} ]]; then + echo "[ERROR] Config file missing; expected ${config_file}" + exit 1 + fi + source "${config_file}" + + # Make sure the config defined the required arrays + for var in include exclude; do + if [[ -z ${!var} ]]; then + echo "[ERROR] ${repo}: The config requires an '${var}' array" + exit 1 + fi + done + + # borg-backup requires --exclude to prepend each excluded item + exclude=( "${exclude[@]/#/--exclude }" ) + + ## Logging + local log_dir="${borg_dir}/log" + local log_month_dir=${log_dir}/$(date --utc "+%Y-%m") + local log_file=${log_month_dir}/$(date --utc "+%Y-%m-%d_%H.%M.%SZ")_${repo}.log + + if [[ ! -d "${log_month_dir}" ]]; then + echo "== Creating log directory" + mkdir --parent "${log_month_dir}" \ + || { echo "[ERROR] Failed to create log directory. Aborting."; exit 1; } + fi + + touch ${log_file} \ + || { echo "[ERROR] Failed to create log file. Aborting"; exit 1; } + + echo "== NOTE: To monitor progress, run this from another terminal:" + echo " $ tail -f ${log_file}" + + # Temporarilly disallow glob + set -o noglob + + local borg_options=(--show-rc --list --stats --one-file-system --exclude-caches) + + if [[ -n $DEBUG ]]; then + borg_options+=(--verbose) + fi + if [ -z $BORG_OLD ]; then + borg_options+=(--show-version --exclude-nodump --keep-exclude-tags) + fi + + local socket_local="/tmp/borg-local.sock" + local socket_remote="/tmp/borg-remote.sock" + + if [ $mode == 'create' ]; then + echo "== CREATING" + # NOTE: "backup-server" is arbitrary and can be anything + # the socat-wrapper will ignore it + server_wrap $socket_local $socket_remote $mode $repo \ + borg create \ + "${borg_options[@]}" \ + "${exclude[@]}" "${volatile[@]}" \ + ssh://backup-server/$BORG_REPO_PATH/$repo::{utcnow:%Y-%m-%d} \ + "${include[@]}" \ + 2>&1 | tee ${log_file} + + # TODO: Test and fix up extract procedure + else + echo "== EXTRACTING" + local archive=$(borg info $BORG_REPO_PATH/$repo --last 1 | grep 'Archive name' | awk '{print $3}') + server_wrap $socket_local $socket_remote $mode $repo \ + borg extract \ + --dry-run \ + ssh://backup-server/$BORG_REPO_PATH/$repo::$archive \ + 2>&1 | tee $log_file + fi + + # Re-allow glob + set +o noglob + + # Tidy up + # NOTE: socket_remote is dealt with in 'server_wrap.sh' + if [ -f $socket_local ]; then + rm $socket_local + fi } -exit_abnormal () { - display_usage - exit 1 +server_wrap() +{ + if [ $# -lt 5 ]; then + echo "[ERROR] Missing argument(s)" + exit 1 + fi + + set -o noglob + + local socket_local="$1" + local socket_remote="$2" + local mode="$3" + local repo="$4" + local borg_command="${@:5}; rm $socket_remote" + local remote_socat_command="'bash -c \"exec socat STDIO UNIX-CONNECT:$socket_remote\"'" + + # Make command more robust against premature expansion + borg_command=`echo $borg_command | sed "s/--exclude\s\(\S\+\)/--exclude \'\1\'/g"` + + # TODO: Handle extract + #if [ $mode == "extract" ]; then + # SH_CMD="cd /mnt" + #fi + + exec socat UNIX-LISTEN:"$socket_local" \ + "EXEC:borg serve --append-only --restrict-to-path $BORG_REPO_PATH --umask 077" & + ssh -t -R "$socket_remote":"$socket_local" $repo \ + sudo BORG_RSH="$remote_socat_command" "$borg_command" + + set +o noglob } -if [ $# -ne 2 ]; then - exit_abnormal -fi -if [ `id -u` -eq 0 ]; then - echo "This script must not be run as root!" - exit_abnormal -fi +display_usage() +{ + echo "Usage: $(basename $0) [-ch] [-p path] create|extract repo-name" + echo " -c Use flags compatible with borg < (version tbd)" + echo " -p Repository path. Overrides BORG_REPO_PATH" + exit $1 +} -#user_name=borg -#if [ "$(id --user --name)" != "$bbbs_user_name" ]; then -# echo "$0 must be run as $bbbs_user_name" -# exit -#fi - -while getopts "h?" option; do - case "$option" in - h|\?) - display_usage - exit 0 - ;; - esac -done - -MODE=$1 -REPO=$2 - -if [ $MODE != "create" ] && [ $MODE != "extract" ]; then - echo "Mode must be either 'create' or 'extract'" - exit_abnormal -fi - -log_base_dir="${HOME}/.borg/logs" - -month_dir=$(date --utc "+%Y-%m") -log_dir=${log_base_dir}/${month_dir} - -log_file=${log_dir}/$(date --utc "+%Y-%m-%d_%H.%M.%SZ")_${REPO}.log - -mkdir --parent "${log_dir}" - -echo "To monitor progress, run this in another terminal:" -echo "$ tail -f ${log_file}" - -# Temporarilly disallow glob -set -o noglob - -BORG_OPTIONS="--show-rc --list --stats --one-file-system --exclude-caches" - -if [ -z $BORG_OLD ]; then - BORG_OPTIONS+=" --show-version --exclude-nodump --keep-exclude-tags" -fi - -[ -n $DEBUG ] && BORG_OPTIONS+=" --verbose" - -COMMON_TARGET="/ /boot /etc /root /home /opt /srv /var /var/log /usr" - -COMMON_EXCLUDE="--exclude /sys/ --exclude /proc/ --exclude /dev/ --exclude /run/ --exclude /var/run/ --exclude /var/lock --exclude /mnt/" - -VOLATILE_EXCLUDE="--exclude /tmp/ --exclude /var/tmp/ --exclude /lost+found --exclude /var/cache/ --exclude /root/.cache --exclude /home/*/.cache" - -LXD_BASE="/var/lib/lxd/*/rootfs" -LXD_EXCLUDE="--exclude $LXD_BASE/lost+found --exclude $LXD_BASE/media/* --exclude $LXD_BASE/mnt/* \ - --exclude $LXD_BASE/proc/* --exclude $LXD_BASE/run/* --exclude $LXD_BASE/sys/* \ - --exclude $LXD_BASE/tmp/*" -LXC_BASE="/home/*/.local/share/lxc/*/rootfs" -LXC_EXCLUDE="--exclude $LXC_BASE/lost+found --exclude $LXC_BASE/media/* --exclude $LXC_BASE/mnt/* \ - --exclude $LXC_BASE/proc/* --exclude $LXC_BASE/run/* --exclude $LXC_BASE/sys/* \ - --exclude $LXC_BASE/tmp/*" - -# Especially when not using --one-file-system -# --exclude /var/run # -> /run - -SOCK_LOCAL="/tmp/borg-local.sock" -SOCK_REMOTE="/tmp/borg-remote.sock" - -if [ $MODE == 'create' ]; then - echo "=== CREATING" - . ./server-wrap.sh -s $SOCK_LOCAL $SOCK_REMOTE $MODE $REPO \ - borg create \ - "$BORG_OPTIONS" \ - "$COMMON_EXCLUDE" "$VOLATILE_EXCLUDE" \ - "$LXD_EXCLUDE" "$LXC_EXCLUDE" \ - ssh://backup-server/$BORG_REPO_PATH/$REPO::{utcnow:%Y-%m-%d} \ - "$COMMON_TARGET" \ - 2>&1 | tee ${log_file} - - # Tidy up Client .sock - dealt with in 'server-wrap.sh' - # . Chance are, on error, the .sock file will not be cleaned up properly - # Current setup requires no specific files on remote machine - # Could perhaps require one small script that just handles any necessary cleanup - - # --exclude /var/lib/lxd/ \ - # --exclude /var/lib/vz/images/ \ - -# TODO: Test and fix up extract procedure -else - echo "=== EXTRACTING" - ARCHIVE=$(borg info $BORG_REPO_PATH/$REPO --last 1 | grep 'Archive name' | awk '{print $3}') - . ./server-wrap.sh -s $SOCK_LOCAL $SOCK_REMOTE $MODE $REPO \ - borg extract \ - --dry-run \ - ssh://backup-server/$BORG_REPO_PATH/$REPO::$ARCHIVE \ - 2>&1 | tee $log_file -fi - -# Re-allow glob -set +o noglob - -# Tidy up - if necessary -if [ -f $SOCK_LOCAL ]; then - rm $SOCK_LOCAL -fi +main "$@" diff --git a/server-wrap.sh b/server-wrap.sh deleted file mode 100755 index f1272b0..0000000 --- a/server-wrap.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -if [ -z ${BORG_REPO_PATH} ]; then - echo "BORG_REPO_PATH is empty or unset. Please set and try again." - exit 1 -fi - -# Temporarilly disallow glob -set -o noglob - -SOCK_SRV="$2" -SOCK_CLI="$3" -CLIENT_SOCAT="'bash -c \"exec socat STDIO UNIX-CONNECT:$SOCK_CLI\"'" -MODE="$4" -REPO="$5" -BORG_CMD="${@:6}" -# Add some cleanup to the command -BORG_CMD+=" && rm $SOCK_CLI" - -# Make command more robust from premature expansion -BORG_CMD=`echo $BORG_CMD | sed "s/--exclude\s\(\S\+\)/--exclude \'\1\'/g"` - -echo $BORG_CMD - -if [ $MODE == "extract" ]; then - SH_CMD="cd /mnt" -fi - -#user_name="borg" - -#if [ "$(id --user --name)" != "$user_name" -o $# -lt 6 ]; then -if [ $# -lt 6 ]; then - echo "$0 must be run as $user_name" - echo "usage: sudo -u $user_name [env vars] $0 [-s|--socket] path-to/local-listening.sock path-to/remote-connecting.sock path-to/socat-wrapper user@sourcehost " - echo "usage: sudo -u $user_name [env vars] $0 [-t|--tcp] local-listening-port remote-connecting-port path-to/socat-wrapper user@sourcehost " - echo - - echo "example: sudo -u $user_name BORGW_RESTRICT_PATH=/path/to/repos $0 -s /tmp/local.sock /tmp/remote.sock /opt/borg/client-wrap"\ - "\"backuped-server -p 22\" sudo borg create ssh://backup-server/./my-repo::{hostname}_{utcnow} paths to backup" - - echo "example: sudo -u $user_name SSH_ARGS=\"-o ProxyCommand=ssh -W %h:%p gateway-server -p 22\" BORGW_RESTRICT_REPOSITORY=/path/to/repos/repo"\ - "$0 -t 12345 12345 /opt/borg/client-wrap backuped-server sudo borg"\ - "create ssh://backup-server/./::{hostname}_{utcnow} paths to backup" - echo - - echo "Note: \"backup-server\" is arbitrary and can be anything - the socat-wrapper will ignore it" -else - exec socat UNIX-LISTEN:"$SOCK_SRV" \ - "EXEC:borg serve --append-only --restrict-to-path $BORG_REPO_PATH --umask 077" & - - ssh -t -R "$SOCK_CLI":"$SOCK_SRV" $REPO sudo BORG_RSH="$CLIENT_SOCAT" "$BORG_CMD" -fi - -# Re-allow glob -set +o noglob diff --git a/server_config/README.md b/server_config/README.md new file mode 100644 index 0000000..2e02d63 --- /dev/null +++ b/server_config/README.md @@ -0,0 +1,38 @@ +# Server Configuration + +Create a file for each of your remote servers to be backed up. +The contents is used to determine what and what not to back up. + + +## File Names + +Each filename must match an entry in your `~/.ssh/config`. +It will be used to connect to the remote machine. +It must also match the name of the target borg-backup repository. + +A minimal example: + +``` +~/.ssh/config + +Host repo-name + HostName + User foo +``` + +## File Content + +We use bash arrays to list `include` and `exclude` paths. +These are then sourced by the backup script and prepared for use with `borg`. + +See the `template` file for a more complete example. + +``` +include=( path/to/include ... ) + +exclude=( + # Comments should be safe to use + path/to/exclude + ... +) +```