diff --git a/RUNBOOK.md b/RUNBOOK.md index a838358..d7bd826 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -115,7 +115,14 @@ BORG_PASSCOMMAND="cat /root/.borg-passphrase" borg break-lock /home/srv/files/ba All `restore.sh` commands accept `--dry-run` to preview exactly what would happen without touching anything, and `--archive NAME` to target a specific archive instead of the latest (see archive names via -`--list-archives`). +`--list-archives`). Root DB credentials come from `MYSQL_ROOT_PASSWORD` in +the environment if set, otherwise from `/root/.mariadb-root.pw` — set +whichever is more convenient for how you're invoking it. Restore logs go +to `/var/log/borg/restore-*.log` (the `RESTORE_LOGDIR` environment +variable overrides the directory, mainly useful for testing). `full` and +`db` share `borg-backup.sh`'s lockfile, so a restore refuses to start +while the nightly backup is mid-run (and vice versa) rather than racing +it. ### 5.1 Full disaster recovery (new or wiped server) @@ -131,13 +138,16 @@ scratch. # /root/.mariadb-root.pw. # 3. Preview: ./restore.sh full --dry-run -# 4. Run for real (refuses if /home/srv/files/content is non-empty): +# 4. Run for real. --force is only required if /home/srv/files/content +# already has data in it (e.g. a stale mount); omit it on a genuinely +# empty/fresh server: ./restore.sh full --force ``` -This extracts the full content tree from the archive, restores every -database dump (users/grants first), and starts the `mariadb` container, -waiting for it to report healthy. +This extracts the full content tree from the archive, starts the +`mariadb` container and waits for it to report healthy, then restores +every database dump (users/grants first) — the container must be running +before any of the dump restores, which is why it starts first. **Verify afterward:** - `docker ps` shows `mariadb` running and healthy. @@ -178,11 +188,18 @@ absolute path it was archived with). Quarterly, run a real `full` restore into a scratch directory (not `/home/srv/files/content`) to confirm backups are actually usable: +`borg extract` always extracts into the current directory (there's no +`--destination` flag — this is why `restore.sh` itself `cd`s into the +destination before extracting), and the archive name must come from +`borg list --short` (plain `borg list` prints a formatted line, not a bare +name), so: + ```bash -mkdir -p /tmp/restore-drill -BORG_PASSCOMMAND="cat /root/.borg-passphrase" \ - borg extract --lock-wait 600 /home/srv/files/backups/borg-2025::$(./restore.sh --list-archives | tail -1) \ - --destination /tmp/restore-drill # (or adapt restore.sh's TARGET for a one-off dry run into scratch) +mkdir -p /tmp/restore-drill && cd /tmp/restore-drill +export BORG_REPO=/home/srv/files/backups/borg-2025 +export BORG_PASSCOMMAND="cat /root/.borg-passphrase" +LATEST=$(borg list --short | tail -1) +borg extract --lock-wait 600 "::$LATEST" ``` Confirm the dump files under `mariadb/dump/` are present, non-empty, and diff --git a/restore.sh b/restore.sh index 89bcc17..b634fa9 100755 --- a/restore.sh +++ b/restore.sh @@ -27,6 +27,11 @@ BORG_PASSPHRASE_FILE="${BORG_PASSPHRASE_FILE:-/root/.borg-passphrase}" ROOT_PASSWORD_FILE="${ROOT_PASSWORD_FILE:-/root/.mariadb-root.pw}" DUMP_SUBDIR="mariadb/dump" +# Same lockfile borg-backup.sh takes (via flock -n 9) before touching $TARGET +# or the repo, so a restore and the nightly backup cron job can never run +# concurrently against each other. +LOCKFILE="${LOCKFILE:-/var/lock/borg-backup.lock}" + LOGDIR="${RESTORE_LOGDIR:-/var/log/borg}" mkdir -p "$LOGDIR" 2>/dev/null || LOGDIR="/tmp" LOGFILE="$LOGDIR/restore-$(date +%Y-%m-%d-%H%M%S).log" @@ -52,7 +57,23 @@ run_cmd() { "$@" } +acquire_lock() { + exec 9>"$LOCKFILE" + if ! flock -n 9; then + die "another backup or restore is already running (lock held on $LOCKFILE)" + fi +} + +preflight() { + [[ -r "$BORG_PASSPHRASE_FILE" ]] || die "passphrase file not readable: $BORG_PASSPHRASE_FILE (chmod 600 it or check the path)" + borg info --lock-wait 60 >/dev/null 2>&1 || die "cannot reach borg repo $REPO (check the passphrase file, permissions, and that the repo exists)" +} + resolve_archive() { + # Deliberately does NOT filter by hostname (unlike borg-backup.sh's + # ARCHIVE_GLOB="$(hostname -s)-*" used for prune/list): disaster recovery + # may need to run from a different host than the one that made the + # backup, so any archive in the repo is a valid restore candidate. if [[ -n "$ARCHIVE_OVERRIDE" ]]; then RESOLVED_ARCHIVE="$ARCHIVE_OVERRIDE" return 0 @@ -81,6 +102,7 @@ cmd_file() { log "[DRY-RUN] would extract ${ARCHIVE_TARGET_PATH}/${rel_path} from ${REPO}::${RESOLVED_ARCHIVE} into $dest" return 0 fi + preflight local final final="$(extract_path "$rel_path" "$dest" "$RESOLVED_ARCHIVE" | tail -n1)" log "Restored file available at: $final" @@ -132,6 +154,7 @@ cmd_db() { return 0 fi confirm_or_abort "$dbname" + preflight dest="/tmp/restore-db-$$" dumpfile="$(extract_path "${DUMP_SUBDIR}/${dbname}.sql" "$dest" "$RESOLVED_ARCHIVE" | tail -n1)" detect_client @@ -171,7 +194,7 @@ start_db() { } cmd_full() { - local staging dumpdir f dbname + local dumpdir f dbname resolve_archive step "Full restore from archive $RESOLVED_ARCHIVE into $TARGET" @@ -181,30 +204,39 @@ cmd_full() { if [[ "$DRY_RUN" == true ]]; then log "[DRY-RUN] would extract full ${ARCHIVE_TARGET_PATH} tree from ${REPO}::${RESOLVED_ARCHIVE} into $TARGET" - log "[DRY-RUN] would restore every *.sql dump under ${DUMP_SUBDIR}/ using root credentials" log "[DRY-RUN] would start $DB_CONTAINER and wait for it to become healthy" + log "[DRY-RUN] would restore every *.sql dump under ${DUMP_SUBDIR}/ using root credentials" return 0 fi - staging="/tmp/restore-full-$$" - mkdir -p "$staging" - ( cd "$staging" && run_cmd borg extract --lock-wait 600 "${REPO}::${RESOLVED_ARCHIVE}" "${ARCHIVE_TARGET_PATH}" ) + preflight - mkdir -p "$(dirname "$TARGET")" - rm -rf "${TARGET:?}"/* 2>/dev/null || true + # rm -rf "$TARGET"/* leaves dotfiles behind (stale .nobackup markers, app + # state) mixed in with the restored tree; find -delete removes everything. + if [[ -d "$TARGET" ]] && [[ -n "$(ls -A "$TARGET" 2>/dev/null)" ]]; then + find "${TARGET:?}" -mindepth 1 -delete + fi mkdir -p "$TARGET" - run_cmd cp -a "${staging}/${ARCHIVE_TARGET_PATH}/." "$TARGET/" - rm -rf "$staging" - detect_client - get_root_creds + # Borg records the absolute path it was given at create time, so + # extracting from / with the leading-slash-stripped path recreates the + # tree directly at $TARGET - no staging copy, no doubled disk usage. + ( cd / && run_cmd borg extract --lock-wait 600 "${REPO}::${RESOLVED_ARCHIVE}" "${ARCHIVE_TARGET_PATH}" ) dumpdir="${TARGET}/${DUMP_SUBDIR}" [[ -d "$dumpdir" ]] || die "no dump directory found after extract: $dumpdir" + # The container must be running before any docker exec against it - on a + # freshly rebuilt server it's created but stopped, so start it first. + step "Starting $DB_CONTAINER" + start_db || die "CRITICAL: $DB_CONTAINER did not come up after extract" + + detect_client + get_root_creds + if [[ -f "${dumpdir}/00-users-and-grants.sql" ]]; then step "Restoring users and grants" - docker exec -i -e MYSQL_PWD="$DB_PASS" "$DB_CONTAINER" \ + run_cmd docker exec -i -e MYSQL_PWD="$DB_PASS" "$DB_CONTAINER" \ "$CLIENT_BIN" -u root < "${dumpdir}/00-users-and-grants.sql" fi @@ -215,9 +247,6 @@ cmd_full() { restore_single_db "$f" "$dbname" done - step "Starting $DB_CONTAINER" - start_db || die "CRITICAL: $DB_CONTAINER did not come up after restore" - log "Full restore complete from archive $RESOLVED_ARCHIVE" } @@ -270,6 +299,7 @@ main() { full) shift parse_common_flags "$@" + acquire_lock cmd_full ;; db) @@ -278,6 +308,7 @@ main() { [[ -n "$dbname" ]] || die "db: missing " shift parse_common_flags "$@" + acquire_lock cmd_db "$dbname" ;; file) @@ -286,6 +317,7 @@ main() { [[ -n "$relpath" ]] || die "file: missing " shift parse_common_flags "$@" + acquire_lock cmd_file "$relpath" ;; *) diff --git a/tests/lib/setup_mocks.sh b/tests/lib/setup_mocks.sh index 0951703..a6f570b 100644 --- a/tests/lib/setup_mocks.sh +++ b/tests/lib/setup_mocks.sh @@ -75,5 +75,36 @@ exit 0 EOF cp "$dir/mysql" "$dir/mariadb" - chmod +x "$dir/borg" "$dir/docker" "$dir/mysql" "$dir/mariadb" + # flock(1) is Linux-only (util-linux) and absent on macOS dev machines; + # restore.sh only ever uses the "flock -n FD" form (never the + # command-wrapping form), so a no-op success is a faithful stand-in for + # exclusion testing purposes - no test here exercises actual contention. + cat > "$dir/flock" <> "$mocklog" +exit 0 +EOF + + chmod +x "$dir/borg" "$dir/docker" "$dir/mysql" "$dir/mariadb" "$dir/flock" +} + +# restore.sh hardens PATH with system dirs (/usr/local/bin etc.) placed +# ahead of $PATH, so `PATH="$mockdir:$PATH" ...` alone does NOT guarantee the +# mock is what actually runs: a real borg/docker/mysql/mariadb installed in +# one of those system dirs would shadow it. BASH_ENV is sourced by bash +# before running a script and shell functions win over PATH lookup for +# simple commands regardless of PATH ordering, so this is what actually +# guarantees isolation on any machine, not just ones that happen to lack +# those binaries in the hardened prefix. +mock_bash_env() { + local dir="$1" + local bashenv="$dir/bash_env.sh" + cat > "$bashenv" < "$passfile" + out="$(PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" \ + BORG_PASSPHRASE_FILE="$passfile" LOCKFILE="$lockfile" \ + bash "$RESTORE" file photos/img.jpg --dest "$dest")" final="$dest/home/srv/files/content/photos/img.jpg" assert_contains "$out" "Restored file available at: $final" "file mode reports final path" if [[ -f "$final" ]]; then @@ -81,7 +93,8 @@ test_file_restore_dry_run_makes_no_borg_call() { mockdir="$(mktemp -d)" dest="$(mktemp -d)" setup_mock_bin "$mockdir" - PATH="$mockdir:$PATH" bash "$RESTORE" file photos/img.jpg --dest "$dest" --dry-run >/dev/null + PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" LOCKFILE="$mockdir/lock" \ + bash "$RESTORE" file photos/img.jpg --dest "$dest" --dry-run >/dev/null if [[ -s "$mockdir/mock.log" ]] && grep -q "^borg extract" "$mockdir/mock.log"; then echo "FAIL: dry-run invoked borg extract" FAILURES=$((FAILURES + 1)) @@ -92,20 +105,16 @@ test_file_restore_dry_run_makes_no_borg_call() { } test_db_restore_with_yes_runs_full_sequence() { - local mockdir out bashenv + local mockdir out passfile mockdir="$(mktemp -d)" setup_mock_bin "$mockdir" echo "rootpass" > "$mockdir/rootpw" - # This host may have a real `docker` CLI in one of the system dirs that - # restore.sh's hardened PATH prepends (e.g. /usr/local/bin), which would - # shadow the mock and make a real (unreachable) docker daemon get called - # instead. A BASH_ENV-sourced function takes precedence over PATH lookup - # regardless of PATH ordering, so force `docker` to the mock that way. - bashenv="$mockdir/bash_env.sh" - cat > "$bashenv" < "$passfile" + out="$(PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" \ + ROOT_PASSWORD_FILE="$mockdir/rootpw" BORG_PASSPHRASE_FILE="$passfile" \ + LOCKFILE="$mockdir/lock" \ + bash "$RESTORE" db shopdb --yes /dev/null 2>&1 + echo "wrongname" | PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" LOCKFILE="$mockdir/lock" \ + bash "$RESTORE" db shopdb >/dev/null 2>&1 rc=$? set -e assert_eq "1" "$rc" "db mode aborts on mismatched confirmation" @@ -147,19 +158,20 @@ test_db_restore_aborts_on_wrong_confirmation() { } test_full_restore_refuses_nonempty_target_without_force() { - local mockdir target rc + local mockdir target out rc mockdir="$(mktemp -d)" target="$(mktemp -d)" touch "$target/existing-file" setup_mock_bin "$mockdir" set +e - PATH="$mockdir:$PATH" TARGET_OVERRIDE=1 bash -c ' + out="$(PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" LOCKFILE="$mockdir/lock" bash -c ' sed "s#^TARGET=\"/home/srv/files/content\"#TARGET=\"'"$target"'\"#; s@^ARCHIVE_TARGET_PATH=.*@ARCHIVE_TARGET_PATH=\"\${TARGET#/}\"@" "'"$RESTORE"'" > "'"$mockdir"'/restore_patched.sh" bash "'"$mockdir"'/restore_patched.sh" full - ' >/dev/null 2>&1 + ' 2>&1)" rc=$? set -e assert_eq "1" "$rc" "full mode refuses non-empty target without --force" + assert_contains "$out" "is not empty - pass --force" "refusal message names the reason" rm -rf "$mockdir" "$target" } @@ -168,7 +180,7 @@ test_full_restore_dry_run_makes_no_calls() { mockdir="$(mktemp -d)" target="$(mktemp -d)" setup_mock_bin "$mockdir" - out="$(PATH="$mockdir:$PATH" bash -c ' + out="$(PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" LOCKFILE="$mockdir/lock" bash -c ' sed "s#^TARGET=\"/home/srv/files/content\"#TARGET=\"'"$target"'\"#; s@^ARCHIVE_TARGET_PATH=.*@ARCHIVE_TARGET_PATH=\"\${TARGET#/}\"@" "'"$RESTORE"'" > "'"$mockdir"'/restore_patched.sh" bash "'"$mockdir"'/restore_patched.sh" full --dry-run ')"