fix: start mariadb before restoring DBs in cmd_full, add lock/preflight
Final review of the restore.sh branch found cmd_full restored every database via `docker exec` before starting the container, which fails immediately in the exact scenario full restore exists for (a freshly rebuilt, stopped container). Reorders to extract -> start container -> restore DBs. Also, while touching cmd_full: - Extract directly into place (cd / && borg extract) instead of staging a full copy under /tmp then cp -a'ing it into $TARGET - halves disk usage and restore time. - Replace `rm -rf "$TARGET"/*` with `find "$TARGET" -mindepth 1 -delete` so dotfiles don't survive a --force wipe. - Add acquire_lock() (shares borg-backup.sh's lockfile so a restore and the nightly backup cron can't run concurrently) and preflight() (passphrase file readable, repo reachable) before any real work in full/db/file. Test isolation: mock borg/docker/mysql/mariadb consistently via a BASH_ENV shim (previously only db-mode's test worked around PATH shadowing by a real docker binary; every mocked test needed it, and a missing `flock` mock broke everything once acquire_lock was added, since flock(1) doesn't exist on macOS). Tests also isolate LOCKFILE and BORG_PASSPHRASE_FILE to throwaway paths. RUNBOOK.md: fix the quarterly drill command (borg extract has no --destination flag, and needs `borg list --short` for a bare archive name), reword the full-restore --force comment which read backwards, and document the MYSQL_ROOT_PASSWORD/RESTORE_LOGDIR env overrides and where restore logs land. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5
+47
-15
@@ -27,6 +27,11 @@ BORG_PASSPHRASE_FILE="${BORG_PASSPHRASE_FILE:-/root/.borg-passphrase}"
|
||||
ROOT_PASSWORD_FILE="${ROOT_PASSWORD_FILE:-/root/.mariadb-root.pw}"
|
||||
DUMP_SUBDIR="mariadb/dump"
|
||||
|
||||
# Same lockfile borg-backup.sh takes (via flock -n 9) before touching $TARGET
|
||||
# or the repo, so a restore and the nightly backup cron job can never run
|
||||
# concurrently against each other.
|
||||
LOCKFILE="${LOCKFILE:-/var/lock/borg-backup.lock}"
|
||||
|
||||
LOGDIR="${RESTORE_LOGDIR:-/var/log/borg}"
|
||||
mkdir -p "$LOGDIR" 2>/dev/null || LOGDIR="/tmp"
|
||||
LOGFILE="$LOGDIR/restore-$(date +%Y-%m-%d-%H%M%S).log"
|
||||
@@ -52,7 +57,23 @@ run_cmd() {
|
||||
"$@"
|
||||
}
|
||||
|
||||
acquire_lock() {
|
||||
exec 9>"$LOCKFILE"
|
||||
if ! flock -n 9; then
|
||||
die "another backup or restore is already running (lock held on $LOCKFILE)"
|
||||
fi
|
||||
}
|
||||
|
||||
preflight() {
|
||||
[[ -r "$BORG_PASSPHRASE_FILE" ]] || die "passphrase file not readable: $BORG_PASSPHRASE_FILE (chmod 600 it or check the path)"
|
||||
borg info --lock-wait 60 >/dev/null 2>&1 || die "cannot reach borg repo $REPO (check the passphrase file, permissions, and that the repo exists)"
|
||||
}
|
||||
|
||||
resolve_archive() {
|
||||
# Deliberately does NOT filter by hostname (unlike borg-backup.sh's
|
||||
# ARCHIVE_GLOB="$(hostname -s)-*" used for prune/list): disaster recovery
|
||||
# may need to run from a different host than the one that made the
|
||||
# backup, so any archive in the repo is a valid restore candidate.
|
||||
if [[ -n "$ARCHIVE_OVERRIDE" ]]; then
|
||||
RESOLVED_ARCHIVE="$ARCHIVE_OVERRIDE"
|
||||
return 0
|
||||
@@ -81,6 +102,7 @@ cmd_file() {
|
||||
log "[DRY-RUN] would extract ${ARCHIVE_TARGET_PATH}/${rel_path} from ${REPO}::${RESOLVED_ARCHIVE} into $dest"
|
||||
return 0
|
||||
fi
|
||||
preflight
|
||||
local final
|
||||
final="$(extract_path "$rel_path" "$dest" "$RESOLVED_ARCHIVE" | tail -n1)"
|
||||
log "Restored file available at: $final"
|
||||
@@ -132,6 +154,7 @@ cmd_db() {
|
||||
return 0
|
||||
fi
|
||||
confirm_or_abort "$dbname"
|
||||
preflight
|
||||
dest="/tmp/restore-db-$$"
|
||||
dumpfile="$(extract_path "${DUMP_SUBDIR}/${dbname}.sql" "$dest" "$RESOLVED_ARCHIVE" | tail -n1)"
|
||||
detect_client
|
||||
@@ -171,7 +194,7 @@ start_db() {
|
||||
}
|
||||
|
||||
cmd_full() {
|
||||
local staging dumpdir f dbname
|
||||
local dumpdir f dbname
|
||||
resolve_archive
|
||||
step "Full restore from archive $RESOLVED_ARCHIVE into $TARGET"
|
||||
|
||||
@@ -181,30 +204,39 @@ cmd_full() {
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "[DRY-RUN] would extract full ${ARCHIVE_TARGET_PATH} tree from ${REPO}::${RESOLVED_ARCHIVE} into $TARGET"
|
||||
log "[DRY-RUN] would restore every *.sql dump under ${DUMP_SUBDIR}/ using root credentials"
|
||||
log "[DRY-RUN] would start $DB_CONTAINER and wait for it to become healthy"
|
||||
log "[DRY-RUN] would restore every *.sql dump under ${DUMP_SUBDIR}/ using root credentials"
|
||||
return 0
|
||||
fi
|
||||
|
||||
staging="/tmp/restore-full-$$"
|
||||
mkdir -p "$staging"
|
||||
( cd "$staging" && run_cmd borg extract --lock-wait 600 "${REPO}::${RESOLVED_ARCHIVE}" "${ARCHIVE_TARGET_PATH}" )
|
||||
preflight
|
||||
|
||||
mkdir -p "$(dirname "$TARGET")"
|
||||
rm -rf "${TARGET:?}"/* 2>/dev/null || true
|
||||
# rm -rf "$TARGET"/* leaves dotfiles behind (stale .nobackup markers, app
|
||||
# state) mixed in with the restored tree; find -delete removes everything.
|
||||
if [[ -d "$TARGET" ]] && [[ -n "$(ls -A "$TARGET" 2>/dev/null)" ]]; then
|
||||
find "${TARGET:?}" -mindepth 1 -delete
|
||||
fi
|
||||
mkdir -p "$TARGET"
|
||||
run_cmd cp -a "${staging}/${ARCHIVE_TARGET_PATH}/." "$TARGET/"
|
||||
rm -rf "$staging"
|
||||
|
||||
detect_client
|
||||
get_root_creds
|
||||
# Borg records the absolute path it was given at create time, so
|
||||
# extracting from / with the leading-slash-stripped path recreates the
|
||||
# tree directly at $TARGET - no staging copy, no doubled disk usage.
|
||||
( cd / && run_cmd borg extract --lock-wait 600 "${REPO}::${RESOLVED_ARCHIVE}" "${ARCHIVE_TARGET_PATH}" )
|
||||
|
||||
dumpdir="${TARGET}/${DUMP_SUBDIR}"
|
||||
[[ -d "$dumpdir" ]] || die "no dump directory found after extract: $dumpdir"
|
||||
|
||||
# The container must be running before any docker exec against it - on a
|
||||
# freshly rebuilt server it's created but stopped, so start it first.
|
||||
step "Starting $DB_CONTAINER"
|
||||
start_db || die "CRITICAL: $DB_CONTAINER did not come up after extract"
|
||||
|
||||
detect_client
|
||||
get_root_creds
|
||||
|
||||
if [[ -f "${dumpdir}/00-users-and-grants.sql" ]]; then
|
||||
step "Restoring users and grants"
|
||||
docker exec -i -e MYSQL_PWD="$DB_PASS" "$DB_CONTAINER" \
|
||||
run_cmd docker exec -i -e MYSQL_PWD="$DB_PASS" "$DB_CONTAINER" \
|
||||
"$CLIENT_BIN" -u root < "${dumpdir}/00-users-and-grants.sql"
|
||||
fi
|
||||
|
||||
@@ -215,9 +247,6 @@ cmd_full() {
|
||||
restore_single_db "$f" "$dbname"
|
||||
done
|
||||
|
||||
step "Starting $DB_CONTAINER"
|
||||
start_db || die "CRITICAL: $DB_CONTAINER did not come up after restore"
|
||||
|
||||
log "Full restore complete from archive $RESOLVED_ARCHIVE"
|
||||
}
|
||||
|
||||
@@ -270,6 +299,7 @@ main() {
|
||||
full)
|
||||
shift
|
||||
parse_common_flags "$@"
|
||||
acquire_lock
|
||||
cmd_full
|
||||
;;
|
||||
db)
|
||||
@@ -278,6 +308,7 @@ main() {
|
||||
[[ -n "$dbname" ]] || die "db: missing <db_name>"
|
||||
shift
|
||||
parse_common_flags "$@"
|
||||
acquire_lock
|
||||
cmd_db "$dbname"
|
||||
;;
|
||||
file)
|
||||
@@ -286,6 +317,7 @@ main() {
|
||||
[[ -n "$relpath" ]] || die "file: missing <path>"
|
||||
shift
|
||||
parse_common_flags "$@"
|
||||
acquire_lock
|
||||
cmd_file "$relpath"
|
||||
;;
|
||||
*)
|
||||
|
||||
Reference in New Issue
Block a user