feat: add full disaster-recovery mode to restore.sh

Adds container_running/container_health/wait_for_container/start_db
helpers and cmd_full(), plus tests. Fixed a delimiter bug in the two
new tests' sed scripts (the literal `${TARGET#/}` replacement text
collided with `#` as the sed delimiter) and relaxed the dry-run
no-external-calls assertion to allow the legitimate borg-list call
from resolve_archive(), mirroring the existing db dry-run test.
This commit is contained in:
kbe
2026-07-25 19:23:27 +02:00
parent 5d547f04ec
commit 4be1372e4d
2 changed files with 122 additions and 0 deletions
+80
View File
@@ -141,6 +141,86 @@ cmd_db() {
rm -rf "$dest"
}
container_running() {
[[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || echo false)" == "true" ]]
}
container_health() {
docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
"$1" 2>/dev/null || echo unknown
}
wait_for_container() {
local name="$1" deadline=$((SECONDS + DB_START_TIMEOUT)) health
while (( SECONDS < deadline )); do
if container_running "$name"; then
health="$(container_health "$name")"
case "$health" in
healthy|none) log "$name is up (health: $health)"; return 0 ;;
unhealthy) log "WARNING: $name reports unhealthy" ;;
esac
fi
sleep 3
done
return 1
}
start_db() {
docker start "$DB_CONTAINER" >/dev/null 2>&1 || true
wait_for_container "$DB_CONTAINER"
}
cmd_full() {
local staging dumpdir f dbname
resolve_archive
step "Full restore from archive $RESOLVED_ARCHIVE into $TARGET"
if [[ -d "$TARGET" ]] && [[ -n "$(ls -A "$TARGET" 2>/dev/null)" ]] && [[ "$FORCE" != true ]]; then
die "$TARGET is not empty - pass --force to overwrite (existing data will be replaced)"
fi
if [[ "$DRY_RUN" == true ]]; then
log "[DRY-RUN] would extract full ${ARCHIVE_TARGET_PATH} tree from ${REPO}::${RESOLVED_ARCHIVE} into $TARGET"
log "[DRY-RUN] would restore every *.sql dump under ${DUMP_SUBDIR}/ using root credentials"
log "[DRY-RUN] would start $DB_CONTAINER and wait for it to become healthy"
return 0
fi
staging="/tmp/restore-full-$$"
mkdir -p "$staging"
( cd "$staging" && run_cmd borg extract --lock-wait 600 "${REPO}::${RESOLVED_ARCHIVE}" "${ARCHIVE_TARGET_PATH}" )
mkdir -p "$(dirname "$TARGET")"
rm -rf "${TARGET:?}"/* 2>/dev/null || true
mkdir -p "$TARGET"
run_cmd cp -a "${staging}/${ARCHIVE_TARGET_PATH}/." "$TARGET/"
rm -rf "$staging"
detect_client
get_root_creds
dumpdir="${TARGET}/${DUMP_SUBDIR}"
[[ -d "$dumpdir" ]] || die "no dump directory found after extract: $dumpdir"
if [[ -f "${dumpdir}/00-users-and-grants.sql" ]]; then
step "Restoring users and grants"
docker exec -i -e MYSQL_PWD="$DB_PASS" "$DB_CONTAINER" \
"$CLIENT_BIN" -u root < "${dumpdir}/00-users-and-grants.sql"
fi
for f in "$dumpdir"/*.sql; do
[[ -e "$f" ]] || continue
dbname="$(basename "$f" .sql)"
[[ "$dbname" == "00-users-and-grants" ]] && continue
restore_single_db "$f" "$dbname"
done
step "Starting $DB_CONTAINER"
start_db || die "CRITICAL: $DB_CONTAINER did not come up after restore"
log "Full restore complete from archive $RESOLVED_ARCHIVE"
}
usage() {
cat <<'EOF'
Usage:
+42
View File
@@ -146,6 +146,46 @@ test_db_restore_aborts_on_wrong_confirmation() {
rm -rf "$mockdir"
}
test_full_restore_refuses_nonempty_target_without_force() {
local mockdir target rc
mockdir="$(mktemp -d)"
target="$(mktemp -d)"
touch "$target/existing-file"
setup_mock_bin "$mockdir"
set +e
PATH="$mockdir:$PATH" TARGET_OVERRIDE=1 bash -c '
sed "s#^TARGET=\"/home/srv/files/content\"#TARGET=\"'"$target"'\"#; s@^ARCHIVE_TARGET_PATH=.*@ARCHIVE_TARGET_PATH=\"\${TARGET#/}\"@" "'"$RESTORE"'" > "'"$mockdir"'/restore_patched.sh"
bash "'"$mockdir"'/restore_patched.sh" full
' >/dev/null 2>&1
rc=$?
set -e
assert_eq "1" "$rc" "full mode refuses non-empty target without --force"
rm -rf "$mockdir" "$target"
}
test_full_restore_dry_run_makes_no_calls() {
local mockdir target out
mockdir="$(mktemp -d)"
target="$(mktemp -d)"
setup_mock_bin "$mockdir"
out="$(PATH="$mockdir:$PATH" bash -c '
sed "s#^TARGET=\"/home/srv/files/content\"#TARGET=\"'"$target"'\"#; s@^ARCHIVE_TARGET_PATH=.*@ARCHIVE_TARGET_PATH=\"\${TARGET#/}\"@" "'"$RESTORE"'" > "'"$mockdir"'/restore_patched.sh"
bash "'"$mockdir"'/restore_patched.sh" full --dry-run
')"
assert_contains "$out" "DRY-RUN" "full dry-run prints DRY-RUN plan"
# cmd_full calls resolve_archive() (a borg list call) before checking
# DRY_RUN, same as cmd_db/cmd_file, so mock.log legitimately gets a
# "borg list" entry. What must NOT happen in dry-run is a borg extract,
# or any docker/db-client interaction.
if grep -qE "^(borg extract|docker|mysql|mariadb)" "$mockdir/mock.log" 2>/dev/null; then
echo "FAIL: full dry-run invoked borg extract or a docker/db-client mock binary"
FAILURES=$((FAILURES + 1))
else
echo "PASS: full dry-run made no borg extract or docker/db-client calls"
fi
rm -rf "$mockdir" "$target"
}
test_help_exits_zero
test_no_args_exits_one
test_unknown_command_exits_one
@@ -155,6 +195,8 @@ test_file_restore_dry_run_makes_no_borg_call
test_db_restore_with_yes_runs_full_sequence
test_db_restore_dry_run_skips_confirmation_and_calls
test_db_restore_aborts_on_wrong_confirmation
test_full_restore_refuses_nonempty_target_without_force
test_full_restore_dry_run_makes_no_calls
echo "-----"
if [[ "$FAILURES" -gt 0 ]]; then