commit d6f30c506e248e16352bce6ecde0e49b9ca0c166 Author: Kevin Bataille Date: Sat Jul 25 18:51:38 2026 +0200 chore: initial import of backup scripts and design docs Existing borg-backup.sh/dump_db.sh, old/ reference scripts, and the brainstormed design spec for restore tooling + runbook. diff --git a/borg-backup.sh b/borg-backup.sh new file mode 100644 index 0000000..7428d63 --- /dev/null +++ b/borg-backup.sh @@ -0,0 +1,318 @@ +#!/bin/bash +# ============================================================================= +# Borg backup: /content + MariaDB -> local repo -> Scaleway S3 mirror +# ============================================================================= +# CHANGES vs. original: +# • Encryption enabled (repokey-blake2 via BORG_PASSCOMMAND) +# • Preflight warns if repo is unencrypted +# • Added borg check (daily archive-only, weekly full --verify-data) +# • Removed BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK +# • No longer stops MariaDB: dump_db.sh's --single-transaction dump is +# already consistent, and the container's raw data directory is +# excluded from the archive (drop a .nobackup file in it - see +# --exclude-if-present below) so no live InnoDB file is ever copied. +# Zero DB downtime during backup. +# ============================================================================= + +set -euo pipefail + +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +umask 077 + +# ========================= CONFIGURATION ========================= + +NAME="borg-2025" +REPO="/home/srv/files/backups/$NAME" +TARGET="/home/srv/files/content" + +REPO_MOUNT="" + +LOGDIR="/var/log/borg" +LOG_RETENTION_DAYS=90 + +HEALTHCHECK_URL="https://hc-ping.com/your-uuid-here" + +RCLONE_REMOTE="scaleway" +RCLONE_PATH="par-backup-1/$NAME" +RCLONE_MAX_DELETE=200 + +LOCKFILE="/var/lock/borg-backup.lock" + +DB_CONTAINER="mariadb" +DB_START_TIMEOUT=180 + +CREATE_TIMEOUT="6h" +PRUNE_TIMEOUT="2h" +SYNC_TIMEOUT="12h" +CHECK_TIMEOUT="4h" + +# Passphrase file: chmod 600, owned by the backup user. +# Create it with: echo 'your-strong-passphrase' > /root/.borg-passphrase +BORG_PASSPHRASE_FILE="${BORG_PASSPHRASE_FILE:-/root/.borg-passphrase}" + +REQUIRED_CMDS=(borg rclone docker curl timeout flock date find) + +# ================================================================= + +SELF="$(readlink -f "$0")" +mkdir -p "$LOGDIR" +LOGFILE="${BORG_BACKUP_LOGFILE:-$LOGDIR/backup-$(date +%Y-%m-%d-%H%M%S).log}" + +# --- Logging: re-exec ourselves through tee ------------------------------ +if [[ -z "${BORG_BACKUP_LOG_WRAPPED:-}" ]]; then + export BORG_BACKUP_LOG_WRAPPED=1 + export BORG_BACKUP_LOGFILE="$LOGFILE" + set +e + "$SELF" "$@" 2>&1 | tee -a "$LOGFILE" + exit "${PIPESTATUS[0]}" +fi + +# --- Borg environment --------------------------------------------------- +export BORG_REPO="$REPO" +export BORG_RELOCATED_REPO_ACCESS_IS_OK=no +export BORG_PASSCOMMAND="cat $BORG_PASSPHRASE_FILE" + +ARCHIVE="$(hostname -s)-$(date +%Y-%m-%dT%H-%M-%S)" +ARCHIVE_GLOB="$(hostname -s)-*" + +# ----------------------- Functions (all before the trap) ----------------- + +log() { echo "[$(date '+%F %T')] $*"; } +step() { echo; echo "=== $* ==="; } + +run_cmd() { + log "[RUN] $*" + "$@" +} + +send_healthcheck() { + local status="${1:-0}" + [[ -n "${HEALTHCHECK_URL:-}" ]] || return 0 + [[ "$HEALTHCHECK_URL" != *"your-uuid-here"* ]] || return 0 + command -v curl >/dev/null 2>&1 || return 0 + + local curl_opts=(-fsS -m 15 --retry 3 --retry-connrefused) + case "$status" in + start) curl "${curl_opts[@]}" "${HEALTHCHECK_URL}/start" >/dev/null || true ;; + 0) curl "${curl_opts[@]}" "$HEALTHCHECK_URL" >/dev/null || true ;; + *) curl "${curl_opts[@]}" "${HEALTHCHECK_URL}/${status}" \ + --data-raw "exit=$status archive=$ARCHIVE log=$LOGFILE" >/dev/null || true ;; + esac +} + +container_running() { + [[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || echo false)" == "true" ]] +} + +container_health() { + docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ + "$1" 2>/dev/null || echo unknown +} + +wait_for_container() { + local name="$1" deadline=$((SECONDS + DB_START_TIMEOUT)) health + while (( SECONDS < deadline )); do + if container_running "$name"; then + health="$(container_health "$name")" + case "$health" in + healthy|none) log "$name is up (health: $health)"; return 0 ;; + unhealthy) log "WARNING: $name reports unhealthy" ;; + esac + fi + sleep 3 + done + return 1 +} + +start_db() { + docker start "$DB_CONTAINER" >/dev/null 2>&1 || true + if wait_for_container "$DB_CONTAINER"; then + return 0 + fi + log "ERROR: $DB_CONTAINER did not come back up within ${DB_START_TIMEOUT}s" + return 1 +} + +cleanup() { + local exit_code=$1 + set +e + + # We never stop the container ourselves anymore, but if it crashed for + # an unrelated reason during the backup window, try to bring it back. + if ! container_running "$DB_CONTAINER"; then + step "Cleanup: $DB_CONTAINER is down, attempting restart" + start_db || { log "CRITICAL: $DB_CONTAINER is DOWN - manual action required" + [ "$exit_code" -eq 0 ] && exit_code=1; } + fi + + find "$LOGDIR" -maxdepth 1 -name 'backup-*.log' -type f \ + -mtime +"$LOG_RETENTION_DAYS" -delete 2>/dev/null + + if [ "$exit_code" -eq 0 ]; then + log "=== Backup completed SUCCESSFULLY (archive: $ARCHIVE) ===" + else + log "=== Backup FAILED (exit $exit_code) ===" + fi + log "Full log: $LOGFILE" + send_healthcheck "$exit_code" +} + +# Returns 0 if the repo is encrypted, 1 if not. +repo_is_encrypted() { + borg info "$REPO" 2>&1 | grep -q "Encrypted:.*Yes" +} + +preflight() { + local missing=() + for c in "${REQUIRED_CMDS[@]}"; do + command -v "$c" >/dev/null 2>&1 || missing+=("$c") + done + if (( ${#missing[@]} )); then + log "ERROR: missing commands: ${missing[*]}"; return 1 + fi + + if [[ -n "$REPO_MOUNT" ]] && ! mountpoint -q "$REPO_MOUNT"; then + log "ERROR: $REPO_MOUNT is not mounted - refusing to run" + return 1 + fi + + [[ -d "$TARGET" ]] || { log "ERROR: target missing: $TARGET"; return 1; } + [[ -n "$(ls -A "$TARGET")" ]] || { log "ERROR: target is empty: $TARGET"; return 1; } + + case "$REPO/" in + "$TARGET"/*) log "ERROR: repo lives inside target - infinite recursion"; return 1 ;; + esac + + if ! borg info --lock-wait 60 >/dev/null 2>&1; then + log "ERROR: cannot open repository $REPO" + log " (wrong path? disk not mounted? stale lock? -> borg break-lock)" + return 1 + fi + + # Warn if the repo is not encrypted. + if ! repo_is_encrypted; then + log "WARNING: Repository is NOT encrypted!" + log " Run: borg init --encryption=repokey-blake2 $REPO" + log " (after backing up any existing data)" + fi + + # Verify passphrase file exists and is readable. + if [[ ! -r "$BORG_PASSPHRASE_FILE" ]]; then + log "ERROR: passphrase file not readable: $BORG_PASSPHRASE_FILE" + log " Create it: echo 'your-passphrase' > $BORG_PASSPHRASE_FILE" + log " chmod 600 $BORG_PASSPHRASE_FILE" + return 1 + fi + + container_running "$DB_CONTAINER" \ + || { log "ERROR: $DB_CONTAINER is not running before we start"; return 1; } + + log "Preflight OK" +} + +# ----------------------- Lock, then trap -------------------------------- + +exec 9>"$LOCKFILE" +if ! flock -n 9; then + echo "ERROR: another backup is already running (lock held on $LOCKFILE)" + exit 1 +fi + +trap 'cleanup $?' EXIT +trap 'log "SIGINT received"; exit 130' INT +trap 'log "SIGTERM received"; exit 143' TERM + +# ================================================================= + +log "=== Backup started ===" +echo "Repository: $REPO" +echo "Target: $TARGET" +echo "Archive: $ARCHIVE" +echo "Log: $LOGFILE" + +send_healthcheck start + +step "Step 0: Preflight" +preflight + +# --- 1. Logical dumps (container stays up the whole time) --------------- +step "Step 1: MariaDB dumps" +DUMP_SCRIPT="${TARGET}/mariadb/dump_db.sh" +DUMP_DIR="${TARGET}/mariadb/dump" + +if [[ ! -x "$DUMP_SCRIPT" ]]; then + log "ERROR: dump script missing or not executable: $DUMP_SCRIPT" + exit 1 +fi +run_cmd "$DUMP_SCRIPT" + +if [[ -d "$DUMP_DIR" ]]; then + fresh=$(find "$DUMP_DIR" -type f -size +1k -mmin -60 | wc -l) + empty=$(find "$DUMP_DIR" -type f -size -1k -mmin -60 | wc -l) + log "Dumps: $fresh fresh non-trivial file(s), $empty suspiciously small" + (( fresh > 0 )) || { log "ERROR: no usable dumps produced"; exit 1; } + (( empty == 0 )) || log "WARNING: $empty near-empty dump file(s) - check $DUMP_DIR" +else + log "WARNING: dump directory not found: $DUMP_DIR" +fi + +# --- 2. Create the archive ----------------------------------------------- +# MariaDB stays up throughout: the dump above is already transactionally +# consistent, and the container's raw data directory carries a .nobackup +# marker (excluded here via --exclude-if-present) so its live files are +# never read by borg. +step "Step 2: Creating archive $ARCHIVE" +container_running "$DB_CONTAINER" \ + || { log "ERROR: $DB_CONTAINER is not running - refusing to archive"; exit 1; } +run_cmd timeout --signal=INT --kill-after=120s "$CREATE_TIMEOUT" \ + borg create \ + --lock-wait 600 \ + --stats \ + --list --filter=AME \ + --compression zstd,8 \ + --exclude-caches \ + --exclude-if-present .nobackup \ + --keep-exclude-tags \ + "::$ARCHIVE" \ + "$TARGET" + +# --- 3/4. Retention ------------------------------------------------------- +step "Step 3: Pruning" +run_cmd timeout "$PRUNE_TIMEOUT" \ + borg prune \ + --lock-wait 600 \ + --list \ + --glob-archives "$ARCHIVE_GLOB" \ + --keep-daily=7 \ + --keep-weekly=4 \ + --keep-monthly=6 + +step "Step 4: Compacting" +run_cmd timeout "$PRUNE_TIMEOUT" borg compact --lock-wait 600 + +# --- 5. Integrity check ------------------------------------------------- +step "Step 5: Verifying archive & repository integrity" +run_cmd borg info --lock-wait 60 "::$ARCHIVE" >/dev/null +ARCHIVE_COUNT=$(borg list --short --glob-archives "$ARCHIVE_GLOB" | wc -l) +log "Repository holds $ARCHIVE_COUNT archive(s) for $ARCHIVE_GLOB" +(( ARCHIVE_COUNT > 0 )) || { log "ERROR: repo unexpectedly empty"; exit 1; } + +# Daily: fast archive-only check. Weekly (Monday): full repo check with data verification. +if [[ "$(date +%u)" == "1" ]]; then + log "Monday — running full repository check with --verify-data (this may take a while)" + run_cmd timeout "$CHECK_TIMEOUT" borg check --verify-data --lock-wait 600 "$REPO" +else + log "Running fast archive-only integrity check" + run_cmd timeout 1h borg check --archives-only --lock-wait 600 "$REPO"::"$ARCHIVE" +fi + +# --- 6. Offsite mirror -------------------------------------------------- +step "Step 6: Syncing to ${RCLONE_REMOTE}:${RCLONE_PATH}" +run_cmd timeout "$SYNC_TIMEOUT" \ + rclone sync -v \ + --fast-list \ + --transfers=8 \ + --checkers=16 \ + --s3-no-check-bucket \ + --max-delete "$RCLONE_MAX_DELETE" \ + "$REPO" "${RCLONE_REMOTE}:${RCLONE_PATH}" diff --git a/codedb.snapshot b/codedb.snapshot new file mode 100644 index 0000000..ec3678d Binary files /dev/null and b/codedb.snapshot differ diff --git a/docs/superpowers/plans/2026-07-25-restore-and-runbook.md b/docs/superpowers/plans/2026-07-25-restore-and-runbook.md new file mode 100644 index 0000000..a2e0d8f --- /dev/null +++ b/docs/superpowers/plans/2026-07-25-restore-and-runbook.md @@ -0,0 +1,1044 @@ +# Restore Tooling & Runbook Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build `restore.sh` (full / single-database / single-file recovery) and `RUNBOOK.md` so the backup system built around `borg-backup.sh` and `dump_db.sh` has a documented, scriptable, fast recovery path. + +**Architecture:** `restore.sh` is a standalone bash script mirroring the config/logging conventions of `borg-backup.sh`. It offers three subcommands (`full`, `db `, `file `) plus `--list-archives`, all reading from the same encrypted Borg repo. Destructive modes require explicit confirmation (`--force` for `full`, typed database name or `--yes` for `db`). `RUNBOOK.md` documents one-time setup, deployment, scheduling, day-2 ops, and step-by-step recovery for all three scenarios. + +**Tech Stack:** bash, borg, docker, mysql/mariadb client, rclone (setup only, not used by restore.sh). + +## Global Constraints + +- Target directory: `/home/srv/files/content` (`TARGET`) +- Repo: `/home/srv/files/backups/borg-2025` (`NAME=borg-2025`) +- DB container name: `mariadb` +- Borg passphrase file: `/root/.borg-passphrase`, chmod 600, read via `BORG_PASSCOMMAND` +- Backup DB user `backup`, password file `/root/.mariadb-backup.pw` (used by `dump_db.sh`, unrelated to restore) +- Dump directory inside target: `mariadb/dump`, one `.sql` file per database, plus `00-users-and-grants.sql` +- MariaDB is never stopped for backups; its raw data directory is excluded from the archive via a `.nobackup` marker file — restores are always driven from the logical dump, never from raw archived DB files +- `restore.sh` must never silently overwrite live data: `full` refuses to run against a non-empty target without `--force`; `db` requires the database name to be typed back for confirmation unless `--yes` is passed +- Restore DB operations authenticate as `root` (env `MYSQL_ROOT_PASSWORD` or a password file, default `/root/.mariadb-root.pw`) since restore needs privileges the `backup` user does not have +- Logs go to `/var/log/borg/restore-*.log` (override via `RESTORE_LOGDIR` for testing), same naming convention as `borg-backup.sh` + +--- + +## File Structure + +- Create: `restore.sh` — the recovery script (repo root, alongside `borg-backup.sh`) +- Create: `RUNBOOK.md` — operational documentation (repo root) +- Create: `tests/test_restore.sh` — bash test script exercising `restore.sh` against mocked `borg`/`docker`/`mysql`/`mariadb` binaries; grows across tasks +- Create: `tests/lib/setup_mocks.sh` — shared helper that builds the mock binaries into a temp `$PATH` directory + +## Task 1: `restore.sh` scaffold — config, logging, arg parsing, usage + +**Files:** +- Create: `restore.sh` +- Create: `tests/test_restore.sh` + +**Interfaces:** +- Produces: `log(msg)`, `step(msg)`, `die(msg)` (exits 1), `run_cmd(cmd...)`, `usage()`, `parse_common_flags(args...)` (sets globals `ARCHIVE_OVERRIDE`, `FORCE`, `YES`, `DRY_RUN`, `DEST`), `main(args...)`. Config globals: `NAME`, `REPO`, `TARGET`, `DB_CONTAINER`, `DB_START_TIMEOUT`, `BORG_PASSPHRASE_FILE`, `ROOT_PASSWORD_FILE`, `DUMP_SUBDIR`, `ARCHIVE_TARGET_PATH`, `LOGDIR`, `LOGFILE`. + +- [ ] **Step 1: Write `restore.sh` scaffold** + +```bash +#!/bin/bash +# ============================================================================= +# Restore tooling for the borg-backup.sh / dump_db.sh backup system. +# Modes: +# restore.sh full [--archive NAME] [--force] [--dry-run] +# restore.sh db [--archive NAME] [--yes] [--dry-run] +# restore.sh file [--archive NAME] [--dest DIR] [--dry-run] +# restore.sh --list-archives +# ============================================================================= + +set -euo pipefail + +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH" +umask 077 + +# ========================= CONFIGURATION ========================= + +NAME="borg-2025" +REPO="/home/srv/files/backups/$NAME" +TARGET="/home/srv/files/content" +ARCHIVE_TARGET_PATH="${TARGET#/}" + +DB_CONTAINER="mariadb" +DB_START_TIMEOUT=180 + +BORG_PASSPHRASE_FILE="${BORG_PASSPHRASE_FILE:-/root/.borg-passphrase}" +ROOT_PASSWORD_FILE="${ROOT_PASSWORD_FILE:-/root/.mariadb-root.pw}" +DUMP_SUBDIR="mariadb/dump" + +LOGDIR="${RESTORE_LOGDIR:-/var/log/borg}" +mkdir -p "$LOGDIR" 2>/dev/null || LOGDIR="/tmp" +LOGFILE="$LOGDIR/restore-$(date +%Y-%m-%d-%H%M%S).log" + +export BORG_REPO="$REPO" +export BORG_PASSCOMMAND="cat $BORG_PASSPHRASE_FILE" + +# ================================================================= + +log() { + local line + line="[$(date '+%F %T')] $*" + echo "$line" + echo "$line" >> "$LOGFILE" 2>/dev/null || true +} + +step() { echo; echo "=== $* ==="; } + +die() { log "ERROR: $*"; exit 1; } + +run_cmd() { + log "[RUN] $*" + "$@" +} + +usage() { + cat <<'EOF' +Usage: + restore.sh full [--archive NAME] [--force] [--dry-run] + restore.sh db [--archive NAME] [--yes] [--dry-run] + restore.sh file [--archive NAME] [--dest DIR] [--dry-run] + restore.sh --list-archives + restore.sh -h | --help +EOF +} + +DRY_RUN=false +FORCE=false +YES=false +ARCHIVE_OVERRIDE="" +DEST="" + +parse_common_flags() { + while [[ $# -gt 0 ]]; do + case "$1" in + --archive) ARCHIVE_OVERRIDE="$2"; shift 2 ;; + --force) FORCE=true; shift ;; + --yes) YES=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + --dest) DEST="$2"; shift 2 ;; + *) die "unknown flag: $1" ;; + esac + done +} + +main() { + local cmd="${1:-}" + case "$cmd" in + "") + usage + exit 1 + ;; + -h|--help) + usage + exit 0 + ;; + --list-archives) + shift + parse_common_flags "$@" + list_archives + ;; + full) + shift + parse_common_flags "$@" + cmd_full + ;; + db) + shift + local dbname="${1:-}" + [[ -n "$dbname" ]] || die "db: missing " + shift + parse_common_flags "$@" + cmd_db "$dbname" + ;; + file) + shift + local relpath="${1:-}" + [[ -n "$relpath" ]] || die "file: missing " + shift + parse_common_flags "$@" + cmd_file "$relpath" + ;; + *) + usage + die "unknown command: $cmd" + ;; + esac +} + +main "$@" +``` + +- [ ] **Step 2: Make it executable and syntax-check** + +Run: `chmod +x restore.sh && bash -n restore.sh` +Expected: no output, exit 0. + +- [ ] **Step 3: Write `tests/test_restore.sh`** + +```bash +#!/bin/bash +set -uo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +RESTORE="$HERE/../restore.sh" +FAILURES=0 + +assert_eq() { + local expected="$1" actual="$2" msg="$3" + if [[ "$expected" != "$actual" ]]; then + echo "FAIL: $msg (expected '$expected', got '$actual')" + FAILURES=$((FAILURES + 1)) + else + echo "PASS: $msg" + fi +} + +assert_contains() { + local haystack="$1" needle="$2" msg="$3" + if [[ "$haystack" != *"$needle"* ]]; then + echo "FAIL: $msg (expected to contain '$needle', got '$haystack')" + FAILURES=$((FAILURES + 1)) + else + echo "PASS: $msg" + fi +} + +test_help_exits_zero() { + local out rc + out="$(bash "$RESTORE" -h)" + rc=$? + assert_eq "0" "$rc" "help exits 0" + assert_contains "$out" "Usage:" "help prints usage" +} + +test_no_args_exits_one() { + local rc + bash "$RESTORE" >/dev/null 2>&1 + rc=$? + assert_eq "1" "$rc" "no args exits 1" +} + +test_unknown_command_exits_one() { + local rc + bash "$RESTORE" bogus >/dev/null 2>&1 + rc=$? + assert_eq "1" "$rc" "unknown command exits 1" +} + +test_help_exits_zero +test_no_args_exits_one +test_unknown_command_exits_one + +echo "-----" +if [[ "$FAILURES" -gt 0 ]]; then + echo "$FAILURES failure(s)" + exit 1 +fi +echo "All tests passed" +``` + +- [ ] **Step 4: Run the tests** + +Run: `chmod +x tests/test_restore.sh && RESTORE_LOGDIR=/tmp/restore-test-logs bash tests/test_restore.sh` +Expected: three `PASS:` lines, then `All tests passed`, exit 0. + +- [ ] **Step 5: Commit** + +```bash +git add restore.sh tests/test_restore.sh +git commit -m "feat: add restore.sh scaffold with arg parsing and usage" +``` + +--- + +## Task 2: Archive listing and resolution + +**Files:** +- Modify: `restore.sh` (add functions before `main()`) +- Modify: `tests/test_restore.sh` +- Create: `tests/lib/setup_mocks.sh` + +**Interfaces:** +- Consumes: `die()`, `log()` from Task 1. +- Produces: `resolve_archive()` (sets global `RESOLVED_ARCHIVE`, honors `ARCHIVE_OVERRIDE`), `list_archives()`. + +- [ ] **Step 1: Write `tests/lib/setup_mocks.sh`** + +```bash +#!/bin/bash +# Builds fake borg/docker/mysql/mariadb executables into $1 so restore.sh +# can be exercised without real infrastructure. Each mock logs its +# invocation to $1/mock.log. + +setup_mock_bin() { + local dir="$1" + mkdir -p "$dir" + local mocklog="$dir/mock.log" + : > "$mocklog" + + cat > "$dir/borg" <> "$mocklog" +case "\$1" in + list) + echo "host-2026-01-01T00-00-00" + echo "host-2026-06-01T00-00-00" + ;; + extract) + shift + paths=() + for a in "\$@"; do + case "\$a" in + -*) ;; + *::*) ;; + *) paths+=("\$a") ;; + esac + done + for p in "\${paths[@]}"; do + if [[ "\$p" == *.* ]]; then + mkdir -p "\$(dirname "\$p")" + printf 'mock-content\n' > "\$p" + else + mkdir -p "\$p" + touch "\$p/RESTORED_MARKER" + fi + done + ;; + *) ;; +esac +exit 0 +EOF + + cat > "$dir/docker" <> "$mocklog" +case "\$1" in + inspect) + if [[ "\$*" == *"State.Running"* ]]; then + echo true + else + echo healthy + fi + ;; + start|stop) + exit 0 + ;; + exec) + case "\$*" in + *"command -v mariadb"*) exit 0 ;; + *) cat >/dev/null 2>&1 || true; exit 0 ;; + esac + ;; + *) ;; +esac +exit 0 +EOF + + cat > "$dir/mysql" <> "$mocklog" +cat >/dev/null 2>&1 || true +exit 0 +EOF + cp "$dir/mysql" "$dir/mariadb" + + chmod +x "$dir/borg" "$dir/docker" "$dir/mysql" "$dir/mariadb" +} +``` + +- [ ] **Step 2: Add `resolve_archive()` and `list_archives()` to `restore.sh`** + +Insert directly above the `usage()` function: + +```bash +resolve_archive() { + if [[ -n "$ARCHIVE_OVERRIDE" ]]; then + RESOLVED_ARCHIVE="$ARCHIVE_OVERRIDE" + return 0 + fi + RESOLVED_ARCHIVE="$(borg list --short --lock-wait 60 2>/dev/null | tail -n1)" + [[ -n "$RESOLVED_ARCHIVE" ]] || die "no archives found in repo $REPO" +} + +list_archives() { + borg list --lock-wait 60 +} +``` + +- [ ] **Step 3: Syntax-check** + +Run: `bash -n restore.sh` +Expected: no output, exit 0. + +- [ ] **Step 4: Add a test for `--list-archives`** + +Append to `tests/test_restore.sh`, just above the `test_help_exits_zero` calls at the bottom: + +```bash +source "$HERE/lib/setup_mocks.sh" + +test_list_archives() { + local mockdir out + mockdir="$(mktemp -d)" + setup_mock_bin "$mockdir" + out="$(PATH="$mockdir:$PATH" bash "$RESTORE" --list-archives)" + assert_contains "$out" "host-2026-01-01T00-00-00" "list-archives shows first archive" + assert_contains "$out" "host-2026-06-01T00-00-00" "list-archives shows latest archive" + rm -rf "$mockdir" +} +``` + +And add the call `test_list_archives` next to the other `test_*` invocations (before the `echo "-----"` line). + +- [ ] **Step 5: Run the tests** + +Run: `RESTORE_LOGDIR=/tmp/restore-test-logs bash tests/test_restore.sh` +Expected: all `PASS:` lines including the two new ones, `All tests passed`, exit 0. + +- [ ] **Step 6: Commit** + +```bash +git add restore.sh tests/test_restore.sh tests/lib/setup_mocks.sh +git commit -m "feat: add archive listing and resolution to restore.sh" +``` + +--- + +## Task 3: `file` mode (non-destructive single path restore) + +**Files:** +- Modify: `restore.sh` +- Modify: `tests/test_restore.sh` + +**Interfaces:** +- Consumes: `resolve_archive()`, `RESOLVED_ARCHIVE`, `ARCHIVE_TARGET_PATH`, `REPO`, `run_cmd()`, `log()`, `step()`, global `DRY_RUN`/`DEST` from Tasks 1-2. +- Produces: `extract_path(rel_path, dest_dir, archive)` (echoes the final absolute path of the extracted item), `cmd_file(rel_path)`. + +- [ ] **Step 1: Add `extract_path()` and `cmd_file()` to `restore.sh`** + +Insert directly above `usage()`, after `list_archives()`: + +```bash +extract_path() { + local rel_path="$1" dest_dir="$2" archive="$3" + mkdir -p "$dest_dir" + ( cd "$dest_dir" && run_cmd borg extract --lock-wait 600 "${REPO}::${archive}" "${ARCHIVE_TARGET_PATH}/${rel_path}" ) + echo "${dest_dir%/}/${ARCHIVE_TARGET_PATH}/${rel_path}" +} + +cmd_file() { + local rel_path="$1" dest + resolve_archive + dest="${DEST:-/tmp/restore-file-$$}" + step "Restoring '$rel_path' from archive $RESOLVED_ARCHIVE into $dest" + if [[ "$DRY_RUN" == true ]]; then + log "[DRY-RUN] would extract ${ARCHIVE_TARGET_PATH}/${rel_path} from ${REPO}::${RESOLVED_ARCHIVE} into $dest" + return 0 + fi + local final + final="$(extract_path "$rel_path" "$dest" "$RESOLVED_ARCHIVE")" + log "Restored file available at: $final" +} +``` + +- [ ] **Step 2: Syntax-check** + +Run: `bash -n restore.sh` +Expected: no output, exit 0. + +- [ ] **Step 3: Add tests for `file` mode** + +Append to `tests/test_restore.sh` (near `test_list_archives`): + +```bash +test_file_restore_extracts_to_dest() { + local mockdir dest out final + mockdir="$(mktemp -d)" + dest="$(mktemp -d)" + setup_mock_bin "$mockdir" + out="$(PATH="$mockdir:$PATH" bash "$RESTORE" file photos/img.jpg --dest "$dest")" + final="$dest/home/srv/files/content/photos/img.jpg" + assert_contains "$out" "Restored file available at: $final" "file mode reports final path" + if [[ -f "$final" ]]; then + echo "PASS: extracted file exists on disk" + else + echo "FAIL: extracted file missing at $final" + FAILURES=$((FAILURES + 1)) + fi + rm -rf "$mockdir" "$dest" +} + +test_file_restore_dry_run_makes_no_borg_call() { + local mockdir dest + mockdir="$(mktemp -d)" + dest="$(mktemp -d)" + setup_mock_bin "$mockdir" + PATH="$mockdir:$PATH" bash "$RESTORE" file photos/img.jpg --dest "$dest" --dry-run >/dev/null + if [[ -s "$mockdir/mock.log" ]] && grep -q "^borg extract" "$mockdir/mock.log"; then + echo "FAIL: dry-run invoked borg extract" + FAILURES=$((FAILURES + 1)) + else + echo "PASS: dry-run made no borg extract call" + fi + rm -rf "$mockdir" "$dest" +} +``` + +Add `test_file_restore_extracts_to_dest` and `test_file_restore_dry_run_makes_no_borg_call` to the list of test invocations. + +- [ ] **Step 4: Run the tests** + +Run: `RESTORE_LOGDIR=/tmp/restore-test-logs bash tests/test_restore.sh` +Expected: all tests pass, including the two new ones. + +- [ ] **Step 5: Commit** + +```bash +git add restore.sh tests/test_restore.sh +git commit -m "feat: add non-destructive file-restore mode to restore.sh" +``` + +--- + +## Task 4: `db` mode (single-database restore with confirmation) + +**Files:** +- Modify: `restore.sh` +- Modify: `tests/test_restore.sh` + +**Interfaces:** +- Consumes: `resolve_archive()`, `RESOLVED_ARCHIVE`, `extract_path()`, `DUMP_SUBDIR`, `DB_CONTAINER`, `ROOT_PASSWORD_FILE`, global `YES`/`DRY_RUN` from Tasks 1-3. +- Produces: `detect_client()` (sets `CLIENT_BIN`), `get_root_creds()` (sets `DB_PASS`), `confirm_or_abort(dbname)`, `restore_single_db(sqlfile, dbname)`, `cmd_db(dbname)`. + +- [ ] **Step 1: Add the db-mode functions to `restore.sh`** + +Insert directly above `usage()`, after `cmd_file()`: + +```bash +detect_client() { + if docker exec "$DB_CONTAINER" sh -c 'command -v mariadb' >/dev/null 2>&1; then + CLIENT_BIN="mariadb" + else + CLIENT_BIN="mysql" + fi +} + +get_root_creds() { + if [[ -n "${MYSQL_ROOT_PASSWORD:-}" ]]; then + DB_PASS="$MYSQL_ROOT_PASSWORD" + elif [[ -r "$ROOT_PASSWORD_FILE" ]]; then + DB_PASS="$(< "$ROOT_PASSWORD_FILE")" + else + die "no root DB password: set MYSQL_ROOT_PASSWORD or create $ROOT_PASSWORD_FILE (chmod 600)" + fi +} + +confirm_or_abort() { + local dbname="$1" typed + [[ "$YES" == true ]] && return 0 + echo "This will DROP/overwrite database '$dbname'. Type the database name to confirm:" + read -r typed + [[ "$typed" == "$dbname" ]] || die "confirmation did not match '$dbname' - aborting" +} + +restore_single_db() { + local sqlfile="$1" dbname="$2" + [[ -s "$sqlfile" ]] || die "dump file missing or empty: $sqlfile" + run_cmd docker exec -e MYSQL_PWD="$DB_PASS" "$DB_CONTAINER" \ + "$CLIENT_BIN" -u root -e "CREATE DATABASE IF NOT EXISTS \`$dbname\`;" + log "[RUN] docker exec -i ... $CLIENT_BIN -u root $dbname < $sqlfile" + docker exec -i -e MYSQL_PWD="$DB_PASS" "$DB_CONTAINER" \ + "$CLIENT_BIN" -u root "$dbname" < "$sqlfile" +} + +cmd_db() { + local dbname="$1" dumpfile dest + resolve_archive + step "Restoring database '$dbname' from archive $RESOLVED_ARCHIVE" + if [[ "$DRY_RUN" == true ]]; then + log "[DRY-RUN] would extract ${ARCHIVE_TARGET_PATH}/${DUMP_SUBDIR}/${dbname}.sql from ${REPO}::${RESOLVED_ARCHIVE}" + log "[DRY-RUN] would DROP/recreate database '$dbname' and import the dump using root credentials" + return 0 + fi + confirm_or_abort "$dbname" + dest="/tmp/restore-db-$$" + dumpfile="$(extract_path "${DUMP_SUBDIR}/${dbname}.sql" "$dest" "$RESOLVED_ARCHIVE")" + detect_client + get_root_creds + restore_single_db "$dumpfile" "$dbname" + log "Database '$dbname' restored from $dumpfile" + rm -rf "$dest" +} +``` + +- [ ] **Step 2: Syntax-check** + +Run: `bash -n restore.sh` +Expected: no output, exit 0. + +- [ ] **Step 3: Add tests for `db` mode** + +Append to `tests/test_restore.sh`: + +```bash +test_db_restore_with_yes_runs_full_sequence() { + local mockdir out + mockdir="$(mktemp -d)" + setup_mock_bin "$mockdir" + out="$(PATH="$mockdir:$PATH" ROOT_PASSWORD_FILE="$mockdir/rootpw" bash -c ' + echo "rootpass" > "'"$mockdir"'/rootpw" + bash "'"$RESTORE"'" db shopdb --yes + ')" + assert_contains "$out" "Database 'shopdb' restored from" "db mode reports success" + if grep -q "CREATE DATABASE IF NOT EXISTS" "$mockdir/mock.log"; then + echo "PASS: db mode issued CREATE DATABASE" + else + echo "FAIL: db mode did not issue CREATE DATABASE" + FAILURES=$((FAILURES + 1)) + fi + rm -rf "$mockdir" +} + +test_db_restore_dry_run_skips_confirmation_and_calls() { + local mockdir out + mockdir="$(mktemp -d)" + setup_mock_bin "$mockdir" + out="$(PATH="$mockdir:$PATH" bash "$RESTORE" db shopdb --dry-run /dev/null 2>&1 + rc=$? + set -e + assert_eq "1" "$rc" "db mode aborts on mismatched confirmation" + rm -rf "$mockdir" +} +``` + +Add all three test names to the invocation list. + +- [ ] **Step 4: Run the tests** + +Run: `RESTORE_LOGDIR=/tmp/restore-test-logs bash tests/test_restore.sh` +Expected: all tests pass, including the three new ones. + +- [ ] **Step 5: Commit** + +```bash +git add restore.sh tests/test_restore.sh +git commit -m "feat: add single-database restore mode to restore.sh" +``` + +--- + +## Task 5: `full` mode (disaster recovery) and shared container helpers + +**Files:** +- Modify: `restore.sh` +- Modify: `tests/test_restore.sh` + +**Interfaces:** +- Consumes: `resolve_archive()`, `detect_client()`, `get_root_creds()`, `restore_single_db()`, `ARCHIVE_TARGET_PATH`, `TARGET`, `DUMP_SUBDIR`, `DB_CONTAINER`, `DB_START_TIMEOUT`, global `FORCE`/`DRY_RUN` from Tasks 1-4. +- Produces: `container_running(name)`, `container_health(name)`, `wait_for_container(name)`, `start_db()`, `cmd_full()`. + +- [ ] **Step 1: Add container helpers and `cmd_full()` to `restore.sh`** + +Insert directly above `usage()`, after `cmd_db()`: + +```bash +container_running() { + [[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || echo false)" == "true" ]] +} + +container_health() { + docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ + "$1" 2>/dev/null || echo unknown +} + +wait_for_container() { + local name="$1" deadline=$((SECONDS + DB_START_TIMEOUT)) health + while (( SECONDS < deadline )); do + if container_running "$name"; then + health="$(container_health "$name")" + case "$health" in + healthy|none) log "$name is up (health: $health)"; return 0 ;; + unhealthy) log "WARNING: $name reports unhealthy" ;; + esac + fi + sleep 3 + done + return 1 +} + +start_db() { + docker start "$DB_CONTAINER" >/dev/null 2>&1 || true + wait_for_container "$DB_CONTAINER" +} + +cmd_full() { + local staging dumpdir f dbname + resolve_archive + step "Full restore from archive $RESOLVED_ARCHIVE into $TARGET" + + if [[ -d "$TARGET" ]] && [[ -n "$(ls -A "$TARGET" 2>/dev/null)" ]] && [[ "$FORCE" != true ]]; then + die "$TARGET is not empty - pass --force to overwrite (existing data will be replaced)" + fi + + if [[ "$DRY_RUN" == true ]]; then + log "[DRY-RUN] would extract full ${ARCHIVE_TARGET_PATH} tree from ${REPO}::${RESOLVED_ARCHIVE} into $TARGET" + log "[DRY-RUN] would restore every *.sql dump under ${DUMP_SUBDIR}/ using root credentials" + log "[DRY-RUN] would start $DB_CONTAINER and wait for it to become healthy" + return 0 + fi + + staging="/tmp/restore-full-$$" + mkdir -p "$staging" + ( cd "$staging" && run_cmd borg extract --lock-wait 600 "${REPO}::${RESOLVED_ARCHIVE}" "${ARCHIVE_TARGET_PATH}" ) + + mkdir -p "$(dirname "$TARGET")" + rm -rf "${TARGET:?}"/* 2>/dev/null || true + mkdir -p "$TARGET" + run_cmd cp -a "${staging}/${ARCHIVE_TARGET_PATH}/." "$TARGET/" + rm -rf "$staging" + + detect_client + get_root_creds + + dumpdir="${TARGET}/${DUMP_SUBDIR}" + [[ -d "$dumpdir" ]] || die "no dump directory found after extract: $dumpdir" + + if [[ -f "${dumpdir}/00-users-and-grants.sql" ]]; then + step "Restoring users and grants" + docker exec -i -e MYSQL_PWD="$DB_PASS" "$DB_CONTAINER" \ + "$CLIENT_BIN" -u root < "${dumpdir}/00-users-and-grants.sql" + fi + + for f in "$dumpdir"/*.sql; do + [[ -e "$f" ]] || continue + dbname="$(basename "$f" .sql)" + [[ "$dbname" == "00-users-and-grants" ]] && continue + restore_single_db "$f" "$dbname" + done + + step "Starting $DB_CONTAINER" + start_db || die "CRITICAL: $DB_CONTAINER did not come up after restore" + + log "Full restore complete from archive $RESOLVED_ARCHIVE" +} +``` + +- [ ] **Step 2: Syntax-check** + +Run: `bash -n restore.sh` +Expected: no output, exit 0. + +- [ ] **Step 3: Add tests for `full` mode** + +Append to `tests/test_restore.sh`: + +```bash +test_full_restore_refuses_nonempty_target_without_force() { + local mockdir target rc + mockdir="$(mktemp -d)" + target="$(mktemp -d)" + touch "$target/existing-file" + setup_mock_bin "$mockdir" + set +e + PATH="$mockdir:$PATH" TARGET_OVERRIDE=1 bash -c ' + sed "s#^TARGET=\"/home/srv/files/content\"#TARGET=\"'"$target"'\"#; s#^ARCHIVE_TARGET_PATH=.*#ARCHIVE_TARGET_PATH=\"\${TARGET#/}\"#" "'"$RESTORE"'" > "'"$mockdir"'/restore_patched.sh" + bash "'"$mockdir"'/restore_patched.sh" full + ' >/dev/null 2>&1 + rc=$? + set -e + assert_eq "1" "$rc" "full mode refuses non-empty target without --force" + rm -rf "$mockdir" "$target" +} + +test_full_restore_dry_run_makes_no_calls() { + local mockdir target out + mockdir="$(mktemp -d)" + target="$(mktemp -d)" + setup_mock_bin "$mockdir" + out="$(PATH="$mockdir:$PATH" bash -c ' + sed "s#^TARGET=\"/home/srv/files/content\"#TARGET=\"'"$target"'\"#; s#^ARCHIVE_TARGET_PATH=.*#ARCHIVE_TARGET_PATH=\"\${TARGET#/}\"#" "'"$RESTORE"'" > "'"$mockdir"'/restore_patched.sh" + bash "'"$mockdir"'/restore_patched.sh" full --dry-run + ')" + assert_contains "$out" "DRY-RUN" "full dry-run prints DRY-RUN plan" + if [[ -s "$mockdir/mock.log" ]]; then + echo "FAIL: full dry-run invoked a mock binary" + FAILURES=$((FAILURES + 1)) + else + echo "PASS: full dry-run made no external calls" + fi + rm -rf "$mockdir" "$target" +} +``` + +Add `test_full_restore_refuses_nonempty_target_without_force` and `test_full_restore_dry_run_makes_no_calls` to the invocation list. + +- [ ] **Step 4: Run the tests** + +Run: `RESTORE_LOGDIR=/tmp/restore-test-logs bash tests/test_restore.sh` +Expected: all tests pass, including the two new ones. + +- [ ] **Step 5: Commit** + +```bash +git add restore.sh tests/test_restore.sh +git commit -m "feat: add full disaster-recovery mode to restore.sh" +``` + +--- + +## Task 6: `RUNBOOK.md` + +**Files:** +- Create: `RUNBOOK.md` + +**Interfaces:** +- Consumes: final `restore.sh` CLI surface from Tasks 1-5 (`full`, `db `, `file `, `--list-archives`, flags `--archive`/`--force`/`--yes`/`--dry-run`/`--dest`), and `borg-backup.sh`'s config (`NAME=borg-2025`, `REPO=/home/srv/files/backups/borg-2025`, `TARGET=/home/srv/files/content`, `BORG_PASSPHRASE_FILE=/root/.borg-passphrase`). + +- [ ] **Step 1: Write `RUNBOOK.md`** + +```markdown +# Backup & Recovery Runbook + +Covers `borg-backup.sh` (daily backup), `dump_db.sh` (MariaDB logical +dumps, invoked by the backup script), and `restore.sh` (recovery). + +## 1. One-Time Setup + +Run once, by hand, on the server: + +1. Initialize the encrypted Borg repo: + ```bash + mkdir -p /home/srv/files/backups + borg init --encryption=repokey-blake2 /home/srv/files/backups/borg-2025 + ``` +2. Create the passphrase file used by both backup and restore: + ```bash + echo 'your-strong-passphrase' > /root/.borg-passphrase + chmod 600 /root/.borg-passphrase + ``` +3. Create the MariaDB `backup` user used by `dump_db.sh` (read-only, no + stop/lock of the server required thanks to `--single-transaction`): + ```sql + CREATE USER 'backup'@'%' IDENTIFIED BY 'a-strong-password'; + GRANT SELECT, LOCK TABLES, SHOW VIEW, TRIGGER, PROCESS, RELOAD ON *.* TO 'backup'@'%'; + ``` + ```bash + echo 'a-strong-password' > /root/.mariadb-backup.pw + chmod 600 /root/.mariadb-backup.pw + ``` +4. Create the root password file used only by `restore.sh` (restore needs + CREATE/DROP privileges the `backup` user does not have): + ```bash + echo 'the-mariadb-root-password' > /root/.mariadb-root.pw + chmod 600 /root/.mariadb-root.pw + ``` +5. Exclude MariaDB's raw data directory from the archive. Find the + directory bind-mounted into the container as its datadir and drop a + marker file in it: + ```bash + touch /home/srv/files/content/mariadb/data/.nobackup + ``` + This is what allows backups to run with the container up: only the + logical dump under `mariadb/dump/` is ever archived or restored from. +6. Configure the `scaleway` rclone remote: + ```bash + rclone config + # create a remote named "scaleway", type S3, matching your Scaleway + # Object Storage credentials and region + ``` +7. Set a real healthcheck URL in `borg-backup.sh` (`HEALTHCHECK_URL=`), for + example from https://healthchecks.io. + +## 2. Deploying the Scripts + +Copy `borg-backup.sh`, `dump_db.sh`, and `restore.sh` to the server (e.g. +`/opt/backup-agent/`), and `dump_db.sh` additionally to +`/home/srv/files/content/mariadb/dump_db.sh` (this exact path is what +`borg-backup.sh` invokes). Make all three executable: + +```bash +chmod +x /opt/backup-agent/borg-backup.sh /opt/backup-agent/restore.sh +chmod +x /home/srv/files/content/mariadb/dump_db.sh +``` + +## 3. Scheduling + +Add a cron entry to run the backup daily, off-peak: + +``` +# /etc/cron.d/borg-backup +30 2 * * * root /opt/backup-agent/borg-backup.sh >> /var/log/borg/cron.log 2>&1 +``` + +Check the last run: + +```bash +ls -lt /var/log/borg/backup-*.log | head -1 # latest log file +tail -50 /var/log/borg/backup-*.log # inspect it +``` + +Or watch the healthcheck dashboard configured in step 1.7 — a missed or +failed run pages/alerts there. + +## 4. Day-2 Operations + +List archives: + +```bash +./restore.sh --list-archives +``` + +Check repo size and health: + +```bash +BORG_PASSCOMMAND="cat /root/.borg-passphrase" borg info /home/srv/files/backups/borg-2025 +``` + +Rotate the passphrase (creates a new key, re-encrypts nothing — old +archives still need the old passphrase to read, so keep both until fully +migrated): + +```bash +BORG_PASSCOMMAND="cat /root/.borg-passphrase" borg key change-passphrase /home/srv/files/backups/borg-2025 +``` + +Stale lockfile (backup or restore aborted mid-run and left the repo +locked): + +```bash +BORG_PASSCOMMAND="cat /root/.borg-passphrase" borg break-lock /home/srv/files/backups/borg-2025 +``` + +## 5. Recovery Procedures + +All `restore.sh` commands accept `--dry-run` to preview exactly what would +happen without touching anything, and `--archive NAME` to target a +specific archive instead of the latest (see archive names via +`--list-archives`). + +### 5.1 Full disaster recovery (new or wiped server) + +Use when the whole server/container is gone and you're rebuilding from +scratch. + +```bash +# 1. Reinstall borg, docker, and the mariadb container image/compose file +# (not covered by restore.sh - this is infra provisioning). +# 2. Restore the passphrase file (from your password manager / secondary +# backup - it is NOT stored in the repo it protects) to +# /root/.borg-passphrase, and the root DB password to +# /root/.mariadb-root.pw. +# 3. Preview: +./restore.sh full --dry-run +# 4. Run for real (refuses if /home/srv/files/content is non-empty): +./restore.sh full --force +``` + +This extracts the full content tree from the archive, restores every +database dump (users/grants first), and starts the `mariadb` container, +waiting for it to report healthy. + +**Verify afterward:** +- `docker ps` shows `mariadb` running and healthy. +- The application responds normally. +- Spot-check row counts on a couple of tables against what you'd expect. + +### 5.2 Single database restore + +Use when one database got corrupted or someone ran a bad migration/query +against it — this **drops and recreates** that database. + +```bash +./restore.sh db shopdb --dry-run # preview +./restore.sh db shopdb # prompts: type "shopdb" to confirm +``` + +Non-interactive (e.g. scripted from a monitoring alert): add `--yes` to +skip the typed confirmation. + +**Verify afterward:** connect to the database and check the tables/row +counts you expect. + +### 5.3 Single file/directory restore + +Use for accidental deletion of a file, or to inspect an old version — this +never touches the running database or container. + +```bash +./restore.sh file path/relative/to/content/some-file.txt --dest /tmp/recovered +``` + +The final location of the recovered item is printed at the end (it lands +under `/tmp/recovered/home/srv/files/content/...` — Borg preserves the +absolute path it was archived with). + +## 6. Restore Drill Cadence + +Quarterly, run a real `full` restore into a scratch directory (not +`/home/srv/files/content`) to confirm backups are actually usable: + +```bash +mkdir -p /tmp/restore-drill +BORG_PASSCOMMAND="cat /root/.borg-passphrase" \ + borg extract --lock-wait 600 /home/srv/files/backups/borg-2025::$(./restore.sh --list-archives | tail -1) \ + --destination /tmp/restore-drill # (or adapt restore.sh's TARGET for a one-off dry run into scratch) +``` + +Confirm the dump files under `mariadb/dump/` are present, non-empty, and +importable (`mysql -u root -p < mariadb/dump/somedb.sql` against a +throwaway MariaDB container). Log the drill date and outcome somewhere +durable (e.g. the healthcheck dashboard's notes, or a team wiki page). +``` + +- [ ] **Step 2: Cross-check the runbook against the script's actual CLI** + +Run: `grep -oE 'restore\.sh [a-z-]+' RUNBOOK.md | sort -u` +Expected output includes `restore.sh --list-archives`, `restore.sh db`, +`restore.sh file`, `restore.sh full` — confirming every implemented +subcommand is documented. Run `./restore.sh -h` and confirm no flag or +subcommand appears there that's missing from `RUNBOOK.md`. + +- [ ] **Step 3: Commit** + +```bash +git add RUNBOOK.md +git commit -m "docs: add operational runbook for backup and recovery" +``` + +--- + +## Self-Review Notes + +- **Spec coverage:** one-time setup (Task 6 §1), deployment (§2), cron + scheduling (§3), day-2 ops (§4), all three recovery scenarios (§5.1-5.3), + quarterly drill (§6) — all present. `restore.sh` covers `full`/`db`/`file` + with `--force`/typed-confirmation/`--yes` safety guards and `--dry-run` + for every mode, per the approved design. +- **No changes** were made to `dump_db.sh`; `borg-backup.sh`'s only change + (removing the DB stop/start) was already implemented and committed + before this plan. +- **Type/name consistency:** `RESOLVED_ARCHIVE`, `CLIENT_BIN`, `DB_PASS`, + `ARCHIVE_TARGET_PATH`, `DUMP_SUBDIR` are defined once (Tasks 1-2) and + reused with the same names through Task 5; `restore_single_db()` is + defined in Task 4 and reused unchanged by `cmd_full()` in Task 5. diff --git a/docs/superpowers/specs/2026-07-25-backup-recovery-system-design.md b/docs/superpowers/specs/2026-07-25-backup-recovery-system-design.md new file mode 100644 index 0000000..55f9b22 --- /dev/null +++ b/docs/superpowers/specs/2026-07-25-backup-recovery-system-design.md @@ -0,0 +1,162 @@ +# Backup & Recovery System Design + +Date: 2026-07-25 + +## Goal + +Reliable, automated backup of `/home/srv/files/content` (including MariaDB +running in Docker) to a local Borg repo + offsite Scaleway S3 mirror, plus a +documented, scriptable recovery path so downtime is short and predictable. + +## Current State + +- `old/borg.sh`, `old/maria.sh` — scripts currently live on the server. +- `borg-backup.sh`, `dump_db.sh` (repo root) — improved replacements, not yet + deployed. Add: encrypted repo (`repokey-blake2` via `BORG_PASSCOMMAND`), + flock-based locking, preflight checks, container health polling, atomic + dump staging/swap, non-transactional table warnings, integrity checks + (`borg check`, daily archive-only / weekly `--verify-data`), retry/timeout + wrapping, log rotation. +- No restore tooling and no scheduling exist yet. + +`dump_db.sh` is kept as-is. `borg-backup.sh` had one functional change made +during this design: it no longer stops the MariaDB container. `dump_db.sh` +already produces a transactionally-consistent dump via +`--single-transaction`, so a live logical dump is safe without stopping +anything (this only covers InnoDB tables — the script already warns if it +finds non-InnoDB tables). The remaining risk was borg also archiving +MariaDB's raw on-disk data directory while it's being written to; that's +resolved by excluding it from the archive entirely (drop a `.nobackup` +marker file in it — `borg-backup.sh` already passes +`--exclude-if-present .nobackup`), so only the logical dump ever gets +backed up, and restore only ever needs the dump anyway. Net effect: daily +backups run with **zero MariaDB downtime**. This exclusion step needs to be +added to the one-time setup, documented in the runbook. + +## Architecture + +``` +cron (daily) → borg-backup.sh + ├─ dump_db.sh (mysqldump per-DB, atomic staging swap; + │ container stays up throughout) + ├─ borg create (encrypted, zstd) → local repo + │ (raw MariaDB data dir excluded via .nobackup; + │ only the logical dump is archived) + ├─ borg prune + compact + ├─ borg check (daily fast / weekly full --verify-data) + └─ rclone sync → Scaleway S3 (offsite mirror) + +restore.sh (manual, run on demand during an incident) + mode=full : fresh/broken server → extract full content dir from archive + → restore all DBs → start container + mode=db : single database → restore one .sql from a chosen archive + mode=file : single file/dir → borg extract path, no DB involved +``` + +## Components + +All at repo root, deployed to the server alongside the existing scripts. + +### `borg-backup.sh` (existing, one change: no longer stops MariaDB) + +Orchestrates the daily backup: dump → archive → prune → compact → check → +offsite sync. The container is never stopped — the dump step alone +produces a consistent backup, and the raw data directory is excluded from +the archive. Already has locking, timeouts, health-check pings, and +cleanup-on-exit logic (cleanup now only intervenes if the container +happens to be down for an unrelated reason, restarting it defensively). + +### `dump_db.sh` (existing, unchanged) + +Runs inside `borg-backup.sh` step 1. Dumps each non-system database plus +users/grants to a staging dir, verifies each dump file, then atomically +swaps staging into place. Deployed at +`/home/srv/files/content/mariadb/dump_db.sh`. + +### `restore.sh` (new) + +``` +restore.sh full [--archive NAME] [--force] +restore.sh db [--archive NAME] +restore.sh file [--archive NAME] [--dest DIR] +restore.sh --dry-run +restore.sh --list-archives +``` + +- Defaults `--archive` to the most recent archive in the repo. +- `full`: refuses to run if `$TARGET` is non-empty unless `--force` is + passed (protects against overwriting a working system by accident). + Extracts the full archive over `$TARGET`, then restores every `.sql` + dump found under `mariadb/dump/` using root DB credentials, then starts + the `mariadb` container and waits for it to become healthy. +- `db `: extracts only `mariadb/dump/.sql` from the archive, + prompts for the database name to be typed again as confirmation (it's + destructive — drops/recreates the DB), restores it with root credentials. +- `file `: `borg extract` of a single path from the archive into + `--dest` (default: a scratch dir under `/tmp`), no DB or container + involvement — safe, non-destructive. +- `--dry-run`: prints exactly what would run (archive chosen, paths + extracted, commands) without touching anything. +- Same logging conventions as `borg-backup.sh` (`/var/log/borg/restore-*.log`), + non-zero exit on any failure, no partial/silent restores — a failed step + aborts before touching the "good" copy where possible. +- Uses the same `BORG_PASSCOMMAND` / passphrase file as backups. +- Restore DB operations authenticate as `root` via `MYSQL_ROOT_PASSWORD` or + a root password file (mirrors how `dump_db.sh` resolves credentials), + since restore needs full privileges (CREATE/DROP) that the limited + `backup` user used for dumping does not have. + +### `RUNBOOK.md` (new) + +Single operational document covering: + +1. **One-time setup** (run once, by hand, not scripted): `borg init + --encryption=repokey-blake2`, create `/root/.borg-passphrase` + (chmod 600), create the `backup` MariaDB user with the grants + `dump_db.sh` needs (SELECT, LOCK TABLES, SHOW VIEW, TRIGGER, PROCESS, + RELOAD) and its password file `/root/.mariadb-backup.pw`, drop a + `.nobackup` marker file into MariaDB's Docker volume data directory + (excludes raw DB files from the archive — only the logical dump under + `mariadb/dump/` gets backed up), configure `rclone` for the `scaleway` + remote, set the real healthcheck URL. +2. **Deploying the scripts**: where each file goes, permissions, + `chmod +x`. +3. **Scheduling**: cron entry (daily, off-peak hours) invoking + `borg-backup.sh`, plus how to check the last run (`/var/log/borg/`, + healthcheck dashboard). +4. **Day-2 operations**: listing archives, checking repo size/health, + rotating the passphrase, what to do if the lockfile is stale. +5. **Recovery procedures** — one clearly-numbered walkthrough per + scenario (full disaster, single DB, single file), written so someone + unfamiliar with the internals can follow it step-by-step under + pressure, including exact `restore.sh` invocations and what to verify + afterward (container healthy, app responds, row counts sane). +6. **Restore drill cadence**: recommend a quarterly test restore into a + scratch location to confirm backups are actually usable. + +## Error Handling & Safety + +- `restore.sh` never overwrites live data without an explicit `--force` + (full) or typed confirmation (db). +- All destructive steps are logged before execution. +- `--dry-run` available for every mode. +- Failures abort immediately (`set -euo pipefail`), matching the existing + scripts' style; partial state is called out explicitly in the log. + +## Testing + +No automated test framework — these are ops scripts against real +infrastructure. Verification consists of: + +- `restore.sh --dry-run` runs against the real repo to sanity-check + archive selection and command construction. +- Quarterly real restore drill into a scratch directory (documented in + the runbook), confirming dumps are complete and importable. + +## Out of Scope + +- No changes to `dump_db.sh`, and no changes to `borg-backup.sh` beyond + removing the DB stop/start (see Components above). +- No monitoring/alerting beyond the existing healthcheck ping. +- No multi-server / multi-target generalization — this is specific to + `/home/srv/files/content`. diff --git a/dump_db.sh b/dump_db.sh new file mode 100644 index 0000000..8cd6f82 --- /dev/null +++ b/dump_db.sh @@ -0,0 +1,202 @@ +#!/bin/bash +# ============================================================================= +# Consistent per-database MariaDB/MySQL dumps from a Docker container. +# Writes to a staging dir, verifies, then swaps atomically. +# ============================================================================= +# CHANGES vs. original: +# • Uses mysqldump --users (MySQL 8.0.31+) instead of MariaDB-only +# --system=users for dumping user accounts as CREATE USER / GRANT. +# • Falls back to full mysql schema dump on older MySQL / MariaDB. +# • Works with both mysql:8.2.0 and MariaDB containers. +# ============================================================================= + +set -euo pipefail +umask 077 + +# ========================= CONFIGURATION ========================= + +CONTAINER="${DOCKER_CONTAINER_NAME:-mariadb}" + +PASSWORD_FILE="${PASSWORD_FILE:-/root/.mariadb-backup.pw}" +DB_USER="${DB_USER:-backup}" + +SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "$0")")" && pwd)" +DUMP_DIR="${DUMP_DIR:-dump}" +[[ "$DUMP_DIR" == /* ]] || DUMP_DIR="$SCRIPT_DIR/$DUMP_DIR" +STAGING="${DUMP_DIR}.staging" + +DEDUP_FRIENDLY="${DEDUP_FRIENDLY:-1}" + +EXCLUDED_SCHEMAS="'information_schema','mysql','performance_schema','sys'" + +# ================================================================= + +die() { echo "ERROR: $*" >&2; exit 1; } +warn() { echo "WARNING: $*" >&2; } + +cleanup() { rm -rf "$STAGING"; } +trap cleanup EXIT + +# --- Preconditions ------------------------------------------------------ + +command -v docker >/dev/null 2>&1 || die "docker not found in PATH" + +[[ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || echo false)" == "true" ]] \ + || die "container '$CONTAINER' is not running" + +if [[ -n "${MYSQL_ROOT_PASSWORD:-}" ]]; then + DB_PASS="$MYSQL_ROOT_PASSWORD" +elif [[ -r "$PASSWORD_FILE" ]]; then + DB_PASS="$(< "$PASSWORD_FILE")" +else + die "no password: set MYSQL_ROOT_PASSWORD or create $PASSWORD_FILE (chmod 600)" +fi + +# Detect client binaries (MariaDB 11+ renamed them; MySQL keeps mysql/*). +if docker exec "$CONTAINER" sh -c 'command -v mariadb-dump' >/dev/null 2>&1; then + CLIENT=mariadb; DUMPER=mariadb-dump + CLIENT_BIN="mariadb" +else + CLIENT=mysql; DUMPER=mysqldump + CLIENT_BIN="mysql" +fi + +# MYSQL_PWD keeps the password out of the process list. +db_exec() { docker exec -e MYSQL_PWD="$DB_PASS" "$CONTAINER" "$@"; } +sql() { db_exec "$CLIENT_BIN" -u "$DB_USER" -N -B -e "$1"; } + +DUMP_OPTS=( + -u "$DB_USER" + --single-transaction + --quick + --routines --triggers --events + --hex-blob + --max-allowed-packet=1G + --default-character-set=utf8mb4 + --skip-dump-date +) +[[ "$DEDUP_FRIENDLY" == "1" ]] && DUMP_OPTS+=(--skip-extended-insert) + +# --- Warn about non-transactional tables -------------------------------- + +NON_TXN="$(sql " + SELECT CONCAT(table_schema,'.',table_name,' [',engine,']') + FROM information_schema.tables + WHERE engine IS NOT NULL + AND engine NOT IN ('InnoDB','MEMORY','SEQUENCE') + AND table_schema NOT IN ($EXCLUDED_SCHEMAS);" || true)" + +if [[ -n "$NON_TXN" ]]; then + warn "non-transactional tables found - NOT covered by --single-transaction:" + echo "$NON_TXN" | sed 's/^/ /' >&2 + warn "convert them: ALTER TABLE ENGINE=InnoDB;" +fi + +# --- Database list ------------------------------------------------------ + +mapfile -t DATABASES < <(sql " + SELECT schema_name FROM information_schema.schemata + WHERE schema_name NOT IN ($EXCLUDED_SCHEMAS) + ORDER BY schema_name;" | tr -d '\r') + +(( ${#DATABASES[@]} > 0 )) || die "no databases returned - check credentials" +echo "Found ${#DATABASES[@]} database(s) to dump" + +# --- Free space sanity check -------------------------------------------- + +if [[ -d "$DUMP_DIR" ]]; then + need_kb=$(( $(du -sk "$DUMP_DIR" | cut -f1) * 3 / 2 )) + free_kb=$(df -Pk "$(dirname "$DUMP_DIR")" | awk 'NR==2{print $4}') + (( free_kb > need_kb )) || die "not enough free space (~${need_kb}K needed, ${free_kb}K free)" +fi + +# --- Detect user-dump method -------------------------------------------- +# mysqldump --users : MySQL 8.0.31+ (dumps CREATE USER + GRANT, no DB tables) +# mariadb-dump --system=users : MariaDB 10.5+ (same idea, different flag) +# fallback : dump full mysql schema (larger, includes grant tables) + +dump_users_and_grants() { + local out="$1" + if "$DUMPER" --help 2>&1 | grep -q -- ' --users '; then + # MySQL 8.0.31+: --users dumps CREATE USER + GRANT statements only + # (when no --databases / --all-databases is given) + log "[dump] using mysqldump --users (MySQL 8.0.31+ style)" + db_exec "$DUMPER" -u "$DB_USER" --users \ + > "$out" 2>/dev/null + elif docker exec "$CONTAINER" sh -c "command -v mariadb-dump" >/dev/null 2>&1 \ + && docker exec "$CONTAINER" mariadb-dump --help 2>&1 | grep -q -- ' --system '; then + # MariaDB: --system=users + log "[dump] using mariadb-dump --system=users" + db_exec "$DUMPER" -u "$DB_USER" --system=users \ + > "$out" 2>/dev/null + else + # Fallback: full mysql schema dump + log "[dump] falling back to full mysql schema dump (older server)" + db_exec "$DUMPER" "${DUMP_OPTS[@]}" --databases mysql \ + > "$out" 2>/dev/null + fi +} + +# --- Dump into staging, verify each file -------------------------------- + +rm -rf "$STAGING" +mkdir -p "$STAGING" + +failed=() + +verify_dump() { + local f="$1" + [[ -s "$f" ]] || { warn "$(basename "$f"): empty"; return 1; } + # mysqldump ends with "-- Dump completed on ..."; mysqlpump ends similarly. + # For the mysql-schema fallback, just check non-empty (it has no marker). + if grep -q '^-- Dump completed' "$f" 2>/dev/null; then + return 0 + fi + # Fallback files (full mysql dump) won't have the marker — accept non-empty. + if [[ "$(basename "$f")" == 00-mysql-schema.sql ]]; then + return 0 + fi + warn "$(basename "$f"): no completion marker" + return 1 +} + +# Users and grants first +USERS_FILE="$STAGING/00-users-and-grants.sql" +echo "Dumping users and grants" +if dump_users_and_grants "$USERS_FILE" && verify_dump "$USERS_FILE"; then + echo " -> ok ($(du -h "$USERS_FILE" | cut -f1))" +else + warn "users-and-grants dump failed or empty" + failed+=("users-and-grants") + rm -f "$USERS_FILE" +fi + +for db in "${DATABASES[@]}"; do + out="$STAGING/${db}.sql" + printf 'Dumping %-32s ' "$db" + if db_exec "$DUMPER" "${DUMP_OPTS[@]}" -- "$db" > "$out" && verify_dump "$out"; then + printf 'ok (%s)\n' "$(du -h "$out" | cut -f1)" + else + printf 'FAILED\n' + failed+=("$db") + rm -f "$out" + fi +done + +# --- Abort before touching the good copy -------------------------------- + +if (( ${#failed[@]} > 0 )); then + die "${#failed[@]} dump(s) failed: ${failed[*]} - previous dumps left intact" +fi + +# --- Atomic swap -------------------------------------------------------- + +rm -rf "${DUMP_DIR}.old" +[[ -d "$DUMP_DIR" ]] && mv "$DUMP_DIR" "${DUMP_DIR}.old" +mv "$STAGING" "$DUMP_DIR" +rm -rf "${DUMP_DIR}.old" +trap - EXIT + +echo "-----------------------------------------------------------" +echo "All ${#DATABASES[@]} database(s) dumped and verified" +echo "Location: $DUMP_DIR ($(du -sh "$DUMP_DIR" | cut -f1))" diff --git a/old/borg.sh b/old/borg.sh new file mode 100644 index 0000000..2fb7b02 --- /dev/null +++ b/old/borg.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# ============================================================================= +# Borg Backup Script - MariaDB Restart Guaranteed +# ============================================================================= + +set -euo pipefail + +# ========================= CONFIGURATION ========================= + +NAME="borg-2025" +REPO="/home/srv/files/backups/$NAME" +TARGET="/home/srv/files/content" + +LOGDIR="/var/log/borg" +LOGFILE="${LOGDIR}/backup-$(date +%Y-%m-%d-%H%M%S).log" + +# Optional but strongly recommended +HEALTHCHECK_URL="https://hc-ping.com/your-uuid-here" + +RCLONE_REMOTE="scaleway" + +# ================================================================= + +mkdir -p "$LOGDIR" +exec > >(tee -a "$LOGFILE") +exec 2>&1 + +echo "=== Backup started at $(date '+%Y-%m-%d %H:%M:%S') ===" +echo "Repository: $REPO" +echo "Target: $TARGET" +echo "Log: $LOGFILE" +echo "-----------------------------------------------------------" + +# ----------------------- Lockfile ----------------------- +LOCKFILE="/var/lock/borg-backup.lock" +MARIADB_STOPPED=false + +cleanup() { + local exit_code=${1:-$?} + + if [ "$MARIADB_STOPPED" = true ]; then + echo "=== Cleanup: Starting MariaDB container ===" + docker start mariadb || echo "WARNING: Failed to start mariadb container" + sleep 3 + echo "MariaDB restart completed." + fi + + rm -f "$LOCKFILE" + + if [ "$exit_code" -eq 0 ]; then + echo "=== Backup completed SUCCESSFULLY at $(date '+%Y-%m-%d %H:%M:%S') ===" + send_healthcheck + else + echo "=== Backup FAILED at $(date '+%Y-%m-%d %H:%M:%S') (exit code $exit_code) ===" + send_healthcheck "fail" + fi + echo "Full log: $LOGFILE" +} + +if [ -e "$LOCKFILE" ]; then + echo "ERROR: Another backup is already running (lockfile exists)" + echo "Remove it manually if stale: $LOCKFILE" + exit 1 +fi + +touch "$LOCKFILE" +trap 'cleanup $?' EXIT + +# ----------------------- Borg & Functions ----------------------- +export BORG_REPO="$REPO" +export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes + +send_healthcheck() { + local status="${1:-}" + if [[ -n "${HEALTHCHECK_URL:-}" ]] && [[ "$HEALTHCHECK_URL" != *"your-uuid-here"* ]]; then + case "$status" in + start) curl -s -m 10 --retry 3 "${HEALTHCHECK_URL}/start" >/dev/null || true ;; + fail) curl -s -m 10 --retry 3 "${HEALTHCHECK_URL}/fail" -d "Backup failed - check $LOGFILE" >/dev/null || true ;; + *) curl -s -m 10 --retry 3 "$HEALTHCHECK_URL" >/dev/null || true ;; + esac + fi +} + +run_cmd() { + echo "[RUN] $*" + "$@" +} + +# ================================================================= + +send_healthcheck "start" + +# 1. MariaDB dump (while running) +echo "=== Step 1: MariaDB dump ===" +DUMP_SCRIPT="${TARGET}/mariadb/dump_db.sh" +if [[ -x "$DUMP_SCRIPT" ]]; then + run_cmd "$DUMP_SCRIPT" +else + echo "WARNING: Dump script not found or not executable: $DUMP_SCRIPT" +fi + +# 2. Stop MariaDB for consistent backup +echo "=== Step 2: Stopping MariaDB container ===" +run_cmd docker stop mariadb +MARIADB_STOPPED=true + +# 3. Create Borg archive +echo "=== Step 3: Creating Borg archive ===" +run_cmd borg create \ + --stats \ + --progress \ + --list \ + --filter=AME \ + --compression zstd,8 \ + --exclude-caches \ + --exclude-if-present .nobackup \ + "::${now:%Y-%m-%dT%H-%M-%S}" \ + "$TARGET" + +# 4. Restart MariaDB (this line is now also in the cleanup trap) +echo "=== Step 4: Starting MariaDB container ===" +run_cmd docker start mariadb +MARIADB_STOPPED=false + +# 5. Offsite sync +echo "=== Step 5: Syncing to Scaleway S3 ===" +run_cmd rclone sync -v \ + --fast-list \ + --transfers=8 \ + --checkers=16 \ + "$REPO" "${RCLONE_REMOTE}:/par-backup-1/$NAME" + +# 6. Prune & Compact +echo "=== Step 6: Pruning and compacting ===" +run_cmd borg prune \ + --list \ + --keep-daily=7 \ + --keep-weekly=4 \ + --keep-monthly=6 \ + --keep-within=7d \ + "$REPO" + +run_cmd borg compact --progress "$REPO" + +# The cleanup trap will run automatically and mark success diff --git a/old/maria.sh b/old/maria.sh new file mode 100644 index 0000000..40b057c --- /dev/null +++ b/old/maria.sh @@ -0,0 +1,36 @@ +root@chaudron:/home/srv/files/content/mariadb# cat dump_db.sh +#!/bin/bash + +# Load configuration from environment variables +MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-MySuperDatabase} +DUMP_DIR=${DUMP_DIR:-dump} +DOCKER_CONTAINER_NAME=${DOCKER_CONTAINER_NAME:-mariadb} + +# Check dependencies +if ! command -v docker &>/dev/null; then + echo "Docker is not installed or not in the PATH" + exit 1 +fi + +if ! docker ps -q -f name=$DOCKER_CONTAINER_NAME; then + echo "Docker container '$DOCKER_CONTAINER_NAME' is not running" + exit 1 +fi + +# Create dump directory if it doesn't exist +mkdir -p "$(dirname "$0")/$DUMP_DIR" + +# Delete old dump files +rm -f "$(dirname "$0")/$DUMP_DIR"/*.sql + +# Get list of databases and exclude system databases +databases=$(docker exec -it $DOCKER_CONTAINER_NAME mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "SHOW DATABASES;" --skip-column-names -s | grep -Ev "(information_schema|mysql|performance_schema|sys)" | tr -d '\r') + +# Iterate through databases and dump them individually +for db in $databases; do + echo "Dumping database: $db" + docker exec -i $DOCKER_CONTAINER_NAME mysqldump -u root -p"$MYSQL_ROOT_PASSWORD" "$db" >"$(dirname "$0")/$DUMP_DIR/$db.sql" + echo "Dumped database: $db" +done + +echo "All non-system databases dumped to individual files in '$DUMP_DIR' directory."