fix: start mariadb before restoring DBs in cmd_full, add lock/preflight

Final review of the restore.sh branch found cmd_full restored every database via `docker exec` before starting the container, which fails immediately in the exact scenario full restore exists for (a freshly rebuilt, stopped container). Reorders to extract -> start container -> restore DBs. Also, while touching cmd_full: - Extract directly into place (cd / && borg extract) instead of staging a full copy under /tmp then cp -a'ing it into $TARGET - halves disk usage and restore time. - Replace `rm -rf "$TARGET"/*` with `find "$TARGET" -mindepth 1 -delete` so dotfiles don't survive a --force wipe. - Add acquire_lock() (shares borg-backup.sh's lockfile so a restore and the nightly backup cron can't run concurrently) and preflight() (passphrase file readable, repo reachable) before any real work in full/db/file. Test isolation: mock borg/docker/mysql/mariadb consistently via a BASH_ENV shim (previously only db-mode's test worked around PATH shadowing by a real docker binary; every mocked test needed it, and a missing `flock` mock broke everything once acquire_lock was added, since flock(1) doesn't exist on macOS). Tests also isolate LOCKFILE and BORG_PASSPHRASE_FILE to throwaway paths. RUNBOOK.md: fix the quarterly drill command (borg extract has no --destination flag, and needs `borg list --short` for a bare archive name), reword the full-restore --force comment which read backwards, and document the MYSQL_ROOT_PASSWORD/RESTORE_LOGDIR env overrides and where restore logs land. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-25 19:42:56 +02:00
parent 8d7a603615
commit 592ef45bab
4 changed files with 138 additions and 46 deletions
@@ -75,5 +75,36 @@ exit 0
 EOF
    cp "$dir/mysql" "$dir/mariadb"

-    chmod +x "$dir/borg" "$dir/docker" "$dir/mysql" "$dir/mariadb"
+    # flock(1) is Linux-only (util-linux) and absent on macOS dev machines;
+    # restore.sh only ever uses the "flock -n FD" form (never the
+    # command-wrapping form), so a no-op success is a faithful stand-in for
+    # exclusion testing purposes - no test here exercises actual contention.
+    cat > "$dir/flock" <<EOF
+#!/bin/bash
+echo "flock \$*" >> "$mocklog"
+exit 0
+EOF
+
+    chmod +x "$dir/borg" "$dir/docker" "$dir/mysql" "$dir/mariadb" "$dir/flock"
+}
+
+# restore.sh hardens PATH with system dirs (/usr/local/bin etc.) placed
+# ahead of $PATH, so `PATH="$mockdir:$PATH" ...` alone does NOT guarantee the
+# mock is what actually runs: a real borg/docker/mysql/mariadb installed in
+# one of those system dirs would shadow it. BASH_ENV is sourced by bash
+# before running a script and shell functions win over PATH lookup for
+# simple commands regardless of PATH ordering, so this is what actually
+# guarantees isolation on any machine, not just ones that happen to lack
+# those binaries in the hardened prefix.
+mock_bash_env() {
+    local dir="$1"
+    local bashenv="$dir/bash_env.sh"
+    cat > "$bashenv" <<EOF
+borg() { "$dir/borg" "\$@"; }
+docker() { "$dir/docker" "\$@"; }
+mysql() { "$dir/mysql" "\$@"; }
+mariadb() { "$dir/mariadb" "\$@"; }
+flock() { "$dir/flock" "\$@"; }
+EOF
+    echo "$bashenv"
 }
@@ -49,22 +49,34 @@ test_unknown_command_exits_one() {

 source "$HERE/lib/setup_mocks.sh"

+# Every test below isolates two things beyond PATH: BASH_ENV forces borg/
+# docker/mysql/mariadb to the mock regardless of what's really installed on
+# this machine's PATH (restore.sh's own hardened PATH would otherwise shadow
+# the mock with any real binary in a system dir), and LOCKFILE/
+# BORG_PASSPHRASE_FILE point at throwaway files so acquire_lock()/preflight()
+# never touch real system paths like /var/lock or /root.
+
 test_list_archives() {
    local mockdir out
    mockdir="$(mktemp -d)"
    setup_mock_bin "$mockdir"
-    out="$(PATH="$mockdir:$PATH" bash "$RESTORE" --list-archives)"
+    out="$(PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" bash "$RESTORE" --list-archives)"
    assert_contains "$out" "host-2026-01-01T00-00-00" "list-archives shows first archive"
    assert_contains "$out" "host-2026-06-01T00-00-00" "list-archives shows latest archive"
    rm -rf "$mockdir"
 }

 test_file_restore_extracts_to_dest() {
-    local mockdir dest out final
+    local mockdir dest out final passfile lockfile
    mockdir="$(mktemp -d)"
    dest="$(mktemp -d)"
    setup_mock_bin "$mockdir"
-    out="$(PATH="$mockdir:$PATH" bash "$RESTORE" file photos/img.jpg --dest "$dest")"
+    passfile="$mockdir/passphrase"
+    lockfile="$mockdir/lock"
+    echo "s3cr3t" > "$passfile"
+    out="$(PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" \
+        BORG_PASSPHRASE_FILE="$passfile" LOCKFILE="$lockfile" \
+        bash "$RESTORE" file photos/img.jpg --dest "$dest")"
    final="$dest/home/srv/files/content/photos/img.jpg"
    assert_contains "$out" "Restored file available at: $final" "file mode reports final path"
    if [[ -f "$final" ]]; then
@@ -81,7 +93,8 @@ test_file_restore_dry_run_makes_no_borg_call() {
    mockdir="$(mktemp -d)"
    dest="$(mktemp -d)"
    setup_mock_bin "$mockdir"
-    PATH="$mockdir:$PATH" bash "$RESTORE" file photos/img.jpg --dest "$dest" --dry-run >/dev/null
+    PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" LOCKFILE="$mockdir/lock" \
+        bash "$RESTORE" file photos/img.jpg --dest "$dest" --dry-run >/dev/null
    if [[ -s "$mockdir/mock.log" ]] && grep -q "^borg extract" "$mockdir/mock.log"; then
        echo "FAIL: dry-run invoked borg extract"
        FAILURES=$((FAILURES + 1))
@@ -92,20 +105,16 @@ test_file_restore_dry_run_makes_no_borg_call() {
 }

 test_db_restore_with_yes_runs_full_sequence() {
-    local mockdir out bashenv
+    local mockdir out passfile
    mockdir="$(mktemp -d)"
    setup_mock_bin "$mockdir"
    echo "rootpass" > "$mockdir/rootpw"
-    # This host may have a real `docker` CLI in one of the system dirs that
-    # restore.sh's hardened PATH prepends (e.g. /usr/local/bin), which would
-    # shadow the mock and make a real (unreachable) docker daemon get called
-    # instead. A BASH_ENV-sourced function takes precedence over PATH lookup
-    # regardless of PATH ordering, so force `docker` to the mock that way.
-    bashenv="$mockdir/bash_env.sh"
-    cat > "$bashenv" <<EOF
-docker() { "$mockdir/docker" "\$@"; }
-EOF
-    out="$(PATH="$mockdir:$PATH" ROOT_PASSWORD_FILE="$mockdir/rootpw" BASH_ENV="$bashenv" bash "$RESTORE" db shopdb --yes </dev/null)"
+    passfile="$mockdir/passphrase"
+    echo "s3cr3t" > "$passfile"
+    out="$(PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" \
+        ROOT_PASSWORD_FILE="$mockdir/rootpw" BORG_PASSPHRASE_FILE="$passfile" \
+        LOCKFILE="$mockdir/lock" \
+        bash "$RESTORE" db shopdb --yes </dev/null)"
    assert_contains "$out" "Database 'shopdb' restored from" "db mode reports success"
    if grep -q "CREATE DATABASE IF NOT EXISTS" "$mockdir/mock.log"; then
        echo "PASS: db mode issued CREATE DATABASE"
@@ -120,7 +129,8 @@ test_db_restore_dry_run_skips_confirmation_and_calls() {
    local mockdir out
    mockdir="$(mktemp -d)"
    setup_mock_bin "$mockdir"
-    out="$(PATH="$mockdir:$PATH" bash "$RESTORE" db shopdb --dry-run </dev/null)"
+    out="$(PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" LOCKFILE="$mockdir/lock" \
+        bash "$RESTORE" db shopdb --dry-run </dev/null)"
    assert_contains "$out" "DRY-RUN" "db dry-run prints DRY-RUN plan"
    # cmd_db calls resolve_archive() (a borg list call) before checking DRY_RUN,
    # same as cmd_file does, so mock.log legitimately gets a "borg list" entry.
@@ -139,7 +149,8 @@ test_db_restore_aborts_on_wrong_confirmation() {
    mockdir="$(mktemp -d)"
    setup_mock_bin "$mockdir"
    set +e
-    echo "wrongname" | PATH="$mockdir:$PATH" bash "$RESTORE" db shopdb >/dev/null 2>&1
+    echo "wrongname" | PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" LOCKFILE="$mockdir/lock" \
+        bash "$RESTORE" db shopdb >/dev/null 2>&1
    rc=$?
    set -e
    assert_eq "1" "$rc" "db mode aborts on mismatched confirmation"
@@ -147,19 +158,20 @@ test_db_restore_aborts_on_wrong_confirmation() {
 }

 test_full_restore_refuses_nonempty_target_without_force() {
-    local mockdir target rc
+    local mockdir target out rc
    mockdir="$(mktemp -d)"
    target="$(mktemp -d)"
    touch "$target/existing-file"
    setup_mock_bin "$mockdir"
    set +e
-    PATH="$mockdir:$PATH" TARGET_OVERRIDE=1 bash -c '
+    out="$(PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" LOCKFILE="$mockdir/lock" bash -c '
        sed "s#^TARGET=\"/home/srv/files/content\"#TARGET=\"'"$target"'\"#; s@^ARCHIVE_TARGET_PATH=.*@ARCHIVE_TARGET_PATH=\"\${TARGET#/}\"@" "'"$RESTORE"'" > "'"$mockdir"'/restore_patched.sh"
        bash "'"$mockdir"'/restore_patched.sh" full
-    ' >/dev/null 2>&1
+    ' 2>&1)"
    rc=$?
    set -e
    assert_eq "1" "$rc" "full mode refuses non-empty target without --force"
+    assert_contains "$out" "is not empty - pass --force" "refusal message names the reason"
    rm -rf "$mockdir" "$target"
 }

@@ -168,7 +180,7 @@ test_full_restore_dry_run_makes_no_calls() {
    mockdir="$(mktemp -d)"
    target="$(mktemp -d)"
    setup_mock_bin "$mockdir"
-    out="$(PATH="$mockdir:$PATH" bash -c '
+    out="$(PATH="$mockdir:$PATH" BASH_ENV="$(mock_bash_env "$mockdir")" LOCKFILE="$mockdir/lock" bash -c '
        sed "s#^TARGET=\"/home/srv/files/content\"#TARGET=\"'"$target"'\"#; s@^ARCHIVE_TARGET_PATH=.*@ARCHIVE_TARGET_PATH=\"\${TARGET#/}\"@" "'"$RESTORE"'" > "'"$mockdir"'/restore_patched.sh"
        bash "'"$mockdir"'/restore_patched.sh" full --dry-run
    ')"