chore: initial import of backup scripts and design docs

Existing borg-backup.sh/dump_db.sh, old/ reference scripts, and the
brainstormed design spec for restore tooling + runbook.

borg-backup.sh

@@ -0,0 +1,318 @@
+#!/bin/bash
+# =============================================================================
+# Borg backup: /content + MariaDB  ->  local repo  ->  Scaleway S3 mirror
+# =============================================================================
+# CHANGES vs. original:
+#   • Encryption enabled (repokey-blake2 via BORG_PASSCOMMAND)
+#   • Preflight warns if repo is unencrypted
+#   • Added borg check (daily archive-only, weekly full --verify-data)
+#   • Removed BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK
+#   • No longer stops MariaDB: dump_db.sh's --single-transaction dump is
+#     already consistent, and the container's raw data directory is
+#     excluded from the archive (drop a .nobackup file in it - see
+#     --exclude-if-present below) so no live InnoDB file is ever copied.
+#     Zero DB downtime during backup.
+# =============================================================================
+
+set -euo pipefail
+
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+umask 077
+
+# ========================= CONFIGURATION =========================
+
+NAME="borg-2025"
+REPO="/home/srv/files/backups/$NAME"
+TARGET="/home/srv/files/content"
+
+REPO_MOUNT=""
+
+LOGDIR="/var/log/borg"
+LOG_RETENTION_DAYS=90
+
+HEALTHCHECK_URL="https://hc-ping.com/your-uuid-here"
+
+RCLONE_REMOTE="scaleway"
+RCLONE_PATH="par-backup-1/$NAME"
+RCLONE_MAX_DELETE=200
+
+LOCKFILE="/var/lock/borg-backup.lock"
+
+DB_CONTAINER="mariadb"
+DB_START_TIMEOUT=180
+
+CREATE_TIMEOUT="6h"
+PRUNE_TIMEOUT="2h"
+SYNC_TIMEOUT="12h"
+CHECK_TIMEOUT="4h"
+
+# Passphrase file: chmod 600, owned by the backup user.
+# Create it with:  echo 'your-strong-passphrase' > /root/.borg-passphrase
+BORG_PASSPHRASE_FILE="${BORG_PASSPHRASE_FILE:-/root/.borg-passphrase}"
+
+REQUIRED_CMDS=(borg rclone docker curl timeout flock date find)
+
+# =================================================================
+
+SELF="$(readlink -f "$0")"
+mkdir -p "$LOGDIR"
+LOGFILE="${BORG_BACKUP_LOGFILE:-$LOGDIR/backup-$(date +%Y-%m-%d-%H%M%S).log}"
+
+# --- Logging: re-exec ourselves through tee ------------------------------
+if [[ -z "${BORG_BACKUP_LOG_WRAPPED:-}" ]]; then
+    export BORG_BACKUP_LOG_WRAPPED=1
+    export BORG_BACKUP_LOGFILE="$LOGFILE"
+    set +e
+    "$SELF" "$@" 2>&1 | tee -a "$LOGFILE"
+    exit "${PIPESTATUS[0]}"
+fi
+
+# --- Borg environment ---------------------------------------------------
+export BORG_REPO="$REPO"
+export BORG_RELOCATED_REPO_ACCESS_IS_OK=no
+export BORG_PASSCOMMAND="cat $BORG_PASSPHRASE_FILE"
+
+ARCHIVE="$(hostname -s)-$(date +%Y-%m-%dT%H-%M-%S)"
+ARCHIVE_GLOB="$(hostname -s)-*"
+
+# ----------------------- Functions (all before the trap) -----------------
+
+log()  { echo "[$(date '+%F %T')] $*"; }
+step() { echo; echo "=== $* ==="; }
+
+run_cmd() {
+    log "[RUN] $*"
+    "$@"
+}
+
+send_healthcheck() {
+    local status="${1:-0}"
+    [[ -n "${HEALTHCHECK_URL:-}"                  ]] || return 0
+    [[ "$HEALTHCHECK_URL" != *"your-uuid-here"*   ]] || return 0
+    command -v curl >/dev/null 2>&1               || return 0
+
+    local curl_opts=(-fsS -m 15 --retry 3 --retry-connrefused)
+    case "$status" in
+        start) curl "${curl_opts[@]}" "${HEALTHCHECK_URL}/start" >/dev/null || true ;;
+        0)     curl "${curl_opts[@]}" "$HEALTHCHECK_URL"         >/dev/null || true ;;
+        *)     curl "${curl_opts[@]}" "${HEALTHCHECK_URL}/${status}" \
+                    --data-raw "exit=$status archive=$ARCHIVE log=$LOGFILE" >/dev/null || true ;;
+    esac
+}
+
+container_running() {
+    [[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || echo false)" == "true" ]]
+}
+
+container_health() {
+    docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
+        "$1" 2>/dev/null || echo unknown
+}
+
+wait_for_container() {
+    local name="$1" deadline=$((SECONDS + DB_START_TIMEOUT)) health
+    while (( SECONDS < deadline )); do
+        if container_running "$name"; then
+            health="$(container_health "$name")"
+            case "$health" in
+                healthy|none) log "$name is up (health: $health)"; return 0 ;;
+                unhealthy)    log "WARNING: $name reports unhealthy" ;;
+            esac
+        fi
+        sleep 3
+    done
+    return 1
+}
+
+start_db() {
+    docker start "$DB_CONTAINER" >/dev/null 2>&1 || true
+    if wait_for_container "$DB_CONTAINER"; then
+        return 0
+    fi
+    log "ERROR: $DB_CONTAINER did not come back up within ${DB_START_TIMEOUT}s"
+    return 1
+}
+
+cleanup() {
+    local exit_code=$1
+    set +e
+
+    # We never stop the container ourselves anymore, but if it crashed for
+    # an unrelated reason during the backup window, try to bring it back.
+    if ! container_running "$DB_CONTAINER"; then
+        step "Cleanup: $DB_CONTAINER is down, attempting restart"
+        start_db || { log "CRITICAL: $DB_CONTAINER is DOWN - manual action required"
+                      [ "$exit_code" -eq 0 ] && exit_code=1; }
+    fi
+
+    find "$LOGDIR" -maxdepth 1 -name 'backup-*.log' -type f \
+         -mtime +"$LOG_RETENTION_DAYS" -delete 2>/dev/null
+
+    if [ "$exit_code" -eq 0 ]; then
+        log "=== Backup completed SUCCESSFULLY (archive: $ARCHIVE) ==="
+    else
+        log "=== Backup FAILED (exit $exit_code) ==="
+    fi
+    log "Full log: $LOGFILE"
+    send_healthcheck "$exit_code"
+}
+
+# Returns 0 if the repo is encrypted, 1 if not.
+repo_is_encrypted() {
+    borg info "$REPO" 2>&1 | grep -q "Encrypted:.*Yes"
+}
+
+preflight() {
+    local missing=()
+    for c in "${REQUIRED_CMDS[@]}"; do
+        command -v "$c" >/dev/null 2>&1 || missing+=("$c")
+    done
+    if (( ${#missing[@]} )); then
+        log "ERROR: missing commands: ${missing[*]}"; return 1
+    fi
+
+    if [[ -n "$REPO_MOUNT" ]] && ! mountpoint -q "$REPO_MOUNT"; then
+        log "ERROR: $REPO_MOUNT is not mounted - refusing to run"
+        return 1
+    fi
+
+    [[ -d "$TARGET" ]]            || { log "ERROR: target missing: $TARGET"; return 1; }
+    [[ -n "$(ls -A "$TARGET")" ]] || { log "ERROR: target is empty: $TARGET"; return 1; }
+
+    case "$REPO/" in
+        "$TARGET"/*) log "ERROR: repo lives inside target - infinite recursion"; return 1 ;;
+    esac
+
+    if ! borg info --lock-wait 60 >/dev/null 2>&1; then
+        log "ERROR: cannot open repository $REPO"
+        log "       (wrong path? disk not mounted? stale lock? -> borg break-lock)"
+        return 1
+    fi
+
+    # Warn if the repo is not encrypted.
+    if ! repo_is_encrypted; then
+        log "WARNING: Repository is NOT encrypted!"
+        log "         Run:  borg init --encryption=repokey-blake2 $REPO"
+        log "         (after backing up any existing data)"
+    fi
+
+    # Verify passphrase file exists and is readable.
+    if [[ ! -r "$BORG_PASSPHRASE_FILE" ]]; then
+        log "ERROR: passphrase file not readable: $BORG_PASSPHRASE_FILE"
+        log "       Create it:  echo 'your-passphrase' > $BORG_PASSPHRASE_FILE"
+        log "                   chmod 600 $BORG_PASSPHRASE_FILE"
+        return 1
+    fi
+
+    container_running "$DB_CONTAINER" \
+        || { log "ERROR: $DB_CONTAINER is not running before we start"; return 1; }
+
+    log "Preflight OK"
+}
+
+# ----------------------- Lock, then trap --------------------------------
+
+exec 9>"$LOCKFILE"
+if ! flock -n 9; then
+    echo "ERROR: another backup is already running (lock held on $LOCKFILE)"
+    exit 1
+fi
+
+trap 'cleanup $?' EXIT
+trap 'log "SIGINT received"; exit 130' INT
+trap 'log "SIGTERM received"; exit 143' TERM
+
+# =================================================================
+
+log "=== Backup started ==="
+echo "Repository: $REPO"
+echo "Target:     $TARGET"
+echo "Archive:    $ARCHIVE"
+echo "Log:        $LOGFILE"
+
+send_healthcheck start
+
+step "Step 0: Preflight"
+preflight
+
+# --- 1. Logical dumps (container stays up the whole time) ---------------
+step "Step 1: MariaDB dumps"
+DUMP_SCRIPT="${TARGET}/mariadb/dump_db.sh"
+DUMP_DIR="${TARGET}/mariadb/dump"
+
+if [[ ! -x "$DUMP_SCRIPT" ]]; then
+    log "ERROR: dump script missing or not executable: $DUMP_SCRIPT"
+    exit 1
+fi
+run_cmd "$DUMP_SCRIPT"
+
+if [[ -d "$DUMP_DIR" ]]; then
+    fresh=$(find "$DUMP_DIR" -type f -size +1k -mmin -60 | wc -l)
+    empty=$(find "$DUMP_DIR" -type f -size -1k -mmin -60 | wc -l)
+    log "Dumps: $fresh fresh non-trivial file(s), $empty suspiciously small"
+    (( fresh > 0 )) || { log "ERROR: no usable dumps produced"; exit 1; }
+    (( empty == 0 )) || log "WARNING: $empty near-empty dump file(s) - check $DUMP_DIR"
+else
+    log "WARNING: dump directory not found: $DUMP_DIR"
+fi
+
+# --- 2. Create the archive -----------------------------------------------
+# MariaDB stays up throughout: the dump above is already transactionally
+# consistent, and the container's raw data directory carries a .nobackup
+# marker (excluded here via --exclude-if-present) so its live files are
+# never read by borg.
+step "Step 2: Creating archive $ARCHIVE"
+container_running "$DB_CONTAINER" \
+    || { log "ERROR: $DB_CONTAINER is not running - refusing to archive"; exit 1; }
+run_cmd timeout --signal=INT --kill-after=120s "$CREATE_TIMEOUT" \
+    borg create \
+        --lock-wait 600 \
+        --stats \
+        --list --filter=AME \
+        --compression zstd,8 \
+        --exclude-caches \
+        --exclude-if-present .nobackup \
+        --keep-exclude-tags \
+        "::$ARCHIVE" \
+        "$TARGET"
+
+# --- 3/4. Retention -------------------------------------------------------
+step "Step 3: Pruning"
+run_cmd timeout "$PRUNE_TIMEOUT" \
+    borg prune \
+        --lock-wait 600 \
+        --list \
+        --glob-archives "$ARCHIVE_GLOB" \
+        --keep-daily=7 \
+        --keep-weekly=4 \
+        --keep-monthly=6
+
+step "Step 4: Compacting"
+run_cmd timeout "$PRUNE_TIMEOUT" borg compact --lock-wait 600
+
+# --- 5. Integrity check -------------------------------------------------
+step "Step 5: Verifying archive & repository integrity"
+run_cmd borg info --lock-wait 60 "::$ARCHIVE" >/dev/null
+ARCHIVE_COUNT=$(borg list --short --glob-archives "$ARCHIVE_GLOB" | wc -l)
+log "Repository holds $ARCHIVE_COUNT archive(s) for $ARCHIVE_GLOB"
+(( ARCHIVE_COUNT > 0 )) || { log "ERROR: repo unexpectedly empty"; exit 1; }
+
+# Daily: fast archive-only check. Weekly (Monday): full repo check with data verification.
+if [[ "$(date +%u)" == "1" ]]; then
+    log "Monday — running full repository check with --verify-data (this may take a while)"
+    run_cmd timeout "$CHECK_TIMEOUT" borg check --verify-data --lock-wait 600 "$REPO"
+else
+    log "Running fast archive-only integrity check"
+    run_cmd timeout 1h borg check --archives-only --lock-wait 600 "$REPO"::"$ARCHIVE"
+fi
+
+# --- 6. Offsite mirror --------------------------------------------------
+step "Step 6: Syncing to ${RCLONE_REMOTE}:${RCLONE_PATH}"
+run_cmd timeout "$SYNC_TIMEOUT" \
+    rclone sync -v \
+        --fast-list \
+        --transfers=8 \
+        --checkers=16 \
+        --s3-no-check-bucket \
+        --max-delete "$RCLONE_MAX_DELETE" \
+        "$REPO" "${RCLONE_REMOTE}:${RCLONE_PATH}"

docs/superpowers/specs/2026-07-25-backup-recovery-system-design.md

@@ -0,0 +1,162 @@
+# Backup & Recovery System Design
+
+Date: 2026-07-25
+
+## Goal
+
+Reliable, automated backup of `/home/srv/files/content` (including MariaDB
+running in Docker) to a local Borg repo + offsite Scaleway S3 mirror, plus a
+documented, scriptable recovery path so downtime is short and predictable.
+
+## Current State
+
+- `old/borg.sh`, `old/maria.sh` — scripts currently live on the server.
+- `borg-backup.sh`, `dump_db.sh` (repo root) — improved replacements, not yet
+  deployed. Add: encrypted repo (`repokey-blake2` via `BORG_PASSCOMMAND`),
+  flock-based locking, preflight checks, container health polling, atomic
+  dump staging/swap, non-transactional table warnings, integrity checks
+  (`borg check`, daily archive-only / weekly `--verify-data`), retry/timeout
+  wrapping, log rotation.
+- No restore tooling and no scheduling exist yet.
+
+`dump_db.sh` is kept as-is. `borg-backup.sh` had one functional change made
+during this design: it no longer stops the MariaDB container. `dump_db.sh`
+already produces a transactionally-consistent dump via
+`--single-transaction`, so a live logical dump is safe without stopping
+anything (this only covers InnoDB tables — the script already warns if it
+finds non-InnoDB tables). The remaining risk was borg also archiving
+MariaDB's raw on-disk data directory while it's being written to; that's
+resolved by excluding it from the archive entirely (drop a `.nobackup`
+marker file in it — `borg-backup.sh` already passes
+`--exclude-if-present .nobackup`), so only the logical dump ever gets
+backed up, and restore only ever needs the dump anyway. Net effect: daily
+backups run with **zero MariaDB downtime**. This exclusion step needs to be
+added to the one-time setup, documented in the runbook.
+
+## Architecture
+
+```
+cron (daily) → borg-backup.sh
+                 ├─ dump_db.sh (mysqldump per-DB, atomic staging swap;
+                 │              container stays up throughout)
+                 ├─ borg create (encrypted, zstd) → local repo
+                 │              (raw MariaDB data dir excluded via .nobackup;
+                 │               only the logical dump is archived)
+                 ├─ borg prune + compact
+                 ├─ borg check (daily fast / weekly full --verify-data)
+                 └─ rclone sync → Scaleway S3 (offsite mirror)
+
+restore.sh (manual, run on demand during an incident)
+  mode=full   : fresh/broken server → extract full content dir from archive
+                → restore all DBs → start container
+  mode=db     : single database → restore one .sql from a chosen archive
+  mode=file   : single file/dir → borg extract path, no DB involved
+```
+
+## Components
+
+All at repo root, deployed to the server alongside the existing scripts.
+
+### `borg-backup.sh` (existing, one change: no longer stops MariaDB)
+
+Orchestrates the daily backup: dump → archive → prune → compact → check →
+offsite sync. The container is never stopped — the dump step alone
+produces a consistent backup, and the raw data directory is excluded from
+the archive. Already has locking, timeouts, health-check pings, and
+cleanup-on-exit logic (cleanup now only intervenes if the container
+happens to be down for an unrelated reason, restarting it defensively).
+
+### `dump_db.sh` (existing, unchanged)
+
+Runs inside `borg-backup.sh` step 1. Dumps each non-system database plus
+users/grants to a staging dir, verifies each dump file, then atomically
+swaps staging into place. Deployed at
+`/home/srv/files/content/mariadb/dump_db.sh`.
+
+### `restore.sh` (new)
+
+```
+restore.sh full  [--archive NAME] [--force]
+restore.sh db    <db_name> [--archive NAME]
+restore.sh file  <path-within-target> [--archive NAME] [--dest DIR]
+restore.sh --dry-run <above args>
+restore.sh --list-archives
+```
+
+- Defaults `--archive` to the most recent archive in the repo.
+- `full`: refuses to run if `$TARGET` is non-empty unless `--force` is
+  passed (protects against overwriting a working system by accident).
+  Extracts the full archive over `$TARGET`, then restores every `.sql`
+  dump found under `mariadb/dump/` using root DB credentials, then starts
+  the `mariadb` container and waits for it to become healthy.
+- `db <name>`: extracts only `mariadb/dump/<name>.sql` from the archive,
+  prompts for the database name to be typed again as confirmation (it's
+  destructive — drops/recreates the DB), restores it with root credentials.
+- `file <path>`: `borg extract` of a single path from the archive into
+  `--dest` (default: a scratch dir under `/tmp`), no DB or container
+  involvement — safe, non-destructive.
+- `--dry-run`: prints exactly what would run (archive chosen, paths
+  extracted, commands) without touching anything.
+- Same logging conventions as `borg-backup.sh` (`/var/log/borg/restore-*.log`),
+  non-zero exit on any failure, no partial/silent restores — a failed step
+  aborts before touching the "good" copy where possible.
+- Uses the same `BORG_PASSCOMMAND` / passphrase file as backups.
+- Restore DB operations authenticate as `root` via `MYSQL_ROOT_PASSWORD` or
+  a root password file (mirrors how `dump_db.sh` resolves credentials),
+  since restore needs full privileges (CREATE/DROP) that the limited
+  `backup` user used for dumping does not have.
+
+### `RUNBOOK.md` (new)
+
+Single operational document covering:
+
+1. **One-time setup** (run once, by hand, not scripted): `borg init
+   --encryption=repokey-blake2`, create `/root/.borg-passphrase`
+   (chmod 600), create the `backup` MariaDB user with the grants
+   `dump_db.sh` needs (SELECT, LOCK TABLES, SHOW VIEW, TRIGGER, PROCESS,
+   RELOAD) and its password file `/root/.mariadb-backup.pw`, drop a
+   `.nobackup` marker file into MariaDB's Docker volume data directory
+   (excludes raw DB files from the archive — only the logical dump under
+   `mariadb/dump/` gets backed up), configure `rclone` for the `scaleway`
+   remote, set the real healthcheck URL.
+2. **Deploying the scripts**: where each file goes, permissions,
+   `chmod +x`.
+3. **Scheduling**: cron entry (daily, off-peak hours) invoking
+   `borg-backup.sh`, plus how to check the last run (`/var/log/borg/`,
+   healthcheck dashboard).
+4. **Day-2 operations**: listing archives, checking repo size/health,
+   rotating the passphrase, what to do if the lockfile is stale.
+5. **Recovery procedures** — one clearly-numbered walkthrough per
+   scenario (full disaster, single DB, single file), written so someone
+   unfamiliar with the internals can follow it step-by-step under
+   pressure, including exact `restore.sh` invocations and what to verify
+   afterward (container healthy, app responds, row counts sane).
+6. **Restore drill cadence**: recommend a quarterly test restore into a
+   scratch location to confirm backups are actually usable.
+
+## Error Handling & Safety
+
+- `restore.sh` never overwrites live data without an explicit `--force`
+  (full) or typed confirmation (db).
+- All destructive steps are logged before execution.
+- `--dry-run` available for every mode.
+- Failures abort immediately (`set -euo pipefail`), matching the existing
+  scripts' style; partial state is called out explicitly in the log.
+
+## Testing
+
+No automated test framework — these are ops scripts against real
+infrastructure. Verification consists of:
+
+- `restore.sh --dry-run` runs against the real repo to sanity-check
+  archive selection and command construction.
+- Quarterly real restore drill into a scratch directory (documented in
+  the runbook), confirming dumps are complete and importable.
+
+## Out of Scope
+
+- No changes to `dump_db.sh`, and no changes to `borg-backup.sh` beyond
+  removing the DB stop/start (see Components above).
+- No monitoring/alerting beyond the existing healthcheck ping.
+- No multi-server / multi-target generalization — this is specific to
+  `/home/srv/files/content`.

dump_db.sh

@@ -0,0 +1,202 @@
+#!/bin/bash
+# =============================================================================
+# Consistent per-database MariaDB/MySQL dumps from a Docker container.
+# Writes to a staging dir, verifies, then swaps atomically.
+# =============================================================================
+# CHANGES vs. original:
+#   • Uses mysqldump --users (MySQL 8.0.31+) instead of MariaDB-only
+#     --system=users for dumping user accounts as CREATE USER / GRANT.
+#   • Falls back to full mysql schema dump on older MySQL / MariaDB.
+#   • Works with both mysql:8.2.0 and MariaDB containers.
+# =============================================================================
+
+set -euo pipefail
+umask 077
+
+# ========================= CONFIGURATION =========================
+
+CONTAINER="${DOCKER_CONTAINER_NAME:-mariadb}"
+
+PASSWORD_FILE="${PASSWORD_FILE:-/root/.mariadb-backup.pw}"
+DB_USER="${DB_USER:-backup}"
+
+SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "$0")")" && pwd)"
+DUMP_DIR="${DUMP_DIR:-dump}"
+[[ "$DUMP_DIR" == /* ]] || DUMP_DIR="$SCRIPT_DIR/$DUMP_DIR"
+STAGING="${DUMP_DIR}.staging"
+
+DEDUP_FRIENDLY="${DEDUP_FRIENDLY:-1}"
+
+EXCLUDED_SCHEMAS="'information_schema','mysql','performance_schema','sys'"
+
+# =================================================================
+
+die()  { echo "ERROR: $*" >&2; exit 1; }
+warn() { echo "WARNING: $*" >&2; }
+
+cleanup() { rm -rf "$STAGING"; }
+trap cleanup EXIT
+
+# --- Preconditions ------------------------------------------------------
+
+command -v docker >/dev/null 2>&1 || die "docker not found in PATH"
+
+[[ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || echo false)" == "true" ]] \
+    || die "container '$CONTAINER' is not running"
+
+if [[ -n "${MYSQL_ROOT_PASSWORD:-}" ]]; then
+    DB_PASS="$MYSQL_ROOT_PASSWORD"
+elif [[ -r "$PASSWORD_FILE" ]]; then
+    DB_PASS="$(< "$PASSWORD_FILE")"
+else
+    die "no password: set MYSQL_ROOT_PASSWORD or create $PASSWORD_FILE (chmod 600)"
+fi
+
+# Detect client binaries (MariaDB 11+ renamed them; MySQL keeps mysql/*).
+if docker exec "$CONTAINER" sh -c 'command -v mariadb-dump' >/dev/null 2>&1; then
+    CLIENT=mariadb; DUMPER=mariadb-dump
+    CLIENT_BIN="mariadb"
+else
+    CLIENT=mysql;   DUMPER=mysqldump
+    CLIENT_BIN="mysql"
+fi
+
+# MYSQL_PWD keeps the password out of the process list.
+db_exec() { docker exec -e MYSQL_PWD="$DB_PASS" "$CONTAINER" "$@"; }
+sql()     { db_exec "$CLIENT_BIN" -u "$DB_USER" -N -B -e "$1"; }
+
+DUMP_OPTS=(
+    -u "$DB_USER"
+    --single-transaction
+    --quick
+    --routines --triggers --events
+    --hex-blob
+    --max-allowed-packet=1G
+    --default-character-set=utf8mb4
+    --skip-dump-date
+)
+[[ "$DEDUP_FRIENDLY" == "1" ]] && DUMP_OPTS+=(--skip-extended-insert)
+
+# --- Warn about non-transactional tables --------------------------------
+
+NON_TXN="$(sql "
+    SELECT CONCAT(table_schema,'.',table_name,' [',engine,']')
+    FROM information_schema.tables
+    WHERE engine IS NOT NULL
+      AND engine NOT IN ('InnoDB','MEMORY','SEQUENCE')
+      AND table_schema NOT IN ($EXCLUDED_SCHEMAS);" || true)"
+
+if [[ -n "$NON_TXN" ]]; then
+    warn "non-transactional tables found - NOT covered by --single-transaction:"
+    echo "$NON_TXN" | sed 's/^/         /' >&2
+    warn "convert them: ALTER TABLE <t> ENGINE=InnoDB;"
+fi
+
+# --- Database list ------------------------------------------------------
+
+mapfile -t DATABASES < <(sql "
+    SELECT schema_name FROM information_schema.schemata
+    WHERE schema_name NOT IN ($EXCLUDED_SCHEMAS)
+    ORDER BY schema_name;" | tr -d '\r')
+
+(( ${#DATABASES[@]} > 0 )) || die "no databases returned - check credentials"
+echo "Found ${#DATABASES[@]} database(s) to dump"
+
+# --- Free space sanity check --------------------------------------------
+
+if [[ -d "$DUMP_DIR" ]]; then
+    need_kb=$(( $(du -sk "$DUMP_DIR" | cut -f1) * 3 / 2 ))
+    free_kb=$(df -Pk "$(dirname "$DUMP_DIR")" | awk 'NR==2{print $4}')
+    (( free_kb > need_kb )) || die "not enough free space (~${need_kb}K needed, ${free_kb}K free)"
+fi
+
+# --- Detect user-dump method --------------------------------------------
+# mysqldump --users  : MySQL 8.0.31+ (dumps CREATE USER + GRANT, no DB tables)
+# mariadb-dump --system=users : MariaDB 10.5+ (same idea, different flag)
+# fallback           : dump full mysql schema (larger, includes grant tables)
+
+dump_users_and_grants() {
+    local out="$1"
+    if "$DUMPER" --help 2>&1 | grep -q -- '  --users '; then
+        # MySQL 8.0.31+: --users dumps CREATE USER + GRANT statements only
+        # (when no --databases / --all-databases is given)
+        log "[dump] using mysqldump --users (MySQL 8.0.31+ style)"
+        db_exec "$DUMPER" -u "$DB_USER" --users \
+            > "$out" 2>/dev/null
+    elif docker exec "$CONTAINER" sh -c "command -v mariadb-dump" >/dev/null 2>&1 \
+         && docker exec "$CONTAINER" mariadb-dump --help 2>&1 | grep -q -- '  --system '; then
+        # MariaDB: --system=users
+        log "[dump] using mariadb-dump --system=users"
+        db_exec "$DUMPER" -u "$DB_USER" --system=users \
+            > "$out" 2>/dev/null
+    else
+        # Fallback: full mysql schema dump
+        log "[dump] falling back to full mysql schema dump (older server)"
+        db_exec "$DUMPER" "${DUMP_OPTS[@]}" --databases mysql \
+            > "$out" 2>/dev/null
+    fi
+}
+
+# --- Dump into staging, verify each file --------------------------------
+
+rm -rf "$STAGING"
+mkdir -p "$STAGING"
+
+failed=()
+
+verify_dump() {
+    local f="$1"
+    [[ -s "$f" ]] || { warn "$(basename "$f"): empty"; return 1; }
+    # mysqldump ends with "-- Dump completed on ..."; mysqlpump ends similarly.
+    # For the mysql-schema fallback, just check non-empty (it has no marker).
+    if grep -q '^-- Dump completed' "$f" 2>/dev/null; then
+        return 0
+    fi
+    # Fallback files (full mysql dump) won't have the marker — accept non-empty.
+    if [[ "$(basename "$f")" == 00-mysql-schema.sql ]]; then
+        return 0
+    fi
+    warn "$(basename "$f"): no completion marker"
+    return 1
+}
+
+# Users and grants first
+USERS_FILE="$STAGING/00-users-and-grants.sql"
+echo "Dumping users and grants"
+if dump_users_and_grants "$USERS_FILE" && verify_dump "$USERS_FILE"; then
+    echo "  -> ok ($(du -h "$USERS_FILE" | cut -f1))"
+else
+    warn "users-and-grants dump failed or empty"
+    failed+=("users-and-grants")
+    rm -f "$USERS_FILE"
+fi
+
+for db in "${DATABASES[@]}"; do
+    out="$STAGING/${db}.sql"
+    printf 'Dumping %-32s ' "$db"
+    if db_exec "$DUMPER" "${DUMP_OPTS[@]}" -- "$db" > "$out" && verify_dump "$out"; then
+        printf 'ok (%s)\n' "$(du -h "$out" | cut -f1)"
+    else
+        printf 'FAILED\n'
+        failed+=("$db")
+        rm -f "$out"
+    fi
+done
+
+# --- Abort before touching the good copy --------------------------------
+
+if (( ${#failed[@]} > 0 )); then
+    die "${#failed[@]} dump(s) failed: ${failed[*]} - previous dumps left intact"
+fi
+
+# --- Atomic swap --------------------------------------------------------
+
+rm -rf "${DUMP_DIR}.old"
+[[ -d "$DUMP_DIR" ]] && mv "$DUMP_DIR" "${DUMP_DIR}.old"
+mv "$STAGING" "$DUMP_DIR"
+rm -rf "${DUMP_DIR}.old"
+trap - EXIT
+
+echo "-----------------------------------------------------------"
+echo "All ${#DATABASES[@]} database(s) dumped and verified"
+echo "Location: $DUMP_DIR  ($(du -sh "$DUMP_DIR" | cut -f1))"

old/borg.sh

@@ -0,0 +1,145 @@
+#!/bin/bash
+# =============================================================================
+# Borg Backup Script - MariaDB Restart Guaranteed
+# =============================================================================
+
+set -euo pipefail
+
+# ========================= CONFIGURATION =========================
+
+NAME="borg-2025"
+REPO="/home/srv/files/backups/$NAME"
+TARGET="/home/srv/files/content"
+
+LOGDIR="/var/log/borg"
+LOGFILE="${LOGDIR}/backup-$(date +%Y-%m-%d-%H%M%S).log"
+
+# Optional but strongly recommended
+HEALTHCHECK_URL="https://hc-ping.com/your-uuid-here"
+
+RCLONE_REMOTE="scaleway"
+
+# =================================================================
+
+mkdir -p "$LOGDIR"
+exec > >(tee -a "$LOGFILE")
+exec 2>&1
+
+echo "=== Backup started at $(date '+%Y-%m-%d %H:%M:%S') ==="
+echo "Repository: $REPO"
+echo "Target: $TARGET"
+echo "Log: $LOGFILE"
+echo "-----------------------------------------------------------"
+
+# ----------------------- Lockfile -----------------------
+LOCKFILE="/var/lock/borg-backup.lock"
+MARIADB_STOPPED=false
+
+cleanup() {
+    local exit_code=${1:-$?}
+
+    if [ "$MARIADB_STOPPED" = true ]; then
+        echo "=== Cleanup: Starting MariaDB container ==="
+        docker start mariadb || echo "WARNING: Failed to start mariadb container"
+        sleep 3
+        echo "MariaDB restart completed."
+    fi
+
+    rm -f "$LOCKFILE"
+
+    if [ "$exit_code" -eq 0 ]; then
+        echo "=== Backup completed SUCCESSFULLY at $(date '+%Y-%m-%d %H:%M:%S') ==="
+        send_healthcheck
+    else
+        echo "=== Backup FAILED at $(date '+%Y-%m-%d %H:%M:%S') (exit code $exit_code) ==="
+        send_healthcheck "fail"
+    fi
+    echo "Full log: $LOGFILE"
+}
+
+if [ -e "$LOCKFILE" ]; then
+    echo "ERROR: Another backup is already running (lockfile exists)"
+    echo "Remove it manually if stale: $LOCKFILE"
+    exit 1
+fi
+
+touch "$LOCKFILE"
+trap 'cleanup $?' EXIT
+
+# ----------------------- Borg & Functions -----------------------
+export BORG_REPO="$REPO"
+export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes
+
+send_healthcheck() {
+    local status="${1:-}"
+    if [[ -n "${HEALTHCHECK_URL:-}" ]] && [[ "$HEALTHCHECK_URL" != *"your-uuid-here"* ]]; then
+        case "$status" in
+            start) curl -s -m 10 --retry 3 "${HEALTHCHECK_URL}/start" >/dev/null || true ;;
+            fail)  curl -s -m 10 --retry 3 "${HEALTHCHECK_URL}/fail" -d "Backup failed - check $LOGFILE" >/dev/null || true ;;
+            *)     curl -s -m 10 --retry 3 "$HEALTHCHECK_URL" >/dev/null || true ;;
+        esac
+    fi
+}
+
+run_cmd() {
+    echo "[RUN] $*"
+    "$@"
+}
+
+# =================================================================
+
+send_healthcheck "start"
+
+# 1. MariaDB dump (while running)
+echo "=== Step 1: MariaDB dump ==="
+DUMP_SCRIPT="${TARGET}/mariadb/dump_db.sh"
+if [[ -x "$DUMP_SCRIPT" ]]; then
+    run_cmd "$DUMP_SCRIPT"
+else
+    echo "WARNING: Dump script not found or not executable: $DUMP_SCRIPT"
+fi
+
+# 2. Stop MariaDB for consistent backup
+echo "=== Step 2: Stopping MariaDB container ==="
+run_cmd docker stop mariadb
+MARIADB_STOPPED=true
+
+# 3. Create Borg archive
+echo "=== Step 3: Creating Borg archive ==="
+run_cmd borg create \
+    --stats \
+    --progress \
+    --list \
+    --filter=AME \
+    --compression zstd,8 \
+    --exclude-caches \
+    --exclude-if-present .nobackup \
+    "::${now:%Y-%m-%dT%H-%M-%S}" \
+    "$TARGET"
+
+# 4. Restart MariaDB (this line is now also in the cleanup trap)
+echo "=== Step 4: Starting MariaDB container ==="
+run_cmd docker start mariadb
+MARIADB_STOPPED=false
+
+# 5. Offsite sync
+echo "=== Step 5: Syncing to Scaleway S3 ==="
+run_cmd rclone sync -v \
+    --fast-list \
+    --transfers=8 \
+    --checkers=16 \
+    "$REPO" "${RCLONE_REMOTE}:/par-backup-1/$NAME"
+
+# 6. Prune & Compact
+echo "=== Step 6: Pruning and compacting ==="
+run_cmd borg prune \
+    --list \
+    --keep-daily=7 \
+    --keep-weekly=4 \
+    --keep-monthly=6 \
+    --keep-within=7d \
+    "$REPO"
+
+run_cmd borg compact --progress "$REPO"
+
+# The cleanup trap will run automatically and mark success

old/maria.sh

@@ -0,0 +1,36 @@
+root@chaudron:/home/srv/files/content/mariadb# cat dump_db.sh
+#!/bin/bash
+
+# Load configuration from environment variables
+MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-MySuperDatabase}
+DUMP_DIR=${DUMP_DIR:-dump}
+DOCKER_CONTAINER_NAME=${DOCKER_CONTAINER_NAME:-mariadb}
+
+# Check dependencies
+if ! command -v docker &>/dev/null; then
+  echo "Docker is not installed or not in the PATH"
+  exit 1
+fi
+
+if ! docker ps -q -f name=$DOCKER_CONTAINER_NAME; then
+  echo "Docker container '$DOCKER_CONTAINER_NAME' is not running"
+  exit 1
+fi
+
+# Create dump directory if it doesn't exist
+mkdir -p "$(dirname "$0")/$DUMP_DIR"
+
+# Delete old dump files
+rm -f "$(dirname "$0")/$DUMP_DIR"/*.sql
+
+# Get list of databases and exclude system databases
+databases=$(docker exec -it $DOCKER_CONTAINER_NAME mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "SHOW DATABASES;" --skip-column-names -s | grep -Ev "(information_schema|mysql|performance_schema|sys)" | tr -d '\r')
+
+# Iterate through databases and dump them individually
+for db in $databases; do
+  echo "Dumping database: $db"
+  docker exec -i $DOCKER_CONTAINER_NAME mysqldump -u root -p"$MYSQL_ROOT_PASSWORD" "$db" >"$(dirname "$0")/$DUMP_DIR/$db.sql"
+  echo "Dumped database: $db"
+done
+
+echo "All non-system databases dumped to individual files in '$DUMP_DIR' directory."