chore: initial import of backup scripts and design docs

Existing borg-backup.sh/dump_db.sh, old/ reference scripts, and the brainstormed design spec for restore tooling + runbook.
2026-07-25 18:51:38 +02:00
commit d6f30c506e
7 changed files with 1907 additions and 0 deletions
@@ -0,0 +1,145 @@
+#!/bin/bash
+# =============================================================================
+# Borg Backup Script - MariaDB Restart Guaranteed
+# =============================================================================
+
+set -euo pipefail
+
+# ========================= CONFIGURATION =========================
+
+NAME="borg-2025"
+REPO="/home/srv/files/backups/$NAME"
+TARGET="/home/srv/files/content"
+
+LOGDIR="/var/log/borg"
+LOGFILE="${LOGDIR}/backup-$(date +%Y-%m-%d-%H%M%S).log"
+
+# Optional but strongly recommended
+HEALTHCHECK_URL="https://hc-ping.com/your-uuid-here"
+
+RCLONE_REMOTE="scaleway"
+
+# =================================================================
+
+mkdir -p "$LOGDIR"
+exec > >(tee -a "$LOGFILE")
+exec 2>&1
+
+echo "=== Backup started at $(date '+%Y-%m-%d %H:%M:%S') ==="
+echo "Repository: $REPO"
+echo "Target: $TARGET"
+echo "Log: $LOGFILE"
+echo "-----------------------------------------------------------"
+
+# ----------------------- Lockfile -----------------------
+LOCKFILE="/var/lock/borg-backup.lock"
+MARIADB_STOPPED=false
+
+cleanup() {
+    local exit_code=${1:-$?}
+
+    if [ "$MARIADB_STOPPED" = true ]; then
+        echo "=== Cleanup: Starting MariaDB container ==="
+        docker start mariadb || echo "WARNING: Failed to start mariadb container"
+        sleep 3
+        echo "MariaDB restart completed."
+    fi
+
+    rm -f "$LOCKFILE"
+
+    if [ "$exit_code" -eq 0 ]; then
+        echo "=== Backup completed SUCCESSFULLY at $(date '+%Y-%m-%d %H:%M:%S') ==="
+        send_healthcheck
+    else
+        echo "=== Backup FAILED at $(date '+%Y-%m-%d %H:%M:%S') (exit code $exit_code) ==="
+        send_healthcheck "fail"
+    fi
+    echo "Full log: $LOGFILE"
+}
+
+if [ -e "$LOCKFILE" ]; then
+    echo "ERROR: Another backup is already running (lockfile exists)"
+    echo "Remove it manually if stale: $LOCKFILE"
+    exit 1
+fi
+
+touch "$LOCKFILE"
+trap 'cleanup $?' EXIT
+
+# ----------------------- Borg & Functions -----------------------
+export BORG_REPO="$REPO"
+export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes
+
+send_healthcheck() {
+    local status="${1:-}"
+    if [[ -n "${HEALTHCHECK_URL:-}" ]] && [[ "$HEALTHCHECK_URL" != *"your-uuid-here"* ]]; then
+        case "$status" in
+            start) curl -s -m 10 --retry 3 "${HEALTHCHECK_URL}/start" >/dev/null || true ;;
+            fail)  curl -s -m 10 --retry 3 "${HEALTHCHECK_URL}/fail" -d "Backup failed - check $LOGFILE" >/dev/null || true ;;
+            *)     curl -s -m 10 --retry 3 "$HEALTHCHECK_URL" >/dev/null || true ;;
+        esac
+    fi
+}
+
+run_cmd() {
+    echo "[RUN] $*"
+    "$@"
+}
+
+# =================================================================
+
+send_healthcheck "start"
+
+# 1. MariaDB dump (while running)
+echo "=== Step 1: MariaDB dump ==="
+DUMP_SCRIPT="${TARGET}/mariadb/dump_db.sh"
+if [[ -x "$DUMP_SCRIPT" ]]; then
+    run_cmd "$DUMP_SCRIPT"
+else
+    echo "WARNING: Dump script not found or not executable: $DUMP_SCRIPT"
+fi
+
+# 2. Stop MariaDB for consistent backup
+echo "=== Step 2: Stopping MariaDB container ==="
+run_cmd docker stop mariadb
+MARIADB_STOPPED=true
+
+# 3. Create Borg archive
+echo "=== Step 3: Creating Borg archive ==="
+run_cmd borg create \
+    --stats \
+    --progress \
+    --list \
+    --filter=AME \
+    --compression zstd,8 \
+    --exclude-caches \
+    --exclude-if-present .nobackup \
+    "::${now:%Y-%m-%dT%H-%M-%S}" \
+    "$TARGET"
+
+# 4. Restart MariaDB (this line is now also in the cleanup trap)
+echo "=== Step 4: Starting MariaDB container ==="
+run_cmd docker start mariadb
+MARIADB_STOPPED=false
+
+# 5. Offsite sync
+echo "=== Step 5: Syncing to Scaleway S3 ==="
+run_cmd rclone sync -v \
+    --fast-list \
+    --transfers=8 \
+    --checkers=16 \
+    "$REPO" "${RCLONE_REMOTE}:/par-backup-1/$NAME"
+
+# 6. Prune & Compact
+echo "=== Step 6: Pruning and compacting ==="
+run_cmd borg prune \
+    --list \
+    --keep-daily=7 \
+    --keep-weekly=4 \
+    --keep-monthly=6 \
+    --keep-within=7d \
+    "$REPO"
+
+run_cmd borg compact --progress "$REPO"
+
+# The cleanup trap will run automatically and mark success
@@ -0,0 +1,36 @@
+root@chaudron:/home/srv/files/content/mariadb# cat dump_db.sh
+#!/bin/bash
+
+# Load configuration from environment variables
+MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-MySuperDatabase}
+DUMP_DIR=${DUMP_DIR:-dump}
+DOCKER_CONTAINER_NAME=${DOCKER_CONTAINER_NAME:-mariadb}
+
+# Check dependencies
+if ! command -v docker &>/dev/null; then
+  echo "Docker is not installed or not in the PATH"
+  exit 1
+fi
+
+if ! docker ps -q -f name=$DOCKER_CONTAINER_NAME; then
+  echo "Docker container '$DOCKER_CONTAINER_NAME' is not running"
+  exit 1
+fi
+
+# Create dump directory if it doesn't exist
+mkdir -p "$(dirname "$0")/$DUMP_DIR"
+
+# Delete old dump files
+rm -f "$(dirname "$0")/$DUMP_DIR"/*.sql
+
+# Get list of databases and exclude system databases
+databases=$(docker exec -it $DOCKER_CONTAINER_NAME mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "SHOW DATABASES;" --skip-column-names -s | grep -Ev "(information_schema|mysql|performance_schema|sys)" | tr -d '\r')
+
+# Iterate through databases and dump them individually
+for db in $databases; do
+  echo "Dumping database: $db"
+  docker exec -i $DOCKER_CONTAINER_NAME mysqldump -u root -p"$MYSQL_ROOT_PASSWORD" "$db" >"$(dirname "$0")/$DUMP_DIR/$db.sql"
+  echo "Dumped database: $db"
+done
+
+echo "All non-system databases dumped to individual files in '$DUMP_DIR' directory."