chore: initial import of backup scripts and design docs
Existing borg-backup.sh/dump_db.sh, old/ reference scripts, and the brainstormed design spec for restore tooling + runbook.
This commit is contained in:
+318
@@ -0,0 +1,318 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# Borg backup: /content + MariaDB -> local repo -> Scaleway S3 mirror
|
||||
# =============================================================================
|
||||
# CHANGES vs. original:
|
||||
# • Encryption enabled (repokey-blake2 via BORG_PASSCOMMAND)
|
||||
# • Preflight warns if repo is unencrypted
|
||||
# • Added borg check (daily archive-only, weekly full --verify-data)
|
||||
# • Removed BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK
|
||||
# • No longer stops MariaDB: dump_db.sh's --single-transaction dump is
|
||||
# already consistent, and the container's raw data directory is
|
||||
# excluded from the archive (drop a .nobackup file in it - see
|
||||
# --exclude-if-present below) so no live InnoDB file is ever copied.
|
||||
# Zero DB downtime during backup.
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
umask 077
|
||||
|
||||
# ========================= CONFIGURATION =========================
|
||||
|
||||
NAME="borg-2025"
|
||||
REPO="/home/srv/files/backups/$NAME"
|
||||
TARGET="/home/srv/files/content"
|
||||
|
||||
REPO_MOUNT=""
|
||||
|
||||
LOGDIR="/var/log/borg"
|
||||
LOG_RETENTION_DAYS=90
|
||||
|
||||
HEALTHCHECK_URL="https://hc-ping.com/your-uuid-here"
|
||||
|
||||
RCLONE_REMOTE="scaleway"
|
||||
RCLONE_PATH="par-backup-1/$NAME"
|
||||
RCLONE_MAX_DELETE=200
|
||||
|
||||
LOCKFILE="/var/lock/borg-backup.lock"
|
||||
|
||||
DB_CONTAINER="mariadb"
|
||||
DB_START_TIMEOUT=180
|
||||
|
||||
CREATE_TIMEOUT="6h"
|
||||
PRUNE_TIMEOUT="2h"
|
||||
SYNC_TIMEOUT="12h"
|
||||
CHECK_TIMEOUT="4h"
|
||||
|
||||
# Passphrase file: chmod 600, owned by the backup user.
|
||||
# Create it with: echo 'your-strong-passphrase' > /root/.borg-passphrase
|
||||
BORG_PASSPHRASE_FILE="${BORG_PASSPHRASE_FILE:-/root/.borg-passphrase}"
|
||||
|
||||
REQUIRED_CMDS=(borg rclone docker curl timeout flock date find)
|
||||
|
||||
# =================================================================
|
||||
|
||||
SELF="$(readlink -f "$0")"
|
||||
mkdir -p "$LOGDIR"
|
||||
LOGFILE="${BORG_BACKUP_LOGFILE:-$LOGDIR/backup-$(date +%Y-%m-%d-%H%M%S).log}"
|
||||
|
||||
# --- Logging: re-exec ourselves through tee ------------------------------
|
||||
if [[ -z "${BORG_BACKUP_LOG_WRAPPED:-}" ]]; then
|
||||
export BORG_BACKUP_LOG_WRAPPED=1
|
||||
export BORG_BACKUP_LOGFILE="$LOGFILE"
|
||||
set +e
|
||||
"$SELF" "$@" 2>&1 | tee -a "$LOGFILE"
|
||||
exit "${PIPESTATUS[0]}"
|
||||
fi
|
||||
|
||||
# --- Borg environment ---------------------------------------------------
|
||||
export BORG_REPO="$REPO"
|
||||
export BORG_RELOCATED_REPO_ACCESS_IS_OK=no
|
||||
export BORG_PASSCOMMAND="cat $BORG_PASSPHRASE_FILE"
|
||||
|
||||
ARCHIVE="$(hostname -s)-$(date +%Y-%m-%dT%H-%M-%S)"
|
||||
ARCHIVE_GLOB="$(hostname -s)-*"
|
||||
|
||||
# ----------------------- Functions (all before the trap) -----------------
|
||||
|
||||
log() { echo "[$(date '+%F %T')] $*"; }
|
||||
step() { echo; echo "=== $* ==="; }
|
||||
|
||||
run_cmd() {
|
||||
log "[RUN] $*"
|
||||
"$@"
|
||||
}
|
||||
|
||||
send_healthcheck() {
|
||||
local status="${1:-0}"
|
||||
[[ -n "${HEALTHCHECK_URL:-}" ]] || return 0
|
||||
[[ "$HEALTHCHECK_URL" != *"your-uuid-here"* ]] || return 0
|
||||
command -v curl >/dev/null 2>&1 || return 0
|
||||
|
||||
local curl_opts=(-fsS -m 15 --retry 3 --retry-connrefused)
|
||||
case "$status" in
|
||||
start) curl "${curl_opts[@]}" "${HEALTHCHECK_URL}/start" >/dev/null || true ;;
|
||||
0) curl "${curl_opts[@]}" "$HEALTHCHECK_URL" >/dev/null || true ;;
|
||||
*) curl "${curl_opts[@]}" "${HEALTHCHECK_URL}/${status}" \
|
||||
--data-raw "exit=$status archive=$ARCHIVE log=$LOGFILE" >/dev/null || true ;;
|
||||
esac
|
||||
}
|
||||
|
||||
container_running() {
|
||||
[[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || echo false)" == "true" ]]
|
||||
}
|
||||
|
||||
container_health() {
|
||||
docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
|
||||
"$1" 2>/dev/null || echo unknown
|
||||
}
|
||||
|
||||
wait_for_container() {
|
||||
local name="$1" deadline=$((SECONDS + DB_START_TIMEOUT)) health
|
||||
while (( SECONDS < deadline )); do
|
||||
if container_running "$name"; then
|
||||
health="$(container_health "$name")"
|
||||
case "$health" in
|
||||
healthy|none) log "$name is up (health: $health)"; return 0 ;;
|
||||
unhealthy) log "WARNING: $name reports unhealthy" ;;
|
||||
esac
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
start_db() {
|
||||
docker start "$DB_CONTAINER" >/dev/null 2>&1 || true
|
||||
if wait_for_container "$DB_CONTAINER"; then
|
||||
return 0
|
||||
fi
|
||||
log "ERROR: $DB_CONTAINER did not come back up within ${DB_START_TIMEOUT}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local exit_code=$1
|
||||
set +e
|
||||
|
||||
# We never stop the container ourselves anymore, but if it crashed for
|
||||
# an unrelated reason during the backup window, try to bring it back.
|
||||
if ! container_running "$DB_CONTAINER"; then
|
||||
step "Cleanup: $DB_CONTAINER is down, attempting restart"
|
||||
start_db || { log "CRITICAL: $DB_CONTAINER is DOWN - manual action required"
|
||||
[ "$exit_code" -eq 0 ] && exit_code=1; }
|
||||
fi
|
||||
|
||||
find "$LOGDIR" -maxdepth 1 -name 'backup-*.log' -type f \
|
||||
-mtime +"$LOG_RETENTION_DAYS" -delete 2>/dev/null
|
||||
|
||||
if [ "$exit_code" -eq 0 ]; then
|
||||
log "=== Backup completed SUCCESSFULLY (archive: $ARCHIVE) ==="
|
||||
else
|
||||
log "=== Backup FAILED (exit $exit_code) ==="
|
||||
fi
|
||||
log "Full log: $LOGFILE"
|
||||
send_healthcheck "$exit_code"
|
||||
}
|
||||
|
||||
# Returns 0 if the repo is encrypted, 1 if not.
|
||||
repo_is_encrypted() {
|
||||
borg info "$REPO" 2>&1 | grep -q "Encrypted:.*Yes"
|
||||
}
|
||||
|
||||
preflight() {
|
||||
local missing=()
|
||||
for c in "${REQUIRED_CMDS[@]}"; do
|
||||
command -v "$c" >/dev/null 2>&1 || missing+=("$c")
|
||||
done
|
||||
if (( ${#missing[@]} )); then
|
||||
log "ERROR: missing commands: ${missing[*]}"; return 1
|
||||
fi
|
||||
|
||||
if [[ -n "$REPO_MOUNT" ]] && ! mountpoint -q "$REPO_MOUNT"; then
|
||||
log "ERROR: $REPO_MOUNT is not mounted - refusing to run"
|
||||
return 1
|
||||
fi
|
||||
|
||||
[[ -d "$TARGET" ]] || { log "ERROR: target missing: $TARGET"; return 1; }
|
||||
[[ -n "$(ls -A "$TARGET")" ]] || { log "ERROR: target is empty: $TARGET"; return 1; }
|
||||
|
||||
case "$REPO/" in
|
||||
"$TARGET"/*) log "ERROR: repo lives inside target - infinite recursion"; return 1 ;;
|
||||
esac
|
||||
|
||||
if ! borg info --lock-wait 60 >/dev/null 2>&1; then
|
||||
log "ERROR: cannot open repository $REPO"
|
||||
log " (wrong path? disk not mounted? stale lock? -> borg break-lock)"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Warn if the repo is not encrypted.
|
||||
if ! repo_is_encrypted; then
|
||||
log "WARNING: Repository is NOT encrypted!"
|
||||
log " Run: borg init --encryption=repokey-blake2 $REPO"
|
||||
log " (after backing up any existing data)"
|
||||
fi
|
||||
|
||||
# Verify passphrase file exists and is readable.
|
||||
if [[ ! -r "$BORG_PASSPHRASE_FILE" ]]; then
|
||||
log "ERROR: passphrase file not readable: $BORG_PASSPHRASE_FILE"
|
||||
log " Create it: echo 'your-passphrase' > $BORG_PASSPHRASE_FILE"
|
||||
log " chmod 600 $BORG_PASSPHRASE_FILE"
|
||||
return 1
|
||||
fi
|
||||
|
||||
container_running "$DB_CONTAINER" \
|
||||
|| { log "ERROR: $DB_CONTAINER is not running before we start"; return 1; }
|
||||
|
||||
log "Preflight OK"
|
||||
}
|
||||
|
||||
# ----------------------- Lock, then trap --------------------------------
|
||||
|
||||
exec 9>"$LOCKFILE"
|
||||
if ! flock -n 9; then
|
||||
echo "ERROR: another backup is already running (lock held on $LOCKFILE)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
trap 'cleanup $?' EXIT
|
||||
trap 'log "SIGINT received"; exit 130' INT
|
||||
trap 'log "SIGTERM received"; exit 143' TERM
|
||||
|
||||
# =================================================================
|
||||
|
||||
log "=== Backup started ==="
|
||||
echo "Repository: $REPO"
|
||||
echo "Target: $TARGET"
|
||||
echo "Archive: $ARCHIVE"
|
||||
echo "Log: $LOGFILE"
|
||||
|
||||
send_healthcheck start
|
||||
|
||||
step "Step 0: Preflight"
|
||||
preflight
|
||||
|
||||
# --- 1. Logical dumps (container stays up the whole time) ---------------
|
||||
step "Step 1: MariaDB dumps"
|
||||
DUMP_SCRIPT="${TARGET}/mariadb/dump_db.sh"
|
||||
DUMP_DIR="${TARGET}/mariadb/dump"
|
||||
|
||||
if [[ ! -x "$DUMP_SCRIPT" ]]; then
|
||||
log "ERROR: dump script missing or not executable: $DUMP_SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
run_cmd "$DUMP_SCRIPT"
|
||||
|
||||
if [[ -d "$DUMP_DIR" ]]; then
|
||||
fresh=$(find "$DUMP_DIR" -type f -size +1k -mmin -60 | wc -l)
|
||||
empty=$(find "$DUMP_DIR" -type f -size -1k -mmin -60 | wc -l)
|
||||
log "Dumps: $fresh fresh non-trivial file(s), $empty suspiciously small"
|
||||
(( fresh > 0 )) || { log "ERROR: no usable dumps produced"; exit 1; }
|
||||
(( empty == 0 )) || log "WARNING: $empty near-empty dump file(s) - check $DUMP_DIR"
|
||||
else
|
||||
log "WARNING: dump directory not found: $DUMP_DIR"
|
||||
fi
|
||||
|
||||
# --- 2. Create the archive -----------------------------------------------
|
||||
# MariaDB stays up throughout: the dump above is already transactionally
|
||||
# consistent, and the container's raw data directory carries a .nobackup
|
||||
# marker (excluded here via --exclude-if-present) so its live files are
|
||||
# never read by borg.
|
||||
step "Step 2: Creating archive $ARCHIVE"
|
||||
container_running "$DB_CONTAINER" \
|
||||
|| { log "ERROR: $DB_CONTAINER is not running - refusing to archive"; exit 1; }
|
||||
run_cmd timeout --signal=INT --kill-after=120s "$CREATE_TIMEOUT" \
|
||||
borg create \
|
||||
--lock-wait 600 \
|
||||
--stats \
|
||||
--list --filter=AME \
|
||||
--compression zstd,8 \
|
||||
--exclude-caches \
|
||||
--exclude-if-present .nobackup \
|
||||
--keep-exclude-tags \
|
||||
"::$ARCHIVE" \
|
||||
"$TARGET"
|
||||
|
||||
# --- 3/4. Retention -------------------------------------------------------
|
||||
step "Step 3: Pruning"
|
||||
run_cmd timeout "$PRUNE_TIMEOUT" \
|
||||
borg prune \
|
||||
--lock-wait 600 \
|
||||
--list \
|
||||
--glob-archives "$ARCHIVE_GLOB" \
|
||||
--keep-daily=7 \
|
||||
--keep-weekly=4 \
|
||||
--keep-monthly=6
|
||||
|
||||
step "Step 4: Compacting"
|
||||
run_cmd timeout "$PRUNE_TIMEOUT" borg compact --lock-wait 600
|
||||
|
||||
# --- 5. Integrity check -------------------------------------------------
|
||||
step "Step 5: Verifying archive & repository integrity"
|
||||
run_cmd borg info --lock-wait 60 "::$ARCHIVE" >/dev/null
|
||||
ARCHIVE_COUNT=$(borg list --short --glob-archives "$ARCHIVE_GLOB" | wc -l)
|
||||
log "Repository holds $ARCHIVE_COUNT archive(s) for $ARCHIVE_GLOB"
|
||||
(( ARCHIVE_COUNT > 0 )) || { log "ERROR: repo unexpectedly empty"; exit 1; }
|
||||
|
||||
# Daily: fast archive-only check. Weekly (Monday): full repo check with data verification.
|
||||
if [[ "$(date +%u)" == "1" ]]; then
|
||||
log "Monday — running full repository check with --verify-data (this may take a while)"
|
||||
run_cmd timeout "$CHECK_TIMEOUT" borg check --verify-data --lock-wait 600 "$REPO"
|
||||
else
|
||||
log "Running fast archive-only integrity check"
|
||||
run_cmd timeout 1h borg check --archives-only --lock-wait 600 "$REPO"::"$ARCHIVE"
|
||||
fi
|
||||
|
||||
# --- 6. Offsite mirror --------------------------------------------------
|
||||
step "Step 6: Syncing to ${RCLONE_REMOTE}:${RCLONE_PATH}"
|
||||
run_cmd timeout "$SYNC_TIMEOUT" \
|
||||
rclone sync -v \
|
||||
--fast-list \
|
||||
--transfers=8 \
|
||||
--checkers=16 \
|
||||
--s3-no-check-bucket \
|
||||
--max-delete "$RCLONE_MAX_DELETE" \
|
||||
"$REPO" "${RCLONE_REMOTE}:${RCLONE_PATH}"
|
||||
Reference in New Issue
Block a user