kbeandClaude Sonnet 5
dd4fd8cb9c
dump_db.sh no longer lives under $TARGET, so its own default dump directory (relative to wherever the script is) would land outside the backed-up tree. Pin DUMP_SCRIPT to /opt/backup-agent/dump_db.sh and export DUMP_DIR explicitly so dumps still land in $TARGET/mariadb/dump regardless of where the script itself is deployed. Also drop the healthcheck integration (HEALTHCHECK_URL, send_healthcheck, curl requirement) from borg-backup.sh per request - no monitoring hook wanted for now. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
303 lines
10 KiB
Bash
Executable File
303 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
|
# =============================================================================
|
|
# Borg backup: /content + MariaDB -> local repo -> Scaleway S3 mirror
|
|
# =============================================================================
|
|
# CHANGES vs. original:
|
|
# • Encryption enabled (repokey-blake2 via BORG_PASSCOMMAND)
|
|
# • Preflight warns if repo is unencrypted
|
|
# • Added borg check (daily archive-only, weekly full --verify-data)
|
|
# • Removed BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK
|
|
# • No longer stops MariaDB: dump_db.sh's --single-transaction dump is
|
|
# already consistent, and the container's raw data directory is
|
|
# excluded from the archive (drop a .nobackup file in it - see
|
|
# --exclude-if-present below) so no live InnoDB file is ever copied.
|
|
# Zero DB downtime during backup.
|
|
# =============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
umask 077
|
|
|
|
# ========================= CONFIGURATION =========================
|
|
|
|
NAME="borg-2025"
|
|
REPO="/home/srv/files/backups/$NAME"
|
|
TARGET="/home/srv/files/content"
|
|
|
|
# dump_db.sh lives with the rest of this toolkit, not inside $TARGET - its
|
|
# own default dump dir is relative to wherever IT lives, so DUMP_DIR must be
|
|
# passed explicitly (below) to keep dumps inside $TARGET where borg can see them.
|
|
DUMP_SCRIPT="/opt/backup-agent/dump_db.sh"
|
|
export DUMP_DIR="${TARGET}/mariadb/dump"
|
|
|
|
REPO_MOUNT=""
|
|
|
|
LOGDIR="/var/log/borg"
|
|
LOG_RETENTION_DAYS=90
|
|
|
|
RCLONE_REMOTE="scaleway"
|
|
RCLONE_PATH="par-backup-1/$NAME"
|
|
RCLONE_MAX_DELETE=200
|
|
|
|
LOCKFILE="/var/lock/borg-backup.lock"
|
|
|
|
DB_CONTAINER="mariadb"
|
|
DB_START_TIMEOUT=180
|
|
|
|
CREATE_TIMEOUT="6h"
|
|
PRUNE_TIMEOUT="2h"
|
|
SYNC_TIMEOUT="12h"
|
|
CHECK_TIMEOUT="4h"
|
|
|
|
# Passphrase file: chmod 600, owned by the backup user.
|
|
# Create it with: echo 'your-strong-passphrase' > /root/.borg-passphrase
|
|
BORG_PASSPHRASE_FILE="${BORG_PASSPHRASE_FILE:-/root/.borg-passphrase}"
|
|
|
|
REQUIRED_CMDS=(borg rclone docker timeout flock date find)
|
|
|
|
# =================================================================
|
|
|
|
SELF="$(readlink -f "$0")"
|
|
mkdir -p "$LOGDIR"
|
|
LOGFILE="${BORG_BACKUP_LOGFILE:-$LOGDIR/backup-$(date +%Y-%m-%d-%H%M%S).log}"
|
|
|
|
# --- Logging: re-exec ourselves through tee ------------------------------
|
|
if [[ -z "${BORG_BACKUP_LOG_WRAPPED:-}" ]]; then
|
|
export BORG_BACKUP_LOG_WRAPPED=1
|
|
export BORG_BACKUP_LOGFILE="$LOGFILE"
|
|
set +e
|
|
"$SELF" "$@" 2>&1 | tee -a "$LOGFILE"
|
|
exit "${PIPESTATUS[0]}"
|
|
fi
|
|
|
|
# --- Borg environment ---------------------------------------------------
|
|
export BORG_REPO="$REPO"
|
|
export BORG_RELOCATED_REPO_ACCESS_IS_OK=no
|
|
export BORG_PASSCOMMAND="cat $BORG_PASSPHRASE_FILE"
|
|
|
|
ARCHIVE="$(hostname -s)-$(date +%Y-%m-%dT%H-%M-%S)"
|
|
ARCHIVE_GLOB="$(hostname -s)-*"
|
|
|
|
# ----------------------- Functions (all before the trap) -----------------
|
|
|
|
log() { echo "[$(date '+%F %T')] $*"; }
|
|
step() { echo; echo "=== $* ==="; }
|
|
|
|
run_cmd() {
|
|
log "[RUN] $*"
|
|
"$@"
|
|
}
|
|
|
|
container_running() {
|
|
[[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || echo false)" == "true" ]]
|
|
}
|
|
|
|
container_health() {
|
|
docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
|
|
"$1" 2>/dev/null || echo unknown
|
|
}
|
|
|
|
wait_for_container() {
|
|
local name="$1" deadline=$((SECONDS + DB_START_TIMEOUT)) health
|
|
while (( SECONDS < deadline )); do
|
|
if container_running "$name"; then
|
|
health="$(container_health "$name")"
|
|
case "$health" in
|
|
healthy|none) log "$name is up (health: $health)"; return 0 ;;
|
|
unhealthy) log "WARNING: $name reports unhealthy" ;;
|
|
esac
|
|
fi
|
|
sleep 3
|
|
done
|
|
return 1
|
|
}
|
|
|
|
start_db() {
|
|
docker start "$DB_CONTAINER" >/dev/null 2>&1 || true
|
|
if wait_for_container "$DB_CONTAINER"; then
|
|
return 0
|
|
fi
|
|
log "ERROR: $DB_CONTAINER did not come back up within ${DB_START_TIMEOUT}s"
|
|
return 1
|
|
}
|
|
|
|
cleanup() {
|
|
local exit_code=$1
|
|
set +e
|
|
|
|
# We never stop the container ourselves anymore, but if it crashed for
|
|
# an unrelated reason during the backup window, try to bring it back.
|
|
if ! container_running "$DB_CONTAINER"; then
|
|
step "Cleanup: $DB_CONTAINER is down, attempting restart"
|
|
start_db || { log "CRITICAL: $DB_CONTAINER is DOWN - manual action required"
|
|
[ "$exit_code" -eq 0 ] && exit_code=1; }
|
|
fi
|
|
|
|
find "$LOGDIR" -maxdepth 1 -name 'backup-*.log' -type f \
|
|
-mtime +"$LOG_RETENTION_DAYS" -delete 2>/dev/null
|
|
|
|
if [ "$exit_code" -eq 0 ]; then
|
|
log "=== Backup completed SUCCESSFULLY (archive: $ARCHIVE) ==="
|
|
else
|
|
log "=== Backup FAILED (exit $exit_code) ==="
|
|
fi
|
|
log "Full log: $LOGFILE"
|
|
}
|
|
|
|
# Returns 0 if the repo is encrypted, 1 if not.
|
|
repo_is_encrypted() {
|
|
borg info "$REPO" 2>&1 | grep -q "Encrypted:.*Yes"
|
|
}
|
|
|
|
preflight() {
|
|
local missing=()
|
|
for c in "${REQUIRED_CMDS[@]}"; do
|
|
command -v "$c" >/dev/null 2>&1 || missing+=("$c")
|
|
done
|
|
if (( ${#missing[@]} )); then
|
|
log "ERROR: missing commands: ${missing[*]}"; return 1
|
|
fi
|
|
|
|
if [[ -n "$REPO_MOUNT" ]] && ! mountpoint -q "$REPO_MOUNT"; then
|
|
log "ERROR: $REPO_MOUNT is not mounted - refusing to run"
|
|
return 1
|
|
fi
|
|
|
|
[[ -d "$TARGET" ]] || { log "ERROR: target missing: $TARGET"; return 1; }
|
|
[[ -n "$(ls -A "$TARGET")" ]] || { log "ERROR: target is empty: $TARGET"; return 1; }
|
|
|
|
case "$REPO/" in
|
|
"$TARGET"/*) log "ERROR: repo lives inside target - infinite recursion"; return 1 ;;
|
|
esac
|
|
|
|
if ! borg info --lock-wait 60 >/dev/null 2>&1; then
|
|
log "ERROR: cannot open repository $REPO"
|
|
log " (wrong path? disk not mounted? stale lock? -> borg break-lock)"
|
|
return 1
|
|
fi
|
|
|
|
# Warn if the repo is not encrypted.
|
|
if ! repo_is_encrypted; then
|
|
log "WARNING: Repository is NOT encrypted!"
|
|
log " Run: borg init --encryption=repokey-blake2 $REPO"
|
|
log " (after backing up any existing data)"
|
|
fi
|
|
|
|
# Verify passphrase file exists and is readable.
|
|
if [[ ! -r "$BORG_PASSPHRASE_FILE" ]]; then
|
|
log "ERROR: passphrase file not readable: $BORG_PASSPHRASE_FILE"
|
|
log " Create it: echo 'your-passphrase' > $BORG_PASSPHRASE_FILE"
|
|
log " chmod 600 $BORG_PASSPHRASE_FILE"
|
|
return 1
|
|
fi
|
|
|
|
container_running "$DB_CONTAINER" \
|
|
|| { log "ERROR: $DB_CONTAINER is not running before we start"; return 1; }
|
|
|
|
log "Preflight OK"
|
|
}
|
|
|
|
# ----------------------- Lock, then trap --------------------------------
|
|
|
|
exec 9>"$LOCKFILE"
|
|
if ! flock -n 9; then
|
|
echo "ERROR: another backup is already running (lock held on $LOCKFILE)"
|
|
exit 1
|
|
fi
|
|
|
|
trap 'cleanup $?' EXIT
|
|
trap 'log "SIGINT received"; exit 130' INT
|
|
trap 'log "SIGTERM received"; exit 143' TERM
|
|
|
|
# =================================================================
|
|
|
|
log "=== Backup started ==="
|
|
echo "Repository: $REPO"
|
|
echo "Target: $TARGET"
|
|
echo "Archive: $ARCHIVE"
|
|
echo "Log: $LOGFILE"
|
|
|
|
step "Step 0: Preflight"
|
|
preflight
|
|
|
|
# --- 1. Logical dumps (container stays up the whole time) ---------------
|
|
step "Step 1: MariaDB dumps"
|
|
|
|
if [[ ! -x "$DUMP_SCRIPT" ]]; then
|
|
log "ERROR: dump script missing or not executable: $DUMP_SCRIPT"
|
|
exit 1
|
|
fi
|
|
run_cmd "$DUMP_SCRIPT"
|
|
|
|
if [[ -d "$DUMP_DIR" ]]; then
|
|
fresh=$(find "$DUMP_DIR" -type f -size +1k -mmin -60 | wc -l)
|
|
empty=$(find "$DUMP_DIR" -type f -size -1k -mmin -60 | wc -l)
|
|
log "Dumps: $fresh fresh non-trivial file(s), $empty suspiciously small"
|
|
(( fresh > 0 )) || { log "ERROR: no usable dumps produced"; exit 1; }
|
|
(( empty == 0 )) || log "WARNING: $empty near-empty dump file(s) - check $DUMP_DIR"
|
|
else
|
|
log "WARNING: dump directory not found: $DUMP_DIR"
|
|
fi
|
|
|
|
# --- 2. Create the archive -----------------------------------------------
|
|
# MariaDB stays up throughout: the dump above is already transactionally
|
|
# consistent, and the container's raw data directory carries a .nobackup
|
|
# marker (excluded here via --exclude-if-present) so its live files are
|
|
# never read by borg.
|
|
step "Step 2: Creating archive $ARCHIVE"
|
|
container_running "$DB_CONTAINER" \
|
|
|| { log "ERROR: $DB_CONTAINER is not running - refusing to archive"; exit 1; }
|
|
run_cmd timeout --signal=INT --kill-after=120s "$CREATE_TIMEOUT" \
|
|
borg create \
|
|
--lock-wait 600 \
|
|
--stats \
|
|
--list --filter=AME \
|
|
--compression zstd,8 \
|
|
--exclude-caches \
|
|
--exclude-if-present .nobackup \
|
|
--keep-exclude-tags \
|
|
"::$ARCHIVE" \
|
|
"$TARGET"
|
|
|
|
# --- 3/4. Retention -------------------------------------------------------
|
|
step "Step 3: Pruning"
|
|
run_cmd timeout "$PRUNE_TIMEOUT" \
|
|
borg prune \
|
|
--lock-wait 600 \
|
|
--list \
|
|
--glob-archives "$ARCHIVE_GLOB" \
|
|
--keep-daily=7 \
|
|
--keep-weekly=4 \
|
|
--keep-monthly=6
|
|
|
|
step "Step 4: Compacting"
|
|
run_cmd timeout "$PRUNE_TIMEOUT" borg compact --lock-wait 600
|
|
|
|
# --- 5. Integrity check -------------------------------------------------
|
|
step "Step 5: Verifying archive & repository integrity"
|
|
run_cmd borg info --lock-wait 60 "::$ARCHIVE" >/dev/null
|
|
ARCHIVE_COUNT=$(borg list --short --glob-archives "$ARCHIVE_GLOB" | wc -l)
|
|
log "Repository holds $ARCHIVE_COUNT archive(s) for $ARCHIVE_GLOB"
|
|
(( ARCHIVE_COUNT > 0 )) || { log "ERROR: repo unexpectedly empty"; exit 1; }
|
|
|
|
# Daily: fast archive-only check. Weekly (Monday): full repo check with data verification.
|
|
if [[ "$(date +%u)" == "1" ]]; then
|
|
log "Monday — running full repository check with --verify-data (this may take a while)"
|
|
run_cmd timeout "$CHECK_TIMEOUT" borg check --verify-data --lock-wait 600 "$REPO"
|
|
else
|
|
log "Running fast archive-only integrity check"
|
|
run_cmd timeout 1h borg check --archives-only --lock-wait 600 "$REPO"::"$ARCHIVE"
|
|
fi
|
|
|
|
# --- 6. Offsite mirror --------------------------------------------------
|
|
step "Step 6: Syncing to ${RCLONE_REMOTE}:${RCLONE_PATH}"
|
|
run_cmd timeout "$SYNC_TIMEOUT" \
|
|
rclone sync -v \
|
|
--fast-list \
|
|
--transfers=8 \
|
|
--checkers=16 \
|
|
--s3-no-check-bucket \
|
|
--max-delete "$RCLONE_MAX_DELETE" \
|
|
"$REPO" "${RCLONE_REMOTE}:${RCLONE_PATH}"
|