#!/bin/bash
# =============================================================================
# Borg backup: /content + MariaDB  ->  local repo  ->  Scaleway S3 mirror
# =============================================================================
# CHANGES vs. original:
#   • Encryption enabled (repokey-blake2 via BORG_PASSCOMMAND)
#   • Preflight warns if repo is unencrypted
#   • Added borg check (daily archive-only, weekly full --verify-data)
#   • Removed BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK
#   • No longer stops MariaDB: dump_db.sh's --single-transaction dump is
#     already consistent, and the container's raw data directory is
#     excluded from the archive (drop a .nobackup file in it - see
#     --exclude-if-present below) so no live InnoDB file is ever copied.
#     Zero DB downtime during backup.
# =============================================================================

set -euo pipefail

export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
umask 077

# ========================= CONFIGURATION =========================

NAME="chaudron"
REPO="/home/srv/files/backups/$NAME"
TARGET="/home/srv/files/content"

# dump_db.sh lives with the rest of this toolkit, not inside $TARGET - its
# own default dump dir is relative to wherever IT lives, so DUMP_DIR must be
# passed explicitly (below) to keep dumps inside $TARGET where borg can see them.
DUMP_SCRIPT="/opt/backup-agent/dump_db.sh"
export DUMP_DIR="${TARGET}/mariadb/dump"

REPO_MOUNT=""

LOGDIR="/var/log/borg"
LOG_RETENTION_DAYS=90

RCLONE_REMOTE="scaleway"
RCLONE_PATH="cyanet-backups-prod/$NAME"

LOCKFILE="/var/lock/borg-backup.lock"

DB_CONTAINER="mariadb"
DB_START_TIMEOUT=180

CREATE_TIMEOUT="6h"
PRUNE_TIMEOUT="2h"
SYNC_TIMEOUT="12h"
CHECK_TIMEOUT="4h"

# Passphrase file: chmod 600, owned by the backup user.
# Create it with:  echo 'your-strong-passphrase' > /root/.borg-passphrase
BORG_PASSPHRASE_FILE="${BORG_PASSPHRASE_FILE:-/root/.borg-passphrase}"

REQUIRED_CMDS=(borg rclone docker timeout flock date find)

# =================================================================

SELF="$(readlink -f "$0")"
mkdir -p "$LOGDIR"
LOGFILE="${BORG_BACKUP_LOGFILE:-$LOGDIR/backup-$(date +%Y-%m-%d-%H%M%S).log}"

# --- Logging: re-exec ourselves through tee ------------------------------
if [[ -z "${BORG_BACKUP_LOG_WRAPPED:-}" ]]; then
    export BORG_BACKUP_LOG_WRAPPED=1
    export BORG_BACKUP_LOGFILE="$LOGFILE"
    set +e
    "$SELF" "$@" 2>&1 | tee -a "$LOGFILE"
    exit "${PIPESTATUS[0]}"
fi

# --- Borg environment ---------------------------------------------------
export BORG_REPO="$REPO"
export BORG_RELOCATED_REPO_ACCESS_IS_OK=no
export BORG_PASSCOMMAND="cat $BORG_PASSPHRASE_FILE"

ARCHIVE="$(hostname -s)-$(date +%Y-%m-%dT%H-%M-%S)"
ARCHIVE_GLOB="$(hostname -s)-*"

# ----------------------- Functions (all before the trap) -----------------

log()  { echo "[$(date '+%F %T')] $*"; }
step() { echo; echo "=== $* ==="; }

run_cmd() {
    log "[RUN] $*"
    "$@"
}

container_running() {
    [[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || echo false)" == "true" ]]
}

container_health() {
    docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
        "$1" 2>/dev/null || echo unknown
}

wait_for_container() {
    local name="$1" deadline=$((SECONDS + DB_START_TIMEOUT)) health
    while (( SECONDS < deadline )); do
        if container_running "$name"; then
            health="$(container_health "$name")"
            case "$health" in
                healthy|none) log "$name is up (health: $health)"; return 0 ;;
                unhealthy)    log "WARNING: $name reports unhealthy" ;;
            esac
        fi
        sleep 3
    done
    return 1
}

start_db() {
    docker start "$DB_CONTAINER" >/dev/null 2>&1 || true
    if wait_for_container "$DB_CONTAINER"; then
        return 0
    fi
    log "ERROR: $DB_CONTAINER did not come back up within ${DB_START_TIMEOUT}s"
    return 1
}

cleanup() {
    local exit_code=$1
    set +e

    # We never stop the container ourselves anymore, but if it crashed for
    # an unrelated reason during the backup window, try to bring it back.
    if ! container_running "$DB_CONTAINER"; then
        step "Cleanup: $DB_CONTAINER is down, attempting restart"
        start_db || { log "CRITICAL: $DB_CONTAINER is DOWN - manual action required"
                      [ "$exit_code" -eq 0 ] && exit_code=1; }
    fi

    find "$LOGDIR" -maxdepth 1 -name 'backup-*.log' -type f \
         -mtime +"$LOG_RETENTION_DAYS" -delete 2>/dev/null

    if [ "$exit_code" -eq 0 ]; then
        log "=== Backup completed SUCCESSFULLY (archive: $ARCHIVE) ==="
    else
        log "=== Backup FAILED (exit $exit_code) ==="
    fi
    log "Full log: $LOGFILE"
}

# Returns 0 if the repo is encrypted, 1 if not.
repo_is_encrypted() {
    borg info "$REPO" 2>&1 | grep -q "Encrypted:.*Yes"
}

preflight() {
    local missing=()
    for c in "${REQUIRED_CMDS[@]}"; do
        command -v "$c" >/dev/null 2>&1 || missing+=("$c")
    done
    if (( ${#missing[@]} )); then
        log "ERROR: missing commands: ${missing[*]}"; return 1
    fi

    if [[ -n "$REPO_MOUNT" ]] && ! mountpoint -q "$REPO_MOUNT"; then
        log "ERROR: $REPO_MOUNT is not mounted - refusing to run"
        return 1
    fi

    [[ -d "$TARGET" ]]            || { log "ERROR: target missing: $TARGET"; return 1; }
    [[ -n "$(ls -A "$TARGET")" ]] || { log "ERROR: target is empty: $TARGET"; return 1; }

    case "$REPO/" in
        "$TARGET"/*) log "ERROR: repo lives inside target - infinite recursion"; return 1 ;;
    esac

    if ! borg info --lock-wait 60 >/dev/null 2>&1; then
        log "ERROR: cannot open repository $REPO"
        log "       (wrong path? disk not mounted? stale lock? -> borg break-lock)"
        return 1
    fi

    # Warn if the repo is not encrypted.
    if ! repo_is_encrypted; then
        log "WARNING: Repository is NOT encrypted!"
        log "         Run:  borg init --encryption=repokey-blake2 $REPO"
        log "         (after backing up any existing data)"
    fi

    # Verify passphrase file exists and is readable.
    if [[ ! -r "$BORG_PASSPHRASE_FILE" ]]; then
        log "ERROR: passphrase file not readable: $BORG_PASSPHRASE_FILE"
        log "       Create it:  echo 'your-passphrase' > $BORG_PASSPHRASE_FILE"
        log "                   chmod 600 $BORG_PASSPHRASE_FILE"
        return 1
    fi

    container_running "$DB_CONTAINER" \
        || { log "ERROR: $DB_CONTAINER is not running before we start"; return 1; }

    log "Preflight OK"
}

# ----------------------- Lock, then trap --------------------------------

exec 9>"$LOCKFILE"
if ! flock -n 9; then
    echo "ERROR: another backup is already running (lock held on $LOCKFILE)"
    exit 1
fi

trap 'cleanup $?' EXIT
trap 'log "SIGINT received"; exit 130' INT
trap 'log "SIGTERM received"; exit 143' TERM

# =================================================================

log "=== Backup started ==="
echo "Repository: $REPO"
echo "Target:     $TARGET"
echo "Archive:    $ARCHIVE"
echo "Log:        $LOGFILE"

step "Step 0: Preflight"
preflight

# --- 1. Logical dumps (container stays up the whole time) ---------------
step "Step 1: MariaDB dumps"

if [[ ! -x "$DUMP_SCRIPT" ]]; then
    log "ERROR: dump script missing or not executable: $DUMP_SCRIPT"
    exit 1
fi
run_cmd "$DUMP_SCRIPT"

if [[ -d "$DUMP_DIR" ]]; then
    fresh=$(find "$DUMP_DIR" -type f -size +1k -mmin -60 | wc -l)
    empty=$(find "$DUMP_DIR" -type f -size -1k -mmin -60 | wc -l)
    log "Dumps: $fresh fresh non-trivial file(s), $empty suspiciously small"
    (( fresh > 0 )) || { log "ERROR: no usable dumps produced"; exit 1; }
    (( empty == 0 )) || log "WARNING: $empty near-empty dump file(s) - check $DUMP_DIR"
else
    log "WARNING: dump directory not found: $DUMP_DIR"
fi

# --- 2. Create the archive -----------------------------------------------
# MariaDB stays up throughout: the dump above is already transactionally
# consistent, and the container's raw data directory carries a .nobackup
# marker (excluded here via --exclude-if-present) so its live files are
# never read by borg. Per-site log/ or logs/ dirs (nginx access/error logs etc.)
# are excluded too - they churn on every request, so backing them up adds
# noise and dedup overhead for no recovery value.
step "Step 2: Creating archive $ARCHIVE"
container_running "$DB_CONTAINER" \
    || { log "ERROR: $DB_CONTAINER is not running - refusing to archive"; exit 1; }
run_cmd timeout --signal=INT --kill-after=120s "$CREATE_TIMEOUT" \
    borg create \
        --lock-wait 600 \
        --stats \
        --list --filter=AME \
        --compression zstd,8 \
        --exclude-caches \
        --exclude-if-present .nobackup \
        --keep-exclude-tags \
        --exclude 're:/logs?/' \
        "::$ARCHIVE" \
        "$TARGET"

# --- 3/4. Retention -------------------------------------------------------
step "Step 3: Pruning"
run_cmd timeout "$PRUNE_TIMEOUT" \
    borg prune \
        --lock-wait 600 \
        --list \
        --glob-archives "$ARCHIVE_GLOB" \
        --keep-daily=7 \
        --keep-weekly=4 \
        --keep-monthly=6

step "Step 4: Compacting"
run_cmd timeout "$PRUNE_TIMEOUT" borg compact --lock-wait 600

# --- 5. Integrity check -------------------------------------------------
step "Step 5: Verifying archive & repository integrity"
run_cmd borg info --lock-wait 60 "::$ARCHIVE" >/dev/null
ARCHIVE_COUNT=$(borg list --short --glob-archives "$ARCHIVE_GLOB" | wc -l)
log "Repository holds $ARCHIVE_COUNT archive(s) for $ARCHIVE_GLOB"
(( ARCHIVE_COUNT > 0 )) || { log "ERROR: repo unexpectedly empty"; exit 1; }

# Daily: fast archive-only check. Weekly (Monday): full repo check with data verification.
if [[ "$(date +%u)" == "1" ]]; then
    log "Monday — running full repository check with --verify-data (this may take a while)"
    run_cmd timeout "$CHECK_TIMEOUT" borg check --verify-data --lock-wait 600 "$REPO"
else
    log "Running fast archive-only integrity check"
    run_cmd timeout 1h borg check --archives-only --lock-wait 600 "$REPO"::"$ARCHIVE"
fi

# --- 6. Offsite mirror --------------------------------------------------
step "Step 6: Syncing to ${RCLONE_REMOTE}:${RCLONE_PATH}"
run_cmd timeout "$SYNC_TIMEOUT" \
    rclone sync -v \
        --fast-list \
        --transfers=4 \
        --checkers=8 \
        --retries 5 \
        --retries-sleep 10s \
        --low-level-retries 20 \
        --s3-no-check-bucket \
        "$REPO" "${RCLONE_REMOTE}:${RCLONE_PATH}"