a62bcb4331
Convert /mnt/pve/oracle-backups from a directory on the pveelite rootfs into a dedicated ZFS dataset rpool/oracle-backups so it can be incrementally replicated to pvemini. zfs-replicate-oracle-backups.sh runs every 15 minutes from cron on pveelite and uses zfs send/recv over the cluster's internal SSH (direct IP, /etc/pve/priv/known_hosts) to avoid Tailscale magicDNS detours that broke the first attempt. The destination dataset is set readonly=on so accidental writes on pvemini cannot diverge it. Snapshot pruning keeps 5 rolling copies. nightly-backup-mirror.sh ships a third copy nightly to pve1's backup-ssd (ext4 SATA) — different physical disk, different filesystem, different node — guarding against the failure mode where both pveelite and pvemini are simultaneously unavailable. The same script tars /etc/pve and rotates 14 days of cluster config archives, since pmxcfs is in-RAM and a multi-node quorum loss would otherwise take cluster config with it. The old directory is kept as oracle-backups.old-DELETE-AFTER-2026-05-02 on pveelite for one week as a safety net. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
64 lines
2.4 KiB
Bash
64 lines
2.4 KiB
Bash
#!/bin/bash
|
|
#
|
|
# Nightly mirror of Oracle backups + cluster config to pve1's backup-ssd.
|
|
#
|
|
# Why two redundant copies are not enough:
|
|
# * ZFS replica pveelite -> pvemini covers pveelite hardware failure.
|
|
# * If both pveelite AND pvemini are down (rare but possible — common
|
|
# storage controller, network rack, electrical fault), pve1 is the
|
|
# last copy. Keeping it on a different physical disk type (SATA
|
|
# ext4) further insulates against ZFS-on-NVMe-specific failures.
|
|
# * /etc/pve is in pmxcfs (in-RAM, replicated cluster-wide). If
|
|
# quorum is lost on multiple nodes simultaneously the config is
|
|
# unrecoverable without a backup.
|
|
#
|
|
# Schedule (cron on pveelite): 0 4 * * *
|
|
|
|
set -euo pipefail
|
|
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
|
|
PVE1_HOST="10.0.20.200"
|
|
PVE1_BACKUP_DIR="/mnt/pve/backup-ssd"
|
|
ORACLE_SRC="/mnt/pve/oracle-backups/"
|
|
ORACLE_DST="${PVE1_BACKUP_DIR}/oracle-backups-mirror/"
|
|
PVE_CFG_DST="${PVE1_BACKUP_DIR}/pve-config-backups"
|
|
LOG="/var/log/oracle-dr/nightly-mirror.log"
|
|
SSH_OPTS="-o UserKnownHostsFile=/etc/pve/priv/known_hosts -o StrictHostKeyChecking=no -o BatchMode=yes"
|
|
KEEP_PVE_CONFIGS=14 # 2 weeks of nightly /etc/pve archives
|
|
|
|
mkdir -p "$(dirname "$LOG")"
|
|
|
|
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >>"$LOG"; }
|
|
|
|
log "=== Starting nightly mirror ==="
|
|
|
|
# 1. Rsync Oracle backups to pve1
|
|
log "Rsync ${ORACLE_SRC} -> ${PVE1_HOST}:${ORACLE_DST}"
|
|
if rsync -aHX --delete -e "ssh ${SSH_OPTS}" \
|
|
"${ORACLE_SRC}" "root@${PVE1_HOST}:${ORACLE_DST}" 2>>"$LOG"; then
|
|
log "Oracle backups rsync OK"
|
|
else
|
|
log "ERROR: Oracle backups rsync failed"
|
|
fi
|
|
|
|
# 2. Tar /etc/pve and ship to pve1
|
|
TS=$(date +%Y%m%d_%H%M%S)
|
|
ARCHIVE="pve-config-${TS}.tar.gz"
|
|
log "Tar /etc/pve -> ${PVE1_HOST}:${PVE_CFG_DST}/${ARCHIVE}"
|
|
if tar czf - -C / etc/pve 2>/dev/null | \
|
|
ssh ${SSH_OPTS} "root@${PVE1_HOST}" \
|
|
"cat > '${PVE_CFG_DST}/${ARCHIVE}'" 2>>"$LOG"; then
|
|
log "pve-config tar OK ($(ssh ${SSH_OPTS} root@${PVE1_HOST} \
|
|
"stat -c %s '${PVE_CFG_DST}/${ARCHIVE}'") bytes)"
|
|
else
|
|
log "ERROR: pve-config tar failed"
|
|
fi
|
|
|
|
# 3. Prune old pve-config archives on pve1 (keep last KEEP_PVE_CONFIGS)
|
|
ssh ${SSH_OPTS} "root@${PVE1_HOST}" "
|
|
cd '${PVE_CFG_DST}' && \
|
|
ls -1t pve-config-*.tar.gz 2>/dev/null | tail -n +$((KEEP_PVE_CONFIGS + 1)) | xargs -r rm -v
|
|
" >>"$LOG" 2>&1 || true
|
|
|
|
log "=== Nightly mirror completed ==="
|