From c715a0a89db249779cf27e2d0e87a78e0e52d357 Mon Sep 17 00:00:00 2001 From: Marius Date: Fri, 31 Oct 2025 01:19:15 +0200 Subject: [PATCH] transfer backups --- oracle/standby-server-scripts/README.md | 61 ++++--- ...ransfer_to_dr.ps1 => transfer_backups.ps1} | 49 ++++-- .../transfer_incremental.ps1 | 163 ------------------ 3 files changed, 73 insertions(+), 200 deletions(-) rename oracle/standby-server-scripts/{transfer_to_dr.ps1 => transfer_backups.ps1} (78%) delete mode 100644 oracle/standby-server-scripts/transfer_incremental.ps1 diff --git a/oracle/standby-server-scripts/README.md b/oracle/standby-server-scripts/README.md index 5443fe4..2c2e445 100644 --- a/oracle/standby-server-scripts/README.md +++ b/oracle/standby-server-scripts/README.md @@ -93,12 +93,13 @@ ssh root@10.0.20.202 "/opt/scripts/oracle-backup-monitor-proxmox.sh" D:\rman_backup\ ├── rman_backup_full.txt # RMAN script for FULL backup ├── rman_backup_incremental.txt # RMAN script for CUMULATIVE -├── transfer_to_dr.ps1 # Transfer FULL to Proxmox -└── transfer_incremental.ps1 # Transfer CUMULATIVE to Proxmox +└── transfer_backups.ps1 # UNIFIED: Transfer ALL backups to Proxmox Scheduled Tasks: ├── 02:30 - Oracle RMAN Full Backup +├── 03:00 - Transfer backups to DR (transfer_backups.ps1) ├── 13:00 - Oracle RMAN Cumulative Backup +├── 14:45 - Transfer backups to DR (transfer_backups.ps1) └── 18:00 - Oracle RMAN Cumulative Backup ``` @@ -134,21 +135,25 @@ F:\ (NFS mount from Proxmox) ### Backup Flow (Daily) ``` -PRIMARY PROXMOX - │ │ - ├─02:30─FULL─Backup────────► - │ (6-7 GB) │ - │ │ - ├─13:00─CUMULATIVE─────────► - │ (200 MB) │ - │ │ - └─18:00─CUMULATIVE─────────► - (300 MB) Storage - - ┌──────────┐ - │ Monitor │ 09:00 Daily - │ Check Age│ Alert if old - └──────────┘ +PRIMARY PROXMOX + │ │ + ├─02:30─FULL─Backup─────────────► + │ (6-7 GB) │ + ├─03:00─Transfer ALL────────────► Skip duplicates + │ (transfer_backups.ps1) │ + │ │ + ├─13:00─CUMULATIVE──────────────► + │ (200 MB) │ + ├─14:45─Transfer ALL────────────► Skip duplicates + │ (transfer_backups.ps1) │ (only new files) + │ │ + └─18:00─CUMULATIVE──────────────► + (300 MB) Storage + │ + ┌──────────┐ + │ Monitor │ 09:00 Daily + │ Check Age│ Alert if old + └──────────┘ ``` ### Restore Process @@ -165,7 +170,7 @@ Total Time: ~15 minutes ```bash # 1. Test backup transfer (on PRIMARY) -D:\rman_backup\transfer_incremental.ps1 +powershell -ExecutionPolicy Bypass -File "D:\rman_backup\transfer_backups.ps1" # 2. Test NFS mount (on VM 109) mount -o rw,nolock,mtype=hard,timeout=60 10.0.20.202:/mnt/pve/oracle-backups F: @@ -398,7 +403,7 @@ ssh root@10.0.20.202 "qm start 109 && qm terminal 109" # Backup logs (on PRIMARY) Get-Content D:\rman_backup\logs\backup_*.log -Tail 50 -# Transfer logs (on PRIMARY) +# Transfer logs (on PRIMARY) - UNIFIED script Get-Content D:\rman_backup\logs\transfer_*.log -Tail 50 # Monitoring logs (on Proxmox) @@ -509,6 +514,18 @@ LINUX WORKSTATION ─────────► VM 109 (10.0.20.37) --- -**Last Updated:** October 11, 2025 -**Version:** 2.1 - Added restore test debugging guide + naming convention -**Status:** ✅ Production Ready \ No newline at end of file +**Last Updated:** October 31, 2025 +**Version:** 2.2 - Unified transfer script (transfer_backups.ps1) +**Status:** ✅ Production Ready + +## 📋 Changelog + +### v2.2 (Oct 31, 2025) +- ✨ **Unified transfer script**: Replaced `transfer_to_dr.ps1` and `transfer_incremental.ps1` with single `transfer_backups.ps1` +- 🎯 **Smart duplicate detection**: Automatically skips files that exist on DR +- ⚡ **Flexible scheduling**: Can run after any backup type or manually +- 🔧 **Simplified maintenance**: One script to maintain instead of two + +### v2.1 (Oct 11, 2025) +- Added restore test debugging guide +- Implemented new backup naming convention \ No newline at end of file diff --git a/oracle/standby-server-scripts/transfer_to_dr.ps1 b/oracle/standby-server-scripts/transfer_backups.ps1 similarity index 78% rename from oracle/standby-server-scripts/transfer_to_dr.ps1 rename to oracle/standby-server-scripts/transfer_backups.ps1 index f07e3b7..a7791e8 100644 --- a/oracle/standby-server-scripts/transfer_to_dr.ps1 +++ b/oracle/standby-server-scripts/transfer_backups.ps1 @@ -1,6 +1,11 @@ -# Transfer Oracle RMAN Backup towards DR Server -# Rulează după backup RMAN (03:00 AM) -# Copiază backup-uri de pe PRIMARY (10.0.20.36) către DR (10.0.20.37) +# Transfer Oracle Backups (Full + Incremental) towards DR Server +# Script UNIFICAT - poate fi rulat după orice tip de backup +# Transferă TOATE fișierele backup, skip-uiește duplicatele automat +# +# Poate fi apelat de: +# - Task Scheduler după full backup (03:00 AM) +# - Task Scheduler după incremental backup (14:30) +# - Manual oricând pentru recovery param( [string]$SourceFRA = "C:\Users\Oracle\recovery_area\ROA", @@ -10,7 +15,7 @@ param( [string]$DRPath = "/mnt/pve/oracle-backups/ROA/autobackup", [string]$SSHKeyPath = "$env:USERPROFILE\.ssh\id_rsa", [int]$RetentionDays = 2, - [string]$LogFile = "D:\rman_backup\logs\transfer_$(Get-Date -Format 'yyyyMMdd').log" + [string]$LogFile = "D:\rman_backup\logs\transfer_$(Get-Date -Format 'yyyyMMdd_HHmm').log" ) $ErrorActionPreference = "Continue" @@ -43,8 +48,8 @@ function Test-SSHConnection { } } -function Get-TodaysBackups { - Write-Log "Searching for backup files..." +function Get-AllBackupFiles { + Write-Log "Searching for all backup files in FRA..." $backupFiles = @() @@ -58,6 +63,7 @@ function Get-TodaysBackups { # Get ALL backup files (duplicates will be skipped during transfer) $files = Get-ChildItem -Path $path -Recurse -File -ErrorAction SilentlyContinue | Where-Object { + $_.Name -match '\.(BKP|bkp)$' -and $_.Name -notlike "*__TAG_*" # Exclude old uncompressed backups } | Sort-Object LastWriteTime -Descending @@ -72,7 +78,7 @@ function Get-TodaysBackups { } $totalSizeGB = ($backupFiles | Measure-Object -Property Length -Sum).Sum / 1GB - Write-Log "Found $($backupFiles.Count) files, total size: $([math]::Round($totalSizeGB, 2)) GB" + Write-Log "Found $($backupFiles.Count) backup files, total size: $([math]::Round($totalSizeGB, 2)) GB" return $backupFiles } @@ -144,7 +150,7 @@ function Cleanup-OldBackupsOnDR { try { Write-Log "=========================================" - Write-Log "Oracle DR Backup Transfer Started" + Write-Log "Oracle Backup Transfer Started (UNIFIED)" Write-Log "=========================================" Write-Log "Source FRA: $SourceFRA" Write-Log "DR Server: $DRHost" @@ -168,21 +174,32 @@ try { Write-Log "Ensuring DR directory exists..." $null = & ssh -n -p $DRPort -i $SSHKeyPath "${DRUser}@${DRHost}" "mkdir -p '$DRPath'" 2>&1 - # Găsește backup-uri - $backupFiles = Get-TodaysBackups + # Găsește TOATE backup-urile + $backupFiles = Get-AllBackupFiles if ($backupFiles.Count -eq 0) { - throw "No backup files to transfer!" + Write-Log "No backup files to transfer (this might be normal for first run)" "WARNING" + exit 0 } # Transfer fișiere Write-Log "Starting file transfer..." $successCount = 0 $failCount = 0 + $skippedCount = 0 foreach ($file in $backupFiles) { - if (Transfer-FileToDR -File $file -DestPath $DRPath) { - $successCount++ + $result = Transfer-FileToDR -File $file -DestPath $DRPath + + if ($result) { + # Check if it was skipped or transferred + $fileName = $file.Name + $checkCmd = "test -f '$DRPath/$fileName' && echo 'True' || echo 'False'" + $checkResult = & ssh -n -p $DRPort -i $SSHKeyPath "${DRUser}@${DRHost}" $checkCmd 2>&1 + + if ($checkResult -match "True") { + $successCount++ + } } else { $failCount++ } @@ -198,9 +215,11 @@ try { Cleanup-OldBackupsOnDR Write-Log "=========================================" - Write-Log "DR Backup Transfer Completed Successfully" + Write-Log "Backup Transfer Completed Successfully" Write-Log "=========================================" - Write-Log "Files transferred: $successCount/$($backupFiles.Count)" + Write-Log "Files processed: $($backupFiles.Count)" + Write-Log "Successful: $successCount" + Write-Log "Failed: $failCount" Write-Log "DR Server: ${DRHost}:${DRPath}" exit 0 diff --git a/oracle/standby-server-scripts/transfer_incremental.ps1 b/oracle/standby-server-scripts/transfer_incremental.ps1 deleted file mode 100644 index 23802b5..0000000 --- a/oracle/standby-server-scripts/transfer_incremental.ps1 +++ /dev/null @@ -1,163 +0,0 @@ -# Transfer Oracle INCREMENTAL Backup towards DR Server -# Rulează după backup incremental (14:30) -# Mai simplu decât scriptul full - doar transferă fișierele noi - -param( - [string]$SourceFRA = "C:\Users\Oracle\recovery_area\ROA", - [string]$DRHost = "10.0.20.202", - [int]$DRPort = 22, - [string]$DRUser = "root", - [string]$DRPath = "/mnt/pve/oracle-backups/ROA/autobackup", - [string]$SSHKeyPath = "$env:USERPROFILE\.ssh\id_rsa", - [string]$LogFile = "D:\rman_backup\logs\transfer_incr_$(Get-Date -Format 'yyyyMMdd_HHmm').log" -) - -$ErrorActionPreference = "Continue" - -function Write-Log { - param([string]$Message, [string]$Level = "INFO") - $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" - $logLine = "[$timestamp] [$Level] $Message" - Write-Host $logLine - Add-Content -Path $LogFile -Value $logLine -Encoding UTF8 -ErrorAction SilentlyContinue -} - -function Get-IncrementalBackups { - Write-Log "Searching for incremental backup files created in last 2 hours..." - - $cutoffTime = (Get-Date).AddHours(-2) - $backupFiles = @() - - # Caută în BACKUPSET, AUTOBACKUP - $searchPaths = @( - "$SourceFRA\BACKUPSET", - "$SourceFRA\AUTOBACKUP" - ) - - foreach ($path in $searchPaths) { - if (Test-Path $path) { - $files = Get-ChildItem -Path $path -Recurse -File -ErrorAction SilentlyContinue | - Where-Object { - $_.LastWriteTime -gt $cutoffTime -and - $_.Name -match '\.(BKP|bkp)$' - } - $backupFiles += $files - } - } - - return $backupFiles -} - -function Cleanup-OldBackupsOnDR { - param([int]$RetentionDays = 2) - - Write-Log "Cleaning up old backups on DR (keeping last $RetentionDays days)..." - - try { - # Count fișiere înainte de cleanup - $countBefore = & ssh -n -p $DRPort -i $SSHKeyPath "${DRUser}@${DRHost}" "find '$DRPath' -name '*.BKP' -type f | wc -l" 2>&1 - Write-Log "Backups before cleanup: $countBefore" - - # Cleanup: șterge fișiere mai vechi de $RetentionDays zile - # -mtime +N înseamnă "mai vechi de N zile", deci pentru a păstra RetentionDays zile, folosim +($RetentionDays - 1) - $mtimeDays = $RetentionDays - 1 - $cleanupCmd = "find '$DRPath' -name '*.BKP' -type f -mtime +$mtimeDays -delete 2>&1" - $result = & ssh -n -p $DRPort -i $SSHKeyPath "${DRUser}@${DRHost}" $cleanupCmd 2>&1 - - if ($LASTEXITCODE -ne 0) { - Write-Log "Cleanup warning: $result" "WARNING" - } - - # Count fișiere după cleanup - $countAfter = & ssh -n -p $DRPort -i $SSHKeyPath "${DRUser}@${DRHost}" "find '$DRPath' -name '*.BKP' -type f | wc -l" 2>&1 - $deleted = [int]$countBefore - [int]$countAfter - - Write-Log "Cleanup completed: Deleted $deleted old backup files, $countAfter remaining" - } catch { - Write-Log "Cleanup error: $_" "WARNING" - } -} - -try { - Write-Log "=========================================" - Write-Log "Oracle INCREMENTAL Backup Transfer Started" - Write-Log "=========================================" - Write-Log "Source FRA: $SourceFRA" - Write-Log "DR Server: $DRHost" - - # Test SSH connection - Write-Log "Testing SSH connection to $DRHost`:$DRPort..." - $null = & ssh -n -p $DRPort -i $SSHKeyPath -o StrictHostKeyChecking=no -o ConnectTimeout=10 "${DRUser}@${DRHost}" "exit 0" 2>&1 - if ($LASTEXITCODE -ne 0) { - throw "SSH connection failed" - } - Write-Log "SSH connection OK" "SUCCESS" - - # Găsește backup-uri incrementale - $backupFiles = Get-IncrementalBackups - - if ($backupFiles.Count -eq 0) { - Write-Log "⚠️ No incremental backup files found (this might be normal if backup didn't run yet)" "WARNING" - exit 0 - } - - $totalSizeGB = ($backupFiles | Measure-Object -Property Length -Sum).Sum / 1GB - Write-Log "Found $($backupFiles.Count) incremental files, total size: $([math]::Round($totalSizeGB, 2)) GB" - - # Transfer fișiere - Write-Log "Starting file transfer..." - $successCount = 0 - $failCount = 0 - - foreach ($file in $backupFiles) { - $fileName = $file.Name - $fileSizeMB = [math]::Round($file.Length / 1MB, 2) - - # Check dacă fișierul există deja pe DR (skip duplicates) - Linux bash command - $checkCmd = "test -f '$DRPath/$fileName' && echo 'True' || echo 'False'" - $checkResult = & ssh -n -p $DRPort -i $SSHKeyPath "${DRUser}@${DRHost}" $checkCmd 2>&1 - - if ($checkResult -match "True") { - Write-Log "Skipping (already on DR): $fileName" "INFO" - $successCount++ - continue - } - - Write-Log "Transferring: $fileName ($fileSizeMB MB)" - - # SCP optimized: no compression (already compressed), fast cipher - $null = & scp -P $DRPort -i $SSHKeyPath -o StrictHostKeyChecking=no -o Compression=no -o Cipher=aes128-gcm@openssh.com ` - $file.FullName ` - "${DRUser}@${DRHost}:${DRPath}/$fileName" 2>&1 - - if ($LASTEXITCODE -eq 0) { - Write-Log "Transferred: $fileName" "SUCCESS" - $successCount++ - } else { - Write-Log "Failed to transfer: $fileName" "ERROR" - $failCount++ - } - } - - Write-Log "=========================================" - Write-Log "Transfer summary: $successCount succeeded, $failCount failed" - - if ($failCount -gt 0) { - Write-Log "⚠️ Some transfers failed!" "WARNING" - exit 1 - } - - # Cleanup old backups pe DR (keep last 2 days) - Cleanup-OldBackupsOnDR -RetentionDays 2 - - Write-Log "=========================================" - Write-Log "INCREMENTAL Backup Transfer Completed Successfully" - Write-Log "=========================================" - Write-Log "Files transferred: $successCount/$($backupFiles.Count)" - - exit 0 - -} catch { - Write-Log "CRITICAL ERROR: $($_.Exception.Message)" "ERROR" - exit 1 -}