transfer backups

2025-10-31 01:19:15 +02:00
parent 63bcdf5c7f
commit c715a0a89d
3 changed files with 73 additions and 200 deletions
--- a/oracle/standby-server-scripts/README.md
+++ b/oracle/standby-server-scripts/README.md
@@ -93,12 +93,13 @@ ssh root@10.0.20.202 "/opt/scripts/oracle-backup-monitor-proxmox.sh"
 D:\rman_backup\
 ├── rman_backup_full.txt          # RMAN script for FULL backup
 ├── rman_backup_incremental.txt   # RMAN script for CUMULATIVE
-├── transfer_to_dr.ps1            # Transfer FULL to Proxmox
-└── transfer_incremental.ps1      # Transfer CUMULATIVE to Proxmox
+└── transfer_backups.ps1          # UNIFIED: Transfer ALL backups to Proxmox

 Scheduled Tasks:
 ├── 02:30 - Oracle RMAN Full Backup
+├── 03:00 - Transfer backups to DR (transfer_backups.ps1)
 ├── 13:00 - Oracle RMAN Cumulative Backup
+├── 14:45 - Transfer backups to DR (transfer_backups.ps1)
 └── 18:00 - Oracle RMAN Cumulative Backup
 ```

@@ -134,21 +135,25 @@ F:\ (NFS mount from Proxmox)

 ### Backup Flow (Daily)
 ```
-PRIMARY                    PROXMOX
-   │                          │
-   ├─02:30─FULL─Backup────────►
-   │         (6-7 GB)         │
-   │                          │
-   ├─13:00─CUMULATIVE─────────►
-   │         (200 MB)         │
-   │                          │
-   └─18:00─CUMULATIVE─────────►
-             (300 MB)      Storage
-
-                        ┌──────────┐
-                        │ Monitor  │ 09:00 Daily
-                        │ Check Age│ Alert if old
-                        └──────────┘
+PRIMARY                         PROXMOX
+   │                               │
+   ├─02:30─FULL─Backup─────────────►
+   │         (6-7 GB)               │
+   ├─03:00─Transfer ALL────────────► Skip duplicates
+   │      (transfer_backups.ps1)   │
+   │                               │
+   ├─13:00─CUMULATIVE──────────────►
+   │         (200 MB)               │
+   ├─14:45─Transfer ALL────────────► Skip duplicates
+   │      (transfer_backups.ps1)   │ (only new files)
+   │                               │
+   └─18:00─CUMULATIVE──────────────►
+             (300 MB)            Storage
+                                    │
+                             ┌──────────┐
+                             │ Monitor  │ 09:00 Daily
+                             │ Check Age│ Alert if old
+                             └──────────┘
 ```

 ### Restore Process
@@ -165,7 +170,7 @@ Total Time: ~15 minutes

 ```bash
 # 1. Test backup transfer (on PRIMARY)
-D:\rman_backup\transfer_incremental.ps1
+powershell -ExecutionPolicy Bypass -File "D:\rman_backup\transfer_backups.ps1"

 # 2. Test NFS mount (on VM 109)
 mount -o rw,nolock,mtype=hard,timeout=60 10.0.20.202:/mnt/pve/oracle-backups F:
@@ -398,7 +403,7 @@ ssh root@10.0.20.202 "qm start 109 && qm terminal 109"
 # Backup logs (on PRIMARY)
 Get-Content D:\rman_backup\logs\backup_*.log -Tail 50

-# Transfer logs (on PRIMARY)
+# Transfer logs (on PRIMARY) - UNIFIED script
 Get-Content D:\rman_backup\logs\transfer_*.log -Tail 50

 # Monitoring logs (on Proxmox)
@@ -509,6 +514,18 @@ LINUX WORKSTATION ─────────► VM 109 (10.0.20.37)

 ---

-**Last Updated:** October 11, 2025
-**Version:** 2.1 - Added restore test debugging guide + naming convention
-**Status:** ✅ Production Ready
+**Last Updated:** October 31, 2025
+**Version:** 2.2 - Unified transfer script (transfer_backups.ps1)
+**Status:** ✅ Production Ready
+
+## 📋 Changelog
+
+### v2.2 (Oct 31, 2025)
+- ✨ **Unified transfer script**: Replaced `transfer_to_dr.ps1` and `transfer_incremental.ps1` with single `transfer_backups.ps1`
+- 🎯 **Smart duplicate detection**: Automatically skips files that exist on DR
+- ⚡ **Flexible scheduling**: Can run after any backup type or manually
+- 🔧 **Simplified maintenance**: One script to maintain instead of two
+
+### v2.1 (Oct 11, 2025)
+- Added restore test debugging guide
+- Implemented new backup naming convention
--- a/oracle/standby-server-scripts/transfer_backups.ps1
+++ b/oracle/standby-server-scripts/transfer_backups.ps1
@@ -1,6 +1,11 @@
-# Transfer Oracle RMAN Backup towards DR Server
-# Rulează după backup RMAN (03:00 AM)
-# Copiază backup-uri de pe PRIMARY (10.0.20.36) către DR (10.0.20.37)
+# Transfer Oracle Backups (Full + Incremental) towards DR Server
+# Script UNIFICAT - poate fi rulat după orice tip de backup
+# Transferă TOATE fișierele backup, skip-uiește duplicatele automat
+#
+# Poate fi apelat de:
+# - Task Scheduler după full backup (03:00 AM)
+# - Task Scheduler după incremental backup (14:30)
+# - Manual oricând pentru recovery

 param(
    [string]$SourceFRA = "C:\Users\Oracle\recovery_area\ROA",
@@ -10,7 +15,7 @@ param(
    [string]$DRPath = "/mnt/pve/oracle-backups/ROA/autobackup",
    [string]$SSHKeyPath = "$env:USERPROFILE\.ssh\id_rsa",
    [int]$RetentionDays = 2,
-    [string]$LogFile = "D:\rman_backup\logs\transfer_$(Get-Date -Format 'yyyyMMdd').log"
+    [string]$LogFile = "D:\rman_backup\logs\transfer_$(Get-Date -Format 'yyyyMMdd_HHmm').log"
 )

 $ErrorActionPreference = "Continue"
@@ -43,8 +48,8 @@ function Test-SSHConnection {
    }
 }

-function Get-TodaysBackups {
-    Write-Log "Searching for backup files..."
+function Get-AllBackupFiles {
+    Write-Log "Searching for all backup files in FRA..."

    $backupFiles = @()

@@ -58,6 +63,7 @@ function Get-TodaysBackups {
            # Get ALL backup files (duplicates will be skipped during transfer)
            $files = Get-ChildItem -Path $path -Recurse -File -ErrorAction SilentlyContinue |
                Where-Object {
+                    $_.Name -match '\.(BKP|bkp)$' -and
                    $_.Name -notlike "*__TAG_*"  # Exclude old uncompressed backups
                } |
                Sort-Object LastWriteTime -Descending
@@ -72,7 +78,7 @@ function Get-TodaysBackups {
    }

    $totalSizeGB = ($backupFiles | Measure-Object -Property Length -Sum).Sum / 1GB
-    Write-Log "Found $($backupFiles.Count) files, total size: $([math]::Round($totalSizeGB, 2)) GB"
+    Write-Log "Found $($backupFiles.Count) backup files, total size: $([math]::Round($totalSizeGB, 2)) GB"

    return $backupFiles
 }
@@ -144,7 +150,7 @@ function Cleanup-OldBackupsOnDR {

 try {
    Write-Log "========================================="
-    Write-Log "Oracle DR Backup Transfer Started"
+    Write-Log "Oracle Backup Transfer Started (UNIFIED)"
    Write-Log "========================================="
    Write-Log "Source FRA: $SourceFRA"
    Write-Log "DR Server: $DRHost"
@@ -168,21 +174,32 @@ try {
    Write-Log "Ensuring DR directory exists..."
    $null = & ssh -n -p $DRPort -i $SSHKeyPath "${DRUser}@${DRHost}" "mkdir -p '$DRPath'" 2>&1

-    # Găsește backup-uri
-    $backupFiles = Get-TodaysBackups
+    # Găsește TOATE backup-urile
+    $backupFiles = Get-AllBackupFiles

    if ($backupFiles.Count -eq 0) {
-        throw "No backup files to transfer!"
+        Write-Log "No backup files to transfer (this might be normal for first run)" "WARNING"
+        exit 0
    }

    # Transfer fișiere
    Write-Log "Starting file transfer..."
    $successCount = 0
    $failCount = 0
+    $skippedCount = 0

    foreach ($file in $backupFiles) {
-        if (Transfer-FileToDR -File $file -DestPath $DRPath) {
-            $successCount++
+        $result = Transfer-FileToDR -File $file -DestPath $DRPath
+
+        if ($result) {
+            # Check if it was skipped or transferred
+            $fileName = $file.Name
+            $checkCmd = "test -f '$DRPath/$fileName' && echo 'True' || echo 'False'"
+            $checkResult = & ssh -n -p $DRPort -i $SSHKeyPath "${DRUser}@${DRHost}" $checkCmd 2>&1
+
+            if ($checkResult -match "True") {
+                $successCount++
+            }
        } else {
            $failCount++
        }
@@ -198,9 +215,11 @@ try {
    Cleanup-OldBackupsOnDR

    Write-Log "========================================="
-    Write-Log "DR Backup Transfer Completed Successfully"
+    Write-Log "Backup Transfer Completed Successfully"
    Write-Log "========================================="
-    Write-Log "Files transferred: $successCount/$($backupFiles.Count)"
+    Write-Log "Files processed: $($backupFiles.Count)"
+    Write-Log "Successful: $successCount"
+    Write-Log "Failed: $failCount"
    Write-Log "DR Server: ${DRHost}:${DRPath}"

    exit 0
--- a/oracle/standby-server-scripts/transfer_incremental.ps1
+++ b/oracle/standby-server-scripts/transfer_incremental.ps1
@@ -1,163 +0,0 @@
-# Transfer Oracle INCREMENTAL Backup towards DR Server
-# Rulează după backup incremental (14:30)
-# Mai simplu decât scriptul full - doar transferă fișierele noi
-
-param(
-    [string]$SourceFRA = "C:\Users\Oracle\recovery_area\ROA",
-    [string]$DRHost = "10.0.20.202",
-    [int]$DRPort = 22,
-    [string]$DRUser = "root",
-    [string]$DRPath = "/mnt/pve/oracle-backups/ROA/autobackup",
-    [string]$SSHKeyPath = "$env:USERPROFILE\.ssh\id_rsa",
-    [string]$LogFile = "D:\rman_backup\logs\transfer_incr_$(Get-Date -Format 'yyyyMMdd_HHmm').log"
-)
-
-$ErrorActionPreference = "Continue"
-
-function Write-Log {
-    param([string]$Message, [string]$Level = "INFO")
-    $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
-    $logLine = "[$timestamp] [$Level] $Message"
-    Write-Host $logLine
-    Add-Content -Path $LogFile -Value $logLine -Encoding UTF8 -ErrorAction SilentlyContinue
-}
-
-function Get-IncrementalBackups {
-    Write-Log "Searching for incremental backup files created in last 2 hours..."
-
-    $cutoffTime = (Get-Date).AddHours(-2)
-    $backupFiles = @()
-
-    # Caută în BACKUPSET, AUTOBACKUP
-    $searchPaths = @(
-        "$SourceFRA\BACKUPSET",
-        "$SourceFRA\AUTOBACKUP"
-    )
-
-    foreach ($path in $searchPaths) {
-        if (Test-Path $path) {
-            $files = Get-ChildItem -Path $path -Recurse -File -ErrorAction SilentlyContinue |
-                Where-Object {
-                    $_.LastWriteTime -gt $cutoffTime -and
-                    $_.Name -match '\.(BKP|bkp)$'
-                }
-            $backupFiles += $files
-        }
-    }
-
-    return $backupFiles
-}
-
-function Cleanup-OldBackupsOnDR {
-    param([int]$RetentionDays = 2)
-
-    Write-Log "Cleaning up old backups on DR (keeping last $RetentionDays days)..."
-
-    try {
-        # Count fișiere înainte de cleanup
-        $countBefore = & ssh -n -p $DRPort -i $SSHKeyPath "${DRUser}@${DRHost}" "find '$DRPath' -name '*.BKP' -type f | wc -l" 2>&1
-        Write-Log "Backups before cleanup: $countBefore"
-
-        # Cleanup: șterge fișiere mai vechi de $RetentionDays zile
-        # -mtime +N înseamnă "mai vechi de N zile", deci pentru a păstra RetentionDays zile, folosim +($RetentionDays - 1)
-        $mtimeDays = $RetentionDays - 1
-        $cleanupCmd = "find '$DRPath' -name '*.BKP' -type f -mtime +$mtimeDays -delete 2>&1"
-        $result = & ssh -n -p $DRPort -i $SSHKeyPath "${DRUser}@${DRHost}" $cleanupCmd 2>&1
-
-        if ($LASTEXITCODE -ne 0) {
-            Write-Log "Cleanup warning: $result" "WARNING"
-        }
-
-        # Count fișiere după cleanup
-        $countAfter = & ssh -n -p $DRPort -i $SSHKeyPath "${DRUser}@${DRHost}" "find '$DRPath' -name '*.BKP' -type f | wc -l" 2>&1
-        $deleted = [int]$countBefore - [int]$countAfter
-
-        Write-Log "Cleanup completed: Deleted $deleted old backup files, $countAfter remaining"
-    } catch {
-        Write-Log "Cleanup error: $_" "WARNING"
-    }
-}
-
-try {
-    Write-Log "========================================="
-    Write-Log "Oracle INCREMENTAL Backup Transfer Started"
-    Write-Log "========================================="
-    Write-Log "Source FRA: $SourceFRA"
-    Write-Log "DR Server: $DRHost"
-
-    # Test SSH connection
-    Write-Log "Testing SSH connection to $DRHost`:$DRPort..."
-    $null = & ssh -n -p $DRPort -i $SSHKeyPath -o StrictHostKeyChecking=no -o ConnectTimeout=10 "${DRUser}@${DRHost}" "exit 0" 2>&1
-    if ($LASTEXITCODE -ne 0) {
-        throw "SSH connection failed"
-    }
-    Write-Log "SSH connection OK" "SUCCESS"
-
-    # Găsește backup-uri incrementale
-    $backupFiles = Get-IncrementalBackups
-
-    if ($backupFiles.Count -eq 0) {
-        Write-Log "⚠️  No incremental backup files found (this might be normal if backup didn't run yet)" "WARNING"
-        exit 0
-    }
-
-    $totalSizeGB = ($backupFiles | Measure-Object -Property Length -Sum).Sum / 1GB
-    Write-Log "Found $($backupFiles.Count) incremental files, total size: $([math]::Round($totalSizeGB, 2)) GB"
-
-    # Transfer fișiere
-    Write-Log "Starting file transfer..."
-    $successCount = 0
-    $failCount = 0
-
-    foreach ($file in $backupFiles) {
-        $fileName = $file.Name
-        $fileSizeMB = [math]::Round($file.Length / 1MB, 2)
-
-        # Check dacă fișierul există deja pe DR (skip duplicates) - Linux bash command
-        $checkCmd = "test -f '$DRPath/$fileName' && echo 'True' || echo 'False'"
-        $checkResult = & ssh -n -p $DRPort -i $SSHKeyPath "${DRUser}@${DRHost}" $checkCmd 2>&1
-
-        if ($checkResult -match "True") {
-            Write-Log "Skipping (already on DR): $fileName" "INFO"
-            $successCount++
-            continue
-        }
-
-        Write-Log "Transferring: $fileName ($fileSizeMB MB)"
-
-        # SCP optimized: no compression (already compressed), fast cipher
-        $null = & scp -P $DRPort -i $SSHKeyPath -o StrictHostKeyChecking=no -o Compression=no -o Cipher=aes128-gcm@openssh.com `
-            $file.FullName `
-            "${DRUser}@${DRHost}:${DRPath}/$fileName" 2>&1
-
-        if ($LASTEXITCODE -eq 0) {
-            Write-Log "Transferred: $fileName" "SUCCESS"
-            $successCount++
-        } else {
-            Write-Log "Failed to transfer: $fileName" "ERROR"
-            $failCount++
-        }
-    }
-
-    Write-Log "========================================="
-    Write-Log "Transfer summary: $successCount succeeded, $failCount failed"
-
-    if ($failCount -gt 0) {
-        Write-Log "⚠️  Some transfers failed!" "WARNING"
-        exit 1
-    }
-
-    # Cleanup old backups pe DR (keep last 2 days)
-    Cleanup-OldBackupsOnDR -RetentionDays 2
-
-    Write-Log "========================================="
-    Write-Log "INCREMENTAL Backup Transfer Completed Successfully"
-    Write-Log "========================================="
-    Write-Log "Files transferred: $successCount/$($backupFiles.Count)"
-
-    exit 0
-
-} catch {
-    Write-Log "CRITICAL ERROR: $($_.Exception.Message)" "ERROR"
-    exit 1
-}