diff --git a/oracle/standby-server-scripts/rman_restore_from_zero.ps1 b/oracle/standby-server-scripts/rman_restore_from_zero.ps1 index 3c9e0a2..b2489c6 100644 --- a/oracle/standby-server-scripts/rman_restore_from_zero.ps1 +++ b/oracle/standby-server-scripts/rman_restore_from_zero.ps1 @@ -45,9 +45,15 @@ if (-not (Test-Path "F:\ROA\autobackup")) { Write-Host "[OK] F:\ROA\autobackup is accessible" Write-Host "" -# Create directories -New-Item -ItemType Directory -Path "D:\oracle\temp" -Force | Out-Null -New-Item -ItemType Directory -Path "D:\oracle\logs" -Force | Out-Null +# Create directories with proper permissions +try { + New-Item -ItemType Directory -Path "D:\oracle\temp" -Force -ErrorAction Stop | Out-Null + New-Item -ItemType Directory -Path "D:\oracle\logs" -Force -ErrorAction Stop | Out-Null + Write-Host "[OK] Created required directories" +} catch { + Write-Host "ERROR: Failed to create directories: $_" -ForegroundColor Red + exit 1 +} Write-Host "============================================" Write-Host "STEP 1: CLEANUP - Delete existing database" @@ -115,12 +121,70 @@ $logFile = "D:\oracle\logs\restore_from_zero.log" New-Item -ItemType Directory -Path "C:\Users\oracle\recovery_area\ROA\autobackup" -Force | Out-Null Write-Host "[INFO] Copying all backups from F:\ROA\autobackup to recovery area..." Write-Host " This may take 1-2 minutes for ~10 GB of backups..." -Copy-Item "F:\ROA\autobackup\*.BKP" "C:\Users\oracle\recovery_area\ROA\autobackup\" -Force -ErrorAction Stop -if ($LASTEXITCODE -ne 0) { - Write-Host "ERROR: Failed to copy backups from F:\" -ForegroundColor Red + +# Check backup files exist on F: drive before copying +try { + $backupFiles = Get-ChildItem "F:\ROA\autobackup\*.BKP" -ErrorAction Continue +} catch { + Write-Host "WARNING: Cannot enumerate backup files on F: drive - $_" -ForegroundColor Yellow + $backupFiles = @() +} + +if ($backupFiles.Count -lt 2) { + Write-Host "ERROR: Insufficient backup files found on F: drive (found: $($backupFiles.Count))" -ForegroundColor Red + Write-Host " At least 2 backup files required for successful restore" + Write-Host " Checking F:\ROA\autobackup directory..." + try { + $dirCheck = Get-ChildItem "F:\ROA\autobackup" -ErrorAction Continue + Write-Host " Directory contents: $($dirCheck.Count) files" + foreach ($file in $dirCheck) { + Write-Host " $($file.Name) - $($file.Length / 1GB) GB" -ForegroundColor Gray + } + } catch { + Write-Host " Cannot access directory: $_" -ForegroundColor Red + } exit 1 } -Write-Host "[OK] All backups copied to recovery area" + +Write-Host "[INFO] Found $($backupFiles.Count) backup files, total size: $([math]::Round(($backupFiles | Measure-Object -Property Length -Sum).Sum / 1GB, 2)) GB" + +# Copy backups with better error handling +Write-Host "[INFO] Starting backup copy operation..." +$copyErrors = @() +foreach ($backupFile in $backupFiles) { + try { + Write-Host "[INFO] Copying $($backupFile.Name)..." + Copy-Item $backupFile.FullName "C:\Users\oracle\recovery_area\ROA\autobackup\" -Force -ErrorAction Stop + Write-Host "[OK] Copied $($backupFile.Name)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Failed to copy $($backupFile.Name) - $_" -ForegroundColor Red + $copyErrors += "$($backupFile.Name): $_" + } +} + +if ($copyErrors.Count -gt 0) { + Write-Host "ERROR: Backup copy failed for $($copyErrors.Count) files" -ForegroundColor Red + foreach ($error in $copyErrors) { + Write-Host " $error" -ForegroundColor Red + } + exit 1 +} + +# Verify copied backups +try { + $copiedFiles = Get-ChildItem "C:\Users\oracle\recovery_area\ROA\autobackup\*.BKP" -ErrorAction Continue +} catch { + Write-Host "ERROR: Cannot verify copied backups - $_" -ForegroundColor Red + exit 1 +} + +if ($copiedFiles.Count -ne $backupFiles.Count) { + Write-Host "ERROR: Backup copy verification failed - file count mismatch" -ForegroundColor Red + Write-Host " Expected: $($backupFiles.Count), Copied: $($copiedFiles.Count)" + exit 1 +} + +Write-Host "[OK] All $($copiedFiles.Count) backups copied and verified to recovery area" # Create RMAN script $rmanContent = @" @@ -134,7 +198,10 @@ RUN { ALTER DATABASE MOUNT; -CATALOG START WITH 'F:/ROA/autobackup' NOPROMPT; +CATALOG START WITH 'C:/USERS/ORACLE/RECOVERY_AREA/ROA/AUTOBACKUP' NOPROMPT; + +CROSSCHECK BACKUP; +DELETE NOPROMPT EXPIRED BACKUP; RUN { ALLOCATE CHANNEL ch1 DEVICE TYPE DISK; @@ -152,6 +219,8 @@ RUN { ALTER DATABASE OPEN RESETLOGS; +DELETE NOPROMPT OBSOLETE; + EXIT; "@ diff --git a/oracle/standby-server-scripts/weekly-dr-test-proxmox.sh b/oracle/standby-server-scripts/weekly-dr-test-proxmox.sh index 46ba35f..e073f9d 100644 --- a/oracle/standby-server-scripts/weekly-dr-test-proxmox.sh +++ b/oracle/standby-server-scripts/weekly-dr-test-proxmox.sh @@ -375,7 +375,7 @@ run_dr_test() { # Use PowerShell to query database status db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ - "powershell -Command \"'SELECT STATUS FROM V\`\$INSTANCE;' | sqlplus -s / as sysdba | Select-String 'OPEN'\"" || echo "") + "powershell -Command \"'SELECT OPEN_MODE FROM V\\\$DATABASE;' | sqlplus -s / as sysdba | findstr 'READ WRITE'\"" || echo "") # Use PowerShell to count tables tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ @@ -383,7 +383,7 @@ run_dr_test() { tables_restored=$(echo "$tables_restored" | tr -cd '0-9') [ -z "$tables_restored" ] && tables_restored=0 - if [[ "$db_status" =~ "OPEN" ]]; then + if [[ "$db_status" =~ "READ WRITE" ]]; then track_step "Database Verification" true "Database OPEN, $tables_restored tables" "$step_start" test_result="PASSED" severity="info" @@ -392,10 +392,26 @@ run_dr_test() { track_step "Database Verification" false "Database not OPEN" "$step_start" fi - # Collect restore log from VM + # Collect restore log from VM (always attempt collection) log "Collecting restore log from DR VM..." restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ "type D:\\oracle\\logs\\restore_from_zero.log 2>nul" | head -200 || echo "Log not available") + + # If not found, try alternate locations + if [[ "$restore_log" == *"Log not available"* ]]; then + restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ + "type D:\\oracle\\temp\\restore_from_zero.log 2>nul" | head -200 || echo "Log not available") + fi + + # Still not found, check if any logs exist + if [[ "$restore_log" == *"Log not available"* ]]; then + log "Checking for any restore logs in DR VM..." + log_check=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ + "dir D:\\oracle\\logs\\*.log 2>nul || dir D:\\oracle\\temp\\*.log 2>nul || echo 'No logs found'" 2>/dev/null || echo "Connection error") + if [[ "$log_check" != *"No logs found"* ]]; then + restore_log="Log files found but could not be read. Available files: $log_check" + fi + fi # Step 6: Cleanup step_start=$(date +%s) @@ -408,6 +424,25 @@ run_dr_test() { track_step "Cleanup" true "Database cleaned, ~${cleanup_freed}GB freed" "$step_start" else + # Collect restore log even when restore fails + log "Collecting restore log after failure..." + restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ + "type D:\\oracle\\logs\\restore_from_zero.log 2>nul" | head -200 || echo "Log not available") + + if [[ "$restore_log" == *"Log not available"* ]]; then + restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ + "type D:\\oracle\\temp\\restore_from_zero.log 2>nul" | head -200 || echo "Log not available") + fi + + # Always try to get some error output + if [[ "$restore_log" == *"Log not available"* ]]; then + last_error=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ + "powershell -Command 'Get-Content D:\\oracle\\temp\\*.rman -Tail 20 || echo \"No RMAN script found\"'" 2>/dev/null || echo "Cannot access RMAN script") + if [[ "$last_error" != *"No RMAN script found"* ]]; then + restore_log="RMAN script content (last 20 lines):$last_error" + fi + fi + track_step "Database Restore" false "Restore failed" "$step_start" fi