#!/bin/bash # # Oracle DR Weekly Test with Proxmox PVE::Notify # Automated DR test with notifications via Proxmox notification system # # Location: /opt/scripts/weekly-dr-test-proxmox.sh (on Proxmox host) # Schedule: Add to cron for weekly execution (Saturdays) # # This script is SELF-SUFFICIENT: # - Automatically creates notification templates if they don't exist # - Uses Proxmox native notification system # - No email configuration needed - uses existing Proxmox setup # # Installation: # cp weekly-dr-test-proxmox.sh /opt/scripts/ # chmod +x /opt/scripts/weekly-dr-test-proxmox.sh # /opt/scripts/weekly-dr-test-proxmox.sh --install # Creates templates # crontab -e # Add: 0 6 * * 6 /opt/scripts/weekly-dr-test-proxmox.sh # # Author: Claude (based on ha-monitor.sh pattern) # Version: 1.0 set -euo pipefail # Configuration DR_VM_ID="109" DR_VM_IP="10.0.20.37" DR_VM_PORT="22122" DR_VM_USER="romfast" BACKUP_PATH="/mnt/pve/oracle-backups/ROA/autobackup" MAX_RESTORE_TIME_MIN=30 TEMPLATE_DIR="/usr/share/pve-manager/templates/default" LOG_DIR="/var/log/oracle-dr" LOG_FILE="$LOG_DIR/dr_test_$(date +%Y%m%d_%H%M%S).log" # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # Create log directory mkdir -p "$LOG_DIR" # Function to create notification templates create_templates() { echo -e "${GREEN}Creating Oracle DR test notification templates...${NC}" # Create templates directory if needed mkdir -p "$TEMPLATE_DIR" # Subject template cat > "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" <<'EOF' Oracle DR Test {{severity}} - {{test_result}} EOF # Text body template cat > "$TEMPLATE_DIR/oracle-dr-test-body.txt.hbs" <<'EOF' Oracle DR Weekly Test Report ============================ Test Result: {{test_result}} Severity: {{severity}} Date: {{timestamp}} Duration: {{total_duration}} minutes {{#if is_success}} ✓ TEST PASSED SUCCESSFULLY {{else}} ✗ TEST FAILED {{/if}} Test Steps Summary: ------------------- {{#each test_steps}} {{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}}: {{this.status}} ({{this.duration}}s) {{/each}} {{#if errors}} ERRORS: {{#each errors}} - {{this}} {{/each}} {{/if}} {{#if warnings}} WARNINGS: {{#each warnings}} - {{this}} {{/each}} {{/if}} Metrics: -------- - Backup Count: {{backup_count}} - Restore Time: {{restore_duration}} minutes - Tables Restored: {{tables_restored}} - Database Status: {{database_status}} - Disk Space Freed: {{disk_freed}} GB VM Details: ----------- - VM ID: {{vm_id}} - VM IP: {{vm_ip}} - NFS Mount: {{nfs_status}} Log File: {{log_file}} EOF # HTML body template cat > "$TEMPLATE_DIR/oracle-dr-test-body.html.hbs" <<'EOF'

Test Summary

Test Result

Restore Time (min)

Tables Restored

Backups Used

Test Steps Timeline

{{this.name}} {{this.duration}}s

{{#if this.passed}} ✓ {{this.status}} {{else}} ✗ {{this.status}} {{/if}}

Errors Encountered

{{this}}

Warnings

{{this}}

System Details

Component	Value	Status
DR VM	ID: {{vm_id}} ({{vm_ip}})	{{vm_status}}
NFS Mount	F:\ drive	{{nfs_status}}
Database	ROA	{{database_status}}
Disk Space Freed	{{disk_freed}} GB	✓

Log File: {{log_file}}
Next Scheduled Test: Next Saturday 06:00

EOF echo -e "${GREEN}Templates created successfully in $TEMPLATE_DIR${NC}" } # Function to send notification via PVE::Notify send_pve_notification() { local severity="$1" local data="$2" # Create Perl script to call PVE::Notify cat > /tmp/oracle-dr-notify.pl <<'PERL_SCRIPT' #!/usr/bin/perl use strict; use warnings; use PVE::Notify; use JSON; my $json_data = do { local $/; }; my $data = decode_json($json_data); my $severity = $data->{severity} // 'info'; my $template_name = 'oracle-dr-test'; # Add fields for matching rules my $fields = { type => 'oracle-dr-test', severity => $severity, test_result => $data->{test_result}, }; # Send notification eval { PVE::Notify::notify( $severity, $template_name, $data, $fields ); }; if ($@) { print "Error sending notification: $@\n"; exit 1; } print "Notification sent successfully\n"; PERL_SCRIPT chmod +x /tmp/oracle-dr-notify.pl # Send notification echo "$data" | perl /tmp/oracle-dr-notify.pl rm -f /tmp/oracle-dr-notify.pl } # Logging functions log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE" } log_error() { echo -e "${RED}[ERROR]${NC} $1" | tee -a "$LOG_FILE" } log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$LOG_FILE" } log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1" | tee -a "$LOG_FILE" } # Test tracking TEST_STEPS=() ERRORS=() WARNINGS=() TEST_START_TIME=$(date +%s) # Function to track test steps track_step() { local name="$1" local passed="$2" local status="$3" local start_time="$4" local end_time=$(date +%s) local duration=$((end_time - start_time)) TEST_STEPS+=("{\"name\":\"$name\",\"passed\":$passed,\"status\":\"$status\",\"duration\":$duration}") if [ "$passed" = "false" ]; then ERRORS+=("$name: $status") fi } # Main test workflow run_dr_test() { local test_result="FAILED" local severity="error" local is_success=false log "==========================================" log "Oracle DR Weekly Test - Starting" log "==========================================" # Step 1: Pre-flight checks local step_start=$(date +%s) log "STEP 1: Pre-flight checks" # Check backups exist local backup_count=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l || echo "0") if [ "$backup_count" -lt 2 ]; then track_step "Pre-flight checks" false "Insufficient backups (found: $backup_count)" "$step_start" test_result="FAILED - No backups" else track_step "Pre-flight checks" true "Found $backup_count backups" "$step_start" # Step 2: Start VM step_start=$(date +%s) log "STEP 2: Starting DR VM" if qm start "$DR_VM_ID" 2>/dev/null; then sleep 180 # Wait for boot track_step "VM Startup" true "VM $DR_VM_ID started" "$step_start" # Step 3: Verify NFS mount step_start=$(date +%s) log "STEP 3: Verifying NFS mount" local nfs_status="Not Mounted" if ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 "$DR_VM_USER@$DR_VM_IP" \ "powershell -Command 'Test-Path F:\\ROA\\autobackup'" 2>/dev/null; then nfs_status="Mounted" track_step "NFS Mount Check" true "F:\\ drive accessible" "$step_start" else track_step "NFS Mount Check" false "F:\\ drive not accessible" "$step_start" WARNINGS+=("NFS mount may need manual intervention") fi # Step 4: Run restore step_start=$(date +%s) local restore_start=$step_start log "STEP 4: Running database restore" if ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ "D:\\oracle\\scripts\\rman_restore_from_zero.cmd" 2>&1 | tee -a "$LOG_FILE"; then local restore_end=$(date +%s) local restore_duration=$(( (restore_end - restore_start) / 60 )) track_step "Database Restore" true "Restored in $restore_duration minutes" "$step_start" # Step 5: Verify database step_start=$(date +%s) log "STEP 5: Verifying database" local db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ "cmd /c 'echo SELECT STATUS FROM V\$INSTANCE; | sqlplus -s / as sysdba' | findstr OPEN" || echo "") local tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ "cmd /c 'echo SELECT COUNT(*) FROM DBA_TABLES WHERE OWNER NOT IN (''SYS'',''SYSTEM''); | sqlplus -s / as sysdba' | grep -o '[0-9]*' | tail -1" || echo "0") if [[ "$db_status" =~ "OPEN" ]]; then track_step "Database Verification" true "Database OPEN, $tables_restored tables" "$step_start" test_result="PASSED" severity="info" is_success=true else track_step "Database Verification" false "Database not OPEN" "$step_start" fi # Step 6: Cleanup step_start=$(date +%s) log "STEP 6: Running cleanup" ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ "D:\\oracle\\scripts\\cleanup_database.cmd" 2>/dev/null track_step "Cleanup" true "Database cleaned, ~8GB freed" "$step_start" else track_step "Database Restore" false "Restore failed" "$step_start" fi # Step 7: Shutdown VM step_start=$(date +%s) log "STEP 7: Shutting down VM" ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" "shutdown /s /t 30" 2>/dev/null sleep 60 qm stop "$DR_VM_ID" 2>/dev/null track_step "VM Shutdown" true "VM stopped" "$step_start" else track_step "VM Startup" false "Failed to start VM $DR_VM_ID" "$step_start" fi fi # Calculate total duration local test_end_time=$(date +%s) local total_duration=$(( (test_end_time - TEST_START_TIME) / 60 )) # Prepare notification data local steps_json=$(printf '%s,' "${TEST_STEPS[@]}" | sed 's/,$//') local errors_json=$(printf '"%s",' "${ERRORS[@]}" | sed 's/,$//') local warnings_json=$(printf '"%s",' "${WARNINGS[@]}" | sed 's/,$//') local json_data=$(cat < Notifications > Add matching rules for 'oracle-dr-test'" ;; --help) echo "Oracle DR Weekly Test for Proxmox" echo "Usage:" echo " $0 - Run DR test" echo " $0 --install - Create notification templates" echo " $0 --help - Show this help" ;; *) # Check if templates exist, create if missing if [ ! -f "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" ]; then echo -e "${YELLOW}Templates not found, creating...${NC}" create_templates echo "" fi # Run DR test run_dr_test ;; esac } # Check dependencies if ! command -v jq &> /dev/null; then echo -e "${RED}Error: jq is not installed${NC}" echo "Install with: apt-get install jq" exit 1 fi main "$@"

Oracle DR Test Report

{{#if is_success}}✓ TEST PASSED{{else}}✗ TEST FAILED{{/if}}

Test Summary

Test Steps Timeline

Errors Encountered

Warnings

System Details