+
+ |
+ Oracle DR Test {{test_result}}
+ |
+
+
+ | {{date}} · Severity: {{severity}} |
+
+
+
+
+ | Outcome | {{test_result}} |
+ | Duration | {{total_duration}} min (restore {{restore_duration}} min) |
+ | Backups used | {{backup_count}} |
+ | Tables restored | {{tables_restored}} |
-
+ |
+
-
-
- Log File: {{log_file}}
- Next Scheduled Test: Next Saturday 06:00
-
-
+
+
+ | Components |
+ | VM {{vm_id}} ({{vm_ip}}): {{vm_status}} |
+ | NFS: {{nfs_status}} |
+ | Database: {{database_status}} |
+ | Cleanup: {{disk_freed}} GB freed |
+
+ |
+
+
+
+
+
+ | Steps |
+ {{#each test_steps}}
+
+ | {{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}} ({{this.duration}}s){{#if this.status}} – {{this.status}}{{/if}} |
+
+ {{/each}}
+
+ |
+
+
+ {{#if has_errors}}
+
+
+
+ | Issues |
+ {{#each errors}}
+ | • {{this}} |
+ {{/each}}
+
+ |
+
+ {{/if}}
+
+ {{#if has_warnings}}
+
+
+
+ | Warnings |
+ {{#each warnings}}
+ | • {{this}} |
+ {{/each}}
+
+ |
+
+ {{/if}}
+
+
+
+
+ | Restore Log (first 200 lines) |
+ | {{restore_log}} |
+
+ |
+
+
+
+ |
+ Log: {{log_file}} · Next test: Saturday 06:00
+ |
+
+
EOF
@@ -421,7 +286,16 @@ track_step() {
local end_time=$(date +%s)
local duration=$((end_time - start_time))
- TEST_STEPS+=("{\"name\":\"$name\",\"passed\":$passed,\"status\":\"$status\",\"duration\":$duration}")
+ local step_json
+ step_json=$(jq -n \
+ --arg name "$name" \
+ --arg status "$status" \
+ --arg duration "$duration" \
+ --arg passed "$passed" \
+ '{name:$name, status:$status, duration:($duration|tonumber), passed:($passed == "true")}'
+ )
+
+ TEST_STEPS+=("$step_json")
if [ "$passed" = "false" ]; then
ERRORS+=("$name: $status")
@@ -433,6 +307,14 @@ run_dr_test() {
local test_result="FAILED"
local severity="error"
local is_success=false
+ local restore_duration=0
+ local tables_restored=0
+ local db_status="UNKNOWN"
+ local nfs_status="Not checked"
+ local vm_status_label="Not started"
+ local cleanup_freed=0
+ local backup_count=0
+ local restore_log="Not collected"
log "=========================================="
log "Oracle DR Weekly Test - Starting"
@@ -443,7 +325,7 @@ run_dr_test() {
log "STEP 1: Pre-flight checks"
# Check backups exist
- local backup_count=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l || echo "0")
+ backup_count=$(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*.BKP' 2>/dev/null | wc -l)
if [ "$backup_count" -lt 2 ]; then
track_step "Pre-flight checks" false "Insufficient backups (found: $backup_count)" "$step_start"
@@ -456,6 +338,7 @@ run_dr_test() {
log "STEP 2: Starting DR VM"
if qm start "$DR_VM_ID" 2>/dev/null; then
+ vm_status_label="Running"
sleep 180 # Wait for boot
track_step "VM Startup" true "VM $DR_VM_ID started" "$step_start"
@@ -463,7 +346,7 @@ run_dr_test() {
step_start=$(date +%s)
log "STEP 3: Verifying NFS mount"
- local nfs_status="Not Mounted"
+ nfs_status="Not Mounted"
if ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 "$DR_VM_USER@$DR_VM_IP" \
"powershell -Command 'Test-Path F:\\ROA\\autobackup'" 2>/dev/null; then
nfs_status="Mounted"
@@ -482,7 +365,7 @@ run_dr_test() {
"D:\\oracle\\scripts\\rman_restore_from_zero.cmd" 2>&1 | tee -a "$LOG_FILE"; then
local restore_end=$(date +%s)
- local restore_duration=$(( (restore_end - restore_start) / 60 ))
+ restore_duration=$(( (restore_end - restore_start) / 60 ))
track_step "Database Restore" true "Restored in $restore_duration minutes" "$step_start"
@@ -490,11 +373,13 @@ run_dr_test() {
step_start=$(date +%s)
log "STEP 5: Verifying database"
- local db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
+ db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
"cmd /c 'echo SELECT STATUS FROM V\$INSTANCE; | sqlplus -s / as sysdba' | findstr OPEN" || echo "")
- local tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
+ tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
"cmd /c 'echo SELECT COUNT(*) FROM DBA_TABLES WHERE OWNER NOT IN (''SYS'',''SYSTEM''); | sqlplus -s / as sysdba' | grep -o '[0-9]*' | tail -1" || echo "0")
+ tables_restored=$(echo "$tables_restored" | tr -cd '0-9')
+ [ -z "$tables_restored" ] && tables_restored=0
if [[ "$db_status" =~ "OPEN" ]]; then
track_step "Database Verification" true "Database OPEN, $tables_restored tables" "$step_start"
@@ -505,6 +390,11 @@ run_dr_test() {
track_step "Database Verification" false "Database not OPEN" "$step_start"
fi
+ # Collect restore log from VM
+ log "Collecting restore log from DR VM..."
+ restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
+ "type D:\\oracle\\logs\\restore_from_zero.log 2>nul" | head -200 || echo "Log not available")
+
# Step 6: Cleanup
step_start=$(date +%s)
log "STEP 6: Running cleanup"
@@ -512,7 +402,8 @@ run_dr_test() {
ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
"D:\\oracle\\scripts\\cleanup_database.cmd" 2>/dev/null
- track_step "Cleanup" true "Database cleaned, ~8GB freed" "$step_start"
+ cleanup_freed=8
+ track_step "Cleanup" true "Database cleaned, ~${cleanup_freed}GB freed" "$step_start"
else
track_step "Database Restore" false "Restore failed" "$step_start"
@@ -527,9 +418,11 @@ run_dr_test() {
qm stop "$DR_VM_ID" 2>/dev/null
track_step "VM Shutdown" true "VM stopped" "$step_start"
+ vm_status_label="Stopped"
else
track_step "VM Startup" false "Failed to start VM $DR_VM_ID" "$step_start"
+ vm_status_label="Failed to start"
fi
fi
@@ -538,9 +431,41 @@ run_dr_test() {
local total_duration=$(( (test_end_time - TEST_START_TIME) / 60 ))
# Prepare notification data
- local steps_json=$(printf '%s,' "${TEST_STEPS[@]}" | sed 's/,$//')
- local errors_json=$(printf '"%s",' "${ERRORS[@]}" | sed 's/,$//')
- local warnings_json=$(printf '"%s",' "${WARNINGS[@]}" | sed 's/,$//')
+ local steps_json
+ if [ ${#TEST_STEPS[@]} -eq 0 ]; then
+ steps_json='[]'
+ else
+ steps_json=$(printf '%s\n' "${TEST_STEPS[@]}" | jq -s '.')
+ fi
+
+ local errors_json
+ if [ ${#ERRORS[@]} -eq 0 ]; then
+ errors_json='[]'
+ else
+ errors_json=$(printf '%s\n' "${ERRORS[@]}" | jq -R . | jq -s .)
+ fi
+
+ local warnings_json
+ if [ ${#WARNINGS[@]} -eq 0 ]; then
+ warnings_json='[]'
+ else
+ warnings_json=$(printf '%s\n' "${WARNINGS[@]}" | jq -R . | jq -s .)
+ fi
+
+ local has_errors=false
+ local has_warnings=false
+ [ ${#ERRORS[@]} -gt 0 ] && has_errors=true
+ [ ${#WARNINGS[@]} -gt 0 ] && has_warnings=true
+
+ if [ "$is_success" = true ] && [ "$has_warnings" = true ]; then
+ severity="warning"
+ fi
+
+ local db_status_clean=$(echo "$db_status" | tr -d '\r' | sed 's/^ *//;s/ *$//')
+
+ # Escape restore log for JSON
+ local restore_log_json
+ restore_log_json=$(echo "$restore_log" | jq -Rs .)
local json_data=$(cat <