94758421c5
Testul DR din 2026-08-08 a raportat "Restore failed" dupa 11 secunde, fara niciun log RMAN. Cauza nu a fost restore-ul: KB5101001 fusese descarcat in timpul testului din 2026-08-01 (singurul moment in care VM 109 e pornit), a ramas staged dupa qm stop si s-a finalizat la boot-ul testului urmator. Cronologie din Event Log-ul guest-ului: 06:00:58 RestartManager 10010 - nu poate reporni powershell.exe (restore-ul) 06:01:04 SCM 7034 - OpenSSH SSH Server terminat neasteptat 06:01:06 pveelite: client_loop: send disconnect: Broken pipe -> FAILED 06:01:38 VM-ul se reboteaza singur Fereastra testului (Sambata 06:00) era in afara Active Hours (08:00-17:00), deci pentru Windows era fereastra de mentenanta valida - iar VM 109 fiind pornit doar in timpul testului, aceea era singura fereastra posibila. Agravant: sshd nu avea acsiuni de recovery (RESET_PERIOD 0), deci dupa ce a murit a ramas mort si au esuat si colectarea logului si shutdown-ul gratios. Masuri: - NoAutoUpdate=1 + AUOptions=2 pe VM 109 (aplicat direct in registry) - actiuni de recovery pentru sshd: restart la 5s/10s/30s, reset=86400 - guard "STEP 3b: Windows servicing" inainte de restore (check_servicing.ps1): asteapta idle 300s, consuma controlat un reboot in asteptare, altfel abandoneaza cu "ABORTED - Windows servicing" in loc de un "Restore failed" inselator. Fail-open daca checkul lipseste - nu are voie sa pice testul. - fereastra lunara de patching (vm109-patch-window.sh + install_updates.ps1), prima duminica 03:00, cu re-armare NoAutoUpdate=1 indiferent de rezultat Adaugat si .gitattributes: cu core.autocrlf=true scripturile .sh ajungeau in working tree cu CRLF, iar ele se deployeaza prin scp direct pe Proxmox. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BhQBTegE4PiMPPaapLHjkc
790 lines
30 KiB
Bash
790 lines
30 KiB
Bash
#!/bin/bash
|
||
#
|
||
# Oracle DR Weekly Test with Proxmox PVE::Notify
|
||
# Automated DR test with notifications via Proxmox notification system
|
||
#
|
||
# Location: /opt/scripts/weekly-dr-test-proxmox.sh (on Proxmox host)
|
||
# Schedule: Add to cron for weekly execution (Saturdays)
|
||
#
|
||
# This script is SELF-SUFFICIENT:
|
||
# - Automatically creates notification templates if they don't exist
|
||
# - Uses Proxmox native notification system
|
||
# - No email configuration needed - uses existing Proxmox setup
|
||
#
|
||
# Installation:
|
||
# cp weekly-dr-test-proxmox.sh /opt/scripts/
|
||
# chmod +x /opt/scripts/weekly-dr-test-proxmox.sh
|
||
# /opt/scripts/weekly-dr-test-proxmox.sh --install # Creates templates
|
||
# crontab -e # Add: 0 6 * * 6 /opt/scripts/weekly-dr-test-proxmox.sh
|
||
#
|
||
# Author: Claude (based on ha-monitor.sh pattern)
|
||
# Version: 1.0
|
||
|
||
set -euo pipefail
|
||
|
||
# Cleanup trap: stop VM 109 on script exit ONLY if this script started it.
|
||
# Fixes incident 2026-04-20: script crashed at SSH step and left VM 109 running
|
||
# for 2.5 days, causing OOM cascade on pveelite after pvemini HA failover.
|
||
# Guard prevents the trap from killing an externally-running VM during
|
||
# --install / --help or when an operator launched it manually for debugging.
|
||
DR_VM_STARTED_BY_US=false
|
||
cleanup_vm() {
|
||
local rc=$?
|
||
if [ "$DR_VM_STARTED_BY_US" = "true" ] \
|
||
&& qm status "${DR_VM_ID:-109}" 2>/dev/null | grep -q running; then
|
||
echo "[trap] VM ${DR_VM_ID:-109} still running at exit (rc=$rc), forcing stop"
|
||
qm stop "${DR_VM_ID:-109}" --skiplock 2>/dev/null || true
|
||
fi
|
||
exit $rc
|
||
}
|
||
trap cleanup_vm EXIT
|
||
|
||
# Set proper PATH for cron execution
|
||
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||
|
||
# Configuration
|
||
DR_VM_ID="109"
|
||
|
||
# Cluster-aware exit: only the node currently hosting VM 109 should run the
|
||
# test. With cron deployed on both pveelite (normal home) and pvemini (DR
|
||
# failover home), this guard ensures only one instance fires.
|
||
if [ ! -f "/etc/pve/qemu-server/${DR_VM_ID}.conf" ] && [ "${1:-}" != "--install" ] && [ "${1:-}" != "--help" ]; then
|
||
exit 0
|
||
fi
|
||
|
||
DR_VM_IP="10.0.20.37"
|
||
DR_VM_PORT="22122"
|
||
DR_VM_USER="romfast"
|
||
BACKUP_PATH="/mnt/pve/oracle-backups/ROA/autobackup"
|
||
MAX_RESTORE_TIME_MIN=30
|
||
TEMPLATE_DIR="/usr/share/pve-manager/templates/default"
|
||
LOG_DIR="/var/log/oracle-dr"
|
||
LOG_FILE="$LOG_DIR/dr_test_$(date +%Y%m%d_%H%M%S).log"
|
||
|
||
# Colors
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
NC='\033[0m'
|
||
|
||
# Create log directory
|
||
mkdir -p "$LOG_DIR"
|
||
|
||
# Function to create notification templates
|
||
create_templates() {
|
||
echo -e "${GREEN}Creating Oracle DR test notification templates...${NC}"
|
||
|
||
# Create templates directory if needed
|
||
mkdir -p "$TEMPLATE_DIR"
|
||
|
||
# Subject template
|
||
cat > "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" <<'EOF'
|
||
Oracle DR Test {{test_result}} | {{date}}
|
||
EOF
|
||
|
||
# Text body template
|
||
cat > "$TEMPLATE_DIR/oracle-dr-test-body.txt.hbs" <<'EOF'
|
||
Oracle DR Test {{test_result}} | {{date}}
|
||
Severity: {{severity}}
|
||
|
||
SUMMARY
|
||
- Outcome: {{test_result}}
|
||
- Duration: {{total_duration}} min (restore {{restore_duration}} min)
|
||
- Backups used: {{backup_count}}
|
||
- Tables restored: {{tables_restored}}
|
||
|
||
COMPONENTS
|
||
- VM {{vm_id}} ({{vm_ip}}): {{vm_status}}
|
||
- NFS: {{nfs_status}}
|
||
- Database: {{database_status}}
|
||
- Cleanup: {{disk_freed}} GB freed
|
||
|
||
STEPS
|
||
{{#each test_steps}}
|
||
- {{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}} ({{this.duration}}s){{#if this.status}} - {{this.status}}{{/if}}
|
||
{{/each}}
|
||
|
||
{{#if has_errors}}
|
||
ISSUES
|
||
{{#each errors}}
|
||
- {{this}}
|
||
{{/each}}
|
||
{{/if}}
|
||
|
||
{{#if has_warnings}}
|
||
WARNINGS
|
||
{{#each warnings}}
|
||
- {{this}}
|
||
{{/each}}
|
||
{{/if}}
|
||
|
||
RMAN RESTORE LOG (complete)
|
||
---
|
||
{{restore_log}}
|
||
---
|
||
|
||
BASH SCRIPT LOG (last 100 lines)
|
||
---
|
||
{{bash_log}}
|
||
---
|
||
|
||
Full log: {{log_file}}
|
||
Next test: Saturday 06:00
|
||
EOF
|
||
|
||
# HTML body template (compact Gmail-friendly layout)
|
||
cat > "$TEMPLATE_DIR/oracle-dr-test-body.html.hbs" <<'EOF'
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<title>Oracle DR Test {{test_result}} | {{date}}</title>
|
||
</head>
|
||
<body style="margin:0;padding:16px;font-family:Arial,Helvetica,sans-serif;background:#ffffff;color:#2c3e50;">
|
||
<table style="width:100%;max-width:640px;margin:0 auto;border-collapse:collapse;">
|
||
<tr>
|
||
<td style="padding:0 0 12px 0;font-size:18px;font-weight:600;">
|
||
Oracle DR Test {{test_result}}
|
||
</td>
|
||
</tr>
|
||
<tr>
|
||
<td style="padding:0 0 8px 0;font-size:13px;color:#6c757d;">{{date}} · Severity: {{severity}}</td>
|
||
</tr>
|
||
<tr>
|
||
<td style="padding:12px;border:1px solid #e1e4e8;border-radius:4px;">
|
||
<table style="width:100%;border-collapse:collapse;font-size:14px;">
|
||
<tr><td style="padding:4px 0;">Outcome</td><td style="padding:4px 0;text-align:right;">{{test_result}}</td></tr>
|
||
<tr><td style="padding:4px 0;">Duration</td><td style="padding:4px 0;text-align:right;">{{total_duration}} min (restore {{restore_duration}} min)</td></tr>
|
||
<tr><td style="padding:4px 0;">Backups used</td><td style="padding:4px 0;text-align:right;">{{backup_count}}</td></tr>
|
||
<tr><td style="padding:4px 0;">Tables restored</td><td style="padding:4px 0;text-align:right;">{{tables_restored}}</td></tr>
|
||
</table>
|
||
</td>
|
||
</tr>
|
||
|
||
<tr>
|
||
<td style="padding:16px 0 0 0;">
|
||
<table style="width:100%;border-collapse:collapse;font-size:14px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
|
||
<tr><td style="padding:8px 12px;font-weight:600;">Components</td></tr>
|
||
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">VM {{vm_id}} ({{vm_ip}}): {{vm_status}}</td></tr>
|
||
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">NFS: {{nfs_status}}</td></tr>
|
||
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">Database: {{database_status}}</td></tr>
|
||
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">Cleanup: {{disk_freed}} GB freed</td></tr>
|
||
</table>
|
||
</td>
|
||
</tr>
|
||
|
||
<tr>
|
||
<td style="padding:16px 0 0 0;">
|
||
<table style="width:100%;border-collapse:collapse;font-size:14px;">
|
||
<tr><td style="padding:0 0 6px 0;font-weight:600;">Steps</td></tr>
|
||
{{#each test_steps}}
|
||
<tr>
|
||
<td style="padding:4px 0;border-bottom:1px solid #f1f1f1;">{{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}} ({{this.duration}}s){{#if this.status}} – {{this.status}}{{/if}}</td>
|
||
</tr>
|
||
{{/each}}
|
||
</table>
|
||
</td>
|
||
</tr>
|
||
|
||
{{#if has_errors}}
|
||
<tr>
|
||
<td style="padding:16px 0 0 0;">
|
||
<table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff5f5;border:1px solid #f1b0b7;border-radius:4px;">
|
||
<tr><td style="padding:8px 12px;font-weight:600;color:#c82333;">Issues</td></tr>
|
||
{{#each errors}}
|
||
<tr><td style="padding:6px 12px;border-top:1px solid #f8d7da;">• {{this}}</td></tr>
|
||
{{/each}}
|
||
</table>
|
||
</td>
|
||
</tr>
|
||
{{/if}}
|
||
|
||
{{#if has_warnings}}
|
||
<tr>
|
||
<td style="padding:16px 0 0 0;">
|
||
<table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff8e5;border:1px solid #ffe8a1;border-radius:4px;">
|
||
<tr><td style="padding:8px 12px;font-weight:600;color:#856404;">Warnings</td></tr>
|
||
{{#each warnings}}
|
||
<tr><td style="padding:6px 12px;border-top:1px solid #ffe8a1;">• {{this}}</td></tr>
|
||
{{/each}}
|
||
</table>
|
||
</td>
|
||
</tr>
|
||
{{/if}}
|
||
|
||
<tr>
|
||
<td style="padding:16px 0 0 0;">
|
||
<table style="width:100%;border-collapse:collapse;font-size:12px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
|
||
<tr><td style="padding:8px 12px;font-weight:600;font-size:13px;">RMAN Restore Log (complete)</td></tr>
|
||
<tr><td style="padding:8px 12px;font-family:monospace;white-space:pre-wrap;word-wrap:break-word;border-top:1px solid #e1e4e8;">{{restore_log}}</td></tr>
|
||
</table>
|
||
</td>
|
||
</tr>
|
||
|
||
<tr>
|
||
<td style="padding:16px 0 0 0;">
|
||
<table style="width:100%;border-collapse:collapse;font-size:12px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
|
||
<tr><td style="padding:8px 12px;font-weight:600;font-size:13px;">Bash Script Log (last 100 lines)</td></tr>
|
||
<tr><td style="padding:8px 12px;font-family:monospace;white-space:pre-wrap;word-wrap:break-word;border-top:1px solid #e1e4e8;">{{bash_log}}</td></tr>
|
||
</table>
|
||
</td>
|
||
</tr>
|
||
|
||
<tr>
|
||
<td style="padding:16px 0 0 0;font-size:12px;color:#6c757d;">
|
||
Full log: {{log_file}} · Next test: Saturday 06:00
|
||
</td>
|
||
</tr>
|
||
</table>
|
||
</body>
|
||
</html>
|
||
EOF
|
||
|
||
echo -e "${GREEN}Templates created successfully in $TEMPLATE_DIR${NC}"
|
||
}
|
||
|
||
# Function to send notification via PVE::Notify
|
||
send_pve_notification() {
|
||
local severity="$1"
|
||
local data="$2"
|
||
|
||
# Create Perl script to call PVE::Notify
|
||
cat > /tmp/oracle-dr-notify.pl <<'PERL_SCRIPT'
|
||
#!/usr/bin/perl
|
||
use strict;
|
||
use warnings;
|
||
use PVE::Notify;
|
||
use JSON;
|
||
|
||
my $json_data = do { local $/; <STDIN> };
|
||
my $data = decode_json($json_data);
|
||
|
||
my $severity = $data->{severity} // 'info';
|
||
my $template_name = 'oracle-dr-test';
|
||
|
||
# Add fields for matching rules
|
||
my $fields = {
|
||
type => 'oracle-dr-test',
|
||
severity => $severity,
|
||
test_result => $data->{test_result},
|
||
};
|
||
|
||
# Send notification
|
||
eval {
|
||
PVE::Notify::notify(
|
||
$severity,
|
||
$template_name,
|
||
$data,
|
||
$fields
|
||
);
|
||
};
|
||
|
||
if ($@) {
|
||
print "Error sending notification: $@\n";
|
||
exit 1;
|
||
}
|
||
|
||
print "Notification sent successfully\n";
|
||
PERL_SCRIPT
|
||
|
||
chmod +x /tmp/oracle-dr-notify.pl
|
||
|
||
# Send notification
|
||
echo "$data" | perl /tmp/oracle-dr-notify.pl
|
||
|
||
rm -f /tmp/oracle-dr-notify.pl
|
||
}
|
||
|
||
# Logging functions
|
||
log() {
|
||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
|
||
}
|
||
|
||
log_error() {
|
||
echo -e "${RED}[ERROR]${NC} $1" | tee -a "$LOG_FILE"
|
||
}
|
||
|
||
log_warning() {
|
||
echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$LOG_FILE"
|
||
}
|
||
|
||
log_success() {
|
||
echo -e "${GREEN}[SUCCESS]${NC} $1" | tee -a "$LOG_FILE"
|
||
}
|
||
|
||
# Test tracking
|
||
TEST_STEPS=()
|
||
ERRORS=()
|
||
WARNINGS=()
|
||
TEST_START_TIME=$(date +%s)
|
||
|
||
# Function to track test steps
|
||
track_step() {
|
||
local name="$1"
|
||
local passed="$2"
|
||
local status="$3"
|
||
local start_time="$4"
|
||
local end_time=$(date +%s)
|
||
local duration=$((end_time - start_time))
|
||
|
||
local step_json
|
||
step_json=$(jq -n \
|
||
--arg name "$name" \
|
||
--arg status "$status" \
|
||
--arg duration "$duration" \
|
||
--arg passed "$passed" \
|
||
'{name:$name, status:$status, duration:($duration|tonumber), passed:($passed == "true")}'
|
||
)
|
||
|
||
TEST_STEPS+=("$step_json")
|
||
|
||
if [ "$passed" = "false" ]; then
|
||
ERRORS+=("$name: $status")
|
||
fi
|
||
}
|
||
|
||
# Main test workflow
|
||
run_dr_test() {
|
||
local test_result="FAILED"
|
||
local severity="error"
|
||
local is_success=false
|
||
local restore_duration=0
|
||
local tables_restored=0
|
||
local db_status="UNKNOWN"
|
||
local nfs_status="Not checked"
|
||
local vm_status_label="Not started"
|
||
local cleanup_freed=0
|
||
local backup_count=0
|
||
local restore_log="Not collected"
|
||
local servicing_abort=false
|
||
|
||
log "=========================================="
|
||
log "Oracle DR Weekly Test - Starting"
|
||
log "=========================================="
|
||
|
||
# Step 1: Pre-flight checks
|
||
local step_start=$(date +%s)
|
||
log "STEP 1: Pre-flight checks"
|
||
|
||
# Check 1a: Cluster quorate and not degraded.
|
||
# Refusing to test during a node outage prevents stacking VM 109 (6 GB)
|
||
# on top of a host already absorbing failover load — the 04-20 trigger.
|
||
local cluster_quorate
|
||
cluster_quorate=$(pvecm status 2>/dev/null | awk '/Quorate:/ {print $2}')
|
||
if [ "$cluster_quorate" != "Yes" ]; then
|
||
track_step "Pre-flight checks" false "Cluster not quorate (degraded?)" "$step_start"
|
||
test_result="FAILED - Cluster degraded"
|
||
backup_count=0
|
||
fi
|
||
|
||
# Check 1b: Memory headroom on this host. Calculated from VM 109 config
|
||
# so it scales automatically if VM 109 memory is later resized.
|
||
local dr_vm_mem_mb avail_mb min_free_mb
|
||
dr_vm_mem_mb=$(qm config "$DR_VM_ID" 2>/dev/null | awk '/^memory:/ {print $2}')
|
||
avail_mb=$(awk '/^MemAvailable:/ {print int($2/1024)}' /proc/meminfo)
|
||
min_free_mb=$((dr_vm_mem_mb + 1024))
|
||
|
||
if [ "$test_result" != "FAILED - Cluster degraded" ] \
|
||
&& [ "$avail_mb" -lt "$min_free_mb" ]; then
|
||
track_step "Pre-flight checks" false \
|
||
"Insufficient memory: ${avail_mb}MB available, need ${min_free_mb}MB" "$step_start"
|
||
test_result="FAILED - Insufficient memory"
|
||
backup_count=0
|
||
fi
|
||
|
||
# Check 1c: Backups exist (only if previous checks passed)
|
||
if [ "$test_result" = "FAILED" ]; then
|
||
backup_count=$(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*.BKP' 2>/dev/null | wc -l)
|
||
fi
|
||
|
||
if [ "$test_result" != "FAILED" ]; then
|
||
: # already failed in cluster/memory check, skip
|
||
elif [ "$backup_count" -lt 2 ]; then
|
||
track_step "Pre-flight checks" false "Insufficient backups (found: $backup_count)" "$step_start"
|
||
test_result="FAILED - No backups"
|
||
else
|
||
track_step "Pre-flight checks" true \
|
||
"Found $backup_count backups, ${avail_mb}MB available" "$step_start"
|
||
|
||
# Step 2: Start VM
|
||
step_start=$(date +%s)
|
||
log "STEP 2: Starting DR VM"
|
||
|
||
local qm_start_output
|
||
if qm_start_output=$(qm start "$DR_VM_ID" 2>&1); then
|
||
DR_VM_STARTED_BY_US=true
|
||
vm_status_label="Running"
|
||
|
||
# Intelligent VM boot wait with polling (max 180s)
|
||
local MAX_BOOT_WAIT=180
|
||
local POLL_INTERVAL=5
|
||
local boot_elapsed=0
|
||
local vm_ready=false
|
||
|
||
log "Waiting for VM to become ready (SSH + PowerShell, max ${MAX_BOOT_WAIT}s)..."
|
||
|
||
while [ $boot_elapsed -lt $MAX_BOOT_WAIT ]; do
|
||
# Check 1: VM running status in Proxmox
|
||
local vm_qm_status
|
||
vm_qm_status=$(qm status "$DR_VM_ID" 2>/dev/null | grep -o "running" || echo "")
|
||
|
||
if [ "$vm_qm_status" = "running" ]; then
|
||
# Check 2: SSH connectivity and PowerShell availability (what we actually need)
|
||
if ssh -p "$DR_VM_PORT" -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes "$DR_VM_USER@$DR_VM_IP" \
|
||
"powershell -Command 'Write-Output ready'" >/dev/null 2>&1; then
|
||
log "VM ready after ${boot_elapsed}s (SSH and PowerShell responding)"
|
||
vm_ready=true
|
||
break
|
||
fi
|
||
fi
|
||
|
||
sleep $POLL_INTERVAL
|
||
boot_elapsed=$((boot_elapsed + POLL_INTERVAL))
|
||
|
||
# Progress logging every 30 seconds
|
||
if [ $((boot_elapsed % 30)) -eq 0 ] && [ $boot_elapsed -lt $MAX_BOOT_WAIT ]; then
|
||
log "Still waiting for VM... (${boot_elapsed}s/${MAX_BOOT_WAIT}s elapsed)"
|
||
fi
|
||
done
|
||
|
||
if [ "$vm_ready" = false ]; then
|
||
log_warning "VM did not respond within ${MAX_BOOT_WAIT}s, continuing anyway (may cause subsequent failures)"
|
||
fi
|
||
|
||
track_step "VM Startup" true "VM $DR_VM_ID started and ready (${boot_elapsed}s)" "$step_start"
|
||
|
||
# Step 3: Verify NFS mount
|
||
step_start=$(date +%s)
|
||
log "STEP 3: Verifying NFS mount"
|
||
|
||
nfs_status="Not Mounted"
|
||
if ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 "$DR_VM_USER@$DR_VM_IP" \
|
||
"powershell -Command 'Test-Path F:\\ROA\\autobackup'" 2>/dev/null; then
|
||
nfs_status="Mounted"
|
||
track_step "NFS Mount Check" true "F:\\ drive accessible" "$step_start"
|
||
else
|
||
track_step "NFS Mount Check" false "F:\\ drive not accessible" "$step_start"
|
||
WARNINGS+=("NFS mount may need manual intervention")
|
||
fi
|
||
|
||
# Step 3b: Windows servicing guard
|
||
#
|
||
# Incident 2026-08-08: KB5101001 descărcat în timpul testului din
|
||
# 2026-08-01 s-a finalizat la boot-ul următorului test. RestartManager
|
||
# a omorât powershell.exe (restore-ul) la 06:00:58, sshd la 06:01:04,
|
||
# iar VM-ul a rebootat la 06:01:38 — testul a raportat "Restore failed"
|
||
# la 11s, fără log RMAN, deși RMAN nu pornise deloc.
|
||
#
|
||
# Măsura principală este NoAutoUpdate=1 pe VM 109 (vezi
|
||
# vm109-patch-window.sh pentru fereastra lunară de patching).
|
||
# Guard-ul de aici este plasa de siguranță: dacă totuși stack-ul de
|
||
# servicing este activ, nu pornim restore-ul într-un VM care e pe
|
||
# cale să se reboteze — raportăm cauza reală în loc de un
|
||
# "Restore failed" care arată ca o problemă de backup.
|
||
step_start=$(date +%s)
|
||
log "STEP 3b: Checking Windows servicing state"
|
||
|
||
local servicing_out=""
|
||
local servicing_idle=false
|
||
local SERVICING_WAIT=300
|
||
local servicing_elapsed=0
|
||
|
||
while [ $servicing_elapsed -lt $SERVICING_WAIT ]; do
|
||
if servicing_out=$(ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$DR_VM_USER@$DR_VM_IP" \
|
||
"powershell -ExecutionPolicy Bypass -File D:\\oracle\\scripts\\check_servicing.ps1" 2>/dev/null | tr -d '\r'); then
|
||
servicing_idle=true
|
||
break
|
||
fi
|
||
|
||
# Check absent (ieșire non-zero fără output) => nu blocăm testul.
|
||
if [ -z "$servicing_out" ]; then
|
||
log_warning "check_servicing.ps1 did not respond, skipping servicing guard"
|
||
WARNINGS+=("Servicing guard skipped: check_servicing.ps1 missing or unreachable on VM $DR_VM_ID")
|
||
servicing_idle=true
|
||
break
|
||
fi
|
||
|
||
log "Windows servicing busy: $servicing_out (${servicing_elapsed}s/${SERVICING_WAIT}s)"
|
||
sleep 15
|
||
servicing_elapsed=$((servicing_elapsed + 15))
|
||
done
|
||
|
||
if [ "$servicing_idle" = true ]; then
|
||
track_step "Windows Servicing Check" true "Servicing stack idle" "$step_start"
|
||
else
|
||
# Reboot în așteptare: îl consumăm controlat, o singură dată,
|
||
# ca testul să poată continua pe un sistem stabil.
|
||
if echo "$servicing_out" | grep -q "REBOOT_PENDING=true"; then
|
||
log_warning "Pending reboot detected, rebooting VM $DR_VM_ID once before restore"
|
||
ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 "$DR_VM_USER@$DR_VM_IP" "shutdown /r /t 5 /f" 2>/dev/null || true
|
||
sleep 45
|
||
|
||
local reboot_elapsed=0
|
||
while [ $reboot_elapsed -lt 300 ]; do
|
||
if servicing_out=$(ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 -o StrictHostKeyChecking=no -o BatchMode=yes "$DR_VM_USER@$DR_VM_IP" \
|
||
"powershell -ExecutionPolicy Bypass -File D:\\oracle\\scripts\\check_servicing.ps1" 2>/dev/null | tr -d '\r'); then
|
||
servicing_idle=true
|
||
break
|
||
fi
|
||
sleep 15
|
||
reboot_elapsed=$((reboot_elapsed + 15))
|
||
done
|
||
fi
|
||
|
||
if [ "$servicing_idle" = true ]; then
|
||
track_step "Windows Servicing Check" true "Servicing completed after controlled reboot" "$step_start"
|
||
WARNINGS+=("VM $DR_VM_ID had a pending servicing reboot; consumed before restore. Check NoAutoUpdate policy.")
|
||
else
|
||
track_step "Windows Servicing Check" false \
|
||
"Windows Update/servicing active, restore not attempted - backups NOT implicated ($servicing_out)" "$step_start"
|
||
test_result="ABORTED - Windows servicing"
|
||
servicing_abort=true
|
||
fi
|
||
fi
|
||
|
||
# Step 4: Run restore
|
||
step_start=$(date +%s)
|
||
local restore_start=$step_start
|
||
log "STEP 4: Running database restore"
|
||
|
||
if [ "$servicing_abort" = true ]; then
|
||
log_error "Skipping restore: Windows servicing active on VM $DR_VM_ID"
|
||
# Fără track_step aici: eșecul e deja raportat de "Windows Servicing
|
||
# Check", iar un al doilea ERRORS ar duplica alarma pentru o singură cauză.
|
||
restore_log="Restore not attempted. Windows Update/servicing was active on VM $DR_VM_ID: $servicing_out"
|
||
elif ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||
"powershell -ExecutionPolicy Bypass -File D:\\oracle\\scripts\\rman_restore_from_zero.ps1 -TestMode" 2>&1 | tee -a "$LOG_FILE"; then
|
||
|
||
local restore_end=$(date +%s)
|
||
restore_duration=$(( (restore_end - restore_start) / 60 ))
|
||
|
||
track_step "Database Restore" true "Restored in $restore_duration minutes" "$step_start"
|
||
|
||
# Step 5: Verify database
|
||
step_start=$(date +%s)
|
||
log "STEP 5: Verifying database"
|
||
|
||
# Parse database status from LOG_FILE (rman_restore_from_zero.ps1 already verified it)
|
||
# Look for "OPEN_MODE: READ WRITE" in the captured output
|
||
if grep -q "OPEN_MODE: READ WRITE" "$LOG_FILE" 2>/dev/null; then
|
||
db_status="READ WRITE"
|
||
else
|
||
db_status=""
|
||
fi
|
||
|
||
# Parse table count from LOG_FILE (already captured in STEP 3 output)
|
||
# Look for "TABLES: <number>" in the output
|
||
tables_restored=$(grep -oP "TABLES:\s*\K\d+" "$LOG_FILE" 2>/dev/null | tail -1 || echo "0")
|
||
tables_restored=$(echo "$tables_restored" | tr -cd '0-9')
|
||
[ -z "$tables_restored" ] && tables_restored=0
|
||
|
||
if [[ "$db_status" == "READ WRITE" ]] && [ "$tables_restored" -gt 0 ]; then
|
||
track_step "Database Verification" true "Database OPEN, $tables_restored tables" "$step_start"
|
||
test_result="PASSED"
|
||
severity="info"
|
||
is_success=true
|
||
else
|
||
track_step "Database Verification" false "Database not OPEN" "$step_start"
|
||
fi
|
||
|
||
# Collect restore log from VM (always attempt collection - FULL log)
|
||
log "Collecting restore log from DR VM..."
|
||
restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||
"powershell -Command \"Get-Content 'D:\\oracle\\logs\\restore_from_zero.log' -ErrorAction SilentlyContinue\"" 2>/dev/null || echo "")
|
||
|
||
# If not found, try alternate locations
|
||
if [ -z "$restore_log" ]; then
|
||
restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||
"powershell -Command \"Get-Content 'D:\\oracle\\temp\\restore_from_zero.log' -ErrorAction SilentlyContinue\"" 2>/dev/null || echo "")
|
||
fi
|
||
|
||
# Still not found, use fallback message
|
||
if [ -z "$restore_log" ]; then
|
||
restore_log="Restore log not available (file may not exist or was not generated)"
|
||
fi
|
||
|
||
# Step 6: Cleanup (AFTER restore - stop service to release file locks)
|
||
step_start=$(date +%s)
|
||
log "STEP 6: Running cleanup"
|
||
|
||
ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||
"powershell -ExecutionPolicy Bypass -File D:\\oracle\\scripts\\cleanup_database.ps1 /SILENT /AFTER" 2>/dev/null
|
||
|
||
cleanup_freed=8
|
||
track_step "Cleanup" true "Database cleaned, ~${cleanup_freed}GB freed" "$step_start"
|
||
|
||
else
|
||
# Collect restore log even when restore fails (FULL log)
|
||
log "Collecting restore log after failure..."
|
||
restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||
"powershell -Command \"Get-Content 'D:\\oracle\\logs\\restore_from_zero.log' -ErrorAction SilentlyContinue\"" 2>/dev/null || echo "")
|
||
|
||
if [ -z "$restore_log" ]; then
|
||
restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||
"powershell -Command \"Get-Content 'D:\\oracle\\temp\\restore_from_zero.log' -ErrorAction SilentlyContinue\"" 2>/dev/null || echo "")
|
||
fi
|
||
|
||
# Always try to get some error output from RMAN script
|
||
if [ -z "$restore_log" ]; then
|
||
last_error=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||
"powershell -Command \"Get-Content 'D:\\oracle\\temp\\*.rman' -Tail 20 -ErrorAction SilentlyContinue\"" 2>/dev/null || echo "")
|
||
if [ -n "$last_error" ]; then
|
||
restore_log="RMAN script content (last 20 lines):\n$last_error"
|
||
else
|
||
restore_log="No restore logs or RMAN scripts found"
|
||
fi
|
||
fi
|
||
|
||
track_step "Database Restore" false "Restore failed" "$step_start"
|
||
fi
|
||
|
||
# Step 7: Shutdown VM
|
||
step_start=$(date +%s)
|
||
log "STEP 7: Shutting down VM"
|
||
|
||
ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 "$DR_VM_USER@$DR_VM_IP" "shutdown /s /t 30" 2>/dev/null \
|
||
|| log_warning "SSH shutdown failed, will force qm stop"
|
||
sleep 60
|
||
qm stop "$DR_VM_ID" 2>/dev/null
|
||
|
||
track_step "VM Shutdown" true "VM stopped" "$step_start"
|
||
vm_status_label="Stopped"
|
||
|
||
else
|
||
log_error "qm start $DR_VM_ID failed: $qm_start_output"
|
||
track_step "VM Startup" false "Failed to start VM $DR_VM_ID: $qm_start_output" "$step_start"
|
||
vm_status_label="Failed to start"
|
||
fi
|
||
fi
|
||
|
||
# Calculate total duration
|
||
local test_end_time=$(date +%s)
|
||
local total_duration=$(( (test_end_time - TEST_START_TIME) / 60 ))
|
||
|
||
# Prepare notification data
|
||
local steps_json
|
||
if [ ${#TEST_STEPS[@]} -eq 0 ]; then
|
||
steps_json='[]'
|
||
else
|
||
steps_json=$(printf '%s\n' "${TEST_STEPS[@]}" | jq -s '.')
|
||
fi
|
||
|
||
local errors_json
|
||
if [ ${#ERRORS[@]} -eq 0 ]; then
|
||
errors_json='[]'
|
||
else
|
||
errors_json=$(printf '%s\n' "${ERRORS[@]}" | jq -R . | jq -s .)
|
||
fi
|
||
|
||
local warnings_json
|
||
if [ ${#WARNINGS[@]} -eq 0 ]; then
|
||
warnings_json='[]'
|
||
else
|
||
warnings_json=$(printf '%s\n' "${WARNINGS[@]}" | jq -R . | jq -s .)
|
||
fi
|
||
|
||
local has_errors=false
|
||
local has_warnings=false
|
||
[ ${#ERRORS[@]} -gt 0 ] && has_errors=true
|
||
[ ${#WARNINGS[@]} -gt 0 ] && has_warnings=true
|
||
|
||
if [ "$is_success" = true ] && [ "$has_warnings" = true ]; then
|
||
severity="warning"
|
||
fi
|
||
|
||
local db_status_clean=$(echo "$db_status" | tr -d '\r' | sed 's/^ *//;s/ *$//')
|
||
|
||
# Escape restore log for JSON
|
||
local restore_log_json
|
||
restore_log_json=$(echo "$restore_log" | jq -Rs .)
|
||
|
||
# Collect last 100 lines of bash script log
|
||
local bash_log
|
||
bash_log=$(tail -100 "$LOG_FILE" 2>/dev/null || echo "Bash log not available")
|
||
local bash_log_json
|
||
bash_log_json=$(echo "$bash_log" | jq -Rs .)
|
||
|
||
local json_data=$(cat <<JSON
|
||
{
|
||
"severity": "$severity",
|
||
"test_result": "$test_result",
|
||
"date": "$(date '+%Y-%m-%d %H:%M:%S')",
|
||
"total_duration": $total_duration,
|
||
"is_success": $is_success,
|
||
"has_errors": $has_errors,
|
||
"has_warnings": $has_warnings,
|
||
"test_steps": $steps_json,
|
||
"errors": $errors_json,
|
||
"warnings": $warnings_json,
|
||
"backup_count": $backup_count,
|
||
"restore_duration": $restore_duration,
|
||
"tables_restored": ${tables_restored:-0},
|
||
"database_status": "${db_status_clean:-UNKNOWN}",
|
||
"disk_freed": $cleanup_freed,
|
||
"vm_id": "$DR_VM_ID",
|
||
"vm_ip": "$DR_VM_IP",
|
||
"vm_status": "$vm_status_label",
|
||
"nfs_status": "${nfs_status:-Unknown}",
|
||
"log_file": "$LOG_FILE",
|
||
"restore_log": $restore_log_json,
|
||
"bash_log": $bash_log_json
|
||
}
|
||
JSON
|
||
)
|
||
|
||
# Send notification
|
||
log "Sending notification..."
|
||
send_pve_notification "$severity" "$json_data"
|
||
|
||
# Final summary
|
||
log "=========================================="
|
||
log "Oracle DR Test Complete: $test_result"
|
||
log "Duration: $total_duration minutes"
|
||
log "Log: $LOG_FILE"
|
||
log "=========================================="
|
||
}
|
||
|
||
# Main execution
|
||
main() {
|
||
case "${1:-}" in
|
||
--install)
|
||
create_templates
|
||
echo ""
|
||
echo -e "${GREEN}Installation complete!${NC}"
|
||
echo "Next steps:"
|
||
echo "1. Test the script: /opt/scripts/weekly-dr-test-proxmox.sh"
|
||
echo "2. Add to cron: crontab -e"
|
||
echo " Add line: 0 6 * * 6 /opt/scripts/weekly-dr-test-proxmox.sh"
|
||
echo "3. Configure notifications in Proxmox GUI if needed:"
|
||
echo " Datacenter > Notifications > Add matching rules for 'oracle-dr-test'"
|
||
;;
|
||
--help)
|
||
echo "Oracle DR Weekly Test for Proxmox"
|
||
echo "Usage:"
|
||
echo " $0 - Run DR test"
|
||
echo " $0 --install - Create notification templates"
|
||
echo " $0 --help - Show this help"
|
||
;;
|
||
*)
|
||
# Check if templates exist, create if missing
|
||
if [ ! -f "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" ]; then
|
||
echo -e "${YELLOW}Templates not found, creating...${NC}"
|
||
create_templates
|
||
echo ""
|
||
fi
|
||
|
||
# Run DR test
|
||
run_dr_test
|
||
;;
|
||
esac
|
||
}
|
||
|
||
# Check dependencies
|
||
if ! command -v jq &> /dev/null; then
|
||
echo -e "${RED}Error: jq is not installed${NC}"
|
||
echo "Install with: apt-get install jq"
|
||
exit 1
|
||
fi
|
||
|
||
main "$@" |