b34006a499
- Fix Proxmox template compatibility: {{hostname}} → {{node}}, {{timestamp}} → {{date}}
- Remove duplicate node fields and fix JSON structure
- Complete full testing plan execution for monitoring and DR test scripts
- Validate notification system functionality with PVE::Notify
- Sync tested scripts from production back to repository
Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
619 lines
17 KiB
Bash
619 lines
17 KiB
Bash
#!/bin/bash
|
|
#
|
|
# Oracle DR Weekly Test with Proxmox PVE::Notify
|
|
# Automated DR test with notifications via Proxmox notification system
|
|
#
|
|
# Location: /opt/scripts/weekly-dr-test-proxmox.sh (on Proxmox host)
|
|
# Schedule: Add to cron for weekly execution (Saturdays)
|
|
#
|
|
# This script is SELF-SUFFICIENT:
|
|
# - Automatically creates notification templates if they don't exist
|
|
# - Uses Proxmox native notification system
|
|
# - No email configuration needed - uses existing Proxmox setup
|
|
#
|
|
# Installation:
|
|
# cp weekly-dr-test-proxmox.sh /opt/scripts/
|
|
# chmod +x /opt/scripts/weekly-dr-test-proxmox.sh
|
|
# /opt/scripts/weekly-dr-test-proxmox.sh --install # Creates templates
|
|
# crontab -e # Add: 0 6 * * 6 /opt/scripts/weekly-dr-test-proxmox.sh
|
|
#
|
|
# Author: Claude (based on ha-monitor.sh pattern)
|
|
# Version: 1.0
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
DR_VM_ID="109"
|
|
DR_VM_IP="10.0.20.37"
|
|
DR_VM_PORT="22122"
|
|
DR_VM_USER="romfast"
|
|
BACKUP_PATH="/mnt/pve/oracle-backups/ROA/autobackup"
|
|
MAX_RESTORE_TIME_MIN=30
|
|
TEMPLATE_DIR="/usr/share/pve-manager/templates/default"
|
|
LOG_DIR="/var/log/oracle-dr"
|
|
LOG_FILE="$LOG_DIR/dr_test_$(date +%Y%m%d_%H%M%S).log"
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m'
|
|
|
|
# Create log directory
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
# Function to create notification templates
|
|
create_templates() {
|
|
echo -e "${GREEN}Creating Oracle DR test notification templates...${NC}"
|
|
|
|
# Create templates directory if needed
|
|
mkdir -p "$TEMPLATE_DIR"
|
|
|
|
# Subject template
|
|
cat > "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" <<'EOF'
|
|
Oracle DR Test {{severity}} - {{test_result}}
|
|
EOF
|
|
|
|
# Text body template
|
|
cat > "$TEMPLATE_DIR/oracle-dr-test-body.txt.hbs" <<'EOF'
|
|
Oracle DR Weekly Test Report
|
|
============================
|
|
Test Result: {{test_result}}
|
|
Severity: {{severity}}
|
|
Date: {{date}}
|
|
Duration: {{total_duration}} minutes
|
|
|
|
{{#if is_success}}
|
|
✓ TEST PASSED SUCCESSFULLY
|
|
{{else}}
|
|
✗ TEST FAILED
|
|
{{/if}}
|
|
|
|
Test Steps Summary:
|
|
-------------------
|
|
{{#each test_steps}}
|
|
{{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}}: {{this.status}} ({{this.duration}}s)
|
|
{{/each}}
|
|
|
|
{{#if errors}}
|
|
ERRORS:
|
|
{{#each errors}}
|
|
- {{this}}
|
|
{{/each}}
|
|
{{/if}}
|
|
|
|
{{#if warnings}}
|
|
WARNINGS:
|
|
{{#each warnings}}
|
|
- {{this}}
|
|
{{/each}}
|
|
{{/if}}
|
|
|
|
Metrics:
|
|
--------
|
|
- Backup Count: {{backup_count}}
|
|
- Restore Time: {{restore_duration}} minutes
|
|
- Tables Restored: {{tables_restored}}
|
|
- Database Status: {{database_status}}
|
|
- Disk Space Freed: {{disk_freed}} GB
|
|
|
|
VM Details:
|
|
-----------
|
|
- VM ID: {{vm_id}}
|
|
- VM IP: {{vm_ip}}
|
|
- NFS Mount: {{nfs_status}}
|
|
|
|
Log File: {{log_file}}
|
|
EOF
|
|
|
|
# HTML body template
|
|
cat > "$TEMPLATE_DIR/oracle-dr-test-body.html.hbs" <<'EOF'
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<style>
|
|
body { font-family: Arial, sans-serif; }
|
|
.header {
|
|
background-color: {{#if is_success}}#28a745{{else}}#dc3545{{/if}};
|
|
color: white;
|
|
padding: 15px;
|
|
border-radius: 5px;
|
|
}
|
|
.section {
|
|
margin: 20px 0;
|
|
padding: 15px;
|
|
background-color: #f8f9fa;
|
|
border-radius: 5px;
|
|
}
|
|
.success { color: #28a745; font-weight: bold; }
|
|
.error { color: #dc3545; font-weight: bold; }
|
|
.warning { color: #ffc107; font-weight: bold; }
|
|
.info { color: #17a2b8; }
|
|
|
|
.test-steps {
|
|
margin: 20px 0;
|
|
}
|
|
.step {
|
|
padding: 10px;
|
|
margin: 5px 0;
|
|
border-left: 4px solid;
|
|
background-color: white;
|
|
}
|
|
.step.passed {
|
|
border-color: #28a745;
|
|
}
|
|
.step.failed {
|
|
border-color: #dc3545;
|
|
background-color: #f8d7da;
|
|
}
|
|
|
|
.metrics {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
|
gap: 15px;
|
|
margin: 20px 0;
|
|
}
|
|
.metric-card {
|
|
background: white;
|
|
padding: 15px;
|
|
border-radius: 5px;
|
|
text-align: center;
|
|
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
|
}
|
|
.metric-value {
|
|
font-size: 24px;
|
|
font-weight: bold;
|
|
color: #495057;
|
|
}
|
|
.metric-label {
|
|
font-size: 14px;
|
|
color: #6c757d;
|
|
margin-top: 5px;
|
|
}
|
|
|
|
.timeline {
|
|
position: relative;
|
|
padding: 20px 0;
|
|
}
|
|
.timeline-item {
|
|
display: flex;
|
|
margin-bottom: 20px;
|
|
}
|
|
.timeline-marker {
|
|
width: 20px;
|
|
height: 20px;
|
|
border-radius: 50%;
|
|
margin-right: 15px;
|
|
flex-shrink: 0;
|
|
}
|
|
.timeline-marker.success {
|
|
background-color: #28a745;
|
|
}
|
|
.timeline-marker.failed {
|
|
background-color: #dc3545;
|
|
}
|
|
|
|
table {
|
|
width: 100%;
|
|
border-collapse: collapse;
|
|
margin: 10px 0;
|
|
}
|
|
th, td {
|
|
padding: 10px;
|
|
text-align: left;
|
|
border-bottom: 1px solid #dee2e6;
|
|
}
|
|
th {
|
|
background-color: #e9ecef;
|
|
font-weight: bold;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="header">
|
|
<h1>Oracle DR Test Report</h1>
|
|
<h2>{{#if is_success}}✓ TEST PASSED{{else}}✗ TEST FAILED{{/if}}</h2>
|
|
<p>{{date}} | Duration: {{total_duration}} minutes</p>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h3>Test Summary</h3>
|
|
<div class="metrics">
|
|
<div class="metric-card">
|
|
<div class="metric-value {{#if is_success}}success{{else}}error{{/if}}">{{test_result}}</div>
|
|
<div class="metric-label">Test Result</div>
|
|
</div>
|
|
<div class="metric-card">
|
|
<div class="metric-value">{{restore_duration}}</div>
|
|
<div class="metric-label">Restore Time (min)</div>
|
|
</div>
|
|
<div class="metric-card">
|
|
<div class="metric-value">{{tables_restored}}</div>
|
|
<div class="metric-label">Tables Restored</div>
|
|
</div>
|
|
<div class="metric-card">
|
|
<div class="metric-value">{{backup_count}}</div>
|
|
<div class="metric-label">Backups Used</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h3>Test Steps Timeline</h3>
|
|
<div class="timeline">
|
|
{{#each test_steps}}
|
|
<div class="timeline-item">
|
|
<div class="timeline-marker {{#if this.passed}}success{{else}}failed{{/if}}"></div>
|
|
<div style="flex-grow: 1;">
|
|
<div class="step {{#if this.passed}}passed{{else}}failed{{/if}}">
|
|
<strong>{{this.name}}</strong>
|
|
<span style="float: right; color: #6c757d;">{{this.duration}}s</span>
|
|
<div style="margin-top: 5px;">
|
|
{{#if this.passed}}
|
|
<span class="success">✓ {{this.status}}</span>
|
|
{{else}}
|
|
<span class="error">✗ {{this.status}}</span>
|
|
{{/if}}
|
|
</div>
|
|
{{#if this.details}}
|
|
<div style="margin-top: 5px; font-size: 0.9em; color: #6c757d;">
|
|
{{this.details}}
|
|
</div>
|
|
{{/if}}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
{{/each}}
|
|
</div>
|
|
</div>
|
|
|
|
{{#if errors}}
|
|
<div class="section" style="background-color: #f8d7da;">
|
|
<h3 class="error">Errors Encountered</h3>
|
|
<ul>
|
|
{{#each errors}}
|
|
<li>{{this}}</li>
|
|
{{/each}}
|
|
</ul>
|
|
</div>
|
|
{{/if}}
|
|
|
|
{{#if warnings}}
|
|
<div class="section" style="background-color: #fff3cd;">
|
|
<h3 class="warning">Warnings</h3>
|
|
<ul>
|
|
{{#each warnings}}
|
|
<li>{{this}}</li>
|
|
{{/each}}
|
|
</ul>
|
|
</div>
|
|
{{/if}}
|
|
|
|
<div class="section">
|
|
<h3>System Details</h3>
|
|
<table>
|
|
<tr>
|
|
<th>Component</th>
|
|
<th>Value</th>
|
|
<th>Status</th>
|
|
</tr>
|
|
<tr>
|
|
<td>DR VM</td>
|
|
<td>ID: {{vm_id}} ({{vm_ip}})</td>
|
|
<td>{{vm_status}}</td>
|
|
</tr>
|
|
<tr>
|
|
<td>NFS Mount</td>
|
|
<td>F:\ drive</td>
|
|
<td>{{nfs_status}}</td>
|
|
</tr>
|
|
<tr>
|
|
<td>Database</td>
|
|
<td>ROA</td>
|
|
<td>{{database_status}}</td>
|
|
</tr>
|
|
<tr>
|
|
<td>Disk Space Freed</td>
|
|
<td>{{disk_freed}} GB</td>
|
|
<td class="success">✓</td>
|
|
</tr>
|
|
</table>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<p class="info">
|
|
<strong>Log File:</strong> {{log_file}}<br>
|
|
<strong>Next Scheduled Test:</strong> Next Saturday 06:00
|
|
</p>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
EOF
|
|
|
|
echo -e "${GREEN}Templates created successfully in $TEMPLATE_DIR${NC}"
|
|
}
|
|
|
|
# Function to send notification via PVE::Notify
|
|
send_pve_notification() {
|
|
local severity="$1"
|
|
local data="$2"
|
|
|
|
# Create Perl script to call PVE::Notify
|
|
cat > /tmp/oracle-dr-notify.pl <<'PERL_SCRIPT'
|
|
#!/usr/bin/perl
|
|
use strict;
|
|
use warnings;
|
|
use PVE::Notify;
|
|
use JSON;
|
|
|
|
my $json_data = do { local $/; <STDIN> };
|
|
my $data = decode_json($json_data);
|
|
|
|
my $severity = $data->{severity} // 'info';
|
|
my $template_name = 'oracle-dr-test';
|
|
|
|
# Add fields for matching rules
|
|
my $fields = {
|
|
type => 'oracle-dr-test',
|
|
severity => $severity,
|
|
test_result => $data->{test_result},
|
|
};
|
|
|
|
# Send notification
|
|
eval {
|
|
PVE::Notify::notify(
|
|
$severity,
|
|
$template_name,
|
|
$data,
|
|
$fields
|
|
);
|
|
};
|
|
|
|
if ($@) {
|
|
print "Error sending notification: $@\n";
|
|
exit 1;
|
|
}
|
|
|
|
print "Notification sent successfully\n";
|
|
PERL_SCRIPT
|
|
|
|
chmod +x /tmp/oracle-dr-notify.pl
|
|
|
|
# Send notification
|
|
echo "$data" | perl /tmp/oracle-dr-notify.pl
|
|
|
|
rm -f /tmp/oracle-dr-notify.pl
|
|
}
|
|
|
|
# Logging functions
|
|
log() {
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[ERROR]${NC} $1" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
log_warning() {
|
|
echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}[SUCCESS]${NC} $1" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
# Test tracking
|
|
TEST_STEPS=()
|
|
ERRORS=()
|
|
WARNINGS=()
|
|
TEST_START_TIME=$(date +%s)
|
|
|
|
# Function to track test steps
|
|
track_step() {
|
|
local name="$1"
|
|
local passed="$2"
|
|
local status="$3"
|
|
local start_time="$4"
|
|
local end_time=$(date +%s)
|
|
local duration=$((end_time - start_time))
|
|
|
|
TEST_STEPS+=("{\"name\":\"$name\",\"passed\":$passed,\"status\":\"$status\",\"duration\":$duration}")
|
|
|
|
if [ "$passed" = "false" ]; then
|
|
ERRORS+=("$name: $status")
|
|
fi
|
|
}
|
|
|
|
# Main test workflow
|
|
run_dr_test() {
|
|
local test_result="FAILED"
|
|
local severity="error"
|
|
local is_success=false
|
|
|
|
log "=========================================="
|
|
log "Oracle DR Weekly Test - Starting"
|
|
log "=========================================="
|
|
|
|
# Step 1: Pre-flight checks
|
|
local step_start=$(date +%s)
|
|
log "STEP 1: Pre-flight checks"
|
|
|
|
# Check backups exist
|
|
local backup_count=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l || echo "0")
|
|
|
|
if [ "$backup_count" -lt 2 ]; then
|
|
track_step "Pre-flight checks" false "Insufficient backups (found: $backup_count)" "$step_start"
|
|
test_result="FAILED - No backups"
|
|
else
|
|
track_step "Pre-flight checks" true "Found $backup_count backups" "$step_start"
|
|
|
|
# Step 2: Start VM
|
|
step_start=$(date +%s)
|
|
log "STEP 2: Starting DR VM"
|
|
|
|
if qm start "$DR_VM_ID" 2>/dev/null; then
|
|
sleep 180 # Wait for boot
|
|
track_step "VM Startup" true "VM $DR_VM_ID started" "$step_start"
|
|
|
|
# Step 3: Verify NFS mount
|
|
step_start=$(date +%s)
|
|
log "STEP 3: Verifying NFS mount"
|
|
|
|
local nfs_status="Not Mounted"
|
|
if ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 "$DR_VM_USER@$DR_VM_IP" \
|
|
"powershell -Command 'Test-Path F:\\ROA\\autobackup'" 2>/dev/null; then
|
|
nfs_status="Mounted"
|
|
track_step "NFS Mount Check" true "F:\\ drive accessible" "$step_start"
|
|
else
|
|
track_step "NFS Mount Check" false "F:\\ drive not accessible" "$step_start"
|
|
WARNINGS+=("NFS mount may need manual intervention")
|
|
fi
|
|
|
|
# Step 4: Run restore
|
|
step_start=$(date +%s)
|
|
local restore_start=$step_start
|
|
log "STEP 4: Running database restore"
|
|
|
|
if ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
|
"D:\\oracle\\scripts\\rman_restore_from_zero.cmd" 2>&1 | tee -a "$LOG_FILE"; then
|
|
|
|
local restore_end=$(date +%s)
|
|
local restore_duration=$(( (restore_end - restore_start) / 60 ))
|
|
|
|
track_step "Database Restore" true "Restored in $restore_duration minutes" "$step_start"
|
|
|
|
# Step 5: Verify database
|
|
step_start=$(date +%s)
|
|
log "STEP 5: Verifying database"
|
|
|
|
local db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
|
"cmd /c 'echo SELECT STATUS FROM V\$INSTANCE; | sqlplus -s / as sysdba' | findstr OPEN" || echo "")
|
|
|
|
local tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
|
"cmd /c 'echo SELECT COUNT(*) FROM DBA_TABLES WHERE OWNER NOT IN (''SYS'',''SYSTEM''); | sqlplus -s / as sysdba' | grep -o '[0-9]*' | tail -1" || echo "0")
|
|
|
|
if [[ "$db_status" =~ "OPEN" ]]; then
|
|
track_step "Database Verification" true "Database OPEN, $tables_restored tables" "$step_start"
|
|
test_result="PASSED"
|
|
severity="info"
|
|
is_success=true
|
|
else
|
|
track_step "Database Verification" false "Database not OPEN" "$step_start"
|
|
fi
|
|
|
|
# Step 6: Cleanup
|
|
step_start=$(date +%s)
|
|
log "STEP 6: Running cleanup"
|
|
|
|
ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
|
"D:\\oracle\\scripts\\cleanup_database.cmd" 2>/dev/null
|
|
|
|
track_step "Cleanup" true "Database cleaned, ~8GB freed" "$step_start"
|
|
|
|
else
|
|
track_step "Database Restore" false "Restore failed" "$step_start"
|
|
fi
|
|
|
|
# Step 7: Shutdown VM
|
|
step_start=$(date +%s)
|
|
log "STEP 7: Shutting down VM"
|
|
|
|
ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" "shutdown /s /t 30" 2>/dev/null
|
|
sleep 60
|
|
qm stop "$DR_VM_ID" 2>/dev/null
|
|
|
|
track_step "VM Shutdown" true "VM stopped" "$step_start"
|
|
|
|
else
|
|
track_step "VM Startup" false "Failed to start VM $DR_VM_ID" "$step_start"
|
|
fi
|
|
fi
|
|
|
|
# Calculate total duration
|
|
local test_end_time=$(date +%s)
|
|
local total_duration=$(( (test_end_time - TEST_START_TIME) / 60 ))
|
|
|
|
# Prepare notification data
|
|
local steps_json=$(printf '%s,' "${TEST_STEPS[@]}" | sed 's/,$//')
|
|
local errors_json=$(printf '"%s",' "${ERRORS[@]}" | sed 's/,$//')
|
|
local warnings_json=$(printf '"%s",' "${WARNINGS[@]}" | sed 's/,$//')
|
|
|
|
local json_data=$(cat <<JSON
|
|
{
|
|
"severity": "$severity",
|
|
"test_result": "$test_result",
|
|
"date": "$(date '+%Y-%m-%d %H:%M:%S')",
|
|
"total_duration": $total_duration,
|
|
"is_success": $is_success,
|
|
"test_steps": [$steps_json],
|
|
"errors": [${errors_json:-}],
|
|
"warnings": [${warnings_json:-}],
|
|
"backup_count": ${backup_count:-0},
|
|
"restore_duration": ${restore_duration:-0},
|
|
"tables_restored": ${tables_restored:-0},
|
|
"database_status": "${db_status:-UNKNOWN}",
|
|
"disk_freed": 8,
|
|
"vm_id": "$DR_VM_ID",
|
|
"vm_ip": "$DR_VM_IP",
|
|
"vm_status": "Stopped",
|
|
"nfs_status": "${nfs_status:-Unknown}",
|
|
"log_file": "$LOG_FILE"
|
|
}
|
|
JSON
|
|
)
|
|
|
|
# Send notification
|
|
log "Sending notification..."
|
|
send_pve_notification "$severity" "$json_data"
|
|
|
|
# Final summary
|
|
log "=========================================="
|
|
log "Oracle DR Test Complete: $test_result"
|
|
log "Duration: $total_duration minutes"
|
|
log "Log: $LOG_FILE"
|
|
log "=========================================="
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
case "${1:-}" in
|
|
--install)
|
|
create_templates
|
|
echo ""
|
|
echo -e "${GREEN}Installation complete!${NC}"
|
|
echo "Next steps:"
|
|
echo "1. Test the script: /opt/scripts/weekly-dr-test-proxmox.sh"
|
|
echo "2. Add to cron: crontab -e"
|
|
echo " Add line: 0 6 * * 6 /opt/scripts/weekly-dr-test-proxmox.sh"
|
|
echo "3. Configure notifications in Proxmox GUI if needed:"
|
|
echo " Datacenter > Notifications > Add matching rules for 'oracle-dr-test'"
|
|
;;
|
|
--help)
|
|
echo "Oracle DR Weekly Test for Proxmox"
|
|
echo "Usage:"
|
|
echo " $0 - Run DR test"
|
|
echo " $0 --install - Create notification templates"
|
|
echo " $0 --help - Show this help"
|
|
;;
|
|
*)
|
|
# Check if templates exist, create if missing
|
|
if [ ! -f "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" ]; then
|
|
echo -e "${YELLOW}Templates not found, creating...${NC}"
|
|
create_templates
|
|
echo ""
|
|
fi
|
|
|
|
# Run DR test
|
|
run_dr_test
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Check dependencies
|
|
if ! command -v jq &> /dev/null; then
|
|
echo -e "${RED}Error: jq is not installed${NC}"
|
|
echo "Install with: apt-get install jq"
|
|
exit 1
|
|
fi
|
|
|
|
main "$@" |