romfast/ROMFASTSQL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Oracle DR: Enhance notification templates with compact HTML layouts and improved data collection

Browse Source 
Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
Marius
2025-10-10 22:41:32 +03:00
parent 6f56e61b04
commit 839f1b6b82
3 changed files with 499 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files

482
oracle/standby-server-scripts/weekly-dr-test-proxmox.sh
View File

@@ -51,285 +51,150 @@ create_templates() {
# Subject template
cat > "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" <<'EOF'
Oracle DR Test {{severity}} - {{test_result}}
Oracle DR Test {{test_result}} | {{date}}
EOF
# Text body template
cat > "$TEMPLATE_DIR/oracle-dr-test-body.txt.hbs" <<'EOF'
Oracle DR Test {{severity}} - {{test_result}}
{{#if is_success}}TEST PASSED✓{{else}}TEST FAILED✗{{/if}}
Oracle DR Test {{test_result}} | {{date}}
Severity: {{severity}}
========================================
SUMMARY: Duration {{total_duration}}min | Tables {{tables_restored}} | Backups {{backup_count}}
SUMMARY
- Outcome: {{test_result}}
- Duration: {{total_duration}} min (restore {{restore_duration}} min)
- Backups used: {{backup_count}}
- Tables restored: {{tables_restored}}
TEST STEPS:
COMPONENTS
- VM {{vm_id}} ({{vm_ip}}): {{vm_status}}
- NFS: {{nfs_status}}
- Database: {{database_status}}
- Cleanup: {{disk_freed}} GB freed
STEPS
{{#each test_steps}}
- {{#if this.passed}}PASS{{else}}FAIL{{/if}}: {{this.name}} ({{this.duration}}s)
{{#if this.details}}
Details: {{this.details}}
- {{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}} ({{this.duration}}s){{#if this.status}} - {{this.status}}{{/if}}
{{/each}}
========================================
COMPONENT STATUS:
DR VM: ID {{vm_id}} ({{vm_ip}}) - {{vm_status}}
NFS Mount: {{nfs_status}} - {{#if nfs_ok}}OK{{else}}FAILED{{/if}}
Database: {{database_status}} - {{#if database_ok}}OK{{else}}FAILED{{/if}}
Disk Space: {{disk_freed}}GB freed - OK
{{#if errors}}
ERRORS:
{{#if has_errors}}
ISSUES
{{#each errors}}
- {{this}}
{{/each}}
{{/if}}
========================================
{{#if has_warnings}}
WARNINGS
{{#each warnings}}
- {{this}}
{{/each}}
{{/if}}
RESTORE LOG (first 200 lines)
---
{{restore_log}}
---
Log: {{log_file}}
Next scheduled test: Next Saturday 06:00
Proxmox DR Monitoring System
Next test: Saturday 06:00
EOF
# HTML body template (identical to text for compatibility)
# HTML body template (compact Gmail-friendly layout)
cat > "$TEMPLATE_DIR/oracle-dr-test-body.html.hbs" <<'EOF'
Oracle DR Test {{severity}} - {{test_result}}
{{#if is_success}}TEST PASSED✓{{else}}TEST FAILED✗{{/if}}
========================================
SUMMARY: Duration {{total_duration}}min | Tables {{tables_restored}} | Backups {{backup_count}}
TEST STEPS:
{{#each test_steps}}
- {{#if this.passed}}PASS{{else}}FAIL{{/if}}: {{this.name}} ({{this.duration}}s)
{{#if this.details}}
Details: {{this.details}}
{{/each}}
========================================
COMPONENT STATUS:
DR VM: ID {{vm_id}} ({{vm_ip}}) - {{vm_status}}
NFS Mount: {{nfs_status}} - {{#if nfs_ok}}OK{{else}}FAILED{{/if}}
Database: {{database_status}} - {{#if database_ok}}OK{{else}}FAILED{{/if}}
Disk Space: {{disk_freed}}GB freed - OK
{{#if errors}}
ERRORS:
{{#each errors}}
- {{this}}
{{/each}}
{{/if}}
========================================
Log: {{log_file}}
Next scheduled test: Next Saturday 06:00
Proxmox DR Monitoring System
EOF
.section {
margin: 20px 0;
padding: 15px;
background-color: #f8f9fa;
border-radius: 5px;
}
.success { color: #28a745; font-weight: bold; }
.error { color: #dc3545; font-weight: bold; }
.warning { color: #ffc107; font-weight: bold; }
.info { color: #17a2b8; }
.test-steps {
margin: 20px 0;
}
.step {
padding: 10px;
margin: 5px 0;
border-left: 4px solid;
background-color: white;
}
.step.passed {
border-color: #28a745;
}
.step.failed {
border-color: #dc3545;
background-color: #f8d7da;
}
.metrics {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 15px;
margin: 20px 0;
}
.metric-card {
background: white;
padding: 15px;
border-radius: 5px;
text-align: center;
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
}
.metric-value {
font-size: 24px;
font-weight: bold;
color: #495057;
}
.metric-label {
font-size: 14px;
color: #6c757d;
margin-top: 5px;
}
.timeline {
position: relative;
padding: 20px 0;
}
.timeline-item {
display: flex;
margin-bottom: 20px;
}
.timeline-marker {
width: 20px;
height: 20px;
border-radius: 50%;
margin-right: 15px;
flex-shrink: 0;
}
.timeline-marker.success {
background-color: #28a745;
}
.timeline-marker.failed {
background-color: #dc3545;
}
table {
width: 100%;
border-collapse: collapse;
margin: 10px 0;
}
th, td {
padding: 10px;
text-align: left;
border-bottom: 1px solid #dee2e6;
}
th {
background-color: #e9ecef;
font-weight: bold;
}
</style>
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Oracle DR Test {{test_result}} | {{date}}</title>
</head>
<body>
<div class="header">
<h1>Oracle DR Test Report</h1>
<h2>{{#if is_success}}✓ TEST PASSED{{else}}✗ TEST FAILED{{/if}}</h2>
<p>{{date}} | Duration: {{total_duration}} minutes</p>
</div>
<div class="section">
<h3>Test Summary</h3>
<div class="metrics">
<div class="metric-card">
<div class="metric-value {{#if is_success}}success{{else}}error{{/if}}">{{test_result}}</div>
<div class="metric-label">Test Result</div>
</div>
<div class="metric-card">
<div class="metric-value">{{restore_duration}}</div>
<div class="metric-label">Restore Time (min)</div>
</div>
<div class="metric-card">
<div class="metric-value">{{tables_restored}}</div>
<div class="metric-label">Tables Restored</div>
</div>
<div class="metric-card">
<div class="metric-value">{{backup_count}}</div>
<div class="metric-label">Backups Used</div>
</div>
</div>
</div>
<div class="section">
<h3>Test Steps Timeline</h3>
<div class="timeline">
{{#each test_steps}}
<div class="timeline-item">
<div class="timeline-marker {{#if this.passed}}success{{else}}failed{{/if}}"></div>
<div style="flex-grow: 1;">
<div class="step {{#if this.passed}}passed{{else}}failed{{/if}}">
<strong>{{this.name}}</strong>
<span style="float: right; color: #6c757d;">{{this.duration}}s</span>
<div style="margin-top: 5px;">
{{#if this.passed}}
<span class="success">✓ {{this.status}}</span>
{{else}}
<span class="error">✗ {{this.status}}</span>
{{/if}}
</div>
{{#if this.details}}
<div style="margin-top: 5px; font-size: 0.9em; color: #6c757d;">
{{this.details}}
</div>
{{/if}}
</div>
</div>
</div>
{{/each}}
</div>
</div>
{{#if errors}}
<div class="section" style="background-color: #f8d7da;">
<h3 class="error">Errors Encountered</h3>
<ul>
{{#each errors}}
<li>{{this}}</li>
{{/each}}
</ul>
</div>
{{/if}}
{{#if warnings}}
<div class="section" style="background-color: #fff3cd;">
<h3 class="warning">Warnings</h3>
<ul>
{{#each warnings}}
<li>{{this}}</li>
{{/each}}
</ul>
</div>
{{/if}}
<div class="section">
<h3>System Details</h3>
<table>
<tr>
<th>Component</th>
<th>Value</th>
<th>Status</th>
</tr>
<tr>
<td>DR VM</td>
<td>ID: {{vm_id}} ({{vm_ip}})</td>
<td>{{vm_status}}</td>
</tr>
<tr>
<td>NFS Mount</td>
<td>F:\ drive</td>
<td>{{nfs_status}}</td>
</tr>
<tr>
<td>Database</td>
<td>ROA</td>
<td>{{database_status}}</td>
</tr>
<tr>
<td>Disk Space Freed</td>
<td>{{disk_freed}} GB</td>
<td class="success">✓</td>
</tr>
<body style="margin:0;padding:16px;font-family:Arial,Helvetica,sans-serif;background:#ffffff;color:#2c3e50;">
<table style="width:100%;max-width:640px;margin:0 auto;border-collapse:collapse;">
<tr>
<td style="padding:0 0 12px 0;font-size:18px;font-weight:600;">
Oracle DR Test {{test_result}}
</td>
</tr>
<tr>
<td style="padding:0 0 8px 0;font-size:13px;color:#6c757d;">{{date}} · Severity: {{severity}}</td>
</tr>
<tr>
<td style="padding:12px;border:1px solid #e1e4e8;border-radius:4px;">
<table style="width:100%;border-collapse:collapse;font-size:14px;">
<tr><td style="padding:4px 0;">Outcome</td><td style="padding:4px 0;text-align:right;">{{test_result}}</td></tr>
<tr><td style="padding:4px 0;">Duration</td><td style="padding:4px 0;text-align:right;">{{total_duration}} min (restore {{restore_duration}} min)</td></tr>
<tr><td style="padding:4px 0;">Backups used</td><td style="padding:4px 0;text-align:right;">{{backup_count}}</td></tr>
<tr><td style="padding:4px 0;">Tables restored</td><td style="padding:4px 0;text-align:right;">{{tables_restored}}</td></tr>
</table>
</div>
</td>
</tr>
<div class="section">
<p class="info">
<strong>Log File:</strong> {{log_file}}<br>
<strong>Next Scheduled Test:</strong> Next Saturday 06:00
</p>
</div>
<tr>
<td style="padding:16px 0 0 0;">
<table style="width:100%;border-collapse:collapse;font-size:14px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
<tr><td style="padding:8px 12px;font-weight:600;">Components</td></tr>
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">VM {{vm_id}} ({{vm_ip}}): {{vm_status}}</td></tr>
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">NFS: {{nfs_status}}</td></tr>
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">Database: {{database_status}}</td></tr>
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">Cleanup: {{disk_freed}} GB freed</td></tr>
</table>
</td>
</tr>
<tr>
<td style="padding:16px 0 0 0;">
<table style="width:100%;border-collapse:collapse;font-size:14px;">
<tr><td style="padding:0 0 6px 0;font-weight:600;">Steps</td></tr>
{{#each test_steps}}
<tr>
<td style="padding:4px 0;border-bottom:1px solid #f1f1f1;">{{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}} ({{this.duration}}s){{#if this.status}} – {{this.status}}{{/if}}</td>
</tr>
{{/each}}
</table>
</td>
</tr>
{{#if has_errors}}
<tr>
<td style="padding:16px 0 0 0;">
<table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff5f5;border:1px solid #f1b0b7;border-radius:4px;">
<tr><td style="padding:8px 12px;font-weight:600;color:#c82333;">Issues</td></tr>
{{#each errors}}
<tr><td style="padding:6px 12px;border-top:1px solid #f8d7da;">• {{this}}</td></tr>
{{/each}}
</table>
</td>
</tr>
{{/if}}
{{#if has_warnings}}
<tr>
<td style="padding:16px 0 0 0;">
<table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff8e5;border:1px solid #ffe8a1;border-radius:4px;">
<tr><td style="padding:8px 12px;font-weight:600;color:#856404;">Warnings</td></tr>
{{#each warnings}}
<tr><td style="padding:6px 12px;border-top:1px solid #ffe8a1;">• {{this}}</td></tr>
{{/each}}
</table>
</td>
</tr>
{{/if}}
<tr>
<td style="padding:16px 0 0 0;">
<table style="width:100%;border-collapse:collapse;font-size:12px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
<tr><td style="padding:8px 12px;font-weight:600;font-size:13px;">Restore Log (first 200 lines)</td></tr>
<tr><td style="padding:8px 12px;font-family:monospace;white-space:pre-wrap;word-wrap:break-word;border-top:1px solid #e1e4e8;">{{restore_log}}</td></tr>
</table>
</td>
</tr>
<tr>
<td style="padding:16px 0 0 0;font-size:12px;color:#6c757d;">
Log: {{log_file}} · Next test: Saturday 06:00
</td>
</tr>
</table>
</body>
</html>
EOF
@@ -421,7 +286,16 @@ track_step() {
local end_time=$(date +%s)
local duration=$((end_time - start_time))
TEST_STEPS+=("{\"name\":\"$name\",\"passed\":$passed,\"status\":\"$status\",\"duration\":$duration}")
local step_json
step_json=$(jq -n \
--arg name "$name" \
--arg status "$status" \
--arg duration "$duration" \
--arg passed "$passed" \
'{name:$name, status:$status, duration:($duration|tonumber), passed:($passed == "true")}'
)
TEST_STEPS+=("$step_json")
if [ "$passed" = "false" ]; then
ERRORS+=("$name: $status")
@@ -433,6 +307,14 @@ run_dr_test() {
local test_result="FAILED"
local severity="error"
local is_success=false
local restore_duration=0
local tables_restored=0
local db_status="UNKNOWN"
local nfs_status="Not checked"
local vm_status_label="Not started"
local cleanup_freed=0
local backup_count=0
local restore_log="Not collected"
log "=========================================="
log "Oracle DR Weekly Test - Starting"
@@ -443,7 +325,7 @@ run_dr_test() {
log "STEP 1: Pre-flight checks"
# Check backups exist
local backup_count=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l || echo "0")
backup_count=$(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*.BKP' 2>/dev/null | wc -l)
if [ "$backup_count" -lt 2 ]; then
track_step "Pre-flight checks" false "Insufficient backups (found: $backup_count)" "$step_start"
@@ -456,6 +338,7 @@ run_dr_test() {
log "STEP 2: Starting DR VM"
if qm start "$DR_VM_ID" 2>/dev/null; then
vm_status_label="Running"
sleep 180 # Wait for boot
track_step "VM Startup" true "VM $DR_VM_ID started" "$step_start"
@@ -463,7 +346,7 @@ run_dr_test() {
step_start=$(date +%s)
log "STEP 3: Verifying NFS mount"
local nfs_status="Not Mounted"
nfs_status="Not Mounted"
if ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 "$DR_VM_USER@$DR_VM_IP" \
"powershell -Command 'Test-Path F:\\ROA\\autobackup'" 2>/dev/null; then
nfs_status="Mounted"
@@ -482,7 +365,7 @@ run_dr_test() {
"D:\\oracle\\scripts\\rman_restore_from_zero.cmd" 2>&1 | tee -a "$LOG_FILE"; then
local restore_end=$(date +%s)
local restore_duration=$(( (restore_end - restore_start) / 60 ))
restore_duration=$(( (restore_end - restore_start) / 60 ))
track_step "Database Restore" true "Restored in $restore_duration minutes" "$step_start"
@@ -490,11 +373,13 @@ run_dr_test() {
step_start=$(date +%s)
log "STEP 5: Verifying database"
local db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
"cmd /c 'echo SELECT STATUS FROM V\$INSTANCE; | sqlplus -s / as sysdba' | findstr OPEN" || echo "")
local tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
"cmd /c 'echo SELECT COUNT(*) FROM DBA_TABLES WHERE OWNER NOT IN (''SYS'',''SYSTEM''); | sqlplus -s / as sysdba' | grep -o '[0-9]*' | tail -1" || echo "0")
tables_restored=$(echo "$tables_restored" | tr -cd '0-9')
[ -z "$tables_restored" ] && tables_restored=0
if [[ "$db_status" =~ "OPEN" ]]; then
track_step "Database Verification" true "Database OPEN, $tables_restored tables" "$step_start"
@@ -505,6 +390,11 @@ run_dr_test() {
track_step "Database Verification" false "Database not OPEN" "$step_start"
fi
# Collect restore log from VM
log "Collecting restore log from DR VM..."
restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
"type D:\\oracle\\logs\\restore_from_zero.log 2>nul" | head -200 || echo "Log not available")
# Step 6: Cleanup
step_start=$(date +%s)
log "STEP 6: Running cleanup"
@@ -512,7 +402,8 @@ run_dr_test() {
ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
"D:\\oracle\\scripts\\cleanup_database.cmd" 2>/dev/null
track_step "Cleanup" true "Database cleaned, ~8GB freed" "$step_start"
cleanup_freed=8
track_step "Cleanup" true "Database cleaned, ~${cleanup_freed}GB freed" "$step_start"
else
track_step "Database Restore" false "Restore failed" "$step_start"
@@ -527,9 +418,11 @@ run_dr_test() {
qm stop "$DR_VM_ID" 2>/dev/null
track_step "VM Shutdown" true "VM stopped" "$step_start"
vm_status_label="Stopped"
else
track_step "VM Startup" false "Failed to start VM $DR_VM_ID" "$step_start"
vm_status_label="Failed to start"
fi
fi
@@ -538,9 +431,41 @@ run_dr_test() {
local total_duration=$(( (test_end_time - TEST_START_TIME) / 60 ))
# Prepare notification data
local steps_json=$(printf '%s,' "${TEST_STEPS[@]}" | sed 's/,$//')
local errors_json=$(printf '"%s",' "${ERRORS[@]}" | sed 's/,$//')
local warnings_json=$(printf '"%s",' "${WARNINGS[@]}" | sed 's/,$//')
local steps_json
if [ ${#TEST_STEPS[@]} -eq 0 ]; then
steps_json='[]'
else
steps_json=$(printf '%s\n' "${TEST_STEPS[@]}" | jq -s '.')
fi
local errors_json
if [ ${#ERRORS[@]} -eq 0 ]; then
errors_json='[]'
else
errors_json=$(printf '%s\n' "${ERRORS[@]}" | jq -R . | jq -s .)
fi
local warnings_json
if [ ${#WARNINGS[@]} -eq 0 ]; then
warnings_json='[]'
else
warnings_json=$(printf '%s\n' "${WARNINGS[@]}" | jq -R . | jq -s .)
fi
local has_errors=false
local has_warnings=false
[ ${#ERRORS[@]} -gt 0 ] && has_errors=true
[ ${#WARNINGS[@]} -gt 0 ] && has_warnings=true
if [ "$is_success" = true ] && [ "$has_warnings" = true ]; then
severity="warning"
fi
local db_status_clean=$(echo "$db_status" | tr -d '\r' | sed 's/^ *//;s/ *$//')
# Escape restore log for JSON
local restore_log_json
restore_log_json=$(echo "$restore_log" | jq -Rs .)
local json_data=$(cat <<JSON
{
@@ -549,19 +474,22 @@ run_dr_test() {
"date": "$(date '+%Y-%m-%d %H:%M:%S')",
"total_duration": $total_duration,
"is_success": $is_success,
"test_steps": [$steps_json],
"errors": [${errors_json:-}],
"warnings": [${warnings_json:-}],
"backup_count": ${backup_count:-0},
"restore_duration": ${restore_duration:-0},
"has_errors": $has_errors,
"has_warnings": $has_warnings,
"test_steps": $steps_json,
"errors": $errors_json,
"warnings": $warnings_json,
"backup_count": $backup_count,
"restore_duration": $restore_duration,
"tables_restored": ${tables_restored:-0},
"database_status": "${db_status:-UNKNOWN}",
"disk_freed": 8,
"database_status": "${db_status_clean:-UNKNOWN}",
"disk_freed": $cleanup_freed,
"vm_id": "$DR_VM_ID",
"vm_ip": "$DR_VM_IP",
"vm_status": "Stopped",
"vm_status": "$vm_status_label",
"nfs_status": "${nfs_status:-Unknown}",
"log_file": "$LOG_FILE"
"log_file": "$LOG_FILE",
"restore_log": $restore_log_json
}
JSON
)