Oracle DR: Enhance notification templates with compact HTML layouts and improved data collection
Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
Marius
@@ -53,7 +53,14 @@
|
||||
3. Validare log-uri și rapoarte generate
|
||||
4. Configurare cron pentru execuție automată
|
||||
|
||||
### Faza 5: Testare Erori și Edge Cases
|
||||
### Faza 5: Validare Format Notificări
|
||||
1. Reinstalare template-uri compacte: `/opt/scripts/oracle-backup-monitor-proxmox.sh --install`
|
||||
2. Generare notificări reale din scripturi (backup monitor + DR test) și analiză în clienți email
|
||||
3. Verificare afișare în client email (text + HTML) și în GUI Proxmox
|
||||
4. Rulare `weekly-dr-test-proxmox.sh` în mediu controlat și validare sumar compact în email (inclusiv componente, pași, timeline)
|
||||
5. Capturare feedback utilizatori finali (Gmail + Outlook) pentru lizibilitate
|
||||
|
||||
### Faza 6: Testare Erori și Edge Cases
|
||||
1. Testare fără conectivitate la VM DR
|
||||
2. Testare director backup-uri gol
|
||||
3. Testare eșec restaurare database
|
||||
|
||||
@@ -46,81 +46,167 @@ create_templates() {
|
||||
|
||||
# Subject template
|
||||
cat > "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" <<'EOF'
|
||||
Oracle Backup {{severity}} - {{node}}
|
||||
Oracle Backup {{status}} | {{node}}
|
||||
EOF
|
||||
|
||||
# Text body template
|
||||
cat > "$TEMPLATE_DIR/oracle-backup-body.txt.hbs" <<'EOF'
|
||||
Oracle Backup {{severity}} - {{node}}
|
||||
{{status}}
|
||||
Oracle Backup {{status}} | {{node}}
|
||||
Date: {{date}}
|
||||
|
||||
========================================
|
||||
{{#if errors}}
|
||||
CRITICAL ISSUES:
|
||||
SUMMARY
|
||||
- Full backup: {{full_backup_age}}h (limit {{full_backup_limit}}h) -> {{#if full_backup_ok}}OK{{else}}CHECK{{/if}}
|
||||
- Incremental: {{cumulative_backup_age}}h (limit {{cumulative_backup_limit}}h) -> {{#if cumulative_backup_ok}}OK{{else}}CHECK{{/if}}
|
||||
- Backups: {{total_backups}} files ({{total_size_label}})
|
||||
- Disk usage: {{disk_usage}}%
|
||||
|
||||
{{#if has_errors}}
|
||||
ISSUES
|
||||
{{#each errors}}
|
||||
- {{this}}
|
||||
{{/each}}
|
||||
{{/if}}
|
||||
|
||||
{{#if warnings}}
|
||||
WARNINGS:
|
||||
{{#if has_warnings}}
|
||||
WARNINGS
|
||||
{{#each warnings}}
|
||||
- {{this}}
|
||||
{{/each}}
|
||||
{{/if}}
|
||||
|
||||
========================================
|
||||
BACKUP STATUS:
|
||||
FULL: {{full_backup_age}}h old {{#if full_backup_ok}}OK{{else}}TOO OLD{{/if}} (limit: 25h)
|
||||
CUMULATIVE: {{cumulative_backup_age}}h old {{#if cumulative_backup_ok}}OK{{else}}TOO OLD{{/if}} (limit: 7h)
|
||||
Total: {{total_backups}} files | Size: {{total_size_gb}}GB | Disk: {{disk_usage}}%
|
||||
|
||||
{{#if backup_list}}
|
||||
LATEST BACKUPS (last 5):
|
||||
{{#each backup_list}}
|
||||
FULL BACKUPS ({{full_backup_count}} files)
|
||||
{{#if has_full_backups}}
|
||||
{{#each full_backup_list}}
|
||||
- {{this}}
|
||||
{{/each}}
|
||||
{{else}}
|
||||
- none detected
|
||||
{{/if}}
|
||||
|
||||
========================================
|
||||
Next check: {{date}} + 24h | Proxmox Monitoring
|
||||
INCREMENTAL BACKUPS ({{incr_backup_count}} files)
|
||||
{{#if has_incr_backups}}
|
||||
{{#each incr_backup_list}}
|
||||
- {{this}}
|
||||
{{/each}}
|
||||
{{else}}
|
||||
- none detected
|
||||
{{/if}}
|
||||
|
||||
Next check: +24h via Proxmox Monitor
|
||||
EOF
|
||||
|
||||
# HTML body template (identical to text for compatibility)
|
||||
# HTML body template (lightweight Gmail-friendly)
|
||||
cat > "$TEMPLATE_DIR/oracle-backup-body.html.hbs" <<'EOF'
|
||||
Oracle Backup {{severity}} - {{node}}
|
||||
{{status}}
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Oracle Backup {{status}} | {{node}}</title>
|
||||
</head>
|
||||
<body style="margin:0;padding:16px;font-family:Arial,Helvetica,sans-serif;background:#ffffff;color:#2c3e50;">
|
||||
<table style="width:100%;max-width:640px;margin:0 auto;border-collapse:collapse;">
|
||||
<tr>
|
||||
<td style="padding:0 0 12px 0;font-size:18px;font-weight:600;">
|
||||
Oracle Backup {{status}} | {{node}}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:0 0 16px 0;font-size:13px;color:#6c757d;">
|
||||
{{date}}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:12px;border:1px solid #e1e4e8;border-radius:4px;">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:14px;">
|
||||
<tr>
|
||||
<td style="padding:4px 0;">Full backup</td>
|
||||
<td style="padding:4px 0;text-align:right;">
|
||||
{{full_backup_age}}h / {{full_backup_limit}}h · {{#if full_backup_ok}}OK{{else}}CHECK{{/if}}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:4px 0;">Incremental</td>
|
||||
<td style="padding:4px 0;text-align:right;">
|
||||
{{cumulative_backup_age}}h / {{cumulative_backup_limit}}h · {{#if cumulative_backup_ok}}OK{{else}}CHECK{{/if}}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:4px 0;">Backups</td>
|
||||
<td style="padding:4px 0;text-align:right;">{{total_backups}} files ({{total_size_label}})</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:4px 0;">Disk usage</td>
|
||||
<td style="padding:4px 0;text-align:right;">{{disk_usage}}%</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
========================================
|
||||
{{#if errors}}
|
||||
CRITICAL ISSUES:
|
||||
{{#if has_errors}}
|
||||
<tr>
|
||||
<td style="padding:16px 0 0 0;">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff5f5;border:1px solid #f1b0b7;border-radius:4px;">
|
||||
<tr><td style="padding:8px 12px;font-weight:600;color:#c82333;">Issues</td></tr>
|
||||
{{#each errors}}
|
||||
- {{this}}
|
||||
<tr><td style="padding:6px 12px;border-top:1px solid #f8d7da;">• {{this}}</td></tr>
|
||||
{{/each}}
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
{{/if}}
|
||||
|
||||
{{#if warnings}}
|
||||
WARNINGS:
|
||||
{{#if has_warnings}}
|
||||
<tr>
|
||||
<td style="padding:16px 0 0 0;">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff8e5;border:1px solid #ffe8a1;border-radius:4px;">
|
||||
<tr><td style="padding:8px 12px;font-weight:600;color:#856404;">Warnings</td></tr>
|
||||
{{#each warnings}}
|
||||
- {{this}}
|
||||
<tr><td style="padding:6px 12px;border-top:1px solid #ffe8a1;">• {{this}}</td></tr>
|
||||
{{/each}}
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
{{/if}}
|
||||
|
||||
========================================
|
||||
BACKUP STATUS:
|
||||
FULL: {{full_backup_age}}h old {{#if full_backup_ok}}OK{{else}}TOO OLD{{/if}} (limit: 25h)
|
||||
CUMULATIVE: {{cumulative_backup_age}}h old {{#if cumulative_backup_ok}}OK{{else}}TOO OLD{{/if}} (limit: 7h)
|
||||
Total: {{total_backups}} files | Size: {{total_size_gb}}GB | Disk: {{disk_usage}}%
|
||||
|
||||
{{#if backup_list}}
|
||||
LATEST BACKUPS (last 5):
|
||||
{{#each backup_list}}
|
||||
- {{this}}
|
||||
<tr>
|
||||
<td style="padding:16px 0 0 0;">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:13px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
|
||||
<tr><td style="padding:8px 12px;font-weight:600;">FULL Backups ({{full_backup_count}} files)</td></tr>
|
||||
{{#if has_full_backups}}
|
||||
{{#each full_backup_list}}
|
||||
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">• {{this}}</td></tr>
|
||||
{{/each}}
|
||||
{{else}}
|
||||
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">• none detected</td></tr>
|
||||
{{/if}}
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
========================================
|
||||
Next check: {{date}} + 24h | Proxmox Monitoring
|
||||
<tr>
|
||||
<td style="padding:16px 0 0 0;">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:13px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
|
||||
<tr><td style="padding:8px 12px;font-weight:600;">INCREMENTAL Backups ({{incr_backup_count}} files)</td></tr>
|
||||
{{#if has_incr_backups}}
|
||||
{{#each incr_backup_list}}
|
||||
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">• {{this}}</td></tr>
|
||||
{{/each}}
|
||||
{{else}}
|
||||
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">• none detected</td></tr>
|
||||
{{/if}}
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td style="padding:16px 0 0 0;font-size:12px;color:#6c757d;">
|
||||
Next automated check: +24h via Proxmox Monitor
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
echo -e "${GREEN}Templates created successfully in $TEMPLATE_DIR${NC}"
|
||||
@@ -150,7 +236,7 @@ my $template_name = 'oracle-backup';
|
||||
my $fields = {
|
||||
type => 'oracle-backup',
|
||||
severity => $severity,
|
||||
hostname => $data->{hostname},
|
||||
hostname => $data->{node} // 'unknown',
|
||||
};
|
||||
|
||||
# Send notification
|
||||
@@ -187,27 +273,32 @@ check_backups() {
|
||||
|
||||
echo "Checking Oracle backups..."
|
||||
|
||||
# Get backup list
|
||||
local backup_files=$(ls -lth "$BACKUP_PATH"/*.BKP 2>/dev/null | head -10 || echo "")
|
||||
|
||||
if [ -z "$backup_files" ]; then
|
||||
status="ERROR"
|
||||
errors+=("No backup files found in $BACKUP_PATH")
|
||||
else
|
||||
# Count backups
|
||||
local total_backups=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l)
|
||||
local total_size=$(du -shc "$BACKUP_PATH"/*.BKP 2>/dev/null | tail -1 | awk '{print $1}')
|
||||
|
||||
# Check FULL backup age
|
||||
local latest_full=$(ls -t "$BACKUP_PATH"/*FULL*.BKP 2>/dev/null | head -1 || echo "")
|
||||
local total_backups=0
|
||||
local total_size_label="0G"
|
||||
local full_age_hours="N/A"
|
||||
local cumulative_age_hours="N/A"
|
||||
local full_backup_ok=false
|
||||
local cumulative_backup_ok=false
|
||||
local disk_usage=0
|
||||
local -a backup_entries=()
|
||||
|
||||
if [ ! -d "$BACKUP_PATH" ]; then
|
||||
status="ERROR"
|
||||
errors+=("Backup path $BACKUP_PATH not accessible")
|
||||
else
|
||||
if compgen -G "$BACKUP_PATH"/*.BKP > /dev/null; then
|
||||
total_backups=$(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*.BKP' | wc -l)
|
||||
total_backups=${total_backups//[[:space:]]/}
|
||||
[ -z "$total_backups" ] && total_backups=0
|
||||
local total_size=$(du -shc "$BACKUP_PATH"/*.BKP 2>/dev/null | tail -1 | awk '{print $1}')
|
||||
[ -z "$total_size" ] && total_size="0G"
|
||||
total_size_label="$total_size"
|
||||
|
||||
local latest_full=$(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*FULL*.BKP' -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)
|
||||
if [ -n "$latest_full" ]; then
|
||||
local full_timestamp=$(stat -c %Y "$latest_full")
|
||||
local current_timestamp=$(date +%s)
|
||||
full_age_hours=$(( (current_timestamp - full_timestamp) / 3600 ))
|
||||
|
||||
if [ "$full_age_hours" -gt "$MAX_FULL_AGE_HOURS" ]; then
|
||||
status="WARNING"
|
||||
warnings+=("FULL backup is $full_age_hours hours old (threshold: $MAX_FULL_AGE_HOURS)")
|
||||
@@ -219,16 +310,11 @@ check_backups() {
|
||||
errors+=("No FULL backup found")
|
||||
fi
|
||||
|
||||
# Check CUMULATIVE backup age
|
||||
local latest_cumulative=$(ls -t "$BACKUP_PATH"/*INCR*.BKP "$BACKUP_PATH"/*INCREMENTAL*.BKP "$BACKUP_PATH"/*CUMULATIVE*.BKP 2>/dev/null | head -1 || echo "")
|
||||
local cumulative_age_hours="N/A"
|
||||
local cumulative_backup_ok=false
|
||||
|
||||
local latest_cumulative=$(find "$BACKUP_PATH" -maxdepth 1 -type f \( -name '*INCR*.BKP' -o -name '*INCREMENTAL*.BKP' -o -name '*CUMULATIVE*.BKP' \) -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)
|
||||
if [ -n "$latest_cumulative" ]; then
|
||||
local cumulative_timestamp=$(stat -c %Y "$latest_cumulative")
|
||||
local current_timestamp=$(date +%s)
|
||||
cumulative_age_hours=$(( (current_timestamp - cumulative_timestamp) / 3600 ))
|
||||
|
||||
if [ "$cumulative_age_hours" -gt "$MAX_CUMULATIVE_AGE_HOURS" ]; then
|
||||
if [ "$status" != "ERROR" ]; then status="WARNING"; fi
|
||||
warnings+=("CUMULATIVE backup is $cumulative_age_hours hours old (threshold: $MAX_CUMULATIVE_AGE_HOURS)")
|
||||
@@ -237,8 +323,46 @@ check_backups() {
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check disk usage
|
||||
local disk_usage=$(df "$BACKUP_PATH" | tail -1 | awk '{print int($5)}')
|
||||
# Collect ALL FULL backups
|
||||
local -a full_backups=()
|
||||
local -a full_backup_entries=()
|
||||
if readarray -t full_backups < <(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*FULL*.BKP' -printf '%T@ %p\n' | sort -nr | cut -d' ' -f2-); then
|
||||
for backup_file in "${full_backups[@]}"; do
|
||||
[ -z "$backup_file" ] && continue
|
||||
local backup_name=$(basename "$backup_file")
|
||||
local backup_time=$(date -r "$backup_file" '+%Y-%m-%d %H:%M')
|
||||
local backup_size=$(du -sh "$backup_file" 2>/dev/null | cut -f1)
|
||||
[ -z "$backup_size" ] && backup_size="N/A"
|
||||
full_backup_entries+=("$backup_time | $backup_name | $backup_size")
|
||||
done
|
||||
fi
|
||||
|
||||
# Collect ALL INCREMENTAL backups
|
||||
local -a incr_backups=()
|
||||
local -a incr_backup_entries=()
|
||||
if readarray -t incr_backups < <(find "$BACKUP_PATH" -maxdepth 1 -type f \( -name '*INCR*.BKP' -o -name '*INCREMENTAL*.BKP' -o -name '*CUMULATIVE*.BKP' \) -printf '%T@ %p\n' | sort -nr | cut -d' ' -f2-); then
|
||||
for backup_file in "${incr_backups[@]}"; do
|
||||
[ -z "$backup_file" ] && continue
|
||||
local backup_name=$(basename "$backup_file")
|
||||
local backup_time=$(date -r "$backup_file" '+%Y-%m-%d %H:%M')
|
||||
local backup_size=$(du -sh "$backup_file" 2>/dev/null | cut -f1)
|
||||
[ -z "$backup_size" ] && backup_size="N/A"
|
||||
incr_backup_entries+=("$backup_time | $backup_name | $backup_size")
|
||||
done
|
||||
fi
|
||||
else
|
||||
status="ERROR"
|
||||
errors+=("No backup files found in $BACKUP_PATH")
|
||||
fi
|
||||
|
||||
local disk_usage_raw=$(df "$BACKUP_PATH" 2>/dev/null | tail -1 | awk '{print int($5)}')
|
||||
if [ -n "$disk_usage_raw" ]; then
|
||||
disk_usage="$disk_usage_raw"
|
||||
else
|
||||
if [ "$status" = "OK" ]; then status="WARNING"; fi
|
||||
warnings+=("Unable to determine disk usage for $BACKUP_PATH")
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$disk_usage" -gt 90 ]; then
|
||||
status="ERROR"
|
||||
@@ -248,17 +372,47 @@ check_backups() {
|
||||
warnings+=("Disk usage high: ${disk_usage}%")
|
||||
fi
|
||||
|
||||
# Prepare notification data
|
||||
local severity="info"
|
||||
[ "$status" = "WARNING" ] && severity="warning"
|
||||
[ "$status" = "ERROR" ] && severity="error"
|
||||
|
||||
# Convert arrays to JSON arrays
|
||||
local errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .)
|
||||
local warnings_json=$(printf '%s\n' "${warnings[@]}" | jq -R . | jq -s .)
|
||||
local backup_list_json=$(echo "$backup_files" | head -5 | jq -R . | jq -s .)
|
||||
local errors_json
|
||||
if [ ${#errors[@]} -eq 0 ]; then
|
||||
errors_json='[]'
|
||||
else
|
||||
errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .)
|
||||
fi
|
||||
|
||||
local warnings_json
|
||||
if [ ${#warnings[@]} -eq 0 ]; then
|
||||
warnings_json='[]'
|
||||
else
|
||||
warnings_json=$(printf '%s\n' "${warnings[@]}" | jq -R . | jq -s .)
|
||||
fi
|
||||
|
||||
local full_backup_list_json
|
||||
if [ ${#full_backup_entries[@]} -eq 0 ]; then
|
||||
full_backup_list_json='[]'
|
||||
else
|
||||
full_backup_list_json=$(printf '%s\n' "${full_backup_entries[@]}" | jq -R . | jq -s .)
|
||||
fi
|
||||
|
||||
local incr_backup_list_json
|
||||
if [ ${#incr_backup_entries[@]} -eq 0 ]; then
|
||||
incr_backup_list_json='[]'
|
||||
else
|
||||
incr_backup_list_json=$(printf '%s\n' "${incr_backup_entries[@]}" | jq -R . | jq -s .)
|
||||
fi
|
||||
|
||||
local has_errors=false
|
||||
local has_warnings=false
|
||||
local has_full_backups=false
|
||||
local has_incr_backups=false
|
||||
[ ${#errors[@]} -gt 0 ] && has_errors=true
|
||||
[ ${#warnings[@]} -gt 0 ] && has_warnings=true
|
||||
[ ${#full_backup_entries[@]} -gt 0 ] && has_full_backups=true
|
||||
[ ${#incr_backup_entries[@]} -gt 0 ] && has_incr_backups=true
|
||||
|
||||
# Create JSON data
|
||||
local json_data=$(cat <<JSON
|
||||
{
|
||||
"severity": "$severity",
|
||||
@@ -267,8 +421,11 @@ check_backups() {
|
||||
"status": "$status",
|
||||
"errors": $errors_json,
|
||||
"warnings": $warnings_json,
|
||||
"has_errors": $has_errors,
|
||||
"has_warnings": $has_warnings,
|
||||
"total_backups": $total_backups,
|
||||
"total_size_gb": "${total_size%G}",
|
||||
"total_size_gb": "${total_size_label%G}",
|
||||
"total_size_label": "$total_size_label",
|
||||
"full_backup_age": "${full_age_hours}",
|
||||
"cumulative_backup_age": "${cumulative_age_hours}",
|
||||
"disk_usage": "${disk_usage}",
|
||||
@@ -276,12 +433,18 @@ check_backups() {
|
||||
"cumulative_backup_ok": $([ "$cumulative_backup_ok" = "true" ] && echo "true" || echo "false"),
|
||||
"is_error": $([ "$status" = "ERROR" ] && echo "true" || echo "false"),
|
||||
"is_warning": $([ "$status" = "WARNING" ] && echo "true" || echo "false"),
|
||||
"backup_list": $backup_list_json
|
||||
"full_backup_list": $full_backup_list_json,
|
||||
"incr_backup_list": $incr_backup_list_json,
|
||||
"has_full_backups": $has_full_backups,
|
||||
"has_incr_backups": $has_incr_backups,
|
||||
"full_backup_count": ${#full_backup_entries[@]},
|
||||
"incr_backup_count": ${#incr_backup_entries[@]},
|
||||
"full_backup_limit": "$MAX_FULL_AGE_HOURS",
|
||||
"cumulative_backup_limit": "$MAX_CUMULATIVE_AGE_HOURS"
|
||||
}
|
||||
JSON
|
||||
)
|
||||
|
||||
# Send notification if there are issues
|
||||
if [ "$status" != "OK" ]; then
|
||||
echo -e "${YELLOW}Issues detected, sending notification...${NC}"
|
||||
send_pve_notification "$severity" "$status" "$json_data"
|
||||
@@ -291,14 +454,12 @@ JSON
|
||||
# send_pve_notification "info" "$status" "$json_data"
|
||||
fi
|
||||
|
||||
# Display summary
|
||||
echo "Status: $status"
|
||||
echo "Total backups: $total_backups"
|
||||
echo "Total size: $total_size"
|
||||
echo "Total size: $total_size_label"
|
||||
echo "FULL backup age: $full_age_hours hours"
|
||||
echo "CUMULATIVE backup age: $cumulative_age_hours hours"
|
||||
echo "Disk usage: ${disk_usage}%"
|
||||
fi
|
||||
}
|
||||
|
||||
# Main execution
|
||||
|
||||
@@ -51,285 +51,150 @@ create_templates() {
|
||||
|
||||
# Subject template
|
||||
cat > "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" <<'EOF'
|
||||
Oracle DR Test {{severity}} - {{test_result}}
|
||||
Oracle DR Test {{test_result}} | {{date}}
|
||||
EOF
|
||||
|
||||
# Text body template
|
||||
cat > "$TEMPLATE_DIR/oracle-dr-test-body.txt.hbs" <<'EOF'
|
||||
Oracle DR Test {{severity}} - {{test_result}}
|
||||
{{#if is_success}}TEST PASSED✓{{else}}TEST FAILED✗{{/if}}
|
||||
Oracle DR Test {{test_result}} | {{date}}
|
||||
Severity: {{severity}}
|
||||
|
||||
========================================
|
||||
SUMMARY: Duration {{total_duration}}min | Tables {{tables_restored}} | Backups {{backup_count}}
|
||||
SUMMARY
|
||||
- Outcome: {{test_result}}
|
||||
- Duration: {{total_duration}} min (restore {{restore_duration}} min)
|
||||
- Backups used: {{backup_count}}
|
||||
- Tables restored: {{tables_restored}}
|
||||
|
||||
TEST STEPS:
|
||||
COMPONENTS
|
||||
- VM {{vm_id}} ({{vm_ip}}): {{vm_status}}
|
||||
- NFS: {{nfs_status}}
|
||||
- Database: {{database_status}}
|
||||
- Cleanup: {{disk_freed}} GB freed
|
||||
|
||||
STEPS
|
||||
{{#each test_steps}}
|
||||
- {{#if this.passed}}PASS{{else}}FAIL{{/if}}: {{this.name}} ({{this.duration}}s)
|
||||
{{#if this.details}}
|
||||
Details: {{this.details}}
|
||||
- {{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}} ({{this.duration}}s){{#if this.status}} - {{this.status}}{{/if}}
|
||||
{{/each}}
|
||||
|
||||
========================================
|
||||
COMPONENT STATUS:
|
||||
DR VM: ID {{vm_id}} ({{vm_ip}}) - {{vm_status}}
|
||||
NFS Mount: {{nfs_status}} - {{#if nfs_ok}}OK{{else}}FAILED{{/if}}
|
||||
Database: {{database_status}} - {{#if database_ok}}OK{{else}}FAILED{{/if}}
|
||||
Disk Space: {{disk_freed}}GB freed - OK
|
||||
|
||||
{{#if errors}}
|
||||
ERRORS:
|
||||
{{#if has_errors}}
|
||||
ISSUES
|
||||
{{#each errors}}
|
||||
- {{this}}
|
||||
{{/each}}
|
||||
{{/if}}
|
||||
|
||||
========================================
|
||||
Log: {{log_file}}
|
||||
Next scheduled test: Next Saturday 06:00
|
||||
Proxmox DR Monitoring System
|
||||
EOF
|
||||
|
||||
# HTML body template (identical to text for compatibility)
|
||||
cat > "$TEMPLATE_DIR/oracle-dr-test-body.html.hbs" <<'EOF'
|
||||
Oracle DR Test {{severity}} - {{test_result}}
|
||||
{{#if is_success}}TEST PASSED✓{{else}}TEST FAILED✗{{/if}}
|
||||
|
||||
========================================
|
||||
SUMMARY: Duration {{total_duration}}min | Tables {{tables_restored}} | Backups {{backup_count}}
|
||||
|
||||
TEST STEPS:
|
||||
{{#each test_steps}}
|
||||
- {{#if this.passed}}PASS{{else}}FAIL{{/if}}: {{this.name}} ({{this.duration}}s)
|
||||
{{#if this.details}}
|
||||
Details: {{this.details}}
|
||||
{{/each}}
|
||||
|
||||
========================================
|
||||
COMPONENT STATUS:
|
||||
DR VM: ID {{vm_id}} ({{vm_ip}}) - {{vm_status}}
|
||||
NFS Mount: {{nfs_status}} - {{#if nfs_ok}}OK{{else}}FAILED{{/if}}
|
||||
Database: {{database_status}} - {{#if database_ok}}OK{{else}}FAILED{{/if}}
|
||||
Disk Space: {{disk_freed}}GB freed - OK
|
||||
|
||||
{{#if errors}}
|
||||
ERRORS:
|
||||
{{#each errors}}
|
||||
- {{this}}
|
||||
{{/each}}
|
||||
{{/if}}
|
||||
|
||||
========================================
|
||||
Log: {{log_file}}
|
||||
Next scheduled test: Next Saturday 06:00
|
||||
Proxmox DR Monitoring System
|
||||
EOF
|
||||
.section {
|
||||
margin: 20px 0;
|
||||
padding: 15px;
|
||||
background-color: #f8f9fa;
|
||||
border-radius: 5px;
|
||||
}
|
||||
.success { color: #28a745; font-weight: bold; }
|
||||
.error { color: #dc3545; font-weight: bold; }
|
||||
.warning { color: #ffc107; font-weight: bold; }
|
||||
.info { color: #17a2b8; }
|
||||
|
||||
.test-steps {
|
||||
margin: 20px 0;
|
||||
}
|
||||
.step {
|
||||
padding: 10px;
|
||||
margin: 5px 0;
|
||||
border-left: 4px solid;
|
||||
background-color: white;
|
||||
}
|
||||
.step.passed {
|
||||
border-color: #28a745;
|
||||
}
|
||||
.step.failed {
|
||||
border-color: #dc3545;
|
||||
background-color: #f8d7da;
|
||||
}
|
||||
|
||||
.metrics {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||
gap: 15px;
|
||||
margin: 20px 0;
|
||||
}
|
||||
.metric-card {
|
||||
background: white;
|
||||
padding: 15px;
|
||||
border-radius: 5px;
|
||||
text-align: center;
|
||||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||||
}
|
||||
.metric-value {
|
||||
font-size: 24px;
|
||||
font-weight: bold;
|
||||
color: #495057;
|
||||
}
|
||||
.metric-label {
|
||||
font-size: 14px;
|
||||
color: #6c757d;
|
||||
margin-top: 5px;
|
||||
}
|
||||
|
||||
.timeline {
|
||||
position: relative;
|
||||
padding: 20px 0;
|
||||
}
|
||||
.timeline-item {
|
||||
display: flex;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.timeline-marker {
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
border-radius: 50%;
|
||||
margin-right: 15px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.timeline-marker.success {
|
||||
background-color: #28a745;
|
||||
}
|
||||
.timeline-marker.failed {
|
||||
background-color: #dc3545;
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin: 10px 0;
|
||||
}
|
||||
th, td {
|
||||
padding: 10px;
|
||||
text-align: left;
|
||||
border-bottom: 1px solid #dee2e6;
|
||||
}
|
||||
th {
|
||||
background-color: #e9ecef;
|
||||
font-weight: bold;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<h1>Oracle DR Test Report</h1>
|
||||
<h2>{{#if is_success}}✓ TEST PASSED{{else}}✗ TEST FAILED{{/if}}</h2>
|
||||
<p>{{date}} | Duration: {{total_duration}} minutes</p>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h3>Test Summary</h3>
|
||||
<div class="metrics">
|
||||
<div class="metric-card">
|
||||
<div class="metric-value {{#if is_success}}success{{else}}error{{/if}}">{{test_result}}</div>
|
||||
<div class="metric-label">Test Result</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-value">{{restore_duration}}</div>
|
||||
<div class="metric-label">Restore Time (min)</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-value">{{tables_restored}}</div>
|
||||
<div class="metric-label">Tables Restored</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-value">{{backup_count}}</div>
|
||||
<div class="metric-label">Backups Used</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h3>Test Steps Timeline</h3>
|
||||
<div class="timeline">
|
||||
{{#each test_steps}}
|
||||
<div class="timeline-item">
|
||||
<div class="timeline-marker {{#if this.passed}}success{{else}}failed{{/if}}"></div>
|
||||
<div style="flex-grow: 1;">
|
||||
<div class="step {{#if this.passed}}passed{{else}}failed{{/if}}">
|
||||
<strong>{{this.name}}</strong>
|
||||
<span style="float: right; color: #6c757d;">{{this.duration}}s</span>
|
||||
<div style="margin-top: 5px;">
|
||||
{{#if this.passed}}
|
||||
<span class="success">✓ {{this.status}}</span>
|
||||
{{else}}
|
||||
<span class="error">✗ {{this.status}}</span>
|
||||
{{/if}}
|
||||
</div>
|
||||
{{#if this.details}}
|
||||
<div style="margin-top: 5px; font-size: 0.9em; color: #6c757d;">
|
||||
{{this.details}}
|
||||
</div>
|
||||
{{/if}}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{{/each}}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{{#if errors}}
|
||||
<div class="section" style="background-color: #f8d7da;">
|
||||
<h3 class="error">Errors Encountered</h3>
|
||||
<ul>
|
||||
{{#each errors}}
|
||||
<li>{{this}}</li>
|
||||
{{/each}}
|
||||
</ul>
|
||||
</div>
|
||||
{{/if}}
|
||||
|
||||
{{#if warnings}}
|
||||
<div class="section" style="background-color: #fff3cd;">
|
||||
<h3 class="warning">Warnings</h3>
|
||||
<ul>
|
||||
{{#if has_warnings}}
|
||||
WARNINGS
|
||||
{{#each warnings}}
|
||||
<li>{{this}}</li>
|
||||
- {{this}}
|
||||
{{/each}}
|
||||
</ul>
|
||||
</div>
|
||||
{{/if}}
|
||||
|
||||
<div class="section">
|
||||
<h3>System Details</h3>
|
||||
<table>
|
||||
RESTORE LOG (first 200 lines)
|
||||
---
|
||||
{{restore_log}}
|
||||
---
|
||||
|
||||
Log: {{log_file}}
|
||||
Next test: Saturday 06:00
|
||||
EOF
|
||||
|
||||
# HTML body template (compact Gmail-friendly layout)
|
||||
cat > "$TEMPLATE_DIR/oracle-dr-test-body.html.hbs" <<'EOF'
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Oracle DR Test {{test_result}} | {{date}}</title>
|
||||
</head>
|
||||
<body style="margin:0;padding:16px;font-family:Arial,Helvetica,sans-serif;background:#ffffff;color:#2c3e50;">
|
||||
<table style="width:100%;max-width:640px;margin:0 auto;border-collapse:collapse;">
|
||||
<tr>
|
||||
<th>Component</th>
|
||||
<th>Value</th>
|
||||
<th>Status</th>
|
||||
<td style="padding:0 0 12px 0;font-size:18px;font-weight:600;">
|
||||
Oracle DR Test {{test_result}}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>DR VM</td>
|
||||
<td>ID: {{vm_id}} ({{vm_ip}})</td>
|
||||
<td>{{vm_status}}</td>
|
||||
<td style="padding:0 0 8px 0;font-size:13px;color:#6c757d;">{{date}} · Severity: {{severity}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>NFS Mount</td>
|
||||
<td>F:\ drive</td>
|
||||
<td>{{nfs_status}}</td>
|
||||
<td style="padding:12px;border:1px solid #e1e4e8;border-radius:4px;">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:14px;">
|
||||
<tr><td style="padding:4px 0;">Outcome</td><td style="padding:4px 0;text-align:right;">{{test_result}}</td></tr>
|
||||
<tr><td style="padding:4px 0;">Duration</td><td style="padding:4px 0;text-align:right;">{{total_duration}} min (restore {{restore_duration}} min)</td></tr>
|
||||
<tr><td style="padding:4px 0;">Backups used</td><td style="padding:4px 0;text-align:right;">{{backup_count}}</td></tr>
|
||||
<tr><td style="padding:4px 0;">Tables restored</td><td style="padding:4px 0;text-align:right;">{{tables_restored}}</td></tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>Database</td>
|
||||
<td>ROA</td>
|
||||
<td>{{database_status}}</td>
|
||||
<td style="padding:16px 0 0 0;">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:14px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
|
||||
<tr><td style="padding:8px 12px;font-weight:600;">Components</td></tr>
|
||||
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">VM {{vm_id}} ({{vm_ip}}): {{vm_status}}</td></tr>
|
||||
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">NFS: {{nfs_status}}</td></tr>
|
||||
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">Database: {{database_status}}</td></tr>
|
||||
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">Cleanup: {{disk_freed}} GB freed</td></tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>Disk Space Freed</td>
|
||||
<td>{{disk_freed}} GB</td>
|
||||
<td class="success">✓</td>
|
||||
<td style="padding:16px 0 0 0;">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:14px;">
|
||||
<tr><td style="padding:0 0 6px 0;font-weight:600;">Steps</td></tr>
|
||||
{{#each test_steps}}
|
||||
<tr>
|
||||
<td style="padding:4px 0;border-bottom:1px solid #f1f1f1;">{{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}} ({{this.duration}}s){{#if this.status}} – {{this.status}}{{/if}}</td>
|
||||
</tr>
|
||||
{{/each}}
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
{{#if has_errors}}
|
||||
<tr>
|
||||
<td style="padding:16px 0 0 0;">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff5f5;border:1px solid #f1b0b7;border-radius:4px;">
|
||||
<tr><td style="padding:8px 12px;font-weight:600;color:#c82333;">Issues</td></tr>
|
||||
{{#each errors}}
|
||||
<tr><td style="padding:6px 12px;border-top:1px solid #f8d7da;">• {{this}}</td></tr>
|
||||
{{/each}}
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
{{/if}}
|
||||
|
||||
{{#if has_warnings}}
|
||||
<tr>
|
||||
<td style="padding:16px 0 0 0;">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff8e5;border:1px solid #ffe8a1;border-radius:4px;">
|
||||
<tr><td style="padding:8px 12px;font-weight:600;color:#856404;">Warnings</td></tr>
|
||||
{{#each warnings}}
|
||||
<tr><td style="padding:6px 12px;border-top:1px solid #ffe8a1;">• {{this}}</td></tr>
|
||||
{{/each}}
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
{{/if}}
|
||||
|
||||
<tr>
|
||||
<td style="padding:16px 0 0 0;">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:12px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
|
||||
<tr><td style="padding:8px 12px;font-weight:600;font-size:13px;">Restore Log (first 200 lines)</td></tr>
|
||||
<tr><td style="padding:8px 12px;font-family:monospace;white-space:pre-wrap;word-wrap:break-word;border-top:1px solid #e1e4e8;">{{restore_log}}</td></tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td style="padding:16px 0 0 0;font-size:12px;color:#6c757d;">
|
||||
Log: {{log_file}} · Next test: Saturday 06:00
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<p class="info">
|
||||
<strong>Log File:</strong> {{log_file}}<br>
|
||||
<strong>Next Scheduled Test:</strong> Next Saturday 06:00
|
||||
</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
@@ -421,7 +286,16 @@ track_step() {
|
||||
local end_time=$(date +%s)
|
||||
local duration=$((end_time - start_time))
|
||||
|
||||
TEST_STEPS+=("{\"name\":\"$name\",\"passed\":$passed,\"status\":\"$status\",\"duration\":$duration}")
|
||||
local step_json
|
||||
step_json=$(jq -n \
|
||||
--arg name "$name" \
|
||||
--arg status "$status" \
|
||||
--arg duration "$duration" \
|
||||
--arg passed "$passed" \
|
||||
'{name:$name, status:$status, duration:($duration|tonumber), passed:($passed == "true")}'
|
||||
)
|
||||
|
||||
TEST_STEPS+=("$step_json")
|
||||
|
||||
if [ "$passed" = "false" ]; then
|
||||
ERRORS+=("$name: $status")
|
||||
@@ -433,6 +307,14 @@ run_dr_test() {
|
||||
local test_result="FAILED"
|
||||
local severity="error"
|
||||
local is_success=false
|
||||
local restore_duration=0
|
||||
local tables_restored=0
|
||||
local db_status="UNKNOWN"
|
||||
local nfs_status="Not checked"
|
||||
local vm_status_label="Not started"
|
||||
local cleanup_freed=0
|
||||
local backup_count=0
|
||||
local restore_log="Not collected"
|
||||
|
||||
log "=========================================="
|
||||
log "Oracle DR Weekly Test - Starting"
|
||||
@@ -443,7 +325,7 @@ run_dr_test() {
|
||||
log "STEP 1: Pre-flight checks"
|
||||
|
||||
# Check backups exist
|
||||
local backup_count=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l || echo "0")
|
||||
backup_count=$(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*.BKP' 2>/dev/null | wc -l)
|
||||
|
||||
if [ "$backup_count" -lt 2 ]; then
|
||||
track_step "Pre-flight checks" false "Insufficient backups (found: $backup_count)" "$step_start"
|
||||
@@ -456,6 +338,7 @@ run_dr_test() {
|
||||
log "STEP 2: Starting DR VM"
|
||||
|
||||
if qm start "$DR_VM_ID" 2>/dev/null; then
|
||||
vm_status_label="Running"
|
||||
sleep 180 # Wait for boot
|
||||
track_step "VM Startup" true "VM $DR_VM_ID started" "$step_start"
|
||||
|
||||
@@ -463,7 +346,7 @@ run_dr_test() {
|
||||
step_start=$(date +%s)
|
||||
log "STEP 3: Verifying NFS mount"
|
||||
|
||||
local nfs_status="Not Mounted"
|
||||
nfs_status="Not Mounted"
|
||||
if ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 "$DR_VM_USER@$DR_VM_IP" \
|
||||
"powershell -Command 'Test-Path F:\\ROA\\autobackup'" 2>/dev/null; then
|
||||
nfs_status="Mounted"
|
||||
@@ -482,7 +365,7 @@ run_dr_test() {
|
||||
"D:\\oracle\\scripts\\rman_restore_from_zero.cmd" 2>&1 | tee -a "$LOG_FILE"; then
|
||||
|
||||
local restore_end=$(date +%s)
|
||||
local restore_duration=$(( (restore_end - restore_start) / 60 ))
|
||||
restore_duration=$(( (restore_end - restore_start) / 60 ))
|
||||
|
||||
track_step "Database Restore" true "Restored in $restore_duration minutes" "$step_start"
|
||||
|
||||
@@ -490,11 +373,13 @@ run_dr_test() {
|
||||
step_start=$(date +%s)
|
||||
log "STEP 5: Verifying database"
|
||||
|
||||
local db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||||
db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||||
"cmd /c 'echo SELECT STATUS FROM V\$INSTANCE; | sqlplus -s / as sysdba' | findstr OPEN" || echo "")
|
||||
|
||||
local tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||||
tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||||
"cmd /c 'echo SELECT COUNT(*) FROM DBA_TABLES WHERE OWNER NOT IN (''SYS'',''SYSTEM''); | sqlplus -s / as sysdba' | grep -o '[0-9]*' | tail -1" || echo "0")
|
||||
tables_restored=$(echo "$tables_restored" | tr -cd '0-9')
|
||||
[ -z "$tables_restored" ] && tables_restored=0
|
||||
|
||||
if [[ "$db_status" =~ "OPEN" ]]; then
|
||||
track_step "Database Verification" true "Database OPEN, $tables_restored tables" "$step_start"
|
||||
@@ -505,6 +390,11 @@ run_dr_test() {
|
||||
track_step "Database Verification" false "Database not OPEN" "$step_start"
|
||||
fi
|
||||
|
||||
# Collect restore log from VM
|
||||
log "Collecting restore log from DR VM..."
|
||||
restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||||
"type D:\\oracle\\logs\\restore_from_zero.log 2>nul" | head -200 || echo "Log not available")
|
||||
|
||||
# Step 6: Cleanup
|
||||
step_start=$(date +%s)
|
||||
log "STEP 6: Running cleanup"
|
||||
@@ -512,7 +402,8 @@ run_dr_test() {
|
||||
ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||||
"D:\\oracle\\scripts\\cleanup_database.cmd" 2>/dev/null
|
||||
|
||||
track_step "Cleanup" true "Database cleaned, ~8GB freed" "$step_start"
|
||||
cleanup_freed=8
|
||||
track_step "Cleanup" true "Database cleaned, ~${cleanup_freed}GB freed" "$step_start"
|
||||
|
||||
else
|
||||
track_step "Database Restore" false "Restore failed" "$step_start"
|
||||
@@ -527,9 +418,11 @@ run_dr_test() {
|
||||
qm stop "$DR_VM_ID" 2>/dev/null
|
||||
|
||||
track_step "VM Shutdown" true "VM stopped" "$step_start"
|
||||
vm_status_label="Stopped"
|
||||
|
||||
else
|
||||
track_step "VM Startup" false "Failed to start VM $DR_VM_ID" "$step_start"
|
||||
vm_status_label="Failed to start"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -538,9 +431,41 @@ run_dr_test() {
|
||||
local total_duration=$(( (test_end_time - TEST_START_TIME) / 60 ))
|
||||
|
||||
# Prepare notification data
|
||||
local steps_json=$(printf '%s,' "${TEST_STEPS[@]}" | sed 's/,$//')
|
||||
local errors_json=$(printf '"%s",' "${ERRORS[@]}" | sed 's/,$//')
|
||||
local warnings_json=$(printf '"%s",' "${WARNINGS[@]}" | sed 's/,$//')
|
||||
local steps_json
|
||||
if [ ${#TEST_STEPS[@]} -eq 0 ]; then
|
||||
steps_json='[]'
|
||||
else
|
||||
steps_json=$(printf '%s\n' "${TEST_STEPS[@]}" | jq -s '.')
|
||||
fi
|
||||
|
||||
local errors_json
|
||||
if [ ${#ERRORS[@]} -eq 0 ]; then
|
||||
errors_json='[]'
|
||||
else
|
||||
errors_json=$(printf '%s\n' "${ERRORS[@]}" | jq -R . | jq -s .)
|
||||
fi
|
||||
|
||||
local warnings_json
|
||||
if [ ${#WARNINGS[@]} -eq 0 ]; then
|
||||
warnings_json='[]'
|
||||
else
|
||||
warnings_json=$(printf '%s\n' "${WARNINGS[@]}" | jq -R . | jq -s .)
|
||||
fi
|
||||
|
||||
local has_errors=false
|
||||
local has_warnings=false
|
||||
[ ${#ERRORS[@]} -gt 0 ] && has_errors=true
|
||||
[ ${#WARNINGS[@]} -gt 0 ] && has_warnings=true
|
||||
|
||||
if [ "$is_success" = true ] && [ "$has_warnings" = true ]; then
|
||||
severity="warning"
|
||||
fi
|
||||
|
||||
local db_status_clean=$(echo "$db_status" | tr -d '\r' | sed 's/^ *//;s/ *$//')
|
||||
|
||||
# Escape restore log for JSON
|
||||
local restore_log_json
|
||||
restore_log_json=$(echo "$restore_log" | jq -Rs .)
|
||||
|
||||
local json_data=$(cat <<JSON
|
||||
{
|
||||
@@ -549,19 +474,22 @@ run_dr_test() {
|
||||
"date": "$(date '+%Y-%m-%d %H:%M:%S')",
|
||||
"total_duration": $total_duration,
|
||||
"is_success": $is_success,
|
||||
"test_steps": [$steps_json],
|
||||
"errors": [${errors_json:-}],
|
||||
"warnings": [${warnings_json:-}],
|
||||
"backup_count": ${backup_count:-0},
|
||||
"restore_duration": ${restore_duration:-0},
|
||||
"has_errors": $has_errors,
|
||||
"has_warnings": $has_warnings,
|
||||
"test_steps": $steps_json,
|
||||
"errors": $errors_json,
|
||||
"warnings": $warnings_json,
|
||||
"backup_count": $backup_count,
|
||||
"restore_duration": $restore_duration,
|
||||
"tables_restored": ${tables_restored:-0},
|
||||
"database_status": "${db_status:-UNKNOWN}",
|
||||
"disk_freed": 8,
|
||||
"database_status": "${db_status_clean:-UNKNOWN}",
|
||||
"disk_freed": $cleanup_freed,
|
||||
"vm_id": "$DR_VM_ID",
|
||||
"vm_ip": "$DR_VM_IP",
|
||||
"vm_status": "Stopped",
|
||||
"vm_status": "$vm_status_label",
|
||||
"nfs_status": "${nfs_status:-Unknown}",
|
||||
"log_file": "$LOG_FILE"
|
||||
"log_file": "$LOG_FILE",
|
||||
"restore_log": $restore_log_json
|
||||
}
|
||||
JSON
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user