Compare commits
2 Commits
b34006a499
...
839f1b6b82
| Author | SHA1 | Date | |
|---|---|---|---|
|
839f1b6b82 | ||
|
|
6f56e61b04 |
@@ -53,7 +53,14 @@
|
|||||||
3. Validare log-uri și rapoarte generate
|
3. Validare log-uri și rapoarte generate
|
||||||
4. Configurare cron pentru execuție automată
|
4. Configurare cron pentru execuție automată
|
||||||
|
|
||||||
### Faza 5: Testare Erori și Edge Cases
|
### Faza 5: Validare Format Notificări
|
||||||
|
1. Reinstalare template-uri compacte: `/opt/scripts/oracle-backup-monitor-proxmox.sh --install`
|
||||||
|
2. Generare notificări reale din scripturi (backup monitor + DR test) și analiză în clienți email
|
||||||
|
3. Verificare afișare în client email (text + HTML) și în GUI Proxmox
|
||||||
|
4. Rulare `weekly-dr-test-proxmox.sh` în mediu controlat și validare sumar compact în email (inclusiv componente, pași, timeline)
|
||||||
|
5. Capturare feedback utilizatori finali (Gmail + Outlook) pentru lizibilitate
|
||||||
|
|
||||||
|
### Faza 6: Testare Erori și Edge Cases
|
||||||
1. Testare fără conectivitate la VM DR
|
1. Testare fără conectivitate la VM DR
|
||||||
2. Testare director backup-uri gol
|
2. Testare director backup-uri gol
|
||||||
3. Testare eșec restaurare database
|
3. Testare eșec restaurare database
|
||||||
|
|||||||
@@ -46,147 +46,165 @@ create_templates() {
|
|||||||
|
|
||||||
# Subject template
|
# Subject template
|
||||||
cat > "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" <<'EOF'
|
cat > "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" <<'EOF'
|
||||||
Oracle Backup {{severity}} - {{node}}
|
Oracle Backup {{status}} | {{node}}
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# Text body template
|
# Text body template
|
||||||
cat > "$TEMPLATE_DIR/oracle-backup-body.txt.hbs" <<'EOF'
|
cat > "$TEMPLATE_DIR/oracle-backup-body.txt.hbs" <<'EOF'
|
||||||
Oracle Backup Monitoring Alert
|
Oracle Backup {{status}} | {{node}}
|
||||||
==============================
|
|
||||||
Severity: {{severity}}
|
|
||||||
Hostname: {{node}}
|
|
||||||
Date: {{date}}
|
Date: {{date}}
|
||||||
Status: {{status}}
|
|
||||||
|
|
||||||
{{#if errors}}
|
SUMMARY
|
||||||
ERRORS:
|
- Full backup: {{full_backup_age}}h (limit {{full_backup_limit}}h) -> {{#if full_backup_ok}}OK{{else}}CHECK{{/if}}
|
||||||
|
- Incremental: {{cumulative_backup_age}}h (limit {{cumulative_backup_limit}}h) -> {{#if cumulative_backup_ok}}OK{{else}}CHECK{{/if}}
|
||||||
|
- Backups: {{total_backups}} files ({{total_size_label}})
|
||||||
|
- Disk usage: {{disk_usage}}%
|
||||||
|
|
||||||
|
{{#if has_errors}}
|
||||||
|
ISSUES
|
||||||
{{#each errors}}
|
{{#each errors}}
|
||||||
- {{this}}
|
- {{this}}
|
||||||
{{/each}}
|
{{/each}}
|
||||||
{{/if}}
|
{{/if}}
|
||||||
|
|
||||||
{{#if warnings}}
|
{{#if has_warnings}}
|
||||||
WARNINGS:
|
WARNINGS
|
||||||
{{#each warnings}}
|
{{#each warnings}}
|
||||||
- {{this}}
|
- {{this}}
|
||||||
{{/each}}
|
{{/each}}
|
||||||
{{/if}}
|
{{/if}}
|
||||||
|
|
||||||
Backup Details:
|
FULL BACKUPS ({{full_backup_count}} files)
|
||||||
- Total Backups: {{total_backups}}
|
{{#if has_full_backups}}
|
||||||
- Total Size: {{total_size_gb}} GB
|
{{#each full_backup_list}}
|
||||||
- FULL Backup Age: {{full_backup_age}} hours
|
- {{this}}
|
||||||
- CUMULATIVE Backup Age: {{cumulative_backup_age}} hours
|
|
||||||
- Disk Usage: {{disk_usage}}%
|
|
||||||
|
|
||||||
{{#if backup_list}}
|
|
||||||
Recent Backups:
|
|
||||||
{{#each backup_list}}
|
|
||||||
{{this}}
|
|
||||||
{{/each}}
|
{{/each}}
|
||||||
|
{{else}}
|
||||||
|
- none detected
|
||||||
{{/if}}
|
{{/if}}
|
||||||
|
|
||||||
|
INCREMENTAL BACKUPS ({{incr_backup_count}} files)
|
||||||
|
{{#if has_incr_backups}}
|
||||||
|
{{#each incr_backup_list}}
|
||||||
|
- {{this}}
|
||||||
|
{{/each}}
|
||||||
|
{{else}}
|
||||||
|
- none detected
|
||||||
|
{{/if}}
|
||||||
|
|
||||||
|
Next check: +24h via Proxmox Monitor
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# HTML body template
|
# HTML body template (lightweight Gmail-friendly)
|
||||||
cat > "$TEMPLATE_DIR/oracle-backup-body.html.hbs" <<'EOF'
|
cat > "$TEMPLATE_DIR/oracle-backup-body.html.hbs" <<'EOF'
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html>
|
<html>
|
||||||
<head>
|
<head>
|
||||||
<style>
|
<meta charset="utf-8">
|
||||||
body { font-family: Arial, sans-serif; }
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
.header {
|
<title>Oracle Backup {{status}} | {{node}}</title>
|
||||||
background-color: {{#if is_error}}#dc3545{{else}}{{#if is_warning}}#ffc107{{else}}#28a745{{/if}}{{/if}};
|
|
||||||
color: white;
|
|
||||||
padding: 10px;
|
|
||||||
border-radius: 5px;
|
|
||||||
}
|
|
||||||
.section { margin: 20px 0; padding: 10px; background-color: #f8f9fa; border-radius: 5px; }
|
|
||||||
.error { color: #dc3545; font-weight: bold; }
|
|
||||||
.warning { color: #ffc107; font-weight: bold; }
|
|
||||||
.success { color: #28a745; }
|
|
||||||
table { width: 100%; border-collapse: collapse; margin: 10px 0; }
|
|
||||||
th, td { padding: 8px; text-align: left; border-bottom: 1px solid #dee2e6; }
|
|
||||||
th { background-color: #e9ecef; }
|
|
||||||
.metric { display: inline-block; margin: 10px 20px 10px 0; }
|
|
||||||
.metric-label { font-size: 0.9em; color: #6c757d; }
|
|
||||||
.metric-value { font-size: 1.5em; font-weight: bold; }
|
|
||||||
</style>
|
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body style="margin:0;padding:16px;font-family:Arial,Helvetica,sans-serif;background:#ffffff;color:#2c3e50;">
|
||||||
<div class="header">
|
<table style="width:100%;max-width:640px;margin:0 auto;border-collapse:collapse;">
|
||||||
<h2>Oracle Backup {{severity}}</h2>
|
<tr>
|
||||||
<p>{{node}} - {{date}}</p>
|
<td style="padding:0 0 12px 0;font-size:18px;font-weight:600;">
|
||||||
</div>
|
Oracle Backup {{status}} | {{node}}
|
||||||
|
</td>
|
||||||
<div class="section">
|
</tr>
|
||||||
<h3>Status: <span class="{{#if is_error}}error{{else}}{{#if is_warning}}warning{{else}}success{{/if}}{{/if}}">{{status}}</span></h3>
|
<tr>
|
||||||
|
<td style="padding:0 0 16px 0;font-size:13px;color:#6c757d;">
|
||||||
{{#if errors}}
|
{{date}}
|
||||||
<div class="error">
|
</td>
|
||||||
<h4>Errors:</h4>
|
</tr>
|
||||||
<ul>
|
<tr>
|
||||||
{{#each errors}}
|
<td style="padding:12px;border:1px solid #e1e4e8;border-radius:4px;">
|
||||||
<li>{{this}}</li>
|
<table style="width:100%;border-collapse:collapse;font-size:14px;">
|
||||||
{{/each}}
|
<tr>
|
||||||
</ul>
|
<td style="padding:4px 0;">Full backup</td>
|
||||||
</div>
|
<td style="padding:4px 0;text-align:right;">
|
||||||
{{/if}}
|
{{full_backup_age}}h / {{full_backup_limit}}h · {{#if full_backup_ok}}OK{{else}}CHECK{{/if}}
|
||||||
|
</td>
|
||||||
{{#if warnings}}
|
</tr>
|
||||||
<div class="warning">
|
<tr>
|
||||||
<h4>Warnings:</h4>
|
<td style="padding:4px 0;">Incremental</td>
|
||||||
<ul>
|
<td style="padding:4px 0;text-align:right;">
|
||||||
{{#each warnings}}
|
{{cumulative_backup_age}}h / {{cumulative_backup_limit}}h · {{#if cumulative_backup_ok}}OK{{else}}CHECK{{/if}}
|
||||||
<li>{{this}}</li>
|
</td>
|
||||||
{{/each}}
|
</tr>
|
||||||
</ul>
|
<tr>
|
||||||
</div>
|
<td style="padding:4px 0;">Backups</td>
|
||||||
{{/if}}
|
<td style="padding:4px 0;text-align:right;">{{total_backups}} files ({{total_size_label}})</td>
|
||||||
</div>
|
</tr>
|
||||||
|
<tr>
|
||||||
<div class="section">
|
<td style="padding:4px 0;">Disk usage</td>
|
||||||
<h3>Backup Metrics</h3>
|
<td style="padding:4px 0;text-align:right;">{{disk_usage}}%</td>
|
||||||
<div>
|
</tr>
|
||||||
<div class="metric">
|
|
||||||
<div class="metric-label">Total Backups</div>
|
|
||||||
<div class="metric-value">{{total_backups}}</div>
|
|
||||||
</div>
|
|
||||||
<div class="metric">
|
|
||||||
<div class="metric-label">Total Size</div>
|
|
||||||
<div class="metric-value">{{total_size_gb}} GB</div>
|
|
||||||
</div>
|
|
||||||
<div class="metric">
|
|
||||||
<div class="metric-label">Disk Usage</div>
|
|
||||||
<div class="metric-value">{{disk_usage}}%</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<table>
|
|
||||||
<tr>
|
|
||||||
<th>Backup Type</th>
|
|
||||||
<th>Age (hours)</th>
|
|
||||||
<th>Status</th>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>FULL</td>
|
|
||||||
<td>{{full_backup_age}}</td>
|
|
||||||
<td>{{#if full_backup_ok}}<span class="success">✓ OK</span>{{else}}<span class="error">✗ Too Old</span>{{/if}}</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>CUMULATIVE</td>
|
|
||||||
<td>{{cumulative_backup_age}}</td>
|
|
||||||
<td>{{#if cumulative_backup_ok}}<span class="success">✓ OK</span>{{else}}<span class="warning">⚠ Check</span>{{/if}}</td>
|
|
||||||
</tr>
|
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</td>
|
||||||
|
</tr>
|
||||||
|
|
||||||
{{#if backup_list}}
|
{{#if has_errors}}
|
||||||
<div class="section">
|
<tr>
|
||||||
<h3>Recent Backups</h3>
|
<td style="padding:16px 0 0 0;">
|
||||||
<pre style="background-color: #f8f9fa; padding: 10px; overflow-x: auto;">{{#each backup_list}}{{this}}
|
<table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff5f5;border:1px solid #f1b0b7;border-radius:4px;">
|
||||||
{{/each}}</pre>
|
<tr><td style="padding:8px 12px;font-weight:600;color:#c82333;">Issues</td></tr>
|
||||||
</div>
|
{{#each errors}}
|
||||||
|
<tr><td style="padding:6px 12px;border-top:1px solid #f8d7da;">• {{this}}</td></tr>
|
||||||
|
{{/each}}
|
||||||
|
</table>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
{{/if}}
|
{{/if}}
|
||||||
|
|
||||||
|
{{#if has_warnings}}
|
||||||
|
<tr>
|
||||||
|
<td style="padding:16px 0 0 0;">
|
||||||
|
<table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff8e5;border:1px solid #ffe8a1;border-radius:4px;">
|
||||||
|
<tr><td style="padding:8px 12px;font-weight:600;color:#856404;">Warnings</td></tr>
|
||||||
|
{{#each warnings}}
|
||||||
|
<tr><td style="padding:6px 12px;border-top:1px solid #ffe8a1;">• {{this}}</td></tr>
|
||||||
|
{{/each}}
|
||||||
|
</table>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{{/if}}
|
||||||
|
|
||||||
|
<tr>
|
||||||
|
<td style="padding:16px 0 0 0;">
|
||||||
|
<table style="width:100%;border-collapse:collapse;font-size:13px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
|
||||||
|
<tr><td style="padding:8px 12px;font-weight:600;">FULL Backups ({{full_backup_count}} files)</td></tr>
|
||||||
|
{{#if has_full_backups}}
|
||||||
|
{{#each full_backup_list}}
|
||||||
|
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">• {{this}}</td></tr>
|
||||||
|
{{/each}}
|
||||||
|
{{else}}
|
||||||
|
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">• none detected</td></tr>
|
||||||
|
{{/if}}
|
||||||
|
</table>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
|
||||||
|
<tr>
|
||||||
|
<td style="padding:16px 0 0 0;">
|
||||||
|
<table style="width:100%;border-collapse:collapse;font-size:13px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
|
||||||
|
<tr><td style="padding:8px 12px;font-weight:600;">INCREMENTAL Backups ({{incr_backup_count}} files)</td></tr>
|
||||||
|
{{#if has_incr_backups}}
|
||||||
|
{{#each incr_backup_list}}
|
||||||
|
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">• {{this}}</td></tr>
|
||||||
|
{{/each}}
|
||||||
|
{{else}}
|
||||||
|
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">• none detected</td></tr>
|
||||||
|
{{/if}}
|
||||||
|
</table>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
|
||||||
|
<tr>
|
||||||
|
<td style="padding:16px 0 0 0;font-size:12px;color:#6c757d;">
|
||||||
|
Next automated check: +24h via Proxmox Monitor
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</table>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
EOF
|
EOF
|
||||||
@@ -218,7 +236,7 @@ my $template_name = 'oracle-backup';
|
|||||||
my $fields = {
|
my $fields = {
|
||||||
type => 'oracle-backup',
|
type => 'oracle-backup',
|
||||||
severity => $severity,
|
severity => $severity,
|
||||||
hostname => $data->{hostname},
|
hostname => $data->{node} // 'unknown',
|
||||||
};
|
};
|
||||||
|
|
||||||
# Send notification
|
# Send notification
|
||||||
@@ -255,119 +273,193 @@ check_backups() {
|
|||||||
|
|
||||||
echo "Checking Oracle backups..."
|
echo "Checking Oracle backups..."
|
||||||
|
|
||||||
# Get backup list
|
local total_backups=0
|
||||||
local backup_files=$(ls -lth "$BACKUP_PATH"/*.BKP 2>/dev/null | head -10 || echo "")
|
local total_size_label="0G"
|
||||||
|
local full_age_hours="N/A"
|
||||||
|
local cumulative_age_hours="N/A"
|
||||||
|
local full_backup_ok=false
|
||||||
|
local cumulative_backup_ok=false
|
||||||
|
local disk_usage=0
|
||||||
|
local -a backup_entries=()
|
||||||
|
|
||||||
if [ -z "$backup_files" ]; then
|
if [ ! -d "$BACKUP_PATH" ]; then
|
||||||
status="ERROR"
|
status="ERROR"
|
||||||
errors+=("No backup files found in $BACKUP_PATH")
|
errors+=("Backup path $BACKUP_PATH not accessible")
|
||||||
else
|
else
|
||||||
# Count backups
|
if compgen -G "$BACKUP_PATH"/*.BKP > /dev/null; then
|
||||||
local total_backups=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l)
|
total_backups=$(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*.BKP' | wc -l)
|
||||||
local total_size=$(du -shc "$BACKUP_PATH"/*.BKP 2>/dev/null | tail -1 | awk '{print $1}')
|
total_backups=${total_backups//[[:space:]]/}
|
||||||
|
[ -z "$total_backups" ] && total_backups=0
|
||||||
|
local total_size=$(du -shc "$BACKUP_PATH"/*.BKP 2>/dev/null | tail -1 | awk '{print $1}')
|
||||||
|
[ -z "$total_size" ] && total_size="0G"
|
||||||
|
total_size_label="$total_size"
|
||||||
|
|
||||||
# Check FULL backup age
|
local latest_full=$(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*FULL*.BKP' -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)
|
||||||
local latest_full=$(ls -t "$BACKUP_PATH"/*FULL*.BKP 2>/dev/null | head -1 || echo "")
|
if [ -n "$latest_full" ]; then
|
||||||
local full_age_hours="N/A"
|
local full_timestamp=$(stat -c %Y "$latest_full")
|
||||||
local full_backup_ok=false
|
local current_timestamp=$(date +%s)
|
||||||
|
full_age_hours=$(( (current_timestamp - full_timestamp) / 3600 ))
|
||||||
if [ -n "$latest_full" ]; then
|
if [ "$full_age_hours" -gt "$MAX_FULL_AGE_HOURS" ]; then
|
||||||
local full_timestamp=$(stat -c %Y "$latest_full")
|
status="WARNING"
|
||||||
local current_timestamp=$(date +%s)
|
warnings+=("FULL backup is $full_age_hours hours old (threshold: $MAX_FULL_AGE_HOURS)")
|
||||||
full_age_hours=$(( (current_timestamp - full_timestamp) / 3600 ))
|
else
|
||||||
|
full_backup_ok=true
|
||||||
if [ "$full_age_hours" -gt "$MAX_FULL_AGE_HOURS" ]; then
|
fi
|
||||||
status="WARNING"
|
|
||||||
warnings+=("FULL backup is $full_age_hours hours old (threshold: $MAX_FULL_AGE_HOURS)")
|
|
||||||
else
|
else
|
||||||
full_backup_ok=true
|
status="ERROR"
|
||||||
|
errors+=("No FULL backup found")
|
||||||
|
fi
|
||||||
|
|
||||||
|
local latest_cumulative=$(find "$BACKUP_PATH" -maxdepth 1 -type f \( -name '*INCR*.BKP' -o -name '*INCREMENTAL*.BKP' -o -name '*CUMULATIVE*.BKP' \) -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)
|
||||||
|
if [ -n "$latest_cumulative" ]; then
|
||||||
|
local cumulative_timestamp=$(stat -c %Y "$latest_cumulative")
|
||||||
|
local current_timestamp=$(date +%s)
|
||||||
|
cumulative_age_hours=$(( (current_timestamp - cumulative_timestamp) / 3600 ))
|
||||||
|
if [ "$cumulative_age_hours" -gt "$MAX_CUMULATIVE_AGE_HOURS" ]; then
|
||||||
|
if [ "$status" != "ERROR" ]; then status="WARNING"; fi
|
||||||
|
warnings+=("CUMULATIVE backup is $cumulative_age_hours hours old (threshold: $MAX_CUMULATIVE_AGE_HOURS)")
|
||||||
|
else
|
||||||
|
cumulative_backup_ok=true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Collect ALL FULL backups
|
||||||
|
local -a full_backups=()
|
||||||
|
local -a full_backup_entries=()
|
||||||
|
if readarray -t full_backups < <(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*FULL*.BKP' -printf '%T@ %p\n' | sort -nr | cut -d' ' -f2-); then
|
||||||
|
for backup_file in "${full_backups[@]}"; do
|
||||||
|
[ -z "$backup_file" ] && continue
|
||||||
|
local backup_name=$(basename "$backup_file")
|
||||||
|
local backup_time=$(date -r "$backup_file" '+%Y-%m-%d %H:%M')
|
||||||
|
local backup_size=$(du -sh "$backup_file" 2>/dev/null | cut -f1)
|
||||||
|
[ -z "$backup_size" ] && backup_size="N/A"
|
||||||
|
full_backup_entries+=("$backup_time | $backup_name | $backup_size")
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Collect ALL INCREMENTAL backups
|
||||||
|
local -a incr_backups=()
|
||||||
|
local -a incr_backup_entries=()
|
||||||
|
if readarray -t incr_backups < <(find "$BACKUP_PATH" -maxdepth 1 -type f \( -name '*INCR*.BKP' -o -name '*INCREMENTAL*.BKP' -o -name '*CUMULATIVE*.BKP' \) -printf '%T@ %p\n' | sort -nr | cut -d' ' -f2-); then
|
||||||
|
for backup_file in "${incr_backups[@]}"; do
|
||||||
|
[ -z "$backup_file" ] && continue
|
||||||
|
local backup_name=$(basename "$backup_file")
|
||||||
|
local backup_time=$(date -r "$backup_file" '+%Y-%m-%d %H:%M')
|
||||||
|
local backup_size=$(du -sh "$backup_file" 2>/dev/null | cut -f1)
|
||||||
|
[ -z "$backup_size" ] && backup_size="N/A"
|
||||||
|
incr_backup_entries+=("$backup_time | $backup_name | $backup_size")
|
||||||
|
done
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
status="ERROR"
|
status="ERROR"
|
||||||
errors+=("No FULL backup found")
|
errors+=("No backup files found in $BACKUP_PATH")
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Check CUMULATIVE backup age
|
local disk_usage_raw=$(df "$BACKUP_PATH" 2>/dev/null | tail -1 | awk '{print int($5)}')
|
||||||
local latest_cumulative=$(ls -t "$BACKUP_PATH"/*INCR*.BKP "$BACKUP_PATH"/*CUMULATIVE*.BKP 2>/dev/null | head -1 || echo "")
|
if [ -n "$disk_usage_raw" ]; then
|
||||||
local cumulative_age_hours="N/A"
|
disk_usage="$disk_usage_raw"
|
||||||
local cumulative_backup_ok=false
|
else
|
||||||
|
if [ "$status" = "OK" ]; then status="WARNING"; fi
|
||||||
if [ -n "$latest_cumulative" ]; then
|
warnings+=("Unable to determine disk usage for $BACKUP_PATH")
|
||||||
local cumulative_timestamp=$(stat -c %Y "$latest_cumulative")
|
|
||||||
local current_timestamp=$(date +%s)
|
|
||||||
cumulative_age_hours=$(( (current_timestamp - cumulative_timestamp) / 3600 ))
|
|
||||||
|
|
||||||
if [ "$cumulative_age_hours" -gt "$MAX_CUMULATIVE_AGE_HOURS" ]; then
|
|
||||||
if [ "$status" != "ERROR" ]; then status="WARNING"; fi
|
|
||||||
warnings+=("CUMULATIVE backup is $cumulative_age_hours hours old (threshold: $MAX_CUMULATIVE_AGE_HOURS)")
|
|
||||||
else
|
|
||||||
cumulative_backup_ok=true
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# Check disk usage
|
if [ "$disk_usage" -gt 90 ]; then
|
||||||
local disk_usage=$(df "$BACKUP_PATH" | tail -1 | awk '{print int($5)}')
|
status="ERROR"
|
||||||
|
errors+=("Disk usage critical: ${disk_usage}%")
|
||||||
|
elif [ "$disk_usage" -gt 80 ]; then
|
||||||
|
if [ "$status" != "ERROR" ]; then status="WARNING"; fi
|
||||||
|
warnings+=("Disk usage high: ${disk_usage}%")
|
||||||
|
fi
|
||||||
|
|
||||||
if [ "$disk_usage" -gt 90 ]; then
|
local severity="info"
|
||||||
status="ERROR"
|
[ "$status" = "WARNING" ] && severity="warning"
|
||||||
errors+=("Disk usage critical: ${disk_usage}%")
|
[ "$status" = "ERROR" ] && severity="error"
|
||||||
elif [ "$disk_usage" -gt 80 ]; then
|
|
||||||
if [ "$status" != "ERROR" ]; then status="WARNING"; fi
|
|
||||||
warnings+=("Disk usage high: ${disk_usage}%")
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Prepare notification data
|
local errors_json
|
||||||
local severity="info"
|
if [ ${#errors[@]} -eq 0 ]; then
|
||||||
[ "$status" = "WARNING" ] && severity="warning"
|
errors_json='[]'
|
||||||
[ "$status" = "ERROR" ] && severity="error"
|
else
|
||||||
|
errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .)
|
||||||
|
fi
|
||||||
|
|
||||||
# Convert arrays to JSON arrays
|
local warnings_json
|
||||||
local errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .)
|
if [ ${#warnings[@]} -eq 0 ]; then
|
||||||
local warnings_json=$(printf '%s\n' "${warnings[@]}" | jq -R . | jq -s .)
|
warnings_json='[]'
|
||||||
local backup_list_json=$(echo "$backup_files" | head -5 | jq -R . | jq -s .)
|
else
|
||||||
|
warnings_json=$(printf '%s\n' "${warnings[@]}" | jq -R . | jq -s .)
|
||||||
|
fi
|
||||||
|
|
||||||
# Create JSON data
|
local full_backup_list_json
|
||||||
local json_data=$(cat <<JSON
|
if [ ${#full_backup_entries[@]} -eq 0 ]; then
|
||||||
|
full_backup_list_json='[]'
|
||||||
|
else
|
||||||
|
full_backup_list_json=$(printf '%s\n' "${full_backup_entries[@]}" | jq -R . | jq -s .)
|
||||||
|
fi
|
||||||
|
|
||||||
|
local incr_backup_list_json
|
||||||
|
if [ ${#incr_backup_entries[@]} -eq 0 ]; then
|
||||||
|
incr_backup_list_json='[]'
|
||||||
|
else
|
||||||
|
incr_backup_list_json=$(printf '%s\n' "${incr_backup_entries[@]}" | jq -R . | jq -s .)
|
||||||
|
fi
|
||||||
|
|
||||||
|
local has_errors=false
|
||||||
|
local has_warnings=false
|
||||||
|
local has_full_backups=false
|
||||||
|
local has_incr_backups=false
|
||||||
|
[ ${#errors[@]} -gt 0 ] && has_errors=true
|
||||||
|
[ ${#warnings[@]} -gt 0 ] && has_warnings=true
|
||||||
|
[ ${#full_backup_entries[@]} -gt 0 ] && has_full_backups=true
|
||||||
|
[ ${#incr_backup_entries[@]} -gt 0 ] && has_incr_backups=true
|
||||||
|
|
||||||
|
local json_data=$(cat <<JSON
|
||||||
{
|
{
|
||||||
"severity": "$severity",
|
"severity": "$severity",
|
||||||
"hostname": "$(hostname)",
|
|
||||||
"node": "$(hostname)",
|
"node": "$(hostname)",
|
||||||
"date": "$(date +'%Y-%m-%d %H:%M:%S')",
|
"date": "$(date +'%Y-%m-%d %H:%M:%S')",
|
||||||
"status": "$status",
|
"status": "$status",
|
||||||
"errors": $errors_json,
|
"errors": $errors_json,
|
||||||
"warnings": $warnings_json,
|
"warnings": $warnings_json,
|
||||||
|
"has_errors": $has_errors,
|
||||||
|
"has_warnings": $has_warnings,
|
||||||
"total_backups": $total_backups,
|
"total_backups": $total_backups,
|
||||||
"total_size_gb": "${total_size%G}",
|
"total_size_gb": "${total_size_label%G}",
|
||||||
"full_backup_age": "$full_age_hours",
|
"total_size_label": "$total_size_label",
|
||||||
"cumulative_backup_age": "$cumulative_age_hours",
|
"full_backup_age": "${full_age_hours}",
|
||||||
"disk_usage": "$disk_usage",
|
"cumulative_backup_age": "${cumulative_age_hours}",
|
||||||
"full_backup_ok": $full_backup_ok,
|
"disk_usage": "${disk_usage}",
|
||||||
"cumulative_backup_ok": $cumulative_backup_ok,
|
"full_backup_ok": $([ "$full_backup_ok" = "true" ] && echo "true" || echo "false"),
|
||||||
|
"cumulative_backup_ok": $([ "$cumulative_backup_ok" = "true" ] && echo "true" || echo "false"),
|
||||||
"is_error": $([ "$status" = "ERROR" ] && echo "true" || echo "false"),
|
"is_error": $([ "$status" = "ERROR" ] && echo "true" || echo "false"),
|
||||||
"is_warning": $([ "$status" = "WARNING" ] && echo "true" || echo "false"),
|
"is_warning": $([ "$status" = "WARNING" ] && echo "true" || echo "false"),
|
||||||
"backup_list": $backup_list_json
|
"full_backup_list": $full_backup_list_json,
|
||||||
|
"incr_backup_list": $incr_backup_list_json,
|
||||||
|
"has_full_backups": $has_full_backups,
|
||||||
|
"has_incr_backups": $has_incr_backups,
|
||||||
|
"full_backup_count": ${#full_backup_entries[@]},
|
||||||
|
"incr_backup_count": ${#incr_backup_entries[@]},
|
||||||
|
"full_backup_limit": "$MAX_FULL_AGE_HOURS",
|
||||||
|
"cumulative_backup_limit": "$MAX_CUMULATIVE_AGE_HOURS"
|
||||||
}
|
}
|
||||||
JSON
|
JSON
|
||||||
)
|
)
|
||||||
|
|
||||||
# Send notification if there are issues
|
if [ "$status" != "OK" ]; then
|
||||||
if [ "$status" != "OK" ]; then
|
echo -e "${YELLOW}Issues detected, sending notification...${NC}"
|
||||||
echo -e "${YELLOW}Issues detected, sending notification...${NC}"
|
send_pve_notification "$severity" "$status" "$json_data"
|
||||||
send_pve_notification "$severity" "$status" "$json_data"
|
else
|
||||||
else
|
echo -e "${GREEN}All backups are healthy${NC}"
|
||||||
echo -e "${GREEN}All backups are healthy${NC}"
|
# Optionally send success notification (uncomment if desired)
|
||||||
# Optionally send success notification (uncomment if desired)
|
# send_pve_notification "info" "$status" "$json_data"
|
||||||
# send_pve_notification "info" "$status" "$json_data"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Display summary
|
|
||||||
echo "Status: $status"
|
|
||||||
echo "Total backups: $total_backups"
|
|
||||||
echo "Total size: $total_size"
|
|
||||||
echo "FULL backup age: $full_age_hours hours"
|
|
||||||
echo "CUMULATIVE backup age: $cumulative_age_hours hours"
|
|
||||||
echo "Disk usage: ${disk_usage}%"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
echo "Status: $status"
|
||||||
|
echo "Total backups: $total_backups"
|
||||||
|
echo "Total size: $total_size_label"
|
||||||
|
echo "FULL backup age: $full_age_hours hours"
|
||||||
|
echo "CUMULATIVE backup age: $cumulative_age_hours hours"
|
||||||
|
echo "Disk usage: ${disk_usage}%"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Main execution
|
# Main execution
|
||||||
|
|||||||
@@ -51,281 +51,150 @@ create_templates() {
|
|||||||
|
|
||||||
# Subject template
|
# Subject template
|
||||||
cat > "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" <<'EOF'
|
cat > "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" <<'EOF'
|
||||||
Oracle DR Test {{severity}} - {{test_result}}
|
Oracle DR Test {{test_result}} | {{date}}
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# Text body template
|
# Text body template
|
||||||
cat > "$TEMPLATE_DIR/oracle-dr-test-body.txt.hbs" <<'EOF'
|
cat > "$TEMPLATE_DIR/oracle-dr-test-body.txt.hbs" <<'EOF'
|
||||||
Oracle DR Weekly Test Report
|
Oracle DR Test {{test_result}} | {{date}}
|
||||||
============================
|
|
||||||
Test Result: {{test_result}}
|
|
||||||
Severity: {{severity}}
|
Severity: {{severity}}
|
||||||
Date: {{date}}
|
|
||||||
Duration: {{total_duration}} minutes
|
|
||||||
|
|
||||||
{{#if is_success}}
|
SUMMARY
|
||||||
✓ TEST PASSED SUCCESSFULLY
|
- Outcome: {{test_result}}
|
||||||
{{else}}
|
- Duration: {{total_duration}} min (restore {{restore_duration}} min)
|
||||||
✗ TEST FAILED
|
- Backups used: {{backup_count}}
|
||||||
{{/if}}
|
- Tables restored: {{tables_restored}}
|
||||||
|
|
||||||
Test Steps Summary:
|
COMPONENTS
|
||||||
-------------------
|
- VM {{vm_id}} ({{vm_ip}}): {{vm_status}}
|
||||||
|
- NFS: {{nfs_status}}
|
||||||
|
- Database: {{database_status}}
|
||||||
|
- Cleanup: {{disk_freed}} GB freed
|
||||||
|
|
||||||
|
STEPS
|
||||||
{{#each test_steps}}
|
{{#each test_steps}}
|
||||||
{{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}}: {{this.status}} ({{this.duration}}s)
|
- {{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}} ({{this.duration}}s){{#if this.status}} - {{this.status}}{{/if}}
|
||||||
{{/each}}
|
{{/each}}
|
||||||
|
|
||||||
{{#if errors}}
|
{{#if has_errors}}
|
||||||
ERRORS:
|
ISSUES
|
||||||
{{#each errors}}
|
{{#each errors}}
|
||||||
- {{this}}
|
- {{this}}
|
||||||
{{/each}}
|
{{/each}}
|
||||||
{{/if}}
|
{{/if}}
|
||||||
|
|
||||||
{{#if warnings}}
|
{{#if has_warnings}}
|
||||||
WARNINGS:
|
WARNINGS
|
||||||
{{#each warnings}}
|
{{#each warnings}}
|
||||||
- {{this}}
|
- {{this}}
|
||||||
{{/each}}
|
{{/each}}
|
||||||
{{/if}}
|
{{/if}}
|
||||||
|
|
||||||
Metrics:
|
RESTORE LOG (first 200 lines)
|
||||||
--------
|
---
|
||||||
- Backup Count: {{backup_count}}
|
{{restore_log}}
|
||||||
- Restore Time: {{restore_duration}} minutes
|
---
|
||||||
- Tables Restored: {{tables_restored}}
|
|
||||||
- Database Status: {{database_status}}
|
|
||||||
- Disk Space Freed: {{disk_freed}} GB
|
|
||||||
|
|
||||||
VM Details:
|
Log: {{log_file}}
|
||||||
-----------
|
Next test: Saturday 06:00
|
||||||
- VM ID: {{vm_id}}
|
|
||||||
- VM IP: {{vm_ip}}
|
|
||||||
- NFS Mount: {{nfs_status}}
|
|
||||||
|
|
||||||
Log File: {{log_file}}
|
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# HTML body template
|
# HTML body template (compact Gmail-friendly layout)
|
||||||
cat > "$TEMPLATE_DIR/oracle-dr-test-body.html.hbs" <<'EOF'
|
cat > "$TEMPLATE_DIR/oracle-dr-test-body.html.hbs" <<'EOF'
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html>
|
<html>
|
||||||
<head>
|
<head>
|
||||||
<style>
|
<meta charset="utf-8">
|
||||||
body { font-family: Arial, sans-serif; }
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
.header {
|
<title>Oracle DR Test {{test_result}} | {{date}}</title>
|
||||||
background-color: {{#if is_success}}#28a745{{else}}#dc3545{{/if}};
|
|
||||||
color: white;
|
|
||||||
padding: 15px;
|
|
||||||
border-radius: 5px;
|
|
||||||
}
|
|
||||||
.section {
|
|
||||||
margin: 20px 0;
|
|
||||||
padding: 15px;
|
|
||||||
background-color: #f8f9fa;
|
|
||||||
border-radius: 5px;
|
|
||||||
}
|
|
||||||
.success { color: #28a745; font-weight: bold; }
|
|
||||||
.error { color: #dc3545; font-weight: bold; }
|
|
||||||
.warning { color: #ffc107; font-weight: bold; }
|
|
||||||
.info { color: #17a2b8; }
|
|
||||||
|
|
||||||
.test-steps {
|
|
||||||
margin: 20px 0;
|
|
||||||
}
|
|
||||||
.step {
|
|
||||||
padding: 10px;
|
|
||||||
margin: 5px 0;
|
|
||||||
border-left: 4px solid;
|
|
||||||
background-color: white;
|
|
||||||
}
|
|
||||||
.step.passed {
|
|
||||||
border-color: #28a745;
|
|
||||||
}
|
|
||||||
.step.failed {
|
|
||||||
border-color: #dc3545;
|
|
||||||
background-color: #f8d7da;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metrics {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
|
||||||
gap: 15px;
|
|
||||||
margin: 20px 0;
|
|
||||||
}
|
|
||||||
.metric-card {
|
|
||||||
background: white;
|
|
||||||
padding: 15px;
|
|
||||||
border-radius: 5px;
|
|
||||||
text-align: center;
|
|
||||||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
|
||||||
}
|
|
||||||
.metric-value {
|
|
||||||
font-size: 24px;
|
|
||||||
font-weight: bold;
|
|
||||||
color: #495057;
|
|
||||||
}
|
|
||||||
.metric-label {
|
|
||||||
font-size: 14px;
|
|
||||||
color: #6c757d;
|
|
||||||
margin-top: 5px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.timeline {
|
|
||||||
position: relative;
|
|
||||||
padding: 20px 0;
|
|
||||||
}
|
|
||||||
.timeline-item {
|
|
||||||
display: flex;
|
|
||||||
margin-bottom: 20px;
|
|
||||||
}
|
|
||||||
.timeline-marker {
|
|
||||||
width: 20px;
|
|
||||||
height: 20px;
|
|
||||||
border-radius: 50%;
|
|
||||||
margin-right: 15px;
|
|
||||||
flex-shrink: 0;
|
|
||||||
}
|
|
||||||
.timeline-marker.success {
|
|
||||||
background-color: #28a745;
|
|
||||||
}
|
|
||||||
.timeline-marker.failed {
|
|
||||||
background-color: #dc3545;
|
|
||||||
}
|
|
||||||
|
|
||||||
table {
|
|
||||||
width: 100%;
|
|
||||||
border-collapse: collapse;
|
|
||||||
margin: 10px 0;
|
|
||||||
}
|
|
||||||
th, td {
|
|
||||||
padding: 10px;
|
|
||||||
text-align: left;
|
|
||||||
border-bottom: 1px solid #dee2e6;
|
|
||||||
}
|
|
||||||
th {
|
|
||||||
background-color: #e9ecef;
|
|
||||||
font-weight: bold;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body style="margin:0;padding:16px;font-family:Arial,Helvetica,sans-serif;background:#ffffff;color:#2c3e50;">
|
||||||
<div class="header">
|
<table style="width:100%;max-width:640px;margin:0 auto;border-collapse:collapse;">
|
||||||
<h1>Oracle DR Test Report</h1>
|
<tr>
|
||||||
<h2>{{#if is_success}}✓ TEST PASSED{{else}}✗ TEST FAILED{{/if}}</h2>
|
<td style="padding:0 0 12px 0;font-size:18px;font-weight:600;">
|
||||||
<p>{{date}} | Duration: {{total_duration}} minutes</p>
|
Oracle DR Test {{test_result}}
|
||||||
</div>
|
</td>
|
||||||
|
</tr>
|
||||||
<div class="section">
|
<tr>
|
||||||
<h3>Test Summary</h3>
|
<td style="padding:0 0 8px 0;font-size:13px;color:#6c757d;">{{date}} · Severity: {{severity}}</td>
|
||||||
<div class="metrics">
|
</tr>
|
||||||
<div class="metric-card">
|
<tr>
|
||||||
<div class="metric-value {{#if is_success}}success{{else}}error{{/if}}">{{test_result}}</div>
|
<td style="padding:12px;border:1px solid #e1e4e8;border-radius:4px;">
|
||||||
<div class="metric-label">Test Result</div>
|
<table style="width:100%;border-collapse:collapse;font-size:14px;">
|
||||||
</div>
|
<tr><td style="padding:4px 0;">Outcome</td><td style="padding:4px 0;text-align:right;">{{test_result}}</td></tr>
|
||||||
<div class="metric-card">
|
<tr><td style="padding:4px 0;">Duration</td><td style="padding:4px 0;text-align:right;">{{total_duration}} min (restore {{restore_duration}} min)</td></tr>
|
||||||
<div class="metric-value">{{restore_duration}}</div>
|
<tr><td style="padding:4px 0;">Backups used</td><td style="padding:4px 0;text-align:right;">{{backup_count}}</td></tr>
|
||||||
<div class="metric-label">Restore Time (min)</div>
|
<tr><td style="padding:4px 0;">Tables restored</td><td style="padding:4px 0;text-align:right;">{{tables_restored}}</td></tr>
|
||||||
</div>
|
|
||||||
<div class="metric-card">
|
|
||||||
<div class="metric-value">{{tables_restored}}</div>
|
|
||||||
<div class="metric-label">Tables Restored</div>
|
|
||||||
</div>
|
|
||||||
<div class="metric-card">
|
|
||||||
<div class="metric-value">{{backup_count}}</div>
|
|
||||||
<div class="metric-label">Backups Used</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="section">
|
|
||||||
<h3>Test Steps Timeline</h3>
|
|
||||||
<div class="timeline">
|
|
||||||
{{#each test_steps}}
|
|
||||||
<div class="timeline-item">
|
|
||||||
<div class="timeline-marker {{#if this.passed}}success{{else}}failed{{/if}}"></div>
|
|
||||||
<div style="flex-grow: 1;">
|
|
||||||
<div class="step {{#if this.passed}}passed{{else}}failed{{/if}}">
|
|
||||||
<strong>{{this.name}}</strong>
|
|
||||||
<span style="float: right; color: #6c757d;">{{this.duration}}s</span>
|
|
||||||
<div style="margin-top: 5px;">
|
|
||||||
{{#if this.passed}}
|
|
||||||
<span class="success">✓ {{this.status}}</span>
|
|
||||||
{{else}}
|
|
||||||
<span class="error">✗ {{this.status}}</span>
|
|
||||||
{{/if}}
|
|
||||||
</div>
|
|
||||||
{{#if this.details}}
|
|
||||||
<div style="margin-top: 5px; font-size: 0.9em; color: #6c757d;">
|
|
||||||
{{this.details}}
|
|
||||||
</div>
|
|
||||||
{{/if}}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{{/each}}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{{#if errors}}
|
|
||||||
<div class="section" style="background-color: #f8d7da;">
|
|
||||||
<h3 class="error">Errors Encountered</h3>
|
|
||||||
<ul>
|
|
||||||
{{#each errors}}
|
|
||||||
<li>{{this}}</li>
|
|
||||||
{{/each}}
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
{{#if warnings}}
|
|
||||||
<div class="section" style="background-color: #fff3cd;">
|
|
||||||
<h3 class="warning">Warnings</h3>
|
|
||||||
<ul>
|
|
||||||
{{#each warnings}}
|
|
||||||
<li>{{this}}</li>
|
|
||||||
{{/each}}
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
{{/if}}
|
|
||||||
|
|
||||||
<div class="section">
|
|
||||||
<h3>System Details</h3>
|
|
||||||
<table>
|
|
||||||
<tr>
|
|
||||||
<th>Component</th>
|
|
||||||
<th>Value</th>
|
|
||||||
<th>Status</th>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>DR VM</td>
|
|
||||||
<td>ID: {{vm_id}} ({{vm_ip}})</td>
|
|
||||||
<td>{{vm_status}}</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>NFS Mount</td>
|
|
||||||
<td>F:\ drive</td>
|
|
||||||
<td>{{nfs_status}}</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>Database</td>
|
|
||||||
<td>ROA</td>
|
|
||||||
<td>{{database_status}}</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>Disk Space Freed</td>
|
|
||||||
<td>{{disk_freed}} GB</td>
|
|
||||||
<td class="success">✓</td>
|
|
||||||
</tr>
|
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</td>
|
||||||
|
</tr>
|
||||||
|
|
||||||
<div class="section">
|
<tr>
|
||||||
<p class="info">
|
<td style="padding:16px 0 0 0;">
|
||||||
<strong>Log File:</strong> {{log_file}}<br>
|
<table style="width:100%;border-collapse:collapse;font-size:14px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
|
||||||
<strong>Next Scheduled Test:</strong> Next Saturday 06:00
|
<tr><td style="padding:8px 12px;font-weight:600;">Components</td></tr>
|
||||||
</p>
|
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">VM {{vm_id}} ({{vm_ip}}): {{vm_status}}</td></tr>
|
||||||
</div>
|
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">NFS: {{nfs_status}}</td></tr>
|
||||||
|
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">Database: {{database_status}}</td></tr>
|
||||||
|
<tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">Cleanup: {{disk_freed}} GB freed</td></tr>
|
||||||
|
</table>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
|
||||||
|
<tr>
|
||||||
|
<td style="padding:16px 0 0 0;">
|
||||||
|
<table style="width:100%;border-collapse:collapse;font-size:14px;">
|
||||||
|
<tr><td style="padding:0 0 6px 0;font-weight:600;">Steps</td></tr>
|
||||||
|
{{#each test_steps}}
|
||||||
|
<tr>
|
||||||
|
<td style="padding:4px 0;border-bottom:1px solid #f1f1f1;">{{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}} ({{this.duration}}s){{#if this.status}} – {{this.status}}{{/if}}</td>
|
||||||
|
</tr>
|
||||||
|
{{/each}}
|
||||||
|
</table>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
|
||||||
|
{{#if has_errors}}
|
||||||
|
<tr>
|
||||||
|
<td style="padding:16px 0 0 0;">
|
||||||
|
<table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff5f5;border:1px solid #f1b0b7;border-radius:4px;">
|
||||||
|
<tr><td style="padding:8px 12px;font-weight:600;color:#c82333;">Issues</td></tr>
|
||||||
|
{{#each errors}}
|
||||||
|
<tr><td style="padding:6px 12px;border-top:1px solid #f8d7da;">• {{this}}</td></tr>
|
||||||
|
{{/each}}
|
||||||
|
</table>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{{/if}}
|
||||||
|
|
||||||
|
{{#if has_warnings}}
|
||||||
|
<tr>
|
||||||
|
<td style="padding:16px 0 0 0;">
|
||||||
|
<table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff8e5;border:1px solid #ffe8a1;border-radius:4px;">
|
||||||
|
<tr><td style="padding:8px 12px;font-weight:600;color:#856404;">Warnings</td></tr>
|
||||||
|
{{#each warnings}}
|
||||||
|
<tr><td style="padding:6px 12px;border-top:1px solid #ffe8a1;">• {{this}}</td></tr>
|
||||||
|
{{/each}}
|
||||||
|
</table>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{{/if}}
|
||||||
|
|
||||||
|
<tr>
|
||||||
|
<td style="padding:16px 0 0 0;">
|
||||||
|
<table style="width:100%;border-collapse:collapse;font-size:12px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
|
||||||
|
<tr><td style="padding:8px 12px;font-weight:600;font-size:13px;">Restore Log (first 200 lines)</td></tr>
|
||||||
|
<tr><td style="padding:8px 12px;font-family:monospace;white-space:pre-wrap;word-wrap:break-word;border-top:1px solid #e1e4e8;">{{restore_log}}</td></tr>
|
||||||
|
</table>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
|
||||||
|
<tr>
|
||||||
|
<td style="padding:16px 0 0 0;font-size:12px;color:#6c757d;">
|
||||||
|
Log: {{log_file}} · Next test: Saturday 06:00
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</table>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
EOF
|
EOF
|
||||||
@@ -417,7 +286,16 @@ track_step() {
|
|||||||
local end_time=$(date +%s)
|
local end_time=$(date +%s)
|
||||||
local duration=$((end_time - start_time))
|
local duration=$((end_time - start_time))
|
||||||
|
|
||||||
TEST_STEPS+=("{\"name\":\"$name\",\"passed\":$passed,\"status\":\"$status\",\"duration\":$duration}")
|
local step_json
|
||||||
|
step_json=$(jq -n \
|
||||||
|
--arg name "$name" \
|
||||||
|
--arg status "$status" \
|
||||||
|
--arg duration "$duration" \
|
||||||
|
--arg passed "$passed" \
|
||||||
|
'{name:$name, status:$status, duration:($duration|tonumber), passed:($passed == "true")}'
|
||||||
|
)
|
||||||
|
|
||||||
|
TEST_STEPS+=("$step_json")
|
||||||
|
|
||||||
if [ "$passed" = "false" ]; then
|
if [ "$passed" = "false" ]; then
|
||||||
ERRORS+=("$name: $status")
|
ERRORS+=("$name: $status")
|
||||||
@@ -429,6 +307,14 @@ run_dr_test() {
|
|||||||
local test_result="FAILED"
|
local test_result="FAILED"
|
||||||
local severity="error"
|
local severity="error"
|
||||||
local is_success=false
|
local is_success=false
|
||||||
|
local restore_duration=0
|
||||||
|
local tables_restored=0
|
||||||
|
local db_status="UNKNOWN"
|
||||||
|
local nfs_status="Not checked"
|
||||||
|
local vm_status_label="Not started"
|
||||||
|
local cleanup_freed=0
|
||||||
|
local backup_count=0
|
||||||
|
local restore_log="Not collected"
|
||||||
|
|
||||||
log "=========================================="
|
log "=========================================="
|
||||||
log "Oracle DR Weekly Test - Starting"
|
log "Oracle DR Weekly Test - Starting"
|
||||||
@@ -439,7 +325,7 @@ run_dr_test() {
|
|||||||
log "STEP 1: Pre-flight checks"
|
log "STEP 1: Pre-flight checks"
|
||||||
|
|
||||||
# Check backups exist
|
# Check backups exist
|
||||||
local backup_count=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l || echo "0")
|
backup_count=$(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*.BKP' 2>/dev/null | wc -l)
|
||||||
|
|
||||||
if [ "$backup_count" -lt 2 ]; then
|
if [ "$backup_count" -lt 2 ]; then
|
||||||
track_step "Pre-flight checks" false "Insufficient backups (found: $backup_count)" "$step_start"
|
track_step "Pre-flight checks" false "Insufficient backups (found: $backup_count)" "$step_start"
|
||||||
@@ -452,6 +338,7 @@ run_dr_test() {
|
|||||||
log "STEP 2: Starting DR VM"
|
log "STEP 2: Starting DR VM"
|
||||||
|
|
||||||
if qm start "$DR_VM_ID" 2>/dev/null; then
|
if qm start "$DR_VM_ID" 2>/dev/null; then
|
||||||
|
vm_status_label="Running"
|
||||||
sleep 180 # Wait for boot
|
sleep 180 # Wait for boot
|
||||||
track_step "VM Startup" true "VM $DR_VM_ID started" "$step_start"
|
track_step "VM Startup" true "VM $DR_VM_ID started" "$step_start"
|
||||||
|
|
||||||
@@ -459,7 +346,7 @@ run_dr_test() {
|
|||||||
step_start=$(date +%s)
|
step_start=$(date +%s)
|
||||||
log "STEP 3: Verifying NFS mount"
|
log "STEP 3: Verifying NFS mount"
|
||||||
|
|
||||||
local nfs_status="Not Mounted"
|
nfs_status="Not Mounted"
|
||||||
if ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 "$DR_VM_USER@$DR_VM_IP" \
|
if ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 "$DR_VM_USER@$DR_VM_IP" \
|
||||||
"powershell -Command 'Test-Path F:\\ROA\\autobackup'" 2>/dev/null; then
|
"powershell -Command 'Test-Path F:\\ROA\\autobackup'" 2>/dev/null; then
|
||||||
nfs_status="Mounted"
|
nfs_status="Mounted"
|
||||||
@@ -478,7 +365,7 @@ run_dr_test() {
|
|||||||
"D:\\oracle\\scripts\\rman_restore_from_zero.cmd" 2>&1 | tee -a "$LOG_FILE"; then
|
"D:\\oracle\\scripts\\rman_restore_from_zero.cmd" 2>&1 | tee -a "$LOG_FILE"; then
|
||||||
|
|
||||||
local restore_end=$(date +%s)
|
local restore_end=$(date +%s)
|
||||||
local restore_duration=$(( (restore_end - restore_start) / 60 ))
|
restore_duration=$(( (restore_end - restore_start) / 60 ))
|
||||||
|
|
||||||
track_step "Database Restore" true "Restored in $restore_duration minutes" "$step_start"
|
track_step "Database Restore" true "Restored in $restore_duration minutes" "$step_start"
|
||||||
|
|
||||||
@@ -486,11 +373,13 @@ run_dr_test() {
|
|||||||
step_start=$(date +%s)
|
step_start=$(date +%s)
|
||||||
log "STEP 5: Verifying database"
|
log "STEP 5: Verifying database"
|
||||||
|
|
||||||
local db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||||||
"cmd /c 'echo SELECT STATUS FROM V\$INSTANCE; | sqlplus -s / as sysdba' | findstr OPEN" || echo "")
|
"cmd /c 'echo SELECT STATUS FROM V\$INSTANCE; | sqlplus -s / as sysdba' | findstr OPEN" || echo "")
|
||||||
|
|
||||||
local tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||||||
"cmd /c 'echo SELECT COUNT(*) FROM DBA_TABLES WHERE OWNER NOT IN (''SYS'',''SYSTEM''); | sqlplus -s / as sysdba' | grep -o '[0-9]*' | tail -1" || echo "0")
|
"cmd /c 'echo SELECT COUNT(*) FROM DBA_TABLES WHERE OWNER NOT IN (''SYS'',''SYSTEM''); | sqlplus -s / as sysdba' | grep -o '[0-9]*' | tail -1" || echo "0")
|
||||||
|
tables_restored=$(echo "$tables_restored" | tr -cd '0-9')
|
||||||
|
[ -z "$tables_restored" ] && tables_restored=0
|
||||||
|
|
||||||
if [[ "$db_status" =~ "OPEN" ]]; then
|
if [[ "$db_status" =~ "OPEN" ]]; then
|
||||||
track_step "Database Verification" true "Database OPEN, $tables_restored tables" "$step_start"
|
track_step "Database Verification" true "Database OPEN, $tables_restored tables" "$step_start"
|
||||||
@@ -501,6 +390,11 @@ run_dr_test() {
|
|||||||
track_step "Database Verification" false "Database not OPEN" "$step_start"
|
track_step "Database Verification" false "Database not OPEN" "$step_start"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Collect restore log from VM
|
||||||
|
log "Collecting restore log from DR VM..."
|
||||||
|
restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||||||
|
"type D:\\oracle\\logs\\restore_from_zero.log 2>nul" | head -200 || echo "Log not available")
|
||||||
|
|
||||||
# Step 6: Cleanup
|
# Step 6: Cleanup
|
||||||
step_start=$(date +%s)
|
step_start=$(date +%s)
|
||||||
log "STEP 6: Running cleanup"
|
log "STEP 6: Running cleanup"
|
||||||
@@ -508,7 +402,8 @@ run_dr_test() {
|
|||||||
ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
|
||||||
"D:\\oracle\\scripts\\cleanup_database.cmd" 2>/dev/null
|
"D:\\oracle\\scripts\\cleanup_database.cmd" 2>/dev/null
|
||||||
|
|
||||||
track_step "Cleanup" true "Database cleaned, ~8GB freed" "$step_start"
|
cleanup_freed=8
|
||||||
|
track_step "Cleanup" true "Database cleaned, ~${cleanup_freed}GB freed" "$step_start"
|
||||||
|
|
||||||
else
|
else
|
||||||
track_step "Database Restore" false "Restore failed" "$step_start"
|
track_step "Database Restore" false "Restore failed" "$step_start"
|
||||||
@@ -523,9 +418,11 @@ run_dr_test() {
|
|||||||
qm stop "$DR_VM_ID" 2>/dev/null
|
qm stop "$DR_VM_ID" 2>/dev/null
|
||||||
|
|
||||||
track_step "VM Shutdown" true "VM stopped" "$step_start"
|
track_step "VM Shutdown" true "VM stopped" "$step_start"
|
||||||
|
vm_status_label="Stopped"
|
||||||
|
|
||||||
else
|
else
|
||||||
track_step "VM Startup" false "Failed to start VM $DR_VM_ID" "$step_start"
|
track_step "VM Startup" false "Failed to start VM $DR_VM_ID" "$step_start"
|
||||||
|
vm_status_label="Failed to start"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -534,9 +431,41 @@ run_dr_test() {
|
|||||||
local total_duration=$(( (test_end_time - TEST_START_TIME) / 60 ))
|
local total_duration=$(( (test_end_time - TEST_START_TIME) / 60 ))
|
||||||
|
|
||||||
# Prepare notification data
|
# Prepare notification data
|
||||||
local steps_json=$(printf '%s,' "${TEST_STEPS[@]}" | sed 's/,$//')
|
local steps_json
|
||||||
local errors_json=$(printf '"%s",' "${ERRORS[@]}" | sed 's/,$//')
|
if [ ${#TEST_STEPS[@]} -eq 0 ]; then
|
||||||
local warnings_json=$(printf '"%s",' "${WARNINGS[@]}" | sed 's/,$//')
|
steps_json='[]'
|
||||||
|
else
|
||||||
|
steps_json=$(printf '%s\n' "${TEST_STEPS[@]}" | jq -s '.')
|
||||||
|
fi
|
||||||
|
|
||||||
|
local errors_json
|
||||||
|
if [ ${#ERRORS[@]} -eq 0 ]; then
|
||||||
|
errors_json='[]'
|
||||||
|
else
|
||||||
|
errors_json=$(printf '%s\n' "${ERRORS[@]}" | jq -R . | jq -s .)
|
||||||
|
fi
|
||||||
|
|
||||||
|
local warnings_json
|
||||||
|
if [ ${#WARNINGS[@]} -eq 0 ]; then
|
||||||
|
warnings_json='[]'
|
||||||
|
else
|
||||||
|
warnings_json=$(printf '%s\n' "${WARNINGS[@]}" | jq -R . | jq -s .)
|
||||||
|
fi
|
||||||
|
|
||||||
|
local has_errors=false
|
||||||
|
local has_warnings=false
|
||||||
|
[ ${#ERRORS[@]} -gt 0 ] && has_errors=true
|
||||||
|
[ ${#WARNINGS[@]} -gt 0 ] && has_warnings=true
|
||||||
|
|
||||||
|
if [ "$is_success" = true ] && [ "$has_warnings" = true ]; then
|
||||||
|
severity="warning"
|
||||||
|
fi
|
||||||
|
|
||||||
|
local db_status_clean=$(echo "$db_status" | tr -d '\r' | sed 's/^ *//;s/ *$//')
|
||||||
|
|
||||||
|
# Escape restore log for JSON
|
||||||
|
local restore_log_json
|
||||||
|
restore_log_json=$(echo "$restore_log" | jq -Rs .)
|
||||||
|
|
||||||
local json_data=$(cat <<JSON
|
local json_data=$(cat <<JSON
|
||||||
{
|
{
|
||||||
@@ -545,19 +474,22 @@ run_dr_test() {
|
|||||||
"date": "$(date '+%Y-%m-%d %H:%M:%S')",
|
"date": "$(date '+%Y-%m-%d %H:%M:%S')",
|
||||||
"total_duration": $total_duration,
|
"total_duration": $total_duration,
|
||||||
"is_success": $is_success,
|
"is_success": $is_success,
|
||||||
"test_steps": [$steps_json],
|
"has_errors": $has_errors,
|
||||||
"errors": [${errors_json:-}],
|
"has_warnings": $has_warnings,
|
||||||
"warnings": [${warnings_json:-}],
|
"test_steps": $steps_json,
|
||||||
"backup_count": ${backup_count:-0},
|
"errors": $errors_json,
|
||||||
"restore_duration": ${restore_duration:-0},
|
"warnings": $warnings_json,
|
||||||
|
"backup_count": $backup_count,
|
||||||
|
"restore_duration": $restore_duration,
|
||||||
"tables_restored": ${tables_restored:-0},
|
"tables_restored": ${tables_restored:-0},
|
||||||
"database_status": "${db_status:-UNKNOWN}",
|
"database_status": "${db_status_clean:-UNKNOWN}",
|
||||||
"disk_freed": 8,
|
"disk_freed": $cleanup_freed,
|
||||||
"vm_id": "$DR_VM_ID",
|
"vm_id": "$DR_VM_ID",
|
||||||
"vm_ip": "$DR_VM_IP",
|
"vm_ip": "$DR_VM_IP",
|
||||||
"vm_status": "Stopped",
|
"vm_status": "$vm_status_label",
|
||||||
"nfs_status": "${nfs_status:-Unknown}",
|
"nfs_status": "${nfs_status:-Unknown}",
|
||||||
"log_file": "$LOG_FILE"
|
"log_file": "$LOG_FILE",
|
||||||
|
"restore_log": $restore_log_json
|
||||||
}
|
}
|
||||||
JSON
|
JSON
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user