Oracle DR: Enhance notification templates with compact HTML layouts and improved data collection

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
2025-10-10 22:41:32 +03:00
parent 6f56e61b04
commit 839f1b6b82
3 changed files with 499 additions and 403 deletions
--- a/oracle/standby-server-scripts/oracle-backup-monitor-proxmox.sh
+++ b/oracle/standby-server-scripts/oracle-backup-monitor-proxmox.sh
@@ -46,81 +46,167 @@ create_templates() {

    # Subject template
    cat > "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" <<'EOF'
-Oracle Backup {{severity}} - {{node}}
+Oracle Backup {{status}} | {{node}}
 EOF

    # Text body template
    cat > "$TEMPLATE_DIR/oracle-backup-body.txt.hbs" <<'EOF'
-Oracle Backup {{severity}} - {{node}}
-{{status}}
+Oracle Backup {{status}} | {{node}}
+Date: {{date}}

-========================================
-{{#if errors}}
-CRITICAL ISSUES:
+SUMMARY
+- Full backup: {{full_backup_age}}h (limit {{full_backup_limit}}h) -> {{#if full_backup_ok}}OK{{else}}CHECK{{/if}}
+- Incremental: {{cumulative_backup_age}}h (limit {{cumulative_backup_limit}}h) -> {{#if cumulative_backup_ok}}OK{{else}}CHECK{{/if}}
+- Backups: {{total_backups}} files ({{total_size_label}})
+- Disk usage: {{disk_usage}}%
+
+{{#if has_errors}}
+ISSUES
 {{#each errors}}
 - {{this}}
 {{/each}}
 {{/if}}

-{{#if warnings}}
-WARNINGS:
+{{#if has_warnings}}
+WARNINGS
 {{#each warnings}}
 - {{this}}
 {{/each}}
 {{/if}}

-========================================
-BACKUP STATUS:
-FULL: {{full_backup_age}}h old {{#if full_backup_ok}}OK{{else}}TOO OLD{{/if}} (limit: 25h)
-CUMULATIVE: {{cumulative_backup_age}}h old {{#if cumulative_backup_ok}}OK{{else}}TOO OLD{{/if}} (limit: 7h)
-Total: {{total_backups}} files | Size: {{total_size_gb}}GB | Disk: {{disk_usage}}%
-
-{{#if backup_list}}
-LATEST BACKUPS (last 5):
-{{#each backup_list}}
+FULL BACKUPS ({{full_backup_count}} files)
+{{#if has_full_backups}}
+{{#each full_backup_list}}
 - {{this}}
 {{/each}}
+{{else}}
+- none detected
 {{/if}}

-========================================
-Next check: {{date}} + 24h | Proxmox Monitoring
+INCREMENTAL BACKUPS ({{incr_backup_count}} files)
+{{#if has_incr_backups}}
+{{#each incr_backup_list}}
+- {{this}}
+{{/each}}
+{{else}}
+- none detected
+{{/if}}
+
+Next check: +24h via Proxmox Monitor
 EOF

-    # HTML body template (identical to text for compatibility)
+    # HTML body template (lightweight Gmail-friendly)
    cat > "$TEMPLATE_DIR/oracle-backup-body.html.hbs" <<'EOF'
-Oracle Backup {{severity}} - {{node}}
-{{status}}
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Oracle Backup {{status}} | {{node}}</title>
+</head>
+<body style="margin:0;padding:16px;font-family:Arial,Helvetica,sans-serif;background:#ffffff;color:#2c3e50;">
+  <table style="width:100%;max-width:640px;margin:0 auto;border-collapse:collapse;">
+    <tr>
+      <td style="padding:0 0 12px 0;font-size:18px;font-weight:600;">
+        Oracle Backup {{status}} | {{node}}
+      </td>
+    </tr>
+    <tr>
+      <td style="padding:0 0 16px 0;font-size:13px;color:#6c757d;">
+        {{date}}
+      </td>
+    </tr>
+    <tr>
+      <td style="padding:12px;border:1px solid #e1e4e8;border-radius:4px;">
+        <table style="width:100%;border-collapse:collapse;font-size:14px;">
+          <tr>
+            <td style="padding:4px 0;">Full backup</td>
+            <td style="padding:4px 0;text-align:right;">
+              {{full_backup_age}}h / {{full_backup_limit}}h · {{#if full_backup_ok}}OK{{else}}CHECK{{/if}}
+            </td>
+          </tr>
+          <tr>
+            <td style="padding:4px 0;">Incremental</td>
+            <td style="padding:4px 0;text-align:right;">
+              {{cumulative_backup_age}}h / {{cumulative_backup_limit}}h · {{#if cumulative_backup_ok}}OK{{else}}CHECK{{/if}}
+            </td>
+          </tr>
+          <tr>
+            <td style="padding:4px 0;">Backups</td>
+            <td style="padding:4px 0;text-align:right;">{{total_backups}} files ({{total_size_label}})</td>
+          </tr>
+          <tr>
+            <td style="padding:4px 0;">Disk usage</td>
+            <td style="padding:4px 0;text-align:right;">{{disk_usage}}%</td>
+          </tr>
+        </table>
+      </td>
+    </tr>

-========================================
-{{#if errors}}
-CRITICAL ISSUES:
-{{#each errors}}
- {{this}}
-{{/each}}
-{{/if}}
+    {{#if has_errors}}
+    <tr>
+      <td style="padding:16px 0 0 0;">
+        <table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff5f5;border:1px solid #f1b0b7;border-radius:4px;">
+          <tr><td style="padding:8px 12px;font-weight:600;color:#c82333;">Issues</td></tr>
+          {{#each errors}}
+          <tr><td style="padding:6px 12px;border-top:1px solid #f8d7da;">• {{this}}</td></tr>
+          {{/each}}
+        </table>
+      </td>
+    </tr>
+    {{/if}}

-{{#if warnings}}
-WARNINGS:
-{{#each warnings}}
- {{this}}
-{{/each}}
-{{/if}}
+    {{#if has_warnings}}
+    <tr>
+      <td style="padding:16px 0 0 0;">
+        <table style="width:100%;border-collapse:collapse;font-size:14px;background:#fff8e5;border:1px solid #ffe8a1;border-radius:4px;">
+          <tr><td style="padding:8px 12px;font-weight:600;color:#856404;">Warnings</td></tr>
+          {{#each warnings}}
+          <tr><td style="padding:6px 12px;border-top:1px solid #ffe8a1;">• {{this}}</td></tr>
+          {{/each}}
+        </table>
+      </td>
+    </tr>
+    {{/if}}

-========================================
-BACKUP STATUS:
-FULL: {{full_backup_age}}h old {{#if full_backup_ok}}OK{{else}}TOO OLD{{/if}} (limit: 25h)
-CUMULATIVE: {{cumulative_backup_age}}h old {{#if cumulative_backup_ok}}OK{{else}}TOO OLD{{/if}} (limit: 7h)
-Total: {{total_backups}} files | Size: {{total_size_gb}}GB | Disk: {{disk_usage}}%
+    <tr>
+      <td style="padding:16px 0 0 0;">
+        <table style="width:100%;border-collapse:collapse;font-size:13px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
+          <tr><td style="padding:8px 12px;font-weight:600;">FULL Backups ({{full_backup_count}} files)</td></tr>
+          {{#if has_full_backups}}
+            {{#each full_backup_list}}
+            <tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">• {{this}}</td></tr>
+            {{/each}}
+          {{else}}
+            <tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">• none detected</td></tr>
+          {{/if}}
+        </table>
+      </td>
+    </tr>

-{{#if backup_list}}
-LATEST BACKUPS (last 5):
-{{#each backup_list}}
- {{this}}
-{{/each}}
-{{/if}}
+    <tr>
+      <td style="padding:16px 0 0 0;">
+        <table style="width:100%;border-collapse:collapse;font-size:13px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
+          <tr><td style="padding:8px 12px;font-weight:600;">INCREMENTAL Backups ({{incr_backup_count}} files)</td></tr>
+          {{#if has_incr_backups}}
+            {{#each incr_backup_list}}
+            <tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">• {{this}}</td></tr>
+            {{/each}}
+          {{else}}
+            <tr><td style="padding:6px 12px;border-top:1px solid #e1e4e8;">• none detected</td></tr>
+          {{/if}}
+        </table>
+      </td>
+    </tr>

-========================================
-Next check: {{date}} + 24h | Proxmox Monitoring
+    <tr>
+      <td style="padding:16px 0 0 0;font-size:12px;color:#6c757d;">
+        Next automated check: +24h via Proxmox Monitor
+      </td>
+    </tr>
+  </table>
+</body>
+</html>
 EOF

    echo -e "${GREEN}Templates created successfully in $TEMPLATE_DIR${NC}"
@@ -150,7 +236,7 @@ my $template_name = 'oracle-backup';
 my $fields = {
    type => 'oracle-backup',
    severity => $severity,
-    hostname => $data->{hostname},
+    hostname => $data->{node} // 'unknown',
 };

 # Send notification
@@ -187,79 +273,147 @@ check_backups() {

    echo "Checking Oracle backups..."

-    # Get backup list
-    local backup_files=$(ls -lth "$BACKUP_PATH"/*.BKP 2>/dev/null | head -10 || echo "")
+    local total_backups=0
+    local total_size_label="0G"
+    local full_age_hours="N/A"
+    local cumulative_age_hours="N/A"
+    local full_backup_ok=false
+    local cumulative_backup_ok=false
+    local disk_usage=0
+    local -a backup_entries=()

-    if [ -z "$backup_files" ]; then
+    if [ ! -d "$BACKUP_PATH" ]; then
        status="ERROR"
-        errors+=("No backup files found in $BACKUP_PATH")
+        errors+=("Backup path $BACKUP_PATH not accessible")
    else
-        # Count backups
-        local total_backups=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l)
-        local total_size=$(du -shc "$BACKUP_PATH"/*.BKP 2>/dev/null | tail -1 | awk '{print $1}')
+        if compgen -G "$BACKUP_PATH"/*.BKP > /dev/null; then
+            total_backups=$(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*.BKP' | wc -l)
+            total_backups=${total_backups//[[:space:]]/}
+            [ -z "$total_backups" ] && total_backups=0
+            local total_size=$(du -shc "$BACKUP_PATH"/*.BKP 2>/dev/null | tail -1 | awk '{print $1}')
+            [ -z "$total_size" ] && total_size="0G"
+            total_size_label="$total_size"

-        # Check FULL backup age
-        local latest_full=$(ls -t "$BACKUP_PATH"/*FULL*.BKP 2>/dev/null | head -1 || echo "")
-        local full_age_hours="N/A"
-        local full_backup_ok=false
-
-        if [ -n "$latest_full" ]; then
-            local full_timestamp=$(stat -c %Y "$latest_full")
-            local current_timestamp=$(date +%s)
-            full_age_hours=$(( (current_timestamp - full_timestamp) / 3600 ))
-
-            if [ "$full_age_hours" -gt "$MAX_FULL_AGE_HOURS" ]; then
-                status="WARNING"
-                warnings+=("FULL backup is $full_age_hours hours old (threshold: $MAX_FULL_AGE_HOURS)")
+            local latest_full=$(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*FULL*.BKP' -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)
+            if [ -n "$latest_full" ]; then
+                local full_timestamp=$(stat -c %Y "$latest_full")
+                local current_timestamp=$(date +%s)
+                full_age_hours=$(( (current_timestamp - full_timestamp) / 3600 ))
+                if [ "$full_age_hours" -gt "$MAX_FULL_AGE_HOURS" ]; then
+                    status="WARNING"
+                    warnings+=("FULL backup is $full_age_hours hours old (threshold: $MAX_FULL_AGE_HOURS)")
+                else
+                    full_backup_ok=true
+                fi
            else
-                full_backup_ok=true
+                status="ERROR"
+                errors+=("No FULL backup found")
+            fi
+
+            local latest_cumulative=$(find "$BACKUP_PATH" -maxdepth 1 -type f \( -name '*INCR*.BKP' -o -name '*INCREMENTAL*.BKP' -o -name '*CUMULATIVE*.BKP' \) -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)
+            if [ -n "$latest_cumulative" ]; then
+                local cumulative_timestamp=$(stat -c %Y "$latest_cumulative")
+                local current_timestamp=$(date +%s)
+                cumulative_age_hours=$(( (current_timestamp - cumulative_timestamp) / 3600 ))
+                if [ "$cumulative_age_hours" -gt "$MAX_CUMULATIVE_AGE_HOURS" ]; then
+                    if [ "$status" != "ERROR" ]; then status="WARNING"; fi
+                    warnings+=("CUMULATIVE backup is $cumulative_age_hours hours old (threshold: $MAX_CUMULATIVE_AGE_HOURS)")
+                else
+                    cumulative_backup_ok=true
+                fi
+            fi
+
+            # Collect ALL FULL backups
+            local -a full_backups=()
+            local -a full_backup_entries=()
+            if readarray -t full_backups < <(find "$BACKUP_PATH" -maxdepth 1 -type f -name '*FULL*.BKP' -printf '%T@ %p\n' | sort -nr | cut -d' ' -f2-); then
+                for backup_file in "${full_backups[@]}"; do
+                    [ -z "$backup_file" ] && continue
+                    local backup_name=$(basename "$backup_file")
+                    local backup_time=$(date -r "$backup_file" '+%Y-%m-%d %H:%M')
+                    local backup_size=$(du -sh "$backup_file" 2>/dev/null | cut -f1)
+                    [ -z "$backup_size" ] && backup_size="N/A"
+                    full_backup_entries+=("$backup_time | $backup_name | $backup_size")
+                done
+            fi
+
+            # Collect ALL INCREMENTAL backups
+            local -a incr_backups=()
+            local -a incr_backup_entries=()
+            if readarray -t incr_backups < <(find "$BACKUP_PATH" -maxdepth 1 -type f \( -name '*INCR*.BKP' -o -name '*INCREMENTAL*.BKP' -o -name '*CUMULATIVE*.BKP' \) -printf '%T@ %p\n' | sort -nr | cut -d' ' -f2-); then
+                for backup_file in "${incr_backups[@]}"; do
+                    [ -z "$backup_file" ] && continue
+                    local backup_name=$(basename "$backup_file")
+                    local backup_time=$(date -r "$backup_file" '+%Y-%m-%d %H:%M')
+                    local backup_size=$(du -sh "$backup_file" 2>/dev/null | cut -f1)
+                    [ -z "$backup_size" ] && backup_size="N/A"
+                    incr_backup_entries+=("$backup_time | $backup_name | $backup_size")
+                done
            fi
        else
            status="ERROR"
-            errors+=("No FULL backup found")
+            errors+=("No backup files found in $BACKUP_PATH")
        fi

-        # Check CUMULATIVE backup age
-        local latest_cumulative=$(ls -t "$BACKUP_PATH"/*INCR*.BKP "$BACKUP_PATH"/*INCREMENTAL*.BKP "$BACKUP_PATH"/*CUMULATIVE*.BKP 2>/dev/null | head -1 || echo "")
-        local cumulative_age_hours="N/A"
-        local cumulative_backup_ok=false
-
-        if [ -n "$latest_cumulative" ]; then
-            local cumulative_timestamp=$(stat -c %Y "$latest_cumulative")
-            local current_timestamp=$(date +%s)
-            cumulative_age_hours=$(( (current_timestamp - cumulative_timestamp) / 3600 ))
-
-            if [ "$cumulative_age_hours" -gt "$MAX_CUMULATIVE_AGE_HOURS" ]; then
-                if [ "$status" != "ERROR" ]; then status="WARNING"; fi
-                warnings+=("CUMULATIVE backup is $cumulative_age_hours hours old (threshold: $MAX_CUMULATIVE_AGE_HOURS)")
-            else
-                cumulative_backup_ok=true
-            fi
+        local disk_usage_raw=$(df "$BACKUP_PATH" 2>/dev/null | tail -1 | awk '{print int($5)}')
+        if [ -n "$disk_usage_raw" ]; then
+            disk_usage="$disk_usage_raw"
+        else
+            if [ "$status" = "OK" ]; then status="WARNING"; fi
+            warnings+=("Unable to determine disk usage for $BACKUP_PATH")
        fi
+    fi

-        # Check disk usage
-        local disk_usage=$(df "$BACKUP_PATH" | tail -1 | awk '{print int($5)}')
+    if [ "$disk_usage" -gt 90 ]; then
+        status="ERROR"
+        errors+=("Disk usage critical: ${disk_usage}%")
+    elif [ "$disk_usage" -gt 80 ]; then
+        if [ "$status" != "ERROR" ]; then status="WARNING"; fi
+        warnings+=("Disk usage high: ${disk_usage}%")
+    fi

-        if [ "$disk_usage" -gt 90 ]; then
-            status="ERROR"
-            errors+=("Disk usage critical: ${disk_usage}%")
-        elif [ "$disk_usage" -gt 80 ]; then
-            if [ "$status" != "ERROR" ]; then status="WARNING"; fi
-            warnings+=("Disk usage high: ${disk_usage}%")
-        fi
+    local severity="info"
+    [ "$status" = "WARNING" ] && severity="warning"
+    [ "$status" = "ERROR" ] && severity="error"

-        # Prepare notification data
-        local severity="info"
-        [ "$status" = "WARNING" ] && severity="warning"
-        [ "$status" = "ERROR" ] && severity="error"
+    local errors_json
+    if [ ${#errors[@]} -eq 0 ]; then
+        errors_json='[]'
+    else
+        errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .)
+    fi

-        # Convert arrays to JSON arrays
-        local errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .)
-        local warnings_json=$(printf '%s\n' "${warnings[@]}" | jq -R . | jq -s .)
-        local backup_list_json=$(echo "$backup_files" | head -5 | jq -R . | jq -s .)
+    local warnings_json
+    if [ ${#warnings[@]} -eq 0 ]; then
+        warnings_json='[]'
+    else
+        warnings_json=$(printf '%s\n' "${warnings[@]}" | jq -R . | jq -s .)
+    fi

-        # Create JSON data
-        local json_data=$(cat <<JSON
+    local full_backup_list_json
+    if [ ${#full_backup_entries[@]} -eq 0 ]; then
+        full_backup_list_json='[]'
+    else
+        full_backup_list_json=$(printf '%s\n' "${full_backup_entries[@]}" | jq -R . | jq -s .)
+    fi
+
+    local incr_backup_list_json
+    if [ ${#incr_backup_entries[@]} -eq 0 ]; then
+        incr_backup_list_json='[]'
+    else
+        incr_backup_list_json=$(printf '%s\n' "${incr_backup_entries[@]}" | jq -R . | jq -s .)
+    fi
+
+    local has_errors=false
+    local has_warnings=false
+    local has_full_backups=false
+    local has_incr_backups=false
+    [ ${#errors[@]} -gt 0 ] && has_errors=true
+    [ ${#warnings[@]} -gt 0 ] && has_warnings=true
+    [ ${#full_backup_entries[@]} -gt 0 ] && has_full_backups=true
+    [ ${#incr_backup_entries[@]} -gt 0 ] && has_incr_backups=true
+
+    local json_data=$(cat <<JSON
 {
    "severity": "$severity",
    "node": "$(hostname)",
@@ -267,8 +421,11 @@ check_backups() {
    "status": "$status",
    "errors": $errors_json,
    "warnings": $warnings_json,
+    "has_errors": $has_errors,
+    "has_warnings": $has_warnings,
    "total_backups": $total_backups,
-    "total_size_gb": "${total_size%G}",
+    "total_size_gb": "${total_size_label%G}",
+    "total_size_label": "$total_size_label",
    "full_backup_age": "${full_age_hours}",
    "cumulative_backup_age": "${cumulative_age_hours}",
    "disk_usage": "${disk_usage}",
@@ -276,29 +433,33 @@ check_backups() {
    "cumulative_backup_ok": $([ "$cumulative_backup_ok" = "true" ] && echo "true" || echo "false"),
    "is_error": $([ "$status" = "ERROR" ] && echo "true" || echo "false"),
    "is_warning": $([ "$status" = "WARNING" ] && echo "true" || echo "false"),
-    "backup_list": $backup_list_json
+    "full_backup_list": $full_backup_list_json,
+    "incr_backup_list": $incr_backup_list_json,
+    "has_full_backups": $has_full_backups,
+    "has_incr_backups": $has_incr_backups,
+    "full_backup_count": ${#full_backup_entries[@]},
+    "incr_backup_count": ${#incr_backup_entries[@]},
+    "full_backup_limit": "$MAX_FULL_AGE_HOURS",
+    "cumulative_backup_limit": "$MAX_CUMULATIVE_AGE_HOURS"
 }
 JSON
 )

-        # Send notification if there are issues
-        if [ "$status" != "OK" ]; then
-            echo -e "${YELLOW}Issues detected, sending notification...${NC}"
-            send_pve_notification "$severity" "$status" "$json_data"
-        else
-            echo -e "${GREEN}All backups are healthy${NC}"
-            # Optionally send success notification (uncomment if desired)
-            # send_pve_notification "info" "$status" "$json_data"
-        fi
-
-        # Display summary
-        echo "Status: $status"
-        echo "Total backups: $total_backups"
-        echo "Total size: $total_size"
-        echo "FULL backup age: $full_age_hours hours"
-        echo "CUMULATIVE backup age: $cumulative_age_hours hours"
-        echo "Disk usage: ${disk_usage}%"
+    if [ "$status" != "OK" ]; then
+        echo -e "${YELLOW}Issues detected, sending notification...${NC}"
+        send_pve_notification "$severity" "$status" "$json_data"
+    else
+        echo -e "${GREEN}All backups are healthy${NC}"
+        # Optionally send success notification (uncomment if desired)
+        # send_pve_notification "info" "$status" "$json_data"
    fi
+
+    echo "Status: $status"
+    echo "Total backups: $total_backups"
+    echo "Total size: $total_size_label"
+    echo "FULL backup age: $full_age_hours hours"
+    echo "CUMULATIVE backup age: $cumulative_age_hours hours"
+    echo "Disk usage: ${disk_usage}%"
 }

 # Main execution