b34006a499
- Fix Proxmox template compatibility: {{hostname}} → {{node}}, {{timestamp}} → {{date}}
- Remove duplicate node fields and fix JSON structure
- Complete full testing plan execution for monitoring and DR test scripts
- Validate notification system functionality with PVE::Notify
- Sync tested scripts from production back to repository
Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
415 lines
12 KiB
Bash
415 lines
12 KiB
Bash
#!/bin/bash
|
|
#
|
|
# Oracle Backup Monitor for Proxmox with PVE::Notify
|
|
# Monitors Oracle backups and sends notifications via Proxmox notification system
|
|
#
|
|
# Location: /opt/scripts/oracle-backup-monitor-proxmox.sh (on Proxmox host)
|
|
# Schedule: Add to cron for daily execution
|
|
#
|
|
# This script is SELF-SUFFICIENT:
|
|
# - Automatically creates notification templates if they don't exist
|
|
# - Uses Proxmox native notification system (same as HA alerts)
|
|
# - No email configuration needed - uses existing Proxmox setup
|
|
#
|
|
# Installation:
|
|
# cp oracle-backup-monitor-proxmox.sh /opt/scripts/
|
|
# chmod +x /opt/scripts/oracle-backup-monitor-proxmox.sh
|
|
# /opt/scripts/oracle-backup-monitor-proxmox.sh --install # Creates templates
|
|
# crontab -e # Add: 0 9 * * * /opt/scripts/oracle-backup-monitor-proxmox.sh
|
|
#
|
|
# Author: Claude (based on ha-monitor.sh pattern)
|
|
# Version: 1.0
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
PRIMARY_HOST="10.0.20.36"
|
|
PRIMARY_PORT="22122"
|
|
PRIMARY_USER="Administrator"
|
|
BACKUP_PATH="/mnt/pve/oracle-backups/ROA/autobackup"
|
|
MAX_FULL_AGE_HOURS=25
|
|
MAX_CUMULATIVE_AGE_HOURS=7
|
|
TEMPLATE_DIR="/usr/share/pve-manager/templates/default"
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m'
|
|
|
|
# Function to create notification templates
|
|
create_templates() {
|
|
echo -e "${GREEN}Creating Oracle backup notification templates...${NC}"
|
|
|
|
# Create templates directory if needed
|
|
mkdir -p "$TEMPLATE_DIR"
|
|
|
|
# Subject template
|
|
cat > "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" <<'EOF'
|
|
Oracle Backup {{severity}} - {{node}}
|
|
EOF
|
|
|
|
# Text body template
|
|
cat > "$TEMPLATE_DIR/oracle-backup-body.txt.hbs" <<'EOF'
|
|
Oracle Backup Monitoring Alert
|
|
==============================
|
|
Severity: {{severity}}
|
|
Hostname: {{node}}
|
|
Date: {{date}}
|
|
Status: {{status}}
|
|
|
|
{{#if errors}}
|
|
ERRORS:
|
|
{{#each errors}}
|
|
- {{this}}
|
|
{{/each}}
|
|
{{/if}}
|
|
|
|
{{#if warnings}}
|
|
WARNINGS:
|
|
{{#each warnings}}
|
|
- {{this}}
|
|
{{/each}}
|
|
{{/if}}
|
|
|
|
Backup Details:
|
|
- Total Backups: {{total_backups}}
|
|
- Total Size: {{total_size_gb}} GB
|
|
- FULL Backup Age: {{full_backup_age}} hours
|
|
- CUMULATIVE Backup Age: {{cumulative_backup_age}} hours
|
|
- Disk Usage: {{disk_usage}}%
|
|
|
|
{{#if backup_list}}
|
|
Recent Backups:
|
|
{{#each backup_list}}
|
|
{{this}}
|
|
{{/each}}
|
|
{{/if}}
|
|
EOF
|
|
|
|
# HTML body template
|
|
cat > "$TEMPLATE_DIR/oracle-backup-body.html.hbs" <<'EOF'
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<style>
|
|
body { font-family: Arial, sans-serif; }
|
|
.header {
|
|
background-color: {{#if is_error}}#dc3545{{else}}{{#if is_warning}}#ffc107{{else}}#28a745{{/if}}{{/if}};
|
|
color: white;
|
|
padding: 10px;
|
|
border-radius: 5px;
|
|
}
|
|
.section { margin: 20px 0; padding: 10px; background-color: #f8f9fa; border-radius: 5px; }
|
|
.error { color: #dc3545; font-weight: bold; }
|
|
.warning { color: #ffc107; font-weight: bold; }
|
|
.success { color: #28a745; }
|
|
table { width: 100%; border-collapse: collapse; margin: 10px 0; }
|
|
th, td { padding: 8px; text-align: left; border-bottom: 1px solid #dee2e6; }
|
|
th { background-color: #e9ecef; }
|
|
.metric { display: inline-block; margin: 10px 20px 10px 0; }
|
|
.metric-label { font-size: 0.9em; color: #6c757d; }
|
|
.metric-value { font-size: 1.5em; font-weight: bold; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="header">
|
|
<h2>Oracle Backup {{severity}}</h2>
|
|
<p>{{node}} - {{date}}</p>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h3>Status: <span class="{{#if is_error}}error{{else}}{{#if is_warning}}warning{{else}}success{{/if}}{{/if}}">{{status}}</span></h3>
|
|
|
|
{{#if errors}}
|
|
<div class="error">
|
|
<h4>Errors:</h4>
|
|
<ul>
|
|
{{#each errors}}
|
|
<li>{{this}}</li>
|
|
{{/each}}
|
|
</ul>
|
|
</div>
|
|
{{/if}}
|
|
|
|
{{#if warnings}}
|
|
<div class="warning">
|
|
<h4>Warnings:</h4>
|
|
<ul>
|
|
{{#each warnings}}
|
|
<li>{{this}}</li>
|
|
{{/each}}
|
|
</ul>
|
|
</div>
|
|
{{/if}}
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h3>Backup Metrics</h3>
|
|
<div>
|
|
<div class="metric">
|
|
<div class="metric-label">Total Backups</div>
|
|
<div class="metric-value">{{total_backups}}</div>
|
|
</div>
|
|
<div class="metric">
|
|
<div class="metric-label">Total Size</div>
|
|
<div class="metric-value">{{total_size_gb}} GB</div>
|
|
</div>
|
|
<div class="metric">
|
|
<div class="metric-label">Disk Usage</div>
|
|
<div class="metric-value">{{disk_usage}}%</div>
|
|
</div>
|
|
</div>
|
|
|
|
<table>
|
|
<tr>
|
|
<th>Backup Type</th>
|
|
<th>Age (hours)</th>
|
|
<th>Status</th>
|
|
</tr>
|
|
<tr>
|
|
<td>FULL</td>
|
|
<td>{{full_backup_age}}</td>
|
|
<td>{{#if full_backup_ok}}<span class="success">✓ OK</span>{{else}}<span class="error">✗ Too Old</span>{{/if}}</td>
|
|
</tr>
|
|
<tr>
|
|
<td>CUMULATIVE</td>
|
|
<td>{{cumulative_backup_age}}</td>
|
|
<td>{{#if cumulative_backup_ok}}<span class="success">✓ OK</span>{{else}}<span class="warning">⚠ Check</span>{{/if}}</td>
|
|
</tr>
|
|
</table>
|
|
</div>
|
|
|
|
{{#if backup_list}}
|
|
<div class="section">
|
|
<h3>Recent Backups</h3>
|
|
<pre style="background-color: #f8f9fa; padding: 10px; overflow-x: auto;">{{#each backup_list}}{{this}}
|
|
{{/each}}</pre>
|
|
</div>
|
|
{{/if}}
|
|
</body>
|
|
</html>
|
|
EOF
|
|
|
|
echo -e "${GREEN}Templates created successfully in $TEMPLATE_DIR${NC}"
|
|
}
|
|
|
|
# Function to send notification via PVE::Notify
|
|
send_pve_notification() {
|
|
local severity="$1"
|
|
local status="$2"
|
|
local data="$3"
|
|
|
|
# Create Perl script to call PVE::Notify
|
|
cat > /tmp/oracle-notify.pl <<'PERL_SCRIPT'
|
|
#!/usr/bin/perl
|
|
use strict;
|
|
use warnings;
|
|
use PVE::Notify;
|
|
use JSON;
|
|
|
|
my $json_data = do { local $/; <STDIN> };
|
|
my $data = decode_json($json_data);
|
|
|
|
my $severity = $data->{severity} // 'info';
|
|
my $template_name = 'oracle-backup';
|
|
|
|
# Add fields for matching rules
|
|
my $fields = {
|
|
type => 'oracle-backup',
|
|
severity => $severity,
|
|
hostname => $data->{hostname},
|
|
};
|
|
|
|
# Send notification
|
|
eval {
|
|
PVE::Notify::notify(
|
|
$severity,
|
|
$template_name,
|
|
$data,
|
|
$fields
|
|
);
|
|
};
|
|
|
|
if ($@) {
|
|
print "Error sending notification: $@\n";
|
|
exit 1;
|
|
}
|
|
|
|
print "Notification sent successfully\n";
|
|
PERL_SCRIPT
|
|
|
|
chmod +x /tmp/oracle-notify.pl
|
|
|
|
# Send notification
|
|
echo "$data" | perl /tmp/oracle-notify.pl
|
|
|
|
rm -f /tmp/oracle-notify.pl
|
|
}
|
|
|
|
# Function to check backups
|
|
check_backups() {
|
|
local status="OK"
|
|
local errors=()
|
|
local warnings=()
|
|
|
|
echo "Checking Oracle backups..."
|
|
|
|
# Get backup list
|
|
local backup_files=$(ls -lth "$BACKUP_PATH"/*.BKP 2>/dev/null | head -10 || echo "")
|
|
|
|
if [ -z "$backup_files" ]; then
|
|
status="ERROR"
|
|
errors+=("No backup files found in $BACKUP_PATH")
|
|
else
|
|
# Count backups
|
|
local total_backups=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l)
|
|
local total_size=$(du -shc "$BACKUP_PATH"/*.BKP 2>/dev/null | tail -1 | awk '{print $1}')
|
|
|
|
# Check FULL backup age
|
|
local latest_full=$(ls -t "$BACKUP_PATH"/*FULL*.BKP 2>/dev/null | head -1 || echo "")
|
|
local full_age_hours="N/A"
|
|
local full_backup_ok=false
|
|
|
|
if [ -n "$latest_full" ]; then
|
|
local full_timestamp=$(stat -c %Y "$latest_full")
|
|
local current_timestamp=$(date +%s)
|
|
full_age_hours=$(( (current_timestamp - full_timestamp) / 3600 ))
|
|
|
|
if [ "$full_age_hours" -gt "$MAX_FULL_AGE_HOURS" ]; then
|
|
status="WARNING"
|
|
warnings+=("FULL backup is $full_age_hours hours old (threshold: $MAX_FULL_AGE_HOURS)")
|
|
else
|
|
full_backup_ok=true
|
|
fi
|
|
else
|
|
status="ERROR"
|
|
errors+=("No FULL backup found")
|
|
fi
|
|
|
|
# Check CUMULATIVE backup age
|
|
local latest_cumulative=$(ls -t "$BACKUP_PATH"/*INCR*.BKP "$BACKUP_PATH"/*CUMULATIVE*.BKP 2>/dev/null | head -1 || echo "")
|
|
local cumulative_age_hours="N/A"
|
|
local cumulative_backup_ok=false
|
|
|
|
if [ -n "$latest_cumulative" ]; then
|
|
local cumulative_timestamp=$(stat -c %Y "$latest_cumulative")
|
|
local current_timestamp=$(date +%s)
|
|
cumulative_age_hours=$(( (current_timestamp - cumulative_timestamp) / 3600 ))
|
|
|
|
if [ "$cumulative_age_hours" -gt "$MAX_CUMULATIVE_AGE_HOURS" ]; then
|
|
if [ "$status" != "ERROR" ]; then status="WARNING"; fi
|
|
warnings+=("CUMULATIVE backup is $cumulative_age_hours hours old (threshold: $MAX_CUMULATIVE_AGE_HOURS)")
|
|
else
|
|
cumulative_backup_ok=true
|
|
fi
|
|
fi
|
|
|
|
# Check disk usage
|
|
local disk_usage=$(df "$BACKUP_PATH" | tail -1 | awk '{print int($5)}')
|
|
|
|
if [ "$disk_usage" -gt 90 ]; then
|
|
status="ERROR"
|
|
errors+=("Disk usage critical: ${disk_usage}%")
|
|
elif [ "$disk_usage" -gt 80 ]; then
|
|
if [ "$status" != "ERROR" ]; then status="WARNING"; fi
|
|
warnings+=("Disk usage high: ${disk_usage}%")
|
|
fi
|
|
|
|
# Prepare notification data
|
|
local severity="info"
|
|
[ "$status" = "WARNING" ] && severity="warning"
|
|
[ "$status" = "ERROR" ] && severity="error"
|
|
|
|
# Convert arrays to JSON arrays
|
|
local errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .)
|
|
local warnings_json=$(printf '%s\n' "${warnings[@]}" | jq -R . | jq -s .)
|
|
local backup_list_json=$(echo "$backup_files" | head -5 | jq -R . | jq -s .)
|
|
|
|
# Create JSON data
|
|
local json_data=$(cat <<JSON
|
|
{
|
|
"severity": "$severity",
|
|
"hostname": "$(hostname)",
|
|
"node": "$(hostname)",
|
|
"date": "$(date +'%Y-%m-%d %H:%M:%S')",
|
|
"status": "$status",
|
|
"errors": $errors_json,
|
|
"warnings": $warnings_json,
|
|
"total_backups": $total_backups,
|
|
"total_size_gb": "${total_size%G}",
|
|
"full_backup_age": "$full_age_hours",
|
|
"cumulative_backup_age": "$cumulative_age_hours",
|
|
"disk_usage": "$disk_usage",
|
|
"full_backup_ok": $full_backup_ok,
|
|
"cumulative_backup_ok": $cumulative_backup_ok,
|
|
"is_error": $([ "$status" = "ERROR" ] && echo "true" || echo "false"),
|
|
"is_warning": $([ "$status" = "WARNING" ] && echo "true" || echo "false"),
|
|
"backup_list": $backup_list_json
|
|
}
|
|
JSON
|
|
)
|
|
|
|
# Send notification if there are issues
|
|
if [ "$status" != "OK" ]; then
|
|
echo -e "${YELLOW}Issues detected, sending notification...${NC}"
|
|
send_pve_notification "$severity" "$status" "$json_data"
|
|
else
|
|
echo -e "${GREEN}All backups are healthy${NC}"
|
|
# Optionally send success notification (uncomment if desired)
|
|
# send_pve_notification "info" "$status" "$json_data"
|
|
fi
|
|
|
|
# Display summary
|
|
echo "Status: $status"
|
|
echo "Total backups: $total_backups"
|
|
echo "Total size: $total_size"
|
|
echo "FULL backup age: $full_age_hours hours"
|
|
echo "CUMULATIVE backup age: $cumulative_age_hours hours"
|
|
echo "Disk usage: ${disk_usage}%"
|
|
fi
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
case "${1:-}" in
|
|
--install)
|
|
create_templates
|
|
echo ""
|
|
echo -e "${GREEN}Installation complete!${NC}"
|
|
echo "Next steps:"
|
|
echo "1. Test the monitor: /opt/scripts/oracle-backup-monitor-proxmox.sh"
|
|
echo "2. Add to cron: crontab -e"
|
|
echo " Add line: 0 9 * * * /opt/scripts/oracle-backup-monitor-proxmox.sh"
|
|
echo "3. Configure notifications in Proxmox GUI if needed:"
|
|
echo " Datacenter > Notifications > Add matching rules for 'oracle-backup'"
|
|
;;
|
|
--help)
|
|
echo "Oracle Backup Monitor for Proxmox"
|
|
echo "Usage:"
|
|
echo " $0 - Check backups and send alerts if issues found"
|
|
echo " $0 --install - Create notification templates"
|
|
echo " $0 --help - Show this help"
|
|
;;
|
|
*)
|
|
# Check if templates exist, create if missing
|
|
if [ ! -f "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" ]; then
|
|
echo -e "${YELLOW}Templates not found, creating...${NC}"
|
|
create_templates
|
|
echo ""
|
|
fi
|
|
|
|
# Run backup check
|
|
check_backups
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Check dependencies
|
|
if ! command -v jq &> /dev/null; then
|
|
echo -e "${RED}Error: jq is not installed${NC}"
|
|
echo "Install with: apt-get install jq"
|
|
exit 1
|
|
fi
|
|
|
|
main "$@" |