Oracle DR: Fix template variables and complete monitoring system testing
- Fix Proxmox template compatibility: {{hostname}} → {{node}}, {{timestamp}} → {{date}}
- Remove duplicate node fields and fix JSON structure
- Complete full testing plan execution for monitoring and DR test scripts
- Validate notification system functionality with PVE::Notify
- Sync tested scripts from production back to repository
Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
Marius
@@ -1,414 +1,415 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Oracle Backup Monitor for Proxmox with PVE::Notify
|
||||
# Monitors Oracle backups and sends notifications via Proxmox notification system
|
||||
#
|
||||
# Location: /opt/scripts/oracle-backup-monitor-proxmox.sh (on Proxmox host)
|
||||
# Schedule: Add to cron for daily execution
|
||||
#
|
||||
# This script is SELF-SUFFICIENT:
|
||||
# - Automatically creates notification templates if they don't exist
|
||||
# - Uses Proxmox native notification system (same as HA alerts)
|
||||
# - No email configuration needed - uses existing Proxmox setup
|
||||
#
|
||||
# Installation:
|
||||
# cp oracle-backup-monitor-proxmox.sh /opt/scripts/
|
||||
# chmod +x /opt/scripts/oracle-backup-monitor-proxmox.sh
|
||||
# /opt/scripts/oracle-backup-monitor-proxmox.sh --install # Creates templates
|
||||
# crontab -e # Add: 0 9 * * * /opt/scripts/oracle-backup-monitor-proxmox.sh
|
||||
#
|
||||
# Author: Claude (based on ha-monitor.sh pattern)
|
||||
# Version: 1.0
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
PRIMARY_HOST="10.0.20.36"
|
||||
PRIMARY_PORT="22122"
|
||||
PRIMARY_USER="Administrator"
|
||||
BACKUP_PATH="/mnt/pve/oracle-backups/ROA/autobackup"
|
||||
MAX_FULL_AGE_HOURS=25
|
||||
MAX_CUMULATIVE_AGE_HOURS=7
|
||||
TEMPLATE_DIR="/usr/share/pve-manager/templates/default"
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Function to create notification templates
|
||||
create_templates() {
|
||||
echo -e "${GREEN}Creating Oracle backup notification templates...${NC}"
|
||||
|
||||
# Create templates directory if needed
|
||||
mkdir -p "$TEMPLATE_DIR"
|
||||
|
||||
# Subject template
|
||||
cat > "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" <<'EOF'
|
||||
Oracle Backup {{severity}} - {{hostname}}
|
||||
EOF
|
||||
|
||||
# Text body template
|
||||
cat > "$TEMPLATE_DIR/oracle-backup-body.txt.hbs" <<'EOF'
|
||||
Oracle Backup Monitoring Alert
|
||||
==============================
|
||||
Severity: {{severity}}
|
||||
Hostname: {{hostname}}
|
||||
Date: {{timestamp}}
|
||||
Status: {{status}}
|
||||
|
||||
{{#if errors}}
|
||||
ERRORS:
|
||||
{{#each errors}}
|
||||
- {{this}}
|
||||
{{/each}}
|
||||
{{/if}}
|
||||
|
||||
{{#if warnings}}
|
||||
WARNINGS:
|
||||
{{#each warnings}}
|
||||
- {{this}}
|
||||
{{/each}}
|
||||
{{/if}}
|
||||
|
||||
Backup Details:
|
||||
- Total Backups: {{total_backups}}
|
||||
- Total Size: {{total_size_gb}} GB
|
||||
- FULL Backup Age: {{full_backup_age}} hours
|
||||
- CUMULATIVE Backup Age: {{cumulative_backup_age}} hours
|
||||
- Disk Usage: {{disk_usage}}%
|
||||
|
||||
{{#if backup_list}}
|
||||
Recent Backups:
|
||||
{{#each backup_list}}
|
||||
{{this}}
|
||||
{{/each}}
|
||||
{{/if}}
|
||||
EOF
|
||||
|
||||
# HTML body template
|
||||
cat > "$TEMPLATE_DIR/oracle-backup-body.html.hbs" <<'EOF'
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
body { font-family: Arial, sans-serif; }
|
||||
.header {
|
||||
background-color: {{#if is_error}}#dc3545{{else}}{{#if is_warning}}#ffc107{{else}}#28a745{{/if}}{{/if}};
|
||||
color: white;
|
||||
padding: 10px;
|
||||
border-radius: 5px;
|
||||
}
|
||||
.section { margin: 20px 0; padding: 10px; background-color: #f8f9fa; border-radius: 5px; }
|
||||
.error { color: #dc3545; font-weight: bold; }
|
||||
.warning { color: #ffc107; font-weight: bold; }
|
||||
.success { color: #28a745; }
|
||||
table { width: 100%; border-collapse: collapse; margin: 10px 0; }
|
||||
th, td { padding: 8px; text-align: left; border-bottom: 1px solid #dee2e6; }
|
||||
th { background-color: #e9ecef; }
|
||||
.metric { display: inline-block; margin: 10px 20px 10px 0; }
|
||||
.metric-label { font-size: 0.9em; color: #6c757d; }
|
||||
.metric-value { font-size: 1.5em; font-weight: bold; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<h2>Oracle Backup {{severity}}</h2>
|
||||
<p>{{hostname}} - {{timestamp}}</p>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h3>Status: <span class="{{#if is_error}}error{{else}}{{#if is_warning}}warning{{else}}success{{/if}}{{/if}}">{{status}}</span></h3>
|
||||
|
||||
{{#if errors}}
|
||||
<div class="error">
|
||||
<h4>Errors:</h4>
|
||||
<ul>
|
||||
{{#each errors}}
|
||||
<li>{{this}}</li>
|
||||
{{/each}}
|
||||
</ul>
|
||||
</div>
|
||||
{{/if}}
|
||||
|
||||
{{#if warnings}}
|
||||
<div class="warning">
|
||||
<h4>Warnings:</h4>
|
||||
<ul>
|
||||
{{#each warnings}}
|
||||
<li>{{this}}</li>
|
||||
{{/each}}
|
||||
</ul>
|
||||
</div>
|
||||
{{/if}}
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h3>Backup Metrics</h3>
|
||||
<div>
|
||||
<div class="metric">
|
||||
<div class="metric-label">Total Backups</div>
|
||||
<div class="metric-value">{{total_backups}}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="metric-label">Total Size</div>
|
||||
<div class="metric-value">{{total_size_gb}} GB</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="metric-label">Disk Usage</div>
|
||||
<div class="metric-value">{{disk_usage}}%</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<th>Backup Type</th>
|
||||
<th>Age (hours)</th>
|
||||
<th>Status</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>FULL</td>
|
||||
<td>{{full_backup_age}}</td>
|
||||
<td>{{#if full_backup_ok}}<span class="success">✓ OK</span>{{else}}<span class="error">✗ Too Old</span>{{/if}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>CUMULATIVE</td>
|
||||
<td>{{cumulative_backup_age}}</td>
|
||||
<td>{{#if cumulative_backup_ok}}<span class="success">✓ OK</span>{{else}}<span class="warning">⚠ Check</span>{{/if}}</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{{#if backup_list}}
|
||||
<div class="section">
|
||||
<h3>Recent Backups</h3>
|
||||
<pre style="background-color: #f8f9fa; padding: 10px; overflow-x: auto;">{{#each backup_list}}{{this}}
|
||||
{{/each}}</pre>
|
||||
</div>
|
||||
{{/if}}
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
echo -e "${GREEN}Templates created successfully in $TEMPLATE_DIR${NC}"
|
||||
}
|
||||
|
||||
# Function to send notification via PVE::Notify
|
||||
send_pve_notification() {
|
||||
local severity="$1"
|
||||
local status="$2"
|
||||
local data="$3"
|
||||
|
||||
# Create Perl script to call PVE::Notify
|
||||
cat > /tmp/oracle-notify.pl <<'PERL_SCRIPT'
|
||||
#!/usr/bin/perl
|
||||
use strict;
|
||||
use warnings;
|
||||
use PVE::Notify;
|
||||
use JSON;
|
||||
|
||||
my $json_data = do { local $/; <STDIN> };
|
||||
my $data = decode_json($json_data);
|
||||
|
||||
my $severity = $data->{severity} // 'info';
|
||||
my $template_name = 'oracle-backup';
|
||||
|
||||
# Add fields for matching rules
|
||||
my $fields = {
|
||||
type => 'oracle-backup',
|
||||
severity => $severity,
|
||||
hostname => $data->{hostname},
|
||||
};
|
||||
|
||||
# Send notification
|
||||
eval {
|
||||
PVE::Notify::notify(
|
||||
$severity,
|
||||
$template_name,
|
||||
$data,
|
||||
$fields
|
||||
);
|
||||
};
|
||||
|
||||
if ($@) {
|
||||
print "Error sending notification: $@\n";
|
||||
exit 1;
|
||||
}
|
||||
|
||||
print "Notification sent successfully\n";
|
||||
PERL_SCRIPT
|
||||
|
||||
chmod +x /tmp/oracle-notify.pl
|
||||
|
||||
# Send notification
|
||||
echo "$data" | perl /tmp/oracle-notify.pl
|
||||
|
||||
rm -f /tmp/oracle-notify.pl
|
||||
}
|
||||
|
||||
# Function to check backups
|
||||
check_backups() {
|
||||
local status="OK"
|
||||
local errors=()
|
||||
local warnings=()
|
||||
|
||||
echo "Checking Oracle backups..."
|
||||
|
||||
# Get backup list
|
||||
local backup_files=$(ls -lth "$BACKUP_PATH"/*.BKP 2>/dev/null | head -10 || echo "")
|
||||
|
||||
if [ -z "$backup_files" ]; then
|
||||
status="ERROR"
|
||||
errors+=("No backup files found in $BACKUP_PATH")
|
||||
else
|
||||
# Count backups
|
||||
local total_backups=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l)
|
||||
local total_size=$(du -shc "$BACKUP_PATH"/*.BKP 2>/dev/null | tail -1 | awk '{print $1}')
|
||||
|
||||
# Check FULL backup age
|
||||
local latest_full=$(ls -t "$BACKUP_PATH"/*FULL*.BKP 2>/dev/null | head -1 || echo "")
|
||||
local full_age_hours="N/A"
|
||||
local full_backup_ok=false
|
||||
|
||||
if [ -n "$latest_full" ]; then
|
||||
local full_timestamp=$(stat -c %Y "$latest_full")
|
||||
local current_timestamp=$(date +%s)
|
||||
full_age_hours=$(( (current_timestamp - full_timestamp) / 3600 ))
|
||||
|
||||
if [ "$full_age_hours" -gt "$MAX_FULL_AGE_HOURS" ]; then
|
||||
status="WARNING"
|
||||
warnings+=("FULL backup is $full_age_hours hours old (threshold: $MAX_FULL_AGE_HOURS)")
|
||||
else
|
||||
full_backup_ok=true
|
||||
fi
|
||||
else
|
||||
status="ERROR"
|
||||
errors+=("No FULL backup found")
|
||||
fi
|
||||
|
||||
# Check CUMULATIVE backup age
|
||||
local latest_cumulative=$(ls -t "$BACKUP_PATH"/*INCR*.BKP "$BACKUP_PATH"/*CUMULATIVE*.BKP 2>/dev/null | head -1 || echo "")
|
||||
local cumulative_age_hours="N/A"
|
||||
local cumulative_backup_ok=false
|
||||
|
||||
if [ -n "$latest_cumulative" ]; then
|
||||
local cumulative_timestamp=$(stat -c %Y "$latest_cumulative")
|
||||
local current_timestamp=$(date +%s)
|
||||
cumulative_age_hours=$(( (current_timestamp - cumulative_timestamp) / 3600 ))
|
||||
|
||||
if [ "$cumulative_age_hours" -gt "$MAX_CUMULATIVE_AGE_HOURS" ]; then
|
||||
if [ "$status" != "ERROR" ]; then status="WARNING"; fi
|
||||
warnings+=("CUMULATIVE backup is $cumulative_age_hours hours old (threshold: $MAX_CUMULATIVE_AGE_HOURS)")
|
||||
else
|
||||
cumulative_backup_ok=true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check disk usage
|
||||
local disk_usage=$(df "$BACKUP_PATH" | tail -1 | awk '{print int($5)}')
|
||||
|
||||
if [ "$disk_usage" -gt 90 ]; then
|
||||
status="ERROR"
|
||||
errors+=("Disk usage critical: ${disk_usage}%")
|
||||
elif [ "$disk_usage" -gt 80 ]; then
|
||||
if [ "$status" != "ERROR" ]; then status="WARNING"; fi
|
||||
warnings+=("Disk usage high: ${disk_usage}%")
|
||||
fi
|
||||
|
||||
# Prepare notification data
|
||||
local severity="info"
|
||||
[ "$status" = "WARNING" ] && severity="warning"
|
||||
[ "$status" = "ERROR" ] && severity="error"
|
||||
|
||||
# Convert arrays to JSON arrays
|
||||
local errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .)
|
||||
local warnings_json=$(printf '%s\n' "${warnings[@]}" | jq -R . | jq -s .)
|
||||
local backup_list_json=$(echo "$backup_files" | head -5 | jq -R . | jq -s .)
|
||||
|
||||
# Create JSON data
|
||||
local json_data=$(cat <<JSON
|
||||
{
|
||||
"severity": "$severity",
|
||||
"hostname": "$(hostname)",
|
||||
"timestamp": "$(date '+%Y-%m-%d %H:%M:%S')",
|
||||
"status": "$status",
|
||||
"errors": $errors_json,
|
||||
"warnings": $warnings_json,
|
||||
"total_backups": $total_backups,
|
||||
"total_size_gb": "${total_size%G}",
|
||||
"full_backup_age": "$full_age_hours",
|
||||
"cumulative_backup_age": "$cumulative_age_hours",
|
||||
"disk_usage": "$disk_usage",
|
||||
"full_backup_ok": $full_backup_ok,
|
||||
"cumulative_backup_ok": $cumulative_backup_ok,
|
||||
"is_error": $([ "$status" = "ERROR" ] && echo "true" || echo "false"),
|
||||
"is_warning": $([ "$status" = "WARNING" ] && echo "true" || echo "false"),
|
||||
"backup_list": $backup_list_json
|
||||
}
|
||||
JSON
|
||||
)
|
||||
|
||||
# Send notification if there are issues
|
||||
if [ "$status" != "OK" ]; then
|
||||
echo -e "${YELLOW}Issues detected, sending notification...${NC}"
|
||||
send_pve_notification "$severity" "$status" "$json_data"
|
||||
else
|
||||
echo -e "${GREEN}All backups are healthy${NC}"
|
||||
# Optionally send success notification (uncomment if desired)
|
||||
# send_pve_notification "info" "$status" "$json_data"
|
||||
fi
|
||||
|
||||
# Display summary
|
||||
echo "Status: $status"
|
||||
echo "Total backups: $total_backups"
|
||||
echo "Total size: $total_size"
|
||||
echo "FULL backup age: $full_age_hours hours"
|
||||
echo "CUMULATIVE backup age: $cumulative_age_hours hours"
|
||||
echo "Disk usage: ${disk_usage}%"
|
||||
fi
|
||||
}
|
||||
|
||||
# Main execution
|
||||
main() {
|
||||
case "${1:-}" in
|
||||
--install)
|
||||
create_templates
|
||||
echo ""
|
||||
echo -e "${GREEN}Installation complete!${NC}"
|
||||
echo "Next steps:"
|
||||
echo "1. Test the monitor: /opt/scripts/oracle-backup-monitor-proxmox.sh"
|
||||
echo "2. Add to cron: crontab -e"
|
||||
echo " Add line: 0 9 * * * /opt/scripts/oracle-backup-monitor-proxmox.sh"
|
||||
echo "3. Configure notifications in Proxmox GUI if needed:"
|
||||
echo " Datacenter > Notifications > Add matching rules for 'oracle-backup'"
|
||||
;;
|
||||
--help)
|
||||
echo "Oracle Backup Monitor for Proxmox"
|
||||
echo "Usage:"
|
||||
echo " $0 - Check backups and send alerts if issues found"
|
||||
echo " $0 --install - Create notification templates"
|
||||
echo " $0 --help - Show this help"
|
||||
;;
|
||||
*)
|
||||
# Check if templates exist, create if missing
|
||||
if [ ! -f "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" ]; then
|
||||
echo -e "${YELLOW}Templates not found, creating...${NC}"
|
||||
create_templates
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Run backup check
|
||||
check_backups
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Check dependencies
|
||||
if ! command -v jq &> /dev/null; then
|
||||
echo -e "${RED}Error: jq is not installed${NC}"
|
||||
echo "Install with: apt-get install jq"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
#!/bin/bash
|
||||
#
|
||||
# Oracle Backup Monitor for Proxmox with PVE::Notify
|
||||
# Monitors Oracle backups and sends notifications via Proxmox notification system
|
||||
#
|
||||
# Location: /opt/scripts/oracle-backup-monitor-proxmox.sh (on Proxmox host)
|
||||
# Schedule: Add to cron for daily execution
|
||||
#
|
||||
# This script is SELF-SUFFICIENT:
|
||||
# - Automatically creates notification templates if they don't exist
|
||||
# - Uses Proxmox native notification system (same as HA alerts)
|
||||
# - No email configuration needed - uses existing Proxmox setup
|
||||
#
|
||||
# Installation:
|
||||
# cp oracle-backup-monitor-proxmox.sh /opt/scripts/
|
||||
# chmod +x /opt/scripts/oracle-backup-monitor-proxmox.sh
|
||||
# /opt/scripts/oracle-backup-monitor-proxmox.sh --install # Creates templates
|
||||
# crontab -e # Add: 0 9 * * * /opt/scripts/oracle-backup-monitor-proxmox.sh
|
||||
#
|
||||
# Author: Claude (based on ha-monitor.sh pattern)
|
||||
# Version: 1.0
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
PRIMARY_HOST="10.0.20.36"
|
||||
PRIMARY_PORT="22122"
|
||||
PRIMARY_USER="Administrator"
|
||||
BACKUP_PATH="/mnt/pve/oracle-backups/ROA/autobackup"
|
||||
MAX_FULL_AGE_HOURS=25
|
||||
MAX_CUMULATIVE_AGE_HOURS=7
|
||||
TEMPLATE_DIR="/usr/share/pve-manager/templates/default"
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Function to create notification templates
|
||||
create_templates() {
|
||||
echo -e "${GREEN}Creating Oracle backup notification templates...${NC}"
|
||||
|
||||
# Create templates directory if needed
|
||||
mkdir -p "$TEMPLATE_DIR"
|
||||
|
||||
# Subject template
|
||||
cat > "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" <<'EOF'
|
||||
Oracle Backup {{severity}} - {{node}}
|
||||
EOF
|
||||
|
||||
# Text body template
|
||||
cat > "$TEMPLATE_DIR/oracle-backup-body.txt.hbs" <<'EOF'
|
||||
Oracle Backup Monitoring Alert
|
||||
==============================
|
||||
Severity: {{severity}}
|
||||
Hostname: {{node}}
|
||||
Date: {{date}}
|
||||
Status: {{status}}
|
||||
|
||||
{{#if errors}}
|
||||
ERRORS:
|
||||
{{#each errors}}
|
||||
- {{this}}
|
||||
{{/each}}
|
||||
{{/if}}
|
||||
|
||||
{{#if warnings}}
|
||||
WARNINGS:
|
||||
{{#each warnings}}
|
||||
- {{this}}
|
||||
{{/each}}
|
||||
{{/if}}
|
||||
|
||||
Backup Details:
|
||||
- Total Backups: {{total_backups}}
|
||||
- Total Size: {{total_size_gb}} GB
|
||||
- FULL Backup Age: {{full_backup_age}} hours
|
||||
- CUMULATIVE Backup Age: {{cumulative_backup_age}} hours
|
||||
- Disk Usage: {{disk_usage}}%
|
||||
|
||||
{{#if backup_list}}
|
||||
Recent Backups:
|
||||
{{#each backup_list}}
|
||||
{{this}}
|
||||
{{/each}}
|
||||
{{/if}}
|
||||
EOF
|
||||
|
||||
# HTML body template
|
||||
cat > "$TEMPLATE_DIR/oracle-backup-body.html.hbs" <<'EOF'
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
body { font-family: Arial, sans-serif; }
|
||||
.header {
|
||||
background-color: {{#if is_error}}#dc3545{{else}}{{#if is_warning}}#ffc107{{else}}#28a745{{/if}}{{/if}};
|
||||
color: white;
|
||||
padding: 10px;
|
||||
border-radius: 5px;
|
||||
}
|
||||
.section { margin: 20px 0; padding: 10px; background-color: #f8f9fa; border-radius: 5px; }
|
||||
.error { color: #dc3545; font-weight: bold; }
|
||||
.warning { color: #ffc107; font-weight: bold; }
|
||||
.success { color: #28a745; }
|
||||
table { width: 100%; border-collapse: collapse; margin: 10px 0; }
|
||||
th, td { padding: 8px; text-align: left; border-bottom: 1px solid #dee2e6; }
|
||||
th { background-color: #e9ecef; }
|
||||
.metric { display: inline-block; margin: 10px 20px 10px 0; }
|
||||
.metric-label { font-size: 0.9em; color: #6c757d; }
|
||||
.metric-value { font-size: 1.5em; font-weight: bold; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<h2>Oracle Backup {{severity}}</h2>
|
||||
<p>{{node}} - {{date}}</p>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h3>Status: <span class="{{#if is_error}}error{{else}}{{#if is_warning}}warning{{else}}success{{/if}}{{/if}}">{{status}}</span></h3>
|
||||
|
||||
{{#if errors}}
|
||||
<div class="error">
|
||||
<h4>Errors:</h4>
|
||||
<ul>
|
||||
{{#each errors}}
|
||||
<li>{{this}}</li>
|
||||
{{/each}}
|
||||
</ul>
|
||||
</div>
|
||||
{{/if}}
|
||||
|
||||
{{#if warnings}}
|
||||
<div class="warning">
|
||||
<h4>Warnings:</h4>
|
||||
<ul>
|
||||
{{#each warnings}}
|
||||
<li>{{this}}</li>
|
||||
{{/each}}
|
||||
</ul>
|
||||
</div>
|
||||
{{/if}}
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h3>Backup Metrics</h3>
|
||||
<div>
|
||||
<div class="metric">
|
||||
<div class="metric-label">Total Backups</div>
|
||||
<div class="metric-value">{{total_backups}}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="metric-label">Total Size</div>
|
||||
<div class="metric-value">{{total_size_gb}} GB</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="metric-label">Disk Usage</div>
|
||||
<div class="metric-value">{{disk_usage}}%</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<th>Backup Type</th>
|
||||
<th>Age (hours)</th>
|
||||
<th>Status</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>FULL</td>
|
||||
<td>{{full_backup_age}}</td>
|
||||
<td>{{#if full_backup_ok}}<span class="success">✓ OK</span>{{else}}<span class="error">✗ Too Old</span>{{/if}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>CUMULATIVE</td>
|
||||
<td>{{cumulative_backup_age}}</td>
|
||||
<td>{{#if cumulative_backup_ok}}<span class="success">✓ OK</span>{{else}}<span class="warning">⚠ Check</span>{{/if}}</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{{#if backup_list}}
|
||||
<div class="section">
|
||||
<h3>Recent Backups</h3>
|
||||
<pre style="background-color: #f8f9fa; padding: 10px; overflow-x: auto;">{{#each backup_list}}{{this}}
|
||||
{{/each}}</pre>
|
||||
</div>
|
||||
{{/if}}
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
echo -e "${GREEN}Templates created successfully in $TEMPLATE_DIR${NC}"
|
||||
}
|
||||
|
||||
# Function to send notification via PVE::Notify
|
||||
send_pve_notification() {
|
||||
local severity="$1"
|
||||
local status="$2"
|
||||
local data="$3"
|
||||
|
||||
# Create Perl script to call PVE::Notify
|
||||
cat > /tmp/oracle-notify.pl <<'PERL_SCRIPT'
|
||||
#!/usr/bin/perl
|
||||
use strict;
|
||||
use warnings;
|
||||
use PVE::Notify;
|
||||
use JSON;
|
||||
|
||||
my $json_data = do { local $/; <STDIN> };
|
||||
my $data = decode_json($json_data);
|
||||
|
||||
my $severity = $data->{severity} // 'info';
|
||||
my $template_name = 'oracle-backup';
|
||||
|
||||
# Add fields for matching rules
|
||||
my $fields = {
|
||||
type => 'oracle-backup',
|
||||
severity => $severity,
|
||||
hostname => $data->{hostname},
|
||||
};
|
||||
|
||||
# Send notification
|
||||
eval {
|
||||
PVE::Notify::notify(
|
||||
$severity,
|
||||
$template_name,
|
||||
$data,
|
||||
$fields
|
||||
);
|
||||
};
|
||||
|
||||
if ($@) {
|
||||
print "Error sending notification: $@\n";
|
||||
exit 1;
|
||||
}
|
||||
|
||||
print "Notification sent successfully\n";
|
||||
PERL_SCRIPT
|
||||
|
||||
chmod +x /tmp/oracle-notify.pl
|
||||
|
||||
# Send notification
|
||||
echo "$data" | perl /tmp/oracle-notify.pl
|
||||
|
||||
rm -f /tmp/oracle-notify.pl
|
||||
}
|
||||
|
||||
# Function to check backups
|
||||
check_backups() {
|
||||
local status="OK"
|
||||
local errors=()
|
||||
local warnings=()
|
||||
|
||||
echo "Checking Oracle backups..."
|
||||
|
||||
# Get backup list
|
||||
local backup_files=$(ls -lth "$BACKUP_PATH"/*.BKP 2>/dev/null | head -10 || echo "")
|
||||
|
||||
if [ -z "$backup_files" ]; then
|
||||
status="ERROR"
|
||||
errors+=("No backup files found in $BACKUP_PATH")
|
||||
else
|
||||
# Count backups
|
||||
local total_backups=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l)
|
||||
local total_size=$(du -shc "$BACKUP_PATH"/*.BKP 2>/dev/null | tail -1 | awk '{print $1}')
|
||||
|
||||
# Check FULL backup age
|
||||
local latest_full=$(ls -t "$BACKUP_PATH"/*FULL*.BKP 2>/dev/null | head -1 || echo "")
|
||||
local full_age_hours="N/A"
|
||||
local full_backup_ok=false
|
||||
|
||||
if [ -n "$latest_full" ]; then
|
||||
local full_timestamp=$(stat -c %Y "$latest_full")
|
||||
local current_timestamp=$(date +%s)
|
||||
full_age_hours=$(( (current_timestamp - full_timestamp) / 3600 ))
|
||||
|
||||
if [ "$full_age_hours" -gt "$MAX_FULL_AGE_HOURS" ]; then
|
||||
status="WARNING"
|
||||
warnings+=("FULL backup is $full_age_hours hours old (threshold: $MAX_FULL_AGE_HOURS)")
|
||||
else
|
||||
full_backup_ok=true
|
||||
fi
|
||||
else
|
||||
status="ERROR"
|
||||
errors+=("No FULL backup found")
|
||||
fi
|
||||
|
||||
# Check CUMULATIVE backup age
|
||||
local latest_cumulative=$(ls -t "$BACKUP_PATH"/*INCR*.BKP "$BACKUP_PATH"/*CUMULATIVE*.BKP 2>/dev/null | head -1 || echo "")
|
||||
local cumulative_age_hours="N/A"
|
||||
local cumulative_backup_ok=false
|
||||
|
||||
if [ -n "$latest_cumulative" ]; then
|
||||
local cumulative_timestamp=$(stat -c %Y "$latest_cumulative")
|
||||
local current_timestamp=$(date +%s)
|
||||
cumulative_age_hours=$(( (current_timestamp - cumulative_timestamp) / 3600 ))
|
||||
|
||||
if [ "$cumulative_age_hours" -gt "$MAX_CUMULATIVE_AGE_HOURS" ]; then
|
||||
if [ "$status" != "ERROR" ]; then status="WARNING"; fi
|
||||
warnings+=("CUMULATIVE backup is $cumulative_age_hours hours old (threshold: $MAX_CUMULATIVE_AGE_HOURS)")
|
||||
else
|
||||
cumulative_backup_ok=true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check disk usage
|
||||
local disk_usage=$(df "$BACKUP_PATH" | tail -1 | awk '{print int($5)}')
|
||||
|
||||
if [ "$disk_usage" -gt 90 ]; then
|
||||
status="ERROR"
|
||||
errors+=("Disk usage critical: ${disk_usage}%")
|
||||
elif [ "$disk_usage" -gt 80 ]; then
|
||||
if [ "$status" != "ERROR" ]; then status="WARNING"; fi
|
||||
warnings+=("Disk usage high: ${disk_usage}%")
|
||||
fi
|
||||
|
||||
# Prepare notification data
|
||||
local severity="info"
|
||||
[ "$status" = "WARNING" ] && severity="warning"
|
||||
[ "$status" = "ERROR" ] && severity="error"
|
||||
|
||||
# Convert arrays to JSON arrays
|
||||
local errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .)
|
||||
local warnings_json=$(printf '%s\n' "${warnings[@]}" | jq -R . | jq -s .)
|
||||
local backup_list_json=$(echo "$backup_files" | head -5 | jq -R . | jq -s .)
|
||||
|
||||
# Create JSON data
|
||||
local json_data=$(cat <<JSON
|
||||
{
|
||||
"severity": "$severity",
|
||||
"hostname": "$(hostname)",
|
||||
"node": "$(hostname)",
|
||||
"date": "$(date +'%Y-%m-%d %H:%M:%S')",
|
||||
"status": "$status",
|
||||
"errors": $errors_json,
|
||||
"warnings": $warnings_json,
|
||||
"total_backups": $total_backups,
|
||||
"total_size_gb": "${total_size%G}",
|
||||
"full_backup_age": "$full_age_hours",
|
||||
"cumulative_backup_age": "$cumulative_age_hours",
|
||||
"disk_usage": "$disk_usage",
|
||||
"full_backup_ok": $full_backup_ok,
|
||||
"cumulative_backup_ok": $cumulative_backup_ok,
|
||||
"is_error": $([ "$status" = "ERROR" ] && echo "true" || echo "false"),
|
||||
"is_warning": $([ "$status" = "WARNING" ] && echo "true" || echo "false"),
|
||||
"backup_list": $backup_list_json
|
||||
}
|
||||
JSON
|
||||
)
|
||||
|
||||
# Send notification if there are issues
|
||||
if [ "$status" != "OK" ]; then
|
||||
echo -e "${YELLOW}Issues detected, sending notification...${NC}"
|
||||
send_pve_notification "$severity" "$status" "$json_data"
|
||||
else
|
||||
echo -e "${GREEN}All backups are healthy${NC}"
|
||||
# Optionally send success notification (uncomment if desired)
|
||||
# send_pve_notification "info" "$status" "$json_data"
|
||||
fi
|
||||
|
||||
# Display summary
|
||||
echo "Status: $status"
|
||||
echo "Total backups: $total_backups"
|
||||
echo "Total size: $total_size"
|
||||
echo "FULL backup age: $full_age_hours hours"
|
||||
echo "CUMULATIVE backup age: $cumulative_age_hours hours"
|
||||
echo "Disk usage: ${disk_usage}%"
|
||||
fi
|
||||
}
|
||||
|
||||
# Main execution
|
||||
main() {
|
||||
case "${1:-}" in
|
||||
--install)
|
||||
create_templates
|
||||
echo ""
|
||||
echo -e "${GREEN}Installation complete!${NC}"
|
||||
echo "Next steps:"
|
||||
echo "1. Test the monitor: /opt/scripts/oracle-backup-monitor-proxmox.sh"
|
||||
echo "2. Add to cron: crontab -e"
|
||||
echo " Add line: 0 9 * * * /opt/scripts/oracle-backup-monitor-proxmox.sh"
|
||||
echo "3. Configure notifications in Proxmox GUI if needed:"
|
||||
echo " Datacenter > Notifications > Add matching rules for 'oracle-backup'"
|
||||
;;
|
||||
--help)
|
||||
echo "Oracle Backup Monitor for Proxmox"
|
||||
echo "Usage:"
|
||||
echo " $0 - Check backups and send alerts if issues found"
|
||||
echo " $0 --install - Create notification templates"
|
||||
echo " $0 --help - Show this help"
|
||||
;;
|
||||
*)
|
||||
# Check if templates exist, create if missing
|
||||
if [ ! -f "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" ]; then
|
||||
echo -e "${YELLOW}Templates not found, creating...${NC}"
|
||||
create_templates
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Run backup check
|
||||
check_backups
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Check dependencies
|
||||
if ! command -v jq &> /dev/null; then
|
||||
echo -e "${RED}Error: jq is not installed${NC}"
|
||||
echo "Install with: apt-get install jq"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
main "$@"
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user