diff --git a/oracle/standby-server-scripts/oracle-backup-monitor-proxmox.sh b/oracle/standby-server-scripts/oracle-backup-monitor-proxmox.sh index 88e5621..d546a9d 100644 --- a/oracle/standby-server-scripts/oracle-backup-monitor-proxmox.sh +++ b/oracle/standby-server-scripts/oracle-backup-monitor-proxmox.sh @@ -1,414 +1,415 @@ -#!/bin/bash -# -# Oracle Backup Monitor for Proxmox with PVE::Notify -# Monitors Oracle backups and sends notifications via Proxmox notification system -# -# Location: /opt/scripts/oracle-backup-monitor-proxmox.sh (on Proxmox host) -# Schedule: Add to cron for daily execution -# -# This script is SELF-SUFFICIENT: -# - Automatically creates notification templates if they don't exist -# - Uses Proxmox native notification system (same as HA alerts) -# - No email configuration needed - uses existing Proxmox setup -# -# Installation: -# cp oracle-backup-monitor-proxmox.sh /opt/scripts/ -# chmod +x /opt/scripts/oracle-backup-monitor-proxmox.sh -# /opt/scripts/oracle-backup-monitor-proxmox.sh --install # Creates templates -# crontab -e # Add: 0 9 * * * /opt/scripts/oracle-backup-monitor-proxmox.sh -# -# Author: Claude (based on ha-monitor.sh pattern) -# Version: 1.0 - -set -euo pipefail - -# Configuration -PRIMARY_HOST="10.0.20.36" -PRIMARY_PORT="22122" -PRIMARY_USER="Administrator" -BACKUP_PATH="/mnt/pve/oracle-backups/ROA/autobackup" -MAX_FULL_AGE_HOURS=25 -MAX_CUMULATIVE_AGE_HOURS=7 -TEMPLATE_DIR="/usr/share/pve-manager/templates/default" - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -# Function to create notification templates -create_templates() { - echo -e "${GREEN}Creating Oracle backup notification templates...${NC}" - - # Create templates directory if needed - mkdir -p "$TEMPLATE_DIR" - - # Subject template - cat > "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" <<'EOF' -Oracle Backup {{severity}} - {{hostname}} -EOF - - # Text body template - cat > "$TEMPLATE_DIR/oracle-backup-body.txt.hbs" <<'EOF' -Oracle Backup Monitoring Alert -============================== -Severity: {{severity}} -Hostname: {{hostname}} -Date: {{timestamp}} -Status: {{status}} - -{{#if errors}} -ERRORS: -{{#each errors}} - - {{this}} -{{/each}} -{{/if}} - -{{#if warnings}} -WARNINGS: -{{#each warnings}} - - {{this}} -{{/each}} -{{/if}} - -Backup Details: -- Total Backups: {{total_backups}} -- Total Size: {{total_size_gb}} GB -- FULL Backup Age: {{full_backup_age}} hours -- CUMULATIVE Backup Age: {{cumulative_backup_age}} hours -- Disk Usage: {{disk_usage}}% - -{{#if backup_list}} -Recent Backups: -{{#each backup_list}} - {{this}} -{{/each}} -{{/if}} -EOF - - # HTML body template - cat > "$TEMPLATE_DIR/oracle-backup-body.html.hbs" <<'EOF' - - - - - - -
-

Oracle Backup {{severity}}

-

{{hostname}} - {{timestamp}}

-
- -
-

Status: {{status}}

- - {{#if errors}} -
-

Errors:

- -
- {{/if}} - - {{#if warnings}} -
-

Warnings:

- -
- {{/if}} -
- -
-

Backup Metrics

-
-
-
Total Backups
-
{{total_backups}}
-
-
-
Total Size
-
{{total_size_gb}} GB
-
-
-
Disk Usage
-
{{disk_usage}}%
-
-
- - - - - - - - - - - - - - - - - -
Backup TypeAge (hours)Status
FULL{{full_backup_age}}{{#if full_backup_ok}}✓ OK{{else}}✗ Too Old{{/if}}
CUMULATIVE{{cumulative_backup_age}}{{#if cumulative_backup_ok}}✓ OK{{else}}⚠ Check{{/if}}
-
- - {{#if backup_list}} -
-

Recent Backups

- 
{{#each backup_list}}{{this}}
-{{/each}}
-
- {{/if}} - - -EOF - - echo -e "${GREEN}Templates created successfully in $TEMPLATE_DIR${NC}" -} - -# Function to send notification via PVE::Notify -send_pve_notification() { - local severity="$1" - local status="$2" - local data="$3" - - # Create Perl script to call PVE::Notify - cat > /tmp/oracle-notify.pl <<'PERL_SCRIPT' -#!/usr/bin/perl -use strict; -use warnings; -use PVE::Notify; -use JSON; - -my $json_data = do { local $/; }; -my $data = decode_json($json_data); - -my $severity = $data->{severity} // 'info'; -my $template_name = 'oracle-backup'; - -# Add fields for matching rules -my $fields = { - type => 'oracle-backup', - severity => $severity, - hostname => $data->{hostname}, -}; - -# Send notification -eval { - PVE::Notify::notify( - $severity, - $template_name, - $data, - $fields - ); -}; - -if ($@) { - print "Error sending notification: $@\n"; - exit 1; -} - -print "Notification sent successfully\n"; -PERL_SCRIPT - - chmod +x /tmp/oracle-notify.pl - - # Send notification - echo "$data" | perl /tmp/oracle-notify.pl - - rm -f /tmp/oracle-notify.pl -} - -# Function to check backups -check_backups() { - local status="OK" - local errors=() - local warnings=() - - echo "Checking Oracle backups..." - - # Get backup list - local backup_files=$(ls -lth "$BACKUP_PATH"/*.BKP 2>/dev/null | head -10 || echo "") - - if [ -z "$backup_files" ]; then - status="ERROR" - errors+=("No backup files found in $BACKUP_PATH") - else - # Count backups - local total_backups=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l) - local total_size=$(du -shc "$BACKUP_PATH"/*.BKP 2>/dev/null | tail -1 | awk '{print $1}') - - # Check FULL backup age - local latest_full=$(ls -t "$BACKUP_PATH"/*FULL*.BKP 2>/dev/null | head -1 || echo "") - local full_age_hours="N/A" - local full_backup_ok=false - - if [ -n "$latest_full" ]; then - local full_timestamp=$(stat -c %Y "$latest_full") - local current_timestamp=$(date +%s) - full_age_hours=$(( (current_timestamp - full_timestamp) / 3600 )) - - if [ "$full_age_hours" -gt "$MAX_FULL_AGE_HOURS" ]; then - status="WARNING" - warnings+=("FULL backup is $full_age_hours hours old (threshold: $MAX_FULL_AGE_HOURS)") - else - full_backup_ok=true - fi - else - status="ERROR" - errors+=("No FULL backup found") - fi - - # Check CUMULATIVE backup age - local latest_cumulative=$(ls -t "$BACKUP_PATH"/*INCR*.BKP "$BACKUP_PATH"/*CUMULATIVE*.BKP 2>/dev/null | head -1 || echo "") - local cumulative_age_hours="N/A" - local cumulative_backup_ok=false - - if [ -n "$latest_cumulative" ]; then - local cumulative_timestamp=$(stat -c %Y "$latest_cumulative") - local current_timestamp=$(date +%s) - cumulative_age_hours=$(( (current_timestamp - cumulative_timestamp) / 3600 )) - - if [ "$cumulative_age_hours" -gt "$MAX_CUMULATIVE_AGE_HOURS" ]; then - if [ "$status" != "ERROR" ]; then status="WARNING"; fi - warnings+=("CUMULATIVE backup is $cumulative_age_hours hours old (threshold: $MAX_CUMULATIVE_AGE_HOURS)") - else - cumulative_backup_ok=true - fi - fi - - # Check disk usage - local disk_usage=$(df "$BACKUP_PATH" | tail -1 | awk '{print int($5)}') - - if [ "$disk_usage" -gt 90 ]; then - status="ERROR" - errors+=("Disk usage critical: ${disk_usage}%") - elif [ "$disk_usage" -gt 80 ]; then - if [ "$status" != "ERROR" ]; then status="WARNING"; fi - warnings+=("Disk usage high: ${disk_usage}%") - fi - - # Prepare notification data - local severity="info" - [ "$status" = "WARNING" ] && severity="warning" - [ "$status" = "ERROR" ] && severity="error" - - # Convert arrays to JSON arrays - local errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .) - local warnings_json=$(printf '%s\n' "${warnings[@]}" | jq -R . | jq -s .) - local backup_list_json=$(echo "$backup_files" | head -5 | jq -R . | jq -s .) - - # Create JSON data - local json_data=$(cat < Notifications > Add matching rules for 'oracle-backup'" - ;; - --help) - echo "Oracle Backup Monitor for Proxmox" - echo "Usage:" - echo " $0 - Check backups and send alerts if issues found" - echo " $0 --install - Create notification templates" - echo " $0 --help - Show this help" - ;; - *) - # Check if templates exist, create if missing - if [ ! -f "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" ]; then - echo -e "${YELLOW}Templates not found, creating...${NC}" - create_templates - echo "" - fi - - # Run backup check - check_backups - ;; - esac -} - -# Check dependencies -if ! command -v jq &> /dev/null; then - echo -e "${RED}Error: jq is not installed${NC}" - echo "Install with: apt-get install jq" - exit 1 -fi - +#!/bin/bash +# +# Oracle Backup Monitor for Proxmox with PVE::Notify +# Monitors Oracle backups and sends notifications via Proxmox notification system +# +# Location: /opt/scripts/oracle-backup-monitor-proxmox.sh (on Proxmox host) +# Schedule: Add to cron for daily execution +# +# This script is SELF-SUFFICIENT: +# - Automatically creates notification templates if they don't exist +# - Uses Proxmox native notification system (same as HA alerts) +# - No email configuration needed - uses existing Proxmox setup +# +# Installation: +# cp oracle-backup-monitor-proxmox.sh /opt/scripts/ +# chmod +x /opt/scripts/oracle-backup-monitor-proxmox.sh +# /opt/scripts/oracle-backup-monitor-proxmox.sh --install # Creates templates +# crontab -e # Add: 0 9 * * * /opt/scripts/oracle-backup-monitor-proxmox.sh +# +# Author: Claude (based on ha-monitor.sh pattern) +# Version: 1.0 + +set -euo pipefail + +# Configuration +PRIMARY_HOST="10.0.20.36" +PRIMARY_PORT="22122" +PRIMARY_USER="Administrator" +BACKUP_PATH="/mnt/pve/oracle-backups/ROA/autobackup" +MAX_FULL_AGE_HOURS=25 +MAX_CUMULATIVE_AGE_HOURS=7 +TEMPLATE_DIR="/usr/share/pve-manager/templates/default" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +# Function to create notification templates +create_templates() { + echo -e "${GREEN}Creating Oracle backup notification templates...${NC}" + + # Create templates directory if needed + mkdir -p "$TEMPLATE_DIR" + + # Subject template + cat > "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" <<'EOF' +Oracle Backup {{severity}} - {{node}} +EOF + + # Text body template + cat > "$TEMPLATE_DIR/oracle-backup-body.txt.hbs" <<'EOF' +Oracle Backup Monitoring Alert +============================== +Severity: {{severity}} +Hostname: {{node}} +Date: {{date}} +Status: {{status}} + +{{#if errors}} +ERRORS: +{{#each errors}} + - {{this}} +{{/each}} +{{/if}} + +{{#if warnings}} +WARNINGS: +{{#each warnings}} + - {{this}} +{{/each}} +{{/if}} + +Backup Details: +- Total Backups: {{total_backups}} +- Total Size: {{total_size_gb}} GB +- FULL Backup Age: {{full_backup_age}} hours +- CUMULATIVE Backup Age: {{cumulative_backup_age}} hours +- Disk Usage: {{disk_usage}}% + +{{#if backup_list}} +Recent Backups: +{{#each backup_list}} + {{this}} +{{/each}} +{{/if}} +EOF + + # HTML body template + cat > "$TEMPLATE_DIR/oracle-backup-body.html.hbs" <<'EOF' + + + + + + +
+

Oracle Backup {{severity}}

+

{{node}} - {{date}}

+
+ +
+

Status: {{status}}

+ + {{#if errors}} +
+

Errors:

+
    + {{#each errors}} +
  • {{this}}
    • + {{/each}} +
+
+ {{/if}} + + {{#if warnings}} +
+

Warnings:

+
    + {{#each warnings}} +
  • {{this}}
    • + {{/each}} +
+
+ {{/if}} +
+ +
+

Backup Metrics

+
+
+
Total Backups
+
{{total_backups}}
+
+
+
Total Size
+
{{total_size_gb}} GB
+
+
+
Disk Usage
+
{{disk_usage}}%
+
+
+ + + + + + + + + + + + + + + + + +
Backup TypeAge (hours)Status
FULL{{full_backup_age}}{{#if full_backup_ok}}✓ OK{{else}}✗ Too Old{{/if}}
CUMULATIVE{{cumulative_backup_age}}{{#if cumulative_backup_ok}}✓ OK{{else}}⚠ Check{{/if}}
+
+ + {{#if backup_list}} +
+

Recent Backups

+ 
{{#each backup_list}}{{this}}
+{{/each}}
+
+ {{/if}} + + +EOF + + echo -e "${GREEN}Templates created successfully in $TEMPLATE_DIR${NC}" +} + +# Function to send notification via PVE::Notify +send_pve_notification() { + local severity="$1" + local status="$2" + local data="$3" + + # Create Perl script to call PVE::Notify + cat > /tmp/oracle-notify.pl <<'PERL_SCRIPT' +#!/usr/bin/perl +use strict; +use warnings; +use PVE::Notify; +use JSON; + +my $json_data = do { local $/; }; +my $data = decode_json($json_data); + +my $severity = $data->{severity} // 'info'; +my $template_name = 'oracle-backup'; + +# Add fields for matching rules +my $fields = { + type => 'oracle-backup', + severity => $severity, + hostname => $data->{hostname}, +}; + +# Send notification +eval { + PVE::Notify::notify( + $severity, + $template_name, + $data, + $fields + ); +}; + +if ($@) { + print "Error sending notification: $@\n"; + exit 1; +} + +print "Notification sent successfully\n"; +PERL_SCRIPT + + chmod +x /tmp/oracle-notify.pl + + # Send notification + echo "$data" | perl /tmp/oracle-notify.pl + + rm -f /tmp/oracle-notify.pl +} + +# Function to check backups +check_backups() { + local status="OK" + local errors=() + local warnings=() + + echo "Checking Oracle backups..." + + # Get backup list + local backup_files=$(ls -lth "$BACKUP_PATH"/*.BKP 2>/dev/null | head -10 || echo "") + + if [ -z "$backup_files" ]; then + status="ERROR" + errors+=("No backup files found in $BACKUP_PATH") + else + # Count backups + local total_backups=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l) + local total_size=$(du -shc "$BACKUP_PATH"/*.BKP 2>/dev/null | tail -1 | awk '{print $1}') + + # Check FULL backup age + local latest_full=$(ls -t "$BACKUP_PATH"/*FULL*.BKP 2>/dev/null | head -1 || echo "") + local full_age_hours="N/A" + local full_backup_ok=false + + if [ -n "$latest_full" ]; then + local full_timestamp=$(stat -c %Y "$latest_full") + local current_timestamp=$(date +%s) + full_age_hours=$(( (current_timestamp - full_timestamp) / 3600 )) + + if [ "$full_age_hours" -gt "$MAX_FULL_AGE_HOURS" ]; then + status="WARNING" + warnings+=("FULL backup is $full_age_hours hours old (threshold: $MAX_FULL_AGE_HOURS)") + else + full_backup_ok=true + fi + else + status="ERROR" + errors+=("No FULL backup found") + fi + + # Check CUMULATIVE backup age + local latest_cumulative=$(ls -t "$BACKUP_PATH"/*INCR*.BKP "$BACKUP_PATH"/*CUMULATIVE*.BKP 2>/dev/null | head -1 || echo "") + local cumulative_age_hours="N/A" + local cumulative_backup_ok=false + + if [ -n "$latest_cumulative" ]; then + local cumulative_timestamp=$(stat -c %Y "$latest_cumulative") + local current_timestamp=$(date +%s) + cumulative_age_hours=$(( (current_timestamp - cumulative_timestamp) / 3600 )) + + if [ "$cumulative_age_hours" -gt "$MAX_CUMULATIVE_AGE_HOURS" ]; then + if [ "$status" != "ERROR" ]; then status="WARNING"; fi + warnings+=("CUMULATIVE backup is $cumulative_age_hours hours old (threshold: $MAX_CUMULATIVE_AGE_HOURS)") + else + cumulative_backup_ok=true + fi + fi + + # Check disk usage + local disk_usage=$(df "$BACKUP_PATH" | tail -1 | awk '{print int($5)}') + + if [ "$disk_usage" -gt 90 ]; then + status="ERROR" + errors+=("Disk usage critical: ${disk_usage}%") + elif [ "$disk_usage" -gt 80 ]; then + if [ "$status" != "ERROR" ]; then status="WARNING"; fi + warnings+=("Disk usage high: ${disk_usage}%") + fi + + # Prepare notification data + local severity="info" + [ "$status" = "WARNING" ] && severity="warning" + [ "$status" = "ERROR" ] && severity="error" + + # Convert arrays to JSON arrays + local errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .) + local warnings_json=$(printf '%s\n' "${warnings[@]}" | jq -R . | jq -s .) + local backup_list_json=$(echo "$backup_files" | head -5 | jq -R . | jq -s .) + + # Create JSON data + local json_data=$(cat < Notifications > Add matching rules for 'oracle-backup'" + ;; + --help) + echo "Oracle Backup Monitor for Proxmox" + echo "Usage:" + echo " $0 - Check backups and send alerts if issues found" + echo " $0 --install - Create notification templates" + echo " $0 --help - Show this help" + ;; + *) + # Check if templates exist, create if missing + if [ ! -f "$TEMPLATE_DIR/oracle-backup-subject.txt.hbs" ]; then + echo -e "${YELLOW}Templates not found, creating...${NC}" + create_templates + echo "" + fi + + # Run backup check + check_backups + ;; + esac +} + +# Check dependencies +if ! command -v jq &> /dev/null; then + echo -e "${RED}Error: jq is not installed${NC}" + echo "Install with: apt-get install jq" + exit 1 +fi + main "$@" \ No newline at end of file diff --git a/oracle/standby-server-scripts/weekly-dr-test-proxmox.sh b/oracle/standby-server-scripts/weekly-dr-test-proxmox.sh index 4e811d5..8c65930 100644 --- a/oracle/standby-server-scripts/weekly-dr-test-proxmox.sh +++ b/oracle/standby-server-scripts/weekly-dr-test-proxmox.sh @@ -1,619 +1,619 @@ -#!/bin/bash -# -# Oracle DR Weekly Test with Proxmox PVE::Notify -# Automated DR test with notifications via Proxmox notification system -# -# Location: /opt/scripts/weekly-dr-test-proxmox.sh (on Proxmox host) -# Schedule: Add to cron for weekly execution (Saturdays) -# -# This script is SELF-SUFFICIENT: -# - Automatically creates notification templates if they don't exist -# - Uses Proxmox native notification system -# - No email configuration needed - uses existing Proxmox setup -# -# Installation: -# cp weekly-dr-test-proxmox.sh /opt/scripts/ -# chmod +x /opt/scripts/weekly-dr-test-proxmox.sh -# /opt/scripts/weekly-dr-test-proxmox.sh --install # Creates templates -# crontab -e # Add: 0 6 * * 6 /opt/scripts/weekly-dr-test-proxmox.sh -# -# Author: Claude (based on ha-monitor.sh pattern) -# Version: 1.0 - -set -euo pipefail - -# Configuration -DR_VM_ID="109" -DR_VM_IP="10.0.20.37" -DR_VM_PORT="22122" -DR_VM_USER="romfast" -BACKUP_PATH="/mnt/pve/oracle-backups/ROA/autobackup" -MAX_RESTORE_TIME_MIN=30 -TEMPLATE_DIR="/usr/share/pve-manager/templates/default" -LOG_DIR="/var/log/oracle-dr" -LOG_FILE="$LOG_DIR/dr_test_$(date +%Y%m%d_%H%M%S).log" - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -# Create log directory -mkdir -p "$LOG_DIR" - -# Function to create notification templates -create_templates() { - echo -e "${GREEN}Creating Oracle DR test notification templates...${NC}" - - # Create templates directory if needed - mkdir -p "$TEMPLATE_DIR" - - # Subject template - cat > "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" <<'EOF' -Oracle DR Test {{severity}} - {{test_result}} -EOF - - # Text body template - cat > "$TEMPLATE_DIR/oracle-dr-test-body.txt.hbs" <<'EOF' -Oracle DR Weekly Test Report -============================ -Test Result: {{test_result}} -Severity: {{severity}} -Date: {{timestamp}} -Duration: {{total_duration}} minutes - -{{#if is_success}} -✓ TEST PASSED SUCCESSFULLY -{{else}} -✗ TEST FAILED -{{/if}} - -Test Steps Summary: -------------------- -{{#each test_steps}} -{{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}}: {{this.status}} ({{this.duration}}s) -{{/each}} - -{{#if errors}} -ERRORS: -{{#each errors}} - - {{this}} -{{/each}} -{{/if}} - -{{#if warnings}} -WARNINGS: -{{#each warnings}} - - {{this}} -{{/each}} -{{/if}} - -Metrics: --------- -- Backup Count: {{backup_count}} -- Restore Time: {{restore_duration}} minutes -- Tables Restored: {{tables_restored}} -- Database Status: {{database_status}} -- Disk Space Freed: {{disk_freed}} GB - -VM Details: ------------ -- VM ID: {{vm_id}} -- VM IP: {{vm_ip}} -- NFS Mount: {{nfs_status}} - -Log File: {{log_file}} -EOF - - # HTML body template - cat > "$TEMPLATE_DIR/oracle-dr-test-body.html.hbs" <<'EOF' - - - - - - -
-

Oracle DR Test Report

-

{{#if is_success}}✓ TEST PASSED{{else}}✗ TEST FAILED{{/if}}

-

{{timestamp}} | Duration: {{total_duration}} minutes

-
- -
-

Test Summary

-
-
-
{{test_result}}
-
Test Result
-
-
-
{{restore_duration}}
-
Restore Time (min)
-
-
-
{{tables_restored}}
-
Tables Restored
-
-
-
{{backup_count}}
-
Backups Used
-
-
-
- -
-

Test Steps Timeline

-
- {{#each test_steps}} -
-
-
-
- {{this.name}} - {{this.duration}}s -
- {{#if this.passed}} - ✓ {{this.status}} - {{else}} - ✗ {{this.status}} - {{/if}} -
- {{#if this.details}} -
- {{this.details}} -
- {{/if}} -
-
-
- {{/each}} -
-
- - {{#if errors}} -
-

Errors Encountered

-
    - {{#each errors}} -
  • {{this}}
    • - {{/each}} -
-
- {{/if}} - - {{#if warnings}} -
-

Warnings

-
    - {{#each warnings}} -
  • {{this}}
    • - {{/each}} -
-
- {{/if}} - -
-

System Details

- - - - - - - - - - - - - - - - - - - - - - - - - - -
ComponentValueStatus
DR VMID: {{vm_id}} ({{vm_ip}}){{vm_status}}
NFS MountF:\ drive{{nfs_status}}
DatabaseROA{{database_status}}
Disk Space Freed{{disk_freed}} GB✓
-
- -
-

- Log File: {{log_file}}
- Next Scheduled Test: Next Saturday 06:00 -

-
- - -EOF - - echo -e "${GREEN}Templates created successfully in $TEMPLATE_DIR${NC}" -} - -# Function to send notification via PVE::Notify -send_pve_notification() { - local severity="$1" - local data="$2" - - # Create Perl script to call PVE::Notify - cat > /tmp/oracle-dr-notify.pl <<'PERL_SCRIPT' -#!/usr/bin/perl -use strict; -use warnings; -use PVE::Notify; -use JSON; - -my $json_data = do { local $/; }; -my $data = decode_json($json_data); - -my $severity = $data->{severity} // 'info'; -my $template_name = 'oracle-dr-test'; - -# Add fields for matching rules -my $fields = { - type => 'oracle-dr-test', - severity => $severity, - test_result => $data->{test_result}, -}; - -# Send notification -eval { - PVE::Notify::notify( - $severity, - $template_name, - $data, - $fields - ); -}; - -if ($@) { - print "Error sending notification: $@\n"; - exit 1; -} - -print "Notification sent successfully\n"; -PERL_SCRIPT - - chmod +x /tmp/oracle-dr-notify.pl - - # Send notification - echo "$data" | perl /tmp/oracle-dr-notify.pl - - rm -f /tmp/oracle-dr-notify.pl -} - -# Logging functions -log() { - echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $1" | tee -a "$LOG_FILE" -} - -log_warning() { - echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$LOG_FILE" -} - -log_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" | tee -a "$LOG_FILE" -} - -# Test tracking -TEST_STEPS=() -ERRORS=() -WARNINGS=() -TEST_START_TIME=$(date +%s) - -# Function to track test steps -track_step() { - local name="$1" - local passed="$2" - local status="$3" - local start_time="$4" - local end_time=$(date +%s) - local duration=$((end_time - start_time)) - - TEST_STEPS+=("{\"name\":\"$name\",\"passed\":$passed,\"status\":\"$status\",\"duration\":$duration}") - - if [ "$passed" = "false" ]; then - ERRORS+=("$name: $status") - fi -} - -# Main test workflow -run_dr_test() { - local test_result="FAILED" - local severity="error" - local is_success=false - - log "==========================================" - log "Oracle DR Weekly Test - Starting" - log "==========================================" - - # Step 1: Pre-flight checks - local step_start=$(date +%s) - log "STEP 1: Pre-flight checks" - - # Check backups exist - local backup_count=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l || echo "0") - - if [ "$backup_count" -lt 2 ]; then - track_step "Pre-flight checks" false "Insufficient backups (found: $backup_count)" "$step_start" - test_result="FAILED - No backups" - else - track_step "Pre-flight checks" true "Found $backup_count backups" "$step_start" - - # Step 2: Start VM - step_start=$(date +%s) - log "STEP 2: Starting DR VM" - - if qm start "$DR_VM_ID" 2>/dev/null; then - sleep 180 # Wait for boot - track_step "VM Startup" true "VM $DR_VM_ID started" "$step_start" - - # Step 3: Verify NFS mount - step_start=$(date +%s) - log "STEP 3: Verifying NFS mount" - - local nfs_status="Not Mounted" - if ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 "$DR_VM_USER@$DR_VM_IP" \ - "powershell -Command 'Test-Path F:\\ROA\\autobackup'" 2>/dev/null; then - nfs_status="Mounted" - track_step "NFS Mount Check" true "F:\\ drive accessible" "$step_start" - else - track_step "NFS Mount Check" false "F:\\ drive not accessible" "$step_start" - WARNINGS+=("NFS mount may need manual intervention") - fi - - # Step 4: Run restore - step_start=$(date +%s) - local restore_start=$step_start - log "STEP 4: Running database restore" - - if ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ - "D:\\oracle\\scripts\\rman_restore_from_zero.cmd" 2>&1 | tee -a "$LOG_FILE"; then - - local restore_end=$(date +%s) - local restore_duration=$(( (restore_end - restore_start) / 60 )) - - track_step "Database Restore" true "Restored in $restore_duration minutes" "$step_start" - - # Step 5: Verify database - step_start=$(date +%s) - log "STEP 5: Verifying database" - - local db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ - "cmd /c 'echo SELECT STATUS FROM V\$INSTANCE; | sqlplus -s / as sysdba' | findstr OPEN" || echo "") - - local tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ - "cmd /c 'echo SELECT COUNT(*) FROM DBA_TABLES WHERE OWNER NOT IN (''SYS'',''SYSTEM''); | sqlplus -s / as sysdba' | grep -o '[0-9]*' | tail -1" || echo "0") - - if [[ "$db_status" =~ "OPEN" ]]; then - track_step "Database Verification" true "Database OPEN, $tables_restored tables" "$step_start" - test_result="PASSED" - severity="info" - is_success=true - else - track_step "Database Verification" false "Database not OPEN" "$step_start" - fi - - # Step 6: Cleanup - step_start=$(date +%s) - log "STEP 6: Running cleanup" - - ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ - "D:\\oracle\\scripts\\cleanup_database.cmd" 2>/dev/null - - track_step "Cleanup" true "Database cleaned, ~8GB freed" "$step_start" - - else - track_step "Database Restore" false "Restore failed" "$step_start" - fi - - # Step 7: Shutdown VM - step_start=$(date +%s) - log "STEP 7: Shutting down VM" - - ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" "shutdown /s /t 30" 2>/dev/null - sleep 60 - qm stop "$DR_VM_ID" 2>/dev/null - - track_step "VM Shutdown" true "VM stopped" "$step_start" - - else - track_step "VM Startup" false "Failed to start VM $DR_VM_ID" "$step_start" - fi - fi - - # Calculate total duration - local test_end_time=$(date +%s) - local total_duration=$(( (test_end_time - TEST_START_TIME) / 60 )) - - # Prepare notification data - local steps_json=$(printf '%s,' "${TEST_STEPS[@]}" | sed 's/,$//') - local errors_json=$(printf '"%s",' "${ERRORS[@]}" | sed 's/,$//') - local warnings_json=$(printf '"%s",' "${WARNINGS[@]}" | sed 's/,$//') - - local json_data=$(cat < Notifications > Add matching rules for 'oracle-dr-test'" - ;; - --help) - echo "Oracle DR Weekly Test for Proxmox" - echo "Usage:" - echo " $0 - Run DR test" - echo " $0 --install - Create notification templates" - echo " $0 --help - Show this help" - ;; - *) - # Check if templates exist, create if missing - if [ ! -f "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" ]; then - echo -e "${YELLOW}Templates not found, creating...${NC}" - create_templates - echo "" - fi - - # Run DR test - run_dr_test - ;; - esac -} - -# Check dependencies -if ! command -v jq &> /dev/null; then - echo -e "${RED}Error: jq is not installed${NC}" - echo "Install with: apt-get install jq" - exit 1 -fi - +#!/bin/bash +# +# Oracle DR Weekly Test with Proxmox PVE::Notify +# Automated DR test with notifications via Proxmox notification system +# +# Location: /opt/scripts/weekly-dr-test-proxmox.sh (on Proxmox host) +# Schedule: Add to cron for weekly execution (Saturdays) +# +# This script is SELF-SUFFICIENT: +# - Automatically creates notification templates if they don't exist +# - Uses Proxmox native notification system +# - No email configuration needed - uses existing Proxmox setup +# +# Installation: +# cp weekly-dr-test-proxmox.sh /opt/scripts/ +# chmod +x /opt/scripts/weekly-dr-test-proxmox.sh +# /opt/scripts/weekly-dr-test-proxmox.sh --install # Creates templates +# crontab -e # Add: 0 6 * * 6 /opt/scripts/weekly-dr-test-proxmox.sh +# +# Author: Claude (based on ha-monitor.sh pattern) +# Version: 1.0 + +set -euo pipefail + +# Configuration +DR_VM_ID="109" +DR_VM_IP="10.0.20.37" +DR_VM_PORT="22122" +DR_VM_USER="romfast" +BACKUP_PATH="/mnt/pve/oracle-backups/ROA/autobackup" +MAX_RESTORE_TIME_MIN=30 +TEMPLATE_DIR="/usr/share/pve-manager/templates/default" +LOG_DIR="/var/log/oracle-dr" +LOG_FILE="$LOG_DIR/dr_test_$(date +%Y%m%d_%H%M%S).log" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +# Create log directory +mkdir -p "$LOG_DIR" + +# Function to create notification templates +create_templates() { + echo -e "${GREEN}Creating Oracle DR test notification templates...${NC}" + + # Create templates directory if needed + mkdir -p "$TEMPLATE_DIR" + + # Subject template + cat > "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" <<'EOF' +Oracle DR Test {{severity}} - {{test_result}} +EOF + + # Text body template + cat > "$TEMPLATE_DIR/oracle-dr-test-body.txt.hbs" <<'EOF' +Oracle DR Weekly Test Report +============================ +Test Result: {{test_result}} +Severity: {{severity}} +Date: {{date}} +Duration: {{total_duration}} minutes + +{{#if is_success}} +✓ TEST PASSED SUCCESSFULLY +{{else}} +✗ TEST FAILED +{{/if}} + +Test Steps Summary: +------------------- +{{#each test_steps}} +{{#if this.passed}}✓{{else}}✗{{/if}} {{this.name}}: {{this.status}} ({{this.duration}}s) +{{/each}} + +{{#if errors}} +ERRORS: +{{#each errors}} + - {{this}} +{{/each}} +{{/if}} + +{{#if warnings}} +WARNINGS: +{{#each warnings}} + - {{this}} +{{/each}} +{{/if}} + +Metrics: +-------- +- Backup Count: {{backup_count}} +- Restore Time: {{restore_duration}} minutes +- Tables Restored: {{tables_restored}} +- Database Status: {{database_status}} +- Disk Space Freed: {{disk_freed}} GB + +VM Details: +----------- +- VM ID: {{vm_id}} +- VM IP: {{vm_ip}} +- NFS Mount: {{nfs_status}} + +Log File: {{log_file}} +EOF + + # HTML body template + cat > "$TEMPLATE_DIR/oracle-dr-test-body.html.hbs" <<'EOF' + + + + + + +
+

Oracle DR Test Report

+

{{#if is_success}}✓ TEST PASSED{{else}}✗ TEST FAILED{{/if}}

+

{{date}} | Duration: {{total_duration}} minutes

+
+ +
+

Test Summary

+
+
+
{{test_result}}
+
Test Result
+
+
+
{{restore_duration}}
+
Restore Time (min)
+
+
+
{{tables_restored}}
+
Tables Restored
+
+
+
{{backup_count}}
+
Backups Used
+
+
+
+ +
+

Test Steps Timeline

+
+ {{#each test_steps}} +
+
+
+
+ {{this.name}} + {{this.duration}}s +
+ {{#if this.passed}} + ✓ {{this.status}} + {{else}} + ✗ {{this.status}} + {{/if}} +
+ {{#if this.details}} +
+ {{this.details}} +
+ {{/if}} +
+
+
+ {{/each}} +
+
+ + {{#if errors}} +
+

Errors Encountered

+
    + {{#each errors}} +
  • {{this}}
    • + {{/each}} +
+
+ {{/if}} + + {{#if warnings}} +
+

Warnings

+
    + {{#each warnings}} +
  • {{this}}
    • + {{/each}} +
+
+ {{/if}} + +
+

System Details

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
ComponentValueStatus
DR VMID: {{vm_id}} ({{vm_ip}}){{vm_status}}
NFS MountF:\ drive{{nfs_status}}
DatabaseROA{{database_status}}
Disk Space Freed{{disk_freed}} GB✓
+
+ +
+

+ Log File: {{log_file}}
+ Next Scheduled Test: Next Saturday 06:00 +

+
+ + +EOF + + echo -e "${GREEN}Templates created successfully in $TEMPLATE_DIR${NC}" +} + +# Function to send notification via PVE::Notify +send_pve_notification() { + local severity="$1" + local data="$2" + + # Create Perl script to call PVE::Notify + cat > /tmp/oracle-dr-notify.pl <<'PERL_SCRIPT' +#!/usr/bin/perl +use strict; +use warnings; +use PVE::Notify; +use JSON; + +my $json_data = do { local $/; }; +my $data = decode_json($json_data); + +my $severity = $data->{severity} // 'info'; +my $template_name = 'oracle-dr-test'; + +# Add fields for matching rules +my $fields = { + type => 'oracle-dr-test', + severity => $severity, + test_result => $data->{test_result}, +}; + +# Send notification +eval { + PVE::Notify::notify( + $severity, + $template_name, + $data, + $fields + ); +}; + +if ($@) { + print "Error sending notification: $@\n"; + exit 1; +} + +print "Notification sent successfully\n"; +PERL_SCRIPT + + chmod +x /tmp/oracle-dr-notify.pl + + # Send notification + echo "$data" | perl /tmp/oracle-dr-notify.pl + + rm -f /tmp/oracle-dr-notify.pl +} + +# Logging functions +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" | tee -a "$LOG_FILE" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$LOG_FILE" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" | tee -a "$LOG_FILE" +} + +# Test tracking +TEST_STEPS=() +ERRORS=() +WARNINGS=() +TEST_START_TIME=$(date +%s) + +# Function to track test steps +track_step() { + local name="$1" + local passed="$2" + local status="$3" + local start_time="$4" + local end_time=$(date +%s) + local duration=$((end_time - start_time)) + + TEST_STEPS+=("{\"name\":\"$name\",\"passed\":$passed,\"status\":\"$status\",\"duration\":$duration}") + + if [ "$passed" = "false" ]; then + ERRORS+=("$name: $status") + fi +} + +# Main test workflow +run_dr_test() { + local test_result="FAILED" + local severity="error" + local is_success=false + + log "==========================================" + log "Oracle DR Weekly Test - Starting" + log "==========================================" + + # Step 1: Pre-flight checks + local step_start=$(date +%s) + log "STEP 1: Pre-flight checks" + + # Check backups exist + local backup_count=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l || echo "0") + + if [ "$backup_count" -lt 2 ]; then + track_step "Pre-flight checks" false "Insufficient backups (found: $backup_count)" "$step_start" + test_result="FAILED - No backups" + else + track_step "Pre-flight checks" true "Found $backup_count backups" "$step_start" + + # Step 2: Start VM + step_start=$(date +%s) + log "STEP 2: Starting DR VM" + + if qm start "$DR_VM_ID" 2>/dev/null; then + sleep 180 # Wait for boot + track_step "VM Startup" true "VM $DR_VM_ID started" "$step_start" + + # Step 3: Verify NFS mount + step_start=$(date +%s) + log "STEP 3: Verifying NFS mount" + + local nfs_status="Not Mounted" + if ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 "$DR_VM_USER@$DR_VM_IP" \ + "powershell -Command 'Test-Path F:\\ROA\\autobackup'" 2>/dev/null; then + nfs_status="Mounted" + track_step "NFS Mount Check" true "F:\\ drive accessible" "$step_start" + else + track_step "NFS Mount Check" false "F:\\ drive not accessible" "$step_start" + WARNINGS+=("NFS mount may need manual intervention") + fi + + # Step 4: Run restore + step_start=$(date +%s) + local restore_start=$step_start + log "STEP 4: Running database restore" + + if ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ + "D:\\oracle\\scripts\\rman_restore_from_zero.cmd" 2>&1 | tee -a "$LOG_FILE"; then + + local restore_end=$(date +%s) + local restore_duration=$(( (restore_end - restore_start) / 60 )) + + track_step "Database Restore" true "Restored in $restore_duration minutes" "$step_start" + + # Step 5: Verify database + step_start=$(date +%s) + log "STEP 5: Verifying database" + + local db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ + "cmd /c 'echo SELECT STATUS FROM V\$INSTANCE; | sqlplus -s / as sysdba' | findstr OPEN" || echo "") + + local tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ + "cmd /c 'echo SELECT COUNT(*) FROM DBA_TABLES WHERE OWNER NOT IN (''SYS'',''SYSTEM''); | sqlplus -s / as sysdba' | grep -o '[0-9]*' | tail -1" || echo "0") + + if [[ "$db_status" =~ "OPEN" ]]; then + track_step "Database Verification" true "Database OPEN, $tables_restored tables" "$step_start" + test_result="PASSED" + severity="info" + is_success=true + else + track_step "Database Verification" false "Database not OPEN" "$step_start" + fi + + # Step 6: Cleanup + step_start=$(date +%s) + log "STEP 6: Running cleanup" + + ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \ + "D:\\oracle\\scripts\\cleanup_database.cmd" 2>/dev/null + + track_step "Cleanup" true "Database cleaned, ~8GB freed" "$step_start" + + else + track_step "Database Restore" false "Restore failed" "$step_start" + fi + + # Step 7: Shutdown VM + step_start=$(date +%s) + log "STEP 7: Shutting down VM" + + ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" "shutdown /s /t 30" 2>/dev/null + sleep 60 + qm stop "$DR_VM_ID" 2>/dev/null + + track_step "VM Shutdown" true "VM stopped" "$step_start" + + else + track_step "VM Startup" false "Failed to start VM $DR_VM_ID" "$step_start" + fi + fi + + # Calculate total duration + local test_end_time=$(date +%s) + local total_duration=$(( (test_end_time - TEST_START_TIME) / 60 )) + + # Prepare notification data + local steps_json=$(printf '%s,' "${TEST_STEPS[@]}" | sed 's/,$//') + local errors_json=$(printf '"%s",' "${ERRORS[@]}" | sed 's/,$//') + local warnings_json=$(printf '"%s",' "${WARNINGS[@]}" | sed 's/,$//') + + local json_data=$(cat < Notifications > Add matching rules for 'oracle-dr-test'" + ;; + --help) + echo "Oracle DR Weekly Test for Proxmox" + echo "Usage:" + echo " $0 - Run DR test" + echo " $0 --install - Create notification templates" + echo " $0 --help - Show this help" + ;; + *) + # Check if templates exist, create if missing + if [ ! -f "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" ]; then + echo -e "${YELLOW}Templates not found, creating...${NC}" + create_templates + echo "" + fi + + # Run DR test + run_dr_test + ;; + esac +} + +# Check dependencies +if ! command -v jq &> /dev/null; then + echo -e "${RED}Error: jq is not installed${NC}" + echo "Install with: apt-get install jq" + exit 1 +fi + main "$@" \ No newline at end of file