#!/bin/bash
#
# Oracle DR Weekly Test with Proxmox PVE::Notify
# Automated DR test with notifications via Proxmox notification system
#
# Location: /opt/scripts/weekly-dr-test-proxmox.sh (on Proxmox host)
# Schedule: Add to cron for weekly execution (Saturdays)
#
# This script is SELF-SUFFICIENT:
# - Automatically creates notification templates if they don't exist
# - Uses Proxmox native notification system
# - No email configuration needed - uses existing Proxmox setup
#
# Installation:
#   cp weekly-dr-test-proxmox.sh /opt/scripts/
#   chmod +x /opt/scripts/weekly-dr-test-proxmox.sh
#   /opt/scripts/weekly-dr-test-proxmox.sh --install  # Creates templates
#   crontab -e  # Add: 0 6 * * 6 /opt/scripts/weekly-dr-test-proxmox.sh
#
# Author: Claude (based on ha-monitor.sh pattern)
# Version: 1.0

set -euo pipefail

# Configuration
DR_VM_ID="109"
DR_VM_IP="10.0.20.37"
DR_VM_PORT="22122"
DR_VM_USER="romfast"
BACKUP_PATH="/mnt/pve/oracle-backups/ROA/autobackup"
MAX_RESTORE_TIME_MIN=30
TEMPLATE_DIR="/usr/share/pve-manager/templates/default"
LOG_DIR="/var/log/oracle-dr"
LOG_FILE="$LOG_DIR/dr_test_$(date +%Y%m%d_%H%M%S).log"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Create log directory
mkdir -p "$LOG_DIR"

# Function to create notification templates
create_templates() {
    echo -e "${GREEN}Creating Oracle DR test notification templates...${NC}"

    # Create templates directory if needed
    mkdir -p "$TEMPLATE_DIR"

    # Subject template
    cat > "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" <<'EOF'
Oracle DR Test {{severity}} - {{test_result}}
EOF

    # Text body template
    cat > "$TEMPLATE_DIR/oracle-dr-test-body.txt.hbs" <<'EOF'
Oracle DR Test {{severity}} - {{test_result}}
{{#if is_success}}TEST PASSED✓{{else}}TEST FAILED✗{{/if}}

========================================
SUMMARY: Duration {{total_duration}}min | Tables {{tables_restored}} | Backups {{backup_count}}

TEST STEPS:
{{#each test_steps}}
- {{#if this.passed}}PASS{{else}}FAIL{{/if}}: {{this.name}} ({{this.duration}}s)
{{#if this.details}}
  Details: {{this.details}}
{{/each}}

========================================
COMPONENT STATUS:
DR VM: ID {{vm_id}} ({{vm_ip}}) - {{vm_status}}
NFS Mount: {{nfs_status}} - {{#if nfs_ok}}OK{{else}}FAILED{{/if}}
Database: {{database_status}} - {{#if database_ok}}OK{{else}}FAILED{{/if}}
Disk Space: {{disk_freed}}GB freed - OK

{{#if errors}}
ERRORS:
{{#each errors}}
- {{this}}
{{/each}}
{{/if}}

========================================
Log: {{log_file}}
Next scheduled test: Next Saturday 06:00
Proxmox DR Monitoring System
EOF

    # HTML body template (identical to text for compatibility)
    cat > "$TEMPLATE_DIR/oracle-dr-test-body.html.hbs" <<'EOF'
Oracle DR Test {{severity}} - {{test_result}}
{{#if is_success}}TEST PASSED✓{{else}}TEST FAILED✗{{/if}}

========================================
SUMMARY: Duration {{total_duration}}min | Tables {{tables_restored}} | Backups {{backup_count}}

TEST STEPS:
{{#each test_steps}}
- {{#if this.passed}}PASS{{else}}FAIL{{/if}}: {{this.name}} ({{this.duration}}s)
{{#if this.details}}
  Details: {{this.details}}
{{/each}}

========================================
COMPONENT STATUS:
DR VM: ID {{vm_id}} ({{vm_ip}}) - {{vm_status}}
NFS Mount: {{nfs_status}} - {{#if nfs_ok}}OK{{else}}FAILED{{/if}}
Database: {{database_status}} - {{#if database_ok}}OK{{else}}FAILED{{/if}}
Disk Space: {{disk_freed}}GB freed - OK

{{#if errors}}
ERRORS:
{{#each errors}}
- {{this}}
{{/each}}
{{/if}}

========================================
Log: {{log_file}}
Next scheduled test: Next Saturday 06:00
Proxmox DR Monitoring System
EOF
    .section {
        margin: 20px 0;
        padding: 15px;
        background-color: #f8f9fa;
        border-radius: 5px;
    }
    .success { color: #28a745; font-weight: bold; }
    .error { color: #dc3545; font-weight: bold; }
    .warning { color: #ffc107; font-weight: bold; }
    .info { color: #17a2b8; }

    .test-steps {
        margin: 20px 0;
    }
    .step {
        padding: 10px;
        margin: 5px 0;
        border-left: 4px solid;
        background-color: white;
    }
    .step.passed {
        border-color: #28a745;
    }
    .step.failed {
        border-color: #dc3545;
        background-color: #f8d7da;
    }

    .metrics {
        display: grid;
        grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
        gap: 15px;
        margin: 20px 0;
    }
    .metric-card {
        background: white;
        padding: 15px;
        border-radius: 5px;
        text-align: center;
        box-shadow: 0 2px 4px rgba(0,0,0,0.1);
    }
    .metric-value {
        font-size: 24px;
        font-weight: bold;
        color: #495057;
    }
    .metric-label {
        font-size: 14px;
        color: #6c757d;
        margin-top: 5px;
    }

    .timeline {
        position: relative;
        padding: 20px 0;
    }
    .timeline-item {
        display: flex;
        margin-bottom: 20px;
    }
    .timeline-marker {
        width: 20px;
        height: 20px;
        border-radius: 50%;
        margin-right: 15px;
        flex-shrink: 0;
    }
    .timeline-marker.success {
        background-color: #28a745;
    }
    .timeline-marker.failed {
        background-color: #dc3545;
    }

    table {
        width: 100%;
        border-collapse: collapse;
        margin: 10px 0;
    }
    th, td {
        padding: 10px;
        text-align: left;
        border-bottom: 1px solid #dee2e6;
    }
    th {
        background-color: #e9ecef;
        font-weight: bold;
    }
</style>
</head>
<body>
    <div class="header">
        <h1>Oracle DR Test Report</h1>
        <h2>{{#if is_success}}✓ TEST PASSED{{else}}✗ TEST FAILED{{/if}}</h2>
        <p>{{date}} | Duration: {{total_duration}} minutes</p>
    </div>

    <div class="section">
        <h3>Test Summary</h3>
        <div class="metrics">
            <div class="metric-card">
                <div class="metric-value {{#if is_success}}success{{else}}error{{/if}}">{{test_result}}</div>
                <div class="metric-label">Test Result</div>
            </div>
            <div class="metric-card">
                <div class="metric-value">{{restore_duration}}</div>
                <div class="metric-label">Restore Time (min)</div>
            </div>
            <div class="metric-card">
                <div class="metric-value">{{tables_restored}}</div>
                <div class="metric-label">Tables Restored</div>
            </div>
            <div class="metric-card">
                <div class="metric-value">{{backup_count}}</div>
                <div class="metric-label">Backups Used</div>
            </div>
        </div>
    </div>

    <div class="section">
        <h3>Test Steps Timeline</h3>
        <div class="timeline">
            {{#each test_steps}}
            <div class="timeline-item">
                <div class="timeline-marker {{#if this.passed}}success{{else}}failed{{/if}}"></div>
                <div style="flex-grow: 1;">
                    <div class="step {{#if this.passed}}passed{{else}}failed{{/if}}">
                        <strong>{{this.name}}</strong>
                        <span style="float: right; color: #6c757d;">{{this.duration}}s</span>
                        <div style="margin-top: 5px;">
                            {{#if this.passed}}
                                <span class="success">✓ {{this.status}}</span>
                            {{else}}
                                <span class="error">✗ {{this.status}}</span>
                            {{/if}}
                        </div>
                        {{#if this.details}}
                        <div style="margin-top: 5px; font-size: 0.9em; color: #6c757d;">
                            {{this.details}}
                        </div>
                        {{/if}}
                    </div>
                </div>
            </div>
            {{/each}}
        </div>
    </div>

    {{#if errors}}
    <div class="section" style="background-color: #f8d7da;">
        <h3 class="error">Errors Encountered</h3>
        <ul>
        {{#each errors}}
            <li>{{this}}</li>
        {{/each}}
        </ul>
    </div>
    {{/if}}

    {{#if warnings}}
    <div class="section" style="background-color: #fff3cd;">
        <h3 class="warning">Warnings</h3>
        <ul>
        {{#each warnings}}
            <li>{{this}}</li>
        {{/each}}
        </ul>
    </div>
    {{/if}}

    <div class="section">
        <h3>System Details</h3>
        <table>
            <tr>
                <th>Component</th>
                <th>Value</th>
                <th>Status</th>
            </tr>
            <tr>
                <td>DR VM</td>
                <td>ID: {{vm_id}} ({{vm_ip}})</td>
                <td>{{vm_status}}</td>
            </tr>
            <tr>
                <td>NFS Mount</td>
                <td>F:\ drive</td>
                <td>{{nfs_status}}</td>
            </tr>
            <tr>
                <td>Database</td>
                <td>ROA</td>
                <td>{{database_status}}</td>
            </tr>
            <tr>
                <td>Disk Space Freed</td>
                <td>{{disk_freed}} GB</td>
                <td class="success">✓</td>
            </tr>
        </table>
    </div>

    <div class="section">
        <p class="info">
            <strong>Log File:</strong> {{log_file}}<br>
            <strong>Next Scheduled Test:</strong> Next Saturday 06:00
        </p>
    </div>
</body>
</html>
EOF

    echo -e "${GREEN}Templates created successfully in $TEMPLATE_DIR${NC}"
}

# Function to send notification via PVE::Notify
send_pve_notification() {
    local severity="$1"
    local data="$2"

    # Create Perl script to call PVE::Notify
    cat > /tmp/oracle-dr-notify.pl <<'PERL_SCRIPT'
#!/usr/bin/perl
use strict;
use warnings;
use PVE::Notify;
use JSON;

my $json_data = do { local $/; <STDIN> };
my $data = decode_json($json_data);

my $severity = $data->{severity} // 'info';
my $template_name = 'oracle-dr-test';

# Add fields for matching rules
my $fields = {
    type => 'oracle-dr-test',
    severity => $severity,
    test_result => $data->{test_result},
};

# Send notification
eval {
    PVE::Notify::notify(
        $severity,
        $template_name,
        $data,
        $fields
    );
};

if ($@) {
    print "Error sending notification: $@\n";
    exit 1;
}

print "Notification sent successfully\n";
PERL_SCRIPT

    chmod +x /tmp/oracle-dr-notify.pl

    # Send notification
    echo "$data" | perl /tmp/oracle-dr-notify.pl

    rm -f /tmp/oracle-dr-notify.pl
}

# Logging functions
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1" | tee -a "$LOG_FILE"
}

log_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$LOG_FILE"
}

log_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1" | tee -a "$LOG_FILE"
}

# Test tracking
TEST_STEPS=()
ERRORS=()
WARNINGS=()
TEST_START_TIME=$(date +%s)

# Function to track test steps
track_step() {
    local name="$1"
    local passed="$2"
    local status="$3"
    local start_time="$4"
    local end_time=$(date +%s)
    local duration=$((end_time - start_time))

    TEST_STEPS+=("{\"name\":\"$name\",\"passed\":$passed,\"status\":\"$status\",\"duration\":$duration}")

    if [ "$passed" = "false" ]; then
        ERRORS+=("$name: $status")
    fi
}

# Main test workflow
run_dr_test() {
    local test_result="FAILED"
    local severity="error"
    local is_success=false

    log "=========================================="
    log "Oracle DR Weekly Test - Starting"
    log "=========================================="

    # Step 1: Pre-flight checks
    local step_start=$(date +%s)
    log "STEP 1: Pre-flight checks"

    # Check backups exist
    local backup_count=$(ls "$BACKUP_PATH"/*.BKP 2>/dev/null | wc -l || echo "0")

    if [ "$backup_count" -lt 2 ]; then
        track_step "Pre-flight checks" false "Insufficient backups (found: $backup_count)" "$step_start"
        test_result="FAILED - No backups"
    else
        track_step "Pre-flight checks" true "Found $backup_count backups" "$step_start"

        # Step 2: Start VM
        step_start=$(date +%s)
        log "STEP 2: Starting DR VM"

        if qm start "$DR_VM_ID" 2>/dev/null; then
            sleep 180  # Wait for boot
            track_step "VM Startup" true "VM $DR_VM_ID started" "$step_start"

            # Step 3: Verify NFS mount
            step_start=$(date +%s)
            log "STEP 3: Verifying NFS mount"

            local nfs_status="Not Mounted"
            if ssh -p "$DR_VM_PORT" -o ConnectTimeout=10 "$DR_VM_USER@$DR_VM_IP" \
                "powershell -Command 'Test-Path F:\\ROA\\autobackup'" 2>/dev/null; then
                nfs_status="Mounted"
                track_step "NFS Mount Check" true "F:\\ drive accessible" "$step_start"
            else
                track_step "NFS Mount Check" false "F:\\ drive not accessible" "$step_start"
                WARNINGS+=("NFS mount may need manual intervention")
            fi

            # Step 4: Run restore
            step_start=$(date +%s)
            local restore_start=$step_start
            log "STEP 4: Running database restore"

            if ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
                "D:\\oracle\\scripts\\rman_restore_from_zero.cmd" 2>&1 | tee -a "$LOG_FILE"; then

                local restore_end=$(date +%s)
                local restore_duration=$(( (restore_end - restore_start) / 60 ))

                track_step "Database Restore" true "Restored in $restore_duration minutes" "$step_start"

                # Step 5: Verify database
                step_start=$(date +%s)
                log "STEP 5: Verifying database"

                local db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
                    "cmd /c 'echo SELECT STATUS FROM V\$INSTANCE; | sqlplus -s / as sysdba' | findstr OPEN" || echo "")

                local tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
                    "cmd /c 'echo SELECT COUNT(*) FROM DBA_TABLES WHERE OWNER NOT IN (''SYS'',''SYSTEM''); | sqlplus -s / as sysdba' | grep -o '[0-9]*' | tail -1" || echo "0")

                if [[ "$db_status" =~ "OPEN" ]]; then
                    track_step "Database Verification" true "Database OPEN, $tables_restored tables" "$step_start"
                    test_result="PASSED"
                    severity="info"
                    is_success=true
                else
                    track_step "Database Verification" false "Database not OPEN" "$step_start"
                fi

                # Step 6: Cleanup
                step_start=$(date +%s)
                log "STEP 6: Running cleanup"

                ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
                    "D:\\oracle\\scripts\\cleanup_database.cmd" 2>/dev/null

                track_step "Cleanup" true "Database cleaned, ~8GB freed" "$step_start"

            else
                track_step "Database Restore" false "Restore failed" "$step_start"
            fi

            # Step 7: Shutdown VM
            step_start=$(date +%s)
            log "STEP 7: Shutting down VM"

            ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" "shutdown /s /t 30" 2>/dev/null
            sleep 60
            qm stop "$DR_VM_ID" 2>/dev/null

            track_step "VM Shutdown" true "VM stopped" "$step_start"

        else
            track_step "VM Startup" false "Failed to start VM $DR_VM_ID" "$step_start"
        fi
    fi

    # Calculate total duration
    local test_end_time=$(date +%s)
    local total_duration=$(( (test_end_time - TEST_START_TIME) / 60 ))

    # Prepare notification data
    local steps_json=$(printf '%s,' "${TEST_STEPS[@]}" | sed 's/,$//')
    local errors_json=$(printf '"%s",' "${ERRORS[@]}" | sed 's/,$//')
    local warnings_json=$(printf '"%s",' "${WARNINGS[@]}" | sed 's/,$//')

    local json_data=$(cat <<JSON
{
    "severity": "$severity",
    "test_result": "$test_result",
    "date": "$(date '+%Y-%m-%d %H:%M:%S')",
    "total_duration": $total_duration,
    "is_success": $is_success,
    "test_steps": [$steps_json],
    "errors": [${errors_json:-}],
    "warnings": [${warnings_json:-}],
    "backup_count": ${backup_count:-0},
    "restore_duration": ${restore_duration:-0},
    "tables_restored": ${tables_restored:-0},
    "database_status": "${db_status:-UNKNOWN}",
    "disk_freed": 8,
    "vm_id": "$DR_VM_ID",
    "vm_ip": "$DR_VM_IP",
    "vm_status": "Stopped",
    "nfs_status": "${nfs_status:-Unknown}",
    "log_file": "$LOG_FILE"
}
JSON
)

    # Send notification
    log "Sending notification..."
    send_pve_notification "$severity" "$json_data"

    # Final summary
    log "=========================================="
    log "Oracle DR Test Complete: $test_result"
    log "Duration: $total_duration minutes"
    log "Log: $LOG_FILE"
    log "=========================================="
}

# Main execution
main() {
    case "${1:-}" in
        --install)
            create_templates
            echo ""
            echo -e "${GREEN}Installation complete!${NC}"
            echo "Next steps:"
            echo "1. Test the script: /opt/scripts/weekly-dr-test-proxmox.sh"
            echo "2. Add to cron: crontab -e"
            echo "   Add line: 0 6 * * 6 /opt/scripts/weekly-dr-test-proxmox.sh"
            echo "3. Configure notifications in Proxmox GUI if needed:"
            echo "   Datacenter > Notifications > Add matching rules for 'oracle-dr-test'"
            ;;
        --help)
            echo "Oracle DR Weekly Test for Proxmox"
            echo "Usage:"
            echo "  $0           - Run DR test"
            echo "  $0 --install - Create notification templates"
            echo "  $0 --help    - Show this help"
            ;;
        *)
            # Check if templates exist, create if missing
            if [ ! -f "$TEMPLATE_DIR/oracle-dr-test-subject.txt.hbs" ]; then
                echo -e "${YELLOW}Templates not found, creating...${NC}"
                create_templates
                echo ""
            fi

            # Run DR test
            run_dr_test
            ;;
    esac
}

# Check dependencies
if ! command -v jq &> /dev/null; then
    echo -e "${RED}Error: jq is not installed${NC}"
    echo "Install with: apt-get install jq"
    exit 1
fi

main "$@"